diff --git a/.github/ISSUE_TEMPLATE/bug.md b/.github/ISSUE_TEMPLATE/bug.md index 1c4663918c18..5384f2d93114 100644 --- a/.github/ISSUE_TEMPLATE/bug.md +++ b/.github/ISSUE_TEMPLATE/bug.md @@ -1,12 +1,17 @@ --- name: Bug Report -about: Create a report to help us improve +about: Report a non-security bug. For suspected security vulnerabilities or crashes, please use "Report a Security Vulnerability", below. labels: 'Type: Bug' --- -Please see the FAQ in our main README.md, then answer the questions below before -submitting your issue. +NOTE: if you are reporting is a potential security vulnerability or a crash, +please follow our CVE process at +https://github.com/grpc/proposal/blob/master/P4-grpc-cve-process.md instead of +filing an issue here. + +Please see the FAQ in our main README.md, then answer the questions below +before submitting your issue. ### What version of gRPC are you using? diff --git a/.github/lock.yml b/.github/lock.yml deleted file mode 100644 index 78f7b19b71d3..000000000000 --- a/.github/lock.yml +++ /dev/null @@ -1,2 +0,0 @@ -daysUntilLock: 180 -lockComment: false diff --git a/.github/mergeable.yml b/.github/mergeable.yml index d647dafb7ab1..187de98277b3 100644 --- a/.github/mergeable.yml +++ b/.github/mergeable.yml @@ -5,32 +5,17 @@ mergeable: - do: label must_include: regex: '^Type:' - fail: - - do: checks - status: 'failure' - payload: - title: 'Need an appropriate "Type:" label' - summary: 'Need an appropriate "Type:" label' - - when: pull_request.* - # This validator requires either the "no release notes" label OR a "Release" milestone - # to be considered successful. However, validators "pass" in mergeable only if all - # checks pass. So it is implemented in reverse. - # I.e.: !(!no_relnotes && !release_milestone) ==> no_relnotes || release_milestone - # If both validators pass, then it is considered a failure, and if either fails, it is - # considered a success. - validate: - - do: label - must_exclude: - regex: '^no release notes$' + - do: description + must_include: + # Allow: + # RELEASE NOTES: none (case insensitive) + # + # RELEASE NOTES: N/A (case insensitive) + # + # RELEASE NOTES: + # * + regex: '^RELEASE NOTES:\s*([Nn][Oo][Nn][Ee]|[Nn]/[Aa]|\n(\*|-)\s*.+)$' + regex_flag: 'm' - do: milestone - must_exclude: + must_include: regex: 'Release$' - pass: - - do: checks - status: 'failure' # fail on pass - payload: - title: 'Need Release milestone or "no release notes" label' - summary: 'Need Release milestone or "no release notes" label' - fail: - - do: checks - status: 'success' # pass on fail diff --git a/.github/stale.yml b/.github/stale.yml deleted file mode 100644 index 04a26a08d2ac..000000000000 --- a/.github/stale.yml +++ /dev/null @@ -1,58 +0,0 @@ -# Configuration for probot-stale - https://github.com/probot/stale - -# Number of days of inactivity before an Issue or Pull Request becomes stale -daysUntilStale: 7 - -# Number of days of inactivity before an Issue or Pull Request with the stale label is closed. -# Set to false to disable. If disabled, issues still need to be closed manually, but will remain marked as stale. -daysUntilClose: 7 - -# Only issues or pull requests with all of these labels are check if stale. Defaults to `[]` (disabled) -onlyLabels: - - "Status: Requires Reporter Clarification" - -# Issues or Pull Requests with these labels will never be considered stale. Set to `[]` to disable -exemptLabels: [] - -# Set to true to ignore issues in a project (defaults to false) -exemptProjects: false - -# Set to true to ignore issues in a milestone (defaults to false) -exemptMilestones: false - -# Set to true to ignore issues with an assignee (defaults to false) -exemptAssignees: false - -# Label to use when marking as stale -staleLabel: "stale" - -# Comment to post when marking as stale. Set to `false` to disable -markComment: > - This issue is labeled as requiring an update from the reporter, and no update has been received - after 7 days. If no update is provided in the next 7 days, this issue will be automatically closed. - -# Comment to post when removing the stale label. -# unmarkComment: > -# Your comment here. - -# Comment to post when closing a stale Issue or Pull Request. -# closeComment: > -# Your comment here. - -# Limit the number of actions per hour, from 1-30. Default is 30 -limitPerRun: 1 - -# Limit to only `issues` or `pulls` -# only: issues - -# Optionally, specify configuration settings that are specific to just 'issues' or 'pulls': -# pulls: -# daysUntilStale: 30 -# markComment: > -# This pull request has been automatically marked as stale because it has not had -# recent activity. It will be closed if no further activity occurs. Thank you -# for your contributions. - -# issues: -# exemptLabels: -# - confirmed diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 000000000000..9ed65e45b91d --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,35 @@ +name: "CodeQL" + +on: + push: + branches: [ master ] + schedule: + - cron: '24 20 * * 3' + +permissions: + contents: read + security-events: write + pull-requests: read + actions: read + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + timeout-minutes: 30 + + strategy: + fail-fast: false + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: go + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/lock.yml b/.github/workflows/lock.yml new file mode 100644 index 000000000000..5f49c7900a3d --- /dev/null +++ b/.github/workflows/lock.yml @@ -0,0 +1,20 @@ +name: 'Lock Threads' + +on: + workflow_dispatch: + schedule: + - cron: '22 1 * * *' + +permissions: + issues: write + pull-requests: write + +jobs: + lock: + runs-on: ubuntu-latest + steps: + - uses: dessant/lock-threads@v2 + with: + github-token: ${{ github.token }} + issue-lock-inactive-days: 180 + pr-lock-inactive-days: 180 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000000..d3dda5376c08 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,64 @@ +name: Release + +on: + release: + types: [published] + +jobs: + release: + permissions: + contents: write # to upload release asset (actions/upload-release-asset) + + name: Release cmd/protoc-gen-go-grpc + runs-on: ubuntu-latest + if: startsWith(github.event.release.tag_name, 'cmd/protoc-gen-go-grpc/') + strategy: + matrix: + goos: [linux, darwin, windows] + goarch: [386, amd64, arm64] + exclude: + - goos: darwin + goarch: 386 + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Set up Go + uses: actions/setup-go@v2 + + - name: Download dependencies + run: | + cd cmd/protoc-gen-go-grpc + go mod download + + - name: Prepare build directory + run: | + mkdir -p build/ + cp README.md build/ + cp LICENSE build/ + + - name: Build + env: + GOOS: ${{ matrix.goos }} + GOARCH: ${{ matrix.goarch }} + run: | + cd cmd/protoc-gen-go-grpc + go build -trimpath -o $GITHUB_WORKSPACE/build + + - name: Create package + id: package + run: | + PACKAGE_NAME=protoc-gen-go-grpc.${GITHUB_REF#refs/tags/cmd/protoc-gen-go-grpc/}.${{ matrix.goos }}.${{ matrix.goarch }}.tar.gz + tar -czvf $PACKAGE_NAME -C build . + echo "name=${PACKAGE_NAME}" >> $GITHUB_OUTPUT + + - name: Upload asset + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ github.event.release.upload_url }} + asset_path: ./${{ steps.package.outputs.name }} + asset_name: ${{ steps.package.outputs.name }} + asset_content_type: application/gzip diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 000000000000..5e01a1e70c45 --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,30 @@ +name: Stale bot + +on: + workflow_dispatch: + schedule: + - cron: "44 */2 * * *" + +jobs: + stale: + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + + steps: + - uses: actions/stale@v4 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + days-before-stale: 6 + days-before-close: 7 + only-labels: 'Status: Requires Reporter Clarification' + stale-issue-label: 'stale' + stale-pr-label: 'stale' + operations-per-run: 999 + stale-issue-message: > + This issue is labeled as requiring an update from the reporter, and no update has been received + after 6 days. If no update is provided in the next 7 days, this issue will be automatically closed. + stale-pr-message: > + This PR is labeled as requiring an update from the reporter, and no update has been received + after 6 days. If no update is provided in the next 7 days, this issue will be automatically closed. diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml new file mode 100644 index 000000000000..3c03f29d0755 --- /dev/null +++ b/.github/workflows/testing.yml @@ -0,0 +1,131 @@ +name: Testing + +# Trigger on pushes, PRs (excluding documentation changes), and nightly. +on: + push: + pull_request: + schedule: + - cron: 0 0 * * * # daily at 00:00 + +permissions: + contents: read + +# Always force the use of Go modules +env: + GO111MODULE: on + +jobs: + # Check generated protos match their source repos (optional for PRs). + vet-proto: + runs-on: ubuntu-latest + timeout-minutes: 20 + env: + VET_ONLY_PROTO: 1 + steps: + # Setup the environment. + - name: Setup Go + uses: actions/setup-go@v3 + with: + go-version: '1.21' + - name: Checkout repo + uses: actions/checkout@v3 + + # Run the vet checks. + - name: vet + run: ./vet.sh -install && ./vet.sh + + # Run the main gRPC-Go tests. + tests: + # Proto checks are run in the above job. + env: + VET_SKIP_PROTO: 1 + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + include: + - type: vet + goversion: '1.21' + + - type: tests + goversion: '1.21' + + - type: tests + goversion: '1.21' + testflags: -race + + - type: tests + goversion: '1.21' + goarch: 386 + + - type: tests + goversion: '1.21' + goarch: arm64 + + - type: tests + goversion: '1.20' + + - type: tests + goversion: '1.19' + + - type: extras + goversion: '1.21' + + steps: + # Setup the environment. + - name: Setup GOARCH + if: matrix.goarch != '' + run: echo "GOARCH=${{ matrix.goarch }}" >> $GITHUB_ENV + + - name: Setup qemu emulator + if: matrix.goarch == 'arm64' + # setup qemu-user-static emulator and register it with binfmt_misc so that aarch64 binaries + # are automatically executed using qemu. + run: docker run --rm --privileged multiarch/qemu-user-static:5.2.0-2 --reset --credential yes --persistent yes + + - name: Setup GRPC environment + if: matrix.grpcenv != '' + run: echo "${{ matrix.grpcenv }}" >> $GITHUB_ENV + + - name: Setup Go + uses: actions/setup-go@v3 + with: + go-version: ${{ matrix.goversion }} + + - name: Checkout repo + uses: actions/checkout@v3 + + # Only run vet for 'vet' runs. + - name: Run vet.sh + if: matrix.type == 'vet' + run: ./vet.sh -install && ./vet.sh + + # Main tests run for everything except when testing "extras" + # (where we run a reduced set of tests). + - name: Run tests + if: matrix.type == 'tests' + run: | + go version + go test ${{ matrix.testflags }} -cpu 1,4 -timeout 7m google.golang.org/grpc/... + cd "${GITHUB_WORKSPACE}" + for MOD_FILE in $(find . -name 'go.mod' | grep -Ev '^\./go\.mod'); do + pushd "$(dirname ${MOD_FILE})" + go test ${{ matrix.testflags }} -cpu 1,4 -timeout 2m ./... + popd + done + + # Non-core gRPC tests (examples, interop, etc) + - name: Run extras tests + if: matrix.type == 'extras' + run: | + export TERM=${TERM:-xterm} + go version + echo -e "\n-- Running Examples --" + examples/examples_test.sh + echo -e "\n-- Running AdvancedTLS Examples --" + security/advancedtls/examples/examples_test.sh + echo -e "\n-- Running Interop Test --" + interop/interop_test.sh + echo -e "\n-- Running xDS E2E Test --" + xds/internal/test/e2e/run.sh diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 9097478f856d..000000000000 --- a/.travis.yml +++ /dev/null @@ -1,40 +0,0 @@ -language: go - -matrix: - include: - - go: 1.14.x - env: VET=1 GO111MODULE=on - - go: 1.14.x - env: RACE=1 GO111MODULE=on - - go: 1.14.x - env: RUN386=1 - - go: 1.14.x - env: GRPC_GO_RETRY=on - - go: 1.14.x - env: TESTEXTRAS=1 - - go: 1.13.x - env: GO111MODULE=on - - go: 1.12.x - env: GO111MODULE=on - - go: 1.11.x # Keep until interop tests no longer require Go1.11 - env: GO111MODULE=on - -go_import_path: google.golang.org/grpc - -before_install: - - if [[ "${GO111MODULE}" = "on" ]]; then mkdir "${HOME}/go"; export GOPATH="${HOME}/go"; fi - - if [[ -n "${RUN386}" ]]; then export GOARCH=386; fi - - if [[ "${TRAVIS_EVENT_TYPE}" = "cron" && -z "${RUN386}" ]]; then RACE=1; fi - - if [[ "${TRAVIS_EVENT_TYPE}" != "cron" ]]; then export VET_SKIP_PROTO=1; fi - -install: - - try3() { eval "$*" || eval "$*" || eval "$*"; } - - try3 'if [[ "${GO111MODULE}" = "on" ]]; then go mod download; else make testdeps; fi' - - if [[ -n "${VET}" ]]; then ./vet.sh -install; fi - -script: - - set -e - - if [[ -n "${TESTEXTRAS}" ]]; then examples/examples_test.sh; interop/interop_test.sh; make testsubmodule; exit 0; fi - - if [[ -n "${VET}" ]]; then ./vet.sh; fi - - if [[ -n "${RACE}" ]]; then make testrace; exit 0; fi - - make test diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cd03f8c76888..608aa6e1ac5e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -20,6 +20,15 @@ How to get your contributions merged smoothly and quickly. both author's & review's time is wasted. Create more PRs to address different concerns and everyone will be happy. +- If you are searching for features to work on, issues labeled [Status: Help + Wanted](https://github.com/grpc/grpc-go/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc+label%3A%22Status%3A+Help+Wanted%22) + is a great place to start. These issues are well-documented and usually can be + resolved with a single pull request. + +- If you are adding a new file, make sure it has the copyright message template + at the top as a comment. You can copy over the message from an existing file + and update the year. + - The grpc package should only depend on standard Go packages and a small number of exceptions. If your contribution introduces new dependencies which are NOT in the [list](https://godoc.org/google.golang.org/grpc?imports), you need a @@ -32,14 +41,18 @@ How to get your contributions merged smoothly and quickly. - Provide a good **PR description** as a record of **what** change is being made and **why** it was made. Link to a github issue if it exists. -- Don't fix code style and formatting unless you are already changing that line - to address an issue. PRs with irrelevant changes won't be merged. If you do - want to fix formatting or style, do that in a separate PR. +- If you want to fix formatting or style, consider whether your changes are an + obvious improvement or might be considered a personal preference. If a style + change is based on preference, it likely will not be accepted. If it corrects + widely agreed-upon anti-patterns, then please do create a PR and explain the + benefits of the change. - Unless your PR is trivial, you should expect there will be reviewer comments - that you'll need to address before merging. We expect you to be reasonably - responsive to those comments, otherwise the PR will be closed after 2-3 weeks - of inactivity. + that you'll need to address before merging. We'll mark it as `Status: Requires + Reporter Clarification` if we expect you to respond to these comments in a + timely manner. If the PR remains inactive for 6 days, it will be marked as + `stale` and automatically close 7 days after that if we don't hear back from + you. - Maintain **clean commit history** and use **meaningful commit messages**. PRs with messy commit history are difficult to review and won't be merged. Use @@ -53,9 +66,8 @@ How to get your contributions merged smoothly and quickly. - **All tests need to be passing** before your change can be merged. We recommend you **run tests locally** before creating your PR to catch breakages early on. - - `make all` to test everything, OR - - `make vet` to catch vet errors - - `make test` to run the tests - - `make testrace` to run tests in race mode + - `VET_SKIP_PROTO=1 ./vet.sh` to catch vet errors + - `go test -cpu 1,4 -timeout 7m ./...` to run the tests + - `go test -race -cpu 1,4 -timeout 7m ./...` to run tests in race mode - Exceptions to the rules can be made if there's a compelling reason for doing so. diff --git a/Documentation/anti-patterns.md b/Documentation/anti-patterns.md new file mode 100644 index 000000000000..08469fc179f7 --- /dev/null +++ b/Documentation/anti-patterns.md @@ -0,0 +1,206 @@ +## Anti-Patterns + +### Dialing in gRPC +[`grpc.Dial`](https://pkg.go.dev/google.golang.org/grpc#Dial) is a function in +the gRPC library that creates a virtual connection from the gRPC client to the +gRPC server. It takes a target URI (which can represent the name of a logical +backend service and could resolve to multiple actual addresses) and a list of +options, and returns a +[`ClientConn`](https://pkg.go.dev/google.golang.org/grpc#ClientConn) object that +represents the connection to the server. The `ClientConn` contains one or more +actual connections to real server backends and attempts to keep these +connections healthy by automatically reconnecting to them when they break. + +The `Dial` function can also be configured with various options to customize the +behavior of the client connection. For example, developers could use options +such a +[`WithTransportCredentials`](https://pkg.go.dev/google.golang.org/grpc#WithTransportCredentials) +to configure the transport credentials to use. + +While `Dial` is commonly referred to as a "dialing" function, it doesn't +actually perform the low-level network dialing operation like +[`net.Dial`](https://pkg.go.dev/net#Dial) would. Instead, it creates a virtual +connection from the gRPC client to the gRPC server. + +`Dial` does initiate the process of connecting to the server, but it uses the +ClientConn object to manage and maintain that connection over time. This is why +errors encountered during the initial connection are no different from those +that occur later on, and why it's important to handle errors from RPCs rather +than relying on options like +[`FailOnNonTempDialError`](https://pkg.go.dev/google.golang.org/grpc#FailOnNonTempDialError), +[`WithBlock`](https://pkg.go.dev/google.golang.org/grpc#WithBlock), and +[`WithReturnConnectionError`](https://pkg.go.dev/google.golang.org/grpc#WithReturnConnectionError). +In fact, `Dial` does not always establish a connection to servers by default. +The connection behavior is determined by the load balancing policy being used. +For instance, an "active" load balancing policy such as Round Robin attempts to +maintain a constant connection, while the default "pick first" policy delays +connection until an RPC is executed. Instead of using the WithBlock option, which +may not be recommended in some cases, you can call the +[`ClientConn.Connect`](https://pkg.go.dev/google.golang.org/grpc#ClientConn.Connect) +method to explicitly initiate a connection. + +### Using `FailOnNonTempDialError`, `WithBlock`, and `WithReturnConnectionError` + +The gRPC API provides several options that can be used to configure the behavior +of dialing and connecting to a gRPC server. Some of these options, such as +`FailOnNonTempDialError`, `WithBlock`, and `WithReturnConnectionError`, rely on +failures at dial time. However, we strongly discourage developers from using +these options, as they can introduce race conditions and result in unreliable +and difficult-to-debug code. + +One of the most important reasons for avoiding these options, which is often +overlooked, is that connections can fail at any point in time. This means that +you need to handle RPC failures caused by connection issues, regardless of +whether a connection was never established in the first place, or if it was +created and then immediately lost. Implementing proper error handling for RPCs +is crucial for maintaining the reliability and stability of your gRPC +communication. + +### Why we discourage using `FailOnNonTempDialError`, `WithBlock`, and `WithReturnConnectionError` + +When a client attempts to connect to a gRPC server, it can encounter a variety +of errors, including network connectivity issues, server-side errors, and +incorrect usage of the gRPC API. The options `FailOnNonTempDialError`, +`WithBlock`, and `WithReturnConnectionError` are designed to handle some of +these errors, but they do so by relying on failures at dial time. This means +that they may not provide reliable or accurate information about the status of +the connection. + +For example, if a client uses `WithBlock` to wait for a connection to be +established, it may end up waiting indefinitely if the server is not responding. +Similarly, if a client uses `WithReturnConnectionError` to return a connection +error if dialing fails, it may miss opportunities to recover from transient +network issues that are resolved shortly after the initial dial attempt. + +## Best practices for error handling in gRPC + +Instead of relying on failures at dial time, we strongly encourage developers to +rely on errors from RPCs. When a client makes an RPC, it can receive an error +response from the server. These errors can provide valuable information about +what went wrong, including information about network issues, server-side errors, +and incorrect usage of the gRPC API. + +By handling errors from RPCs correctly, developers can write more reliable and +robust gRPC applications. Here are some best practices for error handling in +gRPC: + +- Always check for error responses from RPCs and handle them appropriately. +- Use the `status` field of the error response to determine the type of error that + occurred. +- When retrying failed RPCs, consider using the built-in retry mechanism + provided by gRPC-Go, if available, instead of manually implementing retries. + Refer to the [gRPC-Go retry example + documentation](https://github.com/grpc/grpc-go/blob/master/examples/features/retry/README.md) + for more information. +- Avoid using `FailOnNonTempDialError`, `WithBlock`, and + `WithReturnConnectionError`, as these options can introduce race conditions and + result in unreliable and difficult-to-debug code. +- If making the outgoing RPC in order to handle an incoming RPC, be sure to + translate the status code before returning the error from your method handler. + For example, if the error is an `INVALID_ARGUMENT` error, that probably means + your service has a bug (otherwise it shouldn't have triggered this error), in + which case `INTERNAL` is more appropriate to return back to your users. + +### Example: Handling errors from an RPC + +The following code snippet demonstrates how to handle errors from an RPC in +gRPC: + +```go +ctx, cancel := context.WithTimeout(context.Background(), time.Second) +defer cancel() + +res, err := client.MyRPC(ctx, &MyRequest{}) +if err != nil { + // Handle the error appropriately, + // log it & return an error to the caller, etc. + log.Printf("Error calling MyRPC: %v", err) + return nil, err +} + +// Use the response as appropriate +log.Printf("MyRPC response: %v", res) +``` + +To determine the type of error that occurred, you can use the status field of +the error response: + + +```go +resp, err := client.MakeRPC(context.Background(), request) +if err != nil { + status, ok := status.FromError(err) + if ok { + // Handle the error based on its status code + if status.Code() == codes.NotFound { + log.Println("Requested resource not found") + } else { + log.Printf("RPC error: %v", status.Message()) + } + } else { + //Handle non-RPC errors + log.Printf("Non-RPC error: %v", err) + } + return +} + +// Use the response as needed +log.Printf("Response received: %v", resp) +``` + +### Example: Using a backoff strategy + + +When retrying failed RPCs, use a backoff strategy to avoid overwhelming the +server or exacerbating network issues: + + +```go +var res *MyResponse +var err error + +// If the user doesn't have a context with a deadline, create one +ctx, cancel := context.WithTimeout(context.Background(), time.Second) +defer cancel() + +// Retry the RPC call a maximum number of times +for i := 0; i < maxRetries; i++ { + + // Make the RPC call + res, err = client.MyRPC(ctx, &MyRequest{}) + + // Check if the RPC call was successful + if err == nil { + // The RPC was successful, so break out of the loop + break + } + + // The RPC failed, so wait for a backoff period before retrying + backoff := time.Duration(i) * time.Second + log.Printf("Error calling MyRPC: %v; retrying in %v", err, backoff) + time.Sleep(backoff) +} + +// Check if the RPC call was successful after all retries +if err != nil { + // All retries failed, so handle the error appropriately + log.Printf("Error calling MyRPC: %v", err) + return nil, err +} + +// Use the response as appropriate +log.Printf("MyRPC response: %v", res) +``` + + +## Conclusion + +The +[`FailOnNonTempDialError`](https://pkg.go.dev/google.golang.org/grpc#FailOnNonTempDialError), +[`WithBlock`](https://pkg.go.dev/google.golang.org/grpc#WithBlock), and +[`WithReturnConnectionError`](https://pkg.go.dev/google.golang.org/grpc#WithReturnConnectionError) +options are designed to handle errors at dial time, but they can introduce race +conditions and result in unreliable and difficult-to-debug code. Instead of +relying on these options, we strongly encourage developers to rely on errors +from RPCs for error handling. By following best practices for error handling in +gRPC, developers can write more reliable and robust gRPC applications. diff --git a/Documentation/encoding.md b/Documentation/encoding.md index 31436609d51c..dd49f55e5d44 100644 --- a/Documentation/encoding.md +++ b/Documentation/encoding.md @@ -132,7 +132,7 @@ As a reminder, all `CallOption`s may be converted into `DialOption`s that become the default for all RPCs sent through a client using `grpc.WithDefaultCallOptions`: ```go - myclient := grpc.Dial(ctx, target, grpc.WithDefaultCallOptions(grpc.UseCompresor("gzip"))) + myclient := grpc.Dial(ctx, target, grpc.WithDefaultCallOptions(grpc.UseCompressor("gzip"))) ``` When specified in either of these ways, messages will be compressed using this diff --git a/Documentation/grpc-auth-support.md b/Documentation/grpc-auth-support.md index 0a6b9f52c1cd..1362eeaa4ae2 100644 --- a/Documentation/grpc-auth-support.md +++ b/Documentation/grpc-auth-support.md @@ -53,7 +53,7 @@ Alternatively, a client may also use the `grpc.CallOption` on each invocation of an RPC. To create a `credentials.PerRPCCredentials`, use -[oauth.NewOauthAccess](https://godoc.org/google.golang.org/grpc/credentials/oauth#NewOauthAccess). +[oauth.TokenSource](https://godoc.org/google.golang.org/grpc/credentials/oauth#TokenSource). Note, the OAuth2 implementation of `grpc.PerRPCCredentials` requires a client to use [grpc.WithTransportCredentials](https://godoc.org/google.golang.org/grpc#WithTransportCredentials) to prevent any insecure transmission of tokens. diff --git a/Documentation/grpc-metadata.md b/Documentation/grpc-metadata.md index c278a98ef8c7..06b36f4ac171 100644 --- a/Documentation/grpc-metadata.md +++ b/Documentation/grpc-metadata.md @@ -110,9 +110,9 @@ md := metadata.Pairs("k1", "v1", "k1", "v2", "k2", "v3") ctx := metadata.NewOutgoingContext(context.Background(), md) // later, add some more metadata to the context (e.g. in an interceptor) -md, _ := metadata.FromOutgoingContext(ctx) +send, _ := metadata.FromOutgoingContext(ctx) newMD := metadata.Pairs("k3", "v3") -ctx = metadata.NewContext(ctx, metadata.Join(metadata.New(send), newMD)) +ctx = metadata.NewOutgoingContext(ctx, metadata.Join(send, newMD)) // make unary RPC response, err := client.SomeRPC(ctx, someRequest) @@ -223,3 +223,8 @@ func (s *server) SomeStreamingRPC(stream pb.Service_SomeStreamingRPCServer) erro stream.SetTrailer(trailer) } ``` + +## Updating metadata from a server interceptor + +An example for updating metadata from a server interceptor is +available [here](../examples/features/metadata_interceptor/server/main.go). diff --git a/Documentation/keepalive.md b/Documentation/keepalive.md index 43cfda82daf7..6bd95360e02d 100644 --- a/Documentation/keepalive.md +++ b/Documentation/keepalive.md @@ -8,6 +8,17 @@ activity on the connection. For how to configure keepalive, see https://godoc.org/google.golang.org/grpc/keepalive for the options. +## Why do I need this? + +Keepalive can be useful to detect TCP level connection failures. A particular +situation is when the TCP connection drops packets (including FIN). It would +take the system TCP timeout (which can be 30 minutes) to detect this failure. +Keepalive would allow gRPC to detect this failure much sooner. + +Another usage is (as the name suggests) to keep the connection alive. For +example in cases where the L4 proxies are configured to kill "idle" connections. +Sending pings would make the connections not "idle". + ## What should I set? It should be sufficient for most users to set [client diff --git a/Documentation/proxy.md b/Documentation/proxy.md index 8fd6ee5248a8..59f1ed27925c 100644 --- a/Documentation/proxy.md +++ b/Documentation/proxy.md @@ -1,8 +1,8 @@ # Proxy HTTP CONNECT proxies are supported by default in gRPC. The proxy address can be -specified by the environment variables HTTP_PROXY, HTTPS_PROXY and NO_PROXY (or -the lowercase versions thereof). +specified by the environment variables `HTTPS_PROXY` and `NO_PROXY`. (Note that +these environment variables are case insensitive.) ## Custom proxy @@ -12,4 +12,4 @@ connection before giving it to gRPC. If the default proxy doesn't work for you, replace the default dialer with your custom proxy dialer. This can be done using -[`WithDialer`](https://godoc.org/google.golang.org/grpc#WithDialer). \ No newline at end of file +[`WithDialer`](https://godoc.org/google.golang.org/grpc#WithDialer). diff --git a/Documentation/server-reflection-tutorial.md b/Documentation/server-reflection-tutorial.md index 212d4b3771a6..6c7dc6cd6a5f 100644 --- a/Documentation/server-reflection-tutorial.md +++ b/Documentation/server-reflection-tutorial.md @@ -2,8 +2,9 @@ gRPC Server Reflection provides information about publicly-accessible gRPC services on a server, and assists clients at runtime to construct RPC requests -and responses without precompiled service information. It is used by gRPC CLI, -which can be used to introspect server protos and send/receive test RPCs. +and responses without precompiled service information. It is used by +[gRPCurl](https://github.com/fullstorydev/grpcurl), which can be used to +introspect server protos and send/receive test RPCs. ## Enable Server Reflection @@ -39,46 +40,41 @@ make the following changes: An example server with reflection registered can be found at `examples/features/reflection/server`. -## gRPC CLI +## gRPCurl -After enabling Server Reflection in a server application, you can use gRPC CLI -to check its services. gRPC CLI is only available in c++. Instructions on how to -use gRPC CLI can be found at -[command_line_tool.md](https://github.com/grpc/grpc/blob/master/doc/command_line_tool.md). +After enabling Server Reflection in a server application, you can use gRPCurl +to check its services. gRPCurl is built with Go and has packages available. +Instructions on how to install and use gRPCurl can be found at +[gRPCurl Installation](https://github.com/fullstorydev/grpcurl#installation). -To build gRPC CLI: - -```sh -git clone https://github.com/grpc/grpc -cd grpc -git submodule update --init -make grpc_cli -cd bins/opt # grpc_cli is in directory bins/opt/ -``` - -## Use gRPC CLI to check services +## Use gRPCurl to check services First, start the helloworld server in grpc-go directory: ```sh -$ cd -$ go run examples/features/reflection/server/main.go +$ cd /examples +$ go run features/reflection/server/main.go ``` -Open a new terminal and make sure you are in the directory where grpc_cli lives: - +output: ```sh -$ cd /bins/opt +server listening at [::]:50051 ``` -### List services +After installing gRPCurl, open a new terminal and run the commands from the new +terminal. -`grpc_cli ls` command lists services and methods exposed at a given port: +**NOTE:** gRPCurl expects a TLS-encrypted connection by default. For all of +the commands below, use the `-plaintext` flag to use an unencrypted connection. + +### List services and methods + +The `list` command lists services exposed at a given port: - List all the services exposed at a given port ```sh - $ ./grpc_cli ls localhost:50051 + $ grpcurl -plaintext localhost:50051 list ``` output: @@ -88,72 +84,88 @@ $ cd /bins/opt helloworld.Greeter ``` -- List one service with details +- List all the methods of a service - `grpc_cli ls` command inspects a service given its full name (in the format of - \.\). It can print information with a long listing format - when `-l` flag is set. This flag can be used to get more details about a - service. + The `list` command lists methods given the full service name (in the format of + \.\). ```sh - $ ./grpc_cli ls localhost:50051 helloworld.Greeter -l + $ grpcurl -plaintext localhost:50051 list helloworld.Greeter ``` output: ```sh - filename: helloworld.proto - package: helloworld; - service Greeter { - rpc SayHello(helloworld.HelloRequest) returns (helloworld.HelloReply) {} - } + helloworld.Greeter.SayHello + ``` +### Describe services and methods + +- Describe all services + + The `describe` command inspects a service given its full name (in the format + of \.\). + + ```sh + $ grpcurl -plaintext localhost:50051 describe helloworld.Greeter ``` -### List methods + output: + ```sh + helloworld.Greeter is a service: + service Greeter { + rpc SayHello ( .helloworld.HelloRequest ) returns ( .helloworld.HelloReply ); + } + ``` -- List one method with details +- Describe all methods of a service - `grpc_cli ls` command also inspects a method given its full name (in the - format of \.\.\). + The `describe` command inspects a method given its full name (in the format of + \.\.\). ```sh - $ ./grpc_cli ls localhost:50051 helloworld.Greeter.SayHello -l + $ grpcurl -plaintext localhost:50051 describe helloworld.Greeter.SayHello ``` output: ```sh - rpc SayHello(helloworld.HelloRequest) returns (helloworld.HelloReply) {} + helloworld.Greeter.SayHello is a method: + rpc SayHello ( .helloworld.HelloRequest ) returns ( .helloworld.HelloReply ); ``` ### Inspect message types -We can use`grpc_cli type` command to inspect request/response types given the +We can use the `describe` command to inspect request/response types given the full name of the type (in the format of \.\). - Get information about the request type ```sh - $ ./grpc_cli type localhost:50051 helloworld.HelloRequest + $ grpcurl -plaintext localhost:50051 describe helloworld.HelloRequest ``` output: ```sh + helloworld.HelloRequest is a message: message HelloRequest { - optional string name = 1[json_name = "name"]; + string name = 1; } ``` ### Call a remote method -We can send RPCs to a server and get responses using `grpc_cli call` command. +We can send RPCs to a server and get responses using the full method name (in +the format of \.\.\). The `-d ` flag +represents the request data and the `-format text` flag indicates that the +request data is in text format. - Call a unary method ```sh - $ ./grpc_cli call localhost:50051 SayHello "name: 'gRPC CLI'" + $ grpcurl -plaintext -format text -d 'name: "gRPCurl"' \ + localhost:50051 helloworld.Greeter.SayHello ``` output: ```sh - message: "Hello gRPC CLI" + message: "Hello gRPCurl" ``` diff --git a/MAINTAINERS.md b/MAINTAINERS.md index 093c82b3afe8..c6672c0a3efe 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -8,17 +8,18 @@ See [CONTRIBUTING.md](https://github.com/grpc/grpc-community/blob/master/CONTRIB for general contribution guidelines. ## Maintainers (in alphabetical order) -- [canguler](https://github.com/canguler), Google LLC + - [cesarghali](https://github.com/cesarghali), Google LLC - [dfawley](https://github.com/dfawley), Google LLC - [easwars](https://github.com/easwars), Google LLC -- [jadekler](https://github.com/jadekler), Google LLC - [menghanl](https://github.com/menghanl), Google LLC - [srini100](https://github.com/srini100), Google LLC ## Emeritus Maintainers (in alphabetical order) - [adelez](https://github.com/adelez), Google LLC +- [canguler](https://github.com/canguler), Google LLC - [iamqizhao](https://github.com/iamqizhao), Google LLC +- [jadekler](https://github.com/jadekler), Google LLC - [jtattermusch](https://github.com/jtattermusch), Google LLC - [lyuxuan](https://github.com/lyuxuan), Google LLC - [makmukhi](https://github.com/makmukhi), Google LLC diff --git a/Makefile b/Makefile index cf474ae2fb97..1f8960922b3b 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,13 @@ all: vet test testrace -build: deps +build: go build google.golang.org/grpc/... clean: go clean -i google.golang.org/grpc/... deps: - go get -d -v google.golang.org/grpc/... + GO111MODULE=on go get -d -v google.golang.org/grpc/... proto: @ if ! which protoc > /dev/null; then \ @@ -16,24 +16,18 @@ proto: fi go generate google.golang.org/grpc/... -test: testdeps +test: go test -cpu 1,4 -timeout 7m google.golang.org/grpc/... -testsubmodule: testdeps +testsubmodule: cd security/advancedtls && go test -cpu 1,4 -timeout 7m google.golang.org/grpc/security/advancedtls/... cd security/authorization && go test -cpu 1,4 -timeout 7m google.golang.org/grpc/security/authorization/... -testdeps: - go get -d -v -t google.golang.org/grpc/... - -testrace: testdeps +testrace: go test -race -cpu 1,4 -timeout 7m google.golang.org/grpc/... -updatedeps: - go get -d -v -u -f google.golang.org/grpc/... - -updatetestdeps: - go get -d -v -t -u -f google.golang.org/grpc/... +testdeps: + GO111MODULE=on go get -d -v -t google.golang.org/grpc/... vet: vetdeps ./vet.sh @@ -45,12 +39,8 @@ vetdeps: all \ build \ clean \ - deps \ proto \ test \ - testdeps \ testrace \ - updatedeps \ - updatetestdeps \ vet \ vetdeps diff --git a/NOTICE.txt b/NOTICE.txt new file mode 100644 index 000000000000..530197749e9d --- /dev/null +++ b/NOTICE.txt @@ -0,0 +1,13 @@ +Copyright 2014 gRPC authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/README.md b/README.md index 3949a683fb58..1bc92248cb47 100644 --- a/README.md +++ b/README.md @@ -14,21 +14,14 @@ RPC framework that puts mobile and HTTP/2 first. For more information see the ## Installation -With [Go module][] support (Go 1.11+), simply add the following import +Simply add the following import to your code, and then `go [build|run|test]` +will automatically fetch the necessary dependencies: + ```go import "google.golang.org/grpc" ``` -to your code, and then `go [build|run|test]` will automatically fetch the -necessary dependencies. - -Otherwise, to install the `grpc-go` package, run the following command: - -```console -$ go get -u google.golang.org/grpc -``` - > **Note:** If you are trying to access `grpc-go` from **China**, see the > [FAQ](#FAQ) below. @@ -56,15 +49,6 @@ To build Go code, there are several options: - Set up a VPN and access google.golang.org through that. -- Without Go module support: `git clone` the repo manually: - - ```sh - git clone https://github.com/grpc/grpc-go.git $GOPATH/src/google.golang.org/grpc - ``` - - You will need to do the same for all of grpc's dependencies in `golang.org`, - e.g. `golang.org/x/net`. - - With Go module support: it is possible to use the `replace` feature of `go mod` to create aliases for golang.org packages. In your project's directory: @@ -76,33 +60,13 @@ To build Go code, there are several options: ``` Again, this will need to be done for all transitive dependencies hosted on - golang.org as well. For details, refer to [golang/go issue #28652](https://github.com/golang/go/issues/28652). + golang.org as well. For details, refer to [golang/go issue + #28652](https://github.com/golang/go/issues/28652). ### Compiling error, undefined: grpc.SupportPackageIsVersion -#### If you are using Go modules: - -Ensure your gRPC-Go version is `require`d at the appropriate version in -the same module containing the generated `.pb.go` files. For example, -`SupportPackageIsVersion6` needs `v1.27.0`, so in your `go.mod` file: - -```go -module - -require ( - google.golang.org/grpc v1.27.0 -) -``` - -#### If you are *not* using Go modules: - -Update the `proto` package, gRPC package, and rebuild the `.proto` files: - -```sh -go get -u github.com/golang/protobuf/{proto,protoc-gen-go} -go get -u google.golang.org/grpc -protoc --go_out=plugins=grpc:. *.proto -``` +Please update to the latest version of gRPC-Go using +`go get google.golang.org/grpc`. ### How to turn on logging @@ -121,9 +85,11 @@ possible reasons, including: 1. mis-configured transport credentials, connection failed on handshaking 1. bytes disrupted, possibly by a proxy in between 1. server shutdown - 1. Keepalive parameters caused connection shutdown, for example if you have configured - your server to terminate connections regularly to [trigger DNS lookups](https://github.com/grpc/grpc-go/issues/3170#issuecomment-552517779). - If this is the case, you may want to increase your [MaxConnectionAgeGrace](https://pkg.go.dev/google.golang.org/grpc/keepalive?tab=doc#ServerParameters), + 1. Keepalive parameters caused connection shutdown, for example if you have + configured your server to terminate connections regularly to [trigger DNS + lookups](https://github.com/grpc/grpc-go/issues/3170#issuecomment-552517779). + If this is the case, you may want to increase your + [MaxConnectionAgeGrace](https://pkg.go.dev/google.golang.org/grpc/keepalive?tab=doc#ServerParameters), to allow longer RPC calls to finish. It can be tricky to debug this because the error happens on the client side but @@ -136,6 +102,6 @@ errors. [Go module]: https://github.com/golang/go/wiki/Modules [gRPC]: https://grpc.io [Go gRPC docs]: https://grpc.io/docs/languages/go -[Performance benchmark]: https://performance-dot-grpc-testing.appspot.com/explore?dashboard=5652536396611584&widget=490377658&container=1286539696 +[Performance benchmark]: https://performance-dot-grpc-testing.appspot.com/explore?dashboard=5180705743044608 [quick start]: https://grpc.io/docs/languages/go/quickstart [go-releases]: https://golang.org/doc/devel/release.html diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 000000000000..be6e108705c4 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,3 @@ +# Security Policy + +For information on gRPC Security Policy and reporting potentional security issues, please see [gRPC CVE Process](https://github.com/grpc/proposal/blob/master/P4-grpc-cve-process.md). diff --git a/admin/admin.go b/admin/admin.go new file mode 100644 index 000000000000..41ae156b4d15 --- /dev/null +++ b/admin/admin.go @@ -0,0 +1,58 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package admin provides a convenient method for registering a collection of +// administration services to a gRPC server. The services registered are: +// +// - Channelz: https://github.com/grpc/proposal/blob/master/A14-channelz.md +// +// - CSDS: https://github.com/grpc/proposal/blob/master/A40-csds-support.md +// +// # Experimental +// +// Notice: All APIs in this package are experimental and may be removed in a +// later release. +package admin + +import ( + "google.golang.org/grpc" + channelzservice "google.golang.org/grpc/channelz/service" + internaladmin "google.golang.org/grpc/internal/admin" +) + +func init() { + // Add a list of default services to admin here. Optional services, like + // CSDS, will be added by other packages. + internaladmin.AddService(func(registrar grpc.ServiceRegistrar) (func(), error) { + channelzservice.RegisterChannelzServiceToServer(registrar) + return nil, nil + }) +} + +// Register registers the set of admin services to the given server. +// +// The returned cleanup function should be called to clean up the resources +// allocated for the service handlers after the server is stopped. +// +// Note that if `s` is not a *grpc.Server or a *xds.GRPCServer, CSDS will not be +// registered because CSDS generated code is old and doesn't support interface +// `grpc.ServiceRegistrar`. +// https://github.com/envoyproxy/go-control-plane/issues/403 +func Register(s grpc.ServiceRegistrar) (cleanup func(), _ error) { + return internaladmin.Register(s) +} diff --git a/admin/admin_test.go b/admin/admin_test.go new file mode 100644 index 000000000000..0ee4aade0f3c --- /dev/null +++ b/admin/admin_test.go @@ -0,0 +1,34 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package admin_test + +import ( + "testing" + + "google.golang.org/grpc/admin/test" + "google.golang.org/grpc/codes" +) + +func TestRegisterNoCSDS(t *testing.T) { + test.RunRegisterTests(t, test.ExpectedStatusCodes{ + ChannelzCode: codes.OK, + // CSDS is not registered because xDS isn't imported. + CSDSCode: codes.Unimplemented, + }) +} diff --git a/admin/test/admin_test.go b/admin/test/admin_test.go new file mode 100644 index 000000000000..f0f784bfdf36 --- /dev/null +++ b/admin/test/admin_test.go @@ -0,0 +1,38 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// This file has the same content as admin_test.go, difference is that this is +// in another package, and it imports "xds", so we can test that csds is +// registered when xds is imported. + +package test_test + +import ( + "testing" + + "google.golang.org/grpc/admin/test" + "google.golang.org/grpc/codes" + _ "google.golang.org/grpc/xds" +) + +func TestRegisterWithCSDS(t *testing.T) { + test.RunRegisterTests(t, test.ExpectedStatusCodes{ + ChannelzCode: codes.OK, + CSDSCode: codes.OK, + }) +} diff --git a/admin/test/utils.go b/admin/test/utils.go new file mode 100644 index 000000000000..086ba2e6e476 --- /dev/null +++ b/admin/test/utils.go @@ -0,0 +1,116 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package test contains test only functions for package admin. It's used by +// admin/admin_test.go and admin/test/admin_test.go. +package test + +import ( + "context" + "net" + "testing" + "time" + + "github.com/google/uuid" + "google.golang.org/grpc" + "google.golang.org/grpc/admin" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/testutils/xds/bootstrap" + "google.golang.org/grpc/status" + + v3statusgrpc "github.com/envoyproxy/go-control-plane/envoy/service/status/v3" + v3statuspb "github.com/envoyproxy/go-control-plane/envoy/service/status/v3" + channelzgrpc "google.golang.org/grpc/channelz/grpc_channelz_v1" + channelzpb "google.golang.org/grpc/channelz/grpc_channelz_v1" +) + +const ( + defaultTestTimeout = 10 * time.Second +) + +// ExpectedStatusCodes contains the expected status code for each RPC (can be +// OK). +type ExpectedStatusCodes struct { + ChannelzCode codes.Code + CSDSCode codes.Code +} + +// RunRegisterTests makes a client, runs the RPCs, and compares the status +// codes. +func RunRegisterTests(t *testing.T, ec ExpectedStatusCodes) { + nodeID := uuid.New().String() + bootstrapCleanup, err := bootstrap.CreateFile(bootstrap.Options{ + NodeID: nodeID, + ServerURI: "no.need.for.a.server", + }) + if err != nil { + t.Fatal(err) + } + defer bootstrapCleanup() + + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("cannot create listener: %v", err) + } + + server := grpc.NewServer() + defer server.Stop() + cleanup, err := admin.Register(server) + if err != nil { + t.Fatalf("failed to register admin: %v", err) + } + defer cleanup() + go func() { + server.Serve(lis) + }() + + conn, err := grpc.Dial(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("cannot connect to server: %v", err) + } + + t.Run("channelz", func(t *testing.T) { + if err := RunChannelz(conn); status.Code(err) != ec.ChannelzCode { + t.Fatalf("%s RPC failed with error %v, want code %v", "channelz", err, ec.ChannelzCode) + } + }) + t.Run("csds", func(t *testing.T) { + if err := RunCSDS(conn); status.Code(err) != ec.CSDSCode { + t.Fatalf("%s RPC failed with error %v, want code %v", "CSDS", err, ec.CSDSCode) + } + }) +} + +// RunChannelz makes a channelz RPC. +func RunChannelz(conn *grpc.ClientConn) error { + c := channelzgrpc.NewChannelzClient(conn) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + _, err := c.GetTopChannels(ctx, &channelzpb.GetTopChannelsRequest{}, grpc.WaitForReady(true)) + return err +} + +// RunCSDS makes a CSDS RPC. +func RunCSDS(conn *grpc.ClientConn) error { + c := v3statusgrpc.NewClientStatusDiscoveryServiceClient(conn) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + _, err := c.FetchClientStatus(ctx, &v3statuspb.ClientStatusRequest{}, grpc.WaitForReady(true)) + return err +} diff --git a/attributes/attributes.go b/attributes/attributes.go index ee5c51e6cdb0..712fef4d0fb9 100644 --- a/attributes/attributes.go +++ b/attributes/attributes.go @@ -19,58 +19,123 @@ // Package attributes defines a generic key/value store used in various gRPC // components. // -// All APIs in this package are EXPERIMENTAL. +// # Experimental +// +// Notice: This package is EXPERIMENTAL and may be changed or removed in a +// later release. package attributes -import "fmt" +import ( + "fmt" + "strings" +) // Attributes is an immutable struct for storing and retrieving generic // key/value pairs. Keys must be hashable, and users should define their own -// types for keys. +// types for keys. Values should not be modified after they are added to an +// Attributes or if they were received from one. If values implement 'Equal(o +// any) bool', it will be called by (*Attributes).Equal to determine whether +// two values with the same key should be considered equal. type Attributes struct { - m map[interface{}]interface{} + m map[any]any } -// New returns a new Attributes containing all key/value pairs in kvs. If the -// same key appears multiple times, the last value overwrites all previous -// values for that key. Panics if len(kvs) is not even. -func New(kvs ...interface{}) *Attributes { - if len(kvs)%2 != 0 { - panic(fmt.Sprintf("attributes.New called with unexpected input: len(kvs) = %v", len(kvs))) - } - a := &Attributes{m: make(map[interface{}]interface{}, len(kvs)/2)} - for i := 0; i < len(kvs)/2; i++ { - a.m[kvs[i*2]] = kvs[i*2+1] - } - return a +// New returns a new Attributes containing the key/value pair. +func New(key, value any) *Attributes { + return &Attributes{m: map[any]any{key: value}} } -// WithValues returns a new Attributes containing all key/value pairs in a and -// kvs. Panics if len(kvs) is not even. If the same key appears multiple -// times, the last value overwrites all previous values for that key. To -// remove an existing key, use a nil value. -func (a *Attributes) WithValues(kvs ...interface{}) *Attributes { +// WithValue returns a new Attributes containing the previous keys and values +// and the new key/value pair. If the same key appears multiple times, the +// last value overwrites all previous values for that key. To remove an +// existing key, use a nil value. value should not be modified later. +func (a *Attributes) WithValue(key, value any) *Attributes { if a == nil { - return New(kvs...) + return New(key, value) } - if len(kvs)%2 != 0 { - panic(fmt.Sprintf("attributes.New called with unexpected input: len(kvs) = %v", len(kvs))) - } - n := &Attributes{m: make(map[interface{}]interface{}, len(a.m)+len(kvs)/2)} + n := &Attributes{m: make(map[any]any, len(a.m)+1)} for k, v := range a.m { n.m[k] = v } - for i := 0; i < len(kvs)/2; i++ { - n.m[kvs[i*2]] = kvs[i*2+1] - } + n.m[key] = value return n } // Value returns the value associated with these attributes for key, or nil if -// no value is associated with key. -func (a *Attributes) Value(key interface{}) interface{} { +// no value is associated with key. The returned value should not be modified. +func (a *Attributes) Value(key any) any { if a == nil { return nil } return a.m[key] } + +// Equal returns whether a and o are equivalent. If 'Equal(o any) bool' is +// implemented for a value in the attributes, it is called to determine if the +// value matches the one stored in the other attributes. If Equal is not +// implemented, standard equality is used to determine if the two values are +// equal. Note that some types (e.g. maps) aren't comparable by default, so +// they must be wrapped in a struct, or in an alias type, with Equal defined. +func (a *Attributes) Equal(o *Attributes) bool { + if a == nil && o == nil { + return true + } + if a == nil || o == nil { + return false + } + if len(a.m) != len(o.m) { + return false + } + for k, v := range a.m { + ov, ok := o.m[k] + if !ok { + // o missing element of a + return false + } + if eq, ok := v.(interface{ Equal(o any) bool }); ok { + if !eq.Equal(ov) { + return false + } + } else if v != ov { + // Fallback to a standard equality check if Value is unimplemented. + return false + } + } + return true +} + +// String prints the attribute map. If any key or values throughout the map +// implement fmt.Stringer, it calls that method and appends. +func (a *Attributes) String() string { + var sb strings.Builder + sb.WriteString("{") + first := true + for k, v := range a.m { + if !first { + sb.WriteString(", ") + } + sb.WriteString(fmt.Sprintf("%q: %q ", str(k), str(v))) + first = false + } + sb.WriteString("}") + return sb.String() +} + +func str(x any) string { + if v, ok := x.(fmt.Stringer); ok { + return v.String() + } else if v, ok := x.(string); ok { + return v + } + return fmt.Sprintf("<%p>", x) +} + +// MarshalJSON helps implement the json.Marshaler interface, thereby rendering +// the Attributes correctly when printing (via pretty.JSON) structs containing +// Attributes as fields. +// +// Is it impossible to unmarshal attributes from a JSON representation and this +// method is meant only for debugging purposes. +func (a *Attributes) MarshalJSON() ([]byte, error) { + return []byte(a.String()), nil +} diff --git a/attributes/attributes_test.go b/attributes/attributes_test.go index 4cca17b55e93..e4db92177a93 100644 --- a/attributes/attributes_test.go +++ b/attributes/attributes_test.go @@ -20,29 +20,71 @@ package attributes_test import ( "fmt" + "testing" "google.golang.org/grpc/attributes" ) +type stringVal struct { + s string +} + +func (s stringVal) Equal(o any) bool { + os, ok := o.(stringVal) + return ok && s.s == os.s +} + func ExampleAttributes() { type keyOne struct{} type keyTwo struct{} - a := attributes.New(keyOne{}, 1, keyTwo{}, "two") + a := attributes.New(keyOne{}, 1).WithValue(keyTwo{}, stringVal{s: "two"}) fmt.Println("Key one:", a.Value(keyOne{})) fmt.Println("Key two:", a.Value(keyTwo{})) // Output: // Key one: 1 - // Key two: two + // Key two: {two} } -func ExampleAttributes_WithValues() { +func ExampleAttributes_WithValue() { type keyOne struct{} type keyTwo struct{} a := attributes.New(keyOne{}, 1) - a = a.WithValues(keyTwo{}, "two") + a = a.WithValue(keyTwo{}, stringVal{s: "two"}) fmt.Println("Key one:", a.Value(keyOne{})) fmt.Println("Key two:", a.Value(keyTwo{})) // Output: // Key one: 1 - // Key two: two + // Key two: {two} +} + +// Test that two attributes with the same content are Equal. +func TestEqual(t *testing.T) { + type keyOne struct{} + type keyTwo struct{} + a1 := attributes.New(keyOne{}, 1).WithValue(keyTwo{}, stringVal{s: "two"}) + a2 := attributes.New(keyOne{}, 1).WithValue(keyTwo{}, stringVal{s: "two"}) + if !a1.Equal(a2) { + t.Fatalf("%+v.Equals(%+v) = false; want true", a1, a2) + } + if !a2.Equal(a1) { + t.Fatalf("%+v.Equals(%+v) = false; want true", a2, a1) + } +} + +// Test that two attributes with different content are not Equal. +func TestNotEqual(t *testing.T) { + type keyOne struct{} + type keyTwo struct{} + a1 := attributes.New(keyOne{}, 1).WithValue(keyTwo{}, stringVal{s: "two"}) + a2 := attributes.New(keyOne{}, 2).WithValue(keyTwo{}, stringVal{s: "two"}) + a3 := attributes.New(keyOne{}, 1).WithValue(keyTwo{}, stringVal{s: "one"}) + if a1.Equal(a2) { + t.Fatalf("%+v.Equals(%+v) = true; want false", a1, a2) + } + if a2.Equal(a1) { + t.Fatalf("%+v.Equals(%+v) = true; want false", a2, a1) + } + if a3.Equal(a1) { + t.Fatalf("%+v.Equals(%+v) = true; want false", a3, a1) + } } diff --git a/authz/audit/audit_logger.go b/authz/audit/audit_logger.go new file mode 100644 index 000000000000..b9b721970387 --- /dev/null +++ b/authz/audit/audit_logger.go @@ -0,0 +1,127 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package audit contains interfaces for audit logging during authorization. +package audit + +import ( + "encoding/json" + "sync" +) + +// loggerBuilderRegistry holds a map of audit logger builders and a mutex +// to facilitate thread-safe reading/writing operations. +type loggerBuilderRegistry struct { + mu sync.Mutex + builders map[string]LoggerBuilder +} + +var ( + registry = loggerBuilderRegistry{ + builders: make(map[string]LoggerBuilder), + } +) + +// RegisterLoggerBuilder registers the builder in a global map +// using b.Name() as the key. +// +// This should only be called during initialization time (i.e. in an init() +// function). If multiple builders are registered with the same name, +// the one registered last will take effect. +func RegisterLoggerBuilder(b LoggerBuilder) { + registry.mu.Lock() + defer registry.mu.Unlock() + registry.builders[b.Name()] = b +} + +// GetLoggerBuilder returns a builder with the given name. +// It returns nil if the builder is not found in the registry. +func GetLoggerBuilder(name string) LoggerBuilder { + registry.mu.Lock() + defer registry.mu.Unlock() + return registry.builders[name] +} + +// Event contains information passed to the audit logger as part of an +// audit logging event. +type Event struct { + // FullMethodName is the full method name of the audited RPC, in the format + // of "/pkg.Service/Method". For example, "/helloworld.Greeter/SayHello". + FullMethodName string + // Principal is the identity of the caller. Currently it will only be + // available in certificate-based TLS authentication. + Principal string + // PolicyName is the authorization policy name or the xDS RBAC filter name. + PolicyName string + // MatchedRule is the matched rule or policy name in the xDS RBAC filter. + // It will be empty if there is no match. + MatchedRule string + // Authorized indicates whether the audited RPC is authorized or not. + Authorized bool +} + +// LoggerConfig represents an opaque data structure holding an audit +// logger configuration. Concrete types representing configuration of specific +// audit loggers must embed this interface to implement it. +type LoggerConfig interface { + loggerConfig() +} + +// Logger is the interface to be implemented by audit loggers. +// +// An audit logger is a logger instance that can be configured via the +// authorization policy API or xDS HTTP RBAC filters. When the authorization +// decision meets the condition for audit, all the configured audit loggers' +// Log() method will be invoked to log that event. +// +// TODO(lwge): Change the link to the merged gRFC once it's ready. +// Please refer to https://github.com/grpc/proposal/pull/346 for more details +// about audit logging. +type Logger interface { + // Log performs audit logging for the provided audit event. + // + // This method is invoked in the RPC path and therefore implementations + // must not block. + Log(*Event) +} + +// LoggerBuilder is the interface to be implemented by audit logger +// builders that are used at runtime to configure and instantiate audit loggers. +// +// Users who want to implement their own audit logging logic should +// implement this interface, along with the Logger interface, and register +// it by calling RegisterLoggerBuilder() at init time. +// +// TODO(lwge): Change the link to the merged gRFC once it's ready. +// Please refer to https://github.com/grpc/proposal/pull/346 for more details +// about audit logging. +type LoggerBuilder interface { + // ParseLoggerConfig parses the given JSON bytes into a structured + // logger config this builder can use to build an audit logger. + ParseLoggerConfig(config json.RawMessage) (LoggerConfig, error) + // Build builds an audit logger with the given logger config. + // This will only be called with valid configs returned from + // ParseLoggerConfig() and any runtime issues such as failing to + // create a file should be handled by the logger implementation instead of + // failing the logger instantiation. So implementers need to make sure it + // can return a logger without error at this stage. + Build(LoggerConfig) Logger + // Name returns the name of logger built by this builder. + // This is used to register and pick the builder. + Name() string +} diff --git a/authz/audit/audit_logging_test.go b/authz/audit/audit_logging_test.go new file mode 100644 index 000000000000..e3a4ef25b021 --- /dev/null +++ b/authz/audit/audit_logging_test.go @@ -0,0 +1,377 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package audit_test + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "io" + "net" + "os" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc" + "google.golang.org/grpc/authz" + "google.golang.org/grpc/authz/audit" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/stubserver" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + "google.golang.org/grpc/status" + "google.golang.org/grpc/testdata" + + _ "google.golang.org/grpc/authz/audit/stdout" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +type statAuditLogger struct { + authzDecisionStat map[bool]int // Map to hold the counts of authorization decisions + lastEvent *audit.Event // Field to store last received event +} + +func (s *statAuditLogger) Log(event *audit.Event) { + s.authzDecisionStat[event.Authorized]++ + *s.lastEvent = *event +} + +type loggerBuilder struct { + authzDecisionStat map[bool]int + lastEvent *audit.Event +} + +func (loggerBuilder) Name() string { + return "stat_logger" +} + +func (lb *loggerBuilder) Build(audit.LoggerConfig) audit.Logger { + return &statAuditLogger{ + authzDecisionStat: lb.authzDecisionStat, + lastEvent: lb.lastEvent, + } +} + +func (*loggerBuilder) ParseLoggerConfig(config json.RawMessage) (audit.LoggerConfig, error) { + return nil, nil +} + +// TestAuditLogger examines audit logging invocations using four different +// authorization policies. It covers scenarios including a disabled audit, +// auditing both 'allow' and 'deny' outcomes, and separately auditing 'allow' +// and 'deny' outcomes. Additionally, it checks if SPIFFE ID from a certificate +// is propagated correctly. +func (s) TestAuditLogger(t *testing.T) { + // Each test data entry contains an authz policy for a grpc server, + // how many 'allow' and 'deny' outcomes we expect (each test case makes 2 + // unary calls and one client-streaming call), and a structure to check if + // the audit.Event fields are properly populated. Additionally, we specify + // directly which authz outcome we expect from each type of call. + tests := []struct { + name string + authzPolicy string + wantAuthzOutcomes map[bool]int + eventContent *audit.Event + wantUnaryCallCode codes.Code + wantStreamingCallCode codes.Code + }{ + { + name: "No audit", + authzPolicy: `{ + "name": "authz", + "allow_rules": [ + { + "name": "allow_UnaryCall", + "request": { + "paths": [ + "/grpc.testing.TestService/UnaryCall" + ] + } + } + ], + "audit_logging_options": { + "audit_condition": "NONE", + "audit_loggers": [ + { + "name": "stat_logger", + "config": {}, + "is_optional": false + } + ] + } + }`, + wantAuthzOutcomes: map[bool]int{true: 0, false: 0}, + wantUnaryCallCode: codes.OK, + wantStreamingCallCode: codes.PermissionDenied, + }, + { + name: "Allow All Deny Streaming - Audit All", + authzPolicy: `{ + "name": "authz", + "allow_rules": [ + { + "name": "allow_all", + "request": { + "paths": [ + "*" + ] + } + } + ], + "deny_rules": [ + { + "name": "deny_all", + "request": { + "paths": [ + "/grpc.testing.TestService/StreamingInputCall" + ] + } + } + ], + "audit_logging_options": { + "audit_condition": "ON_DENY_AND_ALLOW", + "audit_loggers": [ + { + "name": "stat_logger", + "config": {}, + "is_optional": false + }, + { + "name": "stdout_logger", + "is_optional": false + } + ] + } + }`, + wantAuthzOutcomes: map[bool]int{true: 2, false: 1}, + eventContent: &audit.Event{ + FullMethodName: "/grpc.testing.TestService/StreamingInputCall", + Principal: "spiffe://foo.bar.com/client/workload/1", + PolicyName: "authz", + MatchedRule: "authz_deny_all", + Authorized: false, + }, + wantUnaryCallCode: codes.OK, + wantStreamingCallCode: codes.PermissionDenied, + }, + { + name: "Allow Unary - Audit Allow", + authzPolicy: `{ + "name": "authz", + "allow_rules": [ + { + "name": "allow_UnaryCall", + "request": { + "paths": [ + "/grpc.testing.TestService/UnaryCall" + ] + } + } + ], + "audit_logging_options": { + "audit_condition": "ON_ALLOW", + "audit_loggers": [ + { + "name": "stat_logger", + "config": {}, + "is_optional": false + } + ] + } + }`, + wantAuthzOutcomes: map[bool]int{true: 2, false: 0}, + wantUnaryCallCode: codes.OK, + wantStreamingCallCode: codes.PermissionDenied, + }, + { + name: "Allow Typo - Audit Deny", + authzPolicy: `{ + "name": "authz", + "allow_rules": [ + { + "name": "allow_UnaryCall", + "request": { + "paths": [ + "/grpc.testing.TestService/UnaryCall_Z" + ] + } + } + ], + "audit_logging_options": { + "audit_condition": "ON_DENY", + "audit_loggers": [ + { + "name": "stat_logger", + "config": {}, + "is_optional": false + } + ] + } + }`, + wantAuthzOutcomes: map[bool]int{true: 0, false: 3}, + wantUnaryCallCode: codes.PermissionDenied, + wantStreamingCallCode: codes.PermissionDenied, + }, + } + // Construct the credentials for the tests and the stub server + serverCreds := loadServerCreds(t) + clientCreds := loadClientCreds(t) + ss := &stubserver.StubServer{ + UnaryCallF: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{}, nil + }, + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + _, err := stream.Recv() + if err != io.EOF { + return err + } + return nil + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + // Setup test statAuditLogger, gRPC test server with authzPolicy, unary + // and stream interceptors. + lb := &loggerBuilder{ + authzDecisionStat: map[bool]int{true: 0, false: 0}, + lastEvent: &audit.Event{}, + } + audit.RegisterLoggerBuilder(lb) + i, _ := authz.NewStatic(test.authzPolicy) + + s := grpc.NewServer( + grpc.Creds(serverCreds), + grpc.ChainUnaryInterceptor(i.UnaryInterceptor), + grpc.ChainStreamInterceptor(i.StreamInterceptor)) + defer s.Stop() + testgrpc.RegisterTestServiceServer(s, ss) + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Error listening: %v", err) + } + go s.Serve(lis) + + // Setup gRPC test client with certificates containing a SPIFFE Id. + clientConn, err := grpc.Dial(lis.Addr().String(), grpc.WithTransportCredentials(clientCreds)) + if err != nil { + t.Fatalf("grpc.Dial(%v) failed: %v", lis.Addr().String(), err) + } + defer clientConn.Close() + client := testgrpc.NewTestServiceClient(clientConn) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + if _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}); status.Code(err) != test.wantUnaryCallCode { + t.Errorf("Unexpected UnaryCall fail: got %v want %v", err, test.wantUnaryCallCode) + } + if _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}); status.Code(err) != test.wantUnaryCallCode { + t.Errorf("Unexpected UnaryCall fail: got %v want %v", err, test.wantUnaryCallCode) + } + stream, err := client.StreamingInputCall(ctx) + if err != nil { + t.Fatalf("StreamingInputCall failed:%v", err) + } + req := &testpb.StreamingInputCallRequest{ + Payload: &testpb.Payload{ + Body: []byte("hi"), + }, + } + if err := stream.Send(req); err != nil && err != io.EOF { + t.Fatalf("stream.Send failed:%v", err) + } + if _, err := stream.CloseAndRecv(); status.Code(err) != test.wantStreamingCallCode { + t.Errorf("Unexpected stream.CloseAndRecv fail: got %v want %v", err, test.wantStreamingCallCode) + } + + // Compare expected number of allows/denies with content of the internal + // map of statAuditLogger. + if diff := cmp.Diff(lb.authzDecisionStat, test.wantAuthzOutcomes); diff != "" { + t.Errorf("Authorization decisions do not match\ndiff (-got +want):\n%s", diff) + } + // Compare last event received by statAuditLogger with expected event. + if test.eventContent != nil { + if diff := cmp.Diff(lb.lastEvent, test.eventContent); diff != "" { + t.Errorf("Unexpected message\ndiff (-got +want):\n%s", diff) + } + } + }) + } +} + +// loadServerCreds constructs TLS containing server certs and CA +func loadServerCreds(t *testing.T) credentials.TransportCredentials { + t.Helper() + cert := loadKeys(t, "x509/server1_cert.pem", "x509/server1_key.pem") + certPool := loadCACerts(t, "x509/client_ca_cert.pem") + return credentials.NewTLS(&tls.Config{ + ClientAuth: tls.RequireAndVerifyClientCert, + Certificates: []tls.Certificate{cert}, + ClientCAs: certPool, + }) +} + +// loadClientCreds constructs TLS containing client certs and CA +func loadClientCreds(t *testing.T) credentials.TransportCredentials { + t.Helper() + cert := loadKeys(t, "x509/client_with_spiffe_cert.pem", "x509/client_with_spiffe_key.pem") + roots := loadCACerts(t, "x509/server_ca_cert.pem") + return credentials.NewTLS(&tls.Config{ + Certificates: []tls.Certificate{cert}, + RootCAs: roots, + ServerName: "x.test.example.com", + }) + +} + +// loadKeys loads X509 key pair from the provided file paths. +// It is used for loading both client and server certificates for the test +func loadKeys(t *testing.T, certPath, key string) tls.Certificate { + t.Helper() + cert, err := tls.LoadX509KeyPair(testdata.Path(certPath), testdata.Path(key)) + if err != nil { + t.Fatalf("tls.LoadX509KeyPair(%q, %q) failed: %v", certPath, key, err) + } + return cert +} + +// loadCACerts loads CA certificates and constructs x509.CertPool +// It is used for loading both client and server CAs for the test +func loadCACerts(t *testing.T, certPath string) *x509.CertPool { + t.Helper() + ca, err := os.ReadFile(testdata.Path(certPath)) + if err != nil { + t.Fatalf("os.ReadFile(%q) failed: %v", certPath, err) + } + roots := x509.NewCertPool() + if !roots.AppendCertsFromPEM(ca) { + t.Fatal("Failed to append certificates") + } + return roots +} diff --git a/authz/audit/stdout/stdout_logger.go b/authz/audit/stdout/stdout_logger.go new file mode 100644 index 000000000000..f9bfc2d7d61e --- /dev/null +++ b/authz/audit/stdout/stdout_logger.go @@ -0,0 +1,110 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package stdout defines an stdout audit logger. +package stdout + +import ( + "encoding/json" + "log" + "os" + "time" + + "google.golang.org/grpc/authz/audit" + "google.golang.org/grpc/grpclog" +) + +var grpcLogger = grpclog.Component("authz-audit") + +// Name is the string to identify this logger type in the registry +const Name = "stdout_logger" + +func init() { + audit.RegisterLoggerBuilder(&loggerBuilder{ + goLogger: log.New(os.Stdout, "", 0), + }) +} + +type event struct { + FullMethodName string `json:"rpc_method"` + Principal string `json:"principal"` + PolicyName string `json:"policy_name"` + MatchedRule string `json:"matched_rule"` + Authorized bool `json:"authorized"` + Timestamp string `json:"timestamp"` // Time when the audit event is logged via Log method +} + +// logger implements the audit.logger interface by logging to standard output. +type logger struct { + goLogger *log.Logger +} + +// Log marshals the audit.Event to json and prints it to standard output. +func (l *logger) Log(event *audit.Event) { + jsonContainer := map[string]any{ + "grpc_audit_log": convertEvent(event), + } + jsonBytes, err := json.Marshal(jsonContainer) + if err != nil { + grpcLogger.Errorf("failed to marshal AuditEvent data to JSON: %v", err) + return + } + l.goLogger.Println(string(jsonBytes)) +} + +// loggerConfig represents the configuration for the stdout logger. +// It is currently empty and implements the audit.Logger interface by embedding it. +type loggerConfig struct { + audit.LoggerConfig +} + +type loggerBuilder struct { + goLogger *log.Logger +} + +func (loggerBuilder) Name() string { + return Name +} + +// Build returns a new instance of the stdout logger. +// Passed in configuration is ignored as the stdout logger does not +// expect any configuration to be provided. +func (lb *loggerBuilder) Build(audit.LoggerConfig) audit.Logger { + return &logger{ + goLogger: lb.goLogger, + } +} + +// ParseLoggerConfig is a no-op since the stdout logger does not accept any configuration. +func (*loggerBuilder) ParseLoggerConfig(config json.RawMessage) (audit.LoggerConfig, error) { + if len(config) != 0 && string(config) != "{}" { + grpcLogger.Warningf("Stdout logger doesn't support custom configs. Ignoring:\n%s", string(config)) + } + return &loggerConfig{}, nil +} + +func convertEvent(auditEvent *audit.Event) *event { + return &event{ + FullMethodName: auditEvent.FullMethodName, + Principal: auditEvent.Principal, + PolicyName: auditEvent.PolicyName, + MatchedRule: auditEvent.MatchedRule, + Authorized: auditEvent.Authorized, + Timestamp: time.Now().Format(time.RFC3339Nano), + } +} diff --git a/authz/audit/stdout/stdout_logger_test.go b/authz/audit/stdout/stdout_logger_test.go new file mode 100644 index 000000000000..02f9860c5315 --- /dev/null +++ b/authz/audit/stdout/stdout_logger_test.go @@ -0,0 +1,140 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package stdout + +import ( + "bytes" + "encoding/json" + "log" + "os" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/authz/audit" + "google.golang.org/grpc/internal/grpctest" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func (s) TestStdoutLogger_Log(t *testing.T) { + tests := map[string]struct { + event *audit.Event + wantMessage string + wantErr string + }{ + "few fields": { + event: &audit.Event{PolicyName: "test policy", Principal: "test principal"}, + wantMessage: `{"fullMethodName":"","principal":"test principal","policyName":"test policy","matchedRule":"","authorized":false`, + }, + "all fields": { + event: &audit.Event{ + FullMethodName: "/helloworld.Greeter/SayHello", + Principal: "spiffe://example.org/ns/default/sa/default/backend", + PolicyName: "example-policy", + MatchedRule: "dev-access", + Authorized: true, + }, + wantMessage: `{"fullMethodName":"/helloworld.Greeter/SayHello",` + + `"principal":"spiffe://example.org/ns/default/sa/default/backend","policyName":"example-policy",` + + `"matchedRule":"dev-access","authorized":true`, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + before := time.Now().Unix() + var buf bytes.Buffer + builder := &loggerBuilder{goLogger: log.New(&buf, "", 0)} + auditLogger := builder.Build(nil) + + auditLogger.Log(test.event) + + var container map[string]any + if err := json.Unmarshal(buf.Bytes(), &container); err != nil { + t.Fatalf("Failed to unmarshal audit log event: %v", err) + } + innerEvent := extractEvent(container["grpc_audit_log"].(map[string]any)) + if innerEvent.Timestamp == "" { + t.Fatalf("Resulted event has no timestamp: %v", innerEvent) + } + after := time.Now().Unix() + innerEventUnixTime, err := time.Parse(time.RFC3339Nano, innerEvent.Timestamp) + if err != nil { + t.Fatalf("Failed to convert event timestamp into Unix time format: %v", err) + } + if before > innerEventUnixTime.Unix() || after < innerEventUnixTime.Unix() { + t.Errorf("The audit event timestamp is outside of the test interval: test start %v, event timestamp %v, test end %v", before, innerEventUnixTime.Unix(), after) + } + if diff := cmp.Diff(trimEvent(innerEvent), test.event); diff != "" { + t.Fatalf("Unexpected message\ndiff (-got +want):\n%s", diff) + } + }) + } +} + +func (s) TestStdoutLoggerBuilder_NilConfig(t *testing.T) { + builder := &loggerBuilder{ + goLogger: log.New(os.Stdout, "", log.LstdFlags), + } + config, err := builder.ParseLoggerConfig(nil) + if err != nil { + t.Fatalf("Failed to parse stdout logger configuration: %v", err) + } + if l := builder.Build(config); l == nil { + t.Fatal("Failed to build stdout audit logger") + } +} + +func (s) TestStdoutLoggerBuilder_Registration(t *testing.T) { + if audit.GetLoggerBuilder("stdout_logger") == nil { + t.Fatal("stdout logger is not registered") + } +} + +// extractEvent extracts an stdout.event from a map +// unmarshalled from a logged json message. +func extractEvent(container map[string]any) event { + return event{ + FullMethodName: container["rpc_method"].(string), + Principal: container["principal"].(string), + PolicyName: container["policy_name"].(string), + MatchedRule: container["matched_rule"].(string), + Authorized: container["authorized"].(bool), + Timestamp: container["timestamp"].(string), + } +} + +// trimEvent converts a logged stdout.event into an audit.Event +// by removing Timestamp field. It is used for comparing events during testing. +func trimEvent(testEvent event) *audit.Event { + return &audit.Event{ + FullMethodName: testEvent.FullMethodName, + Principal: testEvent.Principal, + PolicyName: testEvent.PolicyName, + MatchedRule: testEvent.MatchedRule, + Authorized: testEvent.Authorized, + } +} diff --git a/authz/grpc_authz_end2end_test.go b/authz/grpc_authz_end2end_test.go new file mode 100644 index 000000000000..0a4cd1862e98 --- /dev/null +++ b/authz/grpc_authz_end2end_test.go @@ -0,0 +1,727 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package authz_test + +import ( + "context" + "crypto/tls" + "crypto/x509" + "io" + "net" + "os" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/authz" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/grpc/testdata" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +type testServer struct { + testgrpc.UnimplementedTestServiceServer +} + +func (s *testServer) UnaryCall(ctx context.Context, req *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{}, nil +} + +func (s *testServer) StreamingInputCall(stream testgrpc.TestService_StreamingInputCallServer) error { + for { + _, err := stream.Recv() + if err == io.EOF { + return stream.SendAndClose(&testpb.StreamingInputCallResponse{}) + } + if err != nil { + return err + } + } +} + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +var authzTests = map[string]struct { + authzPolicy string + md metadata.MD + wantStatus *status.Status +}{ + "DeniesRPCMatchInDenyNoMatchInAllow": { + authzPolicy: `{ + "name": "authz", + "allow_rules": + [ + { + "name": "allow_StreamingOutputCall", + "request": { + "paths": + [ + "/grpc.testing.TestService/StreamingOutputCall" + ] + } + } + ], + "deny_rules": + [ + { + "name": "deny_TestServiceCalls", + "request": { + "paths": + [ + "/grpc.testing.TestService/*" + ], + "headers": + [ + { + "key": "key-abc", + "values": + [ + "val-abc", + "val-def" + ] + } + ] + } + } + ] + }`, + md: metadata.Pairs("key-abc", "val-abc"), + wantStatus: status.New(codes.PermissionDenied, "unauthorized RPC request rejected"), + }, + "DeniesRPCMatchInDenyAndAllow": { + authzPolicy: `{ + "name": "authz", + "allow_rules": + [ + { + "name": "allow_all", + "request": { + "paths": + [ + "*" + ] + } + } + ], + "deny_rules": + [ + { + "name": "deny_all", + "request": { + "paths": + [ + "*" + ] + } + } + ] + }`, + wantStatus: status.New(codes.PermissionDenied, "unauthorized RPC request rejected"), + }, + "AllowsRPCNoMatchInDenyMatchInAllow": { + authzPolicy: `{ + "name": "authz", + "allow_rules": + [ + { + "name": "allow_all" + } + ], + "deny_rules": + [ + { + "name": "deny_TestServiceCalls", + "request": { + "paths": + [ + "/grpc.testing.TestService/UnaryCall", + "/grpc.testing.TestService/StreamingInputCall" + ], + "headers": + [ + { + "key": "key-abc", + "values": + [ + "val-abc", + "val-def" + ] + } + ] + } + } + ] + }`, + md: metadata.Pairs("key-xyz", "val-xyz"), + wantStatus: status.New(codes.OK, ""), + }, + "DeniesRPCNoMatchInDenyAndAllow": { + authzPolicy: `{ + "name": "authz", + "allow_rules": + [ + { + "name": "allow_some_user", + "source": { + "principals": + [ + "some_user" + ] + } + } + ], + "deny_rules": + [ + { + "name": "deny_StreamingOutputCall", + "request": { + "paths": + [ + "/grpc.testing.TestService/StreamingOutputCall" + ] + } + } + ] + }`, + wantStatus: status.New(codes.PermissionDenied, "unauthorized RPC request rejected"), + }, + "AllowsRPCEmptyDenyMatchInAllow": { + authzPolicy: `{ + "name": "authz", + "allow_rules": + [ + { + "name": "allow_UnaryCall", + "request": + { + "paths": + [ + "/grpc.testing.TestService/UnaryCall" + ] + } + }, + { + "name": "allow_StreamingInputCall", + "request": + { + "paths": + [ + "/grpc.testing.TestService/StreamingInputCall" + ] + } + } + ] + }`, + wantStatus: status.New(codes.OK, ""), + }, + "DeniesRPCEmptyDenyNoMatchInAllow": { + authzPolicy: `{ + "name": "authz", + "allow_rules": + [ + { + "name": "allow_StreamingOutputCall", + "request": + { + "paths": + [ + "/grpc.testing.TestService/StreamingOutputCall" + ] + } + } + ] + }`, + wantStatus: status.New(codes.PermissionDenied, "unauthorized RPC request rejected"), + }, + "DeniesRPCRequestWithPrincipalsFieldOnUnauthenticatedConnection": { + authzPolicy: `{ + "name": "authz", + "allow_rules": + [ + { + "name": "allow_authenticated", + "source": { + "principals": ["*", ""] + } + } + ] + }`, + wantStatus: status.New(codes.PermissionDenied, "unauthorized RPC request rejected"), + }, + "DeniesRPCRequestNoMatchInAllowFailsPresenceMatch": { + authzPolicy: `{ + "name": "authz", + "allow_rules": + [ + { + "name": "allow_TestServiceCalls", + "request": { + "paths": + [ + "/grpc.testing.TestService/*" + ], + "headers": + [ + { + "key": "key-abc", + "values": + [ + "*" + ] + } + ] + } + } + ] + }`, + md: metadata.Pairs("key-abc", ""), + wantStatus: status.New(codes.PermissionDenied, "unauthorized RPC request rejected"), + }, +} + +func (s) TestStaticPolicyEnd2End(t *testing.T) { + for name, test := range authzTests { + t.Run(name, func(t *testing.T) { + // Start a gRPC server with gRPC authz unary and stream server interceptors. + i, _ := authz.NewStatic(test.authzPolicy) + s := grpc.NewServer( + grpc.ChainUnaryInterceptor(i.UnaryInterceptor), + grpc.ChainStreamInterceptor(i.StreamInterceptor)) + defer s.Stop() + testgrpc.RegisterTestServiceServer(s, &testServer{}) + + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("error listening: %v", err) + } + go s.Serve(lis) + + // Establish a connection to the server. + clientConn, err := grpc.Dial(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.Dial(%v) failed: %v", lis.Addr().String(), err) + } + defer clientConn.Close() + client := testgrpc.NewTestServiceClient(clientConn) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + ctx = metadata.NewOutgoingContext(ctx, test.md) + + // Verifying authorization decision for Unary RPC. + _, err = client.UnaryCall(ctx, &testpb.SimpleRequest{}) + if got := status.Convert(err); got.Code() != test.wantStatus.Code() || got.Message() != test.wantStatus.Message() { + t.Fatalf("[UnaryCall] error want:{%v} got:{%v}", test.wantStatus.Err(), got.Err()) + } + + // Verifying authorization decision for Streaming RPC. + stream, err := client.StreamingInputCall(ctx) + if err != nil { + t.Fatalf("failed StreamingInputCall err: %v", err) + } + req := &testpb.StreamingInputCallRequest{ + Payload: &testpb.Payload{ + Body: []byte("hi"), + }, + } + if err := stream.Send(req); err != nil && err != io.EOF { + t.Fatalf("failed stream.Send err: %v", err) + } + _, err = stream.CloseAndRecv() + if got := status.Convert(err); got.Code() != test.wantStatus.Code() || got.Message() != test.wantStatus.Message() { + t.Fatalf("[StreamingCall] error want:{%v} got:{%v}", test.wantStatus.Err(), got.Err()) + } + }) + } +} + +func (s) TestAllowsRPCRequestWithPrincipalsFieldOnTLSAuthenticatedConnection(t *testing.T) { + authzPolicy := `{ + "name": "authz", + "allow_rules": + [ + { + "name": "allow_authenticated", + "source": { + "principals": ["*", ""] + } + } + ] + }` + // Start a gRPC server with gRPC authz unary server interceptor. + i, _ := authz.NewStatic(authzPolicy) + creds, err := credentials.NewServerTLSFromFile(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem")) + if err != nil { + t.Fatalf("failed to generate credentials: %v", err) + } + s := grpc.NewServer( + grpc.Creds(creds), + grpc.ChainUnaryInterceptor(i.UnaryInterceptor)) + defer s.Stop() + testgrpc.RegisterTestServiceServer(s, &testServer{}) + + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("error listening: %v", err) + } + go s.Serve(lis) + + // Establish a connection to the server. + creds, err = credentials.NewClientTLSFromFile(testdata.Path("x509/server_ca_cert.pem"), "x.test.example.com") + if err != nil { + t.Fatalf("failed to load credentials: %v", err) + } + clientConn, err := grpc.Dial(lis.Addr().String(), grpc.WithTransportCredentials(creds)) + if err != nil { + t.Fatalf("grpc.Dial(%v) failed: %v", lis.Addr().String(), err) + } + defer clientConn.Close() + client := testgrpc.NewTestServiceClient(clientConn) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + // Verifying authorization decision. + if _, err = client.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil { + t.Fatalf("client.UnaryCall(_, _) = %v; want nil", err) + } +} + +func (s) TestAllowsRPCRequestWithPrincipalsFieldOnMTLSAuthenticatedConnection(t *testing.T) { + authzPolicy := `{ + "name": "authz", + "allow_rules": + [ + { + "name": "allow_authenticated", + "source": { + "principals": ["*", ""] + } + } + ] + }` + // Start a gRPC server with gRPC authz unary server interceptor. + i, _ := authz.NewStatic(authzPolicy) + cert, err := tls.LoadX509KeyPair(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem")) + if err != nil { + t.Fatalf("tls.LoadX509KeyPair(x509/server1_cert.pem, x509/server1_key.pem) failed: %v", err) + } + ca, err := os.ReadFile(testdata.Path("x509/client_ca_cert.pem")) + if err != nil { + t.Fatalf("os.ReadFile(x509/client_ca_cert.pem) failed: %v", err) + } + certPool := x509.NewCertPool() + if !certPool.AppendCertsFromPEM(ca) { + t.Fatal("failed to append certificates") + } + creds := credentials.NewTLS(&tls.Config{ + ClientAuth: tls.RequireAndVerifyClientCert, + Certificates: []tls.Certificate{cert}, + ClientCAs: certPool, + }) + s := grpc.NewServer( + grpc.Creds(creds), + grpc.ChainUnaryInterceptor(i.UnaryInterceptor)) + defer s.Stop() + testgrpc.RegisterTestServiceServer(s, &testServer{}) + + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("error listening: %v", err) + } + go s.Serve(lis) + + // Establish a connection to the server. + cert, err = tls.LoadX509KeyPair(testdata.Path("x509/client1_cert.pem"), testdata.Path("x509/client1_key.pem")) + if err != nil { + t.Fatalf("tls.LoadX509KeyPair(x509/client1_cert.pem, x509/client1_key.pem) failed: %v", err) + } + ca, err = os.ReadFile(testdata.Path("x509/server_ca_cert.pem")) + if err != nil { + t.Fatalf("os.ReadFile(x509/server_ca_cert.pem) failed: %v", err) + } + roots := x509.NewCertPool() + if !roots.AppendCertsFromPEM(ca) { + t.Fatal("failed to append certificates") + } + creds = credentials.NewTLS(&tls.Config{ + Certificates: []tls.Certificate{cert}, + RootCAs: roots, + ServerName: "x.test.example.com", + }) + clientConn, err := grpc.Dial(lis.Addr().String(), grpc.WithTransportCredentials(creds)) + if err != nil { + t.Fatalf("grpc.Dial(%v) failed: %v", lis.Addr().String(), err) + } + defer clientConn.Close() + client := testgrpc.NewTestServiceClient(clientConn) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + // Verifying authorization decision. + if _, err = client.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil { + t.Fatalf("client.UnaryCall(_, _) = %v; want nil", err) + } +} + +func (s) TestFileWatcherEnd2End(t *testing.T) { + for name, test := range authzTests { + t.Run(name, func(t *testing.T) { + file := createTmpPolicyFile(t, name, []byte(test.authzPolicy)) + i, _ := authz.NewFileWatcher(file, 1*time.Second) + defer i.Close() + + // Start a gRPC server with gRPC authz unary and stream server interceptors. + s := grpc.NewServer( + grpc.ChainUnaryInterceptor(i.UnaryInterceptor), + grpc.ChainStreamInterceptor(i.StreamInterceptor)) + defer s.Stop() + testgrpc.RegisterTestServiceServer(s, &testServer{}) + + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("error listening: %v", err) + } + defer lis.Close() + go s.Serve(lis) + + // Establish a connection to the server. + clientConn, err := grpc.Dial(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.Dial(%v) failed: %v", lis.Addr().String(), err) + } + defer clientConn.Close() + client := testgrpc.NewTestServiceClient(clientConn) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + ctx = metadata.NewOutgoingContext(ctx, test.md) + + // Verifying authorization decision for Unary RPC. + _, err = client.UnaryCall(ctx, &testpb.SimpleRequest{}) + if got := status.Convert(err); got.Code() != test.wantStatus.Code() || got.Message() != test.wantStatus.Message() { + t.Fatalf("[UnaryCall] error want:{%v} got:{%v}", test.wantStatus.Err(), got.Err()) + } + + // Verifying authorization decision for Streaming RPC. + stream, err := client.StreamingInputCall(ctx) + if err != nil { + t.Fatalf("failed StreamingInputCall err: %v", err) + } + req := &testpb.StreamingInputCallRequest{ + Payload: &testpb.Payload{ + Body: []byte("hi"), + }, + } + if err := stream.Send(req); err != nil && err != io.EOF { + t.Fatalf("failed stream.Send err: %v", err) + } + _, err = stream.CloseAndRecv() + if got := status.Convert(err); got.Code() != test.wantStatus.Code() || got.Message() != test.wantStatus.Message() { + t.Fatalf("[StreamingCall] error want:{%v} got:{%v}", test.wantStatus.Err(), got.Err()) + } + }) + } +} + +func retryUntil(ctx context.Context, tsc testgrpc.TestServiceClient, want *status.Status) (lastErr error) { + for ctx.Err() == nil { + _, lastErr = tsc.UnaryCall(ctx, &testpb.SimpleRequest{}) + if s := status.Convert(lastErr); s.Code() == want.Code() && s.Message() == want.Message() { + return nil + } + time.Sleep(20 * time.Millisecond) + } + return lastErr +} + +func (s) TestFileWatcher_ValidPolicyRefresh(t *testing.T) { + valid1 := authzTests["DeniesRPCMatchInDenyAndAllow"] + file := createTmpPolicyFile(t, "valid_policy_refresh", []byte(valid1.authzPolicy)) + i, _ := authz.NewFileWatcher(file, 100*time.Millisecond) + defer i.Close() + + // Start a gRPC server with gRPC authz unary server interceptor. + s := grpc.NewServer( + grpc.ChainUnaryInterceptor(i.UnaryInterceptor)) + defer s.Stop() + testgrpc.RegisterTestServiceServer(s, &testServer{}) + + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("error listening: %v", err) + } + defer lis.Close() + go s.Serve(lis) + + // Establish a connection to the server. + clientConn, err := grpc.Dial(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.Dial(%v) failed: %v", lis.Addr().String(), err) + } + defer clientConn.Close() + client := testgrpc.NewTestServiceClient(clientConn) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + // Verifying authorization decision. + _, err = client.UnaryCall(ctx, &testpb.SimpleRequest{}) + if got := status.Convert(err); got.Code() != valid1.wantStatus.Code() || got.Message() != valid1.wantStatus.Message() { + t.Fatalf("client.UnaryCall(_, _) = %v; want = %v", got.Err(), valid1.wantStatus.Err()) + } + + // Rewrite the file with a different valid authorization policy. + valid2 := authzTests["AllowsRPCEmptyDenyMatchInAllow"] + if err := os.WriteFile(file, []byte(valid2.authzPolicy), os.ModePerm); err != nil { + t.Fatalf("os.WriteFile(%q) failed: %v", file, err) + } + + // Verifying authorization decision. + if got := retryUntil(ctx, client, valid2.wantStatus); got != nil { + t.Fatalf("client.UnaryCall(_, _) = %v; want = %v", got, valid2.wantStatus.Err()) + } +} + +func (s) TestFileWatcher_InvalidPolicySkipReload(t *testing.T) { + valid := authzTests["DeniesRPCMatchInDenyAndAllow"] + file := createTmpPolicyFile(t, "invalid_policy_skip_reload", []byte(valid.authzPolicy)) + i, _ := authz.NewFileWatcher(file, 20*time.Millisecond) + defer i.Close() + + // Start a gRPC server with gRPC authz unary server interceptors. + s := grpc.NewServer( + grpc.ChainUnaryInterceptor(i.UnaryInterceptor)) + defer s.Stop() + testgrpc.RegisterTestServiceServer(s, &testServer{}) + + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("error listening: %v", err) + } + defer lis.Close() + go s.Serve(lis) + + // Establish a connection to the server. + clientConn, err := grpc.Dial(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.Dial(%v) failed: %v", lis.Addr().String(), err) + } + defer clientConn.Close() + client := testgrpc.NewTestServiceClient(clientConn) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + // Verifying authorization decision. + _, err = client.UnaryCall(ctx, &testpb.SimpleRequest{}) + if got := status.Convert(err); got.Code() != valid.wantStatus.Code() || got.Message() != valid.wantStatus.Message() { + t.Fatalf("client.UnaryCall(_, _) = %v; want = %v", got.Err(), valid.wantStatus.Err()) + } + + // Skips the invalid policy update, and continues to use the valid policy. + if err := os.WriteFile(file, []byte("{}"), os.ModePerm); err != nil { + t.Fatalf("os.WriteFile(%q) failed: %v", file, err) + } + + // Wait 40 ms for background go routine to read updated files. + time.Sleep(40 * time.Millisecond) + + // Verifying authorization decision. + _, err = client.UnaryCall(ctx, &testpb.SimpleRequest{}) + if got := status.Convert(err); got.Code() != valid.wantStatus.Code() || got.Message() != valid.wantStatus.Message() { + t.Fatalf("client.UnaryCall(_, _) = %v; want = %v", got.Err(), valid.wantStatus.Err()) + } +} + +func (s) TestFileWatcher_RecoversFromReloadFailure(t *testing.T) { + valid1 := authzTests["DeniesRPCMatchInDenyAndAllow"] + file := createTmpPolicyFile(t, "recovers_from_reload_failure", []byte(valid1.authzPolicy)) + i, _ := authz.NewFileWatcher(file, 100*time.Millisecond) + defer i.Close() + + // Start a gRPC server with gRPC authz unary server interceptors. + s := grpc.NewServer( + grpc.ChainUnaryInterceptor(i.UnaryInterceptor)) + defer s.Stop() + testgrpc.RegisterTestServiceServer(s, &testServer{}) + + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("error listening: %v", err) + } + defer lis.Close() + go s.Serve(lis) + + // Establish a connection to the server. + clientConn, err := grpc.Dial(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.Dial(%v) failed: %v", lis.Addr().String(), err) + } + defer clientConn.Close() + client := testgrpc.NewTestServiceClient(clientConn) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + // Verifying authorization decision. + _, err = client.UnaryCall(ctx, &testpb.SimpleRequest{}) + if got := status.Convert(err); got.Code() != valid1.wantStatus.Code() || got.Message() != valid1.wantStatus.Message() { + t.Fatalf("client.UnaryCall(_, _) = %v; want = %v", got.Err(), valid1.wantStatus.Err()) + } + + // Skips the invalid policy update, and continues to use the valid policy. + if err := os.WriteFile(file, []byte("{}"), os.ModePerm); err != nil { + t.Fatalf("os.WriteFile(%q) failed: %v", file, err) + } + + // Wait 120 ms for background go routine to read updated files. + time.Sleep(120 * time.Millisecond) + + // Verifying authorization decision. + _, err = client.UnaryCall(ctx, &testpb.SimpleRequest{}) + if got := status.Convert(err); got.Code() != valid1.wantStatus.Code() || got.Message() != valid1.wantStatus.Message() { + t.Fatalf("client.UnaryCall(_, _) = %v; want = %v", got.Err(), valid1.wantStatus.Err()) + } + + // Rewrite the file with a different valid authorization policy. + valid2 := authzTests["AllowsRPCEmptyDenyMatchInAllow"] + if err := os.WriteFile(file, []byte(valid2.authzPolicy), os.ModePerm); err != nil { + t.Fatalf("os.WriteFile(%q) failed: %v", file, err) + } + + // Verifying authorization decision. + if got := retryUntil(ctx, client, valid2.wantStatus); got != nil { + t.Fatalf("client.UnaryCall(_, _) = %v; want = %v", got, valid2.wantStatus.Err()) + } +} diff --git a/authz/grpc_authz_server_interceptors.go b/authz/grpc_authz_server_interceptors.go new file mode 100644 index 000000000000..8dfc373fdb66 --- /dev/null +++ b/authz/grpc_authz_server_interceptors.go @@ -0,0 +1,178 @@ +/* + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package authz + +import ( + "bytes" + "context" + "fmt" + "os" + "sync/atomic" + "time" + "unsafe" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal/xds/rbac" + "google.golang.org/grpc/status" +) + +var logger = grpclog.Component("authz") + +// StaticInterceptor contains engines used to make authorization decisions. It +// either contains two engines deny engine followed by an allow engine or only +// one allow engine. +type StaticInterceptor struct { + engines rbac.ChainEngine +} + +// NewStatic returns a new StaticInterceptor from a static authorization policy +// JSON string. +func NewStatic(authzPolicy string) (*StaticInterceptor, error) { + rbacs, policyName, err := translatePolicy(authzPolicy) + if err != nil { + return nil, err + } + chainEngine, err := rbac.NewChainEngine(rbacs, policyName) + if err != nil { + return nil, err + } + return &StaticInterceptor{*chainEngine}, nil +} + +// UnaryInterceptor intercepts incoming Unary RPC requests. +// Only authorized requests are allowed to pass. Otherwise, an unauthorized +// error is returned to the client. +func (i *StaticInterceptor) UnaryInterceptor(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + err := i.engines.IsAuthorized(ctx) + if err != nil { + if status.Code(err) == codes.PermissionDenied { + if logger.V(2) { + logger.Infof("unauthorized RPC request rejected: %v", err) + } + return nil, status.Errorf(codes.PermissionDenied, "unauthorized RPC request rejected") + } + return nil, err + } + return handler(ctx, req) +} + +// StreamInterceptor intercepts incoming Stream RPC requests. +// Only authorized requests are allowed to pass. Otherwise, an unauthorized +// error is returned to the client. +func (i *StaticInterceptor) StreamInterceptor(srv any, ss grpc.ServerStream, _ *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + err := i.engines.IsAuthorized(ss.Context()) + if err != nil { + if status.Code(err) == codes.PermissionDenied { + if logger.V(2) { + logger.Infof("unauthorized RPC request rejected: %v", err) + } + return status.Errorf(codes.PermissionDenied, "unauthorized RPC request rejected") + } + return err + } + return handler(srv, ss) +} + +// FileWatcherInterceptor contains details used to make authorization decisions +// by watching a file path that contains authorization policy in JSON format. +type FileWatcherInterceptor struct { + internalInterceptor unsafe.Pointer // *StaticInterceptor + policyFile string + policyContents []byte + refreshDuration time.Duration + cancel context.CancelFunc +} + +// NewFileWatcher returns a new FileWatcherInterceptor from a policy file +// that contains JSON string of authorization policy and a refresh duration to +// specify the amount of time between policy refreshes. +func NewFileWatcher(file string, duration time.Duration) (*FileWatcherInterceptor, error) { + if file == "" { + return nil, fmt.Errorf("authorization policy file path is empty") + } + if duration <= time.Duration(0) { + return nil, fmt.Errorf("requires refresh interval(%v) greater than 0s", duration) + } + i := &FileWatcherInterceptor{policyFile: file, refreshDuration: duration} + if err := i.updateInternalInterceptor(); err != nil { + return nil, err + } + ctx, cancel := context.WithCancel(context.Background()) + i.cancel = cancel + // Create a background go routine for policy refresh. + go i.run(ctx) + return i, nil +} + +func (i *FileWatcherInterceptor) run(ctx context.Context) { + ticker := time.NewTicker(i.refreshDuration) + for { + if err := i.updateInternalInterceptor(); err != nil { + logger.Warningf("authorization policy reload status err: %v", err) + } + select { + case <-ctx.Done(): + ticker.Stop() + return + case <-ticker.C: + } + } +} + +// updateInternalInterceptor checks if the policy file that is watching has changed, +// and if so, updates the internalInterceptor with the policy. Unlike the +// constructor, if there is an error in reading the file or parsing the policy, the +// previous internalInterceptors will not be replaced. +func (i *FileWatcherInterceptor) updateInternalInterceptor() error { + policyContents, err := os.ReadFile(i.policyFile) + if err != nil { + return fmt.Errorf("policyFile(%s) read failed: %v", i.policyFile, err) + } + if bytes.Equal(i.policyContents, policyContents) { + return nil + } + i.policyContents = policyContents + policyContentsString := string(policyContents) + interceptor, err := NewStatic(policyContentsString) + if err != nil { + return err + } + atomic.StorePointer(&i.internalInterceptor, unsafe.Pointer(interceptor)) + logger.Infof("authorization policy reload status: successfully loaded new policy %v", policyContentsString) + return nil +} + +// Close cleans up resources allocated by the interceptor. +func (i *FileWatcherInterceptor) Close() { + i.cancel() +} + +// UnaryInterceptor intercepts incoming Unary RPC requests. +// Only authorized requests are allowed to pass. Otherwise, an unauthorized +// error is returned to the client. +func (i *FileWatcherInterceptor) UnaryInterceptor(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + return ((*StaticInterceptor)(atomic.LoadPointer(&i.internalInterceptor))).UnaryInterceptor(ctx, req, info, handler) +} + +// StreamInterceptor intercepts incoming Stream RPC requests. +// Only authorized requests are allowed to pass. Otherwise, an unauthorized +// error is returned to the client. +func (i *FileWatcherInterceptor) StreamInterceptor(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + return ((*StaticInterceptor)(atomic.LoadPointer(&i.internalInterceptor))).StreamInterceptor(srv, ss, info, handler) +} diff --git a/authz/grpc_authz_server_interceptors_test.go b/authz/grpc_authz_server_interceptors_test.go new file mode 100644 index 000000000000..ae74c896d960 --- /dev/null +++ b/authz/grpc_authz_server_interceptors_test.go @@ -0,0 +1,120 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package authz_test + +import ( + "fmt" + "os" + "path" + "testing" + "time" + + "google.golang.org/grpc/authz" +) + +func createTmpPolicyFile(t *testing.T, dirSuffix string, policy []byte) string { + t.Helper() + + // Create a temp directory. Passing an empty string for the first argument + // uses the system temp directory. + dir, err := os.MkdirTemp("", dirSuffix) + if err != nil { + t.Fatalf("os.MkdirTemp() failed: %v", err) + } + t.Logf("Using tmpdir: %s", dir) + // Write policy into file. + filename := path.Join(dir, "policy.json") + if err := os.WriteFile(filename, policy, os.ModePerm); err != nil { + t.Fatalf("os.WriteFile(%q) failed: %v", filename, err) + } + t.Logf("Wrote policy %s to file at %s", string(policy), filename) + return filename +} + +func (s) TestNewStatic(t *testing.T) { + tests := map[string]struct { + authzPolicy string + wantErr error + }{ + "InvalidPolicyFailsToCreateInterceptor": { + authzPolicy: `{}`, + wantErr: fmt.Errorf(`"name" is not present`), + }, + "ValidPolicyCreatesInterceptor": { + authzPolicy: `{ + "name": "authz", + "allow_rules": + [ + { + "name": "allow_all" + } + ] + }`, + }, + } + for name, test := range tests { + t.Run(name, func(t *testing.T) { + if _, err := authz.NewStatic(test.authzPolicy); fmt.Sprint(err) != fmt.Sprint(test.wantErr) { + t.Fatalf("NewStatic(%v) returned err: %v, want err: %v", test.authzPolicy, err, test.wantErr) + } + }) + } +} + +func (s) TestNewFileWatcher(t *testing.T) { + tests := map[string]struct { + authzPolicy string + refreshDuration time.Duration + wantErr error + }{ + "InvalidRefreshDurationFailsToCreateInterceptor": { + refreshDuration: time.Duration(0), + wantErr: fmt.Errorf("requires refresh interval(0s) greater than 0s"), + }, + "InvalidPolicyFailsToCreateInterceptor": { + authzPolicy: `{}`, + refreshDuration: time.Duration(1), + wantErr: fmt.Errorf(`"name" is not present`), + }, + "ValidPolicyCreatesInterceptor": { + authzPolicy: `{ + "name": "authz", + "allow_rules": + [ + { + "name": "allow_all" + } + ] + }`, + refreshDuration: time.Duration(1), + }, + } + for name, test := range tests { + t.Run(name, func(t *testing.T) { + file := createTmpPolicyFile(t, name, []byte(test.authzPolicy)) + i, err := authz.NewFileWatcher(file, test.refreshDuration) + if fmt.Sprint(err) != fmt.Sprint(test.wantErr) { + t.Fatalf("NewFileWatcher(%v) returned err: %v, want err: %v", test.authzPolicy, err, test.wantErr) + } + if i != nil { + i.Close() + } + }) + } +} diff --git a/authz/rbac_translator.go b/authz/rbac_translator.go new file mode 100644 index 000000000000..730ec9dc426a --- /dev/null +++ b/authz/rbac_translator.go @@ -0,0 +1,398 @@ +/* + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package authz exposes methods to manage authorization within gRPC. +// +// # Experimental +// +// Notice: This package is EXPERIMENTAL and may be changed or removed +// in a later release. +package authz + +import ( + "bytes" + "encoding/json" + "fmt" + "strings" + + v1xdsudpatypepb "github.com/cncf/xds/go/udpa/type/v1" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3rbacpb "github.com/envoyproxy/go-control-plane/envoy/config/rbac/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/structpb" +) + +// This is used when converting a custom config from raw JSON to a TypedStruct +// The TypeURL of the TypeStruct will be "grpc.authz.audit_logging/" +const typeURLPrefix = "grpc.authz.audit_logging/" + +type header struct { + Key string + Values []string +} + +type peer struct { + Principals []string +} + +type request struct { + Paths []string + Headers []header +} + +type rule struct { + Name string + Source peer + Request request +} + +type auditLogger struct { + Name string `json:"name"` + Config *structpb.Struct `json:"config"` + IsOptional bool `json:"is_optional"` +} + +type auditLoggingOptions struct { + AuditCondition string `json:"audit_condition"` + AuditLoggers []*auditLogger `json:"audit_loggers"` +} + +// Represents the SDK authorization policy provided by user. +type authorizationPolicy struct { + Name string + DenyRules []rule `json:"deny_rules"` + AllowRules []rule `json:"allow_rules"` + AuditLoggingOptions auditLoggingOptions `json:"audit_logging_options"` +} + +func principalOr(principals []*v3rbacpb.Principal) *v3rbacpb.Principal { + return &v3rbacpb.Principal{ + Identifier: &v3rbacpb.Principal_OrIds{ + OrIds: &v3rbacpb.Principal_Set{ + Ids: principals, + }, + }, + } +} + +func permissionOr(permission []*v3rbacpb.Permission) *v3rbacpb.Permission { + return &v3rbacpb.Permission{ + Rule: &v3rbacpb.Permission_OrRules{ + OrRules: &v3rbacpb.Permission_Set{ + Rules: permission, + }, + }, + } +} + +func permissionAnd(permission []*v3rbacpb.Permission) *v3rbacpb.Permission { + return &v3rbacpb.Permission{ + Rule: &v3rbacpb.Permission_AndRules{ + AndRules: &v3rbacpb.Permission_Set{ + Rules: permission, + }, + }, + } +} + +func getStringMatcher(value string) *v3matcherpb.StringMatcher { + switch { + case value == "*": + return &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_SafeRegex{ + SafeRegex: &v3matcherpb.RegexMatcher{Regex: ".+"}}, + } + case strings.HasSuffix(value, "*"): + prefix := strings.TrimSuffix(value, "*") + return &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: prefix}, + } + case strings.HasPrefix(value, "*"): + suffix := strings.TrimPrefix(value, "*") + return &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Suffix{Suffix: suffix}, + } + default: + return &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: value}, + } + } +} + +func getHeaderMatcher(key, value string) *v3routepb.HeaderMatcher { + switch { + case value == "*": + return &v3routepb.HeaderMatcher{ + Name: key, + HeaderMatchSpecifier: &v3routepb.HeaderMatcher_SafeRegexMatch{ + SafeRegexMatch: &v3matcherpb.RegexMatcher{Regex: ".+"}}, + } + case strings.HasSuffix(value, "*"): + prefix := strings.TrimSuffix(value, "*") + return &v3routepb.HeaderMatcher{ + Name: key, + HeaderMatchSpecifier: &v3routepb.HeaderMatcher_PrefixMatch{PrefixMatch: prefix}, + } + case strings.HasPrefix(value, "*"): + suffix := strings.TrimPrefix(value, "*") + return &v3routepb.HeaderMatcher{ + Name: key, + HeaderMatchSpecifier: &v3routepb.HeaderMatcher_SuffixMatch{SuffixMatch: suffix}, + } + default: + return &v3routepb.HeaderMatcher{ + Name: key, + HeaderMatchSpecifier: &v3routepb.HeaderMatcher_ExactMatch{ExactMatch: value}, + } + } +} + +func parsePrincipalNames(principalNames []string) []*v3rbacpb.Principal { + ps := make([]*v3rbacpb.Principal, 0, len(principalNames)) + for _, principalName := range principalNames { + newPrincipalName := &v3rbacpb.Principal{ + Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{ + PrincipalName: getStringMatcher(principalName), + }, + }} + ps = append(ps, newPrincipalName) + } + return ps +} + +func parsePeer(source peer) *v3rbacpb.Principal { + if len(source.Principals) == 0 { + return &v3rbacpb.Principal{ + Identifier: &v3rbacpb.Principal_Any{ + Any: true, + }, + } + } + return principalOr(parsePrincipalNames(source.Principals)) +} + +func parsePaths(paths []string) []*v3rbacpb.Permission { + ps := make([]*v3rbacpb.Permission, 0, len(paths)) + for _, path := range paths { + newPath := &v3rbacpb.Permission{ + Rule: &v3rbacpb.Permission_UrlPath{ + UrlPath: &v3matcherpb.PathMatcher{ + Rule: &v3matcherpb.PathMatcher_Path{Path: getStringMatcher(path)}}}} + ps = append(ps, newPath) + } + return ps +} + +func parseHeaderValues(key string, values []string) []*v3rbacpb.Permission { + vs := make([]*v3rbacpb.Permission, 0, len(values)) + for _, value := range values { + newHeader := &v3rbacpb.Permission{ + Rule: &v3rbacpb.Permission_Header{ + Header: getHeaderMatcher(key, value)}} + vs = append(vs, newHeader) + } + return vs +} + +var unsupportedHeaders = map[string]bool{ + "host": true, + "connection": true, + "keep-alive": true, + "proxy-authenticate": true, + "proxy-authorization": true, + "te": true, + "trailer": true, + "transfer-encoding": true, + "upgrade": true, +} + +func unsupportedHeader(key string) bool { + return key[0] == ':' || strings.HasPrefix(key, "grpc-") || unsupportedHeaders[key] +} + +func parseHeaders(headers []header) ([]*v3rbacpb.Permission, error) { + hs := make([]*v3rbacpb.Permission, 0, len(headers)) + for i, header := range headers { + if header.Key == "" { + return nil, fmt.Errorf(`"headers" %d: "key" is not present`, i) + } + header.Key = strings.ToLower(header.Key) + if unsupportedHeader(header.Key) { + return nil, fmt.Errorf(`"headers" %d: unsupported "key" %s`, i, header.Key) + } + if len(header.Values) == 0 { + return nil, fmt.Errorf(`"headers" %d: "values" is not present`, i) + } + values := parseHeaderValues(header.Key, header.Values) + hs = append(hs, permissionOr(values)) + } + return hs, nil +} + +func parseRequest(request request) (*v3rbacpb.Permission, error) { + var and []*v3rbacpb.Permission + if len(request.Paths) > 0 { + and = append(and, permissionOr(parsePaths(request.Paths))) + } + if len(request.Headers) > 0 { + headers, err := parseHeaders(request.Headers) + if err != nil { + return nil, err + } + and = append(and, permissionAnd(headers)) + } + if len(and) > 0 { + return permissionAnd(and), nil + } + return &v3rbacpb.Permission{ + Rule: &v3rbacpb.Permission_Any{ + Any: true, + }, + }, nil +} + +func parseRules(rules []rule, prefixName string) (map[string]*v3rbacpb.Policy, error) { + policies := make(map[string]*v3rbacpb.Policy) + for i, rule := range rules { + if rule.Name == "" { + return policies, fmt.Errorf(`%d: "name" is not present`, i) + } + permission, err := parseRequest(rule.Request) + if err != nil { + return nil, fmt.Errorf("%d: %v", i, err) + } + policyName := prefixName + "_" + rule.Name + policies[policyName] = &v3rbacpb.Policy{ + Principals: []*v3rbacpb.Principal{parsePeer(rule.Source)}, + Permissions: []*v3rbacpb.Permission{permission}, + } + } + return policies, nil +} + +// Parse auditLoggingOptions to the associated RBAC protos. The single +// auditLoggingOptions results in two different parsed protos, one for the allow +// policy and one for the deny policy +func (options *auditLoggingOptions) toProtos() (allow *v3rbacpb.RBAC_AuditLoggingOptions, deny *v3rbacpb.RBAC_AuditLoggingOptions, err error) { + allow = &v3rbacpb.RBAC_AuditLoggingOptions{} + deny = &v3rbacpb.RBAC_AuditLoggingOptions{} + + if options.AuditCondition != "" { + rbacCondition, ok := v3rbacpb.RBAC_AuditLoggingOptions_AuditCondition_value[options.AuditCondition] + if !ok { + return nil, nil, fmt.Errorf("failed to parse AuditCondition %v. Allowed values {NONE, ON_DENY, ON_ALLOW, ON_DENY_AND_ALLOW}", options.AuditCondition) + } + allow.AuditCondition = v3rbacpb.RBAC_AuditLoggingOptions_AuditCondition(rbacCondition) + deny.AuditCondition = toDenyCondition(v3rbacpb.RBAC_AuditLoggingOptions_AuditCondition(rbacCondition)) + } + + for i, config := range options.AuditLoggers { + if config.Name == "" { + return nil, nil, fmt.Errorf("missing required field: name in audit_logging_options.audit_loggers[%v]", i) + } + if config.Config == nil { + config.Config = &structpb.Struct{} + } + typedStruct := &v1xdsudpatypepb.TypedStruct{ + TypeUrl: typeURLPrefix + config.Name, + Value: config.Config, + } + customConfig, err := anypb.New(typedStruct) + if err != nil { + return nil, nil, fmt.Errorf("error parsing custom audit logger config: %v", err) + } + + logger := &v3corepb.TypedExtensionConfig{Name: config.Name, TypedConfig: customConfig} + rbacConfig := v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + IsOptional: config.IsOptional, + AuditLogger: logger, + } + allow.LoggerConfigs = append(allow.LoggerConfigs, &rbacConfig) + deny.LoggerConfigs = append(deny.LoggerConfigs, &rbacConfig) + } + + return allow, deny, nil +} + +// Maps the AuditCondition coming from AuditLoggingOptions to the proper +// condition for the deny policy RBAC proto +func toDenyCondition(condition v3rbacpb.RBAC_AuditLoggingOptions_AuditCondition) v3rbacpb.RBAC_AuditLoggingOptions_AuditCondition { + // Mapping the overall policy AuditCondition to what it must be for the Deny and Allow RBAC + // See gRPC A59 for details - https://github.com/grpc/proposal/pull/346/files + // |Authorization Policy |DENY RBAC |ALLOW RBAC | + // |----------------------|-------------------|---------------------| + // |NONE |NONE |NONE | + // |ON_DENY |ON_DENY |ON_DENY | + // |ON_ALLOW |NONE |ON_ALLOW | + // |ON_DENY_AND_ALLOW |ON_DENY |ON_DENY_AND_ALLOW | + switch condition { + case v3rbacpb.RBAC_AuditLoggingOptions_NONE: + return v3rbacpb.RBAC_AuditLoggingOptions_NONE + case v3rbacpb.RBAC_AuditLoggingOptions_ON_DENY: + return v3rbacpb.RBAC_AuditLoggingOptions_ON_DENY + case v3rbacpb.RBAC_AuditLoggingOptions_ON_ALLOW: + return v3rbacpb.RBAC_AuditLoggingOptions_NONE + case v3rbacpb.RBAC_AuditLoggingOptions_ON_DENY_AND_ALLOW: + return v3rbacpb.RBAC_AuditLoggingOptions_ON_DENY + default: + return v3rbacpb.RBAC_AuditLoggingOptions_NONE + } +} + +// translatePolicy translates SDK authorization policy in JSON format to two +// Envoy RBAC polices (deny followed by allow policy) or only one Envoy RBAC +// allow policy. Also returns the overall policy name. If the input policy +// cannot be parsed or is invalid, an error will be returned. +func translatePolicy(policyStr string) ([]*v3rbacpb.RBAC, string, error) { + policy := &authorizationPolicy{} + d := json.NewDecoder(bytes.NewReader([]byte(policyStr))) + d.DisallowUnknownFields() + if err := d.Decode(policy); err != nil { + return nil, "", fmt.Errorf("failed to unmarshal policy: %v", err) + } + if policy.Name == "" { + return nil, "", fmt.Errorf(`"name" is not present`) + } + if len(policy.AllowRules) == 0 { + return nil, "", fmt.Errorf(`"allow_rules" is not present`) + } + allowLogger, denyLogger, err := policy.AuditLoggingOptions.toProtos() + if err != nil { + return nil, "", err + } + rbacs := make([]*v3rbacpb.RBAC, 0, 2) + if len(policy.DenyRules) > 0 { + denyPolicies, err := parseRules(policy.DenyRules, policy.Name) + if err != nil { + return nil, "", fmt.Errorf(`"deny_rules" %v`, err) + } + denyRBAC := &v3rbacpb.RBAC{ + Action: v3rbacpb.RBAC_DENY, + Policies: denyPolicies, + AuditLoggingOptions: denyLogger, + } + rbacs = append(rbacs, denyRBAC) + } + allowPolicies, err := parseRules(policy.AllowRules, policy.Name) + if err != nil { + return nil, "", fmt.Errorf(`"allow_rules" %v`, err) + } + allowRBAC := &v3rbacpb.RBAC{Action: v3rbacpb.RBAC_ALLOW, Policies: allowPolicies, AuditLoggingOptions: allowLogger} + return append(rbacs, allowRBAC), policy.Name, nil +} diff --git a/authz/rbac_translator_test.go b/authz/rbac_translator_test.go new file mode 100644 index 000000000000..0de0d503252e --- /dev/null +++ b/authz/rbac_translator_test.go @@ -0,0 +1,1052 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package authz + +import ( + "strings" + "testing" + + v1xdsudpatypepb "github.com/cncf/xds/go/udpa/type/v1" + "github.com/google/go-cmp/cmp" + "google.golang.org/protobuf/testing/protocmp" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/structpb" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3rbacpb "github.com/envoyproxy/go-control-plane/envoy/config/rbac/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3" +) + +func TestTranslatePolicy(t *testing.T) { + tests := map[string]struct { + authzPolicy string + wantErr string + wantPolicies []*v3rbacpb.RBAC + wantPolicyName string + }{ + "valid policy": { + authzPolicy: `{ + "name": "authz", + "deny_rules": [ + { + "name": "deny_policy_1", + "source": { + "principals":[ + "spiffe://foo.abc", + "spiffe://bar*", + "*baz", + "spiffe://abc.*.com" + ] + } + }], + "allow_rules": [ + { + "name": "allow_policy_1", + "source": { + "principals":["*"] + }, + "request": { + "paths": ["path-foo*"] + } + }, + { + "name": "allow_policy_2", + "request": { + "paths": [ + "path-bar", + "*baz" + ], + "headers": [ + { + "key": "key-1", + "values": ["foo", "*bar"] + }, + { + "key": "key-2", + "values": ["baz*"] + } + ] + } + }] + }`, + wantPolicies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_DENY, + Policies: map[string]*v3rbacpb.Policy{ + "authz_deny_policy_1": { + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_OrIds{OrIds: &v3rbacpb.Principal_Set{ + Ids: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "spiffe://foo.abc"}, + }}, + }}, + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: "spiffe://bar"}, + }}, + }}, + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Suffix{Suffix: "baz"}, + }}, + }}, + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "spiffe://abc.*.com"}, + }}, + }}, + }, + }}}, + }, + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + }, + }, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{}, + }, + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "authz_allow_policy_1": { + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_OrIds{OrIds: &v3rbacpb.Principal_Set{ + Ids: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_SafeRegex{SafeRegex: &v3matcherpb.RegexMatcher{Regex: ".+"}}, + }}, + }}, + }, + }}}, + }, + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_AndRules{AndRules: &v3rbacpb.Permission_Set{ + Rules: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_OrRules{OrRules: &v3rbacpb.Permission_Set{ + Rules: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_UrlPath{ + UrlPath: &v3matcherpb.PathMatcher{Rule: &v3matcherpb.PathMatcher_Path{Path: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: "path-foo"}, + }}}, + }}, + }, + }}}, + }, + }}}, + }, + }, + "authz_allow_policy_2": { + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_AndRules{AndRules: &v3rbacpb.Permission_Set{ + Rules: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_OrRules{OrRules: &v3rbacpb.Permission_Set{ + Rules: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_UrlPath{ + UrlPath: &v3matcherpb.PathMatcher{Rule: &v3matcherpb.PathMatcher_Path{Path: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "path-bar"}, + }}}, + }}, + {Rule: &v3rbacpb.Permission_UrlPath{ + UrlPath: &v3matcherpb.PathMatcher{Rule: &v3matcherpb.PathMatcher_Path{Path: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Suffix{Suffix: "baz"}, + }}}, + }}, + }, + }}}, + {Rule: &v3rbacpb.Permission_AndRules{AndRules: &v3rbacpb.Permission_Set{ + Rules: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_OrRules{OrRules: &v3rbacpb.Permission_Set{ + Rules: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Header{ + Header: &v3routepb.HeaderMatcher{ + Name: "key-1", HeaderMatchSpecifier: &v3routepb.HeaderMatcher_ExactMatch{ExactMatch: "foo"}, + }, + }}, + {Rule: &v3rbacpb.Permission_Header{ + Header: &v3routepb.HeaderMatcher{ + Name: "key-1", HeaderMatchSpecifier: &v3routepb.HeaderMatcher_SuffixMatch{SuffixMatch: "bar"}, + }, + }}, + }, + }}}, + {Rule: &v3rbacpb.Permission_OrRules{OrRules: &v3rbacpb.Permission_Set{ + Rules: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Header{ + Header: &v3routepb.HeaderMatcher{ + Name: "key-2", HeaderMatchSpecifier: &v3routepb.HeaderMatcher_PrefixMatch{PrefixMatch: "baz"}, + }, + }}, + }, + }}}, + }, + }}}, + }, + }}}, + }, + }, + }, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{}, + }, + }, + wantPolicyName: "authz", + }, + "allow authenticated": { + authzPolicy: `{ + "name": "authz", + "allow_rules": [ + { + "name": "allow_authenticated", + "source": { + "principals":["*", ""] + } + }] + }`, + wantPolicies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "authz_allow_authenticated": { + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_OrIds{OrIds: &v3rbacpb.Principal_Set{ + Ids: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_SafeRegex{SafeRegex: &v3matcherpb.RegexMatcher{Regex: ".+"}}, + }}, + }}, + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: ""}, + }}, + }}, + }, + }}}, + }, + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + }, + }, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{}, + }, + }, + }, + "audit_logging_ALLOW empty config": { + authzPolicy: `{ + "name": "authz", + "allow_rules": [ + { + "name": "allow_authenticated", + "source": { + "principals":["*", ""] + } + }], + "deny_rules": [ + { + "name": "deny_policy_1", + "source": { + "principals":[ + "spiffe://foo.abc" + ] + } + }], + "audit_logging_options": { + "audit_condition": "ON_ALLOW", + "audit_loggers": [ + { + "name": "stdout_logger", + "config": {}, + "is_optional": false + } + ] + } + }`, + wantPolicies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_DENY, + Policies: map[string]*v3rbacpb.Policy{ + "authz_deny_policy_1": { + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_OrIds{OrIds: &v3rbacpb.Principal_Set{ + Ids: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "spiffe://foo.abc"}, + }}, + }}, + }, + }}}, + }, + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + }, + }, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_NONE, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{Name: "stdout_logger", TypedConfig: anyPbHelper(t, map[string]any{}, "stdout_logger")}, + IsOptional: false, + }, + }, + }, + }, + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "authz_allow_authenticated": { + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_OrIds{OrIds: &v3rbacpb.Principal_Set{ + Ids: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_SafeRegex{SafeRegex: &v3matcherpb.RegexMatcher{Regex: ".+"}}, + }}, + }}, + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: ""}, + }}, + }}, + }, + }}}, + }, + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + }, + }, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_ON_ALLOW, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{Name: "stdout_logger", TypedConfig: anyPbHelper(t, map[string]any{}, "stdout_logger")}, + IsOptional: false, + }, + }, + }, + }, + }, + }, + "audit_logging_DENY_AND_ALLOW": { + authzPolicy: `{ + "name": "authz", + "allow_rules": [ + { + "name": "allow_authenticated", + "source": { + "principals":["*", ""] + } + }], + "deny_rules": [ + { + "name": "deny_policy_1", + "source": { + "principals":[ + "spiffe://foo.abc" + ] + } + }], + "audit_logging_options": { + "audit_condition": "ON_DENY_AND_ALLOW", + "audit_loggers": [ + { + "name": "stdout_logger", + "config": {}, + "is_optional": false + } + ] + } + }`, + wantPolicies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_DENY, + Policies: map[string]*v3rbacpb.Policy{ + "authz_deny_policy_1": { + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_OrIds{OrIds: &v3rbacpb.Principal_Set{ + Ids: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "spiffe://foo.abc"}, + }}, + }}, + }, + }}}, + }, + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + }, + }, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_ON_DENY, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{Name: "stdout_logger", TypedConfig: anyPbHelper(t, map[string]any{}, "stdout_logger")}, + IsOptional: false, + }, + }, + }, + }, + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "authz_allow_authenticated": { + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_OrIds{OrIds: &v3rbacpb.Principal_Set{ + Ids: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_SafeRegex{SafeRegex: &v3matcherpb.RegexMatcher{Regex: ".+"}}, + }}, + }}, + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: ""}, + }}, + }}, + }, + }}}, + }, + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + }, + }, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_ON_DENY_AND_ALLOW, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{Name: "stdout_logger", TypedConfig: anyPbHelper(t, map[string]any{}, "stdout_logger")}, + IsOptional: false, + }, + }, + }, + }, + }, + }, + "audit_logging_NONE": { + authzPolicy: `{ + "name": "authz", + "allow_rules": [ + { + "name": "allow_authenticated", + "source": { + "principals":["*", ""] + } + }], + "deny_rules": [ + { + "name": "deny_policy_1", + "source": { + "principals":[ + "spiffe://foo.abc" + ] + } + }], + "audit_logging_options": { + "audit_condition": "NONE", + "audit_loggers": [ + { + "name": "stdout_logger", + "config": {}, + "is_optional": false + } + ] + } + }`, + wantPolicies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_DENY, + Policies: map[string]*v3rbacpb.Policy{ + "authz_deny_policy_1": { + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_OrIds{OrIds: &v3rbacpb.Principal_Set{ + Ids: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "spiffe://foo.abc"}, + }}, + }}, + }, + }}}, + }, + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + }, + }, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_NONE, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{Name: "stdout_logger", TypedConfig: anyPbHelper(t, map[string]any{}, "stdout_logger")}, + IsOptional: false, + }, + }, + }, + }, + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "authz_allow_authenticated": { + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_OrIds{OrIds: &v3rbacpb.Principal_Set{ + Ids: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_SafeRegex{SafeRegex: &v3matcherpb.RegexMatcher{Regex: ".+"}}, + }}, + }}, + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: ""}, + }}, + }}, + }, + }}}, + }, + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + }, + }, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_NONE, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{Name: "stdout_logger", TypedConfig: anyPbHelper(t, map[string]any{}, "stdout_logger")}, + IsOptional: false, + }, + }, + }, + }, + }, + }, + "audit_logging_custom_config simple": { + authzPolicy: `{ + "name": "authz", + "allow_rules": [ + { + "name": "allow_authenticated", + "source": { + "principals":["*", ""] + } + }], + "deny_rules": [ + { + "name": "deny_policy_1", + "source": { + "principals":[ + "spiffe://foo.abc" + ] + } + }], + "audit_logging_options": { + "audit_condition": "NONE", + "audit_loggers": [ + { + "name": "stdout_logger", + "config": {"abc":123, "xyz":"123"}, + "is_optional": false + } + ] + } + }`, + wantPolicies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_DENY, + Policies: map[string]*v3rbacpb.Policy{ + "authz_deny_policy_1": { + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_OrIds{OrIds: &v3rbacpb.Principal_Set{ + Ids: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "spiffe://foo.abc"}, + }}, + }}, + }, + }}}, + }, + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + }, + }, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_NONE, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{Name: "stdout_logger", TypedConfig: anyPbHelper(t, map[string]any{"abc": 123, "xyz": "123"}, "stdout_logger")}, + IsOptional: false, + }, + }, + }, + }, + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "authz_allow_authenticated": { + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_OrIds{OrIds: &v3rbacpb.Principal_Set{ + Ids: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_SafeRegex{SafeRegex: &v3matcherpb.RegexMatcher{Regex: ".+"}}, + }}, + }}, + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: ""}, + }}, + }}, + }, + }}}, + }, + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + }, + }, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_NONE, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{Name: "stdout_logger", TypedConfig: anyPbHelper(t, map[string]any{"abc": 123, "xyz": "123"}, "stdout_logger")}, + IsOptional: false, + }, + }, + }, + }, + }, + }, + "audit_logging_custom_config nested": { + authzPolicy: `{ + "name": "authz", + "allow_rules": [ + { + "name": "allow_authenticated", + "source": { + "principals":["*", ""] + } + }], + "audit_logging_options": { + "audit_condition": "NONE", + "audit_loggers": [ + { + "name": "stdout_logger", + "config": {"abc":123, "xyz":{"abc":123}}, + "is_optional": false + } + ] + } + }`, + wantPolicies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "authz_allow_authenticated": { + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_OrIds{OrIds: &v3rbacpb.Principal_Set{ + Ids: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_SafeRegex{SafeRegex: &v3matcherpb.RegexMatcher{Regex: ".+"}}, + }}, + }}, + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: ""}, + }}, + }}, + }, + }}}, + }, + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + }, + }, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_NONE, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{Name: "stdout_logger", TypedConfig: anyPbHelper(t, map[string]any{"abc": 123, "xyz": map[string]any{"abc": 123}}, "stdout_logger")}, + IsOptional: false, + }, + }, + }, + }, + }, + }, + "missing audit logger config": { + authzPolicy: `{ + "name": "authz", + "allow_rules": [ + { + "name": "allow_authenticated", + "source": { + "principals":["*", ""] + } + }], + "audit_logging_options": { + "audit_condition": "NONE" + } + }`, + wantPolicies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "authz_allow_authenticated": { + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_OrIds{OrIds: &v3rbacpb.Principal_Set{ + Ids: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_SafeRegex{SafeRegex: &v3matcherpb.RegexMatcher{Regex: ".+"}}, + }}, + }}, + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: ""}, + }}, + }}, + }, + }}}, + }, + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + }, + }, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_NONE, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{}, + }, + }, + }, + }, + "missing audit condition": { + authzPolicy: `{ + "name": "authz", + "allow_rules": [ + { + "name": "allow_authenticated", + "source": { + "principals":["*", ""] + } + }], + "audit_logging_options": { + "audit_loggers": [ + { + "name": "stdout_logger", + "config": {}, + "is_optional": false + } + ] + } + }`, + wantPolicies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "authz_allow_authenticated": { + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_OrIds{OrIds: &v3rbacpb.Principal_Set{ + Ids: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_SafeRegex{SafeRegex: &v3matcherpb.RegexMatcher{Regex: ".+"}}, + }}, + }}, + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: ""}, + }}, + }}, + }, + }}}, + }, + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + }, + }, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_NONE, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{Name: "stdout_logger", TypedConfig: anyPbHelper(t, map[string]any{}, "stdout_logger")}, + IsOptional: false, + }, + }, + }, + }, + }, + }, + "missing custom config audit logger": { + authzPolicy: `{ + "name": "authz", + "allow_rules": [ + { + "name": "allow_authenticated", + "source": { + "principals":["*", ""] + } + }], + "deny_rules": [ + { + "name": "deny_policy_1", + "source": { + "principals":[ + "spiffe://foo.abc" + ] + } + }], + "audit_logging_options": { + "audit_condition": "ON_DENY", + "audit_loggers": [ + { + "name": "stdout_logger", + "is_optional": false + } + ] + } + }`, + wantPolicies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_DENY, + Policies: map[string]*v3rbacpb.Policy{ + "authz_deny_policy_1": { + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_OrIds{OrIds: &v3rbacpb.Principal_Set{ + Ids: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "spiffe://foo.abc"}, + }}, + }}, + }, + }}}, + }, + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + }, + }, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_ON_DENY, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{Name: "stdout_logger", TypedConfig: anyPbHelper(t, map[string]any{}, "stdout_logger")}, + IsOptional: false, + }, + }, + }, + }, + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "authz_allow_authenticated": { + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_OrIds{OrIds: &v3rbacpb.Principal_Set{ + Ids: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_SafeRegex{SafeRegex: &v3matcherpb.RegexMatcher{Regex: ".+"}}, + }}, + }}, + {Identifier: &v3rbacpb.Principal_Authenticated_{ + Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: ""}, + }}, + }}, + }, + }}}, + }, + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + }, + }, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_ON_DENY, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{Name: "stdout_logger", TypedConfig: anyPbHelper(t, map[string]any{}, "stdout_logger")}, + IsOptional: false, + }, + }, + }, + }, + }, + }, + "unknown field": { + authzPolicy: `{"random": 123}`, + wantErr: "failed to unmarshal policy", + }, + "missing name field": { + authzPolicy: `{}`, + wantErr: `"name" is not present`, + }, + "invalid field type": { + authzPolicy: `{"name": 123}`, + wantErr: "failed to unmarshal policy", + }, + "missing allow rules field": { + authzPolicy: `{"name": "authz-foo"}`, + wantErr: `"allow_rules" is not present`, + }, + "missing rule name field": { + authzPolicy: `{ + "name": "authz-foo", + "allow_rules": [{}] + }`, + wantErr: `"allow_rules" 0: "name" is not present`, + }, + "missing header key": { + authzPolicy: `{ + "name": "authz", + "allow_rules": [{ + "name": "allow_policy_1", + "request": {"headers":[{"key":"key-a", "values": ["value-a"]}, {}]} + }] + }`, + wantErr: `"allow_rules" 0: "headers" 1: "key" is not present`, + }, + "missing header values": { + authzPolicy: `{ + "name": "authz", + "allow_rules": [{ + "name": "allow_policy_1", + "request": {"headers":[{"key":"key-a"}]} + }] + }`, + wantErr: `"allow_rules" 0: "headers" 0: "values" is not present`, + }, + "unsupported header": { + authzPolicy: `{ + "name": "authz", + "allow_rules": [{ + "name": "allow_policy_1", + "request": {"headers":[{"key":":method", "values":["GET"]}]} + }] + }`, + wantErr: `"allow_rules" 0: "headers" 0: unsupported "key" :method`, + }, + "bad audit condition": { + authzPolicy: `{ + "name": "authz", + "allow_rules": [ + { + "name": "allow_authenticated", + "source": { + "principals":["*", ""] + } + }], + "audit_logging_options": { + "audit_condition": "ABC", + "audit_loggers": [ + { + "name": "stdout_logger", + "config": {}, + "is_optional": false + } + ] + } + }`, + wantErr: `failed to parse AuditCondition ABC`, + }, + "bad audit logger config": { + authzPolicy: `{ + "name": "authz", + "allow_rules": [ + { + "name": "allow_authenticated", + "source": { + "principals":["*", ""] + } + }], + "audit_logging_options": { + "audit_condition": "NONE", + "audit_loggers": [ + { + "name": "stdout_logger", + "config": "abc", + "is_optional": false + } + ] + } + }`, + wantErr: `failed to unmarshal policy`, + }, + "missing audit logger name": { + authzPolicy: `{ + "name": "authz", + "allow_rules": [ + { + "name": "allow_authenticated", + "source": { + "principals":["*", ""] + } + }], + "audit_logging_options": { + "audit_condition": "NONE", + "audit_loggers": [ + { + "name": "", + "config": {}, + "is_optional": false + } + ] + } + }`, + wantErr: `missing required field: name`, + }, + } + for name, test := range tests { + t.Run(name, func(t *testing.T) { + gotPolicies, gotPolicyName, gotErr := translatePolicy(test.authzPolicy) + if gotErr != nil && !strings.HasPrefix(gotErr.Error(), test.wantErr) { + t.Fatalf("unexpected error\nwant:%v\ngot:%v", test.wantErr, gotErr) + } + if diff := cmp.Diff(gotPolicies, test.wantPolicies, protocmp.Transform()); diff != "" { + t.Fatalf("unexpected policy\ndiff (-want +got):\n%s", diff) + } + if test.wantPolicyName != "" && gotPolicyName != test.wantPolicyName { + t.Fatalf("unexpected policy name\nwant:%v\ngot:%v", test.wantPolicyName, gotPolicyName) + } + }) + } +} + +func anyPbHelper(t *testing.T, in map[string]any, name string) *anypb.Any { + t.Helper() + pb, err := structpb.NewStruct(in) + typedStruct := &v1xdsudpatypepb.TypedStruct{ + TypeUrl: typeURLPrefix + name, + Value: pb, + } + if err != nil { + t.Fatal(err) + } + customConfig, err := anypb.New(typedStruct) + if err != nil { + t.Fatal(err) + } + return customConfig +} diff --git a/backoff.go b/backoff.go index ff7c3ee6f482..29475e31c979 100644 --- a/backoff.go +++ b/backoff.go @@ -48,7 +48,10 @@ type BackoffConfig struct { // here for more details: // https://github.com/grpc/grpc/blob/master/doc/connection-backoff.md. // -// This API is EXPERIMENTAL. +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. type ConnectParams struct { // Backoff specifies the configuration options for connection backoff. Backoff backoff.Config diff --git a/balancer/balancer.go b/balancer/balancer.go index 8bf359dbfda3..b6377f445ad2 100644 --- a/balancer/balancer.go +++ b/balancer/balancer.go @@ -27,6 +27,7 @@ import ( "net" "strings" + "google.golang.org/grpc/channelz" "google.golang.org/grpc/connectivity" "google.golang.org/grpc/credentials" "google.golang.org/grpc/internal" @@ -75,24 +76,26 @@ func Get(name string) Builder { return nil } -// SubConn represents a gRPC sub connection. -// Each sub connection contains a list of addresses. gRPC will -// try to connect to them (in sequence), and stop trying the -// remainder once one connection is successful. +// A SubConn represents a single connection to a gRPC backend service. // -// The reconnect backoff will be applied on the list, not a single address. -// For example, try_on_all_addresses -> backoff -> try_on_all_addresses. +// Each SubConn contains a list of addresses. // -// All SubConns start in IDLE, and will not try to connect. To trigger -// the connecting, Balancers must call Connect. -// When the connection encounters an error, it will reconnect immediately. -// When the connection becomes IDLE, it will not reconnect unless Connect is -// called. +// All SubConns start in IDLE, and will not try to connect. To trigger the +// connecting, Balancers must call Connect. If a connection re-enters IDLE, +// Balancers must call Connect again to trigger a new connection attempt. // -// This interface is to be implemented by gRPC. Users should not need a -// brand new implementation of this interface. For the situations like -// testing, the new implementation should embed this interface. This allows -// gRPC to add new methods to this interface. +// gRPC will try to connect to the addresses in sequence, and stop trying the +// remainder once the first connection is successful. If an attempt to connect +// to all addresses encounters an error, the SubConn will enter +// TRANSIENT_FAILURE for a backoff period, and then transition to IDLE. +// +// Once established, if a connection is lost, the SubConn will transition +// directly to IDLE. +// +// This interface is to be implemented by gRPC. Users should not need their own +// implementation of this interface. For situations like testing, any +// implementations should embed this interface. This allows gRPC to add new +// methods to this interface. type SubConn interface { // UpdateAddresses updates the addresses used in this SubConn. // gRPC checks if currently-connected address is still in the new list. @@ -101,9 +104,24 @@ type SubConn interface { // a new connection will be created. // // This will trigger a state transition for the SubConn. + // + // Deprecated: this method will be removed. Create new SubConns for new + // addresses instead. UpdateAddresses([]resolver.Address) // Connect starts the connecting for this SubConn. Connect() + // GetOrBuildProducer returns a reference to the existing Producer for this + // ProducerBuilder in this SubConn, or, if one does not currently exist, + // creates a new one and returns it. Returns a close function which must + // be called when the Producer is no longer needed. + GetOrBuildProducer(ProducerBuilder) (p Producer, close func()) + // Shutdown shuts down the SubConn gracefully. Any started RPCs will be + // allowed to complete. No future calls should be made on the SubConn. + // One final state update will be delivered to the StateListener (or + // UpdateSubConnState; deprecated) with ConnectivityState of Shutdown to + // indicate the shutdown operation. This may be delivered before + // in-progress RPCs are complete and the actual connection is closed. + Shutdown() } // NewSubConnOptions contains options to create new SubConn. @@ -118,6 +136,11 @@ type NewSubConnOptions struct { // HealthCheckEnabled indicates whether health check service should be // enabled on this SubConn HealthCheckEnabled bool + // StateListener is called when the state of the subconn changes. If nil, + // Balancer.UpdateSubConnState will be called instead. Will never be + // invoked until after Connect() is called on the SubConn created with + // these options. + StateListener func(SubConnState) } // State contains the balancer's state relevant to the gRPC ClientConn. @@ -139,10 +162,25 @@ type ClientConn interface { // NewSubConn is called by balancer to create a new SubConn. // It doesn't block and wait for the connections to be established. // Behaviors of the SubConn can be controlled by options. + // + // Deprecated: please be aware that in a future version, SubConns will only + // support one address per SubConn. NewSubConn([]resolver.Address, NewSubConnOptions) (SubConn, error) // RemoveSubConn removes the SubConn from ClientConn. // The SubConn will be shutdown. + // + // Deprecated: use SubConn.Shutdown instead. RemoveSubConn(SubConn) + // UpdateAddresses updates the addresses used in the passed in SubConn. + // gRPC checks if the currently connected address is still in the new list. + // If so, the connection will be kept. Else, the connection will be + // gracefully closed, and a new connection will be created. + // + // This may trigger a state transition for the SubConn. + // + // Deprecated: this method will be removed. Create new SubConns for new + // addresses instead. + UpdateAddresses(SubConn, []resolver.Address) // UpdateState notifies gRPC that the balancer's internal state has // changed. @@ -162,21 +200,32 @@ type ClientConn interface { // BuildOptions contains additional information for Build. type BuildOptions struct { - // DialCreds is the transport credential the Balancer implementation can - // use to dial to a remote load balancer server. The Balancer implementations - // can ignore this if it does not need to talk to another party securely. + // DialCreds is the transport credentials to use when communicating with a + // remote load balancer server. Balancer implementations which do not + // communicate with a remote load balancer server can ignore this field. DialCreds credentials.TransportCredentials - // CredsBundle is the credentials bundle that the Balancer can use. + // CredsBundle is the credentials bundle to use when communicating with a + // remote load balancer server. Balancer implementations which do not + // communicate with a remote load balancer server can ignore this field. CredsBundle credentials.Bundle - // Dialer is the custom dialer the Balancer implementation can use to dial - // to a remote load balancer server. The Balancer implementations - // can ignore this if it doesn't need to talk to remote balancer. + // Dialer is the custom dialer to use when communicating with a remote load + // balancer server. Balancer implementations which do not communicate with a + // remote load balancer server can ignore this field. Dialer func(context.Context, string) (net.Conn, error) - // ChannelzParentID is the entity parent's channelz unique identification number. - ChannelzParentID int64 - // Target contains the parsed address info of the dial target. It is the same resolver.Target as - // passed to the resolver. - // See the documentation for the resolver.Target type for details about what it contains. + // Authority is the server name to use as part of the authentication + // handshake when communicating with a remote load balancer server. Balancer + // implementations which do not communicate with a remote load balancer + // server can ignore this field. + Authority string + // ChannelzParentID is the parent ClientConn's channelz ID. + ChannelzParentID *channelz.Identifier + // CustomUserAgent is the custom user agent set on the parent ClientConn. + // The balancer should set the same custom user agent if it creates a + // ClientConn. + CustomUserAgent string + // Target contains the parsed address info of the dial target. It is the + // same resolver.Target as passed to the resolver. See the documentation for + // the resolver.Target type for details about what it contains. Target resolver.Target } @@ -220,8 +269,8 @@ type DoneInfo struct { // ServerLoad is the load received from server. It's usually sent as part of // trailing metadata. // - // The only supported type now is *orca_v1.LoadReport. - ServerLoad interface{} + // The only supported type now is *orca_v3.LoadReport. + ServerLoad any } var ( @@ -250,6 +299,14 @@ type PickResult struct { // type, Done may not be called. May be nil if the balancer does not wish // to be notified when the RPC completes. Done func(DoneInfo) + + // Metadata provides a way for LB policies to inject arbitrary per-call + // metadata. Any metadata returned here will be merged with existing + // metadata added by the client application. + // + // LB policies with child policies are responsible for propagating metadata + // injected by their children to the ClientConn, as part of Pick(). + Metadata metadata.MD } // TransientFailureError returns e. It exists for backward compatibility and @@ -306,12 +363,30 @@ type Balancer interface { ResolverError(error) // UpdateSubConnState is called by gRPC when the state of a SubConn // changes. + // + // Deprecated: Use NewSubConnOptions.StateListener when creating the + // SubConn instead. UpdateSubConnState(SubConn, SubConnState) - // Close closes the balancer. The balancer is not required to call - // ClientConn.RemoveSubConn for its existing SubConns. + // Close closes the balancer. The balancer is not currently required to + // call SubConn.Shutdown for its existing SubConns; however, this will be + // required in a future release, so it is recommended. Close() } +// ExitIdler is an optional interface for balancers to implement. If +// implemented, ExitIdle will be called when ClientConn.Connect is called, if +// the ClientConn is idle. If unimplemented, ClientConn.Connect will cause +// all SubConns to connect. +// +// Notice: it will be required for all balancers to implement this in a future +// release. +type ExitIdler interface { + // ExitIdle instructs the LB policy to reconnect to backends / exit the + // IDLE state, if appropriate and possible. Note that SubConns that enter + // the IDLE state will not reconnect until SubConn.Connect is called. + ExitIdle() +} + // SubConnState describes the state of a SubConn. type SubConnState struct { // ConnectivityState is the connectivity state of the SubConn. @@ -334,41 +409,19 @@ type ClientConnState struct { // problem with the provided name resolver data. var ErrBadResolverState = errors.New("bad resolver state") -// ConnectivityStateEvaluator takes the connectivity states of multiple SubConns -// and returns one aggregated connectivity state. -// -// It's not thread safe. -type ConnectivityStateEvaluator struct { - numReady uint64 // Number of addrConns in ready state. - numConnecting uint64 // Number of addrConns in connecting state. +// A ProducerBuilder is a simple constructor for a Producer. It is used by the +// SubConn to create producers when needed. +type ProducerBuilder interface { + // Build creates a Producer. The first parameter is always a + // grpc.ClientConnInterface (a type to allow creating RPCs/streams on the + // associated SubConn), but is declared as `any` to avoid a dependency + // cycle. Should also return a close function that will be called when all + // references to the Producer have been given up. + Build(grpcClientConnInterface any) (p Producer, close func()) } -// RecordTransition records state change happening in subConn and based on that -// it evaluates what aggregated state should be. -// -// - If at least one SubConn in Ready, the aggregated state is Ready; -// - Else if at least one SubConn in Connecting, the aggregated state is Connecting; -// - Else the aggregated state is TransientFailure. -// -// Idle and Shutdown are not considered. -func (cse *ConnectivityStateEvaluator) RecordTransition(oldState, newState connectivity.State) connectivity.State { - // Update counters. - for idx, state := range []connectivity.State{oldState, newState} { - updateVal := 2*uint64(idx) - 1 // -1 for oldState and +1 for new. - switch state { - case connectivity.Ready: - cse.numReady += updateVal - case connectivity.Connecting: - cse.numConnecting += updateVal - } - } - - // Evaluate. - if cse.numReady > 0 { - return connectivity.Ready - } - if cse.numConnecting > 0 { - return connectivity.Connecting - } - return connectivity.TransientFailure -} +// A Producer is a type shared among potentially many consumers. It is +// associated with a SubConn, and an implementation will typically contain +// other methods to provide additional functionality, e.g. configuration or +// subscription registration. +type Producer any diff --git a/balancer/base/balancer.go b/balancer/base/balancer.go index 32d782f1cf5c..a7f1eeec8e6a 100644 --- a/balancer/base/balancer.go +++ b/balancer/base/balancer.go @@ -41,10 +41,11 @@ func (bb *baseBuilder) Build(cc balancer.ClientConn, opt balancer.BuildOptions) cc: cc, pickerBuilder: bb.pickerBuilder, - subConns: make(map[resolver.Address]balancer.SubConn), + subConns: resolver.NewAddressMap(), scStates: make(map[balancer.SubConn]connectivity.State), csEvltr: &balancer.ConnectivityStateEvaluator{}, config: bb.config, + state: connectivity.Connecting, } // Initialize picker to a picker that always returns // ErrNoSubConnAvailable, because when state of a SubConn changes, we @@ -64,7 +65,7 @@ type baseBalancer struct { csEvltr *balancer.ConnectivityStateEvaluator state connectivity.State - subConns map[resolver.Address]balancer.SubConn + subConns *resolver.AddressMap scStates map[balancer.SubConn]connectivity.State picker balancer.Picker config Config @@ -75,7 +76,7 @@ type baseBalancer struct { func (b *baseBalancer) ResolverError(err error) { b.resolverErr = err - if len(b.subConns) == 0 { + if b.subConns.Len() == 0 { b.state = connectivity.TransientFailure } @@ -99,28 +100,36 @@ func (b *baseBalancer) UpdateClientConnState(s balancer.ClientConnState) error { // Successful resolution; clear resolver error and ensure we return nil. b.resolverErr = nil // addrsSet is the set converted from addrs, it's used for quick lookup of an address. - addrsSet := make(map[resolver.Address]struct{}) + addrsSet := resolver.NewAddressMap() for _, a := range s.ResolverState.Addresses { - addrsSet[a] = struct{}{} - if _, ok := b.subConns[a]; !ok { + addrsSet.Set(a, nil) + if _, ok := b.subConns.Get(a); !ok { // a is a new address (not existing in b.subConns). - sc, err := b.cc.NewSubConn([]resolver.Address{a}, balancer.NewSubConnOptions{HealthCheckEnabled: b.config.HealthCheck}) + var sc balancer.SubConn + opts := balancer.NewSubConnOptions{ + HealthCheckEnabled: b.config.HealthCheck, + StateListener: func(scs balancer.SubConnState) { b.updateSubConnState(sc, scs) }, + } + sc, err := b.cc.NewSubConn([]resolver.Address{a}, opts) if err != nil { logger.Warningf("base.baseBalancer: failed to create new SubConn: %v", err) continue } - b.subConns[a] = sc + b.subConns.Set(a, sc) b.scStates[sc] = connectivity.Idle + b.csEvltr.RecordTransition(connectivity.Shutdown, connectivity.Idle) sc.Connect() } } - for a, sc := range b.subConns { + for _, a := range b.subConns.Keys() { + sci, _ := b.subConns.Get(a) + sc := sci.(balancer.SubConn) // a was removed by resolver. - if _, ok := addrsSet[a]; !ok { - b.cc.RemoveSubConn(sc) - delete(b.subConns, a) + if _, ok := addrsSet.Get(a); !ok { + sc.Shutdown() + b.subConns.Delete(a) // Keep the state of this sc in b.scStates until sc's state becomes Shutdown. - // The entry will be deleted in UpdateSubConnState. + // The entry will be deleted in updateSubConnState. } } // If resolver state contains no addresses, return an error so ClientConn @@ -131,6 +140,9 @@ func (b *baseBalancer) UpdateClientConnState(s balancer.ClientConnState) error { b.ResolverError(errors.New("produced zero addresses")) return balancer.ErrBadResolverState } + + b.regeneratePicker() + b.cc.UpdateState(balancer.State{ConnectivityState: b.state, Picker: b.picker}) return nil } @@ -150,8 +162,8 @@ func (b *baseBalancer) mergeErrors() error { // regeneratePicker takes a snapshot of the balancer, and generates a picker // from it. The picker is -// - errPicker if the balancer is in TransientFailure, -// - built by the pickerBuilder with all READY SubConns otherwise. +// - errPicker if the balancer is in TransientFailure, +// - built by the pickerBuilder with all READY SubConns otherwise. func (b *baseBalancer) regeneratePicker() { if b.state == connectivity.TransientFailure { b.picker = NewErrPicker(b.mergeErrors()) @@ -160,7 +172,9 @@ func (b *baseBalancer) regeneratePicker() { readySCs := make(map[balancer.SubConn]SubConnInfo) // Filter out all ready SCs from full subConn map. - for addr, sc := range b.subConns { + for _, addr := range b.subConns.Keys() { + sci, _ := b.subConns.Get(addr) + sc := sci.(balancer.SubConn) if st, ok := b.scStates[sc]; ok && st == connectivity.Ready { readySCs[sc] = SubConnInfo{Address: addr} } @@ -168,7 +182,12 @@ func (b *baseBalancer) regeneratePicker() { b.picker = b.pickerBuilder.Build(PickerBuildInfo{ReadySCs: readySCs}) } +// UpdateSubConnState is a nop because a StateListener is always set in NewSubConn. func (b *baseBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { + logger.Errorf("base.baseBalancer: UpdateSubConnState(%v, %+v) called unexpectedly", sc, state) +} + +func (b *baseBalancer) updateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { s := state.ConnectivityState if logger.V(2) { logger.Infof("base.baseBalancer: handle SubConn state change: %p, %v", sc, s) @@ -180,10 +199,14 @@ func (b *baseBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.Su } return } - if oldS == connectivity.TransientFailure && s == connectivity.Connecting { - // Once a subconn enters TRANSIENT_FAILURE, ignore subsequent + if oldS == connectivity.TransientFailure && + (s == connectivity.Connecting || s == connectivity.Idle) { + // Once a subconn enters TRANSIENT_FAILURE, ignore subsequent IDLE or // CONNECTING transitions to prevent the aggregated state from being // always CONNECTING when many backends exist but are all down. + if s == connectivity.Idle { + sc.Connect() + } return } b.scStates[sc] = s @@ -191,8 +214,8 @@ func (b *baseBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.Su case connectivity.Idle: sc.Connect() case connectivity.Shutdown: - // When an address was removed by resolver, b called RemoveSubConn but - // kept the sc's state in scStates. Remove state for this sc here. + // When an address was removed by resolver, b called Shutdown but kept + // the sc's state in scStates. Remove state for this sc here. delete(b.scStates, sc) case connectivity.TransientFailure: // Save error to be reported via picker. @@ -209,15 +232,19 @@ func (b *baseBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.Su b.state == connectivity.TransientFailure { b.regeneratePicker() } - b.cc.UpdateState(balancer.State{ConnectivityState: b.state, Picker: b.picker}) } // Close is a nop because base balancer doesn't have internal state to clean up, -// and it doesn't need to call RemoveSubConn for the SubConns. +// and it doesn't need to call Shutdown for the SubConns. func (b *baseBalancer) Close() { } +// ExitIdle is a nop because the base balancer attempts to stay connected to +// all SubConns at all times. +func (b *baseBalancer) ExitIdle() { +} + // NewErrPicker returns a Picker that always returns err on Pick(). func NewErrPicker(err error) balancer.Picker { return &errPicker{err: err} diff --git a/balancer/base/balancer_test.go b/balancer/base/balancer_test.go new file mode 100644 index 000000000000..8a97b4220a5c --- /dev/null +++ b/balancer/base/balancer_test.go @@ -0,0 +1,118 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package base + +import ( + "context" + "testing" + "time" + + "google.golang.org/grpc/attributes" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/resolver" +) + +type testClientConn struct { + balancer.ClientConn + newSubConn func([]resolver.Address, balancer.NewSubConnOptions) (balancer.SubConn, error) +} + +func (c *testClientConn) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) { + return c.newSubConn(addrs, opts) +} + +func (c *testClientConn) UpdateState(balancer.State) {} + +type testSubConn struct { + updateState func(balancer.SubConnState) +} + +func (sc *testSubConn) UpdateAddresses(addresses []resolver.Address) {} + +func (sc *testSubConn) Connect() {} + +func (sc *testSubConn) Shutdown() {} + +func (sc *testSubConn) GetOrBuildProducer(balancer.ProducerBuilder) (balancer.Producer, func()) { + return nil, nil +} + +// testPickBuilder creates balancer.Picker for test. +type testPickBuilder struct { + validate func(info PickerBuildInfo) +} + +func (p *testPickBuilder) Build(info PickerBuildInfo) balancer.Picker { + p.validate(info) + return nil +} + +func TestBaseBalancerReserveAttributes(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + validated := make(chan struct{}, 1) + v := func(info PickerBuildInfo) { + defer func() { validated <- struct{}{} }() + for _, sc := range info.ReadySCs { + if sc.Address.Addr == "1.1.1.1" { + if sc.Address.Attributes == nil { + t.Errorf("in picker.validate, got address %+v with nil attributes, want not nil", sc.Address) + } + foo, ok := sc.Address.Attributes.Value("foo").(string) + if !ok || foo != "2233niang" { + t.Errorf("in picker.validate, got address[1.1.1.1] with invalid attributes value %v, want 2233niang", sc.Address.Attributes.Value("foo")) + } + } else if sc.Address.Addr == "2.2.2.2" { + if sc.Address.Attributes != nil { + t.Error("in b.subConns, got address[2.2.2.2] with not nil attributes, want nil") + } + } + } + } + pickBuilder := &testPickBuilder{validate: v} + b := (&baseBuilder{pickerBuilder: pickBuilder}).Build(&testClientConn{ + newSubConn: func(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) { + return &testSubConn{updateState: opts.StateListener}, nil + }, + }, balancer.BuildOptions{}).(*baseBalancer) + + b.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: []resolver.Address{ + {Addr: "1.1.1.1", Attributes: attributes.New("foo", "2233niang")}, + {Addr: "2.2.2.2", Attributes: nil}, + }, + }, + }) + select { + case <-validated: + case <-ctx.Done(): + t.Fatalf("timed out waiting for UpdateClientConnState to call picker.Build") + } + + for sc := range b.scStates { + sc.(*testSubConn).updateState(balancer.SubConnState{ConnectivityState: connectivity.Ready, ConnectionError: nil}) + select { + case <-validated: + case <-ctx.Done(): + t.Fatalf("timed out waiting for UpdateClientConnState to call picker.Build") + } + } +} diff --git a/balancer/conn_state_evaluator.go b/balancer/conn_state_evaluator.go new file mode 100644 index 000000000000..c33413581091 --- /dev/null +++ b/balancer/conn_state_evaluator.go @@ -0,0 +1,74 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package balancer + +import "google.golang.org/grpc/connectivity" + +// ConnectivityStateEvaluator takes the connectivity states of multiple SubConns +// and returns one aggregated connectivity state. +// +// It's not thread safe. +type ConnectivityStateEvaluator struct { + numReady uint64 // Number of addrConns in ready state. + numConnecting uint64 // Number of addrConns in connecting state. + numTransientFailure uint64 // Number of addrConns in transient failure state. + numIdle uint64 // Number of addrConns in idle state. +} + +// RecordTransition records state change happening in subConn and based on that +// it evaluates what aggregated state should be. +// +// - If at least one SubConn in Ready, the aggregated state is Ready; +// - Else if at least one SubConn in Connecting, the aggregated state is Connecting; +// - Else if at least one SubConn is Idle, the aggregated state is Idle; +// - Else if at least one SubConn is TransientFailure (or there are no SubConns), the aggregated state is Transient Failure. +// +// Shutdown is not considered. +func (cse *ConnectivityStateEvaluator) RecordTransition(oldState, newState connectivity.State) connectivity.State { + // Update counters. + for idx, state := range []connectivity.State{oldState, newState} { + updateVal := 2*uint64(idx) - 1 // -1 for oldState and +1 for new. + switch state { + case connectivity.Ready: + cse.numReady += updateVal + case connectivity.Connecting: + cse.numConnecting += updateVal + case connectivity.TransientFailure: + cse.numTransientFailure += updateVal + case connectivity.Idle: + cse.numIdle += updateVal + } + } + return cse.CurrentState() +} + +// CurrentState returns the current aggregate conn state by evaluating the counters +func (cse *ConnectivityStateEvaluator) CurrentState() connectivity.State { + // Evaluate. + if cse.numReady > 0 { + return connectivity.Ready + } + if cse.numConnecting > 0 { + return connectivity.Connecting + } + if cse.numIdle > 0 { + return connectivity.Idle + } + return connectivity.TransientFailure +} diff --git a/balancer/conn_state_evaluator_test.go b/balancer/conn_state_evaluator_test.go new file mode 100644 index 000000000000..d82ddf84c240 --- /dev/null +++ b/balancer/conn_state_evaluator_test.go @@ -0,0 +1,245 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package balancer + +import ( + "testing" + + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/internal/grpctest" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// TestRecordTransition_FirstStateChange tests the first call to +// RecordTransition where the `oldState` is usually set to `Shutdown` (a state +// that the ConnectivityStateEvaluator is set to ignore). +func (s) TestRecordTransition_FirstStateChange(t *testing.T) { + tests := []struct { + newState connectivity.State + wantState connectivity.State + }{ + { + newState: connectivity.Idle, + wantState: connectivity.Idle, + }, + { + newState: connectivity.Connecting, + wantState: connectivity.Connecting, + }, + { + newState: connectivity.Ready, + wantState: connectivity.Ready, + }, + { + newState: connectivity.TransientFailure, + wantState: connectivity.TransientFailure, + }, + { + newState: connectivity.Shutdown, + wantState: connectivity.TransientFailure, + }, + } + for _, test := range tests { + cse := &ConnectivityStateEvaluator{} + if gotState := cse.RecordTransition(connectivity.Shutdown, test.newState); gotState != test.wantState { + t.Fatalf("RecordTransition(%v, %v) = %v, want %v", connectivity.Shutdown, test.newState, gotState, test.wantState) + } + } +} + +// TestRecordTransition_SameState tests the scenario where state transitions to +// the same state are recorded multiple times. +func (s) TestRecordTransition_SameState(t *testing.T) { + tests := []struct { + newState connectivity.State + wantState connectivity.State + }{ + { + newState: connectivity.Idle, + wantState: connectivity.Idle, + }, + { + newState: connectivity.Connecting, + wantState: connectivity.Connecting, + }, + { + newState: connectivity.Ready, + wantState: connectivity.Ready, + }, + { + newState: connectivity.TransientFailure, + wantState: connectivity.TransientFailure, + }, + { + newState: connectivity.Shutdown, + wantState: connectivity.TransientFailure, + }, + } + const numStateChanges = 5 + for _, test := range tests { + cse := &ConnectivityStateEvaluator{} + var prevState, gotState connectivity.State + prevState = connectivity.Shutdown + for i := 0; i < numStateChanges; i++ { + gotState = cse.RecordTransition(prevState, test.newState) + prevState = test.newState + } + if gotState != test.wantState { + t.Fatalf("RecordTransition() = %v, want %v", gotState, test.wantState) + } + } +} + +// TestRecordTransition_SingleSubConn_DifferentStates tests some common +// connectivity state change scenarios, on a single subConn. +func (s) TestRecordTransition_SingleSubConn_DifferentStates(t *testing.T) { + tests := []struct { + name string + states []connectivity.State + wantState connectivity.State + }{ + { + name: "regular transition to ready", + states: []connectivity.State{connectivity.Idle, connectivity.Connecting, connectivity.Ready}, + wantState: connectivity.Ready, + }, + { + name: "regular transition to transient failure", + states: []connectivity.State{connectivity.Idle, connectivity.Connecting, connectivity.TransientFailure}, + wantState: connectivity.TransientFailure, + }, + { + name: "regular transition to ready", + states: []connectivity.State{connectivity.Idle, connectivity.Connecting, connectivity.Ready, connectivity.Idle}, + wantState: connectivity.Idle, + }, + { + name: "transition from ready to transient failure", + states: []connectivity.State{connectivity.Idle, connectivity.Connecting, connectivity.Ready, connectivity.TransientFailure}, + wantState: connectivity.TransientFailure, + }, + { + name: "transition from transient failure back to ready", + states: []connectivity.State{connectivity.Idle, connectivity.Connecting, connectivity.Ready, connectivity.TransientFailure, connectivity.Ready}, + wantState: connectivity.Ready, + }, + { + // This state transition is usually suppressed at the LB policy level, by + // not calling RecordTransition. + name: "transition from transient failure back to idle", + states: []connectivity.State{connectivity.Idle, connectivity.Connecting, connectivity.Ready, connectivity.TransientFailure, connectivity.Idle}, + wantState: connectivity.Idle, + }, + { + // This state transition is usually suppressed at the LB policy level, by + // not calling RecordTransition. + name: "transition from transient failure back to connecting", + states: []connectivity.State{connectivity.Idle, connectivity.Connecting, connectivity.Ready, connectivity.TransientFailure, connectivity.Connecting}, + wantState: connectivity.Connecting, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cse := &ConnectivityStateEvaluator{} + var prevState, gotState connectivity.State + prevState = connectivity.Shutdown + for _, newState := range test.states { + gotState = cse.RecordTransition(prevState, newState) + prevState = newState + } + if gotState != test.wantState { + t.Fatalf("RecordTransition() = %v, want %v", gotState, test.wantState) + } + }) + } +} + +// TestRecordTransition_MultipleSubConns_DifferentStates tests state transitions +// among multiple subConns, and verifies that the connectivity state aggregation +// algorithm produces the expected aggregate connectivity state. +func (s) TestRecordTransition_MultipleSubConns_DifferentStates(t *testing.T) { + tests := []struct { + name string + // Each entry in this slice corresponds to the state changes happening on an + // individual subConn. + subConnStates [][]connectivity.State + wantState connectivity.State + }{ + { + name: "atleast one ready", + subConnStates: [][]connectivity.State{ + {connectivity.Idle, connectivity.Connecting, connectivity.Ready}, + {connectivity.Idle}, + {connectivity.Idle, connectivity.Connecting}, + {connectivity.Idle, connectivity.Connecting, connectivity.TransientFailure}, + }, + wantState: connectivity.Ready, + }, + { + name: "atleast one connecting", + subConnStates: [][]connectivity.State{ + {connectivity.Idle, connectivity.Connecting, connectivity.Ready, connectivity.Connecting}, + {connectivity.Idle}, + {connectivity.Idle, connectivity.Connecting, connectivity.TransientFailure}, + }, + wantState: connectivity.Connecting, + }, + { + name: "atleast one idle", + subConnStates: [][]connectivity.State{ + {connectivity.Idle, connectivity.Connecting, connectivity.Ready, connectivity.Idle}, + {connectivity.Idle, connectivity.Connecting, connectivity.TransientFailure}, + }, + wantState: connectivity.Idle, + }, + { + name: "atleast one transient failure", + subConnStates: [][]connectivity.State{ + {connectivity.Idle, connectivity.Connecting, connectivity.Ready, connectivity.TransientFailure}, + {connectivity.TransientFailure}, + }, + wantState: connectivity.TransientFailure, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cse := &ConnectivityStateEvaluator{} + var prevState, gotState connectivity.State + for _, scStates := range test.subConnStates { + prevState = connectivity.Shutdown + for _, newState := range scStates { + gotState = cse.RecordTransition(prevState, newState) + prevState = newState + } + } + if gotState != test.wantState { + t.Fatalf("RecordTransition() = %v, want %v", gotState, test.wantState) + } + }) + } +} diff --git a/balancer/grpclb/grpc_lb_v1/load_balancer.pb.go b/balancer/grpclb/grpc_lb_v1/load_balancer.pb.go index b59191ac5825..f3545302899f 100644 --- a/balancer/grpclb/grpc_lb_v1/load_balancer.pb.go +++ b/balancer/grpclb/grpc_lb_v1/load_balancer.pb.go @@ -1,82 +1,90 @@ +// Copyright 2015 The gRPC Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file defines the GRPCLB LoadBalancing protocol. +// +// The canonical version of this proto can be found at +// https://github.com/grpc/grpc-proto/blob/master/grpc/lb/v1/load_balancer.proto + // Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.31.0 +// protoc v4.22.0 // source: grpc/lb/v1/load_balancer.proto package grpc_lb_v1 import ( - context "context" - fmt "fmt" - proto "github.com/golang/protobuf/proto" - duration "github.com/golang/protobuf/ptypes/duration" - timestamp "github.com/golang/protobuf/ptypes/timestamp" - grpc "google.golang.org/grpc" - codes "google.golang.org/grpc/codes" - status "google.golang.org/grpc/status" - math "math" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + durationpb "google.golang.org/protobuf/types/known/durationpb" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" + reflect "reflect" + sync "sync" ) -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) type LoadBalanceRequest struct { - // Types that are valid to be assigned to LoadBalanceRequestType: + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Types that are assignable to LoadBalanceRequestType: + // // *LoadBalanceRequest_InitialRequest // *LoadBalanceRequest_ClientStats LoadBalanceRequestType isLoadBalanceRequest_LoadBalanceRequestType `protobuf_oneof:"load_balance_request_type"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` } -func (m *LoadBalanceRequest) Reset() { *m = LoadBalanceRequest{} } -func (m *LoadBalanceRequest) String() string { return proto.CompactTextString(m) } -func (*LoadBalanceRequest) ProtoMessage() {} -func (*LoadBalanceRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_7cd3f6d792743fdf, []int{0} +func (x *LoadBalanceRequest) Reset() { + *x = LoadBalanceRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *LoadBalanceRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_LoadBalanceRequest.Unmarshal(m, b) -} -func (m *LoadBalanceRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_LoadBalanceRequest.Marshal(b, m, deterministic) -} -func (m *LoadBalanceRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_LoadBalanceRequest.Merge(m, src) -} -func (m *LoadBalanceRequest) XXX_Size() int { - return xxx_messageInfo_LoadBalanceRequest.Size(m) -} -func (m *LoadBalanceRequest) XXX_DiscardUnknown() { - xxx_messageInfo_LoadBalanceRequest.DiscardUnknown(m) +func (x *LoadBalanceRequest) String() string { + return protoimpl.X.MessageStringOf(x) } -var xxx_messageInfo_LoadBalanceRequest proto.InternalMessageInfo +func (*LoadBalanceRequest) ProtoMessage() {} -type isLoadBalanceRequest_LoadBalanceRequestType interface { - isLoadBalanceRequest_LoadBalanceRequestType() -} - -type LoadBalanceRequest_InitialRequest struct { - InitialRequest *InitialLoadBalanceRequest `protobuf:"bytes,1,opt,name=initial_request,json=initialRequest,proto3,oneof"` +func (x *LoadBalanceRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -type LoadBalanceRequest_ClientStats struct { - ClientStats *ClientStats `protobuf:"bytes,2,opt,name=client_stats,json=clientStats,proto3,oneof"` +// Deprecated: Use LoadBalanceRequest.ProtoReflect.Descriptor instead. +func (*LoadBalanceRequest) Descriptor() ([]byte, []int) { + return file_grpc_lb_v1_load_balancer_proto_rawDescGZIP(), []int{0} } -func (*LoadBalanceRequest_InitialRequest) isLoadBalanceRequest_LoadBalanceRequestType() {} - -func (*LoadBalanceRequest_ClientStats) isLoadBalanceRequest_LoadBalanceRequestType() {} - func (m *LoadBalanceRequest) GetLoadBalanceRequestType() isLoadBalanceRequest_LoadBalanceRequestType { if m != nil { return m.LoadBalanceRequestType @@ -84,117 +92,144 @@ func (m *LoadBalanceRequest) GetLoadBalanceRequestType() isLoadBalanceRequest_Lo return nil } -func (m *LoadBalanceRequest) GetInitialRequest() *InitialLoadBalanceRequest { - if x, ok := m.GetLoadBalanceRequestType().(*LoadBalanceRequest_InitialRequest); ok { +func (x *LoadBalanceRequest) GetInitialRequest() *InitialLoadBalanceRequest { + if x, ok := x.GetLoadBalanceRequestType().(*LoadBalanceRequest_InitialRequest); ok { return x.InitialRequest } return nil } -func (m *LoadBalanceRequest) GetClientStats() *ClientStats { - if x, ok := m.GetLoadBalanceRequestType().(*LoadBalanceRequest_ClientStats); ok { +func (x *LoadBalanceRequest) GetClientStats() *ClientStats { + if x, ok := x.GetLoadBalanceRequestType().(*LoadBalanceRequest_ClientStats); ok { return x.ClientStats } return nil } -// XXX_OneofWrappers is for the internal use of the proto package. -func (*LoadBalanceRequest) XXX_OneofWrappers() []interface{} { - return []interface{}{ - (*LoadBalanceRequest_InitialRequest)(nil), - (*LoadBalanceRequest_ClientStats)(nil), - } +type isLoadBalanceRequest_LoadBalanceRequestType interface { + isLoadBalanceRequest_LoadBalanceRequestType() +} + +type LoadBalanceRequest_InitialRequest struct { + // This message should be sent on the first request to the load balancer. + InitialRequest *InitialLoadBalanceRequest `protobuf:"bytes,1,opt,name=initial_request,json=initialRequest,proto3,oneof"` +} + +type LoadBalanceRequest_ClientStats struct { + // The client stats should be periodically reported to the load balancer + // based on the duration defined in the InitialLoadBalanceResponse. + ClientStats *ClientStats `protobuf:"bytes,2,opt,name=client_stats,json=clientStats,proto3,oneof"` } +func (*LoadBalanceRequest_InitialRequest) isLoadBalanceRequest_LoadBalanceRequestType() {} + +func (*LoadBalanceRequest_ClientStats) isLoadBalanceRequest_LoadBalanceRequestType() {} + type InitialLoadBalanceRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // The name of the load balanced service (e.g., service.googleapis.com). Its // length should be less than 256 bytes. // The name might include a port number. How to handle the port number is up // to the balancer. - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` } -func (m *InitialLoadBalanceRequest) Reset() { *m = InitialLoadBalanceRequest{} } -func (m *InitialLoadBalanceRequest) String() string { return proto.CompactTextString(m) } -func (*InitialLoadBalanceRequest) ProtoMessage() {} -func (*InitialLoadBalanceRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_7cd3f6d792743fdf, []int{1} +func (x *InitialLoadBalanceRequest) Reset() { + *x = InitialLoadBalanceRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *InitialLoadBalanceRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_InitialLoadBalanceRequest.Unmarshal(m, b) +func (x *InitialLoadBalanceRequest) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *InitialLoadBalanceRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_InitialLoadBalanceRequest.Marshal(b, m, deterministic) -} -func (m *InitialLoadBalanceRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_InitialLoadBalanceRequest.Merge(m, src) -} -func (m *InitialLoadBalanceRequest) XXX_Size() int { - return xxx_messageInfo_InitialLoadBalanceRequest.Size(m) -} -func (m *InitialLoadBalanceRequest) XXX_DiscardUnknown() { - xxx_messageInfo_InitialLoadBalanceRequest.DiscardUnknown(m) + +func (*InitialLoadBalanceRequest) ProtoMessage() {} + +func (x *InitialLoadBalanceRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_InitialLoadBalanceRequest proto.InternalMessageInfo +// Deprecated: Use InitialLoadBalanceRequest.ProtoReflect.Descriptor instead. +func (*InitialLoadBalanceRequest) Descriptor() ([]byte, []int) { + return file_grpc_lb_v1_load_balancer_proto_rawDescGZIP(), []int{1} +} -func (m *InitialLoadBalanceRequest) GetName() string { - if m != nil { - return m.Name +func (x *InitialLoadBalanceRequest) GetName() string { + if x != nil { + return x.Name } return "" } // Contains the number of calls finished for a particular load balance token. type ClientStatsPerToken struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // See Server.load_balance_token. LoadBalanceToken string `protobuf:"bytes,1,opt,name=load_balance_token,json=loadBalanceToken,proto3" json:"load_balance_token,omitempty"` // The total number of RPCs that finished associated with the token. - NumCalls int64 `protobuf:"varint,2,opt,name=num_calls,json=numCalls,proto3" json:"num_calls,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + NumCalls int64 `protobuf:"varint,2,opt,name=num_calls,json=numCalls,proto3" json:"num_calls,omitempty"` } -func (m *ClientStatsPerToken) Reset() { *m = ClientStatsPerToken{} } -func (m *ClientStatsPerToken) String() string { return proto.CompactTextString(m) } -func (*ClientStatsPerToken) ProtoMessage() {} -func (*ClientStatsPerToken) Descriptor() ([]byte, []int) { - return fileDescriptor_7cd3f6d792743fdf, []int{2} +func (x *ClientStatsPerToken) Reset() { + *x = ClientStatsPerToken{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *ClientStatsPerToken) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ClientStatsPerToken.Unmarshal(m, b) -} -func (m *ClientStatsPerToken) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ClientStatsPerToken.Marshal(b, m, deterministic) -} -func (m *ClientStatsPerToken) XXX_Merge(src proto.Message) { - xxx_messageInfo_ClientStatsPerToken.Merge(m, src) +func (x *ClientStatsPerToken) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *ClientStatsPerToken) XXX_Size() int { - return xxx_messageInfo_ClientStatsPerToken.Size(m) -} -func (m *ClientStatsPerToken) XXX_DiscardUnknown() { - xxx_messageInfo_ClientStatsPerToken.DiscardUnknown(m) + +func (*ClientStatsPerToken) ProtoMessage() {} + +func (x *ClientStatsPerToken) ProtoReflect() protoreflect.Message { + mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_ClientStatsPerToken proto.InternalMessageInfo +// Deprecated: Use ClientStatsPerToken.ProtoReflect.Descriptor instead. +func (*ClientStatsPerToken) Descriptor() ([]byte, []int) { + return file_grpc_lb_v1_load_balancer_proto_rawDescGZIP(), []int{2} +} -func (m *ClientStatsPerToken) GetLoadBalanceToken() string { - if m != nil { - return m.LoadBalanceToken +func (x *ClientStatsPerToken) GetLoadBalanceToken() string { + if x != nil { + return x.LoadBalanceToken } return "" } -func (m *ClientStatsPerToken) GetNumCalls() int64 { - if m != nil { - return m.NumCalls +func (x *ClientStatsPerToken) GetNumCalls() int64 { + if x != nil { + return x.NumCalls } return 0 } @@ -202,8 +237,12 @@ func (m *ClientStatsPerToken) GetNumCalls() int64 { // Contains client level statistics that are useful to load balancing. Each // count except the timestamp should be reset to zero after reporting the stats. type ClientStats struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // The timestamp of generating the report. - Timestamp *timestamp.Timestamp `protobuf:"bytes,1,opt,name=timestamp,proto3" json:"timestamp,omitempty"` + Timestamp *timestamppb.Timestamp `protobuf:"bytes,1,opt,name=timestamp,proto3" json:"timestamp,omitempty"` // The total number of RPCs that started. NumCallsStarted int64 `protobuf:"varint,2,opt,name=num_calls_started,json=numCallsStarted,proto3" json:"num_calls_started,omitempty"` // The total number of RPCs that finished. @@ -215,127 +254,173 @@ type ClientStats struct { NumCallsFinishedKnownReceived int64 `protobuf:"varint,7,opt,name=num_calls_finished_known_received,json=numCallsFinishedKnownReceived,proto3" json:"num_calls_finished_known_received,omitempty"` // The list of dropped calls. CallsFinishedWithDrop []*ClientStatsPerToken `protobuf:"bytes,8,rep,name=calls_finished_with_drop,json=callsFinishedWithDrop,proto3" json:"calls_finished_with_drop,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` } -func (m *ClientStats) Reset() { *m = ClientStats{} } -func (m *ClientStats) String() string { return proto.CompactTextString(m) } -func (*ClientStats) ProtoMessage() {} -func (*ClientStats) Descriptor() ([]byte, []int) { - return fileDescriptor_7cd3f6d792743fdf, []int{3} +func (x *ClientStats) Reset() { + *x = ClientStats{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *ClientStats) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ClientStats.Unmarshal(m, b) -} -func (m *ClientStats) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ClientStats.Marshal(b, m, deterministic) -} -func (m *ClientStats) XXX_Merge(src proto.Message) { - xxx_messageInfo_ClientStats.Merge(m, src) -} -func (m *ClientStats) XXX_Size() int { - return xxx_messageInfo_ClientStats.Size(m) +func (x *ClientStats) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *ClientStats) XXX_DiscardUnknown() { - xxx_messageInfo_ClientStats.DiscardUnknown(m) + +func (*ClientStats) ProtoMessage() {} + +func (x *ClientStats) ProtoReflect() protoreflect.Message { + mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_ClientStats proto.InternalMessageInfo +// Deprecated: Use ClientStats.ProtoReflect.Descriptor instead. +func (*ClientStats) Descriptor() ([]byte, []int) { + return file_grpc_lb_v1_load_balancer_proto_rawDescGZIP(), []int{3} +} -func (m *ClientStats) GetTimestamp() *timestamp.Timestamp { - if m != nil { - return m.Timestamp +func (x *ClientStats) GetTimestamp() *timestamppb.Timestamp { + if x != nil { + return x.Timestamp } return nil } -func (m *ClientStats) GetNumCallsStarted() int64 { - if m != nil { - return m.NumCallsStarted +func (x *ClientStats) GetNumCallsStarted() int64 { + if x != nil { + return x.NumCallsStarted } return 0 } -func (m *ClientStats) GetNumCallsFinished() int64 { - if m != nil { - return m.NumCallsFinished +func (x *ClientStats) GetNumCallsFinished() int64 { + if x != nil { + return x.NumCallsFinished } return 0 } -func (m *ClientStats) GetNumCallsFinishedWithClientFailedToSend() int64 { - if m != nil { - return m.NumCallsFinishedWithClientFailedToSend +func (x *ClientStats) GetNumCallsFinishedWithClientFailedToSend() int64 { + if x != nil { + return x.NumCallsFinishedWithClientFailedToSend } return 0 } -func (m *ClientStats) GetNumCallsFinishedKnownReceived() int64 { - if m != nil { - return m.NumCallsFinishedKnownReceived +func (x *ClientStats) GetNumCallsFinishedKnownReceived() int64 { + if x != nil { + return x.NumCallsFinishedKnownReceived } return 0 } -func (m *ClientStats) GetCallsFinishedWithDrop() []*ClientStatsPerToken { - if m != nil { - return m.CallsFinishedWithDrop +func (x *ClientStats) GetCallsFinishedWithDrop() []*ClientStatsPerToken { + if x != nil { + return x.CallsFinishedWithDrop } return nil } type LoadBalanceResponse struct { - // Types that are valid to be assigned to LoadBalanceResponseType: + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Types that are assignable to LoadBalanceResponseType: + // // *LoadBalanceResponse_InitialResponse // *LoadBalanceResponse_ServerList // *LoadBalanceResponse_FallbackResponse LoadBalanceResponseType isLoadBalanceResponse_LoadBalanceResponseType `protobuf_oneof:"load_balance_response_type"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` } -func (m *LoadBalanceResponse) Reset() { *m = LoadBalanceResponse{} } -func (m *LoadBalanceResponse) String() string { return proto.CompactTextString(m) } -func (*LoadBalanceResponse) ProtoMessage() {} -func (*LoadBalanceResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_7cd3f6d792743fdf, []int{4} +func (x *LoadBalanceResponse) Reset() { + *x = LoadBalanceResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *LoadBalanceResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_LoadBalanceResponse.Unmarshal(m, b) +func (x *LoadBalanceResponse) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *LoadBalanceResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_LoadBalanceResponse.Marshal(b, m, deterministic) + +func (*LoadBalanceResponse) ProtoMessage() {} + +func (x *LoadBalanceResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -func (m *LoadBalanceResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_LoadBalanceResponse.Merge(m, src) + +// Deprecated: Use LoadBalanceResponse.ProtoReflect.Descriptor instead. +func (*LoadBalanceResponse) Descriptor() ([]byte, []int) { + return file_grpc_lb_v1_load_balancer_proto_rawDescGZIP(), []int{4} } -func (m *LoadBalanceResponse) XXX_Size() int { - return xxx_messageInfo_LoadBalanceResponse.Size(m) + +func (m *LoadBalanceResponse) GetLoadBalanceResponseType() isLoadBalanceResponse_LoadBalanceResponseType { + if m != nil { + return m.LoadBalanceResponseType + } + return nil } -func (m *LoadBalanceResponse) XXX_DiscardUnknown() { - xxx_messageInfo_LoadBalanceResponse.DiscardUnknown(m) + +func (x *LoadBalanceResponse) GetInitialResponse() *InitialLoadBalanceResponse { + if x, ok := x.GetLoadBalanceResponseType().(*LoadBalanceResponse_InitialResponse); ok { + return x.InitialResponse + } + return nil } -var xxx_messageInfo_LoadBalanceResponse proto.InternalMessageInfo +func (x *LoadBalanceResponse) GetServerList() *ServerList { + if x, ok := x.GetLoadBalanceResponseType().(*LoadBalanceResponse_ServerList); ok { + return x.ServerList + } + return nil +} + +func (x *LoadBalanceResponse) GetFallbackResponse() *FallbackResponse { + if x, ok := x.GetLoadBalanceResponseType().(*LoadBalanceResponse_FallbackResponse); ok { + return x.FallbackResponse + } + return nil +} type isLoadBalanceResponse_LoadBalanceResponseType interface { isLoadBalanceResponse_LoadBalanceResponseType() } type LoadBalanceResponse_InitialResponse struct { + // This message should be sent on the first response to the client. InitialResponse *InitialLoadBalanceResponse `protobuf:"bytes,1,opt,name=initial_response,json=initialResponse,proto3,oneof"` } type LoadBalanceResponse_ServerList struct { + // Contains the list of servers selected by the load balancer. The client + // should send requests to these servers in the specified order. ServerList *ServerList `protobuf:"bytes,2,opt,name=server_list,json=serverList,proto3,oneof"` } type LoadBalanceResponse_FallbackResponse struct { + // If this field is set, then the client should eagerly enter fallback + // mode (even if there are existing, healthy connections to backends). FallbackResponse *FallbackResponse `protobuf:"bytes,3,opt,name=fallback_response,json=fallbackResponse,proto3,oneof"` } @@ -345,155 +430,141 @@ func (*LoadBalanceResponse_ServerList) isLoadBalanceResponse_LoadBalanceResponse func (*LoadBalanceResponse_FallbackResponse) isLoadBalanceResponse_LoadBalanceResponseType() {} -func (m *LoadBalanceResponse) GetLoadBalanceResponseType() isLoadBalanceResponse_LoadBalanceResponseType { - if m != nil { - return m.LoadBalanceResponseType - } - return nil +type FallbackResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields } -func (m *LoadBalanceResponse) GetInitialResponse() *InitialLoadBalanceResponse { - if x, ok := m.GetLoadBalanceResponseType().(*LoadBalanceResponse_InitialResponse); ok { - return x.InitialResponse +func (x *FallbackResponse) Reset() { + *x = FallbackResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } - return nil } -func (m *LoadBalanceResponse) GetServerList() *ServerList { - if x, ok := m.GetLoadBalanceResponseType().(*LoadBalanceResponse_ServerList); ok { - return x.ServerList - } - return nil +func (x *FallbackResponse) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *LoadBalanceResponse) GetFallbackResponse() *FallbackResponse { - if x, ok := m.GetLoadBalanceResponseType().(*LoadBalanceResponse_FallbackResponse); ok { - return x.FallbackResponse - } - return nil -} +func (*FallbackResponse) ProtoMessage() {} -// XXX_OneofWrappers is for the internal use of the proto package. -func (*LoadBalanceResponse) XXX_OneofWrappers() []interface{} { - return []interface{}{ - (*LoadBalanceResponse_InitialResponse)(nil), - (*LoadBalanceResponse_ServerList)(nil), - (*LoadBalanceResponse_FallbackResponse)(nil), +func (x *FallbackResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms } + return mi.MessageOf(x) } -type FallbackResponse struct { - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *FallbackResponse) Reset() { *m = FallbackResponse{} } -func (m *FallbackResponse) String() string { return proto.CompactTextString(m) } -func (*FallbackResponse) ProtoMessage() {} +// Deprecated: Use FallbackResponse.ProtoReflect.Descriptor instead. func (*FallbackResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_7cd3f6d792743fdf, []int{5} -} - -func (m *FallbackResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_FallbackResponse.Unmarshal(m, b) -} -func (m *FallbackResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_FallbackResponse.Marshal(b, m, deterministic) + return file_grpc_lb_v1_load_balancer_proto_rawDescGZIP(), []int{5} } -func (m *FallbackResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_FallbackResponse.Merge(m, src) -} -func (m *FallbackResponse) XXX_Size() int { - return xxx_messageInfo_FallbackResponse.Size(m) -} -func (m *FallbackResponse) XXX_DiscardUnknown() { - xxx_messageInfo_FallbackResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_FallbackResponse proto.InternalMessageInfo type InitialLoadBalanceResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // This interval defines how often the client should send the client stats // to the load balancer. Stats should only be reported when the duration is // positive. - ClientStatsReportInterval *duration.Duration `protobuf:"bytes,2,opt,name=client_stats_report_interval,json=clientStatsReportInterval,proto3" json:"client_stats_report_interval,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + ClientStatsReportInterval *durationpb.Duration `protobuf:"bytes,2,opt,name=client_stats_report_interval,json=clientStatsReportInterval,proto3" json:"client_stats_report_interval,omitempty"` } -func (m *InitialLoadBalanceResponse) Reset() { *m = InitialLoadBalanceResponse{} } -func (m *InitialLoadBalanceResponse) String() string { return proto.CompactTextString(m) } -func (*InitialLoadBalanceResponse) ProtoMessage() {} -func (*InitialLoadBalanceResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_7cd3f6d792743fdf, []int{6} +func (x *InitialLoadBalanceResponse) Reset() { + *x = InitialLoadBalanceResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *InitialLoadBalanceResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_InitialLoadBalanceResponse.Unmarshal(m, b) +func (x *InitialLoadBalanceResponse) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *InitialLoadBalanceResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_InitialLoadBalanceResponse.Marshal(b, m, deterministic) -} -func (m *InitialLoadBalanceResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_InitialLoadBalanceResponse.Merge(m, src) -} -func (m *InitialLoadBalanceResponse) XXX_Size() int { - return xxx_messageInfo_InitialLoadBalanceResponse.Size(m) -} -func (m *InitialLoadBalanceResponse) XXX_DiscardUnknown() { - xxx_messageInfo_InitialLoadBalanceResponse.DiscardUnknown(m) + +func (*InitialLoadBalanceResponse) ProtoMessage() {} + +func (x *InitialLoadBalanceResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_InitialLoadBalanceResponse proto.InternalMessageInfo +// Deprecated: Use InitialLoadBalanceResponse.ProtoReflect.Descriptor instead. +func (*InitialLoadBalanceResponse) Descriptor() ([]byte, []int) { + return file_grpc_lb_v1_load_balancer_proto_rawDescGZIP(), []int{6} +} -func (m *InitialLoadBalanceResponse) GetClientStatsReportInterval() *duration.Duration { - if m != nil { - return m.ClientStatsReportInterval +func (x *InitialLoadBalanceResponse) GetClientStatsReportInterval() *durationpb.Duration { + if x != nil { + return x.ClientStatsReportInterval } return nil } type ServerList struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // Contains a list of servers selected by the load balancer. The list will // be updated when server resolutions change or as needed to balance load // across more servers. The client should consume the server list in order // unless instructed otherwise via the client_config. - Servers []*Server `protobuf:"bytes,1,rep,name=servers,proto3" json:"servers,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Servers []*Server `protobuf:"bytes,1,rep,name=servers,proto3" json:"servers,omitempty"` } -func (m *ServerList) Reset() { *m = ServerList{} } -func (m *ServerList) String() string { return proto.CompactTextString(m) } -func (*ServerList) ProtoMessage() {} -func (*ServerList) Descriptor() ([]byte, []int) { - return fileDescriptor_7cd3f6d792743fdf, []int{7} +func (x *ServerList) Reset() { + *x = ServerList{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *ServerList) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ServerList.Unmarshal(m, b) -} -func (m *ServerList) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ServerList.Marshal(b, m, deterministic) -} -func (m *ServerList) XXX_Merge(src proto.Message) { - xxx_messageInfo_ServerList.Merge(m, src) +func (x *ServerList) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *ServerList) XXX_Size() int { - return xxx_messageInfo_ServerList.Size(m) -} -func (m *ServerList) XXX_DiscardUnknown() { - xxx_messageInfo_ServerList.DiscardUnknown(m) + +func (*ServerList) ProtoMessage() {} + +func (x *ServerList) ProtoReflect() protoreflect.Message { + mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[7] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_ServerList proto.InternalMessageInfo +// Deprecated: Use ServerList.ProtoReflect.Descriptor instead. +func (*ServerList) Descriptor() ([]byte, []int) { + return file_grpc_lb_v1_load_balancer_proto_rawDescGZIP(), []int{7} +} -func (m *ServerList) GetServers() []*Server { - if m != nil { - return m.Servers +func (x *ServerList) GetServers() []*Server { + if x != nil { + return x.Servers } return nil } @@ -501,6 +572,10 @@ func (m *ServerList) GetServers() []*Server { // Contains server information. When the drop field is not true, use the other // fields. type Server struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // A resolved address for the server, serialized in network-byte-order. It may // either be an IPv4 or IPv6 address. IpAddress []byte `protobuf:"bytes,1,opt,name=ip_address,json=ipAddress,proto3" json:"ip_address,omitempty"` @@ -517,242 +592,366 @@ type Server struct { // Indicates whether this particular request should be dropped by the client. // If the request is dropped, there will be a corresponding entry in // ClientStats.calls_finished_with_drop. - Drop bool `protobuf:"varint,4,opt,name=drop,proto3" json:"drop,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Drop bool `protobuf:"varint,4,opt,name=drop,proto3" json:"drop,omitempty"` } -func (m *Server) Reset() { *m = Server{} } -func (m *Server) String() string { return proto.CompactTextString(m) } -func (*Server) ProtoMessage() {} -func (*Server) Descriptor() ([]byte, []int) { - return fileDescriptor_7cd3f6d792743fdf, []int{8} +func (x *Server) Reset() { + *x = Server{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *Server) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Server.Unmarshal(m, b) -} -func (m *Server) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Server.Marshal(b, m, deterministic) +func (x *Server) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *Server) XXX_Merge(src proto.Message) { - xxx_messageInfo_Server.Merge(m, src) -} -func (m *Server) XXX_Size() int { - return xxx_messageInfo_Server.Size(m) -} -func (m *Server) XXX_DiscardUnknown() { - xxx_messageInfo_Server.DiscardUnknown(m) + +func (*Server) ProtoMessage() {} + +func (x *Server) ProtoReflect() protoreflect.Message { + mi := &file_grpc_lb_v1_load_balancer_proto_msgTypes[8] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_Server proto.InternalMessageInfo +// Deprecated: Use Server.ProtoReflect.Descriptor instead. +func (*Server) Descriptor() ([]byte, []int) { + return file_grpc_lb_v1_load_balancer_proto_rawDescGZIP(), []int{8} +} -func (m *Server) GetIpAddress() []byte { - if m != nil { - return m.IpAddress +func (x *Server) GetIpAddress() []byte { + if x != nil { + return x.IpAddress } return nil } -func (m *Server) GetPort() int32 { - if m != nil { - return m.Port +func (x *Server) GetPort() int32 { + if x != nil { + return x.Port } return 0 } -func (m *Server) GetLoadBalanceToken() string { - if m != nil { - return m.LoadBalanceToken +func (x *Server) GetLoadBalanceToken() string { + if x != nil { + return x.LoadBalanceToken } return "" } -func (m *Server) GetDrop() bool { - if m != nil { - return m.Drop +func (x *Server) GetDrop() bool { + if x != nil { + return x.Drop } return false } -func init() { - proto.RegisterType((*LoadBalanceRequest)(nil), "grpc.lb.v1.LoadBalanceRequest") - proto.RegisterType((*InitialLoadBalanceRequest)(nil), "grpc.lb.v1.InitialLoadBalanceRequest") - proto.RegisterType((*ClientStatsPerToken)(nil), "grpc.lb.v1.ClientStatsPerToken") - proto.RegisterType((*ClientStats)(nil), "grpc.lb.v1.ClientStats") - proto.RegisterType((*LoadBalanceResponse)(nil), "grpc.lb.v1.LoadBalanceResponse") - proto.RegisterType((*FallbackResponse)(nil), "grpc.lb.v1.FallbackResponse") - proto.RegisterType((*InitialLoadBalanceResponse)(nil), "grpc.lb.v1.InitialLoadBalanceResponse") - proto.RegisterType((*ServerList)(nil), "grpc.lb.v1.ServerList") - proto.RegisterType((*Server)(nil), "grpc.lb.v1.Server") -} - -func init() { proto.RegisterFile("grpc/lb/v1/load_balancer.proto", fileDescriptor_7cd3f6d792743fdf) } - -var fileDescriptor_7cd3f6d792743fdf = []byte{ - // 769 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x55, 0xdd, 0x6e, 0xdb, 0x36, - 0x14, 0x8e, 0x62, 0x25, 0x75, 0x8e, 0xb3, 0x45, 0x61, 0xb1, 0xcd, 0x71, 0xd3, 0x24, 0x13, 0xb0, - 0x22, 0x18, 0x3a, 0x79, 0xc9, 0x6e, 0x36, 0x60, 0x17, 0x9b, 0x5b, 0x04, 0x69, 0xda, 0x8b, 0x80, - 0x0e, 0xd0, 0xa1, 0xc0, 0xc0, 0x51, 0x12, 0xed, 0x10, 0xa1, 0x49, 0x8d, 0xa2, 0x5d, 0xec, 0x66, - 0x37, 0x7b, 0x81, 0x3d, 0xca, 0x5e, 0x61, 0x6f, 0x36, 0x88, 0xa4, 0x2c, 0xd5, 0xae, 0xb1, 0x2b, - 0x91, 0xe7, 0x7c, 0xfc, 0xce, 0xff, 0x11, 0x9c, 0x4c, 0x75, 0x91, 0x0d, 0x45, 0x3a, 0x5c, 0x5c, - 0x0c, 0x85, 0xa2, 0x39, 0x49, 0xa9, 0xa0, 0x32, 0x63, 0x3a, 0x29, 0xb4, 0x32, 0x0a, 0x41, 0xa5, - 0x4f, 0x44, 0x9a, 0x2c, 0x2e, 0x06, 0x27, 0x53, 0xa5, 0xa6, 0x82, 0x0d, 0xad, 0x26, 0x9d, 0x4f, - 0x86, 0xf9, 0x5c, 0x53, 0xc3, 0x95, 0x74, 0xd8, 0xc1, 0xe9, 0xaa, 0xde, 0xf0, 0x19, 0x2b, 0x0d, - 0x9d, 0x15, 0x0e, 0x10, 0xff, 0x1b, 0x00, 0x7a, 0xa3, 0x68, 0x3e, 0x72, 0x36, 0x30, 0xfb, 0x7d, - 0xce, 0x4a, 0x83, 0x6e, 0xe1, 0x80, 0x4b, 0x6e, 0x38, 0x15, 0x44, 0x3b, 0x51, 0x3f, 0x38, 0x0b, - 0xce, 0x7b, 0x97, 0x5f, 0x25, 0x8d, 0xf5, 0xe4, 0x95, 0x83, 0xac, 0xbf, 0xbf, 0xde, 0xc2, 0x9f, - 0xfa, 0xf7, 0x35, 0xe3, 0x8f, 0xb0, 0x9f, 0x09, 0xce, 0xa4, 0x21, 0xa5, 0xa1, 0xa6, 0xec, 0x6f, - 0x5b, 0xba, 0x2f, 0xda, 0x74, 0x2f, 0xac, 0x7e, 0x5c, 0xa9, 0xaf, 0xb7, 0x70, 0x2f, 0x6b, 0xae, - 0xa3, 0x27, 0x70, 0xd4, 0x4e, 0x45, 0xed, 0x14, 0x31, 0x7f, 0x14, 0x2c, 0x1e, 0xc2, 0xd1, 0x46, - 0x4f, 0x10, 0x82, 0x50, 0xd2, 0x19, 0xb3, 0xee, 0xef, 0x61, 0x7b, 0x8e, 0x7f, 0x83, 0xc7, 0x2d, - 0x5b, 0xb7, 0x4c, 0xdf, 0xa9, 0x07, 0x26, 0xd1, 0x73, 0x40, 0x1f, 0x18, 0x31, 0x95, 0xd4, 0x3f, - 0x8c, 0x44, 0x43, 0xed, 0xd0, 0x4f, 0x60, 0x4f, 0xce, 0x67, 0x24, 0xa3, 0x42, 0xb8, 0x68, 0x3a, - 0xb8, 0x2b, 0xe7, 0xb3, 0x17, 0xd5, 0x3d, 0xfe, 0xa7, 0x03, 0xbd, 0x96, 0x09, 0xf4, 0x3d, 0xec, - 0x2d, 0x33, 0xef, 0x33, 0x39, 0x48, 0x5c, 0x6d, 0x92, 0xba, 0x36, 0xc9, 0x5d, 0x8d, 0xc0, 0x0d, - 0x18, 0x7d, 0x0d, 0x87, 0x4b, 0x33, 0x55, 0xea, 0xb4, 0x61, 0xb9, 0x37, 0x77, 0x50, 0x9b, 0x1b, - 0x3b, 0x71, 0x15, 0x40, 0x83, 0x9d, 0x70, 0xc9, 0xcb, 0x7b, 0x96, 0xf7, 0x3b, 0x16, 0x1c, 0xd5, - 0xe0, 0x2b, 0x2f, 0x47, 0xbf, 0xc2, 0x37, 0xeb, 0x68, 0xf2, 0x9e, 0x9b, 0x7b, 0xe2, 0x2b, 0x35, - 0xa1, 0x5c, 0xb0, 0x9c, 0x18, 0x45, 0x4a, 0x26, 0xf3, 0xfe, 0xae, 0x25, 0x7a, 0xb6, 0x4a, 0xf4, - 0x96, 0x9b, 0x7b, 0x17, 0xeb, 0x95, 0xc5, 0xdf, 0xa9, 0x31, 0x93, 0x39, 0xba, 0x86, 0x2f, 0x3f, - 0x42, 0xff, 0x20, 0xd5, 0x7b, 0x49, 0x34, 0xcb, 0x18, 0x5f, 0xb0, 0xbc, 0xff, 0xc8, 0x52, 0x3e, - 0x5d, 0xa5, 0x7c, 0x5d, 0xa1, 0xb0, 0x07, 0xa1, 0x5f, 0xa0, 0xff, 0x31, 0x27, 0x73, 0xad, 0x8a, - 0x7e, 0xf7, 0xac, 0x73, 0xde, 0xbb, 0x3c, 0xdd, 0xd0, 0x46, 0x75, 0x69, 0xf1, 0x67, 0xd9, 0xaa, - 0xc7, 0x2f, 0xb5, 0x2a, 0x6e, 0xc2, 0x6e, 0x18, 0xed, 0xdc, 0x84, 0xdd, 0x9d, 0x68, 0x37, 0xfe, - 0x7b, 0x1b, 0x1e, 0x7f, 0xd0, 0x3f, 0x65, 0xa1, 0x64, 0xc9, 0xd0, 0x18, 0xa2, 0x66, 0x14, 0x9c, - 0xcc, 0x57, 0xf0, 0xd9, 0xff, 0xcd, 0x82, 0x43, 0x5f, 0x6f, 0xe1, 0x83, 0xe5, 0x30, 0x78, 0xd2, - 0x1f, 0xa0, 0x57, 0x32, 0xbd, 0x60, 0x9a, 0x08, 0x5e, 0x1a, 0x3f, 0x0c, 0x9f, 0xb7, 0xf9, 0xc6, - 0x56, 0xfd, 0x86, 0xdb, 0x61, 0x82, 0x72, 0x79, 0x43, 0xaf, 0xe1, 0x70, 0x42, 0x85, 0x48, 0x69, - 0xf6, 0xd0, 0x38, 0xd4, 0xb1, 0x04, 0xc7, 0x6d, 0x82, 0x2b, 0x0f, 0x6a, 0xb9, 0x11, 0x4d, 0x56, - 0x64, 0xa3, 0x63, 0x18, 0xac, 0xcc, 0x95, 0x53, 0xb8, 0xc1, 0x42, 0x10, 0xad, 0xb2, 0xc4, 0x7f, - 0xc2, 0x60, 0x73, 0xa8, 0xe8, 0x1d, 0x1c, 0xb7, 0xa7, 0x9c, 0x68, 0x56, 0x28, 0x6d, 0x08, 0x97, - 0x86, 0xe9, 0x05, 0x15, 0x3e, 0xd0, 0xa3, 0xb5, 0xd6, 0x7f, 0xe9, 0xd7, 0x16, 0x3e, 0x6a, 0x4d, - 0x3d, 0xb6, 0x8f, 0x5f, 0xf9, 0xb7, 0x37, 0x61, 0x37, 0x88, 0xb6, 0xe3, 0x9f, 0x00, 0x9a, 0xd4, - 0xa0, 0xe7, 0xf0, 0xc8, 0xa5, 0xa6, 0xec, 0x07, 0xb6, 0x13, 0xd0, 0x7a, 0x0e, 0x71, 0x0d, 0xb9, - 0x09, 0xbb, 0x9d, 0x28, 0x8c, 0xff, 0x0a, 0x60, 0xd7, 0x69, 0xd0, 0x53, 0x00, 0x5e, 0x10, 0x9a, - 0xe7, 0x9a, 0x95, 0xa5, 0xad, 0xea, 0x3e, 0xde, 0xe3, 0xc5, 0xcf, 0x4e, 0x50, 0xed, 0x8e, 0xca, - 0x03, 0xeb, 0xf5, 0x0e, 0xb6, 0xe7, 0x0d, 0x4b, 0xa2, 0xb3, 0x61, 0x49, 0x20, 0x08, 0x6d, 0x9b, - 0x86, 0x67, 0xc1, 0x79, 0x17, 0xdb, 0xb3, 0x6b, 0xb7, 0xcb, 0x14, 0xf6, 0x5b, 0x09, 0xd4, 0x08, - 0x43, 0xcf, 0x9f, 0x2b, 0x31, 0x3a, 0x69, 0xc7, 0xb1, 0xbe, 0xd6, 0x06, 0xa7, 0x1b, 0xf5, 0xae, - 0x12, 0xe7, 0xc1, 0xb7, 0xc1, 0xe8, 0x2d, 0x7c, 0xc2, 0x55, 0x0b, 0x38, 0x3a, 0x6c, 0x9b, 0xbc, - 0xad, 0x92, 0x7f, 0x1b, 0xbc, 0xbb, 0xf0, 0xc5, 0x98, 0x2a, 0x41, 0xe5, 0x34, 0x51, 0x7a, 0x3a, - 0xb4, 0x7f, 0xa0, 0xfa, 0xb7, 0x63, 0x6f, 0x22, 0xb5, 0x1f, 0x22, 0x52, 0xb2, 0xb8, 0x48, 0x77, - 0x6d, 0xe1, 0xbe, 0xfb, 0x2f, 0x00, 0x00, 0xff, 0xff, 0x3f, 0x47, 0x55, 0xac, 0xab, 0x06, 0x00, - 0x00, -} - -// Reference imports to suppress errors if they are not otherwise used. -var _ context.Context -var _ grpc.ClientConnInterface - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the grpc package it is being compiled against. -const _ = grpc.SupportPackageIsVersion6 - -// LoadBalancerClient is the client API for LoadBalancer service. -// -// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. -type LoadBalancerClient interface { - // Bidirectional rpc to get a list of servers. - BalanceLoad(ctx context.Context, opts ...grpc.CallOption) (LoadBalancer_BalanceLoadClient, error) -} - -type loadBalancerClient struct { - cc grpc.ClientConnInterface -} - -func NewLoadBalancerClient(cc grpc.ClientConnInterface) LoadBalancerClient { - return &loadBalancerClient{cc} -} +var File_grpc_lb_v1_load_balancer_proto protoreflect.FileDescriptor + +var file_grpc_lb_v1_load_balancer_proto_rawDesc = []byte{ + 0x0a, 0x1e, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x6c, 0x62, 0x2f, 0x76, 0x31, 0x2f, 0x6c, 0x6f, 0x61, + 0x64, 0x5f, 0x62, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x12, 0x0a, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x62, 0x2e, 0x76, 0x31, 0x1a, 0x1e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x64, 0x75, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1f, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, + 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xc1, 0x01, + 0x0a, 0x12, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x12, 0x50, 0x0a, 0x0f, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x5f, + 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e, + 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x62, 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x69, + 0x61, 0x6c, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x48, 0x00, 0x52, 0x0e, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3c, 0x0a, 0x0c, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, + 0x5f, 0x73, 0x74, 0x61, 0x74, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x67, + 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x62, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, + 0x53, 0x74, 0x61, 0x74, 0x73, 0x48, 0x00, 0x52, 0x0b, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x53, + 0x74, 0x61, 0x74, 0x73, 0x42, 0x1b, 0x0a, 0x19, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x62, 0x61, 0x6c, + 0x61, 0x6e, 0x63, 0x65, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x74, 0x79, 0x70, + 0x65, 0x22, 0x2f, 0x0a, 0x19, 0x49, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x4c, 0x6f, 0x61, 0x64, + 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, + 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, + 0x6d, 0x65, 0x22, 0x60, 0x0a, 0x13, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, + 0x73, 0x50, 0x65, 0x72, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x2c, 0x0a, 0x12, 0x6c, 0x6f, 0x61, + 0x64, 0x5f, 0x62, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x6c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, + 0x63, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1b, 0x0a, 0x09, 0x6e, 0x75, 0x6d, 0x5f, 0x63, + 0x61, 0x6c, 0x6c, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x6e, 0x75, 0x6d, 0x43, + 0x61, 0x6c, 0x6c, 0x73, 0x22, 0xb0, 0x03, 0x0a, 0x0b, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x53, + 0x74, 0x61, 0x74, 0x73, 0x12, 0x38, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, + 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, + 0x61, 0x6d, 0x70, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x2a, + 0x0a, 0x11, 0x6e, 0x75, 0x6d, 0x5f, 0x63, 0x61, 0x6c, 0x6c, 0x73, 0x5f, 0x73, 0x74, 0x61, 0x72, + 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0f, 0x6e, 0x75, 0x6d, 0x43, 0x61, + 0x6c, 0x6c, 0x73, 0x53, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x12, 0x2c, 0x0a, 0x12, 0x6e, 0x75, + 0x6d, 0x5f, 0x63, 0x61, 0x6c, 0x6c, 0x73, 0x5f, 0x66, 0x69, 0x6e, 0x69, 0x73, 0x68, 0x65, 0x64, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x10, 0x6e, 0x75, 0x6d, 0x43, 0x61, 0x6c, 0x6c, 0x73, + 0x46, 0x69, 0x6e, 0x69, 0x73, 0x68, 0x65, 0x64, 0x12, 0x5d, 0x0a, 0x2d, 0x6e, 0x75, 0x6d, 0x5f, + 0x63, 0x61, 0x6c, 0x6c, 0x73, 0x5f, 0x66, 0x69, 0x6e, 0x69, 0x73, 0x68, 0x65, 0x64, 0x5f, 0x77, + 0x69, 0x74, 0x68, 0x5f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x66, 0x61, 0x69, 0x6c, 0x65, + 0x64, 0x5f, 0x74, 0x6f, 0x5f, 0x73, 0x65, 0x6e, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, + 0x26, 0x6e, 0x75, 0x6d, 0x43, 0x61, 0x6c, 0x6c, 0x73, 0x46, 0x69, 0x6e, 0x69, 0x73, 0x68, 0x65, + 0x64, 0x57, 0x69, 0x74, 0x68, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x46, 0x61, 0x69, 0x6c, 0x65, + 0x64, 0x54, 0x6f, 0x53, 0x65, 0x6e, 0x64, 0x12, 0x48, 0x0a, 0x21, 0x6e, 0x75, 0x6d, 0x5f, 0x63, + 0x61, 0x6c, 0x6c, 0x73, 0x5f, 0x66, 0x69, 0x6e, 0x69, 0x73, 0x68, 0x65, 0x64, 0x5f, 0x6b, 0x6e, + 0x6f, 0x77, 0x6e, 0x5f, 0x72, 0x65, 0x63, 0x65, 0x69, 0x76, 0x65, 0x64, 0x18, 0x07, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x1d, 0x6e, 0x75, 0x6d, 0x43, 0x61, 0x6c, 0x6c, 0x73, 0x46, 0x69, 0x6e, 0x69, + 0x73, 0x68, 0x65, 0x64, 0x4b, 0x6e, 0x6f, 0x77, 0x6e, 0x52, 0x65, 0x63, 0x65, 0x69, 0x76, 0x65, + 0x64, 0x12, 0x58, 0x0a, 0x18, 0x63, 0x61, 0x6c, 0x6c, 0x73, 0x5f, 0x66, 0x69, 0x6e, 0x69, 0x73, + 0x68, 0x65, 0x64, 0x5f, 0x77, 0x69, 0x74, 0x68, 0x5f, 0x64, 0x72, 0x6f, 0x70, 0x18, 0x08, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x62, 0x2e, 0x76, 0x31, + 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x73, 0x50, 0x65, 0x72, 0x54, + 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x15, 0x63, 0x61, 0x6c, 0x6c, 0x73, 0x46, 0x69, 0x6e, 0x69, 0x73, + 0x68, 0x65, 0x64, 0x57, 0x69, 0x74, 0x68, 0x44, 0x72, 0x6f, 0x70, 0x4a, 0x04, 0x08, 0x04, 0x10, + 0x05, 0x4a, 0x04, 0x08, 0x05, 0x10, 0x06, 0x22, 0x90, 0x02, 0x0a, 0x13, 0x4c, 0x6f, 0x61, 0x64, + 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x53, 0x0a, 0x10, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x5f, 0x72, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x6c, 0x62, 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x4c, 0x6f, + 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x48, 0x00, 0x52, 0x0f, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x39, 0x0a, 0x0b, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x6c, + 0x69, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x6c, 0x62, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x4c, 0x69, 0x73, + 0x74, 0x48, 0x00, 0x52, 0x0a, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x4c, 0x69, 0x73, 0x74, 0x12, + 0x4b, 0x0a, 0x11, 0x66, 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x5f, 0x72, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x72, 0x70, + 0x63, 0x2e, 0x6c, 0x62, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x61, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x00, 0x52, 0x10, 0x66, 0x61, 0x6c, 0x6c, + 0x62, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x1c, 0x0a, 0x1a, + 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x62, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x5f, 0x72, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x22, 0x12, 0x0a, 0x10, 0x46, 0x61, + 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x7e, + 0x0a, 0x1a, 0x49, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, + 0x61, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5a, 0x0a, 0x1c, + 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x73, 0x5f, 0x72, 0x65, 0x70, + 0x6f, 0x72, 0x74, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x19, 0x63, + 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, + 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x4a, 0x04, 0x08, 0x01, 0x10, 0x02, 0x22, 0x40, + 0x0a, 0x0a, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x2c, 0x0a, 0x07, + 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, + 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x62, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, + 0x72, 0x52, 0x07, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x4a, 0x04, 0x08, 0x03, 0x10, 0x04, + 0x22, 0x83, 0x01, 0x0a, 0x06, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x1d, 0x0a, 0x0a, 0x69, + 0x70, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x09, 0x69, 0x70, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, + 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x2c, + 0x0a, 0x12, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x62, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x5f, 0x74, + 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x6c, 0x6f, 0x61, 0x64, + 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x12, 0x0a, 0x04, + 0x64, 0x72, 0x6f, 0x70, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x04, 0x64, 0x72, 0x6f, 0x70, + 0x4a, 0x04, 0x08, 0x05, 0x10, 0x06, 0x32, 0x62, 0x0a, 0x0c, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, + 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x12, 0x52, 0x0a, 0x0b, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, + 0x65, 0x4c, 0x6f, 0x61, 0x64, 0x12, 0x1e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x62, 0x2e, + 0x76, 0x31, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x62, 0x2e, + 0x76, 0x31, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x42, 0x57, 0x0a, 0x0d, 0x69, 0x6f, + 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x62, 0x2e, 0x76, 0x31, 0x42, 0x11, 0x4c, 0x6f, 0x61, + 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, + 0x5a, 0x31, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e, + 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x62, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, + 0x72, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x6c, 0x62, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x6c, 0x62, + 0x5f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_grpc_lb_v1_load_balancer_proto_rawDescOnce sync.Once + file_grpc_lb_v1_load_balancer_proto_rawDescData = file_grpc_lb_v1_load_balancer_proto_rawDesc +) -func (c *loadBalancerClient) BalanceLoad(ctx context.Context, opts ...grpc.CallOption) (LoadBalancer_BalanceLoadClient, error) { - stream, err := c.cc.NewStream(ctx, &_LoadBalancer_serviceDesc.Streams[0], "/grpc.lb.v1.LoadBalancer/BalanceLoad", opts...) - if err != nil { - return nil, err +func file_grpc_lb_v1_load_balancer_proto_rawDescGZIP() []byte { + file_grpc_lb_v1_load_balancer_proto_rawDescOnce.Do(func() { + file_grpc_lb_v1_load_balancer_proto_rawDescData = protoimpl.X.CompressGZIP(file_grpc_lb_v1_load_balancer_proto_rawDescData) + }) + return file_grpc_lb_v1_load_balancer_proto_rawDescData +} + +var file_grpc_lb_v1_load_balancer_proto_msgTypes = make([]protoimpl.MessageInfo, 9) +var file_grpc_lb_v1_load_balancer_proto_goTypes = []interface{}{ + (*LoadBalanceRequest)(nil), // 0: grpc.lb.v1.LoadBalanceRequest + (*InitialLoadBalanceRequest)(nil), // 1: grpc.lb.v1.InitialLoadBalanceRequest + (*ClientStatsPerToken)(nil), // 2: grpc.lb.v1.ClientStatsPerToken + (*ClientStats)(nil), // 3: grpc.lb.v1.ClientStats + (*LoadBalanceResponse)(nil), // 4: grpc.lb.v1.LoadBalanceResponse + (*FallbackResponse)(nil), // 5: grpc.lb.v1.FallbackResponse + (*InitialLoadBalanceResponse)(nil), // 6: grpc.lb.v1.InitialLoadBalanceResponse + (*ServerList)(nil), // 7: grpc.lb.v1.ServerList + (*Server)(nil), // 8: grpc.lb.v1.Server + (*timestamppb.Timestamp)(nil), // 9: google.protobuf.Timestamp + (*durationpb.Duration)(nil), // 10: google.protobuf.Duration +} +var file_grpc_lb_v1_load_balancer_proto_depIdxs = []int32{ + 1, // 0: grpc.lb.v1.LoadBalanceRequest.initial_request:type_name -> grpc.lb.v1.InitialLoadBalanceRequest + 3, // 1: grpc.lb.v1.LoadBalanceRequest.client_stats:type_name -> grpc.lb.v1.ClientStats + 9, // 2: grpc.lb.v1.ClientStats.timestamp:type_name -> google.protobuf.Timestamp + 2, // 3: grpc.lb.v1.ClientStats.calls_finished_with_drop:type_name -> grpc.lb.v1.ClientStatsPerToken + 6, // 4: grpc.lb.v1.LoadBalanceResponse.initial_response:type_name -> grpc.lb.v1.InitialLoadBalanceResponse + 7, // 5: grpc.lb.v1.LoadBalanceResponse.server_list:type_name -> grpc.lb.v1.ServerList + 5, // 6: grpc.lb.v1.LoadBalanceResponse.fallback_response:type_name -> grpc.lb.v1.FallbackResponse + 10, // 7: grpc.lb.v1.InitialLoadBalanceResponse.client_stats_report_interval:type_name -> google.protobuf.Duration + 8, // 8: grpc.lb.v1.ServerList.servers:type_name -> grpc.lb.v1.Server + 0, // 9: grpc.lb.v1.LoadBalancer.BalanceLoad:input_type -> grpc.lb.v1.LoadBalanceRequest + 4, // 10: grpc.lb.v1.LoadBalancer.BalanceLoad:output_type -> grpc.lb.v1.LoadBalanceResponse + 10, // [10:11] is the sub-list for method output_type + 9, // [9:10] is the sub-list for method input_type + 9, // [9:9] is the sub-list for extension type_name + 9, // [9:9] is the sub-list for extension extendee + 0, // [0:9] is the sub-list for field type_name +} + +func init() { file_grpc_lb_v1_load_balancer_proto_init() } +func file_grpc_lb_v1_load_balancer_proto_init() { + if File_grpc_lb_v1_load_balancer_proto != nil { + return } - x := &loadBalancerBalanceLoadClient{stream} - return x, nil -} - -type LoadBalancer_BalanceLoadClient interface { - Send(*LoadBalanceRequest) error - Recv() (*LoadBalanceResponse, error) - grpc.ClientStream -} - -type loadBalancerBalanceLoadClient struct { - grpc.ClientStream -} - -func (x *loadBalancerBalanceLoadClient) Send(m *LoadBalanceRequest) error { - return x.ClientStream.SendMsg(m) -} - -func (x *loadBalancerBalanceLoadClient) Recv() (*LoadBalanceResponse, error) { - m := new(LoadBalanceResponse) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err + if !protoimpl.UnsafeEnabled { + file_grpc_lb_v1_load_balancer_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*LoadBalanceRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_lb_v1_load_balancer_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*InitialLoadBalanceRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_lb_v1_load_balancer_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ClientStatsPerToken); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_lb_v1_load_balancer_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ClientStats); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_lb_v1_load_balancer_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*LoadBalanceResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_lb_v1_load_balancer_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*FallbackResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_lb_v1_load_balancer_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*InitialLoadBalanceResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_lb_v1_load_balancer_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ServerList); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_lb_v1_load_balancer_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Server); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } - return m, nil -} - -// LoadBalancerServer is the server API for LoadBalancer service. -type LoadBalancerServer interface { - // Bidirectional rpc to get a list of servers. - BalanceLoad(LoadBalancer_BalanceLoadServer) error -} - -// UnimplementedLoadBalancerServer can be embedded to have forward compatible implementations. -type UnimplementedLoadBalancerServer struct { -} - -func (*UnimplementedLoadBalancerServer) BalanceLoad(srv LoadBalancer_BalanceLoadServer) error { - return status.Errorf(codes.Unimplemented, "method BalanceLoad not implemented") -} - -func RegisterLoadBalancerServer(s *grpc.Server, srv LoadBalancerServer) { - s.RegisterService(&_LoadBalancer_serviceDesc, srv) -} - -func _LoadBalancer_BalanceLoad_Handler(srv interface{}, stream grpc.ServerStream) error { - return srv.(LoadBalancerServer).BalanceLoad(&loadBalancerBalanceLoadServer{stream}) -} - -type LoadBalancer_BalanceLoadServer interface { - Send(*LoadBalanceResponse) error - Recv() (*LoadBalanceRequest, error) - grpc.ServerStream -} - -type loadBalancerBalanceLoadServer struct { - grpc.ServerStream -} - -func (x *loadBalancerBalanceLoadServer) Send(m *LoadBalanceResponse) error { - return x.ServerStream.SendMsg(m) -} - -func (x *loadBalancerBalanceLoadServer) Recv() (*LoadBalanceRequest, error) { - m := new(LoadBalanceRequest) - if err := x.ServerStream.RecvMsg(m); err != nil { - return nil, err + file_grpc_lb_v1_load_balancer_proto_msgTypes[0].OneofWrappers = []interface{}{ + (*LoadBalanceRequest_InitialRequest)(nil), + (*LoadBalanceRequest_ClientStats)(nil), } - return m, nil -} - -var _LoadBalancer_serviceDesc = grpc.ServiceDesc{ - ServiceName: "grpc.lb.v1.LoadBalancer", - HandlerType: (*LoadBalancerServer)(nil), - Methods: []grpc.MethodDesc{}, - Streams: []grpc.StreamDesc{ - { - StreamName: "BalanceLoad", - Handler: _LoadBalancer_BalanceLoad_Handler, - ServerStreams: true, - ClientStreams: true, + file_grpc_lb_v1_load_balancer_proto_msgTypes[4].OneofWrappers = []interface{}{ + (*LoadBalanceResponse_InitialResponse)(nil), + (*LoadBalanceResponse_ServerList)(nil), + (*LoadBalanceResponse_FallbackResponse)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_grpc_lb_v1_load_balancer_proto_rawDesc, + NumEnums: 0, + NumMessages: 9, + NumExtensions: 0, + NumServices: 1, }, - }, - Metadata: "grpc/lb/v1/load_balancer.proto", + GoTypes: file_grpc_lb_v1_load_balancer_proto_goTypes, + DependencyIndexes: file_grpc_lb_v1_load_balancer_proto_depIdxs, + MessageInfos: file_grpc_lb_v1_load_balancer_proto_msgTypes, + }.Build() + File_grpc_lb_v1_load_balancer_proto = out.File + file_grpc_lb_v1_load_balancer_proto_rawDesc = nil + file_grpc_lb_v1_load_balancer_proto_goTypes = nil + file_grpc_lb_v1_load_balancer_proto_depIdxs = nil } diff --git a/balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go b/balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go index 0079d0fad8df..00d0954b38a5 100644 --- a/balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go +++ b/balancer/grpclb/grpc_lb_v1/load_balancer_grpc.pb.go @@ -1,8 +1,32 @@ +// Copyright 2015 The gRPC Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file defines the GRPCLB LoadBalancing protocol. +// +// The canonical version of this proto can be found at +// https://github.com/grpc/grpc-proto/blob/master/grpc/lb/v1/load_balancer.proto + // Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.3.0 +// - protoc v4.22.0 +// source: grpc/lb/v1/load_balancer.proto package grpc_lb_v1 import ( + context "context" grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" @@ -10,64 +34,127 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. const _ = grpc.SupportPackageIsVersion7 -// LoadBalancerService is the service API for LoadBalancer service. -// Fields should be assigned to their respective handler implementations only before -// RegisterLoadBalancerService is called. Any unassigned fields will result in the -// handler for that method returning an Unimplemented error. -type LoadBalancerService struct { +const ( + LoadBalancer_BalanceLoad_FullMethodName = "/grpc.lb.v1.LoadBalancer/BalanceLoad" +) + +// LoadBalancerClient is the client API for LoadBalancer service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type LoadBalancerClient interface { // Bidirectional rpc to get a list of servers. - BalanceLoad func(LoadBalancer_BalanceLoadServer) error + BalanceLoad(ctx context.Context, opts ...grpc.CallOption) (LoadBalancer_BalanceLoadClient, error) } -func (s *LoadBalancerService) balanceLoad(_ interface{}, stream grpc.ServerStream) error { - if s.BalanceLoad == nil { - return status.Errorf(codes.Unimplemented, "method BalanceLoad not implemented") - } - return s.BalanceLoad(&loadBalancerBalanceLoadServer{stream}) -} - -// RegisterLoadBalancerService registers a service implementation with a gRPC server. -func RegisterLoadBalancerService(s grpc.ServiceRegistrar, srv *LoadBalancerService) { - sd := grpc.ServiceDesc{ - ServiceName: "grpc.lb.v1.LoadBalancer", - Methods: []grpc.MethodDesc{}, - Streams: []grpc.StreamDesc{ - { - StreamName: "BalanceLoad", - Handler: srv.balanceLoad, - ServerStreams: true, - ClientStreams: true, - }, - }, - Metadata: "grpc/lb/v1/load_balancer.proto", +type loadBalancerClient struct { + cc grpc.ClientConnInterface +} + +func NewLoadBalancerClient(cc grpc.ClientConnInterface) LoadBalancerClient { + return &loadBalancerClient{cc} +} + +func (c *loadBalancerClient) BalanceLoad(ctx context.Context, opts ...grpc.CallOption) (LoadBalancer_BalanceLoadClient, error) { + stream, err := c.cc.NewStream(ctx, &LoadBalancer_ServiceDesc.Streams[0], LoadBalancer_BalanceLoad_FullMethodName, opts...) + if err != nil { + return nil, err } + x := &loadBalancerBalanceLoadClient{stream} + return x, nil +} + +type LoadBalancer_BalanceLoadClient interface { + Send(*LoadBalanceRequest) error + Recv() (*LoadBalanceResponse, error) + grpc.ClientStream +} + +type loadBalancerBalanceLoadClient struct { + grpc.ClientStream +} - s.RegisterService(&sd, nil) -} - -// NewLoadBalancerService creates a new LoadBalancerService containing the -// implemented methods of the LoadBalancer service in s. Any unimplemented -// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client. -// This includes situations where the method handler is misspelled or has the wrong -// signature. For this reason, this function should be used with great care and -// is not recommended to be used by most users. -func NewLoadBalancerService(s interface{}) *LoadBalancerService { - ns := &LoadBalancerService{} - if h, ok := s.(interface { - BalanceLoad(LoadBalancer_BalanceLoadServer) error - }); ok { - ns.BalanceLoad = h.BalanceLoad +func (x *loadBalancerBalanceLoadClient) Send(m *LoadBalanceRequest) error { + return x.ClientStream.SendMsg(m) +} + +func (x *loadBalancerBalanceLoadClient) Recv() (*LoadBalanceResponse, error) { + m := new(LoadBalanceResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err } - return ns + return m, nil } -// UnstableLoadBalancerService is the service API for LoadBalancer service. -// New methods may be added to this interface if they are added to the service -// definition, which is not a backward-compatible change. For this reason, -// use of this type is not recommended. -type UnstableLoadBalancerService interface { +// LoadBalancerServer is the server API for LoadBalancer service. +// All implementations should embed UnimplementedLoadBalancerServer +// for forward compatibility +type LoadBalancerServer interface { // Bidirectional rpc to get a list of servers. BalanceLoad(LoadBalancer_BalanceLoadServer) error } + +// UnimplementedLoadBalancerServer should be embedded to have forward compatible implementations. +type UnimplementedLoadBalancerServer struct { +} + +func (UnimplementedLoadBalancerServer) BalanceLoad(LoadBalancer_BalanceLoadServer) error { + return status.Errorf(codes.Unimplemented, "method BalanceLoad not implemented") +} + +// UnsafeLoadBalancerServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to LoadBalancerServer will +// result in compilation errors. +type UnsafeLoadBalancerServer interface { + mustEmbedUnimplementedLoadBalancerServer() +} + +func RegisterLoadBalancerServer(s grpc.ServiceRegistrar, srv LoadBalancerServer) { + s.RegisterService(&LoadBalancer_ServiceDesc, srv) +} + +func _LoadBalancer_BalanceLoad_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(LoadBalancerServer).BalanceLoad(&loadBalancerBalanceLoadServer{stream}) +} + +type LoadBalancer_BalanceLoadServer interface { + Send(*LoadBalanceResponse) error + Recv() (*LoadBalanceRequest, error) + grpc.ServerStream +} + +type loadBalancerBalanceLoadServer struct { + grpc.ServerStream +} + +func (x *loadBalancerBalanceLoadServer) Send(m *LoadBalanceResponse) error { + return x.ServerStream.SendMsg(m) +} + +func (x *loadBalancerBalanceLoadServer) Recv() (*LoadBalanceRequest, error) { + m := new(LoadBalanceRequest) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +// LoadBalancer_ServiceDesc is the grpc.ServiceDesc for LoadBalancer service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var LoadBalancer_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.lb.v1.LoadBalancer", + HandlerType: (*LoadBalancerServer)(nil), + Methods: []grpc.MethodDesc{}, + Streams: []grpc.StreamDesc{ + { + StreamName: "BalanceLoad", + Handler: _LoadBalancer_BalanceLoad_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "grpc/lb/v1/load_balancer.proto", +} diff --git a/balancer/grpclb/grpclb.go b/balancer/grpclb/grpclb.go index a7424cf8d2d7..f2ddfc3788ed 100644 --- a/balancer/grpclb/grpclb.go +++ b/balancer/grpclb/grpclb.go @@ -19,12 +19,14 @@ // Package grpclb defines a grpclb balancer. // // To install grpclb balancer, import this package as: -// import _ "google.golang.org/grpc/balancer/grpclb" +// +// import _ "google.golang.org/grpc/balancer/grpclb" package grpclb import ( "context" "errors" + "fmt" "sync" "time" @@ -134,7 +136,8 @@ func (b *lbBuilder) Build(cc balancer.ClientConn, opt balancer.BuildOptions) bal lb := &lbBalancer{ cc: newLBCacheClientConn(cc), - target: opt.Target.Endpoint, + dialTarget: opt.Target.Endpoint(), + target: opt.Target.Endpoint(), opt: opt, fallbackTimeout: b.fallbackTimeout, doneCh: make(chan struct{}), @@ -163,9 +166,10 @@ func (b *lbBuilder) Build(cc balancer.ClientConn, opt balancer.BuildOptions) bal } type lbBalancer struct { - cc *lbCacheClientConn - target string - opt balancer.BuildOptions + cc *lbCacheClientConn + dialTarget string // user's dial target + target string // same as dialTarget unless overridden in service config + opt balancer.BuildOptions usePickFirst bool @@ -209,7 +213,7 @@ type lbBalancer struct { backendAddrsWithoutMetadata []resolver.Address // Roundrobin functionalities. state connectivity.State - subConns map[resolver.Address]balancer.SubConn // Used to new/remove SubConn. + subConns map[resolver.Address]balancer.SubConn // Used to new/shutdown SubConn. scStates map[balancer.SubConn]connectivity.State // Used to filter READY SubConns. picker balancer.Picker // Support fallback to resolved backend addresses if there's no response @@ -221,16 +225,18 @@ type lbBalancer struct { // when resolved address updates are received, and read in the goroutine // handling fallback. resolvedBackendAddrs []resolver.Address + connErr error // the last connection error } // regeneratePicker takes a snapshot of the balancer, and generates a picker from // it. The picker -// - always returns ErrTransientFailure if the balancer is in TransientFailure, -// - does two layer roundrobin pick otherwise. +// - always returns ErrTransientFailure if the balancer is in TransientFailure, +// - does two layer roundrobin pick otherwise. +// // Caller must hold lb.mu. func (lb *lbBalancer) regeneratePicker(resetDrop bool) { if lb.state == connectivity.TransientFailure { - lb.picker = &errPicker{err: balancer.ErrTransientFailure} + lb.picker = &errPicker{err: fmt.Errorf("all SubConns are in TransientFailure, last connection error: %v", lb.connErr)} return } @@ -284,12 +290,16 @@ func (lb *lbBalancer) regeneratePicker(resetDrop bool) { // aggregateSubConnStats calculate the aggregated state of SubConns in // lb.SubConns. These SubConns are subconns in use (when switching between // fallback and grpclb). lb.scState contains states for all SubConns, including -// those in cache (SubConns are cached for 10 seconds after remove). +// those in cache (SubConns are cached for 10 seconds after shutdown). // -// The aggregated state is: -// - If at least one SubConn in Ready, the aggregated state is Ready; -// - Else if at least one SubConn in Connecting, the aggregated state is Connecting; -// - Else the aggregated state is TransientFailure. +// The aggregated state is: +// - If at least one SubConn in Ready, the aggregated state is Ready; +// - Else if at least one SubConn in Connecting or IDLE, the aggregated state is Connecting; +// - It's OK to consider IDLE as Connecting. SubConns never stay in IDLE, +// they start to connect immediately. But there's a race between the overall +// state is reported, and when the new SubConn state arrives. And SubConns +// never go back to IDLE. +// - Else the aggregated state is TransientFailure. func (lb *lbBalancer) aggregateSubConnStates() connectivity.State { var numConnecting uint64 @@ -298,7 +308,7 @@ func (lb *lbBalancer) aggregateSubConnStates() connectivity.State { switch state { case connectivity.Ready: return connectivity.Ready - case connectivity.Connecting: + case connectivity.Connecting, connectivity.Idle: numConnecting++ } } @@ -309,7 +319,13 @@ func (lb *lbBalancer) aggregateSubConnStates() connectivity.State { return connectivity.TransientFailure } +// UpdateSubConnState is unused; NewSubConn's options always specifies +// updateSubConnState as the listener. func (lb *lbBalancer) UpdateSubConnState(sc balancer.SubConn, scs balancer.SubConnState) { + logger.Errorf("grpclb: UpdateSubConnState(%v, %+v) called unexpectedly", sc, scs) +} + +func (lb *lbBalancer) updateSubConnState(sc balancer.SubConn, scs balancer.SubConnState) { s := scs.ConnectivityState if logger.V(2) { logger.Infof("lbBalancer: handle SubConn state change: %p, %v", sc, s) @@ -329,9 +345,11 @@ func (lb *lbBalancer) UpdateSubConnState(sc balancer.SubConn, scs balancer.SubCo case connectivity.Idle: sc.Connect() case connectivity.Shutdown: - // When an address was removed by resolver, b called RemoveSubConn but - // kept the sc's state in scStates. Remove state for this sc here. + // When an address was removed by resolver, b called Shutdown but kept + // the sc's state in scStates. Remove state for this sc here. delete(lb.scStates, sc) + case connectivity.TransientFailure: + lb.connErr = scs.ConnectionError } // Force regenerate picker if // - this sc became ready from not-ready @@ -361,8 +379,13 @@ func (lb *lbBalancer) updateStateAndPicker(forceRegeneratePicker bool, resetDrop if forceRegeneratePicker || (lb.state != oldAggrState) { lb.regeneratePicker(resetDrop) } + var cc balancer.ClientConn = lb.cc + if lb.usePickFirst { + // Bypass the caching layer that would wrap the picker. + cc = lb.cc.ClientConn + } - lb.cc.UpdateState(balancer.State{ConnectivityState: lb.state, Picker: lb.picker}) + cc.UpdateState(balancer.State{ConnectivityState: lb.state, Picker: lb.picker}) } // fallbackToBackendsAfter blocks for fallbackTimeout and falls back to use @@ -390,6 +413,30 @@ func (lb *lbBalancer) handleServiceConfig(gc *grpclbServiceConfig) { lb.mu.Lock() defer lb.mu.Unlock() + // grpclb uses the user's dial target to populate the `Name` field of the + // `InitialLoadBalanceRequest` message sent to the remote balancer. But when + // grpclb is used a child policy in the context of RLS, we want the `Name` + // field to be populated with the value received from the RLS server. To + // support this use case, an optional "target_name" field has been added to + // the grpclb LB policy's config. If specified, it overrides the name of + // the target to be sent to the remote balancer; if not, the target to be + // sent to the balancer will continue to be obtained from the target URI + // passed to the gRPC client channel. Whenever that target to be sent to the + // balancer is updated, we need to restart the stream to the balancer as + // this target is sent in the first message on the stream. + if gc != nil { + target := lb.dialTarget + if gc.ServiceName != "" { + target = gc.ServiceName + } + if target != lb.target { + lb.target = target + if lb.ccRemoteLB != nil { + lb.ccRemoteLB.cancelRemoteBalancerCall() + } + } + } + newUsePickFirst := childIsPickFirst(gc) if lb.usePickFirst == newUsePickFirst { return @@ -412,17 +459,9 @@ func (lb *lbBalancer) UpdateClientConnState(ccs balancer.ClientConnState) error gc, _ := ccs.BalancerConfig.(*grpclbServiceConfig) lb.handleServiceConfig(gc) - addrs := ccs.ResolverState.Addresses + backendAddrs := ccs.ResolverState.Addresses - var remoteBalancerAddrs, backendAddrs []resolver.Address - for _, a := range addrs { - if a.Type == resolver.GRPCLB { - a.Type = resolver.Backend - remoteBalancerAddrs = append(remoteBalancerAddrs, a) - } else { - backendAddrs = append(backendAddrs, a) - } - } + var remoteBalancerAddrs []resolver.Address if sd := grpclbstate.Get(ccs.ResolverState); sd != nil { // Override any balancer addresses provided via // ccs.ResolverState.Addresses. @@ -480,3 +519,5 @@ func (lb *lbBalancer) Close() { } lb.cc.close() } + +func (lb *lbBalancer) ExitIdle() {} diff --git a/balancer/grpclb/grpclb_config.go b/balancer/grpclb/grpclb_config.go index aac3719631b4..8942c31310af 100644 --- a/balancer/grpclb/grpclb_config.go +++ b/balancer/grpclb/grpclb_config.go @@ -34,6 +34,7 @@ const ( type grpclbServiceConfig struct { serviceconfig.LoadBalancingConfig ChildPolicy *[]map[string]json.RawMessage + ServiceName string } func (b *lbBuilder) ParseConfig(lbConfig json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { diff --git a/balancer/grpclb/grpclb_config_test.go b/balancer/grpclb/grpclb_config_test.go index 5a45de90494b..040908728793 100644 --- a/balancer/grpclb/grpclb_config_test.go +++ b/balancer/grpclb/grpclb_config_test.go @@ -20,52 +20,68 @@ package grpclb import ( "encoding/json" - "errors" - "fmt" - "reflect" - "strings" "testing" + "github.com/google/go-cmp/cmp" "google.golang.org/grpc/serviceconfig" ) func (s) TestParse(t *testing.T) { tests := []struct { name string - s string + sc string want serviceconfig.LoadBalancingConfig - wantErr error + wantErr bool }{ { name: "empty", - s: "", + sc: "", want: nil, - wantErr: errors.New("unexpected end of JSON input"), + wantErr: true, }, { name: "success1", - s: `{"childPolicy":[{"pick_first":{}}]}`, + sc: ` +{ + "childPolicy": [ + {"pick_first":{}} + ], + "serviceName": "foo-service" +}`, want: &grpclbServiceConfig{ ChildPolicy: &[]map[string]json.RawMessage{ {"pick_first": json.RawMessage("{}")}, }, + ServiceName: "foo-service", }, }, { name: "success2", - s: `{"childPolicy":[{"round_robin":{}},{"pick_first":{}}]}`, + sc: ` +{ + "childPolicy": [ + {"round_robin":{}}, + {"pick_first":{}} + ], + "serviceName": "foo-service" +}`, want: &grpclbServiceConfig{ ChildPolicy: &[]map[string]json.RawMessage{ {"round_robin": json.RawMessage("{}")}, {"pick_first": json.RawMessage("{}")}, }, + ServiceName: "foo-service", }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got, err := (&lbBuilder{}).ParseConfig(json.RawMessage(tt.s)); !reflect.DeepEqual(got, tt.want) || !strings.Contains(fmt.Sprint(err), fmt.Sprint(tt.wantErr)) { - t.Errorf("parseFullServiceConfig() = %+v, %+v, want %+v, ", got, err, tt.want, tt.wantErr) + got, err := (&lbBuilder{}).ParseConfig(json.RawMessage(tt.sc)) + if (err != nil) != (tt.wantErr) { + t.Fatalf("ParseConfig(%q) returned error: %v, wantErr: %v", tt.sc, err, tt.wantErr) + } + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Fatalf("ParseConfig(%q) returned unexpected difference (-want +got):\n%s", tt.sc, diff) } }) } diff --git a/balancer/grpclb/grpclb_remote_balancer.go b/balancer/grpclb/grpclb_remote_balancer.go index 8eb45be28e32..edb66a90a3b1 100644 --- a/balancer/grpclb/grpclb_remote_balancer.go +++ b/balancer/grpclb/grpclb_remote_balancer.go @@ -33,8 +33,9 @@ import ( "google.golang.org/grpc/balancer" lbpb "google.golang.org/grpc/balancer/grpclb/grpc_lb_v1" "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/internal/backoff" - "google.golang.org/grpc/internal/channelz" + imetadata "google.golang.org/grpc/internal/metadata" "google.golang.org/grpc/keepalive" "google.golang.org/grpc/metadata" "google.golang.org/grpc/resolver" @@ -76,10 +77,7 @@ func (lb *lbBalancer) processServerList(l *lbpb.ServerList) { // net.SplitHostPort() will return too many colons error. ipStr = fmt.Sprintf("[%s]", ipStr) } - addr := resolver.Address{ - Addr: fmt.Sprintf("%s:%d", ipStr, s.Port), - Metadata: &md, - } + addr := imetadata.Set(resolver.Address{Addr: fmt.Sprintf("%s:%d", ipStr, s.Port)}, md) if logger.V(2) { logger.Infof("lbBalancer: server list entry[%d]: ipStr:|%s|, port:|%d|, load balancer token:|%v|", i, ipStr, s.Port, s.LoadBalanceToken) @@ -115,7 +113,6 @@ func (lb *lbBalancer) refreshSubConns(backendAddrs []resolver.Address, fallback } balancingPolicyChanged := lb.usePickFirst != pickFirst - oldUsePickFirst := lb.usePickFirst lb.usePickFirst = pickFirst if fallbackModeChanged || balancingPolicyChanged { @@ -125,29 +122,32 @@ func (lb *lbBalancer) refreshSubConns(backendAddrs []resolver.Address, fallback // For fallback mode switching with pickfirst, we want to recreate the // SubConn because the creds could be different. for a, sc := range lb.subConns { - if oldUsePickFirst { - // If old SubConn were created for pickfirst, bypass cache and - // remove directly. - lb.cc.cc.RemoveSubConn(sc) - } else { - lb.cc.RemoveSubConn(sc) - } + sc.Shutdown() delete(lb.subConns, a) } } if lb.usePickFirst { - var sc balancer.SubConn - for _, sc = range lb.subConns { + var ( + scKey resolver.Address + sc balancer.SubConn + ) + for scKey, sc = range lb.subConns { break } if sc != nil { - sc.UpdateAddresses(backendAddrs) + if len(backendAddrs) == 0 { + sc.Shutdown() + delete(lb.subConns, scKey) + return + } + lb.cc.ClientConn.UpdateAddresses(sc, backendAddrs) sc.Connect() return } + opts.StateListener = func(scs balancer.SubConnState) { lb.updateSubConnState(sc, scs) } // This bypasses the cc wrapper with SubConn cache. - sc, err := lb.cc.cc.NewSubConn(backendAddrs, opts) + sc, err := lb.cc.ClientConn.NewSubConn(backendAddrs, opts) if err != nil { logger.Warningf("grpclb: failed to create new SubConn: %v", err) return @@ -163,19 +163,21 @@ func (lb *lbBalancer) refreshSubConns(backendAddrs []resolver.Address, fallback addrsSet := make(map[resolver.Address]struct{}) // Create new SubConns. for _, addr := range backendAddrs { - addrWithoutMD := addr - addrWithoutMD.Metadata = nil - addrsSet[addrWithoutMD] = struct{}{} - lb.backendAddrsWithoutMetadata = append(lb.backendAddrsWithoutMetadata, addrWithoutMD) + addrWithoutAttrs := addr + addrWithoutAttrs.Attributes = nil + addrsSet[addrWithoutAttrs] = struct{}{} + lb.backendAddrsWithoutMetadata = append(lb.backendAddrsWithoutMetadata, addrWithoutAttrs) - if _, ok := lb.subConns[addrWithoutMD]; !ok { + if _, ok := lb.subConns[addrWithoutAttrs]; !ok { // Use addrWithMD to create the SubConn. + var sc balancer.SubConn + opts.StateListener = func(scs balancer.SubConnState) { lb.updateSubConnState(sc, scs) } sc, err := lb.cc.NewSubConn([]resolver.Address{addr}, opts) if err != nil { logger.Warningf("grpclb: failed to create new SubConn: %v", err) continue } - lb.subConns[addrWithoutMD] = sc // Use the addr without MD as key for the map. + lb.subConns[addrWithoutAttrs] = sc // Use the addr without MD as key for the map. if _, ok := lb.scStates[sc]; !ok { // Only set state of new sc to IDLE. The state could already be // READY for cached SubConns. @@ -188,7 +190,7 @@ func (lb *lbBalancer) refreshSubConns(backendAddrs []resolver.Address, fallback for a, sc := range lb.subConns { // a was removed by resolver. if _, ok := addrsSet[a]; !ok { - lb.cc.RemoveSubConn(sc) + sc.Shutdown() delete(lb.subConns, a) // Keep the state of this sc in b.scStates until sc's state becomes Shutdown. // The entry will be deleted in UpdateSubConnState. @@ -208,6 +210,9 @@ type remoteBalancerCCWrapper struct { backoff backoff.Strategy done chan struct{} + streamMu sync.Mutex + streamCancel func() + // waitgroup to wait for all goroutines to exit. wg sync.WaitGroup } @@ -219,17 +224,18 @@ func (lb *lbBalancer) newRemoteBalancerCCWrapper() { } else if bundle := lb.grpclbClientConnCreds; bundle != nil { dopts = append(dopts, grpc.WithCredentialsBundle(bundle)) } else { - dopts = append(dopts, grpc.WithInsecure()) + dopts = append(dopts, grpc.WithTransportCredentials(insecure.NewCredentials())) } if lb.opt.Dialer != nil { dopts = append(dopts, grpc.WithContextDialer(lb.opt.Dialer)) } + if lb.opt.CustomUserAgent != "" { + dopts = append(dopts, grpc.WithUserAgent(lb.opt.CustomUserAgent)) + } // Explicitly set pickfirst as the balancer. dopts = append(dopts, grpc.WithDefaultServiceConfig(`{"loadBalancingPolicy":"pick_first"}`)) dopts = append(dopts, grpc.WithResolvers(lb.manualResolver)) - if channelz.IsOn() { - dopts = append(dopts, grpc.WithChannelzParentID(lb.opt.ChannelzParentID)) - } + dopts = append(dopts, grpc.WithChannelzParentID(lb.opt.ChannelzParentID)) // Enable Keepalive for grpclb client. dopts = append(dopts, grpc.WithKeepaliveParams(keepalive.ClientParameters{ @@ -318,13 +324,11 @@ func (ccw *remoteBalancerCCWrapper) sendLoadReport(s *balanceLoadClientStream, i } } -func (ccw *remoteBalancerCCWrapper) callRemoteBalancer() (backoff bool, _ error) { +func (ccw *remoteBalancerCCWrapper) callRemoteBalancer(ctx context.Context) (backoff bool, _ error) { lbClient := &loadBalancerClient{cc: ccw.cc} - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() stream, err := lbClient.BalanceLoad(ctx, grpc.WaitForReady(true)) if err != nil { - return true, fmt.Errorf("grpclb: failed to perform RPC to the remote balancer %v", err) + return true, fmt.Errorf("grpclb: failed to perform RPC to the remote balancer: %v", err) } ccw.lb.mu.Lock() ccw.lb.remoteBalancerConnected = true @@ -361,11 +365,43 @@ func (ccw *remoteBalancerCCWrapper) callRemoteBalancer() (backoff bool, _ error) return false, ccw.readServerList(stream) } +// cancelRemoteBalancerCall cancels the context used by the stream to the remote +// balancer. watchRemoteBalancer() takes care of restarting this call after the +// stream fails. +func (ccw *remoteBalancerCCWrapper) cancelRemoteBalancerCall() { + ccw.streamMu.Lock() + if ccw.streamCancel != nil { + ccw.streamCancel() + ccw.streamCancel = nil + } + ccw.streamMu.Unlock() +} + func (ccw *remoteBalancerCCWrapper) watchRemoteBalancer() { - defer ccw.wg.Done() + defer func() { + ccw.wg.Done() + ccw.streamMu.Lock() + if ccw.streamCancel != nil { + // This is to make sure that we don't leak the context when we are + // directly returning from inside of the below `for` loop. + ccw.streamCancel() + ccw.streamCancel = nil + } + ccw.streamMu.Unlock() + }() + var retryCount int + var ctx context.Context for { - doBackoff, err := ccw.callRemoteBalancer() + ccw.streamMu.Lock() + if ccw.streamCancel != nil { + ccw.streamCancel() + ccw.streamCancel = nil + } + ctx, ccw.streamCancel = context.WithCancel(context.Background()) + ccw.streamMu.Unlock() + + doBackoff, err := ccw.callRemoteBalancer(ctx) select { case <-ccw.done: return @@ -379,7 +415,7 @@ func (ccw *remoteBalancerCCWrapper) watchRemoteBalancer() { } } // Trigger a re-resolve when the stream errors. - ccw.lb.cc.cc.ResolveNow(resolver.ResolveNowOptions{}) + ccw.lb.cc.ClientConn.ResolveNow(resolver.ResolveNowOptions{}) ccw.lb.mu.Lock() ccw.lb.remoteBalancerConnected = false diff --git a/balancer/grpclb/grpclb_test.go b/balancer/grpclb/grpclb_test.go index 5a2297d43ec8..69e842cb435d 100644 --- a/balancer/grpclb/grpclb_test.go +++ b/balancer/grpclb/grpclb_test.go @@ -31,22 +31,31 @@ import ( "testing" "time" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "google.golang.org/grpc" "google.golang.org/grpc/balancer" grpclbstate "google.golang.org/grpc/balancer/grpclb/state" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/internal" "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/pickfirst" + "google.golang.org/grpc/internal/testutils/roundrobin" "google.golang.org/grpc/metadata" "google.golang.org/grpc/peer" "google.golang.org/grpc/resolver" "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/serviceconfig" "google.golang.org/grpc/status" durationpb "github.com/golang/protobuf/ptypes/duration" lbgrpc "google.golang.org/grpc/balancer/grpclb/grpc_lb_v1" lbpb "google.golang.org/grpc/balancer/grpclb/grpc_lb_v1" - testpb "google.golang.org/grpc/test/grpc_testing" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" ) var ( @@ -60,6 +69,13 @@ var ( fakeName = "fake.Name" ) +const ( + defaultTestTimeout = 10 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond + testUserAgent = "test-user-agent" + grpclbConfig = `{"loadBalancingConfig": [{"grpclb": {}}]}` +) + type s struct { grpctest.Tester } @@ -136,18 +152,6 @@ func (s *rpcStats) merge(cs *lbpb.ClientStats) { s.mu.Unlock() } -func mapsEqual(a, b map[string]int64) bool { - if len(a) != len(b) { - return false - } - for k, v1 := range a { - if v2, ok := b[k]; !ok || v1 != v2 { - return false - } - } - return true -} - func atomicEqual(a, b *int64) bool { return atomic.LoadInt64(a) == atomic.LoadInt64(b) } @@ -172,7 +176,7 @@ func (s *rpcStats) equal(o *rpcStats) bool { defer s.mu.Unlock() o.mu.Lock() defer o.mu.Unlock() - return mapsEqual(s.numCallsDropped, o.numCallsDropped) + return cmp.Equal(s.numCallsDropped, o.numCallsDropped, cmpopts.EquateEmpty()) } func (s *rpcStats) String() string { @@ -188,21 +192,28 @@ func (s *rpcStats) String() string { type remoteBalancer struct { lbgrpc.UnimplementedLoadBalancerServer - sls chan *lbpb.ServerList - statsDura time.Duration - done chan struct{} - stats *rpcStats - statsChan chan *lbpb.ClientStats - fbChan chan struct{} + sls chan *lbpb.ServerList + statsDura time.Duration + done chan struct{} + stats *rpcStats + statsChan chan *lbpb.ClientStats + fbChan chan struct{} + balanceLoadCh chan struct{} // notify successful invocation of BalanceLoad + + wantUserAgent string // expected user-agent in metadata of BalancerLoad + wantServerName string // expected server name in InitialLoadBalanceRequest } -func newRemoteBalancer(intervals []time.Duration, statsChan chan *lbpb.ClientStats) *remoteBalancer { +func newRemoteBalancer(wantUserAgent, wantServerName string, statsChan chan *lbpb.ClientStats) *remoteBalancer { return &remoteBalancer{ - sls: make(chan *lbpb.ServerList, 1), - done: make(chan struct{}), - stats: newRPCStats(), - statsChan: statsChan, - fbChan: make(chan struct{}), + sls: make(chan *lbpb.ServerList, 1), + done: make(chan struct{}), + stats: newRPCStats(), + statsChan: statsChan, + fbChan: make(chan struct{}), + balanceLoadCh: make(chan struct{}, 1), + wantUserAgent: wantUserAgent, + wantServerName: wantServerName, } } @@ -215,15 +226,30 @@ func (b *remoteBalancer) fallbackNow() { b.fbChan <- struct{}{} } +func (b *remoteBalancer) updateServerName(name string) { + b.wantServerName = name +} + func (b *remoteBalancer) BalanceLoad(stream lbgrpc.LoadBalancer_BalanceLoadServer) error { + md, ok := metadata.FromIncomingContext(stream.Context()) + if !ok { + return status.Error(codes.Internal, "failed to receive metadata") + } + if b.wantUserAgent != "" { + if ua := md["user-agent"]; len(ua) == 0 || !strings.HasPrefix(ua[0], b.wantUserAgent) { + return status.Errorf(codes.InvalidArgument, "received unexpected user-agent: %v, want prefix %q", ua, b.wantUserAgent) + } + } + req, err := stream.Recv() if err != nil { return err } initReq := req.GetInitialRequest() - if initReq.Name != beServerName { - return status.Errorf(codes.InvalidArgument, "invalid service name: %v", initReq.Name) + if initReq.Name != b.wantServerName { + return status.Errorf(codes.InvalidArgument, "invalid service name: %q, want: %q", initReq.Name, b.wantServerName) } + b.balanceLoadCh <- struct{}{} resp := &lbpb.LoadBalanceResponse{ LoadBalanceResponseType: &lbpb.LoadBalanceResponse_InitialResponse{ InitialResponse: &lbpb.InitialLoadBalanceResponse{ @@ -239,11 +265,8 @@ func (b *remoteBalancer) BalanceLoad(stream lbgrpc.LoadBalancer_BalanceLoadServe } go func() { for { - var ( - req *lbpb.LoadBalanceRequest - err error - ) - if req, err = stream.Recv(); err != nil { + req, err := stream.Recv() + if err != nil { return } b.stats.merge(req.GetClientStats()) @@ -276,6 +299,8 @@ func (b *remoteBalancer) BalanceLoad(stream lbgrpc.LoadBalancer_BalanceLoadServe } type testServer struct { + testgrpc.UnimplementedTestServiceServer + addr string fallback bool } @@ -287,32 +312,29 @@ func (s *testServer) EmptyCall(ctx context.Context, in *testpb.Empty) (*testpb.E if !ok { return nil, status.Error(codes.Internal, "failed to receive metadata") } - if !s.fallback && (md == nil || md["lb-token"][0] != lbToken) { + if !s.fallback && (md == nil || len(md["lb-token"]) == 0 || md["lb-token"][0] != lbToken) { return nil, status.Errorf(codes.Internal, "received unexpected metadata: %v", md) } grpc.SetTrailer(ctx, metadata.Pairs(testmdkey, s.addr)) return &testpb.Empty{}, nil } -func (s *testServer) FullDuplexCall(stream testpb.TestService_FullDuplexCallServer) error { +func (s *testServer) FullDuplexCall(stream testgrpc.TestService_FullDuplexCallServer) error { return nil } -func startBackends(sn string, fallback bool, lis ...net.Listener) (servers []*grpc.Server) { +func startBackends(t *testing.T, sn string, fallback bool, lis ...net.Listener) (servers []*grpc.Server) { for _, l := range lis { creds := &serverNameCheckCreds{ sn: sn, } s := grpc.NewServer(grpc.Creds(creds)) - ts := &testServer{addr: l.Addr().String(), fallback: fallback} - testpb.RegisterTestServiceService(s, &testpb.TestServiceService{ - EmptyCall: ts.EmptyCall, - FullDuplexCall: ts.FullDuplexCall, - }) + testgrpc.RegisterTestServiceServer(s, &testServer{addr: l.Addr().String(), fallback: fallback}) servers = append(servers, s) go func(s *grpc.Server, l net.Listener) { s.Serve(l) }(s, l) + t.Logf("Started backend server listening on %s", l.Addr().String()) } return } @@ -335,7 +357,7 @@ type testServers struct { beListeners []net.Listener } -func newLoadBalancer(numberOfBackends int, statsChan chan *lbpb.ClientStats) (tss *testServers, cleanup func(), err error) { +func startBackendsAndRemoteLoadBalancer(t *testing.T, numberOfBackends int, customUserAgent string, statsChan chan *lbpb.ClientStats) (tss *testServers, cleanup func(), err error) { var ( beListeners []net.Listener ls *remoteBalancer @@ -344,7 +366,6 @@ func newLoadBalancer(numberOfBackends int, statsChan chan *lbpb.ClientStats) (ts bePorts []int ) for i := 0; i < numberOfBackends; i++ { - // Start a backend. beLis, e := net.Listen("tcp", "localhost:0") if e != nil { err = fmt.Errorf("failed to listen %v", err) @@ -353,26 +374,26 @@ func newLoadBalancer(numberOfBackends int, statsChan chan *lbpb.ClientStats) (ts beIPs = append(beIPs, beLis.Addr().(*net.TCPAddr).IP) bePorts = append(bePorts, beLis.Addr().(*net.TCPAddr).Port) - beListeners = append(beListeners, newRestartableListener(beLis)) + beListeners = append(beListeners, testutils.NewRestartableListener(beLis)) } - backends := startBackends(beServerName, false, beListeners...) + backends := startBackends(t, beServerName, false, beListeners...) - // Start a load balancer. lbLis, err := net.Listen("tcp", "localhost:0") if err != nil { err = fmt.Errorf("failed to create the listener for the load balancer %v", err) return } - lbLis = newRestartableListener(lbLis) + lbLis = testutils.NewRestartableListener(lbLis) lbCreds := &serverNameCheckCreds{ sn: lbServerName, } lb = grpc.NewServer(grpc.Creds(lbCreds)) - ls = newRemoteBalancer(nil, statsChan) + ls = newRemoteBalancer(customUserAgent, beServerName, statsChan) lbgrpc.RegisterLoadBalancerServer(lb, ls) go func() { lb.Serve(lbLis) }() + t.Logf("Started remote load balancer server listening on %s", lbLis.Addr().String()) tss = &testServers{ lbAddr: net.JoinHostPort(fakeName, strconv.Itoa(lbLis.Addr().(*net.TCPAddr).Port)), @@ -395,59 +416,69 @@ func newLoadBalancer(numberOfBackends int, statsChan chan *lbpb.ClientStats) (ts return } -var grpclbConfig = `{"loadBalancingConfig": [{"grpclb": {}}]}` - -func (s) TestGRPCLB(t *testing.T) { - r := manual.NewBuilderWithScheme("whatever") - - tss, cleanup, err := newLoadBalancer(1, nil) +// TestGRPCLB_Basic tests the basic case of a channel being configured with +// grpclb as the load balancing policy. +func (s) TestGRPCLB_Basic(t *testing.T) { + tss, cleanup, err := startBackendsAndRemoteLoadBalancer(t, 1, testUserAgent, nil) if err != nil { t.Fatalf("failed to create new load balancer: %v", err) } defer cleanup() - be := &lbpb.Server{ - IpAddress: tss.beIPs[0], - Port: int32(tss.bePorts[0]), - LoadBalanceToken: lbToken, + // Push the test backend address to the remote balancer. + tss.ls.sls <- &lbpb.ServerList{ + Servers: []*lbpb.Server{ + { + IpAddress: tss.beIPs[0], + Port: int32(tss.bePorts[0]), + LoadBalanceToken: lbToken, + }, + }, } - var bes []*lbpb.Server - bes = append(bes, be) - sl := &lbpb.ServerList{ - Servers: bes, + + // Configure the manual resolver with an initial state containing a service + // config with grpclb as the load balancing policy and the remote balancer + // address specified via attributes. + r := manual.NewBuilderWithScheme("whatever") + s := &grpclbstate.State{ + BalancerAddresses: []resolver.Address{ + { + Addr: tss.lbAddr, + ServerName: lbServerName, + }, + }, } - tss.ls.sls <- sl - creds := serverNameCheckCreds{} - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - cc, err := grpc.DialContext(ctx, r.Scheme()+":///"+beServerName, grpc.WithResolvers(r), - grpc.WithTransportCredentials(&creds), grpc.WithContextDialer(fakeNameDialer)) + rs := grpclbstate.Set(resolver.State{ServiceConfig: internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(grpclbConfig)}, s) + r.InitialState(rs) + + // Connect to the test backend. + dopts := []grpc.DialOption{ + grpc.WithResolvers(r), + grpc.WithTransportCredentials(&serverNameCheckCreds{}), + grpc.WithContextDialer(fakeNameDialer), + grpc.WithUserAgent(testUserAgent), + } + cc, err := grpc.Dial(r.Scheme()+":///"+beServerName, dopts...) if err != nil { t.Fatalf("Failed to dial to the backend %v", err) } defer cc.Close() - testC := testpb.NewTestServiceClient(cc) - - rs := grpclbstate.Set(resolver.State{ServiceConfig: r.CC.ParseServiceConfig(grpclbConfig)}, - &grpclbstate.State{BalancerAddresses: []resolver.Address{{ - Addr: tss.lbAddr, - Type: resolver.Backend, - ServerName: lbServerName, - }}}) - r.UpdateState(rs) - ctx, cancel = context.WithTimeout(context.Background(), 5*time.Second) + // Make one successful RPC. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() - if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + testC := testgrpc.NewTestServiceClient(cc) + if _, err := testC.EmptyCall(ctx, &testpb.Empty{}); err != nil { t.Fatalf("%v.EmptyCall(_, _) = _, %v, want _, ", testC, err) } } -// The remote balancer sends response with duplicates to grpclb client. -func (s) TestGRPCLBWeighted(t *testing.T) { - r := manual.NewBuilderWithScheme("whatever") - - tss, cleanup, err := newLoadBalancer(2, nil) +// TestGRPCLB_Weighted tests weighted roundrobin. The remote balancer is +// configured to send a response with duplicate backend addresses (to simulate +// weights) to the grpclb client. The test verifies that RPCs are weighted +// roundrobin-ed across these backends. +func (s) TestGRPCLB_Weighted(t *testing.T) { + tss, cleanup, err := startBackendsAndRemoteLoadBalancer(t, 2, "", nil) if err != nil { t.Fatalf("failed to create new load balancer: %v", err) } @@ -462,57 +493,67 @@ func (s) TestGRPCLBWeighted(t *testing.T) { Port: int32(tss.bePorts[1]), LoadBalanceToken: lbToken, }} - portsToIndex := make(map[int]int) - for i := range beServers { - portsToIndex[tss.bePorts[i]] = i + + // Configure the manual resolver with an initial state containing a service + // config with grpclb as the load balancing policy and the remote balancer + // address specified via attributes. + r := manual.NewBuilderWithScheme("whatever") + s := &grpclbstate.State{ + BalancerAddresses: []resolver.Address{ + { + Addr: tss.lbAddr, + ServerName: lbServerName, + }, + }, } + rs := grpclbstate.Set(resolver.State{ServiceConfig: internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(grpclbConfig)}, s) + r.InitialState(rs) - creds := serverNameCheckCreds{} - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - cc, err := grpc.DialContext(ctx, r.Scheme()+":///"+beServerName, grpc.WithResolvers(r), - grpc.WithTransportCredentials(&creds), grpc.WithContextDialer(fakeNameDialer)) + // Connect to test backends. + dopts := []grpc.DialOption{ + grpc.WithResolvers(r), + grpc.WithTransportCredentials(&serverNameCheckCreds{}), + grpc.WithContextDialer(fakeNameDialer), + } + cc, err := grpc.Dial(r.Scheme()+":///"+beServerName, dopts...) if err != nil { t.Fatalf("Failed to dial to the backend %v", err) } defer cc.Close() - testC := testpb.NewTestServiceClient(cc) - r.UpdateState(resolver.State{Addresses: []resolver.Address{{ - Addr: tss.lbAddr, - Type: resolver.GRPCLB, - ServerName: lbServerName, - }}}) - - sequences := []string{"00101", "00011"} + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + // Sequence represents the sequence of backends to be returned from the + // remote load balancer. + sequences := [][]int{ + {0, 0, 1, 0, 1}, + {0, 0, 0, 1, 1}, + } for _, seq := range sequences { - var ( - bes []*lbpb.Server - p peer.Peer - result string - ) + // Push the configured sequence of backend to the remote balancer, and + // compute the expected addresses to which RPCs should be routed. + var backends []*lbpb.Server + var wantAddrs []resolver.Address for _, s := range seq { - bes = append(bes, beServers[s-'0']) + backends = append(backends, beServers[s]) + wantAddrs = append(wantAddrs, resolver.Address{Addr: tss.beListeners[s].Addr().String()}) } - tss.ls.sls <- &lbpb.ServerList{Servers: bes} + tss.ls.sls <- &lbpb.ServerList{Servers: backends} - for i := 0; i < 1000; i++ { - if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil { - t.Fatalf("%v.EmptyCall(_, _) = _, %v, want _, ", testC, err) - } - result += strconv.Itoa(portsToIndex[p.Addr.(*net.TCPAddr).Port]) - } - // The generated result will be in format of "0010100101". - if !strings.Contains(result, strings.Repeat(seq, 2)) { - t.Errorf("got result sequence %q, want patten %q", result, seq) + testC := testgrpc.NewTestServiceClient(cc) + if err := roundrobin.CheckWeightedRoundRobinRPCs(ctx, testC, wantAddrs); err != nil { + t.Fatal(err) } } } -func (s) TestDropRequest(t *testing.T) { - r := manual.NewBuilderWithScheme("whatever") - - tss, cleanup, err := newLoadBalancer(2, nil) +// TestGRPCLB_DropRequest tests grpclb support for dropping requests based on +// configuration received from the remote balancer. +// +// TODO: Rewrite this test to verify drop behavior using the +// ClientStats.CallsFinishedWithDrop field instead. +func (s) TestGRPCLB_DropRequest(t *testing.T) { + tss, cleanup, err := startBackendsAndRemoteLoadBalancer(t, 2, "", nil) if err != nil { t.Fatalf("failed to create new load balancer: %v", err) } @@ -532,22 +573,34 @@ func (s) TestDropRequest(t *testing.T) { Drop: true, }}, } - creds := serverNameCheckCreds{} - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - cc, err := grpc.DialContext(ctx, r.Scheme()+":///"+beServerName, grpc.WithResolvers(r), - grpc.WithTransportCredentials(&creds), grpc.WithContextDialer(fakeNameDialer)) + + // Configure the manual resolver with an initial state containing a service + // config with grpclb as the load balancing policy and the remote balancer + // address specified via attributes. + r := manual.NewBuilderWithScheme("whatever") + s := &grpclbstate.State{ + BalancerAddresses: []resolver.Address{ + { + Addr: tss.lbAddr, + ServerName: lbServerName, + }, + }, + } + rs := grpclbstate.Set(resolver.State{ServiceConfig: internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(grpclbConfig)}, s) + r.InitialState(rs) + + // Connect to test backends. + dopts := []grpc.DialOption{ + grpc.WithResolvers(r), + grpc.WithTransportCredentials(&serverNameCheckCreds{}), + grpc.WithContextDialer(fakeNameDialer), + } + cc, err := grpc.Dial(r.Scheme()+":///"+beServerName, dopts...) if err != nil { t.Fatalf("Failed to dial to the backend %v", err) } defer cc.Close() - testC := testpb.NewTestServiceClient(cc) - - r.UpdateState(resolver.State{Addresses: []resolver.Address{{ - Addr: tss.lbAddr, - Type: resolver.GRPCLB, - ServerName: lbServerName, - }}}) + testC := testgrpc.NewTestServiceClient(cc) var ( i int @@ -559,6 +612,8 @@ func (s) TestDropRequest(t *testing.T) { sleepEachLoop = time.Millisecond loopCount = int(time.Second / sleepEachLoop) ) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() // Make a non-fail-fast RPC and wait for it to succeed. for i = 0; i < loopCount; i++ { if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err == nil { @@ -607,18 +662,18 @@ func (s) TestDropRequest(t *testing.T) { // 1st RPCs pick the first item in server list. They should succeed // since they choose the non-drop-request backend according to the // round robin policy. - if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(!failfast)); err != nil { + if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(!failfast)); err != nil { t.Errorf("%v.EmptyCall(_, _) = _, %v, want _, ", testC, err) } // 2nd RPCs pick the second item in server list. They should succeed // since they choose the non-drop-request backend according to the // round robin policy. - if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(!failfast)); err != nil { + if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(!failfast)); err != nil { t.Errorf("%v.EmptyCall(_, _) = _, %v, want _, ", testC, err) } // 3rd RPCs should fail, because they pick last item in server list, // with Drop set to true. - if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(!failfast)); status.Code(err) != codes.Unavailable { + if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(!failfast)); status.Code(err) != codes.Unavailable { t.Errorf("%v.EmptyCall(_, _) = _, %v, want _, %s", testC, err, codes.Unavailable) } } @@ -627,7 +682,7 @@ func (s) TestDropRequest(t *testing.T) { // Make one more RPC to move the picker index one step further, so it's not // 0. The following RPCs will test that drop index is not reset. If picker // index is at 0, we cannot tell whether it's reset or not. - if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { t.Errorf("%v.EmptyCall(_, _) = _, %v, want _, ", testC, err) } @@ -638,19 +693,19 @@ func (s) TestDropRequest(t *testing.T) { time.Sleep(time.Second) for i := 0; i < 3; i++ { var p peer.Peer - if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil { - t.Errorf("%v.EmptyCall(_, _) = _, %v, want _, ", testC, err) + if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil { + t.Fatalf("%v.EmptyCall(_, _) = _, %v, want _, ", testC, err) } if want := tss.bePorts[1]; p.Addr.(*net.TCPAddr).Port != want { t.Errorf("got peer: %v, want peer port: %v", p.Addr, want) } - if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true)); status.Code(err) != codes.Unavailable { + if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); status.Code(err) != codes.Unavailable { t.Errorf("%v.EmptyCall(_, _) = _, %v, want _, %s", testC, err, codes.Unavailable) } - if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil { - t.Errorf("%v.EmptyCall(_, _) = _, %v, want _, ", testC, err) + if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil { + t.Fatalf("%v.EmptyCall(_, _) = _, %v, want _, ", testC, err) } if want := tss.bePorts[1]; p.Addr.(*net.TCPAddr).Port != want { t.Errorf("got peer: %v, want peer port: %v", p.Addr, want) @@ -658,436 +713,365 @@ func (s) TestDropRequest(t *testing.T) { } } -// When the balancer in use disconnects, grpclb should connect to the next address from resolved balancer address list. -func (s) TestBalancerDisconnects(t *testing.T) { - r := manual.NewBuilderWithScheme("whatever") - +// TestGRPCLB_BalancerDisconnects tests the case where the remote balancer in +// use disconnects. The test verifies that grpclb connects to the next remote +// balancer address specified in attributes, and RPCs get routed to the backends +// returned by the new balancer. +func (s) TestGRPCLB_BalancerDisconnects(t *testing.T) { var ( tests []*testServers lbs []*grpc.Server ) for i := 0; i < 2; i++ { - tss, cleanup, err := newLoadBalancer(1, nil) + tss, cleanup, err := startBackendsAndRemoteLoadBalancer(t, 1, "", nil) if err != nil { t.Fatalf("failed to create new load balancer: %v", err) } defer cleanup() - be := &lbpb.Server{ - IpAddress: tss.beIPs[0], - Port: int32(tss.bePorts[0]), - LoadBalanceToken: lbToken, - } - var bes []*lbpb.Server - bes = append(bes, be) - sl := &lbpb.ServerList{ - Servers: bes, + tss.ls.sls <- &lbpb.ServerList{ + Servers: []*lbpb.Server{ + { + IpAddress: tss.beIPs[0], + Port: int32(tss.bePorts[0]), + LoadBalanceToken: lbToken, + }, + }, } - tss.ls.sls <- sl tests = append(tests, tss) lbs = append(lbs, tss.lb) } - creds := serverNameCheckCreds{} - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - cc, err := grpc.DialContext(ctx, r.Scheme()+":///"+beServerName, grpc.WithResolvers(r), - grpc.WithTransportCredentials(&creds), grpc.WithContextDialer(fakeNameDialer)) + // Configure the manual resolver with an initial state containing a service + // config with grpclb as the load balancing policy and the remote balancer + // addresses specified via attributes. + r := manual.NewBuilderWithScheme("whatever") + s := &grpclbstate.State{ + BalancerAddresses: []resolver.Address{ + { + Addr: tests[0].lbAddr, + ServerName: lbServerName, + }, + { + Addr: tests[1].lbAddr, + ServerName: lbServerName, + }, + }, + } + rs := grpclbstate.Set(resolver.State{ServiceConfig: internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(grpclbConfig)}, s) + r.InitialState(rs) + + dopts := []grpc.DialOption{ + grpc.WithResolvers(r), + grpc.WithTransportCredentials(&serverNameCheckCreds{}), + grpc.WithContextDialer(fakeNameDialer), + } + cc, err := grpc.Dial(r.Scheme()+":///"+beServerName, dopts...) if err != nil { t.Fatalf("Failed to dial to the backend %v", err) } defer cc.Close() - testC := testpb.NewTestServiceClient(cc) - - r.UpdateState(resolver.State{Addresses: []resolver.Address{{ - Addr: tests[0].lbAddr, - Type: resolver.GRPCLB, - ServerName: lbServerName, - }, { - Addr: tests[1].lbAddr, - Type: resolver.GRPCLB, - ServerName: lbServerName, - }}}) + testC := testgrpc.NewTestServiceClient(cc) - var p peer.Peer - if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil { - t.Fatalf("%v.EmptyCall(_, _) = _, %v, want _, ", testC, err) - } - if p.Addr.(*net.TCPAddr).Port != tests[0].bePorts[0] { - t.Fatalf("got peer: %v, want peer port: %v", p.Addr, tests[0].bePorts[0]) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := roundrobin.CheckRoundRobinRPCs(ctx, testC, []resolver.Address{{Addr: tests[0].beListeners[0].Addr().String()}}); err != nil { + t.Fatal(err) } - lbs[0].Stop() // Stop balancer[0], balancer[1] should be used by grpclb. // Check peer address to see if that happened. - for i := 0; i < 1000; i++ { - if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil { - t.Fatalf("%v.EmptyCall(_, _) = _, %v, want _, ", testC, err) - } - if p.Addr.(*net.TCPAddr).Port == tests[1].bePorts[0] { - return - } - time.Sleep(time.Millisecond) + lbs[0].Stop() + if err := roundrobin.CheckRoundRobinRPCs(ctx, testC, []resolver.Address{{Addr: tests[1].beListeners[0].Addr().String()}}); err != nil { + t.Fatal(err) } - t.Fatalf("No RPC sent to second backend after 1 second") } -func (s) TestFallback(t *testing.T) { +// TestGRPCLB_Fallback tests the following fallback scenarios: +// - when the remote balancer address specified in attributes is invalid, the +// test verifies that RPCs are routed to the fallback backend. +// - when the remote balancer address specified in attributes is changed to a +// valid one, the test verifies that RPCs are routed to the backend returned +// by the remote balancer. +// - when the configured remote balancer goes down, the test verifies that +// RPCs are routed to the fallback backend. +func (s) TestGRPCLB_Fallback(t *testing.T) { balancer.Register(newLBBuilderWithFallbackTimeout(100 * time.Millisecond)) defer balancer.Register(newLBBuilder()) - r := manual.NewBuilderWithScheme("whatever") - - tss, cleanup, err := newLoadBalancer(1, nil) + tss, cleanup, err := startBackendsAndRemoteLoadBalancer(t, 1, "", nil) if err != nil { t.Fatalf("failed to create new load balancer: %v", err) } defer cleanup() + sl := &lbpb.ServerList{ + Servers: []*lbpb.Server{ + { + IpAddress: tss.beIPs[0], + Port: int32(tss.bePorts[0]), + LoadBalanceToken: lbToken, + }, + }, + } + // Push the backend address to the remote balancer. + tss.ls.sls <- sl - // Start a standalone backend. + // Start a standalone backend for fallback. beLis, err := net.Listen("tcp", "localhost:0") if err != nil { t.Fatalf("Failed to listen %v", err) } defer beLis.Close() - standaloneBEs := startBackends(beServerName, true, beLis) + standaloneBEs := startBackends(t, beServerName, true, beLis) defer stopBackends(standaloneBEs) - be := &lbpb.Server{ - IpAddress: tss.beIPs[0], - Port: int32(tss.bePorts[0]), - LoadBalanceToken: lbToken, - } - var bes []*lbpb.Server - bes = append(bes, be) - sl := &lbpb.ServerList{ - Servers: bes, + r := manual.NewBuilderWithScheme("whatever") + dopts := []grpc.DialOption{ + grpc.WithResolvers(r), + grpc.WithTransportCredentials(&serverNameCheckCreds{}), + grpc.WithContextDialer(fakeNameDialer), } - tss.ls.sls <- sl - creds := serverNameCheckCreds{} - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - cc, err := grpc.DialContext(ctx, r.Scheme()+":///"+beServerName, grpc.WithResolvers(r), - grpc.WithTransportCredentials(&creds), grpc.WithContextDialer(fakeNameDialer)) + cc, err := grpc.Dial(r.Scheme()+":///"+beServerName, dopts...) if err != nil { t.Fatalf("Failed to dial to the backend %v", err) } defer cc.Close() - testC := testpb.NewTestServiceClient(cc) + testC := testgrpc.NewTestServiceClient(cc) - r.UpdateState(resolver.State{Addresses: []resolver.Address{{ - Addr: "invalid.address", - Type: resolver.GRPCLB, - ServerName: lbServerName, - }, { - Addr: beLis.Addr().String(), - Type: resolver.Backend, - }}}) - - var p peer.Peer - if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil { - t.Fatalf("_.EmptyCall(_, _) = _, %v, want _, ", err) - } - if p.Addr.String() != beLis.Addr().String() { - t.Fatalf("got peer: %v, want peer: %v", p.Addr, beLis.Addr()) + // Push an update to the resolver with fallback backend address stored in + // the `Addresses` field and an invalid remote balancer address stored in + // attributes, which will cause fallback behavior to be invoked. + rs := resolver.State{ + Addresses: []resolver.Address{{Addr: beLis.Addr().String()}}, + ServiceConfig: r.CC.ParseServiceConfig(grpclbConfig), } + rs = grpclbstate.Set(rs, &grpclbstate.State{BalancerAddresses: []resolver.Address{{Addr: "invalid.address", ServerName: lbServerName}}}) + r.UpdateState(rs) - r.UpdateState(resolver.State{Addresses: []resolver.Address{{ - Addr: tss.lbAddr, - Type: resolver.GRPCLB, - ServerName: lbServerName, - }, { - Addr: beLis.Addr().String(), - Type: resolver.Backend, - }}}) + // Make an RPC and verify that it got routed to the fallback backend. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := roundrobin.CheckRoundRobinRPCs(ctx, testC, []resolver.Address{{Addr: beLis.Addr().String()}}); err != nil { + t.Fatal(err) + } - var backendUsed bool - for i := 0; i < 1000; i++ { - if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil { - t.Fatalf("%v.EmptyCall(_, _) = _, %v, want _, ", testC, err) - } - if p.Addr.(*net.TCPAddr).Port == tss.bePorts[0] { - backendUsed = true - break - } - time.Sleep(time.Millisecond) + // Push another update to the resolver, this time with a valid balancer + // address in the attributes field. + rs = resolver.State{ + ServiceConfig: r.CC.ParseServiceConfig(grpclbConfig), + Addresses: []resolver.Address{{Addr: beLis.Addr().String()}}, } - if !backendUsed { - t.Fatalf("No RPC sent to backend behind remote balancer after 1 second") + rs = grpclbstate.Set(rs, &grpclbstate.State{BalancerAddresses: []resolver.Address{{Addr: tss.lbAddr, ServerName: lbServerName}}}) + r.UpdateState(rs) + select { + case <-ctx.Done(): + t.Fatalf("timeout when waiting for BalanceLoad RPC to be called on the remote balancer") + case <-tss.ls.balanceLoadCh: } - // Close backend and remote balancer connections, should use fallback. - tss.beListeners[0].(*restartableListener).stopPreviousConns() - tss.lbListener.(*restartableListener).stopPreviousConns() - - var fallbackUsed bool - for i := 0; i < 2000; i++ { - if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil { - // Because we are hard-closing the connection, above, it's possible - // for the first RPC attempt to be sent on the old connection, - // which will lead to an Unavailable error when it is closed. - // Ignore unavailable errors. - if status.Code(err) != codes.Unavailable { - t.Fatalf("%v.EmptyCall(_, _) = _, %v, want _, ", testC, err) - } - } - if p.Addr.String() == beLis.Addr().String() { - fallbackUsed = true - break - } - time.Sleep(time.Millisecond) + // Wait for RPCs to get routed to the backend behind the remote balancer. + if err := roundrobin.CheckRoundRobinRPCs(ctx, testC, []resolver.Address{{Addr: tss.beListeners[0].Addr().String()}}); err != nil { + t.Fatal(err) } - if !fallbackUsed { - t.Fatalf("No RPC sent to fallback after 2 seconds") + + // Close backend and remote balancer connections, should use fallback. + tss.beListeners[0].(*testutils.RestartableListener).Stop() + tss.lbListener.(*testutils.RestartableListener).Stop() + if err := roundrobin.CheckRoundRobinRPCs(ctx, testC, []resolver.Address{{Addr: beLis.Addr().String()}}); err != nil { + t.Fatal(err) } - // Restart backend and remote balancer, should not use backends. - tss.beListeners[0].(*restartableListener).restart() - tss.lbListener.(*restartableListener).restart() + // Restart backend and remote balancer, should not use fallback backend. + tss.beListeners[0].(*testutils.RestartableListener).Restart() + tss.lbListener.(*testutils.RestartableListener).Restart() tss.ls.sls <- sl - - var backendUsed2 bool - for i := 0; i < 2000; i++ { - if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil { - t.Fatalf("%v.EmptyCall(_, _) = _, %v, want _, ", testC, err) - } - if p.Addr.(*net.TCPAddr).Port == tss.bePorts[0] { - backendUsed2 = true - break - } - time.Sleep(time.Millisecond) - } - if !backendUsed2 { - t.Fatalf("No RPC sent to backend behind remote balancer after 2 seconds") + if err := roundrobin.CheckRoundRobinRPCs(ctx, testC, []resolver.Address{{Addr: tss.beListeners[0].Addr().String()}}); err != nil { + t.Fatal(err) } } -func (s) TestExplicitFallback(t *testing.T) { - r := manual.NewBuilderWithScheme("whatever") - - tss, cleanup, err := newLoadBalancer(1, nil) +// TestGRPCLB_ExplicitFallback tests the case where the remote balancer sends an +// explicit fallback signal to the grpclb client, and the test verifies that +// RPCs are routed to the fallback backend. +func (s) TestGRPCLB_ExplicitFallback(t *testing.T) { + tss, cleanup, err := startBackendsAndRemoteLoadBalancer(t, 1, "", nil) if err != nil { t.Fatalf("failed to create new load balancer: %v", err) } defer cleanup() + sl := &lbpb.ServerList{ + Servers: []*lbpb.Server{ + { + IpAddress: tss.beIPs[0], + Port: int32(tss.bePorts[0]), + LoadBalanceToken: lbToken, + }, + }, + } + // Push the backend address to the remote balancer. + tss.ls.sls <- sl - // Start a standalone backend. + // Start a standalone backend for fallback. beLis, err := net.Listen("tcp", "localhost:0") if err != nil { t.Fatalf("Failed to listen %v", err) } defer beLis.Close() - standaloneBEs := startBackends(beServerName, true, beLis) + standaloneBEs := startBackends(t, beServerName, true, beLis) defer stopBackends(standaloneBEs) - be := &lbpb.Server{ - IpAddress: tss.beIPs[0], - Port: int32(tss.bePorts[0]), - LoadBalanceToken: lbToken, + // Configure the manual resolver with an initial state containing a service + // config with grpclb as the load balancing policy and the address of the + // fallback backend. The remote balancer address is specified via + // attributes. + r := manual.NewBuilderWithScheme("whatever") + rs := resolver.State{ + Addresses: []resolver.Address{{Addr: beLis.Addr().String()}}, + ServiceConfig: internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(grpclbConfig), } - var bes []*lbpb.Server - bes = append(bes, be) - sl := &lbpb.ServerList{ - Servers: bes, + rs = grpclbstate.Set(rs, &grpclbstate.State{BalancerAddresses: []resolver.Address{{Addr: tss.lbAddr, ServerName: lbServerName}}}) + r.InitialState(rs) + + dopts := []grpc.DialOption{ + grpc.WithResolvers(r), + grpc.WithTransportCredentials(&serverNameCheckCreds{}), + grpc.WithContextDialer(fakeNameDialer), } - tss.ls.sls <- sl - creds := serverNameCheckCreds{} - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - cc, err := grpc.DialContext(ctx, r.Scheme()+":///"+beServerName, grpc.WithResolvers(r), - grpc.WithTransportCredentials(&creds), grpc.WithContextDialer(fakeNameDialer)) + cc, err := grpc.Dial(r.Scheme()+":///"+beServerName, dopts...) if err != nil { t.Fatalf("Failed to dial to the backend %v", err) } defer cc.Close() - testC := testpb.NewTestServiceClient(cc) + testC := testgrpc.NewTestServiceClient(cc) - r.UpdateState(resolver.State{Addresses: []resolver.Address{{ - Addr: tss.lbAddr, - Type: resolver.GRPCLB, - ServerName: lbServerName, - }, { - Addr: beLis.Addr().String(), - Type: resolver.Backend, - }}}) - - var p peer.Peer - var backendUsed bool - for i := 0; i < 2000; i++ { - if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil { - t.Fatalf("%v.EmptyCall(_, _) = _, %v, want _, ", testC, err) - } - if p.Addr.(*net.TCPAddr).Port == tss.bePorts[0] { - backendUsed = true - break - } - time.Sleep(time.Millisecond) - } - if !backendUsed { - t.Fatalf("No RPC sent to backend behind remote balancer after 2 seconds") + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := roundrobin.CheckRoundRobinRPCs(ctx, testC, []resolver.Address{{Addr: tss.beListeners[0].Addr().String()}}); err != nil { + t.Fatal(err) } // Send fallback signal from remote balancer; should use fallback. tss.ls.fallbackNow() - - var fallbackUsed bool - for i := 0; i < 2000; i++ { - if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil { - t.Fatalf("%v.EmptyCall(_, _) = _, %v, want _, ", testC, err) - } - if p.Addr.String() == beLis.Addr().String() { - fallbackUsed = true - break - } - time.Sleep(time.Millisecond) - } - if !fallbackUsed { - t.Fatalf("No RPC sent to fallback after 2 seconds") + if err := roundrobin.CheckRoundRobinRPCs(ctx, testC, []resolver.Address{{Addr: beLis.Addr().String()}}); err != nil { + t.Fatal(err) } // Send another server list; should use backends again. tss.ls.sls <- sl - - backendUsed = false - for i := 0; i < 2000; i++ { - if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil { - t.Fatalf("%v.EmptyCall(_, _) = _, %v, want _, ", testC, err) - } - if p.Addr.(*net.TCPAddr).Port == tss.bePorts[0] { - backendUsed = true - break - } - time.Sleep(time.Millisecond) - } - if !backendUsed { - t.Fatalf("No RPC sent to backend behind remote balancer after 2 seconds") + if err := roundrobin.CheckRoundRobinRPCs(ctx, testC, []resolver.Address{{Addr: tss.beListeners[0].Addr().String()}}); err != nil { + t.Fatal(err) } } -func (s) TestFallBackWithNoServerAddress(t *testing.T) { - resolveNowCh := make(chan struct{}, 1) +// TestGRPCLB_FallBackWithNoServerAddress tests the fallback case where no +// backend addresses are returned by the remote balancer. +func (s) TestGRPCLB_FallBackWithNoServerAddress(t *testing.T) { + resolveNowCh := testutils.NewChannel() r := manual.NewBuilderWithScheme("whatever") r.ResolveNowCallback = func(resolver.ResolveNowOptions) { - select { - case <-resolveNowCh: - default: + ctx, cancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer cancel() + if err := resolveNowCh.SendContext(ctx, nil); err != nil { + t.Error("timeout when attempting to send on resolverNowCh") } - resolveNowCh <- struct{}{} } - tss, cleanup, err := newLoadBalancer(1, nil) + // Start a remote balancer and a backend. Don't push the backend address to + // the remote balancer yet. + tss, cleanup, err := startBackendsAndRemoteLoadBalancer(t, 1, "", nil) if err != nil { t.Fatalf("failed to create new load balancer: %v", err) } defer cleanup() + sl := &lbpb.ServerList{ + Servers: []*lbpb.Server{ + { + IpAddress: tss.beIPs[0], + Port: int32(tss.bePorts[0]), + LoadBalanceToken: lbToken, + }, + }, + } - // Start a standalone backend. + // Start a standalone backend for fallback. beLis, err := net.Listen("tcp", "localhost:0") if err != nil { t.Fatalf("Failed to listen %v", err) } defer beLis.Close() - standaloneBEs := startBackends(beServerName, true, beLis) + standaloneBEs := startBackends(t, beServerName, true, beLis) defer stopBackends(standaloneBEs) - be := &lbpb.Server{ - IpAddress: tss.beIPs[0], - Port: int32(tss.bePorts[0]), - LoadBalanceToken: lbToken, - } - var bes []*lbpb.Server - bes = append(bes, be) - sl := &lbpb.ServerList{ - Servers: bes, + dopts := []grpc.DialOption{ + grpc.WithResolvers(r), + grpc.WithTransportCredentials(&serverNameCheckCreds{}), + grpc.WithContextDialer(fakeNameDialer), } - creds := serverNameCheckCreds{} - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - cc, err := grpc.DialContext(ctx, r.Scheme()+":///"+beServerName, grpc.WithResolvers(r), - grpc.WithTransportCredentials(&creds), grpc.WithContextDialer(fakeNameDialer)) + cc, err := grpc.Dial(r.Scheme()+":///"+beServerName, dopts...) if err != nil { t.Fatalf("Failed to dial to the backend %v", err) } defer cc.Close() - testC := testpb.NewTestServiceClient(cc) - - // Select grpclb with service config. - const pfc = `{"loadBalancingConfig":[{"grpclb":{"childPolicy":[{"round_robin":{}}]}}]}` - scpr := r.CC.ParseServiceConfig(pfc) - if scpr.Err != nil { - t.Fatalf("Error parsing config %q: %v", pfc, scpr.Err) - } + testC := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() for i := 0; i < 2; i++ { - // Send an update with only backend address. grpclb should enter fallback - // and use the fallback backend. + // Send an update with only backend address. grpclb should enter + // fallback and use the fallback backend. r.UpdateState(resolver.State{ - Addresses: []resolver.Address{{ - Addr: beLis.Addr().String(), - Type: resolver.Backend, - }}, - ServiceConfig: scpr, + Addresses: []resolver.Address{{Addr: beLis.Addr().String()}}, + ServiceConfig: r.CC.ParseServiceConfig(grpclbConfig), }) - select { - case <-resolveNowCh: - t.Errorf("unexpected resolveNow when grpclb gets no balancer address 1111, %d", i) - case <-time.After(time.Second): + sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + if _, err := resolveNowCh.Receive(sCtx); err != context.DeadlineExceeded { + t.Fatalf("unexpected resolveNow when grpclb gets no balancer address 1111, %d", i) } var p peer.Peer - rpcCtx, rpcCancel := context.WithTimeout(context.Background(), time.Second) - defer rpcCancel() - if _, err := testC.EmptyCall(rpcCtx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil { + if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil { t.Fatalf("_.EmptyCall(_, _) = _, %v, want _, ", err) } if p.Addr.String() != beLis.Addr().String() { t.Fatalf("got peer: %v, want peer: %v", p.Addr, beLis.Addr()) } - select { - case <-resolveNowCh: + sCtx, sCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + if _, err := resolveNowCh.Receive(sCtx); err != context.DeadlineExceeded { t.Errorf("unexpected resolveNow when grpclb gets no balancer address 2222, %d", i) - case <-time.After(time.Second): } tss.ls.sls <- sl // Send an update with balancer address. The backends behind grpclb should // be used. - r.UpdateState(resolver.State{ - Addresses: []resolver.Address{{ - Addr: tss.lbAddr, - Type: resolver.GRPCLB, - ServerName: lbServerName, - }, { - Addr: beLis.Addr().String(), - Type: resolver.Backend, - }}, - ServiceConfig: scpr, - }) + rs := resolver.State{ + Addresses: []resolver.Address{{Addr: beLis.Addr().String()}}, + ServiceConfig: r.CC.ParseServiceConfig(grpclbConfig), + } + rs = grpclbstate.Set(rs, &grpclbstate.State{BalancerAddresses: []resolver.Address{{Addr: tss.lbAddr, ServerName: lbServerName}}}) + r.UpdateState(rs) - var backendUsed bool - for i := 0; i < 1000; i++ { - if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil { - t.Fatalf("%v.EmptyCall(_, _) = _, %v, want _, ", testC, err) - } - if p.Addr.(*net.TCPAddr).Port == tss.bePorts[0] { - backendUsed = true - break - } - time.Sleep(time.Millisecond) + select { + case <-ctx.Done(): + t.Fatalf("timeout when waiting for BalanceLoad RPC to be called on the remote balancer") + case <-tss.ls.balanceLoadCh: } - if !backendUsed { - t.Fatalf("No RPC sent to backend behind remote balancer after 1 second") + + if err := roundrobin.CheckRoundRobinRPCs(ctx, testC, []resolver.Address{{Addr: tss.beListeners[0].Addr().String()}}); err != nil { + t.Fatal(err) } } } -func (s) TestGRPCLBPickFirst(t *testing.T) { - r := manual.NewBuilderWithScheme("whatever") - - tss, cleanup, err := newLoadBalancer(3, nil) +// TestGRPCLB_PickFirst configures grpclb with pick_first as the child policy. +// The test changes the list of backend addresses returned by the remote +// balancer and verifies that RPCs are sent to the first address returned. +func (s) TestGRPCLB_PickFirst(t *testing.T) { + tss, cleanup, err := startBackendsAndRemoteLoadBalancer(t, 3, "", nil) if err != nil { t.Fatalf("failed to create new load balancer: %v", err) } @@ -1106,115 +1090,289 @@ func (s) TestGRPCLBPickFirst(t *testing.T) { Port: int32(tss.bePorts[2]), LoadBalanceToken: lbToken, }} - portsToIndex := make(map[int]int) - for i := range beServers { - portsToIndex[tss.bePorts[i]] = i + beServerAddrs := []resolver.Address{} + for _, lis := range tss.beListeners { + beServerAddrs = append(beServerAddrs, resolver.Address{Addr: lis.Addr().String()}) } - creds := serverNameCheckCreds{} - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - cc, err := grpc.DialContext(ctx, r.Scheme()+":///"+beServerName, grpc.WithResolvers(r), - grpc.WithTransportCredentials(&creds), grpc.WithContextDialer(fakeNameDialer)) + // Connect to the test backends. + r := manual.NewBuilderWithScheme("whatever") + dopts := []grpc.DialOption{ + grpc.WithResolvers(r), + grpc.WithTransportCredentials(&serverNameCheckCreds{}), + grpc.WithContextDialer(fakeNameDialer), + } + cc, err := grpc.Dial(r.Scheme()+":///"+beServerName, dopts...) if err != nil { t.Fatalf("Failed to dial to the backend %v", err) } defer cc.Close() - testC := testpb.NewTestServiceClient(cc) - var ( - p peer.Peer - result string - ) + // Push a service config with grpclb as the load balancing policy and + // configure pick_first as its child policy. + rs := resolver.State{ServiceConfig: r.CC.ParseServiceConfig(`{"loadBalancingConfig":[{"grpclb":{"childPolicy":[{"pick_first":{}}]}}]}`)} + + // Push a resolver update with the remote balancer address specified via + // attributes. + r.UpdateState(grpclbstate.Set(rs, &grpclbstate.State{BalancerAddresses: []resolver.Address{{Addr: tss.lbAddr, ServerName: lbServerName}}})) + + // Push all three backend addresses to the remote balancer, and verify that + // RPCs are routed to the first backend. tss.ls.sls <- &lbpb.ServerList{Servers: beServers[0:3]} + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := pickfirst.CheckRPCsToBackend(ctx, cc, beServerAddrs[0]); err != nil { + t.Fatal(err) + } - // Start with sub policy pick_first. - const pfc = `{"loadBalancingConfig":[{"grpclb":{"childPolicy":[{"pick_first":{}}]}}]}` - scpr := r.CC.ParseServiceConfig(pfc) - if scpr.Err != nil { - t.Fatalf("Error parsing config %q: %v", pfc, scpr.Err) + // Update the address list with the remote balancer and verify pick_first + // behavior based on the new backends. + tss.ls.sls <- &lbpb.ServerList{Servers: beServers[2:]} + if err := pickfirst.CheckRPCsToBackend(ctx, cc, beServerAddrs[2]); err != nil { + t.Fatal(err) } - r.UpdateState(resolver.State{ - Addresses: []resolver.Address{{ - Addr: tss.lbAddr, - Type: resolver.GRPCLB, - ServerName: lbServerName, - }}, - ServiceConfig: scpr, - }) + // Update the address list with the remote balancer and verify pick_first + // behavior based on the new backends. Since the currently connected backend + // is in the new list (even though it is not the first one on the list), + // pick_first will continue to use it. + tss.ls.sls <- &lbpb.ServerList{Servers: beServers[1:]} + if err := pickfirst.CheckRPCsToBackend(ctx, cc, beServerAddrs[2]); err != nil { + t.Fatal(err) + } - result = "" - for i := 0; i < 1000; i++ { - if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil { - t.Fatalf("_.EmptyCall(_, _) = _, %v, want _, ", err) - } - result += strconv.Itoa(portsToIndex[p.Addr.(*net.TCPAddr).Port]) + // Switch child policy to roundrobin. + s := &grpclbstate.State{ + BalancerAddresses: []resolver.Address{ + { + Addr: tss.lbAddr, + ServerName: lbServerName, + }, + }, } - if seq := "00000"; !strings.Contains(result, strings.Repeat(seq, 100)) { - t.Errorf("got result sequence %q, want patten %q", result, seq) + rs = grpclbstate.Set(resolver.State{ServiceConfig: r.CC.ParseServiceConfig(grpclbConfig)}, s) + r.UpdateState(rs) + testC := testgrpc.NewTestServiceClient(cc) + if err := roundrobin.CheckRoundRobinRPCs(ctx, testC, beServerAddrs[1:]); err != nil { + t.Fatal(err) } - tss.ls.sls <- &lbpb.ServerList{Servers: beServers[2:]} - result = "" - for i := 0; i < 1000; i++ { - if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil { - t.Fatalf("_.EmptyCall(_, _) = _, %v, want _, ", err) - } - result += strconv.Itoa(portsToIndex[p.Addr.(*net.TCPAddr).Port]) + tss.ls.sls <- &lbpb.ServerList{Servers: beServers[0:3]} + if err := roundrobin.CheckRoundRobinRPCs(ctx, testC, beServerAddrs[0:3]); err != nil { + t.Fatal(err) } - if seq := "22222"; !strings.Contains(result, strings.Repeat(seq, 100)) { - t.Errorf("got result sequence %q, want patten %q", result, seq) +} + +// TestGRPCLB_BackendConnectionErrorPropagation tests the case where grpclb +// falls back to a backend which returns an error and the test verifies that the +// error is propagated to the RPC. +func (s) TestGRPCLB_BackendConnectionErrorPropagation(t *testing.T) { + r := manual.NewBuilderWithScheme("whatever") + + // Start up an LB which will tells the client to fall back right away. + tss, cleanup, err := startBackendsAndRemoteLoadBalancer(t, 0, "", nil) + if err != nil { + t.Fatalf("failed to create new load balancer: %v", err) } + defer cleanup() - tss.ls.sls <- &lbpb.ServerList{Servers: beServers[1:]} - result = "" - for i := 0; i < 1000; i++ { - if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil { - t.Fatalf("_.EmptyCall(_, _) = _, %v, want _, ", err) + // Start a standalone backend, to be used during fallback. The creds + // are intentionally misconfigured in order to simulate failure of a + // security handshake. + beLis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Failed to listen %v", err) + } + defer beLis.Close() + standaloneBEs := startBackends(t, "arbitrary.invalid.name", true, beLis) + defer stopBackends(standaloneBEs) + + cc, err := grpc.Dial(r.Scheme()+":///"+beServerName, + grpc.WithResolvers(r), + grpc.WithTransportCredentials(&serverNameCheckCreds{}), + grpc.WithContextDialer(fakeNameDialer)) + if err != nil { + t.Fatalf("Failed to dial to the backend %v", err) + } + defer cc.Close() + testC := testgrpc.NewTestServiceClient(cc) + + rs := resolver.State{ + Addresses: []resolver.Address{{Addr: beLis.Addr().String()}}, + ServiceConfig: r.CC.ParseServiceConfig(grpclbConfig), + } + rs = grpclbstate.Set(rs, &grpclbstate.State{BalancerAddresses: []resolver.Address{{Addr: tss.lbAddr, ServerName: lbServerName}}}) + r.UpdateState(rs) + + // If https://github.com/grpc/grpc-go/blob/65cabd74d8e18d7347fecd414fa8d83a00035f5f/balancer/grpclb/grpclb_test.go#L103 + // changes, then expectedErrMsg may need to be updated. + const expectedErrMsg = "received unexpected server name" + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + var wg sync.WaitGroup + wg.Add(1) + go func() { + tss.ls.fallbackNow() + wg.Done() + }() + if _, err := testC.EmptyCall(ctx, &testpb.Empty{}); err == nil || !strings.Contains(err.Error(), expectedErrMsg) { + t.Fatalf("%v.EmptyCall(_, _) = _, %v, want _, rpc error containing substring: %q", testC, err, expectedErrMsg) + } + wg.Wait() +} + +func testGRPCLBEmptyServerList(t *testing.T, svcfg string) { + tss, cleanup, err := startBackendsAndRemoteLoadBalancer(t, 1, "", nil) + if err != nil { + t.Fatalf("failed to create new load balancer: %v", err) + } + defer cleanup() + + beServers := []*lbpb.Server{{ + IpAddress: tss.beIPs[0], + Port: int32(tss.bePorts[0]), + LoadBalanceToken: lbToken, + }} + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + r := manual.NewBuilderWithScheme("whatever") + dopts := []grpc.DialOption{ + grpc.WithResolvers(r), + grpc.WithTransportCredentials(&serverNameCheckCreds{}), + grpc.WithContextDialer(fakeNameDialer), + } + cc, err := grpc.DialContext(ctx, r.Scheme()+":///"+beServerName, dopts...) + if err != nil { + t.Fatalf("Failed to dial to the backend %v", err) + } + defer cc.Close() + testC := testgrpc.NewTestServiceClient(cc) + + tss.ls.sls <- &lbpb.ServerList{Servers: beServers} + + s := &grpclbstate.State{ + BalancerAddresses: []resolver.Address{ + { + Addr: tss.lbAddr, + ServerName: lbServerName, + }, + }, + } + rs := grpclbstate.Set(resolver.State{ServiceConfig: r.CC.ParseServiceConfig(svcfg)}, s) + r.UpdateState(rs) + t.Log("Perform an initial RPC and expect it to succeed...") + if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("Initial _.EmptyCall(_, _) = _, %v, want _, ", err) + } + t.Log("Now send an empty server list. Wait until we see an RPC failure to make sure the client got it...") + tss.ls.sls <- &lbpb.ServerList{} + gotError := false + for ; ctx.Err() == nil; <-time.After(time.Millisecond) { + if _, err := testC.EmptyCall(ctx, &testpb.Empty{}); err != nil { + gotError = true + break } - result += strconv.Itoa(portsToIndex[p.Addr.(*net.TCPAddr).Port]) } - if seq := "22222"; !strings.Contains(result, strings.Repeat(seq, 100)) { - t.Errorf("got result sequence %q, want patten %q", result, seq) + if !gotError { + t.Fatalf("Expected to eventually see an RPC fail after the grpclb sends an empty server list, but none did.") + } + t.Log("Now send a non-empty server list. A wait-for-ready RPC should now succeed...") + tss.ls.sls <- &lbpb.ServerList{Servers: beServers} + if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("Final _.EmptyCall(_, _) = _, %v, want _, ", err) } +} - // Switch sub policy to roundrobin. - grpclbServiceConfigEmpty := r.CC.ParseServiceConfig(`{}`) - if grpclbServiceConfigEmpty.Err != nil { - t.Fatalf("Error parsing config %q: %v", `{}`, grpclbServiceConfigEmpty.Err) +func (s) TestGRPCLBEmptyServerListRoundRobin(t *testing.T) { + testGRPCLBEmptyServerList(t, `{"loadBalancingConfig":[{"grpclb":{"childPolicy":[{"round_robin":{}}]}}]}`) +} + +func (s) TestGRPCLBEmptyServerListPickFirst(t *testing.T) { + testGRPCLBEmptyServerList(t, `{"loadBalancingConfig":[{"grpclb":{"childPolicy":[{"pick_first":{}}]}}]}`) +} + +func (s) TestGRPCLBWithTargetNameFieldInConfig(t *testing.T) { + r := manual.NewBuilderWithScheme("whatever") + + tss, cleanup, err := startBackendsAndRemoteLoadBalancer(t, 1, "", nil) + if err != nil { + t.Fatalf("failed to create new load balancer: %v", err) } + defer cleanup() + sl := &lbpb.ServerList{ + Servers: []*lbpb.Server{ + { + IpAddress: tss.beIPs[0], + Port: int32(tss.bePorts[0]), + LoadBalanceToken: lbToken, + }, + }, + } + // Push the backend address to the remote balancer. + tss.ls.sls <- sl + + cc, err := grpc.Dial(r.Scheme()+":///"+beServerName, + grpc.WithResolvers(r), + grpc.WithTransportCredentials(&serverNameCheckCreds{}), + grpc.WithContextDialer(fakeNameDialer), + grpc.WithUserAgent(testUserAgent)) + if err != nil { + t.Fatalf("Failed to dial to the backend %v", err) + } + defer cc.Close() + testC := testgrpc.NewTestServiceClient(cc) - r.UpdateState(resolver.State{ - Addresses: []resolver.Address{{ + // Push a resolver update with grpclb configuration which does not contain the + // target_name field. Our fake remote balancer is configured to always + // expect `beServerName` as the server name in the initial request. + rs := grpclbstate.Set(resolver.State{ServiceConfig: r.CC.ParseServiceConfig(grpclbConfig)}, + &grpclbstate.State{BalancerAddresses: []resolver.Address{{ Addr: tss.lbAddr, - Type: resolver.GRPCLB, ServerName: lbServerName, - }}, - ServiceConfig: grpclbServiceConfigEmpty, - }) + }}}) + r.UpdateState(rs) - result = "" - for i := 0; i < 1000; i++ { - if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil { - t.Fatalf("_.EmptyCall(_, _) = _, %v, want _, ", err) - } - result += strconv.Itoa(portsToIndex[p.Addr.(*net.TCPAddr).Port]) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + select { + case <-ctx.Done(): + t.Fatalf("timeout when waiting for BalanceLoad RPC to be called on the remote balancer") + case <-tss.ls.balanceLoadCh: } - if seq := "121212"; !strings.Contains(result, strings.Repeat(seq, 100)) { - t.Errorf("got result sequence %q, want patten %q", result, seq) + if _, err := testC.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("%v.EmptyCall(_, _) = _, %v, want _, ", testC, err) } - tss.ls.sls <- &lbpb.ServerList{Servers: beServers[0:3]} - result = "" - for i := 0; i < 1000; i++ { - if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(&p)); err != nil { - t.Fatalf("%v.EmptyCall(_, _) = _, %v, want _, ", testC, err) - } - result += strconv.Itoa(portsToIndex[p.Addr.(*net.TCPAddr).Port]) + // When the value of target_field changes, grpclb will recreate the stream + // to the remote balancer. So, we need to update the fake remote balancer to + // expect a new server name in the initial request. + const newServerName = "new-server-name" + tss.ls.updateServerName(newServerName) + tss.ls.sls <- sl + + // Push the resolver update with target_field changed. + // Push a resolver update with grpclb configuration containing the + // target_name field. Our fake remote balancer has been updated above to expect the newServerName in the initial request. + lbCfg := fmt.Sprintf(`{"loadBalancingConfig": [{"grpclb": {"serviceName": "%s"}}]}`, newServerName) + s := &grpclbstate.State{ + BalancerAddresses: []resolver.Address{ + { + Addr: tss.lbAddr, + ServerName: lbServerName, + }, + }, } - if seq := "012012012"; !strings.Contains(result, strings.Repeat(seq, 2)) { - t.Errorf("got result sequence %q, want patten %q", result, seq) + rs = grpclbstate.Set(resolver.State{ServiceConfig: r.CC.ParseServiceConfig(lbCfg)}, s) + r.UpdateState(rs) + select { + case <-ctx.Done(): + t.Fatalf("timeout when waiting for BalanceLoad RPC to be called on the remote balancer") + case <-tss.ls.balanceLoadCh: + } + + if _, err := testC.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("%v.EmptyCall(_, _) = _, %v, want _, ", testC, err) } } @@ -1241,7 +1399,7 @@ func checkStats(stats, expected *rpcStats) error { func runAndCheckStats(t *testing.T, drop bool, statsChan chan *lbpb.ClientStats, runRPCs func(*grpc.ClientConn), statsWant *rpcStats) error { r := manual.NewBuilderWithScheme("whatever") - tss, cleanup, err := newLoadBalancer(1, statsChan) + tss, cleanup, err := startBackendsAndRemoteLoadBalancer(t, 1, "", statsChan) if err != nil { t.Fatalf("failed to create new load balancer: %v", err) } @@ -1272,11 +1430,11 @@ func runAndCheckStats(t *testing.T, drop bool, statsChan chan *lbpb.ClientStats, } defer cc.Close() - r.UpdateState(resolver.State{Addresses: []resolver.Address{{ + rstate := resolver.State{ServiceConfig: r.CC.ParseServiceConfig(grpclbConfig)} + r.UpdateState(grpclbstate.Set(rstate, &grpclbstate.State{BalancerAddresses: []resolver.Address{{ Addr: tss.lbAddr, - Type: resolver.GRPCLB, ServerName: lbServerName, - }}}) + }}})) runRPCs(cc) end := time.Now().Add(time.Second) @@ -1296,13 +1454,15 @@ const ( func (s) TestGRPCLBStatsUnarySuccess(t *testing.T) { if err := runAndCheckStats(t, false, nil, func(cc *grpc.ClientConn) { - testC := testpb.NewTestServiceClient(cc) + testC := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultFallbackTimeout) + defer cancel() // The first non-failfast RPC succeeds, all connections are up. - if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { t.Fatalf("%v.EmptyCall(_, _) = _, %v, want _, ", testC, err) } for i := 0; i < countRPC-1; i++ { - testC.EmptyCall(context.Background(), &testpb.Empty{}) + testC.EmptyCall(ctx, &testpb.Empty{}) } }, &rpcStats{ numCallsStarted: int64(countRPC), @@ -1315,13 +1475,15 @@ func (s) TestGRPCLBStatsUnarySuccess(t *testing.T) { func (s) TestGRPCLBStatsUnaryDrop(t *testing.T) { if err := runAndCheckStats(t, true, nil, func(cc *grpc.ClientConn) { - testC := testpb.NewTestServiceClient(cc) + testC := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultFallbackTimeout) + defer cancel() // The first non-failfast RPC succeeds, all connections are up. - if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { t.Fatalf("%v.EmptyCall(_, _) = _, %v, want _, ", testC, err) } for i := 0; i < countRPC-1; i++ { - testC.EmptyCall(context.Background(), &testpb.Empty{}) + testC.EmptyCall(ctx, &testpb.Empty{}) } }, &rpcStats{ numCallsStarted: int64(countRPC), @@ -1335,13 +1497,15 @@ func (s) TestGRPCLBStatsUnaryDrop(t *testing.T) { func (s) TestGRPCLBStatsUnaryFailedToSend(t *testing.T) { if err := runAndCheckStats(t, false, nil, func(cc *grpc.ClientConn) { - testC := testpb.NewTestServiceClient(cc) + testC := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultFallbackTimeout) + defer cancel() // The first non-failfast RPC succeeds, all connections are up. - if _, err := testC.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + if _, err := testC.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { t.Fatalf("%v.EmptyCall(_, _) = _, %v, want _, ", testC, err) } for i := 0; i < countRPC-1; i++ { - cc.Invoke(context.Background(), failtosendURI, &testpb.Empty{}, nil) + cc.Invoke(ctx, failtosendURI, &testpb.Empty{}, nil) } }, &rpcStats{ numCallsStarted: int64(countRPC), @@ -1355,9 +1519,11 @@ func (s) TestGRPCLBStatsUnaryFailedToSend(t *testing.T) { func (s) TestGRPCLBStatsStreamingSuccess(t *testing.T) { if err := runAndCheckStats(t, false, nil, func(cc *grpc.ClientConn) { - testC := testpb.NewTestServiceClient(cc) + testC := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultFallbackTimeout) + defer cancel() // The first non-failfast RPC succeeds, all connections are up. - stream, err := testC.FullDuplexCall(context.Background(), grpc.WaitForReady(true)) + stream, err := testC.FullDuplexCall(ctx, grpc.WaitForReady(true)) if err != nil { t.Fatalf("%v.FullDuplexCall(_, _) = _, %v, want _, ", testC, err) } @@ -1367,7 +1533,7 @@ func (s) TestGRPCLBStatsStreamingSuccess(t *testing.T) { } } for i := 0; i < countRPC-1; i++ { - stream, err = testC.FullDuplexCall(context.Background()) + stream, err = testC.FullDuplexCall(ctx) if err == nil { // Wait for stream to end if err is nil. for { @@ -1388,9 +1554,11 @@ func (s) TestGRPCLBStatsStreamingSuccess(t *testing.T) { func (s) TestGRPCLBStatsStreamingDrop(t *testing.T) { if err := runAndCheckStats(t, true, nil, func(cc *grpc.ClientConn) { - testC := testpb.NewTestServiceClient(cc) + testC := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultFallbackTimeout) + defer cancel() // The first non-failfast RPC succeeds, all connections are up. - stream, err := testC.FullDuplexCall(context.Background(), grpc.WaitForReady(true)) + stream, err := testC.FullDuplexCall(ctx, grpc.WaitForReady(true)) if err != nil { t.Fatalf("%v.FullDuplexCall(_, _) = _, %v, want _, ", testC, err) } @@ -1400,7 +1568,7 @@ func (s) TestGRPCLBStatsStreamingDrop(t *testing.T) { } } for i := 0; i < countRPC-1; i++ { - stream, err = testC.FullDuplexCall(context.Background()) + stream, err = testC.FullDuplexCall(ctx) if err == nil { // Wait for stream to end if err is nil. for { @@ -1422,9 +1590,11 @@ func (s) TestGRPCLBStatsStreamingDrop(t *testing.T) { func (s) TestGRPCLBStatsStreamingFailedToSend(t *testing.T) { if err := runAndCheckStats(t, false, nil, func(cc *grpc.ClientConn) { - testC := testpb.NewTestServiceClient(cc) + testC := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultFallbackTimeout) + defer cancel() // The first non-failfast RPC succeeds, all connections are up. - stream, err := testC.FullDuplexCall(context.Background(), grpc.WaitForReady(true)) + stream, err := testC.FullDuplexCall(ctx, grpc.WaitForReady(true)) if err != nil { t.Fatalf("%v.FullDuplexCall(_, _) = _, %v, want _, ", testC, err) } @@ -1434,7 +1604,7 @@ func (s) TestGRPCLBStatsStreamingFailedToSend(t *testing.T) { } } for i := 0; i < countRPC-1; i++ { - cc.NewStream(context.Background(), &grpc.StreamDesc{}, failtosendURI) + cc.NewStream(ctx, &grpc.StreamDesc{}, failtosendURI) } }, &rpcStats{ numCallsStarted: int64(countRPC), diff --git a/balancer/grpclb/grpclb_test_util_test.go b/balancer/grpclb/grpclb_test_util_test.go deleted file mode 100644 index 5d3e6ba7fed9..000000000000 --- a/balancer/grpclb/grpclb_test_util_test.go +++ /dev/null @@ -1,85 +0,0 @@ -/* - * - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package grpclb - -import ( - "net" - "sync" -) - -type tempError struct{} - -func (*tempError) Error() string { - return "grpclb test temporary error" -} -func (*tempError) Temporary() bool { - return true -} - -type restartableListener struct { - net.Listener - addr string - - mu sync.Mutex - closed bool - conns []net.Conn -} - -func newRestartableListener(l net.Listener) *restartableListener { - return &restartableListener{ - Listener: l, - addr: l.Addr().String(), - } -} - -func (l *restartableListener) Accept() (conn net.Conn, err error) { - conn, err = l.Listener.Accept() - if err == nil { - l.mu.Lock() - if l.closed { - conn.Close() - l.mu.Unlock() - return nil, &tempError{} - } - l.conns = append(l.conns, conn) - l.mu.Unlock() - } - return -} - -func (l *restartableListener) Close() error { - return l.Listener.Close() -} - -func (l *restartableListener) stopPreviousConns() { - l.mu.Lock() - l.closed = true - tmp := l.conns - l.conns = nil - l.mu.Unlock() - for _, conn := range tmp { - conn.Close() - } -} - -func (l *restartableListener) restart() { - l.mu.Lock() - l.closed = false - l.mu.Unlock() -} diff --git a/balancer/grpclb/grpclb_util.go b/balancer/grpclb/grpclb_util.go index 636725e5416d..680779f1c82e 100644 --- a/balancer/grpclb/grpclb_util.go +++ b/balancer/grpclb/grpclb_util.go @@ -91,11 +91,12 @@ func (r *lbManualResolver) UpdateState(s resolver.State) { const subConnCacheTime = time.Second * 10 // lbCacheClientConn is a wrapper balancer.ClientConn with a SubConn cache. -// SubConns will be kept in cache for subConnCacheTime before being removed. +// SubConns will be kept in cache for subConnCacheTime before being shut down. // -// Its new and remove methods are updated to do cache first. +// Its NewSubconn and SubConn.Shutdown methods are updated to do cache first. type lbCacheClientConn struct { - cc balancer.ClientConn + balancer.ClientConn + timeout time.Duration mu sync.Mutex @@ -113,7 +114,7 @@ type subConnCacheEntry struct { func newLBCacheClientConn(cc balancer.ClientConn) *lbCacheClientConn { return &lbCacheClientConn{ - cc: cc, + ClientConn: cc, timeout: subConnCacheTime, subConnCache: make(map[resolver.Address]*subConnCacheEntry), subConnToAddr: make(map[balancer.SubConn]resolver.Address), @@ -124,29 +125,40 @@ func (ccc *lbCacheClientConn) NewSubConn(addrs []resolver.Address, opts balancer if len(addrs) != 1 { return nil, fmt.Errorf("grpclb calling NewSubConn with addrs of length %v", len(addrs)) } - addrWithoutMD := addrs[0] - addrWithoutMD.Metadata = nil + addrWithoutAttrs := addrs[0] + addrWithoutAttrs.Attributes = nil ccc.mu.Lock() defer ccc.mu.Unlock() - if entry, ok := ccc.subConnCache[addrWithoutMD]; ok { + if entry, ok := ccc.subConnCache[addrWithoutAttrs]; ok { // If entry is in subConnCache, the SubConn was being deleted. // cancel function will never be nil. entry.cancel() - delete(ccc.subConnCache, addrWithoutMD) + delete(ccc.subConnCache, addrWithoutAttrs) return entry.sc, nil } - scNew, err := ccc.cc.NewSubConn(addrs, opts) + scNew, err := ccc.ClientConn.NewSubConn(addrs, opts) if err != nil { return nil, err } + scNew = &lbCacheSubConn{SubConn: scNew, ccc: ccc} - ccc.subConnToAddr[scNew] = addrWithoutMD + ccc.subConnToAddr[scNew] = addrWithoutAttrs return scNew, nil } func (ccc *lbCacheClientConn) RemoveSubConn(sc balancer.SubConn) { + logger.Errorf("RemoveSubConn(%v) called unexpectedly", sc) +} + +type lbCacheSubConn struct { + balancer.SubConn + ccc *lbCacheClientConn +} + +func (sc *lbCacheSubConn) Shutdown() { + ccc := sc.ccc ccc.mu.Lock() defer ccc.mu.Unlock() addr, ok := ccc.subConnToAddr[sc] @@ -156,11 +168,11 @@ func (ccc *lbCacheClientConn) RemoveSubConn(sc balancer.SubConn) { if entry, ok := ccc.subConnCache[addr]; ok { if entry.sc != sc { - // This could happen if NewSubConn was called multiple times for the - // same address, and those SubConns are all removed. We remove sc - // immediately here. + // This could happen if NewSubConn was called multiple times for + // the same address, and those SubConns are all shut down. We + // remove sc immediately here. delete(ccc.subConnToAddr, sc) - ccc.cc.RemoveSubConn(sc) + sc.SubConn.Shutdown() } return } @@ -176,7 +188,7 @@ func (ccc *lbCacheClientConn) RemoveSubConn(sc balancer.SubConn) { if entry.abortDeleting { return } - ccc.cc.RemoveSubConn(sc) + sc.SubConn.Shutdown() delete(ccc.subConnToAddr, sc) delete(ccc.subConnCache, addr) }) @@ -195,14 +207,28 @@ func (ccc *lbCacheClientConn) RemoveSubConn(sc balancer.SubConn) { } func (ccc *lbCacheClientConn) UpdateState(s balancer.State) { - ccc.cc.UpdateState(s) + s.Picker = &lbCachePicker{Picker: s.Picker} + ccc.ClientConn.UpdateState(s) } func (ccc *lbCacheClientConn) close() { ccc.mu.Lock() - // Only cancel all existing timers. There's no need to remove SubConns. + defer ccc.mu.Unlock() + // Only cancel all existing timers. There's no need to shut down SubConns. for _, entry := range ccc.subConnCache { entry.cancel() } - ccc.mu.Unlock() +} + +type lbCachePicker struct { + balancer.Picker +} + +func (cp *lbCachePicker) Pick(i balancer.PickInfo) (balancer.PickResult, error) { + res, err := cp.Picker.Pick(i) + if err != nil { + return res, err + } + res.SubConn = res.SubConn.(*lbCacheSubConn).SubConn + return res, nil } diff --git a/balancer/grpclb/grpclb_util_test.go b/balancer/grpclb/grpclb_util_test.go index 463d7ba23c75..c09edc324e15 100644 --- a/balancer/grpclb/grpclb_util_test.go +++ b/balancer/grpclb/grpclb_util_test.go @@ -30,6 +30,13 @@ import ( type mockSubConn struct { balancer.SubConn + mcc *mockClientConn +} + +func (msc *mockSubConn) Shutdown() { + msc.mcc.mu.Lock() + defer msc.mcc.mu.Unlock() + delete(msc.mcc.subConns, msc) } type mockClientConn struct { @@ -46,7 +53,7 @@ func newMockClientConn() *mockClientConn { } func (mcc *mockClientConn) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) { - sc := &mockSubConn{} + sc := &mockSubConn{mcc: mcc} mcc.mu.Lock() defer mcc.mu.Unlock() mcc.subConns[sc] = addrs[0] @@ -54,9 +61,7 @@ func (mcc *mockClientConn) NewSubConn(addrs []resolver.Address, opts balancer.Ne } func (mcc *mockClientConn) RemoveSubConn(sc balancer.SubConn) { - mcc.mu.Lock() - defer mcc.mu.Unlock() - delete(mcc.subConns, sc) + panic(fmt.Sprintf("RemoveSubConn(%v) called unexpectedly", sc)) } const testCacheTimeout = 100 * time.Millisecond @@ -82,7 +87,7 @@ func checkCacheCC(ccc *lbCacheClientConn, sccLen, sctaLen int) error { return nil } -// Test that SubConn won't be immediately removed. +// Test that SubConn won't be immediately shut down. func (s) TestLBCacheClientConnExpire(t *testing.T) { mcc := newMockClientConn() if err := checkMockCC(mcc, 0); err != nil { @@ -105,7 +110,7 @@ func (s) TestLBCacheClientConnExpire(t *testing.T) { t.Fatal(err) } - ccc.RemoveSubConn(sc) + sc.Shutdown() // One subconn in MockCC before timeout. if err := checkMockCC(mcc, 1); err != nil { t.Fatal(err) @@ -133,7 +138,7 @@ func (s) TestLBCacheClientConnExpire(t *testing.T) { } } -// Test that NewSubConn with the same address of a SubConn being removed will +// Test that NewSubConn with the same address of a SubConn being shut down will // reuse the SubConn and cancel the removing. func (s) TestLBCacheClientConnReuse(t *testing.T) { mcc := newMockClientConn() @@ -157,7 +162,7 @@ func (s) TestLBCacheClientConnReuse(t *testing.T) { t.Fatal(err) } - ccc.RemoveSubConn(sc) + sc.Shutdown() // One subconn in MockCC before timeout. if err := checkMockCC(mcc, 1); err != nil { t.Fatal(err) @@ -190,8 +195,8 @@ func (s) TestLBCacheClientConnReuse(t *testing.T) { t.Fatal(err) } - // Call remove again, will delete after timeout. - ccc.RemoveSubConn(sc) + // Call Shutdown again, will delete after timeout. + sc.Shutdown() // One subconn in MockCC before timeout. if err := checkMockCC(mcc, 1); err != nil { t.Fatal(err) @@ -218,9 +223,9 @@ func (s) TestLBCacheClientConnReuse(t *testing.T) { } } -// Test that if the timer to remove a SubConn fires at the same time NewSubConn -// cancels the timer, it doesn't cause deadlock. -func (s) TestLBCache_RemoveTimer_New_Race(t *testing.T) { +// Test that if the timer to shut down a SubConn fires at the same time +// NewSubConn cancels the timer, it doesn't cause deadlock. +func (s) TestLBCache_ShutdownTimer_New_Race(t *testing.T) { mcc := newMockClientConn() if err := checkMockCC(mcc, 0); err != nil { t.Fatal(err) @@ -246,9 +251,9 @@ func (s) TestLBCache_RemoveTimer_New_Race(t *testing.T) { go func() { for i := 0; i < 1000; i++ { - // Remove starts a timer with 1 ns timeout, the NewSubConn will race - // with with the timer. - ccc.RemoveSubConn(sc) + // Shutdown starts a timer with 1 ns timeout, the NewSubConn will + // race with with the timer. + sc.Shutdown() sc, _ = ccc.NewSubConn([]resolver.Address{{Addr: "address1"}}, balancer.NewSubConnOptions{}) } close(done) diff --git a/balancer/grpclb/state/state.go b/balancer/grpclb/state/state.go index a24264a34f5f..4ecfa1c21511 100644 --- a/balancer/grpclb/state/state.go +++ b/balancer/grpclb/state/state.go @@ -39,7 +39,7 @@ type State struct { // Set returns a copy of the provided state with attributes containing s. s's // data should not be mutated after calling Set. func Set(state resolver.State, s *State) resolver.State { - state.Attributes = state.Attributes.WithValues(key, s) + state.Attributes = state.Attributes.WithValue(key, s) return state } diff --git a/balancer/leastrequest/balancer_test.go b/balancer/leastrequest/balancer_test.go new file mode 100644 index 000000000000..44bb21c9e9ff --- /dev/null +++ b/balancer/leastrequest/balancer_test.go @@ -0,0 +1,512 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package leastrequest + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "sync" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/stubserver" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/serviceconfig" +) + +const ( + defaultTestTimeout = 5 * time.Second +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func (s) TestParseConfig(t *testing.T) { + parser := bb{} + tests := []struct { + name string + input string + wantCfg serviceconfig.LoadBalancingConfig + wantErr string + }{ + { + name: "happy-case-default", + input: `{}`, + wantCfg: &LBConfig{ + ChoiceCount: 2, + }, + }, + { + name: "happy-case-choice-count-set", + input: `{"choiceCount": 3}`, + wantCfg: &LBConfig{ + ChoiceCount: 3, + }, + }, + { + name: "happy-case-choice-count-greater-than-ten", + input: `{"choiceCount": 11}`, + wantCfg: &LBConfig{ + ChoiceCount: 10, + }, + }, + { + name: "choice-count-less-than-2", + input: `{"choiceCount": 1}`, + wantErr: "must be >= 2", + }, + { + name: "invalid-json", + input: "{{invalidjson{{", + wantErr: "invalid character", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + gotCfg, gotErr := parser.ParseConfig(json.RawMessage(test.input)) + // Substring match makes this very tightly coupled to the + // internalserviceconfig.BalancerConfig error strings. However, it + // is important to distinguish the different types of error messages + // possible as the parser has a few defined buckets of ways it can + // error out. + if (gotErr != nil) != (test.wantErr != "") { + t.Fatalf("ParseConfig(%v) = %v, wantErr %v", test.input, gotErr, test.wantErr) + } + if gotErr != nil && !strings.Contains(gotErr.Error(), test.wantErr) { + t.Fatalf("ParseConfig(%v) = %v, wantErr %v", test.input, gotErr, test.wantErr) + } + if test.wantErr != "" { + return + } + if diff := cmp.Diff(gotCfg, test.wantCfg); diff != "" { + t.Fatalf("ParseConfig(%v) got unexpected output, diff (-got +want): %v", test.input, diff) + } + }) + } +} + +// setupBackends spins up three test backends, each listening on a port on +// localhost. The three backends always reply with an empty response with no +// error, and for streaming receive until hitting an EOF error. +func setupBackends(t *testing.T) []string { + t.Helper() + const numBackends = 3 + addresses := make([]string, numBackends) + // Construct and start three working backends. + for i := 0; i < numBackends; i++ { + backend := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + <-stream.Context().Done() + return nil + }, + } + if err := backend.StartServer(); err != nil { + t.Fatalf("Failed to start backend: %v", err) + } + t.Logf("Started good TestService backend at: %q", backend.Address) + t.Cleanup(func() { backend.Stop() }) + addresses[i] = backend.Address + } + return addresses +} + +// checkRoundRobinRPCs verifies that EmptyCall RPCs on the given ClientConn, +// connected to a server exposing the test.grpc_testing.TestService, are +// roundrobined across the given backend addresses. +// +// Returns a non-nil error if context deadline expires before RPCs start to get +// roundrobined across the given backends. +func checkRoundRobinRPCs(ctx context.Context, client testgrpc.TestServiceClient, addrs []resolver.Address) error { + wantAddrCount := make(map[string]int) + for _, addr := range addrs { + wantAddrCount[addr.Addr]++ + } + gotAddrCount := make(map[string]int) + for ; ctx.Err() == nil; <-time.After(time.Millisecond) { + gotAddrCount = make(map[string]int) + // Perform 3 iterations. + var iterations [][]string + for i := 0; i < 3; i++ { + iteration := make([]string, len(addrs)) + for c := 0; c < len(addrs); c++ { + var peer peer.Peer + client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&peer)) + iteration[c] = peer.Addr.String() + } + iterations = append(iterations, iteration) + } + // Ensure the the first iteration contains all addresses in addrs. + for _, addr := range iterations[0] { + gotAddrCount[addr]++ + } + if !cmp.Equal(gotAddrCount, wantAddrCount) { + continue + } + // Ensure all three iterations contain the same addresses. + if !cmp.Equal(iterations[0], iterations[1]) || !cmp.Equal(iterations[0], iterations[2]) { + continue + } + return nil + } + return fmt.Errorf("timeout when waiting for roundrobin distribution of RPCs across addresses: %v; got: %v", addrs, gotAddrCount) +} + +// TestLeastRequestE2E tests the Least Request LB policy in an e2e style. The +// Least Request balancer is configured as the top level balancer of the +// channel, and is passed three addresses. Eventually, the test creates three +// streams, which should be on certain backends according to the least request +// algorithm. The randomness in the picker is injected in the test to be +// deterministic, allowing the test to make assertions on the distribution. +func (s) TestLeastRequestE2E(t *testing.T) { + defer func(u func() uint32) { + grpcranduint32 = u + }(grpcranduint32) + var index int + indexes := []uint32{ + 0, 0, 1, 1, 2, 2, // Triggers a round robin distribution. + } + grpcranduint32 = func() uint32 { + ret := indexes[index%len(indexes)] + index++ + return ret + } + addresses := setupBackends(t) + + mr := manual.NewBuilderWithScheme("lr-e2e") + defer mr.Close() + + // Configure least request as top level balancer of channel. + lrscJSON := ` +{ + "loadBalancingConfig": [ + { + "least_request_experimental": { + "choiceCount": 2 + } + } + ] +}` + sc := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(lrscJSON) + firstThreeAddresses := []resolver.Address{ + {Addr: addresses[0]}, + {Addr: addresses[1]}, + {Addr: addresses[2]}, + } + mr.InitialState(resolver.State{ + Addresses: firstThreeAddresses, + ServiceConfig: sc, + }) + + cc, err := grpc.Dial(mr.Scheme()+":///", grpc.WithResolvers(mr), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) + } + defer cc.Close() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testServiceClient := testgrpc.NewTestServiceClient(cc) + + // Wait for all 3 backends to round robin across. The happens because a + // SubConn transitioning into READY causes a new picker update. Once the + // picker update with all 3 backends is present, this test can start to make + // assertions based on those backends. + if err := checkRoundRobinRPCs(ctx, testServiceClient, firstThreeAddresses); err != nil { + t.Fatalf("error in expected round robin: %v", err) + } + + // Map ordering of READY SubConns is non deterministic. Thus, perform 3 RPCs + // mocked from the random to each index to learn the addresses of SubConns + // at each index. + index = 0 + peerAtIndex := make([]string, 3) + var peer0 peer.Peer + if _, err := testServiceClient.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&peer0)); err != nil { + t.Fatalf("testServiceClient.EmptyCall failed: %v", err) + } + peerAtIndex[0] = peer0.Addr.String() + if _, err := testServiceClient.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&peer0)); err != nil { + t.Fatalf("testServiceClient.EmptyCall failed: %v", err) + } + peerAtIndex[1] = peer0.Addr.String() + if _, err := testServiceClient.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&peer0)); err != nil { + t.Fatalf("testServiceClient.EmptyCall failed: %v", err) + } + peerAtIndex[2] = peer0.Addr.String() + + // Start streaming RPCs, but do not finish them. Each subsequent stream + // should be started according to the least request algorithm, and chosen + // between the indexes provided. + index = 0 + indexes = []uint32{ + 0, 0, // Causes first stream to be on first address. + 0, 1, // Compares first address (one RPC) to second (no RPCs), so choose second. + 1, 2, // Compares second address (one RPC) to third (no RPCs), so choose third. + 0, 3, // Causes another stream on first address. + 1, 0, // Compares second address (one RPC) to first (two RPCs), so choose second. + 2, 0, // Compares third address (one RPC) to first (two RPCs), so choose third. + 0, 0, // Causes another stream on first address. + 2, 2, // Causes a stream on third address. + 2, 1, // Compares third address (three RPCs) to second (two RPCs), so choose third. + } + wantIndex := []uint32{0, 1, 2, 0, 1, 2, 0, 2, 1} + + // Start streaming RPC's, but do not finish them. Each created stream should + // be started based on the least request algorithm and injected randomness + // (see indexes slice above for exact expectations). + for _, wantIndex := range wantIndex { + stream, err := testServiceClient.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("testServiceClient.FullDuplexCall failed: %v", err) + } + p, ok := peer.FromContext(stream.Context()) + if !ok { + t.Fatalf("testServiceClient.FullDuplexCall has no Peer") + } + if p.Addr.String() != peerAtIndex[wantIndex] { + t.Fatalf("testServiceClient.FullDuplexCall's Peer got: %v, want: %v", p.Addr.String(), peerAtIndex[wantIndex]) + } + } +} + +// TestLeastRequestPersistsCounts tests that the Least Request Balancer persists +// counts once it gets a new picker update. It first updates the Least Request +// Balancer with two backends, and creates a bunch of streams on them. Then, it +// updates the Least Request Balancer with three backends, including the two +// previous. Any created streams should then be started on the new backend. +func (s) TestLeastRequestPersistsCounts(t *testing.T) { + defer func(u func() uint32) { + grpcranduint32 = u + }(grpcranduint32) + var index int + indexes := []uint32{ + 0, 0, 1, 1, + } + grpcranduint32 = func() uint32 { + ret := indexes[index%len(indexes)] + index++ + return ret + } + addresses := setupBackends(t) + + mr := manual.NewBuilderWithScheme("lr-e2e") + defer mr.Close() + + // Configure least request as top level balancer of channel. + lrscJSON := ` +{ + "loadBalancingConfig": [ + { + "least_request_experimental": { + "choiceCount": 2 + } + } + ] +}` + sc := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(lrscJSON) + firstTwoAddresses := []resolver.Address{ + {Addr: addresses[0]}, + {Addr: addresses[1]}, + } + mr.InitialState(resolver.State{ + Addresses: firstTwoAddresses, + ServiceConfig: sc, + }) + + cc, err := grpc.Dial(mr.Scheme()+":///", grpc.WithResolvers(mr), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) + } + defer cc.Close() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testServiceClient := testgrpc.NewTestServiceClient(cc) + + // Wait for the two backends to round robin across. The happens because a + // SubConn transitioning into READY causes a new picker update. Once the + // picker update with the two backends is present, this test can start to + // populate those backends with streams. + if err := checkRoundRobinRPCs(ctx, testServiceClient, firstTwoAddresses); err != nil { + t.Fatalf("error in expected round robin: %v", err) + } + + // Start 50 streaming RPCs, and leave them unfinished for the duration of + // the test. This will populate the first two addresses with many active + // RPCs. + for i := 0; i < 50; i++ { + _, err := testServiceClient.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("testServiceClient.FullDuplexCall failed: %v", err) + } + } + + // Update the least request balancer to choice count 3. Also update the + // address list adding a third address. Alongside the injected randomness, + // this should trigger the least request balancer to search all created + // SubConns. Thus, since address 3 is the new address and the first two + // addresses are populated with RPCs, once the picker update of all 3 READY + // SubConns takes effect, all new streams should be started on address 3. + index = 0 + indexes = []uint32{ + 0, 1, 2, 3, 4, 5, + } + lrscJSON = ` +{ + "loadBalancingConfig": [ + { + "least_request_experimental": { + "choiceCount": 3 + } + } + ] +}` + sc = internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(lrscJSON) + fullAddresses := []resolver.Address{ + {Addr: addresses[0]}, + {Addr: addresses[1]}, + {Addr: addresses[2]}, + } + mr.UpdateState(resolver.State{ + Addresses: fullAddresses, + ServiceConfig: sc, + }) + newAddress := fullAddresses[2] + // Poll for only address 3 to show up. This requires a polling loop because + // picker update with all three SubConns doesn't take into effect + // immediately, needs the third SubConn to become READY. + if err := checkRoundRobinRPCs(ctx, testServiceClient, []resolver.Address{newAddress}); err != nil { + t.Fatalf("error in expected round robin: %v", err) + } + + // Start 25 rpcs, but don't finish them. They should all start on address 3, + // since the first two addresses both have 25 RPCs (and randomness + // injection/choiceCount causes all 3 to be compared every iteration). + for i := 0; i < 25; i++ { + stream, err := testServiceClient.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("testServiceClient.FullDuplexCall failed: %v", err) + } + p, ok := peer.FromContext(stream.Context()) + if !ok { + t.Fatalf("testServiceClient.FullDuplexCall has no Peer") + } + if p.Addr.String() != addresses[2] { + t.Fatalf("testServiceClient.FullDuplexCall's Peer got: %v, want: %v", p.Addr.String(), addresses[2]) + } + } + + // Now 25 RPC's are active on each address, the next three RPC's should + // round robin, since choiceCount is three and the injected random indexes + // cause it to search all three addresses for fewest outstanding requests on + // each iteration. + wantAddrCount := map[string]int{ + addresses[0]: 1, + addresses[1]: 1, + addresses[2]: 1, + } + gotAddrCount := make(map[string]int) + for i := 0; i < len(addresses); i++ { + stream, err := testServiceClient.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("testServiceClient.FullDuplexCall failed: %v", err) + } + p, ok := peer.FromContext(stream.Context()) + if !ok { + t.Fatalf("testServiceClient.FullDuplexCall has no Peer") + } + if p.Addr != nil { + gotAddrCount[p.Addr.String()]++ + } + } + if diff := cmp.Diff(gotAddrCount, wantAddrCount); diff != "" { + t.Fatalf("addr count (-got:, +want): %v", diff) + } +} + +// TestConcurrentRPCs tests concurrent RPCs on the least request balancer. It +// configures a channel with a least request balancer as the top level balancer, +// and makes 100 RPCs asynchronously. This makes sure no race conditions happen +// in this scenario. +func (s) TestConcurrentRPCs(t *testing.T) { + addresses := setupBackends(t) + + mr := manual.NewBuilderWithScheme("lr-e2e") + defer mr.Close() + + // Configure least request as top level balancer of channel. + lrscJSON := ` +{ + "loadBalancingConfig": [ + { + "least_request_experimental": { + "choiceCount": 2 + } + } + ] +}` + sc := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(lrscJSON) + firstTwoAddresses := []resolver.Address{ + {Addr: addresses[0]}, + {Addr: addresses[1]}, + } + mr.InitialState(resolver.State{ + Addresses: firstTwoAddresses, + ServiceConfig: sc, + }) + + cc, err := grpc.Dial(mr.Scheme()+":///", grpc.WithResolvers(mr), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) + } + defer cc.Close() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testServiceClient := testgrpc.NewTestServiceClient(cc) + + var wg sync.WaitGroup + for i := 0; i < 100; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < 5; j++ { + testServiceClient.EmptyCall(ctx, &testpb.Empty{}) + } + }() + } + wg.Wait() + +} diff --git a/balancer/leastrequest/leastrequest.go b/balancer/leastrequest/leastrequest.go new file mode 100644 index 000000000000..3289f2869f88 --- /dev/null +++ b/balancer/leastrequest/leastrequest.go @@ -0,0 +1,181 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package leastrequest implements a least request load balancer. +package leastrequest + +import ( + "encoding/json" + "fmt" + "sync/atomic" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/base" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal/grpcrand" + "google.golang.org/grpc/serviceconfig" +) + +// grpcranduint32 is a global to stub out in tests. +var grpcranduint32 = grpcrand.Uint32 + +// Name is the name of the least request balancer. +const Name = "least_request_experimental" + +var logger = grpclog.Component("least-request") + +func init() { + balancer.Register(bb{}) +} + +// LBConfig is the balancer config for least_request_experimental balancer. +type LBConfig struct { + serviceconfig.LoadBalancingConfig `json:"-"` + + // ChoiceCount is the number of random SubConns to sample to find the one + // with the fewest outstanding requests. If unset, defaults to 2. If set to + // < 2, the config will be rejected, and if set to > 10, will become 10. + ChoiceCount uint32 `json:"choiceCount,omitempty"` +} + +type bb struct{} + +func (bb) ParseConfig(s json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + lbConfig := &LBConfig{ + ChoiceCount: 2, + } + if err := json.Unmarshal(s, lbConfig); err != nil { + return nil, fmt.Errorf("least-request: unable to unmarshal LBConfig: %v", err) + } + // "If `choice_count < 2`, the config will be rejected." - A48 + if lbConfig.ChoiceCount < 2 { // sweet + return nil, fmt.Errorf("least-request: lbConfig.choiceCount: %v, must be >= 2", lbConfig.ChoiceCount) + } + // "If a LeastRequestLoadBalancingConfig with a choice_count > 10 is + // received, the least_request_experimental policy will set choice_count = + // 10." - A48 + if lbConfig.ChoiceCount > 10 { + lbConfig.ChoiceCount = 10 + } + return lbConfig, nil +} + +func (bb) Name() string { + return Name +} + +func (bb) Build(cc balancer.ClientConn, bOpts balancer.BuildOptions) balancer.Balancer { + b := &leastRequestBalancer{scRPCCounts: make(map[balancer.SubConn]*atomic.Int32)} + baseBuilder := base.NewBalancerBuilder(Name, b, base.Config{HealthCheck: true}) + b.Balancer = baseBuilder.Build(cc, bOpts) + return b +} + +type leastRequestBalancer struct { + // Embeds balancer.Balancer because needs to intercept UpdateClientConnState + // to learn about choiceCount. + balancer.Balancer + + choiceCount uint32 + scRPCCounts map[balancer.SubConn]*atomic.Int32 // Hold onto RPC counts to keep track for subsequent picker updates. +} + +func (lrb *leastRequestBalancer) UpdateClientConnState(s balancer.ClientConnState) error { + lrCfg, ok := s.BalancerConfig.(*LBConfig) + if !ok { + logger.Errorf("least-request: received config with unexpected type %T: %v", s.BalancerConfig, s.BalancerConfig) + return balancer.ErrBadResolverState + } + + lrb.choiceCount = lrCfg.ChoiceCount + return lrb.Balancer.UpdateClientConnState(s) +} + +type scWithRPCCount struct { + sc balancer.SubConn + numRPCs *atomic.Int32 +} + +func (lrb *leastRequestBalancer) Build(info base.PickerBuildInfo) balancer.Picker { + logger.Infof("least-request: Build called with info: %v", info) + if len(info.ReadySCs) == 0 { + return base.NewErrPicker(balancer.ErrNoSubConnAvailable) + } + + for sc := range lrb.scRPCCounts { + if _, ok := info.ReadySCs[sc]; !ok { // If no longer ready, no more need for the ref to count active RPCs. + delete(lrb.scRPCCounts, sc) + } + } + + // Create new refs if needed. + for sc := range info.ReadySCs { + if _, ok := lrb.scRPCCounts[sc]; !ok { + lrb.scRPCCounts[sc] = new(atomic.Int32) + } + } + + // Copy refs to counters into picker. + scs := make([]scWithRPCCount, 0, len(info.ReadySCs)) + for sc := range info.ReadySCs { + scs = append(scs, scWithRPCCount{ + sc: sc, + numRPCs: lrb.scRPCCounts[sc], // guaranteed to be present due to algorithm + }) + } + + return &picker{ + choiceCount: lrb.choiceCount, + subConns: scs, + } +} + +type picker struct { + // choiceCount is the number of random SubConns to find the one with + // the least request. + choiceCount uint32 + // Built out when receives list of ready RPCs. + subConns []scWithRPCCount +} + +func (p *picker) Pick(balancer.PickInfo) (balancer.PickResult, error) { + var pickedSC *scWithRPCCount + var pickedSCNumRPCs int32 + for i := 0; i < int(p.choiceCount); i++ { + index := grpcranduint32() % uint32(len(p.subConns)) + sc := p.subConns[index] + n := sc.numRPCs.Load() + if pickedSC == nil || n < pickedSCNumRPCs { + pickedSC = &sc + pickedSCNumRPCs = n + } + } + // "The counter for a subchannel should be atomically incremented by one + // after it has been successfully picked by the picker." - A48 + pickedSC.numRPCs.Add(1) + // "the picker should add a callback for atomically decrementing the + // subchannel counter once the RPC finishes (regardless of Status code)." - + // A48. + done := func(balancer.DoneInfo) { + pickedSC.numRPCs.Add(-1) + } + return balancer.PickResult{ + SubConn: pickedSC.sc, + Done: done, + }, nil +} diff --git a/balancer/rls/balancer.go b/balancer/rls/balancer.go new file mode 100644 index 000000000000..3ac28271618b --- /dev/null +++ b/balancer/rls/balancer.go @@ -0,0 +1,664 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package rls implements the RLS LB policy. +package rls + +import ( + "encoding/json" + "errors" + "fmt" + "sync" + "sync/atomic" + "time" + "unsafe" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/backoff" + "google.golang.org/grpc/internal/balancergroup" + "google.golang.org/grpc/internal/buffer" + internalgrpclog "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/resolver" +) + +const ( + // Name is the name of the RLS LB policy. + // + // It currently has an experimental suffix which would be removed once + // end-to-end testing of the policy is completed. + Name = internal.RLSLoadBalancingPolicyName + // Default frequency for data cache purging. + periodicCachePurgeFreq = time.Minute +) + +var ( + logger = grpclog.Component("rls") + errBalancerClosed = errors.New("rls LB policy is closed") + + // Below defined vars for overriding in unit tests. + + // Default exponential backoff strategy for data cache entries. + defaultBackoffStrategy = backoff.Strategy(backoff.DefaultExponential) + // Ticker used for periodic data cache purging. + dataCachePurgeTicker = func() *time.Ticker { return time.NewTicker(periodicCachePurgeFreq) } + // We want every cache entry to live in the cache for at least this + // duration. If we encounter a cache entry whose minimum expiration time is + // in the future, we abort the LRU pass, which may temporarily leave the + // cache being too large. This is necessary to ensure that in cases where + // the cache is too small, when we receive an RLS Response, we keep the + // resulting cache entry around long enough for the pending incoming + // requests to be re-processed through the new Picker. If we didn't do this, + // then we'd risk throwing away each RLS response as we receive it, in which + // case we would fail to actually route any of our incoming requests. + minEvictDuration = 5 * time.Second + + // Following functions are no-ops in actual code, but can be overridden in + // tests to give tests visibility into exactly when certain events happen. + clientConnUpdateHook = func() {} + dataCachePurgeHook = func() {} + resetBackoffHook = func() {} +) + +func init() { + balancer.Register(&rlsBB{}) +} + +type rlsBB struct{} + +func (rlsBB) Name() string { + return Name +} + +func (rlsBB) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { + lb := &rlsBalancer{ + closed: grpcsync.NewEvent(), + done: grpcsync.NewEvent(), + cc: cc, + bopts: opts, + purgeTicker: dataCachePurgeTicker(), + dataCachePurgeHook: dataCachePurgeHook, + lbCfg: &lbConfig{}, + pendingMap: make(map[cacheKey]*backoffState), + childPolicies: make(map[string]*childPolicyWrapper), + updateCh: buffer.NewUnbounded(), + } + lb.logger = internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf("[rls-experimental-lb %p] ", lb)) + lb.dataCache = newDataCache(maxCacheSize, lb.logger) + lb.bg = balancergroup.New(balancergroup.Options{ + CC: cc, + BuildOpts: opts, + StateAggregator: lb, + Logger: lb.logger, + SubBalancerCloseTimeout: time.Duration(0), // Disable caching of removed child policies + }) + lb.bg.Start() + go lb.run() + return lb +} + +// rlsBalancer implements the RLS LB policy. +type rlsBalancer struct { + closed *grpcsync.Event // Fires when Close() is invoked. Guarded by stateMu. + done *grpcsync.Event // Fires when Close() is done. + cc balancer.ClientConn + bopts balancer.BuildOptions + purgeTicker *time.Ticker + dataCachePurgeHook func() + logger *internalgrpclog.PrefixLogger + + // If both cacheMu and stateMu need to be acquired, the former must be + // acquired first to prevent a deadlock. This order restriction is due to the + // fact that in places where we need to acquire both the locks, we always + // start off reading the cache. + + // cacheMu guards access to the data cache and pending requests map. We + // cannot use an RWMutex here since even an operation like + // dataCache.getEntry() modifies the underlying LRU, which is implemented as + // a doubly linked list. + cacheMu sync.Mutex + dataCache *dataCache // Cache of RLS data. + pendingMap map[cacheKey]*backoffState // Map of pending RLS requests. + + // stateMu guards access to all LB policy state. + stateMu sync.Mutex + lbCfg *lbConfig // Most recently received service config. + childPolicyBuilder balancer.Builder // Cached child policy builder. + resolverState resolver.State // Cached resolver state. + ctrlCh *controlChannel // Control channel to the RLS server. + bg *balancergroup.BalancerGroup + childPolicies map[string]*childPolicyWrapper + defaultPolicy *childPolicyWrapper + // A reference to the most recent picker sent to gRPC as part of a state + // update is cached in this field so that we can release the reference to the + // default child policy wrapper when a new picker is created. See + // sendNewPickerLocked() for details. + lastPicker *rlsPicker + // Set during UpdateClientConnState when pushing updates to child policies. + // Prevents state updates from child policies causing new pickers to be sent + // up the channel. Cleared after all child policies have processed the + // updates sent to them, after which a new picker is sent up the channel. + inhibitPickerUpdates bool + + // Channel on which all updates are pushed. Processed in run(). + updateCh *buffer.Unbounded +} + +type resumePickerUpdates struct { + done chan struct{} +} + +// childPolicyIDAndState wraps a child policy id and its state update. +type childPolicyIDAndState struct { + id string + state balancer.State +} + +type controlChannelReady struct{} + +// run is a long-running goroutine which handles all the updates that the +// balancer wishes to handle. The appropriate updateHandler will push the update +// on to a channel that this goroutine will select on, thereby the handling of +// the update will happen asynchronously. +func (b *rlsBalancer) run() { + // We exit out of the for loop below only after `Close()` has been invoked. + // Firing the done event here will ensure that Close() returns only after + // all goroutines are done. + defer func() { b.done.Fire() }() + + // Wait for purgeDataCache() goroutine to exit before returning from here. + doneCh := make(chan struct{}) + defer func() { + <-doneCh + }() + go b.purgeDataCache(doneCh) + + for { + select { + case u, ok := <-b.updateCh.Get(): + if !ok { + return + } + b.updateCh.Load() + switch update := u.(type) { + case childPolicyIDAndState: + b.handleChildPolicyStateUpdate(update.id, update.state) + case controlChannelReady: + b.logger.Infof("Resetting backoff state after control channel getting back to READY") + b.cacheMu.Lock() + updatePicker := b.dataCache.resetBackoffState(&backoffState{bs: defaultBackoffStrategy}) + b.cacheMu.Unlock() + if updatePicker { + b.sendNewPicker() + } + resetBackoffHook() + case resumePickerUpdates: + b.stateMu.Lock() + b.logger.Infof("Resuming picker updates after config propagation to child policies") + b.inhibitPickerUpdates = false + b.sendNewPickerLocked() + close(update.done) + b.stateMu.Unlock() + default: + b.logger.Errorf("Unsupported update type %T", update) + } + case <-b.closed.Done(): + return + } + } +} + +// purgeDataCache is a long-running goroutine which periodically deletes expired +// entries. An expired entry is one for which both the expiryTime and +// backoffExpiryTime are in the past. +func (b *rlsBalancer) purgeDataCache(doneCh chan struct{}) { + defer close(doneCh) + + for { + select { + case <-b.closed.Done(): + return + case <-b.purgeTicker.C: + b.cacheMu.Lock() + updatePicker := b.dataCache.evictExpiredEntries() + b.cacheMu.Unlock() + if updatePicker { + b.sendNewPicker() + } + b.dataCachePurgeHook() + } + } +} + +func (b *rlsBalancer) UpdateClientConnState(ccs balancer.ClientConnState) error { + defer clientConnUpdateHook() + + b.stateMu.Lock() + if b.closed.HasFired() { + b.stateMu.Unlock() + b.logger.Warningf("Received service config after balancer close: %s", pretty.ToJSON(ccs.BalancerConfig)) + return errBalancerClosed + } + + newCfg := ccs.BalancerConfig.(*lbConfig) + if b.lbCfg.Equal(newCfg) { + b.stateMu.Unlock() + b.logger.Infof("New service config matches existing config") + return nil + } + + b.logger.Infof("Delaying picker updates until config is propagated to and processed by child policies") + b.inhibitPickerUpdates = true + + // When the RLS server name changes, the old control channel needs to be + // swapped out for a new one. All state associated with the throttling + // algorithm is stored on a per-control-channel basis; when we swap out + // channels, we also swap out the throttling state. + b.handleControlChannelUpdate(newCfg) + + // Any changes to child policy name or configuration needs to be handled by + // either creating new child policies or pushing updates to existing ones. + b.resolverState = ccs.ResolverState + b.handleChildPolicyConfigUpdate(newCfg, &ccs) + + // Resize the cache if the size in the config has changed. + resizeCache := newCfg.cacheSizeBytes != b.lbCfg.cacheSizeBytes + + // Update the copy of the config in the LB policy before releasing the lock. + b.lbCfg = newCfg + + // Enqueue an event which will notify us when the above update has been + // propagated to all child policies, and the child policies have all + // processed their updates, and we have sent a picker update. + done := make(chan struct{}) + b.updateCh.Put(resumePickerUpdates{done: done}) + b.stateMu.Unlock() + <-done + + if resizeCache { + // If the new config changes reduces the size of the data cache, we + // might have to evict entries to get the cache size down to the newly + // specified size. + // + // And we cannot do this operation above (where we compute the + // `resizeCache` boolean) because `cacheMu` needs to be grabbed before + // `stateMu` if we are to hold both locks at the same time. + b.cacheMu.Lock() + b.dataCache.resize(newCfg.cacheSizeBytes) + b.cacheMu.Unlock() + } + return nil +} + +// handleControlChannelUpdate handles updates to service config fields which +// influence the control channel to the RLS server. +// +// Caller must hold lb.stateMu. +func (b *rlsBalancer) handleControlChannelUpdate(newCfg *lbConfig) { + if newCfg.lookupService == b.lbCfg.lookupService && newCfg.lookupServiceTimeout == b.lbCfg.lookupServiceTimeout { + return + } + + // Create a new control channel and close the existing one. + b.logger.Infof("Creating control channel to RLS server at: %v", newCfg.lookupService) + backToReadyFn := func() { + b.updateCh.Put(controlChannelReady{}) + } + ctrlCh, err := newControlChannel(newCfg.lookupService, newCfg.controlChannelServiceConfig, newCfg.lookupServiceTimeout, b.bopts, backToReadyFn) + if err != nil { + // This is very uncommon and usually represents a non-transient error. + // There is not much we can do here other than wait for another update + // which might fix things. + b.logger.Errorf("Failed to create control channel to %q: %v", newCfg.lookupService, err) + return + } + if b.ctrlCh != nil { + b.ctrlCh.close() + } + b.ctrlCh = ctrlCh +} + +// handleChildPolicyConfigUpdate handles updates to service config fields which +// influence child policy configuration. +// +// Caller must hold lb.stateMu. +func (b *rlsBalancer) handleChildPolicyConfigUpdate(newCfg *lbConfig, ccs *balancer.ClientConnState) { + // Update child policy builder first since other steps are dependent on this. + if b.childPolicyBuilder == nil || b.childPolicyBuilder.Name() != newCfg.childPolicyName { + b.logger.Infof("Child policy changed to %q", newCfg.childPolicyName) + b.childPolicyBuilder = balancer.Get(newCfg.childPolicyName) + for _, cpw := range b.childPolicies { + // If the child policy has changed, we need to remove the old policy + // from the BalancerGroup and add a new one. The BalancerGroup takes + // care of closing the old one in this case. + b.bg.Remove(cpw.target) + b.bg.Add(cpw.target, b.childPolicyBuilder) + } + } + + configSentToDefault := false + if b.lbCfg.defaultTarget != newCfg.defaultTarget { + // If the default target has changed, create a new childPolicyWrapper for + // the new target if required. If a new wrapper is created, add it to the + // childPolicies map and the BalancerGroup. + b.logger.Infof("Default target in LB config changing from %q to %q", b.lbCfg.defaultTarget, newCfg.defaultTarget) + cpw := b.childPolicies[newCfg.defaultTarget] + if cpw == nil { + cpw = newChildPolicyWrapper(newCfg.defaultTarget) + b.childPolicies[newCfg.defaultTarget] = cpw + b.bg.Add(newCfg.defaultTarget, b.childPolicyBuilder) + b.logger.Infof("Child policy %q added to BalancerGroup", newCfg.defaultTarget) + } + if err := b.buildAndPushChildPolicyConfigs(newCfg.defaultTarget, newCfg, ccs); err != nil { + cpw.lamify(err) + } + + // If an old default exists, release its reference. If this was the last + // reference, remove the child policy from the BalancerGroup and remove the + // corresponding entry the childPolicies map. + if b.defaultPolicy != nil { + if b.defaultPolicy.releaseRef() { + delete(b.childPolicies, b.lbCfg.defaultTarget) + b.bg.Remove(b.defaultPolicy.target) + } + } + b.defaultPolicy = cpw + configSentToDefault = true + } + + // No change in configuration affecting child policies. Return early. + if b.lbCfg.childPolicyName == newCfg.childPolicyName && b.lbCfg.childPolicyTargetField == newCfg.childPolicyTargetField && childPolicyConfigEqual(b.lbCfg.childPolicyConfig, newCfg.childPolicyConfig) { + return + } + + // If fields affecting child policy configuration have changed, the changes + // are pushed to the childPolicyWrapper which handles them appropriately. + for _, cpw := range b.childPolicies { + if configSentToDefault && cpw.target == newCfg.defaultTarget { + // Default target has already been taken care of. + continue + } + if err := b.buildAndPushChildPolicyConfigs(cpw.target, newCfg, ccs); err != nil { + cpw.lamify(err) + } + } +} + +// buildAndPushChildPolicyConfigs builds the final child policy configuration by +// adding the `targetField` to the base child policy configuration received in +// RLS LB policy configuration. The `targetField` is set to target and +// configuration is pushed to the child policy through the BalancerGroup. +// +// Caller must hold lb.stateMu. +func (b *rlsBalancer) buildAndPushChildPolicyConfigs(target string, newCfg *lbConfig, ccs *balancer.ClientConnState) error { + jsonTarget, err := json.Marshal(target) + if err != nil { + return fmt.Errorf("failed to marshal child policy target %q: %v", target, err) + } + + config := newCfg.childPolicyConfig + targetField := newCfg.childPolicyTargetField + config[targetField] = jsonTarget + jsonCfg, err := json.Marshal(config) + if err != nil { + return fmt.Errorf("failed to marshal child policy config %+v: %v", config, err) + } + + parser, _ := b.childPolicyBuilder.(balancer.ConfigParser) + parsedCfg, err := parser.ParseConfig(jsonCfg) + if err != nil { + return fmt.Errorf("childPolicy config parsing failed: %v", err) + } + + state := balancer.ClientConnState{ResolverState: ccs.ResolverState, BalancerConfig: parsedCfg} + b.logger.Infof("Pushing new state to child policy %q: %+v", target, state) + if err := b.bg.UpdateClientConnState(target, state); err != nil { + b.logger.Warningf("UpdateClientConnState(%q, %+v) failed : %v", target, ccs, err) + } + return nil +} + +func (b *rlsBalancer) ResolverError(err error) { + b.bg.ResolverError(err) +} + +func (b *rlsBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { + b.logger.Errorf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, state) +} + +func (b *rlsBalancer) Close() { + b.stateMu.Lock() + b.closed.Fire() + b.purgeTicker.Stop() + if b.ctrlCh != nil { + b.ctrlCh.close() + } + b.bg.Close() + b.stateMu.Unlock() + + b.cacheMu.Lock() + b.dataCache.stop() + b.cacheMu.Unlock() + + b.updateCh.Close() + + <-b.done.Done() +} + +func (b *rlsBalancer) ExitIdle() { + b.bg.ExitIdle() +} + +// sendNewPickerLocked pushes a new picker on to the channel. +// +// Note that regardless of what connectivity state is reported, the policy will +// return its own picker, and not a picker that unconditionally queues +// (typically used for IDLE or CONNECTING) or a picker that unconditionally +// fails (typically used for TRANSIENT_FAILURE). This is required because, +// irrespective of the connectivity state, we need to able to perform RLS +// lookups for incoming RPCs and affect the status of queued RPCs based on the +// receipt of RLS responses. +// +// Caller must hold lb.stateMu. +func (b *rlsBalancer) sendNewPickerLocked() { + aggregatedState := b.aggregatedConnectivityState() + + // Acquire a separate reference for the picker. This is required to ensure + // that the wrapper held by the old picker is not closed when the default + // target changes in the config, and a new wrapper is created for the new + // default target. See handleChildPolicyConfigUpdate() for how config changes + // affecting the default target are handled. + if b.defaultPolicy != nil { + b.defaultPolicy.acquireRef() + } + picker := &rlsPicker{ + kbm: b.lbCfg.kbMap, + origEndpoint: b.bopts.Target.Endpoint(), + lb: b, + defaultPolicy: b.defaultPolicy, + ctrlCh: b.ctrlCh, + maxAge: b.lbCfg.maxAge, + staleAge: b.lbCfg.staleAge, + bg: b.bg, + } + picker.logger = internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf("[rls-picker %p] ", picker)) + state := balancer.State{ + ConnectivityState: aggregatedState, + Picker: picker, + } + + if !b.inhibitPickerUpdates { + b.logger.Infof("New balancer.State: %+v", state) + b.cc.UpdateState(state) + } else { + b.logger.Infof("Delaying picker update: %+v", state) + } + + if b.lastPicker != nil { + if b.defaultPolicy != nil { + b.defaultPolicy.releaseRef() + } + } + b.lastPicker = picker +} + +func (b *rlsBalancer) sendNewPicker() { + b.stateMu.Lock() + defer b.stateMu.Unlock() + if b.closed.HasFired() { + return + } + b.sendNewPickerLocked() +} + +// The aggregated connectivity state reported is determined as follows: +// - If there is at least one child policy in state READY, the connectivity +// state is READY. +// - Otherwise, if there is at least one child policy in state CONNECTING, the +// connectivity state is CONNECTING. +// - Otherwise, if there is at least one child policy in state IDLE, the +// connectivity state is IDLE. +// - Otherwise, all child policies are in TRANSIENT_FAILURE, and the +// connectivity state is TRANSIENT_FAILURE. +// +// If the RLS policy has no child policies and no configured default target, +// then we will report connectivity state IDLE. +// +// Caller must hold lb.stateMu. +func (b *rlsBalancer) aggregatedConnectivityState() connectivity.State { + if len(b.childPolicies) == 0 && b.lbCfg.defaultTarget == "" { + return connectivity.Idle + } + + var readyN, connectingN, idleN int + for _, cpw := range b.childPolicies { + state := (*balancer.State)(atomic.LoadPointer(&cpw.state)) + switch state.ConnectivityState { + case connectivity.Ready: + readyN++ + case connectivity.Connecting: + connectingN++ + case connectivity.Idle: + idleN++ + } + } + + switch { + case readyN > 0: + return connectivity.Ready + case connectingN > 0: + return connectivity.Connecting + case idleN > 0: + return connectivity.Idle + default: + return connectivity.TransientFailure + } +} + +// UpdateState is a implementation of the balancergroup.BalancerStateAggregator +// interface. The actual state aggregation functionality is handled +// asynchronously. This method only pushes the state update on to channel read +// and dispatched by the run() goroutine. +func (b *rlsBalancer) UpdateState(id string, state balancer.State) { + b.updateCh.Put(childPolicyIDAndState{id: id, state: state}) +} + +// handleChildPolicyStateUpdate provides the state aggregator functionality for +// the BalancerGroup. +// +// This method is invoked by the BalancerGroup whenever a child policy sends a +// state update. We cache the child policy's connectivity state and picker for +// two reasons: +// - to suppress connectivity state transitions from TRANSIENT_FAILURE to states +// other than READY +// - to delegate picks to child policies +func (b *rlsBalancer) handleChildPolicyStateUpdate(id string, newState balancer.State) { + b.stateMu.Lock() + defer b.stateMu.Unlock() + + cpw := b.childPolicies[id] + if cpw == nil { + // All child policies start with an entry in the map. If ID is not in + // map, it's either been removed, or never existed. + b.logger.Warningf("Received state update %+v for missing child policy %q", newState, id) + return + } + + oldState := (*balancer.State)(atomic.LoadPointer(&cpw.state)) + if oldState.ConnectivityState == connectivity.TransientFailure && newState.ConnectivityState == connectivity.Connecting { + // Ignore state transitions from TRANSIENT_FAILURE to CONNECTING, and thus + // fail pending RPCs instead of queuing them indefinitely when all + // subChannels are failing, even if the subChannels are bouncing back and + // forth between CONNECTING and TRANSIENT_FAILURE. + return + } + atomic.StorePointer(&cpw.state, unsafe.Pointer(&newState)) + b.logger.Infof("Child policy %q has new state %+v", id, newState) + b.sendNewPickerLocked() +} + +// acquireChildPolicyReferences attempts to acquire references to +// childPolicyWrappers corresponding to the passed in targets. If there is no +// childPolicyWrapper corresponding to one of the targets, a new one is created +// and added to the BalancerGroup. +func (b *rlsBalancer) acquireChildPolicyReferences(targets []string) []*childPolicyWrapper { + b.stateMu.Lock() + var newChildPolicies []*childPolicyWrapper + for _, target := range targets { + // If the target exists in the LB policy's childPolicies map. a new + // reference is taken here and added to the new list. + if cpw := b.childPolicies[target]; cpw != nil { + cpw.acquireRef() + newChildPolicies = append(newChildPolicies, cpw) + continue + } + + // If the target does not exist in the child policy map, then a new + // child policy wrapper is created and added to the new list. + cpw := newChildPolicyWrapper(target) + b.childPolicies[target] = cpw + b.bg.Add(target, b.childPolicyBuilder) + b.logger.Infof("Child policy %q added to BalancerGroup", target) + newChildPolicies = append(newChildPolicies, cpw) + if err := b.buildAndPushChildPolicyConfigs(target, b.lbCfg, &balancer.ClientConnState{ + ResolverState: b.resolverState, + }); err != nil { + cpw.lamify(err) + } + } + b.stateMu.Unlock() + return newChildPolicies +} + +// releaseChildPolicyReferences releases references to childPolicyWrappers +// corresponding to the passed in targets. If the release reference was the last +// one, the child policy is removed from the BalancerGroup. +func (b *rlsBalancer) releaseChildPolicyReferences(targets []string) { + b.stateMu.Lock() + for _, target := range targets { + if cpw := b.childPolicies[target]; cpw.releaseRef() { + delete(b.childPolicies, cpw.target) + b.bg.Remove(cpw.target) + } + } + b.stateMu.Unlock() +} diff --git a/balancer/rls/balancer_test.go b/balancer/rls/balancer_test.go new file mode 100644 index 000000000000..444b8b99d4a3 --- /dev/null +++ b/balancer/rls/balancer_test.go @@ -0,0 +1,1081 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package rls + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sync" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/rls/internal/test/e2e" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/balancer/stub" + rlspb "google.golang.org/grpc/internal/proto/grpc_lookup_v1" + internalserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/internal/testutils" + rlstest "google.golang.org/grpc/internal/testutils/rls" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/serviceconfig" + "google.golang.org/grpc/testdata" + "google.golang.org/protobuf/types/known/durationpb" +) + +// TestConfigUpdate_ControlChannel tests the scenario where a config update +// changes the RLS server name. Verifies that the new control channel is created +// and the old one is closed. +func (s) TestConfigUpdate_ControlChannel(t *testing.T) { + // Start two RLS servers. + lis1 := testutils.NewListenerWrapper(t, nil) + rlsServer1, rlsReqCh1 := rlstest.SetupFakeRLSServer(t, lis1) + lis2 := testutils.NewListenerWrapper(t, nil) + rlsServer2, rlsReqCh2 := rlstest.SetupFakeRLSServer(t, lis2) + + // Build RLS service config with the RLS server pointing to the first one. + // Set a very low value for maxAge to ensure that the entry expires soon. + rlsConfig := buildBasicRLSConfigWithChildPolicy(t, t.Name(), rlsServer1.Address) + rlsConfig.RouteLookupConfig.MaxAge = durationpb.New(defaultTestShortTimeout) + + // Start a couple of test backends, and set up the fake RLS servers to return + // these as a target in the RLS response. + backendCh1, backendAddress1 := startBackend(t) + rlsServer1.SetResponseCallback(func(_ context.Context, req *rlspb.RouteLookupRequest) *rlstest.RouteLookupResponse { + return &rlstest.RouteLookupResponse{Resp: &rlspb.RouteLookupResponse{Targets: []string{backendAddress1}}} + }) + backendCh2, backendAddress2 := startBackend(t) + rlsServer2.SetResponseCallback(func(_ context.Context, req *rlspb.RouteLookupRequest) *rlstest.RouteLookupResponse { + return &rlstest.RouteLookupResponse{Resp: &rlspb.RouteLookupResponse{Targets: []string{backendAddress2}}} + }) + + // Register a manual resolver and push the RLS service config through it. + r := startManualResolverWithConfig(t, rlsConfig) + + cc, err := grpc.Dial(r.Scheme()+":///", grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) + } + defer cc.Close() + + // Make an RPC and ensure it gets routed to the test backend. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + makeTestRPCAndExpectItToReachBackend(ctx, t, cc, backendCh1) + + // Ensure a connection is established to the first RLS server. + val, err := lis1.NewConnCh.Receive(ctx) + if err != nil { + t.Fatal("Timeout expired when waiting for LB policy to create control channel") + } + conn1 := val.(*testutils.ConnWrapper) + + // Make sure an RLS request is sent out. + verifyRLSRequest(t, rlsReqCh1, true) + + // Change lookup_service field of the RLS config to point to the second one. + rlsConfig.RouteLookupConfig.LookupService = rlsServer2.Address + + // Push the config update through the manual resolver. + scJSON, err := rlsConfig.ServiceConfigJSON() + if err != nil { + t.Fatal(err) + } + sc := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(scJSON) + r.UpdateState(resolver.State{ServiceConfig: sc}) + + // Ensure a connection is established to the second RLS server. + if _, err := lis2.NewConnCh.Receive(ctx); err != nil { + t.Fatal("Timeout expired when waiting for LB policy to create control channel") + } + + // Ensure the connection to the old one is closed. + if _, err := conn1.CloseCh.Receive(ctx); err != nil { + t.Fatal("Timeout expired when waiting for LB policy to close control channel") + } + + // Make an RPC and expect it to get routed to the second test backend through + // the second RLS server. + makeTestRPCAndExpectItToReachBackend(ctx, t, cc, backendCh2) + verifyRLSRequest(t, rlsReqCh2, true) +} + +// TestConfigUpdate_ControlChannelWithCreds tests the scenario where a config +// update specified an RLS server name, and the parent ClientConn specifies +// transport credentials. The RLS server and the test backend are configured to +// accept those transport credentials. This test verifies that the parent +// channel credentials are correctly propagated to the control channel. +func (s) TestConfigUpdate_ControlChannelWithCreds(t *testing.T) { + serverCreds, err := credentials.NewServerTLSFromFile(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem")) + if err != nil { + t.Fatalf("credentials.NewServerTLSFromFile(server1.pem, server1.key) = %v", err) + } + clientCreds, err := credentials.NewClientTLSFromFile(testdata.Path("x509/server_ca_cert.pem"), "") + if err != nil { + t.Fatalf("credentials.NewClientTLSFromFile(ca.pem) = %v", err) + } + + // Start an RLS server with the wrapped listener and credentials. + lis := testutils.NewListenerWrapper(t, nil) + rlsServer, rlsReqCh := rlstest.SetupFakeRLSServer(t, lis, grpc.Creds(serverCreds)) + overrideAdaptiveThrottler(t, neverThrottlingThrottler()) + + // Build RLS service config. + rlsConfig := buildBasicRLSConfigWithChildPolicy(t, t.Name(), rlsServer.Address) + + // Start a test backend which uses the same credentials as the RLS server, + // and set up the fake RLS server to return this as the target in the RLS + // response. + backendCh, backendAddress := startBackend(t, grpc.Creds(serverCreds)) + rlsServer.SetResponseCallback(func(_ context.Context, req *rlspb.RouteLookupRequest) *rlstest.RouteLookupResponse { + return &rlstest.RouteLookupResponse{Resp: &rlspb.RouteLookupResponse{Targets: []string{backendAddress}}} + }) + + // Register a manual resolver and push the RLS service config through it. + r := startManualResolverWithConfig(t, rlsConfig) + + // Dial with credentials and expect the RLS server to receive the same. The + // server certificate used for the RLS server and the backend specifies a + // DNS SAN of "*.test.example.com". Hence we use a dial target which is a + // subdomain of the same here. + cc, err := grpc.Dial(r.Scheme()+":///rls.test.example.com", grpc.WithResolvers(r), grpc.WithTransportCredentials(clientCreds)) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) + } + defer cc.Close() + + // Make an RPC and ensure it gets routed to the test backend. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + makeTestRPCAndExpectItToReachBackend(ctx, t, cc, backendCh) + + // Make sure an RLS request is sent out. + verifyRLSRequest(t, rlsReqCh, true) + + // Ensure a connection is established to the first RLS server. + if _, err := lis.NewConnCh.Receive(ctx); err != nil { + t.Fatal("Timeout expired when waiting for LB policy to create control channel") + } +} + +// TestConfigUpdate_ControlChannelServiceConfig tests the scenario where RLS LB +// policy's configuration specifies the service config for the control channel +// via the `routeLookupChannelServiceConfig` field. This test verifies that the +// provided service config is applied for the control channel. +func (s) TestConfigUpdate_ControlChannelServiceConfig(t *testing.T) { + // Start an RLS server and set the throttler to never throttle requests. + rlsServer, rlsReqCh := rlstest.SetupFakeRLSServer(t, nil) + overrideAdaptiveThrottler(t, neverThrottlingThrottler()) + + // Register a balancer to be used for the control channel, and set up a + // callback to get notified when the balancer receives a clientConn updates. + ccUpdateCh := testutils.NewChannel() + bf := &e2e.BalancerFuncs{ + UpdateClientConnState: func(cfg *e2e.RLSChildPolicyConfig) error { + if cfg.Backend != rlsServer.Address { + return fmt.Errorf("control channel LB policy received config with backend %q, want %q", cfg.Backend, rlsServer.Address) + } + ccUpdateCh.Replace(nil) + return nil + }, + } + controlChannelPolicyName := "test-control-channel-" + t.Name() + e2e.RegisterRLSChildPolicy(controlChannelPolicyName, bf) + t.Logf("Registered child policy with name %q", controlChannelPolicyName) + + // Build RLS service config and set the `routeLookupChannelServiceConfig` + // field to a service config which uses the above balancer. + rlsConfig := buildBasicRLSConfigWithChildPolicy(t, t.Name(), rlsServer.Address) + rlsConfig.RouteLookupChannelServiceConfig = fmt.Sprintf(`{"loadBalancingConfig" : [{%q: {"backend": %q} }]}`, controlChannelPolicyName, rlsServer.Address) + + // Start a test backend, and set up the fake RLS server to return this as a + // target in the RLS response. + backendCh, backendAddress := startBackend(t) + rlsServer.SetResponseCallback(func(_ context.Context, req *rlspb.RouteLookupRequest) *rlstest.RouteLookupResponse { + return &rlstest.RouteLookupResponse{Resp: &rlspb.RouteLookupResponse{Targets: []string{backendAddress}}} + }) + + // Register a manual resolver and push the RLS service config through it. + r := startManualResolverWithConfig(t, rlsConfig) + + cc, err := grpc.Dial(r.Scheme()+":///rls.test.example.com", grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) + } + defer cc.Close() + + // Make an RPC and ensure it gets routed to the test backend. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + makeTestRPCAndExpectItToReachBackend(ctx, t, cc, backendCh) + + // Make sure an RLS request is sent out. + verifyRLSRequest(t, rlsReqCh, true) + + // Verify that the control channel is using the LB policy we injected via the + // routeLookupChannelServiceConfig field. + if _, err := ccUpdateCh.Receive(ctx); err != nil { + t.Fatalf("timeout when waiting for control channel LB policy to receive a clientConn update") + } +} + +// TestConfigUpdate_DefaultTarget tests the scenario where a config update +// changes the default target. Verifies that RPCs get routed to the new default +// target after the config has been applied. +func (s) TestConfigUpdate_DefaultTarget(t *testing.T) { + // Start an RLS server and set the throttler to always throttle requests. + rlsServer, _ := rlstest.SetupFakeRLSServer(t, nil) + overrideAdaptiveThrottler(t, alwaysThrottlingThrottler()) + + // Build RLS service config with a default target. + rlsConfig := buildBasicRLSConfigWithChildPolicy(t, t.Name(), rlsServer.Address) + backendCh1, backendAddress1 := startBackend(t) + rlsConfig.RouteLookupConfig.DefaultTarget = backendAddress1 + + // Register a manual resolver and push the RLS service config through it. + r := startManualResolverWithConfig(t, rlsConfig) + + cc, err := grpc.Dial(r.Scheme()+":///", grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) + } + defer cc.Close() + + // Make an RPC and ensure it gets routed to the default target. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + makeTestRPCAndExpectItToReachBackend(ctx, t, cc, backendCh1) + + // Change default_target field of the RLS config. + backendCh2, backendAddress2 := startBackend(t) + rlsConfig.RouteLookupConfig.DefaultTarget = backendAddress2 + + // Push the config update through the manual resolver. + scJSON, err := rlsConfig.ServiceConfigJSON() + if err != nil { + t.Fatal(err) + } + sc := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(scJSON) + r.UpdateState(resolver.State{ServiceConfig: sc}) + makeTestRPCAndExpectItToReachBackend(ctx, t, cc, backendCh2) +} + +// TestConfigUpdate_ChildPolicyConfigs verifies that config changes which affect +// child policy configuration are propagated correctly. +func (s) TestConfigUpdate_ChildPolicyConfigs(t *testing.T) { + // Start an RLS server and set the throttler to never throttle requests. + rlsServer, rlsReqCh := rlstest.SetupFakeRLSServer(t, nil) + overrideAdaptiveThrottler(t, neverThrottlingThrottler()) + + // Start a default backend and a test backend. + _, defBackendAddress := startBackend(t) + testBackendCh, testBackendAddress := startBackend(t) + + // Set up the RLS server to respond with the test backend. + rlsServer.SetResponseCallback(func(_ context.Context, req *rlspb.RouteLookupRequest) *rlstest.RouteLookupResponse { + return &rlstest.RouteLookupResponse{Resp: &rlspb.RouteLookupResponse{Targets: []string{testBackendAddress}}} + }) + + // Set up a test balancer callback to push configs received by child policies. + defBackendConfigsCh := make(chan *e2e.RLSChildPolicyConfig, 1) + testBackendConfigsCh := make(chan *e2e.RLSChildPolicyConfig, 1) + bf := &e2e.BalancerFuncs{ + UpdateClientConnState: func(cfg *e2e.RLSChildPolicyConfig) error { + switch cfg.Backend { + case defBackendAddress: + defBackendConfigsCh <- cfg + case testBackendAddress: + testBackendConfigsCh <- cfg + default: + t.Errorf("Received child policy configs for unknown target %q", cfg.Backend) + } + return nil + }, + } + + // Register an LB policy to act as the child policy for RLS LB policy. + childPolicyName := "test-child-policy" + t.Name() + e2e.RegisterRLSChildPolicy(childPolicyName, bf) + t.Logf("Registered child policy with name %q", childPolicyName) + + // Build RLS service config with default target. + rlsConfig := buildBasicRLSConfig(childPolicyName, rlsServer.Address) + rlsConfig.RouteLookupConfig.DefaultTarget = defBackendAddress + + // Register a manual resolver and push the RLS service config through it. + r := startManualResolverWithConfig(t, rlsConfig) + + cc, err := grpc.Dial(r.Scheme()+":///", grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) + } + defer cc.Close() + + // At this point, the RLS LB policy should have received its config, and + // should have created a child policy for the default target. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + wantCfg := &e2e.RLSChildPolicyConfig{Backend: defBackendAddress} + select { + case <-ctx.Done(): + t.Fatal("Timed out when waiting for the default target child policy to receive its config") + case gotCfg := <-defBackendConfigsCh: + if !cmp.Equal(gotCfg, wantCfg) { + t.Fatalf("Default target child policy received config %+v, want %+v", gotCfg, wantCfg) + } + } + + // Make an RPC and ensure it gets routed to the test backend. + makeTestRPCAndExpectItToReachBackend(ctx, t, cc, testBackendCh) + + // Make sure an RLS request is sent out. + verifyRLSRequest(t, rlsReqCh, true) + + // As part of handling the above RPC, the RLS LB policy should have created + // a child policy for the test target. + wantCfg = &e2e.RLSChildPolicyConfig{Backend: testBackendAddress} + select { + case <-ctx.Done(): + t.Fatal("Timed out when waiting for the test target child policy to receive its config") + case gotCfg := <-testBackendConfigsCh: + if !cmp.Equal(gotCfg, wantCfg) { + t.Fatalf("Test target child policy received config %+v, want %+v", gotCfg, wantCfg) + } + } + + // Push an RLS config update with a change in the child policy config. + childPolicyBuilder := balancer.Get(childPolicyName) + childPolicyParser := childPolicyBuilder.(balancer.ConfigParser) + lbCfg, err := childPolicyParser.ParseConfig([]byte(`{"Random": "random"}`)) + if err != nil { + t.Fatal(err) + } + rlsConfig.ChildPolicy.Config = lbCfg + scJSON, err := rlsConfig.ServiceConfigJSON() + if err != nil { + t.Fatal(err) + } + sc := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(scJSON) + r.UpdateState(resolver.State{ServiceConfig: sc}) + + // Expect the child policy for the test backend to receive the update. + wantCfg = &e2e.RLSChildPolicyConfig{ + Backend: testBackendAddress, + Random: "random", + } + select { + case <-ctx.Done(): + t.Fatal("Timed out when waiting for the test target child policy to receive its config") + case gotCfg := <-testBackendConfigsCh: + if !cmp.Equal(gotCfg, wantCfg) { + t.Fatalf("Test target child policy received config %+v, want %+v", gotCfg, wantCfg) + } + } + + // Expect the child policy for the default backend to receive the update. + wantCfg = &e2e.RLSChildPolicyConfig{ + Backend: defBackendAddress, + Random: "random", + } + select { + case <-ctx.Done(): + t.Fatal("Timed out when waiting for the default target child policy to receive its config") + case gotCfg := <-defBackendConfigsCh: + if !cmp.Equal(gotCfg, wantCfg) { + t.Fatalf("Default target child policy received config %+v, want %+v", gotCfg, wantCfg) + } + } +} + +// TestConfigUpdate_ChildPolicyChange verifies that a child policy change is +// handled by closing the old balancer and creating a new one. +func (s) TestConfigUpdate_ChildPolicyChange(t *testing.T) { + // Start an RLS server and set the throttler to never throttle requests. + rlsServer, _ := rlstest.SetupFakeRLSServer(t, nil) + overrideAdaptiveThrottler(t, neverThrottlingThrottler()) + + // Set up balancer callbacks. + configsCh1 := make(chan *e2e.RLSChildPolicyConfig, 1) + closeCh1 := make(chan struct{}, 1) + bf := &e2e.BalancerFuncs{ + UpdateClientConnState: func(cfg *e2e.RLSChildPolicyConfig) error { + configsCh1 <- cfg + return nil + }, + Close: func() { + closeCh1 <- struct{}{} + }, + } + + // Register an LB policy to act as the child policy for RLS LB policy. + childPolicyName1 := "test-child-policy-1" + t.Name() + e2e.RegisterRLSChildPolicy(childPolicyName1, bf) + t.Logf("Registered child policy with name %q", childPolicyName1) + + // Build RLS service config with a dummy default target. + const defaultBackend = "default-backend" + rlsConfig := buildBasicRLSConfig(childPolicyName1, rlsServer.Address) + rlsConfig.RouteLookupConfig.DefaultTarget = defaultBackend + + // Register a manual resolver and push the RLS service config through it. + r := startManualResolverWithConfig(t, rlsConfig) + + cc, err := grpc.Dial(r.Scheme()+":///", grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) + } + defer cc.Close() + + // At this point, the RLS LB policy should have received its config, and + // should have created a child policy for the default target. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + wantCfg := &e2e.RLSChildPolicyConfig{Backend: defaultBackend} + select { + case <-ctx.Done(): + t.Fatal("Timed out when waiting for the first child policy to receive its config") + case gotCfg := <-configsCh1: + if !cmp.Equal(gotCfg, wantCfg) { + t.Fatalf("First child policy received config %+v, want %+v", gotCfg, wantCfg) + } + } + + // Set up balancer callbacks for the second policy. + configsCh2 := make(chan *e2e.RLSChildPolicyConfig, 1) + bf = &e2e.BalancerFuncs{ + UpdateClientConnState: func(cfg *e2e.RLSChildPolicyConfig) error { + configsCh2 <- cfg + return nil + }, + } + + // Register a second LB policy to act as the child policy for RLS LB policy. + childPolicyName2 := "test-child-policy-2" + t.Name() + e2e.RegisterRLSChildPolicy(childPolicyName2, bf) + t.Logf("Registered child policy with name %q", childPolicyName2) + + // Push an RLS config update with a change in the child policy name. + rlsConfig.ChildPolicy = &internalserviceconfig.BalancerConfig{Name: childPolicyName2} + scJSON, err := rlsConfig.ServiceConfigJSON() + if err != nil { + t.Fatal(err) + } + sc := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(scJSON) + r.UpdateState(resolver.State{ServiceConfig: sc}) + + // The above update should result in the first LB policy being shutdown and + // the second LB policy receiving a config update. + select { + case <-ctx.Done(): + t.Fatal("Timed out when waiting for the first child policy to be shutdown") + case <-closeCh1: + } + + select { + case <-ctx.Done(): + t.Fatal("Timed out when waiting for the second child policy to receive its config") + case gotCfg := <-configsCh2: + if !cmp.Equal(gotCfg, wantCfg) { + t.Fatalf("First child policy received config %+v, want %+v", gotCfg, wantCfg) + } + } +} + +// TestConfigUpdate_BadChildPolicyConfigs tests the scenario where a config +// update is rejected by the child policy. Verifies that the child policy +// wrapper goes "lame" and the error from the child policy is reported back to +// the caller of the RPC. +func (s) TestConfigUpdate_BadChildPolicyConfigs(t *testing.T) { + // Start an RLS server and set the throttler to never throttle requests. + rlsServer, rlsReqCh := rlstest.SetupFakeRLSServer(t, nil) + overrideAdaptiveThrottler(t, neverThrottlingThrottler()) + + // Set up the RLS server to respond with a bad target field which is expected + // to cause the child policy's ParseTarget to fail and should result in the LB + // policy creating a lame child policy wrapper. + rlsServer.SetResponseCallback(func(_ context.Context, req *rlspb.RouteLookupRequest) *rlstest.RouteLookupResponse { + return &rlstest.RouteLookupResponse{Resp: &rlspb.RouteLookupResponse{Targets: []string{e2e.RLSChildPolicyBadTarget}}} + }) + + // Build RLS service config with a default target. This default backend is + // expected to be healthy (even though we don't attempt to route RPCs to it) + // and ensures that the overall connectivity state of the RLS LB policy is not + // TRANSIENT_FAILURE. This is required to make sure that the pick for the bad + // child policy actually gets delegated to the child policy picker. + rlsConfig := buildBasicRLSConfigWithChildPolicy(t, t.Name(), rlsServer.Address) + _, addr := startBackend(t) + rlsConfig.RouteLookupConfig.DefaultTarget = addr + + // Register a manual resolver and push the RLS service config through it. + r := startManualResolverWithConfig(t, rlsConfig) + + cc, err := grpc.Dial(r.Scheme()+":///", grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) + } + defer cc.Close() + + // Make an RPC and ensure that if fails with the expected error. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + makeTestRPCAndVerifyError(ctx, t, cc, codes.Unavailable, e2e.ErrParseConfigBadTarget) + + // Make sure an RLS request is sent out. + verifyRLSRequest(t, rlsReqCh, true) +} + +// TestConfigUpdate_DataCacheSizeDecrease tests the scenario where a config +// update decreases the data cache size. Verifies that entries are evicted from +// the cache. +func (s) TestConfigUpdate_DataCacheSizeDecrease(t *testing.T) { + // Override the clientConn update hook to get notified. + clientConnUpdateDone := make(chan struct{}, 1) + origClientConnUpdateHook := clientConnUpdateHook + clientConnUpdateHook = func() { clientConnUpdateDone <- struct{}{} } + defer func() { clientConnUpdateHook = origClientConnUpdateHook }() + + // Override the cache entry size func, and always return 1. + origEntrySizeFunc := computeDataCacheEntrySize + computeDataCacheEntrySize = func(cacheKey, *cacheEntry) int64 { return 1 } + defer func() { computeDataCacheEntrySize = origEntrySizeFunc }() + + // Override the minEvictionDuration to ensure that when the config update + // reduces the cache size, the resize operation is not stopped because + // we find an entry whose minExpiryDuration has not elapsed. + origMinEvictDuration := minEvictDuration + minEvictDuration = time.Duration(0) + defer func() { minEvictDuration = origMinEvictDuration }() + + // Start an RLS server and set the throttler to never throttle requests. + rlsServer, rlsReqCh := rlstest.SetupFakeRLSServer(t, nil) + overrideAdaptiveThrottler(t, neverThrottlingThrottler()) + + // Register an LB policy to act as the child policy for RLS LB policy. + childPolicyName := "test-child-policy" + t.Name() + e2e.RegisterRLSChildPolicy(childPolicyName, nil) + t.Logf("Registered child policy with name %q", childPolicyName) + + // Build RLS service config with header matchers. + rlsConfig := buildBasicRLSConfig(childPolicyName, rlsServer.Address) + + // Start a couple of test backends, and set up the fake RLS server to return + // these as targets in the RLS response, based on request keys. + backendCh1, backendAddress1 := startBackend(t) + backendCh2, backendAddress2 := startBackend(t) + rlsServer.SetResponseCallback(func(ctx context.Context, req *rlspb.RouteLookupRequest) *rlstest.RouteLookupResponse { + if req.KeyMap["k1"] == "v1" { + return &rlstest.RouteLookupResponse{Resp: &rlspb.RouteLookupResponse{Targets: []string{backendAddress1}}} + } + if req.KeyMap["k2"] == "v2" { + return &rlstest.RouteLookupResponse{Resp: &rlspb.RouteLookupResponse{Targets: []string{backendAddress2}}} + } + return &rlstest.RouteLookupResponse{Err: errors.New("no keys in request metadata")} + }) + + // Register a manual resolver and push the RLS service config through it. + r := startManualResolverWithConfig(t, rlsConfig) + + cc, err := grpc.Dial(r.Scheme()+":///", grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) + } + defer cc.Close() + + <-clientConnUpdateDone + + // Make an RPC and ensure it gets routed to the first backend. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctxOutgoing := metadata.AppendToOutgoingContext(ctx, "n1", "v1") + makeTestRPCAndExpectItToReachBackend(ctxOutgoing, t, cc, backendCh1) + + // Make sure an RLS request is sent out. + verifyRLSRequest(t, rlsReqCh, true) + + // Make another RPC with a different set of headers. This will force the LB + // policy to send out a new RLS request, resulting in a new data cache + // entry. + ctxOutgoing = metadata.AppendToOutgoingContext(ctx, "n2", "v2") + makeTestRPCAndExpectItToReachBackend(ctxOutgoing, t, cc, backendCh2) + + // Make sure an RLS request is sent out. + verifyRLSRequest(t, rlsReqCh, true) + + // We currently have two cache entries. Setting the size to 1, will cause + // the entry corresponding to backend1 to be evicted. + rlsConfig.RouteLookupConfig.CacheSizeBytes = 1 + + // Push the config update through the manual resolver. + scJSON, err := rlsConfig.ServiceConfigJSON() + if err != nil { + t.Fatal(err) + } + sc := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(scJSON) + r.UpdateState(resolver.State{ServiceConfig: sc}) + + <-clientConnUpdateDone + + // Make an RPC to match the cache entry which got evicted above, and expect + // an RLS request to be made to fetch the targets. + ctxOutgoing = metadata.AppendToOutgoingContext(ctx, "n1", "v1") + makeTestRPCAndExpectItToReachBackend(ctxOutgoing, t, cc, backendCh1) + + // Make sure an RLS request is sent out. + verifyRLSRequest(t, rlsReqCh, true) +} + +// TestDataCachePurging verifies that the LB policy periodically evicts expired +// entries from the data cache. +func (s) TestDataCachePurging(t *testing.T) { + // Override the frequency of the data cache purger to a small one. + origDataCachePurgeTicker := dataCachePurgeTicker + ticker := time.NewTicker(defaultTestShortTimeout) + defer ticker.Stop() + dataCachePurgeTicker = func() *time.Ticker { return ticker } + defer func() { dataCachePurgeTicker = origDataCachePurgeTicker }() + + // Override the data cache purge hook to get notified. + dataCachePurgeDone := make(chan struct{}, 1) + origDataCachePurgeHook := dataCachePurgeHook + dataCachePurgeHook = func() { dataCachePurgeDone <- struct{}{} } + defer func() { dataCachePurgeHook = origDataCachePurgeHook }() + + // Start an RLS server and set the throttler to never throttle requests. + rlsServer, rlsReqCh := rlstest.SetupFakeRLSServer(t, nil) + overrideAdaptiveThrottler(t, neverThrottlingThrottler()) + + // Register an LB policy to act as the child policy for RLS LB policy. + childPolicyName := "test-child-policy" + t.Name() + e2e.RegisterRLSChildPolicy(childPolicyName, nil) + t.Logf("Registered child policy with name %q", childPolicyName) + + // Build RLS service config with header matchers and lookupService pointing to + // the fake RLS server created above. Set a very low value for maxAge to + // ensure that the entry expires soon. + rlsConfig := buildBasicRLSConfig(childPolicyName, rlsServer.Address) + rlsConfig.RouteLookupConfig.MaxAge = durationpb.New(time.Millisecond) + + // Start a test backend, and set up the fake RLS server to return this as a + // target in the RLS response. + backendCh, backendAddress := startBackend(t) + rlsServer.SetResponseCallback(func(_ context.Context, req *rlspb.RouteLookupRequest) *rlstest.RouteLookupResponse { + return &rlstest.RouteLookupResponse{Resp: &rlspb.RouteLookupResponse{Targets: []string{backendAddress}}} + }) + + // Register a manual resolver and push the RLS service config through it. + r := startManualResolverWithConfig(t, rlsConfig) + + cc, err := grpc.Dial(r.Scheme()+":///", grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) + } + defer cc.Close() + + // Make an RPC and ensure it gets routed to the test backend. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctxOutgoing := metadata.AppendToOutgoingContext(ctx, "n1", "v1") + makeTestRPCAndExpectItToReachBackend(ctxOutgoing, t, cc, backendCh) + + // Make sure an RLS request is sent out. + verifyRLSRequest(t, rlsReqCh, true) + + // Make another RPC with different headers. This will force the LB policy to + // send out a new RLS request, resulting in a new data cache entry. + ctxOutgoing = metadata.AppendToOutgoingContext(ctx, "n2", "v2") + makeTestRPCAndExpectItToReachBackend(ctxOutgoing, t, cc, backendCh) + + // Make sure an RLS request is sent out. + verifyRLSRequest(t, rlsReqCh, true) + + // Wait for the data cache purging to happen before proceeding. + <-dataCachePurgeDone + + // Perform the same RPCs again and verify that they result in RLS requests. + ctxOutgoing = metadata.AppendToOutgoingContext(ctx, "n1", "v1") + makeTestRPCAndExpectItToReachBackend(ctxOutgoing, t, cc, backendCh) + + // Make sure an RLS request is sent out. + verifyRLSRequest(t, rlsReqCh, true) + + // Make another RPC with different headers. This will force the LB policy to + // send out a new RLS request, resulting in a new data cache entry. + ctxOutgoing = metadata.AppendToOutgoingContext(ctx, "n2", "v2") + makeTestRPCAndExpectItToReachBackend(ctxOutgoing, t, cc, backendCh) + + // Make sure an RLS request is sent out. + verifyRLSRequest(t, rlsReqCh, true) +} + +// TestControlChannelConnectivityStateMonitoring tests the scenario where the +// control channel goes down and comes back up again and verifies that backoff +// state is reset for cache entries in this scenario. +func (s) TestControlChannelConnectivityStateMonitoring(t *testing.T) { + // Create a restartable listener which can close existing connections. + l, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("net.Listen() failed: %v", err) + } + lis := testutils.NewRestartableListener(l) + + // Start an RLS server with the restartable listener and set the throttler to + // never throttle requests. + rlsServer, rlsReqCh := rlstest.SetupFakeRLSServer(t, lis) + overrideAdaptiveThrottler(t, neverThrottlingThrottler()) + + // Override the reset backoff hook to get notified. + resetBackoffDone := make(chan struct{}, 1) + origResetBackoffHook := resetBackoffHook + resetBackoffHook = func() { resetBackoffDone <- struct{}{} } + defer func() { resetBackoffHook = origResetBackoffHook }() + + // Override the backoff strategy to return a large backoff which + // will make sure the date cache entry remains in backoff for the + // duration of the test. + origBackoffStrategy := defaultBackoffStrategy + defaultBackoffStrategy = &fakeBackoffStrategy{backoff: defaultTestTimeout} + defer func() { defaultBackoffStrategy = origBackoffStrategy }() + + // Register an LB policy to act as the child policy for RLS LB policy. + childPolicyName := "test-child-policy" + t.Name() + e2e.RegisterRLSChildPolicy(childPolicyName, nil) + t.Logf("Registered child policy with name %q", childPolicyName) + + // Build RLS service config with header matchers, and a very low value for + // maxAge to ensure that cache entries become invalid very soon. + rlsConfig := buildBasicRLSConfig(childPolicyName, rlsServer.Address) + rlsConfig.RouteLookupConfig.MaxAge = durationpb.New(defaultTestShortTimeout) + + // Start a test backend, and set up the fake RLS server to return this as a + // target in the RLS response. + backendCh, backendAddress := startBackend(t) + rlsServer.SetResponseCallback(func(_ context.Context, req *rlspb.RouteLookupRequest) *rlstest.RouteLookupResponse { + return &rlstest.RouteLookupResponse{Resp: &rlspb.RouteLookupResponse{Targets: []string{backendAddress}}} + }) + + // Register a manual resolver and push the RLS service config through it. + r := startManualResolverWithConfig(t, rlsConfig) + + cc, err := grpc.Dial(r.Scheme()+":///", grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) + } + defer cc.Close() + + // Make an RPC and ensure it gets routed to the test backend. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + makeTestRPCAndExpectItToReachBackend(ctx, t, cc, backendCh) + + // Make sure an RLS request is sent out. + verifyRLSRequest(t, rlsReqCh, true) + + // Stop the RLS server. + lis.Stop() + + // Make another RPC similar to the first one. Since the above cache entry + // would have expired by now, this should trigger another RLS request. And + // since the RLS server is down, RLS request will fail and the cache entry + // will enter backoff, and we have overridden the default backoff strategy to + // return a value which will keep this entry in backoff for the whole duration + // of the test. + makeTestRPCAndVerifyError(ctx, t, cc, codes.Unavailable, nil) + + // Restart the RLS server. + lis.Restart() + + // When we closed the RLS server earlier, the existing transport to the RLS + // server would have closed, and the RLS control channel would have moved to + // TRANSIENT_FAILURE with a subConn backoff before moving to IDLE. This + // backoff will last for about a second. We need to keep retrying RPCs for the + // subConn to eventually come out of backoff and attempt to reconnect. + // + // Make this RPC with a different set of headers leading to the creation of + // a new cache entry and a new RLS request. This RLS request will also fail + // till the control channel comes moves back to READY. So, override the + // backoff strategy to perform a small backoff on this entry. + defaultBackoffStrategy = &fakeBackoffStrategy{backoff: defaultTestShortTimeout} + ctxOutgoing := metadata.AppendToOutgoingContext(ctx, "n1", "v1") + makeTestRPCAndExpectItToReachBackend(ctxOutgoing, t, cc, backendCh) + + select { + case <-ctx.Done(): + t.Fatalf("Timed out waiting for resetBackoffDone") + case <-resetBackoffDone: + } + + // The fact that the above RPC succeeded indicates that the control channel + // has moved back to READY. The connectivity state monitoring code should have + // realized this and should have reset all backoff timers (which in this case + // is the cache entry corresponding to the first RPC). Retrying that RPC now + // should succeed with an RLS request being sent out. + makeTestRPCAndExpectItToReachBackend(ctx, t, cc, backendCh) + verifyRLSRequest(t, rlsReqCh, true) +} + +// testCCWrapper wraps a balancer.ClientConn and overrides UpdateState and +// stores all state updates pushed by the RLS LB policy. +type testCCWrapper struct { + balancer.ClientConn + + mu sync.Mutex + states []balancer.State +} + +func (t *testCCWrapper) UpdateState(bs balancer.State) { + t.mu.Lock() + t.states = append(t.states, bs) + t.mu.Unlock() + t.ClientConn.UpdateState(bs) +} + +func (t *testCCWrapper) getStates() []balancer.State { + t.mu.Lock() + defer t.mu.Unlock() + + states := make([]balancer.State, len(t.states)) + copy(states, t.states) + return states +} + +// TestUpdateStatePauses tests the scenario where a config update received by +// the RLS LB policy results in multiple UpdateState calls from the child +// policies. This test verifies that picker updates are paused when the config +// update is being processed by RLS LB policy and its child policies. +// +// The test uses a wrapping balancer as the top-level LB policy on the channel. +// The wrapping balancer wraps an RLS LB policy as a child policy and forwards +// all calls to it. It also records the UpdateState() calls from the RLS LB +// policy and makes it available for inspection by the test. +// +// The test uses another wrapped balancer (which wraps a pickfirst balancer) as +// the child policy of the RLS LB policy. This balancer makes multiple +// UpdateState calls when handling an update from its parent in +// UpdateClientConnState. +func (s) TestUpdateStatePauses(t *testing.T) { + // Override the hook to get notified when UpdateClientConnState is done. + clientConnUpdateDone := make(chan struct{}, 1) + origClientConnUpdateHook := clientConnUpdateHook + clientConnUpdateHook = func() { clientConnUpdateDone <- struct{}{} } + defer func() { clientConnUpdateHook = origClientConnUpdateHook }() + + // Register the top-level wrapping balancer which forwards calls to RLS. + topLevelBalancerName := t.Name() + "top-level" + var ccWrapper *testCCWrapper + stub.Register(topLevelBalancerName, stub.BalancerFuncs{ + Init: func(bd *stub.BalancerData) { + ccWrapper = &testCCWrapper{ClientConn: bd.ClientConn} + bd.Data = balancer.Get(Name).Build(ccWrapper, bd.BuildOptions) + }, + ParseConfig: func(sc json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + parser := balancer.Get(Name).(balancer.ConfigParser) + return parser.ParseConfig(sc) + }, + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + bal := bd.Data.(balancer.Balancer) + return bal.UpdateClientConnState(ccs) + }, + Close: func(bd *stub.BalancerData) { + bal := bd.Data.(balancer.Balancer) + bal.Close() + }, + }) + + // Register a child policy that wraps a pickfirst balancer and makes multiple calls + // to UpdateState when handling a config update in UpdateClientConnState. When + // this policy is used as a child policy of the RLS LB policy, it is expected + // that the latter suppress these updates and push a single picker update on the + // channel (after the config has been processed by all child policies). + childPolicyName := t.Name() + "child" + type childPolicyConfig struct { + serviceconfig.LoadBalancingConfig + Backend string // `json:"backend,omitempty"` + } + stub.Register(childPolicyName, stub.BalancerFuncs{ + Init: func(bd *stub.BalancerData) { + bd.Data = balancer.Get(grpc.PickFirstBalancerName).Build(bd.ClientConn, bd.BuildOptions) + }, + ParseConfig: func(sc json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + cfg := &childPolicyConfig{} + if err := json.Unmarshal(sc, cfg); err != nil { + return nil, err + } + return cfg, nil + }, + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + bal := bd.Data.(balancer.Balancer) + bd.ClientConn.UpdateState(balancer.State{ConnectivityState: connectivity.Idle, Picker: &testutils.TestConstPicker{Err: balancer.ErrNoSubConnAvailable}}) + bd.ClientConn.UpdateState(balancer.State{ConnectivityState: connectivity.Connecting, Picker: &testutils.TestConstPicker{Err: balancer.ErrNoSubConnAvailable}}) + + cfg := ccs.BalancerConfig.(*childPolicyConfig) + return bal.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Addresses: []resolver.Address{{Addr: cfg.Backend}}}, + }) + }, + }) + + // Start an RLS server and set the throttler to never throttle requests. + rlsServer, rlsReqCh := rlstest.SetupFakeRLSServer(t, nil) + overrideAdaptiveThrottler(t, neverThrottlingThrottler()) + + // Start a test backend and set the RLS server to respond with it. + testBackendCh, testBackendAddress := startBackend(t) + rlsServer.SetResponseCallback(func(_ context.Context, req *rlspb.RouteLookupRequest) *rlstest.RouteLookupResponse { + return &rlstest.RouteLookupResponse{Resp: &rlspb.RouteLookupResponse{Targets: []string{testBackendAddress}}} + }) + + // Register a manual resolver and push the RLS service config through it. + r := manual.NewBuilderWithScheme("rls-e2e") + scJSON := fmt.Sprintf(` +{ + "loadBalancingConfig": [ + { + "%s": { + "routeLookupConfig": { + "grpcKeybuilders": [{ + "names": [{"service": "grpc.testing.TestService"}] + }], + "lookupService": "%s", + "cacheSizeBytes": 1000 + }, + "childPolicy": [{"%s": {}}], + "childPolicyConfigTargetFieldName": "Backend" + } + } + ] +}`, topLevelBalancerName, rlsServer.Address, childPolicyName) + sc := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(scJSON) + r.InitialState(resolver.State{ServiceConfig: sc}) + + cc, err := grpc.Dial(r.Scheme()+":///", grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) + } + defer cc.Close() + + // Wait for the clientconn update to be processed by the RLS LB policy. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + select { + case <-ctx.Done(): + case <-clientConnUpdateDone: + } + + // It is important to note that at this point no child policies have been + // created because we have not attempted any RPC so far. When we attempt an + // RPC (below), child policies will be created and their configs will be + // pushed to them. But this config update will not happen in the context of + // a config update on the parent. + + // Make an RPC and ensure it gets routed to the test backend. + makeTestRPCAndExpectItToReachBackend(ctx, t, cc, testBackendCh) + + // Make sure an RLS request is sent out. + verifyRLSRequest(t, rlsReqCh, true) + + // Wait for the control channel to become READY, before reading the states + // out of the wrapping top-level balancer. + // + // makeTestRPCAndExpectItToReachBackend repeatedly sends RPCs with short + // deadlines until one succeeds. See its docstring for details. + // + // The following sequence of events is possible: + // 1. When the first RPC is attempted above, a pending cache entry is + // created, an RLS request is sent out, and the pick is queued. The + // channel is in CONNECTING state. + // 2. When the RLS response arrives, the pending cache entry is moved to the + // data cache, a child policy is created for the target specified in the + // response and a new picker is returned. The channel is still in + // CONNECTING, and retried pick is again queued. + // 3. The child policy moves through the standard set of states, IDLE --> + // CONNECTING --> READY. And for each of these state changes, a new + // picker is sent on the channel. But the overall connectivity state of + // the channel is still CONNECTING. + // 4. Right around the time when the child policy becomes READY, the + // deadline associated with the first RPC made by + // makeTestRPCAndExpectItToReachBackend() could expire, and it could send + // a new one. And because the internal state of the LB policy now + // contains a child policy which is READY, this RPC will succeed. But the + // RLS LB policy has yet to push a new picker on the channel. + // 5. If we read the states seen by the top-level wrapping LB policy without + // waiting for the channel to become READY, there is a possibility that we + // might not see the READY state in there. And if that happens, we will + // see two extra states in the last check made in the test, and thereby + // the test would fail. Waiting for the channel to become READY here + // ensures that the test does not flake because of this rare sequence of + // events. + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + + // Cache the state changes seen up to this point. + states0 := ccWrapper.getStates() + + // Push an updated service config. As mentioned earlier, the previous config + // updates on the child policies did not happen in the context of a config + // update on the parent. Hence, this update is required to force the + // scenario which we are interesting in testing here, i.e child policies get + // config updates as part of the parent policy getting its config update. + scJSON = fmt.Sprintf(` +{ + "loadBalancingConfig": [ + { + "%s": { + "routeLookupConfig": { + "grpcKeybuilders": [{ + "names": [ + {"service": "grpc.testing.TestService"}, + {"service": "grpc.health.v1.Health"} + ] + }], + "lookupService": "%s", + "cacheSizeBytes": 1000 + }, + "childPolicy": [{"%s": {}}], + "childPolicyConfigTargetFieldName": "Backend" + } + } + ] +}`, topLevelBalancerName, rlsServer.Address, childPolicyName) + sc = internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(scJSON) + r.UpdateState(resolver.State{ServiceConfig: sc}) + + // Wait for the clientconn update to be processed by the RLS LB policy. + select { + case <-ctx.Done(): + case <-clientConnUpdateDone: + } + + // Even though the child policies used in this test make multiple calls to + // UpdateState as part of handling their configs, we expect the RLS policy + // to inhibit picker updates during this time frame, and send a single + // picker once the config update is completely handled. + states1 := ccWrapper.getStates() + if len(states1) != len(states0)+1 { + t.Fatalf("more than one state update seen. before %v, after %v", states0, states1) + } +} diff --git a/balancer/rls/cache.go b/balancer/rls/cache.go new file mode 100644 index 000000000000..d7a6a1a436c6 --- /dev/null +++ b/balancer/rls/cache.go @@ -0,0 +1,361 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package rls + +import ( + "container/list" + "time" + + "google.golang.org/grpc/internal/backoff" + internalgrpclog "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/grpcsync" +) + +// cacheKey represents the key used to uniquely identify an entry in the data +// cache and in the pending requests map. +type cacheKey struct { + // path is the full path of the incoming RPC request. + path string + // keys is a stringified version of the RLS request key map built using the + // RLS keyBuilder. Since maps are not a type which is comparable in Go, it + // cannot be part of the key for another map (entries in the data cache and + // pending requests map are stored in maps). + keys string +} + +// cacheEntry wraps all the data to be stored in a data cache entry. +type cacheEntry struct { + // childPolicyWrappers contains the list of child policy wrappers + // corresponding to the targets returned by the RLS server for this entry. + childPolicyWrappers []*childPolicyWrapper + // headerData is received in the RLS response and is to be sent in the + // X-Google-RLS-Data header for matching RPCs. + headerData string + // expiryTime is the absolute time at which this cache entry entry stops + // being valid. When an RLS request succeeds, this is set to the current + // time plus the max_age field from the LB policy config. + expiryTime time.Time + // staleTime is the absolute time after which this cache entry will be + // proactively refreshed if an incoming RPC matches this entry. When an RLS + // request succeeds, this is set to the current time plus the stale_age from + // the LB policy config. + staleTime time.Time + // earliestEvictTime is the absolute time before which this entry should not + // be evicted from the cache. When a cache entry is created, this is set to + // the current time plus a default value of 5 seconds. This is required to + // make sure that a new entry added to the cache is not evicted before the + // RLS response arrives (usually when the cache is too small). + earliestEvictTime time.Time + + // status stores the RPC status of the previous RLS request for this + // entry. Picks for entries with a non-nil value for this field are failed + // with the error stored here. + status error + // backoffState contains all backoff related state. When an RLS request + // succeeds, backoffState is reset. This state moves between the data cache + // and the pending requests map. + backoffState *backoffState + // backoffTime is the absolute time at which the backoff period for this + // entry ends. When an RLS request fails, this is set to the current time + // plus the backoff value returned by the backoffState. The backoff timer is + // also setup with this value. No new RLS requests are sent out for this + // entry until the backoff period ends. + // + // Set to zero time instant upon a successful RLS response. + backoffTime time.Time + // backoffExpiryTime is the absolute time at which an entry which has gone + // through backoff stops being valid. When an RLS request fails, this is + // set to the current time plus twice the backoff time. The cache expiry + // timer will only delete entries for which both expiryTime and + // backoffExpiryTime are in the past. + // + // Set to zero time instant upon a successful RLS response. + backoffExpiryTime time.Time + + // size stores the size of this cache entry. Used to enforce the cache size + // specified in the LB policy configuration. + size int64 +} + +// backoffState wraps all backoff related state associated with a cache entry. +type backoffState struct { + // retries keeps track of the number of RLS failures, to be able to + // determine the amount of time to backoff before the next attempt. + retries int + // bs is the exponential backoff implementation which returns the amount of + // time to backoff, given the number of retries. + bs backoff.Strategy + // timer fires when the backoff period ends and incoming requests after this + // will trigger a new RLS request. + timer *time.Timer +} + +// lru is a cache implementation with a least recently used eviction policy. +// Internally it uses a doubly linked list, with the least recently used element +// at the front of the list and the most recently used element at the back of +// the list. The value stored in this cache will be of type `cacheKey`. +// +// It is not safe for concurrent access. +type lru struct { + ll *list.List + + // A map from the value stored in the lru to its underlying list element is + // maintained to have a clean API. Without this, a subset of the lru's API + // would accept/return cacheKey while another subset would accept/return + // list elements. + m map[cacheKey]*list.Element +} + +// newLRU creates a new cache with a least recently used eviction policy. +func newLRU() *lru { + return &lru{ + ll: list.New(), + m: make(map[cacheKey]*list.Element), + } +} + +func (l *lru) addEntry(key cacheKey) { + e := l.ll.PushBack(key) + l.m[key] = e +} + +func (l *lru) makeRecent(key cacheKey) { + e := l.m[key] + l.ll.MoveToBack(e) +} + +func (l *lru) removeEntry(key cacheKey) { + e := l.m[key] + l.ll.Remove(e) + delete(l.m, key) +} + +func (l *lru) getLeastRecentlyUsed() cacheKey { + e := l.ll.Front() + if e == nil { + return cacheKey{} + } + return e.Value.(cacheKey) +} + +// dataCache contains a cache of RLS data used by the LB policy to make routing +// decisions. +// +// The dataCache will be keyed by the request's path and keys, represented by +// the `cacheKey` type. It will maintain the cache keys in an `lru` and the +// cache data, represented by the `cacheEntry` type, in a native map. +// +// It is not safe for concurrent access. +type dataCache struct { + maxSize int64 // Maximum allowed size. + currentSize int64 // Current size. + keys *lru // Cache keys maintained in lru order. + entries map[cacheKey]*cacheEntry + logger *internalgrpclog.PrefixLogger + shutdown *grpcsync.Event +} + +func newDataCache(size int64, logger *internalgrpclog.PrefixLogger) *dataCache { + return &dataCache{ + maxSize: size, + keys: newLRU(), + entries: make(map[cacheKey]*cacheEntry), + logger: logger, + shutdown: grpcsync.NewEvent(), + } +} + +// resize changes the maximum allowed size of the data cache. +// +// The return value indicates if an entry with a valid backoff timer was +// evicted. This is important to the RLS LB policy which would send a new picker +// on the channel to re-process any RPCs queued as a result of this backoff +// timer. +func (dc *dataCache) resize(size int64) (backoffCancelled bool) { + if dc.shutdown.HasFired() { + return false + } + + backoffCancelled = false + for dc.currentSize > size { + key := dc.keys.getLeastRecentlyUsed() + entry, ok := dc.entries[key] + if !ok { + // This should never happen. + dc.logger.Errorf("cacheKey %+v not found in the cache while attempting to resize it", key) + break + } + + // When we encounter a cache entry whose minimum expiration time is in + // the future, we abort the LRU pass, which may temporarily leave the + // cache being too large. This is necessary to ensure that in cases + // where the cache is too small, when we receive an RLS Response, we + // keep the resulting cache entry around long enough for the pending + // incoming requests to be re-processed through the new Picker. If we + // didn't do this, then we'd risk throwing away each RLS response as we + // receive it, in which case we would fail to actually route any of our + // incoming requests. + if entry.earliestEvictTime.After(time.Now()) { + dc.logger.Warningf("cachekey %+v is too recent to be evicted. Stopping cache resizing for now", key) + break + } + + // Stop the backoff timer before evicting the entry. + if entry.backoffState != nil && entry.backoffState.timer != nil { + if entry.backoffState.timer.Stop() { + entry.backoffState.timer = nil + backoffCancelled = true + } + } + dc.deleteAndcleanup(key, entry) + } + dc.maxSize = size + return backoffCancelled +} + +// evictExpiredEntries sweeps through the cache and deletes expired entries. An +// expired entry is one for which both the `expiryTime` and `backoffExpiryTime` +// fields are in the past. +// +// The return value indicates if any expired entries were evicted. +// +// The LB policy invokes this method periodically to purge expired entries. +func (dc *dataCache) evictExpiredEntries() bool { + if dc.shutdown.HasFired() { + return false + } + + evicted := false + for key, entry := range dc.entries { + // Only evict entries for which both the data expiration time and + // backoff expiration time fields are in the past. + now := time.Now() + if entry.expiryTime.After(now) || entry.backoffExpiryTime.After(now) { + continue + } + dc.deleteAndcleanup(key, entry) + evicted = true + } + return evicted +} + +// resetBackoffState sweeps through the cache and for entries with a backoff +// state, the backoff timer is cancelled and the backoff state is reset. The +// return value indicates if any entries were mutated in this fashion. +// +// The LB policy invokes this method when the control channel moves from READY +// to TRANSIENT_FAILURE back to READY. See `monitorConnectivityState` method on +// the `controlChannel` type for more details. +func (dc *dataCache) resetBackoffState(newBackoffState *backoffState) bool { + if dc.shutdown.HasFired() { + return false + } + + backoffReset := false + for _, entry := range dc.entries { + if entry.backoffState == nil { + continue + } + if entry.backoffState.timer != nil { + entry.backoffState.timer.Stop() + entry.backoffState.timer = nil + } + entry.backoffState = &backoffState{bs: newBackoffState.bs} + entry.backoffTime = time.Time{} + entry.backoffExpiryTime = time.Time{} + backoffReset = true + } + return backoffReset +} + +// addEntry adds a cache entry for the given key. +// +// Return value backoffCancelled indicates if a cache entry with a valid backoff +// timer was evicted to make space for the current entry. This is important to +// the RLS LB policy which would send a new picker on the channel to re-process +// any RPCs queued as a result of this backoff timer. +// +// Return value ok indicates if entry was successfully added to the cache. +func (dc *dataCache) addEntry(key cacheKey, entry *cacheEntry) (backoffCancelled bool, ok bool) { + if dc.shutdown.HasFired() { + return false, false + } + + // Handle the extremely unlikely case that a single entry is bigger than the + // size of the cache. + if entry.size > dc.maxSize { + return false, false + } + dc.entries[key] = entry + dc.currentSize += entry.size + dc.keys.addEntry(key) + // If the new entry makes the cache go over its configured size, remove some + // old entries. + if dc.currentSize > dc.maxSize { + backoffCancelled = dc.resize(dc.maxSize) + } + return backoffCancelled, true +} + +// updateEntrySize updates the size of a cache entry and the current size of the +// data cache. An entry's size can change upon receipt of an RLS response. +func (dc *dataCache) updateEntrySize(entry *cacheEntry, newSize int64) { + dc.currentSize -= entry.size + entry.size = newSize + dc.currentSize += entry.size +} + +func (dc *dataCache) getEntry(key cacheKey) *cacheEntry { + if dc.shutdown.HasFired() { + return nil + } + + entry, ok := dc.entries[key] + if !ok { + return nil + } + dc.keys.makeRecent(key) + return entry +} + +func (dc *dataCache) removeEntryForTesting(key cacheKey) { + entry, ok := dc.entries[key] + if !ok { + return + } + dc.deleteAndcleanup(key, entry) +} + +// deleteAndCleanup performs actions required at the time of deleting an entry +// from the data cache. +// - the entry is removed from the map of entries +// - current size of the data cache is update +// - the key is removed from the LRU +func (dc *dataCache) deleteAndcleanup(key cacheKey, entry *cacheEntry) { + delete(dc.entries, key) + dc.currentSize -= entry.size + dc.keys.removeEntry(key) +} + +func (dc *dataCache) stop() { + for key, entry := range dc.entries { + dc.deleteAndcleanup(key, entry) + } + dc.shutdown.Fire() +} diff --git a/balancer/rls/cache_test.go b/balancer/rls/cache_test.go new file mode 100644 index 000000000000..80185f39c929 --- /dev/null +++ b/balancer/rls/cache_test.go @@ -0,0 +1,243 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package rls + +import ( + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "google.golang.org/grpc/internal/backoff" +) + +var ( + cacheKeys = []cacheKey{ + {path: "0", keys: "a"}, + {path: "1", keys: "b"}, + {path: "2", keys: "c"}, + {path: "3", keys: "d"}, + {path: "4", keys: "e"}, + } + + longDuration = 10 * time.Minute + shortDuration = 1 * time.Millisecond + cacheEntries []*cacheEntry +) + +func initCacheEntries() { + // All entries have a dummy size of 1 to simplify resize operations. + cacheEntries = []*cacheEntry{ + { + // Entry is valid and minimum expiry time has not expired. + expiryTime: time.Now().Add(longDuration), + earliestEvictTime: time.Now().Add(longDuration), + size: 1, + }, + { + // Entry is valid and is in backoff. + expiryTime: time.Now().Add(longDuration), + backoffTime: time.Now().Add(longDuration), + backoffState: &backoffState{timer: time.NewTimer(longDuration)}, + size: 1, + }, + { + // Entry is valid, and not in backoff. + expiryTime: time.Now().Add(longDuration), + size: 1, + }, + { + // Entry is invalid. + expiryTime: time.Time{}.Add(shortDuration), + size: 1, + }, + { + // Entry is invalid valid and backoff has expired. + expiryTime: time.Time{}.Add(shortDuration), + backoffExpiryTime: time.Time{}.Add(shortDuration), + size: 1, + }, + } +} + +func (s) TestLRU_BasicOperations(t *testing.T) { + initCacheEntries() + // Create an LRU and add some entries to it. + lru := newLRU() + for _, k := range cacheKeys { + lru.addEntry(k) + } + + // Get the least recent entry. This should be the first entry we added. + if got, want := lru.getLeastRecentlyUsed(), cacheKeys[0]; got != want { + t.Fatalf("lru.getLeastRecentlyUsed() = %v, want %v", got, want) + } + + // Iterate through the slice of keys we added earlier, making them the most + // recent entry, one at a time. The least recent entry at that point should + // be the next entry from our slice of keys. + for i, k := range cacheKeys { + lru.makeRecent(k) + + lruIndex := (i + 1) % len(cacheKeys) + if got, want := lru.getLeastRecentlyUsed(), cacheKeys[lruIndex]; got != want { + t.Fatalf("lru.getLeastRecentlyUsed() = %v, want %v", got, want) + } + } + + // Iterate through the slice of keys we added earlier, removing them one at + // a time The least recent entry at that point should be the next entry from + // our slice of keys, except for the last one because the lru will be empty. + for i, k := range cacheKeys { + lru.removeEntry(k) + + var want cacheKey + if i < len(cacheKeys)-1 { + want = cacheKeys[i+1] + } + if got := lru.getLeastRecentlyUsed(); got != want { + t.Fatalf("lru.getLeastRecentlyUsed() = %v, want %v", got, want) + } + } +} + +func (s) TestDataCache_BasicOperations(t *testing.T) { + initCacheEntries() + dc := newDataCache(5, nil) + for i, k := range cacheKeys { + dc.addEntry(k, cacheEntries[i]) + } + for i, k := range cacheKeys { + entry := dc.getEntry(k) + if !cmp.Equal(entry, cacheEntries[i], cmp.AllowUnexported(cacheEntry{}, backoffState{}), cmpopts.IgnoreUnexported(time.Timer{})) { + t.Fatalf("Data cache lookup for key %v returned entry %v, want %v", k, entry, cacheEntries[i]) + } + } +} + +func (s) TestDataCache_AddForcesResize(t *testing.T) { + initCacheEntries() + dc := newDataCache(1, nil) + + // The first entry in cacheEntries has a minimum expiry time in the future. + // This entry would stop the resize operation since we do not evict entries + // whose minimum expiration time is in the future. So, we do not use that + // entry in this test. The entry being added has a running backoff timer. + evicted, ok := dc.addEntry(cacheKeys[1], cacheEntries[1]) + if evicted || !ok { + t.Fatalf("dataCache.addEntry() returned (%v, %v) want (false, true)", evicted, ok) + } + + // Add another entry leading to the eviction of the above entry which has a + // running backoff timer. The first return value is expected to be true. + backoffCancelled, ok := dc.addEntry(cacheKeys[2], cacheEntries[2]) + if !backoffCancelled || !ok { + t.Fatalf("dataCache.addEntry() returned (%v, %v) want (true, true)", backoffCancelled, ok) + } + + // Add another entry leading to the eviction of the above entry which does not + // have a running backoff timer. This should evict the above entry, but the + // first return value is expected to be false. + backoffCancelled, ok = dc.addEntry(cacheKeys[3], cacheEntries[3]) + if backoffCancelled || !ok { + t.Fatalf("dataCache.addEntry() returned (%v, %v) want (false, true)", backoffCancelled, ok) + } +} + +func (s) TestDataCache_Resize(t *testing.T) { + initCacheEntries() + dc := newDataCache(5, nil) + for i, k := range cacheKeys { + dc.addEntry(k, cacheEntries[i]) + } + + // The first cache entry (with a key of cacheKeys[0]) that we added has an + // earliestEvictTime in the future. As part of the resize operation, we + // traverse the cache in least recently used order, and this will be first + // entry that we will encounter. And since the earliestEvictTime is in the + // future, the resize operation will stop, leaving the cache bigger than + // what was asked for. + if dc.resize(1) { + t.Fatalf("dataCache.resize() returned true, want false") + } + if dc.currentSize != 5 { + t.Fatalf("dataCache.size is %d, want 5", dc.currentSize) + } + + // Remove the entry with earliestEvictTime in the future and retry the + // resize operation. + dc.removeEntryForTesting(cacheKeys[0]) + if !dc.resize(1) { + t.Fatalf("dataCache.resize() returned false, want true") + } + if dc.currentSize != 1 { + t.Fatalf("dataCache.size is %d, want 1", dc.currentSize) + } +} + +func (s) TestDataCache_EvictExpiredEntries(t *testing.T) { + initCacheEntries() + dc := newDataCache(5, nil) + for i, k := range cacheKeys { + dc.addEntry(k, cacheEntries[i]) + } + + // The last two entries in the cacheEntries list have expired, and will be + // evicted. The first three should still remain in the cache. + if !dc.evictExpiredEntries() { + t.Fatal("dataCache.evictExpiredEntries() returned false, want true") + } + if dc.currentSize != 3 { + t.Fatalf("dataCache.size is %d, want 3", dc.currentSize) + } + for i := 0; i < 3; i++ { + entry := dc.getEntry(cacheKeys[i]) + if !cmp.Equal(entry, cacheEntries[i], cmp.AllowUnexported(cacheEntry{}, backoffState{}), cmpopts.IgnoreUnexported(time.Timer{})) { + t.Fatalf("Data cache lookup for key %v returned entry %v, want %v", cacheKeys[i], entry, cacheEntries[i]) + } + } +} + +func (s) TestDataCache_ResetBackoffState(t *testing.T) { + type fakeBackoff struct { + backoff.Strategy + } + + initCacheEntries() + dc := newDataCache(5, nil) + for i, k := range cacheKeys { + dc.addEntry(k, cacheEntries[i]) + } + + newBackoffState := &backoffState{bs: &fakeBackoff{}} + if updatePicker := dc.resetBackoffState(newBackoffState); !updatePicker { + t.Fatal("dataCache.resetBackoffState() returned updatePicker is false, want true") + } + + // Make sure that the entry with no backoff state was not touched. + if entry := dc.getEntry(cacheKeys[0]); cmp.Equal(entry.backoffState, newBackoffState, cmp.AllowUnexported(backoffState{})) { + t.Fatal("dataCache.resetBackoffState() touched entries without a valid backoffState") + } + + // Make sure that the entry with a valid backoff state was reset. + entry := dc.getEntry(cacheKeys[1]) + if diff := cmp.Diff(entry.backoffState, newBackoffState, cmp.AllowUnexported(backoffState{})); diff != "" { + t.Fatalf("unexpected diff in backoffState for cache entry after dataCache.resetBackoffState(): %s", diff) + } +} diff --git a/balancer/rls/child_policy.go b/balancer/rls/child_policy.go new file mode 100644 index 000000000000..c74184cac238 --- /dev/null +++ b/balancer/rls/child_policy.go @@ -0,0 +1,109 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package rls + +import ( + "fmt" + "sync/atomic" + "unsafe" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/base" + "google.golang.org/grpc/connectivity" + internalgrpclog "google.golang.org/grpc/internal/grpclog" +) + +// childPolicyWrapper is a reference counted wrapper around a child policy. +// +// The LB policy maintains a map of these wrappers keyed by the target returned +// by RLS. When a target is seen for the first time, a child policy wrapper is +// created for it and the wrapper is added to the child policy map. Each entry +// in the data cache holds references to the corresponding child policy +// wrappers. The LB policy also holds a reference to the child policy wrapper +// for the default target specified in the LB Policy Configuration +// +// When a cache entry is evicted, it releases references to the child policy +// wrappers that it contains. When all references have been released, the +// wrapper is removed from the child policy map and is destroyed. +// +// The child policy wrapper also caches the connectivity state and most recent +// picker from the child policy. Once the child policy wrapper reports +// TRANSIENT_FAILURE, it will continue reporting that state until it goes READY; +// transitions from TRANSIENT_FAILURE to CONNECTING are ignored. +// +// Whenever a child policy wrapper changes its connectivity state, the LB policy +// returns a new picker to the channel, since the channel may need to re-process +// the picks for queued RPCs. +// +// It is not safe for concurrent access. +type childPolicyWrapper struct { + logger *internalgrpclog.PrefixLogger + target string // RLS target corresponding to this child policy. + refCnt int // Reference count. + + // Balancer state reported by the child policy. The RLS LB policy maintains + // these child policies in a BalancerGroup. The state reported by the child + // policy is pushed to the state aggregator (which is also implemented by the + // RLS LB policy) and cached here. See handleChildPolicyStateUpdate() for + // details on how the state aggregation is performed. + // + // While this field is written to by the LB policy, it is read by the picker + // at Pick time. Making this an atomic to enable the picker to read this value + // without a mutex. + state unsafe.Pointer // *balancer.State +} + +// newChildPolicyWrapper creates a child policy wrapper for the given target, +// and is initialized with one reference and starts off in CONNECTING state. +func newChildPolicyWrapper(target string) *childPolicyWrapper { + c := &childPolicyWrapper{ + target: target, + refCnt: 1, + state: unsafe.Pointer(&balancer.State{ + ConnectivityState: connectivity.Connecting, + Picker: base.NewErrPicker(balancer.ErrNoSubConnAvailable), + }), + } + c.logger = internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf("[rls-child-policy-wrapper %s %p] ", c.target, c)) + c.logger.Infof("Created") + return c +} + +// acquireRef increments the reference count on the child policy wrapper. +func (c *childPolicyWrapper) acquireRef() { + c.refCnt++ +} + +// releaseRef decrements the reference count on the child policy wrapper. The +// return value indicates whether the released reference was the last one. +func (c *childPolicyWrapper) releaseRef() bool { + c.refCnt-- + return c.refCnt == 0 +} + +// lamify causes the child policy wrapper to return a picker which will always +// fail requests. This is used when the wrapper runs into errors when trying to +// build and parse the child policy configuration. +func (c *childPolicyWrapper) lamify(err error) { + c.logger.Warningf("Entering lame mode: %v", err) + atomic.StorePointer(&c.state, unsafe.Pointer(&balancer.State{ + ConnectivityState: connectivity.TransientFailure, + Picker: base.NewErrPicker(err), + })) +} diff --git a/balancer/rls/config.go b/balancer/rls/config.go new file mode 100644 index 000000000000..77b6bdcd1cca --- /dev/null +++ b/balancer/rls/config.go @@ -0,0 +1,312 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package rls + +import ( + "bytes" + "encoding/json" + "fmt" + "net/url" + "time" + + "github.com/golang/protobuf/ptypes" + durationpb "github.com/golang/protobuf/ptypes/duration" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/rls/internal/keys" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/pretty" + rlspb "google.golang.org/grpc/internal/proto/grpc_lookup_v1" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" + "google.golang.org/protobuf/encoding/protojson" +) + +const ( + // Default max_age if not specified (or greater than this value) in the + // service config. + maxMaxAge = 5 * time.Minute + // Upper limit for cache_size since we don't fully trust the service config. + maxCacheSize = 5 * 1024 * 1024 * 8 // 5MB in bytes + // Default lookup_service_timeout if not specified in the service config. + defaultLookupServiceTimeout = 10 * time.Second + // Default value for targetNameField in the child policy config during + // service config validation. + dummyChildPolicyTarget = "target_name_to_be_filled_in_later" +) + +// lbConfig is the internal representation of the RLS LB policy's config. +type lbConfig struct { + serviceconfig.LoadBalancingConfig + + cacheSizeBytes int64 // Keep this field 64-bit aligned. + kbMap keys.BuilderMap + lookupService string + lookupServiceTimeout time.Duration + maxAge time.Duration + staleAge time.Duration + defaultTarget string + + childPolicyName string + childPolicyConfig map[string]json.RawMessage + childPolicyTargetField string + controlChannelServiceConfig string +} + +func (lbCfg *lbConfig) Equal(other *lbConfig) bool { + return lbCfg.kbMap.Equal(other.kbMap) && + lbCfg.lookupService == other.lookupService && + lbCfg.lookupServiceTimeout == other.lookupServiceTimeout && + lbCfg.maxAge == other.maxAge && + lbCfg.staleAge == other.staleAge && + lbCfg.cacheSizeBytes == other.cacheSizeBytes && + lbCfg.defaultTarget == other.defaultTarget && + lbCfg.childPolicyName == other.childPolicyName && + lbCfg.childPolicyTargetField == other.childPolicyTargetField && + lbCfg.controlChannelServiceConfig == other.controlChannelServiceConfig && + childPolicyConfigEqual(lbCfg.childPolicyConfig, other.childPolicyConfig) +} + +func childPolicyConfigEqual(a, b map[string]json.RawMessage) bool { + if (b == nil) != (a == nil) { + return false + } + if len(b) != len(a) { + return false + } + for k, jsonA := range a { + jsonB, ok := b[k] + if !ok { + return false + } + if !bytes.Equal(jsonA, jsonB) { + return false + } + } + return true +} + +// This struct resembles the JSON representation of the loadBalancing config +// and makes it easier to unmarshal. +type lbConfigJSON struct { + RouteLookupConfig json.RawMessage + RouteLookupChannelServiceConfig json.RawMessage + ChildPolicy []map[string]json.RawMessage + ChildPolicyConfigTargetFieldName string +} + +// ParseConfig parses the JSON load balancer config provided into an +// internal form or returns an error if the config is invalid. +// +// When parsing a config update, the following validations are performed: +// - routeLookupConfig: +// - grpc_keybuilders field: +// - must have at least one entry +// - must not have two entries with the same `Name` +// - within each entry: +// - must have at least one `Name` +// - must not have a `Name` with the `service` field unset or empty +// - within each `headers` entry: +// - must not have `required_match` set +// - must not have `key` unset or empty +// - across all `headers`, `constant_keys` and `extra_keys` fields: +// - must not have the same `key` specified twice +// - no `key` must be the empty string +// - `lookup_service` field must be set and must parse as a target URI +// - if `max_age` > 5m, it should be set to 5 minutes +// - if `stale_age` > `max_age`, ignore it +// - if `stale_age` is set, then `max_age` must also be set +// - ignore `valid_targets` field +// - `cache_size_bytes` field must have a value greater than 0, and if its +// value is greater than 5M, we cap it at 5M +// +// - routeLookupChannelServiceConfig: +// - if specified, must parse as valid service config +// +// - childPolicy: +// - must find a valid child policy with a valid config +// +// - childPolicyConfigTargetFieldName: +// - must be set and non-empty +func (rlsBB) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + logger.Infof("Received JSON service config: %v", pretty.ToJSON(c)) + cfgJSON := &lbConfigJSON{} + if err := json.Unmarshal(c, cfgJSON); err != nil { + return nil, fmt.Errorf("rls: json unmarshal failed for service config %+v: %v", string(c), err) + } + + m := protojson.UnmarshalOptions{DiscardUnknown: true} + rlsProto := &rlspb.RouteLookupConfig{} + if err := m.Unmarshal(cfgJSON.RouteLookupConfig, rlsProto); err != nil { + return nil, fmt.Errorf("rls: bad RouteLookupConfig proto %+v: %v", string(cfgJSON.RouteLookupConfig), err) + } + lbCfg, err := parseRLSProto(rlsProto) + if err != nil { + return nil, err + } + + if sc := string(cfgJSON.RouteLookupChannelServiceConfig); sc != "" { + parsed := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(sc) + if parsed.Err != nil { + return nil, fmt.Errorf("rls: bad control channel service config %q: %v", sc, parsed.Err) + } + lbCfg.controlChannelServiceConfig = sc + } + + if cfgJSON.ChildPolicyConfigTargetFieldName == "" { + return nil, fmt.Errorf("rls: childPolicyConfigTargetFieldName field is not set in service config %+v", string(c)) + } + name, config, err := parseChildPolicyConfigs(cfgJSON.ChildPolicy, cfgJSON.ChildPolicyConfigTargetFieldName) + if err != nil { + return nil, err + } + lbCfg.childPolicyName = name + lbCfg.childPolicyConfig = config + lbCfg.childPolicyTargetField = cfgJSON.ChildPolicyConfigTargetFieldName + return lbCfg, nil +} + +func parseRLSProto(rlsProto *rlspb.RouteLookupConfig) (*lbConfig, error) { + // Validations specified on the `grpc_keybuilders` field are performed here. + kbMap, err := keys.MakeBuilderMap(rlsProto) + if err != nil { + return nil, err + } + + // `lookup_service` field must be set and must parse as a target URI. + lookupService := rlsProto.GetLookupService() + if lookupService == "" { + return nil, fmt.Errorf("rls: empty lookup_service in route lookup config %+v", rlsProto) + } + parsedTarget, err := url.Parse(lookupService) + if err != nil { + // url.Parse() fails if scheme is missing. Retry with default scheme. + parsedTarget, err = url.Parse(resolver.GetDefaultScheme() + ":///" + lookupService) + if err != nil { + return nil, fmt.Errorf("rls: invalid target URI in lookup_service %s", lookupService) + } + } + if parsedTarget.Scheme == "" { + parsedTarget.Scheme = resolver.GetDefaultScheme() + } + if resolver.Get(parsedTarget.Scheme) == nil { + return nil, fmt.Errorf("rls: unregistered scheme in lookup_service %s", lookupService) + } + + lookupServiceTimeout, err := convertDuration(rlsProto.GetLookupServiceTimeout()) + if err != nil { + return nil, fmt.Errorf("rls: failed to parse lookup_service_timeout in route lookup config %+v: %v", rlsProto, err) + } + if lookupServiceTimeout == 0 { + lookupServiceTimeout = defaultLookupServiceTimeout + } + + // Validations performed here: + // - if `max_age` > 5m, it should be set to 5 minutes + // - if `stale_age` > `max_age`, ignore it + // - if `stale_age` is set, then `max_age` must also be set + maxAge, err := convertDuration(rlsProto.GetMaxAge()) + if err != nil { + return nil, fmt.Errorf("rls: failed to parse max_age in route lookup config %+v: %v", rlsProto, err) + } + staleAge, err := convertDuration(rlsProto.GetStaleAge()) + if err != nil { + return nil, fmt.Errorf("rls: failed to parse staleAge in route lookup config %+v: %v", rlsProto, err) + } + if staleAge != 0 && maxAge == 0 { + return nil, fmt.Errorf("rls: stale_age is set, but max_age is not in route lookup config %+v", rlsProto) + } + if staleAge >= maxAge { + logger.Infof("rls: stale_age %v is not less than max_age %v, ignoring it", staleAge, maxAge) + staleAge = 0 + } + if maxAge == 0 || maxAge > maxMaxAge { + logger.Infof("rls: max_age in route lookup config is %v, using %v", maxAge, maxMaxAge) + maxAge = maxMaxAge + } + + // `cache_size_bytes` field must have a value greater than 0, and if its + // value is greater than 5M, we cap it at 5M + cacheSizeBytes := rlsProto.GetCacheSizeBytes() + if cacheSizeBytes <= 0 { + return nil, fmt.Errorf("rls: cache_size_bytes must be set to a non-zero value: %+v", rlsProto) + } + if cacheSizeBytes > maxCacheSize { + logger.Info("rls: cache_size_bytes %v is too large, setting it to: %v", cacheSizeBytes, maxCacheSize) + cacheSizeBytes = maxCacheSize + } + return &lbConfig{ + kbMap: kbMap, + lookupService: lookupService, + lookupServiceTimeout: lookupServiceTimeout, + maxAge: maxAge, + staleAge: staleAge, + cacheSizeBytes: cacheSizeBytes, + defaultTarget: rlsProto.GetDefaultTarget(), + }, nil +} + +// parseChildPolicyConfigs iterates through the list of child policies and picks +// the first registered policy and validates its config. +func parseChildPolicyConfigs(childPolicies []map[string]json.RawMessage, targetFieldName string) (string, map[string]json.RawMessage, error) { + for i, config := range childPolicies { + if len(config) != 1 { + return "", nil, fmt.Errorf("rls: invalid childPolicy: entry %v does not contain exactly 1 policy/config pair: %q", i, config) + } + + var name string + var rawCfg json.RawMessage + for name, rawCfg = range config { + } + builder := balancer.Get(name) + if builder == nil { + continue + } + parser, ok := builder.(balancer.ConfigParser) + if !ok { + return "", nil, fmt.Errorf("rls: childPolicy %q with config %q does not support config parsing", name, string(rawCfg)) + } + + // To validate child policy configs we do the following: + // - unmarshal the raw JSON bytes of the child policy config into a map + // - add an entry with key set to `target_field_name` and a dummy value + // - marshal the map back to JSON and parse the config using the parser + // retrieved previously + var childConfig map[string]json.RawMessage + if err := json.Unmarshal(rawCfg, &childConfig); err != nil { + return "", nil, fmt.Errorf("rls: json unmarshal failed for child policy config %q: %v", string(rawCfg), err) + } + childConfig[targetFieldName], _ = json.Marshal(dummyChildPolicyTarget) + jsonCfg, err := json.Marshal(childConfig) + if err != nil { + return "", nil, fmt.Errorf("rls: json marshal failed for child policy config {%+v}: %v", childConfig, err) + } + if _, err := parser.ParseConfig(jsonCfg); err != nil { + return "", nil, fmt.Errorf("rls: childPolicy config validation failed: %v", err) + } + return name, childConfig, nil + } + return "", nil, fmt.Errorf("rls: invalid childPolicy config: no supported policies found in %+v", childPolicies) +} + +func convertDuration(d *durationpb.Duration) (time.Duration, error) { + if d == nil { + return 0, nil + } + return ptypes.Duration(d) +} diff --git a/balancer/rls/internal/config_test.go b/balancer/rls/config_test.go similarity index 60% rename from balancer/rls/internal/config_test.go rename to balancer/rls/config_test.go index 1efd054512b2..86cfcad74935 100644 --- a/balancer/rls/internal/config_test.go +++ b/balancer/rls/config_test.go @@ -25,27 +25,10 @@ import ( "testing" "time" - "github.com/google/go-cmp/cmp" - - "google.golang.org/grpc/balancer" _ "google.golang.org/grpc/balancer/grpclb" // grpclb for config parsing. _ "google.golang.org/grpc/internal/resolver/passthrough" // passthrough resolver. ) -const balancerWithoutConfigParserName = "dummy_balancer" - -type dummyBB struct { - balancer.Builder -} - -func (*dummyBB) Name() string { - return balancerWithoutConfigParserName -} - -func init() { - balancer.Register(&dummyBB{}) -} - // testEqual reports whether the lbCfgs a and b are equal. This is to be used // only from tests. This ignores the keyBuilderMap field because its internals // are not exported, and hence not possible to specify in the want section of @@ -58,25 +41,29 @@ func testEqual(a, b *lbConfig) bool { a.staleAge == b.staleAge && a.cacheSizeBytes == b.cacheSizeBytes && a.defaultTarget == b.defaultTarget && - a.cpName == b.cpName && - a.cpTargetField == b.cpTargetField && - cmp.Equal(a.cpConfig, b.cpConfig) + a.controlChannelServiceConfig == b.controlChannelServiceConfig && + a.childPolicyName == b.childPolicyName && + a.childPolicyTargetField == b.childPolicyTargetField && + childPolicyConfigEqual(a.childPolicyConfig, b.childPolicyConfig) } -func TestParseConfig(t *testing.T) { +// TestParseConfig verifies successful config parsing scenarios. +func (s) TestParseConfig(t *testing.T) { + childPolicyTargetFieldVal, _ := json.Marshal(dummyChildPolicyTarget) tests := []struct { desc string input []byte wantCfg *lbConfig }{ - // This input validates a few cases: - // - A top-level unknown field should not fail. - // - An unknown field in routeLookupConfig proto should not fail. - // - lookupServiceTimeout is set to its default value, since it is not specified in the input. - // - maxAge is set to maxMaxAge since the value is too large in the input. - // - staleAge is ignore because it is higher than maxAge in the input. { - desc: "with transformations", + // This input validates a few cases: + // - A top-level unknown field should not fail. + // - An unknown field in routeLookupConfig proto should not fail. + // - lookupServiceTimeout is set to its default value, since it is not specified in the input. + // - maxAge is set to maxMaxAge since the value is too large in the input. + // - staleAge is ignore because it is higher than maxAge in the input. + // - cacheSizeBytes is greater than the hard upper limit of 5MB + desc: "with transformations 1", input: []byte(`{ "top-level-unknown-field": "unknown-value", "routeLookupConfig": { @@ -85,10 +72,10 @@ func TestParseConfig(t *testing.T) { "names": [{"service": "service", "method": "method"}], "headers": [{"key": "k1", "names": ["v1"]}] }], - "lookupService": "passthrough:///target", + "lookupService": ":///target", "maxAge" : "500s", "staleAge": "600s", - "cacheSizeBytes": 1000, + "cacheSizeBytes": 100000000, "defaultTarget": "passthrough:///default" }, "childPolicy": [ @@ -96,18 +83,21 @@ func TestParseConfig(t *testing.T) { {"unknown-policy": {"unknown-field": "unknown-value"}}, {"grpclb": {"childPolicy": [{"pickfirst": {}}]}} ], - "childPolicyConfigTargetFieldName": "service_name" + "childPolicyConfigTargetFieldName": "serviceName" }`), wantCfg: &lbConfig{ - lookupService: "passthrough:///target", - lookupServiceTimeout: 10 * time.Second, // This is the default value. - maxAge: 5 * time.Minute, // This is max maxAge. - staleAge: time.Duration(0), // StaleAge is ignore because it was higher than maxAge. - cacheSizeBytes: 1000, - defaultTarget: "passthrough:///default", - cpName: "grpclb", - cpTargetField: "service_name", - cpConfig: map[string]json.RawMessage{"childPolicy": json.RawMessage(`[{"pickfirst": {}}]`)}, + lookupService: ":///target", + lookupServiceTimeout: 10 * time.Second, // This is the default value. + maxAge: 5 * time.Minute, // This is max maxAge. + staleAge: time.Duration(0), // StaleAge is ignore because it was higher than maxAge. + cacheSizeBytes: maxCacheSize, + defaultTarget: "passthrough:///default", + childPolicyName: "grpclb", + childPolicyTargetField: "serviceName", + childPolicyConfig: map[string]json.RawMessage{ + "childPolicy": json.RawMessage(`[{"pickfirst": {}}]`), + "serviceName": json.RawMessage(childPolicyTargetFieldVal), + }, }, }, { @@ -118,31 +108,36 @@ func TestParseConfig(t *testing.T) { "names": [{"service": "service", "method": "method"}], "headers": [{"key": "k1", "names": ["v1"]}] }], - "lookupService": "passthrough:///target", + "lookupService": "target", "lookupServiceTimeout" : "100s", "maxAge": "60s", "staleAge" : "50s", "cacheSizeBytes": 1000, "defaultTarget": "passthrough:///default" }, + "routeLookupChannelServiceConfig": {"loadBalancingConfig": [{"grpclb": {"childPolicy": [{"pickfirst": {}}]}}]}, "childPolicy": [{"grpclb": {"childPolicy": [{"pickfirst": {}}]}}], - "childPolicyConfigTargetFieldName": "service_name" + "childPolicyConfigTargetFieldName": "serviceName" }`), wantCfg: &lbConfig{ - lookupService: "passthrough:///target", - lookupServiceTimeout: 100 * time.Second, - maxAge: 60 * time.Second, - staleAge: 50 * time.Second, - cacheSizeBytes: 1000, - defaultTarget: "passthrough:///default", - cpName: "grpclb", - cpTargetField: "service_name", - cpConfig: map[string]json.RawMessage{"childPolicy": json.RawMessage(`[{"pickfirst": {}}]`)}, + lookupService: "target", + lookupServiceTimeout: 100 * time.Second, + maxAge: 60 * time.Second, + staleAge: 50 * time.Second, + cacheSizeBytes: 1000, + defaultTarget: "passthrough:///default", + controlChannelServiceConfig: `{"loadBalancingConfig": [{"grpclb": {"childPolicy": [{"pickfirst": {}}]}}]}`, + childPolicyName: "grpclb", + childPolicyTargetField: "serviceName", + childPolicyConfig: map[string]json.RawMessage{ + "childPolicy": json.RawMessage(`[{"pickfirst": {}}]`), + "serviceName": json.RawMessage(childPolicyTargetFieldVal), + }, }, }, } - builder := &rlsBB{} + builder := rlsBB{} for _, test := range tests { t.Run(test.desc, func(t *testing.T) { lbCfg, err := builder.ParseConfig(test.input) @@ -153,7 +148,8 @@ func TestParseConfig(t *testing.T) { } } -func TestParseConfigErrors(t *testing.T) { +// TestParseConfigErrors verifies config parsing failure scenarios. +func (s) TestParseConfigErrors(t *testing.T) { tests := []struct { desc string input []byte @@ -191,10 +187,10 @@ func TestParseConfigErrors(t *testing.T) { }] } }`), - wantErr: "rls: empty lookup_service in service config", + wantErr: "rls: empty lookup_service in route lookup config", }, { - desc: "invalid lookup service URI", + desc: "unregistered scheme in lookup service URI", input: []byte(`{ "routeLookupConfig": { "grpcKeybuilders": [{ @@ -204,7 +200,7 @@ func TestParseConfigErrors(t *testing.T) { "lookupService": "badScheme:///target" } }`), - wantErr: "rls: invalid target URI in lookup_service", + wantErr: "rls: unregistered scheme in lookup_service", }, { desc: "invalid lookup service timeout", @@ -218,7 +214,7 @@ func TestParseConfigErrors(t *testing.T) { "lookupServiceTimeout" : "315576000001s" } }`), - wantErr: "bad Duration: time: invalid duration", + wantErr: "google.protobuf.Duration value out of range", }, { desc: "invalid max age", @@ -233,7 +229,7 @@ func TestParseConfigErrors(t *testing.T) { "maxAge" : "315576000001s" } }`), - wantErr: "bad Duration: time: invalid duration", + wantErr: "google.protobuf.Duration value out of range", }, { desc: "invalid stale age", @@ -249,7 +245,7 @@ func TestParseConfigErrors(t *testing.T) { "staleAge" : "315576000001s" } }`), - wantErr: "bad Duration: time: invalid duration", + wantErr: "google.protobuf.Duration value out of range", }, { desc: "invalid max age stale age combo", @@ -264,10 +260,10 @@ func TestParseConfigErrors(t *testing.T) { "staleAge" : "10s" } }`), - wantErr: "rls: stale_age is set, but max_age is not in service config", + wantErr: "rls: stale_age is set, but max_age is not in route lookup config", }, { - desc: "invalid cache size", + desc: "cache_size_bytes field is not set", input: []byte(`{ "routeLookupConfig": { "grpcKeybuilders": [{ @@ -277,10 +273,56 @@ func TestParseConfigErrors(t *testing.T) { "lookupService": "passthrough:///target", "lookupServiceTimeout" : "10s", "maxAge": "30s", - "staleAge" : "25s" - } + "staleAge" : "25s", + "defaultTarget": "passthrough:///default" + }, + "childPolicyConfigTargetFieldName": "serviceName" + }`), + wantErr: "rls: cache_size_bytes must be set to a non-zero value", + }, + { + desc: "routeLookupChannelServiceConfig is not in service config format", + input: []byte(`{ + "routeLookupConfig": { + "grpcKeybuilders": [{ + "names": [{"service": "service", "method": "method"}], + "headers": [{"key": "k1", "names": ["v1"]}] + }], + "lookupService": "target", + "lookupServiceTimeout" : "100s", + "maxAge": "60s", + "staleAge" : "50s", + "cacheSizeBytes": 1000, + "defaultTarget": "passthrough:///default" + }, + "routeLookupChannelServiceConfig": "unknown", + "childPolicy": [{"grpclb": {"childPolicy": [{"pickfirst": {}}]}}], + "childPolicyConfigTargetFieldName": "serviceName" + }`), + wantErr: "cannot unmarshal string into Go value of type grpc.jsonSC", + }, + { + desc: "routeLookupChannelServiceConfig contains unknown LB policy", + input: []byte(`{ + "routeLookupConfig": { + "grpcKeybuilders": [{ + "names": [{"service": "service", "method": "method"}], + "headers": [{"key": "k1", "names": ["v1"]}] + }], + "lookupService": "target", + "lookupServiceTimeout" : "100s", + "maxAge": "60s", + "staleAge" : "50s", + "cacheSizeBytes": 1000, + "defaultTarget": "passthrough:///default" + }, + "routeLookupChannelServiceConfig": { + "loadBalancingConfig": [{"not_a_balancer1": {} }, {"not_a_balancer2": {}}] + }, + "childPolicy": [{"grpclb": {"childPolicy": [{"pickfirst": {}}]}}], + "childPolicyConfigTargetFieldName": "serviceName" }`), - wantErr: "rls: cache_size_bytes must be greater than 0 in service config", + wantErr: "invalid loadBalancingConfig: no supported policies found", }, { desc: "no child policy", @@ -296,9 +338,10 @@ func TestParseConfigErrors(t *testing.T) { "staleAge" : "25s", "cacheSizeBytes": 1000, "defaultTarget": "passthrough:///default" - } + }, + "childPolicyConfigTargetFieldName": "serviceName" }`), - wantErr: "rls: childPolicy is invalid in service config", + wantErr: "rls: invalid childPolicy config: no supported policies found", }, { desc: "no known child policy", @@ -318,9 +361,35 @@ func TestParseConfigErrors(t *testing.T) { "childPolicy": [ {"cds_experimental": {"Cluster": "my-fav-cluster"}}, {"unknown-policy": {"unknown-field": "unknown-value"}} - ] + ], + "childPolicyConfigTargetFieldName": "serviceName" }`), - wantErr: "rls: childPolicy is invalid in service config", + wantErr: "rls: invalid childPolicy config: no supported policies found", + }, + { + desc: "invalid child policy config - more than one entry in map", + input: []byte(`{ + "routeLookupConfig": { + "grpcKeybuilders": [{ + "names": [{"service": "service", "method": "method"}], + "headers": [{"key": "k1", "names": ["v1"]}] + }], + "lookupService": "passthrough:///target", + "lookupServiceTimeout" : "10s", + "maxAge": "30s", + "staleAge" : "25s", + "cacheSizeBytes": 1000, + "defaultTarget": "passthrough:///default" + }, + "childPolicy": [ + { + "cds_experimental": {"Cluster": "my-fav-cluster"}, + "unknown-policy": {"unknown-field": "unknown-value"} + } + ], + "childPolicyConfigTargetFieldName": "serviceName" + }`), + wantErr: "does not contain exactly 1 policy/config pair", }, { desc: "no childPolicyConfigTargetFieldName", @@ -365,13 +434,13 @@ func TestParseConfigErrors(t *testing.T) { {"unknown-policy": {"unknown-field": "unknown-value"}}, {"grpclb": {"childPolicy": "not-an-array"}} ], - "childPolicyConfigTargetFieldName": "service_name" + "childPolicyConfigTargetFieldName": "serviceName" }`), wantErr: "rls: childPolicy config validation failed", }, } - builder := &rlsBB{} + builder := rlsBB{} for _, test := range tests { t.Run(test.desc, func(t *testing.T) { lbCfg, err := builder.ParseConfig(test.input) @@ -381,60 +450,3 @@ func TestParseConfigErrors(t *testing.T) { }) } } - -func TestValidateChildPolicyConfig(t *testing.T) { - jsonCfg := json.RawMessage(`[{"round_robin" : {}}, {"pick_first" : {}}]`) - wantChildConfig := map[string]json.RawMessage{"childPolicy": jsonCfg} - cp := &loadBalancingConfig{ - Name: "grpclb", - Config: []byte(`{"childPolicy": [{"round_robin" : {}}, {"pick_first" : {}}]}`), - } - cpTargetField := "serviceName" - - gotChildConfig, err := validateChildPolicyConfig(cp, cpTargetField) - if err != nil || !cmp.Equal(gotChildConfig, wantChildConfig) { - t.Errorf("validateChildPolicyConfig(%v, %v) = {%v, %v}, want {%v, nil}", cp, cpTargetField, gotChildConfig, err, wantChildConfig) - } -} - -func TestValidateChildPolicyConfigErrors(t *testing.T) { - tests := []struct { - desc string - cp *loadBalancingConfig - wantErrPrefix string - }{ - { - desc: "unknown child policy", - cp: &loadBalancingConfig{ - Name: "unknown", - Config: []byte(`{}`), - }, - wantErrPrefix: "rls: balancer builder not found for child_policy", - }, - { - desc: "balancer builder does not implement ConfigParser", - cp: &loadBalancingConfig{ - Name: balancerWithoutConfigParserName, - Config: []byte(`{}`), - }, - wantErrPrefix: "rls: balancer builder for child_policy does not implement balancer.ConfigParser", - }, - { - desc: "child policy config parsing failure", - cp: &loadBalancingConfig{ - Name: "grpclb", - Config: []byte(`{"childPolicy": "not-an-array"}`), - }, - wantErrPrefix: "rls: childPolicy config validation failed", - }, - } - - for _, test := range tests { - t.Run(test.desc, func(t *testing.T) { - gotChildConfig, gotErr := validateChildPolicyConfig(test.cp, "") - if gotChildConfig != nil || !strings.HasPrefix(fmt.Sprint(gotErr), test.wantErrPrefix) { - t.Errorf("validateChildPolicyConfig(%v) = {%v, %v}, want {nil, %v}", test.cp, gotChildConfig, gotErr, test.wantErrPrefix) - } - }) - } -} diff --git a/balancer/rls/control_channel.go b/balancer/rls/control_channel.go new file mode 100644 index 000000000000..4acc11d90e94 --- /dev/null +++ b/balancer/rls/control_channel.go @@ -0,0 +1,220 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package rls + +import ( + "context" + "fmt" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/rls/internal/adaptive" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + internalgrpclog "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/pretty" + rlsgrpc "google.golang.org/grpc/internal/proto/grpc_lookup_v1" + rlspb "google.golang.org/grpc/internal/proto/grpc_lookup_v1" +) + +var newAdaptiveThrottler = func() adaptiveThrottler { return adaptive.New() } + +type adaptiveThrottler interface { + ShouldThrottle() bool + RegisterBackendResponse(throttled bool) +} + +// controlChannel is a wrapper around the gRPC channel to the RLS server +// specified in the service config. +type controlChannel struct { + // rpcTimeout specifies the timeout for the RouteLookup RPC call. The LB + // policy receives this value in its service config. + rpcTimeout time.Duration + // backToReadyFunc is a callback to be invoked when the connectivity state + // changes from READY --> TRANSIENT_FAILURE --> READY. + backToReadyFunc func() + // throttler in an adaptive throttling implementation used to avoid + // hammering the RLS service while it is overloaded or down. + throttler adaptiveThrottler + + cc *grpc.ClientConn + client rlsgrpc.RouteLookupServiceClient + logger *internalgrpclog.PrefixLogger +} + +// newControlChannel creates a controlChannel to rlsServerName and uses +// serviceConfig, if non-empty, as the default service config for the underlying +// gRPC channel. +func newControlChannel(rlsServerName, serviceConfig string, rpcTimeout time.Duration, bOpts balancer.BuildOptions, backToReadyFunc func()) (*controlChannel, error) { + ctrlCh := &controlChannel{ + rpcTimeout: rpcTimeout, + backToReadyFunc: backToReadyFunc, + throttler: newAdaptiveThrottler(), + } + ctrlCh.logger = internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf("[rls-control-channel %p] ", ctrlCh)) + + dopts, err := ctrlCh.dialOpts(bOpts, serviceConfig) + if err != nil { + return nil, err + } + ctrlCh.cc, err = grpc.Dial(rlsServerName, dopts...) + if err != nil { + return nil, err + } + ctrlCh.client = rlsgrpc.NewRouteLookupServiceClient(ctrlCh.cc) + ctrlCh.logger.Infof("Control channel created to RLS server at: %v", rlsServerName) + + go ctrlCh.monitorConnectivityState() + return ctrlCh, nil +} + +// dialOpts constructs the dial options for the control plane channel. +func (cc *controlChannel) dialOpts(bOpts balancer.BuildOptions, serviceConfig string) ([]grpc.DialOption, error) { + // The control plane channel will use the same authority as the parent + // channel for server authorization. This ensures that the identity of the + // RLS server and the identity of the backends is the same, so if the RLS + // config is injected by an attacker, it cannot cause leakage of private + // information contained in headers set by the application. + dopts := []grpc.DialOption{grpc.WithAuthority(bOpts.Authority)} + if bOpts.Dialer != nil { + dopts = append(dopts, grpc.WithContextDialer(bOpts.Dialer)) + } + + // The control channel will use the channel credentials from the parent + // channel, including any call creds associated with the channel creds. + var credsOpt grpc.DialOption + switch { + case bOpts.DialCreds != nil: + credsOpt = grpc.WithTransportCredentials(bOpts.DialCreds.Clone()) + case bOpts.CredsBundle != nil: + // The "fallback" mode in google default credentials (which is the only + // type of credentials we expect to be used with RLS) uses TLS/ALTS + // creds for transport and uses the same call creds as that on the + // parent bundle. + bundle, err := bOpts.CredsBundle.NewWithMode(internal.CredsBundleModeFallback) + if err != nil { + return nil, err + } + credsOpt = grpc.WithCredentialsBundle(bundle) + default: + cc.logger.Warningf("no credentials available, using Insecure") + credsOpt = grpc.WithTransportCredentials(insecure.NewCredentials()) + } + dopts = append(dopts, credsOpt) + + // If the RLS LB policy's configuration specified a service config for the + // control channel, use that and disable service config fetching via the name + // resolver for the control channel. + if serviceConfig != "" { + cc.logger.Infof("Disabling service config from the name resolver and instead using: %s", serviceConfig) + dopts = append(dopts, grpc.WithDisableServiceConfig(), grpc.WithDefaultServiceConfig(serviceConfig)) + } + + return dopts, nil +} + +func (cc *controlChannel) monitorConnectivityState() { + cc.logger.Infof("Starting connectivity state monitoring goroutine") + // Since we use two mechanisms to deal with RLS server being down: + // - adaptive throttling for the channel as a whole + // - exponential backoff on a per-request basis + // we need a way to avoid double-penalizing requests by counting failures + // toward both mechanisms when the RLS server is unreachable. + // + // To accomplish this, we monitor the state of the control plane channel. If + // the state has been TRANSIENT_FAILURE since the last time it was in state + // READY, and it then transitions into state READY, we push on a channel + // which is being read by the LB policy. + // + // The LB the policy will iterate through the cache to reset the backoff + // timeouts in all cache entries. Specifically, this means that it will + // reset the backoff state and cancel the pending backoff timer. Note that + // when cancelling the backoff timer, just like when the backoff timer fires + // normally, a new picker is returned to the channel, to force it to + // re-process any wait-for-ready RPCs that may still be queued if we failed + // them while we were in backoff. However, we should optimize this case by + // returning only one new picker, regardless of how many backoff timers are + // cancelled. + + // Using the background context is fine here since we check for the ClientConn + // entering SHUTDOWN and return early in that case. + ctx := context.Background() + + first := true + for { + // Wait for the control channel to become READY. + for s := cc.cc.GetState(); s != connectivity.Ready; s = cc.cc.GetState() { + if s == connectivity.Shutdown { + return + } + cc.cc.WaitForStateChange(ctx, s) + } + cc.logger.Infof("Connectivity state is READY") + + if !first { + cc.logger.Infof("Control channel back to READY") + cc.backToReadyFunc() + } + first = false + + // Wait for the control channel to move out of READY. + cc.cc.WaitForStateChange(ctx, connectivity.Ready) + if cc.cc.GetState() == connectivity.Shutdown { + return + } + cc.logger.Infof("Connectivity state is %s", cc.cc.GetState()) + } +} + +func (cc *controlChannel) close() { + cc.logger.Infof("Closing control channel") + cc.cc.Close() +} + +type lookupCallback func(targets []string, headerData string, err error) + +// lookup starts a RouteLookup RPC in a separate goroutine and returns the +// results (and error, if any) in the provided callback. +// +// The returned boolean indicates whether the request was throttled by the +// client-side adaptive throttling algorithm in which case the provided callback +// will not be invoked. +func (cc *controlChannel) lookup(reqKeys map[string]string, reason rlspb.RouteLookupRequest_Reason, staleHeaders string, cb lookupCallback) (throttled bool) { + if cc.throttler.ShouldThrottle() { + cc.logger.Infof("RLS request throttled by client-side adaptive throttling") + return true + } + go func() { + req := &rlspb.RouteLookupRequest{ + TargetType: "grpc", + KeyMap: reqKeys, + Reason: reason, + StaleHeaderData: staleHeaders, + } + cc.logger.Infof("Sending RLS request %+v", pretty.ToJSON(req)) + + ctx, cancel := context.WithTimeout(context.Background(), cc.rpcTimeout) + defer cancel() + resp, err := cc.client.RouteLookup(ctx, req) + cb(resp.GetTargets(), resp.GetHeaderData(), err) + }() + return false +} diff --git a/balancer/rls/control_channel_test.go b/balancer/rls/control_channel_test.go new file mode 100644 index 000000000000..dc7acd32c863 --- /dev/null +++ b/balancer/rls/control_channel_test.go @@ -0,0 +1,465 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package rls + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "os" + "regexp" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/internal" + rlspb "google.golang.org/grpc/internal/proto/grpc_lookup_v1" + rlstest "google.golang.org/grpc/internal/testutils/rls" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/grpc/testdata" + "google.golang.org/protobuf/proto" +) + +// TestControlChannelThrottled tests the case where the adaptive throttler +// indicates that the control channel needs to be throttled. +func (s) TestControlChannelThrottled(t *testing.T) { + // Start an RLS server and set the throttler to always throttle requests. + rlsServer, rlsReqCh := rlstest.SetupFakeRLSServer(t, nil) + overrideAdaptiveThrottler(t, alwaysThrottlingThrottler()) + + // Create a control channel to the fake RLS server. + ctrlCh, err := newControlChannel(rlsServer.Address, "", defaultTestTimeout, balancer.BuildOptions{}, nil) + if err != nil { + t.Fatalf("Failed to create control channel to RLS server: %v", err) + } + defer ctrlCh.close() + + // Perform the lookup and expect the attempt to be throttled. + ctrlCh.lookup(nil, rlspb.RouteLookupRequest_REASON_MISS, staleHeaderData, nil) + + select { + case <-rlsReqCh: + t.Fatal("RouteLookup RPC invoked when control channel is throtlled") + case <-time.After(defaultTestShortTimeout): + } +} + +// TestLookupFailure tests the case where the RLS server responds with an error. +func (s) TestLookupFailure(t *testing.T) { + // Start an RLS server and set the throttler to never throttle requests. + rlsServer, _ := rlstest.SetupFakeRLSServer(t, nil) + overrideAdaptiveThrottler(t, neverThrottlingThrottler()) + + // Setup the RLS server to respond with errors. + rlsServer.SetResponseCallback(func(_ context.Context, req *rlspb.RouteLookupRequest) *rlstest.RouteLookupResponse { + return &rlstest.RouteLookupResponse{Err: errors.New("rls failure")} + }) + + // Create a control channel to the fake RLS server. + ctrlCh, err := newControlChannel(rlsServer.Address, "", defaultTestTimeout, balancer.BuildOptions{}, nil) + if err != nil { + t.Fatalf("Failed to create control channel to RLS server: %v", err) + } + defer ctrlCh.close() + + // Perform the lookup and expect the callback to be invoked with an error. + errCh := make(chan error, 1) + ctrlCh.lookup(nil, rlspb.RouteLookupRequest_REASON_MISS, staleHeaderData, func(_ []string, _ string, err error) { + if err == nil { + errCh <- errors.New("rlsClient.lookup() succeeded, should have failed") + return + } + errCh <- nil + }) + + select { + case <-time.After(defaultTestTimeout): + t.Fatal("timeout when waiting for lookup callback to be invoked") + case err := <-errCh: + if err != nil { + t.Fatal(err) + } + } +} + +// TestLookupDeadlineExceeded tests the case where the RLS server does not +// respond within the configured rpc timeout. +func (s) TestLookupDeadlineExceeded(t *testing.T) { + // A unary interceptor which returns a status error with DeadlineExceeded. + interceptor := func(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp any, err error) { + return nil, status.Error(codes.DeadlineExceeded, "deadline exceeded") + } + + // Start an RLS server and set the throttler to never throttle. + rlsServer, _ := rlstest.SetupFakeRLSServer(t, nil, grpc.UnaryInterceptor(interceptor)) + overrideAdaptiveThrottler(t, neverThrottlingThrottler()) + + // Create a control channel with a small deadline. + ctrlCh, err := newControlChannel(rlsServer.Address, "", defaultTestShortTimeout, balancer.BuildOptions{}, nil) + if err != nil { + t.Fatalf("Failed to create control channel to RLS server: %v", err) + } + defer ctrlCh.close() + + // Perform the lookup and expect the callback to be invoked with an error. + errCh := make(chan error) + ctrlCh.lookup(nil, rlspb.RouteLookupRequest_REASON_MISS, staleHeaderData, func(_ []string, _ string, err error) { + if st, ok := status.FromError(err); !ok || st.Code() != codes.DeadlineExceeded { + errCh <- fmt.Errorf("rlsClient.lookup() returned error: %v, want %v", err, codes.DeadlineExceeded) + return + } + errCh <- nil + }) + + select { + case <-time.After(defaultTestTimeout): + t.Fatal("timeout when waiting for lookup callback to be invoked") + case err := <-errCh: + if err != nil { + t.Fatal(err) + } + } +} + +// testCredsBundle wraps a test call creds and real transport creds. +type testCredsBundle struct { + transportCreds credentials.TransportCredentials + callCreds credentials.PerRPCCredentials +} + +func (f *testCredsBundle) TransportCredentials() credentials.TransportCredentials { + return f.transportCreds +} + +func (f *testCredsBundle) PerRPCCredentials() credentials.PerRPCCredentials { + return f.callCreds +} + +func (f *testCredsBundle) NewWithMode(mode string) (credentials.Bundle, error) { + if mode != internal.CredsBundleModeFallback { + return nil, fmt.Errorf("unsupported mode: %v", mode) + } + return &testCredsBundle{ + transportCreds: f.transportCreds, + callCreds: f.callCreds, + }, nil +} + +var ( + // Call creds sent by the testPerRPCCredentials on the client, and verified + // by an interceptor on the server. + perRPCCredsData = map[string]string{ + "test-key": "test-value", + "test-key-bin": string([]byte{1, 2, 3}), + } +) + +type testPerRPCCredentials struct { + callCreds map[string]string +} + +func (f *testPerRPCCredentials) GetRequestMetadata(context.Context, ...string) (map[string]string, error) { + return f.callCreds, nil +} + +func (f *testPerRPCCredentials) RequireTransportSecurity() bool { + return true +} + +// Unary server interceptor which validates if the RPC contains call credentials +// which match `perRPCCredsData +func callCredsValidatingServerInterceptor(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp any, err error) { + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return nil, status.Error(codes.PermissionDenied, "didn't find metadata in context") + } + for k, want := range perRPCCredsData { + got, ok := md[k] + if !ok { + return ctx, status.Errorf(codes.PermissionDenied, "didn't find call creds key %v in context", k) + } + if got[0] != want { + return ctx, status.Errorf(codes.PermissionDenied, "for key %v, got value %v, want %v", k, got, want) + } + } + return handler(ctx, req) +} + +// makeTLSCreds is a test helper which creates a TLS based transport credentials +// from files specified in the arguments. +func makeTLSCreds(t *testing.T, certPath, keyPath, rootsPath string) credentials.TransportCredentials { + cert, err := tls.LoadX509KeyPair(testdata.Path(certPath), testdata.Path(keyPath)) + if err != nil { + t.Fatalf("tls.LoadX509KeyPair(%q, %q) failed: %v", certPath, keyPath, err) + } + b, err := os.ReadFile(testdata.Path(rootsPath)) + if err != nil { + t.Fatalf("os.ReadFile(%q) failed: %v", rootsPath, err) + } + roots := x509.NewCertPool() + if !roots.AppendCertsFromPEM(b) { + t.Fatal("failed to append certificates") + } + return credentials.NewTLS(&tls.Config{ + Certificates: []tls.Certificate{cert}, + RootCAs: roots, + }) +} + +const ( + wantHeaderData = "headerData" + staleHeaderData = "staleHeaderData" +) + +var ( + keyMap = map[string]string{ + "k1": "v1", + "k2": "v2", + } + wantTargets = []string{"us_east_1.firestore.googleapis.com"} + lookupRequest = &rlspb.RouteLookupRequest{ + TargetType: "grpc", + KeyMap: keyMap, + Reason: rlspb.RouteLookupRequest_REASON_MISS, + StaleHeaderData: staleHeaderData, + } + lookupResponse = &rlstest.RouteLookupResponse{ + Resp: &rlspb.RouteLookupResponse{ + Targets: wantTargets, + HeaderData: wantHeaderData, + }, + } +) + +func testControlChannelCredsSuccess(t *testing.T, sopts []grpc.ServerOption, bopts balancer.BuildOptions) { + // Start an RLS server and set the throttler to never throttle requests. + rlsServer, _ := rlstest.SetupFakeRLSServer(t, nil, sopts...) + overrideAdaptiveThrottler(t, neverThrottlingThrottler()) + + // Setup the RLS server to respond with a valid response. + rlsServer.SetResponseCallback(func(_ context.Context, req *rlspb.RouteLookupRequest) *rlstest.RouteLookupResponse { + return lookupResponse + }) + + // Verify that the request received by the RLS matches the expected one. + rlsServer.SetRequestCallback(func(got *rlspb.RouteLookupRequest) { + if diff := cmp.Diff(lookupRequest, got, cmp.Comparer(proto.Equal)); diff != "" { + t.Errorf("RouteLookupRequest diff (-want, +got):\n%s", diff) + } + }) + + // Create a control channel to the fake server. + ctrlCh, err := newControlChannel(rlsServer.Address, "", defaultTestTimeout, bopts, nil) + if err != nil { + t.Fatalf("Failed to create control channel to RLS server: %v", err) + } + defer ctrlCh.close() + + // Perform the lookup and expect a successful callback invocation. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + errCh := make(chan error, 1) + ctrlCh.lookup(keyMap, rlspb.RouteLookupRequest_REASON_MISS, staleHeaderData, func(targets []string, headerData string, err error) { + if err != nil { + errCh <- fmt.Errorf("rlsClient.lookup() failed with err: %v", err) + return + } + if !cmp.Equal(targets, wantTargets) || headerData != wantHeaderData { + errCh <- fmt.Errorf("rlsClient.lookup() = (%v, %s), want (%v, %s)", targets, headerData, wantTargets, wantHeaderData) + return + } + errCh <- nil + }) + + select { + case <-ctx.Done(): + t.Fatal("timeout when waiting for lookup callback to be invoked") + case err := <-errCh: + if err != nil { + t.Fatal(err) + } + } +} + +// TestControlChannelCredsSuccess tests creation of the control channel with +// different credentials, which are expected to succeed. +func (s) TestControlChannelCredsSuccess(t *testing.T) { + serverCreds := makeTLSCreds(t, "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem") + clientCreds := makeTLSCreds(t, "x509/client1_cert.pem", "x509/client1_key.pem", "x509/server_ca_cert.pem") + + tests := []struct { + name string + sopts []grpc.ServerOption + bopts balancer.BuildOptions + }{ + { + name: "insecure", + sopts: nil, + bopts: balancer.BuildOptions{}, + }, + { + name: "transport creds only", + sopts: []grpc.ServerOption{grpc.Creds(serverCreds)}, + bopts: balancer.BuildOptions{ + DialCreds: clientCreds, + Authority: "x.test.example.com", + }, + }, + { + name: "creds bundle", + sopts: []grpc.ServerOption{ + grpc.Creds(serverCreds), + grpc.UnaryInterceptor(callCredsValidatingServerInterceptor), + }, + bopts: balancer.BuildOptions{ + CredsBundle: &testCredsBundle{ + transportCreds: clientCreds, + callCreds: &testPerRPCCredentials{callCreds: perRPCCredsData}, + }, + Authority: "x.test.example.com", + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + testControlChannelCredsSuccess(t, test.sopts, test.bopts) + }) + } +} + +func testControlChannelCredsFailure(t *testing.T, sopts []grpc.ServerOption, bopts balancer.BuildOptions, wantCode codes.Code, wantErrRegex *regexp.Regexp) { + // StartFakeRouteLookupServer a fake server. + // + // Start an RLS server and set the throttler to never throttle requests. The + // creds failures happen before the RPC handler on the server is invoked. + // So, there is need to setup the request and responses on the fake server. + rlsServer, _ := rlstest.SetupFakeRLSServer(t, nil, sopts...) + overrideAdaptiveThrottler(t, neverThrottlingThrottler()) + + // Create the control channel to the fake server. + ctrlCh, err := newControlChannel(rlsServer.Address, "", defaultTestTimeout, bopts, nil) + if err != nil { + t.Fatalf("Failed to create control channel to RLS server: %v", err) + } + defer ctrlCh.close() + + // Perform the lookup and expect the callback to be invoked with an error. + errCh := make(chan error) + ctrlCh.lookup(nil, rlspb.RouteLookupRequest_REASON_MISS, staleHeaderData, func(_ []string, _ string, err error) { + if st, ok := status.FromError(err); !ok || st.Code() != wantCode || !wantErrRegex.MatchString(st.String()) { + errCh <- fmt.Errorf("rlsClient.lookup() returned error: %v, wantCode: %v, wantErr: %s", err, wantCode, wantErrRegex.String()) + return + } + errCh <- nil + }) + + select { + case <-time.After(defaultTestTimeout): + t.Fatal("timeout when waiting for lookup callback to be invoked") + case err := <-errCh: + if err != nil { + t.Fatal(err) + } + } +} + +// TestControlChannelCredsFailure tests creation of the control channel with +// different credentials, which are expected to fail. +func (s) TestControlChannelCredsFailure(t *testing.T) { + serverCreds := makeTLSCreds(t, "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem") + clientCreds := makeTLSCreds(t, "x509/client1_cert.pem", "x509/client1_key.pem", "x509/server_ca_cert.pem") + + tests := []struct { + name string + sopts []grpc.ServerOption + bopts balancer.BuildOptions + wantCode codes.Code + wantErrRegex *regexp.Regexp + }{ + { + name: "transport creds authority mismatch", + sopts: []grpc.ServerOption{grpc.Creds(serverCreds)}, + bopts: balancer.BuildOptions{ + DialCreds: clientCreds, + Authority: "authority-mismatch", + }, + wantCode: codes.Unavailable, + wantErrRegex: regexp.MustCompile(`transport: authentication handshake failed: .* \*\.test\.example\.com.*authority-mismatch`), + }, + { + name: "transport creds handshake failure", + sopts: nil, // server expects insecure connection + bopts: balancer.BuildOptions{ + DialCreds: clientCreds, + Authority: "x.test.example.com", + }, + wantCode: codes.Unavailable, + wantErrRegex: regexp.MustCompile("transport: authentication handshake failed: .*"), + }, + { + name: "call creds mismatch", + sopts: []grpc.ServerOption{ + grpc.Creds(serverCreds), + grpc.UnaryInterceptor(callCredsValidatingServerInterceptor), // server expects call creds + }, + bopts: balancer.BuildOptions{ + CredsBundle: &testCredsBundle{ + transportCreds: clientCreds, + callCreds: &testPerRPCCredentials{}, // sends no call creds + }, + Authority: "x.test.example.com", + }, + wantCode: codes.PermissionDenied, + wantErrRegex: regexp.MustCompile("didn't find call creds"), + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + testControlChannelCredsFailure(t, test.sopts, test.bopts, test.wantCode, test.wantErrRegex) + }) + } +} + +type unsupportedCredsBundle struct { + credentials.Bundle +} + +func (*unsupportedCredsBundle) NewWithMode(mode string) (credentials.Bundle, error) { + return nil, fmt.Errorf("unsupported mode: %v", mode) +} + +// TestNewControlChannelUnsupportedCredsBundle tests the case where the control +// channel is configured with a bundle which does not support the mode we use. +func (s) TestNewControlChannelUnsupportedCredsBundle(t *testing.T) { + rlsServer, _ := rlstest.SetupFakeRLSServer(t, nil) + + // Create the control channel to the fake server. + ctrlCh, err := newControlChannel(rlsServer.Address, "", defaultTestTimeout, balancer.BuildOptions{CredsBundle: &unsupportedCredsBundle{}}, nil) + if err == nil { + ctrlCh.close() + t.Fatal("newControlChannel succeeded when expected to fail") + } +} diff --git a/balancer/rls/helpers_test.go b/balancer/rls/helpers_test.go new file mode 100644 index 000000000000..e9dee2dbe173 --- /dev/null +++ b/balancer/rls/helpers_test.go @@ -0,0 +1,299 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package rls + +import ( + "context" + "strings" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/balancer/rls/internal/test/e2e" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/grpctest" + rlspb "google.golang.org/grpc/internal/proto/grpc_lookup_v1" + internalserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/internal/stubserver" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/serviceconfig" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/durationpb" +) + +const ( + defaultTestTimeout = 5 * time.Second + defaultTestShortTimeout = 100 * time.Millisecond +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// fakeBackoffStrategy is a fake implementation of the backoff.Strategy +// interface, for tests to inject the backoff duration. +type fakeBackoffStrategy struct { + backoff time.Duration +} + +func (f *fakeBackoffStrategy) Backoff(retries int) time.Duration { + return f.backoff +} + +// fakeThrottler is a fake implementation of the adaptiveThrottler interface. +type fakeThrottler struct { + throttleFunc func() bool // Fake throttler implementation. + throttleCh chan struct{} // Invocation of ShouldThrottle signals here. +} + +func (f *fakeThrottler) ShouldThrottle() bool { + select { + case <-f.throttleCh: + default: + } + f.throttleCh <- struct{}{} + + return f.throttleFunc() +} + +func (f *fakeThrottler) RegisterBackendResponse(bool) {} + +// alwaysThrottlingThrottler returns a fake throttler which always throttles. +func alwaysThrottlingThrottler() *fakeThrottler { + return &fakeThrottler{ + throttleFunc: func() bool { return true }, + throttleCh: make(chan struct{}, 1), + } +} + +// neverThrottlingThrottler returns a fake throttler which never throttles. +func neverThrottlingThrottler() *fakeThrottler { + return &fakeThrottler{ + throttleFunc: func() bool { return false }, + throttleCh: make(chan struct{}, 1), + } +} + +// oneTimeAllowingThrottler returns a fake throttler which does not throttle +// requests until the client RPC succeeds, but throttles everything that comes +// after. This is useful for tests which need to set up a valid cache entry +// before testing other cases. +func oneTimeAllowingThrottler(firstRPCDone *grpcsync.Event) *fakeThrottler { + return &fakeThrottler{ + throttleFunc: firstRPCDone.HasFired, + throttleCh: make(chan struct{}, 1), + } +} + +func overrideAdaptiveThrottler(t *testing.T, f *fakeThrottler) { + origAdaptiveThrottler := newAdaptiveThrottler + newAdaptiveThrottler = func() adaptiveThrottler { return f } + t.Cleanup(func() { newAdaptiveThrottler = origAdaptiveThrottler }) +} + +// buildBasicRLSConfig constructs a basic service config for the RLS LB policy +// with header matching rules. This expects the passed child policy name to +// have been registered by the caller. +func buildBasicRLSConfig(childPolicyName, rlsServerAddress string) *e2e.RLSConfig { + return &e2e.RLSConfig{ + RouteLookupConfig: &rlspb.RouteLookupConfig{ + GrpcKeybuilders: []*rlspb.GrpcKeyBuilder{ + { + Names: []*rlspb.GrpcKeyBuilder_Name{{Service: "grpc.testing.TestService"}}, + Headers: []*rlspb.NameMatcher{ + {Key: "k1", Names: []string{"n1"}}, + {Key: "k2", Names: []string{"n2"}}, + }, + }, + }, + LookupService: rlsServerAddress, + LookupServiceTimeout: durationpb.New(defaultTestTimeout), + CacheSizeBytes: 1024, + }, + RouteLookupChannelServiceConfig: `{"loadBalancingConfig": [{"pick_first": {}}]}`, + ChildPolicy: &internalserviceconfig.BalancerConfig{Name: childPolicyName}, + ChildPolicyConfigTargetFieldName: e2e.RLSChildPolicyTargetNameField, + } +} + +// buildBasicRLSConfigWithChildPolicy constructs a very basic service config for +// the RLS LB policy. It also registers a test LB policy which is capable of +// being a child of the RLS LB policy. +func buildBasicRLSConfigWithChildPolicy(t *testing.T, childPolicyName, rlsServerAddress string) *e2e.RLSConfig { + childPolicyName = "test-child-policy" + childPolicyName + e2e.RegisterRLSChildPolicy(childPolicyName, nil) + t.Logf("Registered child policy with name %q", childPolicyName) + + return &e2e.RLSConfig{ + RouteLookupConfig: &rlspb.RouteLookupConfig{ + GrpcKeybuilders: []*rlspb.GrpcKeyBuilder{{Names: []*rlspb.GrpcKeyBuilder_Name{{Service: "grpc.testing.TestService"}}}}, + LookupService: rlsServerAddress, + LookupServiceTimeout: durationpb.New(defaultTestTimeout), + CacheSizeBytes: 1024, + }, + RouteLookupChannelServiceConfig: `{"loadBalancingConfig": [{"pick_first": {}}]}`, + ChildPolicy: &internalserviceconfig.BalancerConfig{Name: childPolicyName}, + ChildPolicyConfigTargetFieldName: e2e.RLSChildPolicyTargetNameField, + } +} + +// startBackend starts a backend implementing the TestService on a local port. +// It returns a channel for tests to get notified whenever an RPC is invoked on +// the backend. This allows tests to ensure that RPCs reach expected backends. +// Also returns the address of the backend. +func startBackend(t *testing.T, sopts ...grpc.ServerOption) (rpcCh chan struct{}, address string) { + t.Helper() + + rpcCh = make(chan struct{}, 1) + backend := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { + select { + case rpcCh <- struct{}{}: + default: + } + return &testpb.Empty{}, nil + }, + } + if err := backend.StartServer(sopts...); err != nil { + t.Fatalf("Failed to start backend: %v", err) + } + t.Logf("Started TestService backend at: %q", backend.Address) + t.Cleanup(func() { backend.Stop() }) + return rpcCh, backend.Address +} + +// startManualResolverWithConfig registers and returns a manual resolver which +// pushes the RLS LB policy's service config on the channel. +func startManualResolverWithConfig(t *testing.T, rlsConfig *e2e.RLSConfig) *manual.Resolver { + t.Helper() + + scJSON, err := rlsConfig.ServiceConfigJSON() + if err != nil { + t.Fatal(err) + } + + sc := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(scJSON) + r := manual.NewBuilderWithScheme("rls-e2e") + r.InitialState(resolver.State{ServiceConfig: sc}) + t.Cleanup(r.Close) + return r +} + +// makeTestRPCAndExpectItToReachBackend is a test helper function which makes +// the EmptyCall RPC on the given ClientConn and verifies that it reaches a +// backend. The latter is accomplished by listening on the provided channel +// which gets pushed to whenever the backend in question gets an RPC. +// +// There are many instances where it can take a while before the attempted RPC +// reaches the expected backend. Examples include, but are not limited to: +// - control channel is changed in a config update. The RLS LB policy creates a +// new control channel, and sends a new picker to gRPC. But it takes a while +// before gRPC actually starts using the new picker. +// - test is waiting for a cache entry to expire after which we expect a +// different behavior because we have configured the fake RLS server to return +// different backends. +// +// Therefore, we do not return an error when the RPC fails. Instead, we wait for +// the context to expire before failing. +func makeTestRPCAndExpectItToReachBackend(ctx context.Context, t *testing.T, cc *grpc.ClientConn, ch chan struct{}) { + t.Helper() + + // Drain the backend channel before performing the RPC to remove any + // notifications from previous RPCs. + select { + case <-ch: + default: + } + + for { + if err := ctx.Err(); err != nil { + t.Fatalf("Timeout when waiting for RPCs to be routed to the given target: %v", err) + } + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + client := testgrpc.NewTestServiceClient(cc) + client.EmptyCall(sCtx, &testpb.Empty{}) + + select { + case <-sCtx.Done(): + case <-ch: + sCancel() + return + } + } +} + +// makeTestRPCAndVerifyError is a test helper function which makes the EmptyCall +// RPC on the given ClientConn and verifies that the RPC fails with the given +// status code and error. +// +// Similar to makeTestRPCAndExpectItToReachBackend, retries until expected +// outcome is reached or the provided context has expired. +func makeTestRPCAndVerifyError(ctx context.Context, t *testing.T, cc *grpc.ClientConn, wantCode codes.Code, wantErr error) { + t.Helper() + + for { + if err := ctx.Err(); err != nil { + t.Fatalf("Timeout when waiting for RPCs to fail with given error: %v", err) + } + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + client := testgrpc.NewTestServiceClient(cc) + _, err := client.EmptyCall(sCtx, &testpb.Empty{}) + + // If the RPC fails with the expected code and expected error message (if + // one was provided), we return. Else we retry after blocking for a little + // while to ensure that we don't keep blasting away with RPCs. + if code := status.Code(err); code == wantCode { + if wantErr == nil || strings.Contains(err.Error(), wantErr.Error()) { + sCancel() + return + } + } + <-sCtx.Done() + } +} + +// verifyRLSRequest is a test helper which listens on a channel to see if an RLS +// request was received by the fake RLS server. Based on whether the test +// expects a request to be sent out or not, it uses a different timeout. +func verifyRLSRequest(t *testing.T, ch chan struct{}, wantRequest bool) { + t.Helper() + + if wantRequest { + select { + case <-time.After(defaultTestTimeout): + t.Fatalf("Timeout when waiting for an RLS request to be sent out") + case <-ch: + } + } else { + select { + case <-time.After(defaultTestShortTimeout): + case <-ch: + t.Fatalf("RLS request sent out when not expecting one") + } + } +} diff --git a/balancer/rls/internal/adaptive/adaptive.go b/balancer/rls/internal/adaptive/adaptive.go index 4adae1bde6b4..a3b0931b2955 100644 --- a/balancer/rls/internal/adaptive/adaptive.go +++ b/balancer/rls/internal/adaptive/adaptive.go @@ -45,21 +45,21 @@ const ( // The throttler has the following knobs for which we will use defaults for // now. If there is a need to make them configurable at a later point in time, // support for the same will be added. -// * Duration: amount of recent history that will be taken into account for -// making client-side throttling decisions. A default of 30 seconds is used. -// * Bins: number of bins to be used for bucketing historical data. A default -// of 100 is used. -// * RatioForAccepts: ratio by which accepts are multiplied, typically a value -// slightly larger than 1.0. This is used to make the throttler behave as if -// the backend had accepted more requests than it actually has, which lets us -// err on the side of sending to the backend more requests than we think it -// will accept for the sake of speeding up the propagation of state. A -// default of 2.0 is used. -// * RequestsPadding: is used to decrease the (client-side) throttling -// probability in the low QPS regime (to speed up propagation of state), as -// well as to safeguard against hitting a client-side throttling probability -// of 100%. The weight of this value decreases as the number of requests in -// recent history grows. A default of 8 is used. +// - Duration: amount of recent history that will be taken into account for +// making client-side throttling decisions. A default of 30 seconds is used. +// - Bins: number of bins to be used for bucketing historical data. A default +// of 100 is used. +// - RatioForAccepts: ratio by which accepts are multiplied, typically a value +// slightly larger than 1.0. This is used to make the throttler behave as if +// the backend had accepted more requests than it actually has, which lets us +// err on the side of sending to the backend more requests than we think it +// will accept for the sake of speeding up the propagation of state. A +// default of 2.0 is used. +// - RequestsPadding: is used to decrease the (client-side) throttling +// probability in the low QPS regime (to speed up propagation of state), as +// well as to safeguard against hitting a client-side throttling probability +// of 100%. The weight of this value decreases as the number of requests in +// recent history grows. A default of 8 is used. // // The adaptive throttler attempts to estimate the probability that a request // will be throttled using recent history. Server requests (both throttled and diff --git a/balancer/rls/internal/adaptive/adaptive_test.go b/balancer/rls/internal/adaptive/adaptive_test.go index 40a846a388a4..2205b533eec7 100644 --- a/balancer/rls/internal/adaptive/adaptive_test.go +++ b/balancer/rls/internal/adaptive/adaptive_test.go @@ -156,7 +156,7 @@ func TestShouldThrottleOptions(t *testing.T) { for _, test := range testcases { t.Run(test.desc, func(t *testing.T) { m.SetNanos(0) - th := newWithArgs(time.Duration(time.Nanosecond), 1, test.ratioForAccepts, test.requestsPadding) + th := newWithArgs(time.Nanosecond, 1, test.ratioForAccepts, test.requestsPadding) for i, response := range responses { if response != E { th.RegisterBackendResponse(response == T) diff --git a/balancer/rls/internal/balancer.go b/balancer/rls/internal/balancer.go deleted file mode 100644 index 7af97b76faf1..000000000000 --- a/balancer/rls/internal/balancer.go +++ /dev/null @@ -1,197 +0,0 @@ -/* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package rls - -import ( - "sync" - - "google.golang.org/grpc" - "google.golang.org/grpc/balancer" - "google.golang.org/grpc/grpclog" - "google.golang.org/grpc/internal/grpcsync" -) - -var ( - _ balancer.Balancer = (*rlsBalancer)(nil) - - // For overriding in tests. - newRLSClientFunc = newRLSClient - logger = grpclog.Component("rls") -) - -// rlsBalancer implements the RLS LB policy. -type rlsBalancer struct { - done *grpcsync.Event - cc balancer.ClientConn - opts balancer.BuildOptions - - // Mutex protects all the state maintained by the LB policy. - // TODO(easwars): Once we add the cache, we will also have another lock for - // the cache alone. - mu sync.Mutex - lbCfg *lbConfig // Most recently received service config. - rlsCC *grpc.ClientConn // ClientConn to the RLS server. - rlsC *rlsClient // RLS client wrapper. - - ccUpdateCh chan *balancer.ClientConnState -} - -// run is a long running goroutine which handles all the updates that the -// balancer wishes to handle. The appropriate updateHandler will push the update -// on to a channel that this goroutine will select on, thereby the handling of -// the update will happen asynchronously. -func (lb *rlsBalancer) run() { - for { - // TODO(easwars): Handle other updates like subConn state changes, RLS - // responses from the server etc. - select { - case u := <-lb.ccUpdateCh: - lb.handleClientConnUpdate(u) - case <-lb.done.Done(): - return - } - } -} - -// handleClientConnUpdate handles updates to the service config. -// If the RLS server name or the RLS RPC timeout changes, it updates the control -// channel accordingly. -// TODO(easwars): Handle updates to other fields in the service config. -func (lb *rlsBalancer) handleClientConnUpdate(ccs *balancer.ClientConnState) { - logger.Infof("rls: service config: %+v", ccs.BalancerConfig) - lb.mu.Lock() - defer lb.mu.Unlock() - - if lb.done.HasFired() { - logger.Warning("rls: received service config after balancer close") - return - } - - newCfg := ccs.BalancerConfig.(*lbConfig) - if lb.lbCfg.Equal(newCfg) { - logger.Info("rls: new service config matches existing config") - return - } - - lb.updateControlChannel(newCfg) - lb.lbCfg = newCfg -} - -// UpdateClientConnState pushes the received ClientConnState update on the -// update channel which will be processed asynchronously by the run goroutine. -// Implements balancer.Balancer interface. -func (lb *rlsBalancer) UpdateClientConnState(ccs balancer.ClientConnState) error { - select { - case lb.ccUpdateCh <- &ccs: - case <-lb.done.Done(): - } - return nil -} - -// ResolverErr implements balancer.Balancer interface. -func (lb *rlsBalancer) ResolverError(error) { - // ResolverError is called by gRPC when the name resolver reports an error. - // TODO(easwars): How do we handle this? - logger.Fatal("rls: ResolverError is not yet unimplemented") -} - -// UpdateSubConnState implements balancer.Balancer interface. -func (lb *rlsBalancer) UpdateSubConnState(_ balancer.SubConn, _ balancer.SubConnState) { - logger.Fatal("rls: UpdateSubConnState is not yet implemented") -} - -// Cleans up the resources allocated by the LB policy including the clientConn -// to the RLS server. -// Implements balancer.Balancer. -func (lb *rlsBalancer) Close() { - lb.mu.Lock() - defer lb.mu.Unlock() - - lb.done.Fire() - if lb.rlsCC != nil { - lb.rlsCC.Close() - } -} - -// updateControlChannel updates the RLS client if required. -// Caller must hold lb.mu. -func (lb *rlsBalancer) updateControlChannel(newCfg *lbConfig) { - oldCfg := lb.lbCfg - if newCfg.lookupService == oldCfg.lookupService && newCfg.lookupServiceTimeout == oldCfg.lookupServiceTimeout { - return - } - - // Use RPC timeout from new config, if different from existing one. - timeout := oldCfg.lookupServiceTimeout - if timeout != newCfg.lookupServiceTimeout { - timeout = newCfg.lookupServiceTimeout - } - - if newCfg.lookupService == oldCfg.lookupService { - // This is the case where only the timeout has changed. We will continue - // to use the existing clientConn. but will create a new rlsClient with - // the new timeout. - lb.rlsC = newRLSClientFunc(lb.rlsCC, lb.opts.Target.Endpoint, timeout) - return - } - - // This is the case where the RLS server name has changed. We need to create - // a new clientConn and close the old one. - var dopts []grpc.DialOption - if dialer := lb.opts.Dialer; dialer != nil { - dopts = append(dopts, grpc.WithContextDialer(dialer)) - } - dopts = append(dopts, dialCreds(lb.opts)) - - cc, err := grpc.Dial(newCfg.lookupService, dopts...) - if err != nil { - logger.Errorf("rls: dialRLS(%s, %v): %v", newCfg.lookupService, lb.opts, err) - // An error from a non-blocking dial indicates something serious. We - // should continue to use the old control channel if one exists, and - // return so that the rest of the config updates can be processes. - return - } - if lb.rlsCC != nil { - lb.rlsCC.Close() - } - lb.rlsCC = cc - lb.rlsC = newRLSClientFunc(cc, lb.opts.Target.Endpoint, timeout) -} - -func dialCreds(opts balancer.BuildOptions) grpc.DialOption { - // The control channel should use the same authority as that of the parent - // channel. This ensures that the identify of the RLS server and that of the - // backend is the same, so if the RLS config is injected by an attacker, it - // cannot cause leakage of private information contained in headers set by - // the application. - server := opts.Target.Authority - switch { - case opts.DialCreds != nil: - if err := opts.DialCreds.OverrideServerName(server); err != nil { - logger.Warningf("rls: OverrideServerName(%s) = (%v), using Insecure", server, err) - return grpc.WithInsecure() - } - return grpc.WithTransportCredentials(opts.DialCreds) - case opts.CredsBundle != nil: - return grpc.WithTransportCredentials(opts.CredsBundle.TransportCredentials()) - default: - logger.Warning("rls: no credentials available, using Insecure") - return grpc.WithInsecure() - } -} diff --git a/balancer/rls/internal/balancer_test.go b/balancer/rls/internal/balancer_test.go deleted file mode 100644 index d6a98aa9ab00..000000000000 --- a/balancer/rls/internal/balancer_test.go +++ /dev/null @@ -1,226 +0,0 @@ -/* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package rls - -import ( - "net" - "testing" - "time" - - "google.golang.org/grpc" - "google.golang.org/grpc/balancer" - "google.golang.org/grpc/balancer/rls/internal/testutils/fakeserver" - "google.golang.org/grpc/credentials" - "google.golang.org/grpc/internal/grpctest" - "google.golang.org/grpc/internal/testutils" - "google.golang.org/grpc/testdata" -) - -type s struct { - grpctest.Tester -} - -func Test(t *testing.T) { - grpctest.RunSubTests(t, s{}) -} - -type listenerWrapper struct { - net.Listener - connCh *testutils.Channel -} - -// Accept waits for and returns the next connection to the listener. -func (l *listenerWrapper) Accept() (net.Conn, error) { - c, err := l.Listener.Accept() - if err != nil { - return nil, err - } - l.connCh.Send(c) - return c, nil -} - -func setupwithListener(t *testing.T, opts ...grpc.ServerOption) (*fakeserver.Server, *listenerWrapper, func()) { - t.Helper() - - l, err := net.Listen("tcp", "localhost:0") - if err != nil { - t.Fatalf("net.Listen(tcp, localhost:0): %v", err) - } - lw := &listenerWrapper{ - Listener: l, - connCh: testutils.NewChannel(), - } - - server, cleanup, err := fakeserver.Start(lw, opts...) - if err != nil { - t.Fatalf("fakeserver.Start(): %v", err) - } - t.Logf("Fake RLS server started at %s ...", server.Address) - - return server, lw, cleanup -} - -type testBalancerCC struct { - balancer.ClientConn -} - -// TestUpdateControlChannelFirstConfig tests the scenario where the LB policy -// receives its first service config and verifies that a control channel to the -// RLS server specified in the serviceConfig is established. -func (s) TestUpdateControlChannelFirstConfig(t *testing.T) { - server, lis, cleanup := setupwithListener(t) - defer cleanup() - - bb := balancer.Get(rlsBalancerName) - if bb == nil { - t.Fatalf("balancer.Get(%s) = nil", rlsBalancerName) - } - rlsB := bb.Build(&testBalancerCC{}, balancer.BuildOptions{}) - defer rlsB.Close() - t.Log("Built RLS LB policy ...") - - lbCfg := &lbConfig{lookupService: server.Address} - t.Logf("Sending service config %+v to RLS LB policy ...", lbCfg) - rlsB.UpdateClientConnState(balancer.ClientConnState{BalancerConfig: lbCfg}) - - if _, err := lis.connCh.Receive(); err != nil { - t.Fatal("Timeout expired when waiting for LB policy to create control channel") - } - - // TODO: Verify channel connectivity state once control channel connectivity - // state monitoring is in place. - - // TODO: Verify RLS RPC can be made once we integrate with the picker. -} - -// TestUpdateControlChannelSwitch tests the scenario where a control channel -// exists and the LB policy receives a new serviceConfig with a different RLS -// server name. Verifies that the new control channel is created and the old one -// is closed (the leakchecker takes care of this). -func (s) TestUpdateControlChannelSwitch(t *testing.T) { - server1, lis1, cleanup1 := setupwithListener(t) - defer cleanup1() - - server2, lis2, cleanup2 := setupwithListener(t) - defer cleanup2() - - bb := balancer.Get(rlsBalancerName) - if bb == nil { - t.Fatalf("balancer.Get(%s) = nil", rlsBalancerName) - } - rlsB := bb.Build(&testBalancerCC{}, balancer.BuildOptions{}) - defer rlsB.Close() - t.Log("Built RLS LB policy ...") - - lbCfg := &lbConfig{lookupService: server1.Address} - t.Logf("Sending service config %+v to RLS LB policy ...", lbCfg) - rlsB.UpdateClientConnState(balancer.ClientConnState{BalancerConfig: lbCfg}) - - if _, err := lis1.connCh.Receive(); err != nil { - t.Fatal("Timeout expired when waiting for LB policy to create control channel") - } - - lbCfg = &lbConfig{lookupService: server2.Address} - t.Logf("Sending service config %+v to RLS LB policy ...", lbCfg) - rlsB.UpdateClientConnState(balancer.ClientConnState{BalancerConfig: lbCfg}) - - if _, err := lis2.connCh.Receive(); err != nil { - t.Fatal("Timeout expired when waiting for LB policy to create control channel") - } - - // TODO: Verify channel connectivity state once control channel connectivity - // state monitoring is in place. - - // TODO: Verify RLS RPC can be made once we integrate with the picker. -} - -// TestUpdateControlChannelTimeout tests the scenario where the LB policy -// receives a service config update with a different lookupServiceTimeout, but -// the lookupService itself remains unchanged. It verifies that the LB policy -// does not create a new control channel in this case. -func (s) TestUpdateControlChannelTimeout(t *testing.T) { - server, lis, cleanup := setupwithListener(t) - defer cleanup() - - bb := balancer.Get(rlsBalancerName) - if bb == nil { - t.Fatalf("balancer.Get(%s) = nil", rlsBalancerName) - } - rlsB := bb.Build(&testBalancerCC{}, balancer.BuildOptions{}) - defer rlsB.Close() - t.Log("Built RLS LB policy ...") - - lbCfg := &lbConfig{lookupService: server.Address, lookupServiceTimeout: 1 * time.Second} - t.Logf("Sending service config %+v to RLS LB policy ...", lbCfg) - rlsB.UpdateClientConnState(balancer.ClientConnState{BalancerConfig: lbCfg}) - if _, err := lis.connCh.Receive(); err != nil { - t.Fatal("Timeout expired when waiting for LB policy to create control channel") - } - - lbCfg = &lbConfig{lookupService: server.Address, lookupServiceTimeout: 2 * time.Second} - t.Logf("Sending service config %+v to RLS LB policy ...", lbCfg) - rlsB.UpdateClientConnState(balancer.ClientConnState{BalancerConfig: lbCfg}) - if _, err := lis.connCh.Receive(); err != testutils.ErrRecvTimeout { - t.Fatal("LB policy created new control channel when only lookupServiceTimeout changed") - } - - // TODO: Verify channel connectivity state once control channel connectivity - // state monitoring is in place. - - // TODO: Verify RLS RPC can be made once we integrate with the picker. -} - -// TestUpdateControlChannelWithCreds tests the scenario where the control -// channel is to established with credentials from the parent channel. -func (s) TestUpdateControlChannelWithCreds(t *testing.T) { - sCreds, err := credentials.NewServerTLSFromFile(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem")) - if err != nil { - t.Fatalf("credentials.NewServerTLSFromFile(server1.pem, server1.key) = %v", err) - } - cCreds, err := credentials.NewClientTLSFromFile(testdata.Path("x509/server_ca_cert.pem"), "") - if err != nil { - t.Fatalf("credentials.NewClientTLSFromFile(ca.pem) = %v", err) - } - - server, lis, cleanup := setupwithListener(t, grpc.Creds(sCreds)) - defer cleanup() - - bb := balancer.Get(rlsBalancerName) - if bb == nil { - t.Fatalf("balancer.Get(%s) = nil", rlsBalancerName) - } - rlsB := bb.Build(&testBalancerCC{}, balancer.BuildOptions{ - DialCreds: cCreds, - }) - defer rlsB.Close() - t.Log("Built RLS LB policy ...") - - lbCfg := &lbConfig{lookupService: server.Address} - t.Logf("Sending service config %+v to RLS LB policy ...", lbCfg) - rlsB.UpdateClientConnState(balancer.ClientConnState{BalancerConfig: lbCfg}) - - if _, err := lis.connCh.Receive(); err != nil { - t.Fatal("Timeout expired when waiting for LB policy to create control channel") - } - - // TODO: Verify channel connectivity state once control channel connectivity - // state monitoring is in place. - - // TODO: Verify RLS RPC can be made once we integrate with the picker. -} diff --git a/balancer/rls/internal/builder.go b/balancer/rls/internal/builder.go deleted file mode 100644 index 7c29caef4047..000000000000 --- a/balancer/rls/internal/builder.go +++ /dev/null @@ -1,53 +0,0 @@ -/* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -// Package rls implements the RLS LB policy. -package rls - -import ( - "google.golang.org/grpc/balancer" - "google.golang.org/grpc/internal/grpcsync" -) - -const rlsBalancerName = "rls" - -func init() { - balancer.Register(&rlsBB{}) -} - -// rlsBB helps build RLS load balancers and parse the service config to be -// passed to the RLS load balancer. -type rlsBB struct{} - -// Name returns the name of the RLS LB policy and helps implement the -// balancer.Balancer interface. -func (*rlsBB) Name() string { - return rlsBalancerName -} - -func (*rlsBB) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { - lb := &rlsBalancer{ - done: grpcsync.NewEvent(), - cc: cc, - opts: opts, - lbCfg: &lbConfig{}, - ccUpdateCh: make(chan *balancer.ClientConnState), - } - go lb.run() - return lb -} diff --git a/balancer/rls/internal/cache/cache.go b/balancer/rls/internal/cache/cache.go deleted file mode 100644 index b975c3078fdb..000000000000 --- a/balancer/rls/internal/cache/cache.go +++ /dev/null @@ -1,244 +0,0 @@ -/* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -// Package cache provides an LRU cache implementation to be used by the RLS LB -// policy to cache RLS response data. -package cache - -import ( - "container/list" - "sync" - "time" - - "google.golang.org/grpc/balancer" - "google.golang.org/grpc/grpclog" - "google.golang.org/grpc/internal/backoff" -) - -var logger = grpclog.Component("rls") - -// Key represents the cache key used to uniquely identify a cache entry. -type Key struct { - // Path is the full path of the incoming RPC request. - Path string - // KeyMap is a stringified version of the RLS request keys built using the - // RLS keyBuilder. Since map is not a Type which is comparable in Go, it - // cannot be part of the key for another map (the LRU cache is implemented - // using a native map type). - KeyMap string -} - -// Entry wraps all the data to be stored in a cache entry. -type Entry struct { - // Mu synchronizes access to this particular cache entry. The LB policy - // will also hold another mutex to synchronize access to the cache as a - // whole. To avoid holding the top-level mutex for the whole duration for - // which one particular cache entry is acted upon, we use this entry mutex. - Mu sync.Mutex - // ExpiryTime is the absolute time at which the data cached as part of this - // entry stops being valid. When an RLS request succeeds, this is set to - // the current time plus the max_age field from the LB policy config. An - // entry with this field in the past is not used to process picks. - ExpiryTime time.Time - // BackoffExpiryTime is the absolute time at which an entry which has gone - // through backoff stops being valid. When an RLS request fails, this is - // set to the current time plus twice the backoff time. The cache expiry - // timer will only delete entries for which both ExpiryTime and - // BackoffExpiryTime are in the past. - BackoffExpiryTime time.Time - // StaleTime is the absolute time after which this entry will be - // proactively refreshed if we receive a request for it. When an RLS - // request succeeds, this is set to the current time plus the stale_age - // from the LB policy config. - StaleTime time.Time - // BackoffTime is the absolute time at which the backoff period for this - // entry ends. The backoff timer is setup with this value. No new RLS - // requests are sent out for this entry until the backoff period ends. - BackoffTime time.Time - // EarliestEvictTime is the absolute time before which this entry should - // not be evicted from the cache. This is set to a default value of 5 - // seconds when the entry is created. This is required to make sure that a - // new entry added to the cache is not evicted before the RLS response - // arrives (usually when the cache is too small). - EarliestEvictTime time.Time - // CallStatus stores the RPC status of the previous RLS request for this - // entry. Picks for entries with a non-nil value for this field are failed - // with the error stored here. - CallStatus error - // Backoff contains all backoff related state. When an RLS request - // succeeds, backoff state is reset. - Backoff BackoffState - // HeaderData is received in an RLS response and is to be sent in the - // X-Google-RLS-Data header for matching RPCs. - HeaderData string - // ChildPicker is a very thin wrapper around the child policy wrapper. - // The type is declared as a Picker interface since the users of - // the cache only care about the picker provided by the child policy, and - // this makes it easy for testing. - ChildPicker balancer.Picker - - // size stores the size of this cache entry. Uses only a subset of the - // fields. See `entrySize` for this is computed. - size int64 - // key contains the cache key corresponding to this entry. This is required - // from methods like `removeElement` which only have a pointer to the - // list.Element which contains a reference to the cache.Entry. But these - // methods need the cache.Key to be able to remove the entry from the - // underlying map. - key Key -} - -// BackoffState wraps all backoff related state associated with a cache entry. -type BackoffState struct { - // Retries keeps track of the number of RLS failures, to be able to - // determine the amount of time to backoff before the next attempt. - Retries int - // Backoff is an exponential backoff implementation which returns the - // amount of time to backoff, given the number of retries. - Backoff backoff.Strategy - // Timer fires when the backoff period ends and incoming requests after - // this will trigger a new RLS request. - Timer *time.Timer - // Callback provided by the LB policy to be notified when the backoff timer - // expires. This will trigger a new picker to be returned to the - // ClientConn, to force queued up RPCs to be retried. - Callback func() -} - -// LRU is a cache with a least recently used eviction policy. It is not safe -// for concurrent access. -type LRU struct { - maxSize int64 - usedSize int64 - onEvicted func(Key, *Entry) - - ll *list.List - cache map[Key]*list.Element -} - -// NewLRU creates a cache.LRU with a size limit of maxSize and the provided -// eviction callback. -// -// Currently, only the cache.Key and the HeaderData field from cache.Entry -// count towards the size of the cache (other overhead per cache entry is not -// counted). The cache could temporarily exceed the configured maxSize because -// we want the entries to spend a configured minimum amount of time in the -// cache before they are LRU evicted (so that all the work performed in sending -// an RLS request and caching the response is not a total waste). -// -// The provided onEvited callback must not attempt to re-add the entry inline -// and the RLS LB policy does not have a need to do that. -// -// The cache package trusts the RLS policy (its only user) to supply a default -// minimum non-zero maxSize, in the event that the ServiceConfig does not -// provide a value for it. -func NewLRU(maxSize int64, onEvicted func(Key, *Entry)) *LRU { - return &LRU{ - maxSize: maxSize, - onEvicted: onEvicted, - ll: list.New(), - cache: make(map[Key]*list.Element), - } -} - -// Resize sets the size limit of the LRU to newMaxSize and removes older -// entries, if required, to comply with the new limit. -func (lru *LRU) Resize(newMaxSize int64) { - lru.maxSize = newMaxSize - lru.removeToFit(0) -} - -// TODO(easwars): If required, make this function more sophisticated. -func entrySize(key Key, value *Entry) int64 { - return int64(len(key.Path) + len(key.KeyMap) + len(value.HeaderData)) -} - -// removeToFit removes older entries from the cache to make room for a new -// entry of size newSize. -func (lru *LRU) removeToFit(newSize int64) { - now := time.Now() - for lru.usedSize+newSize > lru.maxSize { - elem := lru.ll.Back() - if elem == nil { - // This is a corner case where the cache is empty, but the new entry - // to be added is bigger than maxSize. - logger.Info("rls: newly added cache entry exceeds cache maxSize") - return - } - - entry := elem.Value.(*Entry) - if t := entry.EarliestEvictTime; !t.IsZero() && t.Before(now) { - // When the oldest entry is too new (it hasn't even spent a default - // minimum amount of time in the cache), we abort and allow the - // cache to grow bigger than the configured maxSize. - logger.Info("rls: LRU eviction finds oldest entry to be too new. Allowing cache to exceed maxSize momentarily") - return - } - lru.removeElement(elem) - } -} - -// Add adds a new entry to the cache. -func (lru *LRU) Add(key Key, value *Entry) { - size := entrySize(key, value) - elem, ok := lru.cache[key] - if !ok { - lru.removeToFit(size) - lru.usedSize += size - value.size = size - value.key = key - elem := lru.ll.PushFront(value) - lru.cache[key] = elem - return - } - - existing := elem.Value.(*Entry) - sizeDiff := size - existing.size - lru.removeToFit(sizeDiff) - value.size = size - elem.Value = value - lru.ll.MoveToFront(elem) - lru.usedSize += sizeDiff -} - -// Remove removes a cache entry wth key key, if one exists. -func (lru *LRU) Remove(key Key) { - if elem, ok := lru.cache[key]; ok { - lru.removeElement(elem) - } -} - -func (lru *LRU) removeElement(e *list.Element) { - entry := e.Value.(*Entry) - lru.ll.Remove(e) - delete(lru.cache, entry.key) - lru.usedSize -= entry.size - if lru.onEvicted != nil { - lru.onEvicted(entry.key, entry) - } -} - -// Get returns a cache entry with key key. -func (lru *LRU) Get(key Key) *Entry { - elem, ok := lru.cache[key] - if !ok { - return nil - } - lru.ll.MoveToFront(elem) - return elem.Value.(*Entry) -} diff --git a/balancer/rls/internal/cache/cache_test.go b/balancer/rls/internal/cache/cache_test.go deleted file mode 100644 index 7c480b64621e..000000000000 --- a/balancer/rls/internal/cache/cache_test.go +++ /dev/null @@ -1,262 +0,0 @@ -/* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package cache - -import ( - "sync" - "testing" - "time" - - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" -) - -const ( - defaultTestCacheSize = 5 - defaultTestCacheMaxSize = 1000000 - defaultTestTimeout = 1 * time.Second -) - -// TestGet verifies the Add and Get methods of cache.LRU. -func TestGet(t *testing.T) { - key1 := Key{Path: "/service1/method1", KeyMap: "k1=v1,k2=v2"} - key2 := Key{Path: "/service2/method2", KeyMap: "k1=v1,k2=v2"} - val1 := Entry{HeaderData: "h1=v1"} - val2 := Entry{HeaderData: "h2=v2"} - - tests := []struct { - desc string - keysToAdd []Key - valsToAdd []*Entry - keyToGet Key - wantEntry *Entry - }{ - { - desc: "Empty cache", - keyToGet: Key{}, - }, - { - desc: "Single entry miss", - keysToAdd: []Key{key1}, - valsToAdd: []*Entry{&val1}, - keyToGet: Key{}, - }, - { - desc: "Single entry hit", - keysToAdd: []Key{key1}, - valsToAdd: []*Entry{&val1}, - keyToGet: key1, - wantEntry: &val1, - }, - { - desc: "Multi entry miss", - keysToAdd: []Key{key1, key2}, - valsToAdd: []*Entry{&val1, &val2}, - keyToGet: Key{}, - }, - { - desc: "Multi entry hit", - keysToAdd: []Key{key1, key2}, - valsToAdd: []*Entry{&val1, &val2}, - keyToGet: key1, - wantEntry: &val1, - }, - } - - for _, test := range tests { - t.Run(test.desc, func(t *testing.T) { - lru := NewLRU(defaultTestCacheMaxSize, nil) - for i, key := range test.keysToAdd { - lru.Add(key, test.valsToAdd[i]) - } - opts := []cmp.Option{ - cmpopts.IgnoreInterfaces(struct{ sync.Locker }{}), - cmpopts.IgnoreUnexported(Entry{}), - } - if gotEntry := lru.Get(test.keyToGet); !cmp.Equal(gotEntry, test.wantEntry, opts...) { - t.Errorf("lru.Get(%+v) = %+v, want %+v", test.keyToGet, gotEntry, test.wantEntry) - } - }) - } -} - -// TestRemove verifies the Add and Remove methods of cache.LRU. -func TestRemove(t *testing.T) { - keys := []Key{ - {Path: "/service1/method1", KeyMap: "k1=v1,k2=v2"}, - {Path: "/service2/method2", KeyMap: "k1=v1,k2=v2"}, - {Path: "/service3/method3", KeyMap: "k1=v1,k2=v2"}, - } - - lru := NewLRU(defaultTestCacheMaxSize, nil) - for _, k := range keys { - lru.Add(k, &Entry{}) - } - for _, k := range keys { - lru.Remove(k) - if entry := lru.Get(k); entry != nil { - t.Fatalf("lru.Get(%+v) after a call to lru.Remove succeeds, should have failed", k) - } - } -} - -// TestExceedingSizeCausesEviction verifies the case where adding a new entry -// to the cache leads to eviction of old entries to make space for the new one. -func TestExceedingSizeCausesEviction(t *testing.T) { - evictCh := make(chan Key, defaultTestCacheSize) - onEvicted := func(k Key, _ *Entry) { - t.Logf("evicted key {%+v} from cache", k) - evictCh <- k - } - - keysToFill := []Key{{Path: "a"}, {Path: "b"}, {Path: "c"}, {Path: "d"}, {Path: "e"}} - keysCausingEviction := []Key{{Path: "f"}, {Path: "g"}, {Path: "h"}, {Path: "i"}, {Path: "j"}} - - lru := NewLRU(defaultTestCacheSize, onEvicted) - for _, key := range keysToFill { - lru.Add(key, &Entry{}) - } - - for i, key := range keysCausingEviction { - lru.Add(key, &Entry{}) - - timer := time.NewTimer(defaultTestTimeout) - select { - case <-timer.C: - t.Fatal("Test timeout waiting for eviction") - case k := <-evictCh: - timer.Stop() - if !cmp.Equal(k, keysToFill[i]) { - t.Fatalf("Evicted key %+v, wanted %+v", k, keysToFill[i]) - } - } - } -} - -// TestAddCausesMultipleEvictions verifies the case where adding one new entry -// causes the eviction of multiple old entries to make space for the new one. -func TestAddCausesMultipleEvictions(t *testing.T) { - evictCh := make(chan Key, defaultTestCacheSize) - onEvicted := func(k Key, _ *Entry) { - evictCh <- k - } - - keysToFill := []Key{{Path: "a"}, {Path: "b"}, {Path: "c"}, {Path: "d"}, {Path: "e"}} - keyCausingEviction := Key{Path: "abcde"} - - lru := NewLRU(defaultTestCacheSize, onEvicted) - for _, key := range keysToFill { - lru.Add(key, &Entry{}) - } - - lru.Add(keyCausingEviction, &Entry{}) - - for i := range keysToFill { - timer := time.NewTimer(defaultTestTimeout) - select { - case <-timer.C: - t.Fatal("Test timeout waiting for eviction") - case k := <-evictCh: - timer.Stop() - if !cmp.Equal(k, keysToFill[i]) { - t.Fatalf("Evicted key %+v, wanted %+v", k, keysToFill[i]) - } - } - } -} - -// TestModifyCausesMultipleEvictions verifies the case where mofiying an -// existing entry to increase its size leads to the eviction of older entries -// to make space for the new one. -func TestModifyCausesMultipleEvictions(t *testing.T) { - evictCh := make(chan Key, defaultTestCacheSize) - onEvicted := func(k Key, _ *Entry) { - evictCh <- k - } - - keysToFill := []Key{{Path: "a"}, {Path: "b"}, {Path: "c"}, {Path: "d"}, {Path: "e"}} - lru := NewLRU(defaultTestCacheSize, onEvicted) - for _, key := range keysToFill { - lru.Add(key, &Entry{}) - } - - lru.Add(keysToFill[len(keysToFill)-1], &Entry{HeaderData: "xxxx"}) - for i := range keysToFill[:len(keysToFill)-1] { - timer := time.NewTimer(defaultTestTimeout) - select { - case <-timer.C: - t.Fatal("Test timeout waiting for eviction") - case k := <-evictCh: - timer.Stop() - if !cmp.Equal(k, keysToFill[i]) { - t.Fatalf("Evicted key %+v, wanted %+v", k, keysToFill[i]) - } - } - } -} - -func TestLRUResize(t *testing.T) { - tests := []struct { - desc string - maxSize int64 - keysToFill []Key - newMaxSize int64 - wantEvictedKeys []Key - }{ - { - desc: "resize causes multiple evictions", - maxSize: 5, - keysToFill: []Key{{Path: "a"}, {Path: "b"}, {Path: "c"}, {Path: "d"}, {Path: "e"}}, - newMaxSize: 3, - wantEvictedKeys: []Key{{Path: "a"}, {Path: "b"}}, - }, - { - desc: "resize causes no evictions", - maxSize: 50, - keysToFill: []Key{{Path: "a"}, {Path: "b"}, {Path: "c"}, {Path: "d"}, {Path: "e"}}, - newMaxSize: 10, - wantEvictedKeys: []Key{}, - }, - { - desc: "resize to higher value", - maxSize: 5, - keysToFill: []Key{{Path: "a"}, {Path: "b"}, {Path: "c"}, {Path: "d"}, {Path: "e"}}, - newMaxSize: 10, - wantEvictedKeys: []Key{}, - }, - } - - for _, test := range tests { - t.Run(test.desc, func(t *testing.T) { - var evictedKeys []Key - onEvicted := func(k Key, _ *Entry) { - evictedKeys = append(evictedKeys, k) - } - - lru := NewLRU(test.maxSize, onEvicted) - for _, key := range test.keysToFill { - lru.Add(key, &Entry{}) - } - lru.Resize(test.newMaxSize) - if !cmp.Equal(evictedKeys, test.wantEvictedKeys, cmpopts.EquateEmpty()) { - t.Fatalf("lru.Resize evicted keys {%v}, should have evicted {%v}", evictedKeys, test.wantEvictedKeys) - } - }) - } -} diff --git a/balancer/rls/internal/client.go b/balancer/rls/internal/client.go deleted file mode 100644 index 0e8a1c932f11..000000000000 --- a/balancer/rls/internal/client.go +++ /dev/null @@ -1,79 +0,0 @@ -/* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package rls - -import ( - "context" - "time" - - "google.golang.org/grpc" - rlspb "google.golang.org/grpc/balancer/rls/internal/proto/grpc_lookup_v1" -) - -// For gRPC services using RLS, the value of target_type in the -// RouteLookupServiceRequest will be set to this. -const grpcTargetType = "grpc" - -// rlsClient is a simple wrapper around a RouteLookupService client which -// provides non-blocking semantics on top of a blocking unary RPC call. -// -// The RLS LB policy creates a new rlsClient object with the following values: -// * a grpc.ClientConn to the RLS server using appropriate credentials from the -// parent channel -// * dialTarget corresponding to the original user dial target, e.g. -// "firestore.googleapis.com". -// -// The RLS LB policy uses an adaptive throttler to perform client side -// throttling and asks this client to make an RPC call only after checking with -// the throttler. -type rlsClient struct { - stub rlspb.RouteLookupServiceClient - // origDialTarget is the original dial target of the user and sent in each - // RouteLookup RPC made to the RLS server. - origDialTarget string - // rpcTimeout specifies the timeout for the RouteLookup RPC call. The LB - // policy receives this value in its service config. - rpcTimeout time.Duration -} - -func newRLSClient(cc *grpc.ClientConn, dialTarget string, rpcTimeout time.Duration) *rlsClient { - return &rlsClient{ - stub: rlspb.NewRouteLookupServiceClient(cc), - origDialTarget: dialTarget, - rpcTimeout: rpcTimeout, - } -} - -type lookupCallback func(targets []string, headerData string, err error) - -// lookup starts a RouteLookup RPC in a separate goroutine and returns the -// results (and error, if any) in the provided callback. -func (c *rlsClient) lookup(path string, keyMap map[string]string, cb lookupCallback) { - go func() { - ctx, cancel := context.WithTimeout(context.Background(), c.rpcTimeout) - resp, err := c.stub.RouteLookup(ctx, &rlspb.RouteLookupRequest{ - Server: c.origDialTarget, - Path: path, - TargetType: grpcTargetType, - KeyMap: keyMap, - }) - cb(resp.GetTargets(), resp.GetHeaderData(), err) - cancel() - }() -} diff --git a/balancer/rls/internal/client_test.go b/balancer/rls/internal/client_test.go deleted file mode 100644 index a45850dce11e..000000000000 --- a/balancer/rls/internal/client_test.go +++ /dev/null @@ -1,174 +0,0 @@ -/* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package rls - -import ( - "errors" - "fmt" - "testing" - "time" - - "github.com/golang/protobuf/proto" - "github.com/google/go-cmp/cmp" - "google.golang.org/grpc" - rlspb "google.golang.org/grpc/balancer/rls/internal/proto/grpc_lookup_v1" - "google.golang.org/grpc/balancer/rls/internal/testutils/fakeserver" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/internal/testutils" - "google.golang.org/grpc/status" -) - -const ( - defaultDialTarget = "dummy" - defaultRPCTimeout = 5 * time.Second -) - -func setup(t *testing.T) (*fakeserver.Server, *grpc.ClientConn, func()) { - t.Helper() - - server, sCleanup, err := fakeserver.Start(nil) - if err != nil { - t.Fatalf("Failed to start fake RLS server: %v", err) - } - - cc, cCleanup, err := server.ClientConn() - if err != nil { - sCleanup() - t.Fatalf("Failed to get a ClientConn to the RLS server: %v", err) - } - - return server, cc, func() { - sCleanup() - cCleanup() - } -} - -// TestLookupFailure verifies the case where the RLS server returns an error. -func (s) TestLookupFailure(t *testing.T) { - server, cc, cleanup := setup(t) - defer cleanup() - - // We setup the fake server to return an error. - server.ResponseChan <- fakeserver.Response{Err: errors.New("rls failure")} - - rlsClient := newRLSClient(cc, defaultDialTarget, defaultRPCTimeout) - - errCh := testutils.NewChannel() - rlsClient.lookup("", nil, func(targets []string, headerData string, err error) { - if err == nil { - errCh.Send(errors.New("rlsClient.lookup() succeeded, should have failed")) - return - } - if len(targets) != 0 || headerData != "" { - errCh.Send(fmt.Errorf("rlsClient.lookup() = (%v, %s), want (nil, \"\")", targets, headerData)) - return - } - errCh.Send(nil) - }) - - if e, err := errCh.Receive(); err != nil || e != nil { - t.Fatalf("lookup error: %v, error receiving from channel: %v", e, err) - } -} - -// TestLookupDeadlineExceeded tests the case where the RPC deadline associated -// with the lookup expires. -func (s) TestLookupDeadlineExceeded(t *testing.T) { - _, cc, cleanup := setup(t) - defer cleanup() - - // Give the Lookup RPC a small deadline, but don't setup the fake server to - // return anything. So the Lookup call will block and eventually expire. - rlsClient := newRLSClient(cc, defaultDialTarget, 100*time.Millisecond) - - errCh := testutils.NewChannel() - rlsClient.lookup("", nil, func(_ []string, _ string, err error) { - if st, ok := status.FromError(err); !ok || st.Code() != codes.DeadlineExceeded { - errCh.Send(fmt.Errorf("rlsClient.lookup() returned error: %v, want %v", err, codes.DeadlineExceeded)) - return - } - errCh.Send(nil) - }) - - if e, err := errCh.Receive(); err != nil || e != nil { - t.Fatalf("lookup error: %v, error receiving from channel: %v", e, err) - } -} - -// TestLookupSuccess verifies the successful Lookup API case. -func (s) TestLookupSuccess(t *testing.T) { - server, cc, cleanup := setup(t) - defer cleanup() - - const ( - rlsReqPath = "/service/method" - wantHeaderData = "headerData" - ) - - rlsReqKeyMap := map[string]string{ - "k1": "v1", - "k2": "v2", - } - wantLookupRequest := &rlspb.RouteLookupRequest{ - Server: defaultDialTarget, - Path: rlsReqPath, - TargetType: "grpc", - KeyMap: rlsReqKeyMap, - } - wantRespTargets := []string{"us_east_1.firestore.googleapis.com"} - - rlsClient := newRLSClient(cc, defaultDialTarget, defaultRPCTimeout) - - errCh := testutils.NewChannel() - rlsClient.lookup(rlsReqPath, rlsReqKeyMap, func(targets []string, hd string, err error) { - if err != nil { - errCh.Send(fmt.Errorf("rlsClient.Lookup() failed: %v", err)) - return - } - if !cmp.Equal(targets, wantRespTargets) || hd != wantHeaderData { - errCh.Send(fmt.Errorf("rlsClient.lookup() = (%v, %s), want (%v, %s)", targets, hd, wantRespTargets, wantHeaderData)) - return - } - errCh.Send(nil) - }) - - // Make sure that the fake server received the expected RouteLookupRequest - // proto. - req, err := server.RequestChan.Receive() - if err != nil { - t.Fatalf("Timed out wile waiting for a RouteLookupRequest") - } - gotLookupRequest := req.(*rlspb.RouteLookupRequest) - if diff := cmp.Diff(wantLookupRequest, gotLookupRequest, cmp.Comparer(proto.Equal)); diff != "" { - t.Fatalf("RouteLookupRequest diff (-want, +got):\n%s", diff) - } - - // We setup the fake server to return this response when it receives a - // request. - server.ResponseChan <- fakeserver.Response{ - Resp: &rlspb.RouteLookupResponse{ - Targets: wantRespTargets, - HeaderData: wantHeaderData, - }, - } - - if e, err := errCh.Receive(); err != nil || e != nil { - t.Fatalf("lookup error: %v, error receiving from channel: %v", e, err) - } -} diff --git a/balancer/rls/internal/config.go b/balancer/rls/internal/config.go deleted file mode 100644 index 305c09106ba3..000000000000 --- a/balancer/rls/internal/config.go +++ /dev/null @@ -1,326 +0,0 @@ -/* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package rls - -import ( - "bytes" - "encoding/json" - "fmt" - "time" - - "github.com/golang/protobuf/jsonpb" - "github.com/golang/protobuf/ptypes" - durationpb "github.com/golang/protobuf/ptypes/duration" - "google.golang.org/grpc/balancer" - "google.golang.org/grpc/balancer/rls/internal/keys" - rlspb "google.golang.org/grpc/balancer/rls/internal/proto/grpc_lookup_v1" - "google.golang.org/grpc/internal/grpcutil" - "google.golang.org/grpc/resolver" - "google.golang.org/grpc/serviceconfig" -) - -const ( - // This is max duration that we are willing to cache RLS responses. If the - // service config doesn't specify a value for max_age or if it specified a - // value greater that this, we will use this value instead. - maxMaxAge = 5 * time.Minute - // If lookup_service_timeout is not specified in the service config, we use - // a default of 10 seconds. - defaultLookupServiceTimeout = 10 * time.Second - // This is set to the targetNameField in the child policy config during - // service config validation. - dummyChildPolicyTarget = "target_name_to_be_filled_in_later" -) - -// lbConfig contains the parsed and validated contents of the -// loadBalancingConfig section of the service config. The RLS LB policy will -// use this to directly access config data instead of ploughing through proto -// fields. -type lbConfig struct { - serviceconfig.LoadBalancingConfig - - kbMap keys.BuilderMap - lookupService string - lookupServiceTimeout time.Duration - maxAge time.Duration - staleAge time.Duration - cacheSizeBytes int64 - defaultTarget string - cpName string - cpTargetField string - cpConfig map[string]json.RawMessage -} - -func (lbCfg *lbConfig) Equal(other *lbConfig) bool { - return lbCfg.kbMap.Equal(other.kbMap) && - lbCfg.lookupService == other.lookupService && - lbCfg.lookupServiceTimeout == other.lookupServiceTimeout && - lbCfg.maxAge == other.maxAge && - lbCfg.staleAge == other.staleAge && - lbCfg.cacheSizeBytes == other.cacheSizeBytes && - lbCfg.defaultTarget == other.defaultTarget && - lbCfg.cpName == other.cpName && - lbCfg.cpTargetField == other.cpTargetField && - cpConfigEqual(lbCfg.cpConfig, other.cpConfig) -} - -func cpConfigEqual(am, bm map[string]json.RawMessage) bool { - if (bm == nil) != (am == nil) { - return false - } - if len(bm) != len(am) { - return false - } - - for k, jsonA := range am { - jsonB, ok := bm[k] - if !ok { - return false - } - if !bytes.Equal(jsonA, jsonB) { - return false - } - } - return true -} - -// This struct resembles the JSON respresentation of the loadBalancing config -// and makes it easier to unmarshal. -type lbConfigJSON struct { - RouteLookupConfig json.RawMessage - ChildPolicy []*loadBalancingConfig - ChildPolicyConfigTargetFieldName string -} - -// loadBalancingConfig represents a single load balancing config, -// stored in JSON format. -// -// TODO(easwars): This code seems to be repeated in a few places -// (service_config.go and in the xds code as well). Refactor and re-use. -type loadBalancingConfig struct { - Name string - Config json.RawMessage -} - -// MarshalJSON returns a JSON encoding of l. -func (l *loadBalancingConfig) MarshalJSON() ([]byte, error) { - return nil, fmt.Errorf("rls: loadBalancingConfig.MarshalJSON() is unimplemented") -} - -// UnmarshalJSON parses the JSON-encoded byte slice in data and stores it in l. -func (l *loadBalancingConfig) UnmarshalJSON(data []byte) error { - var cfg map[string]json.RawMessage - if err := json.Unmarshal(data, &cfg); err != nil { - return err - } - for name, config := range cfg { - l.Name = name - l.Config = config - } - return nil -} - -// ParseConfig parses and validates the JSON representation of the service -// config and returns the loadBalancingConfig to be used by the RLS LB policy. -// -// Helps implement the balancer.ConfigParser interface. -// -// The following validation checks are performed: -// * routeLookupConfig: -// ** grpc_keybuilders field: -// - must have at least one entry -// - must not have two entries with the same Name -// - must not have any entry with a Name with the service field unset or -// empty -// - must not have any entries without a Name -// - must not have a headers entry that has required_match set -// - must not have two headers entries with the same key within one entry -// ** lookup_service field: -// - must be set and non-empty and must parse as a target URI -// ** max_age field: -// - if not specified or is greater than maxMaxAge, it will be reset to -// maxMaxAge -// ** stale_age field: -// - if the value is greater than or equal to max_age, it is ignored -// - if set, then max_age must also be set -// ** valid_targets field: -// - will be ignored -// ** cache_size_bytes field: -// - must be greater than zero -// - TODO(easwars): Define a minimum value for this field, to be used when -// left unspecified -// * childPolicy field: -// - must find a valid child policy with a valid config (the child policy must -// be able to parse the provided config successfully when we pass it a dummy -// target name in the target_field provided by the -// childPolicyConfigTargetFieldName field) -// * childPolicyConfigTargetFieldName field: -// - must be set and non-empty -func (*rlsBB) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { - cfgJSON := &lbConfigJSON{} - if err := json.Unmarshal(c, cfgJSON); err != nil { - return nil, fmt.Errorf("rls: json unmarshal failed for service config {%+v}: %v", string(c), err) - } - - m := jsonpb.Unmarshaler{AllowUnknownFields: true} - rlsProto := &rlspb.RouteLookupConfig{} - if err := m.Unmarshal(bytes.NewReader(cfgJSON.RouteLookupConfig), rlsProto); err != nil { - return nil, fmt.Errorf("rls: bad RouteLookupConfig proto {%+v}: %v", string(cfgJSON.RouteLookupConfig), err) - } - - var childPolicy *loadBalancingConfig - for _, lbcfg := range cfgJSON.ChildPolicy { - if balancer.Get(lbcfg.Name) != nil { - childPolicy = lbcfg - break - } - } - - kbMap, err := keys.MakeBuilderMap(rlsProto) - if err != nil { - return nil, err - } - - lookupService := rlsProto.GetLookupService() - if lookupService == "" { - return nil, fmt.Errorf("rls: empty lookup_service in service config {%+v}", string(c)) - } - parsedTarget := grpcutil.ParseTarget(lookupService) - if parsedTarget.Scheme == "" { - parsedTarget.Scheme = resolver.GetDefaultScheme() - } - if resolver.Get(parsedTarget.Scheme) == nil { - return nil, fmt.Errorf("rls: invalid target URI in lookup_service {%s}", lookupService) - } - - lookupServiceTimeout, err := convertDuration(rlsProto.GetLookupServiceTimeout()) - if err != nil { - return nil, fmt.Errorf("rls: failed to parse lookup_service_timeout in service config {%+v}: %v", string(c), err) - } - if lookupServiceTimeout == 0 { - lookupServiceTimeout = defaultLookupServiceTimeout - } - maxAge, err := convertDuration(rlsProto.GetMaxAge()) - if err != nil { - return nil, fmt.Errorf("rls: failed to parse max_age in service config {%+v}: %v", string(c), err) - } - staleAge, err := convertDuration(rlsProto.GetStaleAge()) - if err != nil { - return nil, fmt.Errorf("rls: failed to parse staleAge in service config {%+v}: %v", string(c), err) - } - if staleAge != 0 && maxAge == 0 { - return nil, fmt.Errorf("rls: stale_age is set, but max_age is not in service config {%+v}", string(c)) - } - if staleAge >= maxAge { - logger.Info("rls: stale_age {%v} is greater than max_age {%v}, ignoring it", staleAge, maxAge) - staleAge = 0 - } - if maxAge == 0 || maxAge > maxMaxAge { - logger.Infof("rls: max_age in service config is %v, using %v", maxAge, maxMaxAge) - maxAge = maxMaxAge - } - cacheSizeBytes := rlsProto.GetCacheSizeBytes() - if cacheSizeBytes <= 0 { - return nil, fmt.Errorf("rls: cache_size_bytes must be greater than 0 in service config {%+v}", string(c)) - } - if childPolicy == nil { - return nil, fmt.Errorf("rls: childPolicy is invalid in service config {%+v}", string(c)) - } - if cfgJSON.ChildPolicyConfigTargetFieldName == "" { - return nil, fmt.Errorf("rls: childPolicyConfigTargetFieldName field is not set in service config {%+v}", string(c)) - } - // TODO(easwars): When we start instantiating the child policy from the - // parent RLS LB policy, we could make this function a method on the - // lbConfig object and share the code. We would be parsing the child policy - // config again during that time. The only difference betweeen now and then - // would be that we would be using real targetField name instead of the - // dummy. So, we could make the targetName field a parameter to this - // function during the refactor. - cpCfg, err := validateChildPolicyConfig(childPolicy, cfgJSON.ChildPolicyConfigTargetFieldName) - if err != nil { - return nil, err - } - - return &lbConfig{ - kbMap: kbMap, - lookupService: lookupService, - lookupServiceTimeout: lookupServiceTimeout, - maxAge: maxAge, - staleAge: staleAge, - cacheSizeBytes: cacheSizeBytes, - defaultTarget: rlsProto.GetDefaultTarget(), - // TODO(easwars): Once we refactor validateChildPolicyConfig and make - // it a method on the lbConfig object, we could directly store the - // balancer.Builder and/or balancer.ConfigParser here instead of the - // Name. That would mean that we would have to create the lbConfig - // object here first before validating the childPolicy config, but - // that's a minor detail. - cpName: childPolicy.Name, - cpTargetField: cfgJSON.ChildPolicyConfigTargetFieldName, - cpConfig: cpCfg, - }, nil -} - -// validateChildPolicyConfig validates the child policy config received in the -// service config. This makes it possible for us to reject service configs -// which contain invalid child policy configs which we know will fail for sure. -// -// It does the following: -// * Unmarshals the provided child policy config into a map of string to -// json.RawMessage. This allows us to add an entry to the map corresponding -// to the targetFieldName that we received in the service config. -// * Marshals the map back into JSON, finds the config parser associated with -// the child policy and asks it to validate the config. -// * If the validation succeeded, removes the dummy entry from the map and -// returns it. If any of the above steps failed, it returns an error. -func validateChildPolicyConfig(cp *loadBalancingConfig, cpTargetField string) (map[string]json.RawMessage, error) { - var childConfig map[string]json.RawMessage - if err := json.Unmarshal(cp.Config, &childConfig); err != nil { - return nil, fmt.Errorf("rls: json unmarshal failed for child policy config {%+v}: %v", cp.Config, err) - } - childConfig[cpTargetField], _ = json.Marshal(dummyChildPolicyTarget) - - jsonCfg, err := json.Marshal(childConfig) - if err != nil { - return nil, fmt.Errorf("rls: json marshal failed for child policy config {%+v}: %v", childConfig, err) - } - builder := balancer.Get(cp.Name) - if builder == nil { - // This should never happen since we already made sure that the child - // policy name mentioned in the service config is a valid one. - return nil, fmt.Errorf("rls: balancer builder not found for child_policy %v", cp.Name) - } - parser, ok := builder.(balancer.ConfigParser) - if !ok { - return nil, fmt.Errorf("rls: balancer builder for child_policy does not implement balancer.ConfigParser: %v", cp.Name) - } - _, err = parser.ParseConfig(jsonCfg) - if err != nil { - return nil, fmt.Errorf("rls: childPolicy config validation failed: %v", err) - } - delete(childConfig, cpTargetField) - return childConfig, nil -} - -func convertDuration(d *durationpb.Duration) (time.Duration, error) { - if d == nil { - return 0, nil - } - return ptypes.Duration(d) -} diff --git a/balancer/rls/internal/keys/builder.go b/balancer/rls/internal/keys/builder.go index 5ce5a9da508a..d010f74456fe 100644 --- a/balancer/rls/internal/keys/builder.go +++ b/balancer/rls/internal/keys/builder.go @@ -25,29 +25,15 @@ import ( "sort" "strings" - rlspb "google.golang.org/grpc/balancer/rls/internal/proto/grpc_lookup_v1" + rlspb "google.golang.org/grpc/internal/proto/grpc_lookup_v1" "google.golang.org/grpc/metadata" ) -// BuilderMap provides a mapping from a request path to the key builder to be -// used for that path. -// The BuilderMap is constructed by parsing the RouteLookupConfig received by -// the RLS balancer as part of its ServiceConfig, and is used by the picker in -// the data path to build the RLS keys to be used for a given request. +// BuilderMap maps from request path to the key builder for that path. type BuilderMap map[string]builder // MakeBuilderMap parses the provided RouteLookupConfig proto and returns a map // from paths to key builders. -// -// The following conditions are validated, and an error is returned if any of -// them is not met: -// grpc_keybuilders field -// * must have at least one entry -// * must not have two entries with the same Name -// * must not have any entry with a Name with the service field unset or empty -// * must not have any entries without a Name -// * must not have a headers entry that has required_match set -// * must not have two headers entries with the same key within one entry func MakeBuilderMap(cfg *rlspb.RouteLookupConfig) (BuilderMap, error) { kbs := cfg.GetGrpcKeybuilders() if len(kbs) == 0 { @@ -56,21 +42,46 @@ func MakeBuilderMap(cfg *rlspb.RouteLookupConfig) (BuilderMap, error) { bm := make(map[string]builder) for _, kb := range kbs { + // Extract keys from `headers`, `constant_keys` and `extra_keys` fields + // and populate appropriate values in the builder struct. Also ensure + // that keys are not repeated. var matchers []matcher seenKeys := make(map[string]bool) + constantKeys := kb.GetConstantKeys() + for k := range kb.GetConstantKeys() { + seenKeys[k] = true + } for _, h := range kb.GetHeaders() { if h.GetRequiredMatch() { return nil, fmt.Errorf("rls: GrpcKeyBuilder in RouteLookupConfig has required_match field set {%+v}", kbs) } key := h.GetKey() if seenKeys[key] { - return nil, fmt.Errorf("rls: GrpcKeyBuilder in RouteLookupConfig contains repeated Key field in headers {%+v}", kbs) + return nil, fmt.Errorf("rls: GrpcKeyBuilder in RouteLookupConfig contains repeated key %q across headers, constant_keys and extra_keys {%+v}", key, kbs) } seenKeys[key] = true matchers = append(matchers, matcher{key: h.GetKey(), names: h.GetNames()}) } - b := builder{matchers: matchers} + if seenKeys[kb.GetExtraKeys().GetHost()] { + return nil, fmt.Errorf("rls: GrpcKeyBuilder in RouteLookupConfig contains repeated key %q in extra_keys from constant_keys or headers {%+v}", kb.GetExtraKeys().GetHost(), kbs) + } + if seenKeys[kb.GetExtraKeys().GetService()] { + return nil, fmt.Errorf("rls: GrpcKeyBuilder in RouteLookupConfig contains repeated key %q in extra_keys from constant_keys or headers {%+v}", kb.GetExtraKeys().GetService(), kbs) + } + if seenKeys[kb.GetExtraKeys().GetMethod()] { + return nil, fmt.Errorf("rls: GrpcKeyBuilder in RouteLookupConfig contains repeated key %q in extra_keys from constant_keys or headers {%+v}", kb.GetExtraKeys().GetMethod(), kbs) + } + b := builder{ + headerKeys: matchers, + constantKeys: constantKeys, + hostKey: kb.GetExtraKeys().GetHost(), + serviceKey: kb.GetExtraKeys().GetService(), + methodKey: kb.GetExtraKeys().GetMethod(), + } + // Store the builder created above in the BuilderMap based on the value + // of the `Names` field, which wraps incoming request's service and + // method. Also, ensure that there are no repeated `Names` field. names := kb.GetNames() if len(names) == 0 { return nil, fmt.Errorf("rls: GrpcKeyBuilder in RouteLookupConfig does not contain any Name {%+v}", kbs) @@ -108,16 +119,35 @@ type KeyMap struct { // RLSKey builds the RLS keys to be used for the given request, identified by // the request path and the request headers stored in metadata. -func (bm BuilderMap) RLSKey(md metadata.MD, path string) KeyMap { +func (bm BuilderMap) RLSKey(md metadata.MD, host, path string) KeyMap { + // The path passed in is of the form "/service/method". The keyBuilderMap is + // indexed with keys of the form "/service/" or "/service/method". The service + // that we set in the keyMap (to be sent out in the RLS request) should not + // include any slashes though. + i := strings.LastIndex(path, "/") + service, method := path[:i+1], path[i+1:] b, ok := bm[path] if !ok { - i := strings.LastIndex(path, "/") - b, ok = bm[path[:i+1]] + b, ok = bm[service] if !ok { return KeyMap{} } } - return b.keys(md) + + kvMap := b.buildHeaderKeys(md) + if b.hostKey != "" { + kvMap[b.hostKey] = host + } + if b.serviceKey != "" { + kvMap[b.serviceKey] = strings.Trim(service, "/") + } + if b.methodKey != "" { + kvMap[b.methodKey] = method + } + for k, v := range b.constantKeys { + kvMap[k] = v + } + return KeyMap{Map: kvMap, Str: mapToString(kvMap)} } // Equal reports whether bm and am represent equivalent BuilderMaps. @@ -141,26 +171,19 @@ func (bm BuilderMap) Equal(am BuilderMap) bool { return true } -// builder provides the actual functionality of building RLS keys. These are -// stored in the BuilderMap. -// While processing a pick, the picker looks in the BuilderMap for the -// appropriate builder to be used for the given RPC. For each of the matchers -// in the found builder, we iterate over the list of request headers (available -// as metadata in the context). Once a header matches one of the names in the -// matcher, we set the value of the header in the keyMap (with the key being -// the one found in the matcher) and move on to the next matcher. If no -// KeyBuilder was found in the map, or no header match was found, an empty -// keyMap is returned. +// builder provides the actual functionality of building RLS keys. type builder struct { - matchers []matcher + headerKeys []matcher + constantKeys map[string]string + // The following keys mirror corresponding fields in `extra_keys`. + hostKey string + serviceKey string + methodKey string } // Equal reports whether b and a represent equivalent key builders. func (b builder) Equal(a builder) bool { - if (b.matchers == nil) != (a.matchers == nil) { - return false - } - if len(b.matchers) != len(a.matchers) { + if len(b.headerKeys) != len(a.headerKeys) { return false } // Protobuf serialization maintains the order of repeated fields. Matchers @@ -168,13 +191,23 @@ func (b builder) Equal(a builder) bool { // order changes, it means that the order in the protobuf changed. We report // this case as not being equal even though the builders could possible be // functionally equal. - for i, bMatcher := range b.matchers { - aMatcher := a.matchers[i] + for i, bMatcher := range b.headerKeys { + aMatcher := a.headerKeys[i] if !bMatcher.Equal(aMatcher) { return false } } - return true + + if len(b.constantKeys) != len(a.constantKeys) { + return false + } + for k, v := range b.constantKeys { + if a.constantKeys[k] != v { + return false + } + } + + return b.hostKey == a.hostKey && b.serviceKey == a.serviceKey && b.methodKey == a.methodKey } // matcher helps extract a key from request headers based on a given name. @@ -185,14 +218,11 @@ type matcher struct { names []string } -// Equal reports if m and are are equivalent matchers. +// Equal reports if m and are are equivalent headerKeys. func (m matcher) Equal(a matcher) bool { if m.key != a.key { return false } - if (m.names == nil) != (a.names == nil) { - return false - } if len(m.names) != len(a.names) { return false } @@ -204,9 +234,12 @@ func (m matcher) Equal(a matcher) bool { return true } -func (b builder) keys(md metadata.MD) KeyMap { +func (b builder) buildHeaderKeys(md metadata.MD) map[string]string { kvMap := make(map[string]string) - for _, m := range b.matchers { + if len(md) == 0 { + return kvMap + } + for _, m := range b.headerKeys { for _, name := range m.names { if vals := md.Get(name); vals != nil { kvMap[m.key] = strings.Join(vals, ",") @@ -214,11 +247,11 @@ func (b builder) keys(md metadata.MD) KeyMap { } } } - return KeyMap{Map: kvMap, Str: mapToString(kvMap)} + return kvMap } func mapToString(kv map[string]string) string { - var keys []string + keys := make([]string, 0, len(kv)) for k := range kv { keys = append(keys, k) } diff --git a/balancer/rls/internal/keys/builder_test.go b/balancer/rls/internal/keys/builder_test.go index a5cad29e0c93..90c132bc9169 100644 --- a/balancer/rls/internal/keys/builder_test.go +++ b/balancer/rls/internal/keys/builder_test.go @@ -24,7 +24,7 @@ import ( "testing" "github.com/google/go-cmp/cmp" - rlspb "google.golang.org/grpc/balancer/rls/internal/proto/grpc_lookup_v1" + rlspb "google.golang.org/grpc/internal/proto/grpc_lookup_v1" "google.golang.org/grpc/metadata" ) @@ -37,6 +37,15 @@ var ( {Key: "k1", Names: []string{"n1"}}, {Key: "k2", Names: []string{"n1"}}, }, + ExtraKeys: &rlspb.GrpcKeyBuilder_ExtraKeys{ + Host: "host", + Service: "service", + Method: "method", + }, + ConstantKeys: map[string]string{ + "const-key-1": "const-val-1", + "const-key-2": "const-val-2", + }, } goodKeyBuilder2 = &rlspb.GrpcKeyBuilder{ Names: []*rlspb.GrpcKeyBuilder_Name{ @@ -50,13 +59,21 @@ var ( ) func TestMakeBuilderMap(t *testing.T) { - wantBuilderMap1 := map[string]builder{ - "/gFoo/": {matchers: []matcher{{key: "k1", names: []string{"n1"}}, {key: "k2", names: []string{"n1"}}}}, + gFooBuilder := builder{ + headerKeys: []matcher{{key: "k1", names: []string{"n1"}}, {key: "k2", names: []string{"n1"}}}, + constantKeys: map[string]string{ + "const-key-1": "const-val-1", + "const-key-2": "const-val-2", + }, + hostKey: "host", + serviceKey: "service", + methodKey: "method", } + wantBuilderMap1 := map[string]builder{"/gFoo/": gFooBuilder} wantBuilderMap2 := map[string]builder{ - "/gFoo/": {matchers: []matcher{{key: "k1", names: []string{"n1"}}, {key: "k2", names: []string{"n1"}}}}, - "/gBar/method1": {matchers: []matcher{{key: "k1", names: []string{"n1", "n2"}}}}, - "/gFoobar/": {matchers: []matcher{{key: "k1", names: []string{"n1", "n2"}}}}, + "/gFoo/": gFooBuilder, + "/gBar/method1": {headerKeys: []matcher{{key: "k1", names: []string{"n1", "n2"}}}}, + "/gFoobar/": {headerKeys: []matcher{{key: "k1", names: []string{"n1", "n2"}}}}, } tests := []struct { @@ -91,33 +108,6 @@ func TestMakeBuilderMap(t *testing.T) { } func TestMakeBuilderMapErrors(t *testing.T) { - emptyServiceKeyBuilder := &rlspb.GrpcKeyBuilder{ - Names: []*rlspb.GrpcKeyBuilder_Name{ - {Service: "bFoo", Method: "method1"}, - {Service: "bBar"}, - {Method: "method1"}, - }, - Headers: []*rlspb.NameMatcher{{Key: "k1", Names: []string{"n1", "n2"}}}, - } - requiredMatchKeyBuilder := &rlspb.GrpcKeyBuilder{ - Names: []*rlspb.GrpcKeyBuilder_Name{{Service: "bFoo", Method: "method1"}}, - Headers: []*rlspb.NameMatcher{{Key: "k1", Names: []string{"n1", "n2"}, RequiredMatch: true}}, - } - repeatedHeadersKeyBuilder := &rlspb.GrpcKeyBuilder{ - Names: []*rlspb.GrpcKeyBuilder_Name{ - {Service: "gBar", Method: "method1"}, - {Service: "gFoobar"}, - }, - Headers: []*rlspb.NameMatcher{ - {Key: "k1", Names: []string{"n1", "n2"}}, - {Key: "k1", Names: []string{"n1", "n2"}}, - }, - } - methodNameWithSlashKeyBuilder := &rlspb.GrpcKeyBuilder{ - Names: []*rlspb.GrpcKeyBuilder_Name{{Service: "gBar", Method: "method1/foo"}}, - Headers: []*rlspb.NameMatcher{{Key: "k1", Names: []string{"n1", "n2"}}}, - } - tests := []struct { desc string cfg *rlspb.RouteLookupConfig @@ -138,7 +128,17 @@ func TestMakeBuilderMapErrors(t *testing.T) { { desc: "GrpcKeyBuilder with empty Service field", cfg: &rlspb.RouteLookupConfig{ - GrpcKeybuilders: []*rlspb.GrpcKeyBuilder{emptyServiceKeyBuilder, goodKeyBuilder1}, + GrpcKeybuilders: []*rlspb.GrpcKeyBuilder{ + { + Names: []*rlspb.GrpcKeyBuilder_Name{ + {Service: "bFoo", Method: "method1"}, + {Service: "bBar"}, + {Method: "method1"}, + }, + Headers: []*rlspb.NameMatcher{{Key: "k1", Names: []string{"n1", "n2"}}}, + }, + goodKeyBuilder1, + }, }, wantErrPrefix: "rls: GrpcKeyBuilder in RouteLookupConfig contains a Name field with no Service", }, @@ -152,21 +152,96 @@ func TestMakeBuilderMapErrors(t *testing.T) { { desc: "GrpcKeyBuilder with requiredMatch field set", cfg: &rlspb.RouteLookupConfig{ - GrpcKeybuilders: []*rlspb.GrpcKeyBuilder{requiredMatchKeyBuilder, goodKeyBuilder1}, + GrpcKeybuilders: []*rlspb.GrpcKeyBuilder{ + { + Names: []*rlspb.GrpcKeyBuilder_Name{{Service: "bFoo", Method: "method1"}}, + Headers: []*rlspb.NameMatcher{{Key: "k1", Names: []string{"n1", "n2"}, RequiredMatch: true}}, + }, + goodKeyBuilder1, + }, }, wantErrPrefix: "rls: GrpcKeyBuilder in RouteLookupConfig has required_match field set", }, { desc: "GrpcKeyBuilder two headers with same key", cfg: &rlspb.RouteLookupConfig{ - GrpcKeybuilders: []*rlspb.GrpcKeyBuilder{repeatedHeadersKeyBuilder, goodKeyBuilder1}, + GrpcKeybuilders: []*rlspb.GrpcKeyBuilder{ + { + Names: []*rlspb.GrpcKeyBuilder_Name{ + {Service: "gBar", Method: "method1"}, + {Service: "gFoobar"}, + }, + Headers: []*rlspb.NameMatcher{ + {Key: "k1", Names: []string{"n1", "n2"}}, + {Key: "k1", Names: []string{"n1", "n2"}}, + }, + }, + goodKeyBuilder1, + }, + }, + wantErrPrefix: "rls: GrpcKeyBuilder in RouteLookupConfig contains repeated key \"k1\" across headers, constant_keys and extra_keys", + }, + { + desc: "GrpcKeyBuilder repeated keys across headers and constant_keys", + cfg: &rlspb.RouteLookupConfig{ + GrpcKeybuilders: []*rlspb.GrpcKeyBuilder{ + { + Names: []*rlspb.GrpcKeyBuilder_Name{ + {Service: "gBar", Method: "method1"}, + {Service: "gFoobar"}, + }, + Headers: []*rlspb.NameMatcher{{Key: "k1", Names: []string{"n1", "n2"}}}, + ConstantKeys: map[string]string{"k1": "v1"}, + }, + goodKeyBuilder1, + }, + }, + wantErrPrefix: "rls: GrpcKeyBuilder in RouteLookupConfig contains repeated key \"k1\" across headers, constant_keys and extra_keys", + }, + { + desc: "GrpcKeyBuilder repeated keys across headers and extra_keys", + cfg: &rlspb.RouteLookupConfig{ + GrpcKeybuilders: []*rlspb.GrpcKeyBuilder{ + { + Names: []*rlspb.GrpcKeyBuilder_Name{ + {Service: "gBar", Method: "method1"}, + {Service: "gFoobar"}, + }, + Headers: []*rlspb.NameMatcher{{Key: "k1", Names: []string{"n1", "n2"}}}, + ExtraKeys: &rlspb.GrpcKeyBuilder_ExtraKeys{Method: "k1"}, + }, + goodKeyBuilder1, + }, + }, + wantErrPrefix: "rls: GrpcKeyBuilder in RouteLookupConfig contains repeated key \"k1\" in extra_keys from constant_keys or headers", + }, + { + desc: "GrpcKeyBuilder repeated keys across constant_keys and extra_keys", + cfg: &rlspb.RouteLookupConfig{ + GrpcKeybuilders: []*rlspb.GrpcKeyBuilder{ + { + Names: []*rlspb.GrpcKeyBuilder_Name{ + {Service: "gBar", Method: "method1"}, + {Service: "gFoobar"}, + }, + Headers: []*rlspb.NameMatcher{{Key: "k1", Names: []string{"n1", "n2"}}}, + ConstantKeys: map[string]string{"host": "v1"}, + ExtraKeys: &rlspb.GrpcKeyBuilder_ExtraKeys{Host: "host"}, + }, + goodKeyBuilder1, + }, }, - wantErrPrefix: "rls: GrpcKeyBuilder in RouteLookupConfig contains repeated Key field in headers", + wantErrPrefix: "rls: GrpcKeyBuilder in RouteLookupConfig contains repeated key \"host\" in extra_keys from constant_keys or headers", }, { desc: "GrpcKeyBuilder with slash in method name", cfg: &rlspb.RouteLookupConfig{ - GrpcKeybuilders: []*rlspb.GrpcKeyBuilder{methodNameWithSlashKeyBuilder}, + GrpcKeybuilders: []*rlspb.GrpcKeyBuilder{ + { + Names: []*rlspb.GrpcKeyBuilder_Name{{Service: "gBar", Method: "method1/foo"}}, + Headers: []*rlspb.NameMatcher{{Key: "k1", Names: []string{"n1", "n2"}}}, + }, + }, }, wantErrPrefix: "rls: GrpcKeyBuilder in RouteLookupConfig contains a method with a slash", }, @@ -257,11 +332,22 @@ func TestRLSKey(t *testing.T) { wantKM: KeyMap{Map: map[string]string{"k1": "v1"}, Str: "k1=v1"}, }, { - // Multiple matchers find hits in the provided request headers. - desc: "multipleMatchers", - path: "/gFoo/method1", - md: metadata.Pairs("n2", "v2", "n1", "v1"), - wantKM: KeyMap{Map: map[string]string{"k1": "v1", "k2": "v1"}, Str: "k1=v1,k2=v1"}, + // Multiple headerKeys find hits in the provided request headers. + desc: "multipleMatchers", + path: "/gFoo/method1", + md: metadata.Pairs("n2", "v2", "n1", "v1"), + wantKM: KeyMap{ + Map: map[string]string{ + "const-key-1": "const-val-1", + "const-key-2": "const-val-2", + "host": "dummy-host", + "service": "gFoo", + "method": "method1", + "k1": "v1", + "k2": "v1", + }, + Str: "const-key-1=const-val-1,const-key-2=const-val-2,host=dummy-host,k1=v1,k2=v1,method=method1,service=gFoo", + }, }, { // A match is found for a header which is specified multiple times. @@ -275,7 +361,7 @@ func TestRLSKey(t *testing.T) { for _, test := range tests { t.Run(test.desc, func(t *testing.T) { - if gotKM := bm.RLSKey(test.md, test.path); !cmp.Equal(gotKM, test.wantKM) { + if gotKM := bm.RLSKey(test.md, "dummy-host", test.path); !cmp.Equal(gotKM, test.wantKM) { t.Errorf("RLSKey(%+v, %s) = %+v, want %+v", test.md, test.path, gotKM, test.wantKM) } }) @@ -351,57 +437,57 @@ func TestBuilderMapEqual(t *testing.T) { { desc: "nil and non-nil builder maps", a: nil, - b: map[string]builder{"/gFoo/": {matchers: []matcher{{key: "k1", names: []string{"n1"}}}}}, + b: map[string]builder{"/gFoo/": {headerKeys: []matcher{{key: "k1", names: []string{"n1"}}}}}, wantEqual: false, }, { desc: "empty and non-empty builder maps", a: make(map[string]builder), - b: map[string]builder{"/gFoo/": {matchers: []matcher{{key: "k1", names: []string{"n1"}}}}}, + b: map[string]builder{"/gFoo/": {headerKeys: []matcher{{key: "k1", names: []string{"n1"}}}}}, wantEqual: false, }, { desc: "different number of map keys", a: map[string]builder{ - "/gFoo/": {matchers: []matcher{{key: "k1", names: []string{"n1"}}}}, - "/gBar/": {matchers: []matcher{{key: "k1", names: []string{"n1"}}}}, + "/gFoo/": {headerKeys: []matcher{{key: "k1", names: []string{"n1"}}}}, + "/gBar/": {headerKeys: []matcher{{key: "k1", names: []string{"n1"}}}}, }, b: map[string]builder{ - "/gFoo/": {matchers: []matcher{{key: "k1", names: []string{"n1"}}}}, + "/gFoo/": {headerKeys: []matcher{{key: "k1", names: []string{"n1"}}}}, }, wantEqual: false, }, { desc: "different map keys", a: map[string]builder{ - "/gBar/": {matchers: []matcher{{key: "k1", names: []string{"n1"}}}}, + "/gBar/": {headerKeys: []matcher{{key: "k1", names: []string{"n1"}}}}, }, b: map[string]builder{ - "/gFoo/": {matchers: []matcher{{key: "k1", names: []string{"n1"}}}}, + "/gFoo/": {headerKeys: []matcher{{key: "k1", names: []string{"n1"}}}}, }, wantEqual: false, }, { desc: "equal keys different values", a: map[string]builder{ - "/gBar/": {matchers: []matcher{{key: "k1", names: []string{"n1"}}}}, - "/gFoo/": {matchers: []matcher{{key: "k1", names: []string{"n1", "n2"}}}}, + "/gBar/": {headerKeys: []matcher{{key: "k1", names: []string{"n1"}}}}, + "/gFoo/": {headerKeys: []matcher{{key: "k1", names: []string{"n1", "n2"}}}}, }, b: map[string]builder{ - "/gBar/": {matchers: []matcher{{key: "k1", names: []string{"n1"}}}}, - "/gFoo/": {matchers: []matcher{{key: "k1", names: []string{"n1"}}}}, + "/gBar/": {headerKeys: []matcher{{key: "k1", names: []string{"n1"}}}}, + "/gFoo/": {headerKeys: []matcher{{key: "k1", names: []string{"n1"}}}}, }, wantEqual: false, }, { desc: "good match", a: map[string]builder{ - "/gBar/": {matchers: []matcher{{key: "k1", names: []string{"n1"}}}}, - "/gFoo/": {matchers: []matcher{{key: "k1", names: []string{"n1"}}}}, + "/gBar/": {headerKeys: []matcher{{key: "k1", names: []string{"n1"}}}}, + "/gFoo/": {headerKeys: []matcher{{key: "k1", names: []string{"n1"}}}}, }, b: map[string]builder{ - "/gBar/": {matchers: []matcher{{key: "k1", names: []string{"n1"}}}}, - "/gFoo/": {matchers: []matcher{{key: "k1", names: []string{"n1"}}}}, + "/gBar/": {headerKeys: []matcher{{key: "k1", names: []string{"n1"}}}}, + "/gFoo/": {headerKeys: []matcher{{key: "k1", names: []string{"n1"}}}}, }, wantEqual: true, }, @@ -425,44 +511,80 @@ func TestBuilderEqual(t *testing.T) { }{ { desc: "nil builders", - a: builder{matchers: nil}, - b: builder{matchers: nil}, + a: builder{headerKeys: nil}, + b: builder{headerKeys: nil}, wantEqual: true, }, { desc: "empty builders", - a: builder{matchers: []matcher{}}, - b: builder{matchers: []matcher{}}, + a: builder{headerKeys: []matcher{}}, + b: builder{headerKeys: []matcher{}}, wantEqual: true, }, { - desc: "nil and non-nil builders", - a: builder{matchers: nil}, - b: builder{matchers: []matcher{}}, + desc: "empty and non-empty builders", + a: builder{headerKeys: []matcher{}}, + b: builder{headerKeys: []matcher{{key: "foo"}}}, wantEqual: false, }, { - desc: "empty and non-empty builders", - a: builder{matchers: []matcher{}}, - b: builder{matchers: []matcher{{key: "foo"}}}, + desc: "different number of headerKeys", + a: builder{headerKeys: []matcher{{key: "foo"}, {key: "bar"}}}, + b: builder{headerKeys: []matcher{{key: "foo"}}}, wantEqual: false, }, { - desc: "different number of matchers", - a: builder{matchers: []matcher{{key: "foo"}, {key: "bar"}}}, - b: builder{matchers: []matcher{{key: "foo"}}}, + desc: "equal number but differing headerKeys", + a: builder{headerKeys: []matcher{{key: "bar"}}}, + b: builder{headerKeys: []matcher{{key: "foo"}}}, wantEqual: false, }, { - desc: "equal number but differing matchers", - a: builder{matchers: []matcher{{key: "bar"}}}, - b: builder{matchers: []matcher{{key: "foo"}}}, + desc: "different number of constantKeys", + a: builder{constantKeys: map[string]string{"k1": "v1"}}, + b: builder{constantKeys: map[string]string{"k1": "v1", "k2": "v2"}}, wantEqual: false, }, { - desc: "good match", - a: builder{matchers: []matcher{{key: "foo"}}}, - b: builder{matchers: []matcher{{key: "foo"}}}, + desc: "equal number but differing constantKeys", + a: builder{constantKeys: map[string]string{"k1": "v1"}}, + b: builder{constantKeys: map[string]string{"k2": "v2"}}, + wantEqual: false, + }, + { + desc: "different hostKey", + a: builder{hostKey: "host1"}, + b: builder{hostKey: "host2"}, + wantEqual: false, + }, + { + desc: "different serviceKey", + a: builder{hostKey: "service1"}, + b: builder{hostKey: "service2"}, + wantEqual: false, + }, + { + desc: "different methodKey", + a: builder{hostKey: "method1"}, + b: builder{hostKey: "method2"}, + wantEqual: false, + }, + { + desc: "equal", + a: builder{ + headerKeys: []matcher{{key: "foo"}}, + constantKeys: map[string]string{"k1": "v1"}, + hostKey: "host", + serviceKey: "/service/", + methodKey: "method", + }, + b: builder{ + headerKeys: []matcher{{key: "foo"}}, + constantKeys: map[string]string{"k1": "v1"}, + hostKey: "host", + serviceKey: "/service/", + methodKey: "method", + }, wantEqual: true, }, } diff --git a/balancer/rls/internal/picker.go b/balancer/rls/internal/picker.go deleted file mode 100644 index 738449446558..000000000000 --- a/balancer/rls/internal/picker.go +++ /dev/null @@ -1,149 +0,0 @@ -/* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package rls - -import ( - "errors" - "time" - - "google.golang.org/grpc/balancer" - "google.golang.org/grpc/balancer/rls/internal/cache" - "google.golang.org/grpc/balancer/rls/internal/keys" - "google.golang.org/grpc/metadata" -) - -var errRLSThrottled = errors.New("RLS call throttled at client side") - -// RLS rlsPicker selects the subConn to be used for a particular RPC. It does -// not manage subConns directly and usually deletegates to pickers provided by -// child policies. -// -// The RLS LB policy creates a new rlsPicker object whenever its ServiceConfig -// is updated and provides a bunch of hooks for the rlsPicker to get the latest -// state that it can used to make its decision. -type rlsPicker struct { - // The keyBuilder map used to generate RLS keys for the RPC. This is built - // by the LB policy based on the received ServiceConfig. - kbm keys.BuilderMap - - // The following hooks are setup by the LB policy to enable the rlsPicker to - // access state stored in the policy. This approach has the following - // advantages: - // 1. The rlsPicker is loosely coupled with the LB policy in the sense that - // updates happening on the LB policy like the receipt of an RLS - // response, or an update to the default rlsPicker etc are not explicitly - // pushed to the rlsPicker, but are readily available to the rlsPicker - // when it invokes these hooks. And the LB policy takes care of - // synchronizing access to these shared state. - // 2. It makes unit testing the rlsPicker easy since any number of these - // hooks could be overridden. - - // readCache is used to read from the data cache and the pending request - // map in an atomic fashion. The first return parameter is the entry in the - // data cache, and the second indicates whether an entry for the same key - // is present in the pending cache. - readCache func(cache.Key) (*cache.Entry, bool) - // shouldThrottle decides if the current RPC should be throttled at the - // client side. It uses an adaptive throttling algorithm. - shouldThrottle func() bool - // startRLS kicks off an RLS request in the background for the provided RPC - // path and keyMap. An entry in the pending request map is created before - // sending out the request and an entry in the data cache is created or - // updated upon receipt of a response. See implementation in the LB policy - // for details. - startRLS func(string, keys.KeyMap) - // defaultPick enables the rlsPicker to delegate the pick decision to the - // rlsPicker returned by the child LB policy pointing to the default target - // specified in the service config. - defaultPick func(balancer.PickInfo) (balancer.PickResult, error) -} - -// Pick makes the routing decision for every outbound RPC. -func (p *rlsPicker) Pick(info balancer.PickInfo) (balancer.PickResult, error) { - // For every incoming request, we first build the RLS keys using the - // keyBuilder we received from the LB policy. If no metadata is present in - // the context, we end up using an empty key. - km := keys.KeyMap{} - md, ok := metadata.FromOutgoingContext(info.Ctx) - if ok { - km = p.kbm.RLSKey(md, info.FullMethodName) - } - - // We use the LB policy hook to read the data cache and the pending request - // map (whether or not an entry exists) for the RPC path and the generated - // RLS keys. We will end up kicking off an RLS request only if there is no - // pending request for the current RPC path and keys, and either we didn't - // find an entry in the data cache or the entry was stale and it wasn't in - // backoff. - startRequest := false - now := time.Now() - entry, pending := p.readCache(cache.Key{Path: info.FullMethodName, KeyMap: km.Str}) - if entry == nil { - startRequest = true - } else { - entry.Mu.Lock() - defer entry.Mu.Unlock() - if entry.StaleTime.Before(now) && entry.BackoffTime.Before(now) { - // This is the proactive cache refresh. - startRequest = true - } - } - - if startRequest && !pending { - if p.shouldThrottle() { - // The entry doesn't exist or has expired and the new RLS request - // has been throttled. Treat it as an error and delegate to default - // pick, if one exists, or fail the pick. - if entry == nil || entry.ExpiryTime.Before(now) { - if p.defaultPick != nil { - return p.defaultPick(info) - } - return balancer.PickResult{}, errRLSThrottled - } - // The proactive refresh has been throttled. Nothing to worry, just - // keep using the existing entry. - } else { - p.startRLS(info.FullMethodName, km) - } - } - - if entry != nil { - if entry.ExpiryTime.After(now) { - // This is the jolly good case where we have found a valid entry in - // the data cache. We delegate to the LB policy associated with - // this cache entry. - return entry.ChildPicker.Pick(info) - } else if entry.BackoffTime.After(now) { - // The entry has expired, but is in backoff. We delegate to the - // default pick, if one exists, or return the error from the last - // failed RLS request for this entry. - if p.defaultPick != nil { - return p.defaultPick(info) - } - return balancer.PickResult{}, entry.CallStatus - } - } - - // We get here only in the following cases: - // * No data cache entry or expired entry, RLS request sent out - // * No valid data cache entry and Pending cache entry exists - // We need to queue to pick which will be handled once the RLS response is - // received. - return balancer.PickResult{}, balancer.ErrNoSubConnAvailable -} diff --git a/balancer/rls/internal/picker_test.go b/balancer/rls/internal/picker_test.go deleted file mode 100644 index 1397bf8085d8..000000000000 --- a/balancer/rls/internal/picker_test.go +++ /dev/null @@ -1,607 +0,0 @@ -/* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package rls - -import ( - "context" - "errors" - "fmt" - "math" - "testing" - "time" - - "github.com/google/go-cmp/cmp" - - "google.golang.org/grpc/balancer" - "google.golang.org/grpc/balancer/rls/internal/cache" - "google.golang.org/grpc/balancer/rls/internal/keys" - rlspb "google.golang.org/grpc/balancer/rls/internal/proto/grpc_lookup_v1" - "google.golang.org/grpc/internal/grpcrand" - "google.golang.org/grpc/internal/testutils" - "google.golang.org/grpc/metadata" -) - -const defaultTestMaxAge = 5 * time.Second - -// initKeyBuilderMap initializes a keyBuilderMap of the form: -// { -// "gFoo": "k1=n1", -// "gBar/method1": "k2=n21,n22" -// "gFoobar": "k3=n3", -// } -func initKeyBuilderMap() (keys.BuilderMap, error) { - kb1 := &rlspb.GrpcKeyBuilder{ - Names: []*rlspb.GrpcKeyBuilder_Name{{Service: "gFoo"}}, - Headers: []*rlspb.NameMatcher{{Key: "k1", Names: []string{"n1"}}}, - } - kb2 := &rlspb.GrpcKeyBuilder{ - Names: []*rlspb.GrpcKeyBuilder_Name{{Service: "gBar", Method: "method1"}}, - Headers: []*rlspb.NameMatcher{{Key: "k2", Names: []string{"n21", "n22"}}}, - } - kb3 := &rlspb.GrpcKeyBuilder{ - Names: []*rlspb.GrpcKeyBuilder_Name{{Service: "gFoobar"}}, - Headers: []*rlspb.NameMatcher{{Key: "k3", Names: []string{"n3"}}}, - } - return keys.MakeBuilderMap(&rlspb.RouteLookupConfig{ - GrpcKeybuilders: []*rlspb.GrpcKeyBuilder{kb1, kb2, kb3}, - }) -} - -// fakeSubConn embeds the balancer.SubConn interface and contains an id which -// helps verify that the expected subConn was returned by the rlsPicker. -type fakeSubConn struct { - balancer.SubConn - id int -} - -// fakePicker sends a PickResult with a fakeSubConn with the configured id. -type fakePicker struct { - id int -} - -func (p *fakePicker) Pick(_ balancer.PickInfo) (balancer.PickResult, error) { - return balancer.PickResult{SubConn: &fakeSubConn{id: p.id}}, nil -} - -// newFakePicker returns a fakePicker configured with a random ID. The subConns -// returned by this picker are of type fakefakeSubConn, and contain the same -// random ID, which tests can use to verify. -func newFakePicker() *fakePicker { - return &fakePicker{id: grpcrand.Intn(math.MaxInt32)} -} - -func verifySubConn(sc balancer.SubConn, wantID int) error { - fsc, ok := sc.(*fakeSubConn) - if !ok { - return fmt.Errorf("Pick() returned a SubConn of type %T, want %T", sc, &fakeSubConn{}) - } - if fsc.id != wantID { - return fmt.Errorf("Pick() returned SubConn %d, want %d", fsc.id, wantID) - } - return nil -} - -// TestPickKeyBuilder verifies the different possible scenarios for forming an -// RLS key for an incoming RPC. -func TestPickKeyBuilder(t *testing.T) { - kbm, err := initKeyBuilderMap() - if err != nil { - t.Fatalf("Failed to create keyBuilderMap: %v", err) - } - - tests := []struct { - desc string - rpcPath string - md metadata.MD - wantKey cache.Key - }{ - { - desc: "non existent service in keyBuilder map", - rpcPath: "/gNonExistentService/method", - md: metadata.New(map[string]string{"n1": "v1", "n3": "v3"}), - wantKey: cache.Key{Path: "/gNonExistentService/method", KeyMap: ""}, - }, - { - desc: "no metadata in incoming context", - rpcPath: "/gFoo/method", - md: metadata.MD{}, - wantKey: cache.Key{Path: "/gFoo/method", KeyMap: ""}, - }, - { - desc: "keyBuilderMatch", - rpcPath: "/gFoo/method", - md: metadata.New(map[string]string{"n1": "v1", "n3": "v3"}), - wantKey: cache.Key{Path: "/gFoo/method", KeyMap: "k1=v1"}, - }, - } - - for _, test := range tests { - t.Run(test.desc, func(t *testing.T) { - randID := grpcrand.Intn(math.MaxInt32) - p := rlsPicker{ - kbm: kbm, - readCache: func(key cache.Key) (*cache.Entry, bool) { - if !cmp.Equal(key, test.wantKey) { - t.Fatalf("rlsPicker using cacheKey %v, want %v", key, test.wantKey) - } - - now := time.Now() - return &cache.Entry{ - ExpiryTime: now.Add(defaultTestMaxAge), - StaleTime: now.Add(defaultTestMaxAge), - // Cache entry is configured with a child policy whose - // rlsPicker always returns an empty PickResult and nil - // error. - ChildPicker: &fakePicker{id: randID}, - }, false - }, - // The other hooks are not set here because they are not expected to be - // invoked for these cases and if they get invoked, they will panic. - } - - gotResult, err := p.Pick(balancer.PickInfo{ - FullMethodName: test.rpcPath, - Ctx: metadata.NewOutgoingContext(context.Background(), test.md), - }) - if err != nil { - t.Fatalf("Pick() failed with error: %v", err) - } - sc, ok := gotResult.SubConn.(*fakeSubConn) - if !ok { - t.Fatalf("Pick() returned a SubConn of type %T, want %T", gotResult.SubConn, &fakeSubConn{}) - } - if sc.id != randID { - t.Fatalf("Pick() returned SubConn %d, want %d", sc.id, randID) - } - }) - } -} - -// TestPick_DataCacheMiss_PendingCacheMiss verifies different Pick scenarios -// where the entry is neither found in the data cache nor in the pending cache. -func TestPick_DataCacheMiss_PendingCacheMiss(t *testing.T) { - const ( - rpcPath = "/gFoo/method" - wantKeyMapStr = "k1=v1" - ) - kbm, err := initKeyBuilderMap() - if err != nil { - t.Fatalf("Failed to create keyBuilderMap: %v", err) - } - md := metadata.New(map[string]string{"n1": "v1", "n3": "v3"}) - wantKey := cache.Key{Path: rpcPath, KeyMap: wantKeyMapStr} - - tests := []struct { - desc string - // Whether or not a default target is configured. - defaultPickExists bool - // Whether or not the RLS request should be throttled. - throttle bool - // Whether or not the test is expected to make a new RLS request. - wantRLSRequest bool - // Expected error returned by the rlsPicker under test. - wantErr error - }{ - { - desc: "rls request throttled with default pick", - defaultPickExists: true, - throttle: true, - }, - { - desc: "rls request throttled without default pick", - throttle: true, - wantErr: errRLSThrottled, - }, - { - desc: "rls request not throttled", - wantRLSRequest: true, - wantErr: balancer.ErrNoSubConnAvailable, - }, - } - - for _, test := range tests { - t.Run(test.desc, func(t *testing.T) { - rlsCh := testutils.NewChannel() - defaultPicker := newFakePicker() - - p := rlsPicker{ - kbm: kbm, - // Cache lookup fails, no pending entry. - readCache: func(key cache.Key) (*cache.Entry, bool) { - if !cmp.Equal(key, wantKey) { - t.Fatalf("cache lookup using cacheKey %v, want %v", key, wantKey) - } - return nil, false - }, - shouldThrottle: func() bool { return test.throttle }, - startRLS: func(path string, km keys.KeyMap) { - if !test.wantRLSRequest { - rlsCh.Send(errors.New("RLS request attempted when none was expected")) - return - } - if path != rpcPath { - rlsCh.Send(fmt.Errorf("RLS request initiated for rpcPath %s, want %s", path, rpcPath)) - return - } - if km.Str != wantKeyMapStr { - rlsCh.Send(fmt.Errorf("RLS request initiated with keys %v, want %v", km.Str, wantKeyMapStr)) - return - } - rlsCh.Send(nil) - }, - } - if test.defaultPickExists { - p.defaultPick = defaultPicker.Pick - } - - gotResult, err := p.Pick(balancer.PickInfo{ - FullMethodName: rpcPath, - Ctx: metadata.NewOutgoingContext(context.Background(), md), - }) - if err != test.wantErr { - t.Fatalf("Pick() returned error {%v}, want {%v}", err, test.wantErr) - } - // If the test specified that a new RLS request should be made, - // verify it. - if test.wantRLSRequest { - if rlsErr, err := rlsCh.Receive(); err != nil || rlsErr != nil { - t.Fatalf("startRLS() = %v, error receiving from channel: %v", rlsErr, err) - } - } - if test.wantErr != nil { - return - } - - // We get here only for cases where we expect the pick to be - // delegated to the default picker. - if err := verifySubConn(gotResult.SubConn, defaultPicker.id); err != nil { - t.Fatal(err) - } - }) - } -} - -// TestPick_DataCacheMiss_PendingCacheMiss verifies different Pick scenarios -// where the entry is not found in the data cache, but there is a entry in the -// pending cache. For all of these scenarios, no new RLS request will be sent. -func TestPick_DataCacheMiss_PendingCacheHit(t *testing.T) { - const ( - rpcPath = "/gFoo/method" - wantKeyMapStr = "k1=v1" - ) - kbm, err := initKeyBuilderMap() - if err != nil { - t.Fatalf("Failed to create keyBuilderMap: %v", err) - } - md := metadata.New(map[string]string{"n1": "v1", "n3": "v3"}) - wantKey := cache.Key{Path: rpcPath, KeyMap: wantKeyMapStr} - - tests := []struct { - desc string - defaultPickExists bool - }{ - { - desc: "default pick exists", - defaultPickExists: true, - }, - { - desc: "default pick does not exists", - }, - } - for _, test := range tests { - t.Run(test.desc, func(t *testing.T) { - rlsCh := testutils.NewChannel() - p := rlsPicker{ - kbm: kbm, - // Cache lookup fails, pending entry exists. - readCache: func(key cache.Key) (*cache.Entry, bool) { - if !cmp.Equal(key, wantKey) { - t.Fatalf("cache lookup using cacheKey %v, want %v", key, wantKey) - } - return nil, true - }, - // Never throttle. We do not expect an RLS request to be sent out anyways. - shouldThrottle: func() bool { return false }, - startRLS: func(_ string, _ keys.KeyMap) { - rlsCh.Send(nil) - }, - } - if test.defaultPickExists { - p.defaultPick = func(info balancer.PickInfo) (balancer.PickResult, error) { - // We do not expect the default picker to be invoked at all. - // So, if we get here, the test will fail, because it - // expects the pick to be queued. - return balancer.PickResult{}, nil - } - } - - if _, err := p.Pick(balancer.PickInfo{ - FullMethodName: rpcPath, - Ctx: metadata.NewOutgoingContext(context.Background(), md), - }); err != balancer.ErrNoSubConnAvailable { - t.Fatalf("Pick() returned error {%v}, want {%v}", err, balancer.ErrNoSubConnAvailable) - } - - // Make sure that no RLS request was sent out. - if _, err := rlsCh.Receive(); err != testutils.ErrRecvTimeout { - t.Fatalf("RLS request sent out when pending entry exists") - } - }) - } -} - -// TestPick_DataCacheHit_PendingCacheMiss verifies different Pick scenarios -// where the entry is found in the data cache, and there is no entry in the -// pending cache. This includes cases where the entry in the data cache is -// stale, expired or in backoff. -func TestPick_DataCacheHit_PendingCacheMiss(t *testing.T) { - const ( - rpcPath = "/gFoo/method" - wantKeyMapStr = "k1=v1" - ) - kbm, err := initKeyBuilderMap() - if err != nil { - t.Fatalf("Failed to create keyBuilderMap: %v", err) - } - md := metadata.New(map[string]string{"n1": "v1", "n3": "v3"}) - wantKey := cache.Key{Path: rpcPath, KeyMap: wantKeyMapStr} - rlsLastErr := errors.New("last RLS request failed") - - tests := []struct { - desc string - // The cache entry, as returned by the overridden readCache hook. - cacheEntry *cache.Entry - // Whether or not a default target is configured. - defaultPickExists bool - // Whether or not the RLS request should be throttled. - throttle bool - // Whether or not the test is expected to make a new RLS request. - wantRLSRequest bool - // Whether or not the rlsPicker should delegate to the child picker. - wantChildPick bool - // Whether or not the rlsPicker should delegate to the default picker. - wantDefaultPick bool - // Expected error returned by the rlsPicker under test. - wantErr error - }{ - { - desc: "valid entry", - cacheEntry: &cache.Entry{ - ExpiryTime: time.Now().Add(defaultTestMaxAge), - StaleTime: time.Now().Add(defaultTestMaxAge), - }, - wantChildPick: true, - }, - { - desc: "entryStale_requestThrottled", - cacheEntry: &cache.Entry{ExpiryTime: time.Now().Add(defaultTestMaxAge)}, - throttle: true, - wantChildPick: true, - }, - { - desc: "entryStale_requestNotThrottled", - cacheEntry: &cache.Entry{ExpiryTime: time.Now().Add(defaultTestMaxAge)}, - wantRLSRequest: true, - wantChildPick: true, - }, - { - desc: "entryExpired_requestThrottled_defaultPickExists", - cacheEntry: &cache.Entry{}, - throttle: true, - defaultPickExists: true, - wantDefaultPick: true, - }, - { - desc: "entryExpired_requestThrottled_defaultPickNotExists", - cacheEntry: &cache.Entry{}, - throttle: true, - wantErr: errRLSThrottled, - }, - { - desc: "entryExpired_requestNotThrottled", - cacheEntry: &cache.Entry{}, - wantRLSRequest: true, - wantErr: balancer.ErrNoSubConnAvailable, - }, - { - desc: "entryExpired_backoffNotExpired_defaultPickExists", - cacheEntry: &cache.Entry{ - BackoffTime: time.Now().Add(defaultTestMaxAge), - CallStatus: rlsLastErr, - }, - defaultPickExists: true, - }, - { - desc: "entryExpired_backoffNotExpired_defaultPickNotExists", - cacheEntry: &cache.Entry{ - BackoffTime: time.Now().Add(defaultTestMaxAge), - CallStatus: rlsLastErr, - }, - wantErr: rlsLastErr, - }, - } - - for _, test := range tests { - t.Run(test.desc, func(t *testing.T) { - rlsCh := testutils.NewChannel() - childPicker := newFakePicker() - defaultPicker := newFakePicker() - - p := rlsPicker{ - kbm: kbm, - readCache: func(key cache.Key) (*cache.Entry, bool) { - if !cmp.Equal(key, wantKey) { - t.Fatalf("cache lookup using cacheKey %v, want %v", key, wantKey) - } - test.cacheEntry.ChildPicker = childPicker - return test.cacheEntry, false - }, - shouldThrottle: func() bool { return test.throttle }, - startRLS: func(path string, km keys.KeyMap) { - if !test.wantRLSRequest { - rlsCh.Send(errors.New("RLS request attempted when none was expected")) - return - } - if path != rpcPath { - rlsCh.Send(fmt.Errorf("RLS request initiated for rpcPath %s, want %s", path, rpcPath)) - return - } - if km.Str != wantKeyMapStr { - rlsCh.Send(fmt.Errorf("RLS request initiated with keys %v, want %v", km.Str, wantKeyMapStr)) - return - } - rlsCh.Send(nil) - }, - } - if test.defaultPickExists { - p.defaultPick = defaultPicker.Pick - } - - gotResult, err := p.Pick(balancer.PickInfo{ - FullMethodName: rpcPath, - Ctx: metadata.NewOutgoingContext(context.Background(), md), - }) - if err != test.wantErr { - t.Fatalf("Pick() returned error {%v}, want {%v}", err, test.wantErr) - } - // If the test specified that a new RLS request should be made, - // verify it. - if test.wantRLSRequest { - if rlsErr, err := rlsCh.Receive(); err != nil || rlsErr != nil { - t.Fatalf("startRLS() = %v, error receiving from channel: %v", rlsErr, err) - } - } - if test.wantErr != nil { - return - } - - // We get here only for cases where we expect the pick to be - // delegated to the child picker or the default picker. - if test.wantChildPick { - if err := verifySubConn(gotResult.SubConn, childPicker.id); err != nil { - t.Fatal(err) - } - - } - if test.wantDefaultPick { - if err := verifySubConn(gotResult.SubConn, defaultPicker.id); err != nil { - t.Fatal(err) - } - } - }) - } -} - -// TestPick_DataCacheHit_PendingCacheHit verifies different Pick scenarios where -// the entry is found both in the data cache and in the pending cache. This -// mostly verifies cases where the entry is stale, but there is already a -// pending RLS request, so no new request should be sent out. -func TestPick_DataCacheHit_PendingCacheHit(t *testing.T) { - const ( - rpcPath = "/gFoo/method" - wantKeyMapStr = "k1=v1" - ) - kbm, err := initKeyBuilderMap() - if err != nil { - t.Fatalf("Failed to create keyBuilderMap: %v", err) - } - md := metadata.New(map[string]string{"n1": "v1", "n3": "v3"}) - wantKey := cache.Key{Path: rpcPath, KeyMap: wantKeyMapStr} - - tests := []struct { - desc string - // The cache entry, as returned by the overridden readCache hook. - cacheEntry *cache.Entry - // Whether or not a default target is configured. - defaultPickExists bool - // Expected error returned by the rlsPicker under test. - wantErr error - }{ - { - desc: "stale entry", - cacheEntry: &cache.Entry{ExpiryTime: time.Now().Add(defaultTestMaxAge)}, - }, - { - desc: "stale entry with default picker", - cacheEntry: &cache.Entry{ExpiryTime: time.Now().Add(defaultTestMaxAge)}, - defaultPickExists: true, - }, - { - desc: "entryExpired_defaultPickExists", - cacheEntry: &cache.Entry{}, - defaultPickExists: true, - wantErr: balancer.ErrNoSubConnAvailable, - }, - { - desc: "entryExpired_defaultPickNotExists", - cacheEntry: &cache.Entry{}, - wantErr: balancer.ErrNoSubConnAvailable, - }, - } - for _, test := range tests { - t.Run(test.desc, func(t *testing.T) { - rlsCh := testutils.NewChannel() - childPicker := newFakePicker() - - p := rlsPicker{ - kbm: kbm, - readCache: func(key cache.Key) (*cache.Entry, bool) { - if !cmp.Equal(key, wantKey) { - t.Fatalf("cache lookup using cacheKey %v, want %v", key, wantKey) - } - test.cacheEntry.ChildPicker = childPicker - return test.cacheEntry, true - }, - // Never throttle. We do not expect an RLS request to be sent out anyways. - shouldThrottle: func() bool { return false }, - startRLS: func(path string, km keys.KeyMap) { - rlsCh.Send(nil) - }, - } - if test.defaultPickExists { - p.defaultPick = func(info balancer.PickInfo) (balancer.PickResult, error) { - // We do not expect the default picker to be invoked at all. - // So, if we get here, we return an error. - return balancer.PickResult{}, errors.New("default picker invoked when expecting a child pick") - } - } - - gotResult, err := p.Pick(balancer.PickInfo{ - FullMethodName: rpcPath, - Ctx: metadata.NewOutgoingContext(context.Background(), md), - }) - if err != test.wantErr { - t.Fatalf("Pick() returned error {%v}, want {%v}", err, test.wantErr) - } - // Make sure that no RLS request was sent out. - if _, err := rlsCh.Receive(); err != testutils.ErrRecvTimeout { - t.Fatalf("RLS request sent out when pending entry exists") - } - if test.wantErr != nil { - return - } - - // We get here only for cases where we expect the pick to be - // delegated to the child picker. - if err := verifySubConn(gotResult.SubConn, childPicker.id); err != nil { - t.Fatal(err) - } - }) - } -} diff --git a/balancer/rls/internal/proto/grpc_lookup_v1/rls.pb.go b/balancer/rls/internal/proto/grpc_lookup_v1/rls.pb.go deleted file mode 100644 index a3a3f8118ce9..000000000000 --- a/balancer/rls/internal/proto/grpc_lookup_v1/rls.pb.go +++ /dev/null @@ -1,264 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. -// source: grpc/lookup/v1/rls.proto - -package grpc_lookup_v1 - -import ( - context "context" - fmt "fmt" - proto "github.com/golang/protobuf/proto" - grpc "google.golang.org/grpc" - codes "google.golang.org/grpc/codes" - status "google.golang.org/grpc/status" - math "math" -) - -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package - -type RouteLookupRequest struct { - // Full host name of the target server, e.g. firestore.googleapis.com. - // Only set for gRPC requests; HTTP requests must use key_map explicitly. - Server string `protobuf:"bytes,1,opt,name=server,proto3" json:"server,omitempty"` - // Full path of the request, i.e. "/service/method". - // Only set for gRPC requests; HTTP requests must use key_map explicitly. - Path string `protobuf:"bytes,2,opt,name=path,proto3" json:"path,omitempty"` - // Target type allows the client to specify what kind of target format it - // would like from RLS to allow it to find the regional server, e.g. "grpc". - TargetType string `protobuf:"bytes,3,opt,name=target_type,json=targetType,proto3" json:"target_type,omitempty"` - // Map of key values extracted via key builders for the gRPC or HTTP request. - KeyMap map[string]string `protobuf:"bytes,4,rep,name=key_map,json=keyMap,proto3" json:"key_map,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *RouteLookupRequest) Reset() { *m = RouteLookupRequest{} } -func (m *RouteLookupRequest) String() string { return proto.CompactTextString(m) } -func (*RouteLookupRequest) ProtoMessage() {} -func (*RouteLookupRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_3bab962d3362f3ca, []int{0} -} - -func (m *RouteLookupRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_RouteLookupRequest.Unmarshal(m, b) -} -func (m *RouteLookupRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_RouteLookupRequest.Marshal(b, m, deterministic) -} -func (m *RouteLookupRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_RouteLookupRequest.Merge(m, src) -} -func (m *RouteLookupRequest) XXX_Size() int { - return xxx_messageInfo_RouteLookupRequest.Size(m) -} -func (m *RouteLookupRequest) XXX_DiscardUnknown() { - xxx_messageInfo_RouteLookupRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_RouteLookupRequest proto.InternalMessageInfo - -func (m *RouteLookupRequest) GetServer() string { - if m != nil { - return m.Server - } - return "" -} - -func (m *RouteLookupRequest) GetPath() string { - if m != nil { - return m.Path - } - return "" -} - -func (m *RouteLookupRequest) GetTargetType() string { - if m != nil { - return m.TargetType - } - return "" -} - -func (m *RouteLookupRequest) GetKeyMap() map[string]string { - if m != nil { - return m.KeyMap - } - return nil -} - -type RouteLookupResponse struct { - // Prioritized list (best one first) of addressable entities to use - // for routing, using syntax requested by the request target_type. - // The targets will be tried in order until a healthy one is found. - Targets []string `protobuf:"bytes,3,rep,name=targets,proto3" json:"targets,omitempty"` - // Optional header value to pass along to AFE in the X-Google-RLS-Data header. - // Cached with "target" and sent with all requests that match the request key. - // Allows the RLS to pass its work product to the eventual target. - HeaderData string `protobuf:"bytes,2,opt,name=header_data,json=headerData,proto3" json:"header_data,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *RouteLookupResponse) Reset() { *m = RouteLookupResponse{} } -func (m *RouteLookupResponse) String() string { return proto.CompactTextString(m) } -func (*RouteLookupResponse) ProtoMessage() {} -func (*RouteLookupResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_3bab962d3362f3ca, []int{1} -} - -func (m *RouteLookupResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_RouteLookupResponse.Unmarshal(m, b) -} -func (m *RouteLookupResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_RouteLookupResponse.Marshal(b, m, deterministic) -} -func (m *RouteLookupResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_RouteLookupResponse.Merge(m, src) -} -func (m *RouteLookupResponse) XXX_Size() int { - return xxx_messageInfo_RouteLookupResponse.Size(m) -} -func (m *RouteLookupResponse) XXX_DiscardUnknown() { - xxx_messageInfo_RouteLookupResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_RouteLookupResponse proto.InternalMessageInfo - -func (m *RouteLookupResponse) GetTargets() []string { - if m != nil { - return m.Targets - } - return nil -} - -func (m *RouteLookupResponse) GetHeaderData() string { - if m != nil { - return m.HeaderData - } - return "" -} - -func init() { - proto.RegisterType((*RouteLookupRequest)(nil), "grpc.lookup.v1.RouteLookupRequest") - proto.RegisterMapType((map[string]string)(nil), "grpc.lookup.v1.RouteLookupRequest.KeyMapEntry") - proto.RegisterType((*RouteLookupResponse)(nil), "grpc.lookup.v1.RouteLookupResponse") -} - -func init() { proto.RegisterFile("grpc/lookup/v1/rls.proto", fileDescriptor_3bab962d3362f3ca) } - -var fileDescriptor_3bab962d3362f3ca = []byte{ - // 341 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x92, 0xd1, 0x4b, 0xeb, 0x30, - 0x14, 0xc6, 0x6f, 0xd7, 0xdd, 0x6e, 0x3b, 0x85, 0xcb, 0x6e, 0xee, 0x45, 0xca, 0x5e, 0x1c, 0xf5, - 0x65, 0x0f, 0x92, 0xb2, 0xf9, 0xa2, 0x3e, 0x0e, 0x45, 0x50, 0x07, 0xa3, 0xfa, 0x20, 0x3e, 0x58, - 0xe2, 0x76, 0xc8, 0x46, 0x6b, 0x13, 0x93, 0xb4, 0xd0, 0x3f, 0xd8, 0xff, 0x43, 0xda, 0x54, 0xd8, - 0x14, 0xf4, 0xed, 0xfb, 0xbe, 0x73, 0x48, 0x7e, 0x27, 0x39, 0x10, 0x70, 0x25, 0x57, 0x51, 0x26, - 0x44, 0x5a, 0xc8, 0xa8, 0x9c, 0x46, 0x2a, 0xd3, 0x54, 0x2a, 0x61, 0x04, 0xf9, 0x53, 0x57, 0xa8, - 0xad, 0xd0, 0x72, 0x1a, 0xbe, 0x39, 0x40, 0x62, 0x51, 0x18, 0xbc, 0x6d, 0xa2, 0x18, 0x5f, 0x0b, - 0xd4, 0x86, 0x1c, 0x80, 0xa7, 0x51, 0x95, 0xa8, 0x02, 0x67, 0xec, 0x4c, 0x06, 0x71, 0xeb, 0x08, - 0x81, 0xae, 0x64, 0x66, 0x13, 0x74, 0x9a, 0xb4, 0xd1, 0xe4, 0x10, 0x7c, 0xc3, 0x14, 0x47, 0x93, - 0x98, 0x4a, 0x62, 0xe0, 0x36, 0x25, 0xb0, 0xd1, 0x7d, 0x25, 0x91, 0x5c, 0x41, 0x2f, 0xc5, 0x2a, - 0x79, 0x61, 0x32, 0xe8, 0x8e, 0xdd, 0x89, 0x3f, 0xa3, 0x74, 0x9f, 0x82, 0x7e, 0x25, 0xa0, 0x37, - 0x58, 0x2d, 0x98, 0xbc, 0xcc, 0x8d, 0xaa, 0x62, 0x2f, 0x6d, 0xcc, 0xe8, 0x0c, 0xfc, 0x9d, 0x98, - 0x0c, 0xc1, 0x4d, 0xb1, 0x6a, 0x09, 0x6b, 0x49, 0xfe, 0xc3, 0xef, 0x92, 0x65, 0x05, 0xb6, 0x7c, - 0xd6, 0x9c, 0x77, 0x4e, 0x9d, 0xf0, 0x09, 0xfe, 0xed, 0x5d, 0xa2, 0xa5, 0xc8, 0x35, 0x92, 0x00, - 0x7a, 0x16, 0x54, 0x07, 0xee, 0xd8, 0x9d, 0x0c, 0xe2, 0x0f, 0x5b, 0x4f, 0xb5, 0x41, 0xb6, 0x46, - 0x95, 0xac, 0x99, 0x61, 0xed, 0x81, 0x60, 0xa3, 0x0b, 0x66, 0xd8, 0x75, 0xb7, 0xef, 0x0c, 0x3b, - 0xb1, 0x67, 0xfb, 0x67, 0xf9, 0xde, 0x33, 0xde, 0xa1, 0x2a, 0xb7, 0x2b, 0x24, 0x0f, 0xe0, 0xef, - 0xa4, 0x24, 0xfc, 0x79, 0xee, 0xd1, 0xd1, 0xb7, 0x3d, 0x16, 0x3b, 0xfc, 0x35, 0x5f, 0xc0, 0xdf, - 0xad, 0xf8, 0xd4, 0x3a, 0xef, 0xc7, 0x99, 0x5e, 0xd6, 0xdf, 0xbc, 0x74, 0x1e, 0x8f, 0xb9, 0x10, - 0x3c, 0x43, 0xca, 0x45, 0xc6, 0x72, 0x4e, 0x85, 0xe2, 0xd1, 0xee, 0x52, 0xd4, 0x3a, 0xb1, 0x3a, - 0x29, 0xa7, 0xcf, 0x5e, 0xb3, 0x1d, 0x27, 0xef, 0x01, 0x00, 0x00, 0xff, 0xff, 0xca, 0x8d, 0x5c, - 0xc7, 0x39, 0x02, 0x00, 0x00, -} - -// Reference imports to suppress errors if they are not otherwise used. -var _ context.Context -var _ grpc.ClientConnInterface - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the grpc package it is being compiled against. -const _ = grpc.SupportPackageIsVersion6 - -// RouteLookupServiceClient is the client API for RouteLookupService service. -// -// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. -type RouteLookupServiceClient interface { - // Lookup returns a target for a single key. - RouteLookup(ctx context.Context, in *RouteLookupRequest, opts ...grpc.CallOption) (*RouteLookupResponse, error) -} - -type routeLookupServiceClient struct { - cc grpc.ClientConnInterface -} - -func NewRouteLookupServiceClient(cc grpc.ClientConnInterface) RouteLookupServiceClient { - return &routeLookupServiceClient{cc} -} - -func (c *routeLookupServiceClient) RouteLookup(ctx context.Context, in *RouteLookupRequest, opts ...grpc.CallOption) (*RouteLookupResponse, error) { - out := new(RouteLookupResponse) - err := c.cc.Invoke(ctx, "/grpc.lookup.v1.RouteLookupService/RouteLookup", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -// RouteLookupServiceServer is the server API for RouteLookupService service. -type RouteLookupServiceServer interface { - // Lookup returns a target for a single key. - RouteLookup(context.Context, *RouteLookupRequest) (*RouteLookupResponse, error) -} - -// UnimplementedRouteLookupServiceServer can be embedded to have forward compatible implementations. -type UnimplementedRouteLookupServiceServer struct { -} - -func (*UnimplementedRouteLookupServiceServer) RouteLookup(ctx context.Context, req *RouteLookupRequest) (*RouteLookupResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method RouteLookup not implemented") -} - -func RegisterRouteLookupServiceServer(s *grpc.Server, srv RouteLookupServiceServer) { - s.RegisterService(&_RouteLookupService_serviceDesc, srv) -} - -func _RouteLookupService_RouteLookup_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(RouteLookupRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(RouteLookupServiceServer).RouteLookup(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/grpc.lookup.v1.RouteLookupService/RouteLookup", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(RouteLookupServiceServer).RouteLookup(ctx, req.(*RouteLookupRequest)) - } - return interceptor(ctx, in, info, handler) -} - -var _RouteLookupService_serviceDesc = grpc.ServiceDesc{ - ServiceName: "grpc.lookup.v1.RouteLookupService", - HandlerType: (*RouteLookupServiceServer)(nil), - Methods: []grpc.MethodDesc{ - { - MethodName: "RouteLookup", - Handler: _RouteLookupService_RouteLookup_Handler, - }, - }, - Streams: []grpc.StreamDesc{}, - Metadata: "grpc/lookup/v1/rls.proto", -} diff --git a/balancer/rls/internal/proto/grpc_lookup_v1/rls_config.pb.go b/balancer/rls/internal/proto/grpc_lookup_v1/rls_config.pb.go deleted file mode 100644 index 01d5b656ebe2..000000000000 --- a/balancer/rls/internal/proto/grpc_lookup_v1/rls_config.pb.go +++ /dev/null @@ -1,505 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. -// source: grpc/lookup/v1/rls_config.proto - -package grpc_lookup_v1 - -import ( - fmt "fmt" - proto "github.com/golang/protobuf/proto" - duration "github.com/golang/protobuf/ptypes/duration" - math "math" -) - -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package - -// Extract a key based on a given name (e.g. header name or query parameter -// name). The name must match one of the names listed in the "name" field. If -// the "required_match" field is true, one of the specified names must be -// present for the keybuilder to match. -type NameMatcher struct { - // The name that will be used in the RLS key_map to refer to this value. - Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` - // Ordered list of names (headers or query parameter names) that can supply - // this value; the first one with a non-empty value is used. - Names []string `protobuf:"bytes,2,rep,name=names,proto3" json:"names,omitempty"` - // If true, make this extraction required; the key builder will not match - // if no value is found. - RequiredMatch bool `protobuf:"varint,3,opt,name=required_match,json=requiredMatch,proto3" json:"required_match,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *NameMatcher) Reset() { *m = NameMatcher{} } -func (m *NameMatcher) String() string { return proto.CompactTextString(m) } -func (*NameMatcher) ProtoMessage() {} -func (*NameMatcher) Descriptor() ([]byte, []int) { - return fileDescriptor_5fe74d4f6e8f01c1, []int{0} -} - -func (m *NameMatcher) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_NameMatcher.Unmarshal(m, b) -} -func (m *NameMatcher) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_NameMatcher.Marshal(b, m, deterministic) -} -func (m *NameMatcher) XXX_Merge(src proto.Message) { - xxx_messageInfo_NameMatcher.Merge(m, src) -} -func (m *NameMatcher) XXX_Size() int { - return xxx_messageInfo_NameMatcher.Size(m) -} -func (m *NameMatcher) XXX_DiscardUnknown() { - xxx_messageInfo_NameMatcher.DiscardUnknown(m) -} - -var xxx_messageInfo_NameMatcher proto.InternalMessageInfo - -func (m *NameMatcher) GetKey() string { - if m != nil { - return m.Key - } - return "" -} - -func (m *NameMatcher) GetNames() []string { - if m != nil { - return m.Names - } - return nil -} - -func (m *NameMatcher) GetRequiredMatch() bool { - if m != nil { - return m.RequiredMatch - } - return false -} - -// A GrpcKeyBuilder applies to a given gRPC service, name, and headers. -type GrpcKeyBuilder struct { - Names []*GrpcKeyBuilder_Name `protobuf:"bytes,1,rep,name=names,proto3" json:"names,omitempty"` - // Extract keys from all listed headers. - // For gRPC, it is an error to specify "required_match" on the NameMatcher - // protos. - Headers []*NameMatcher `protobuf:"bytes,2,rep,name=headers,proto3" json:"headers,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *GrpcKeyBuilder) Reset() { *m = GrpcKeyBuilder{} } -func (m *GrpcKeyBuilder) String() string { return proto.CompactTextString(m) } -func (*GrpcKeyBuilder) ProtoMessage() {} -func (*GrpcKeyBuilder) Descriptor() ([]byte, []int) { - return fileDescriptor_5fe74d4f6e8f01c1, []int{1} -} - -func (m *GrpcKeyBuilder) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_GrpcKeyBuilder.Unmarshal(m, b) -} -func (m *GrpcKeyBuilder) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_GrpcKeyBuilder.Marshal(b, m, deterministic) -} -func (m *GrpcKeyBuilder) XXX_Merge(src proto.Message) { - xxx_messageInfo_GrpcKeyBuilder.Merge(m, src) -} -func (m *GrpcKeyBuilder) XXX_Size() int { - return xxx_messageInfo_GrpcKeyBuilder.Size(m) -} -func (m *GrpcKeyBuilder) XXX_DiscardUnknown() { - xxx_messageInfo_GrpcKeyBuilder.DiscardUnknown(m) -} - -var xxx_messageInfo_GrpcKeyBuilder proto.InternalMessageInfo - -func (m *GrpcKeyBuilder) GetNames() []*GrpcKeyBuilder_Name { - if m != nil { - return m.Names - } - return nil -} - -func (m *GrpcKeyBuilder) GetHeaders() []*NameMatcher { - if m != nil { - return m.Headers - } - return nil -} - -// To match, one of the given Name fields must match; the service and method -// fields are specified as fixed strings. The service name is required and -// includes the proto package name. The method name may be omitted, in -// which case any method on the given service is matched. -type GrpcKeyBuilder_Name struct { - Service string `protobuf:"bytes,1,opt,name=service,proto3" json:"service,omitempty"` - Method string `protobuf:"bytes,2,opt,name=method,proto3" json:"method,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *GrpcKeyBuilder_Name) Reset() { *m = GrpcKeyBuilder_Name{} } -func (m *GrpcKeyBuilder_Name) String() string { return proto.CompactTextString(m) } -func (*GrpcKeyBuilder_Name) ProtoMessage() {} -func (*GrpcKeyBuilder_Name) Descriptor() ([]byte, []int) { - return fileDescriptor_5fe74d4f6e8f01c1, []int{1, 0} -} - -func (m *GrpcKeyBuilder_Name) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_GrpcKeyBuilder_Name.Unmarshal(m, b) -} -func (m *GrpcKeyBuilder_Name) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_GrpcKeyBuilder_Name.Marshal(b, m, deterministic) -} -func (m *GrpcKeyBuilder_Name) XXX_Merge(src proto.Message) { - xxx_messageInfo_GrpcKeyBuilder_Name.Merge(m, src) -} -func (m *GrpcKeyBuilder_Name) XXX_Size() int { - return xxx_messageInfo_GrpcKeyBuilder_Name.Size(m) -} -func (m *GrpcKeyBuilder_Name) XXX_DiscardUnknown() { - xxx_messageInfo_GrpcKeyBuilder_Name.DiscardUnknown(m) -} - -var xxx_messageInfo_GrpcKeyBuilder_Name proto.InternalMessageInfo - -func (m *GrpcKeyBuilder_Name) GetService() string { - if m != nil { - return m.Service - } - return "" -} - -func (m *GrpcKeyBuilder_Name) GetMethod() string { - if m != nil { - return m.Method - } - return "" -} - -// An HttpKeyBuilder applies to a given HTTP URL and headers. -// -// Path and host patterns use the matching syntax from gRPC transcoding to -// extract named key/value pairs from the path and host components of the URL: -// https://github.com/googleapis/googleapis/blob/master/google/api/http.proto -// -// It is invalid to specify the same key name in multiple places in a pattern. -// -// For a service where the project id can be expressed either as a subdomain or -// in the path, separate HttpKeyBuilders must be used: -// host_pattern: 'example.com' path_pattern: '/{id}/{object}/**' -// host_pattern: '{id}.example.com' path_pattern: '/{object}/**' -// If the host is exactly 'example.com', the first path segment will be used as -// the id and the second segment as the object. If the host has a subdomain, the -// subdomain will be used as the id and the first segment as the object. If -// neither pattern matches, no keys will be extracted. -type HttpKeyBuilder struct { - // host_pattern is an ordered list of host template patterns for the desired - // value. If any host_pattern values are specified, then at least one must - // match, and the last one wins and sets any specified variables. A host - // consists of labels separated by dots. Each label is matched against the - // label in the pattern as follows: - // - "*": Matches any single label. - // - "**": Matches zero or more labels (first or last part of host only). - // - "{=...}": One or more label capture, where "..." can be any - // template that does not include a capture. - // - "{}": A single label capture. Identical to {=*}. - // - // Examples: - // - "example.com": Only applies to the exact host example.com. - // - "*.example.com": Matches subdomains of example.com. - // - "**.example.com": matches example.com, and all levels of subdomains. - // - "{project}.example.com": Extracts the third level subdomain. - // - "{project=**}.example.com": Extracts the third level+ subdomains. - // - "{project=**}": Extracts the entire host. - HostPatterns []string `protobuf:"bytes,1,rep,name=host_patterns,json=hostPatterns,proto3" json:"host_patterns,omitempty"` - // path_pattern is an ordered list of path template patterns for the desired - // value. If any path_pattern values are specified, then at least one must - // match, and the last one wins and sets any specified variables. A path - // consists of segments separated by slashes. Each segment is matched against - // the segment in the pattern as follows: - // - "*": Matches any single segment. - // - "**": Matches zero or more segments (first or last part of path only). - // - "{=...}": One or more segment capture, where "..." can be any - // template that does not include a capture. - // - "{}": A single segment capture. Identical to {=*}. - // A custom method may also be specified by appending ":" and the custom - // method name or "*" to indicate any custom method (including no custom - // method). For example, "/*/projects/{project_id}/**:*" extracts - // `{project_id}` for any version, resource and custom method that includes - // it. By default, any custom method will be matched. - // - // Examples: - // - "/v1/{name=messages/*}": extracts a name like "messages/12345". - // - "/v1/messages/{message_id}": extracts a message_id like "12345". - // - "/v1/users/{user_id}/messages/{message_id}": extracts two key values. - PathPatterns []string `protobuf:"bytes,2,rep,name=path_patterns,json=pathPatterns,proto3" json:"path_patterns,omitempty"` - // List of query parameter names to try to match. - // For example: ["parent", "name", "resource.name"] - // We extract all the specified query_parameters (case-sensitively). If any - // are marked as "required_match" and are not present, this keybuilder fails - // to match. If a given parameter appears multiple times (?foo=a&foo=b) we - // will report it as a comma-separated string (foo=a,b). - QueryParameters []*NameMatcher `protobuf:"bytes,3,rep,name=query_parameters,json=queryParameters,proto3" json:"query_parameters,omitempty"` - // List of headers to try to match. - // We extract all the specified header values (case-insensitively). If any - // are marked as "required_match" and are not present, this keybuilder fails - // to match. If a given header appears multiple times in the request we will - // report it as a comma-separated string, in standard HTTP fashion. - Headers []*NameMatcher `protobuf:"bytes,4,rep,name=headers,proto3" json:"headers,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *HttpKeyBuilder) Reset() { *m = HttpKeyBuilder{} } -func (m *HttpKeyBuilder) String() string { return proto.CompactTextString(m) } -func (*HttpKeyBuilder) ProtoMessage() {} -func (*HttpKeyBuilder) Descriptor() ([]byte, []int) { - return fileDescriptor_5fe74d4f6e8f01c1, []int{2} -} - -func (m *HttpKeyBuilder) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_HttpKeyBuilder.Unmarshal(m, b) -} -func (m *HttpKeyBuilder) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_HttpKeyBuilder.Marshal(b, m, deterministic) -} -func (m *HttpKeyBuilder) XXX_Merge(src proto.Message) { - xxx_messageInfo_HttpKeyBuilder.Merge(m, src) -} -func (m *HttpKeyBuilder) XXX_Size() int { - return xxx_messageInfo_HttpKeyBuilder.Size(m) -} -func (m *HttpKeyBuilder) XXX_DiscardUnknown() { - xxx_messageInfo_HttpKeyBuilder.DiscardUnknown(m) -} - -var xxx_messageInfo_HttpKeyBuilder proto.InternalMessageInfo - -func (m *HttpKeyBuilder) GetHostPatterns() []string { - if m != nil { - return m.HostPatterns - } - return nil -} - -func (m *HttpKeyBuilder) GetPathPatterns() []string { - if m != nil { - return m.PathPatterns - } - return nil -} - -func (m *HttpKeyBuilder) GetQueryParameters() []*NameMatcher { - if m != nil { - return m.QueryParameters - } - return nil -} - -func (m *HttpKeyBuilder) GetHeaders() []*NameMatcher { - if m != nil { - return m.Headers - } - return nil -} - -type RouteLookupConfig struct { - // Ordered specifications for constructing keys for HTTP requests. Last - // match wins. If no HttpKeyBuilder matches, an empty key_map will be sent to - // the lookup service; it should likely reply with a global default route - // and raise an alert. - HttpKeybuilders []*HttpKeyBuilder `protobuf:"bytes,1,rep,name=http_keybuilders,json=httpKeybuilders,proto3" json:"http_keybuilders,omitempty"` - // Unordered specifications for constructing keys for gRPC requests. All - // GrpcKeyBuilders on this list must have unique "name" fields so that the - // client is free to prebuild a hash map keyed by name. If no GrpcKeyBuilder - // matches, an empty key_map will be sent to the lookup service; it should - // likely reply with a global default route and raise an alert. - GrpcKeybuilders []*GrpcKeyBuilder `protobuf:"bytes,2,rep,name=grpc_keybuilders,json=grpcKeybuilders,proto3" json:"grpc_keybuilders,omitempty"` - // The name of the lookup service as a gRPC URI. Typically, this will be - // a subdomain of the target, such as "lookup.datastore.googleapis.com". - LookupService string `protobuf:"bytes,3,opt,name=lookup_service,json=lookupService,proto3" json:"lookup_service,omitempty"` - // Configure a timeout value for lookup service requests. - // Defaults to 10 seconds if not specified. - LookupServiceTimeout *duration.Duration `protobuf:"bytes,4,opt,name=lookup_service_timeout,json=lookupServiceTimeout,proto3" json:"lookup_service_timeout,omitempty"` - // How long are responses valid for (like HTTP Cache-Control). - // If omitted or zero, the longest valid cache time is used. - // This value is clamped to 5 minutes to avoid unflushable bad responses. - MaxAge *duration.Duration `protobuf:"bytes,5,opt,name=max_age,json=maxAge,proto3" json:"max_age,omitempty"` - // After a response has been in the client cache for this amount of time - // and is re-requested, start an asynchronous RPC to re-validate it. - // This value should be less than max_age by at least the length of a - // typical RTT to the Route Lookup Service to fully mask the RTT latency. - // If omitted, keys are only re-requested after they have expired. - StaleAge *duration.Duration `protobuf:"bytes,6,opt,name=stale_age,json=staleAge,proto3" json:"stale_age,omitempty"` - // Rough indicator of amount of memory to use for the client cache. Some of - // the data structure overhead is not accounted for, so actual memory consumed - // will be somewhat greater than this value. If this field is omitted or set - // to zero, a client default will be used. The value may be capped to a lower - // amount based on client configuration. - CacheSizeBytes int64 `protobuf:"varint,7,opt,name=cache_size_bytes,json=cacheSizeBytes,proto3" json:"cache_size_bytes,omitempty"` - // This is a list of all the possible targets that can be returned by the - // lookup service. If a target not on this list is returned, it will be - // treated the same as an unhealthy target. - ValidTargets []string `protobuf:"bytes,8,rep,name=valid_targets,json=validTargets,proto3" json:"valid_targets,omitempty"` - // This value provides a default target to use if needed. If set, it will be - // used if RLS returns an error, times out, or returns an invalid response. - // Note that requests can be routed only to a subdomain of the original - // target, e.g. "us_east_1.cloudbigtable.googleapis.com". - DefaultTarget string `protobuf:"bytes,9,opt,name=default_target,json=defaultTarget,proto3" json:"default_target,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *RouteLookupConfig) Reset() { *m = RouteLookupConfig{} } -func (m *RouteLookupConfig) String() string { return proto.CompactTextString(m) } -func (*RouteLookupConfig) ProtoMessage() {} -func (*RouteLookupConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_5fe74d4f6e8f01c1, []int{3} -} - -func (m *RouteLookupConfig) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_RouteLookupConfig.Unmarshal(m, b) -} -func (m *RouteLookupConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_RouteLookupConfig.Marshal(b, m, deterministic) -} -func (m *RouteLookupConfig) XXX_Merge(src proto.Message) { - xxx_messageInfo_RouteLookupConfig.Merge(m, src) -} -func (m *RouteLookupConfig) XXX_Size() int { - return xxx_messageInfo_RouteLookupConfig.Size(m) -} -func (m *RouteLookupConfig) XXX_DiscardUnknown() { - xxx_messageInfo_RouteLookupConfig.DiscardUnknown(m) -} - -var xxx_messageInfo_RouteLookupConfig proto.InternalMessageInfo - -func (m *RouteLookupConfig) GetHttpKeybuilders() []*HttpKeyBuilder { - if m != nil { - return m.HttpKeybuilders - } - return nil -} - -func (m *RouteLookupConfig) GetGrpcKeybuilders() []*GrpcKeyBuilder { - if m != nil { - return m.GrpcKeybuilders - } - return nil -} - -func (m *RouteLookupConfig) GetLookupService() string { - if m != nil { - return m.LookupService - } - return "" -} - -func (m *RouteLookupConfig) GetLookupServiceTimeout() *duration.Duration { - if m != nil { - return m.LookupServiceTimeout - } - return nil -} - -func (m *RouteLookupConfig) GetMaxAge() *duration.Duration { - if m != nil { - return m.MaxAge - } - return nil -} - -func (m *RouteLookupConfig) GetStaleAge() *duration.Duration { - if m != nil { - return m.StaleAge - } - return nil -} - -func (m *RouteLookupConfig) GetCacheSizeBytes() int64 { - if m != nil { - return m.CacheSizeBytes - } - return 0 -} - -func (m *RouteLookupConfig) GetValidTargets() []string { - if m != nil { - return m.ValidTargets - } - return nil -} - -func (m *RouteLookupConfig) GetDefaultTarget() string { - if m != nil { - return m.DefaultTarget - } - return "" -} - -func init() { - proto.RegisterType((*NameMatcher)(nil), "grpc.lookup.v1.NameMatcher") - proto.RegisterType((*GrpcKeyBuilder)(nil), "grpc.lookup.v1.GrpcKeyBuilder") - proto.RegisterType((*GrpcKeyBuilder_Name)(nil), "grpc.lookup.v1.GrpcKeyBuilder.Name") - proto.RegisterType((*HttpKeyBuilder)(nil), "grpc.lookup.v1.HttpKeyBuilder") - proto.RegisterType((*RouteLookupConfig)(nil), "grpc.lookup.v1.RouteLookupConfig") -} - -func init() { proto.RegisterFile("grpc/lookup/v1/rls_config.proto", fileDescriptor_5fe74d4f6e8f01c1) } - -var fileDescriptor_5fe74d4f6e8f01c1 = []byte{ - // 615 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0xed, 0x6a, 0xdb, 0x30, - 0x14, 0xc5, 0x75, 0x9a, 0x0f, 0xa5, 0x75, 0x53, 0x53, 0x8a, 0xd7, 0x42, 0x17, 0x52, 0x0a, 0xfe, - 0x31, 0x1c, 0x9a, 0xb1, 0xb1, 0xfd, 0x5c, 0x36, 0xf6, 0xfd, 0x11, 0xdc, 0xfe, 0x1a, 0x03, 0xa1, - 0xd8, 0xb7, 0xb2, 0xa9, 0x1d, 0xb9, 0x92, 0x1c, 0x9a, 0x3e, 0xd0, 0x9e, 0x62, 0x2f, 0xb2, 0xb7, - 0x19, 0x92, 0xec, 0x2e, 0x2e, 0x83, 0xec, 0x5f, 0xee, 0xd1, 0x39, 0x27, 0x3a, 0xf7, 0x5e, 0x19, - 0x3d, 0xa6, 0xbc, 0x88, 0xc6, 0x19, 0x63, 0xd7, 0x65, 0x31, 0x5e, 0x9e, 0x8f, 0x79, 0x26, 0x70, - 0xc4, 0x16, 0x57, 0x29, 0x0d, 0x0a, 0xce, 0x24, 0x73, 0x1d, 0x45, 0x08, 0x0c, 0x21, 0x58, 0x9e, - 0x1f, 0x9d, 0x50, 0xc6, 0x68, 0x06, 0x63, 0x7d, 0x3a, 0x2f, 0xaf, 0xc6, 0x71, 0xc9, 0x89, 0x4c, - 0xd9, 0xc2, 0xf0, 0x47, 0x3f, 0x50, 0xff, 0x2b, 0xc9, 0xe1, 0x0b, 0x91, 0x51, 0x02, 0xdc, 0x1d, - 0x20, 0xfb, 0x1a, 0x56, 0x9e, 0x35, 0xb4, 0xfc, 0x5e, 0xa8, 0x7e, 0xba, 0x07, 0x68, 0x7b, 0x41, - 0x72, 0x10, 0xde, 0xd6, 0xd0, 0xf6, 0x7b, 0xa1, 0x29, 0xdc, 0x33, 0xe4, 0x70, 0xb8, 0x29, 0x53, - 0x0e, 0x31, 0xce, 0x95, 0xd6, 0xb3, 0x87, 0x96, 0xdf, 0x0d, 0x77, 0x6b, 0x54, 0x1b, 0x8e, 0x7e, - 0x59, 0xc8, 0x79, 0xc7, 0x8b, 0xe8, 0x13, 0xac, 0xa6, 0x65, 0x9a, 0xc5, 0xc0, 0xdd, 0x97, 0xb5, - 0x9f, 0x35, 0xb4, 0xfd, 0xfe, 0xe4, 0x34, 0x68, 0x5e, 0x38, 0x68, 0xd2, 0x03, 0x75, 0xb9, 0xfa, - 0x4f, 0x9f, 0xa1, 0x4e, 0x02, 0x24, 0x06, 0x6e, 0x2e, 0xd3, 0x9f, 0x1c, 0x3f, 0x14, 0xaf, 0x45, - 0x09, 0x6b, 0xee, 0xd1, 0x0b, 0xd4, 0x52, 0xb8, 0xeb, 0xa1, 0x8e, 0x00, 0xbe, 0x4c, 0x23, 0xa8, - 0xf2, 0xd5, 0xa5, 0x7b, 0x88, 0xda, 0x39, 0xc8, 0x84, 0xc5, 0xde, 0x96, 0x3e, 0xa8, 0xaa, 0xd1, - 0x6f, 0x0b, 0x39, 0xef, 0xa5, 0x2c, 0xd6, 0xae, 0x7f, 0x8a, 0x76, 0x13, 0x26, 0x24, 0x2e, 0x88, - 0x94, 0xc0, 0x17, 0x26, 0x46, 0x2f, 0xdc, 0x51, 0xe0, 0xac, 0xc2, 0x14, 0xa9, 0x20, 0x32, 0xf9, - 0x4b, 0x32, 0xbd, 0xdb, 0x51, 0xe0, 0x3d, 0xe9, 0x2d, 0x1a, 0xdc, 0x94, 0xc0, 0x57, 0xb8, 0x20, - 0x9c, 0xe4, 0x20, 0x55, 0x2c, 0x7b, 0x73, 0xac, 0x3d, 0x2d, 0x9a, 0xdd, 0x6b, 0xd6, 0xbb, 0xd2, - 0xfa, 0xff, 0xae, 0x8c, 0x7e, 0xb6, 0xd0, 0x7e, 0xc8, 0x4a, 0x09, 0x9f, 0x35, 0xef, 0xb5, 0x5e, - 0x22, 0xf7, 0x03, 0x1a, 0x24, 0x52, 0x16, 0xf8, 0x1a, 0x56, 0x73, 0x93, 0xb8, 0x1e, 0xd4, 0xc9, - 0x43, 0xd7, 0x66, 0x63, 0xc2, 0xbd, 0xc4, 0xd4, 0xb5, 0x4c, 0x59, 0x29, 0x45, 0xc3, 0x6a, 0xeb, - 0xdf, 0x56, 0xcd, 0x99, 0x87, 0x7b, 0xd4, 0xd4, 0xf7, 0x56, 0x67, 0xc8, 0x31, 0x64, 0x5c, 0x0f, - 0xd0, 0xd6, 0x73, 0xda, 0x35, 0xe8, 0x45, 0x35, 0xc6, 0x6f, 0xe8, 0xb0, 0x49, 0xc3, 0x32, 0xcd, - 0x81, 0x95, 0xd2, 0x6b, 0x0d, 0x2d, 0xbf, 0x3f, 0x79, 0x14, 0x98, 0xc7, 0x10, 0xd4, 0x8f, 0x21, - 0x78, 0x53, 0x3d, 0x86, 0xf0, 0xa0, 0xe1, 0x74, 0x69, 0x64, 0xee, 0x04, 0x75, 0x72, 0x72, 0x8b, - 0x09, 0x05, 0x6f, 0x7b, 0x93, 0x43, 0x3b, 0x27, 0xb7, 0xaf, 0x28, 0xb8, 0xcf, 0x51, 0x4f, 0x48, - 0x92, 0x81, 0x56, 0xb5, 0x37, 0xa9, 0xba, 0x9a, 0xab, 0x74, 0x3e, 0x1a, 0x44, 0x24, 0x4a, 0x00, - 0x8b, 0xf4, 0x0e, 0xf0, 0x7c, 0x25, 0x41, 0x78, 0x9d, 0xa1, 0xe5, 0xdb, 0xa1, 0xa3, 0xf1, 0x8b, - 0xf4, 0x0e, 0xa6, 0x0a, 0x55, 0xdb, 0xb5, 0x24, 0x59, 0x1a, 0x63, 0x49, 0x38, 0x05, 0x29, 0xbc, - 0xae, 0xd9, 0x2e, 0x0d, 0x5e, 0x1a, 0x4c, 0xb5, 0x2c, 0x86, 0x2b, 0x52, 0x66, 0xb2, 0xa2, 0x79, - 0x3d, 0xd3, 0xb2, 0x0a, 0x35, 0xbc, 0x8f, 0xad, 0x2e, 0x1a, 0xf4, 0xc3, 0x63, 0xf5, 0x6a, 0x41, - 0x6d, 0x35, 0x67, 0x11, 0x08, 0x91, 0x2e, 0x28, 0x16, 0x92, 0x13, 0x09, 0x74, 0x35, 0xbd, 0x40, - 0xfb, 0x29, 0x7b, 0x30, 0xb1, 0xa9, 0x13, 0x66, 0xc2, 0xac, 0xcc, 0x4c, 0x65, 0x9a, 0x59, 0xdf, - 0x9f, 0x54, 0x19, 0x29, 0xcb, 0xc8, 0x82, 0x06, 0x8c, 0xd3, 0xf1, 0xfa, 0xb7, 0x4a, 0xaf, 0x42, - 0x35, 0x9d, 0xe5, 0xf9, 0xbc, 0xad, 0x5b, 0xf1, 0xf4, 0x4f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, - 0xaa, 0xbd, 0xb2, 0xd0, 0x04, 0x00, 0x00, -} diff --git a/balancer/rls/internal/proto/grpc_lookup_v1/rls_grpc.pb.go b/balancer/rls/internal/proto/grpc_lookup_v1/rls_grpc.pb.go deleted file mode 100644 index fe1027c71ec7..000000000000 --- a/balancer/rls/internal/proto/grpc_lookup_v1/rls_grpc.pb.go +++ /dev/null @@ -1,86 +0,0 @@ -// Code generated by protoc-gen-go-grpc. DO NOT EDIT. - -package grpc_lookup_v1 - -import ( - context "context" - grpc "google.golang.org/grpc" - codes "google.golang.org/grpc/codes" - status "google.golang.org/grpc/status" -) - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the grpc package it is being compiled against. -const _ = grpc.SupportPackageIsVersion7 - -// RouteLookupServiceService is the service API for RouteLookupService service. -// Fields should be assigned to their respective handler implementations only before -// RegisterRouteLookupServiceService is called. Any unassigned fields will result in the -// handler for that method returning an Unimplemented error. -type RouteLookupServiceService struct { - // Lookup returns a target for a single key. - RouteLookup func(context.Context, *RouteLookupRequest) (*RouteLookupResponse, error) -} - -func (s *RouteLookupServiceService) routeLookup(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - if s.RouteLookup == nil { - return nil, status.Errorf(codes.Unimplemented, "method RouteLookup not implemented") - } - in := new(RouteLookupRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return s.RouteLookup(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: s, - FullMethod: "/grpc.lookup.v1.RouteLookupService/RouteLookup", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return s.RouteLookup(ctx, req.(*RouteLookupRequest)) - } - return interceptor(ctx, in, info, handler) -} - -// RegisterRouteLookupServiceService registers a service implementation with a gRPC server. -func RegisterRouteLookupServiceService(s grpc.ServiceRegistrar, srv *RouteLookupServiceService) { - sd := grpc.ServiceDesc{ - ServiceName: "grpc.lookup.v1.RouteLookupService", - Methods: []grpc.MethodDesc{ - { - MethodName: "RouteLookup", - Handler: srv.routeLookup, - }, - }, - Streams: []grpc.StreamDesc{}, - Metadata: "grpc/lookup/v1/rls.proto", - } - - s.RegisterService(&sd, nil) -} - -// NewRouteLookupServiceService creates a new RouteLookupServiceService containing the -// implemented methods of the RouteLookupService service in s. Any unimplemented -// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client. -// This includes situations where the method handler is misspelled or has the wrong -// signature. For this reason, this function should be used with great care and -// is not recommended to be used by most users. -func NewRouteLookupServiceService(s interface{}) *RouteLookupServiceService { - ns := &RouteLookupServiceService{} - if h, ok := s.(interface { - RouteLookup(context.Context, *RouteLookupRequest) (*RouteLookupResponse, error) - }); ok { - ns.RouteLookup = h.RouteLookup - } - return ns -} - -// UnstableRouteLookupServiceService is the service API for RouteLookupService service. -// New methods may be added to this interface if they are added to the service -// definition, which is not a backward-compatible change. For this reason, -// use of this type is not recommended. -type UnstableRouteLookupServiceService interface { - // Lookup returns a target for a single key. - RouteLookup(context.Context, *RouteLookupRequest) (*RouteLookupResponse, error) -} diff --git a/xds/internal/balancer/xdsrouting/doc.go b/balancer/rls/internal/test/e2e/e2e.go similarity index 82% rename from xds/internal/balancer/xdsrouting/doc.go rename to balancer/rls/internal/test/e2e/e2e.go index 2f00649a0eb0..7b8a8bbde138 100644 --- a/xds/internal/balancer/xdsrouting/doc.go +++ b/balancer/rls/internal/test/e2e/e2e.go @@ -1,6 +1,6 @@ /* * - * Copyright 2020 gRPC authors. + * Copyright 2021 gRPC authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,5 +16,5 @@ * */ -// Package xdsrouting implements the routing balancer for xds. -package xdsrouting +// Package e2e contains utilities for end-to-end RouteLookupService tests. +package e2e diff --git a/balancer/rls/internal/test/e2e/rls_child_policy.go b/balancer/rls/internal/test/e2e/rls_child_policy.go new file mode 100644 index 000000000000..5a6e3e69175a --- /dev/null +++ b/balancer/rls/internal/test/e2e/rls_child_policy.go @@ -0,0 +1,131 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package e2e + +import ( + "encoding/json" + "errors" + "fmt" + + "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" +) + +const ( + // RLSChildPolicyTargetNameField is a top-level field name to add to the child + // policy's config, whose value is set to the target for the child policy. + RLSChildPolicyTargetNameField = "Backend" + // RLSChildPolicyBadTarget is a value which is considered a bad target by the + // child policy. This is useful to test bad child policy configuration. + RLSChildPolicyBadTarget = "bad-target" +) + +// ErrParseConfigBadTarget is the error returned from ParseConfig when the +// backend field is set to RLSChildPolicyBadTarget. +var ErrParseConfigBadTarget = errors.New("backend field set to RLSChildPolicyBadTarget") + +// BalancerFuncs is a set of callbacks which get invoked when the corresponding +// method on the child policy is invoked. +type BalancerFuncs struct { + UpdateClientConnState func(cfg *RLSChildPolicyConfig) error + Close func() +} + +// RegisterRLSChildPolicy registers a balancer builder with the given name, to +// be used as a child policy for the RLS LB policy. +// +// The child policy uses a pickfirst balancer under the hood to send all traffic +// to the single backend specified by the `RLSChildPolicyTargetNameField` field +// in its configuration which looks like: {"Backend": "Backend-address"}. +func RegisterRLSChildPolicy(name string, bf *BalancerFuncs) { + balancer.Register(bb{name: name, bf: bf}) +} + +type bb struct { + name string + bf *BalancerFuncs +} + +func (bb bb) Name() string { return bb.name } + +func (bb bb) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { + pf := balancer.Get(grpc.PickFirstBalancerName) + b := &bal{ + Balancer: pf.Build(cc, opts), + bf: bb.bf, + done: grpcsync.NewEvent(), + } + go b.run() + return b +} + +func (bb bb) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + cfg := &RLSChildPolicyConfig{} + if err := json.Unmarshal(c, cfg); err != nil { + return nil, err + } + if cfg.Backend == RLSChildPolicyBadTarget { + return nil, ErrParseConfigBadTarget + } + return cfg, nil +} + +type bal struct { + balancer.Balancer + bf *BalancerFuncs + done *grpcsync.Event +} + +// RLSChildPolicyConfig is the LB config for the test child policy. +type RLSChildPolicyConfig struct { + serviceconfig.LoadBalancingConfig + Backend string // The target for which this child policy was created. + Random string // A random field to test child policy config changes. +} + +func (b *bal) UpdateClientConnState(c balancer.ClientConnState) error { + cfg, ok := c.BalancerConfig.(*RLSChildPolicyConfig) + if !ok { + return fmt.Errorf("received balancer config of type %T, want %T", c.BalancerConfig, &RLSChildPolicyConfig{}) + } + if b.bf != nil && b.bf.UpdateClientConnState != nil { + b.bf.UpdateClientConnState(cfg) + } + return b.Balancer.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Addresses: []resolver.Address{{Addr: cfg.Backend}}}, + }) +} + +func (b *bal) Close() { + b.Balancer.Close() + if b.bf != nil && b.bf.Close != nil { + b.bf.Close() + } + b.done.Fire() +} + +// run is a dummy goroutine to make sure that child policies are closed at the +// end of tests. If they are not closed, these goroutines will be picked up by +// the leakcheker and tests will fail. +func (b *bal) run() { + <-b.done.Done() +} diff --git a/balancer/rls/internal/test/e2e/rls_lb_config.go b/balancer/rls/internal/test/e2e/rls_lb_config.go new file mode 100644 index 000000000000..0a5993d795c0 --- /dev/null +++ b/balancer/rls/internal/test/e2e/rls_lb_config.go @@ -0,0 +1,103 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package e2e + +import ( + "errors" + "fmt" + + "google.golang.org/grpc/balancer" + rlspb "google.golang.org/grpc/internal/proto/grpc_lookup_v1" + internalserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/serviceconfig" + + "google.golang.org/protobuf/encoding/protojson" +) + +// RLSConfig is a utility type to build service config for the RLS LB policy. +type RLSConfig struct { + RouteLookupConfig *rlspb.RouteLookupConfig + RouteLookupChannelServiceConfig string + ChildPolicy *internalserviceconfig.BalancerConfig + ChildPolicyConfigTargetFieldName string +} + +// ServiceConfigJSON generates service config with a load balancing config +// corresponding to the RLS LB policy. +func (c *RLSConfig) ServiceConfigJSON() (string, error) { + m := protojson.MarshalOptions{ + Multiline: true, + Indent: " ", + UseProtoNames: true, + } + routeLookupCfg, err := m.Marshal(c.RouteLookupConfig) + if err != nil { + return "", err + } + childPolicy, err := c.ChildPolicy.MarshalJSON() + if err != nil { + return "", err + } + + return fmt.Sprintf(` +{ + "loadBalancingConfig": [ + { + "rls_experimental": { + "routeLookupConfig": %s, + "routeLookupChannelServiceConfig": %s, + "childPolicy": %s, + "childPolicyConfigTargetFieldName": %q + } + } + ] +}`, string(routeLookupCfg), c.RouteLookupChannelServiceConfig, string(childPolicy), c.ChildPolicyConfigTargetFieldName), nil +} + +// LoadBalancingConfig generates load balancing config which can used as part of +// a ClientConnState update to the RLS LB policy. +func (c *RLSConfig) LoadBalancingConfig() (serviceconfig.LoadBalancingConfig, error) { + m := protojson.MarshalOptions{ + Multiline: true, + Indent: " ", + UseProtoNames: true, + } + routeLookupCfg, err := m.Marshal(c.RouteLookupConfig) + if err != nil { + return nil, err + } + childPolicy, err := c.ChildPolicy.MarshalJSON() + if err != nil { + return nil, err + } + lbConfigJSON := fmt.Sprintf(` +{ + "routeLookupConfig": %s, + "routeLookupChannelServiceConfig": %s, + "childPolicy": %s, + "childPolicyConfigTargetFieldName": %q +}`, string(routeLookupCfg), c.RouteLookupChannelServiceConfig, string(childPolicy), c.ChildPolicyConfigTargetFieldName) + + builder := balancer.Get("rls_experimental") + if builder == nil { + return nil, errors.New("balancer builder not found for RLS LB policy") + } + parser := builder.(balancer.ConfigParser) + return parser.ParseConfig([]byte(lbConfigJSON)) +} diff --git a/balancer/rls/internal/testutils/fakeserver/fakeserver.go b/balancer/rls/internal/testutils/fakeserver/fakeserver.go deleted file mode 100644 index 479e3036468f..000000000000 --- a/balancer/rls/internal/testutils/fakeserver/fakeserver.go +++ /dev/null @@ -1,109 +0,0 @@ -/* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -// Package fakeserver provides a fake implementation of the RouteLookupService, -// to be used in unit tests. -package fakeserver - -import ( - "context" - "errors" - "fmt" - "net" - "time" - - "google.golang.org/grpc" - rlsgrpc "google.golang.org/grpc/balancer/rls/internal/proto/grpc_lookup_v1" - rlspb "google.golang.org/grpc/balancer/rls/internal/proto/grpc_lookup_v1" - "google.golang.org/grpc/internal/testutils" -) - -const ( - defaultDialTimeout = 5 * time.Second - defaultRPCTimeout = 5 * time.Second - defaultChannelBufferSize = 50 -) - -// Response wraps the response protobuf (xds/LRS) and error that the Server -// should send out to the client through a call to stream.Send() -type Response struct { - Resp *rlspb.RouteLookupResponse - Err error -} - -// Server is a fake implementation of RLS. It exposes channels to send/receive -// RLS requests and responses. -type Server struct { - rlsgrpc.UnimplementedRouteLookupServiceServer - RequestChan *testutils.Channel - ResponseChan chan Response - Address string -} - -// Start makes a new Server which uses the provided net.Listener. If lis is nil, -// it creates a new net.Listener on a local port. The returned cancel function -// should be invoked by the caller upon completion of the test. -func Start(lis net.Listener, opts ...grpc.ServerOption) (*Server, func(), error) { - if lis == nil { - var err error - lis, err = net.Listen("tcp", "localhost:0") - if err != nil { - return nil, func() {}, fmt.Errorf("net.Listen() failed: %v", err) - } - } - s := &Server{ - // Give the channels a buffer size of 1 so that we can setup - // expectations for one lookup call, without blocking. - RequestChan: testutils.NewChannelWithSize(defaultChannelBufferSize), - ResponseChan: make(chan Response, 1), - Address: lis.Addr().String(), - } - - server := grpc.NewServer(opts...) - rlsgrpc.RegisterRouteLookupServiceServer(server, s) - go server.Serve(lis) - - return s, func() { server.Stop() }, nil -} - -// RouteLookup implements the RouteLookupService. -func (s *Server) RouteLookup(ctx context.Context, req *rlspb.RouteLookupRequest) (*rlspb.RouteLookupResponse, error) { - s.RequestChan.Send(req) - - // The leakchecker fails if we don't exit out of here in a reasonable time. - timer := time.NewTimer(defaultRPCTimeout) - select { - case <-timer.C: - return nil, errors.New("default RPC timeout exceeded") - case resp := <-s.ResponseChan: - timer.Stop() - return resp.Resp, resp.Err - } -} - -// ClientConn returns a grpc.ClientConn connected to the fakeServer. -func (s *Server) ClientConn() (*grpc.ClientConn, func(), error) { - ctx, cancel := context.WithTimeout(context.Background(), defaultDialTimeout) - defer cancel() - - cc, err := grpc.DialContext(ctx, s.Address, grpc.WithInsecure(), grpc.WithBlock()) - if err != nil { - return nil, nil, fmt.Errorf("grpc.DialContext(%s) failed: %v", s.Address, err) - } - return cc, func() { cc.Close() }, nil -} diff --git a/balancer/rls/picker.go b/balancer/rls/picker.go new file mode 100644 index 000000000000..c2d972739689 --- /dev/null +++ b/balancer/rls/picker.go @@ -0,0 +1,331 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package rls + +import ( + "errors" + "fmt" + "strings" + "sync/atomic" + "time" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/rls/internal/keys" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + internalgrpclog "google.golang.org/grpc/internal/grpclog" + rlspb "google.golang.org/grpc/internal/proto/grpc_lookup_v1" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" +) + +var ( + errRLSThrottled = errors.New("RLS call throttled at client side") + + // Function to compute data cache entry size. + computeDataCacheEntrySize = dcEntrySize +) + +// exitIdler wraps the only method on the BalancerGroup that the picker calls. +type exitIdler interface { + ExitIdleOne(id string) +} + +// rlsPicker selects the subConn to be used for a particular RPC. It does not +// manage subConns directly and delegates to pickers provided by child policies. +type rlsPicker struct { + // The keyBuilder map used to generate RLS keys for the RPC. This is built + // by the LB policy based on the received ServiceConfig. + kbm keys.BuilderMap + // Endpoint from the user's original dial target. Used to set the `host_key` + // field in `extra_keys`. + origEndpoint string + + lb *rlsBalancer + + // The picker is given its own copy of the below fields from the RLS LB policy + // to avoid having to grab the mutex on the latter. + defaultPolicy *childPolicyWrapper // Child policy for the default target. + ctrlCh *controlChannel // Control channel to the RLS server. + maxAge time.Duration // Cache max age from LB config. + staleAge time.Duration // Cache stale age from LB config. + bg exitIdler + logger *internalgrpclog.PrefixLogger +} + +// isFullMethodNameValid return true if name is of the form `/service/method`. +func isFullMethodNameValid(name string) bool { + return strings.HasPrefix(name, "/") && strings.Count(name, "/") == 2 +} + +// Pick makes the routing decision for every outbound RPC. +func (p *rlsPicker) Pick(info balancer.PickInfo) (balancer.PickResult, error) { + if name := info.FullMethodName; !isFullMethodNameValid(name) { + return balancer.PickResult{}, fmt.Errorf("rls: method name %q is not of the form '/service/method", name) + } + + // Build the request's keys using the key builders from LB config. + md, _ := metadata.FromOutgoingContext(info.Ctx) + reqKeys := p.kbm.RLSKey(md, p.origEndpoint, info.FullMethodName) + + p.lb.cacheMu.Lock() + defer p.lb.cacheMu.Unlock() + + // Lookup data cache and pending request map using request path and keys. + cacheKey := cacheKey{path: info.FullMethodName, keys: reqKeys.Str} + dcEntry := p.lb.dataCache.getEntry(cacheKey) + pendingEntry := p.lb.pendingMap[cacheKey] + now := time.Now() + + switch { + // No data cache entry. No pending request. + case dcEntry == nil && pendingEntry == nil: + throttled := p.sendRouteLookupRequestLocked(cacheKey, &backoffState{bs: defaultBackoffStrategy}, reqKeys.Map, rlspb.RouteLookupRequest_REASON_MISS, "") + if throttled { + return p.useDefaultPickIfPossible(info, errRLSThrottled) + } + return balancer.PickResult{}, balancer.ErrNoSubConnAvailable + + // No data cache entry. Pending request exits. + case dcEntry == nil && pendingEntry != nil: + return balancer.PickResult{}, balancer.ErrNoSubConnAvailable + + // Data cache hit. No pending request. + case dcEntry != nil && pendingEntry == nil: + if dcEntry.expiryTime.After(now) { + if !dcEntry.staleTime.IsZero() && dcEntry.staleTime.Before(now) && dcEntry.backoffTime.Before(now) { + p.sendRouteLookupRequestLocked(cacheKey, dcEntry.backoffState, reqKeys.Map, rlspb.RouteLookupRequest_REASON_STALE, dcEntry.headerData) + } + // Delegate to child policies. + res, err := p.delegateToChildPoliciesLocked(dcEntry, info) + return res, err + } + + // We get here only if the data cache entry has expired. If entry is in + // backoff, delegate to default target or fail the pick. + if dcEntry.backoffState != nil && dcEntry.backoffTime.After(now) { + // Avoid propagating the status code received on control plane RPCs to the + // data plane which can lead to unexpected outcomes as we do not control + // the status code sent by the control plane. Propagating the status + // message received from the control plane is still fine, as it could be + // useful for debugging purposes. + st := dcEntry.status + return p.useDefaultPickIfPossible(info, status.Error(codes.Unavailable, fmt.Sprintf("most recent error from RLS server: %v", st.Error()))) + } + + // We get here only if the entry has expired and is not in backoff. + throttled := p.sendRouteLookupRequestLocked(cacheKey, dcEntry.backoffState, reqKeys.Map, rlspb.RouteLookupRequest_REASON_MISS, "") + if throttled { + return p.useDefaultPickIfPossible(info, errRLSThrottled) + } + return balancer.PickResult{}, balancer.ErrNoSubConnAvailable + + // Data cache hit. Pending request exists. + default: + if dcEntry.expiryTime.After(now) { + res, err := p.delegateToChildPoliciesLocked(dcEntry, info) + return res, err + } + // Data cache entry has expired and pending request exists. Queue pick. + return balancer.PickResult{}, balancer.ErrNoSubConnAvailable + } +} + +// delegateToChildPoliciesLocked is a helper function which iterates through the +// list of child policy wrappers in a cache entry and attempts to find a child +// policy to which this RPC can be routed to. If all child policies are in +// TRANSIENT_FAILURE, we delegate to the last child policy arbitrarily. +func (p *rlsPicker) delegateToChildPoliciesLocked(dcEntry *cacheEntry, info balancer.PickInfo) (balancer.PickResult, error) { + const rlsDataHeaderName = "x-google-rls-data" + for i, cpw := range dcEntry.childPolicyWrappers { + state := (*balancer.State)(atomic.LoadPointer(&cpw.state)) + // Delegate to the child policy if it is not in TRANSIENT_FAILURE, or if + // it is the last one (which handles the case of delegating to the last + // child picker if all child polcies are in TRANSIENT_FAILURE). + if state.ConnectivityState != connectivity.TransientFailure || i == len(dcEntry.childPolicyWrappers)-1 { + // Any header data received from the RLS server is stored in the + // cache entry and needs to be sent to the actual backend in the + // X-Google-RLS-Data header. + res, err := state.Picker.Pick(info) + if err != nil { + return res, err + } + if res.Metadata == nil { + res.Metadata = metadata.Pairs(rlsDataHeaderName, dcEntry.headerData) + } else { + res.Metadata.Append(rlsDataHeaderName, dcEntry.headerData) + } + return res, nil + } + } + // In the unlikely event that we have a cache entry with no targets, we end up + // queueing the RPC. + return balancer.PickResult{}, balancer.ErrNoSubConnAvailable +} + +// useDefaultPickIfPossible is a helper method which delegates to the default +// target if one is configured, or fails the pick with the given error. +func (p *rlsPicker) useDefaultPickIfPossible(info balancer.PickInfo, errOnNoDefault error) (balancer.PickResult, error) { + if p.defaultPolicy != nil { + state := (*balancer.State)(atomic.LoadPointer(&p.defaultPolicy.state)) + return state.Picker.Pick(info) + } + return balancer.PickResult{}, errOnNoDefault +} + +// sendRouteLookupRequestLocked adds an entry to the pending request map and +// sends out an RLS request using the passed in arguments. Returns a value +// indicating if the request was throttled by the client-side adaptive +// throttler. +func (p *rlsPicker) sendRouteLookupRequestLocked(cacheKey cacheKey, bs *backoffState, reqKeys map[string]string, reason rlspb.RouteLookupRequest_Reason, staleHeaders string) bool { + if p.lb.pendingMap[cacheKey] != nil { + return false + } + + p.lb.pendingMap[cacheKey] = bs + throttled := p.ctrlCh.lookup(reqKeys, reason, staleHeaders, func(targets []string, headerData string, err error) { + p.handleRouteLookupResponse(cacheKey, targets, headerData, err) + }) + if throttled { + delete(p.lb.pendingMap, cacheKey) + } + return throttled +} + +// handleRouteLookupResponse is the callback invoked by the control channel upon +// receipt of an RLS response. Modifies the data cache and pending requests map +// and sends a new picker. +// +// Acquires the write-lock on the cache. Caller must not hold p.lb.cacheMu. +func (p *rlsPicker) handleRouteLookupResponse(cacheKey cacheKey, targets []string, headerData string, err error) { + p.logger.Infof("Received RLS response for key %+v with targets %+v, headerData %q, err: %v", cacheKey, targets, headerData, err) + + p.lb.cacheMu.Lock() + defer func() { + // Pending request map entry is unconditionally deleted since the request is + // no longer pending. + p.logger.Infof("Removing pending request entry for key %+v", cacheKey) + delete(p.lb.pendingMap, cacheKey) + p.lb.sendNewPicker() + p.lb.cacheMu.Unlock() + }() + + // Lookup the data cache entry or create a new one. + dcEntry := p.lb.dataCache.getEntry(cacheKey) + if dcEntry == nil { + dcEntry = &cacheEntry{} + if _, ok := p.lb.dataCache.addEntry(cacheKey, dcEntry); !ok { + // This is a very unlikely case where we are unable to add a + // data cache entry. Log and leave. + p.logger.Warningf("Failed to add data cache entry for %+v", cacheKey) + return + } + } + + // For failed requests, the data cache entry is modified as follows: + // - status is set to error returned from the control channel + // - current backoff state is available in the pending entry + // - `retries` field is incremented and + // - backoff state is moved to the data cache + // - backoffTime is set to the time indicated by the backoff state + // - backoffExpirationTime is set to twice the backoff time + // - backoffTimer is set to fire after backoffTime + // + // When a proactive cache refresh fails, this would leave the targets and the + // expiry time from the old entry unchanged. And this mean that the old valid + // entry would be used until expiration, and a new picker would be sent upon + // backoff expiry. + now := time.Now() + if err != nil { + dcEntry.status = err + pendingEntry := p.lb.pendingMap[cacheKey] + pendingEntry.retries++ + backoffTime := pendingEntry.bs.Backoff(pendingEntry.retries) + dcEntry.backoffState = pendingEntry + dcEntry.backoffTime = now.Add(backoffTime) + dcEntry.backoffExpiryTime = now.Add(2 * backoffTime) + if dcEntry.backoffState.timer != nil { + dcEntry.backoffState.timer.Stop() + } + dcEntry.backoffState.timer = time.AfterFunc(backoffTime, p.lb.sendNewPicker) + return + } + + // For successful requests, the cache entry is modified as follows: + // - childPolicyWrappers is set to point to the child policy wrappers + // associated with the targets specified in the received response + // - headerData is set to the value received in the response + // - expiryTime, stateTime and earliestEvictionTime are set + // - status is set to nil (OK status) + // - backoff state is cleared + p.setChildPolicyWrappersInCacheEntry(dcEntry, targets) + dcEntry.headerData = headerData + dcEntry.expiryTime = now.Add(p.maxAge) + if p.staleAge != 0 { + dcEntry.staleTime = now.Add(p.staleAge) + } + dcEntry.earliestEvictTime = now.Add(minEvictDuration) + dcEntry.status = nil + dcEntry.backoffState = &backoffState{bs: defaultBackoffStrategy} + dcEntry.backoffTime = time.Time{} + dcEntry.backoffExpiryTime = time.Time{} + p.lb.dataCache.updateEntrySize(dcEntry, computeDataCacheEntrySize(cacheKey, dcEntry)) +} + +// setChildPolicyWrappersInCacheEntry sets up the childPolicyWrappers field in +// the cache entry to point to the child policy wrappers for the targets +// specified in the RLS response. +// +// Caller must hold a write-lock on p.lb.cacheMu. +func (p *rlsPicker) setChildPolicyWrappersInCacheEntry(dcEntry *cacheEntry, newTargets []string) { + // If the childPolicyWrappers field is already pointing to the right targets, + // then the field's value does not need to change. + targetsChanged := true + func() { + if cpws := dcEntry.childPolicyWrappers; cpws != nil { + if len(newTargets) != len(cpws) { + return + } + for i, target := range newTargets { + if cpws[i].target != target { + return + } + } + targetsChanged = false + } + }() + if !targetsChanged { + return + } + + // If the childPolicyWrappers field is not already set to the right targets, + // then it must be reset. We construct a new list of child policies and + // then swap out the old list for the new one. + newChildPolicies := p.lb.acquireChildPolicyReferences(newTargets) + oldChildPolicyTargets := make([]string, len(dcEntry.childPolicyWrappers)) + for i, cpw := range dcEntry.childPolicyWrappers { + oldChildPolicyTargets[i] = cpw.target + } + p.lb.releaseChildPolicyReferences(oldChildPolicyTargets) + dcEntry.childPolicyWrappers = newChildPolicies +} + +func dcEntrySize(key cacheKey, entry *cacheEntry) int64 { + return int64(len(key.path) + len(key.keys) + len(entry.headerData)) +} diff --git a/balancer/rls/picker_test.go b/balancer/rls/picker_test.go new file mode 100644 index 000000000000..c0970e809f7a --- /dev/null +++ b/balancer/rls/picker_test.go @@ -0,0 +1,848 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package rls + +import ( + "context" + "fmt" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/stubserver" + rlstest "google.golang.org/grpc/internal/testutils/rls" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/durationpb" + + rlspb "google.golang.org/grpc/internal/proto/grpc_lookup_v1" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +// Test verifies the scenario where there is no matching entry in the data cache +// and no pending request either, and the ensuing RLS request is throttled. +func (s) TestPick_DataCacheMiss_NoPendingEntry_ThrottledWithDefaultTarget(t *testing.T) { + // Start an RLS server and set the throttler to always throttle requests. + rlsServer, rlsReqCh := rlstest.SetupFakeRLSServer(t, nil) + overrideAdaptiveThrottler(t, alwaysThrottlingThrottler()) + + // Build RLS service config with a default target. + rlsConfig := buildBasicRLSConfigWithChildPolicy(t, t.Name(), rlsServer.Address) + defBackendCh, defBackendAddress := startBackend(t) + rlsConfig.RouteLookupConfig.DefaultTarget = defBackendAddress + + // Register a manual resolver and push the RLS service config through it. + r := startManualResolverWithConfig(t, rlsConfig) + + cc, err := grpc.Dial(r.Scheme()+":///", grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) + } + defer cc.Close() + + // Make an RPC and ensure it gets routed to the default target. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + makeTestRPCAndExpectItToReachBackend(ctx, t, cc, defBackendCh) + + // Make sure no RLS request is sent out. + verifyRLSRequest(t, rlsReqCh, false) +} + +// Test verifies the scenario where there is no matching entry in the data cache +// and no pending request either, and the ensuing RLS request is throttled. +// There is no default target configured in the service config, so the RPC is +// expected to fail with an RLS throttled error. +func (s) TestPick_DataCacheMiss_NoPendingEntry_ThrottledWithoutDefaultTarget(t *testing.T) { + // Start an RLS server and set the throttler to always throttle requests. + rlsServer, rlsReqCh := rlstest.SetupFakeRLSServer(t, nil) + overrideAdaptiveThrottler(t, alwaysThrottlingThrottler()) + + // Build an RLS config without a default target. + rlsConfig := buildBasicRLSConfigWithChildPolicy(t, t.Name(), rlsServer.Address) + + // Register a manual resolver and push the RLS service config through it. + r := startManualResolverWithConfig(t, rlsConfig) + + // Dial the backend. + cc, err := grpc.Dial(r.Scheme()+":///", grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) + } + defer cc.Close() + + // Make an RPC and expect it to fail with RLS throttled error. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + makeTestRPCAndVerifyError(ctx, t, cc, codes.Unavailable, errRLSThrottled) + + // Make sure no RLS request is sent out. + verifyRLSRequest(t, rlsReqCh, false) +} + +// Test verifies the scenario where there is no matching entry in the data cache +// and no pending request either, and the ensuing RLS request is not throttled. +// The RLS response does not contain any backends, so the RPC fails with a +// deadline exceeded error. +func (s) TestPick_DataCacheMiss_NoPendingEntry_NotThrottled(t *testing.T) { + // Start an RLS server and set the throttler to never throttle requests. + rlsServer, rlsReqCh := rlstest.SetupFakeRLSServer(t, nil) + overrideAdaptiveThrottler(t, neverThrottlingThrottler()) + + // Build an RLS config without a default target. + rlsConfig := buildBasicRLSConfigWithChildPolicy(t, t.Name(), rlsServer.Address) + + // Register a manual resolver and push the RLS service config through it. + r := startManualResolverWithConfig(t, rlsConfig) + + // Dial the backend. + cc, err := grpc.Dial(r.Scheme()+":///", grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) + } + defer cc.Close() + + // Make an RPC and expect it to fail with deadline exceeded error. We use a + // smaller timeout to ensure that the test doesn't run very long. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer cancel() + makeTestRPCAndVerifyError(ctx, t, cc, codes.DeadlineExceeded, context.DeadlineExceeded) + + // Make sure an RLS request is sent out. + verifyRLSRequest(t, rlsReqCh, true) +} + +// Test verifies the scenario where there is no matching entry in the data +// cache, but there is a pending request. So, we expect no RLS request to be +// sent out. The pick should be queued and not delegated to the default target. +func (s) TestPick_DataCacheMiss_PendingEntryExists(t *testing.T) { + tests := []struct { + name string + withDefaultTarget bool + }{ + { + name: "withDefaultTarget", + withDefaultTarget: true, + }, + { + name: "withoutDefaultTarget", + withDefaultTarget: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + // A unary interceptor which blocks the RouteLookup RPC on the fake + // RLS server until the test is done. The first RPC by the client + // will cause the LB policy to send out an RLS request. This will + // also lead to creation of a pending entry, and further RPCs by the + // client should not result in RLS requests being sent out. + rlsReqCh := make(chan struct{}, 1) + interceptor := func(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp any, err error) { + rlsReqCh <- struct{}{} + <-ctx.Done() + return nil, ctx.Err() + } + + // Start an RLS server and set the throttler to never throttle. + rlsServer, _ := rlstest.SetupFakeRLSServer(t, nil, grpc.UnaryInterceptor(interceptor)) + overrideAdaptiveThrottler(t, neverThrottlingThrottler()) + + // Build RLS service config with an optional default target. + rlsConfig := buildBasicRLSConfigWithChildPolicy(t, t.Name(), rlsServer.Address) + if test.withDefaultTarget { + _, defBackendAddress := startBackend(t) + rlsConfig.RouteLookupConfig.DefaultTarget = defBackendAddress + } + + // Register a manual resolver and push the RLS service config + // through it. + r := startManualResolverWithConfig(t, rlsConfig) + + // Dial the backend. + cc, err := grpc.Dial(r.Scheme()+":///", grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) + } + defer cc.Close() + + // Make an RPC that results in the RLS request being sent out. And + // since the RLS server is configured to block on the first request, + // this RPC will block until its context expires. This ensures that + // we have a pending cache entry for the duration of the test. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + go func() { + client := testgrpc.NewTestServiceClient(cc) + client.EmptyCall(ctx, &testpb.Empty{}) + }() + + // Make sure an RLS request is sent out. + verifyRLSRequest(t, rlsReqCh, true) + + // Make another RPC and expect it to fail the same way. + ctx, cancel = context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer cancel() + makeTestRPCAndVerifyError(ctx, t, cc, codes.DeadlineExceeded, context.DeadlineExceeded) + + // Make sure no RLS request is sent out this time around. + verifyRLSRequest(t, rlsReqCh, false) + }) + } +} + +// Test verifies the scenario where there is a matching entry in the data cache +// which is valid and there is no pending request. The pick is expected to be +// delegated to the child policy. +func (s) TestPick_DataCacheHit_NoPendingEntry_ValidEntry(t *testing.T) { + // Start an RLS server and set the throttler to never throttle requests. + rlsServer, rlsReqCh := rlstest.SetupFakeRLSServer(t, nil) + overrideAdaptiveThrottler(t, neverThrottlingThrottler()) + + // Build the RLS config without a default target. + rlsConfig := buildBasicRLSConfigWithChildPolicy(t, t.Name(), rlsServer.Address) + + // Start a test backend, and setup the fake RLS server to return this as a + // target in the RLS response. + testBackendCh, testBackendAddress := startBackend(t) + rlsServer.SetResponseCallback(func(_ context.Context, req *rlspb.RouteLookupRequest) *rlstest.RouteLookupResponse { + return &rlstest.RouteLookupResponse{Resp: &rlspb.RouteLookupResponse{Targets: []string{testBackendAddress}}} + }) + + // Register a manual resolver and push the RLS service config through it. + r := startManualResolverWithConfig(t, rlsConfig) + + // Dial the backend. + cc, err := grpc.Dial(r.Scheme()+":///", grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) + } + defer cc.Close() + + // Make an RPC and ensure it gets routed to the test backend. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + makeTestRPCAndExpectItToReachBackend(ctx, t, cc, testBackendCh) + + // Make sure an RLS request is sent out. + verifyRLSRequest(t, rlsReqCh, true) + + // Make another RPC and expect it to find the target in the data cache. + makeTestRPCAndExpectItToReachBackend(ctx, t, cc, testBackendCh) + + // Make sure no RLS request is sent out this time around. + verifyRLSRequest(t, rlsReqCh, false) +} + +// Test verifies the scenario where there is a matching entry in the data cache +// which is valid and there is no pending request. The pick is expected to be +// delegated to the child policy. +func (s) TestPick_DataCacheHit_NoPendingEntry_ValidEntry_WithHeaderData(t *testing.T) { + // Start an RLS server and set the throttler to never throttle requests. + rlsServer, _ := rlstest.SetupFakeRLSServer(t, nil) + overrideAdaptiveThrottler(t, neverThrottlingThrottler()) + + // Build the RLS config without a default target. + rlsConfig := buildBasicRLSConfigWithChildPolicy(t, t.Name(), rlsServer.Address) + + // Start a test backend which expects the header data contents sent from the + // RLS server to be part of RPC metadata as X-Google-RLS-Data header. + const headerDataContents = "foo,bar,baz" + backend := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { + gotHeaderData := metadata.ValueFromIncomingContext(ctx, "x-google-rls-data") + if len(gotHeaderData) != 1 || gotHeaderData[0] != headerDataContents { + return nil, fmt.Errorf("got metadata in `X-Google-RLS-Data` is %v, want %s", gotHeaderData, headerDataContents) + } + return &testpb.Empty{}, nil + }, + } + if err := backend.StartServer(); err != nil { + t.Fatalf("Failed to start backend: %v", err) + } + t.Logf("Started TestService backend at: %q", backend.Address) + defer backend.Stop() + + // Setup the fake RLS server to return the above backend as a target in the + // RLS response. Also, populate the header data field in the response. + rlsServer.SetResponseCallback(func(_ context.Context, req *rlspb.RouteLookupRequest) *rlstest.RouteLookupResponse { + return &rlstest.RouteLookupResponse{Resp: &rlspb.RouteLookupResponse{ + Targets: []string{backend.Address}, + HeaderData: headerDataContents, + }} + }) + + // Register a manual resolver and push the RLS service config through it. + r := startManualResolverWithConfig(t, rlsConfig) + + // Dial the backend. + cc, err := grpc.Dial(r.Scheme()+":///", grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) + } + defer cc.Close() + + // Make an RPC and ensure it gets routed to the test backend with the header + // data sent by the RLS server. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := testgrpc.NewTestServiceClient(cc).EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() RPC: %v", err) + } +} + +// Test verifies the scenario where there is a matching entry in the data cache +// which is stale and there is no pending request. The pick is expected to be +// delegated to the child policy with a proactive cache refresh. +func (s) TestPick_DataCacheHit_NoPendingEntry_StaleEntry(t *testing.T) { + // We expect the same pick behavior (i.e delegated to the child policy) for + // a proactive refresh whether the control channel is throttled or not. + tests := []struct { + name string + throttled bool + }{ + { + name: "throttled", + throttled: true, + }, + { + name: "notThrottled", + throttled: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + // Start an RLS server and setup the throttler appropriately. + rlsServer, rlsReqCh := rlstest.SetupFakeRLSServer(t, nil) + var throttler *fakeThrottler + firstRPCDone := grpcsync.NewEvent() + if test.throttled { + throttler = oneTimeAllowingThrottler(firstRPCDone) + overrideAdaptiveThrottler(t, throttler) + } else { + throttler = neverThrottlingThrottler() + overrideAdaptiveThrottler(t, throttler) + } + + // Build the RLS config without a default target. Set the stale age + // to a very low value to force entries to become stale quickly. + rlsConfig := buildBasicRLSConfigWithChildPolicy(t, t.Name(), rlsServer.Address) + rlsConfig.RouteLookupConfig.MaxAge = durationpb.New(time.Minute) + rlsConfig.RouteLookupConfig.StaleAge = durationpb.New(defaultTestShortTimeout) + + // Start a test backend, and setup the fake RLS server to return + // this as a target in the RLS response. + testBackendCh, testBackendAddress := startBackend(t) + rlsServer.SetResponseCallback(func(_ context.Context, req *rlspb.RouteLookupRequest) *rlstest.RouteLookupResponse { + return &rlstest.RouteLookupResponse{Resp: &rlspb.RouteLookupResponse{Targets: []string{testBackendAddress}}} + }) + + // Register a manual resolver and push the RLS service config + // through it. + r := startManualResolverWithConfig(t, rlsConfig) + + // Dial the backend. + cc, err := grpc.Dial(r.Scheme()+":///", grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) + } + defer cc.Close() + + // Make an RPC and ensure it gets routed to the test backend. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + makeTestRPCAndExpectItToReachBackend(ctx, t, cc, testBackendCh) + + // Make sure an RLS request is sent out. + verifyRLSRequest(t, rlsReqCh, true) + firstRPCDone.Fire() + + // The cache entry has a large maxAge, but a small stateAge. We keep + // retrying until the cache entry becomes stale, in which case we expect a + // proactive cache refresh. + // + // If the control channel is not throttled, then we expect an RLS request + // to be sent out. If the control channel is throttled, we expect the fake + // throttler's channel to be signalled. + for { + // Make another RPC and expect it to find the target in the data cache. + makeTestRPCAndExpectItToReachBackend(ctx, t, cc, testBackendCh) + + if !test.throttled { + select { + case <-time.After(defaultTestShortTimeout): + // Go back and retry the RPC. + case <-rlsReqCh: + return + } + } else { + select { + case <-time.After(defaultTestShortTimeout): + // Go back and retry the RPC. + case <-throttler.throttleCh: + return + } + } + } + }) + } +} + +// Test verifies scenarios where there is a matching entry in the data cache +// which has expired and there is no pending request. +func (s) TestPick_DataCacheHit_NoPendingEntry_ExpiredEntry(t *testing.T) { + tests := []struct { + name string + throttled bool + withDefaultTarget bool + }{ + { + name: "throttledWithDefaultTarget", + throttled: true, + withDefaultTarget: true, + }, + { + name: "throttledWithoutDefaultTarget", + throttled: true, + withDefaultTarget: false, + }, + { + name: "notThrottled", + throttled: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + // Start an RLS server and setup the throttler appropriately. + rlsServer, rlsReqCh := rlstest.SetupFakeRLSServer(t, nil) + var throttler *fakeThrottler + firstRPCDone := grpcsync.NewEvent() + if test.throttled { + throttler = oneTimeAllowingThrottler(firstRPCDone) + overrideAdaptiveThrottler(t, throttler) + } else { + throttler = neverThrottlingThrottler() + overrideAdaptiveThrottler(t, throttler) + } + + // Build the RLS config with a very low value for maxAge. This will + // ensure that cache entries become invalid very soon. + rlsConfig := buildBasicRLSConfigWithChildPolicy(t, t.Name(), rlsServer.Address) + rlsConfig.RouteLookupConfig.MaxAge = durationpb.New(defaultTestShortTimeout) + + // Start a default backend if needed. + var defBackendCh chan struct{} + if test.withDefaultTarget { + var defBackendAddress string + defBackendCh, defBackendAddress = startBackend(t) + rlsConfig.RouteLookupConfig.DefaultTarget = defBackendAddress + } + + // Start a test backend, and setup the fake RLS server to return + // this as a target in the RLS response. + testBackendCh, testBackendAddress := startBackend(t) + rlsServer.SetResponseCallback(func(_ context.Context, req *rlspb.RouteLookupRequest) *rlstest.RouteLookupResponse { + return &rlstest.RouteLookupResponse{Resp: &rlspb.RouteLookupResponse{Targets: []string{testBackendAddress}}} + }) + + // Register a manual resolver and push the RLS service config + // through it. + r := startManualResolverWithConfig(t, rlsConfig) + + // Dial the backend. + cc, err := grpc.Dial(r.Scheme()+":///", grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) + } + defer cc.Close() + + // Make an RPC and ensure it gets routed to the test backend. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + makeTestRPCAndExpectItToReachBackend(ctx, t, cc, testBackendCh) + + // Make sure an RLS request is sent out. + verifyRLSRequest(t, rlsReqCh, true) + firstRPCDone.Fire() + + // Keep retrying the RPC until the cache entry expires. Expected behavior + // is dependent on the scenario being tested. + switch { + case test.throttled && test.withDefaultTarget: + makeTestRPCAndExpectItToReachBackend(ctx, t, cc, defBackendCh) + <-throttler.throttleCh + case test.throttled && !test.withDefaultTarget: + makeTestRPCAndVerifyError(ctx, t, cc, codes.Unavailable, errRLSThrottled) + <-throttler.throttleCh + case !test.throttled: + for { + // The backend to which the RPC is routed does not change after the + // cache entry expires because the control channel is not throttled. + // So, we need to keep retrying until the cache entry expires, at + // which point we expect an RLS request to be sent out and the RPC to + // get routed to the same testBackend. + makeTestRPCAndExpectItToReachBackend(ctx, t, cc, testBackendCh) + select { + case <-time.After(defaultTestShortTimeout): + // Go back and retry the RPC. + case <-rlsReqCh: + return + } + } + } + }) + } +} + +// Test verifies scenarios where there is a matching entry in the data cache +// which has expired and is in backoff and there is no pending request. +func (s) TestPick_DataCacheHit_NoPendingEntry_ExpiredEntryInBackoff(t *testing.T) { + tests := []struct { + name string + withDefaultTarget bool + }{ + { + name: "withDefaultTarget", + withDefaultTarget: true, + }, + { + name: "withoutDefaultTarget", + withDefaultTarget: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + // Start an RLS server and set the throttler to never throttle requests. + rlsServer, rlsReqCh := rlstest.SetupFakeRLSServer(t, nil) + overrideAdaptiveThrottler(t, neverThrottlingThrottler()) + + // Override the backoff strategy to return a large backoff which + // will make sure the date cache entry remains in backoff for the + // duration of the test. + origBackoffStrategy := defaultBackoffStrategy + defaultBackoffStrategy = &fakeBackoffStrategy{backoff: defaultTestTimeout} + defer func() { defaultBackoffStrategy = origBackoffStrategy }() + + // Build the RLS config with a very low value for maxAge. This will + // ensure that cache entries become invalid very soon. + rlsConfig := buildBasicRLSConfigWithChildPolicy(t, t.Name(), rlsServer.Address) + rlsConfig.RouteLookupConfig.MaxAge = durationpb.New(defaultTestShortTimeout) + + // Start a default backend if needed. + var defBackendCh chan struct{} + if test.withDefaultTarget { + var defBackendAddress string + defBackendCh, defBackendAddress = startBackend(t) + rlsConfig.RouteLookupConfig.DefaultTarget = defBackendAddress + } + + // Start a test backend, and set up the fake RLS server to return this as + // a target in the RLS response. + testBackendCh, testBackendAddress := startBackend(t) + rlsServer.SetResponseCallback(func(_ context.Context, req *rlspb.RouteLookupRequest) *rlstest.RouteLookupResponse { + return &rlstest.RouteLookupResponse{Resp: &rlspb.RouteLookupResponse{Targets: []string{testBackendAddress}}} + }) + + // Register a manual resolver and push the RLS service config through it. + r := startManualResolverWithConfig(t, rlsConfig) + + // Dial the backend. + cc, err := grpc.Dial(r.Scheme()+":///", grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) + } + defer cc.Close() + + // Make an RPC and ensure it gets routed to the test backend. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + makeTestRPCAndExpectItToReachBackend(ctx, t, cc, testBackendCh) + + // Make sure an RLS request is sent out. + verifyRLSRequest(t, rlsReqCh, true) + + // Set up the fake RLS server to return errors. This will push the cache + // entry into backoff. + var rlsLastErr = status.Error(codes.DeadlineExceeded, "last RLS request failed") + rlsServer.SetResponseCallback(func(_ context.Context, req *rlspb.RouteLookupRequest) *rlstest.RouteLookupResponse { + return &rlstest.RouteLookupResponse{Err: rlsLastErr} + }) + + // Since the RLS server is now configured to return errors, this will push + // the cache entry into backoff. The pick will be delegated to the default + // backend if one exits, and will fail with the error returned by the RLS + // server otherwise. + if test.withDefaultTarget { + makeTestRPCAndExpectItToReachBackend(ctx, t, cc, defBackendCh) + } else { + makeTestRPCAndVerifyError(ctx, t, cc, codes.Unavailable, rlsLastErr) + } + }) + } +} + +// Test verifies scenarios where there is a matching entry in the data cache +// which is stale and there is a pending request. +func (s) TestPick_DataCacheHit_PendingEntryExists_StaleEntry(t *testing.T) { + tests := []struct { + name string + withDefaultTarget bool + }{ + { + name: "withDefaultTarget", + withDefaultTarget: true, + }, + { + name: "withoutDefaultTarget", + withDefaultTarget: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + // A unary interceptor which simply calls the underlying handler + // until the first client RPC is done. We want one client RPC to + // succeed to ensure that a data cache entry is created. For + // subsequent client RPCs which result in RLS requests, this + // interceptor blocks until the test's context expires. And since we + // configure the RLS LB policy with a really low value for max age, + // this allows us to simulate the condition where the it has an + // expired entry and a pending entry in the cache. + rlsReqCh := make(chan struct{}, 1) + firstRPCDone := grpcsync.NewEvent() + interceptor := func(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp any, err error) { + select { + case rlsReqCh <- struct{}{}: + default: + } + if firstRPCDone.HasFired() { + <-ctx.Done() + return nil, ctx.Err() + } + return handler(ctx, req) + } + + // Start an RLS server and set the throttler to never throttle. + rlsServer, _ := rlstest.SetupFakeRLSServer(t, nil, grpc.UnaryInterceptor(interceptor)) + overrideAdaptiveThrottler(t, neverThrottlingThrottler()) + + // Build RLS service config with an optional default target. + rlsConfig := buildBasicRLSConfigWithChildPolicy(t, t.Name(), rlsServer.Address) + if test.withDefaultTarget { + _, defBackendAddress := startBackend(t) + rlsConfig.RouteLookupConfig.DefaultTarget = defBackendAddress + } + + // Low value for stale age to force entries to become stale quickly. + rlsConfig.RouteLookupConfig.MaxAge = durationpb.New(time.Minute) + rlsConfig.RouteLookupConfig.StaleAge = durationpb.New(defaultTestShortTimeout) + + // Start a test backend, and setup the fake RLS server to return + // this as a target in the RLS response. + testBackendCh, testBackendAddress := startBackend(t) + rlsServer.SetResponseCallback(func(_ context.Context, req *rlspb.RouteLookupRequest) *rlstest.RouteLookupResponse { + return &rlstest.RouteLookupResponse{Resp: &rlspb.RouteLookupResponse{Targets: []string{testBackendAddress}}} + }) + + // Register a manual resolver and push the RLS service config + // through it. + r := startManualResolverWithConfig(t, rlsConfig) + + // Dial the backend. + cc, err := grpc.Dial(r.Scheme()+":///", grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) + } + defer cc.Close() + + // Make an RPC and ensure it gets routed to the test backend. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + makeTestRPCAndExpectItToReachBackend(ctx, t, cc, testBackendCh) + + // Make sure an RLS request is sent out. + verifyRLSRequest(t, rlsReqCh, true) + firstRPCDone.Fire() + + // The cache entry has a large maxAge, but a small stateAge. We keep + // retrying until the cache entry becomes stale, in which case we expect a + // proactive cache refresh. + for { + makeTestRPCAndExpectItToReachBackend(ctx, t, cc, testBackendCh) + + select { + case <-time.After(defaultTestShortTimeout): + // Go back and retry the RPC. + case <-rlsReqCh: + return + } + } + }) + } +} + +// Test verifies scenarios where there is a matching entry in the data cache +// which is expired and there is a pending request. +func (s) TestPick_DataCacheHit_PendingEntryExists_ExpiredEntry(t *testing.T) { + tests := []struct { + name string + withDefaultTarget bool + }{ + { + name: "withDefaultTarget", + withDefaultTarget: true, + }, + { + name: "withoutDefaultTarget", + withDefaultTarget: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + // A unary interceptor which simply calls the underlying handler + // until the first client RPC is done. We want one client RPC to + // succeed to ensure that a data cache entry is created. For + // subsequent client RPCs which result in RLS requests, this + // interceptor blocks until the test's context expires. And since we + // configure the RLS LB policy with a really low value for max age, + // this allows us to simulate the condition where the it has an + // expired entry and a pending entry in the cache. + rlsReqCh := make(chan struct{}, 1) + firstRPCDone := grpcsync.NewEvent() + interceptor := func(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp any, err error) { + select { + case rlsReqCh <- struct{}{}: + default: + } + if firstRPCDone.HasFired() { + <-ctx.Done() + return nil, ctx.Err() + } + return handler(ctx, req) + } + + // Start an RLS server and set the throttler to never throttle. + rlsServer, _ := rlstest.SetupFakeRLSServer(t, nil, grpc.UnaryInterceptor(interceptor)) + overrideAdaptiveThrottler(t, neverThrottlingThrottler()) + + // Build RLS service config with an optional default target. + rlsConfig := buildBasicRLSConfigWithChildPolicy(t, t.Name(), rlsServer.Address) + if test.withDefaultTarget { + _, defBackendAddress := startBackend(t) + rlsConfig.RouteLookupConfig.DefaultTarget = defBackendAddress + } + // Set a low value for maxAge to ensure cache entries expire soon. + rlsConfig.RouteLookupConfig.MaxAge = durationpb.New(defaultTestShortTimeout) + + // Start a test backend, and setup the fake RLS server to return + // this as a target in the RLS response. + testBackendCh, testBackendAddress := startBackend(t) + rlsServer.SetResponseCallback(func(_ context.Context, req *rlspb.RouteLookupRequest) *rlstest.RouteLookupResponse { + return &rlstest.RouteLookupResponse{Resp: &rlspb.RouteLookupResponse{Targets: []string{testBackendAddress}}} + }) + + // Register a manual resolver and push the RLS service config + // through it. + r := startManualResolverWithConfig(t, rlsConfig) + + // Dial the backend. + cc, err := grpc.Dial(r.Scheme()+":///", grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) + } + defer cc.Close() + + // Make an RPC and ensure it gets routed to the test backend. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + makeTestRPCAndExpectItToReachBackend(ctx, t, cc, testBackendCh) + + // Make sure an RLS request is sent out. + verifyRLSRequest(t, rlsReqCh, true) + firstRPCDone.Fire() + + // At this point, we have a cache entry with a small maxAge, and the + // RLS server is configured to block on further RLS requests. As we + // retry the RPC, at some point the cache entry would expire and + // force us to send an RLS request which would block on the server, + // giving us a pending cache entry for the duration of the test. + go func() { + for client := testgrpc.NewTestServiceClient(cc); ctx.Err() == nil; <-time.After(defaultTestShortTimeout) { + client.EmptyCall(ctx, &testpb.Empty{}) + } + }() + verifyRLSRequest(t, rlsReqCh, true) + + // Another RPC at this point should find the pending entry and be queued. + // But since we pass a small deadline, this RPC should fail with a + // deadline exceeded error since the pending request does not return until + // the test is done. And since we have a pending entry, we expect no RLS + // request to be sent out. + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + makeTestRPCAndVerifyError(sCtx, t, cc, codes.DeadlineExceeded, context.DeadlineExceeded) + verifyRLSRequest(t, rlsReqCh, false) + }) + } +} + +func TestIsFullMethodNameValid(t *testing.T) { + tests := []struct { + desc string + methodName string + want bool + }{ + { + desc: "does not start with a slash", + methodName: "service/method", + want: false, + }, + { + desc: "does not contain a method", + methodName: "/service", + want: false, + }, + { + desc: "path has more elements", + methodName: "/service/path/to/method", + want: false, + }, + { + desc: "valid", + methodName: "/service/method", + want: true, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + if got := isFullMethodNameValid(test.methodName); got != test.want { + t.Fatalf("isFullMethodNameValid(%q) = %v, want %v", test.methodName, got, test.want) + } + }) + } +} diff --git a/balancer/roundrobin/roundrobin.go b/balancer/roundrobin/roundrobin.go index 43c2a15373a1..f7031ad2251b 100644 --- a/balancer/roundrobin/roundrobin.go +++ b/balancer/roundrobin/roundrobin.go @@ -22,7 +22,7 @@ package roundrobin import ( - "sync" + "sync/atomic" "google.golang.org/grpc/balancer" "google.golang.org/grpc/balancer/base" @@ -47,11 +47,11 @@ func init() { type rrPickerBuilder struct{} func (*rrPickerBuilder) Build(info base.PickerBuildInfo) balancer.Picker { - logger.Infof("roundrobinPicker: newPicker called with info: %v", info) + logger.Infof("roundrobinPicker: Build called with info: %v", info) if len(info.ReadySCs) == 0 { return base.NewErrPicker(balancer.ErrNoSubConnAvailable) } - var scs []balancer.SubConn + scs := make([]balancer.SubConn, 0, len(info.ReadySCs)) for sc := range info.ReadySCs { scs = append(scs, sc) } @@ -60,7 +60,7 @@ func (*rrPickerBuilder) Build(info base.PickerBuildInfo) balancer.Picker { // Start at a random index, as the same RR balancer rebuilds a new // picker when SubConn states change, and we don't want to apply excess // load to the first server in the list. - next: grpcrand.Intn(len(scs)), + next: uint32(grpcrand.Intn(len(scs))), } } @@ -69,15 +69,13 @@ type rrPicker struct { // created. The slice is immutable. Each Get() will do a round robin // selection from it and return the selected SubConn. subConns []balancer.SubConn - - mu sync.Mutex - next int + next uint32 } func (p *rrPicker) Pick(balancer.PickInfo) (balancer.PickResult, error) { - p.mu.Lock() - sc := p.subConns[p.next] - p.next = (p.next + 1) % len(p.subConns) - p.mu.Unlock() + subConnsLen := uint32(len(p.subConns)) + nextIndex := atomic.AddUint32(&p.next, 1) + + sc := p.subConns[nextIndex%subConnsLen] return balancer.PickResult{SubConn: sc}, nil } diff --git a/balancer/roundrobin/roundrobin_test.go b/balancer/roundrobin/roundrobin_test.go deleted file mode 100644 index 0c54465a6af2..000000000000 --- a/balancer/roundrobin/roundrobin_test.go +++ /dev/null @@ -1,474 +0,0 @@ -/* - * - * Copyright 2017 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package roundrobin_test - -import ( - "context" - "fmt" - "net" - "strings" - "sync" - "testing" - "time" - - "google.golang.org/grpc" - "google.golang.org/grpc/balancer/roundrobin" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/connectivity" - "google.golang.org/grpc/internal/grpctest" - "google.golang.org/grpc/peer" - "google.golang.org/grpc/resolver" - "google.golang.org/grpc/resolver/manual" - "google.golang.org/grpc/status" - testpb "google.golang.org/grpc/test/grpc_testing" -) - -type s struct { - grpctest.Tester -} - -func Test(t *testing.T) { - grpctest.RunSubTests(t, s{}) -} - -func emptyCall(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { - return &testpb.Empty{}, nil -} - -func fullDuplexCall(stream testpb.TestService_FullDuplexCallServer) error { - return nil -} - -type test struct { - servers []*grpc.Server - addresses []string -} - -func (t *test) cleanup() { - for _, s := range t.servers { - s.Stop() - } -} - -func startTestServers(count int) (_ *test, err error) { - t := &test{} - - defer func() { - if err != nil { - t.cleanup() - } - }() - for i := 0; i < count; i++ { - lis, err := net.Listen("tcp", "localhost:0") - if err != nil { - return nil, fmt.Errorf("failed to listen %v", err) - } - - s := grpc.NewServer() - testpb.RegisterTestServiceService(s, &testpb.TestServiceService{ - EmptyCall: emptyCall, - FullDuplexCall: fullDuplexCall, - }) - t.servers = append(t.servers, s) - t.addresses = append(t.addresses, lis.Addr().String()) - - go func(s *grpc.Server, l net.Listener) { - s.Serve(l) - }(s, lis) - } - - return t, nil -} - -func (s) TestOneBackend(t *testing.T) { - r := manual.NewBuilderWithScheme("whatever") - - test, err := startTestServers(1) - if err != nil { - t.Fatalf("failed to start servers: %v", err) - } - defer test.cleanup() - - cc, err := grpc.Dial(r.Scheme()+":///test.server", grpc.WithInsecure(), grpc.WithResolvers(r), grpc.WithBalancerName(roundrobin.Name)) - if err != nil { - t.Fatalf("failed to dial: %v", err) - } - defer cc.Close() - testc := testpb.NewTestServiceClient(cc) - // The first RPC should fail because there's no address. - ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond) - defer cancel() - if _, err := testc.EmptyCall(ctx, &testpb.Empty{}); err == nil || status.Code(err) != codes.DeadlineExceeded { - t.Fatalf("EmptyCall() = _, %v, want _, DeadlineExceeded", err) - } - - r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: test.addresses[0]}}}) - // The second RPC should succeed. - if _, err := testc.EmptyCall(context.Background(), &testpb.Empty{}); err != nil { - t.Fatalf("EmptyCall() = _, %v, want _, ", err) - } -} - -func (s) TestBackendsRoundRobin(t *testing.T) { - r := manual.NewBuilderWithScheme("whatever") - - backendCount := 5 - test, err := startTestServers(backendCount) - if err != nil { - t.Fatalf("failed to start servers: %v", err) - } - defer test.cleanup() - - cc, err := grpc.Dial(r.Scheme()+":///test.server", grpc.WithInsecure(), grpc.WithResolvers(r), grpc.WithBalancerName(roundrobin.Name)) - if err != nil { - t.Fatalf("failed to dial: %v", err) - } - defer cc.Close() - testc := testpb.NewTestServiceClient(cc) - // The first RPC should fail because there's no address. - ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond) - defer cancel() - if _, err := testc.EmptyCall(ctx, &testpb.Empty{}); err == nil || status.Code(err) != codes.DeadlineExceeded { - t.Fatalf("EmptyCall() = _, %v, want _, DeadlineExceeded", err) - } - - var resolvedAddrs []resolver.Address - for i := 0; i < backendCount; i++ { - resolvedAddrs = append(resolvedAddrs, resolver.Address{Addr: test.addresses[i]}) - } - - r.UpdateState(resolver.State{Addresses: resolvedAddrs}) - var p peer.Peer - // Make sure connections to all servers are up. - for si := 0; si < backendCount; si++ { - var connected bool - for i := 0; i < 1000; i++ { - if _, err := testc.EmptyCall(context.Background(), &testpb.Empty{}, grpc.Peer(&p)); err != nil { - t.Fatalf("EmptyCall() = _, %v, want _, ", err) - } - if p.Addr.String() == test.addresses[si] { - connected = true - break - } - time.Sleep(time.Millisecond) - } - if !connected { - t.Fatalf("Connection to %v was not up after more than 1 second", test.addresses[si]) - } - } - - for i := 0; i < 3*backendCount; i++ { - if _, err := testc.EmptyCall(context.Background(), &testpb.Empty{}, grpc.Peer(&p)); err != nil { - t.Fatalf("EmptyCall() = _, %v, want _, ", err) - } - if p.Addr.String() != test.addresses[i%backendCount] { - t.Fatalf("Index %d: want peer %v, got peer %v", i, test.addresses[i%backendCount], p.Addr.String()) - } - } -} - -func (s) TestAddressesRemoved(t *testing.T) { - r := manual.NewBuilderWithScheme("whatever") - - test, err := startTestServers(1) - if err != nil { - t.Fatalf("failed to start servers: %v", err) - } - defer test.cleanup() - - cc, err := grpc.Dial(r.Scheme()+":///test.server", grpc.WithInsecure(), grpc.WithResolvers(r), grpc.WithBalancerName(roundrobin.Name)) - if err != nil { - t.Fatalf("failed to dial: %v", err) - } - defer cc.Close() - testc := testpb.NewTestServiceClient(cc) - // The first RPC should fail because there's no address. - ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond) - defer cancel() - if _, err := testc.EmptyCall(ctx, &testpb.Empty{}); err == nil || status.Code(err) != codes.DeadlineExceeded { - t.Fatalf("EmptyCall() = _, %v, want _, DeadlineExceeded", err) - } - - r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: test.addresses[0]}}}) - // The second RPC should succeed. - if _, err := testc.EmptyCall(context.Background(), &testpb.Empty{}); err != nil { - t.Fatalf("EmptyCall() = _, %v, want _, ", err) - } - - r.UpdateState(resolver.State{Addresses: []resolver.Address{}}) - - ctx2, cancel2 := context.WithTimeout(context.Background(), 500*time.Millisecond) - defer cancel2() - // Wait for state to change to transient failure. - for src := cc.GetState(); src != connectivity.TransientFailure; src = cc.GetState() { - if !cc.WaitForStateChange(ctx2, src) { - t.Fatalf("timed out waiting for state change. got %v; want %v", src, connectivity.TransientFailure) - } - } - - const msgWant = "produced zero addresses" - if _, err := testc.EmptyCall(ctx2, &testpb.Empty{}); err == nil || !strings.Contains(status.Convert(err).Message(), msgWant) { - t.Fatalf("EmptyCall() = _, %v, want _, Contains(Message(), %q)", err, msgWant) - } -} - -func (s) TestCloseWithPendingRPC(t *testing.T) { - r := manual.NewBuilderWithScheme("whatever") - - test, err := startTestServers(1) - if err != nil { - t.Fatalf("failed to start servers: %v", err) - } - defer test.cleanup() - - cc, err := grpc.Dial(r.Scheme()+":///test.server", grpc.WithInsecure(), grpc.WithResolvers(r), grpc.WithBalancerName(roundrobin.Name)) - if err != nil { - t.Fatalf("failed to dial: %v", err) - } - testc := testpb.NewTestServiceClient(cc) - - var wg sync.WaitGroup - for i := 0; i < 3; i++ { - wg.Add(1) - go func() { - defer wg.Done() - // This RPC blocks until cc is closed. - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - if _, err := testc.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) == codes.DeadlineExceeded { - t.Errorf("RPC failed because of deadline after cc is closed; want error the client connection is closing") - } - cancel() - }() - } - cc.Close() - wg.Wait() -} - -func (s) TestNewAddressWhileBlocking(t *testing.T) { - r := manual.NewBuilderWithScheme("whatever") - - test, err := startTestServers(1) - if err != nil { - t.Fatalf("failed to start servers: %v", err) - } - defer test.cleanup() - - cc, err := grpc.Dial(r.Scheme()+":///test.server", grpc.WithInsecure(), grpc.WithResolvers(r), grpc.WithBalancerName(roundrobin.Name)) - if err != nil { - t.Fatalf("failed to dial: %v", err) - } - defer cc.Close() - testc := testpb.NewTestServiceClient(cc) - // The first RPC should fail because there's no address. - ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond) - defer cancel() - if _, err := testc.EmptyCall(ctx, &testpb.Empty{}); err == nil || status.Code(err) != codes.DeadlineExceeded { - t.Fatalf("EmptyCall() = _, %v, want _, DeadlineExceeded", err) - } - - r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: test.addresses[0]}}}) - // The second RPC should succeed. - ctx, cancel = context.WithTimeout(context.Background(), 2*time.Second) - defer cancel() - if _, err := testc.EmptyCall(ctx, &testpb.Empty{}); err != nil { - t.Fatalf("EmptyCall() = _, %v, want _, nil", err) - } - - r.UpdateState(resolver.State{Addresses: []resolver.Address{}}) - - var wg sync.WaitGroup - for i := 0; i < 3; i++ { - wg.Add(1) - go func() { - defer wg.Done() - // This RPC blocks until NewAddress is called. - testc.EmptyCall(context.Background(), &testpb.Empty{}) - }() - } - time.Sleep(50 * time.Millisecond) - r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: test.addresses[0]}}}) - wg.Wait() -} - -func (s) TestOneServerDown(t *testing.T) { - r := manual.NewBuilderWithScheme("whatever") - - backendCount := 3 - test, err := startTestServers(backendCount) - if err != nil { - t.Fatalf("failed to start servers: %v", err) - } - defer test.cleanup() - - cc, err := grpc.Dial(r.Scheme()+":///test.server", grpc.WithInsecure(), grpc.WithResolvers(r), grpc.WithBalancerName(roundrobin.Name)) - if err != nil { - t.Fatalf("failed to dial: %v", err) - } - defer cc.Close() - testc := testpb.NewTestServiceClient(cc) - // The first RPC should fail because there's no address. - ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond) - defer cancel() - if _, err := testc.EmptyCall(ctx, &testpb.Empty{}); err == nil || status.Code(err) != codes.DeadlineExceeded { - t.Fatalf("EmptyCall() = _, %v, want _, DeadlineExceeded", err) - } - - var resolvedAddrs []resolver.Address - for i := 0; i < backendCount; i++ { - resolvedAddrs = append(resolvedAddrs, resolver.Address{Addr: test.addresses[i]}) - } - - r.UpdateState(resolver.State{Addresses: resolvedAddrs}) - var p peer.Peer - // Make sure connections to all servers are up. - for si := 0; si < backendCount; si++ { - var connected bool - for i := 0; i < 1000; i++ { - if _, err := testc.EmptyCall(context.Background(), &testpb.Empty{}, grpc.Peer(&p)); err != nil { - t.Fatalf("EmptyCall() = _, %v, want _, ", err) - } - if p.Addr.String() == test.addresses[si] { - connected = true - break - } - time.Sleep(time.Millisecond) - } - if !connected { - t.Fatalf("Connection to %v was not up after more than 1 second", test.addresses[si]) - } - } - - for i := 0; i < 3*backendCount; i++ { - if _, err := testc.EmptyCall(context.Background(), &testpb.Empty{}, grpc.Peer(&p)); err != nil { - t.Fatalf("EmptyCall() = _, %v, want _, ", err) - } - if p.Addr.String() != test.addresses[i%backendCount] { - t.Fatalf("Index %d: want peer %v, got peer %v", i, test.addresses[i%backendCount], p.Addr.String()) - } - } - - // Stop one server, RPCs should roundrobin among the remaining servers. - backendCount-- - test.servers[backendCount].Stop() - // Loop until see server[backendCount-1] twice without seeing server[backendCount]. - var targetSeen int - for i := 0; i < 1000; i++ { - if _, err := testc.EmptyCall(context.Background(), &testpb.Empty{}, grpc.Peer(&p)); err != nil { - targetSeen = 0 - t.Logf("EmptyCall() = _, %v, want _, ", err) - // Due to a race, this RPC could possibly get the connection that - // was closing, and this RPC may fail. Keep trying when this - // happens. - continue - } - switch p.Addr.String() { - case test.addresses[backendCount-1]: - targetSeen++ - case test.addresses[backendCount]: - // Reset targetSeen if peer is server[backendCount]. - targetSeen = 0 - } - // Break to make sure the last picked address is server[-1], so the following for loop won't be flaky. - if targetSeen >= 2 { - break - } - } - if targetSeen != 2 { - t.Fatal("Failed to see server[backendCount-1] twice without seeing server[backendCount]") - } - for i := 0; i < 3*backendCount; i++ { - if _, err := testc.EmptyCall(context.Background(), &testpb.Empty{}, grpc.Peer(&p)); err != nil { - t.Fatalf("EmptyCall() = _, %v, want _, ", err) - } - if p.Addr.String() != test.addresses[i%backendCount] { - t.Errorf("Index %d: want peer %v, got peer %v", i, test.addresses[i%backendCount], p.Addr.String()) - } - } -} - -func (s) TestAllServersDown(t *testing.T) { - r := manual.NewBuilderWithScheme("whatever") - - backendCount := 3 - test, err := startTestServers(backendCount) - if err != nil { - t.Fatalf("failed to start servers: %v", err) - } - defer test.cleanup() - - cc, err := grpc.Dial(r.Scheme()+":///test.server", grpc.WithInsecure(), grpc.WithResolvers(r), grpc.WithBalancerName(roundrobin.Name)) - if err != nil { - t.Fatalf("failed to dial: %v", err) - } - defer cc.Close() - testc := testpb.NewTestServiceClient(cc) - // The first RPC should fail because there's no address. - ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond) - defer cancel() - if _, err := testc.EmptyCall(ctx, &testpb.Empty{}); err == nil || status.Code(err) != codes.DeadlineExceeded { - t.Fatalf("EmptyCall() = _, %v, want _, DeadlineExceeded", err) - } - - var resolvedAddrs []resolver.Address - for i := 0; i < backendCount; i++ { - resolvedAddrs = append(resolvedAddrs, resolver.Address{Addr: test.addresses[i]}) - } - - r.UpdateState(resolver.State{Addresses: resolvedAddrs}) - var p peer.Peer - // Make sure connections to all servers are up. - for si := 0; si < backendCount; si++ { - var connected bool - for i := 0; i < 1000; i++ { - if _, err := testc.EmptyCall(context.Background(), &testpb.Empty{}, grpc.Peer(&p)); err != nil { - t.Fatalf("EmptyCall() = _, %v, want _, ", err) - } - if p.Addr.String() == test.addresses[si] { - connected = true - break - } - time.Sleep(time.Millisecond) - } - if !connected { - t.Fatalf("Connection to %v was not up after more than 1 second", test.addresses[si]) - } - } - - for i := 0; i < 3*backendCount; i++ { - if _, err := testc.EmptyCall(context.Background(), &testpb.Empty{}, grpc.Peer(&p)); err != nil { - t.Fatalf("EmptyCall() = _, %v, want _, ", err) - } - if p.Addr.String() != test.addresses[i%backendCount] { - t.Fatalf("Index %d: want peer %v, got peer %v", i, test.addresses[i%backendCount], p.Addr.String()) - } - } - - // All servers are stopped, failfast RPC should fail with unavailable. - for i := 0; i < backendCount; i++ { - test.servers[i].Stop() - } - time.Sleep(100 * time.Millisecond) - for i := 0; i < 1000; i++ { - if _, err := testc.EmptyCall(context.Background(), &testpb.Empty{}); status.Code(err) == codes.Unavailable { - return - } - time.Sleep(time.Millisecond) - } - t.Fatalf("Failfast RPCs didn't fail with Unavailable after all servers are stopped") -} diff --git a/balancer/weightedroundrobin/balancer.go b/balancer/weightedroundrobin/balancer.go new file mode 100644 index 000000000000..1c515b6de719 --- /dev/null +++ b/balancer/weightedroundrobin/balancer.go @@ -0,0 +1,546 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package weightedroundrobin + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sync" + "sync/atomic" + "time" + "unsafe" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/base" + "google.golang.org/grpc/balancer/weightedroundrobin/internal" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/grpcrand" + iserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/orca" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" + + v3orcapb "github.com/cncf/xds/go/xds/data/orca/v3" +) + +// Name is the name of the weighted round robin balancer. +const Name = "weighted_round_robin" + +func init() { + balancer.Register(bb{}) +} + +type bb struct{} + +func (bb) Build(cc balancer.ClientConn, bOpts balancer.BuildOptions) balancer.Balancer { + b := &wrrBalancer{ + cc: cc, + subConns: resolver.NewAddressMap(), + csEvltr: &balancer.ConnectivityStateEvaluator{}, + scMap: make(map[balancer.SubConn]*weightedSubConn), + connectivityState: connectivity.Connecting, + } + b.logger = prefixLogger(b) + b.logger.Infof("Created") + return b +} + +func (bb) ParseConfig(js json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + lbCfg := &lbConfig{ + // Default values as documented in A58. + OOBReportingPeriod: iserviceconfig.Duration(10 * time.Second), + BlackoutPeriod: iserviceconfig.Duration(10 * time.Second), + WeightExpirationPeriod: iserviceconfig.Duration(3 * time.Minute), + WeightUpdatePeriod: iserviceconfig.Duration(time.Second), + ErrorUtilizationPenalty: 1, + } + if err := json.Unmarshal(js, lbCfg); err != nil { + return nil, fmt.Errorf("wrr: unable to unmarshal LB policy config: %s, error: %v", string(js), err) + } + + if lbCfg.ErrorUtilizationPenalty < 0 { + return nil, fmt.Errorf("wrr: errorUtilizationPenalty must be non-negative") + } + + // For easier comparisons later, ensure the OOB reporting period is unset + // (0s) when OOB reports are disabled. + if !lbCfg.EnableOOBLoadReport { + lbCfg.OOBReportingPeriod = 0 + } + + // Impose lower bound of 100ms on weightUpdatePeriod. + if !internal.AllowAnyWeightUpdatePeriod && lbCfg.WeightUpdatePeriod < iserviceconfig.Duration(100*time.Millisecond) { + lbCfg.WeightUpdatePeriod = iserviceconfig.Duration(100 * time.Millisecond) + } + + return lbCfg, nil +} + +func (bb) Name() string { + return Name +} + +// wrrBalancer implements the weighted round robin LB policy. +type wrrBalancer struct { + cc balancer.ClientConn + logger *grpclog.PrefixLogger + + // The following fields are only accessed on calls into the LB policy, and + // do not need a mutex. + cfg *lbConfig // active config + subConns *resolver.AddressMap // active weightedSubConns mapped by address + scMap map[balancer.SubConn]*weightedSubConn + connectivityState connectivity.State // aggregate state + csEvltr *balancer.ConnectivityStateEvaluator + resolverErr error // the last error reported by the resolver; cleared on successful resolution + connErr error // the last connection error; cleared upon leaving TransientFailure + stopPicker func() +} + +func (b *wrrBalancer) UpdateClientConnState(ccs balancer.ClientConnState) error { + b.logger.Infof("UpdateCCS: %v", ccs) + b.resolverErr = nil + cfg, ok := ccs.BalancerConfig.(*lbConfig) + if !ok { + return fmt.Errorf("wrr: received nil or illegal BalancerConfig (type %T): %v", ccs.BalancerConfig, ccs.BalancerConfig) + } + + b.cfg = cfg + b.updateAddresses(ccs.ResolverState.Addresses) + + if len(ccs.ResolverState.Addresses) == 0 { + b.ResolverError(errors.New("resolver produced zero addresses")) // will call regeneratePicker + return balancer.ErrBadResolverState + } + + b.regeneratePicker() + + return nil +} + +func (b *wrrBalancer) updateAddresses(addrs []resolver.Address) { + addrsSet := resolver.NewAddressMap() + + // Loop through new address list and create subconns for any new addresses. + for _, addr := range addrs { + if _, ok := addrsSet.Get(addr); ok { + // Redundant address; skip. + continue + } + addrsSet.Set(addr, nil) + + var wsc *weightedSubConn + wsci, ok := b.subConns.Get(addr) + if ok { + wsc = wsci.(*weightedSubConn) + } else { + // addr is a new address (not existing in b.subConns). + var sc balancer.SubConn + sc, err := b.cc.NewSubConn([]resolver.Address{addr}, balancer.NewSubConnOptions{ + StateListener: func(state balancer.SubConnState) { + b.updateSubConnState(sc, state) + }, + }) + if err != nil { + b.logger.Warningf("Failed to create new SubConn for address %v: %v", addr, err) + continue + } + wsc = &weightedSubConn{ + SubConn: sc, + logger: b.logger, + connectivityState: connectivity.Idle, + // Initially, we set load reports to off, because they are not + // running upon initial weightedSubConn creation. + cfg: &lbConfig{EnableOOBLoadReport: false}, + } + b.subConns.Set(addr, wsc) + b.scMap[sc] = wsc + b.csEvltr.RecordTransition(connectivity.Shutdown, connectivity.Idle) + sc.Connect() + } + // Update config for existing weightedSubConn or send update for first + // time to new one. Ensures an OOB listener is running if needed + // (and stops the existing one if applicable). + wsc.updateConfig(b.cfg) + } + + // Loop through existing subconns and remove ones that are not in addrs. + for _, addr := range b.subConns.Keys() { + if _, ok := addrsSet.Get(addr); ok { + // Existing address also in new address list; skip. + continue + } + // addr was removed by resolver. Remove. + wsci, _ := b.subConns.Get(addr) + wsc := wsci.(*weightedSubConn) + wsc.SubConn.Shutdown() + b.subConns.Delete(addr) + } +} + +func (b *wrrBalancer) ResolverError(err error) { + b.resolverErr = err + if b.subConns.Len() == 0 { + b.connectivityState = connectivity.TransientFailure + } + if b.connectivityState != connectivity.TransientFailure { + // No need to update the picker since no error is being returned. + return + } + b.regeneratePicker() +} + +func (b *wrrBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { + b.logger.Errorf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, state) +} + +func (b *wrrBalancer) updateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { + wsc := b.scMap[sc] + if wsc == nil { + b.logger.Errorf("UpdateSubConnState called with an unknown SubConn: %p, %v", sc, state) + return + } + if b.logger.V(2) { + logger.Infof("UpdateSubConnState(%+v, %+v)", sc, state) + } + + cs := state.ConnectivityState + + if cs == connectivity.TransientFailure { + // Save error to be reported via picker. + b.connErr = state.ConnectionError + } + + if cs == connectivity.Shutdown { + delete(b.scMap, sc) + // The subconn was removed from b.subConns when the address was removed + // in updateAddresses. + } + + oldCS := wsc.updateConnectivityState(cs) + b.connectivityState = b.csEvltr.RecordTransition(oldCS, cs) + + // Regenerate picker when one of the following happens: + // - this sc entered or left ready + // - the aggregated state of balancer is TransientFailure + // (may need to update error message) + if (cs == connectivity.Ready) != (oldCS == connectivity.Ready) || + b.connectivityState == connectivity.TransientFailure { + b.regeneratePicker() + } +} + +// Close stops the balancer. It cancels any ongoing scheduler updates and +// stops any ORCA listeners. +func (b *wrrBalancer) Close() { + if b.stopPicker != nil { + b.stopPicker() + b.stopPicker = nil + } + for _, wsc := range b.scMap { + // Ensure any lingering OOB watchers are stopped. + wsc.updateConnectivityState(connectivity.Shutdown) + } +} + +// ExitIdle is ignored; we always connect to all backends. +func (b *wrrBalancer) ExitIdle() {} + +func (b *wrrBalancer) readySubConns() []*weightedSubConn { + var ret []*weightedSubConn + for _, v := range b.subConns.Values() { + wsc := v.(*weightedSubConn) + if wsc.connectivityState == connectivity.Ready { + ret = append(ret, wsc) + } + } + return ret +} + +// mergeErrors builds an error from the last connection error and the last +// resolver error. Must only be called if b.connectivityState is +// TransientFailure. +func (b *wrrBalancer) mergeErrors() error { + // connErr must always be non-nil unless there are no SubConns, in which + // case resolverErr must be non-nil. + if b.connErr == nil { + return fmt.Errorf("last resolver error: %v", b.resolverErr) + } + if b.resolverErr == nil { + return fmt.Errorf("last connection error: %v", b.connErr) + } + return fmt.Errorf("last connection error: %v; last resolver error: %v", b.connErr, b.resolverErr) +} + +func (b *wrrBalancer) regeneratePicker() { + if b.stopPicker != nil { + b.stopPicker() + b.stopPicker = nil + } + + switch b.connectivityState { + case connectivity.TransientFailure: + b.cc.UpdateState(balancer.State{ + ConnectivityState: connectivity.TransientFailure, + Picker: base.NewErrPicker(b.mergeErrors()), + }) + return + case connectivity.Connecting, connectivity.Idle: + // Idle could happen very briefly if all subconns are Idle and we've + // asked them to connect but they haven't reported Connecting yet. + // Report the same as Connecting since this is temporary. + b.cc.UpdateState(balancer.State{ + ConnectivityState: connectivity.Connecting, + Picker: base.NewErrPicker(balancer.ErrNoSubConnAvailable), + }) + return + case connectivity.Ready: + b.connErr = nil + } + + p := &picker{ + v: grpcrand.Uint32(), // start the scheduler at a random point + cfg: b.cfg, + subConns: b.readySubConns(), + } + var ctx context.Context + ctx, b.stopPicker = context.WithCancel(context.Background()) + p.start(ctx) + b.cc.UpdateState(balancer.State{ + ConnectivityState: b.connectivityState, + Picker: p, + }) +} + +// picker is the WRR policy's picker. It uses live-updating backend weights to +// update the scheduler periodically and ensure picks are routed proportional +// to those weights. +type picker struct { + scheduler unsafe.Pointer // *scheduler; accessed atomically + v uint32 // incrementing value used by the scheduler; accessed atomically + cfg *lbConfig // active config when picker created + subConns []*weightedSubConn // all READY subconns +} + +// scWeights returns a slice containing the weights from p.subConns in the same +// order as p.subConns. +func (p *picker) scWeights() []float64 { + ws := make([]float64, len(p.subConns)) + now := internal.TimeNow() + for i, wsc := range p.subConns { + ws[i] = wsc.weight(now, time.Duration(p.cfg.WeightExpirationPeriod), time.Duration(p.cfg.BlackoutPeriod)) + } + return ws +} + +func (p *picker) inc() uint32 { + return atomic.AddUint32(&p.v, 1) +} + +func (p *picker) regenerateScheduler() { + s := newScheduler(p.scWeights(), p.inc) + atomic.StorePointer(&p.scheduler, unsafe.Pointer(&s)) +} + +func (p *picker) start(ctx context.Context) { + p.regenerateScheduler() + if len(p.subConns) == 1 { + // No need to regenerate weights with only one backend. + return + } + go func() { + ticker := time.NewTicker(time.Duration(p.cfg.WeightUpdatePeriod)) + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + p.regenerateScheduler() + } + } + }() +} + +func (p *picker) Pick(info balancer.PickInfo) (balancer.PickResult, error) { + // Read the scheduler atomically. All scheduler operations are threadsafe, + // and if the scheduler is replaced during this usage, we want to use the + // scheduler that was live when the pick started. + sched := *(*scheduler)(atomic.LoadPointer(&p.scheduler)) + + pickedSC := p.subConns[sched.nextIndex()] + pr := balancer.PickResult{SubConn: pickedSC.SubConn} + if !p.cfg.EnableOOBLoadReport { + pr.Done = func(info balancer.DoneInfo) { + if load, ok := info.ServerLoad.(*v3orcapb.OrcaLoadReport); ok && load != nil { + pickedSC.OnLoadReport(load) + } + } + } + return pr, nil +} + +// weightedSubConn is the wrapper of a subconn that holds the subconn and its +// weight (and other parameters relevant to computing the effective weight). +// When needed, it also tracks connectivity state, listens for metrics updates +// by implementing the orca.OOBListener interface and manages that listener. +type weightedSubConn struct { + balancer.SubConn + logger *grpclog.PrefixLogger + + // The following fields are only accessed on calls into the LB policy, and + // do not need a mutex. + connectivityState connectivity.State + stopORCAListener func() + + // The following fields are accessed asynchronously and are protected by + // mu. Note that mu may not be held when calling into the stopORCAListener + // or when registering a new listener, as those calls require the ORCA + // producer mu which is held when calling the listener, and the listener + // holds mu. + mu sync.Mutex + weightVal float64 + nonEmptySince time.Time + lastUpdated time.Time + cfg *lbConfig +} + +func (w *weightedSubConn) OnLoadReport(load *v3orcapb.OrcaLoadReport) { + if w.logger.V(2) { + w.logger.Infof("Received load report for subchannel %v: %v", w.SubConn, load) + } + // Update weights of this subchannel according to the reported load + utilization := load.ApplicationUtilization + if utilization == 0 { + utilization = load.CpuUtilization + } + if utilization == 0 || load.RpsFractional == 0 { + if w.logger.V(2) { + w.logger.Infof("Ignoring empty load report for subchannel %v", w.SubConn) + } + return + } + + w.mu.Lock() + defer w.mu.Unlock() + + errorRate := load.Eps / load.RpsFractional + w.weightVal = load.RpsFractional / (utilization + errorRate*w.cfg.ErrorUtilizationPenalty) + if w.logger.V(2) { + w.logger.Infof("New weight for subchannel %v: %v", w.SubConn, w.weightVal) + } + + w.lastUpdated = internal.TimeNow() + if w.nonEmptySince == (time.Time{}) { + w.nonEmptySince = w.lastUpdated + } +} + +// updateConfig updates the parameters of the WRR policy and +// stops/starts/restarts the ORCA OOB listener. +func (w *weightedSubConn) updateConfig(cfg *lbConfig) { + w.mu.Lock() + oldCfg := w.cfg + w.cfg = cfg + w.mu.Unlock() + + newPeriod := cfg.OOBReportingPeriod + if cfg.EnableOOBLoadReport == oldCfg.EnableOOBLoadReport && + newPeriod == oldCfg.OOBReportingPeriod { + // Load reporting wasn't enabled before or after, or load reporting was + // enabled before and after, and had the same period. (Note that with + // load reporting disabled, OOBReportingPeriod is always 0.) + return + } + // (Optionally stop and) start the listener to use the new config's + // settings for OOB reporting. + + if w.stopORCAListener != nil { + w.stopORCAListener() + } + if !cfg.EnableOOBLoadReport { + w.stopORCAListener = nil + return + } + if w.logger.V(2) { + w.logger.Infof("Registering ORCA listener for %v with interval %v", w.SubConn, newPeriod) + } + opts := orca.OOBListenerOptions{ReportInterval: time.Duration(newPeriod)} + w.stopORCAListener = orca.RegisterOOBListener(w.SubConn, w, opts) +} + +func (w *weightedSubConn) updateConnectivityState(cs connectivity.State) connectivity.State { + switch cs { + case connectivity.Idle: + // Always reconnect when idle. + w.SubConn.Connect() + case connectivity.Ready: + // If we transition back to READY state, reset nonEmptySince so that we + // apply the blackout period after we start receiving load data. Note + // that we cannot guarantee that we will never receive lingering + // callbacks for backend metric reports from the previous connection + // after the new connection has been established, but they should be + // masked by new backend metric reports from the new connection by the + // time the blackout period ends. + w.mu.Lock() + w.nonEmptySince = time.Time{} + w.mu.Unlock() + case connectivity.Shutdown: + if w.stopORCAListener != nil { + w.stopORCAListener() + } + } + + oldCS := w.connectivityState + + if oldCS == connectivity.TransientFailure && + (cs == connectivity.Connecting || cs == connectivity.Idle) { + // Once a subconn enters TRANSIENT_FAILURE, ignore subsequent IDLE or + // CONNECTING transitions to prevent the aggregated state from being + // always CONNECTING when many backends exist but are all down. + return oldCS + } + + w.connectivityState = cs + + return oldCS +} + +// weight returns the current effective weight of the subconn, taking into +// account the parameters. Returns 0 for blacked out or expired data, which +// will cause the backend weight to be treated as the mean of the weights of +// the other backends. +func (w *weightedSubConn) weight(now time.Time, weightExpirationPeriod, blackoutPeriod time.Duration) float64 { + w.mu.Lock() + defer w.mu.Unlock() + // If the most recent update was longer ago than the expiration period, + // reset nonEmptySince so that we apply the blackout period again if we + // start getting data again in the future, and return 0. + if now.Sub(w.lastUpdated) >= weightExpirationPeriod { + w.nonEmptySince = time.Time{} + return 0 + } + // If we don't have at least blackoutPeriod worth of data, return 0. + if blackoutPeriod != 0 && (w.nonEmptySince == (time.Time{}) || now.Sub(w.nonEmptySince) < blackoutPeriod) { + return 0 + } + return w.weightVal +} diff --git a/balancer/weightedroundrobin/balancer_test.go b/balancer/weightedroundrobin/balancer_test.go new file mode 100644 index 000000000000..1d67bcf1f008 --- /dev/null +++ b/balancer/weightedroundrobin/balancer_test.go @@ -0,0 +1,756 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package weightedroundrobin_test + +import ( + "context" + "encoding/json" + "fmt" + "sync" + "sync/atomic" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils/roundrobin" + "google.golang.org/grpc/orca" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/resolver" + + wrr "google.golang.org/grpc/balancer/weightedroundrobin" + iwrr "google.golang.org/grpc/balancer/weightedroundrobin/internal" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +const defaultTestTimeout = 10 * time.Second +const weightUpdatePeriod = 50 * time.Millisecond +const weightExpirationPeriod = time.Minute +const oobReportingInterval = 10 * time.Millisecond + +func init() { + iwrr.AllowAnyWeightUpdatePeriod = true +} + +func boolp(b bool) *bool { return &b } +func float64p(f float64) *float64 { return &f } +func stringp(s string) *string { return &s } + +var ( + perCallConfig = iwrr.LBConfig{ + EnableOOBLoadReport: boolp(false), + OOBReportingPeriod: stringp("0.005s"), + BlackoutPeriod: stringp("0s"), + WeightExpirationPeriod: stringp("60s"), + WeightUpdatePeriod: stringp(".050s"), + ErrorUtilizationPenalty: float64p(0), + } + oobConfig = iwrr.LBConfig{ + EnableOOBLoadReport: boolp(true), + OOBReportingPeriod: stringp("0.005s"), + BlackoutPeriod: stringp("0s"), + WeightExpirationPeriod: stringp("60s"), + WeightUpdatePeriod: stringp(".050s"), + ErrorUtilizationPenalty: float64p(0), + } +) + +type testServer struct { + *stubserver.StubServer + + oobMetrics orca.ServerMetricsRecorder // Attached to the OOB stream. + callMetrics orca.CallMetricsRecorder // Attached to per-call metrics. +} + +type reportType int + +const ( + reportNone reportType = iota + reportOOB + reportCall + reportBoth +) + +func startServer(t *testing.T, r reportType) *testServer { + t.Helper() + + smr := orca.NewServerMetricsRecorder() + cmr := orca.NewServerMetricsRecorder().(orca.CallMetricsRecorder) + + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { + if r := orca.CallMetricsRecorderFromContext(ctx); r != nil { + // Copy metrics from what the test set in cmr into r. + sm := cmr.(orca.ServerMetricsProvider).ServerMetrics() + r.SetApplicationUtilization(sm.AppUtilization) + r.SetQPS(sm.QPS) + r.SetEPS(sm.EPS) + } + return &testpb.Empty{}, nil + }, + } + + var sopts []grpc.ServerOption + if r == reportCall || r == reportBoth { + sopts = append(sopts, orca.CallMetricsServerOption(nil)) + } + + if r == reportOOB || r == reportBoth { + oso := orca.ServiceOptions{ + ServerMetricsProvider: smr, + MinReportingInterval: 10 * time.Millisecond, + } + internal.ORCAAllowAnyMinReportingInterval.(func(so *orca.ServiceOptions))(&oso) + sopts = append(sopts, stubserver.RegisterServiceServerOption(func(s *grpc.Server) { + if err := orca.Register(s, oso); err != nil { + t.Fatalf("Failed to register orca service: %v", err) + } + })) + } + + if err := ss.StartServer(sopts...); err != nil { + t.Fatalf("Error starting server: %v", err) + } + t.Cleanup(ss.Stop) + + return &testServer{ + StubServer: ss, + oobMetrics: smr, + callMetrics: cmr, + } +} + +func svcConfig(t *testing.T, wrrCfg iwrr.LBConfig) string { + t.Helper() + m, err := json.Marshal(wrrCfg) + if err != nil { + t.Fatalf("Error marshaling JSON %v: %v", wrrCfg, err) + } + sc := fmt.Sprintf(`{"loadBalancingConfig": [ {%q:%v} ] }`, wrr.Name, string(m)) + t.Logf("Marshaled service config: %v", sc) + return sc +} + +// Tests basic functionality with one address. With only one address, load +// reporting doesn't affect routing at all. +func (s) TestBalancer_OneAddress(t *testing.T) { + testCases := []struct { + rt reportType + cfg iwrr.LBConfig + }{ + {rt: reportNone, cfg: perCallConfig}, + {rt: reportCall, cfg: perCallConfig}, + {rt: reportOOB, cfg: oobConfig}, + } + + for _, tc := range testCases { + t.Run(fmt.Sprintf("reportType:%v", tc.rt), func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + srv := startServer(t, tc.rt) + + sc := svcConfig(t, tc.cfg) + if err := srv.StartClient(grpc.WithDefaultServiceConfig(sc)); err != nil { + t.Fatalf("Error starting client: %v", err) + } + + // Perform many RPCs to ensure the LB policy works with 1 address. + for i := 0; i < 100; i++ { + srv.callMetrics.SetQPS(float64(i)) + srv.oobMetrics.SetQPS(float64(i)) + if _, err := srv.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("Error from EmptyCall: %v", err) + } + time.Sleep(time.Millisecond) // Delay; test will run 100ms and should perform ~10 weight updates + } + }) + } +} + +// Tests two addresses with ORCA reporting disabled (should fall back to pure +// RR). +func (s) TestBalancer_TwoAddresses_ReportingDisabled(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + srv1 := startServer(t, reportNone) + srv2 := startServer(t, reportNone) + + sc := svcConfig(t, perCallConfig) + if err := srv1.StartClient(grpc.WithDefaultServiceConfig(sc)); err != nil { + t.Fatalf("Error starting client: %v", err) + } + addrs := []resolver.Address{{Addr: srv1.Address}, {Addr: srv2.Address}} + srv1.R.UpdateState(resolver.State{Addresses: addrs}) + + // Perform many RPCs to ensure the LB policy works with 2 addresses. + for i := 0; i < 20; i++ { + roundrobin.CheckRoundRobinRPCs(ctx, srv1.Client, addrs) + } +} + +// Tests two addresses with per-call ORCA reporting enabled. Checks the +// backends are called in the appropriate ratios. +func (s) TestBalancer_TwoAddresses_ReportingEnabledPerCall(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + srv1 := startServer(t, reportCall) + srv2 := startServer(t, reportCall) + + // srv1 starts loaded and srv2 starts without load; ensure RPCs are routed + // disproportionately to srv2 (10:1). + srv1.callMetrics.SetQPS(10.0) + srv1.callMetrics.SetApplicationUtilization(1.0) + + srv2.callMetrics.SetQPS(10.0) + srv2.callMetrics.SetApplicationUtilization(.1) + + sc := svcConfig(t, perCallConfig) + if err := srv1.StartClient(grpc.WithDefaultServiceConfig(sc)); err != nil { + t.Fatalf("Error starting client: %v", err) + } + addrs := []resolver.Address{{Addr: srv1.Address}, {Addr: srv2.Address}} + srv1.R.UpdateState(resolver.State{Addresses: addrs}) + + // Call each backend once to ensure the weights have been received. + ensureReached(ctx, t, srv1.Client, 2) + + // Wait for the weight update period to allow the new weights to be processed. + time.Sleep(weightUpdatePeriod) + checkWeights(ctx, t, srvWeight{srv1, 1}, srvWeight{srv2, 10}) +} + +// Tests two addresses with OOB ORCA reporting enabled. Checks the backends +// are called in the appropriate ratios. +func (s) TestBalancer_TwoAddresses_ReportingEnabledOOB(t *testing.T) { + testCases := []struct { + name string + utilSetter func(orca.ServerMetricsRecorder, float64) + }{{ + name: "application_utilization", + utilSetter: func(smr orca.ServerMetricsRecorder, val float64) { + smr.SetApplicationUtilization(val) + }, + }, { + name: "cpu_utilization", + utilSetter: func(smr orca.ServerMetricsRecorder, val float64) { + smr.SetCPUUtilization(val) + }, + }, { + name: "application over cpu", + utilSetter: func(smr orca.ServerMetricsRecorder, val float64) { + smr.SetApplicationUtilization(val) + smr.SetCPUUtilization(2.0) // ignored because ApplicationUtilization is set + }, + }} + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + srv1 := startServer(t, reportOOB) + srv2 := startServer(t, reportOOB) + + // srv1 starts loaded and srv2 starts without load; ensure RPCs are routed + // disproportionately to srv2 (10:1). + srv1.oobMetrics.SetQPS(10.0) + tc.utilSetter(srv1.oobMetrics, 1.0) + + srv2.oobMetrics.SetQPS(10.0) + tc.utilSetter(srv2.oobMetrics, 0.1) + + sc := svcConfig(t, oobConfig) + if err := srv1.StartClient(grpc.WithDefaultServiceConfig(sc)); err != nil { + t.Fatalf("Error starting client: %v", err) + } + addrs := []resolver.Address{{Addr: srv1.Address}, {Addr: srv2.Address}} + srv1.R.UpdateState(resolver.State{Addresses: addrs}) + + // Call each backend once to ensure the weights have been received. + ensureReached(ctx, t, srv1.Client, 2) + + // Wait for the weight update period to allow the new weights to be processed. + time.Sleep(weightUpdatePeriod) + checkWeights(ctx, t, srvWeight{srv1, 1}, srvWeight{srv2, 10}) + }) + } +} + +// Tests two addresses with OOB ORCA reporting enabled, where the reports +// change over time. Checks the backends are called in the appropriate ratios +// before and after modifying the reports. +func (s) TestBalancer_TwoAddresses_UpdateLoads(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + srv1 := startServer(t, reportOOB) + srv2 := startServer(t, reportOOB) + + // srv1 starts loaded and srv2 starts without load; ensure RPCs are routed + // disproportionately to srv2 (10:1). + srv1.oobMetrics.SetQPS(10.0) + srv1.oobMetrics.SetApplicationUtilization(1.0) + + srv2.oobMetrics.SetQPS(10.0) + srv2.oobMetrics.SetApplicationUtilization(.1) + + sc := svcConfig(t, oobConfig) + if err := srv1.StartClient(grpc.WithDefaultServiceConfig(sc)); err != nil { + t.Fatalf("Error starting client: %v", err) + } + addrs := []resolver.Address{{Addr: srv1.Address}, {Addr: srv2.Address}} + srv1.R.UpdateState(resolver.State{Addresses: addrs}) + + // Call each backend once to ensure the weights have been received. + ensureReached(ctx, t, srv1.Client, 2) + + // Wait for the weight update period to allow the new weights to be processed. + time.Sleep(weightUpdatePeriod) + checkWeights(ctx, t, srvWeight{srv1, 1}, srvWeight{srv2, 10}) + + // Update the loads so srv2 is loaded and srv1 is not; ensure RPCs are + // routed disproportionately to srv1. + srv1.oobMetrics.SetQPS(10.0) + srv1.oobMetrics.SetApplicationUtilization(.1) + + srv2.oobMetrics.SetQPS(10.0) + srv2.oobMetrics.SetApplicationUtilization(1.0) + + // Wait for the weight update period to allow the new weights to be processed. + time.Sleep(weightUpdatePeriod + oobReportingInterval) + checkWeights(ctx, t, srvWeight{srv1, 10}, srvWeight{srv2, 1}) +} + +// Tests two addresses with OOB ORCA reporting enabled, then with switching to +// per-call reporting. Checks the backends are called in the appropriate +// ratios before and after the change. +func (s) TestBalancer_TwoAddresses_OOBThenPerCall(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + srv1 := startServer(t, reportBoth) + srv2 := startServer(t, reportBoth) + + // srv1 starts loaded and srv2 starts without load; ensure RPCs are routed + // disproportionately to srv2 (10:1). + srv1.oobMetrics.SetQPS(10.0) + srv1.oobMetrics.SetApplicationUtilization(1.0) + + srv2.oobMetrics.SetQPS(10.0) + srv2.oobMetrics.SetApplicationUtilization(.1) + + // For per-call metrics (not used initially), srv2 reports that it is + // loaded and srv1 reports low load. After confirming OOB works, switch to + // per-call and confirm the new routing weights are applied. + srv1.callMetrics.SetQPS(10.0) + srv1.callMetrics.SetApplicationUtilization(.1) + + srv2.callMetrics.SetQPS(10.0) + srv2.callMetrics.SetApplicationUtilization(1.0) + + sc := svcConfig(t, oobConfig) + if err := srv1.StartClient(grpc.WithDefaultServiceConfig(sc)); err != nil { + t.Fatalf("Error starting client: %v", err) + } + addrs := []resolver.Address{{Addr: srv1.Address}, {Addr: srv2.Address}} + srv1.R.UpdateState(resolver.State{Addresses: addrs}) + + // Call each backend once to ensure the weights have been received. + ensureReached(ctx, t, srv1.Client, 2) + + // Wait for the weight update period to allow the new weights to be processed. + time.Sleep(weightUpdatePeriod) + checkWeights(ctx, t, srvWeight{srv1, 1}, srvWeight{srv2, 10}) + + // Update to per-call weights. + c := svcConfig(t, perCallConfig) + parsedCfg := srv1.R.CC.ParseServiceConfig(c) + if parsedCfg.Err != nil { + panic(fmt.Sprintf("Error parsing config %q: %v", c, parsedCfg.Err)) + } + srv1.R.UpdateState(resolver.State{Addresses: addrs, ServiceConfig: parsedCfg}) + + // Wait for the weight update period to allow the new weights to be processed. + time.Sleep(weightUpdatePeriod) + checkWeights(ctx, t, srvWeight{srv1, 10}, srvWeight{srv2, 1}) +} + +// Tests two addresses with OOB ORCA reporting enabled and a non-zero error +// penalty applied. +func (s) TestBalancer_TwoAddresses_ErrorPenalty(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + srv1 := startServer(t, reportOOB) + srv2 := startServer(t, reportOOB) + + // srv1 starts loaded and srv2 starts without load; ensure RPCs are routed + // disproportionately to srv2 (10:1). EPS values are set (but ignored + // initially due to ErrorUtilizationPenalty=0). Later EUP will be updated + // to 0.9 which will cause the weights to be equal and RPCs to be routed + // 50/50. + srv1.oobMetrics.SetQPS(10.0) + srv1.oobMetrics.SetApplicationUtilization(1.0) + srv1.oobMetrics.SetEPS(0) + // srv1 weight before: 10.0 / 1.0 = 10.0 + // srv1 weight after: 10.0 / 1.0 = 10.0 + + srv2.oobMetrics.SetQPS(10.0) + srv2.oobMetrics.SetApplicationUtilization(.1) + srv2.oobMetrics.SetEPS(10.0) + // srv2 weight before: 10.0 / 0.1 = 100.0 + // srv2 weight after: 10.0 / 1.0 = 10.0 + + sc := svcConfig(t, oobConfig) + if err := srv1.StartClient(grpc.WithDefaultServiceConfig(sc)); err != nil { + t.Fatalf("Error starting client: %v", err) + } + addrs := []resolver.Address{{Addr: srv1.Address}, {Addr: srv2.Address}} + srv1.R.UpdateState(resolver.State{Addresses: addrs}) + + // Call each backend once to ensure the weights have been received. + ensureReached(ctx, t, srv1.Client, 2) + + // Wait for the weight update period to allow the new weights to be processed. + time.Sleep(weightUpdatePeriod) + checkWeights(ctx, t, srvWeight{srv1, 1}, srvWeight{srv2, 10}) + + // Update to include an error penalty in the weights. + newCfg := oobConfig + newCfg.ErrorUtilizationPenalty = float64p(0.9) + c := svcConfig(t, newCfg) + parsedCfg := srv1.R.CC.ParseServiceConfig(c) + if parsedCfg.Err != nil { + panic(fmt.Sprintf("Error parsing config %q: %v", c, parsedCfg.Err)) + } + srv1.R.UpdateState(resolver.State{Addresses: addrs, ServiceConfig: parsedCfg}) + + // Wait for the weight update period to allow the new weights to be processed. + time.Sleep(weightUpdatePeriod + oobReportingInterval) + checkWeights(ctx, t, srvWeight{srv1, 1}, srvWeight{srv2, 1}) +} + +// Tests that the blackout period causes backends to use 0 as their weight +// (meaning to use the average weight) until the blackout period elapses. +func (s) TestBalancer_TwoAddresses_BlackoutPeriod(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + var mu sync.Mutex + start := time.Now() + now := start + setNow := func(t time.Time) { + mu.Lock() + defer mu.Unlock() + now = t + } + + setTimeNow(func() time.Time { + mu.Lock() + defer mu.Unlock() + return now + }) + t.Cleanup(func() { setTimeNow(time.Now) }) + + testCases := []struct { + blackoutPeriodCfg *string + blackoutPeriod time.Duration + }{{ + blackoutPeriodCfg: stringp("1s"), + blackoutPeriod: time.Second, + }, { + blackoutPeriodCfg: nil, + blackoutPeriod: 10 * time.Second, // the default + }} + for _, tc := range testCases { + setNow(start) + srv1 := startServer(t, reportOOB) + srv2 := startServer(t, reportOOB) + + // srv1 starts loaded and srv2 starts without load; ensure RPCs are routed + // disproportionately to srv2 (10:1). + srv1.oobMetrics.SetQPS(10.0) + srv1.oobMetrics.SetApplicationUtilization(1.0) + + srv2.oobMetrics.SetQPS(10.0) + srv2.oobMetrics.SetApplicationUtilization(.1) + + cfg := oobConfig + cfg.BlackoutPeriod = tc.blackoutPeriodCfg + sc := svcConfig(t, cfg) + if err := srv1.StartClient(grpc.WithDefaultServiceConfig(sc)); err != nil { + t.Fatalf("Error starting client: %v", err) + } + addrs := []resolver.Address{{Addr: srv1.Address}, {Addr: srv2.Address}} + srv1.R.UpdateState(resolver.State{Addresses: addrs}) + + // Call each backend once to ensure the weights have been received. + ensureReached(ctx, t, srv1.Client, 2) + + // Wait for the weight update period to allow the new weights to be processed. + time.Sleep(weightUpdatePeriod) + // During the blackout period (1s) we should route roughly 50/50. + checkWeights(ctx, t, srvWeight{srv1, 1}, srvWeight{srv2, 1}) + + // Advance time to right before the blackout period ends and the weights + // should still be zero. + setNow(start.Add(tc.blackoutPeriod - time.Nanosecond)) + // Wait for the weight update period to allow the new weights to be processed. + time.Sleep(weightUpdatePeriod) + checkWeights(ctx, t, srvWeight{srv1, 1}, srvWeight{srv2, 1}) + + // Advance time to right after the blackout period ends and the weights + // should now activate. + setNow(start.Add(tc.blackoutPeriod)) + // Wait for the weight update period to allow the new weights to be processed. + time.Sleep(weightUpdatePeriod) + checkWeights(ctx, t, srvWeight{srv1, 1}, srvWeight{srv2, 10}) + } +} + +// Tests that the weight expiration period causes backends to use 0 as their +// weight (meaning to use the average weight) once the expiration period +// elapses. +func (s) TestBalancer_TwoAddresses_WeightExpiration(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + var mu sync.Mutex + start := time.Now() + now := start + setNow := func(t time.Time) { + mu.Lock() + defer mu.Unlock() + now = t + } + setTimeNow(func() time.Time { + mu.Lock() + defer mu.Unlock() + return now + }) + t.Cleanup(func() { setTimeNow(time.Now) }) + + srv1 := startServer(t, reportBoth) + srv2 := startServer(t, reportBoth) + + // srv1 starts loaded and srv2 starts without load; ensure RPCs are routed + // disproportionately to srv2 (10:1). Because the OOB reporting interval + // is 1 minute but the weights expire in 1 second, routing will go to 50/50 + // after the weights expire. + srv1.oobMetrics.SetQPS(10.0) + srv1.oobMetrics.SetApplicationUtilization(1.0) + + srv2.oobMetrics.SetQPS(10.0) + srv2.oobMetrics.SetApplicationUtilization(.1) + + cfg := oobConfig + cfg.OOBReportingPeriod = stringp("60s") + sc := svcConfig(t, cfg) + if err := srv1.StartClient(grpc.WithDefaultServiceConfig(sc)); err != nil { + t.Fatalf("Error starting client: %v", err) + } + addrs := []resolver.Address{{Addr: srv1.Address}, {Addr: srv2.Address}} + srv1.R.UpdateState(resolver.State{Addresses: addrs}) + + // Call each backend once to ensure the weights have been received. + ensureReached(ctx, t, srv1.Client, 2) + + // Wait for the weight update period to allow the new weights to be processed. + time.Sleep(weightUpdatePeriod) + checkWeights(ctx, t, srvWeight{srv1, 1}, srvWeight{srv2, 10}) + + // Advance what time.Now returns to the weight expiration time minus 1s to + // ensure all weights are still honored. + setNow(start.Add(weightExpirationPeriod - time.Second)) + + // Wait for the weight update period to allow the new weights to be processed. + time.Sleep(weightUpdatePeriod) + checkWeights(ctx, t, srvWeight{srv1, 1}, srvWeight{srv2, 10}) + + // Advance what time.Now returns to the weight expiration time plus 1s to + // ensure all weights expired and addresses are routed evenly. + setNow(start.Add(weightExpirationPeriod + time.Second)) + + // Wait for the weight expiration period so the weights have expired. + time.Sleep(weightUpdatePeriod) + checkWeights(ctx, t, srvWeight{srv1, 1}, srvWeight{srv2, 1}) +} + +// Tests logic surrounding subchannel management. +func (s) TestBalancer_AddressesChanging(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + srv1 := startServer(t, reportBoth) + srv2 := startServer(t, reportBoth) + srv3 := startServer(t, reportBoth) + srv4 := startServer(t, reportBoth) + + // srv1: weight 10 + srv1.oobMetrics.SetQPS(10.0) + srv1.oobMetrics.SetApplicationUtilization(1.0) + // srv2: weight 100 + srv2.oobMetrics.SetQPS(10.0) + srv2.oobMetrics.SetApplicationUtilization(.1) + // srv3: weight 20 + srv3.oobMetrics.SetQPS(20.0) + srv3.oobMetrics.SetApplicationUtilization(1.0) + // srv4: weight 200 + srv4.oobMetrics.SetQPS(20.0) + srv4.oobMetrics.SetApplicationUtilization(.1) + + sc := svcConfig(t, oobConfig) + if err := srv1.StartClient(grpc.WithDefaultServiceConfig(sc)); err != nil { + t.Fatalf("Error starting client: %v", err) + } + srv2.Client = srv1.Client + addrs := []resolver.Address{{Addr: srv1.Address}, {Addr: srv2.Address}, {Addr: srv3.Address}} + srv1.R.UpdateState(resolver.State{Addresses: addrs}) + + // Call each backend once to ensure the weights have been received. + ensureReached(ctx, t, srv1.Client, 3) + time.Sleep(weightUpdatePeriod) + checkWeights(ctx, t, srvWeight{srv1, 1}, srvWeight{srv2, 10}, srvWeight{srv3, 2}) + + // Add backend 4 + addrs = append(addrs, resolver.Address{Addr: srv4.Address}) + srv1.R.UpdateState(resolver.State{Addresses: addrs}) + time.Sleep(weightUpdatePeriod) + checkWeights(ctx, t, srvWeight{srv1, 1}, srvWeight{srv2, 10}, srvWeight{srv3, 2}, srvWeight{srv4, 20}) + + // Shutdown backend 3. RPCs will no longer be routed to it. + srv3.Stop() + time.Sleep(weightUpdatePeriod) + checkWeights(ctx, t, srvWeight{srv1, 1}, srvWeight{srv2, 10}, srvWeight{srv4, 20}) + + // Remove addresses 2 and 3. RPCs will no longer be routed to 2 either. + addrs = []resolver.Address{{Addr: srv1.Address}, {Addr: srv4.Address}} + srv1.R.UpdateState(resolver.State{Addresses: addrs}) + time.Sleep(weightUpdatePeriod) + checkWeights(ctx, t, srvWeight{srv1, 1}, srvWeight{srv4, 20}) + + // Re-add 2 and remove the rest. + addrs = []resolver.Address{{Addr: srv2.Address}} + srv1.R.UpdateState(resolver.State{Addresses: addrs}) + time.Sleep(weightUpdatePeriod) + checkWeights(ctx, t, srvWeight{srv2, 10}) + + // Re-add 4. + addrs = append(addrs, resolver.Address{Addr: srv4.Address}) + srv1.R.UpdateState(resolver.State{Addresses: addrs}) + time.Sleep(weightUpdatePeriod) + checkWeights(ctx, t, srvWeight{srv2, 10}, srvWeight{srv4, 20}) +} + +func ensureReached(ctx context.Context, t *testing.T, c testgrpc.TestServiceClient, n int) { + t.Helper() + reached := make(map[string]struct{}) + for len(reached) != n { + var peer peer.Peer + if _, err := c.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&peer)); err != nil { + t.Fatalf("Error from EmptyCall: %v", err) + } + reached[peer.Addr.String()] = struct{}{} + } +} + +type srvWeight struct { + srv *testServer + w int +} + +const rrIterations = 100 + +// checkWeights does rrIterations RPCs and expects the different backends to be +// routed in a ratio as deterimined by the srvWeights passed in. Allows for +// some variance (+/- 2 RPCs per backend). +func checkWeights(ctx context.Context, t *testing.T, sws ...srvWeight) { + t.Helper() + + c := sws[0].srv.Client + + // Replace the weights with approximate counts of RPCs wanted given the + // iterations performed. + weightSum := 0 + for _, sw := range sws { + weightSum += sw.w + } + for i := range sws { + sws[i].w = rrIterations * sws[i].w / weightSum + } + + for attempts := 0; attempts < 10; attempts++ { + serverCounts := make(map[string]int) + for i := 0; i < rrIterations; i++ { + var peer peer.Peer + if _, err := c.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&peer)); err != nil { + t.Fatalf("Error from EmptyCall: %v; timed out waiting for weighted RR behavior?", err) + } + serverCounts[peer.Addr.String()]++ + } + if len(serverCounts) != len(sws) { + continue + } + success := true + for _, sw := range sws { + c := serverCounts[sw.srv.Address] + if c < sw.w-2 || c > sw.w+2 { + success = false + break + } + } + if success { + t.Logf("Passed iteration %v; counts: %v", attempts, serverCounts) + return + } + t.Logf("Failed iteration %v; counts: %v; want %+v", attempts, serverCounts, sws) + time.Sleep(5 * time.Millisecond) + } + t.Fatalf("Failed to route RPCs with proper ratio") +} + +func init() { + setTimeNow(time.Now) + iwrr.TimeNow = timeNow +} + +var timeNowFunc atomic.Value // func() time.Time + +func timeNow() time.Time { + return timeNowFunc.Load().(func() time.Time)() +} + +func setTimeNow(f func() time.Time) { + timeNowFunc.Store(f) +} diff --git a/balancer/weightedroundrobin/config.go b/balancer/weightedroundrobin/config.go new file mode 100644 index 000000000000..38f89d32fb43 --- /dev/null +++ b/balancer/weightedroundrobin/config.go @@ -0,0 +1,59 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package weightedroundrobin + +import ( + iserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/serviceconfig" +) + +type lbConfig struct { + serviceconfig.LoadBalancingConfig `json:"-"` + + // Whether to enable out-of-band utilization reporting collection from the + // endpoints. By default, per-request utilization reporting is used. + EnableOOBLoadReport bool `json:"enableOobLoadReport,omitempty"` + + // Load reporting interval to request from the server. Note that the + // server may not provide reports as frequently as the client requests. + // Used only when enable_oob_load_report is true. Default is 10 seconds. + OOBReportingPeriod iserviceconfig.Duration `json:"oobReportingPeriod,omitempty"` + + // A given endpoint must report load metrics continuously for at least this + // long before the endpoint weight will be used. This avoids churn when + // the set of endpoint addresses changes. Takes effect both immediately + // after we establish a connection to an endpoint and after + // weight_expiration_period has caused us to stop using the most recent + // load metrics. Default is 10 seconds. + BlackoutPeriod iserviceconfig.Duration `json:"blackoutPeriod,omitempty"` + + // If a given endpoint has not reported load metrics in this long, + // then we stop using the reported weight. This ensures that we do + // not continue to use very stale weights. Once we stop using a stale + // value, if we later start seeing fresh reports again, the + // blackout_period applies. Defaults to 3 minutes. + WeightExpirationPeriod iserviceconfig.Duration `json:"weightExpirationPeriod,omitempty"` + + // How often endpoint weights are recalculated. Default is 1 second. + WeightUpdatePeriod iserviceconfig.Duration `json:"weightUpdatePeriod,omitempty"` + + // The multiplier used to adjust endpoint weights with the error rate + // calculated as eps/qps. Default is 1.0. + ErrorUtilizationPenalty float64 `json:"errorUtilizationPenalty,omitempty"` +} diff --git a/balancer/weightedroundrobin/internal/internal.go b/balancer/weightedroundrobin/internal/internal.go new file mode 100644 index 000000000000..7b64fbf4e574 --- /dev/null +++ b/balancer/weightedroundrobin/internal/internal.go @@ -0,0 +1,44 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package internal allows for easier testing of the weightedroundrobin +// package. +package internal + +import ( + "time" +) + +// AllowAnyWeightUpdatePeriod permits any setting of WeightUpdatePeriod for +// testing. Normally a minimum of 100ms is applied. +var AllowAnyWeightUpdatePeriod bool + +// LBConfig allows tests to produce a JSON form of the config from the struct +// instead of using a string. +type LBConfig struct { + EnableOOBLoadReport *bool `json:"enableOobLoadReport,omitempty"` + OOBReportingPeriod *string `json:"oobReportingPeriod,omitempty"` + BlackoutPeriod *string `json:"blackoutPeriod,omitempty"` + WeightExpirationPeriod *string `json:"weightExpirationPeriod,omitempty"` + WeightUpdatePeriod *string `json:"weightUpdatePeriod,omitempty"` + ErrorUtilizationPenalty *float64 `json:"errorUtilizationPenalty,omitempty"` +} + +// TimeNow can be overridden by tests to return a different value for the +// current iserviceconfig. +var TimeNow = time.Now diff --git a/balancer/weightedroundrobin/logging.go b/balancer/weightedroundrobin/logging.go new file mode 100644 index 000000000000..43184ca9ab91 --- /dev/null +++ b/balancer/weightedroundrobin/logging.go @@ -0,0 +1,34 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package weightedroundrobin + +import ( + "fmt" + + "google.golang.org/grpc/grpclog" + internalgrpclog "google.golang.org/grpc/internal/grpclog" +) + +const prefix = "[%p] " + +var logger = grpclog.Component("weighted-round-robin") + +func prefixLogger(p *wrrBalancer) *internalgrpclog.PrefixLogger { + return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p)) +} diff --git a/balancer/weightedroundrobin/scheduler.go b/balancer/weightedroundrobin/scheduler.go new file mode 100644 index 000000000000..e19428112e1e --- /dev/null +++ b/balancer/weightedroundrobin/scheduler.go @@ -0,0 +1,138 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package weightedroundrobin + +import ( + "math" +) + +type scheduler interface { + nextIndex() int +} + +// newScheduler uses scWeights to create a new scheduler for selecting subconns +// in a picker. It will return a round robin implementation if at least +// len(scWeights)-1 are zero or there is only a single subconn, otherwise it +// will return an Earliest Deadline First (EDF) scheduler implementation that +// selects the subchannels according to their weights. +func newScheduler(scWeights []float64, inc func() uint32) scheduler { + n := len(scWeights) + if n == 0 { + return nil + } + if n == 1 { + return &rrScheduler{numSCs: 1, inc: inc} + } + sum := float64(0) + numZero := 0 + max := float64(0) + for _, w := range scWeights { + sum += w + if w > max { + max = w + } + if w == 0 { + numZero++ + } + } + if numZero >= n-1 { + return &rrScheduler{numSCs: uint32(n), inc: inc} + } + unscaledMean := sum / float64(n-numZero) + scalingFactor := maxWeight / max + mean := uint16(math.Round(scalingFactor * unscaledMean)) + + weights := make([]uint16, n) + allEqual := true + for i, w := range scWeights { + if w == 0 { + // Backends with weight = 0 use the mean. + weights[i] = mean + } else { + scaledWeight := uint16(math.Round(scalingFactor * w)) + weights[i] = scaledWeight + if scaledWeight != mean { + allEqual = false + } + } + } + + if allEqual { + return &rrScheduler{numSCs: uint32(n), inc: inc} + } + + logger.Infof("using edf scheduler with weights: %v", weights) + return &edfScheduler{weights: weights, inc: inc} +} + +const maxWeight = math.MaxUint16 + +// edfScheduler implements EDF using the same algorithm as grpc-c++ here: +// +// https://github.com/grpc/grpc/blob/master/src/core/ext/filters/client_channel/lb_policy/weighted_round_robin/static_stride_scheduler.cc +type edfScheduler struct { + inc func() uint32 + weights []uint16 +} + +// Returns the index in s.weights for the picker to choose. +func (s *edfScheduler) nextIndex() int { + const offset = maxWeight / 2 + + for { + idx := uint64(s.inc()) + + // The sequence number (idx) is split in two: the lower %n gives the + // index of the backend, and the rest gives the number of times we've + // iterated through all backends. `generation` is used to + // deterministically decide whether we pick or skip the backend on this + // iteration, in proportion to the backend's weight. + + backendIndex := idx % uint64(len(s.weights)) + generation := idx / uint64(len(s.weights)) + weight := uint64(s.weights[backendIndex]) + + // We pick a backend `weight` times per `maxWeight` generations. The + // multiply and modulus ~evenly spread out the picks for a given + // backend between different generations. The offset by `backendIndex` + // helps to reduce the chance of multiple consecutive non-picks: if we + // have two consecutive backends with an equal, say, 80% weight of the + // max, with no offset we would see 1/5 generations that skipped both. + // TODO(b/190488683): add test for offset efficacy. + mod := uint64(weight*generation+backendIndex*offset) % maxWeight + + if mod < maxWeight-weight { + continue + } + return int(backendIndex) + } +} + +// A simple RR scheduler to use for fallback when fewer than two backends have +// non-zero weights, or all backends have the the same weight, or when only one +// subconn exists. +type rrScheduler struct { + inc func() uint32 + numSCs uint32 +} + +func (s *rrScheduler) nextIndex() int { + idx := s.inc() + return int(idx % s.numSCs) +} diff --git a/balancer/weightedroundrobin/weightedroundrobin.go b/balancer/weightedroundrobin/weightedroundrobin.go index d232491aef28..8741fdad19dc 100644 --- a/balancer/weightedroundrobin/weightedroundrobin.go +++ b/balancer/weightedroundrobin/weightedroundrobin.go @@ -16,40 +16,54 @@ * */ -// Package weightedroundrobin defines a weighted roundrobin balancer. +// Package weightedroundrobin provides an implementation of the weighted round +// robin LB policy, as defined in [gRFC A58]. +// +// # Experimental +// +// Notice: This package is EXPERIMENTAL and may be changed or removed in a +// later release. +// +// [gRFC A58]: https://github.com/grpc/proposal/blob/master/A58-client-side-weighted-round-robin-lb-policy.md package weightedroundrobin import ( + "fmt" + "google.golang.org/grpc/resolver" ) -// Name is the name of weighted_round_robin balancer. -const Name = "weighted_round_robin" - -// attributeKey is the type used as the key to store AddrInfo in the Attributes -// field of resolver.Address. +// attributeKey is the type used as the key to store AddrInfo in the +// BalancerAttributes field of resolver.Address. type attributeKey struct{} -// AddrInfo will be stored inside Address metadata in order to use weighted -// roundrobin balancer. +// AddrInfo will be stored in the BalancerAttributes field of Address in order +// to use weighted roundrobin balancer. type AddrInfo struct { Weight uint32 } -// SetAddrInfo returns a copy of addr in which the Attributes field is updated -// with addrInfo. -// -// This is an EXPERIMENTAL API. +// Equal allows the values to be compared by Attributes.Equal. +func (a AddrInfo) Equal(o any) bool { + oa, ok := o.(AddrInfo) + return ok && oa.Weight == a.Weight +} + +// SetAddrInfo returns a copy of addr in which the BalancerAttributes field is +// updated with addrInfo. func SetAddrInfo(addr resolver.Address, addrInfo AddrInfo) resolver.Address { - addr.Attributes = addr.Attributes.WithValues(attributeKey{}, addrInfo) + addr.BalancerAttributes = addr.BalancerAttributes.WithValue(attributeKey{}, addrInfo) return addr } -// GetAddrInfo returns the AddrInfo stored in the Attributes fields of addr. -// -// This is an EXPERIMENTAL API. +// GetAddrInfo returns the AddrInfo stored in the BalancerAttributes field of +// addr. func GetAddrInfo(addr resolver.Address) AddrInfo { - v := addr.Attributes.Value(attributeKey{}) + v := addr.BalancerAttributes.Value(attributeKey{}) ai, _ := v.(AddrInfo) return ai } + +func (a AddrInfo) String() string { + return fmt.Sprintf("Weight: %d", a.Weight) +} diff --git a/balancer/weightedroundrobin/weightedwoundrobin_test.go b/balancer/weightedroundrobin/weightedroundrobin_test.go similarity index 97% rename from balancer/weightedroundrobin/weightedwoundrobin_test.go rename to balancer/weightedroundrobin/weightedroundrobin_test.go index aa46c449a13d..d83619da2e6a 100644 --- a/balancer/weightedroundrobin/weightedwoundrobin_test.go +++ b/balancer/weightedroundrobin/weightedroundrobin_test.go @@ -73,7 +73,7 @@ func TestAddrInfoToAndFromAttributes(t *testing.T) { } func TestGetAddInfoEmpty(t *testing.T) { - addr := resolver.Address{Attributes: attributes.New()} + addr := resolver.Address{} gotAddrInfo := GetAddrInfo(addr) wantAddrInfo := AddrInfo{} if !cmp.Equal(gotAddrInfo, wantAddrInfo) { diff --git a/xds/internal/balancer/weightedtarget/logging.go b/balancer/weightedtarget/logging.go similarity index 100% rename from xds/internal/balancer/weightedtarget/logging.go rename to balancer/weightedtarget/logging.go diff --git a/xds/internal/balancer/weightedtarget/weightedaggregator/aggregator.go b/balancer/weightedtarget/weightedaggregator/aggregator.go similarity index 63% rename from xds/internal/balancer/weightedtarget/weightedaggregator/aggregator.go rename to balancer/weightedtarget/weightedaggregator/aggregator.go index a77a7c3f495f..27279257ed13 100644 --- a/xds/internal/balancer/weightedtarget/weightedaggregator/aggregator.go +++ b/balancer/weightedtarget/weightedaggregator/aggregator.go @@ -34,7 +34,6 @@ import ( "google.golang.org/grpc/connectivity" "google.golang.org/grpc/internal/grpclog" "google.golang.org/grpc/internal/wrr" - "google.golang.org/grpc/xds/internal" ) type weightedPickerState struct { @@ -58,6 +57,8 @@ type Aggregator struct { logger *grpclog.PrefixLogger newWRR func() wrr.WRR + csEvltr *balancer.ConnectivityStateEvaluator + mu sync.Mutex // If started is false, no updates should be sent to the parent cc. A closed // sub-balancer could still send pickers to this aggregator. This makes sure @@ -68,7 +69,12 @@ type Aggregator struct { // started. // // If an ID is not in map, it's either removed or never added. - idToPickerState map[internal.LocalityID]*weightedPickerState + idToPickerState map[string]*weightedPickerState + // Set when UpdateState call propagation is paused. + pauseUpdateState bool + // Set when UpdateState call propagation is paused and an UpdateState call + // is suppressed. + needUpdateStateOnResume bool } // New creates a new weighted balancer state aggregator. @@ -77,11 +83,12 @@ func New(cc balancer.ClientConn, logger *grpclog.PrefixLogger, newWRR func() wrr cc: cc, logger: logger, newWRR: newWRR, - idToPickerState: make(map[internal.LocalityID]*weightedPickerState), + csEvltr: &balancer.ConnectivityStateEvaluator{}, + idToPickerState: make(map[string]*weightedPickerState), } } -// Start starts the aggregator. It can be called after Close to restart the +// Start starts the aggregator. It can be called after Stop to restart the // aggretator. func (wbsa *Aggregator) Start() { wbsa.mu.Lock() @@ -89,7 +96,7 @@ func (wbsa *Aggregator) Start() { wbsa.started = true } -// Stop stops the aggregator. When the aggregator is closed, it won't call +// Stop stops the aggregator. When the aggregator is stopped, it won't call // parent ClientConn to update balancer state. func (wbsa *Aggregator) Stop() { wbsa.mu.Lock() @@ -100,7 +107,7 @@ func (wbsa *Aggregator) Stop() { // Add adds a sub-balancer state with weight. It adds a place holder, and waits for // the real sub-balancer to update state. -func (wbsa *Aggregator) Add(id internal.LocalityID, weight uint32) { +func (wbsa *Aggregator) Add(id string, weight uint32) { wbsa.mu.Lock() defer wbsa.mu.Unlock() wbsa.idToPickerState[id] = &weightedPickerState{ @@ -114,25 +121,33 @@ func (wbsa *Aggregator) Add(id internal.LocalityID, weight uint32) { }, stateToAggregate: connectivity.Connecting, } + wbsa.csEvltr.RecordTransition(connectivity.Shutdown, connectivity.Connecting) + + wbsa.buildAndUpdateLocked() } // Remove removes the sub-balancer state. Future updates from this sub-balancer, // if any, will be ignored. -func (wbsa *Aggregator) Remove(id internal.LocalityID) { +func (wbsa *Aggregator) Remove(id string) { wbsa.mu.Lock() defer wbsa.mu.Unlock() if _, ok := wbsa.idToPickerState[id]; !ok { return } + // Setting the state of the deleted sub-balancer to Shutdown will get csEvltr + // to remove the previous state for any aggregated state evaluations. + // transitions to and from connectivity.Shutdown are ignored by csEvltr. + wbsa.csEvltr.RecordTransition(wbsa.idToPickerState[id].stateToAggregate, connectivity.Shutdown) // Remove id and picker from picker map. This also results in future updates // for this ID to be ignored. delete(wbsa.idToPickerState, id) + wbsa.buildAndUpdateLocked() } // UpdateWeight updates the weight for the given id. Note that this doesn't // trigger an update to the parent ClientConn. The caller should decide when // it's necessary, and call BuildAndUpdate. -func (wbsa *Aggregator) UpdateWeight(id internal.LocalityID, newWeight uint32) { +func (wbsa *Aggregator) UpdateWeight(id string, newWeight uint32) { wbsa.mu.Lock() defer wbsa.mu.Unlock() pState, ok := wbsa.idToPickerState[id] @@ -142,32 +157,60 @@ func (wbsa *Aggregator) UpdateWeight(id internal.LocalityID, newWeight uint32) { pState.weight = newWeight } +// PauseStateUpdates causes UpdateState calls to not propagate to the parent +// ClientConn. The last state will be remembered and propagated when +// ResumeStateUpdates is called. +func (wbsa *Aggregator) PauseStateUpdates() { + wbsa.mu.Lock() + defer wbsa.mu.Unlock() + wbsa.pauseUpdateState = true + wbsa.needUpdateStateOnResume = false +} + +// ResumeStateUpdates will resume propagating UpdateState calls to the parent, +// and call UpdateState on the parent if any UpdateState call was suppressed. +func (wbsa *Aggregator) ResumeStateUpdates() { + wbsa.mu.Lock() + defer wbsa.mu.Unlock() + wbsa.pauseUpdateState = false + if wbsa.needUpdateStateOnResume { + wbsa.cc.UpdateState(wbsa.build()) + } +} + +// NeedUpdateStateOnResume sets the UpdateStateOnResume bool to true, letting a +// picker update be sent once ResumeStateUpdates is called. +func (wbsa *Aggregator) NeedUpdateStateOnResume() { + wbsa.mu.Lock() + defer wbsa.mu.Unlock() + wbsa.needUpdateStateOnResume = true +} + // UpdateState is called to report a balancer state change from sub-balancer. // It's usually called by the balancer group. // // It calls parent ClientConn's UpdateState with the new aggregated state. -func (wbsa *Aggregator) UpdateState(id internal.LocalityID, newState balancer.State) { +func (wbsa *Aggregator) UpdateState(id string, newState balancer.State) { wbsa.mu.Lock() defer wbsa.mu.Unlock() - oldState, ok := wbsa.idToPickerState[id] + state, ok := wbsa.idToPickerState[id] if !ok { // All state starts with an entry in pickStateMap. If ID is not in map, // it's either removed, or never existed. return } - if !(oldState.state.ConnectivityState == connectivity.TransientFailure && newState.ConnectivityState == connectivity.Connecting) { + + if !(state.state.ConnectivityState == connectivity.TransientFailure && newState.ConnectivityState == connectivity.Connecting) { // If old state is TransientFailure, and new state is Connecting, don't // update the state, to prevent the aggregated state from being always // CONNECTING. Otherwise, stateToAggregate is the same as // state.ConnectivityState. - oldState.stateToAggregate = newState.ConnectivityState + wbsa.csEvltr.RecordTransition(state.stateToAggregate, newState.ConnectivityState) + state.stateToAggregate = newState.ConnectivityState } - oldState.state = newState + state.state = newState - if !wbsa.started { - return - } - wbsa.cc.UpdateState(wbsa.build()) + wbsa.buildAndUpdateLocked() } // clearState Reset everything to init state (Connecting) but keep the entry in @@ -184,14 +227,21 @@ func (wbsa *Aggregator) clearStates() { } } -// BuildAndUpdate combines the sub-state from each sub-balancer into one state, -// and update it to parent ClientConn. -func (wbsa *Aggregator) BuildAndUpdate() { - wbsa.mu.Lock() - defer wbsa.mu.Unlock() +// buildAndUpdateLocked aggregates the connectivity states of the sub-balancers, +// builds a new picker and sends an update to the parent ClientConn. +// +// Caller must hold wbsa.mu. +func (wbsa *Aggregator) buildAndUpdateLocked() { if !wbsa.started { return } + if wbsa.pauseUpdateState { + // If updates are paused, do not call UpdateState, but remember that we + // need to call it when they are resumed. + wbsa.needUpdateStateOnResume = true + return + } + wbsa.cc.UpdateState(wbsa.build()) } @@ -200,39 +250,34 @@ func (wbsa *Aggregator) BuildAndUpdate() { // Caller must hold wbsa.mu. func (wbsa *Aggregator) build() balancer.State { wbsa.logger.Infof("Child pickers with config: %+v", wbsa.idToPickerState) - m := wbsa.idToPickerState - var readyN, connectingN int - readyPickerWithWeights := make([]weightedPickerState, 0, len(m)) - for _, ps := range m { - switch ps.stateToAggregate { - case connectivity.Ready: - readyN++ - readyPickerWithWeights = append(readyPickerWithWeights, *ps) - case connectivity.Connecting: - connectingN++ - } - } - var aggregatedState connectivity.State - switch { - case readyN > 0: - aggregatedState = connectivity.Ready - case connectingN > 0: - aggregatedState = connectivity.Connecting - default: - aggregatedState = connectivity.TransientFailure - } // Make sure picker's return error is consistent with the aggregatedState. - var picker balancer.Picker - switch aggregatedState { - case connectivity.TransientFailure: - picker = base.NewErrPicker(balancer.ErrTransientFailure) + pickers := make([]weightedPickerState, 0, len(wbsa.idToPickerState)) + + switch aggState := wbsa.csEvltr.CurrentState(); aggState { case connectivity.Connecting: - picker = base.NewErrPicker(balancer.ErrNoSubConnAvailable) + return balancer.State{ + ConnectivityState: aggState, + Picker: base.NewErrPicker(balancer.ErrNoSubConnAvailable)} + case connectivity.TransientFailure: + // this means that all sub-balancers are now in TransientFailure. + for _, ps := range wbsa.idToPickerState { + pickers = append(pickers, *ps) + } + return balancer.State{ + ConnectivityState: aggState, + Picker: newWeightedPickerGroup(pickers, wbsa.newWRR)} default: - picker = newWeightedPickerGroup(readyPickerWithWeights, wbsa.newWRR) + for _, ps := range wbsa.idToPickerState { + if ps.stateToAggregate == connectivity.Ready { + pickers = append(pickers, *ps) + } + } + return balancer.State{ + ConnectivityState: aggState, + Picker: newWeightedPickerGroup(pickers, wbsa.newWRR)} } - return balancer.State{ConnectivityState: aggregatedState, Picker: picker} + } type weightedPickerGroup struct { diff --git a/balancer/weightedtarget/weightedtarget.go b/balancer/weightedtarget/weightedtarget.go new file mode 100644 index 000000000000..220f4e555674 --- /dev/null +++ b/balancer/weightedtarget/weightedtarget.go @@ -0,0 +1,183 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package weightedtarget implements the weighted_target balancer. +// +// All APIs in this package are experimental. +package weightedtarget + +import ( + "encoding/json" + "fmt" + "time" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/weightedtarget/weightedaggregator" + "google.golang.org/grpc/internal/balancergroup" + "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/hierarchy" + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/internal/wrr" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" +) + +// Name is the name of the weighted_target balancer. +const Name = "weighted_target_experimental" + +// NewRandomWRR is the WRR constructor used to pick sub-pickers from +// sub-balancers. It's to be modified in tests. +var NewRandomWRR = wrr.NewRandom + +func init() { + balancer.Register(bb{}) +} + +type bb struct{} + +func (bb) Build(cc balancer.ClientConn, bOpts balancer.BuildOptions) balancer.Balancer { + b := &weightedTargetBalancer{} + b.logger = prefixLogger(b) + b.stateAggregator = weightedaggregator.New(cc, b.logger, NewRandomWRR) + b.stateAggregator.Start() + b.bg = balancergroup.New(balancergroup.Options{ + CC: cc, + BuildOpts: bOpts, + StateAggregator: b.stateAggregator, + Logger: b.logger, + SubBalancerCloseTimeout: time.Duration(0), // Disable caching of removed child policies + }) + b.bg.Start() + b.logger.Infof("Created") + return b +} + +func (bb) Name() string { + return Name +} + +func (bb) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + return parseConfig(c) +} + +type weightedTargetBalancer struct { + logger *grpclog.PrefixLogger + + bg *balancergroup.BalancerGroup + stateAggregator *weightedaggregator.Aggregator + + targets map[string]Target +} + +// UpdateClientConnState takes the new targets in balancer group, +// creates/deletes sub-balancers and sends them update. addresses are split into +// groups based on hierarchy path. +func (b *weightedTargetBalancer) UpdateClientConnState(s balancer.ClientConnState) error { + b.logger.Infof("Received update from resolver, balancer config: %+v", pretty.ToJSON(s.BalancerConfig)) + newConfig, ok := s.BalancerConfig.(*LBConfig) + if !ok { + return fmt.Errorf("unexpected balancer config with type: %T", s.BalancerConfig) + } + addressesSplit := hierarchy.Group(s.ResolverState.Addresses) + + b.stateAggregator.PauseStateUpdates() + defer b.stateAggregator.ResumeStateUpdates() + + // Remove sub-pickers and sub-balancers that are not in the new config. + for name := range b.targets { + if _, ok := newConfig.Targets[name]; !ok { + b.stateAggregator.Remove(name) + b.bg.Remove(name) + } + } + + // For sub-balancers in the new config + // - if it's new. add to balancer group, + // - if it's old, but has a new weight, update weight in balancer group. + // + // For all sub-balancers, forward the address/balancer config update. + for name, newT := range newConfig.Targets { + oldT, ok := b.targets[name] + if !ok { + // If this is a new sub-balancer, add weights to the picker map. + b.stateAggregator.Add(name, newT.Weight) + // Then add to the balancer group. + b.bg.Add(name, balancer.Get(newT.ChildPolicy.Name)) + // Not trigger a state/picker update. Wait for the new sub-balancer + // to send its updates. + } else if newT.ChildPolicy.Name != oldT.ChildPolicy.Name { + // If the child policy name is different, remove from balancer group + // and re-add. + b.stateAggregator.Remove(name) + b.bg.Remove(name) + b.stateAggregator.Add(name, newT.Weight) + b.bg.Add(name, balancer.Get(newT.ChildPolicy.Name)) + } else if newT.Weight != oldT.Weight { + // If this is an existing sub-balancer, update weight if necessary. + b.stateAggregator.UpdateWeight(name, newT.Weight) + } + + // Forwards all the update: + // - addresses are from the map after splitting with hierarchy path, + // - Top level service config and attributes are the same, + // - Balancer config comes from the targets map. + // + // TODO: handle error? How to aggregate errors and return? + _ = b.bg.UpdateClientConnState(name, balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: addressesSplit[name], + ServiceConfig: s.ResolverState.ServiceConfig, + Attributes: s.ResolverState.Attributes, + }, + BalancerConfig: newT.ChildPolicy.Config, + }) + } + + b.targets = newConfig.Targets + + // If the targets length is zero, it means we have removed all child + // policies from the balancer group and aggregator. + // At the start of this UpdateClientConnState() operation, a call to + // b.stateAggregator.ResumeStateUpdates() is deferred. Thus, setting the + // needUpdateStateOnResume bool to true here will ensure a new picker is + // built as part of that deferred function. Since there are now no child + // policies, the aggregated connectivity state reported form the Aggregator + // will be TRANSIENT_FAILURE. + if len(b.targets) == 0 { + b.stateAggregator.NeedUpdateStateOnResume() + } + + return nil +} + +func (b *weightedTargetBalancer) ResolverError(err error) { + b.bg.ResolverError(err) +} + +func (b *weightedTargetBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { + b.logger.Errorf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, state) +} + +func (b *weightedTargetBalancer) Close() { + b.stateAggregator.Stop() + b.bg.Close() +} + +func (b *weightedTargetBalancer) ExitIdle() { + b.bg.ExitIdle() +} diff --git a/xds/internal/balancer/lrs/config.go b/balancer/weightedtarget/weightedtarget_config.go similarity index 52% rename from xds/internal/balancer/lrs/config.go rename to balancer/weightedtarget/weightedtarget_config.go index 3d39961401b5..52090cd67b02 100644 --- a/xds/internal/balancer/lrs/config.go +++ b/balancer/weightedtarget/weightedtarget_config.go @@ -16,39 +16,34 @@ * */ -package lrs +package weightedtarget import ( "encoding/json" - "fmt" internalserviceconfig "google.golang.org/grpc/internal/serviceconfig" "google.golang.org/grpc/serviceconfig" - "google.golang.org/grpc/xds/internal" ) -type lbConfig struct { - serviceconfig.LoadBalancingConfig - ClusterName string - EdsServiceName string - LrsLoadReportingServerName string - Locality *internal.LocalityID - ChildPolicy *internalserviceconfig.BalancerConfig +// Target represents one target with the weight and the child policy. +type Target struct { + // Weight is the weight of the child policy. + Weight uint32 `json:"weight,omitempty"` + // ChildPolicy is the child policy and it's config. + ChildPolicy *internalserviceconfig.BalancerConfig `json:"childPolicy,omitempty"` } -func parseConfig(c json.RawMessage) (*lbConfig, error) { - var cfg lbConfig +// LBConfig is the balancer config for weighted_target. +type LBConfig struct { + serviceconfig.LoadBalancingConfig `json:"-"` + + Targets map[string]Target `json:"targets,omitempty"` +} + +func parseConfig(c json.RawMessage) (*LBConfig, error) { + var cfg LBConfig if err := json.Unmarshal(c, &cfg); err != nil { return nil, err } - if cfg.ClusterName == "" { - return nil, fmt.Errorf("required ClusterName is not set in %+v", cfg) - } - if cfg.LrsLoadReportingServerName == "" { - return nil, fmt.Errorf("required LrsLoadReportingServerName is not set in %+v", cfg) - } - if cfg.Locality == nil { - return nil, fmt.Errorf("required Locality is not set in %+v", cfg) - } return &cfg, nil } diff --git a/xds/internal/balancer/weightedtarget/weightedtarget_config_test.go b/balancer/weightedtarget/weightedtarget_config_test.go similarity index 65% rename from xds/internal/balancer/weightedtarget/weightedtarget_config_test.go rename to balancer/weightedtarget/weightedtarget_config_test.go index 2208117f60e1..25bbee836abe 100644 --- a/xds/internal/balancer/weightedtarget/weightedtarget_config_test.go +++ b/balancer/weightedtarget/weightedtarget_config_test.go @@ -23,40 +23,42 @@ import ( "github.com/google/go-cmp/cmp" "google.golang.org/grpc/balancer" + _ "google.golang.org/grpc/balancer/grpclb" + "google.golang.org/grpc/balancer/roundrobin" internalserviceconfig "google.golang.org/grpc/internal/serviceconfig" - _ "google.golang.org/grpc/xds/internal/balancer/cdsbalancer" ) const ( testJSONConfig = `{ "targets": { - "cluster_1" : { - "weight":75, - "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}] + "cluster_1": { + "weight": 75, + "childPolicy": [{ + "grpclb": { + "childPolicy": [{"pick_first":{}}], + "targetName": "foo-service" + } + }] }, - "cluster_2" : { - "weight":25, - "childPolicy":[{"cds_experimental":{"cluster":"cluster_2"}}] + "cluster_2": { + "weight": 25, + "childPolicy": [{"round_robin": ""}] } } }` - - cdsName = "cds_experimental" ) var ( - cdsConfigParser = balancer.Get(cdsName).(balancer.ConfigParser) - cdsConfigJSON1 = `{"cluster":"cluster_1"}` - cdsConfig1, _ = cdsConfigParser.ParseConfig([]byte(cdsConfigJSON1)) - cdsConfigJSON2 = `{"cluster":"cluster_2"}` - cdsConfig2, _ = cdsConfigParser.ParseConfig([]byte(cdsConfigJSON2)) + grpclbConfigParser = balancer.Get("grpclb").(balancer.ConfigParser) + grpclbConfigJSON = `{"childPolicy": [{"pick_first":{}}], "targetName": "foo-service"}` + grpclbConfig, _ = grpclbConfigParser.ParseConfig([]byte(grpclbConfigJSON)) ) -func Test_parseConfig(t *testing.T) { +func (s) TestParseConfig(t *testing.T) { tests := []struct { name string js string - want *lbConfig + want *LBConfig wantErr bool }{ { @@ -68,20 +70,19 @@ func Test_parseConfig(t *testing.T) { { name: "OK", js: testJSONConfig, - want: &lbConfig{ - Targets: map[string]target{ + want: &LBConfig{ + Targets: map[string]Target{ "cluster_1": { Weight: 75, ChildPolicy: &internalserviceconfig.BalancerConfig{ - Name: cdsName, - Config: cdsConfig1, + Name: "grpclb", + Config: grpclbConfig, }, }, "cluster_2": { Weight: 25, ChildPolicy: &internalserviceconfig.BalancerConfig{ - Name: cdsName, - Config: cdsConfig2, + Name: roundrobin.Name, }, }, }, @@ -93,8 +94,7 @@ func Test_parseConfig(t *testing.T) { t.Run(tt.name, func(t *testing.T) { got, err := parseConfig([]byte(tt.js)) if (err != nil) != tt.wantErr { - t.Errorf("parseConfig() error = %v, wantErr %v", err, tt.wantErr) - return + t.Fatalf("parseConfig() error = %v, wantErr %v", err, tt.wantErr) } if !cmp.Equal(got, tt.want) { t.Errorf("parseConfig() got unexpected result, diff: %v", cmp.Diff(got, tt.want)) diff --git a/balancer/weightedtarget/weightedtarget_test.go b/balancer/weightedtarget/weightedtarget_test.go new file mode 100644 index 000000000000..1fe16039e341 --- /dev/null +++ b/balancer/weightedtarget/weightedtarget_test.go @@ -0,0 +1,1367 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package weightedtarget + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/attributes" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/roundrobin" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/internal/balancer/stub" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/hierarchy" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" +) + +const ( + defaultTestTimeout = 5 * time.Second +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +type testConfigBalancerBuilder struct { + balancer.Builder +} + +func newTestConfigBalancerBuilder() *testConfigBalancerBuilder { + return &testConfigBalancerBuilder{ + Builder: balancer.Get(roundrobin.Name), + } +} + +// pickAndCheckError returns a function which takes a picker, invokes the Pick() method +// multiple times and ensures that the error returned by the picker matches the provided error. +func pickAndCheckError(want error) func(balancer.Picker) error { + const rpcCount = 5 + return func(p balancer.Picker) error { + for i := 0; i < rpcCount; i++ { + if _, err := p.Pick(balancer.PickInfo{}); err == nil || !strings.Contains(err.Error(), want.Error()) { + return fmt.Errorf("picker.Pick() returned error: %v, want: %v", err, want) + } + } + return nil + } +} + +func (t *testConfigBalancerBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { + rr := t.Builder.Build(cc, opts) + return &testConfigBalancer{ + Balancer: rr, + } +} + +const testConfigBalancerName = "test_config_balancer" + +func (t *testConfigBalancerBuilder) Name() string { + return testConfigBalancerName +} + +type stringBalancerConfig struct { + serviceconfig.LoadBalancingConfig + configStr string +} + +func (t *testConfigBalancerBuilder) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + var cfg string + if err := json.Unmarshal(c, &cfg); err != nil { + return nil, fmt.Errorf("failed to unmarshal config in %q: %v", testConfigBalancerName, err) + } + return stringBalancerConfig{configStr: cfg}, nil +} + +// testConfigBalancer is a roundrobin balancer, but it takes the balancer config +// string and adds it as an address attribute to the backend addresses. +type testConfigBalancer struct { + balancer.Balancer +} + +// configKey is the type used as the key to store balancer config in the +// Attributes field of resolver.Address. +type configKey struct{} + +func setConfigKey(addr resolver.Address, config string) resolver.Address { + addr.Attributes = addr.Attributes.WithValue(configKey{}, config) + return addr +} + +func getConfigKey(attr *attributes.Attributes) (string, bool) { + v := attr.Value(configKey{}) + name, ok := v.(string) + return name, ok +} + +func (b *testConfigBalancer) UpdateClientConnState(s balancer.ClientConnState) error { + c, ok := s.BalancerConfig.(stringBalancerConfig) + if !ok { + return fmt.Errorf("unexpected balancer config with type %T", s.BalancerConfig) + } + + addrsWithAttr := make([]resolver.Address, len(s.ResolverState.Addresses)) + for i, addr := range s.ResolverState.Addresses { + addrsWithAttr[i] = setConfigKey(addr, c.configStr) + } + s.BalancerConfig = nil + s.ResolverState.Addresses = addrsWithAttr + return b.Balancer.UpdateClientConnState(s) +} + +func (b *testConfigBalancer) Close() { + b.Balancer.Close() +} + +var ( + wtbBuilder balancer.Builder + wtbParser balancer.ConfigParser + testBackendAddrStrs []string +) + +const testBackendAddrsCount = 12 + +func init() { + balancer.Register(newTestConfigBalancerBuilder()) + for i := 0; i < testBackendAddrsCount; i++ { + testBackendAddrStrs = append(testBackendAddrStrs, fmt.Sprintf("%d.%d.%d.%d:%d", i, i, i, i, i)) + } + wtbBuilder = balancer.Get(Name) + wtbParser = wtbBuilder.(balancer.ConfigParser) + + NewRandomWRR = testutils.NewTestWRR +} + +// TestWeightedTarget covers the cases that a sub-balancer is added and a +// sub-balancer is removed. It verifies that the addresses and balancer configs +// are forwarded to the right sub-balancer. This test is intended to test the +// glue code in weighted_target. It also tests an empty target config update, +// which should trigger a transient failure state update. +func (s) TestWeightedTarget(t *testing.T) { + cc := testutils.NewTestClientConn(t) + wtb := wtbBuilder.Build(cc, balancer.BuildOptions{}) + defer wtb.Close() + + // Start with "cluster_1: round_robin". + config1, err := wtbParser.ParseConfig([]byte(` +{ + "targets": { + "cluster_1": { + "weight":1, + "childPolicy": [{"round_robin": ""}] + } + } +}`)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + + // Send the config, and an address with hierarchy path ["cluster_1"]. + addr1 := resolver.Address{Addr: testBackendAddrStrs[1], Attributes: nil} + if err := wtb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Addresses: []resolver.Address{hierarchy.Set(addr1, []string{"cluster_1"})}}, + BalancerConfig: config1, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + verifyAddressInNewSubConn(t, cc, addr1) + + // Send subconn state change. + sc1 := <-cc.NewSubConnCh + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + <-cc.NewPickerCh + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + p := <-cc.NewPickerCh + + // Test pick with one backend. + for i := 0; i < 5; i++ { + gotSCSt, _ := p.Pick(balancer.PickInfo{}) + if gotSCSt.SubConn != sc1 { + t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc1) + } + } + + // Remove cluster_1, and add "cluster_2: test_config_balancer". The + // test_config_balancer adds an address attribute whose value is set to the + // config that is passed to it. + config2, err := wtbParser.ParseConfig([]byte(` +{ + "targets": { + "cluster_2": { + "weight":1, + "childPolicy": [{"test_config_balancer": "cluster_2"}] + } + } +}`)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + + // Send the config, and one address with hierarchy path "cluster_2". + addr2 := resolver.Address{Addr: testBackendAddrStrs[2], Attributes: nil} + if err := wtb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Addresses: []resolver.Address{hierarchy.Set(addr2, []string{"cluster_2"})}}, + BalancerConfig: config2, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // Expect a new subConn from the test_config_balancer which has an address + // attribute set to the config that was passed to it. + verifyAddressInNewSubConn(t, cc, setConfigKey(addr2, "cluster_2")) + + // The subconn for cluster_1 should be shut down. + scShutdown := <-cc.ShutdownSubConnCh + if scShutdown != sc1 { + t.Fatalf("ShutdownSubConn, want %v, got %v", sc1, scShutdown) + } + scShutdown.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Shutdown}) + + sc2 := <-cc.NewSubConnCh + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + <-cc.NewPickerCh + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + p = <-cc.NewPickerCh + + // Test pick with one backend. + for i := 0; i < 5; i++ { + gotSCSt, _ := p.Pick(balancer.PickInfo{}) + if gotSCSt.SubConn != sc2 { + t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc2) + } + } + + // Replace child policy of "cluster_1" to "round_robin". + config3, err := wtbParser.ParseConfig([]byte(` +{ + "targets": { + "cluster_2": { + "weight":1, + "childPolicy": [{"round_robin": ""}] + } + } +}`)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + + // Send the config, and an address with hierarchy path ["cluster_2"]. + addr3 := resolver.Address{Addr: testBackendAddrStrs[3], Attributes: nil} + if err := wtb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Addresses: []resolver.Address{hierarchy.Set(addr3, []string{"cluster_2"})}}, + BalancerConfig: config3, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + verifyAddressInNewSubConn(t, cc, addr3) + + // The subconn from the test_config_balancer should be shut down. + scShutdown = <-cc.ShutdownSubConnCh + if scShutdown != sc2 { + t.Fatalf("ShutdownSubConn, want %v, got %v", sc1, scShutdown) + } + scShutdown.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Shutdown}) + + // Send subconn state change. + sc3 := <-cc.NewSubConnCh + sc3.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + <-cc.NewPickerCh + sc3.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + p = <-cc.NewPickerCh + + // Test pick with one backend. + for i := 0; i < 5; i++ { + gotSCSt, _ := p.Pick(balancer.PickInfo{}) + if gotSCSt.SubConn != sc3 { + t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc3) + } + } + // Update the Weighted Target Balancer with an empty address list and no + // targets. This should cause a Transient Failure State update to the Client + // Conn. + emptyConfig, err := wtbParser.ParseConfig([]byte(`{}`)) + if err != nil { + t.Fatalf("Failed to parse balancer config: %v", err) + } + if err := wtb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{}, + BalancerConfig: emptyConfig, + }); err != nil { + t.Fatalf("Failed to update ClientConn state: %v", err) + } + + state := <-cc.NewStateCh + if state != connectivity.TransientFailure { + t.Fatalf("Empty target update should have triggered a TF state update, got: %v", state) + } +} + +// TestWeightedTarget_OneSubBalancer_AddRemoveBackend tests the case where we +// have a weighted target balancer will one sub-balancer, and we add and remove +// backends from the subBalancer. +func (s) TestWeightedTarget_OneSubBalancer_AddRemoveBackend(t *testing.T) { + cc := testutils.NewTestClientConn(t) + wtb := wtbBuilder.Build(cc, balancer.BuildOptions{}) + defer wtb.Close() + + // Start with "cluster_1: round_robin". + config, err := wtbParser.ParseConfig([]byte(` +{ + "targets": { + "cluster_1": { + "weight":1, + "childPolicy": [{"round_robin": ""}] + } + } +}`)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + + // Send the config, and an address with hierarchy path ["cluster_1"]. + addr1 := resolver.Address{Addr: testBackendAddrStrs[1]} + if err := wtb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Addresses: []resolver.Address{hierarchy.Set(addr1, []string{"cluster_1"})}}, + BalancerConfig: config, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + verifyAddressInNewSubConn(t, cc, addr1) + + // Expect one SubConn, and move it to READY. + sc1 := <-cc.NewSubConnCh + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + <-cc.NewPickerCh + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + p := <-cc.NewPickerCh + + // Test pick with one backend. + for i := 0; i < 5; i++ { + gotSCSt, _ := p.Pick(balancer.PickInfo{}) + if gotSCSt.SubConn != sc1 { + t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc1) + } + } + + // Send two addresses. + addr2 := resolver.Address{Addr: testBackendAddrStrs[2]} + if err := wtb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Addresses: []resolver.Address{ + hierarchy.Set(addr1, []string{"cluster_1"}), + hierarchy.Set(addr2, []string{"cluster_1"}), + }}, + BalancerConfig: config, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + verifyAddressInNewSubConn(t, cc, addr2) + + // Expect one new SubConn, and move it to READY. + sc2 := <-cc.NewSubConnCh + // Update the SubConn to become READY. + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + <-cc.NewPickerCh + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + p = <-cc.NewPickerCh + + // Test round robin pick. + want := []balancer.SubConn{sc1, sc2} + if err := testutils.IsRoundRobin(want, testutils.SubConnFromPicker(p)); err != nil { + t.Fatalf("want %v, got %v", want, err) + } + + // Remove the first address. + if err := wtb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Addresses: []resolver.Address{hierarchy.Set(addr2, []string{"cluster_1"})}}, + BalancerConfig: config, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // Expect one SubConn to be shut down. + scShutdown := <-cc.ShutdownSubConnCh + if scShutdown != sc1 { + t.Fatalf("ShutdownSubConn, want %v, got %v", sc1, scShutdown) + } + scShutdown.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Shutdown}) + p = <-cc.NewPickerCh + + // Test pick with only the second SubConn. + for i := 0; i < 5; i++ { + gotSC, _ := p.Pick(balancer.PickInfo{}) + if gotSC.SubConn != sc2 { + t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSC, sc2) + } + } +} + +// TestWeightedTarget_TwoSubBalancers_OneBackend tests the case where we have a +// weighted target balancer with two sub-balancers, each with one backend. +func (s) TestWeightedTarget_TwoSubBalancers_OneBackend(t *testing.T) { + cc := testutils.NewTestClientConn(t) + wtb := wtbBuilder.Build(cc, balancer.BuildOptions{}) + defer wtb.Close() + + // Start with "cluster_1: test_config_balancer, cluster_2: test_config_balancer". + config, err := wtbParser.ParseConfig([]byte(` +{ + "targets": { + "cluster_1": { + "weight":1, + "childPolicy": [{"test_config_balancer": "cluster_1"}] + }, + "cluster_2": { + "weight":1, + "childPolicy": [{"test_config_balancer": "cluster_2"}] + } + } +}`)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + + // Send the config with one address for each cluster. + addr1 := resolver.Address{Addr: testBackendAddrStrs[1]} + addr2 := resolver.Address{Addr: testBackendAddrStrs[2]} + if err := wtb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Addresses: []resolver.Address{ + hierarchy.Set(addr1, []string{"cluster_1"}), + hierarchy.Set(addr2, []string{"cluster_2"}), + }}, + BalancerConfig: config, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + scs := waitForNewSubConns(t, cc, 2) + verifySubConnAddrs(t, scs, map[string][]resolver.Address{ + "cluster_1": {addr1}, + "cluster_2": {addr2}, + }) + + // We expect a single subConn on each subBalancer. + sc1 := scs["cluster_1"][0].sc.(*testutils.TestSubConn) + sc2 := scs["cluster_2"][0].sc.(*testutils.TestSubConn) + + // Send state changes for both SubConns, and wait for the picker. + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + <-cc.NewPickerCh + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + <-cc.NewPickerCh + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + <-cc.NewPickerCh + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + p := <-cc.NewPickerCh + + // Test roundrobin on the last picker. + want := []balancer.SubConn{sc1, sc2} + if err := testutils.IsRoundRobin(want, testutils.SubConnFromPicker(p)); err != nil { + t.Fatalf("want %v, got %v", want, err) + } +} + +// TestWeightedTarget_TwoSubBalancers_MoreBackends tests the case where we have +// a weighted target balancer with two sub-balancers, each with more than one +// backend. +func (s) TestWeightedTarget_TwoSubBalancers_MoreBackends(t *testing.T) { + cc := testutils.NewTestClientConn(t) + wtb := wtbBuilder.Build(cc, balancer.BuildOptions{}) + defer wtb.Close() + + // Start with "cluster_1: round_robin, cluster_2: round_robin". + config, err := wtbParser.ParseConfig([]byte(` +{ + "targets": { + "cluster_1": { + "weight":1, + "childPolicy": [{"test_config_balancer": "cluster_1"}] + }, + "cluster_2": { + "weight":1, + "childPolicy": [{"test_config_balancer": "cluster_2"}] + } + } +}`)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + + // Send the config with two backends for each cluster. + addr1 := resolver.Address{Addr: testBackendAddrStrs[1]} + addr2 := resolver.Address{Addr: testBackendAddrStrs[2]} + addr3 := resolver.Address{Addr: testBackendAddrStrs[3]} + addr4 := resolver.Address{Addr: testBackendAddrStrs[4]} + if err := wtb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Addresses: []resolver.Address{ + hierarchy.Set(addr1, []string{"cluster_1"}), + hierarchy.Set(addr2, []string{"cluster_1"}), + hierarchy.Set(addr3, []string{"cluster_2"}), + hierarchy.Set(addr4, []string{"cluster_2"}), + }}, + BalancerConfig: config, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + scs := waitForNewSubConns(t, cc, 4) + verifySubConnAddrs(t, scs, map[string][]resolver.Address{ + "cluster_1": {addr1, addr2}, + "cluster_2": {addr3, addr4}, + }) + + // We expect two subConns on each subBalancer. + sc1 := scs["cluster_1"][0].sc.(*testutils.TestSubConn) + sc2 := scs["cluster_1"][1].sc.(*testutils.TestSubConn) + sc3 := scs["cluster_2"][0].sc.(*testutils.TestSubConn) + sc4 := scs["cluster_2"][1].sc.(*testutils.TestSubConn) + + // Send state changes for all SubConns, and wait for the picker. + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + <-cc.NewPickerCh + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + <-cc.NewPickerCh + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + <-cc.NewPickerCh + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + <-cc.NewPickerCh + sc3.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + <-cc.NewPickerCh + sc3.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + <-cc.NewPickerCh + sc4.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + <-cc.NewPickerCh + sc4.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + p := <-cc.NewPickerCh + + // Test roundrobin on the last picker. RPCs should be sent equally to all + // backends. + want := []balancer.SubConn{sc1, sc2, sc3, sc4} + if err := testutils.IsRoundRobin(want, testutils.SubConnFromPicker(p)); err != nil { + t.Fatalf("want %v, got %v", want, err) + } + + // Turn sc2's connection down, should be RR between balancers. + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) + p = <-cc.NewPickerCh + want = []balancer.SubConn{sc1, sc1, sc3, sc4} + if err := testutils.IsRoundRobin(want, testutils.SubConnFromPicker(p)); err != nil { + t.Fatalf("want %v, got %v", want, err) + } + + // Shut down subConn corresponding to addr3. + if err := wtb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Addresses: []resolver.Address{ + hierarchy.Set(addr1, []string{"cluster_1"}), + hierarchy.Set(addr2, []string{"cluster_1"}), + hierarchy.Set(addr4, []string{"cluster_2"}), + }}, + BalancerConfig: config, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + scShutdown := <-cc.ShutdownSubConnCh + if scShutdown != sc3 { + t.Fatalf("ShutdownSubConn, want %v, got %v", sc3, scShutdown) + } + scShutdown.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Shutdown}) + p = <-cc.NewPickerCh + want = []balancer.SubConn{sc1, sc4} + if err := testutils.IsRoundRobin(want, testutils.SubConnFromPicker(p)); err != nil { + t.Fatalf("want %v, got %v", want, err) + } + + // Turn sc1's connection down. + wantSubConnErr := errors.New("subConn connection error") + sc1.UpdateState(balancer.SubConnState{ + ConnectivityState: connectivity.TransientFailure, + ConnectionError: wantSubConnErr, + }) + p = <-cc.NewPickerCh + want = []balancer.SubConn{sc4} + if err := testutils.IsRoundRobin(want, testutils.SubConnFromPicker(p)); err != nil { + t.Fatalf("want %v, got %v", want, err) + } + + // Turn last connection to connecting. + sc4.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + p = <-cc.NewPickerCh + for i := 0; i < 5; i++ { + if _, err := p.Pick(balancer.PickInfo{}); err != balancer.ErrNoSubConnAvailable { + t.Fatalf("want pick error %v, got %v", balancer.ErrNoSubConnAvailable, err) + } + } + + // Turn all connections down. + sc4.UpdateState(balancer.SubConnState{ + ConnectivityState: connectivity.TransientFailure, + ConnectionError: wantSubConnErr, + }) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := cc.WaitForPicker(ctx, pickAndCheckError(wantSubConnErr)); err != nil { + t.Fatal(err) + } +} + +// TestWeightedTarget_TwoSubBalancers_DifferentWeight_MoreBackends tests the +// case where we have a weighted target balancer with two sub-balancers of +// differing weights. +func (s) TestWeightedTarget_TwoSubBalancers_DifferentWeight_MoreBackends(t *testing.T) { + cc := testutils.NewTestClientConn(t) + wtb := wtbBuilder.Build(cc, balancer.BuildOptions{}) + defer wtb.Close() + + // Start with two subBalancers, one with twice the weight of the other. + config, err := wtbParser.ParseConfig([]byte(` +{ + "targets": { + "cluster_1": { + "weight": 2, + "childPolicy": [{"test_config_balancer": "cluster_1"}] + }, + "cluster_2": { + "weight": 1, + "childPolicy": [{"test_config_balancer": "cluster_2"}] + } + } +}`)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + + // Send the config with two backends for each cluster. + addr1 := resolver.Address{Addr: testBackendAddrStrs[1]} + addr2 := resolver.Address{Addr: testBackendAddrStrs[2]} + addr3 := resolver.Address{Addr: testBackendAddrStrs[3]} + addr4 := resolver.Address{Addr: testBackendAddrStrs[4]} + if err := wtb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Addresses: []resolver.Address{ + hierarchy.Set(addr1, []string{"cluster_1"}), + hierarchy.Set(addr2, []string{"cluster_1"}), + hierarchy.Set(addr3, []string{"cluster_2"}), + hierarchy.Set(addr4, []string{"cluster_2"}), + }}, + BalancerConfig: config, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + scs := waitForNewSubConns(t, cc, 4) + verifySubConnAddrs(t, scs, map[string][]resolver.Address{ + "cluster_1": {addr1, addr2}, + "cluster_2": {addr3, addr4}, + }) + + // We expect two subConns on each subBalancer. + sc1 := scs["cluster_1"][0].sc.(*testutils.TestSubConn) + sc2 := scs["cluster_1"][1].sc.(*testutils.TestSubConn) + sc3 := scs["cluster_2"][0].sc.(*testutils.TestSubConn) + sc4 := scs["cluster_2"][1].sc.(*testutils.TestSubConn) + + // Send state changes for all SubConns, and wait for the picker. + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + <-cc.NewPickerCh + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + <-cc.NewPickerCh + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + <-cc.NewPickerCh + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + <-cc.NewPickerCh + sc3.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + <-cc.NewPickerCh + sc3.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + <-cc.NewPickerCh + sc4.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + <-cc.NewPickerCh + sc4.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + p := <-cc.NewPickerCh + + // Test roundrobin on the last picker. Twice the number of RPCs should be + // sent to cluster_1 when compared to cluster_2. + want := []balancer.SubConn{sc1, sc1, sc2, sc2, sc3, sc4} + if err := testutils.IsRoundRobin(want, testutils.SubConnFromPicker(p)); err != nil { + t.Fatalf("want %v, got %v", want, err) + } +} + +// TestWeightedTarget_ThreeSubBalancers_RemoveBalancer tests the case where we +// have a weighted target balancer with three sub-balancers and we remove one of +// the subBalancers. +func (s) TestWeightedTarget_ThreeSubBalancers_RemoveBalancer(t *testing.T) { + cc := testutils.NewTestClientConn(t) + wtb := wtbBuilder.Build(cc, balancer.BuildOptions{}) + defer wtb.Close() + + // Start with two subBalancers, one with twice the weight of the other. + config, err := wtbParser.ParseConfig([]byte(` +{ + "targets": { + "cluster_1": { + "weight": 1, + "childPolicy": [{"test_config_balancer": "cluster_1"}] + }, + "cluster_2": { + "weight": 1, + "childPolicy": [{"test_config_balancer": "cluster_2"}] + }, + "cluster_3": { + "weight": 1, + "childPolicy": [{"test_config_balancer": "cluster_3"}] + } + } +}`)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + + // Send the config with one backend for each cluster. + addr1 := resolver.Address{Addr: testBackendAddrStrs[1]} + addr2 := resolver.Address{Addr: testBackendAddrStrs[2]} + addr3 := resolver.Address{Addr: testBackendAddrStrs[3]} + if err := wtb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Addresses: []resolver.Address{ + hierarchy.Set(addr1, []string{"cluster_1"}), + hierarchy.Set(addr2, []string{"cluster_2"}), + hierarchy.Set(addr3, []string{"cluster_3"}), + }}, + BalancerConfig: config, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + scs := waitForNewSubConns(t, cc, 3) + verifySubConnAddrs(t, scs, map[string][]resolver.Address{ + "cluster_1": {addr1}, + "cluster_2": {addr2}, + "cluster_3": {addr3}, + }) + + // We expect one subConn on each subBalancer. + sc1 := scs["cluster_1"][0].sc.(*testutils.TestSubConn) + sc2 := scs["cluster_2"][0].sc.(*testutils.TestSubConn) + sc3 := scs["cluster_3"][0].sc.(*testutils.TestSubConn) + + // Send state changes for all SubConns, and wait for the picker. + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + <-cc.NewPickerCh + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + <-cc.NewPickerCh + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + <-cc.NewPickerCh + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + <-cc.NewPickerCh + sc3.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + <-cc.NewPickerCh + sc3.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + p := <-cc.NewPickerCh + + want := []balancer.SubConn{sc1, sc2, sc3} + if err := testutils.IsRoundRobin(want, testutils.SubConnFromPicker(p)); err != nil { + t.Fatalf("want %v, got %v", want, err) + } + + // Remove the second balancer, while the others two are ready. + config, err = wtbParser.ParseConfig([]byte(` +{ + "targets": { + "cluster_1": { + "weight": 1, + "childPolicy": [{"test_config_balancer": "cluster_1"}] + }, + "cluster_3": { + "weight": 1, + "childPolicy": [{"test_config_balancer": "cluster_3"}] + } + } +}`)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + if err := wtb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Addresses: []resolver.Address{ + hierarchy.Set(addr1, []string{"cluster_1"}), + hierarchy.Set(addr3, []string{"cluster_3"}), + }}, + BalancerConfig: config, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // Removing a subBalancer causes the weighted target LB policy to push a new + // picker which ensures that the removed subBalancer is not picked for RPCs. + p = <-cc.NewPickerCh + + scShutdown := <-cc.ShutdownSubConnCh + if scShutdown != sc2 { + t.Fatalf("ShutdownSubConn, want %v, got %v", sc2, scShutdown) + } + want = []balancer.SubConn{sc1, sc3} + if err := testutils.IsRoundRobin(want, testutils.SubConnFromPicker(p)); err != nil { + t.Fatalf("want %v, got %v", want, err) + } + + // Move balancer 3 into transient failure. + wantSubConnErr := errors.New("subConn connection error") + sc3.UpdateState(balancer.SubConnState{ + ConnectivityState: connectivity.TransientFailure, + ConnectionError: wantSubConnErr, + }) + <-cc.NewPickerCh + + // Remove the first balancer, while the third is transient failure. + config, err = wtbParser.ParseConfig([]byte(` +{ + "targets": { + "cluster_3": { + "weight": 1, + "childPolicy": [{"test_config_balancer": "cluster_3"}] + } + } +}`)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + if err := wtb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Addresses: []resolver.Address{ + hierarchy.Set(addr3, []string{"cluster_3"}), + }}, + BalancerConfig: config, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // Removing a subBalancer causes the weighted target LB policy to push a new + // picker which ensures that the removed subBalancer is not picked for RPCs. + + scShutdown = <-cc.ShutdownSubConnCh + if scShutdown != sc1 { + t.Fatalf("ShutdownSubConn, want %v, got %v", sc1, scShutdown) + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := cc.WaitForPicker(ctx, pickAndCheckError(wantSubConnErr)); err != nil { + t.Fatal(err) + } +} + +// TestWeightedTarget_TwoSubBalancers_ChangeWeight_MoreBackends tests the case +// where we have a weighted target balancer with two sub-balancers, and we +// change the weight of these subBalancers. +func (s) TestWeightedTarget_TwoSubBalancers_ChangeWeight_MoreBackends(t *testing.T) { + cc := testutils.NewTestClientConn(t) + wtb := wtbBuilder.Build(cc, balancer.BuildOptions{}) + defer wtb.Close() + + // Start with two subBalancers, one with twice the weight of the other. + config, err := wtbParser.ParseConfig([]byte(` +{ + "targets": { + "cluster_1": { + "weight": 2, + "childPolicy": [{"test_config_balancer": "cluster_1"}] + }, + "cluster_2": { + "weight": 1, + "childPolicy": [{"test_config_balancer": "cluster_2"}] + } + } +}`)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + + // Send the config with two backends for each cluster. + addr1 := resolver.Address{Addr: testBackendAddrStrs[1]} + addr2 := resolver.Address{Addr: testBackendAddrStrs[2]} + addr3 := resolver.Address{Addr: testBackendAddrStrs[3]} + addr4 := resolver.Address{Addr: testBackendAddrStrs[4]} + if err := wtb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Addresses: []resolver.Address{ + hierarchy.Set(addr1, []string{"cluster_1"}), + hierarchy.Set(addr2, []string{"cluster_1"}), + hierarchy.Set(addr3, []string{"cluster_2"}), + hierarchy.Set(addr4, []string{"cluster_2"}), + }}, + BalancerConfig: config, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + scs := waitForNewSubConns(t, cc, 4) + verifySubConnAddrs(t, scs, map[string][]resolver.Address{ + "cluster_1": {addr1, addr2}, + "cluster_2": {addr3, addr4}, + }) + + // We expect two subConns on each subBalancer. + sc1 := scs["cluster_1"][0].sc.(*testutils.TestSubConn) + sc2 := scs["cluster_1"][1].sc.(*testutils.TestSubConn) + sc3 := scs["cluster_2"][0].sc.(*testutils.TestSubConn) + sc4 := scs["cluster_2"][1].sc.(*testutils.TestSubConn) + + // Send state changes for all SubConns, and wait for the picker. + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + <-cc.NewPickerCh + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + <-cc.NewPickerCh + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + <-cc.NewPickerCh + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + <-cc.NewPickerCh + sc3.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + <-cc.NewPickerCh + sc3.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + <-cc.NewPickerCh + sc4.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + <-cc.NewPickerCh + sc4.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + p := <-cc.NewPickerCh + + // Test roundrobin on the last picker. Twice the number of RPCs should be + // sent to cluster_1 when compared to cluster_2. + want := []balancer.SubConn{sc1, sc1, sc2, sc2, sc3, sc4} + if err := testutils.IsRoundRobin(want, testutils.SubConnFromPicker(p)); err != nil { + t.Fatalf("want %v, got %v", want, err) + } + + // Change the weight of cluster_1. + config, err = wtbParser.ParseConfig([]byte(` +{ + "targets": { + "cluster_1": { + "weight": 3, + "childPolicy": [{"test_config_balancer": "cluster_1"}] + }, + "cluster_2": { + "weight": 1, + "childPolicy": [{"test_config_balancer": "cluster_2"}] + } + } +}`)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + if err := wtb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Addresses: []resolver.Address{ + hierarchy.Set(addr1, []string{"cluster_1"}), + hierarchy.Set(addr2, []string{"cluster_1"}), + hierarchy.Set(addr3, []string{"cluster_2"}), + hierarchy.Set(addr4, []string{"cluster_2"}), + }}, + BalancerConfig: config, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // Weight change causes a new picker to be pushed to the channel. + p = <-cc.NewPickerCh + want = []balancer.SubConn{sc1, sc1, sc1, sc2, sc2, sc2, sc3, sc4} + if err := testutils.IsRoundRobin(want, testutils.SubConnFromPicker(p)); err != nil { + t.Fatalf("want %v, got %v", want, err) + } +} + +// TestWeightedTarget_InitOneSubBalancerTransientFailure tests that at init +// time, with two sub-balancers, if one sub-balancer reports transient_failure, +// the picks won't fail with transient_failure, and should instead wait for the +// other sub-balancer. +func (s) TestWeightedTarget_InitOneSubBalancerTransientFailure(t *testing.T) { + cc := testutils.NewTestClientConn(t) + wtb := wtbBuilder.Build(cc, balancer.BuildOptions{}) + defer wtb.Close() + + // Start with "cluster_1: test_config_balancer, cluster_2: test_config_balancer". + config, err := wtbParser.ParseConfig([]byte(` +{ + "targets": { + "cluster_1": { + "weight":1, + "childPolicy": [{"test_config_balancer": "cluster_1"}] + }, + "cluster_2": { + "weight":1, + "childPolicy": [{"test_config_balancer": "cluster_2"}] + } + } +}`)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + + // Send the config with one address for each cluster. + addr1 := resolver.Address{Addr: testBackendAddrStrs[1]} + addr2 := resolver.Address{Addr: testBackendAddrStrs[2]} + if err := wtb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Addresses: []resolver.Address{ + hierarchy.Set(addr1, []string{"cluster_1"}), + hierarchy.Set(addr2, []string{"cluster_2"}), + }}, + BalancerConfig: config, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + scs := waitForNewSubConns(t, cc, 2) + verifySubConnAddrs(t, scs, map[string][]resolver.Address{ + "cluster_1": {addr1}, + "cluster_2": {addr2}, + }) + + // We expect a single subConn on each subBalancer. + sc1 := scs["cluster_1"][0].sc.(*testutils.TestSubConn) + _ = scs["cluster_2"][0].sc + + // Set one subconn to TransientFailure, this will trigger one sub-balancer + // to report transient failure. + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) + + p := <-cc.NewPickerCh + for i := 0; i < 5; i++ { + r, err := p.Pick(balancer.PickInfo{}) + if err != balancer.ErrNoSubConnAvailable { + t.Fatalf("want pick to fail with %v, got result %v, err %v", balancer.ErrNoSubConnAvailable, r, err) + } + } +} + +// Test that with two sub-balancers, both in transient_failure, if one turns +// connecting, the overall state stays in transient_failure, and all picks +// return transient failure error. +func (s) TestBalancerGroup_SubBalancerTurnsConnectingFromTransientFailure(t *testing.T) { + cc := testutils.NewTestClientConn(t) + wtb := wtbBuilder.Build(cc, balancer.BuildOptions{}) + defer wtb.Close() + + // Start with "cluster_1: test_config_balancer, cluster_2: test_config_balancer". + config, err := wtbParser.ParseConfig([]byte(` +{ + "targets": { + "cluster_1": { + "weight":1, + "childPolicy": [{"test_config_balancer": "cluster_1"}] + }, + "cluster_2": { + "weight":1, + "childPolicy": [{"test_config_balancer": "cluster_2"}] + } + } +}`)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + + // Send the config with one address for each cluster. + addr1 := resolver.Address{Addr: testBackendAddrStrs[1]} + addr2 := resolver.Address{Addr: testBackendAddrStrs[2]} + if err := wtb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Addresses: []resolver.Address{ + hierarchy.Set(addr1, []string{"cluster_1"}), + hierarchy.Set(addr2, []string{"cluster_2"}), + }}, + BalancerConfig: config, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + scs := waitForNewSubConns(t, cc, 2) + verifySubConnAddrs(t, scs, map[string][]resolver.Address{ + "cluster_1": {addr1}, + "cluster_2": {addr2}, + }) + + // We expect a single subConn on each subBalancer. + sc1 := scs["cluster_1"][0].sc.(*testutils.TestSubConn) + sc2 := scs["cluster_2"][0].sc.(*testutils.TestSubConn) + + // Set both subconn to TransientFailure, this will put both sub-balancers in + // transient failure. + wantSubConnErr := errors.New("subConn connection error") + sc1.UpdateState(balancer.SubConnState{ + ConnectivityState: connectivity.TransientFailure, + ConnectionError: wantSubConnErr, + }) + <-cc.NewPickerCh + sc2.UpdateState(balancer.SubConnState{ + ConnectivityState: connectivity.TransientFailure, + ConnectionError: wantSubConnErr, + }) + p := <-cc.NewPickerCh + + for i := 0; i < 5; i++ { + if _, err := p.Pick(balancer.PickInfo{}); err == nil || !strings.Contains(err.Error(), wantSubConnErr.Error()) { + t.Fatalf("picker.Pick() returned error: %v, want: %v", err, wantSubConnErr) + } + } + + // Set one subconn to Connecting, it shouldn't change the overall state. + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + select { + case <-time.After(100 * time.Millisecond): + case <-cc.NewPickerCh: + t.Fatal("received new picker from the LB policy when expecting none") + } + + for i := 0; i < 5; i++ { + if _, err := p.Pick(balancer.PickInfo{}); err == nil || !strings.Contains(err.Error(), wantSubConnErr.Error()) { + t.Fatalf("picker.Pick() returned error: %v, want: %v", err, wantSubConnErr) + } + } +} + +// Verify that a SubConn is created with the expected address and hierarchy +// path cleared. +func verifyAddressInNewSubConn(t *testing.T, cc *testutils.TestClientConn, addr resolver.Address) { + t.Helper() + + gotAddr := <-cc.NewSubConnAddrsCh + wantAddr := []resolver.Address{hierarchy.Set(addr, []string{})} + if diff := cmp.Diff(gotAddr, wantAddr, cmp.AllowUnexported(attributes.Attributes{})); diff != "" { + t.Fatalf("got unexpected new subconn addrs: %v", diff) + } +} + +// subConnWithAddr wraps a subConn and the address for which it was created. +type subConnWithAddr struct { + sc balancer.SubConn + addr resolver.Address +} + +// waitForNewSubConns waits for `num` number of subConns to be created. This is +// expected to be used from tests using the "test_config_balancer" LB policy, +// which adds an address attribute with value set to the balancer config. +// +// Returned value is a map from subBalancer (identified by its config) to +// subConns created by it. +func waitForNewSubConns(t *testing.T, cc *testutils.TestClientConn, num int) map[string][]subConnWithAddr { + t.Helper() + + scs := make(map[string][]subConnWithAddr) + for i := 0; i < num; i++ { + addrs := <-cc.NewSubConnAddrsCh + if len(addrs) != 1 { + t.Fatalf("received subConns with %d addresses, want 1", len(addrs)) + } + cfg, ok := getConfigKey(addrs[0].Attributes) + if !ok { + t.Fatalf("received subConn address %v contains no attribute for balancer config", addrs[0]) + } + sc := <-cc.NewSubConnCh + scWithAddr := subConnWithAddr{sc: sc, addr: addrs[0]} + scs[cfg] = append(scs[cfg], scWithAddr) + } + return scs +} + +func verifySubConnAddrs(t *testing.T, scs map[string][]subConnWithAddr, wantSubConnAddrs map[string][]resolver.Address) { + t.Helper() + + if len(scs) != len(wantSubConnAddrs) { + t.Fatalf("got new subConns %+v, want %v", scs, wantSubConnAddrs) + } + for cfg, scsWithAddr := range scs { + if len(scsWithAddr) != len(wantSubConnAddrs[cfg]) { + t.Fatalf("got new subConns %+v, want %v", scs, wantSubConnAddrs) + } + wantAddrs := wantSubConnAddrs[cfg] + for i, scWithAddr := range scsWithAddr { + if diff := cmp.Diff(wantAddrs[i].Addr, scWithAddr.addr.Addr); diff != "" { + t.Fatalf("got unexpected new subconn addrs: %v", diff) + } + } + } +} + +const initIdleBalancerName = "test-init-Idle-balancer" + +var errTestInitIdle = fmt.Errorf("init Idle balancer error 0") + +func init() { + stub.Register(initIdleBalancerName, stub.BalancerFuncs{ + UpdateClientConnState: func(bd *stub.BalancerData, opts balancer.ClientConnState) error { + sc, err := bd.ClientConn.NewSubConn(opts.ResolverState.Addresses, balancer.NewSubConnOptions{ + StateListener: func(state balancer.SubConnState) { + err := fmt.Errorf("wrong picker error") + if state.ConnectivityState == connectivity.Idle { + err = errTestInitIdle + } + bd.ClientConn.UpdateState(balancer.State{ + ConnectivityState: state.ConnectivityState, + Picker: &testutils.TestConstPicker{Err: err}, + }) + }, + }) + if err != nil { + return err + } + sc.Connect() + return nil + }, + }) +} + +// TestInitialIdle covers the case that if the child reports Idle, the overall +// state will be Idle. +func (s) TestInitialIdle(t *testing.T) { + cc := testutils.NewTestClientConn(t) + wtb := wtbBuilder.Build(cc, balancer.BuildOptions{}) + defer wtb.Close() + + config, err := wtbParser.ParseConfig([]byte(` +{ + "targets": { + "cluster_1": { + "weight":1, + "childPolicy": [{"test-init-Idle-balancer": ""}] + } + } +}`)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + + // Send the config, and an address with hierarchy path ["cluster_1"]. + addrs := []resolver.Address{{Addr: testBackendAddrStrs[0], Attributes: nil}} + if err := wtb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Addresses: []resolver.Address{hierarchy.Set(addrs[0], []string{"cds:cluster_1"})}}, + BalancerConfig: config, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // Verify that a subconn is created with the address, and the hierarchy path + // in the address is cleared. + for range addrs { + sc := <-cc.NewSubConnCh + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Idle}) + } + + if state := <-cc.NewStateCh; state != connectivity.Idle { + t.Fatalf("Received aggregated state: %v, want Idle", state) + } +} + +// TestIgnoreSubBalancerStateTransitions covers the case that if the child reports a +// transition from TF to Connecting, the overall state will still be TF. +func (s) TestIgnoreSubBalancerStateTransitions(t *testing.T) { + cc := &tcc{TestClientConn: testutils.NewTestClientConn(t)} + + wtb := wtbBuilder.Build(cc, balancer.BuildOptions{}) + defer wtb.Close() + + config, err := wtbParser.ParseConfig([]byte(` +{ + "targets": { + "cluster_1": { + "weight":1, + "childPolicy": [{"round_robin": ""}] + } + } +}`)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + + // Send the config, and an address with hierarchy path ["cluster_1"]. + addr := resolver.Address{Addr: testBackendAddrStrs[0], Attributes: nil} + if err := wtb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Addresses: []resolver.Address{hierarchy.Set(addr, []string{"cluster_1"})}}, + BalancerConfig: config, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + sc := <-cc.NewSubConnCh + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + + // Verify that the SubConnState update from TF to Connecting is ignored. + if len(cc.states) != 2 || cc.states[0].ConnectivityState != connectivity.Connecting || cc.states[1].ConnectivityState != connectivity.TransientFailure { + t.Fatalf("cc.states = %v; want [Connecting, TransientFailure]", cc.states) + } +} + +// tcc wraps a testutils.TestClientConn but stores all state transitions in a +// slice. +type tcc struct { + *testutils.TestClientConn + states []balancer.State +} + +func (t *tcc) UpdateState(bs balancer.State) { + t.states = append(t.states, bs) + t.TestClientConn.UpdateState(bs) +} + +func (s) TestUpdateStatePauses(t *testing.T) { + cc := &tcc{TestClientConn: testutils.NewTestClientConn(t)} + + balFuncs := stub.BalancerFuncs{ + UpdateClientConnState: func(bd *stub.BalancerData, s balancer.ClientConnState) error { + bd.ClientConn.UpdateState(balancer.State{ConnectivityState: connectivity.TransientFailure, Picker: nil}) + bd.ClientConn.UpdateState(balancer.State{ConnectivityState: connectivity.Ready, Picker: nil}) + return nil + }, + } + stub.Register("update_state_balancer", balFuncs) + + wtb := wtbBuilder.Build(cc, balancer.BuildOptions{}) + defer wtb.Close() + + config, err := wtbParser.ParseConfig([]byte(` +{ + "targets": { + "cluster_1": { + "weight":1, + "childPolicy": [{"update_state_balancer": ""}] + } + } +}`)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + + // Send the config, and an address with hierarchy path ["cluster_1"]. + addrs := []resolver.Address{{Addr: testBackendAddrStrs[0], Attributes: nil}} + if err := wtb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Addresses: []resolver.Address{hierarchy.Set(addrs[0], []string{"cds:cluster_1"})}}, + BalancerConfig: config, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // Verify that the only state update is the second one called by the child. + if len(cc.states) != 1 || cc.states[0].ConnectivityState != connectivity.Ready { + t.Fatalf("cc.states = %v; want [connectivity.Ready]", cc.states) + } +} diff --git a/balancer_conn_wrappers.go b/balancer_conn_wrappers.go index 11e592aabb01..a4411c22bfc8 100644 --- a/balancer_conn_wrappers.go +++ b/balancer_conn_wrappers.go @@ -19,156 +19,319 @@ package grpc import ( + "context" "fmt" + "strings" "sync" "google.golang.org/grpc/balancer" "google.golang.org/grpc/connectivity" - "google.golang.org/grpc/internal/buffer" + "google.golang.org/grpc/internal/balancer/gracefulswitch" "google.golang.org/grpc/internal/channelz" "google.golang.org/grpc/internal/grpcsync" "google.golang.org/grpc/resolver" ) -// scStateUpdate contains the subConn and the new state it changed to. -type scStateUpdate struct { - sc balancer.SubConn - state connectivity.State - err error -} +type ccbMode int + +const ( + ccbModeActive = iota + ccbModeIdle + ccbModeClosed + ccbModeExitingIdle +) -// ccBalancerWrapper is a wrapper on top of cc for balancers. -// It implements balancer.ClientConn interface. +// ccBalancerWrapper sits between the ClientConn and the Balancer. +// +// ccBalancerWrapper implements methods corresponding to the ones on the +// balancer.Balancer interface. The ClientConn is free to call these methods +// concurrently and the ccBalancerWrapper ensures that calls from the ClientConn +// to the Balancer happen synchronously and in order. +// +// ccBalancerWrapper also implements the balancer.ClientConn interface and is +// passed to the Balancer implementations. It invokes unexported methods on the +// ClientConn to handle these calls from the Balancer. +// +// It uses the gracefulswitch.Balancer internally to ensure that balancer +// switches happen in a graceful manner. type ccBalancerWrapper struct { - cc *ClientConn - balancerMu sync.Mutex // synchronizes calls to the balancer - balancer balancer.Balancer - scBuffer *buffer.Unbounded - done *grpcsync.Event + // The following fields are initialized when the wrapper is created and are + // read-only afterwards, and therefore can be accessed without a mutex. + cc *ClientConn + opts balancer.BuildOptions + + // Outgoing (gRPC --> balancer) calls are guaranteed to execute in a + // mutually exclusive manner as they are scheduled in the serializer. Fields + // accessed *only* in these serializer callbacks, can therefore be accessed + // without a mutex. + balancer *gracefulswitch.Balancer + curBalancerName string - mu sync.Mutex - subConns map[*acBalancerWrapper]struct{} + // mu guards access to the below fields. Access to the serializer and its + // cancel function needs to be mutex protected because they are overwritten + // when the wrapper exits idle mode. + mu sync.Mutex + serializer *grpcsync.CallbackSerializer // To serialize all outoing calls. + serializerCancel context.CancelFunc // To close the seralizer at close/enterIdle time. + mode ccbMode // Tracks the current mode of the wrapper. } -func newCCBalancerWrapper(cc *ClientConn, b balancer.Builder, bopts balancer.BuildOptions) *ccBalancerWrapper { +// newCCBalancerWrapper creates a new balancer wrapper. The underlying balancer +// is not created until the switchTo() method is invoked. +func newCCBalancerWrapper(cc *ClientConn, bopts balancer.BuildOptions) *ccBalancerWrapper { + ctx, cancel := context.WithCancel(context.Background()) ccb := &ccBalancerWrapper{ - cc: cc, - scBuffer: buffer.NewUnbounded(), - done: grpcsync.NewEvent(), - subConns: make(map[*acBalancerWrapper]struct{}), + cc: cc, + opts: bopts, + serializer: grpcsync.NewCallbackSerializer(ctx), + serializerCancel: cancel, } - go ccb.watcher() - ccb.balancer = b.Build(ccb, bopts) + ccb.balancer = gracefulswitch.NewBalancer(ccb, bopts) return ccb } -// watcher balancer functions sequentially, so the balancer can be implemented -// lock-free. -func (ccb *ccBalancerWrapper) watcher() { - for { - select { - case t := <-ccb.scBuffer.Get(): - ccb.scBuffer.Load() - if ccb.done.HasFired() { - break - } - ccb.balancerMu.Lock() - su := t.(*scStateUpdate) - ccb.balancer.UpdateSubConnState(su.sc, balancer.SubConnState{ConnectivityState: su.state, ConnectionError: su.err}) - ccb.balancerMu.Unlock() - case <-ccb.done.Done(): - } +// updateClientConnState is invoked by grpc to push a ClientConnState update to +// the underlying balancer. +func (ccb *ccBalancerWrapper) updateClientConnState(ccs *balancer.ClientConnState) error { + ccb.mu.Lock() + errCh := make(chan error, 1) + // Here and everywhere else where Schedule() is called, it is done with the + // lock held. But the lock guards only the scheduling part. The actual + // callback is called asynchronously without the lock being held. + ok := ccb.serializer.Schedule(func(_ context.Context) { + errCh <- ccb.balancer.UpdateClientConnState(*ccs) + }) + if !ok { + // If we are unable to schedule a function with the serializer, it + // indicates that it has been closed. A serializer is only closed when + // the wrapper is closed or is in idle. + ccb.mu.Unlock() + return fmt.Errorf("grpc: cannot send state update to a closed or idle balancer") + } + ccb.mu.Unlock() + + // We get here only if the above call to Schedule succeeds, in which case it + // is guaranteed that the scheduled function will run. Therefore it is safe + // to block on this channel. + err := <-errCh + if logger.V(2) && err != nil { + logger.Infof("error from balancer.UpdateClientConnState: %v", err) + } + return err +} + +// updateSubConnState is invoked by grpc to push a subConn state update to the +// underlying balancer. +func (ccb *ccBalancerWrapper) updateSubConnState(sc balancer.SubConn, s connectivity.State, err error) { + ccb.mu.Lock() + ccb.serializer.Schedule(func(_ context.Context) { + // Even though it is optional for balancers, gracefulswitch ensures + // opts.StateListener is set, so this cannot ever be nil. + sc.(*acBalancerWrapper).stateListener(balancer.SubConnState{ConnectivityState: s, ConnectionError: err}) + }) + ccb.mu.Unlock() +} + +func (ccb *ccBalancerWrapper) resolverError(err error) { + ccb.mu.Lock() + ccb.serializer.Schedule(func(_ context.Context) { + ccb.balancer.ResolverError(err) + }) + ccb.mu.Unlock() +} - if ccb.done.HasFired() { - ccb.balancer.Close() - ccb.mu.Lock() - scs := ccb.subConns - ccb.subConns = nil - ccb.mu.Unlock() - for acbw := range scs { - ccb.cc.removeAddrConn(acbw.getAddrConn(), errConnDrain) - } - ccb.UpdateState(balancer.State{ConnectivityState: connectivity.Connecting, Picker: nil}) +// switchTo is invoked by grpc to instruct the balancer wrapper to switch to the +// LB policy identified by name. +// +// ClientConn calls newCCBalancerWrapper() at creation time. Upon receipt of the +// first good update from the name resolver, it determines the LB policy to use +// and invokes the switchTo() method. Upon receipt of every subsequent update +// from the name resolver, it invokes this method. +// +// the ccBalancerWrapper keeps track of the current LB policy name, and skips +// the graceful balancer switching process if the name does not change. +func (ccb *ccBalancerWrapper) switchTo(name string) { + ccb.mu.Lock() + ccb.serializer.Schedule(func(_ context.Context) { + // TODO: Other languages use case-sensitive balancer registries. We should + // switch as well. See: https://github.com/grpc/grpc-go/issues/5288. + if strings.EqualFold(ccb.curBalancerName, name) { return } + ccb.buildLoadBalancingPolicy(name) + }) + ccb.mu.Unlock() +} + +// buildLoadBalancingPolicy performs the following: +// - retrieve a balancer builder for the given name. Use the default LB +// policy, pick_first, if no LB policy with name is found in the registry. +// - instruct the gracefulswitch balancer to switch to the above builder. This +// will actually build the new balancer. +// - update the `curBalancerName` field +// +// Must be called from a serializer callback. +func (ccb *ccBalancerWrapper) buildLoadBalancingPolicy(name string) { + builder := balancer.Get(name) + if builder == nil { + channelz.Warningf(logger, ccb.cc.channelzID, "Channel switches to new LB policy %q, since the specified LB policy %q was not registered", PickFirstBalancerName, name) + builder = newPickfirstBuilder() + } else { + channelz.Infof(logger, ccb.cc.channelzID, "Channel switches to new LB policy %q", name) + } + + if err := ccb.balancer.SwitchTo(builder); err != nil { + channelz.Errorf(logger, ccb.cc.channelzID, "Channel failed to build new LB policy %q: %v", name, err) + return } + ccb.curBalancerName = builder.Name() } func (ccb *ccBalancerWrapper) close() { - ccb.done.Fire() -} - -func (ccb *ccBalancerWrapper) handleSubConnStateChange(sc balancer.SubConn, s connectivity.State, err error) { - // When updating addresses for a SubConn, if the address in use is not in - // the new addresses, the old ac will be tearDown() and a new ac will be - // created. tearDown() generates a state change with Shutdown state, we - // don't want the balancer to receive this state change. So before - // tearDown() on the old ac, ac.acbw (acWrapper) will be set to nil, and - // this function will be called with (nil, Shutdown). We don't need to call - // balancer method in this case. - if sc == nil { + channelz.Info(logger, ccb.cc.channelzID, "ccBalancerWrapper: closing") + ccb.closeBalancer(ccbModeClosed) +} + +// enterIdleMode is invoked by grpc when the channel enters idle mode upon +// expiry of idle_timeout. This call blocks until the balancer is closed. +func (ccb *ccBalancerWrapper) enterIdleMode() { + channelz.Info(logger, ccb.cc.channelzID, "ccBalancerWrapper: entering idle mode") + ccb.closeBalancer(ccbModeIdle) +} + +// closeBalancer is invoked when the channel is being closed or when it enters +// idle mode upon expiry of idle_timeout. +func (ccb *ccBalancerWrapper) closeBalancer(m ccbMode) { + ccb.mu.Lock() + if ccb.mode == ccbModeClosed || ccb.mode == ccbModeIdle { + ccb.mu.Unlock() return } - ccb.scBuffer.Put(&scStateUpdate{ - sc: sc, - state: s, - err: err, + + ccb.mode = m + done := ccb.serializer.Done() + b := ccb.balancer + ok := ccb.serializer.Schedule(func(_ context.Context) { + // Close the serializer to ensure that no more calls from gRPC are sent + // to the balancer. + ccb.serializerCancel() + // Empty the current balancer name because we don't have a balancer + // anymore and also so that we act on the next call to switchTo by + // creating a new balancer specified by the new resolver. + ccb.curBalancerName = "" }) + if !ok { + ccb.mu.Unlock() + return + } + ccb.mu.Unlock() + + // Give enqueued callbacks a chance to finish before closing the balancer. + <-done + b.Close() } -func (ccb *ccBalancerWrapper) updateClientConnState(ccs *balancer.ClientConnState) error { - ccb.balancerMu.Lock() - defer ccb.balancerMu.Unlock() - return ccb.balancer.UpdateClientConnState(*ccs) +// exitIdleMode is invoked by grpc when the channel exits idle mode either +// because of an RPC or because of an invocation of the Connect() API. This +// recreates the balancer that was closed previously when entering idle mode. +// +// If the channel is not in idle mode, we know for a fact that we are here as a +// result of the user calling the Connect() method on the ClientConn. In this +// case, we can simply forward the call to the underlying balancer, instructing +// it to reconnect to the backends. +func (ccb *ccBalancerWrapper) exitIdleMode() { + ccb.mu.Lock() + if ccb.mode == ccbModeClosed { + // Request to exit idle is a no-op when wrapper is already closed. + ccb.mu.Unlock() + return + } + + if ccb.mode == ccbModeIdle { + // Recreate the serializer which was closed when we entered idle. + ctx, cancel := context.WithCancel(context.Background()) + ccb.serializer = grpcsync.NewCallbackSerializer(ctx) + ccb.serializerCancel = cancel + } + + // The ClientConn guarantees that mutual exclusion between close() and + // exitIdleMode(), and since we just created a new serializer, we can be + // sure that the below function will be scheduled. + done := make(chan struct{}) + ccb.serializer.Schedule(func(_ context.Context) { + defer close(done) + + ccb.mu.Lock() + defer ccb.mu.Unlock() + + if ccb.mode != ccbModeIdle { + ccb.balancer.ExitIdle() + return + } + + // Gracefulswitch balancer does not support a switchTo operation after + // being closed. Hence we need to create a new one here. + ccb.balancer = gracefulswitch.NewBalancer(ccb, ccb.opts) + ccb.mode = ccbModeActive + channelz.Info(logger, ccb.cc.channelzID, "ccBalancerWrapper: exiting idle mode") + + }) + ccb.mu.Unlock() + + <-done } -func (ccb *ccBalancerWrapper) resolverError(err error) { - ccb.balancerMu.Lock() - ccb.balancer.ResolverError(err) - ccb.balancerMu.Unlock() +func (ccb *ccBalancerWrapper) isIdleOrClosed() bool { + ccb.mu.Lock() + defer ccb.mu.Unlock() + return ccb.mode == ccbModeIdle || ccb.mode == ccbModeClosed } func (ccb *ccBalancerWrapper) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) { - if len(addrs) <= 0 { - return nil, fmt.Errorf("grpc: cannot create SubConn with empty address list") + if ccb.isIdleOrClosed() { + return nil, fmt.Errorf("grpc: cannot create SubConn when balancer is closed or idle") } - ccb.mu.Lock() - defer ccb.mu.Unlock() - if ccb.subConns == nil { - return nil, fmt.Errorf("grpc: ClientConn balancer wrapper was closed") + + if len(addrs) == 0 { + return nil, fmt.Errorf("grpc: cannot create SubConn with empty address list") } ac, err := ccb.cc.newAddrConn(addrs, opts) if err != nil { + channelz.Warningf(logger, ccb.cc.channelzID, "acBalancerWrapper: NewSubConn: failed to newAddrConn: %v", err) return nil, err } - acbw := &acBalancerWrapper{ac: ac} - acbw.ac.mu.Lock() + acbw := &acBalancerWrapper{ + ccb: ccb, + ac: ac, + producers: make(map[balancer.ProducerBuilder]*refCountedProducer), + stateListener: opts.StateListener, + } ac.acbw = acbw - acbw.ac.mu.Unlock() - ccb.subConns[acbw] = struct{}{} return acbw, nil } func (ccb *ccBalancerWrapper) RemoveSubConn(sc balancer.SubConn) { - acbw, ok := sc.(*acBalancerWrapper) - if !ok { + // The graceful switch balancer will never call this. + logger.Errorf("ccb RemoveSubConn(%v) called unexpectedly, sc") +} + +func (ccb *ccBalancerWrapper) UpdateAddresses(sc balancer.SubConn, addrs []resolver.Address) { + if ccb.isIdleOrClosed() { return } - ccb.mu.Lock() - defer ccb.mu.Unlock() - if ccb.subConns == nil { + + acbw, ok := sc.(*acBalancerWrapper) + if !ok { return } - delete(ccb.subConns, acbw) - ccb.cc.removeAddrConn(acbw.getAddrConn(), errConnDrain) + acbw.UpdateAddresses(addrs) } func (ccb *ccBalancerWrapper) UpdateState(s balancer.State) { - ccb.mu.Lock() - defer ccb.mu.Unlock() - if ccb.subConns == nil { + if ccb.isIdleOrClosed() { return } + // Update picker before updating state. Even though the ordering here does // not matter, it can lead to multiple calls of Pick in the common start-up // case where we wait for ready and then perform an RPC. If the picker is @@ -179,6 +342,10 @@ func (ccb *ccBalancerWrapper) UpdateState(s balancer.State) { } func (ccb *ccBalancerWrapper) ResolveNow(o resolver.ResolveNowOptions) { + if ccb.isIdleOrClosed() { + return + } + ccb.cc.resolveNow(o) } @@ -189,58 +356,99 @@ func (ccb *ccBalancerWrapper) Target() string { // acBalancerWrapper is a wrapper on top of ac for balancers. // It implements balancer.SubConn interface. type acBalancerWrapper struct { - mu sync.Mutex - ac *addrConn + ac *addrConn // read-only + ccb *ccBalancerWrapper // read-only + stateListener func(balancer.SubConnState) + + mu sync.Mutex + producers map[balancer.ProducerBuilder]*refCountedProducer +} + +func (acbw *acBalancerWrapper) String() string { + return fmt.Sprintf("SubConn(id:%d)", acbw.ac.channelzID.Int()) } func (acbw *acBalancerWrapper) UpdateAddresses(addrs []resolver.Address) { - acbw.mu.Lock() - defer acbw.mu.Unlock() - if len(addrs) <= 0 { - acbw.ac.tearDown(errConnDrain) + acbw.ac.updateAddrs(addrs) +} + +func (acbw *acBalancerWrapper) Connect() { + go acbw.ac.connect() +} + +func (acbw *acBalancerWrapper) Shutdown() { + ccb := acbw.ccb + if ccb.isIdleOrClosed() { + // It it safe to ignore this call when the balancer is closed or in idle + // because the ClientConn takes care of closing the connections. + // + // Not returning early from here when the balancer is closed or in idle + // leads to a deadlock though, because of the following sequence of + // calls when holding cc.mu: + // cc.exitIdleMode --> ccb.enterIdleMode --> gsw.Close --> + // ccb.RemoveAddrConn --> cc.removeAddrConn return } - if !acbw.ac.tryUpdateAddrs(addrs) { - cc := acbw.ac.cc - opts := acbw.ac.scopts - acbw.ac.mu.Lock() - // Set old ac.acbw to nil so the Shutdown state update will be ignored - // by balancer. - // - // TODO(bar) the state transition could be wrong when tearDown() old ac - // and creating new ac, fix the transition. - acbw.ac.acbw = nil - acbw.ac.mu.Unlock() - acState := acbw.ac.getState() - acbw.ac.tearDown(errConnDrain) - - if acState == connectivity.Shutdown { - return - } - ac, err := cc.newAddrConn(addrs, opts) - if err != nil { - channelz.Warningf(logger, acbw.ac.channelzID, "acBalancerWrapper: UpdateAddresses: failed to newAddrConn: %v", err) - return - } - acbw.ac = ac - ac.mu.Lock() - ac.acbw = acbw - ac.mu.Unlock() - if acState != connectivity.Idle { - ac.connect() - } + ccb.cc.removeAddrConn(acbw.ac, errConnDrain) +} + +// NewStream begins a streaming RPC on the addrConn. If the addrConn is not +// ready, blocks until it is or ctx expires. Returns an error when the context +// expires or the addrConn is shut down. +func (acbw *acBalancerWrapper) NewStream(ctx context.Context, desc *StreamDesc, method string, opts ...CallOption) (ClientStream, error) { + transport, err := acbw.ac.getTransport(ctx) + if err != nil { + return nil, err } + return newNonRetryClientStream(ctx, desc, method, transport, acbw.ac, opts...) } -func (acbw *acBalancerWrapper) Connect() { - acbw.mu.Lock() - defer acbw.mu.Unlock() - acbw.ac.connect() +// Invoke performs a unary RPC. If the addrConn is not ready, returns +// errSubConnNotReady. +func (acbw *acBalancerWrapper) Invoke(ctx context.Context, method string, args any, reply any, opts ...CallOption) error { + cs, err := acbw.NewStream(ctx, unaryStreamDesc, method, opts...) + if err != nil { + return err + } + if err := cs.SendMsg(args); err != nil { + return err + } + return cs.RecvMsg(reply) } -func (acbw *acBalancerWrapper) getAddrConn() *addrConn { +type refCountedProducer struct { + producer balancer.Producer + refs int // number of current refs to the producer + close func() // underlying producer's close function +} + +func (acbw *acBalancerWrapper) GetOrBuildProducer(pb balancer.ProducerBuilder) (balancer.Producer, func()) { acbw.mu.Lock() defer acbw.mu.Unlock() - return acbw.ac + + // Look up existing producer from this builder. + pData := acbw.producers[pb] + if pData == nil { + // Not found; create a new one and add it to the producers map. + p, close := pb.Build(acbw) + pData = &refCountedProducer{producer: p, close: close} + acbw.producers[pb] = pData + } + // Account for this new reference. + pData.refs++ + + // Return a cleanup function wrapped in a OnceFunc to remove this reference + // and delete the refCountedProducer from the map if the total reference + // count goes to zero. + unref := func() { + acbw.mu.Lock() + pData.refs-- + if pData.refs == 0 { + defer pData.close() // Run outside the acbw mutex + delete(acbw.producers, pb) + } + acbw.mu.Unlock() + } + return pData.producer, grpcsync.OnceFunc(unref) } diff --git a/balancer_conn_wrappers_test.go b/balancer_conn_wrappers_test.go deleted file mode 100644 index 935d11d1d391..000000000000 --- a/balancer_conn_wrappers_test.go +++ /dev/null @@ -1,90 +0,0 @@ -/* - * - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package grpc - -import ( - "fmt" - "net" - "testing" - - "google.golang.org/grpc/balancer" - "google.golang.org/grpc/balancer/roundrobin" - "google.golang.org/grpc/internal/balancer/stub" - "google.golang.org/grpc/resolver" - "google.golang.org/grpc/resolver/manual" -) - -// TestBalancerErrorResolverPolling injects balancer errors and verifies -// ResolveNow is called on the resolver with the appropriate backoff strategy -// being consulted between ResolveNow calls. -func (s) TestBalancerErrorResolverPolling(t *testing.T) { - // The test balancer will return ErrBadResolverState iff the - // ClientConnState contains no addresses. - bf := stub.BalancerFuncs{ - UpdateClientConnState: func(_ *stub.BalancerData, s balancer.ClientConnState) error { - if len(s.ResolverState.Addresses) == 0 { - return balancer.ErrBadResolverState - } - return nil - }, - } - const balName = "BalancerErrorResolverPolling" - stub.Register(balName, bf) - - testResolverErrorPolling(t, - func(r *manual.Resolver) { - // No addresses so the balancer will fail. - r.CC.UpdateState(resolver.State{}) - }, func(r *manual.Resolver) { - // UpdateState will block if ResolveNow is being called (which blocks on - // rn), so call it in a goroutine. Include some address so the balancer - // will be happy. - go r.CC.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: "x"}}}) - }, - WithDefaultServiceConfig(fmt.Sprintf(`{ "loadBalancingConfig": [{"%v": {}}] }`, balName))) -} - -// TestRoundRobinZeroAddressesResolverPolling reports no addresses to the round -// robin balancer and verifies ResolveNow is called on the resolver with the -// appropriate backoff strategy being consulted between ResolveNow calls. -func (s) TestRoundRobinZeroAddressesResolverPolling(t *testing.T) { - // We need to start a real server or else the connecting loop will call - // ResolveNow after every iteration, even after a valid resolver result is - // returned. - lis, err := net.Listen("tcp", "localhost:0") - if err != nil { - t.Fatalf("Error while listening. Err: %v", err) - } - defer lis.Close() - s := NewServer() - defer s.Stop() - go s.Serve(lis) - - testResolverErrorPolling(t, - func(r *manual.Resolver) { - // No addresses so the balancer will fail. - r.CC.UpdateState(resolver.State{}) - }, func(r *manual.Resolver) { - // UpdateState will block if ResolveNow is being called (which - // blocks on rn), so call it in a goroutine. Include a valid - // address so the balancer will be happy. - go r.CC.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: lis.Addr().String()}}}) - }, - WithDefaultServiceConfig(fmt.Sprintf(`{ "loadBalancingConfig": [{"%v": {}}] }`, roundrobin.Name))) -} diff --git a/balancer_switching_test.go b/balancer_switching_test.go deleted file mode 100644 index ed132121280c..000000000000 --- a/balancer_switching_test.go +++ /dev/null @@ -1,535 +0,0 @@ -/* - * - * Copyright 2017 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package grpc - -import ( - "context" - "fmt" - "math" - "testing" - "time" - - "google.golang.org/grpc/balancer" - "google.golang.org/grpc/balancer/roundrobin" - "google.golang.org/grpc/internal" - "google.golang.org/grpc/resolver" - "google.golang.org/grpc/resolver/manual" - "google.golang.org/grpc/serviceconfig" -) - -var _ balancer.Builder = &magicalLB{} -var _ balancer.Balancer = &magicalLB{} - -// magicalLB is a ringer for grpclb. It is used to avoid circular dependencies on the grpclb package -type magicalLB struct{} - -func (b *magicalLB) Name() string { - return "grpclb" -} - -func (b *magicalLB) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { - return b -} - -func (b *magicalLB) ResolverError(error) {} - -func (b *magicalLB) UpdateSubConnState(balancer.SubConn, balancer.SubConnState) {} - -func (b *magicalLB) UpdateClientConnState(balancer.ClientConnState) error { - return nil -} - -func (b *magicalLB) Close() {} - -func init() { - balancer.Register(&magicalLB{}) -} - -func startServers(t *testing.T, numServers int, maxStreams uint32) ([]*server, func()) { - var servers []*server - for i := 0; i < numServers; i++ { - s := newTestServer() - servers = append(servers, s) - go s.start(t, 0, maxStreams) - s.wait(t, 2*time.Second) - } - return servers, func() { - for i := 0; i < numServers; i++ { - servers[i].stop() - } - } -} - -func checkPickFirst(cc *ClientConn, servers []*server) error { - var ( - req = "port" - reply string - err error - ) - connected := false - for i := 0; i < 5000; i++ { - if err = cc.Invoke(context.Background(), "/foo/bar", &req, &reply); errorDesc(err) == servers[0].port { - if connected { - // connected is set to false if peer is not server[0]. So if - // connected is true here, this is the second time we saw - // server[0] in a row. Break because pickfirst is in effect. - break - } - connected = true - } else { - connected = false - } - time.Sleep(time.Millisecond) - } - if !connected { - return fmt.Errorf("pickfirst is not in effect after 5 second, EmptyCall() = _, %v, want _, %v", err, servers[0].port) - } - // The following RPCs should all succeed with the first server. - for i := 0; i < 3; i++ { - err = cc.Invoke(context.Background(), "/foo/bar", &req, &reply) - if errorDesc(err) != servers[0].port { - return fmt.Errorf("index %d: want peer %v, got peer %v", i, servers[0].port, err) - } - } - return nil -} - -func checkRoundRobin(cc *ClientConn, servers []*server) error { - var ( - req = "port" - reply string - err error - ) - - // Make sure connections to all servers are up. - for i := 0; i < 2; i++ { - // Do this check twice, otherwise the first RPC's transport may still be - // picked by the closing pickfirst balancer, and the test becomes flaky. - for _, s := range servers { - var up bool - for i := 0; i < 5000; i++ { - if err = cc.Invoke(context.Background(), "/foo/bar", &req, &reply); errorDesc(err) == s.port { - up = true - break - } - time.Sleep(time.Millisecond) - } - if !up { - return fmt.Errorf("server %v is not up within 5 second", s.port) - } - } - } - - serverCount := len(servers) - for i := 0; i < 3*serverCount; i++ { - err = cc.Invoke(context.Background(), "/foo/bar", &req, &reply) - if errorDesc(err) != servers[i%serverCount].port { - return fmt.Errorf("index %d: want peer %v, got peer %v", i, servers[i%serverCount].port, err) - } - } - return nil -} - -func (s) TestSwitchBalancer(t *testing.T) { - r := manual.NewBuilderWithScheme("whatever") - - const numServers = 2 - servers, scleanup := startServers(t, numServers, math.MaxInt32) - defer scleanup() - - cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithResolvers(r), WithCodec(testCodec{})) - if err != nil { - t.Fatalf("failed to dial: %v", err) - } - defer cc.Close() - addrs := []resolver.Address{{Addr: servers[0].addr}, {Addr: servers[1].addr}} - r.UpdateState(resolver.State{Addresses: addrs}) - // The default balancer is pickfirst. - if err := checkPickFirst(cc, servers); err != nil { - t.Fatalf("check pickfirst returned non-nil error: %v", err) - } - // Switch to roundrobin. - cc.updateResolverState(resolver.State{ServiceConfig: parseCfg(r, `{"loadBalancingPolicy": "round_robin"}`), Addresses: addrs}, nil) - if err := checkRoundRobin(cc, servers); err != nil { - t.Fatalf("check roundrobin returned non-nil error: %v", err) - } - // Switch to pickfirst. - cc.updateResolverState(resolver.State{ServiceConfig: parseCfg(r, `{"loadBalancingPolicy": "pick_first"}`), Addresses: addrs}, nil) - if err := checkPickFirst(cc, servers); err != nil { - t.Fatalf("check pickfirst returned non-nil error: %v", err) - } -} - -// Test that balancer specified by dial option will not be overridden. -func (s) TestBalancerDialOption(t *testing.T) { - r := manual.NewBuilderWithScheme("whatever") - - const numServers = 2 - servers, scleanup := startServers(t, numServers, math.MaxInt32) - defer scleanup() - - cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithResolvers(r), WithCodec(testCodec{}), WithBalancerName(roundrobin.Name)) - if err != nil { - t.Fatalf("failed to dial: %v", err) - } - defer cc.Close() - addrs := []resolver.Address{{Addr: servers[0].addr}, {Addr: servers[1].addr}} - r.UpdateState(resolver.State{Addresses: addrs}) - // The init balancer is roundrobin. - if err := checkRoundRobin(cc, servers); err != nil { - t.Fatalf("check roundrobin returned non-nil error: %v", err) - } - // Switch to pickfirst. - cc.updateResolverState(resolver.State{ServiceConfig: parseCfg(r, `{"loadBalancingPolicy": "pick_first"}`), Addresses: addrs}, nil) - // Balancer is still roundrobin. - if err := checkRoundRobin(cc, servers); err != nil { - t.Fatalf("check roundrobin returned non-nil error: %v", err) - } -} - -// First addr update contains grpclb. -func (s) TestSwitchBalancerGRPCLBFirst(t *testing.T) { - r := manual.NewBuilderWithScheme("whatever") - - cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithResolvers(r), WithCodec(testCodec{})) - if err != nil { - t.Fatalf("failed to dial: %v", err) - } - defer cc.Close() - - // ClientConn will switch balancer to grpclb when receives an address of - // type GRPCLB. - r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: "backend"}, {Addr: "grpclb", Type: resolver.GRPCLB}}}) - var isGRPCLB bool - for i := 0; i < 5000; i++ { - cc.mu.Lock() - isGRPCLB = cc.curBalancerName == "grpclb" - cc.mu.Unlock() - if isGRPCLB { - break - } - time.Sleep(time.Millisecond) - } - if !isGRPCLB { - t.Fatalf("after 5 second, cc.balancer is of type %v, not grpclb", cc.curBalancerName) - } - - // New update containing new backend and new grpclb. Should not switch - // balancer. - r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: "backend2"}, {Addr: "grpclb2", Type: resolver.GRPCLB}}}) - for i := 0; i < 200; i++ { - cc.mu.Lock() - isGRPCLB = cc.curBalancerName == "grpclb" - cc.mu.Unlock() - if !isGRPCLB { - break - } - time.Sleep(time.Millisecond) - } - if !isGRPCLB { - t.Fatalf("within 200 ms, cc.balancer switched to !grpclb, want grpclb") - } - - var isPickFirst bool - // Switch balancer to pickfirst. - r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: "backend"}}}) - for i := 0; i < 5000; i++ { - cc.mu.Lock() - isPickFirst = cc.curBalancerName == PickFirstBalancerName - cc.mu.Unlock() - if isPickFirst { - break - } - time.Sleep(time.Millisecond) - } - if !isPickFirst { - t.Fatalf("after 5 second, cc.balancer is of type %v, not pick_first", cc.curBalancerName) - } -} - -// First addr update does not contain grpclb. -func (s) TestSwitchBalancerGRPCLBSecond(t *testing.T) { - r := manual.NewBuilderWithScheme("whatever") - - cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithResolvers(r), WithCodec(testCodec{})) - if err != nil { - t.Fatalf("failed to dial: %v", err) - } - defer cc.Close() - - r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: "backend"}}}) - var isPickFirst bool - for i := 0; i < 5000; i++ { - cc.mu.Lock() - isPickFirst = cc.curBalancerName == PickFirstBalancerName - cc.mu.Unlock() - if isPickFirst { - break - } - time.Sleep(time.Millisecond) - } - if !isPickFirst { - t.Fatalf("after 5 second, cc.balancer is of type %v, not pick_first", cc.curBalancerName) - } - - // ClientConn will switch balancer to grpclb when receives an address of - // type GRPCLB. - r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: "backend"}, {Addr: "grpclb", Type: resolver.GRPCLB}}}) - var isGRPCLB bool - for i := 0; i < 5000; i++ { - cc.mu.Lock() - isGRPCLB = cc.curBalancerName == "grpclb" - cc.mu.Unlock() - if isGRPCLB { - break - } - time.Sleep(time.Millisecond) - } - if !isGRPCLB { - t.Fatalf("after 5 second, cc.balancer is of type %v, not grpclb", cc.curBalancerName) - } - - // New update containing new backend and new grpclb. Should not switch - // balancer. - r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: "backend2"}, {Addr: "grpclb2", Type: resolver.GRPCLB}}}) - for i := 0; i < 200; i++ { - cc.mu.Lock() - isGRPCLB = cc.curBalancerName == "grpclb" - cc.mu.Unlock() - if !isGRPCLB { - break - } - time.Sleep(time.Millisecond) - } - if !isGRPCLB { - t.Fatalf("within 200 ms, cc.balancer switched to !grpclb, want grpclb") - } - - // Switch balancer back. - r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: "backend"}}}) - for i := 0; i < 5000; i++ { - cc.mu.Lock() - isPickFirst = cc.curBalancerName == PickFirstBalancerName - cc.mu.Unlock() - if isPickFirst { - break - } - time.Sleep(time.Millisecond) - } - if !isPickFirst { - t.Fatalf("after 5 second, cc.balancer is of type %v, not pick_first", cc.curBalancerName) - } -} - -// Test that if the current balancer is roundrobin, after switching to grpclb, -// when the resolved address doesn't contain grpclb addresses, balancer will be -// switched back to roundrobin. -func (s) TestSwitchBalancerGRPCLBRoundRobin(t *testing.T) { - r := manual.NewBuilderWithScheme("whatever") - - cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithResolvers(r), WithCodec(testCodec{})) - if err != nil { - t.Fatalf("failed to dial: %v", err) - } - defer cc.Close() - - sc := parseCfg(r, `{"loadBalancingPolicy": "round_robin"}`) - - r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: "backend"}}, ServiceConfig: sc}) - var isRoundRobin bool - for i := 0; i < 5000; i++ { - cc.mu.Lock() - isRoundRobin = cc.curBalancerName == "round_robin" - cc.mu.Unlock() - if isRoundRobin { - break - } - time.Sleep(time.Millisecond) - } - if !isRoundRobin { - t.Fatalf("after 5 second, cc.balancer is of type %v, not round_robin", cc.curBalancerName) - } - - // ClientConn will switch balancer to grpclb when receives an address of - // type GRPCLB. - r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: "grpclb", Type: resolver.GRPCLB}}, ServiceConfig: sc}) - var isGRPCLB bool - for i := 0; i < 5000; i++ { - cc.mu.Lock() - isGRPCLB = cc.curBalancerName == "grpclb" - cc.mu.Unlock() - if isGRPCLB { - break - } - time.Sleep(time.Millisecond) - } - if !isGRPCLB { - t.Fatalf("after 5 second, cc.balancer is of type %v, not grpclb", cc.curBalancerName) - } - - // Switch balancer back. - r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: "backend"}}, ServiceConfig: sc}) - for i := 0; i < 5000; i++ { - cc.mu.Lock() - isRoundRobin = cc.curBalancerName == "round_robin" - cc.mu.Unlock() - if isRoundRobin { - break - } - time.Sleep(time.Millisecond) - } - if !isRoundRobin { - t.Fatalf("after 5 second, cc.balancer is of type %v, not round_robin", cc.curBalancerName) - } -} - -// Test that if resolved address list contains grpclb, the balancer option in -// service config won't take effect. But when there's no grpclb address in a new -// resolved address list, balancer will be switched to the new one. -func (s) TestSwitchBalancerGRPCLBServiceConfig(t *testing.T) { - r := manual.NewBuilderWithScheme("whatever") - - cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithResolvers(r), WithCodec(testCodec{})) - if err != nil { - t.Fatalf("failed to dial: %v", err) - } - defer cc.Close() - - r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: "backend"}}}) - var isPickFirst bool - for i := 0; i < 5000; i++ { - cc.mu.Lock() - isPickFirst = cc.curBalancerName == PickFirstBalancerName - cc.mu.Unlock() - if isPickFirst { - break - } - time.Sleep(time.Millisecond) - } - if !isPickFirst { - t.Fatalf("after 5 second, cc.balancer is of type %v, not pick_first", cc.curBalancerName) - } - - // ClientConn will switch balancer to grpclb when receives an address of - // type GRPCLB. - addrs := []resolver.Address{{Addr: "grpclb", Type: resolver.GRPCLB}} - r.UpdateState(resolver.State{Addresses: addrs}) - var isGRPCLB bool - for i := 0; i < 5000; i++ { - cc.mu.Lock() - isGRPCLB = cc.curBalancerName == "grpclb" - cc.mu.Unlock() - if isGRPCLB { - break - } - time.Sleep(time.Millisecond) - } - if !isGRPCLB { - t.Fatalf("after 5 second, cc.balancer is of type %v, not grpclb", cc.curBalancerName) - } - - sc := parseCfg(r, `{"loadBalancingPolicy": "round_robin"}`) - r.UpdateState(resolver.State{Addresses: addrs, ServiceConfig: sc}) - var isRoundRobin bool - for i := 0; i < 200; i++ { - cc.mu.Lock() - isRoundRobin = cc.curBalancerName == "round_robin" - cc.mu.Unlock() - if isRoundRobin { - break - } - time.Sleep(time.Millisecond) - } - // Balancer should NOT switch to round_robin because resolved list contains - // grpclb. - if isRoundRobin { - t.Fatalf("within 200 ms, cc.balancer switched to round_robin, want grpclb") - } - - // Switch balancer back. - r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: "backend"}}, ServiceConfig: sc}) - for i := 0; i < 5000; i++ { - cc.mu.Lock() - isRoundRobin = cc.curBalancerName == "round_robin" - cc.mu.Unlock() - if isRoundRobin { - break - } - time.Sleep(time.Millisecond) - } - if !isRoundRobin { - t.Fatalf("after 5 second, cc.balancer is of type %v, not round_robin", cc.curBalancerName) - } -} - -// Test that when switching to grpclb fails because grpclb is not registered, -// the fallback balancer will only get backend addresses, not the grpclb server -// address. -// -// The tests sends 3 server addresses (all backends) as resolved addresses, but -// claim the first one is grpclb server. The all RPCs should all be send to the -// other addresses, not the first one. -func (s) TestSwitchBalancerGRPCLBWithGRPCLBNotRegistered(t *testing.T) { - internal.BalancerUnregister("grpclb") - defer balancer.Register(&magicalLB{}) - - r := manual.NewBuilderWithScheme("whatever") - - const numServers = 3 - servers, scleanup := startServers(t, numServers, math.MaxInt32) - defer scleanup() - - cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithResolvers(r), WithCodec(testCodec{})) - if err != nil { - t.Fatalf("failed to dial: %v", err) - } - defer cc.Close() - r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: servers[1].addr}, {Addr: servers[2].addr}}}) - // The default balancer is pickfirst. - if err := checkPickFirst(cc, servers[1:]); err != nil { - t.Fatalf("check pickfirst returned non-nil error: %v", err) - } - // Try switching to grpclb by sending servers[0] as grpclb address. It's - // expected that servers[0] will be filtered out, so it will not be used by - // the balancer. - // - // If the filtering failed, servers[0] will be used for RPCs and the RPCs - // will succeed. The following checks will catch this and fail. - addrs := []resolver.Address{ - {Addr: servers[0].addr, Type: resolver.GRPCLB}, - {Addr: servers[1].addr}, {Addr: servers[2].addr}} - r.UpdateState(resolver.State{Addresses: addrs}) - // Still check for pickfirst, but only with server[1] and server[2]. - if err := checkPickFirst(cc, servers[1:]); err != nil { - t.Fatalf("check pickfirst returned non-nil error: %v", err) - } - // Switch to roundrobin, and check against server[1] and server[2]. - cc.updateResolverState(resolver.State{ServiceConfig: parseCfg(r, `{"loadBalancingPolicy": "round_robin"}`), Addresses: addrs}, nil) - if err := checkRoundRobin(cc, servers[1:]); err != nil { - t.Fatalf("check roundrobin returned non-nil error: %v", err) - } -} - -func parseCfg(r *manual.Resolver, s string) *serviceconfig.ParseResult { - scpr := r.CC.ParseServiceConfig(s) - if scpr.Err != nil { - panic(fmt.Sprintf("Error parsing config %q: %v", s, scpr.Err)) - } - return scpr -} diff --git a/benchmark/benchmain/main.go b/benchmark/benchmain/main.go index 956024c7f047..ec26909ba79d 100644 --- a/benchmark/benchmain/main.go +++ b/benchmark/benchmain/main.go @@ -21,10 +21,10 @@ Package main provides benchmark with setting flags. An example to run some benchmarks with profiling enabled: -go run benchmark/benchmain/main.go -benchtime=10s -workloads=all \ - -compression=gzip -maxConcurrentCalls=1 -trace=off \ - -reqSizeBytes=1,1048576 -respSizeBytes=1,1048576 -networkMode=Local \ - -cpuProfile=cpuProf -memProfile=memProf -memProfileRate=10000 -resultFile=result + go run benchmark/benchmain/main.go -benchtime=10s -workloads=all \ + -compression=gzip -maxConcurrentCalls=1 -trace=off \ + -reqSizeBytes=1,1048576 -respSizeBytes=1,1048576 -networkMode=Local \ + -cpuProfile=cpuProf -memProfile=memProf -memProfileRate=10000 -resultFile=result As a suggestion, when creating a branch, you can run this benchmark and save the result file "-resultFile=basePerf", and later when you at the middle of the work or finish the @@ -32,10 +32,11 @@ work, you can get the benchmark result and compare it with the base anytime. Assume there are two result files names as "basePerf" and "curPerf" created by adding -resultFile=basePerf and -resultFile=curPerf. - To format the curPerf, run: - go run benchmark/benchresult/main.go curPerf - To observe how the performance changes based on a base result, run: - go run benchmark/benchresult/main.go basePerf curPerf + + To format the curPerf, run: + go run benchmark/benchresult/main.go curPerf + To observe how the performance changes based on a base result, run: + go run benchmark/benchresult/main.go basePerf curPerf */ package main @@ -45,28 +46,34 @@ import ( "flag" "fmt" "io" - "io/ioutil" "log" + "math/rand" "net" "os" "reflect" "runtime" "runtime/pprof" + "strconv" "strings" "sync" "sync/atomic" "time" "google.golang.org/grpc" + "google.golang.org/grpc/benchmark" bm "google.golang.org/grpc/benchmark" "google.golang.org/grpc/benchmark/flags" - testpb "google.golang.org/grpc/benchmark/grpc_testing" "google.golang.org/grpc/benchmark/latency" "google.golang.org/grpc/benchmark/stats" + "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/grpclog" "google.golang.org/grpc/internal/channelz" "google.golang.org/grpc/keepalive" + "google.golang.org/grpc/metadata" "google.golang.org/grpc/test/bufconn" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" ) var ( @@ -75,7 +82,8 @@ var ( traceMode = flags.StringWithAllowedValues("trace", toggleModeOff, fmt.Sprintf("Trace mode - One of: %v", strings.Join(allToggleModes, ", ")), allToggleModes) preloaderMode = flags.StringWithAllowedValues("preloader", toggleModeOff, - fmt.Sprintf("Preloader mode - One of: %v", strings.Join(allToggleModes, ", ")), allToggleModes) + fmt.Sprintf("Preloader mode - One of: %v, preloader works only in streaming and unconstrained modes and will be ignored in unary mode", + strings.Join(allToggleModes, ", ")), allToggleModes) channelzOn = flags.StringWithAllowedValues("channelz", toggleModeOff, fmt.Sprintf("Channelz mode - One of: %v", strings.Join(allToggleModes, ", ")), allToggleModes) compressorMode = flags.StringWithAllowedValues("compression", compModeOff, @@ -100,6 +108,15 @@ var ( useBufconn = flag.Bool("bufconn", false, "Use in-memory connection instead of system network I/O") enableKeepalive = flag.Bool("enable_keepalive", false, "Enable client keepalive. \n"+ "Keepalive.Time is set to 10s, Keepalive.Timeout is set to 1s, Keepalive.PermitWithoutStream is set to true.") + clientReadBufferSize = flags.IntSlice("clientReadBufferSize", []int{-1}, "Configures the client read buffer size in bytes. If negative, use the default - may be a a comma-separated list") + clientWriteBufferSize = flags.IntSlice("clientWriteBufferSize", []int{-1}, "Configures the client write buffer size in bytes. If negative, use the default - may be a a comma-separated list") + serverReadBufferSize = flags.IntSlice("serverReadBufferSize", []int{-1}, "Configures the server read buffer size in bytes. If negative, use the default - may be a a comma-separated list") + serverWriteBufferSize = flags.IntSlice("serverWriteBufferSize", []int{-1}, "Configures the server write buffer size in bytes. If negative, use the default - may be a a comma-separated list") + sleepBetweenRPCs = flags.DurationSlice("sleepBetweenRPCs", []time.Duration{0}, "Configures the maximum amount of time the client should sleep between consecutive RPCs - may be a a comma-separated list") + connections = flag.Int("connections", 1, "The number of connections. Each connection will handle maxConcurrentCalls RPC streams") + recvBufferPool = flags.StringWithAllowedValues("recvBufferPool", recvBufferPoolNil, "Configures the shared receive buffer pool. One of: nil, simple, all", allRecvBufferPools) + sharedWriteBuffer = flags.StringWithAllowedValues("sharedWriteBuffer", toggleModeOff, + fmt.Sprintf("Configures both client and server to share write buffer - One of: %v", strings.Join(allToggleModes, ", ")), allToggleModes) logger = grpclog.Component("benchmark") ) @@ -124,6 +141,10 @@ const ( networkModeLAN = "LAN" networkModeWAN = "WAN" networkLongHaul = "Longhaul" + // Shared recv buffer pool + recvBufferPoolNil = "nil" + recvBufferPoolSimple = "simple" + recvBufferPoolAll = "all" numStatsBuckets = 10 warmupCallCount = 10 @@ -135,6 +156,7 @@ var ( allCompModes = []string{compModeOff, compModeGzip, compModeNop, compModeAll} allToggleModes = []string{toggleModeOff, toggleModeOn, toggleModeBoth} allNetworkModes = []string{networkModeNone, networkModeLocal, networkModeLAN, networkModeWAN, networkLongHaul} + allRecvBufferPools = []string{recvBufferPoolNil, recvBufferPoolSimple, recvBufferPoolAll} defaultReadLatency = []time.Duration{0, 40 * time.Millisecond} // if non-positive, no delay. defaultReadKbps = []int{0, 10240} // if non-positive, infinite defaultReadMTU = []int{0} // if non-positive, infinite @@ -185,9 +207,9 @@ func runModesFromWorkloads(workload string) runModes { type startFunc func(mode string, bf stats.Features) type stopFunc func(count uint64) type ucStopFunc func(req uint64, resp uint64) -type rpcCallFunc func(pos int) -type rpcSendFunc func(pos int) -type rpcRecvFunc func(pos int) +type rpcCallFunc func(cn, pos int) +type rpcSendFunc func(cn, pos int) +type rpcRecvFunc func(cn, pos int) type rpcCleanupFunc func() func unaryBenchmark(start startFunc, stop stopFunc, bf stats.Features, s *stats.Stats) { @@ -202,7 +224,7 @@ func streamBenchmark(start startFunc, stop stopFunc, bf stats.Features, s *stats runBenchmark(caller, start, stop, bf, s, workloadsStreaming) } -func unconstrainedStreamBenchmark(start startFunc, stop ucStopFunc, bf stats.Features, s *stats.Stats) { +func unconstrainedStreamBenchmark(start startFunc, stop ucStopFunc, bf stats.Features) { var sender rpcSendFunc var recver rpcRecvFunc var cleanup rpcCleanupFunc @@ -224,40 +246,46 @@ func unconstrainedStreamBenchmark(start startFunc, stop ucStopFunc, bf stats.Fea bmEnd := time.Now().Add(bf.BenchTime + warmuptime) var wg sync.WaitGroup - wg.Add(2 * bf.MaxConcurrentCalls) - for i := 0; i < bf.MaxConcurrentCalls; i++ { - go func(pos int) { - defer wg.Done() - for { - t := time.Now() - if t.After(bmEnd) { - return + wg.Add(2 * bf.Connections * bf.MaxConcurrentCalls) + maxSleep := int(bf.SleepBetweenRPCs) + for cn := 0; cn < bf.Connections; cn++ { + for pos := 0; pos < bf.MaxConcurrentCalls; pos++ { + go func(cn, pos int) { + defer wg.Done() + for { + if maxSleep > 0 { + time.Sleep(time.Duration(rand.Intn(maxSleep))) + } + t := time.Now() + if t.After(bmEnd) { + return + } + sender(cn, pos) + atomic.AddUint64(&req, 1) } - sender(pos) - atomic.AddUint64(&req, 1) - } - }(i) - go func(pos int) { - defer wg.Done() - for { - t := time.Now() - if t.After(bmEnd) { - return + }(cn, pos) + go func(cn, pos int) { + defer wg.Done() + for { + t := time.Now() + if t.After(bmEnd) { + return + } + recver(cn, pos) + atomic.AddUint64(&resp, 1) } - recver(pos) - atomic.AddUint64(&resp, 1) - } - }(i) + }(cn, pos) + } } wg.Wait() stop(req, resp) } -// makeClient returns a gRPC client for the grpc.testing.BenchmarkService +// makeClients returns a gRPC client (or multiple clients) for the grpc.testing.BenchmarkService // service. The client is configured using the different options in the passed // 'bf'. Also returns a cleanup function to close the client and release // resources. -func makeClient(bf stats.Features) (testpb.BenchmarkServiceClient, func()) { +func makeClients(bf stats.Features) ([]testgrpc.BenchmarkServiceClient, func()) { nw := &latency.Network{Kbps: bf.Kbps, Latency: bf.Latency, MTU: bf.MTU} opts := []grpc.DialOption{} sopts := []grpc.ServerOption{} @@ -300,8 +328,34 @@ func makeClient(bf stats.Features) (testpb.BenchmarkServiceClient, func()) { }), ) } + if bf.ClientReadBufferSize >= 0 { + opts = append(opts, grpc.WithReadBufferSize(bf.ClientReadBufferSize)) + } + if bf.ClientWriteBufferSize >= 0 { + opts = append(opts, grpc.WithWriteBufferSize(bf.ClientWriteBufferSize)) + } + if bf.ServerReadBufferSize >= 0 { + sopts = append(sopts, grpc.ReadBufferSize(bf.ServerReadBufferSize)) + } + if bf.SharedWriteBuffer { + opts = append(opts, grpc.WithSharedWriteBuffer(true)) + sopts = append(sopts, grpc.SharedWriteBuffer(true)) + } + if bf.ServerWriteBufferSize >= 0 { + sopts = append(sopts, grpc.WriteBufferSize(bf.ServerWriteBufferSize)) + } + switch bf.RecvBufferPool { + case recvBufferPoolNil: + // Do nothing. + case recvBufferPoolSimple: + opts = append(opts, grpc.WithRecvBufferPool(grpc.NewSharedBufferPool())) + sopts = append(sopts, grpc.RecvBufferPool(grpc.NewSharedBufferPool())) + default: + logger.Fatalf("Unknown shared recv buffer pool type: %v", bf.RecvBufferPool) + } + sopts = append(sopts, grpc.MaxConcurrentStreams(uint32(bf.MaxConcurrentCalls+1))) - opts = append(opts, grpc.WithInsecure()) + opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) var lis net.Listener if bf.UseBufConn { @@ -324,16 +378,24 @@ func makeClient(bf stats.Features) (testpb.BenchmarkServiceClient, func()) { } lis = nw.Listener(lis) stopper := bm.StartServer(bm.ServerInfo{Type: "protobuf", Listener: lis}, sopts...) - conn := bm.NewClientConn("" /* target not used */, opts...) - return testpb.NewBenchmarkServiceClient(conn), func() { - conn.Close() + conns := make([]*grpc.ClientConn, bf.Connections) + clients := make([]testgrpc.BenchmarkServiceClient, bf.Connections) + for cn := 0; cn < bf.Connections; cn++ { + conns[cn] = bm.NewClientConn("" /* target not used */, opts...) + clients[cn] = testgrpc.NewBenchmarkServiceClient(conns[cn]) + } + + return clients, func() { + for _, conn := range conns { + conn.Close() + } stopper() } } func makeFuncUnary(bf stats.Features) (rpcCallFunc, rpcCleanupFunc) { - tc, cleanup := makeClient(bf) - return func(int) { + clients, cleanup := makeClients(bf) + return func(cn, pos int) { reqSizeBytes := bf.ReqSizeBytes respSizeBytes := bf.RespSizeBytes if bf.ReqPayloadCurve != nil { @@ -342,23 +404,19 @@ func makeFuncUnary(bf stats.Features) (rpcCallFunc, rpcCleanupFunc) { if bf.RespPayloadCurve != nil { respSizeBytes = bf.RespPayloadCurve.ChooseRandom() } - unaryCaller(tc, reqSizeBytes, respSizeBytes) + unaryCaller(clients[cn], reqSizeBytes, respSizeBytes) }, cleanup } func makeFuncStream(bf stats.Features) (rpcCallFunc, rpcCleanupFunc) { - tc, cleanup := makeClient(bf) + streams, req, cleanup := setupStream(bf, false) - streams := make([]testpb.BenchmarkService_StreamingCallClient, bf.MaxConcurrentCalls) - for i := 0; i < bf.MaxConcurrentCalls; i++ { - stream, err := tc.StreamingCall(context.Background()) - if err != nil { - logger.Fatalf("%v.StreamingCall(_) = _, %v", tc, err) - } - streams[i] = stream + var preparedMsg [][]*grpc.PreparedMsg + if bf.EnablePreloader { + preparedMsg = prepareMessages(streams, req) } - return func(pos int) { + return func(cn, pos int) { reqSizeBytes := bf.ReqSizeBytes respSizeBytes := bf.RespSizeBytes if bf.ReqPayloadCurve != nil { @@ -367,49 +425,66 @@ func makeFuncStream(bf stats.Features) (rpcCallFunc, rpcCleanupFunc) { if bf.RespPayloadCurve != nil { respSizeBytes = bf.RespPayloadCurve.ChooseRandom() } - streamCaller(streams[pos], reqSizeBytes, respSizeBytes) + var req any + if bf.EnablePreloader { + req = preparedMsg[cn][pos] + } else { + pl := bm.NewPayload(testpb.PayloadType_COMPRESSABLE, reqSizeBytes) + req = &testpb.SimpleRequest{ + ResponseType: pl.Type, + ResponseSize: int32(respSizeBytes), + Payload: pl, + } + } + streamCaller(streams[cn][pos], req) }, cleanup } func makeFuncUnconstrainedStreamPreloaded(bf stats.Features) (rpcSendFunc, rpcRecvFunc, rpcCleanupFunc) { - streams, req, cleanup := setupUnconstrainedStream(bf) + streams, req, cleanup := setupStream(bf, true) - preparedMsg := make([]*grpc.PreparedMsg, len(streams)) - for i, stream := range streams { - preparedMsg[i] = &grpc.PreparedMsg{} - err := preparedMsg[i].Encode(stream, req) - if err != nil { - logger.Fatalf("%v.Encode(%v, %v) = %v", preparedMsg[i], req, stream, err) - } - } + preparedMsg := prepareMessages(streams, req) - return func(pos int) { - streams[pos].SendMsg(preparedMsg[pos]) - }, func(pos int) { - streams[pos].Recv() + return func(cn, pos int) { + streams[cn][pos].SendMsg(preparedMsg[cn][pos]) + }, func(cn, pos int) { + streams[cn][pos].Recv() }, cleanup } func makeFuncUnconstrainedStream(bf stats.Features) (rpcSendFunc, rpcRecvFunc, rpcCleanupFunc) { - streams, req, cleanup := setupUnconstrainedStream(bf) + streams, req, cleanup := setupStream(bf, true) - return func(pos int) { - streams[pos].Send(req) - }, func(pos int) { - streams[pos].Recv() + return func(cn, pos int) { + streams[cn][pos].Send(req) + }, func(cn, pos int) { + streams[cn][pos].Recv() }, cleanup } -func setupUnconstrainedStream(bf stats.Features) ([]testpb.BenchmarkService_StreamingCallClient, *testpb.SimpleRequest, rpcCleanupFunc) { - tc, cleanup := makeClient(bf) +func setupStream(bf stats.Features, unconstrained bool) ([][]testgrpc.BenchmarkService_StreamingCallClient, *testpb.SimpleRequest, rpcCleanupFunc) { + clients, cleanup := makeClients(bf) - streams := make([]testpb.BenchmarkService_StreamingCallClient, bf.MaxConcurrentCalls) - for i := 0; i < bf.MaxConcurrentCalls; i++ { - stream, err := tc.UnconstrainedStreamingCall(context.Background()) - if err != nil { - logger.Fatalf("%v.UnconstrainedStreamingCall(_) = _, %v", tc, err) + streams := make([][]testgrpc.BenchmarkService_StreamingCallClient, bf.Connections) + ctx := context.Background() + if unconstrained { + md := metadata.Pairs(benchmark.UnconstrainedStreamingHeader, "1", benchmark.UnconstrainedStreamingDelayHeader, bf.SleepBetweenRPCs.String()) + ctx = metadata.NewOutgoingContext(ctx, md) + } + if bf.EnablePreloader { + md := metadata.Pairs(benchmark.PreloadMsgSizeHeader, strconv.Itoa(bf.RespSizeBytes), benchmark.UnconstrainedStreamingDelayHeader, bf.SleepBetweenRPCs.String()) + ctx = metadata.NewOutgoingContext(ctx, md) + } + for cn := 0; cn < bf.Connections; cn++ { + tc := clients[cn] + streams[cn] = make([]testgrpc.BenchmarkService_StreamingCallClient, bf.MaxConcurrentCalls) + for pos := 0; pos < bf.MaxConcurrentCalls; pos++ { + stream, err := tc.StreamingCall(ctx) + if err != nil { + logger.Fatalf("%v.StreamingCall(_) = _, %v", tc, err) + } + streams[cn][pos] = stream } - streams[i] = stream } pl := bm.NewPayload(testpb.PayloadType_COMPRESSABLE, bf.ReqSizeBytes) @@ -422,47 +497,74 @@ func setupUnconstrainedStream(bf stats.Features) ([]testpb.BenchmarkService_Stre return streams, req, cleanup } +func prepareMessages(streams [][]testgrpc.BenchmarkService_StreamingCallClient, req *testpb.SimpleRequest) [][]*grpc.PreparedMsg { + preparedMsg := make([][]*grpc.PreparedMsg, len(streams)) + for cn, connStreams := range streams { + preparedMsg[cn] = make([]*grpc.PreparedMsg, len(connStreams)) + for pos, stream := range connStreams { + preparedMsg[cn][pos] = &grpc.PreparedMsg{} + if err := preparedMsg[cn][pos].Encode(stream, req); err != nil { + logger.Fatalf("%v.Encode(%v, %v) = %v", preparedMsg[cn][pos], req, stream, err) + } + } + } + return preparedMsg +} + // Makes a UnaryCall gRPC request using the given BenchmarkServiceClient and // request and response sizes. -func unaryCaller(client testpb.BenchmarkServiceClient, reqSize, respSize int) { +func unaryCaller(client testgrpc.BenchmarkServiceClient, reqSize, respSize int) { if err := bm.DoUnaryCall(client, reqSize, respSize); err != nil { logger.Fatalf("DoUnaryCall failed: %v", err) } } -func streamCaller(stream testpb.BenchmarkService_StreamingCallClient, reqSize, respSize int) { - if err := bm.DoStreamingRoundTrip(stream, reqSize, respSize); err != nil { +func streamCaller(stream testgrpc.BenchmarkService_StreamingCallClient, req any) { + if err := bm.DoStreamingRoundTripPreloaded(stream, req); err != nil { logger.Fatalf("DoStreamingRoundTrip failed: %v", err) } } func runBenchmark(caller rpcCallFunc, start startFunc, stop stopFunc, bf stats.Features, s *stats.Stats, mode string) { - // Warm up connection. - for i := 0; i < warmupCallCount; i++ { - caller(0) + // if SleepBetweenRPCs > 0 we skip the warmup because otherwise + // we are going to send a set of simultaneous requests on every connection, + // which is something we are trying to avoid when using SleepBetweenRPCs. + if bf.SleepBetweenRPCs == 0 { + // Warm up connections. + for i := 0; i < warmupCallCount; i++ { + for cn := 0; cn < bf.Connections; cn++ { + caller(cn, 0) + } + } } // Run benchmark. start(mode, bf) var wg sync.WaitGroup - wg.Add(bf.MaxConcurrentCalls) + wg.Add(bf.Connections * bf.MaxConcurrentCalls) bmEnd := time.Now().Add(bf.BenchTime) + maxSleep := int(bf.SleepBetweenRPCs) var count uint64 - for i := 0; i < bf.MaxConcurrentCalls; i++ { - go func(pos int) { - defer wg.Done() - for { - t := time.Now() - if t.After(bmEnd) { - return + for cn := 0; cn < bf.Connections; cn++ { + for pos := 0; pos < bf.MaxConcurrentCalls; pos++ { + go func(cn, pos int) { + defer wg.Done() + for { + if maxSleep > 0 { + time.Sleep(time.Duration(rand.Intn(maxSleep))) + } + t := time.Now() + if t.After(bmEnd) { + return + } + start := time.Now() + caller(cn, pos) + elapse := time.Since(start) + atomic.AddUint64(&count, 1) + s.AddDuration(elapse) } - start := time.Now() - caller(pos) - elapse := time.Since(start) - atomic.AddUint64(&count, 1) - s.AddDuration(elapse) - } - }(i) + }(cn, pos) + } } wg.Wait() stop(count) @@ -480,6 +582,7 @@ type benchOpts struct { benchmarkResultFile string useBufconn bool enableKeepalive bool + connections int features *featureOpts } @@ -488,18 +591,25 @@ type benchOpts struct { // features through command line flags. We generate all possible combinations // for the provided values and run the benchmarks for each combination. type featureOpts struct { - enableTrace []bool - readLatencies []time.Duration - readKbps []int - readMTU []int - maxConcurrentCalls []int - reqSizeBytes []int - respSizeBytes []int - reqPayloadCurves []*stats.PayloadCurve - respPayloadCurves []*stats.PayloadCurve - compModes []string - enableChannelz []bool - enablePreloader []bool + enableTrace []bool + readLatencies []time.Duration + readKbps []int + readMTU []int + maxConcurrentCalls []int + reqSizeBytes []int + respSizeBytes []int + reqPayloadCurves []*stats.PayloadCurve + respPayloadCurves []*stats.PayloadCurve + compModes []string + enableChannelz []bool + enablePreloader []bool + clientReadBufferSize []int + clientWriteBufferSize []int + serverReadBufferSize []int + serverWriteBufferSize []int + sleepBetweenRPCs []time.Duration + recvBufferPools []string + sharedWriteBuffer []bool } // makeFeaturesNum returns a slice of ints of size 'maxFeatureIndex' where each @@ -536,6 +646,20 @@ func makeFeaturesNum(b *benchOpts) []int { featuresNum[i] = len(b.features.enableChannelz) case stats.EnablePreloaderIndex: featuresNum[i] = len(b.features.enablePreloader) + case stats.ClientReadBufferSize: + featuresNum[i] = len(b.features.clientReadBufferSize) + case stats.ClientWriteBufferSize: + featuresNum[i] = len(b.features.clientWriteBufferSize) + case stats.ServerReadBufferSize: + featuresNum[i] = len(b.features.serverReadBufferSize) + case stats.ServerWriteBufferSize: + featuresNum[i] = len(b.features.serverWriteBufferSize) + case stats.SleepBetweenRPCs: + featuresNum[i] = len(b.features.sleepBetweenRPCs) + case stats.RecvBufferPool: + featuresNum[i] = len(b.features.recvBufferPools) + case stats.SharedWriteBuffer: + featuresNum[i] = len(b.features.sharedWriteBuffer) default: log.Fatalf("Unknown feature index %v in generateFeatures. maxFeatureIndex is %v", i, stats.MaxFeatureIndex) } @@ -589,15 +713,23 @@ func (b *benchOpts) generateFeatures(featuresNum []int) []stats.Features { UseBufConn: b.useBufconn, EnableKeepalive: b.enableKeepalive, BenchTime: b.benchTime, + Connections: b.connections, // These features can potentially change for each iteration. - EnableTrace: b.features.enableTrace[curPos[stats.EnableTraceIndex]], - Latency: b.features.readLatencies[curPos[stats.ReadLatenciesIndex]], - Kbps: b.features.readKbps[curPos[stats.ReadKbpsIndex]], - MTU: b.features.readMTU[curPos[stats.ReadMTUIndex]], - MaxConcurrentCalls: b.features.maxConcurrentCalls[curPos[stats.MaxConcurrentCallsIndex]], - ModeCompressor: b.features.compModes[curPos[stats.CompModesIndex]], - EnableChannelz: b.features.enableChannelz[curPos[stats.EnableChannelzIndex]], - EnablePreloader: b.features.enablePreloader[curPos[stats.EnablePreloaderIndex]], + EnableTrace: b.features.enableTrace[curPos[stats.EnableTraceIndex]], + Latency: b.features.readLatencies[curPos[stats.ReadLatenciesIndex]], + Kbps: b.features.readKbps[curPos[stats.ReadKbpsIndex]], + MTU: b.features.readMTU[curPos[stats.ReadMTUIndex]], + MaxConcurrentCalls: b.features.maxConcurrentCalls[curPos[stats.MaxConcurrentCallsIndex]], + ModeCompressor: b.features.compModes[curPos[stats.CompModesIndex]], + EnableChannelz: b.features.enableChannelz[curPos[stats.EnableChannelzIndex]], + EnablePreloader: b.features.enablePreloader[curPos[stats.EnablePreloaderIndex]], + ClientReadBufferSize: b.features.clientReadBufferSize[curPos[stats.ClientReadBufferSize]], + ClientWriteBufferSize: b.features.clientWriteBufferSize[curPos[stats.ClientWriteBufferSize]], + ServerReadBufferSize: b.features.serverReadBufferSize[curPos[stats.ServerReadBufferSize]], + ServerWriteBufferSize: b.features.serverWriteBufferSize[curPos[stats.ServerWriteBufferSize]], + SleepBetweenRPCs: b.features.sleepBetweenRPCs[curPos[stats.SleepBetweenRPCs]], + RecvBufferPool: b.features.recvBufferPools[curPos[stats.RecvBufferPool]], + SharedWriteBuffer: b.features.sharedWriteBuffer[curPos[stats.SharedWriteBuffer]], } if len(b.features.reqPayloadCurves) == 0 { f.ReqSizeBytes = b.features.reqSizeBytes[curPos[stats.ReqSizeBytesIndex]] @@ -653,17 +785,25 @@ func processFlags() *benchOpts { benchmarkResultFile: *benchmarkResultFile, useBufconn: *useBufconn, enableKeepalive: *enableKeepalive, + connections: *connections, features: &featureOpts{ - enableTrace: setToggleMode(*traceMode), - readLatencies: append([]time.Duration(nil), *readLatency...), - readKbps: append([]int(nil), *readKbps...), - readMTU: append([]int(nil), *readMTU...), - maxConcurrentCalls: append([]int(nil), *maxConcurrentCalls...), - reqSizeBytes: append([]int(nil), *readReqSizeBytes...), - respSizeBytes: append([]int(nil), *readRespSizeBytes...), - compModes: setCompressorMode(*compressorMode), - enableChannelz: setToggleMode(*channelzOn), - enablePreloader: setToggleMode(*preloaderMode), + enableTrace: setToggleMode(*traceMode), + readLatencies: append([]time.Duration(nil), *readLatency...), + readKbps: append([]int(nil), *readKbps...), + readMTU: append([]int(nil), *readMTU...), + maxConcurrentCalls: append([]int(nil), *maxConcurrentCalls...), + reqSizeBytes: append([]int(nil), *readReqSizeBytes...), + respSizeBytes: append([]int(nil), *readRespSizeBytes...), + compModes: setCompressorMode(*compressorMode), + enableChannelz: setToggleMode(*channelzOn), + enablePreloader: setToggleMode(*preloaderMode), + clientReadBufferSize: append([]int(nil), *clientReadBufferSize...), + clientWriteBufferSize: append([]int(nil), *clientWriteBufferSize...), + serverReadBufferSize: append([]int(nil), *serverReadBufferSize...), + serverWriteBufferSize: append([]int(nil), *serverWriteBufferSize...), + sleepBetweenRPCs: append([]time.Duration(nil), *sleepBetweenRPCs...), + recvBufferPools: setRecvBufferPool(*recvBufferPool), + sharedWriteBuffer: setToggleMode(*sharedWriteBuffer), }, } @@ -675,6 +815,9 @@ func processFlags() *benchOpts { if len(opts.features.reqSizeBytes) != 0 { log.Fatalf("you may not specify -reqPayloadCurveFiles and -reqSizeBytes at the same time") } + if len(opts.features.enablePreloader) != 0 { + log.Fatalf("you may not specify -reqPayloadCurveFiles and -preloader at the same time") + } for _, file := range *reqPayloadCurveFiles { pc, err := stats.NewPayloadCurve(file) if err != nil { @@ -692,6 +835,9 @@ func processFlags() *benchOpts { if len(opts.features.respSizeBytes) != 0 { log.Fatalf("you may not specify -respPayloadCurveFiles and -respSizeBytes at the same time") } + if len(opts.features.enablePreloader) != 0 { + log.Fatalf("you may not specify -respPayloadCurveFiles and -preloader at the same time") + } for _, file := range *respPayloadCurveFiles { pc, err := stats.NewPayloadCurve(file) if err != nil { @@ -739,6 +885,19 @@ func setCompressorMode(val string) []string { } } +func setRecvBufferPool(val string) []string { + switch val { + case recvBufferPoolNil, recvBufferPoolSimple: + return []string{val} + case recvBufferPoolAll: + return []string{recvBufferPoolNil, recvBufferPoolSimple} + default: + // This should never happen because a wrong value passed to this flag would + // be caught during flag.Parse(). + return []string{} + } +} + func main() { opts := processFlags() before(opts) @@ -765,7 +924,7 @@ func main() { streamBenchmark(start, stop, bf, s) } if opts.rModes.unconstrained { - unconstrainedStreamBenchmark(start, ucStop, bf, s) + unconstrainedStreamBenchmark(start, ucStop, bf) } } after(opts, s.GetResults()) @@ -826,7 +985,7 @@ func (nopCompressor) Do(w io.Writer, p []byte) error { return err } if n != len(p) { - return fmt.Errorf("nopCompressor.Write: wrote %v bytes; want %v", n, len(p)) + return fmt.Errorf("nopCompressor.Write: wrote %d bytes; want %d", n, len(p)) } return nil } @@ -836,5 +995,5 @@ func (nopCompressor) Type() string { return compModeNop } // nopDecompressor is a decompressor that just copies data. type nopDecompressor struct{} -func (nopDecompressor) Do(r io.Reader) ([]byte, error) { return ioutil.ReadAll(r) } +func (nopDecompressor) Do(r io.Reader) ([]byte, error) { return io.ReadAll(r) } func (nopDecompressor) Type() string { return compModeNop } diff --git a/benchmark/benchmark.go b/benchmark/benchmark.go index d2783a87e00a..df2d2126439b 100644 --- a/benchmark/benchmark.go +++ b/benchmark/benchmark.go @@ -26,13 +26,19 @@ import ( "fmt" "io" "log" + "math/rand" "net" + "strconv" + "time" "google.golang.org/grpc" - testpb "google.golang.org/grpc/benchmark/grpc_testing" "google.golang.org/grpc/codes" "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" ) var logger = grpclog.Component("benchmark") @@ -45,8 +51,6 @@ func setPayload(p *testpb.Payload, t testpb.PayloadType, size int) { body := make([]byte, size) switch t { case testpb.PayloadType_COMPRESSABLE: - case testpb.PayloadType_UNCOMPRESSABLE: - logger.Fatalf("PayloadType UNCOMPRESSABLE is not supported") default: logger.Fatalf("Unsupported payload type: %d", t) } @@ -61,9 +65,9 @@ func NewPayload(t testpb.PayloadType, size int) *testpb.Payload { return p } -type testServer struct{} - -var _ testpb.UnstableBenchmarkServiceService = (*testServer)(nil) +type testServer struct { + testgrpc.UnimplementedBenchmarkServiceServer +} func (s *testServer) UnaryCall(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { return &testpb.SimpleResponse{ @@ -71,10 +75,44 @@ func (s *testServer) UnaryCall(ctx context.Context, in *testpb.SimpleRequest) (* }, nil } -func (s *testServer) StreamingCall(stream testpb.BenchmarkService_StreamingCallServer) error { +// UnconstrainedStreamingHeader indicates to the StreamingCall handler that its +// behavior should be unconstrained (constant send/receive in parallel) instead +// of ping-pong. +const UnconstrainedStreamingHeader = "unconstrained-streaming" + +// UnconstrainedStreamingDelayHeader is used to pass the maximum amount of time +// the server should sleep between consecutive RPC responses. +const UnconstrainedStreamingDelayHeader = "unconstrained-streaming-delay" + +// PreloadMsgSizeHeader indicates that the client is going to ask for +// a fixed response size and passes this size to the server. +// The server is expected to preload the response on startup. +const PreloadMsgSizeHeader = "preload-msg-size" + +func (s *testServer) StreamingCall(stream testgrpc.BenchmarkService_StreamingCallServer) error { + preloadMsgSize := 0 + if md, ok := metadata.FromIncomingContext(stream.Context()); ok && len(md[PreloadMsgSizeHeader]) != 0 { + val := md[PreloadMsgSizeHeader][0] + var err error + preloadMsgSize, err = strconv.Atoi(val) + if err != nil { + return fmt.Errorf("%q header value is not an integer: %s", PreloadMsgSizeHeader, err) + } + } + + if md, ok := metadata.FromIncomingContext(stream.Context()); ok && len(md[UnconstrainedStreamingHeader]) != 0 { + return s.UnconstrainedStreamingCall(stream, preloadMsgSize) + } response := &testpb.SimpleResponse{ Payload: new(testpb.Payload), } + preloadedResponse := &grpc.PreparedMsg{} + if preloadMsgSize > 0 { + setPayload(response.Payload, testpb.PayloadType_COMPRESSABLE, preloadMsgSize) + if err := preloadedResponse.Encode(stream, response); err != nil { + return err + } + } in := new(testpb.SimpleRequest) for { // use ServerStream directly to reuse the same testpb.SimpleRequest object @@ -86,14 +124,29 @@ func (s *testServer) StreamingCall(stream testpb.BenchmarkService_StreamingCallS if err != nil { return err } - setPayload(response.Payload, in.ResponseType, int(in.ResponseSize)) - if err := stream.Send(response); err != nil { + if preloadMsgSize > 0 { + err = stream.SendMsg(preloadedResponse) + } else { + setPayload(response.Payload, in.ResponseType, int(in.ResponseSize)) + err = stream.Send(response) + } + if err != nil { return err } } } -func (s *testServer) UnconstrainedStreamingCall(stream testpb.BenchmarkService_UnconstrainedStreamingCallServer) error { +func (s *testServer) UnconstrainedStreamingCall(stream testgrpc.BenchmarkService_StreamingCallServer, preloadMsgSize int) error { + maxSleep := 0 + if md, ok := metadata.FromIncomingContext(stream.Context()); ok && len(md[UnconstrainedStreamingDelayHeader]) != 0 { + val := md[UnconstrainedStreamingDelayHeader][0] + d, err := time.ParseDuration(val) + if err != nil { + return fmt.Errorf("can't parse %q header: %s", UnconstrainedStreamingDelayHeader, err) + } + maxSleep = int(d) + } + in := new(testpb.SimpleRequest) // Receive a message to learn response type and size. err := stream.RecvMsg(in) @@ -110,6 +163,13 @@ func (s *testServer) UnconstrainedStreamingCall(stream testpb.BenchmarkService_U } setPayload(response.Payload, in.ResponseType, int(in.ResponseSize)) + preloadedResponse := &grpc.PreparedMsg{} + if preloadMsgSize > 0 { + if err := preloadedResponse.Encode(stream, response); err != nil { + return err + } + } + go func() { for { // Using RecvMsg rather than Recv to prevent reallocation of SimpleRequest. @@ -126,9 +186,17 @@ func (s *testServer) UnconstrainedStreamingCall(stream testpb.BenchmarkService_U go func() { for { - err := stream.Send(response) + if maxSleep > 0 { + time.Sleep(time.Duration(rand.Intn(maxSleep))) + } + var err error + if preloadMsgSize > 0 { + err = stream.SendMsg(preloadedResponse) + } else { + err = stream.Send(response) + } switch status.Code(err) { - case codes.Unavailable: + case codes.Unavailable, codes.Canceled: return case codes.OK: default: @@ -144,35 +212,17 @@ func (s *testServer) UnconstrainedStreamingCall(stream testpb.BenchmarkService_U // byteBufServer is a gRPC server that sends and receives byte buffer. // The purpose is to benchmark the gRPC performance without protobuf serialization/deserialization overhead. type byteBufServer struct { + testgrpc.UnimplementedBenchmarkServiceServer respSize int32 } -var _ testpb.UnstableBenchmarkServiceService = (*byteBufServer)(nil) - // UnaryCall is an empty function and is not used for benchmark. // If bytebuf UnaryCall benchmark is needed later, the function body needs to be updated. func (s *byteBufServer) UnaryCall(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { return &testpb.SimpleResponse{}, nil } -func (s *byteBufServer) StreamingCall(stream testpb.BenchmarkService_StreamingCallServer) error { - for { - var in []byte - err := stream.(grpc.ServerStream).RecvMsg(&in) - if err == io.EOF { - return nil - } - if err != nil { - return err - } - out := make([]byte, s.respSize) - if err := stream.(grpc.ServerStream).SendMsg(&out); err != nil { - return err - } - } -} - -func (s *byteBufServer) UnconstrainedStreamingCall(stream testpb.BenchmarkService_UnconstrainedStreamingCallServer) error { +func (s *byteBufServer) StreamingCall(stream testgrpc.BenchmarkService_StreamingCallServer) error { for { var in []byte err := stream.(grpc.ServerStream).RecvMsg(&in) @@ -198,7 +248,7 @@ type ServerInfo struct { // Metadata is an optional configuration. // For "protobuf", it's ignored. // For "bytebuf", it should be an int representing response size. - Metadata interface{} + Metadata any // Listener is the network listener for the server to use Listener net.Listener @@ -207,18 +257,16 @@ type ServerInfo struct { // StartServer starts a gRPC server serving a benchmark service according to info. // It returns a function to stop the server. func StartServer(info ServerInfo, opts ...grpc.ServerOption) func() { - opts = append(opts, grpc.WriteBufferSize(128*1024)) - opts = append(opts, grpc.ReadBufferSize(128*1024)) s := grpc.NewServer(opts...) switch info.Type { case "protobuf": - testpb.RegisterBenchmarkServiceService(s, testpb.NewBenchmarkServiceService(&testServer{})) + testgrpc.RegisterBenchmarkServiceServer(s, &testServer{}) case "bytebuf": respSize, ok := info.Metadata.(int32) if !ok { logger.Fatalf("failed to StartServer, invalid metadata: %v, for Type: %v", info.Metadata, info.Type) } - testpb.RegisterBenchmarkServiceService(s, testpb.NewBenchmarkServiceService(&byteBufServer{respSize: respSize})) + testgrpc.RegisterBenchmarkServiceServer(s, &byteBufServer{respSize: respSize}) default: logger.Fatalf("failed to StartServer, unknown Type: %v", info.Type) } @@ -229,7 +277,7 @@ func StartServer(info ServerInfo, opts ...grpc.ServerOption) func() { } // DoUnaryCall performs an unary RPC with given stub and request and response sizes. -func DoUnaryCall(tc testpb.BenchmarkServiceClient, reqSize, respSize int) error { +func DoUnaryCall(tc testgrpc.BenchmarkServiceClient, reqSize, respSize int) error { pl := NewPayload(testpb.PayloadType_COMPRESSABLE, reqSize) req := &testpb.SimpleRequest{ ResponseType: pl.Type, @@ -243,14 +291,20 @@ func DoUnaryCall(tc testpb.BenchmarkServiceClient, reqSize, respSize int) error } // DoStreamingRoundTrip performs a round trip for a single streaming rpc. -func DoStreamingRoundTrip(stream testpb.BenchmarkService_StreamingCallClient, reqSize, respSize int) error { +func DoStreamingRoundTrip(stream testgrpc.BenchmarkService_StreamingCallClient, reqSize, respSize int) error { pl := NewPayload(testpb.PayloadType_COMPRESSABLE, reqSize) req := &testpb.SimpleRequest{ ResponseType: pl.Type, ResponseSize: int32(respSize), Payload: pl, } - if err := stream.Send(req); err != nil { + return DoStreamingRoundTripPreloaded(stream, req) +} + +// DoStreamingRoundTripPreloaded performs a round trip for a single streaming rpc with preloaded payload. +func DoStreamingRoundTripPreloaded(stream testgrpc.BenchmarkService_StreamingCallClient, req any) error { + // req could be either *testpb.SimpleRequest or *grpc.PreparedMsg + if err := stream.SendMsg(req); err != nil { return fmt.Errorf("/BenchmarkService/StreamingCall.Send(_) = %v, want ", err) } if _, err := stream.Recv(); err != nil { @@ -264,7 +318,7 @@ func DoStreamingRoundTrip(stream testpb.BenchmarkService_StreamingCallClient, re } // DoByteBufStreamingRoundTrip performs a round trip for a single streaming rpc, using a custom codec for byte buffer. -func DoByteBufStreamingRoundTrip(stream testpb.BenchmarkService_StreamingCallClient, reqSize, respSize int) error { +func DoByteBufStreamingRoundTrip(stream testgrpc.BenchmarkService_StreamingCallClient, reqSize, respSize int) error { out := make([]byte, reqSize) if err := stream.(grpc.ClientStream).SendMsg(&out); err != nil { return fmt.Errorf("/BenchmarkService/StreamingCall.(ClientStream).SendMsg(_) = %v, want ", err) @@ -287,11 +341,9 @@ func NewClientConn(addr string, opts ...grpc.DialOption) *grpc.ClientConn { // NewClientConnWithContext creates a gRPC client connection to addr using ctx. func NewClientConnWithContext(ctx context.Context, addr string, opts ...grpc.DialOption) *grpc.ClientConn { - opts = append(opts, grpc.WithWriteBufferSize(128*1024)) - opts = append(opts, grpc.WithReadBufferSize(128*1024)) conn, err := grpc.DialContext(ctx, addr, opts...) if err != nil { - logger.Fatalf("NewClientConn(%q) failed to create a ClientConn %v", addr, err) + logger.Fatalf("NewClientConn(%q) failed to create a ClientConn: %v", addr, err) } return conn } diff --git a/benchmark/benchresult/main.go b/benchmark/benchresult/main.go index 587a0f6bda32..5bd9ce6ff891 100644 --- a/benchmark/benchresult/main.go +++ b/benchmark/benchresult/main.go @@ -18,12 +18,14 @@ /* To format the benchmark result: - go run benchmark/benchresult/main.go resultfile + + go run benchmark/benchresult/main.go resultfile To see the performance change based on a old result: - go run benchmark/benchresult/main.go resultfile_old resultfile -It will print the comparison result of intersection benchmarks between two files. + go run benchmark/benchresult/main.go resultfile_old resultfile + +It will print the comparison result of intersection benchmarks between two files. */ package main diff --git a/benchmark/client/main.go b/benchmark/client/main.go index f750c5c30721..d742e1a29f48 100644 --- a/benchmark/client/main.go +++ b/benchmark/client/main.go @@ -50,10 +50,13 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/benchmark" - testpb "google.golang.org/grpc/benchmark/grpc_testing" "google.golang.org/grpc/benchmark/stats" + "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/grpclog" "google.golang.org/grpc/internal/syscall" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" ) var ( @@ -83,7 +86,7 @@ var ( func main() { flag.Parse() if *testName == "" { - logger.Fatalf("test_name not set") + logger.Fatal("-test_name not set") } req := &testpb.SimpleRequest{ ResponseType: testpb.PayloadType_COMPRESSABLE, @@ -133,7 +136,12 @@ func main() { func buildConnections(ctx context.Context) []*grpc.ClientConn { ccs := make([]*grpc.ClientConn, *numConn) for i := range ccs { - ccs[i] = benchmark.NewClientConnWithContext(ctx, "localhost:"+*port, grpc.WithInsecure(), grpc.WithBlock()) + ccs[i] = benchmark.NewClientConnWithContext(ctx, "localhost:"+*port, + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithBlock(), + grpc.WithWriteBufferSize(128*1024), + grpc.WithReadBufferSize(128*1024), + ) } return ccs } @@ -164,7 +172,7 @@ func runWithConn(cc *grpc.ClientConn, req *testpb.SimpleRequest, warmDeadline, e } func makeCaller(cc *grpc.ClientConn, req *testpb.SimpleRequest) func() { - client := testpb.NewBenchmarkServiceClient(cc) + client := testgrpc.NewBenchmarkServiceClient(cc) if *rpcType == "unary" { return func() { if _, err := client.UnaryCall(context.Background(), req); err != nil { diff --git a/benchmark/grpc_testing/control.pb.go b/benchmark/grpc_testing/control.pb.go deleted file mode 100644 index 7a109d8e9cb8..000000000000 --- a/benchmark/grpc_testing/control.pb.go +++ /dev/null @@ -1,1360 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. -// source: benchmark/grpc_testing/control.proto - -package grpc_testing - -import ( - fmt "fmt" - proto "github.com/golang/protobuf/proto" - math "math" -) - -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package - -type ClientType int32 - -const ( - ClientType_SYNC_CLIENT ClientType = 0 - ClientType_ASYNC_CLIENT ClientType = 1 -) - -var ClientType_name = map[int32]string{ - 0: "SYNC_CLIENT", - 1: "ASYNC_CLIENT", -} - -var ClientType_value = map[string]int32{ - "SYNC_CLIENT": 0, - "ASYNC_CLIENT": 1, -} - -func (x ClientType) String() string { - return proto.EnumName(ClientType_name, int32(x)) -} - -func (ClientType) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_6f4e2bf9f888bddb, []int{0} -} - -type ServerType int32 - -const ( - ServerType_SYNC_SERVER ServerType = 0 - ServerType_ASYNC_SERVER ServerType = 1 - ServerType_ASYNC_GENERIC_SERVER ServerType = 2 -) - -var ServerType_name = map[int32]string{ - 0: "SYNC_SERVER", - 1: "ASYNC_SERVER", - 2: "ASYNC_GENERIC_SERVER", -} - -var ServerType_value = map[string]int32{ - "SYNC_SERVER": 0, - "ASYNC_SERVER": 1, - "ASYNC_GENERIC_SERVER": 2, -} - -func (x ServerType) String() string { - return proto.EnumName(ServerType_name, int32(x)) -} - -func (ServerType) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_6f4e2bf9f888bddb, []int{1} -} - -type RpcType int32 - -const ( - RpcType_UNARY RpcType = 0 - RpcType_STREAMING RpcType = 1 -) - -var RpcType_name = map[int32]string{ - 0: "UNARY", - 1: "STREAMING", -} - -var RpcType_value = map[string]int32{ - "UNARY": 0, - "STREAMING": 1, -} - -func (x RpcType) String() string { - return proto.EnumName(RpcType_name, int32(x)) -} - -func (RpcType) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_6f4e2bf9f888bddb, []int{2} -} - -// Parameters of poisson process distribution, which is a good representation -// of activity coming in from independent identical stationary sources. -type PoissonParams struct { - // The rate of arrivals (a.k.a. lambda parameter of the exp distribution). - OfferedLoad float64 `protobuf:"fixed64,1,opt,name=offered_load,json=offeredLoad,proto3" json:"offered_load,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *PoissonParams) Reset() { *m = PoissonParams{} } -func (m *PoissonParams) String() string { return proto.CompactTextString(m) } -func (*PoissonParams) ProtoMessage() {} -func (*PoissonParams) Descriptor() ([]byte, []int) { - return fileDescriptor_6f4e2bf9f888bddb, []int{0} -} - -func (m *PoissonParams) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_PoissonParams.Unmarshal(m, b) -} -func (m *PoissonParams) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_PoissonParams.Marshal(b, m, deterministic) -} -func (m *PoissonParams) XXX_Merge(src proto.Message) { - xxx_messageInfo_PoissonParams.Merge(m, src) -} -func (m *PoissonParams) XXX_Size() int { - return xxx_messageInfo_PoissonParams.Size(m) -} -func (m *PoissonParams) XXX_DiscardUnknown() { - xxx_messageInfo_PoissonParams.DiscardUnknown(m) -} - -var xxx_messageInfo_PoissonParams proto.InternalMessageInfo - -func (m *PoissonParams) GetOfferedLoad() float64 { - if m != nil { - return m.OfferedLoad - } - return 0 -} - -type UniformParams struct { - InterarrivalLo float64 `protobuf:"fixed64,1,opt,name=interarrival_lo,json=interarrivalLo,proto3" json:"interarrival_lo,omitempty"` - InterarrivalHi float64 `protobuf:"fixed64,2,opt,name=interarrival_hi,json=interarrivalHi,proto3" json:"interarrival_hi,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *UniformParams) Reset() { *m = UniformParams{} } -func (m *UniformParams) String() string { return proto.CompactTextString(m) } -func (*UniformParams) ProtoMessage() {} -func (*UniformParams) Descriptor() ([]byte, []int) { - return fileDescriptor_6f4e2bf9f888bddb, []int{1} -} - -func (m *UniformParams) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_UniformParams.Unmarshal(m, b) -} -func (m *UniformParams) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_UniformParams.Marshal(b, m, deterministic) -} -func (m *UniformParams) XXX_Merge(src proto.Message) { - xxx_messageInfo_UniformParams.Merge(m, src) -} -func (m *UniformParams) XXX_Size() int { - return xxx_messageInfo_UniformParams.Size(m) -} -func (m *UniformParams) XXX_DiscardUnknown() { - xxx_messageInfo_UniformParams.DiscardUnknown(m) -} - -var xxx_messageInfo_UniformParams proto.InternalMessageInfo - -func (m *UniformParams) GetInterarrivalLo() float64 { - if m != nil { - return m.InterarrivalLo - } - return 0 -} - -func (m *UniformParams) GetInterarrivalHi() float64 { - if m != nil { - return m.InterarrivalHi - } - return 0 -} - -type DeterministicParams struct { - OfferedLoad float64 `protobuf:"fixed64,1,opt,name=offered_load,json=offeredLoad,proto3" json:"offered_load,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *DeterministicParams) Reset() { *m = DeterministicParams{} } -func (m *DeterministicParams) String() string { return proto.CompactTextString(m) } -func (*DeterministicParams) ProtoMessage() {} -func (*DeterministicParams) Descriptor() ([]byte, []int) { - return fileDescriptor_6f4e2bf9f888bddb, []int{2} -} - -func (m *DeterministicParams) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_DeterministicParams.Unmarshal(m, b) -} -func (m *DeterministicParams) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_DeterministicParams.Marshal(b, m, deterministic) -} -func (m *DeterministicParams) XXX_Merge(src proto.Message) { - xxx_messageInfo_DeterministicParams.Merge(m, src) -} -func (m *DeterministicParams) XXX_Size() int { - return xxx_messageInfo_DeterministicParams.Size(m) -} -func (m *DeterministicParams) XXX_DiscardUnknown() { - xxx_messageInfo_DeterministicParams.DiscardUnknown(m) -} - -var xxx_messageInfo_DeterministicParams proto.InternalMessageInfo - -func (m *DeterministicParams) GetOfferedLoad() float64 { - if m != nil { - return m.OfferedLoad - } - return 0 -} - -type ParetoParams struct { - InterarrivalBase float64 `protobuf:"fixed64,1,opt,name=interarrival_base,json=interarrivalBase,proto3" json:"interarrival_base,omitempty"` - Alpha float64 `protobuf:"fixed64,2,opt,name=alpha,proto3" json:"alpha,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *ParetoParams) Reset() { *m = ParetoParams{} } -func (m *ParetoParams) String() string { return proto.CompactTextString(m) } -func (*ParetoParams) ProtoMessage() {} -func (*ParetoParams) Descriptor() ([]byte, []int) { - return fileDescriptor_6f4e2bf9f888bddb, []int{3} -} - -func (m *ParetoParams) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ParetoParams.Unmarshal(m, b) -} -func (m *ParetoParams) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ParetoParams.Marshal(b, m, deterministic) -} -func (m *ParetoParams) XXX_Merge(src proto.Message) { - xxx_messageInfo_ParetoParams.Merge(m, src) -} -func (m *ParetoParams) XXX_Size() int { - return xxx_messageInfo_ParetoParams.Size(m) -} -func (m *ParetoParams) XXX_DiscardUnknown() { - xxx_messageInfo_ParetoParams.DiscardUnknown(m) -} - -var xxx_messageInfo_ParetoParams proto.InternalMessageInfo - -func (m *ParetoParams) GetInterarrivalBase() float64 { - if m != nil { - return m.InterarrivalBase - } - return 0 -} - -func (m *ParetoParams) GetAlpha() float64 { - if m != nil { - return m.Alpha - } - return 0 -} - -// Once an RPC finishes, immediately start a new one. -// No configuration parameters needed. -type ClosedLoopParams struct { - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *ClosedLoopParams) Reset() { *m = ClosedLoopParams{} } -func (m *ClosedLoopParams) String() string { return proto.CompactTextString(m) } -func (*ClosedLoopParams) ProtoMessage() {} -func (*ClosedLoopParams) Descriptor() ([]byte, []int) { - return fileDescriptor_6f4e2bf9f888bddb, []int{4} -} - -func (m *ClosedLoopParams) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ClosedLoopParams.Unmarshal(m, b) -} -func (m *ClosedLoopParams) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ClosedLoopParams.Marshal(b, m, deterministic) -} -func (m *ClosedLoopParams) XXX_Merge(src proto.Message) { - xxx_messageInfo_ClosedLoopParams.Merge(m, src) -} -func (m *ClosedLoopParams) XXX_Size() int { - return xxx_messageInfo_ClosedLoopParams.Size(m) -} -func (m *ClosedLoopParams) XXX_DiscardUnknown() { - xxx_messageInfo_ClosedLoopParams.DiscardUnknown(m) -} - -var xxx_messageInfo_ClosedLoopParams proto.InternalMessageInfo - -type LoadParams struct { - // Types that are valid to be assigned to Load: - // *LoadParams_ClosedLoop - // *LoadParams_Poisson - // *LoadParams_Uniform - // *LoadParams_Determ - // *LoadParams_Pareto - Load isLoadParams_Load `protobuf_oneof:"load"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *LoadParams) Reset() { *m = LoadParams{} } -func (m *LoadParams) String() string { return proto.CompactTextString(m) } -func (*LoadParams) ProtoMessage() {} -func (*LoadParams) Descriptor() ([]byte, []int) { - return fileDescriptor_6f4e2bf9f888bddb, []int{5} -} - -func (m *LoadParams) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_LoadParams.Unmarshal(m, b) -} -func (m *LoadParams) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_LoadParams.Marshal(b, m, deterministic) -} -func (m *LoadParams) XXX_Merge(src proto.Message) { - xxx_messageInfo_LoadParams.Merge(m, src) -} -func (m *LoadParams) XXX_Size() int { - return xxx_messageInfo_LoadParams.Size(m) -} -func (m *LoadParams) XXX_DiscardUnknown() { - xxx_messageInfo_LoadParams.DiscardUnknown(m) -} - -var xxx_messageInfo_LoadParams proto.InternalMessageInfo - -type isLoadParams_Load interface { - isLoadParams_Load() -} - -type LoadParams_ClosedLoop struct { - ClosedLoop *ClosedLoopParams `protobuf:"bytes,1,opt,name=closed_loop,json=closedLoop,proto3,oneof"` -} - -type LoadParams_Poisson struct { - Poisson *PoissonParams `protobuf:"bytes,2,opt,name=poisson,proto3,oneof"` -} - -type LoadParams_Uniform struct { - Uniform *UniformParams `protobuf:"bytes,3,opt,name=uniform,proto3,oneof"` -} - -type LoadParams_Determ struct { - Determ *DeterministicParams `protobuf:"bytes,4,opt,name=determ,proto3,oneof"` -} - -type LoadParams_Pareto struct { - Pareto *ParetoParams `protobuf:"bytes,5,opt,name=pareto,proto3,oneof"` -} - -func (*LoadParams_ClosedLoop) isLoadParams_Load() {} - -func (*LoadParams_Poisson) isLoadParams_Load() {} - -func (*LoadParams_Uniform) isLoadParams_Load() {} - -func (*LoadParams_Determ) isLoadParams_Load() {} - -func (*LoadParams_Pareto) isLoadParams_Load() {} - -func (m *LoadParams) GetLoad() isLoadParams_Load { - if m != nil { - return m.Load - } - return nil -} - -func (m *LoadParams) GetClosedLoop() *ClosedLoopParams { - if x, ok := m.GetLoad().(*LoadParams_ClosedLoop); ok { - return x.ClosedLoop - } - return nil -} - -func (m *LoadParams) GetPoisson() *PoissonParams { - if x, ok := m.GetLoad().(*LoadParams_Poisson); ok { - return x.Poisson - } - return nil -} - -func (m *LoadParams) GetUniform() *UniformParams { - if x, ok := m.GetLoad().(*LoadParams_Uniform); ok { - return x.Uniform - } - return nil -} - -func (m *LoadParams) GetDeterm() *DeterministicParams { - if x, ok := m.GetLoad().(*LoadParams_Determ); ok { - return x.Determ - } - return nil -} - -func (m *LoadParams) GetPareto() *ParetoParams { - if x, ok := m.GetLoad().(*LoadParams_Pareto); ok { - return x.Pareto - } - return nil -} - -// XXX_OneofWrappers is for the internal use of the proto package. -func (*LoadParams) XXX_OneofWrappers() []interface{} { - return []interface{}{ - (*LoadParams_ClosedLoop)(nil), - (*LoadParams_Poisson)(nil), - (*LoadParams_Uniform)(nil), - (*LoadParams_Determ)(nil), - (*LoadParams_Pareto)(nil), - } -} - -// presence of SecurityParams implies use of TLS -type SecurityParams struct { - UseTestCa bool `protobuf:"varint,1,opt,name=use_test_ca,json=useTestCa,proto3" json:"use_test_ca,omitempty"` - ServerHostOverride string `protobuf:"bytes,2,opt,name=server_host_override,json=serverHostOverride,proto3" json:"server_host_override,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *SecurityParams) Reset() { *m = SecurityParams{} } -func (m *SecurityParams) String() string { return proto.CompactTextString(m) } -func (*SecurityParams) ProtoMessage() {} -func (*SecurityParams) Descriptor() ([]byte, []int) { - return fileDescriptor_6f4e2bf9f888bddb, []int{6} -} - -func (m *SecurityParams) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_SecurityParams.Unmarshal(m, b) -} -func (m *SecurityParams) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_SecurityParams.Marshal(b, m, deterministic) -} -func (m *SecurityParams) XXX_Merge(src proto.Message) { - xxx_messageInfo_SecurityParams.Merge(m, src) -} -func (m *SecurityParams) XXX_Size() int { - return xxx_messageInfo_SecurityParams.Size(m) -} -func (m *SecurityParams) XXX_DiscardUnknown() { - xxx_messageInfo_SecurityParams.DiscardUnknown(m) -} - -var xxx_messageInfo_SecurityParams proto.InternalMessageInfo - -func (m *SecurityParams) GetUseTestCa() bool { - if m != nil { - return m.UseTestCa - } - return false -} - -func (m *SecurityParams) GetServerHostOverride() string { - if m != nil { - return m.ServerHostOverride - } - return "" -} - -type ClientConfig struct { - // List of targets to connect to. At least one target needs to be specified. - ServerTargets []string `protobuf:"bytes,1,rep,name=server_targets,json=serverTargets,proto3" json:"server_targets,omitempty"` - ClientType ClientType `protobuf:"varint,2,opt,name=client_type,json=clientType,proto3,enum=grpc.testing.ClientType" json:"client_type,omitempty"` - SecurityParams *SecurityParams `protobuf:"bytes,3,opt,name=security_params,json=securityParams,proto3" json:"security_params,omitempty"` - // How many concurrent RPCs to start for each channel. - // For synchronous client, use a separate thread for each outstanding RPC. - OutstandingRpcsPerChannel int32 `protobuf:"varint,4,opt,name=outstanding_rpcs_per_channel,json=outstandingRpcsPerChannel,proto3" json:"outstanding_rpcs_per_channel,omitempty"` - // Number of independent client channels to create. - // i-th channel will connect to server_target[i % server_targets.size()] - ClientChannels int32 `protobuf:"varint,5,opt,name=client_channels,json=clientChannels,proto3" json:"client_channels,omitempty"` - // Only for async client. Number of threads to use to start/manage RPCs. - AsyncClientThreads int32 `protobuf:"varint,7,opt,name=async_client_threads,json=asyncClientThreads,proto3" json:"async_client_threads,omitempty"` - RpcType RpcType `protobuf:"varint,8,opt,name=rpc_type,json=rpcType,proto3,enum=grpc.testing.RpcType" json:"rpc_type,omitempty"` - // The requested load for the entire client (aggregated over all the threads). - LoadParams *LoadParams `protobuf:"bytes,10,opt,name=load_params,json=loadParams,proto3" json:"load_params,omitempty"` - PayloadConfig *PayloadConfig `protobuf:"bytes,11,opt,name=payload_config,json=payloadConfig,proto3" json:"payload_config,omitempty"` - HistogramParams *HistogramParams `protobuf:"bytes,12,opt,name=histogram_params,json=histogramParams,proto3" json:"histogram_params,omitempty"` - // Specify the cores we should run the client on, if desired - CoreList []int32 `protobuf:"varint,13,rep,packed,name=core_list,json=coreList,proto3" json:"core_list,omitempty"` - CoreLimit int32 `protobuf:"varint,14,opt,name=core_limit,json=coreLimit,proto3" json:"core_limit,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *ClientConfig) Reset() { *m = ClientConfig{} } -func (m *ClientConfig) String() string { return proto.CompactTextString(m) } -func (*ClientConfig) ProtoMessage() {} -func (*ClientConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_6f4e2bf9f888bddb, []int{7} -} - -func (m *ClientConfig) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ClientConfig.Unmarshal(m, b) -} -func (m *ClientConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ClientConfig.Marshal(b, m, deterministic) -} -func (m *ClientConfig) XXX_Merge(src proto.Message) { - xxx_messageInfo_ClientConfig.Merge(m, src) -} -func (m *ClientConfig) XXX_Size() int { - return xxx_messageInfo_ClientConfig.Size(m) -} -func (m *ClientConfig) XXX_DiscardUnknown() { - xxx_messageInfo_ClientConfig.DiscardUnknown(m) -} - -var xxx_messageInfo_ClientConfig proto.InternalMessageInfo - -func (m *ClientConfig) GetServerTargets() []string { - if m != nil { - return m.ServerTargets - } - return nil -} - -func (m *ClientConfig) GetClientType() ClientType { - if m != nil { - return m.ClientType - } - return ClientType_SYNC_CLIENT -} - -func (m *ClientConfig) GetSecurityParams() *SecurityParams { - if m != nil { - return m.SecurityParams - } - return nil -} - -func (m *ClientConfig) GetOutstandingRpcsPerChannel() int32 { - if m != nil { - return m.OutstandingRpcsPerChannel - } - return 0 -} - -func (m *ClientConfig) GetClientChannels() int32 { - if m != nil { - return m.ClientChannels - } - return 0 -} - -func (m *ClientConfig) GetAsyncClientThreads() int32 { - if m != nil { - return m.AsyncClientThreads - } - return 0 -} - -func (m *ClientConfig) GetRpcType() RpcType { - if m != nil { - return m.RpcType - } - return RpcType_UNARY -} - -func (m *ClientConfig) GetLoadParams() *LoadParams { - if m != nil { - return m.LoadParams - } - return nil -} - -func (m *ClientConfig) GetPayloadConfig() *PayloadConfig { - if m != nil { - return m.PayloadConfig - } - return nil -} - -func (m *ClientConfig) GetHistogramParams() *HistogramParams { - if m != nil { - return m.HistogramParams - } - return nil -} - -func (m *ClientConfig) GetCoreList() []int32 { - if m != nil { - return m.CoreList - } - return nil -} - -func (m *ClientConfig) GetCoreLimit() int32 { - if m != nil { - return m.CoreLimit - } - return 0 -} - -type ClientStatus struct { - Stats *ClientStats `protobuf:"bytes,1,opt,name=stats,proto3" json:"stats,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *ClientStatus) Reset() { *m = ClientStatus{} } -func (m *ClientStatus) String() string { return proto.CompactTextString(m) } -func (*ClientStatus) ProtoMessage() {} -func (*ClientStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_6f4e2bf9f888bddb, []int{8} -} - -func (m *ClientStatus) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ClientStatus.Unmarshal(m, b) -} -func (m *ClientStatus) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ClientStatus.Marshal(b, m, deterministic) -} -func (m *ClientStatus) XXX_Merge(src proto.Message) { - xxx_messageInfo_ClientStatus.Merge(m, src) -} -func (m *ClientStatus) XXX_Size() int { - return xxx_messageInfo_ClientStatus.Size(m) -} -func (m *ClientStatus) XXX_DiscardUnknown() { - xxx_messageInfo_ClientStatus.DiscardUnknown(m) -} - -var xxx_messageInfo_ClientStatus proto.InternalMessageInfo - -func (m *ClientStatus) GetStats() *ClientStats { - if m != nil { - return m.Stats - } - return nil -} - -// Request current stats -type Mark struct { - // if true, the stats will be reset after taking their snapshot. - Reset_ bool `protobuf:"varint,1,opt,name=reset,proto3" json:"reset,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *Mark) Reset() { *m = Mark{} } -func (m *Mark) String() string { return proto.CompactTextString(m) } -func (*Mark) ProtoMessage() {} -func (*Mark) Descriptor() ([]byte, []int) { - return fileDescriptor_6f4e2bf9f888bddb, []int{9} -} - -func (m *Mark) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Mark.Unmarshal(m, b) -} -func (m *Mark) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Mark.Marshal(b, m, deterministic) -} -func (m *Mark) XXX_Merge(src proto.Message) { - xxx_messageInfo_Mark.Merge(m, src) -} -func (m *Mark) XXX_Size() int { - return xxx_messageInfo_Mark.Size(m) -} -func (m *Mark) XXX_DiscardUnknown() { - xxx_messageInfo_Mark.DiscardUnknown(m) -} - -var xxx_messageInfo_Mark proto.InternalMessageInfo - -func (m *Mark) GetReset_() bool { - if m != nil { - return m.Reset_ - } - return false -} - -type ClientArgs struct { - // Types that are valid to be assigned to Argtype: - // *ClientArgs_Setup - // *ClientArgs_Mark - Argtype isClientArgs_Argtype `protobuf_oneof:"argtype"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *ClientArgs) Reset() { *m = ClientArgs{} } -func (m *ClientArgs) String() string { return proto.CompactTextString(m) } -func (*ClientArgs) ProtoMessage() {} -func (*ClientArgs) Descriptor() ([]byte, []int) { - return fileDescriptor_6f4e2bf9f888bddb, []int{10} -} - -func (m *ClientArgs) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ClientArgs.Unmarshal(m, b) -} -func (m *ClientArgs) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ClientArgs.Marshal(b, m, deterministic) -} -func (m *ClientArgs) XXX_Merge(src proto.Message) { - xxx_messageInfo_ClientArgs.Merge(m, src) -} -func (m *ClientArgs) XXX_Size() int { - return xxx_messageInfo_ClientArgs.Size(m) -} -func (m *ClientArgs) XXX_DiscardUnknown() { - xxx_messageInfo_ClientArgs.DiscardUnknown(m) -} - -var xxx_messageInfo_ClientArgs proto.InternalMessageInfo - -type isClientArgs_Argtype interface { - isClientArgs_Argtype() -} - -type ClientArgs_Setup struct { - Setup *ClientConfig `protobuf:"bytes,1,opt,name=setup,proto3,oneof"` -} - -type ClientArgs_Mark struct { - Mark *Mark `protobuf:"bytes,2,opt,name=mark,proto3,oneof"` -} - -func (*ClientArgs_Setup) isClientArgs_Argtype() {} - -func (*ClientArgs_Mark) isClientArgs_Argtype() {} - -func (m *ClientArgs) GetArgtype() isClientArgs_Argtype { - if m != nil { - return m.Argtype - } - return nil -} - -func (m *ClientArgs) GetSetup() *ClientConfig { - if x, ok := m.GetArgtype().(*ClientArgs_Setup); ok { - return x.Setup - } - return nil -} - -func (m *ClientArgs) GetMark() *Mark { - if x, ok := m.GetArgtype().(*ClientArgs_Mark); ok { - return x.Mark - } - return nil -} - -// XXX_OneofWrappers is for the internal use of the proto package. -func (*ClientArgs) XXX_OneofWrappers() []interface{} { - return []interface{}{ - (*ClientArgs_Setup)(nil), - (*ClientArgs_Mark)(nil), - } -} - -type ServerConfig struct { - ServerType ServerType `protobuf:"varint,1,opt,name=server_type,json=serverType,proto3,enum=grpc.testing.ServerType" json:"server_type,omitempty"` - SecurityParams *SecurityParams `protobuf:"bytes,2,opt,name=security_params,json=securityParams,proto3" json:"security_params,omitempty"` - // Port on which to listen. Zero means pick unused port. - Port int32 `protobuf:"varint,4,opt,name=port,proto3" json:"port,omitempty"` - // Only for async server. Number of threads used to serve the requests. - AsyncServerThreads int32 `protobuf:"varint,7,opt,name=async_server_threads,json=asyncServerThreads,proto3" json:"async_server_threads,omitempty"` - // Specify the number of cores to limit server to, if desired - CoreLimit int32 `protobuf:"varint,8,opt,name=core_limit,json=coreLimit,proto3" json:"core_limit,omitempty"` - // payload config, used in generic server - PayloadConfig *PayloadConfig `protobuf:"bytes,9,opt,name=payload_config,json=payloadConfig,proto3" json:"payload_config,omitempty"` - // Specify the cores we should run the server on, if desired - CoreList []int32 `protobuf:"varint,10,rep,packed,name=core_list,json=coreList,proto3" json:"core_list,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *ServerConfig) Reset() { *m = ServerConfig{} } -func (m *ServerConfig) String() string { return proto.CompactTextString(m) } -func (*ServerConfig) ProtoMessage() {} -func (*ServerConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_6f4e2bf9f888bddb, []int{11} -} - -func (m *ServerConfig) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ServerConfig.Unmarshal(m, b) -} -func (m *ServerConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ServerConfig.Marshal(b, m, deterministic) -} -func (m *ServerConfig) XXX_Merge(src proto.Message) { - xxx_messageInfo_ServerConfig.Merge(m, src) -} -func (m *ServerConfig) XXX_Size() int { - return xxx_messageInfo_ServerConfig.Size(m) -} -func (m *ServerConfig) XXX_DiscardUnknown() { - xxx_messageInfo_ServerConfig.DiscardUnknown(m) -} - -var xxx_messageInfo_ServerConfig proto.InternalMessageInfo - -func (m *ServerConfig) GetServerType() ServerType { - if m != nil { - return m.ServerType - } - return ServerType_SYNC_SERVER -} - -func (m *ServerConfig) GetSecurityParams() *SecurityParams { - if m != nil { - return m.SecurityParams - } - return nil -} - -func (m *ServerConfig) GetPort() int32 { - if m != nil { - return m.Port - } - return 0 -} - -func (m *ServerConfig) GetAsyncServerThreads() int32 { - if m != nil { - return m.AsyncServerThreads - } - return 0 -} - -func (m *ServerConfig) GetCoreLimit() int32 { - if m != nil { - return m.CoreLimit - } - return 0 -} - -func (m *ServerConfig) GetPayloadConfig() *PayloadConfig { - if m != nil { - return m.PayloadConfig - } - return nil -} - -func (m *ServerConfig) GetCoreList() []int32 { - if m != nil { - return m.CoreList - } - return nil -} - -type ServerArgs struct { - // Types that are valid to be assigned to Argtype: - // *ServerArgs_Setup - // *ServerArgs_Mark - Argtype isServerArgs_Argtype `protobuf_oneof:"argtype"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *ServerArgs) Reset() { *m = ServerArgs{} } -func (m *ServerArgs) String() string { return proto.CompactTextString(m) } -func (*ServerArgs) ProtoMessage() {} -func (*ServerArgs) Descriptor() ([]byte, []int) { - return fileDescriptor_6f4e2bf9f888bddb, []int{12} -} - -func (m *ServerArgs) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ServerArgs.Unmarshal(m, b) -} -func (m *ServerArgs) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ServerArgs.Marshal(b, m, deterministic) -} -func (m *ServerArgs) XXX_Merge(src proto.Message) { - xxx_messageInfo_ServerArgs.Merge(m, src) -} -func (m *ServerArgs) XXX_Size() int { - return xxx_messageInfo_ServerArgs.Size(m) -} -func (m *ServerArgs) XXX_DiscardUnknown() { - xxx_messageInfo_ServerArgs.DiscardUnknown(m) -} - -var xxx_messageInfo_ServerArgs proto.InternalMessageInfo - -type isServerArgs_Argtype interface { - isServerArgs_Argtype() -} - -type ServerArgs_Setup struct { - Setup *ServerConfig `protobuf:"bytes,1,opt,name=setup,proto3,oneof"` -} - -type ServerArgs_Mark struct { - Mark *Mark `protobuf:"bytes,2,opt,name=mark,proto3,oneof"` -} - -func (*ServerArgs_Setup) isServerArgs_Argtype() {} - -func (*ServerArgs_Mark) isServerArgs_Argtype() {} - -func (m *ServerArgs) GetArgtype() isServerArgs_Argtype { - if m != nil { - return m.Argtype - } - return nil -} - -func (m *ServerArgs) GetSetup() *ServerConfig { - if x, ok := m.GetArgtype().(*ServerArgs_Setup); ok { - return x.Setup - } - return nil -} - -func (m *ServerArgs) GetMark() *Mark { - if x, ok := m.GetArgtype().(*ServerArgs_Mark); ok { - return x.Mark - } - return nil -} - -// XXX_OneofWrappers is for the internal use of the proto package. -func (*ServerArgs) XXX_OneofWrappers() []interface{} { - return []interface{}{ - (*ServerArgs_Setup)(nil), - (*ServerArgs_Mark)(nil), - } -} - -type ServerStatus struct { - Stats *ServerStats `protobuf:"bytes,1,opt,name=stats,proto3" json:"stats,omitempty"` - // the port bound by the server - Port int32 `protobuf:"varint,2,opt,name=port,proto3" json:"port,omitempty"` - // Number of cores available to the server - Cores int32 `protobuf:"varint,3,opt,name=cores,proto3" json:"cores,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *ServerStatus) Reset() { *m = ServerStatus{} } -func (m *ServerStatus) String() string { return proto.CompactTextString(m) } -func (*ServerStatus) ProtoMessage() {} -func (*ServerStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_6f4e2bf9f888bddb, []int{13} -} - -func (m *ServerStatus) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ServerStatus.Unmarshal(m, b) -} -func (m *ServerStatus) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ServerStatus.Marshal(b, m, deterministic) -} -func (m *ServerStatus) XXX_Merge(src proto.Message) { - xxx_messageInfo_ServerStatus.Merge(m, src) -} -func (m *ServerStatus) XXX_Size() int { - return xxx_messageInfo_ServerStatus.Size(m) -} -func (m *ServerStatus) XXX_DiscardUnknown() { - xxx_messageInfo_ServerStatus.DiscardUnknown(m) -} - -var xxx_messageInfo_ServerStatus proto.InternalMessageInfo - -func (m *ServerStatus) GetStats() *ServerStats { - if m != nil { - return m.Stats - } - return nil -} - -func (m *ServerStatus) GetPort() int32 { - if m != nil { - return m.Port - } - return 0 -} - -func (m *ServerStatus) GetCores() int32 { - if m != nil { - return m.Cores - } - return 0 -} - -type CoreRequest struct { - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *CoreRequest) Reset() { *m = CoreRequest{} } -func (m *CoreRequest) String() string { return proto.CompactTextString(m) } -func (*CoreRequest) ProtoMessage() {} -func (*CoreRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_6f4e2bf9f888bddb, []int{14} -} - -func (m *CoreRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_CoreRequest.Unmarshal(m, b) -} -func (m *CoreRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_CoreRequest.Marshal(b, m, deterministic) -} -func (m *CoreRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreRequest.Merge(m, src) -} -func (m *CoreRequest) XXX_Size() int { - return xxx_messageInfo_CoreRequest.Size(m) -} -func (m *CoreRequest) XXX_DiscardUnknown() { - xxx_messageInfo_CoreRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_CoreRequest proto.InternalMessageInfo - -type CoreResponse struct { - // Number of cores available on the server - Cores int32 `protobuf:"varint,1,opt,name=cores,proto3" json:"cores,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *CoreResponse) Reset() { *m = CoreResponse{} } -func (m *CoreResponse) String() string { return proto.CompactTextString(m) } -func (*CoreResponse) ProtoMessage() {} -func (*CoreResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_6f4e2bf9f888bddb, []int{15} -} - -func (m *CoreResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_CoreResponse.Unmarshal(m, b) -} -func (m *CoreResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_CoreResponse.Marshal(b, m, deterministic) -} -func (m *CoreResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreResponse.Merge(m, src) -} -func (m *CoreResponse) XXX_Size() int { - return xxx_messageInfo_CoreResponse.Size(m) -} -func (m *CoreResponse) XXX_DiscardUnknown() { - xxx_messageInfo_CoreResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_CoreResponse proto.InternalMessageInfo - -func (m *CoreResponse) GetCores() int32 { - if m != nil { - return m.Cores - } - return 0 -} - -type Void struct { - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *Void) Reset() { *m = Void{} } -func (m *Void) String() string { return proto.CompactTextString(m) } -func (*Void) ProtoMessage() {} -func (*Void) Descriptor() ([]byte, []int) { - return fileDescriptor_6f4e2bf9f888bddb, []int{16} -} - -func (m *Void) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Void.Unmarshal(m, b) -} -func (m *Void) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Void.Marshal(b, m, deterministic) -} -func (m *Void) XXX_Merge(src proto.Message) { - xxx_messageInfo_Void.Merge(m, src) -} -func (m *Void) XXX_Size() int { - return xxx_messageInfo_Void.Size(m) -} -func (m *Void) XXX_DiscardUnknown() { - xxx_messageInfo_Void.DiscardUnknown(m) -} - -var xxx_messageInfo_Void proto.InternalMessageInfo - -// A single performance scenario: input to qps_json_driver -type Scenario struct { - // Human readable name for this scenario - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - // Client configuration - ClientConfig *ClientConfig `protobuf:"bytes,2,opt,name=client_config,json=clientConfig,proto3" json:"client_config,omitempty"` - // Number of clients to start for the test - NumClients int32 `protobuf:"varint,3,opt,name=num_clients,json=numClients,proto3" json:"num_clients,omitempty"` - // Server configuration - ServerConfig *ServerConfig `protobuf:"bytes,4,opt,name=server_config,json=serverConfig,proto3" json:"server_config,omitempty"` - // Number of servers to start for the test - NumServers int32 `protobuf:"varint,5,opt,name=num_servers,json=numServers,proto3" json:"num_servers,omitempty"` - // Warmup period, in seconds - WarmupSeconds int32 `protobuf:"varint,6,opt,name=warmup_seconds,json=warmupSeconds,proto3" json:"warmup_seconds,omitempty"` - // Benchmark time, in seconds - BenchmarkSeconds int32 `protobuf:"varint,7,opt,name=benchmark_seconds,json=benchmarkSeconds,proto3" json:"benchmark_seconds,omitempty"` - // Number of workers to spawn locally (usually zero) - SpawnLocalWorkerCount int32 `protobuf:"varint,8,opt,name=spawn_local_worker_count,json=spawnLocalWorkerCount,proto3" json:"spawn_local_worker_count,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *Scenario) Reset() { *m = Scenario{} } -func (m *Scenario) String() string { return proto.CompactTextString(m) } -func (*Scenario) ProtoMessage() {} -func (*Scenario) Descriptor() ([]byte, []int) { - return fileDescriptor_6f4e2bf9f888bddb, []int{17} -} - -func (m *Scenario) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Scenario.Unmarshal(m, b) -} -func (m *Scenario) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Scenario.Marshal(b, m, deterministic) -} -func (m *Scenario) XXX_Merge(src proto.Message) { - xxx_messageInfo_Scenario.Merge(m, src) -} -func (m *Scenario) XXX_Size() int { - return xxx_messageInfo_Scenario.Size(m) -} -func (m *Scenario) XXX_DiscardUnknown() { - xxx_messageInfo_Scenario.DiscardUnknown(m) -} - -var xxx_messageInfo_Scenario proto.InternalMessageInfo - -func (m *Scenario) GetName() string { - if m != nil { - return m.Name - } - return "" -} - -func (m *Scenario) GetClientConfig() *ClientConfig { - if m != nil { - return m.ClientConfig - } - return nil -} - -func (m *Scenario) GetNumClients() int32 { - if m != nil { - return m.NumClients - } - return 0 -} - -func (m *Scenario) GetServerConfig() *ServerConfig { - if m != nil { - return m.ServerConfig - } - return nil -} - -func (m *Scenario) GetNumServers() int32 { - if m != nil { - return m.NumServers - } - return 0 -} - -func (m *Scenario) GetWarmupSeconds() int32 { - if m != nil { - return m.WarmupSeconds - } - return 0 -} - -func (m *Scenario) GetBenchmarkSeconds() int32 { - if m != nil { - return m.BenchmarkSeconds - } - return 0 -} - -func (m *Scenario) GetSpawnLocalWorkerCount() int32 { - if m != nil { - return m.SpawnLocalWorkerCount - } - return 0 -} - -// A set of scenarios to be run with qps_json_driver -type Scenarios struct { - Scenarios []*Scenario `protobuf:"bytes,1,rep,name=scenarios,proto3" json:"scenarios,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *Scenarios) Reset() { *m = Scenarios{} } -func (m *Scenarios) String() string { return proto.CompactTextString(m) } -func (*Scenarios) ProtoMessage() {} -func (*Scenarios) Descriptor() ([]byte, []int) { - return fileDescriptor_6f4e2bf9f888bddb, []int{18} -} - -func (m *Scenarios) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Scenarios.Unmarshal(m, b) -} -func (m *Scenarios) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Scenarios.Marshal(b, m, deterministic) -} -func (m *Scenarios) XXX_Merge(src proto.Message) { - xxx_messageInfo_Scenarios.Merge(m, src) -} -func (m *Scenarios) XXX_Size() int { - return xxx_messageInfo_Scenarios.Size(m) -} -func (m *Scenarios) XXX_DiscardUnknown() { - xxx_messageInfo_Scenarios.DiscardUnknown(m) -} - -var xxx_messageInfo_Scenarios proto.InternalMessageInfo - -func (m *Scenarios) GetScenarios() []*Scenario { - if m != nil { - return m.Scenarios - } - return nil -} - -func init() { - proto.RegisterEnum("grpc.testing.ClientType", ClientType_name, ClientType_value) - proto.RegisterEnum("grpc.testing.ServerType", ServerType_name, ServerType_value) - proto.RegisterEnum("grpc.testing.RpcType", RpcType_name, RpcType_value) - proto.RegisterType((*PoissonParams)(nil), "grpc.testing.PoissonParams") - proto.RegisterType((*UniformParams)(nil), "grpc.testing.UniformParams") - proto.RegisterType((*DeterministicParams)(nil), "grpc.testing.DeterministicParams") - proto.RegisterType((*ParetoParams)(nil), "grpc.testing.ParetoParams") - proto.RegisterType((*ClosedLoopParams)(nil), "grpc.testing.ClosedLoopParams") - proto.RegisterType((*LoadParams)(nil), "grpc.testing.LoadParams") - proto.RegisterType((*SecurityParams)(nil), "grpc.testing.SecurityParams") - proto.RegisterType((*ClientConfig)(nil), "grpc.testing.ClientConfig") - proto.RegisterType((*ClientStatus)(nil), "grpc.testing.ClientStatus") - proto.RegisterType((*Mark)(nil), "grpc.testing.Mark") - proto.RegisterType((*ClientArgs)(nil), "grpc.testing.ClientArgs") - proto.RegisterType((*ServerConfig)(nil), "grpc.testing.ServerConfig") - proto.RegisterType((*ServerArgs)(nil), "grpc.testing.ServerArgs") - proto.RegisterType((*ServerStatus)(nil), "grpc.testing.ServerStatus") - proto.RegisterType((*CoreRequest)(nil), "grpc.testing.CoreRequest") - proto.RegisterType((*CoreResponse)(nil), "grpc.testing.CoreResponse") - proto.RegisterType((*Void)(nil), "grpc.testing.Void") - proto.RegisterType((*Scenario)(nil), "grpc.testing.Scenario") - proto.RegisterType((*Scenarios)(nil), "grpc.testing.Scenarios") -} - -func init() { - proto.RegisterFile("benchmark/grpc_testing/control.proto", fileDescriptor_6f4e2bf9f888bddb) -} - -var fileDescriptor_6f4e2bf9f888bddb = []byte{ - // 1219 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x56, 0xed, 0x6e, 0xdb, 0x36, - 0x17, 0xb6, 0x5d, 0xdb, 0xb1, 0x8e, 0x3f, 0xe2, 0x97, 0x6f, 0x3a, 0xa8, 0xe9, 0xc7, 0x5a, 0xad, - 0xc5, 0x82, 0x0e, 0xb3, 0x0b, 0xaf, 0x40, 0x57, 0xec, 0x47, 0xe1, 0x78, 0x46, 0x1d, 0x20, 0xcd, - 0x32, 0x3a, 0xed, 0xd0, 0xfd, 0x11, 0x18, 0x99, 0x91, 0x85, 0xca, 0xa2, 0x46, 0x52, 0x2d, 0x72, - 0x0b, 0xbb, 0xa6, 0x5d, 0xc7, 0x6e, 0x63, 0xb7, 0x30, 0xf0, 0x43, 0x8e, 0xe4, 0xba, 0x68, 0xb6, - 0xfd, 0x13, 0xcf, 0x79, 0x1e, 0x1e, 0xf2, 0x3c, 0xe7, 0x1c, 0x0a, 0x1e, 0x9e, 0xd3, 0x24, 0x58, - 0xae, 0x08, 0x7f, 0x37, 0x0c, 0x79, 0x1a, 0xf8, 0x92, 0x0a, 0x19, 0x25, 0xe1, 0x30, 0x60, 0x89, - 0xe4, 0x2c, 0x1e, 0xa4, 0x9c, 0x49, 0x86, 0x3a, 0xca, 0x37, 0xb0, 0xbe, 0xfd, 0x47, 0x9f, 0xe0, - 0xa4, 0xe4, 0x32, 0x66, 0x64, 0x21, 0x0c, 0x69, 0xdf, 0xfb, 0x04, 0x4c, 0x48, 0x22, 0x2d, 0xc6, - 0x1b, 0x41, 0xf7, 0x94, 0x45, 0x42, 0xb0, 0xe4, 0x94, 0x70, 0xb2, 0x12, 0xe8, 0x01, 0x74, 0xd8, - 0xc5, 0x05, 0xe5, 0x74, 0xe1, 0xab, 0xbd, 0xdc, 0xea, 0xfd, 0xea, 0x41, 0x15, 0xb7, 0xad, 0xed, - 0x98, 0x91, 0x85, 0x47, 0xa0, 0xfb, 0x3a, 0x89, 0x2e, 0x18, 0x5f, 0x59, 0xce, 0xd7, 0xb0, 0x1b, - 0x25, 0x92, 0x72, 0xc2, 0x79, 0xf4, 0x9e, 0xc4, 0x7e, 0xcc, 0x2c, 0xad, 0x57, 0x34, 0x1f, 0xb3, - 0x8f, 0x80, 0xcb, 0xc8, 0xad, 0x7d, 0x0c, 0x9c, 0x45, 0xde, 0xf7, 0xf0, 0xff, 0x1f, 0xa9, 0xa4, - 0x7c, 0x15, 0x25, 0x91, 0x90, 0x51, 0x70, 0xfd, 0xc3, 0xfd, 0x0c, 0x9d, 0x53, 0xc2, 0xa9, 0x64, - 0x96, 0xf2, 0x0d, 0xfc, 0xaf, 0x14, 0xf2, 0x9c, 0x08, 0x6a, 0x79, 0xfd, 0xa2, 0xe3, 0x90, 0x08, - 0x8a, 0xf6, 0xa0, 0x41, 0xe2, 0x74, 0x49, 0xec, 0xa9, 0xcc, 0xc2, 0x43, 0xd0, 0x9f, 0xc4, 0x4c, - 0xa8, 0x00, 0x2c, 0x35, 0xdb, 0x7a, 0x7f, 0xd4, 0x00, 0x54, 0x3c, 0x1b, 0x65, 0x0c, 0xed, 0x40, - 0x43, 0xfc, 0x98, 0xb1, 0x54, 0xef, 0xdf, 0x1e, 0xdd, 0x1b, 0x14, 0x55, 0x1b, 0x6c, 0xee, 0x31, - 0xab, 0x60, 0x08, 0xd6, 0x36, 0xf4, 0x0c, 0x76, 0x52, 0xa3, 0x84, 0x8e, 0xde, 0x1e, 0xdd, 0x2e, - 0xd3, 0x4b, 0x32, 0xcd, 0x2a, 0x38, 0x47, 0x2b, 0x62, 0x66, 0xe4, 0x70, 0x6f, 0x6c, 0x23, 0x96, - 0xb4, 0x52, 0x44, 0x8b, 0x46, 0x3f, 0x40, 0x73, 0xa1, 0x93, 0xec, 0xd6, 0x35, 0xef, 0x41, 0x99, - 0xb7, 0x45, 0x80, 0x59, 0x05, 0x5b, 0x0a, 0x7a, 0x0a, 0xcd, 0x54, 0xe7, 0xd9, 0x6d, 0x68, 0xf2, - 0xfe, 0xc6, 0x69, 0x0b, 0x1a, 0x28, 0x96, 0xc1, 0x1e, 0x36, 0xa1, 0xae, 0x84, 0xf3, 0xce, 0xa1, - 0x37, 0xa7, 0x41, 0xc6, 0x23, 0x79, 0x69, 0x33, 0x78, 0x0f, 0xda, 0x99, 0xa0, 0xba, 0x46, 0xfd, - 0x80, 0xe8, 0x0c, 0xb6, 0xb0, 0x93, 0x09, 0x7a, 0x46, 0x85, 0x9c, 0x10, 0xf4, 0x04, 0xf6, 0x04, - 0xe5, 0xef, 0x29, 0xf7, 0x97, 0x4c, 0x48, 0x9f, 0xbd, 0xa7, 0x9c, 0x47, 0x0b, 0xaa, 0x73, 0xe5, - 0x60, 0x64, 0x7c, 0x33, 0x26, 0xe4, 0x4f, 0xd6, 0xe3, 0xfd, 0xde, 0x80, 0xce, 0x24, 0x8e, 0x68, - 0x22, 0x27, 0x2c, 0xb9, 0x88, 0x42, 0xf4, 0x08, 0x7a, 0x76, 0x0b, 0x49, 0x78, 0x48, 0xa5, 0x70, - 0xab, 0xf7, 0x6f, 0x1c, 0x38, 0xb8, 0x6b, 0xac, 0x67, 0xc6, 0x88, 0x9e, 0x2b, 0x2d, 0x15, 0xcd, - 0x97, 0x97, 0xa9, 0x09, 0xd0, 0x1b, 0xb9, 0x9b, 0x5a, 0x2a, 0xc0, 0xd9, 0x65, 0x4a, 0x95, 0x86, - 0xf9, 0x37, 0x9a, 0xc2, 0xae, 0xb0, 0xd7, 0xf2, 0x53, 0x7d, 0x2f, 0x2b, 0xc9, 0x9d, 0x32, 0xbd, - 0x7c, 0x77, 0xdc, 0x13, 0xe5, 0x5c, 0xbc, 0x80, 0x3b, 0x2c, 0x93, 0x42, 0x92, 0x64, 0x11, 0x25, - 0xa1, 0xcf, 0xd3, 0x40, 0xf8, 0x29, 0xe5, 0x7e, 0xb0, 0x24, 0x49, 0x42, 0x63, 0x2d, 0x57, 0x03, - 0xdf, 0x2a, 0x60, 0x70, 0x1a, 0x88, 0x53, 0xca, 0x27, 0x06, 0xa0, 0xfa, 0xcc, 0x5e, 0xc1, 0x52, - 0x84, 0x56, 0xa9, 0x81, 0x7b, 0xc6, 0x6c, 0x71, 0x42, 0x65, 0x95, 0x88, 0xcb, 0x24, 0xf0, 0xf3, - 0x1b, 0x2f, 0x39, 0x25, 0x0b, 0xe1, 0xee, 0x68, 0x34, 0xd2, 0x3e, 0x7b, 0x57, 0xe3, 0x41, 0x4f, - 0xa0, 0xa5, 0x67, 0x89, 0x4a, 0x4d, 0x4b, 0xa7, 0xe6, 0x66, 0xf9, 0x6e, 0x38, 0x0d, 0x74, 0x5e, - 0x76, 0xb8, 0xf9, 0x50, 0xf9, 0x54, 0x9a, 0xe7, 0x09, 0x01, 0x9d, 0x90, 0x8d, 0x7c, 0x5e, 0xb5, - 0x12, 0x86, 0xf8, 0xaa, 0xad, 0x0e, 0xa1, 0x67, 0x67, 0x9a, 0x1f, 0x68, 0x0d, 0xdd, 0xf6, 0xd6, - 0xd6, 0x30, 0x18, 0x23, 0x33, 0xee, 0xa6, 0xc5, 0x25, 0x9a, 0x41, 0x7f, 0x19, 0x09, 0xc9, 0x42, - 0x4e, 0x56, 0xf9, 0x19, 0x3a, 0x7a, 0x97, 0xbb, 0xe5, 0x5d, 0x66, 0x39, 0xca, 0x1e, 0x64, 0x77, - 0x59, 0x36, 0xa0, 0xdb, 0xe0, 0x04, 0x8c, 0x53, 0x3f, 0x8e, 0x84, 0x74, 0xbb, 0xf7, 0x6f, 0x1c, - 0x34, 0x70, 0x4b, 0x19, 0x8e, 0x23, 0x21, 0xd1, 0x5d, 0x00, 0xeb, 0x5c, 0x45, 0xd2, 0xed, 0xe9, - 0xfc, 0x39, 0xc6, 0xbb, 0x8a, 0xa4, 0xf7, 0x22, 0xaf, 0xc5, 0xb9, 0x24, 0x32, 0x13, 0x68, 0x08, - 0x0d, 0x3d, 0x86, 0xed, 0xa8, 0xb8, 0xb5, 0xad, 0xbc, 0x14, 0x54, 0x60, 0x83, 0xf3, 0xee, 0x40, - 0xfd, 0x15, 0xe1, 0xef, 0xd4, 0x88, 0xe2, 0x54, 0x50, 0x69, 0x3b, 0xc4, 0x2c, 0xbc, 0x0c, 0xc0, - 0x70, 0xc6, 0x3c, 0x14, 0x68, 0x04, 0x0d, 0x41, 0x65, 0x96, 0xcf, 0xa1, 0xfd, 0x6d, 0x9b, 0x9b, - 0xec, 0xcc, 0x2a, 0xd8, 0x40, 0xd1, 0x01, 0xd4, 0xd5, 0x4b, 0x61, 0x67, 0x0f, 0x2a, 0x53, 0x54, - 0xe4, 0x59, 0x05, 0x6b, 0xc4, 0xa1, 0x03, 0x3b, 0x84, 0x87, 0xaa, 0x00, 0xbc, 0x3f, 0x6b, 0xd0, - 0x99, 0xeb, 0xe6, 0xb1, 0xc9, 0x7e, 0x0e, 0xed, 0xbc, 0xc5, 0x54, 0x81, 0x54, 0xb7, 0xf5, 0x8e, - 0x21, 0x98, 0xde, 0x11, 0xeb, 0xef, 0x6d, 0xbd, 0x53, 0xfb, 0x17, 0xbd, 0x83, 0xa0, 0x9e, 0x32, - 0x2e, 0x6d, 0x8f, 0xe8, 0xef, 0xab, 0x2a, 0xcf, 0xcf, 0xb6, 0xa5, 0xca, 0xed, 0xa9, 0x6c, 0x95, - 0x97, 0xd5, 0x6c, 0x6d, 0xa8, 0xb9, 0xa5, 0x2e, 0x9d, 0x7f, 0x5c, 0x97, 0xa5, 0x6a, 0x82, 0x72, - 0x35, 0x29, 0x3d, 0xcd, 0x81, 0xae, 0xa1, 0x67, 0x51, 0x80, 0xff, 0xa8, 0x67, 0x94, 0xcb, 0x79, - 0xad, 0x2a, 0xbd, 0x82, 0xe6, 0x55, 0xba, 0xce, 0x7e, 0xad, 0x90, 0xfd, 0x3d, 0x68, 0xa8, 0x7b, - 0x99, 0x51, 0xd8, 0xc0, 0x66, 0xe1, 0x75, 0xa1, 0x3d, 0x61, 0x9c, 0x62, 0xfa, 0x5b, 0x46, 0x85, - 0xf4, 0x1e, 0x42, 0xc7, 0x2c, 0x45, 0xca, 0x12, 0xf3, 0x12, 0x1b, 0x52, 0xb5, 0x48, 0x6a, 0x42, - 0xfd, 0x0d, 0x8b, 0x16, 0xde, 0x5f, 0x35, 0x68, 0xcd, 0x03, 0x9a, 0x10, 0x1e, 0x31, 0x15, 0x33, - 0x21, 0x2b, 0x53, 0x6c, 0x0e, 0xd6, 0xdf, 0xe8, 0x05, 0x74, 0xf3, 0x01, 0x68, 0xf4, 0xa9, 0x7d, - 0xae, 0x13, 0x70, 0x27, 0x28, 0xbe, 0x15, 0x5f, 0x42, 0x3b, 0xc9, 0x56, 0x76, 0x2c, 0xe6, 0x47, - 0x87, 0x24, 0x5b, 0x19, 0x8e, 0x9a, 0xd1, 0xf6, 0xd9, 0xc8, 0x23, 0xd4, 0x3f, 0xa7, 0x0d, 0xee, - 0x88, 0x62, 0xab, 0xd8, 0x08, 0xc6, 0x96, 0xcf, 0x67, 0x15, 0xc1, 0x70, 0x84, 0x7a, 0xae, 0x3e, - 0x10, 0xbe, 0xca, 0x52, 0x5f, 0xd0, 0x80, 0x25, 0x0b, 0xe1, 0x36, 0x35, 0xa6, 0x6b, 0xac, 0x73, - 0x63, 0x54, 0x3f, 0x38, 0xeb, 0xff, 0xbc, 0x35, 0xd2, 0x54, 0x76, 0x7f, 0xed, 0xc8, 0xc1, 0xcf, - 0xc0, 0x15, 0x29, 0xf9, 0x90, 0xf8, 0x31, 0x0b, 0x48, 0xec, 0x7f, 0x60, 0xfc, 0x9d, 0xbe, 0x41, - 0x96, 0xe4, 0x55, 0x7e, 0x53, 0xfb, 0x8f, 0x95, 0xfb, 0x17, 0xed, 0x9d, 0x28, 0xa7, 0x37, 0x06, - 0x27, 0x4f, 0xb8, 0x40, 0x4f, 0xc1, 0x11, 0xf9, 0x42, 0xbf, 0xa1, 0xed, 0xd1, 0x17, 0x1b, 0xf7, - 0xb6, 0x6e, 0x7c, 0x05, 0x7c, 0x3c, 0xcc, 0x67, 0x94, 0x6e, 0xf7, 0x5d, 0x68, 0xcf, 0xdf, 0x9e, - 0x4c, 0xfc, 0xc9, 0xf1, 0xd1, 0xf4, 0xe4, 0xac, 0x5f, 0x41, 0x7d, 0xe8, 0x8c, 0x8b, 0x96, 0xea, - 0xe3, 0xa3, 0xbc, 0x09, 0x4a, 0x84, 0xf9, 0x14, 0xbf, 0x99, 0xe2, 0x22, 0xc1, 0x5a, 0xaa, 0xc8, - 0x85, 0x3d, 0x63, 0x79, 0x39, 0x3d, 0x99, 0xe2, 0xa3, 0xb5, 0xa7, 0xf6, 0xf8, 0x2b, 0xd8, 0xb1, - 0xef, 0x12, 0x72, 0xa0, 0xf1, 0xfa, 0x64, 0x8c, 0xdf, 0xf6, 0x2b, 0xa8, 0x0b, 0xce, 0xfc, 0x0c, - 0x4f, 0xc7, 0xaf, 0x8e, 0x4e, 0x5e, 0xf6, 0xab, 0x87, 0xc3, 0x5f, 0xbf, 0x0d, 0x19, 0x0b, 0x63, - 0x3a, 0x08, 0x59, 0x4c, 0x92, 0x70, 0xc0, 0x78, 0xa8, 0xff, 0x9c, 0x87, 0xdb, 0x7f, 0xa4, 0xcf, - 0x9b, 0xfa, 0x1f, 0xfa, 0xbb, 0xbf, 0x03, 0x00, 0x00, 0xff, 0xff, 0x0d, 0x89, 0xaf, 0xc1, 0xc4, - 0x0b, 0x00, 0x00, -} diff --git a/benchmark/grpc_testing/control.proto b/benchmark/grpc_testing/control.proto deleted file mode 100644 index e9ee3484d19f..000000000000 --- a/benchmark/grpc_testing/control.proto +++ /dev/null @@ -1,188 +0,0 @@ -// Copyright 2016 gRPC authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -option go_package = "google.golang.org/grpc/benchmark/grpc_testing"; - -import "benchmark/grpc_testing/payloads.proto"; -import "benchmark/grpc_testing/stats.proto"; - -package grpc.testing; - -enum ClientType { - SYNC_CLIENT = 0; - ASYNC_CLIENT = 1; -} - -enum ServerType { - SYNC_SERVER = 0; - ASYNC_SERVER = 1; - ASYNC_GENERIC_SERVER = 2; -} - -enum RpcType { - UNARY = 0; - STREAMING = 1; -} - -// Parameters of poisson process distribution, which is a good representation -// of activity coming in from independent identical stationary sources. -message PoissonParams { - // The rate of arrivals (a.k.a. lambda parameter of the exp distribution). - double offered_load = 1; -} - -message UniformParams { - double interarrival_lo = 1; - double interarrival_hi = 2; -} - -message DeterministicParams { - double offered_load = 1; -} - -message ParetoParams { - double interarrival_base = 1; - double alpha = 2; -} - -// Once an RPC finishes, immediately start a new one. -// No configuration parameters needed. -message ClosedLoopParams { -} - -message LoadParams { - oneof load { - ClosedLoopParams closed_loop = 1; - PoissonParams poisson = 2; - UniformParams uniform = 3; - DeterministicParams determ = 4; - ParetoParams pareto = 5; - }; -} - -// presence of SecurityParams implies use of TLS -message SecurityParams { - bool use_test_ca = 1; - string server_host_override = 2; -} - -message ClientConfig { - // List of targets to connect to. At least one target needs to be specified. - repeated string server_targets = 1; - ClientType client_type = 2; - SecurityParams security_params = 3; - // How many concurrent RPCs to start for each channel. - // For synchronous client, use a separate thread for each outstanding RPC. - int32 outstanding_rpcs_per_channel = 4; - // Number of independent client channels to create. - // i-th channel will connect to server_target[i % server_targets.size()] - int32 client_channels = 5; - // Only for async client. Number of threads to use to start/manage RPCs. - int32 async_client_threads = 7; - RpcType rpc_type = 8; - // The requested load for the entire client (aggregated over all the threads). - LoadParams load_params = 10; - PayloadConfig payload_config = 11; - HistogramParams histogram_params = 12; - - // Specify the cores we should run the client on, if desired - repeated int32 core_list = 13; - int32 core_limit = 14; -} - -message ClientStatus { - ClientStats stats = 1; -} - -// Request current stats -message Mark { - // if true, the stats will be reset after taking their snapshot. - bool reset = 1; -} - -message ClientArgs { - oneof argtype { - ClientConfig setup = 1; - Mark mark = 2; - } -} - -message ServerConfig { - ServerType server_type = 1; - SecurityParams security_params = 2; - // Port on which to listen. Zero means pick unused port. - int32 port = 4; - // Only for async server. Number of threads used to serve the requests. - int32 async_server_threads = 7; - // Specify the number of cores to limit server to, if desired - int32 core_limit = 8; - // payload config, used in generic server - PayloadConfig payload_config = 9; - - // Specify the cores we should run the server on, if desired - repeated int32 core_list = 10; -} - -message ServerArgs { - oneof argtype { - ServerConfig setup = 1; - Mark mark = 2; - } -} - -message ServerStatus { - ServerStats stats = 1; - // the port bound by the server - int32 port = 2; - // Number of cores available to the server - int32 cores = 3; -} - -message CoreRequest { -} - -message CoreResponse { - // Number of cores available on the server - int32 cores = 1; -} - -message Void { -} - -// A single performance scenario: input to qps_json_driver -message Scenario { - // Human readable name for this scenario - string name = 1; - // Client configuration - ClientConfig client_config = 2; - // Number of clients to start for the test - int32 num_clients = 3; - // Server configuration - ServerConfig server_config = 4; - // Number of servers to start for the test - int32 num_servers = 5; - // Warmup period, in seconds - int32 warmup_seconds = 6; - // Benchmark time, in seconds - int32 benchmark_seconds = 7; - // Number of workers to spawn locally (usually zero) - int32 spawn_local_worker_count = 8; -} - -// A set of scenarios to be run with qps_json_driver -message Scenarios { - repeated Scenario scenarios = 1; -} diff --git a/benchmark/grpc_testing/messages.pb.go b/benchmark/grpc_testing/messages.pb.go deleted file mode 100644 index 0fddbc137fe2..000000000000 --- a/benchmark/grpc_testing/messages.pb.go +++ /dev/null @@ -1,753 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. -// source: benchmark/grpc_testing/messages.proto - -package grpc_testing - -import ( - fmt "fmt" - proto "github.com/golang/protobuf/proto" - math "math" -) - -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package - -// The type of payload that should be returned. -type PayloadType int32 - -const ( - // Compressable text format. - PayloadType_COMPRESSABLE PayloadType = 0 - // Uncompressable binary format. - PayloadType_UNCOMPRESSABLE PayloadType = 1 - // Randomly chosen from all other formats defined in this enum. - PayloadType_RANDOM PayloadType = 2 -) - -var PayloadType_name = map[int32]string{ - 0: "COMPRESSABLE", - 1: "UNCOMPRESSABLE", - 2: "RANDOM", -} - -var PayloadType_value = map[string]int32{ - "COMPRESSABLE": 0, - "UNCOMPRESSABLE": 1, - "RANDOM": 2, -} - -func (x PayloadType) String() string { - return proto.EnumName(PayloadType_name, int32(x)) -} - -func (PayloadType) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_7e3146bec7fa4346, []int{0} -} - -// Compression algorithms -type CompressionType int32 - -const ( - // No compression - CompressionType_NONE CompressionType = 0 - CompressionType_GZIP CompressionType = 1 - CompressionType_DEFLATE CompressionType = 2 -) - -var CompressionType_name = map[int32]string{ - 0: "NONE", - 1: "GZIP", - 2: "DEFLATE", -} - -var CompressionType_value = map[string]int32{ - "NONE": 0, - "GZIP": 1, - "DEFLATE": 2, -} - -func (x CompressionType) String() string { - return proto.EnumName(CompressionType_name, int32(x)) -} - -func (CompressionType) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_7e3146bec7fa4346, []int{1} -} - -// A block of data, to simply increase gRPC message size. -type Payload struct { - // The type of data in body. - Type PayloadType `protobuf:"varint,1,opt,name=type,proto3,enum=grpc.testing.PayloadType" json:"type,omitempty"` - // Primary contents of payload. - Body []byte `protobuf:"bytes,2,opt,name=body,proto3" json:"body,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *Payload) Reset() { *m = Payload{} } -func (m *Payload) String() string { return proto.CompactTextString(m) } -func (*Payload) ProtoMessage() {} -func (*Payload) Descriptor() ([]byte, []int) { - return fileDescriptor_7e3146bec7fa4346, []int{0} -} - -func (m *Payload) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Payload.Unmarshal(m, b) -} -func (m *Payload) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Payload.Marshal(b, m, deterministic) -} -func (m *Payload) XXX_Merge(src proto.Message) { - xxx_messageInfo_Payload.Merge(m, src) -} -func (m *Payload) XXX_Size() int { - return xxx_messageInfo_Payload.Size(m) -} -func (m *Payload) XXX_DiscardUnknown() { - xxx_messageInfo_Payload.DiscardUnknown(m) -} - -var xxx_messageInfo_Payload proto.InternalMessageInfo - -func (m *Payload) GetType() PayloadType { - if m != nil { - return m.Type - } - return PayloadType_COMPRESSABLE -} - -func (m *Payload) GetBody() []byte { - if m != nil { - return m.Body - } - return nil -} - -// A protobuf representation for grpc status. This is used by test -// clients to specify a status that the server should attempt to return. -type EchoStatus struct { - Code int32 `protobuf:"varint,1,opt,name=code,proto3" json:"code,omitempty"` - Message string `protobuf:"bytes,2,opt,name=message,proto3" json:"message,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *EchoStatus) Reset() { *m = EchoStatus{} } -func (m *EchoStatus) String() string { return proto.CompactTextString(m) } -func (*EchoStatus) ProtoMessage() {} -func (*EchoStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_7e3146bec7fa4346, []int{1} -} - -func (m *EchoStatus) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_EchoStatus.Unmarshal(m, b) -} -func (m *EchoStatus) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_EchoStatus.Marshal(b, m, deterministic) -} -func (m *EchoStatus) XXX_Merge(src proto.Message) { - xxx_messageInfo_EchoStatus.Merge(m, src) -} -func (m *EchoStatus) XXX_Size() int { - return xxx_messageInfo_EchoStatus.Size(m) -} -func (m *EchoStatus) XXX_DiscardUnknown() { - xxx_messageInfo_EchoStatus.DiscardUnknown(m) -} - -var xxx_messageInfo_EchoStatus proto.InternalMessageInfo - -func (m *EchoStatus) GetCode() int32 { - if m != nil { - return m.Code - } - return 0 -} - -func (m *EchoStatus) GetMessage() string { - if m != nil { - return m.Message - } - return "" -} - -// Unary request. -type SimpleRequest struct { - // Desired payload type in the response from the server. - // If response_type is RANDOM, server randomly chooses one from other formats. - ResponseType PayloadType `protobuf:"varint,1,opt,name=response_type,json=responseType,proto3,enum=grpc.testing.PayloadType" json:"response_type,omitempty"` - // Desired payload size in the response from the server. - // If response_type is COMPRESSABLE, this denotes the size before compression. - ResponseSize int32 `protobuf:"varint,2,opt,name=response_size,json=responseSize,proto3" json:"response_size,omitempty"` - // Optional input payload sent along with the request. - Payload *Payload `protobuf:"bytes,3,opt,name=payload,proto3" json:"payload,omitempty"` - // Whether SimpleResponse should include username. - FillUsername bool `protobuf:"varint,4,opt,name=fill_username,json=fillUsername,proto3" json:"fill_username,omitempty"` - // Whether SimpleResponse should include OAuth scope. - FillOauthScope bool `protobuf:"varint,5,opt,name=fill_oauth_scope,json=fillOauthScope,proto3" json:"fill_oauth_scope,omitempty"` - // Compression algorithm to be used by the server for the response (stream) - ResponseCompression CompressionType `protobuf:"varint,6,opt,name=response_compression,json=responseCompression,proto3,enum=grpc.testing.CompressionType" json:"response_compression,omitempty"` - // Whether server should return a given status - ResponseStatus *EchoStatus `protobuf:"bytes,7,opt,name=response_status,json=responseStatus,proto3" json:"response_status,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *SimpleRequest) Reset() { *m = SimpleRequest{} } -func (m *SimpleRequest) String() string { return proto.CompactTextString(m) } -func (*SimpleRequest) ProtoMessage() {} -func (*SimpleRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_7e3146bec7fa4346, []int{2} -} - -func (m *SimpleRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_SimpleRequest.Unmarshal(m, b) -} -func (m *SimpleRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_SimpleRequest.Marshal(b, m, deterministic) -} -func (m *SimpleRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_SimpleRequest.Merge(m, src) -} -func (m *SimpleRequest) XXX_Size() int { - return xxx_messageInfo_SimpleRequest.Size(m) -} -func (m *SimpleRequest) XXX_DiscardUnknown() { - xxx_messageInfo_SimpleRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_SimpleRequest proto.InternalMessageInfo - -func (m *SimpleRequest) GetResponseType() PayloadType { - if m != nil { - return m.ResponseType - } - return PayloadType_COMPRESSABLE -} - -func (m *SimpleRequest) GetResponseSize() int32 { - if m != nil { - return m.ResponseSize - } - return 0 -} - -func (m *SimpleRequest) GetPayload() *Payload { - if m != nil { - return m.Payload - } - return nil -} - -func (m *SimpleRequest) GetFillUsername() bool { - if m != nil { - return m.FillUsername - } - return false -} - -func (m *SimpleRequest) GetFillOauthScope() bool { - if m != nil { - return m.FillOauthScope - } - return false -} - -func (m *SimpleRequest) GetResponseCompression() CompressionType { - if m != nil { - return m.ResponseCompression - } - return CompressionType_NONE -} - -func (m *SimpleRequest) GetResponseStatus() *EchoStatus { - if m != nil { - return m.ResponseStatus - } - return nil -} - -// Unary response, as configured by the request. -type SimpleResponse struct { - // Payload to increase message size. - Payload *Payload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"` - // The user the request came from, for verifying authentication was - // successful when the client expected it. - Username string `protobuf:"bytes,2,opt,name=username,proto3" json:"username,omitempty"` - // OAuth scope. - OauthScope string `protobuf:"bytes,3,opt,name=oauth_scope,json=oauthScope,proto3" json:"oauth_scope,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *SimpleResponse) Reset() { *m = SimpleResponse{} } -func (m *SimpleResponse) String() string { return proto.CompactTextString(m) } -func (*SimpleResponse) ProtoMessage() {} -func (*SimpleResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_7e3146bec7fa4346, []int{3} -} - -func (m *SimpleResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_SimpleResponse.Unmarshal(m, b) -} -func (m *SimpleResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_SimpleResponse.Marshal(b, m, deterministic) -} -func (m *SimpleResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_SimpleResponse.Merge(m, src) -} -func (m *SimpleResponse) XXX_Size() int { - return xxx_messageInfo_SimpleResponse.Size(m) -} -func (m *SimpleResponse) XXX_DiscardUnknown() { - xxx_messageInfo_SimpleResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_SimpleResponse proto.InternalMessageInfo - -func (m *SimpleResponse) GetPayload() *Payload { - if m != nil { - return m.Payload - } - return nil -} - -func (m *SimpleResponse) GetUsername() string { - if m != nil { - return m.Username - } - return "" -} - -func (m *SimpleResponse) GetOauthScope() string { - if m != nil { - return m.OauthScope - } - return "" -} - -// Client-streaming request. -type StreamingInputCallRequest struct { - // Optional input payload sent along with the request. - Payload *Payload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *StreamingInputCallRequest) Reset() { *m = StreamingInputCallRequest{} } -func (m *StreamingInputCallRequest) String() string { return proto.CompactTextString(m) } -func (*StreamingInputCallRequest) ProtoMessage() {} -func (*StreamingInputCallRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_7e3146bec7fa4346, []int{4} -} - -func (m *StreamingInputCallRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_StreamingInputCallRequest.Unmarshal(m, b) -} -func (m *StreamingInputCallRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_StreamingInputCallRequest.Marshal(b, m, deterministic) -} -func (m *StreamingInputCallRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_StreamingInputCallRequest.Merge(m, src) -} -func (m *StreamingInputCallRequest) XXX_Size() int { - return xxx_messageInfo_StreamingInputCallRequest.Size(m) -} -func (m *StreamingInputCallRequest) XXX_DiscardUnknown() { - xxx_messageInfo_StreamingInputCallRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_StreamingInputCallRequest proto.InternalMessageInfo - -func (m *StreamingInputCallRequest) GetPayload() *Payload { - if m != nil { - return m.Payload - } - return nil -} - -// Client-streaming response. -type StreamingInputCallResponse struct { - // Aggregated size of payloads received from the client. - AggregatedPayloadSize int32 `protobuf:"varint,1,opt,name=aggregated_payload_size,json=aggregatedPayloadSize,proto3" json:"aggregated_payload_size,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *StreamingInputCallResponse) Reset() { *m = StreamingInputCallResponse{} } -func (m *StreamingInputCallResponse) String() string { return proto.CompactTextString(m) } -func (*StreamingInputCallResponse) ProtoMessage() {} -func (*StreamingInputCallResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_7e3146bec7fa4346, []int{5} -} - -func (m *StreamingInputCallResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_StreamingInputCallResponse.Unmarshal(m, b) -} -func (m *StreamingInputCallResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_StreamingInputCallResponse.Marshal(b, m, deterministic) -} -func (m *StreamingInputCallResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_StreamingInputCallResponse.Merge(m, src) -} -func (m *StreamingInputCallResponse) XXX_Size() int { - return xxx_messageInfo_StreamingInputCallResponse.Size(m) -} -func (m *StreamingInputCallResponse) XXX_DiscardUnknown() { - xxx_messageInfo_StreamingInputCallResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_StreamingInputCallResponse proto.InternalMessageInfo - -func (m *StreamingInputCallResponse) GetAggregatedPayloadSize() int32 { - if m != nil { - return m.AggregatedPayloadSize - } - return 0 -} - -// Configuration for a particular response. -type ResponseParameters struct { - // Desired payload sizes in responses from the server. - // If response_type is COMPRESSABLE, this denotes the size before compression. - Size int32 `protobuf:"varint,1,opt,name=size,proto3" json:"size,omitempty"` - // Desired interval between consecutive responses in the response stream in - // microseconds. - IntervalUs int32 `protobuf:"varint,2,opt,name=interval_us,json=intervalUs,proto3" json:"interval_us,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *ResponseParameters) Reset() { *m = ResponseParameters{} } -func (m *ResponseParameters) String() string { return proto.CompactTextString(m) } -func (*ResponseParameters) ProtoMessage() {} -func (*ResponseParameters) Descriptor() ([]byte, []int) { - return fileDescriptor_7e3146bec7fa4346, []int{6} -} - -func (m *ResponseParameters) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ResponseParameters.Unmarshal(m, b) -} -func (m *ResponseParameters) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ResponseParameters.Marshal(b, m, deterministic) -} -func (m *ResponseParameters) XXX_Merge(src proto.Message) { - xxx_messageInfo_ResponseParameters.Merge(m, src) -} -func (m *ResponseParameters) XXX_Size() int { - return xxx_messageInfo_ResponseParameters.Size(m) -} -func (m *ResponseParameters) XXX_DiscardUnknown() { - xxx_messageInfo_ResponseParameters.DiscardUnknown(m) -} - -var xxx_messageInfo_ResponseParameters proto.InternalMessageInfo - -func (m *ResponseParameters) GetSize() int32 { - if m != nil { - return m.Size - } - return 0 -} - -func (m *ResponseParameters) GetIntervalUs() int32 { - if m != nil { - return m.IntervalUs - } - return 0 -} - -// Server-streaming request. -type StreamingOutputCallRequest struct { - // Desired payload type in the response from the server. - // If response_type is RANDOM, the payload from each response in the stream - // might be of different types. This is to simulate a mixed type of payload - // stream. - ResponseType PayloadType `protobuf:"varint,1,opt,name=response_type,json=responseType,proto3,enum=grpc.testing.PayloadType" json:"response_type,omitempty"` - // Configuration for each expected response message. - ResponseParameters []*ResponseParameters `protobuf:"bytes,2,rep,name=response_parameters,json=responseParameters,proto3" json:"response_parameters,omitempty"` - // Optional input payload sent along with the request. - Payload *Payload `protobuf:"bytes,3,opt,name=payload,proto3" json:"payload,omitempty"` - // Compression algorithm to be used by the server for the response (stream) - ResponseCompression CompressionType `protobuf:"varint,6,opt,name=response_compression,json=responseCompression,proto3,enum=grpc.testing.CompressionType" json:"response_compression,omitempty"` - // Whether server should return a given status - ResponseStatus *EchoStatus `protobuf:"bytes,7,opt,name=response_status,json=responseStatus,proto3" json:"response_status,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *StreamingOutputCallRequest) Reset() { *m = StreamingOutputCallRequest{} } -func (m *StreamingOutputCallRequest) String() string { return proto.CompactTextString(m) } -func (*StreamingOutputCallRequest) ProtoMessage() {} -func (*StreamingOutputCallRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_7e3146bec7fa4346, []int{7} -} - -func (m *StreamingOutputCallRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_StreamingOutputCallRequest.Unmarshal(m, b) -} -func (m *StreamingOutputCallRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_StreamingOutputCallRequest.Marshal(b, m, deterministic) -} -func (m *StreamingOutputCallRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_StreamingOutputCallRequest.Merge(m, src) -} -func (m *StreamingOutputCallRequest) XXX_Size() int { - return xxx_messageInfo_StreamingOutputCallRequest.Size(m) -} -func (m *StreamingOutputCallRequest) XXX_DiscardUnknown() { - xxx_messageInfo_StreamingOutputCallRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_StreamingOutputCallRequest proto.InternalMessageInfo - -func (m *StreamingOutputCallRequest) GetResponseType() PayloadType { - if m != nil { - return m.ResponseType - } - return PayloadType_COMPRESSABLE -} - -func (m *StreamingOutputCallRequest) GetResponseParameters() []*ResponseParameters { - if m != nil { - return m.ResponseParameters - } - return nil -} - -func (m *StreamingOutputCallRequest) GetPayload() *Payload { - if m != nil { - return m.Payload - } - return nil -} - -func (m *StreamingOutputCallRequest) GetResponseCompression() CompressionType { - if m != nil { - return m.ResponseCompression - } - return CompressionType_NONE -} - -func (m *StreamingOutputCallRequest) GetResponseStatus() *EchoStatus { - if m != nil { - return m.ResponseStatus - } - return nil -} - -// Server-streaming response, as configured by the request and parameters. -type StreamingOutputCallResponse struct { - // Payload to increase response size. - Payload *Payload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *StreamingOutputCallResponse) Reset() { *m = StreamingOutputCallResponse{} } -func (m *StreamingOutputCallResponse) String() string { return proto.CompactTextString(m) } -func (*StreamingOutputCallResponse) ProtoMessage() {} -func (*StreamingOutputCallResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_7e3146bec7fa4346, []int{8} -} - -func (m *StreamingOutputCallResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_StreamingOutputCallResponse.Unmarshal(m, b) -} -func (m *StreamingOutputCallResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_StreamingOutputCallResponse.Marshal(b, m, deterministic) -} -func (m *StreamingOutputCallResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_StreamingOutputCallResponse.Merge(m, src) -} -func (m *StreamingOutputCallResponse) XXX_Size() int { - return xxx_messageInfo_StreamingOutputCallResponse.Size(m) -} -func (m *StreamingOutputCallResponse) XXX_DiscardUnknown() { - xxx_messageInfo_StreamingOutputCallResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_StreamingOutputCallResponse proto.InternalMessageInfo - -func (m *StreamingOutputCallResponse) GetPayload() *Payload { - if m != nil { - return m.Payload - } - return nil -} - -// For reconnect interop test only. -// Client tells server what reconnection parameters it used. -type ReconnectParams struct { - MaxReconnectBackoffMs int32 `protobuf:"varint,1,opt,name=max_reconnect_backoff_ms,json=maxReconnectBackoffMs,proto3" json:"max_reconnect_backoff_ms,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *ReconnectParams) Reset() { *m = ReconnectParams{} } -func (m *ReconnectParams) String() string { return proto.CompactTextString(m) } -func (*ReconnectParams) ProtoMessage() {} -func (*ReconnectParams) Descriptor() ([]byte, []int) { - return fileDescriptor_7e3146bec7fa4346, []int{9} -} - -func (m *ReconnectParams) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ReconnectParams.Unmarshal(m, b) -} -func (m *ReconnectParams) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ReconnectParams.Marshal(b, m, deterministic) -} -func (m *ReconnectParams) XXX_Merge(src proto.Message) { - xxx_messageInfo_ReconnectParams.Merge(m, src) -} -func (m *ReconnectParams) XXX_Size() int { - return xxx_messageInfo_ReconnectParams.Size(m) -} -func (m *ReconnectParams) XXX_DiscardUnknown() { - xxx_messageInfo_ReconnectParams.DiscardUnknown(m) -} - -var xxx_messageInfo_ReconnectParams proto.InternalMessageInfo - -func (m *ReconnectParams) GetMaxReconnectBackoffMs() int32 { - if m != nil { - return m.MaxReconnectBackoffMs - } - return 0 -} - -// For reconnect interop test only. -// Server tells client whether its reconnects are following the spec and the -// reconnect backoffs it saw. -type ReconnectInfo struct { - Passed bool `protobuf:"varint,1,opt,name=passed,proto3" json:"passed,omitempty"` - BackoffMs []int32 `protobuf:"varint,2,rep,packed,name=backoff_ms,json=backoffMs,proto3" json:"backoff_ms,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *ReconnectInfo) Reset() { *m = ReconnectInfo{} } -func (m *ReconnectInfo) String() string { return proto.CompactTextString(m) } -func (*ReconnectInfo) ProtoMessage() {} -func (*ReconnectInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_7e3146bec7fa4346, []int{10} -} - -func (m *ReconnectInfo) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ReconnectInfo.Unmarshal(m, b) -} -func (m *ReconnectInfo) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ReconnectInfo.Marshal(b, m, deterministic) -} -func (m *ReconnectInfo) XXX_Merge(src proto.Message) { - xxx_messageInfo_ReconnectInfo.Merge(m, src) -} -func (m *ReconnectInfo) XXX_Size() int { - return xxx_messageInfo_ReconnectInfo.Size(m) -} -func (m *ReconnectInfo) XXX_DiscardUnknown() { - xxx_messageInfo_ReconnectInfo.DiscardUnknown(m) -} - -var xxx_messageInfo_ReconnectInfo proto.InternalMessageInfo - -func (m *ReconnectInfo) GetPassed() bool { - if m != nil { - return m.Passed - } - return false -} - -func (m *ReconnectInfo) GetBackoffMs() []int32 { - if m != nil { - return m.BackoffMs - } - return nil -} - -func init() { - proto.RegisterEnum("grpc.testing.PayloadType", PayloadType_name, PayloadType_value) - proto.RegisterEnum("grpc.testing.CompressionType", CompressionType_name, CompressionType_value) - proto.RegisterType((*Payload)(nil), "grpc.testing.Payload") - proto.RegisterType((*EchoStatus)(nil), "grpc.testing.EchoStatus") - proto.RegisterType((*SimpleRequest)(nil), "grpc.testing.SimpleRequest") - proto.RegisterType((*SimpleResponse)(nil), "grpc.testing.SimpleResponse") - proto.RegisterType((*StreamingInputCallRequest)(nil), "grpc.testing.StreamingInputCallRequest") - proto.RegisterType((*StreamingInputCallResponse)(nil), "grpc.testing.StreamingInputCallResponse") - proto.RegisterType((*ResponseParameters)(nil), "grpc.testing.ResponseParameters") - proto.RegisterType((*StreamingOutputCallRequest)(nil), "grpc.testing.StreamingOutputCallRequest") - proto.RegisterType((*StreamingOutputCallResponse)(nil), "grpc.testing.StreamingOutputCallResponse") - proto.RegisterType((*ReconnectParams)(nil), "grpc.testing.ReconnectParams") - proto.RegisterType((*ReconnectInfo)(nil), "grpc.testing.ReconnectInfo") -} - -func init() { - proto.RegisterFile("benchmark/grpc_testing/messages.proto", fileDescriptor_7e3146bec7fa4346) -} - -var fileDescriptor_7e3146bec7fa4346 = []byte{ - // 690 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x55, 0x4b, 0x6f, 0xd3, 0x4c, - 0x14, 0xfd, 0x9c, 0x77, 0x6f, 0xd2, 0x34, 0x9a, 0x8f, 0x82, 0x5b, 0x54, 0x11, 0x19, 0x21, 0x45, - 0x95, 0x9a, 0x48, 0x41, 0x02, 0x89, 0x05, 0x28, 0x6d, 0x53, 0x14, 0x94, 0x26, 0xc1, 0x6e, 0x37, - 0xdd, 0x58, 0x13, 0x67, 0xe2, 0x5a, 0xb5, 0x3d, 0x66, 0x66, 0x82, 0x9a, 0x2e, 0xd8, 0xf3, 0x83, - 0xd9, 0x23, 0x8f, 0x1f, 0x71, 0xda, 0x2c, 0x5a, 0xd8, 0xb0, 0x9b, 0x39, 0xf7, 0xdc, 0xe3, 0xfb, - 0x38, 0xd6, 0xc0, 0x9b, 0x29, 0xf1, 0xad, 0x6b, 0x0f, 0xb3, 0x9b, 0x8e, 0xcd, 0x02, 0xcb, 0x14, - 0x84, 0x0b, 0xc7, 0xb7, 0x3b, 0x1e, 0xe1, 0x1c, 0xdb, 0x84, 0xb7, 0x03, 0x46, 0x05, 0x45, 0xb5, - 0x30, 0xd8, 0x8e, 0x83, 0xda, 0x10, 0xca, 0x13, 0xbc, 0x74, 0x29, 0x9e, 0xa1, 0x23, 0x28, 0x88, - 0x65, 0x40, 0x54, 0xa5, 0xa9, 0xb4, 0xea, 0xdd, 0xbd, 0x76, 0x96, 0xd7, 0x8e, 0x49, 0x17, 0xcb, - 0x80, 0xe8, 0x92, 0x86, 0x10, 0x14, 0xa6, 0x74, 0xb6, 0x54, 0x73, 0x4d, 0xa5, 0x55, 0xd3, 0xe5, - 0x59, 0xfb, 0x00, 0xd0, 0xb7, 0xae, 0xa9, 0x21, 0xb0, 0x58, 0xf0, 0x90, 0x61, 0xd1, 0x59, 0x24, - 0x58, 0xd4, 0xe5, 0x19, 0xa9, 0x50, 0x8e, 0xeb, 0x91, 0x89, 0x5b, 0x7a, 0x72, 0xd5, 0x7e, 0xe6, - 0x61, 0xdb, 0x70, 0xbc, 0xc0, 0x25, 0x3a, 0xf9, 0xb6, 0x20, 0x5c, 0xa0, 0x8f, 0xb0, 0xcd, 0x08, - 0x0f, 0xa8, 0xcf, 0x89, 0xf9, 0xb8, 0xca, 0x6a, 0x09, 0x3f, 0xbc, 0xa1, 0xd7, 0x99, 0x7c, 0xee, - 0xdc, 0x45, 0x5f, 0x2c, 0xae, 0x48, 0x86, 0x73, 0x47, 0x50, 0x07, 0xca, 0x41, 0xa4, 0xa0, 0xe6, - 0x9b, 0x4a, 0xab, 0xda, 0xdd, 0xdd, 0x28, 0xaf, 0x27, 0xac, 0x50, 0x75, 0xee, 0xb8, 0xae, 0xb9, - 0xe0, 0x84, 0xf9, 0xd8, 0x23, 0x6a, 0xa1, 0xa9, 0xb4, 0x2a, 0x7a, 0x2d, 0x04, 0x2f, 0x63, 0x0c, - 0xb5, 0xa0, 0x21, 0x49, 0x14, 0x2f, 0xc4, 0xb5, 0xc9, 0x2d, 0x1a, 0x10, 0xb5, 0x28, 0x79, 0xf5, - 0x10, 0x1f, 0x87, 0xb0, 0x11, 0xa2, 0x68, 0x02, 0xcf, 0xd2, 0x22, 0x2d, 0xea, 0x05, 0x8c, 0x70, - 0xee, 0x50, 0x5f, 0x2d, 0xc9, 0x5e, 0x0f, 0xd6, 0x8b, 0x39, 0x59, 0x11, 0x64, 0xbf, 0xff, 0x27, - 0xa9, 0x99, 0x00, 0xea, 0xc1, 0xce, 0xaa, 0x6d, 0xb9, 0x09, 0xb5, 0x2c, 0x3b, 0x53, 0xd7, 0xc5, - 0x56, 0x9b, 0xd2, 0xeb, 0xe9, 0x48, 0xe4, 0x5d, 0xfb, 0x01, 0xf5, 0x64, 0x15, 0x11, 0x9e, 0x1d, - 0x93, 0xf2, 0xa8, 0x31, 0xed, 0x43, 0x25, 0x9d, 0x50, 0xb4, 0xe9, 0xf4, 0x8e, 0x5e, 0x41, 0x35, - 0x3b, 0x98, 0xbc, 0x0c, 0x03, 0x4d, 0x87, 0xa2, 0x0d, 0x61, 0xcf, 0x10, 0x8c, 0x60, 0xcf, 0xf1, - 0xed, 0x81, 0x1f, 0x2c, 0xc4, 0x09, 0x76, 0xdd, 0xc4, 0x16, 0x4f, 0x2d, 0x45, 0xbb, 0x80, 0xfd, - 0x4d, 0x6a, 0x71, 0x67, 0xef, 0xe0, 0x05, 0xb6, 0x6d, 0x46, 0x6c, 0x2c, 0xc8, 0xcc, 0x8c, 0x73, - 0x22, 0xbf, 0x44, 0xc6, 0xdd, 0x5d, 0x85, 0x63, 0xe9, 0xd0, 0x38, 0xda, 0x00, 0x50, 0xa2, 0x31, - 0xc1, 0x0c, 0x7b, 0x44, 0x10, 0x26, 0x3d, 0x9f, 0x49, 0x95, 0xe7, 0xb0, 0x5d, 0xc7, 0x17, 0x84, - 0x7d, 0xc7, 0xa1, 0x6b, 0x62, 0x17, 0x42, 0x02, 0x5d, 0x72, 0xed, 0x57, 0x2e, 0x53, 0xe1, 0x78, - 0x21, 0xee, 0x35, 0xfc, 0xb7, 0xff, 0xc1, 0x57, 0x48, 0x7d, 0x62, 0x06, 0x69, 0xa9, 0x6a, 0xae, - 0x99, 0x6f, 0x55, 0xbb, 0xcd, 0x75, 0x95, 0x87, 0x2d, 0xe9, 0x88, 0x3d, 0x6c, 0xf3, 0xc9, 0x7f, - 0xcd, 0x3f, 0x69, 0xf3, 0x11, 0xbc, 0xdc, 0x38, 0xf6, 0x3f, 0xf4, 0xbc, 0xf6, 0x05, 0x76, 0x74, - 0x62, 0x51, 0xdf, 0x27, 0x96, 0x90, 0xc3, 0xe2, 0xe8, 0x3d, 0xa8, 0x1e, 0xbe, 0x35, 0x59, 0x02, - 0x9b, 0x53, 0x6c, 0xdd, 0xd0, 0xf9, 0xdc, 0xf4, 0x78, 0x62, 0x2f, 0x0f, 0xdf, 0xa6, 0x59, 0xc7, - 0x51, 0xf4, 0x9c, 0x6b, 0x67, 0xb0, 0x9d, 0xa2, 0x03, 0x7f, 0x4e, 0xd1, 0x73, 0x28, 0x05, 0x98, - 0x73, 0x12, 0x15, 0x53, 0xd1, 0xe3, 0x1b, 0x3a, 0x00, 0xc8, 0x68, 0x86, 0x4b, 0x2d, 0xea, 0x5b, - 0xd3, 0x44, 0xe7, 0xf0, 0x13, 0x54, 0x33, 0xce, 0x40, 0x0d, 0xa8, 0x9d, 0x8c, 0xcf, 0x27, 0x7a, - 0xdf, 0x30, 0x7a, 0xc7, 0xc3, 0x7e, 0xe3, 0x3f, 0x84, 0xa0, 0x7e, 0x39, 0x5a, 0xc3, 0x14, 0x04, - 0x50, 0xd2, 0x7b, 0xa3, 0xd3, 0xf1, 0x79, 0x23, 0x77, 0xd8, 0x85, 0x9d, 0x7b, 0xfb, 0x40, 0x15, - 0x28, 0x8c, 0xc6, 0xa3, 0x30, 0xb9, 0x02, 0x85, 0xcf, 0x57, 0x83, 0x49, 0x43, 0x41, 0x55, 0x28, - 0x9f, 0xf6, 0xcf, 0x86, 0xbd, 0x8b, 0x7e, 0x23, 0x77, 0xdc, 0xb9, 0x3a, 0xb2, 0x29, 0xb5, 0x5d, - 0xd2, 0xb6, 0xa9, 0x8b, 0x7d, 0xbb, 0x4d, 0x99, 0x2d, 0x1f, 0xa5, 0xce, 0xe6, 0x37, 0x6a, 0x5a, - 0x92, 0x6f, 0xd3, 0xdb, 0xdf, 0x01, 0x00, 0x00, 0xff, 0xff, 0xea, 0xde, 0x4b, 0x12, 0xc4, 0x06, - 0x00, 0x00, -} diff --git a/benchmark/grpc_testing/messages.proto b/benchmark/grpc_testing/messages.proto deleted file mode 100644 index c48cdae9a29e..000000000000 --- a/benchmark/grpc_testing/messages.proto +++ /dev/null @@ -1,159 +0,0 @@ -// Copyright 2016 gRPC authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Message definitions to be used by integration test service definitions. - -syntax = "proto3"; - -option go_package = "google.golang.org/grpc/benchmark/grpc_testing"; - -package grpc.testing; - -// The type of payload that should be returned. -enum PayloadType { - // Compressable text format. - COMPRESSABLE = 0; - - // Uncompressable binary format. - UNCOMPRESSABLE = 1; - - // Randomly chosen from all other formats defined in this enum. - RANDOM = 2; -} - -// Compression algorithms -enum CompressionType { - // No compression - NONE = 0; - GZIP = 1; - DEFLATE = 2; -} - -// A block of data, to simply increase gRPC message size. -message Payload { - // The type of data in body. - PayloadType type = 1; - // Primary contents of payload. - bytes body = 2; -} - -// A protobuf representation for grpc status. This is used by test -// clients to specify a status that the server should attempt to return. -message EchoStatus { - int32 code = 1; - string message = 2; -} - -// Unary request. -message SimpleRequest { - // Desired payload type in the response from the server. - // If response_type is RANDOM, server randomly chooses one from other formats. - PayloadType response_type = 1; - - // Desired payload size in the response from the server. - // If response_type is COMPRESSABLE, this denotes the size before compression. - int32 response_size = 2; - - // Optional input payload sent along with the request. - Payload payload = 3; - - // Whether SimpleResponse should include username. - bool fill_username = 4; - - // Whether SimpleResponse should include OAuth scope. - bool fill_oauth_scope = 5; - - // Compression algorithm to be used by the server for the response (stream) - CompressionType response_compression = 6; - - // Whether server should return a given status - EchoStatus response_status = 7; -} - -// Unary response, as configured by the request. -message SimpleResponse { - // Payload to increase message size. - Payload payload = 1; - // The user the request came from, for verifying authentication was - // successful when the client expected it. - string username = 2; - // OAuth scope. - string oauth_scope = 3; -} - -// Client-streaming request. -message StreamingInputCallRequest { - // Optional input payload sent along with the request. - Payload payload = 1; - - // Not expecting any payload from the response. -} - -// Client-streaming response. -message StreamingInputCallResponse { - // Aggregated size of payloads received from the client. - int32 aggregated_payload_size = 1; -} - -// Configuration for a particular response. -message ResponseParameters { - // Desired payload sizes in responses from the server. - // If response_type is COMPRESSABLE, this denotes the size before compression. - int32 size = 1; - - // Desired interval between consecutive responses in the response stream in - // microseconds. - int32 interval_us = 2; -} - -// Server-streaming request. -message StreamingOutputCallRequest { - // Desired payload type in the response from the server. - // If response_type is RANDOM, the payload from each response in the stream - // might be of different types. This is to simulate a mixed type of payload - // stream. - PayloadType response_type = 1; - - // Configuration for each expected response message. - repeated ResponseParameters response_parameters = 2; - - // Optional input payload sent along with the request. - Payload payload = 3; - - // Compression algorithm to be used by the server for the response (stream) - CompressionType response_compression = 6; - - // Whether server should return a given status - EchoStatus response_status = 7; -} - -// Server-streaming response, as configured by the request and parameters. -message StreamingOutputCallResponse { - // Payload to increase response size. - Payload payload = 1; -} - -// For reconnect interop test only. -// Client tells server what reconnection parameters it used. -message ReconnectParams { - int32 max_reconnect_backoff_ms = 1; -} - -// For reconnect interop test only. -// Server tells client whether its reconnects are following the spec and the -// reconnect backoffs it saw. -message ReconnectInfo { - bool passed = 1; - repeated int32 backoff_ms = 2; -} diff --git a/benchmark/grpc_testing/payloads.pb.go b/benchmark/grpc_testing/payloads.pb.go deleted file mode 100644 index c96b7b6c4213..000000000000 --- a/benchmark/grpc_testing/payloads.pb.go +++ /dev/null @@ -1,275 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. -// source: benchmark/grpc_testing/payloads.proto - -package grpc_testing - -import ( - fmt "fmt" - proto "github.com/golang/protobuf/proto" - math "math" -) - -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package - -type ByteBufferParams struct { - ReqSize int32 `protobuf:"varint,1,opt,name=req_size,json=reqSize,proto3" json:"req_size,omitempty"` - RespSize int32 `protobuf:"varint,2,opt,name=resp_size,json=respSize,proto3" json:"resp_size,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *ByteBufferParams) Reset() { *m = ByteBufferParams{} } -func (m *ByteBufferParams) String() string { return proto.CompactTextString(m) } -func (*ByteBufferParams) ProtoMessage() {} -func (*ByteBufferParams) Descriptor() ([]byte, []int) { - return fileDescriptor_69438956f5d73a41, []int{0} -} - -func (m *ByteBufferParams) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ByteBufferParams.Unmarshal(m, b) -} -func (m *ByteBufferParams) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ByteBufferParams.Marshal(b, m, deterministic) -} -func (m *ByteBufferParams) XXX_Merge(src proto.Message) { - xxx_messageInfo_ByteBufferParams.Merge(m, src) -} -func (m *ByteBufferParams) XXX_Size() int { - return xxx_messageInfo_ByteBufferParams.Size(m) -} -func (m *ByteBufferParams) XXX_DiscardUnknown() { - xxx_messageInfo_ByteBufferParams.DiscardUnknown(m) -} - -var xxx_messageInfo_ByteBufferParams proto.InternalMessageInfo - -func (m *ByteBufferParams) GetReqSize() int32 { - if m != nil { - return m.ReqSize - } - return 0 -} - -func (m *ByteBufferParams) GetRespSize() int32 { - if m != nil { - return m.RespSize - } - return 0 -} - -type SimpleProtoParams struct { - ReqSize int32 `protobuf:"varint,1,opt,name=req_size,json=reqSize,proto3" json:"req_size,omitempty"` - RespSize int32 `protobuf:"varint,2,opt,name=resp_size,json=respSize,proto3" json:"resp_size,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *SimpleProtoParams) Reset() { *m = SimpleProtoParams{} } -func (m *SimpleProtoParams) String() string { return proto.CompactTextString(m) } -func (*SimpleProtoParams) ProtoMessage() {} -func (*SimpleProtoParams) Descriptor() ([]byte, []int) { - return fileDescriptor_69438956f5d73a41, []int{1} -} - -func (m *SimpleProtoParams) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_SimpleProtoParams.Unmarshal(m, b) -} -func (m *SimpleProtoParams) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_SimpleProtoParams.Marshal(b, m, deterministic) -} -func (m *SimpleProtoParams) XXX_Merge(src proto.Message) { - xxx_messageInfo_SimpleProtoParams.Merge(m, src) -} -func (m *SimpleProtoParams) XXX_Size() int { - return xxx_messageInfo_SimpleProtoParams.Size(m) -} -func (m *SimpleProtoParams) XXX_DiscardUnknown() { - xxx_messageInfo_SimpleProtoParams.DiscardUnknown(m) -} - -var xxx_messageInfo_SimpleProtoParams proto.InternalMessageInfo - -func (m *SimpleProtoParams) GetReqSize() int32 { - if m != nil { - return m.ReqSize - } - return 0 -} - -func (m *SimpleProtoParams) GetRespSize() int32 { - if m != nil { - return m.RespSize - } - return 0 -} - -type ComplexProtoParams struct { - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *ComplexProtoParams) Reset() { *m = ComplexProtoParams{} } -func (m *ComplexProtoParams) String() string { return proto.CompactTextString(m) } -func (*ComplexProtoParams) ProtoMessage() {} -func (*ComplexProtoParams) Descriptor() ([]byte, []int) { - return fileDescriptor_69438956f5d73a41, []int{2} -} - -func (m *ComplexProtoParams) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ComplexProtoParams.Unmarshal(m, b) -} -func (m *ComplexProtoParams) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ComplexProtoParams.Marshal(b, m, deterministic) -} -func (m *ComplexProtoParams) XXX_Merge(src proto.Message) { - xxx_messageInfo_ComplexProtoParams.Merge(m, src) -} -func (m *ComplexProtoParams) XXX_Size() int { - return xxx_messageInfo_ComplexProtoParams.Size(m) -} -func (m *ComplexProtoParams) XXX_DiscardUnknown() { - xxx_messageInfo_ComplexProtoParams.DiscardUnknown(m) -} - -var xxx_messageInfo_ComplexProtoParams proto.InternalMessageInfo - -type PayloadConfig struct { - // Types that are valid to be assigned to Payload: - // *PayloadConfig_BytebufParams - // *PayloadConfig_SimpleParams - // *PayloadConfig_ComplexParams - Payload isPayloadConfig_Payload `protobuf_oneof:"payload"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *PayloadConfig) Reset() { *m = PayloadConfig{} } -func (m *PayloadConfig) String() string { return proto.CompactTextString(m) } -func (*PayloadConfig) ProtoMessage() {} -func (*PayloadConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_69438956f5d73a41, []int{3} -} - -func (m *PayloadConfig) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_PayloadConfig.Unmarshal(m, b) -} -func (m *PayloadConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_PayloadConfig.Marshal(b, m, deterministic) -} -func (m *PayloadConfig) XXX_Merge(src proto.Message) { - xxx_messageInfo_PayloadConfig.Merge(m, src) -} -func (m *PayloadConfig) XXX_Size() int { - return xxx_messageInfo_PayloadConfig.Size(m) -} -func (m *PayloadConfig) XXX_DiscardUnknown() { - xxx_messageInfo_PayloadConfig.DiscardUnknown(m) -} - -var xxx_messageInfo_PayloadConfig proto.InternalMessageInfo - -type isPayloadConfig_Payload interface { - isPayloadConfig_Payload() -} - -type PayloadConfig_BytebufParams struct { - BytebufParams *ByteBufferParams `protobuf:"bytes,1,opt,name=bytebuf_params,json=bytebufParams,proto3,oneof"` -} - -type PayloadConfig_SimpleParams struct { - SimpleParams *SimpleProtoParams `protobuf:"bytes,2,opt,name=simple_params,json=simpleParams,proto3,oneof"` -} - -type PayloadConfig_ComplexParams struct { - ComplexParams *ComplexProtoParams `protobuf:"bytes,3,opt,name=complex_params,json=complexParams,proto3,oneof"` -} - -func (*PayloadConfig_BytebufParams) isPayloadConfig_Payload() {} - -func (*PayloadConfig_SimpleParams) isPayloadConfig_Payload() {} - -func (*PayloadConfig_ComplexParams) isPayloadConfig_Payload() {} - -func (m *PayloadConfig) GetPayload() isPayloadConfig_Payload { - if m != nil { - return m.Payload - } - return nil -} - -func (m *PayloadConfig) GetBytebufParams() *ByteBufferParams { - if x, ok := m.GetPayload().(*PayloadConfig_BytebufParams); ok { - return x.BytebufParams - } - return nil -} - -func (m *PayloadConfig) GetSimpleParams() *SimpleProtoParams { - if x, ok := m.GetPayload().(*PayloadConfig_SimpleParams); ok { - return x.SimpleParams - } - return nil -} - -func (m *PayloadConfig) GetComplexParams() *ComplexProtoParams { - if x, ok := m.GetPayload().(*PayloadConfig_ComplexParams); ok { - return x.ComplexParams - } - return nil -} - -// XXX_OneofWrappers is for the internal use of the proto package. -func (*PayloadConfig) XXX_OneofWrappers() []interface{} { - return []interface{}{ - (*PayloadConfig_BytebufParams)(nil), - (*PayloadConfig_SimpleParams)(nil), - (*PayloadConfig_ComplexParams)(nil), - } -} - -func init() { - proto.RegisterType((*ByteBufferParams)(nil), "grpc.testing.ByteBufferParams") - proto.RegisterType((*SimpleProtoParams)(nil), "grpc.testing.SimpleProtoParams") - proto.RegisterType((*ComplexProtoParams)(nil), "grpc.testing.ComplexProtoParams") - proto.RegisterType((*PayloadConfig)(nil), "grpc.testing.PayloadConfig") -} - -func init() { - proto.RegisterFile("benchmark/grpc_testing/payloads.proto", fileDescriptor_69438956f5d73a41) -} - -var fileDescriptor_69438956f5d73a41 = []byte{ - // 289 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x91, 0xc1, 0x4b, 0xc3, 0x30, - 0x18, 0xc5, 0xd7, 0x89, 0x6e, 0xfb, 0x5c, 0x87, 0x16, 0x0f, 0x8a, 0xa0, 0xa3, 0x20, 0x78, 0x31, - 0x05, 0xfd, 0x0f, 0x3a, 0xd0, 0xa9, 0x97, 0xd2, 0xdd, 0xbc, 0x94, 0xb4, 0x7e, 0x8d, 0xc5, 0xb6, - 0xc9, 0x92, 0x0e, 0xec, 0xfe, 0x70, 0xcf, 0x92, 0x34, 0x83, 0xcd, 0x79, 0xf3, 0x9a, 0x97, 0xf7, - 0xfb, 0xde, 0xe3, 0xc1, 0x4d, 0x8a, 0x75, 0xf6, 0x51, 0x51, 0xf9, 0x19, 0x30, 0x29, 0xb2, 0xa4, - 0x41, 0xd5, 0x14, 0x35, 0x0b, 0x04, 0x6d, 0x4b, 0x4e, 0xdf, 0x15, 0x11, 0x92, 0x37, 0xdc, 0x1b, - 0x6b, 0x91, 0x58, 0xd1, 0x7f, 0x81, 0x93, 0xb0, 0x6d, 0x30, 0x5c, 0xe5, 0x39, 0xca, 0x88, 0x4a, - 0x5a, 0x29, 0xef, 0x02, 0x86, 0x12, 0x97, 0x89, 0x2a, 0xd6, 0x78, 0xee, 0x4c, 0x9d, 0xdb, 0xc3, - 0x78, 0x20, 0x71, 0xb9, 0x28, 0xd6, 0xe8, 0x5d, 0xc2, 0x48, 0xa2, 0x12, 0x9d, 0xd6, 0x37, 0xda, - 0x50, 0x3f, 0x68, 0xd1, 0x7f, 0x85, 0xd3, 0x45, 0x51, 0x89, 0x12, 0x23, 0x7d, 0xe8, 0x9f, 0xb0, - 0x33, 0xf0, 0x66, 0x5c, 0xc3, 0xbe, 0xb6, 0x68, 0xfe, 0xb7, 0x03, 0x6e, 0xd4, 0xf5, 0x99, 0xf1, - 0x3a, 0x2f, 0x98, 0xf7, 0x04, 0x93, 0xb4, 0x6d, 0x30, 0x5d, 0xe5, 0x89, 0x30, 0x7f, 0xcc, 0x95, - 0xe3, 0xfb, 0x2b, 0xb2, 0xdd, 0x93, 0xfc, 0x2e, 0x39, 0xef, 0xc5, 0xae, 0xf5, 0xd9, 0xa0, 0x8f, - 0xe0, 0x2a, 0x93, 0x7e, 0xc3, 0xe9, 0x1b, 0xce, 0xf5, 0x2e, 0x67, 0xaf, 0xe0, 0xbc, 0x17, 0x8f, - 0x3b, 0x9f, 0xe5, 0x3c, 0xc3, 0x24, 0xeb, 0x82, 0x6f, 0x40, 0x07, 0x06, 0x34, 0xdd, 0x05, 0xed, - 0x97, 0xd3, 0x91, 0xac, 0xb3, 0x7b, 0x08, 0x47, 0x30, 0xb0, 0xe3, 0x85, 0xc1, 0xdb, 0x1d, 0xe3, - 0x9c, 0x95, 0x48, 0x18, 0x2f, 0x69, 0xcd, 0x08, 0x97, 0xcc, 0xcc, 0x1c, 0xfc, 0xbd, 0x7a, 0x7a, - 0x64, 0xd6, 0x7e, 0xf8, 0x09, 0x00, 0x00, 0xff, 0xff, 0x15, 0x79, 0xad, 0xbf, 0x16, 0x02, 0x00, - 0x00, -} diff --git a/benchmark/grpc_testing/payloads.proto b/benchmark/grpc_testing/payloads.proto deleted file mode 100644 index 862fb71bc135..000000000000 --- a/benchmark/grpc_testing/payloads.proto +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2016 gRPC authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -option go_package = "google.golang.org/grpc/benchmark/grpc_testing"; - -package grpc.testing; - -message ByteBufferParams { - int32 req_size = 1; - int32 resp_size = 2; -} - -message SimpleProtoParams { - int32 req_size = 1; - int32 resp_size = 2; -} - -message ComplexProtoParams { - // TODO (vpai): Fill this in once the details of complex, representative - // protos are decided -} - -message PayloadConfig { - oneof payload { - ByteBufferParams bytebuf_params = 1; - SimpleProtoParams simple_params = 2; - ComplexProtoParams complex_params = 3; - } -} diff --git a/benchmark/grpc_testing/services.pb.go b/benchmark/grpc_testing/services.pb.go deleted file mode 100644 index 4d0acd35d4fb..000000000000 --- a/benchmark/grpc_testing/services.pb.go +++ /dev/null @@ -1,49 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. -// source: benchmark/grpc_testing/services.proto - -package grpc_testing - -import ( - fmt "fmt" - proto "github.com/golang/protobuf/proto" - math "math" -) - -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package - -func init() { - proto.RegisterFile("benchmark/grpc_testing/services.proto", fileDescriptor_e86b6b5d31c265e4) -} - -var fileDescriptor_e86b6b5d31c265e4 = []byte{ - // 309 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x92, 0xc1, 0x4a, 0x3b, 0x31, - 0x10, 0xc6, 0x69, 0x0f, 0x7f, 0x68, 0xf8, 0x17, 0x24, 0x27, 0x8d, 0x1e, 0x15, 0xbc, 0xb8, 0x2b, - 0xd5, 0x17, 0xb0, 0x8b, 0x1e, 0x05, 0xbb, 0x54, 0x41, 0x0f, 0x92, 0x6e, 0x87, 0x18, 0x9a, 0x9d, - 0x59, 0x27, 0xb3, 0x82, 0x4f, 0xe0, 0x23, 0xf8, 0xba, 0xd2, 0xdd, 0x56, 0x6a, 0xd9, 0x9e, 0xf4, - 0x98, 0xf9, 0xbe, 0xf9, 0x4d, 0x26, 0xf9, 0xd4, 0xc9, 0x0c, 0xb0, 0x78, 0x29, 0x2d, 0x2f, 0x52, - 0xc7, 0x55, 0xf1, 0x2c, 0x10, 0xc5, 0xa3, 0x4b, 0x23, 0xf0, 0x9b, 0x2f, 0x20, 0x26, 0x15, 0x93, - 0x90, 0xfe, 0xbf, 0x14, 0x93, 0x95, 0x68, 0x76, 0x35, 0x95, 0x10, 0xa3, 0x75, 0xeb, 0x26, 0x73, - 0xbc, 0xc3, 0x56, 0x10, 0x0a, 0x53, 0x68, 0x5d, 0xa3, 0x8f, 0xbe, 0xda, 0x1b, 0xaf, 0x8d, 0x79, - 0x3b, 0x56, 0xdf, 0xa8, 0xc1, 0x14, 0x2d, 0xbf, 0x67, 0x36, 0x04, 0x7d, 0x98, 0x6c, 0x4e, 0x4f, - 0x72, 0x5f, 0x56, 0x01, 0x26, 0xf0, 0x5a, 0x43, 0x14, 0x73, 0xd4, 0x2d, 0xc6, 0x8a, 0x30, 0x82, - 0xbe, 0x55, 0xc3, 0x5c, 0x18, 0x6c, 0xe9, 0xd1, 0xfd, 0x92, 0x75, 0xda, 0x3b, 0xef, 0xe9, 0x27, - 0x65, 0xa6, 0x58, 0x10, 0x46, 0x61, 0xeb, 0x11, 0xe6, 0x7f, 0x09, 0x1f, 0x7d, 0xf6, 0xd5, 0xf0, - 0x81, 0x78, 0x01, 0xbc, 0x7e, 0x86, 0x6b, 0x35, 0x98, 0xd4, 0xb8, 0x3c, 0x01, 0xeb, 0xfd, 0x2d, - 0x40, 0x53, 0xbd, 0x62, 0x17, 0x8d, 0xe9, 0x52, 0x72, 0xb1, 0x52, 0xc7, 0xe6, 0xd6, 0x2d, 0x26, - 0x0b, 0x1e, 0x50, 0xb6, 0x31, 0x6d, 0xb5, 0x0b, 0xd3, 0x2a, 0x1b, 0x98, 0xb1, 0x1a, 0x64, 0xc4, - 0x90, 0x51, 0x8d, 0xa2, 0x0f, 0xb6, 0xcc, 0xc4, 0xdf, 0x9b, 0x9a, 0x2e, 0x69, 0xf5, 0x21, 0x97, - 0x4a, 0xdd, 0xd5, 0x5e, 0xda, 0x35, 0xb5, 0xfe, 0xe9, 0xbc, 0x27, 0x3f, 0x37, 0x1d, 0xb5, 0x71, - 0xfa, 0x78, 0xe6, 0x88, 0x5c, 0x80, 0xc4, 0x51, 0xb0, 0xe8, 0x12, 0x62, 0xd7, 0x64, 0x2a, 0xed, - 0x8e, 0xd8, 0xec, 0x5f, 0x93, 0xad, 0x8b, 0xaf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x77, 0x96, 0x9e, - 0xdb, 0xdf, 0x02, 0x00, 0x00, -} diff --git a/benchmark/grpc_testing/services.proto b/benchmark/grpc_testing/services.proto deleted file mode 100644 index 9028c9cfe113..000000000000 --- a/benchmark/grpc_testing/services.proto +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright 2016 gRPC authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// An integration test service that covers all the method signature permutations -// of unary/streaming requests/responses. -syntax = "proto3"; - -option go_package = "google.golang.org/grpc/benchmark/grpc_testing"; - -import "benchmark/grpc_testing/messages.proto"; -import "benchmark/grpc_testing/control.proto"; - -package grpc.testing; - -service BenchmarkService { - // One request followed by one response. - // The server returns the client payload as-is. - rpc UnaryCall(SimpleRequest) returns (SimpleResponse); - - // One request followed by one response. - // The server returns the client payload as-is. - rpc StreamingCall(stream SimpleRequest) returns (stream SimpleResponse); - // Unconstrainted streaming. - // Both server and client keep sending & receiving simultaneously. - rpc UnconstrainedStreamingCall(stream SimpleRequest) returns (stream SimpleResponse); -} - -service WorkerService { - // Start server with specified workload. - // First request sent specifies the ServerConfig followed by ServerStatus - // response. After that, a "Mark" can be sent anytime to request the latest - // stats. Closing the stream will initiate shutdown of the test server - // and once the shutdown has finished, the OK status is sent to terminate - // this RPC. - rpc RunServer(stream ServerArgs) returns (stream ServerStatus); - - // Start client with specified workload. - // First request sent specifies the ClientConfig followed by ClientStatus - // response. After that, a "Mark" can be sent anytime to request the latest - // stats. Closing the stream will initiate shutdown of the test client - // and once the shutdown has finished, the OK status is sent to terminate - // this RPC. - rpc RunClient(stream ClientArgs) returns (stream ClientStatus); - - // Just return the core count - unary call - rpc CoreCount(CoreRequest) returns (CoreResponse); - - // Quit this worker - rpc QuitWorker(Void) returns (Void); -} diff --git a/benchmark/grpc_testing/services_grpc.pb.go b/benchmark/grpc_testing/services_grpc.pb.go deleted file mode 100644 index 60be6cd689a2..000000000000 --- a/benchmark/grpc_testing/services_grpc.pb.go +++ /dev/null @@ -1,633 +0,0 @@ -// Code generated by protoc-gen-go-grpc. DO NOT EDIT. - -package grpc_testing - -import ( - context "context" - grpc "google.golang.org/grpc" - codes "google.golang.org/grpc/codes" - status "google.golang.org/grpc/status" -) - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the grpc package it is being compiled against. -const _ = grpc.SupportPackageIsVersion7 - -// BenchmarkServiceClient is the client API for BenchmarkService service. -// -// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. -type BenchmarkServiceClient interface { - // One request followed by one response. - // The server returns the client payload as-is. - UnaryCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (*SimpleResponse, error) - // One request followed by one response. - // The server returns the client payload as-is. - StreamingCall(ctx context.Context, opts ...grpc.CallOption) (BenchmarkService_StreamingCallClient, error) - // Unconstrainted streaming. - // Both server and client keep sending & receiving simultaneously. - UnconstrainedStreamingCall(ctx context.Context, opts ...grpc.CallOption) (BenchmarkService_UnconstrainedStreamingCallClient, error) -} - -type benchmarkServiceClient struct { - cc grpc.ClientConnInterface -} - -func NewBenchmarkServiceClient(cc grpc.ClientConnInterface) BenchmarkServiceClient { - return &benchmarkServiceClient{cc} -} - -var benchmarkServiceUnaryCallStreamDesc = &grpc.StreamDesc{ - StreamName: "UnaryCall", -} - -func (c *benchmarkServiceClient) UnaryCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (*SimpleResponse, error) { - out := new(SimpleResponse) - err := c.cc.Invoke(ctx, "/grpc.testing.BenchmarkService/UnaryCall", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -var benchmarkServiceStreamingCallStreamDesc = &grpc.StreamDesc{ - StreamName: "StreamingCall", - ServerStreams: true, - ClientStreams: true, -} - -func (c *benchmarkServiceClient) StreamingCall(ctx context.Context, opts ...grpc.CallOption) (BenchmarkService_StreamingCallClient, error) { - stream, err := c.cc.NewStream(ctx, benchmarkServiceStreamingCallStreamDesc, "/grpc.testing.BenchmarkService/StreamingCall", opts...) - if err != nil { - return nil, err - } - x := &benchmarkServiceStreamingCallClient{stream} - return x, nil -} - -type BenchmarkService_StreamingCallClient interface { - Send(*SimpleRequest) error - Recv() (*SimpleResponse, error) - grpc.ClientStream -} - -type benchmarkServiceStreamingCallClient struct { - grpc.ClientStream -} - -func (x *benchmarkServiceStreamingCallClient) Send(m *SimpleRequest) error { - return x.ClientStream.SendMsg(m) -} - -func (x *benchmarkServiceStreamingCallClient) Recv() (*SimpleResponse, error) { - m := new(SimpleResponse) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -var benchmarkServiceUnconstrainedStreamingCallStreamDesc = &grpc.StreamDesc{ - StreamName: "UnconstrainedStreamingCall", - ServerStreams: true, - ClientStreams: true, -} - -func (c *benchmarkServiceClient) UnconstrainedStreamingCall(ctx context.Context, opts ...grpc.CallOption) (BenchmarkService_UnconstrainedStreamingCallClient, error) { - stream, err := c.cc.NewStream(ctx, benchmarkServiceUnconstrainedStreamingCallStreamDesc, "/grpc.testing.BenchmarkService/UnconstrainedStreamingCall", opts...) - if err != nil { - return nil, err - } - x := &benchmarkServiceUnconstrainedStreamingCallClient{stream} - return x, nil -} - -type BenchmarkService_UnconstrainedStreamingCallClient interface { - Send(*SimpleRequest) error - Recv() (*SimpleResponse, error) - grpc.ClientStream -} - -type benchmarkServiceUnconstrainedStreamingCallClient struct { - grpc.ClientStream -} - -func (x *benchmarkServiceUnconstrainedStreamingCallClient) Send(m *SimpleRequest) error { - return x.ClientStream.SendMsg(m) -} - -func (x *benchmarkServiceUnconstrainedStreamingCallClient) Recv() (*SimpleResponse, error) { - m := new(SimpleResponse) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -// BenchmarkServiceService is the service API for BenchmarkService service. -// Fields should be assigned to their respective handler implementations only before -// RegisterBenchmarkServiceService is called. Any unassigned fields will result in the -// handler for that method returning an Unimplemented error. -type BenchmarkServiceService struct { - // One request followed by one response. - // The server returns the client payload as-is. - UnaryCall func(context.Context, *SimpleRequest) (*SimpleResponse, error) - // One request followed by one response. - // The server returns the client payload as-is. - StreamingCall func(BenchmarkService_StreamingCallServer) error - // Unconstrainted streaming. - // Both server and client keep sending & receiving simultaneously. - UnconstrainedStreamingCall func(BenchmarkService_UnconstrainedStreamingCallServer) error -} - -func (s *BenchmarkServiceService) unaryCall(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - if s.UnaryCall == nil { - return nil, status.Errorf(codes.Unimplemented, "method UnaryCall not implemented") - } - in := new(SimpleRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return s.UnaryCall(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: s, - FullMethod: "/grpc.testing.BenchmarkService/UnaryCall", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return s.UnaryCall(ctx, req.(*SimpleRequest)) - } - return interceptor(ctx, in, info, handler) -} -func (s *BenchmarkServiceService) streamingCall(_ interface{}, stream grpc.ServerStream) error { - if s.StreamingCall == nil { - return status.Errorf(codes.Unimplemented, "method StreamingCall not implemented") - } - return s.StreamingCall(&benchmarkServiceStreamingCallServer{stream}) -} -func (s *BenchmarkServiceService) unconstrainedStreamingCall(_ interface{}, stream grpc.ServerStream) error { - if s.UnconstrainedStreamingCall == nil { - return status.Errorf(codes.Unimplemented, "method UnconstrainedStreamingCall not implemented") - } - return s.UnconstrainedStreamingCall(&benchmarkServiceUnconstrainedStreamingCallServer{stream}) -} - -type BenchmarkService_StreamingCallServer interface { - Send(*SimpleResponse) error - Recv() (*SimpleRequest, error) - grpc.ServerStream -} - -type benchmarkServiceStreamingCallServer struct { - grpc.ServerStream -} - -func (x *benchmarkServiceStreamingCallServer) Send(m *SimpleResponse) error { - return x.ServerStream.SendMsg(m) -} - -func (x *benchmarkServiceStreamingCallServer) Recv() (*SimpleRequest, error) { - m := new(SimpleRequest) - if err := x.ServerStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -type BenchmarkService_UnconstrainedStreamingCallServer interface { - Send(*SimpleResponse) error - Recv() (*SimpleRequest, error) - grpc.ServerStream -} - -type benchmarkServiceUnconstrainedStreamingCallServer struct { - grpc.ServerStream -} - -func (x *benchmarkServiceUnconstrainedStreamingCallServer) Send(m *SimpleResponse) error { - return x.ServerStream.SendMsg(m) -} - -func (x *benchmarkServiceUnconstrainedStreamingCallServer) Recv() (*SimpleRequest, error) { - m := new(SimpleRequest) - if err := x.ServerStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -// RegisterBenchmarkServiceService registers a service implementation with a gRPC server. -func RegisterBenchmarkServiceService(s grpc.ServiceRegistrar, srv *BenchmarkServiceService) { - sd := grpc.ServiceDesc{ - ServiceName: "grpc.testing.BenchmarkService", - Methods: []grpc.MethodDesc{ - { - MethodName: "UnaryCall", - Handler: srv.unaryCall, - }, - }, - Streams: []grpc.StreamDesc{ - { - StreamName: "StreamingCall", - Handler: srv.streamingCall, - ServerStreams: true, - ClientStreams: true, - }, - { - StreamName: "UnconstrainedStreamingCall", - Handler: srv.unconstrainedStreamingCall, - ServerStreams: true, - ClientStreams: true, - }, - }, - Metadata: "benchmark/grpc_testing/services.proto", - } - - s.RegisterService(&sd, nil) -} - -// NewBenchmarkServiceService creates a new BenchmarkServiceService containing the -// implemented methods of the BenchmarkService service in s. Any unimplemented -// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client. -// This includes situations where the method handler is misspelled or has the wrong -// signature. For this reason, this function should be used with great care and -// is not recommended to be used by most users. -func NewBenchmarkServiceService(s interface{}) *BenchmarkServiceService { - ns := &BenchmarkServiceService{} - if h, ok := s.(interface { - UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error) - }); ok { - ns.UnaryCall = h.UnaryCall - } - if h, ok := s.(interface { - StreamingCall(BenchmarkService_StreamingCallServer) error - }); ok { - ns.StreamingCall = h.StreamingCall - } - if h, ok := s.(interface { - UnconstrainedStreamingCall(BenchmarkService_UnconstrainedStreamingCallServer) error - }); ok { - ns.UnconstrainedStreamingCall = h.UnconstrainedStreamingCall - } - return ns -} - -// UnstableBenchmarkServiceService is the service API for BenchmarkService service. -// New methods may be added to this interface if they are added to the service -// definition, which is not a backward-compatible change. For this reason, -// use of this type is not recommended. -type UnstableBenchmarkServiceService interface { - // One request followed by one response. - // The server returns the client payload as-is. - UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error) - // One request followed by one response. - // The server returns the client payload as-is. - StreamingCall(BenchmarkService_StreamingCallServer) error - // Unconstrainted streaming. - // Both server and client keep sending & receiving simultaneously. - UnconstrainedStreamingCall(BenchmarkService_UnconstrainedStreamingCallServer) error -} - -// WorkerServiceClient is the client API for WorkerService service. -// -// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. -type WorkerServiceClient interface { - // Start server with specified workload. - // First request sent specifies the ServerConfig followed by ServerStatus - // response. After that, a "Mark" can be sent anytime to request the latest - // stats. Closing the stream will initiate shutdown of the test server - // and once the shutdown has finished, the OK status is sent to terminate - // this RPC. - RunServer(ctx context.Context, opts ...grpc.CallOption) (WorkerService_RunServerClient, error) - // Start client with specified workload. - // First request sent specifies the ClientConfig followed by ClientStatus - // response. After that, a "Mark" can be sent anytime to request the latest - // stats. Closing the stream will initiate shutdown of the test client - // and once the shutdown has finished, the OK status is sent to terminate - // this RPC. - RunClient(ctx context.Context, opts ...grpc.CallOption) (WorkerService_RunClientClient, error) - // Just return the core count - unary call - CoreCount(ctx context.Context, in *CoreRequest, opts ...grpc.CallOption) (*CoreResponse, error) - // Quit this worker - QuitWorker(ctx context.Context, in *Void, opts ...grpc.CallOption) (*Void, error) -} - -type workerServiceClient struct { - cc grpc.ClientConnInterface -} - -func NewWorkerServiceClient(cc grpc.ClientConnInterface) WorkerServiceClient { - return &workerServiceClient{cc} -} - -var workerServiceRunServerStreamDesc = &grpc.StreamDesc{ - StreamName: "RunServer", - ServerStreams: true, - ClientStreams: true, -} - -func (c *workerServiceClient) RunServer(ctx context.Context, opts ...grpc.CallOption) (WorkerService_RunServerClient, error) { - stream, err := c.cc.NewStream(ctx, workerServiceRunServerStreamDesc, "/grpc.testing.WorkerService/RunServer", opts...) - if err != nil { - return nil, err - } - x := &workerServiceRunServerClient{stream} - return x, nil -} - -type WorkerService_RunServerClient interface { - Send(*ServerArgs) error - Recv() (*ServerStatus, error) - grpc.ClientStream -} - -type workerServiceRunServerClient struct { - grpc.ClientStream -} - -func (x *workerServiceRunServerClient) Send(m *ServerArgs) error { - return x.ClientStream.SendMsg(m) -} - -func (x *workerServiceRunServerClient) Recv() (*ServerStatus, error) { - m := new(ServerStatus) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -var workerServiceRunClientStreamDesc = &grpc.StreamDesc{ - StreamName: "RunClient", - ServerStreams: true, - ClientStreams: true, -} - -func (c *workerServiceClient) RunClient(ctx context.Context, opts ...grpc.CallOption) (WorkerService_RunClientClient, error) { - stream, err := c.cc.NewStream(ctx, workerServiceRunClientStreamDesc, "/grpc.testing.WorkerService/RunClient", opts...) - if err != nil { - return nil, err - } - x := &workerServiceRunClientClient{stream} - return x, nil -} - -type WorkerService_RunClientClient interface { - Send(*ClientArgs) error - Recv() (*ClientStatus, error) - grpc.ClientStream -} - -type workerServiceRunClientClient struct { - grpc.ClientStream -} - -func (x *workerServiceRunClientClient) Send(m *ClientArgs) error { - return x.ClientStream.SendMsg(m) -} - -func (x *workerServiceRunClientClient) Recv() (*ClientStatus, error) { - m := new(ClientStatus) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -var workerServiceCoreCountStreamDesc = &grpc.StreamDesc{ - StreamName: "CoreCount", -} - -func (c *workerServiceClient) CoreCount(ctx context.Context, in *CoreRequest, opts ...grpc.CallOption) (*CoreResponse, error) { - out := new(CoreResponse) - err := c.cc.Invoke(ctx, "/grpc.testing.WorkerService/CoreCount", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -var workerServiceQuitWorkerStreamDesc = &grpc.StreamDesc{ - StreamName: "QuitWorker", -} - -func (c *workerServiceClient) QuitWorker(ctx context.Context, in *Void, opts ...grpc.CallOption) (*Void, error) { - out := new(Void) - err := c.cc.Invoke(ctx, "/grpc.testing.WorkerService/QuitWorker", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -// WorkerServiceService is the service API for WorkerService service. -// Fields should be assigned to their respective handler implementations only before -// RegisterWorkerServiceService is called. Any unassigned fields will result in the -// handler for that method returning an Unimplemented error. -type WorkerServiceService struct { - // Start server with specified workload. - // First request sent specifies the ServerConfig followed by ServerStatus - // response. After that, a "Mark" can be sent anytime to request the latest - // stats. Closing the stream will initiate shutdown of the test server - // and once the shutdown has finished, the OK status is sent to terminate - // this RPC. - RunServer func(WorkerService_RunServerServer) error - // Start client with specified workload. - // First request sent specifies the ClientConfig followed by ClientStatus - // response. After that, a "Mark" can be sent anytime to request the latest - // stats. Closing the stream will initiate shutdown of the test client - // and once the shutdown has finished, the OK status is sent to terminate - // this RPC. - RunClient func(WorkerService_RunClientServer) error - // Just return the core count - unary call - CoreCount func(context.Context, *CoreRequest) (*CoreResponse, error) - // Quit this worker - QuitWorker func(context.Context, *Void) (*Void, error) -} - -func (s *WorkerServiceService) runServer(_ interface{}, stream grpc.ServerStream) error { - if s.RunServer == nil { - return status.Errorf(codes.Unimplemented, "method RunServer not implemented") - } - return s.RunServer(&workerServiceRunServerServer{stream}) -} -func (s *WorkerServiceService) runClient(_ interface{}, stream grpc.ServerStream) error { - if s.RunClient == nil { - return status.Errorf(codes.Unimplemented, "method RunClient not implemented") - } - return s.RunClient(&workerServiceRunClientServer{stream}) -} -func (s *WorkerServiceService) coreCount(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - if s.CoreCount == nil { - return nil, status.Errorf(codes.Unimplemented, "method CoreCount not implemented") - } - in := new(CoreRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return s.CoreCount(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: s, - FullMethod: "/grpc.testing.WorkerService/CoreCount", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return s.CoreCount(ctx, req.(*CoreRequest)) - } - return interceptor(ctx, in, info, handler) -} -func (s *WorkerServiceService) quitWorker(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - if s.QuitWorker == nil { - return nil, status.Errorf(codes.Unimplemented, "method QuitWorker not implemented") - } - in := new(Void) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return s.QuitWorker(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: s, - FullMethod: "/grpc.testing.WorkerService/QuitWorker", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return s.QuitWorker(ctx, req.(*Void)) - } - return interceptor(ctx, in, info, handler) -} - -type WorkerService_RunServerServer interface { - Send(*ServerStatus) error - Recv() (*ServerArgs, error) - grpc.ServerStream -} - -type workerServiceRunServerServer struct { - grpc.ServerStream -} - -func (x *workerServiceRunServerServer) Send(m *ServerStatus) error { - return x.ServerStream.SendMsg(m) -} - -func (x *workerServiceRunServerServer) Recv() (*ServerArgs, error) { - m := new(ServerArgs) - if err := x.ServerStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -type WorkerService_RunClientServer interface { - Send(*ClientStatus) error - Recv() (*ClientArgs, error) - grpc.ServerStream -} - -type workerServiceRunClientServer struct { - grpc.ServerStream -} - -func (x *workerServiceRunClientServer) Send(m *ClientStatus) error { - return x.ServerStream.SendMsg(m) -} - -func (x *workerServiceRunClientServer) Recv() (*ClientArgs, error) { - m := new(ClientArgs) - if err := x.ServerStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -// RegisterWorkerServiceService registers a service implementation with a gRPC server. -func RegisterWorkerServiceService(s grpc.ServiceRegistrar, srv *WorkerServiceService) { - sd := grpc.ServiceDesc{ - ServiceName: "grpc.testing.WorkerService", - Methods: []grpc.MethodDesc{ - { - MethodName: "CoreCount", - Handler: srv.coreCount, - }, - { - MethodName: "QuitWorker", - Handler: srv.quitWorker, - }, - }, - Streams: []grpc.StreamDesc{ - { - StreamName: "RunServer", - Handler: srv.runServer, - ServerStreams: true, - ClientStreams: true, - }, - { - StreamName: "RunClient", - Handler: srv.runClient, - ServerStreams: true, - ClientStreams: true, - }, - }, - Metadata: "benchmark/grpc_testing/services.proto", - } - - s.RegisterService(&sd, nil) -} - -// NewWorkerServiceService creates a new WorkerServiceService containing the -// implemented methods of the WorkerService service in s. Any unimplemented -// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client. -// This includes situations where the method handler is misspelled or has the wrong -// signature. For this reason, this function should be used with great care and -// is not recommended to be used by most users. -func NewWorkerServiceService(s interface{}) *WorkerServiceService { - ns := &WorkerServiceService{} - if h, ok := s.(interface { - RunServer(WorkerService_RunServerServer) error - }); ok { - ns.RunServer = h.RunServer - } - if h, ok := s.(interface { - RunClient(WorkerService_RunClientServer) error - }); ok { - ns.RunClient = h.RunClient - } - if h, ok := s.(interface { - CoreCount(context.Context, *CoreRequest) (*CoreResponse, error) - }); ok { - ns.CoreCount = h.CoreCount - } - if h, ok := s.(interface { - QuitWorker(context.Context, *Void) (*Void, error) - }); ok { - ns.QuitWorker = h.QuitWorker - } - return ns -} - -// UnstableWorkerServiceService is the service API for WorkerService service. -// New methods may be added to this interface if they are added to the service -// definition, which is not a backward-compatible change. For this reason, -// use of this type is not recommended. -type UnstableWorkerServiceService interface { - // Start server with specified workload. - // First request sent specifies the ServerConfig followed by ServerStatus - // response. After that, a "Mark" can be sent anytime to request the latest - // stats. Closing the stream will initiate shutdown of the test server - // and once the shutdown has finished, the OK status is sent to terminate - // this RPC. - RunServer(WorkerService_RunServerServer) error - // Start client with specified workload. - // First request sent specifies the ClientConfig followed by ClientStatus - // response. After that, a "Mark" can be sent anytime to request the latest - // stats. Closing the stream will initiate shutdown of the test client - // and once the shutdown has finished, the OK status is sent to terminate - // this RPC. - RunClient(WorkerService_RunClientServer) error - // Just return the core count - unary call - CoreCount(context.Context, *CoreRequest) (*CoreResponse, error) - // Quit this worker - QuitWorker(context.Context, *Void) (*Void, error) -} diff --git a/benchmark/grpc_testing/stats.pb.go b/benchmark/grpc_testing/stats.pb.go deleted file mode 100644 index ebe310c41dcc..000000000000 --- a/benchmark/grpc_testing/stats.pb.go +++ /dev/null @@ -1,312 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. -// source: benchmark/grpc_testing/stats.proto - -package grpc_testing - -import ( - fmt "fmt" - proto "github.com/golang/protobuf/proto" - math "math" -) - -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package - -type ServerStats struct { - // wall clock time change in seconds since last reset - TimeElapsed float64 `protobuf:"fixed64,1,opt,name=time_elapsed,json=timeElapsed,proto3" json:"time_elapsed,omitempty"` - // change in user time (in seconds) used by the server since last reset - TimeUser float64 `protobuf:"fixed64,2,opt,name=time_user,json=timeUser,proto3" json:"time_user,omitempty"` - // change in server time (in seconds) used by the server process and all - // threads since last reset - TimeSystem float64 `protobuf:"fixed64,3,opt,name=time_system,json=timeSystem,proto3" json:"time_system,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *ServerStats) Reset() { *m = ServerStats{} } -func (m *ServerStats) String() string { return proto.CompactTextString(m) } -func (*ServerStats) ProtoMessage() {} -func (*ServerStats) Descriptor() ([]byte, []int) { - return fileDescriptor_a0658d5f374c86d6, []int{0} -} - -func (m *ServerStats) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ServerStats.Unmarshal(m, b) -} -func (m *ServerStats) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ServerStats.Marshal(b, m, deterministic) -} -func (m *ServerStats) XXX_Merge(src proto.Message) { - xxx_messageInfo_ServerStats.Merge(m, src) -} -func (m *ServerStats) XXX_Size() int { - return xxx_messageInfo_ServerStats.Size(m) -} -func (m *ServerStats) XXX_DiscardUnknown() { - xxx_messageInfo_ServerStats.DiscardUnknown(m) -} - -var xxx_messageInfo_ServerStats proto.InternalMessageInfo - -func (m *ServerStats) GetTimeElapsed() float64 { - if m != nil { - return m.TimeElapsed - } - return 0 -} - -func (m *ServerStats) GetTimeUser() float64 { - if m != nil { - return m.TimeUser - } - return 0 -} - -func (m *ServerStats) GetTimeSystem() float64 { - if m != nil { - return m.TimeSystem - } - return 0 -} - -// Histogram params based on grpc/support/histogram.c -type HistogramParams struct { - Resolution float64 `protobuf:"fixed64,1,opt,name=resolution,proto3" json:"resolution,omitempty"` - MaxPossible float64 `protobuf:"fixed64,2,opt,name=max_possible,json=maxPossible,proto3" json:"max_possible,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *HistogramParams) Reset() { *m = HistogramParams{} } -func (m *HistogramParams) String() string { return proto.CompactTextString(m) } -func (*HistogramParams) ProtoMessage() {} -func (*HistogramParams) Descriptor() ([]byte, []int) { - return fileDescriptor_a0658d5f374c86d6, []int{1} -} - -func (m *HistogramParams) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_HistogramParams.Unmarshal(m, b) -} -func (m *HistogramParams) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_HistogramParams.Marshal(b, m, deterministic) -} -func (m *HistogramParams) XXX_Merge(src proto.Message) { - xxx_messageInfo_HistogramParams.Merge(m, src) -} -func (m *HistogramParams) XXX_Size() int { - return xxx_messageInfo_HistogramParams.Size(m) -} -func (m *HistogramParams) XXX_DiscardUnknown() { - xxx_messageInfo_HistogramParams.DiscardUnknown(m) -} - -var xxx_messageInfo_HistogramParams proto.InternalMessageInfo - -func (m *HistogramParams) GetResolution() float64 { - if m != nil { - return m.Resolution - } - return 0 -} - -func (m *HistogramParams) GetMaxPossible() float64 { - if m != nil { - return m.MaxPossible - } - return 0 -} - -// Histogram data based on grpc/support/histogram.c -type HistogramData struct { - Bucket []uint32 `protobuf:"varint,1,rep,packed,name=bucket,proto3" json:"bucket,omitempty"` - MinSeen float64 `protobuf:"fixed64,2,opt,name=min_seen,json=minSeen,proto3" json:"min_seen,omitempty"` - MaxSeen float64 `protobuf:"fixed64,3,opt,name=max_seen,json=maxSeen,proto3" json:"max_seen,omitempty"` - Sum float64 `protobuf:"fixed64,4,opt,name=sum,proto3" json:"sum,omitempty"` - SumOfSquares float64 `protobuf:"fixed64,5,opt,name=sum_of_squares,json=sumOfSquares,proto3" json:"sum_of_squares,omitempty"` - Count float64 `protobuf:"fixed64,6,opt,name=count,proto3" json:"count,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *HistogramData) Reset() { *m = HistogramData{} } -func (m *HistogramData) String() string { return proto.CompactTextString(m) } -func (*HistogramData) ProtoMessage() {} -func (*HistogramData) Descriptor() ([]byte, []int) { - return fileDescriptor_a0658d5f374c86d6, []int{2} -} - -func (m *HistogramData) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_HistogramData.Unmarshal(m, b) -} -func (m *HistogramData) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_HistogramData.Marshal(b, m, deterministic) -} -func (m *HistogramData) XXX_Merge(src proto.Message) { - xxx_messageInfo_HistogramData.Merge(m, src) -} -func (m *HistogramData) XXX_Size() int { - return xxx_messageInfo_HistogramData.Size(m) -} -func (m *HistogramData) XXX_DiscardUnknown() { - xxx_messageInfo_HistogramData.DiscardUnknown(m) -} - -var xxx_messageInfo_HistogramData proto.InternalMessageInfo - -func (m *HistogramData) GetBucket() []uint32 { - if m != nil { - return m.Bucket - } - return nil -} - -func (m *HistogramData) GetMinSeen() float64 { - if m != nil { - return m.MinSeen - } - return 0 -} - -func (m *HistogramData) GetMaxSeen() float64 { - if m != nil { - return m.MaxSeen - } - return 0 -} - -func (m *HistogramData) GetSum() float64 { - if m != nil { - return m.Sum - } - return 0 -} - -func (m *HistogramData) GetSumOfSquares() float64 { - if m != nil { - return m.SumOfSquares - } - return 0 -} - -func (m *HistogramData) GetCount() float64 { - if m != nil { - return m.Count - } - return 0 -} - -type ClientStats struct { - // Latency histogram. Data points are in nanoseconds. - Latencies *HistogramData `protobuf:"bytes,1,opt,name=latencies,proto3" json:"latencies,omitempty"` - // See ServerStats for details. - TimeElapsed float64 `protobuf:"fixed64,2,opt,name=time_elapsed,json=timeElapsed,proto3" json:"time_elapsed,omitempty"` - TimeUser float64 `protobuf:"fixed64,3,opt,name=time_user,json=timeUser,proto3" json:"time_user,omitempty"` - TimeSystem float64 `protobuf:"fixed64,4,opt,name=time_system,json=timeSystem,proto3" json:"time_system,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *ClientStats) Reset() { *m = ClientStats{} } -func (m *ClientStats) String() string { return proto.CompactTextString(m) } -func (*ClientStats) ProtoMessage() {} -func (*ClientStats) Descriptor() ([]byte, []int) { - return fileDescriptor_a0658d5f374c86d6, []int{3} -} - -func (m *ClientStats) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ClientStats.Unmarshal(m, b) -} -func (m *ClientStats) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ClientStats.Marshal(b, m, deterministic) -} -func (m *ClientStats) XXX_Merge(src proto.Message) { - xxx_messageInfo_ClientStats.Merge(m, src) -} -func (m *ClientStats) XXX_Size() int { - return xxx_messageInfo_ClientStats.Size(m) -} -func (m *ClientStats) XXX_DiscardUnknown() { - xxx_messageInfo_ClientStats.DiscardUnknown(m) -} - -var xxx_messageInfo_ClientStats proto.InternalMessageInfo - -func (m *ClientStats) GetLatencies() *HistogramData { - if m != nil { - return m.Latencies - } - return nil -} - -func (m *ClientStats) GetTimeElapsed() float64 { - if m != nil { - return m.TimeElapsed - } - return 0 -} - -func (m *ClientStats) GetTimeUser() float64 { - if m != nil { - return m.TimeUser - } - return 0 -} - -func (m *ClientStats) GetTimeSystem() float64 { - if m != nil { - return m.TimeSystem - } - return 0 -} - -func init() { - proto.RegisterType((*ServerStats)(nil), "grpc.testing.ServerStats") - proto.RegisterType((*HistogramParams)(nil), "grpc.testing.HistogramParams") - proto.RegisterType((*HistogramData)(nil), "grpc.testing.HistogramData") - proto.RegisterType((*ClientStats)(nil), "grpc.testing.ClientStats") -} - -func init() { - proto.RegisterFile("benchmark/grpc_testing/stats.proto", fileDescriptor_a0658d5f374c86d6) -} - -var fileDescriptor_a0658d5f374c86d6 = []byte{ - // 376 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x92, 0xcf, 0x0b, 0xd3, 0x30, - 0x14, 0xc7, 0xe9, 0xba, 0xcd, 0xed, 0x75, 0x53, 0x09, 0x22, 0x95, 0x81, 0xce, 0xe2, 0x61, 0x17, - 0x5b, 0xd0, 0x93, 0x57, 0x7f, 0x80, 0x37, 0xc7, 0xaa, 0x17, 0x2f, 0x25, 0xad, 0x6f, 0x31, 0xac, - 0x49, 0x6a, 0x5e, 0x22, 0xf5, 0x4f, 0x12, 0xff, 0x49, 0x69, 0x5a, 0x74, 0xfe, 0x40, 0x6f, 0xcd, - 0xe7, 0xf3, 0x25, 0x69, 0xf2, 0x7d, 0x90, 0xd5, 0xa8, 0x9b, 0x8f, 0x8a, 0xdb, 0x4b, 0x21, 0x6c, - 0xd7, 0x54, 0x0e, 0xc9, 0x49, 0x2d, 0x0a, 0x72, 0xdc, 0x51, 0xde, 0x59, 0xe3, 0x0c, 0xdb, 0x0c, - 0x26, 0x9f, 0x4c, 0xa6, 0x21, 0x29, 0xd1, 0x7e, 0x46, 0x5b, 0x0e, 0x11, 0xf6, 0x10, 0x36, 0x4e, - 0x2a, 0xac, 0xb0, 0xe5, 0x1d, 0xe1, 0x87, 0x34, 0xda, 0x47, 0x87, 0xe8, 0x94, 0x0c, 0xec, 0xd5, - 0x88, 0xd8, 0x0e, 0xd6, 0x21, 0xe2, 0x09, 0x6d, 0x3a, 0x0b, 0x7e, 0x35, 0x80, 0x77, 0x84, 0x96, - 0x3d, 0x80, 0x90, 0xad, 0xe8, 0x0b, 0x39, 0x54, 0x69, 0x1c, 0x34, 0x0c, 0xa8, 0x0c, 0x24, 0x7b, - 0x0b, 0xb7, 0x5e, 0x4b, 0x72, 0x46, 0x58, 0xae, 0x8e, 0xdc, 0x72, 0x45, 0xec, 0x3e, 0x80, 0x45, - 0x32, 0xad, 0x77, 0xd2, 0xe8, 0xe9, 0xc4, 0x2b, 0x32, 0xfc, 0x93, 0xe2, 0x7d, 0xd5, 0x19, 0x22, - 0x59, 0xb7, 0x38, 0x9d, 0x99, 0x28, 0xde, 0x1f, 0x27, 0x94, 0x7d, 0x8b, 0x60, 0xfb, 0x63, 0xdb, - 0x97, 0xdc, 0x71, 0x76, 0x17, 0x96, 0xb5, 0x6f, 0x2e, 0xe8, 0xd2, 0x68, 0x1f, 0x1f, 0xb6, 0xa7, - 0x69, 0xc5, 0xee, 0xc1, 0x4a, 0x49, 0x5d, 0x11, 0xa2, 0x9e, 0x36, 0xba, 0xa1, 0xa4, 0x2e, 0x11, - 0x75, 0x50, 0xbc, 0x1f, 0x55, 0x3c, 0x29, 0xde, 0x07, 0x75, 0x1b, 0x62, 0xf2, 0x2a, 0x9d, 0x07, - 0x3a, 0x7c, 0xb2, 0x47, 0x70, 0x93, 0xbc, 0xaa, 0xcc, 0xb9, 0xa2, 0x4f, 0x9e, 0x5b, 0xa4, 0x74, - 0x11, 0xe4, 0x86, 0xbc, 0x7a, 0x73, 0x2e, 0x47, 0xc6, 0xee, 0xc0, 0xa2, 0x31, 0x5e, 0xbb, 0x74, - 0x19, 0xe4, 0xb8, 0xc8, 0xbe, 0x46, 0x90, 0xbc, 0x68, 0x25, 0x6a, 0x37, 0x3e, 0xfa, 0x33, 0x58, - 0xb7, 0xdc, 0xa1, 0x6e, 0x24, 0x52, 0xb8, 0x7f, 0xf2, 0x64, 0x97, 0x5f, 0xb7, 0x94, 0xff, 0x72, - 0xb7, 0xd3, 0xcf, 0xf4, 0x1f, 0x7d, 0xcd, 0xfe, 0xd3, 0x57, 0xfc, 0xef, 0xbe, 0xe6, 0xbf, 0xf7, - 0xf5, 0xbc, 0x78, 0xff, 0x58, 0x18, 0x23, 0x5a, 0xcc, 0x85, 0x69, 0xb9, 0x16, 0xb9, 0xb1, 0x22, - 0xcc, 0x56, 0xf1, 0xf7, 0x51, 0xab, 0x97, 0x61, 0xca, 0x9e, 0x7e, 0x0f, 0x00, 0x00, 0xff, 0xff, - 0xcc, 0xda, 0x99, 0x65, 0x8b, 0x02, 0x00, 0x00, -} diff --git a/benchmark/grpc_testing/stats.proto b/benchmark/grpc_testing/stats.proto deleted file mode 100644 index 1517e7f6d2ef..000000000000 --- a/benchmark/grpc_testing/stats.proto +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright 2016 gRPC authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -option go_package = "google.golang.org/grpc/benchmark/grpc_testing"; - -package grpc.testing; - -message ServerStats { - // wall clock time change in seconds since last reset - double time_elapsed = 1; - - // change in user time (in seconds) used by the server since last reset - double time_user = 2; - - // change in server time (in seconds) used by the server process and all - // threads since last reset - double time_system = 3; -} - -// Histogram params based on grpc/support/histogram.c -message HistogramParams { - double resolution = 1; // first bucket is [0, 1 + resolution) - double max_possible = 2; // use enough buckets to allow this value -} - -// Histogram data based on grpc/support/histogram.c -message HistogramData { - repeated uint32 bucket = 1; - double min_seen = 2; - double max_seen = 3; - double sum = 4; - double sum_of_squares = 5; - double count = 6; -} - -message ClientStats { - // Latency histogram. Data points are in nanoseconds. - HistogramData latencies = 1; - - // See ServerStats for details. - double time_elapsed = 2; - double time_user = 3; - double time_system = 4; -} diff --git a/benchmark/latency/latency_test.go b/benchmark/latency/latency_test.go index 5d08b90b4fa2..787373ca30be 100644 --- a/benchmark/latency/latency_test.go +++ b/benchmark/latency/latency_test.go @@ -86,7 +86,7 @@ func (s) TestConn(t *testing.T) { wantSleeps(latency) // Connection creation delay. // 1 kbps = 128 Bps. Divides evenly by 1 second using nanos. - byteLatency := time.Duration(time.Second / 128) + byteLatency := time.Second / 128 write := func(b []byte) { n, err := c.Write(b) diff --git a/benchmark/primitives/code_string_test.go b/benchmark/primitives/code_string_test.go index 51b1ee48cf3c..095d0045c188 100644 --- a/benchmark/primitives/code_string_test.go +++ b/benchmark/primitives/code_string_test.go @@ -87,49 +87,39 @@ func (i codeBench) StringUsingMap() string { } func BenchmarkCodeStringStringer(b *testing.B) { - b.ResetTimer() for i := 0; i < b.N; i++ { c := codeBench(uint32(i % 17)) _ = c.String() } - b.StopTimer() } func BenchmarkCodeStringMap(b *testing.B) { - b.ResetTimer() for i := 0; i < b.N; i++ { c := codeBench(uint32(i % 17)) _ = c.StringUsingMap() } - b.StopTimer() } // codes.Code.String() does a switch. func BenchmarkCodeStringSwitch(b *testing.B) { - b.ResetTimer() for i := 0; i < b.N; i++ { c := codes.Code(uint32(i % 17)) _ = c.String() } - b.StopTimer() } // Testing all codes (0<=c<=16) and also one overflow (17). func BenchmarkCodeStringStringerWithOverflow(b *testing.B) { - b.ResetTimer() for i := 0; i < b.N; i++ { c := codeBench(uint32(i % 18)) _ = c.String() } - b.StopTimer() } // Testing all codes (0<=c<=16) and also one overflow (17). func BenchmarkCodeStringSwitchWithOverflow(b *testing.B) { - b.ResetTimer() for i := 0; i < b.N; i++ { c := codes.Code(uint32(i % 18)) _ = c.String() } - b.StopTimer() } diff --git a/benchmark/primitives/context_test.go b/benchmark/primitives/context_test.go index c94acd74597c..faae50703e7d 100644 --- a/benchmark/primitives/context_test.go +++ b/benchmark/primitives/context_test.go @@ -24,6 +24,8 @@ import ( "time" ) +const defaultTestTimeout = 10 * time.Second + func BenchmarkCancelContextErrNoErr(b *testing.B) { ctx, cancel := context.WithCancel(context.Background()) for i := 0; i < b.N; i++ { @@ -72,7 +74,7 @@ func BenchmarkCancelContextChannelGotErr(b *testing.B) { } func BenchmarkTimerContextErrNoErr(b *testing.B) { - ctx, cancel := context.WithTimeout(context.Background(), 24*time.Hour) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) for i := 0; i < b.N; i++ { if err := ctx.Err(); err != nil { b.Fatal("error") @@ -92,7 +94,7 @@ func BenchmarkTimerContextErrGotErr(b *testing.B) { } func BenchmarkTimerContextChannelNoErr(b *testing.B) { - ctx, cancel := context.WithTimeout(context.Background(), 24*time.Hour) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) for i := 0; i < b.N; i++ { select { case <-ctx.Done(): diff --git a/benchmark/primitives/primitives_test.go b/benchmark/primitives/primitives_test.go index 71fc26e29418..dbbb313e8dcc 100644 --- a/benchmark/primitives/primitives_test.go +++ b/benchmark/primitives/primitives_test.go @@ -367,7 +367,7 @@ func BenchmarkInterfaceTypeAssertion(b *testing.B) { runInterfaceTypeAssertion(b, myFooer{}) } -func runInterfaceTypeAssertion(b *testing.B, fer interface{}) { +func runInterfaceTypeAssertion(b *testing.B, fer any) { x := 0 b.ResetTimer() for i := 0; i < b.N; i++ { @@ -386,7 +386,7 @@ func BenchmarkStructTypeAssertion(b *testing.B) { runStructTypeAssertion(b, myFooer{}) } -func runStructTypeAssertion(b *testing.B, fer interface{}) { +func runStructTypeAssertion(b *testing.B, fer any) { x := 0 b.ResetTimer() for i := 0; i < b.N; i++ { @@ -399,3 +399,29 @@ func runStructTypeAssertion(b *testing.B, fer interface{}) { b.Fatal("error") } } + +func BenchmarkWaitGroupAddDone(b *testing.B) { + wg := sync.WaitGroup{} + b.RunParallel(func(pb *testing.PB) { + i := 0 + for ; pb.Next(); i++ { + wg.Add(1) + } + for ; i > 0; i-- { + wg.Done() + } + }) +} + +func BenchmarkRLockUnlock(b *testing.B) { + mu := sync.RWMutex{} + b.RunParallel(func(pb *testing.PB) { + i := 0 + for ; pb.Next(); i++ { + mu.RLock() + } + for ; i > 0; i-- { + mu.RUnlock() + } + }) +} diff --git a/benchmark/primitives/safe_config_selector_test.go b/benchmark/primitives/safe_config_selector_test.go new file mode 100644 index 000000000000..3d368d8d7cf3 --- /dev/null +++ b/benchmark/primitives/safe_config_selector_test.go @@ -0,0 +1,114 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Benchmark options for safe config selector type. + +package primitives_test + +import ( + "sync" + "sync/atomic" + "testing" + "time" + "unsafe" +) + +type safeUpdaterAtomicAndCounter struct { + ptr unsafe.Pointer // *countingFunc +} + +type countingFunc struct { + mu sync.RWMutex + f func() +} + +func (s *safeUpdaterAtomicAndCounter) call() { + cfPtr := atomic.LoadPointer(&s.ptr) + var cf *countingFunc + for { + cf = (*countingFunc)(cfPtr) + cf.mu.RLock() + cfPtr2 := atomic.LoadPointer(&s.ptr) + if cfPtr == cfPtr2 { + // Use cf with confidence! + break + } + // cf changed; try to use the new one instead, because the old one is + // no longer valid to use. + cf.mu.RUnlock() + cfPtr = cfPtr2 + } + defer cf.mu.RUnlock() + cf.f() +} + +func (s *safeUpdaterAtomicAndCounter) update(f func()) { + newCF := &countingFunc{f: f} + oldCFPtr := atomic.SwapPointer(&s.ptr, unsafe.Pointer(newCF)) + if oldCFPtr == nil { + return + } + (*countingFunc)(oldCFPtr).mu.Lock() + (*countingFunc)(oldCFPtr).mu.Unlock() //lint:ignore SA2001 necessary to unlock after locking to unblock any RLocks +} + +type safeUpdaterRWMutex struct { + mu sync.RWMutex + f func() +} + +func (s *safeUpdaterRWMutex) call() { + s.mu.RLock() + defer s.mu.RUnlock() + s.f() +} + +func (s *safeUpdaterRWMutex) update(f func()) { + s.mu.Lock() + defer s.mu.Unlock() + s.f = f +} + +type updater interface { + call() + update(f func()) +} + +func benchmarkSafeUpdater(b *testing.B, u updater) { + t := time.NewTicker(time.Second) + go func() { + for range t.C { + u.update(func() {}) + } + }() + b.RunParallel(func(pb *testing.PB) { + u.update(func() {}) + for pb.Next() { + u.call() + } + }) + t.Stop() +} + +func BenchmarkSafeUpdaterAtomicAndCounter(b *testing.B) { + benchmarkSafeUpdater(b, &safeUpdaterAtomicAndCounter{}) +} + +func BenchmarkSafeUpdaterRWMutex(b *testing.B) { + benchmarkSafeUpdater(b, &safeUpdaterRWMutex{}) +} diff --git a/benchmark/server/main.go b/benchmark/server/main.go index 5a82b1c78012..1093ee9eb56b 100644 --- a/benchmark/server/main.go +++ b/benchmark/server/main.go @@ -20,6 +20,7 @@ Package main provides a server used for benchmarking. It launches a server which is listening on port 50051. An example to start the server can be found at: + go run benchmark/server/main.go -test_name=grpc_test After starting the server, the client can be run separately and used to test @@ -38,6 +39,7 @@ import ( "runtime/pprof" "time" + "google.golang.org/grpc" "google.golang.org/grpc/benchmark" "google.golang.org/grpc/grpclog" "google.golang.org/grpc/internal/syscall" @@ -53,7 +55,7 @@ var ( func main() { flag.Parse() if *testName == "" { - logger.Fatalf("test name not set") + logger.Fatal("-test_name not set") } lis, err := net.Listen("tcp", ":"+*port) if err != nil { @@ -69,7 +71,10 @@ func main() { pprof.StartCPUProfile(cf) cpuBeg := syscall.GetCPUTime() // Launch server in a separate goroutine. - stop := benchmark.StartServer(benchmark.ServerInfo{Type: "protobuf", Listener: lis}) + stop := benchmark.StartServer(benchmark.ServerInfo{Type: "protobuf", Listener: lis}, + grpc.WriteBufferSize(128*1024), + grpc.ReadBufferSize(128*1024), + ) // Wait on OS terminate signal. ch := make(chan os.Signal, 1) signal.Notify(ch, os.Interrupt) diff --git a/benchmark/stats/curve.go b/benchmark/stats/curve.go index 68821bcc2690..124183dac2ea 100644 --- a/benchmark/stats/curve.go +++ b/benchmark/stats/curve.go @@ -23,7 +23,6 @@ import ( "encoding/csv" "encoding/hex" "fmt" - "io/ioutil" "math" "math/rand" "os" @@ -81,7 +80,7 @@ func (pcr *payloadCurveRange) chooseRandom() int { // sha256file is a helper function that returns a hex string matching the // SHA-256 sum of the input file. func sha256file(file string) (string, error) { - data, err := ioutil.ReadFile(file) + data, err := os.ReadFile(file) if err != nil { return "", err } diff --git a/benchmark/stats/histogram.go b/benchmark/stats/histogram.go index f038d26ed0aa..461135f0125c 100644 --- a/benchmark/stats/histogram.go +++ b/benchmark/stats/histogram.go @@ -118,10 +118,6 @@ func (h *Histogram) PrintWithUnit(w io.Writer, unit float64) { } maxBucketDigitLen := len(strconv.FormatFloat(h.Buckets[len(h.Buckets)-1].LowBound, 'f', 6, 64)) - if maxBucketDigitLen < 3 { - // For "inf". - maxBucketDigitLen = 3 - } maxCountDigitLen := len(strconv.FormatInt(h.Count, 10)) percentMulti := 100 / float64(h.Count) @@ -131,9 +127,9 @@ func (h *Histogram) PrintWithUnit(w io.Writer, unit float64) { if i+1 < len(h.Buckets) { fmt.Fprintf(w, "%*f)", maxBucketDigitLen, h.Buckets[i+1].LowBound/unit) } else { - fmt.Fprintf(w, "%*s)", maxBucketDigitLen, "inf") + upperBound := float64(h.opts.MinValue) + (b.LowBound-float64(h.opts.MinValue))*(1.0+h.opts.GrowthFactor) + fmt.Fprintf(w, "%*f)", maxBucketDigitLen, upperBound/unit) } - accCount += b.Count fmt.Fprintf(w, " %*d %5.1f%% %5.1f%%", maxCountDigitLen, b.Count, float64(b.Count)*percentMulti, float64(accCount)*percentMulti) @@ -188,6 +184,9 @@ func (h *Histogram) Add(value int64) error { func (h *Histogram) findBucket(value int64) (int, error) { delta := float64(value - h.opts.MinValue) + if delta < 0 { + return 0, fmt.Errorf("no bucket for value: %d", value) + } var b int if delta >= h.opts.BaseBucketSize { // b = log_{1+growthFactor} (delta / baseBucketSize) + 1 diff --git a/benchmark/stats/stats.go b/benchmark/stats/stats.go index 6275c3c3a71c..e42c5b6c0f24 100644 --- a/benchmark/stats/stats.go +++ b/benchmark/stats/stats.go @@ -52,6 +52,13 @@ const ( CompModesIndex EnableChannelzIndex EnablePreloaderIndex + ClientReadBufferSize + ClientWriteBufferSize + ServerReadBufferSize + ServerWriteBufferSize + SleepBetweenRPCs + RecvBufferPool + SharedWriteBuffer // MaxFeatureIndex is a place holder to indicate the total number of feature // indices we have. Any new feature indices should be added above this. @@ -74,6 +81,8 @@ type Features struct { EnableKeepalive bool // BenchTime indicates the duration of the benchmark run. BenchTime time.Duration + // Connections configures the number of grpc connections between client and server. + Connections int // Features defined above are usually the same for all benchmark runs in a // particular invocation, while the features defined below could vary from @@ -109,6 +118,20 @@ type Features struct { EnableChannelz bool // EnablePreloader indicates if preloading was turned on. EnablePreloader bool + // ClientReadBufferSize is the size of the client read buffer in bytes. If negative, use the default buffer size. + ClientReadBufferSize int + // ClientWriteBufferSize is the size of the client write buffer in bytes. If negative, use the default buffer size. + ClientWriteBufferSize int + // ServerReadBufferSize is the size of the server read buffer in bytes. If negative, use the default buffer size. + ServerReadBufferSize int + // ServerWriteBufferSize is the size of the server write buffer in bytes. If negative, use the default buffer size. + ServerWriteBufferSize int + // SleepBetweenRPCs configures optional delay between RPCs. + SleepBetweenRPCs time.Duration + // RecvBufferPool represents the shared recv buffer pool used. + RecvBufferPool string + // SharedWriteBuffer configures whether both client and server share per-connection write buffer + SharedWriteBuffer bool } // String returns all the feature values as a string. @@ -126,10 +149,15 @@ func (f Features) String() string { } return fmt.Sprintf("networkMode_%v-bufConn_%v-keepalive_%v-benchTime_%v-"+ "trace_%v-latency_%v-kbps_%v-MTU_%v-maxConcurrentCalls_%v-%s-%s-"+ - "compressor_%v-channelz_%v-preloader_%v", + "compressor_%v-channelz_%v-preloader_%v-clientReadBufferSize_%v-"+ + "clientWriteBufferSize_%v-serverReadBufferSize_%v-serverWriteBufferSize_%v-"+ + "sleepBetweenRPCs_%v-connections_%v-recvBufferPool_%v-sharedWriteBuffer_%v", f.NetworkMode, f.UseBufConn, f.EnableKeepalive, f.BenchTime, f.EnableTrace, f.Latency, f.Kbps, f.MTU, f.MaxConcurrentCalls, reqPayloadString, - respPayloadString, f.ModeCompressor, f.EnableChannelz, f.EnablePreloader) + respPayloadString, f.ModeCompressor, f.EnableChannelz, f.EnablePreloader, + f.ClientReadBufferSize, f.ClientWriteBufferSize, f.ServerReadBufferSize, + f.ServerWriteBufferSize, f.SleepBetweenRPCs, f.Connections, + f.RecvBufferPool, f.SharedWriteBuffer) } // SharedFeatures returns the shared features as a pretty printable string. @@ -193,6 +221,20 @@ func (f Features) partialString(b *bytes.Buffer, wantFeatures []bool, sep, delim b.WriteString(fmt.Sprintf("Channelz%v%v%v", sep, f.EnableChannelz, delim)) case EnablePreloaderIndex: b.WriteString(fmt.Sprintf("Preloader%v%v%v", sep, f.EnablePreloader, delim)) + case ClientReadBufferSize: + b.WriteString(fmt.Sprintf("ClientReadBufferSize%v%v%v", sep, f.ClientReadBufferSize, delim)) + case ClientWriteBufferSize: + b.WriteString(fmt.Sprintf("ClientWriteBufferSize%v%v%v", sep, f.ClientWriteBufferSize, delim)) + case ServerReadBufferSize: + b.WriteString(fmt.Sprintf("ServerReadBufferSize%v%v%v", sep, f.ServerReadBufferSize, delim)) + case ServerWriteBufferSize: + b.WriteString(fmt.Sprintf("ServerWriteBufferSize%v%v%v", sep, f.ServerWriteBufferSize, delim)) + case SleepBetweenRPCs: + b.WriteString(fmt.Sprintf("SleepBetweenRPCs%v%v%v", sep, f.SleepBetweenRPCs, delim)) + case RecvBufferPool: + b.WriteString(fmt.Sprintf("RecvBufferPool%v%v%v", sep, f.RecvBufferPool, delim)) + case SharedWriteBuffer: + b.WriteString(fmt.Sprintf("SharedWriteBuffer%v%v%v", sep, f.SharedWriteBuffer, delim)) default: log.Fatalf("Unknown feature index %v. maxFeatureIndex is %v", i, MaxFeatureIndex) } diff --git a/benchmark/worker/benchmark_client.go b/benchmark/worker/benchmark_client.go index 3fa682d5a77e..bd8a6e9ee49f 100644 --- a/benchmark/worker/benchmark_client.go +++ b/benchmark/worker/benchmark_client.go @@ -28,13 +28,19 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/benchmark" - testpb "google.golang.org/grpc/benchmark/grpc_testing" "google.golang.org/grpc/benchmark/stats" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/grpcrand" "google.golang.org/grpc/internal/syscall" "google.golang.org/grpc/status" "google.golang.org/grpc/testdata" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + + _ "google.golang.org/grpc/xds" // To install the xds resolvers and balancers. ) var caFile = flag.String("ca_file", "", "The file containing the CA root cert file") @@ -110,7 +116,10 @@ func setupClientEnv(config *testpb.ClientConfig) { // It returns the connections and corresponding function to close them. // It returns non-nil error if there is anything wrong. func createConns(config *testpb.ClientConfig) ([]*grpc.ClientConn, func(), error) { - var opts []grpc.DialOption + opts := []grpc.DialOption{ + grpc.WithWriteBufferSize(128 * 1024), + grpc.WithReadBufferSize(128 * 1024), + } // Sanity check for client type. switch config.ClientType { @@ -123,15 +132,15 @@ func createConns(config *testpb.ClientConfig) ([]*grpc.ClientConn, func(), error // Check and set security options. if config.SecurityParams != nil { if *caFile == "" { - *caFile = testdata.Path("x509/server_ca.pem") + *caFile = testdata.Path("ca.pem") } creds, err := credentials.NewClientTLSFromFile(*caFile, config.SecurityParams.ServerHostOverride) if err != nil { - return nil, nil, status.Errorf(codes.InvalidArgument, "failed to create TLS credentials %v", err) + return nil, nil, status.Errorf(codes.InvalidArgument, "failed to create TLS credentials: %v", err) } opts = append(opts, grpc.WithTransportCredentials(creds)) } else { - opts = append(opts, grpc.WithInsecure()) + opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) } // Use byteBufCodec if it is required. @@ -180,11 +189,21 @@ func performRPCs(config *testpb.ClientConfig, conns []*grpc.ClientConn, bc *benc } } - // TODO add open loop distribution. - switch config.LoadParams.Load.(type) { + // If set, perform an open loop, if not perform a closed loop. An open loop + // asynchronously starts RPCs based on random start times derived from a + // Poisson distribution. A closed loop performs RPCs in a blocking manner, + // and runs the next RPC after the previous RPC completes and returns. + var poissonLambda *float64 + switch t := config.LoadParams.Load.(type) { case *testpb.LoadParams_ClosedLoop: case *testpb.LoadParams_Poisson: - return status.Errorf(codes.Unimplemented, "unsupported load params: %v", config.LoadParams) + if t.Poisson == nil { + return status.Errorf(codes.InvalidArgument, "poisson is nil, needs to be set") + } + if t.Poisson.OfferedLoad <= 0 { + return status.Errorf(codes.InvalidArgument, "poisson.offered is <= 0: %v, needs to be >0", t.Poisson.OfferedLoad) + } + poissonLambda = &t.Poisson.OfferedLoad default: return status.Errorf(codes.InvalidArgument, "unknown load params: %v", config.LoadParams) } @@ -193,11 +212,9 @@ func performRPCs(config *testpb.ClientConfig, conns []*grpc.ClientConn, bc *benc switch config.RpcType { case testpb.RpcType_UNARY: - bc.doCloseLoopUnary(conns, rpcCountPerConn, payloadReqSize, payloadRespSize) - // TODO open loop. + bc.unaryLoop(conns, rpcCountPerConn, payloadReqSize, payloadRespSize, poissonLambda) case testpb.RpcType_STREAMING: - bc.doCloseLoopStreaming(conns, rpcCountPerConn, payloadReqSize, payloadRespSize, payloadType) - // TODO open loop. + bc.streamingLoop(conns, rpcCountPerConn, payloadReqSize, payloadRespSize, payloadType, poissonLambda) default: return status.Errorf(codes.InvalidArgument, "unknown rpc type: %v", config.RpcType) } @@ -241,9 +258,9 @@ func startBenchmarkClient(config *testpb.ClientConfig) (*benchmarkClient, error) return bc, nil } -func (bc *benchmarkClient) doCloseLoopUnary(conns []*grpc.ClientConn, rpcCountPerConn int, reqSize int, respSize int) { +func (bc *benchmarkClient) unaryLoop(conns []*grpc.ClientConn, rpcCountPerConn int, reqSize int, respSize int, poissonLambda *float64) { for ic, conn := range conns { - client := testpb.NewBenchmarkServiceClient(conn) + client := testgrpc.NewBenchmarkServiceClient(conn) // For each connection, create rpcCountPerConn goroutines to do rpc. for j := 0; j < rpcCountPerConn; j++ { // Create histogram for each goroutine. @@ -255,37 +272,45 @@ func (bc *benchmarkClient) doCloseLoopUnary(conns []*grpc.ClientConn, rpcCountPe // Now relying on worker client to reserve time to do warm up. // The worker client needs to wait for some time after client is created, // before starting benchmark. - done := make(chan bool) - for { - go func() { - start := time.Now() - if err := benchmark.DoUnaryCall(client, reqSize, respSize); err != nil { + if poissonLambda == nil { // Closed loop. + done := make(chan bool) + for { + go func() { + start := time.Now() + if err := benchmark.DoUnaryCall(client, reqSize, respSize); err != nil { + select { + case <-bc.stop: + case done <- false: + } + return + } + elapse := time.Since(start) + bc.lockingHistograms[idx].add(int64(elapse)) select { case <-bc.stop: - case done <- false: + case done <- true: } - return - } - elapse := time.Since(start) - bc.lockingHistograms[idx].add(int64(elapse)) + }() select { case <-bc.stop: - case done <- true: + return + case <-done: } - }() - select { - case <-bc.stop: - return - case <-done: } + } else { // Open loop. + timeBetweenRPCs := time.Duration((grpcrand.ExpFloat64() / *poissonLambda) * float64(time.Second)) + time.AfterFunc(timeBetweenRPCs, func() { + bc.poissonUnary(client, idx, reqSize, respSize, *poissonLambda) + }) } + }(idx) } } } -func (bc *benchmarkClient) doCloseLoopStreaming(conns []*grpc.ClientConn, rpcCountPerConn int, reqSize int, respSize int, payloadType string) { - var doRPC func(testpb.BenchmarkService_StreamingCallClient, int, int) error +func (bc *benchmarkClient) streamingLoop(conns []*grpc.ClientConn, rpcCountPerConn int, reqSize int, respSize int, payloadType string, poissonLambda *float64) { + var doRPC func(testgrpc.BenchmarkService_StreamingCallClient, int, int) error if payloadType == "bytebuf" { doRPC = benchmark.DoByteBufStreamingRoundTrip } else { @@ -294,38 +319,74 @@ func (bc *benchmarkClient) doCloseLoopStreaming(conns []*grpc.ClientConn, rpcCou for ic, conn := range conns { // For each connection, create rpcCountPerConn goroutines to do rpc. for j := 0; j < rpcCountPerConn; j++ { - c := testpb.NewBenchmarkServiceClient(conn) + c := testgrpc.NewBenchmarkServiceClient(conn) stream, err := c.StreamingCall(context.Background()) if err != nil { logger.Fatalf("%v.StreamingCall(_) = _, %v", c, err) } - // Create histogram for each goroutine. idx := ic*rpcCountPerConn + j bc.lockingHistograms[idx].histogram = stats.NewHistogram(bc.histogramOptions) - // Start goroutine on the created mutex and histogram. - go func(idx int) { - // TODO: do warm up if necessary. - // Now relying on worker client to reserve time to do warm up. - // The worker client needs to wait for some time after client is created, - // before starting benchmark. - for { - start := time.Now() - if err := doRPC(stream, reqSize, respSize); err != nil { - return - } - elapse := time.Since(start) - bc.lockingHistograms[idx].add(int64(elapse)) - select { - case <-bc.stop: - return - default: + if poissonLambda == nil { // Closed loop. + // Start goroutine on the created mutex and histogram. + go func(idx int) { + // TODO: do warm up if necessary. + // Now relying on worker client to reserve time to do warm up. + // The worker client needs to wait for some time after client is created, + // before starting benchmark. + for { + start := time.Now() + if err := doRPC(stream, reqSize, respSize); err != nil { + return + } + elapse := time.Since(start) + bc.lockingHistograms[idx].add(int64(elapse)) + select { + case <-bc.stop: + return + default: + } } - } - }(idx) + }(idx) + } else { // Open loop. + timeBetweenRPCs := time.Duration((grpcrand.ExpFloat64() / *poissonLambda) * float64(time.Second)) + time.AfterFunc(timeBetweenRPCs, func() { + bc.poissonStreaming(stream, idx, reqSize, respSize, *poissonLambda, doRPC) + }) + } } } } +func (bc *benchmarkClient) poissonUnary(client testgrpc.BenchmarkServiceClient, idx int, reqSize int, respSize int, lambda float64) { + go func() { + start := time.Now() + if err := benchmark.DoUnaryCall(client, reqSize, respSize); err != nil { + return + } + elapse := time.Since(start) + bc.lockingHistograms[idx].add(int64(elapse)) + }() + timeBetweenRPCs := time.Duration((grpcrand.ExpFloat64() / lambda) * float64(time.Second)) + time.AfterFunc(timeBetweenRPCs, func() { + bc.poissonUnary(client, idx, reqSize, respSize, lambda) + }) +} + +func (bc *benchmarkClient) poissonStreaming(stream testgrpc.BenchmarkService_StreamingCallClient, idx int, reqSize int, respSize int, lambda float64, doRPC func(testgrpc.BenchmarkService_StreamingCallClient, int, int) error) { + go func() { + start := time.Now() + if err := doRPC(stream, reqSize, respSize); err != nil { + return + } + elapse := time.Since(start) + bc.lockingHistograms[idx].add(int64(elapse)) + }() + timeBetweenRPCs := time.Duration((grpcrand.ExpFloat64() / lambda) * float64(time.Second)) + time.AfterFunc(timeBetweenRPCs, func() { + bc.poissonStreaming(stream, idx, reqSize, respSize, lambda, doRPC) + }) +} + // getStats returns the stats for benchmark client. // It resets lastResetTime and all histograms if argument reset is true. func (bc *benchmarkClient) getStats(reset bool) *testpb.ClientStats { diff --git a/benchmark/worker/benchmark_server.go b/benchmark/worker/benchmark_server.go index c2e5e2609786..01b3a5ce8a9f 100644 --- a/benchmark/worker/benchmark_server.go +++ b/benchmark/worker/benchmark_server.go @@ -30,10 +30,10 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/benchmark" - testpb "google.golang.org/grpc/benchmark/grpc_testing" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" "google.golang.org/grpc/internal/syscall" + testpb "google.golang.org/grpc/interop/grpc_testing" "google.golang.org/grpc/status" "google.golang.org/grpc/testdata" ) @@ -80,7 +80,10 @@ func startBenchmarkServer(config *testpb.ServerConfig, serverPort int) (*benchma } runtime.GOMAXPROCS(numOfCores) - var opts []grpc.ServerOption + opts := []grpc.ServerOption{ + grpc.WriteBufferSize(128 * 1024), + grpc.ReadBufferSize(128 * 1024), + } // Sanity check for server type. switch config.ServerType { @@ -94,14 +97,14 @@ func startBenchmarkServer(config *testpb.ServerConfig, serverPort int) (*benchma // Set security options. if config.SecurityParams != nil { if *certFile == "" { - *certFile = testdata.Path("x509/server1_cert.pem") + *certFile = testdata.Path("server1.pem") } if *keyFile == "" { - *keyFile = testdata.Path("x509/server1_key.pem") + *keyFile = testdata.Path("server1.key") } creds, err := credentials.NewServerTLSFromFile(*certFile, *keyFile) if err != nil { - logger.Fatalf("failed to generate credentials %v", err) + logger.Fatalf("failed to generate credentials: %v", err) } opts = append(opts, grpc.Creds(creds)) } diff --git a/benchmark/worker/main.go b/benchmark/worker/main.go index 891cd01fc92b..793fd76bcd40 100644 --- a/benchmark/worker/main.go +++ b/benchmark/worker/main.go @@ -33,10 +33,12 @@ import ( "time" "google.golang.org/grpc" - testpb "google.golang.org/grpc/benchmark/grpc_testing" "google.golang.org/grpc/codes" "google.golang.org/grpc/grpclog" "google.golang.org/grpc/status" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" ) var ( @@ -51,7 +53,7 @@ var ( type byteBufCodec struct { } -func (byteBufCodec) Marshal(v interface{}) ([]byte, error) { +func (byteBufCodec) Marshal(v any) ([]byte, error) { b, ok := v.(*[]byte) if !ok { return nil, fmt.Errorf("failed to marshal: %v is not type of *[]byte", v) @@ -59,7 +61,7 @@ func (byteBufCodec) Marshal(v interface{}) ([]byte, error) { return *b, nil } -func (byteBufCodec) Unmarshal(data []byte, v interface{}) error { +func (byteBufCodec) Unmarshal(data []byte, v any) error { b, ok := v.(*[]byte) if !ok { return fmt.Errorf("failed to marshal: %v is not type of *[]byte", v) @@ -75,13 +77,12 @@ func (byteBufCodec) String() string { // workerServer implements WorkerService rpc handlers. // It can create benchmarkServer or benchmarkClient on demand. type workerServer struct { + testgrpc.UnimplementedWorkerServiceServer stop chan<- bool serverPort int } -var _ testpb.UnstableWorkerServiceService = (*workerServer)(nil) - -func (s *workerServer) RunServer(stream testpb.WorkerService_RunServerServer) error { +func (s *workerServer) RunServer(stream testgrpc.WorkerService_RunServerServer) error { var bs *benchmarkServer defer func() { // Close benchmark server when stream ends. @@ -136,7 +137,7 @@ func (s *workerServer) RunServer(stream testpb.WorkerService_RunServerServer) er } } -func (s *workerServer) RunClient(stream testpb.WorkerService_RunClientServer) error { +func (s *workerServer) RunClient(stream testgrpc.WorkerService_RunClientServer) error { var bc *benchmarkClient defer func() { // Shut down benchmark client when stream ends. @@ -210,10 +211,10 @@ func main() { s := grpc.NewServer() stop := make(chan bool) - testpb.RegisterWorkerServiceService(s, testpb.NewWorkerServiceService(&workerServer{ + testgrpc.RegisterWorkerServiceServer(s, &workerServer{ stop: stop, serverPort: *serverPort, - })) + }) go func() { <-stop diff --git a/internal/binarylog/binarylog_end2end_test.go b/binarylog/binarylog_end2end_test.go similarity index 71% rename from internal/binarylog/binarylog_end2end_test.go rename to binarylog/binarylog_end2end_test.go index bdce754a57d5..277c17a10726 100644 --- a/internal/binarylog/binarylog_end2end_test.go +++ b/binarylog/binarylog_end2end_test.go @@ -29,15 +29,20 @@ import ( "time" "github.com/golang/protobuf/proto" - "google.golang.org/grpc" - pb "google.golang.org/grpc/binarylog/grpc_binarylog_v1" + "google.golang.org/grpc/binarylog" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/grpclog" - "google.golang.org/grpc/internal/binarylog" + iblog "google.golang.org/grpc/internal/binarylog" "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/stubserver" "google.golang.org/grpc/metadata" - testpb "google.golang.org/grpc/stats/grpc_testing" "google.golang.org/grpc/status" + + binlogpb "google.golang.org/grpc/binarylog/grpc_binarylog_v1" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" ) var grpclogLogger = grpclog.Component("binarylog") @@ -53,18 +58,18 @@ func Test(t *testing.T) { func init() { // Setting environment variable in tests doesn't work because of the init // orders. Set the loggers directly here. - binarylog.SetLogger(binarylog.AllLogger) - binarylog.SetDefaultSink(testSink) + iblog.SetLogger(iblog.AllLogger) + binarylog.SetSink(testSink) } var testSink = &testBinLogSink{} type testBinLogSink struct { mu sync.Mutex - buf []*pb.GrpcLogEntry + buf []*binlogpb.GrpcLogEntry } -func (s *testBinLogSink) Write(e *pb.GrpcLogEntry) error { +func (s *testBinLogSink) Write(e *binlogpb.GrpcLogEntry) error { s.mu.Lock() s.buf = append(s.buf, e) s.mu.Unlock() @@ -75,12 +80,12 @@ func (s *testBinLogSink) Close() error { return nil } // Returns all client entris if client is true, otherwise return all server // entries. -func (s *testBinLogSink) logEntries(client bool) []*pb.GrpcLogEntry { - logger := pb.GrpcLogEntry_LOGGER_SERVER +func (s *testBinLogSink) logEntries(client bool) []*binlogpb.GrpcLogEntry { + logger := binlogpb.GrpcLogEntry_LOGGER_SERVER if client { - logger = pb.GrpcLogEntry_LOGGER_CLIENT + logger = binlogpb.GrpcLogEntry_LOGGER_CLIENT } - var ret []*pb.GrpcLogEntry + var ret []*binlogpb.GrpcLogEntry s.mu.Lock() for _, e := range s.buf { if e.Logger == logger { @@ -114,12 +119,22 @@ var ( globalRPCID uint64 // RPC id starts with 1, but we do ++ at the beginning of each test. ) +func idToPayload(id int32) *testpb.Payload { + return &testpb.Payload{Body: []byte{byte(id), byte(id >> 8), byte(id >> 16), byte(id >> 24)}} +} + +func payloadToID(p *testpb.Payload) int32 { + if p == nil || len(p.Body) != 4 { + panic("invalid payload") + } + return int32(p.Body[0]) + int32(p.Body[1])<<8 + int32(p.Body[2])<<16 + int32(p.Body[3])<<24 +} + type testServer struct { + testgrpc.UnimplementedTestServiceServer te *test } -var _ testpb.UnstableTestServiceService = (*testServer)(nil) - func (s *testServer) UnaryCall(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { md, ok := metadata.FromIncomingContext(ctx) if ok { @@ -131,14 +146,14 @@ func (s *testServer) UnaryCall(ctx context.Context, in *testpb.SimpleRequest) (* } } - if in.Id == errorID { - return nil, fmt.Errorf("got error id: %v", in.Id) + if id := payloadToID(in.Payload); id == errorID { + return nil, fmt.Errorf("got error id: %v", id) } - return &testpb.SimpleResponse{Id: in.Id}, nil + return &testpb.SimpleResponse{Payload: in.Payload}, nil } -func (s *testServer) FullDuplexCall(stream testpb.TestService_FullDuplexCallServer) error { +func (s *testServer) FullDuplexCall(stream testgrpc.TestService_FullDuplexCallServer) error { md, ok := metadata.FromIncomingContext(stream.Context()) if ok { if err := stream.SendHeader(md); err != nil { @@ -156,17 +171,17 @@ func (s *testServer) FullDuplexCall(stream testpb.TestService_FullDuplexCallServ return err } - if in.Id == errorID { - return fmt.Errorf("got error id: %v", in.Id) + if id := payloadToID(in.Payload); id == errorID { + return fmt.Errorf("got error id: %v", id) } - if err := stream.Send(&testpb.SimpleResponse{Id: in.Id}); err != nil { + if err := stream.Send(&testpb.StreamingOutputCallResponse{Payload: in.Payload}); err != nil { return err } } } -func (s *testServer) ClientStreamCall(stream testpb.TestService_ClientStreamCallServer) error { +func (s *testServer) StreamingInputCall(stream testgrpc.TestService_StreamingInputCallServer) error { md, ok := metadata.FromIncomingContext(stream.Context()) if ok { if err := stream.SendHeader(md); err != nil { @@ -178,19 +193,19 @@ func (s *testServer) ClientStreamCall(stream testpb.TestService_ClientStreamCall in, err := stream.Recv() if err == io.EOF { // read done. - return stream.SendAndClose(&testpb.SimpleResponse{Id: int32(0)}) + return stream.SendAndClose(&testpb.StreamingInputCallResponse{AggregatedPayloadSize: 0}) } if err != nil { return err } - if in.Id == errorID { - return fmt.Errorf("got error id: %v", in.Id) + if id := payloadToID(in.Payload); id == errorID { + return fmt.Errorf("got error id: %v", id) } } } -func (s *testServer) ServerStreamCall(in *testpb.SimpleRequest, stream testpb.TestService_ServerStreamCallServer) error { +func (s *testServer) StreamingOutputCall(in *testpb.StreamingOutputCallRequest, stream testgrpc.TestService_StreamingOutputCallServer) error { md, ok := metadata.FromIncomingContext(stream.Context()) if ok { if err := stream.SendHeader(md); err != nil { @@ -199,12 +214,12 @@ func (s *testServer) ServerStreamCall(in *testpb.SimpleRequest, stream testpb.Te stream.SetTrailer(testTrailerMetadata) } - if in.Id == errorID { - return fmt.Errorf("got error id: %v", in.Id) + if id := payloadToID(in.Payload); id == errorID { + return fmt.Errorf("got error id: %v", id) } for i := 0; i < 5; i++ { - if err := stream.Send(&testpb.SimpleResponse{Id: in.Id}); err != nil { + if err := stream.Send(&testpb.StreamingOutputCallResponse{Payload: in.Payload}); err != nil { return err } } @@ -217,7 +232,7 @@ func (s *testServer) ServerStreamCall(in *testpb.SimpleRequest, stream testpb.Te type test struct { t *testing.T - testService *testpb.TestServiceService // nil means none + testService testgrpc.TestServiceServer // nil means none // srv and srvAddr are set once startServer is called. srv *grpc.Server srvAddr string // Server IP without port. @@ -240,13 +255,10 @@ func (te *test) tearDown() { te.srv.Stop() } -type testConfig struct { -} - // newTest returns a new test using the provided testing.T and // environment. It is returned with default values. Tests should // modify it before calling its startServer and clientConn methods. -func newTest(t *testing.T, tc *testConfig) *test { +func newTest(t *testing.T) *test { te := &test{ t: t, } @@ -272,7 +284,7 @@ func (lw *listenerWrapper) Accept() (net.Conn, error) { // startServer starts a gRPC server listening. Callers should defer a // call to te.tearDown to clean up. -func (te *test) startServer(ts *testpb.TestServiceService) { +func (te *test) startServer(ts testgrpc.TestServiceServer) { te.testService = ts lis, err := net.Listen("tcp", "localhost:0") @@ -288,7 +300,7 @@ func (te *test) startServer(ts *testpb.TestServiceService) { s := grpc.NewServer(opts...) te.srv = s if te.testService != nil { - testpb.RegisterTestServiceService(s, te.testService) + testgrpc.RegisterTestServiceServer(s, te.testService) } go s.Serve(lis) @@ -301,7 +313,7 @@ func (te *test) clientConn() *grpc.ClientConn { if te.cc != nil { return te.cc } - opts := []grpc.DialOption{grpc.WithInsecure(), grpc.WithBlock()} + opts := []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithBlock()} var err error te.cc, err = grpc.Dial(te.srvAddr, opts...) @@ -333,11 +345,11 @@ func (te *test) doUnaryCall(c *rpcConfig) (*testpb.SimpleRequest, *testpb.Simple req *testpb.SimpleRequest err error ) - tc := testpb.NewTestServiceClient(te.clientConn()) + tc := testgrpc.NewTestServiceClient(te.clientConn()) if c.success { - req = &testpb.SimpleRequest{Id: errorID + 1} + req = &testpb.SimpleRequest{Payload: idToPayload(errorID + 1)} } else { - req = &testpb.SimpleRequest{Id: errorID} + req = &testpb.SimpleRequest{Payload: idToPayload(errorID)} } ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() @@ -347,13 +359,13 @@ func (te *test) doUnaryCall(c *rpcConfig) (*testpb.SimpleRequest, *testpb.Simple return req, resp, err } -func (te *test) doFullDuplexCallRoundtrip(c *rpcConfig) ([]*testpb.SimpleRequest, []*testpb.SimpleResponse, error) { +func (te *test) doFullDuplexCallRoundtrip(c *rpcConfig) ([]proto.Message, []proto.Message, error) { var ( - reqs []*testpb.SimpleRequest - resps []*testpb.SimpleResponse + reqs []proto.Message + resps []proto.Message err error ) - tc := testpb.NewTestServiceClient(te.clientConn()) + tc := testgrpc.NewTestServiceClient(te.clientConn()) ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() ctx = metadata.NewOutgoingContext(ctx, testMetadata) @@ -373,14 +385,14 @@ func (te *test) doFullDuplexCallRoundtrip(c *rpcConfig) ([]*testpb.SimpleRequest startID = errorID } for i := 0; i < c.count; i++ { - req := &testpb.SimpleRequest{ - Id: int32(i) + startID, + req := &testpb.StreamingOutputCallRequest{ + Payload: idToPayload(int32(i) + startID), } reqs = append(reqs, req) if err = stream.Send(req); err != nil { return reqs, resps, err } - var resp *testpb.SimpleResponse + var resp *testpb.StreamingOutputCallResponse if resp, err = stream.Recv(); err != nil { return reqs, resps, err } @@ -396,18 +408,18 @@ func (te *test) doFullDuplexCallRoundtrip(c *rpcConfig) ([]*testpb.SimpleRequest return reqs, resps, nil } -func (te *test) doClientStreamCall(c *rpcConfig) ([]*testpb.SimpleRequest, *testpb.SimpleResponse, error) { +func (te *test) doClientStreamCall(c *rpcConfig) ([]proto.Message, proto.Message, error) { var ( - reqs []*testpb.SimpleRequest - resp *testpb.SimpleResponse + reqs []proto.Message + resp *testpb.StreamingInputCallResponse err error ) - tc := testpb.NewTestServiceClient(te.clientConn()) + tc := testgrpc.NewTestServiceClient(te.clientConn()) ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() ctx = metadata.NewOutgoingContext(ctx, testMetadata) - stream, err := tc.ClientStreamCall(ctx) + stream, err := tc.StreamingInputCall(ctx) if err != nil { return reqs, resp, err } @@ -416,8 +428,8 @@ func (te *test) doClientStreamCall(c *rpcConfig) ([]*testpb.SimpleRequest, *test startID = errorID } for i := 0; i < c.count; i++ { - req := &testpb.SimpleRequest{ - Id: int32(i) + startID, + req := &testpb.StreamingInputCallRequest{ + Payload: idToPayload(int32(i) + startID), } reqs = append(reqs, req) if err = stream.Send(req); err != nil { @@ -428,14 +440,14 @@ func (te *test) doClientStreamCall(c *rpcConfig) ([]*testpb.SimpleRequest, *test return reqs, resp, err } -func (te *test) doServerStreamCall(c *rpcConfig) (*testpb.SimpleRequest, []*testpb.SimpleResponse, error) { +func (te *test) doServerStreamCall(c *rpcConfig) (proto.Message, []proto.Message, error) { var ( - req *testpb.SimpleRequest - resps []*testpb.SimpleResponse + req *testpb.StreamingOutputCallRequest + resps []proto.Message err error ) - tc := testpb.NewTestServiceClient(te.clientConn()) + tc := testgrpc.NewTestServiceClient(te.clientConn()) ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() ctx = metadata.NewOutgoingContext(ctx, testMetadata) @@ -444,13 +456,13 @@ func (te *test) doServerStreamCall(c *rpcConfig) (*testpb.SimpleRequest, []*test if !c.success { startID = errorID } - req = &testpb.SimpleRequest{Id: startID} - stream, err := tc.ServerStreamCall(ctx, req) + req = &testpb.StreamingOutputCallRequest{Payload: idToPayload(startID)} + stream, err := tc.StreamingOutputCall(ctx, req) if err != nil { return req, resps, err } for { - var resp *testpb.SimpleResponse + var resp *testpb.StreamingOutputCallResponse resp, err := stream.Recv() if err == io.EOF { return req, resps, nil @@ -466,37 +478,37 @@ type expectedData struct { cc *rpcConfig method string - requests []*testpb.SimpleRequest - responses []*testpb.SimpleResponse + requests []proto.Message + responses []proto.Message err error } -func (ed *expectedData) newClientHeaderEntry(client bool, rpcID, inRPCID uint64) *pb.GrpcLogEntry { - logger := pb.GrpcLogEntry_LOGGER_CLIENT - var peer *pb.Address +func (ed *expectedData) newClientHeaderEntry(client bool, rpcID, inRPCID uint64) *binlogpb.GrpcLogEntry { + logger := binlogpb.GrpcLogEntry_LOGGER_CLIENT + var peer *binlogpb.Address if !client { - logger = pb.GrpcLogEntry_LOGGER_SERVER + logger = binlogpb.GrpcLogEntry_LOGGER_SERVER ed.te.clientAddrMu.Lock() - peer = &pb.Address{ + peer = &binlogpb.Address{ Address: ed.te.clientIP.String(), IpPort: uint32(ed.te.clientPort), } if ed.te.clientIP.To4() != nil { - peer.Type = pb.Address_TYPE_IPV4 + peer.Type = binlogpb.Address_TYPE_IPV4 } else { - peer.Type = pb.Address_TYPE_IPV6 + peer.Type = binlogpb.Address_TYPE_IPV6 } ed.te.clientAddrMu.Unlock() } - return &pb.GrpcLogEntry{ + return &binlogpb.GrpcLogEntry{ Timestamp: nil, CallId: rpcID, SequenceIdWithinCall: inRPCID, - Type: pb.GrpcLogEntry_EVENT_TYPE_CLIENT_HEADER, + Type: binlogpb.GrpcLogEntry_EVENT_TYPE_CLIENT_HEADER, Logger: logger, - Payload: &pb.GrpcLogEntry_ClientHeader{ - ClientHeader: &pb.ClientHeader{ - Metadata: binarylog.MdToMetadataProto(testMetadata), + Payload: &binlogpb.GrpcLogEntry_ClientHeader{ + ClientHeader: &binlogpb.ClientHeader{ + Metadata: iblog.MdToMetadataProto(testMetadata), MethodName: ed.method, Authority: ed.te.srvAddr, }, @@ -505,53 +517,53 @@ func (ed *expectedData) newClientHeaderEntry(client bool, rpcID, inRPCID uint64) } } -func (ed *expectedData) newServerHeaderEntry(client bool, rpcID, inRPCID uint64) *pb.GrpcLogEntry { - logger := pb.GrpcLogEntry_LOGGER_SERVER - var peer *pb.Address +func (ed *expectedData) newServerHeaderEntry(client bool, rpcID, inRPCID uint64) *binlogpb.GrpcLogEntry { + logger := binlogpb.GrpcLogEntry_LOGGER_SERVER + var peer *binlogpb.Address if client { - logger = pb.GrpcLogEntry_LOGGER_CLIENT - peer = &pb.Address{ + logger = binlogpb.GrpcLogEntry_LOGGER_CLIENT + peer = &binlogpb.Address{ Address: ed.te.srvIP.String(), IpPort: uint32(ed.te.srvPort), } if ed.te.srvIP.To4() != nil { - peer.Type = pb.Address_TYPE_IPV4 + peer.Type = binlogpb.Address_TYPE_IPV4 } else { - peer.Type = pb.Address_TYPE_IPV6 + peer.Type = binlogpb.Address_TYPE_IPV6 } } - return &pb.GrpcLogEntry{ + return &binlogpb.GrpcLogEntry{ Timestamp: nil, CallId: rpcID, SequenceIdWithinCall: inRPCID, - Type: pb.GrpcLogEntry_EVENT_TYPE_SERVER_HEADER, + Type: binlogpb.GrpcLogEntry_EVENT_TYPE_SERVER_HEADER, Logger: logger, - Payload: &pb.GrpcLogEntry_ServerHeader{ - ServerHeader: &pb.ServerHeader{ - Metadata: binarylog.MdToMetadataProto(testMetadata), + Payload: &binlogpb.GrpcLogEntry_ServerHeader{ + ServerHeader: &binlogpb.ServerHeader{ + Metadata: iblog.MdToMetadataProto(testMetadata), }, }, Peer: peer, } } -func (ed *expectedData) newClientMessageEntry(client bool, rpcID, inRPCID uint64, msg *testpb.SimpleRequest) *pb.GrpcLogEntry { - logger := pb.GrpcLogEntry_LOGGER_CLIENT +func (ed *expectedData) newClientMessageEntry(client bool, rpcID, inRPCID uint64, msg proto.Message) *binlogpb.GrpcLogEntry { + logger := binlogpb.GrpcLogEntry_LOGGER_CLIENT if !client { - logger = pb.GrpcLogEntry_LOGGER_SERVER + logger = binlogpb.GrpcLogEntry_LOGGER_SERVER } data, err := proto.Marshal(msg) if err != nil { grpclogLogger.Infof("binarylogging_testing: failed to marshal proto message: %v", err) } - return &pb.GrpcLogEntry{ + return &binlogpb.GrpcLogEntry{ Timestamp: nil, CallId: rpcID, SequenceIdWithinCall: inRPCID, - Type: pb.GrpcLogEntry_EVENT_TYPE_CLIENT_MESSAGE, + Type: binlogpb.GrpcLogEntry_EVENT_TYPE_CLIENT_MESSAGE, Logger: logger, - Payload: &pb.GrpcLogEntry_Message{ - Message: &pb.Message{ + Payload: &binlogpb.GrpcLogEntry_Message{ + Message: &binlogpb.Message{ Length: uint32(len(data)), Data: data, }, @@ -559,23 +571,23 @@ func (ed *expectedData) newClientMessageEntry(client bool, rpcID, inRPCID uint64 } } -func (ed *expectedData) newServerMessageEntry(client bool, rpcID, inRPCID uint64, msg *testpb.SimpleResponse) *pb.GrpcLogEntry { - logger := pb.GrpcLogEntry_LOGGER_CLIENT +func (ed *expectedData) newServerMessageEntry(client bool, rpcID, inRPCID uint64, msg proto.Message) *binlogpb.GrpcLogEntry { + logger := binlogpb.GrpcLogEntry_LOGGER_CLIENT if !client { - logger = pb.GrpcLogEntry_LOGGER_SERVER + logger = binlogpb.GrpcLogEntry_LOGGER_SERVER } data, err := proto.Marshal(msg) if err != nil { grpclogLogger.Infof("binarylogging_testing: failed to marshal proto message: %v", err) } - return &pb.GrpcLogEntry{ + return &binlogpb.GrpcLogEntry{ Timestamp: nil, CallId: rpcID, SequenceIdWithinCall: inRPCID, - Type: pb.GrpcLogEntry_EVENT_TYPE_SERVER_MESSAGE, + Type: binlogpb.GrpcLogEntry_EVENT_TYPE_SERVER_MESSAGE, Logger: logger, - Payload: &pb.GrpcLogEntry_Message{ - Message: &pb.Message{ + Payload: &binlogpb.GrpcLogEntry_Message{ + Message: &binlogpb.Message{ Length: uint32(len(data)), Data: data, }, @@ -583,34 +595,34 @@ func (ed *expectedData) newServerMessageEntry(client bool, rpcID, inRPCID uint64 } } -func (ed *expectedData) newHalfCloseEntry(client bool, rpcID, inRPCID uint64) *pb.GrpcLogEntry { - logger := pb.GrpcLogEntry_LOGGER_CLIENT +func (ed *expectedData) newHalfCloseEntry(client bool, rpcID, inRPCID uint64) *binlogpb.GrpcLogEntry { + logger := binlogpb.GrpcLogEntry_LOGGER_CLIENT if !client { - logger = pb.GrpcLogEntry_LOGGER_SERVER + logger = binlogpb.GrpcLogEntry_LOGGER_SERVER } - return &pb.GrpcLogEntry{ + return &binlogpb.GrpcLogEntry{ Timestamp: nil, CallId: rpcID, SequenceIdWithinCall: inRPCID, - Type: pb.GrpcLogEntry_EVENT_TYPE_CLIENT_HALF_CLOSE, + Type: binlogpb.GrpcLogEntry_EVENT_TYPE_CLIENT_HALF_CLOSE, Payload: nil, // No payload here. Logger: logger, } } -func (ed *expectedData) newServerTrailerEntry(client bool, rpcID, inRPCID uint64, stErr error) *pb.GrpcLogEntry { - logger := pb.GrpcLogEntry_LOGGER_SERVER - var peer *pb.Address +func (ed *expectedData) newServerTrailerEntry(client bool, rpcID, inRPCID uint64, stErr error) *binlogpb.GrpcLogEntry { + logger := binlogpb.GrpcLogEntry_LOGGER_SERVER + var peer *binlogpb.Address if client { - logger = pb.GrpcLogEntry_LOGGER_CLIENT - peer = &pb.Address{ + logger = binlogpb.GrpcLogEntry_LOGGER_CLIENT + peer = &binlogpb.Address{ Address: ed.te.srvIP.String(), IpPort: uint32(ed.te.srvPort), } if ed.te.srvIP.To4() != nil { - peer.Type = pb.Address_TYPE_IPV4 + peer.Type = binlogpb.Address_TYPE_IPV4 } else { - peer.Type = pb.Address_TYPE_IPV6 + peer.Type = binlogpb.Address_TYPE_IPV6 } } st, ok := status.FromError(stErr) @@ -628,15 +640,15 @@ func (ed *expectedData) newServerTrailerEntry(client bool, rpcID, inRPCID uint64 grpclogLogger.Infof("binarylogging: failed to marshal status proto: %v", err) } } - return &pb.GrpcLogEntry{ + return &binlogpb.GrpcLogEntry{ Timestamp: nil, CallId: rpcID, SequenceIdWithinCall: inRPCID, - Type: pb.GrpcLogEntry_EVENT_TYPE_SERVER_TRAILER, + Type: binlogpb.GrpcLogEntry_EVENT_TYPE_SERVER_TRAILER, Logger: logger, - Payload: &pb.GrpcLogEntry_Trailer{ - Trailer: &pb.Trailer{ - Metadata: binarylog.MdToMetadataProto(testTrailerMetadata), + Payload: &binlogpb.GrpcLogEntry_Trailer{ + Trailer: &binlogpb.Trailer{ + Metadata: iblog.MdToMetadataProto(testTrailerMetadata), // st will be nil if err was not a status error, but nil is ok. StatusCode: uint32(st.Code()), StatusMessage: st.Message(), @@ -647,20 +659,20 @@ func (ed *expectedData) newServerTrailerEntry(client bool, rpcID, inRPCID uint64 } } -func (ed *expectedData) newCancelEntry(rpcID, inRPCID uint64) *pb.GrpcLogEntry { - return &pb.GrpcLogEntry{ +func (ed *expectedData) newCancelEntry(rpcID, inRPCID uint64) *binlogpb.GrpcLogEntry { + return &binlogpb.GrpcLogEntry{ Timestamp: nil, CallId: rpcID, SequenceIdWithinCall: inRPCID, - Type: pb.GrpcLogEntry_EVENT_TYPE_CANCEL, - Logger: pb.GrpcLogEntry_LOGGER_CLIENT, + Type: binlogpb.GrpcLogEntry_EVENT_TYPE_CANCEL, + Logger: binlogpb.GrpcLogEntry_LOGGER_CLIENT, Payload: nil, } } -func (ed *expectedData) toClientLogEntries() []*pb.GrpcLogEntry { +func (ed *expectedData) toClientLogEntries() []*binlogpb.GrpcLogEntry { var ( - ret []*pb.GrpcLogEntry + ret []*binlogpb.GrpcLogEntry idInRPC uint64 = 1 ) ret = append(ret, ed.newClientHeaderEntry(true, globalRPCID, idInRPC)) @@ -716,9 +728,9 @@ func (ed *expectedData) toClientLogEntries() []*pb.GrpcLogEntry { return ret } -func (ed *expectedData) toServerLogEntries() []*pb.GrpcLogEntry { +func (ed *expectedData) toServerLogEntries() []*binlogpb.GrpcLogEntry { var ( - ret []*pb.GrpcLogEntry + ret []*binlogpb.GrpcLogEntry idInRPC uint64 = 1 ) ret = append(ret, ed.newClientHeaderEntry(false, globalRPCID, idInRPC)) @@ -782,9 +794,9 @@ func (ed *expectedData) toServerLogEntries() []*pb.GrpcLogEntry { return ret } -func runRPCs(t *testing.T, tc *testConfig, cc *rpcConfig) *expectedData { - te := newTest(t, tc) - te.startServer(testpb.NewTestServiceService(&testServer{te: te})) +func runRPCs(t *testing.T, cc *rpcConfig) *expectedData { + te := newTest(t) + te.startServer(&testServer{te: te}) defer te.tearDown() expect := &expectedData{ @@ -796,20 +808,20 @@ func runRPCs(t *testing.T, tc *testConfig, cc *rpcConfig) *expectedData { case unaryRPC: expect.method = "/grpc.testing.TestService/UnaryCall" req, resp, err := te.doUnaryCall(cc) - expect.requests = []*testpb.SimpleRequest{req} - expect.responses = []*testpb.SimpleResponse{resp} + expect.requests = []proto.Message{req} + expect.responses = []proto.Message{resp} expect.err = err case clientStreamRPC: - expect.method = "/grpc.testing.TestService/ClientStreamCall" + expect.method = "/grpc.testing.TestService/StreamingInputCall" reqs, resp, err := te.doClientStreamCall(cc) expect.requests = reqs - expect.responses = []*testpb.SimpleResponse{resp} + expect.responses = []proto.Message{resp} expect.err = err case serverStreamRPC: - expect.method = "/grpc.testing.TestService/ServerStreamCall" + expect.method = "/grpc.testing.TestService/StreamingOutputCall" req, resps, err := te.doServerStreamCall(cc) expect.responses = resps - expect.requests = []*testpb.SimpleRequest{req} + expect.requests = []proto.Message{req} expect.err = err case fullDuplexStreamRPC, cancelRPC: expect.method = "/grpc.testing.TestService/FullDuplexCall" @@ -828,7 +840,7 @@ func runRPCs(t *testing.T, tc *testConfig, cc *rpcConfig) *expectedData { // // This function is typically called with only two entries. It's written in this // way so the code can be put in a for loop instead of copied twice. -func equalLogEntry(entries ...*pb.GrpcLogEntry) (equal bool) { +func equalLogEntry(entries ...*binlogpb.GrpcLogEntry) (equal bool) { for i, e := range entries { // Clear out some fields we don't compare. e.Timestamp = nil @@ -857,9 +869,9 @@ func equalLogEntry(entries ...*pb.GrpcLogEntry) (equal bool) { func testClientBinaryLog(t *testing.T, c *rpcConfig) error { defer testSink.clear() - expect := runRPCs(t, &testConfig{}, c) + expect := runRPCs(t, c) want := expect.toClientLogEntries() - var got []*pb.GrpcLogEntry + var got []*binlogpb.GrpcLogEntry // In racy cases, some entries are not logged when the RPC is finished (e.g. // context.Cancel). // @@ -957,9 +969,9 @@ func (s) TestClientBinaryLogCancel(t *testing.T) { func testServerBinaryLog(t *testing.T, c *rpcConfig) error { defer testSink.clear() - expect := runRPCs(t, &testConfig{}, c) + expect := runRPCs(t, c) want := expect.toServerLogEntries() - var got []*pb.GrpcLogEntry + var got []*binlogpb.GrpcLogEntry // In racy cases, some entries are not logged when the RPC is finished (e.g. // context.Cancel). This is unlikely to happen on server side, but it does // no harm to retry. @@ -1049,3 +1061,39 @@ func (s) TestServerBinaryLogFullDuplexError(t *testing.T) { t.Fatal(err) } } + +// TestCanceledStatus ensures a server that responds with a Canceled status has +// its trailers logged appropriately and is not treated as a canceled RPC. +func (s) TestCanceledStatus(t *testing.T) { + defer testSink.clear() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + const statusMsgWant = "server returned Canceled" + ss := &stubserver.StubServer{ + UnaryCallF: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + grpc.SetTrailer(ctx, metadata.Pairs("key", "value")) + return nil, status.Error(codes.Canceled, statusMsgWant) + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{}); status.Code(err) != codes.Canceled { + t.Fatalf("Received unexpected error from UnaryCall: %v; want Canceled", err) + } + + got := testSink.logEntries(true) + last := got[len(got)-1] + if last.Type != binlogpb.GrpcLogEntry_EVENT_TYPE_SERVER_TRAILER || + last.GetTrailer().GetStatusCode() != uint32(codes.Canceled) || + last.GetTrailer().GetStatusMessage() != statusMsgWant || + len(last.GetTrailer().GetMetadata().GetEntry()) != 1 || + last.GetTrailer().GetMetadata().GetEntry()[0].GetKey() != "key" || + string(last.GetTrailer().GetMetadata().GetEntry()[0].GetValue()) != "value" { + t.Fatalf("Got binary log: %+v; want last entry is server trailing with status Canceled", got) + } +} diff --git a/binarylog/grpc_binarylog_v1/binarylog.pb.go b/binarylog/grpc_binarylog_v1/binarylog.pb.go index f826ec76984d..5954801122ad 100644 --- a/binarylog/grpc_binarylog_v1/binarylog.pb.go +++ b/binarylog/grpc_binarylog_v1/binarylog.pb.go @@ -1,26 +1,44 @@ +// Copyright 2018 The gRPC Authors +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The canonical version of this proto can be found at +// https://github.com/grpc/grpc-proto/blob/master/grpc/binlog/v1/binarylog.proto + // Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.31.0 +// protoc v4.22.0 // source: grpc/binlog/v1/binarylog.proto package grpc_binarylog_v1 import ( - fmt "fmt" - proto "github.com/golang/protobuf/proto" - duration "github.com/golang/protobuf/ptypes/duration" - timestamp "github.com/golang/protobuf/ptypes/timestamp" - math "math" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + durationpb "google.golang.org/protobuf/types/known/durationpb" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" + reflect "reflect" + sync "sync" ) -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) // Enumerates the type of event // Note the terminology is different from the RPC semantics @@ -56,34 +74,55 @@ const ( GrpcLogEntry_EVENT_TYPE_CANCEL GrpcLogEntry_EventType = 7 ) -var GrpcLogEntry_EventType_name = map[int32]string{ - 0: "EVENT_TYPE_UNKNOWN", - 1: "EVENT_TYPE_CLIENT_HEADER", - 2: "EVENT_TYPE_SERVER_HEADER", - 3: "EVENT_TYPE_CLIENT_MESSAGE", - 4: "EVENT_TYPE_SERVER_MESSAGE", - 5: "EVENT_TYPE_CLIENT_HALF_CLOSE", - 6: "EVENT_TYPE_SERVER_TRAILER", - 7: "EVENT_TYPE_CANCEL", -} +// Enum value maps for GrpcLogEntry_EventType. +var ( + GrpcLogEntry_EventType_name = map[int32]string{ + 0: "EVENT_TYPE_UNKNOWN", + 1: "EVENT_TYPE_CLIENT_HEADER", + 2: "EVENT_TYPE_SERVER_HEADER", + 3: "EVENT_TYPE_CLIENT_MESSAGE", + 4: "EVENT_TYPE_SERVER_MESSAGE", + 5: "EVENT_TYPE_CLIENT_HALF_CLOSE", + 6: "EVENT_TYPE_SERVER_TRAILER", + 7: "EVENT_TYPE_CANCEL", + } + GrpcLogEntry_EventType_value = map[string]int32{ + "EVENT_TYPE_UNKNOWN": 0, + "EVENT_TYPE_CLIENT_HEADER": 1, + "EVENT_TYPE_SERVER_HEADER": 2, + "EVENT_TYPE_CLIENT_MESSAGE": 3, + "EVENT_TYPE_SERVER_MESSAGE": 4, + "EVENT_TYPE_CLIENT_HALF_CLOSE": 5, + "EVENT_TYPE_SERVER_TRAILER": 6, + "EVENT_TYPE_CANCEL": 7, + } +) -var GrpcLogEntry_EventType_value = map[string]int32{ - "EVENT_TYPE_UNKNOWN": 0, - "EVENT_TYPE_CLIENT_HEADER": 1, - "EVENT_TYPE_SERVER_HEADER": 2, - "EVENT_TYPE_CLIENT_MESSAGE": 3, - "EVENT_TYPE_SERVER_MESSAGE": 4, - "EVENT_TYPE_CLIENT_HALF_CLOSE": 5, - "EVENT_TYPE_SERVER_TRAILER": 6, - "EVENT_TYPE_CANCEL": 7, +func (x GrpcLogEntry_EventType) Enum() *GrpcLogEntry_EventType { + p := new(GrpcLogEntry_EventType) + *p = x + return p } func (x GrpcLogEntry_EventType) String() string { - return proto.EnumName(GrpcLogEntry_EventType_name, int32(x)) + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (GrpcLogEntry_EventType) Descriptor() protoreflect.EnumDescriptor { + return file_grpc_binlog_v1_binarylog_proto_enumTypes[0].Descriptor() } +func (GrpcLogEntry_EventType) Type() protoreflect.EnumType { + return &file_grpc_binlog_v1_binarylog_proto_enumTypes[0] +} + +func (x GrpcLogEntry_EventType) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use GrpcLogEntry_EventType.Descriptor instead. func (GrpcLogEntry_EventType) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_b7972e58de45083a, []int{0, 0} + return file_grpc_binlog_v1_binarylog_proto_rawDescGZIP(), []int{0, 0} } // Enumerates the entity that generates the log entry @@ -95,24 +134,45 @@ const ( GrpcLogEntry_LOGGER_SERVER GrpcLogEntry_Logger = 2 ) -var GrpcLogEntry_Logger_name = map[int32]string{ - 0: "LOGGER_UNKNOWN", - 1: "LOGGER_CLIENT", - 2: "LOGGER_SERVER", -} +// Enum value maps for GrpcLogEntry_Logger. +var ( + GrpcLogEntry_Logger_name = map[int32]string{ + 0: "LOGGER_UNKNOWN", + 1: "LOGGER_CLIENT", + 2: "LOGGER_SERVER", + } + GrpcLogEntry_Logger_value = map[string]int32{ + "LOGGER_UNKNOWN": 0, + "LOGGER_CLIENT": 1, + "LOGGER_SERVER": 2, + } +) -var GrpcLogEntry_Logger_value = map[string]int32{ - "LOGGER_UNKNOWN": 0, - "LOGGER_CLIENT": 1, - "LOGGER_SERVER": 2, +func (x GrpcLogEntry_Logger) Enum() *GrpcLogEntry_Logger { + p := new(GrpcLogEntry_Logger) + *p = x + return p } func (x GrpcLogEntry_Logger) String() string { - return proto.EnumName(GrpcLogEntry_Logger_name, int32(x)) + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (GrpcLogEntry_Logger) Descriptor() protoreflect.EnumDescriptor { + return file_grpc_binlog_v1_binarylog_proto_enumTypes[1].Descriptor() +} + +func (GrpcLogEntry_Logger) Type() protoreflect.EnumType { + return &file_grpc_binlog_v1_binarylog_proto_enumTypes[1] +} + +func (x GrpcLogEntry_Logger) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) } +// Deprecated: Use GrpcLogEntry_Logger.Descriptor instead. func (GrpcLogEntry_Logger) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_b7972e58de45083a, []int{0, 1} + return file_grpc_binlog_v1_binarylog_proto_rawDescGZIP(), []int{0, 1} } type Address_Type int32 @@ -128,32 +188,57 @@ const ( Address_TYPE_UNIX Address_Type = 3 ) -var Address_Type_name = map[int32]string{ - 0: "TYPE_UNKNOWN", - 1: "TYPE_IPV4", - 2: "TYPE_IPV6", - 3: "TYPE_UNIX", -} +// Enum value maps for Address_Type. +var ( + Address_Type_name = map[int32]string{ + 0: "TYPE_UNKNOWN", + 1: "TYPE_IPV4", + 2: "TYPE_IPV6", + 3: "TYPE_UNIX", + } + Address_Type_value = map[string]int32{ + "TYPE_UNKNOWN": 0, + "TYPE_IPV4": 1, + "TYPE_IPV6": 2, + "TYPE_UNIX": 3, + } +) -var Address_Type_value = map[string]int32{ - "TYPE_UNKNOWN": 0, - "TYPE_IPV4": 1, - "TYPE_IPV6": 2, - "TYPE_UNIX": 3, +func (x Address_Type) Enum() *Address_Type { + p := new(Address_Type) + *p = x + return p } func (x Address_Type) String() string { - return proto.EnumName(Address_Type_name, int32(x)) + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (Address_Type) Descriptor() protoreflect.EnumDescriptor { + return file_grpc_binlog_v1_binarylog_proto_enumTypes[2].Descriptor() +} + +func (Address_Type) Type() protoreflect.EnumType { + return &file_grpc_binlog_v1_binarylog_proto_enumTypes[2] +} + +func (x Address_Type) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) } +// Deprecated: Use Address_Type.Descriptor instead. func (Address_Type) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_b7972e58de45083a, []int{7, 0} + return file_grpc_binlog_v1_binarylog_proto_rawDescGZIP(), []int{7, 0} } // Log entry we store in binary logs type GrpcLogEntry struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // The timestamp of the binary log message - Timestamp *timestamp.Timestamp `protobuf:"bytes,1,opt,name=timestamp,proto3" json:"timestamp,omitempty"` + Timestamp *timestamppb.Timestamp `protobuf:"bytes,1,opt,name=timestamp,proto3" json:"timestamp,omitempty"` // Uniquely identifies a call. The value must not be 0 in order to disambiguate // from an unset value. // Each call may have several log entries, they will all have the same call_id. @@ -166,11 +251,12 @@ type GrpcLogEntry struct { // durability or ordering is not guaranteed. SequenceIdWithinCall uint64 `protobuf:"varint,3,opt,name=sequence_id_within_call,json=sequenceIdWithinCall,proto3" json:"sequence_id_within_call,omitempty"` Type GrpcLogEntry_EventType `protobuf:"varint,4,opt,name=type,proto3,enum=grpc.binarylog.v1.GrpcLogEntry_EventType" json:"type,omitempty"` - Logger GrpcLogEntry_Logger `protobuf:"varint,5,opt,name=logger,proto3,enum=grpc.binarylog.v1.GrpcLogEntry_Logger" json:"logger,omitempty"` + Logger GrpcLogEntry_Logger `protobuf:"varint,5,opt,name=logger,proto3,enum=grpc.binarylog.v1.GrpcLogEntry_Logger" json:"logger,omitempty"` // One of the above Logger enum // The logger uses one of the following fields to record the payload, // according to the type of the log entry. // - // Types that are valid to be assigned to Payload: + // Types that are assignable to Payload: + // // *GrpcLogEntry_ClientHeader // *GrpcLogEntry_ServerHeader // *GrpcLogEntry_Message @@ -183,100 +269,76 @@ type GrpcLogEntry struct { // EVENT_TYPE_SERVER_HEADER normally or EVENT_TYPE_SERVER_TRAILER in // the case of trailers-only. On server side, peer is always // logged on EVENT_TYPE_CLIENT_HEADER. - Peer *Address `protobuf:"bytes,11,opt,name=peer,proto3" json:"peer,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Peer *Address `protobuf:"bytes,11,opt,name=peer,proto3" json:"peer,omitempty"` } -func (m *GrpcLogEntry) Reset() { *m = GrpcLogEntry{} } -func (m *GrpcLogEntry) String() string { return proto.CompactTextString(m) } -func (*GrpcLogEntry) ProtoMessage() {} -func (*GrpcLogEntry) Descriptor() ([]byte, []int) { - return fileDescriptor_b7972e58de45083a, []int{0} +func (x *GrpcLogEntry) Reset() { + *x = GrpcLogEntry{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_binlog_v1_binarylog_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *GrpcLogEntry) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_GrpcLogEntry.Unmarshal(m, b) +func (x *GrpcLogEntry) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *GrpcLogEntry) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_GrpcLogEntry.Marshal(b, m, deterministic) -} -func (m *GrpcLogEntry) XXX_Merge(src proto.Message) { - xxx_messageInfo_GrpcLogEntry.Merge(m, src) -} -func (m *GrpcLogEntry) XXX_Size() int { - return xxx_messageInfo_GrpcLogEntry.Size(m) -} -func (m *GrpcLogEntry) XXX_DiscardUnknown() { - xxx_messageInfo_GrpcLogEntry.DiscardUnknown(m) + +func (*GrpcLogEntry) ProtoMessage() {} + +func (x *GrpcLogEntry) ProtoReflect() protoreflect.Message { + mi := &file_grpc_binlog_v1_binarylog_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_GrpcLogEntry proto.InternalMessageInfo +// Deprecated: Use GrpcLogEntry.ProtoReflect.Descriptor instead. +func (*GrpcLogEntry) Descriptor() ([]byte, []int) { + return file_grpc_binlog_v1_binarylog_proto_rawDescGZIP(), []int{0} +} -func (m *GrpcLogEntry) GetTimestamp() *timestamp.Timestamp { - if m != nil { - return m.Timestamp +func (x *GrpcLogEntry) GetTimestamp() *timestamppb.Timestamp { + if x != nil { + return x.Timestamp } return nil } -func (m *GrpcLogEntry) GetCallId() uint64 { - if m != nil { - return m.CallId +func (x *GrpcLogEntry) GetCallId() uint64 { + if x != nil { + return x.CallId } return 0 } -func (m *GrpcLogEntry) GetSequenceIdWithinCall() uint64 { - if m != nil { - return m.SequenceIdWithinCall +func (x *GrpcLogEntry) GetSequenceIdWithinCall() uint64 { + if x != nil { + return x.SequenceIdWithinCall } return 0 } -func (m *GrpcLogEntry) GetType() GrpcLogEntry_EventType { - if m != nil { - return m.Type +func (x *GrpcLogEntry) GetType() GrpcLogEntry_EventType { + if x != nil { + return x.Type } return GrpcLogEntry_EVENT_TYPE_UNKNOWN } -func (m *GrpcLogEntry) GetLogger() GrpcLogEntry_Logger { - if m != nil { - return m.Logger +func (x *GrpcLogEntry) GetLogger() GrpcLogEntry_Logger { + if x != nil { + return x.Logger } return GrpcLogEntry_LOGGER_UNKNOWN } -type isGrpcLogEntry_Payload interface { - isGrpcLogEntry_Payload() -} - -type GrpcLogEntry_ClientHeader struct { - ClientHeader *ClientHeader `protobuf:"bytes,6,opt,name=client_header,json=clientHeader,proto3,oneof"` -} - -type GrpcLogEntry_ServerHeader struct { - ServerHeader *ServerHeader `protobuf:"bytes,7,opt,name=server_header,json=serverHeader,proto3,oneof"` -} - -type GrpcLogEntry_Message struct { - Message *Message `protobuf:"bytes,8,opt,name=message,proto3,oneof"` -} - -type GrpcLogEntry_Trailer struct { - Trailer *Trailer `protobuf:"bytes,9,opt,name=trailer,proto3,oneof"` -} - -func (*GrpcLogEntry_ClientHeader) isGrpcLogEntry_Payload() {} - -func (*GrpcLogEntry_ServerHeader) isGrpcLogEntry_Payload() {} - -func (*GrpcLogEntry_Message) isGrpcLogEntry_Payload() {} - -func (*GrpcLogEntry_Trailer) isGrpcLogEntry_Payload() {} - func (m *GrpcLogEntry) GetPayload() isGrpcLogEntry_Payload { if m != nil { return m.Payload @@ -284,59 +346,82 @@ func (m *GrpcLogEntry) GetPayload() isGrpcLogEntry_Payload { return nil } -func (m *GrpcLogEntry) GetClientHeader() *ClientHeader { - if x, ok := m.GetPayload().(*GrpcLogEntry_ClientHeader); ok { +func (x *GrpcLogEntry) GetClientHeader() *ClientHeader { + if x, ok := x.GetPayload().(*GrpcLogEntry_ClientHeader); ok { return x.ClientHeader } return nil } -func (m *GrpcLogEntry) GetServerHeader() *ServerHeader { - if x, ok := m.GetPayload().(*GrpcLogEntry_ServerHeader); ok { +func (x *GrpcLogEntry) GetServerHeader() *ServerHeader { + if x, ok := x.GetPayload().(*GrpcLogEntry_ServerHeader); ok { return x.ServerHeader } return nil } -func (m *GrpcLogEntry) GetMessage() *Message { - if x, ok := m.GetPayload().(*GrpcLogEntry_Message); ok { +func (x *GrpcLogEntry) GetMessage() *Message { + if x, ok := x.GetPayload().(*GrpcLogEntry_Message); ok { return x.Message } return nil } -func (m *GrpcLogEntry) GetTrailer() *Trailer { - if x, ok := m.GetPayload().(*GrpcLogEntry_Trailer); ok { +func (x *GrpcLogEntry) GetTrailer() *Trailer { + if x, ok := x.GetPayload().(*GrpcLogEntry_Trailer); ok { return x.Trailer } return nil } -func (m *GrpcLogEntry) GetPayloadTruncated() bool { - if m != nil { - return m.PayloadTruncated +func (x *GrpcLogEntry) GetPayloadTruncated() bool { + if x != nil { + return x.PayloadTruncated } return false } -func (m *GrpcLogEntry) GetPeer() *Address { - if m != nil { - return m.Peer +func (x *GrpcLogEntry) GetPeer() *Address { + if x != nil { + return x.Peer } return nil } -// XXX_OneofWrappers is for the internal use of the proto package. -func (*GrpcLogEntry) XXX_OneofWrappers() []interface{} { - return []interface{}{ - (*GrpcLogEntry_ClientHeader)(nil), - (*GrpcLogEntry_ServerHeader)(nil), - (*GrpcLogEntry_Message)(nil), - (*GrpcLogEntry_Trailer)(nil), - } +type isGrpcLogEntry_Payload interface { + isGrpcLogEntry_Payload() } +type GrpcLogEntry_ClientHeader struct { + ClientHeader *ClientHeader `protobuf:"bytes,6,opt,name=client_header,json=clientHeader,proto3,oneof"` +} + +type GrpcLogEntry_ServerHeader struct { + ServerHeader *ServerHeader `protobuf:"bytes,7,opt,name=server_header,json=serverHeader,proto3,oneof"` +} + +type GrpcLogEntry_Message struct { + // Used by EVENT_TYPE_CLIENT_MESSAGE, EVENT_TYPE_SERVER_MESSAGE + Message *Message `protobuf:"bytes,8,opt,name=message,proto3,oneof"` +} + +type GrpcLogEntry_Trailer struct { + Trailer *Trailer `protobuf:"bytes,9,opt,name=trailer,proto3,oneof"` +} + +func (*GrpcLogEntry_ClientHeader) isGrpcLogEntry_Payload() {} + +func (*GrpcLogEntry_ServerHeader) isGrpcLogEntry_Payload() {} + +func (*GrpcLogEntry_Message) isGrpcLogEntry_Payload() {} + +func (*GrpcLogEntry_Trailer) isGrpcLogEntry_Payload() {} + type ClientHeader struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // This contains only the metadata from the application. Metadata *Metadata `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"` // The name of the RPC method, which looks something like: @@ -350,106 +435,122 @@ type ClientHeader struct { // or : . Authority string `protobuf:"bytes,3,opt,name=authority,proto3" json:"authority,omitempty"` // the RPC timeout - Timeout *duration.Duration `protobuf:"bytes,4,opt,name=timeout,proto3" json:"timeout,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Timeout *durationpb.Duration `protobuf:"bytes,4,opt,name=timeout,proto3" json:"timeout,omitempty"` } -func (m *ClientHeader) Reset() { *m = ClientHeader{} } -func (m *ClientHeader) String() string { return proto.CompactTextString(m) } -func (*ClientHeader) ProtoMessage() {} -func (*ClientHeader) Descriptor() ([]byte, []int) { - return fileDescriptor_b7972e58de45083a, []int{1} +func (x *ClientHeader) Reset() { + *x = ClientHeader{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_binlog_v1_binarylog_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *ClientHeader) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ClientHeader.Unmarshal(m, b) -} -func (m *ClientHeader) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ClientHeader.Marshal(b, m, deterministic) -} -func (m *ClientHeader) XXX_Merge(src proto.Message) { - xxx_messageInfo_ClientHeader.Merge(m, src) +func (x *ClientHeader) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *ClientHeader) XXX_Size() int { - return xxx_messageInfo_ClientHeader.Size(m) -} -func (m *ClientHeader) XXX_DiscardUnknown() { - xxx_messageInfo_ClientHeader.DiscardUnknown(m) + +func (*ClientHeader) ProtoMessage() {} + +func (x *ClientHeader) ProtoReflect() protoreflect.Message { + mi := &file_grpc_binlog_v1_binarylog_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_ClientHeader proto.InternalMessageInfo +// Deprecated: Use ClientHeader.ProtoReflect.Descriptor instead. +func (*ClientHeader) Descriptor() ([]byte, []int) { + return file_grpc_binlog_v1_binarylog_proto_rawDescGZIP(), []int{1} +} -func (m *ClientHeader) GetMetadata() *Metadata { - if m != nil { - return m.Metadata +func (x *ClientHeader) GetMetadata() *Metadata { + if x != nil { + return x.Metadata } return nil } -func (m *ClientHeader) GetMethodName() string { - if m != nil { - return m.MethodName +func (x *ClientHeader) GetMethodName() string { + if x != nil { + return x.MethodName } return "" } -func (m *ClientHeader) GetAuthority() string { - if m != nil { - return m.Authority +func (x *ClientHeader) GetAuthority() string { + if x != nil { + return x.Authority } return "" } -func (m *ClientHeader) GetTimeout() *duration.Duration { - if m != nil { - return m.Timeout +func (x *ClientHeader) GetTimeout() *durationpb.Duration { + if x != nil { + return x.Timeout } return nil } type ServerHeader struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // This contains only the metadata from the application. - Metadata *Metadata `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Metadata *Metadata `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"` } -func (m *ServerHeader) Reset() { *m = ServerHeader{} } -func (m *ServerHeader) String() string { return proto.CompactTextString(m) } -func (*ServerHeader) ProtoMessage() {} -func (*ServerHeader) Descriptor() ([]byte, []int) { - return fileDescriptor_b7972e58de45083a, []int{2} +func (x *ServerHeader) Reset() { + *x = ServerHeader{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_binlog_v1_binarylog_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *ServerHeader) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ServerHeader.Unmarshal(m, b) -} -func (m *ServerHeader) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ServerHeader.Marshal(b, m, deterministic) -} -func (m *ServerHeader) XXX_Merge(src proto.Message) { - xxx_messageInfo_ServerHeader.Merge(m, src) -} -func (m *ServerHeader) XXX_Size() int { - return xxx_messageInfo_ServerHeader.Size(m) +func (x *ServerHeader) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *ServerHeader) XXX_DiscardUnknown() { - xxx_messageInfo_ServerHeader.DiscardUnknown(m) + +func (*ServerHeader) ProtoMessage() {} + +func (x *ServerHeader) ProtoReflect() protoreflect.Message { + mi := &file_grpc_binlog_v1_binarylog_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_ServerHeader proto.InternalMessageInfo +// Deprecated: Use ServerHeader.ProtoReflect.Descriptor instead. +func (*ServerHeader) Descriptor() ([]byte, []int) { + return file_grpc_binlog_v1_binarylog_proto_rawDescGZIP(), []int{2} +} -func (m *ServerHeader) GetMetadata() *Metadata { - if m != nil { - return m.Metadata +func (x *ServerHeader) GetMetadata() *Metadata { + if x != nil { + return x.Metadata } return nil } type Trailer struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // This contains only the metadata from the application. Metadata *Metadata `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"` // The gRPC status code. @@ -459,112 +560,124 @@ type Trailer struct { StatusMessage string `protobuf:"bytes,3,opt,name=status_message,json=statusMessage,proto3" json:"status_message,omitempty"` // The value of the 'grpc-status-details-bin' metadata key. If // present, this is always an encoded 'google.rpc.Status' message. - StatusDetails []byte `protobuf:"bytes,4,opt,name=status_details,json=statusDetails,proto3" json:"status_details,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + StatusDetails []byte `protobuf:"bytes,4,opt,name=status_details,json=statusDetails,proto3" json:"status_details,omitempty"` } -func (m *Trailer) Reset() { *m = Trailer{} } -func (m *Trailer) String() string { return proto.CompactTextString(m) } -func (*Trailer) ProtoMessage() {} -func (*Trailer) Descriptor() ([]byte, []int) { - return fileDescriptor_b7972e58de45083a, []int{3} +func (x *Trailer) Reset() { + *x = Trailer{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_binlog_v1_binarylog_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *Trailer) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Trailer.Unmarshal(m, b) -} -func (m *Trailer) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Trailer.Marshal(b, m, deterministic) -} -func (m *Trailer) XXX_Merge(src proto.Message) { - xxx_messageInfo_Trailer.Merge(m, src) +func (x *Trailer) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *Trailer) XXX_Size() int { - return xxx_messageInfo_Trailer.Size(m) -} -func (m *Trailer) XXX_DiscardUnknown() { - xxx_messageInfo_Trailer.DiscardUnknown(m) + +func (*Trailer) ProtoMessage() {} + +func (x *Trailer) ProtoReflect() protoreflect.Message { + mi := &file_grpc_binlog_v1_binarylog_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_Trailer proto.InternalMessageInfo +// Deprecated: Use Trailer.ProtoReflect.Descriptor instead. +func (*Trailer) Descriptor() ([]byte, []int) { + return file_grpc_binlog_v1_binarylog_proto_rawDescGZIP(), []int{3} +} -func (m *Trailer) GetMetadata() *Metadata { - if m != nil { - return m.Metadata +func (x *Trailer) GetMetadata() *Metadata { + if x != nil { + return x.Metadata } return nil } -func (m *Trailer) GetStatusCode() uint32 { - if m != nil { - return m.StatusCode +func (x *Trailer) GetStatusCode() uint32 { + if x != nil { + return x.StatusCode } return 0 } -func (m *Trailer) GetStatusMessage() string { - if m != nil { - return m.StatusMessage +func (x *Trailer) GetStatusMessage() string { + if x != nil { + return x.StatusMessage } return "" } -func (m *Trailer) GetStatusDetails() []byte { - if m != nil { - return m.StatusDetails +func (x *Trailer) GetStatusDetails() []byte { + if x != nil { + return x.StatusDetails } return nil } // Message payload, used by CLIENT_MESSAGE and SERVER_MESSAGE type Message struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // Length of the message. It may not be the same as the length of the // data field, as the logging payload can be truncated or omitted. Length uint32 `protobuf:"varint,1,opt,name=length,proto3" json:"length,omitempty"` // May be truncated or omitted. - Data []byte `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Data []byte `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"` } -func (m *Message) Reset() { *m = Message{} } -func (m *Message) String() string { return proto.CompactTextString(m) } -func (*Message) ProtoMessage() {} -func (*Message) Descriptor() ([]byte, []int) { - return fileDescriptor_b7972e58de45083a, []int{4} +func (x *Message) Reset() { + *x = Message{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_binlog_v1_binarylog_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *Message) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Message.Unmarshal(m, b) -} -func (m *Message) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Message.Marshal(b, m, deterministic) -} -func (m *Message) XXX_Merge(src proto.Message) { - xxx_messageInfo_Message.Merge(m, src) -} -func (m *Message) XXX_Size() int { - return xxx_messageInfo_Message.Size(m) +func (x *Message) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *Message) XXX_DiscardUnknown() { - xxx_messageInfo_Message.DiscardUnknown(m) + +func (*Message) ProtoMessage() {} + +func (x *Message) ProtoReflect() protoreflect.Message { + mi := &file_grpc_binlog_v1_binarylog_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_Message proto.InternalMessageInfo +// Deprecated: Use Message.ProtoReflect.Descriptor instead. +func (*Message) Descriptor() ([]byte, []int) { + return file_grpc_binlog_v1_binarylog_proto_rawDescGZIP(), []int{4} +} -func (m *Message) GetLength() uint32 { - if m != nil { - return m.Length +func (x *Message) GetLength() uint32 { + if x != nil { + return x.Length } return 0 } -func (m *Message) GetData() []byte { - if m != nil { - return m.Data +func (x *Message) GetData() []byte { + if x != nil { + return x.Data } return nil } @@ -577,12 +690,12 @@ func (m *Message) GetData() []byte { // Header keys added by gRPC are omitted. To be more specific, // implementations will not log the following entries, and this is // not to be treated as a truncation: -// - entries handled by grpc that are not user visible, such as those -// that begin with 'grpc-' (with exception of grpc-trace-bin) -// or keys like 'lb-token' -// - transport specific entries, including but not limited to: -// ':path', ':authority', 'content-encoding', 'user-agent', 'te', etc -// - entries added for call credentials +// - entries handled by grpc that are not user visible, such as those +// that begin with 'grpc-' (with exception of grpc-trace-bin) +// or keys like 'lb-token' +// - transport specific entries, including but not limited to: +// ':path', ':authority', 'content-encoding', 'user-agent', 'te', etc +// - entries added for call credentials // // Implementations must always log grpc-trace-bin if it is present. // Practically speaking it will only be visible on server side because @@ -591,222 +704,480 @@ func (m *Message) GetData() []byte { // header is just a normal metadata key. // The pair will not count towards the size limit. type Metadata struct { - Entry []*MetadataEntry `protobuf:"bytes,1,rep,name=entry,proto3" json:"entry,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields -func (m *Metadata) Reset() { *m = Metadata{} } -func (m *Metadata) String() string { return proto.CompactTextString(m) } -func (*Metadata) ProtoMessage() {} -func (*Metadata) Descriptor() ([]byte, []int) { - return fileDescriptor_b7972e58de45083a, []int{5} + Entry []*MetadataEntry `protobuf:"bytes,1,rep,name=entry,proto3" json:"entry,omitempty"` } -func (m *Metadata) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Metadata.Unmarshal(m, b) -} -func (m *Metadata) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Metadata.Marshal(b, m, deterministic) -} -func (m *Metadata) XXX_Merge(src proto.Message) { - xxx_messageInfo_Metadata.Merge(m, src) +func (x *Metadata) Reset() { + *x = Metadata{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_binlog_v1_binarylog_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *Metadata) XXX_Size() int { - return xxx_messageInfo_Metadata.Size(m) + +func (x *Metadata) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *Metadata) XXX_DiscardUnknown() { - xxx_messageInfo_Metadata.DiscardUnknown(m) + +func (*Metadata) ProtoMessage() {} + +func (x *Metadata) ProtoReflect() protoreflect.Message { + mi := &file_grpc_binlog_v1_binarylog_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_Metadata proto.InternalMessageInfo +// Deprecated: Use Metadata.ProtoReflect.Descriptor instead. +func (*Metadata) Descriptor() ([]byte, []int) { + return file_grpc_binlog_v1_binarylog_proto_rawDescGZIP(), []int{5} +} -func (m *Metadata) GetEntry() []*MetadataEntry { - if m != nil { - return m.Entry +func (x *Metadata) GetEntry() []*MetadataEntry { + if x != nil { + return x.Entry } return nil } // A metadata key value pair type MetadataEntry struct { - Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` - Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields -func (m *MetadataEntry) Reset() { *m = MetadataEntry{} } -func (m *MetadataEntry) String() string { return proto.CompactTextString(m) } -func (*MetadataEntry) ProtoMessage() {} -func (*MetadataEntry) Descriptor() ([]byte, []int) { - return fileDescriptor_b7972e58de45083a, []int{6} + Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` } -func (m *MetadataEntry) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_MetadataEntry.Unmarshal(m, b) -} -func (m *MetadataEntry) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_MetadataEntry.Marshal(b, m, deterministic) -} -func (m *MetadataEntry) XXX_Merge(src proto.Message) { - xxx_messageInfo_MetadataEntry.Merge(m, src) +func (x *MetadataEntry) Reset() { + *x = MetadataEntry{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_binlog_v1_binarylog_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *MetadataEntry) XXX_Size() int { - return xxx_messageInfo_MetadataEntry.Size(m) + +func (x *MetadataEntry) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *MetadataEntry) XXX_DiscardUnknown() { - xxx_messageInfo_MetadataEntry.DiscardUnknown(m) + +func (*MetadataEntry) ProtoMessage() {} + +func (x *MetadataEntry) ProtoReflect() protoreflect.Message { + mi := &file_grpc_binlog_v1_binarylog_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_MetadataEntry proto.InternalMessageInfo +// Deprecated: Use MetadataEntry.ProtoReflect.Descriptor instead. +func (*MetadataEntry) Descriptor() ([]byte, []int) { + return file_grpc_binlog_v1_binarylog_proto_rawDescGZIP(), []int{6} +} -func (m *MetadataEntry) GetKey() string { - if m != nil { - return m.Key +func (x *MetadataEntry) GetKey() string { + if x != nil { + return x.Key } return "" } -func (m *MetadataEntry) GetValue() []byte { - if m != nil { - return m.Value +func (x *MetadataEntry) GetValue() []byte { + if x != nil { + return x.Value } return nil } // Address information type Address struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + Type Address_Type `protobuf:"varint,1,opt,name=type,proto3,enum=grpc.binarylog.v1.Address_Type" json:"type,omitempty"` Address string `protobuf:"bytes,2,opt,name=address,proto3" json:"address,omitempty"` // only for TYPE_IPV4 and TYPE_IPV6 - IpPort uint32 `protobuf:"varint,3,opt,name=ip_port,json=ipPort,proto3" json:"ip_port,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + IpPort uint32 `protobuf:"varint,3,opt,name=ip_port,json=ipPort,proto3" json:"ip_port,omitempty"` } -func (m *Address) Reset() { *m = Address{} } -func (m *Address) String() string { return proto.CompactTextString(m) } -func (*Address) ProtoMessage() {} -func (*Address) Descriptor() ([]byte, []int) { - return fileDescriptor_b7972e58de45083a, []int{7} +func (x *Address) Reset() { + *x = Address{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_binlog_v1_binarylog_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *Address) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Address.Unmarshal(m, b) +func (x *Address) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *Address) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Address.Marshal(b, m, deterministic) -} -func (m *Address) XXX_Merge(src proto.Message) { - xxx_messageInfo_Address.Merge(m, src) -} -func (m *Address) XXX_Size() int { - return xxx_messageInfo_Address.Size(m) -} -func (m *Address) XXX_DiscardUnknown() { - xxx_messageInfo_Address.DiscardUnknown(m) + +func (*Address) ProtoMessage() {} + +func (x *Address) ProtoReflect() protoreflect.Message { + mi := &file_grpc_binlog_v1_binarylog_proto_msgTypes[7] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_Address proto.InternalMessageInfo +// Deprecated: Use Address.ProtoReflect.Descriptor instead. +func (*Address) Descriptor() ([]byte, []int) { + return file_grpc_binlog_v1_binarylog_proto_rawDescGZIP(), []int{7} +} -func (m *Address) GetType() Address_Type { - if m != nil { - return m.Type +func (x *Address) GetType() Address_Type { + if x != nil { + return x.Type } return Address_TYPE_UNKNOWN } -func (m *Address) GetAddress() string { - if m != nil { - return m.Address +func (x *Address) GetAddress() string { + if x != nil { + return x.Address } return "" } -func (m *Address) GetIpPort() uint32 { - if m != nil { - return m.IpPort +func (x *Address) GetIpPort() uint32 { + if x != nil { + return x.IpPort } return 0 } -func init() { - proto.RegisterEnum("grpc.binarylog.v1.GrpcLogEntry_EventType", GrpcLogEntry_EventType_name, GrpcLogEntry_EventType_value) - proto.RegisterEnum("grpc.binarylog.v1.GrpcLogEntry_Logger", GrpcLogEntry_Logger_name, GrpcLogEntry_Logger_value) - proto.RegisterEnum("grpc.binarylog.v1.Address_Type", Address_Type_name, Address_Type_value) - proto.RegisterType((*GrpcLogEntry)(nil), "grpc.binarylog.v1.GrpcLogEntry") - proto.RegisterType((*ClientHeader)(nil), "grpc.binarylog.v1.ClientHeader") - proto.RegisterType((*ServerHeader)(nil), "grpc.binarylog.v1.ServerHeader") - proto.RegisterType((*Trailer)(nil), "grpc.binarylog.v1.Trailer") - proto.RegisterType((*Message)(nil), "grpc.binarylog.v1.Message") - proto.RegisterType((*Metadata)(nil), "grpc.binarylog.v1.Metadata") - proto.RegisterType((*MetadataEntry)(nil), "grpc.binarylog.v1.MetadataEntry") - proto.RegisterType((*Address)(nil), "grpc.binarylog.v1.Address") -} - -func init() { proto.RegisterFile("grpc/binlog/v1/binarylog.proto", fileDescriptor_b7972e58de45083a) } - -var fileDescriptor_b7972e58de45083a = []byte{ - // 904 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x55, 0x51, 0x6f, 0xe3, 0x44, - 0x10, 0xae, 0xdb, 0x34, 0x6e, 0x26, 0x49, 0xe5, 0xae, 0xca, 0x9d, 0xaf, 0x94, 0x6b, 0x64, 0x09, - 0x14, 0x84, 0xe4, 0xa8, 0x29, 0xd7, 0xe3, 0x05, 0xa4, 0x24, 0xf5, 0xa5, 0x11, 0xb9, 0x34, 0xda, - 0xe4, 0x7a, 0x80, 0x90, 0xac, 0x6d, 0xbc, 0x38, 0x16, 0x8e, 0xd7, 0xac, 0x37, 0x41, 0xf9, 0x59, - 0xbc, 0x21, 0xdd, 0xef, 0xe2, 0x1d, 0x79, 0xd7, 0x4e, 0x4d, 0xd3, 0x82, 0xc4, 0xbd, 0xed, 0x7c, - 0xf3, 0xcd, 0x37, 0xbb, 0xe3, 0x99, 0x31, 0xbc, 0xf4, 0x79, 0x3c, 0x6b, 0xdd, 0x05, 0x51, 0xc8, - 0xfc, 0xd6, 0xea, 0x3c, 0x3d, 0x11, 0xbe, 0x0e, 0x99, 0x6f, 0xc7, 0x9c, 0x09, 0x86, 0x8e, 0x52, - 0xbf, 0x7d, 0x8f, 0xae, 0xce, 0x4f, 0x5e, 0xfa, 0x8c, 0xf9, 0x21, 0x6d, 0x49, 0xc2, 0xdd, 0xf2, - 0x97, 0x96, 0xb7, 0xe4, 0x44, 0x04, 0x2c, 0x52, 0x21, 0x27, 0x67, 0x0f, 0xfd, 0x22, 0x58, 0xd0, - 0x44, 0x90, 0x45, 0xac, 0x08, 0xd6, 0x07, 0x1d, 0x6a, 0x7d, 0x1e, 0xcf, 0x86, 0xcc, 0x77, 0x22, - 0xc1, 0xd7, 0xe8, 0x1b, 0xa8, 0x6c, 0x38, 0xa6, 0xd6, 0xd0, 0x9a, 0xd5, 0xf6, 0x89, 0xad, 0x54, - 0xec, 0x5c, 0xc5, 0x9e, 0xe6, 0x0c, 0x7c, 0x4f, 0x46, 0xcf, 0x41, 0x9f, 0x91, 0x30, 0x74, 0x03, - 0xcf, 0xdc, 0x6d, 0x68, 0xcd, 0x12, 0x2e, 0xa7, 0xe6, 0xc0, 0x43, 0xaf, 0xe0, 0x79, 0x42, 0x7f, - 0x5b, 0xd2, 0x68, 0x46, 0xdd, 0xc0, 0x73, 0x7f, 0x0f, 0xc4, 0x3c, 0x88, 0xdc, 0xd4, 0x69, 0xee, - 0x49, 0xe2, 0x71, 0xee, 0x1e, 0x78, 0xef, 0xa5, 0xb3, 0x47, 0xc2, 0x10, 0x7d, 0x0b, 0x25, 0xb1, - 0x8e, 0xa9, 0x59, 0x6a, 0x68, 0xcd, 0xc3, 0xf6, 0x97, 0xf6, 0xd6, 0xeb, 0xed, 0xe2, 0xc5, 0x6d, - 0x67, 0x45, 0x23, 0x31, 0x5d, 0xc7, 0x14, 0xcb, 0x30, 0xf4, 0x1d, 0x94, 0x43, 0xe6, 0xfb, 0x94, - 0x9b, 0xfb, 0x52, 0xe0, 0x8b, 0xff, 0x12, 0x18, 0x4a, 0x36, 0xce, 0xa2, 0xd0, 0x1b, 0xa8, 0xcf, - 0xc2, 0x80, 0x46, 0xc2, 0x9d, 0x53, 0xe2, 0x51, 0x6e, 0x96, 0x65, 0x31, 0xce, 0x1e, 0x91, 0xe9, - 0x49, 0xde, 0xb5, 0xa4, 0x5d, 0xef, 0xe0, 0xda, 0xac, 0x60, 0xa7, 0x3a, 0x09, 0xe5, 0x2b, 0xca, - 0x73, 0x1d, 0xfd, 0x49, 0x9d, 0x89, 0xe4, 0xdd, 0xeb, 0x24, 0x05, 0x1b, 0x5d, 0x82, 0xbe, 0xa0, - 0x49, 0x42, 0x7c, 0x6a, 0x1e, 0xe4, 0x9f, 0x65, 0x4b, 0xe1, 0xad, 0x62, 0x5c, 0xef, 0xe0, 0x9c, - 0x9c, 0xc6, 0x09, 0x4e, 0x82, 0x90, 0x72, 0xb3, 0xf2, 0x64, 0xdc, 0x54, 0x31, 0xd2, 0xb8, 0x8c, - 0x8c, 0xbe, 0x82, 0xa3, 0x98, 0xac, 0x43, 0x46, 0x3c, 0x57, 0xf0, 0x65, 0x34, 0x23, 0x82, 0x7a, - 0x26, 0x34, 0xb4, 0xe6, 0x01, 0x36, 0x32, 0xc7, 0x34, 0xc7, 0x91, 0x0d, 0xa5, 0x98, 0x52, 0x6e, - 0x56, 0x9f, 0xcc, 0xd0, 0xf1, 0x3c, 0x4e, 0x93, 0x04, 0x4b, 0x9e, 0xf5, 0x97, 0x06, 0x95, 0xcd, - 0x07, 0x43, 0xcf, 0x00, 0x39, 0xb7, 0xce, 0x68, 0xea, 0x4e, 0x7f, 0x1c, 0x3b, 0xee, 0xbb, 0xd1, - 0xf7, 0xa3, 0x9b, 0xf7, 0x23, 0x63, 0x07, 0x9d, 0x82, 0x59, 0xc0, 0x7b, 0xc3, 0x41, 0x7a, 0xbe, - 0x76, 0x3a, 0x57, 0x0e, 0x36, 0xb4, 0x07, 0xde, 0x89, 0x83, 0x6f, 0x1d, 0x9c, 0x7b, 0x77, 0xd1, - 0x67, 0xf0, 0x62, 0x3b, 0xf6, 0xad, 0x33, 0x99, 0x74, 0xfa, 0x8e, 0xb1, 0xf7, 0xc0, 0x9d, 0x05, - 0xe7, 0xee, 0x12, 0x6a, 0xc0, 0xe9, 0x23, 0x99, 0x3b, 0xc3, 0x37, 0x6e, 0x6f, 0x78, 0x33, 0x71, - 0x8c, 0xfd, 0xc7, 0x05, 0xa6, 0xb8, 0x33, 0x18, 0x3a, 0xd8, 0x28, 0xa3, 0x4f, 0xe0, 0xa8, 0x28, - 0xd0, 0x19, 0xf5, 0x9c, 0xa1, 0xa1, 0x5b, 0x5d, 0x28, 0xab, 0x36, 0x43, 0x08, 0x0e, 0x87, 0x37, - 0xfd, 0xbe, 0x83, 0x0b, 0xef, 0x3d, 0x82, 0x7a, 0x86, 0xa9, 0x8c, 0x86, 0x56, 0x80, 0x54, 0x0a, - 0x63, 0xb7, 0x5b, 0x01, 0x3d, 0xab, 0xbf, 0xf5, 0x41, 0x83, 0x5a, 0xb1, 0xf9, 0xd0, 0x6b, 0x38, - 0x58, 0x50, 0x41, 0x3c, 0x22, 0x48, 0x36, 0xbc, 0x9f, 0x3e, 0xda, 0x25, 0x8a, 0x82, 0x37, 0x64, - 0x74, 0x06, 0xd5, 0x05, 0x15, 0x73, 0xe6, 0xb9, 0x11, 0x59, 0x50, 0x39, 0xc0, 0x15, 0x0c, 0x0a, - 0x1a, 0x91, 0x05, 0x45, 0xa7, 0x50, 0x21, 0x4b, 0x31, 0x67, 0x3c, 0x10, 0x6b, 0x39, 0xb6, 0x15, - 0x7c, 0x0f, 0xa0, 0x0b, 0xd0, 0xd3, 0x45, 0xc0, 0x96, 0x42, 0x8e, 0x6b, 0xb5, 0xfd, 0x62, 0x6b, - 0x67, 0x5c, 0x65, 0x9b, 0x09, 0xe7, 0x4c, 0xab, 0x0f, 0xb5, 0x62, 0xc7, 0xff, 0xef, 0xcb, 0x5b, - 0x7f, 0x68, 0xa0, 0x67, 0x1d, 0xfc, 0x51, 0x15, 0x48, 0x04, 0x11, 0xcb, 0xc4, 0x9d, 0x31, 0x4f, - 0x55, 0xa0, 0x8e, 0x41, 0x41, 0x3d, 0xe6, 0x51, 0xf4, 0x39, 0x1c, 0x66, 0x84, 0x7c, 0x0e, 0x55, - 0x19, 0xea, 0x0a, 0xcd, 0x46, 0xaf, 0x40, 0xf3, 0xa8, 0x20, 0x41, 0x98, 0xc8, 0x8a, 0xd4, 0x72, - 0xda, 0x95, 0x02, 0xad, 0x57, 0xa0, 0xe7, 0x11, 0xcf, 0xa0, 0x1c, 0xd2, 0xc8, 0x17, 0x73, 0x79, - 0xe1, 0x3a, 0xce, 0x2c, 0x84, 0xa0, 0x24, 0x9f, 0xb1, 0x2b, 0xe3, 0xe5, 0xd9, 0xea, 0xc2, 0x41, - 0x7e, 0x77, 0x74, 0x09, 0xfb, 0x34, 0xdd, 0x5c, 0xa6, 0xd6, 0xd8, 0x6b, 0x56, 0xdb, 0x8d, 0x7f, - 0x79, 0xa7, 0xdc, 0x70, 0x58, 0xd1, 0xad, 0xd7, 0x50, 0xff, 0x07, 0x8e, 0x0c, 0xd8, 0xfb, 0x95, - 0xae, 0x65, 0xf6, 0x0a, 0x4e, 0x8f, 0xe8, 0x18, 0xf6, 0x57, 0x24, 0x5c, 0xd2, 0x2c, 0xb7, 0x32, - 0xac, 0x3f, 0x35, 0xd0, 0xb3, 0x39, 0x46, 0x17, 0xd9, 0x76, 0xd6, 0xe4, 0x72, 0x3d, 0x7b, 0x7a, - 0xe2, 0xed, 0xc2, 0x4e, 0x36, 0x41, 0x27, 0x0a, 0xcd, 0x3a, 0x2c, 0x37, 0xd3, 0x9f, 0x47, 0x10, - 0xbb, 0x31, 0xe3, 0x42, 0x56, 0xb5, 0x8e, 0xcb, 0x41, 0x3c, 0x66, 0x5c, 0x58, 0x0e, 0x94, 0xe4, - 0x8e, 0x30, 0xa0, 0xf6, 0x60, 0x3b, 0xd4, 0xa1, 0x22, 0x91, 0xc1, 0xf8, 0xf6, 0x6b, 0x43, 0x2b, - 0x9a, 0x97, 0xc6, 0xee, 0xc6, 0x7c, 0x37, 0x1a, 0xfc, 0x60, 0xec, 0x75, 0x7f, 0x86, 0xe3, 0x80, - 0x6d, 0x5f, 0xb2, 0x7b, 0xd8, 0x95, 0xd6, 0x90, 0xf9, 0xe3, 0xb4, 0x51, 0xc7, 0xda, 0x4f, 0xed, - 0xac, 0x71, 0x7d, 0x16, 0x92, 0xc8, 0xb7, 0x19, 0xf7, 0x5b, 0xf9, 0x7f, 0x59, 0x85, 0x49, 0xd3, - 0xdd, 0x98, 0xee, 0xea, 0xfc, 0xae, 0x2c, 0xbb, 0xfc, 0xe2, 0xef, 0x00, 0x00, 0x00, 0xff, 0xff, - 0x10, 0x93, 0x68, 0x41, 0xc2, 0x07, 0x00, 0x00, +var File_grpc_binlog_v1_binarylog_proto protoreflect.FileDescriptor + +var file_grpc_binlog_v1_binarylog_proto_rawDesc = []byte{ + 0x0a, 0x1e, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x62, 0x69, 0x6e, 0x6c, 0x6f, 0x67, 0x2f, 0x76, 0x31, + 0x2f, 0x62, 0x69, 0x6e, 0x61, 0x72, 0x79, 0x6c, 0x6f, 0x67, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x12, 0x11, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x62, 0x69, 0x6e, 0x61, 0x72, 0x79, 0x6c, 0x6f, 0x67, + 0x2e, 0x76, 0x31, 0x1a, 0x1e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xbb, 0x07, 0x0a, 0x0c, 0x47, 0x72, 0x70, 0x63, 0x4c, 0x6f, 0x67, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x38, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, + 0x6d, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, + 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, + 0x17, 0x0a, 0x07, 0x63, 0x61, 0x6c, 0x6c, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, + 0x52, 0x06, 0x63, 0x61, 0x6c, 0x6c, 0x49, 0x64, 0x12, 0x35, 0x0a, 0x17, 0x73, 0x65, 0x71, 0x75, + 0x65, 0x6e, 0x63, 0x65, 0x5f, 0x69, 0x64, 0x5f, 0x77, 0x69, 0x74, 0x68, 0x69, 0x6e, 0x5f, 0x63, + 0x61, 0x6c, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x14, 0x73, 0x65, 0x71, 0x75, 0x65, + 0x6e, 0x63, 0x65, 0x49, 0x64, 0x57, 0x69, 0x74, 0x68, 0x69, 0x6e, 0x43, 0x61, 0x6c, 0x6c, 0x12, + 0x3d, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x29, 0x2e, + 0x67, 0x72, 0x70, 0x63, 0x2e, 0x62, 0x69, 0x6e, 0x61, 0x72, 0x79, 0x6c, 0x6f, 0x67, 0x2e, 0x76, + 0x31, 0x2e, 0x47, 0x72, 0x70, 0x63, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x2e, 0x45, + 0x76, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x3e, + 0x0a, 0x06, 0x6c, 0x6f, 0x67, 0x67, 0x65, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x26, + 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x62, 0x69, 0x6e, 0x61, 0x72, 0x79, 0x6c, 0x6f, 0x67, 0x2e, + 0x76, 0x31, 0x2e, 0x47, 0x72, 0x70, 0x63, 0x4c, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x2e, + 0x4c, 0x6f, 0x67, 0x67, 0x65, 0x72, 0x52, 0x06, 0x6c, 0x6f, 0x67, 0x67, 0x65, 0x72, 0x12, 0x46, + 0x0a, 0x0d, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x18, + 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x62, 0x69, 0x6e, + 0x61, 0x72, 0x79, 0x6c, 0x6f, 0x67, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, + 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x48, 0x00, 0x52, 0x0c, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, + 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x12, 0x46, 0x0a, 0x0d, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, + 0x5f, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, + 0x67, 0x72, 0x70, 0x63, 0x2e, 0x62, 0x69, 0x6e, 0x61, 0x72, 0x79, 0x6c, 0x6f, 0x67, 0x2e, 0x76, + 0x31, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x48, 0x00, + 0x52, 0x0c, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x12, 0x36, + 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x62, 0x69, 0x6e, 0x61, 0x72, 0x79, 0x6c, 0x6f, 0x67, + 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x48, 0x00, 0x52, 0x07, 0x6d, + 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x36, 0x0a, 0x07, 0x74, 0x72, 0x61, 0x69, 0x6c, 0x65, + 0x72, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x62, + 0x69, 0x6e, 0x61, 0x72, 0x79, 0x6c, 0x6f, 0x67, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x72, 0x61, 0x69, + 0x6c, 0x65, 0x72, 0x48, 0x00, 0x52, 0x07, 0x74, 0x72, 0x61, 0x69, 0x6c, 0x65, 0x72, 0x12, 0x2b, + 0x0a, 0x11, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x74, 0x72, 0x75, 0x6e, 0x63, 0x61, + 0x74, 0x65, 0x64, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x70, 0x61, 0x79, 0x6c, 0x6f, + 0x61, 0x64, 0x54, 0x72, 0x75, 0x6e, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x2e, 0x0a, 0x04, 0x70, + 0x65, 0x65, 0x72, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x62, 0x69, 0x6e, 0x61, 0x72, 0x79, 0x6c, 0x6f, 0x67, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x64, + 0x64, 0x72, 0x65, 0x73, 0x73, 0x52, 0x04, 0x70, 0x65, 0x65, 0x72, 0x22, 0xf5, 0x01, 0x0a, 0x09, + 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x12, 0x45, 0x56, 0x45, + 0x4e, 0x54, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, + 0x00, 0x12, 0x1c, 0x0a, 0x18, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, + 0x43, 0x4c, 0x49, 0x45, 0x4e, 0x54, 0x5f, 0x48, 0x45, 0x41, 0x44, 0x45, 0x52, 0x10, 0x01, 0x12, + 0x1c, 0x0a, 0x18, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x45, + 0x52, 0x56, 0x45, 0x52, 0x5f, 0x48, 0x45, 0x41, 0x44, 0x45, 0x52, 0x10, 0x02, 0x12, 0x1d, 0x0a, + 0x19, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x4c, 0x49, 0x45, + 0x4e, 0x54, 0x5f, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x10, 0x03, 0x12, 0x1d, 0x0a, 0x19, + 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x45, + 0x52, 0x5f, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x10, 0x04, 0x12, 0x20, 0x0a, 0x1c, 0x45, + 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x4c, 0x49, 0x45, 0x4e, 0x54, + 0x5f, 0x48, 0x41, 0x4c, 0x46, 0x5f, 0x43, 0x4c, 0x4f, 0x53, 0x45, 0x10, 0x05, 0x12, 0x1d, 0x0a, + 0x19, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x45, 0x52, 0x56, + 0x45, 0x52, 0x5f, 0x54, 0x52, 0x41, 0x49, 0x4c, 0x45, 0x52, 0x10, 0x06, 0x12, 0x15, 0x0a, 0x11, + 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x41, 0x4e, 0x43, 0x45, + 0x4c, 0x10, 0x07, 0x22, 0x42, 0x0a, 0x06, 0x4c, 0x6f, 0x67, 0x67, 0x65, 0x72, 0x12, 0x12, 0x0a, + 0x0e, 0x4c, 0x4f, 0x47, 0x47, 0x45, 0x52, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, + 0x00, 0x12, 0x11, 0x0a, 0x0d, 0x4c, 0x4f, 0x47, 0x47, 0x45, 0x52, 0x5f, 0x43, 0x4c, 0x49, 0x45, + 0x4e, 0x54, 0x10, 0x01, 0x12, 0x11, 0x0a, 0x0d, 0x4c, 0x4f, 0x47, 0x47, 0x45, 0x52, 0x5f, 0x53, + 0x45, 0x52, 0x56, 0x45, 0x52, 0x10, 0x02, 0x42, 0x09, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, + 0x61, 0x64, 0x22, 0xbb, 0x01, 0x0a, 0x0c, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x48, 0x65, 0x61, + 0x64, 0x65, 0x72, 0x12, 0x37, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x62, 0x69, 0x6e, + 0x61, 0x72, 0x79, 0x6c, 0x6f, 0x67, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, + 0x74, 0x61, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x1f, 0x0a, 0x0b, + 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0a, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x0a, + 0x09, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x09, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x12, 0x33, 0x0a, 0x07, 0x74, + 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, + 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x07, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, + 0x22, 0x47, 0x0a, 0x0c, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, + 0x12, 0x37, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x62, 0x69, 0x6e, 0x61, 0x72, 0x79, + 0x6c, 0x6f, 0x67, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, + 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x22, 0xb1, 0x01, 0x0a, 0x07, 0x54, 0x72, + 0x61, 0x69, 0x6c, 0x65, 0x72, 0x12, 0x37, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, + 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x62, + 0x69, 0x6e, 0x61, 0x72, 0x79, 0x6c, 0x6f, 0x67, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x65, 0x74, 0x61, + 0x64, 0x61, 0x74, 0x61, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x1f, + 0x0a, 0x0b, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0d, 0x52, 0x0a, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x43, 0x6f, 0x64, 0x65, 0x12, + 0x25, 0x0a, 0x0e, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, + 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x4d, + 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x25, 0x0a, 0x0e, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x5f, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0d, + 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x22, 0x35, 0x0a, + 0x07, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x6c, 0x65, 0x6e, 0x67, + 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, + 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, + 0x64, 0x61, 0x74, 0x61, 0x22, 0x42, 0x0a, 0x08, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x12, 0x36, 0x0a, 0x05, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x20, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x62, 0x69, 0x6e, 0x61, 0x72, 0x79, 0x6c, 0x6f, 0x67, + 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x52, 0x05, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x22, 0x37, 0x0a, 0x0d, 0x4d, 0x65, 0x74, 0x61, + 0x64, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x22, 0xb8, 0x01, 0x0a, 0x07, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x33, 0x0a, + 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1f, 0x2e, 0x67, 0x72, + 0x70, 0x63, 0x2e, 0x62, 0x69, 0x6e, 0x61, 0x72, 0x79, 0x6c, 0x6f, 0x67, 0x2e, 0x76, 0x31, 0x2e, + 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x2e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, + 0x70, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x17, 0x0a, 0x07, + 0x69, 0x70, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x69, + 0x70, 0x50, 0x6f, 0x72, 0x74, 0x22, 0x45, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, 0x10, 0x0a, + 0x0c, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, + 0x0d, 0x0a, 0x09, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x49, 0x50, 0x56, 0x34, 0x10, 0x01, 0x12, 0x0d, + 0x0a, 0x09, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x49, 0x50, 0x56, 0x36, 0x10, 0x02, 0x12, 0x0d, 0x0a, + 0x09, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x49, 0x58, 0x10, 0x03, 0x42, 0x5c, 0x0a, 0x14, + 0x69, 0x6f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x62, 0x69, 0x6e, 0x61, 0x72, 0x79, 0x6c, 0x6f, + 0x67, 0x2e, 0x76, 0x31, 0x42, 0x0e, 0x42, 0x69, 0x6e, 0x61, 0x72, 0x79, 0x4c, 0x6f, 0x67, 0x50, + 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x32, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67, + 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x62, + 0x69, 0x6e, 0x61, 0x72, 0x79, 0x6c, 0x6f, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x62, 0x69, + 0x6e, 0x61, 0x72, 0x79, 0x6c, 0x6f, 0x67, 0x5f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x33, +} + +var ( + file_grpc_binlog_v1_binarylog_proto_rawDescOnce sync.Once + file_grpc_binlog_v1_binarylog_proto_rawDescData = file_grpc_binlog_v1_binarylog_proto_rawDesc +) + +func file_grpc_binlog_v1_binarylog_proto_rawDescGZIP() []byte { + file_grpc_binlog_v1_binarylog_proto_rawDescOnce.Do(func() { + file_grpc_binlog_v1_binarylog_proto_rawDescData = protoimpl.X.CompressGZIP(file_grpc_binlog_v1_binarylog_proto_rawDescData) + }) + return file_grpc_binlog_v1_binarylog_proto_rawDescData +} + +var file_grpc_binlog_v1_binarylog_proto_enumTypes = make([]protoimpl.EnumInfo, 3) +var file_grpc_binlog_v1_binarylog_proto_msgTypes = make([]protoimpl.MessageInfo, 8) +var file_grpc_binlog_v1_binarylog_proto_goTypes = []interface{}{ + (GrpcLogEntry_EventType)(0), // 0: grpc.binarylog.v1.GrpcLogEntry.EventType + (GrpcLogEntry_Logger)(0), // 1: grpc.binarylog.v1.GrpcLogEntry.Logger + (Address_Type)(0), // 2: grpc.binarylog.v1.Address.Type + (*GrpcLogEntry)(nil), // 3: grpc.binarylog.v1.GrpcLogEntry + (*ClientHeader)(nil), // 4: grpc.binarylog.v1.ClientHeader + (*ServerHeader)(nil), // 5: grpc.binarylog.v1.ServerHeader + (*Trailer)(nil), // 6: grpc.binarylog.v1.Trailer + (*Message)(nil), // 7: grpc.binarylog.v1.Message + (*Metadata)(nil), // 8: grpc.binarylog.v1.Metadata + (*MetadataEntry)(nil), // 9: grpc.binarylog.v1.MetadataEntry + (*Address)(nil), // 10: grpc.binarylog.v1.Address + (*timestamppb.Timestamp)(nil), // 11: google.protobuf.Timestamp + (*durationpb.Duration)(nil), // 12: google.protobuf.Duration +} +var file_grpc_binlog_v1_binarylog_proto_depIdxs = []int32{ + 11, // 0: grpc.binarylog.v1.GrpcLogEntry.timestamp:type_name -> google.protobuf.Timestamp + 0, // 1: grpc.binarylog.v1.GrpcLogEntry.type:type_name -> grpc.binarylog.v1.GrpcLogEntry.EventType + 1, // 2: grpc.binarylog.v1.GrpcLogEntry.logger:type_name -> grpc.binarylog.v1.GrpcLogEntry.Logger + 4, // 3: grpc.binarylog.v1.GrpcLogEntry.client_header:type_name -> grpc.binarylog.v1.ClientHeader + 5, // 4: grpc.binarylog.v1.GrpcLogEntry.server_header:type_name -> grpc.binarylog.v1.ServerHeader + 7, // 5: grpc.binarylog.v1.GrpcLogEntry.message:type_name -> grpc.binarylog.v1.Message + 6, // 6: grpc.binarylog.v1.GrpcLogEntry.trailer:type_name -> grpc.binarylog.v1.Trailer + 10, // 7: grpc.binarylog.v1.GrpcLogEntry.peer:type_name -> grpc.binarylog.v1.Address + 8, // 8: grpc.binarylog.v1.ClientHeader.metadata:type_name -> grpc.binarylog.v1.Metadata + 12, // 9: grpc.binarylog.v1.ClientHeader.timeout:type_name -> google.protobuf.Duration + 8, // 10: grpc.binarylog.v1.ServerHeader.metadata:type_name -> grpc.binarylog.v1.Metadata + 8, // 11: grpc.binarylog.v1.Trailer.metadata:type_name -> grpc.binarylog.v1.Metadata + 9, // 12: grpc.binarylog.v1.Metadata.entry:type_name -> grpc.binarylog.v1.MetadataEntry + 2, // 13: grpc.binarylog.v1.Address.type:type_name -> grpc.binarylog.v1.Address.Type + 14, // [14:14] is the sub-list for method output_type + 14, // [14:14] is the sub-list for method input_type + 14, // [14:14] is the sub-list for extension type_name + 14, // [14:14] is the sub-list for extension extendee + 0, // [0:14] is the sub-list for field type_name +} + +func init() { file_grpc_binlog_v1_binarylog_proto_init() } +func file_grpc_binlog_v1_binarylog_proto_init() { + if File_grpc_binlog_v1_binarylog_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_grpc_binlog_v1_binarylog_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GrpcLogEntry); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_binlog_v1_binarylog_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ClientHeader); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_binlog_v1_binarylog_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ServerHeader); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_binlog_v1_binarylog_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Trailer); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_binlog_v1_binarylog_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Message); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_binlog_v1_binarylog_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Metadata); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_binlog_v1_binarylog_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*MetadataEntry); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_binlog_v1_binarylog_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Address); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + file_grpc_binlog_v1_binarylog_proto_msgTypes[0].OneofWrappers = []interface{}{ + (*GrpcLogEntry_ClientHeader)(nil), + (*GrpcLogEntry_ServerHeader)(nil), + (*GrpcLogEntry_Message)(nil), + (*GrpcLogEntry_Trailer)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_grpc_binlog_v1_binarylog_proto_rawDesc, + NumEnums: 3, + NumMessages: 8, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_grpc_binlog_v1_binarylog_proto_goTypes, + DependencyIndexes: file_grpc_binlog_v1_binarylog_proto_depIdxs, + EnumInfos: file_grpc_binlog_v1_binarylog_proto_enumTypes, + MessageInfos: file_grpc_binlog_v1_binarylog_proto_msgTypes, + }.Build() + File_grpc_binlog_v1_binarylog_proto = out.File + file_grpc_binlog_v1_binarylog_proto_rawDesc = nil + file_grpc_binlog_v1_binarylog_proto_goTypes = nil + file_grpc_binlog_v1_binarylog_proto_depIdxs = nil } diff --git a/binarylog/sink.go b/binarylog/sink.go new file mode 100644 index 000000000000..d924e4c91867 --- /dev/null +++ b/binarylog/sink.go @@ -0,0 +1,68 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package binarylog implementation binary logging as defined in +// https://github.com/grpc/proposal/blob/master/A16-binary-logging.md. +// +// Notice: All APIs in this package are experimental. +package binarylog + +import ( + "fmt" + "os" + + binlogpb "google.golang.org/grpc/binarylog/grpc_binarylog_v1" + iblog "google.golang.org/grpc/internal/binarylog" +) + +// SetSink sets the destination for the binary log entries. +// +// NOTE: this function must only be called during initialization time (i.e. in +// an init() function), and is not thread-safe. +func SetSink(s Sink) { + if iblog.DefaultSink != nil { + iblog.DefaultSink.Close() + } + iblog.DefaultSink = s +} + +// Sink represents the destination for the binary log entries. +type Sink interface { + // Write marshals the log entry and writes it to the destination. The format + // is not specified, but should have sufficient information to rebuild the + // entry. Some options are: proto bytes, or proto json. + // + // Note this function needs to be thread-safe. + Write(*binlogpb.GrpcLogEntry) error + // Close closes this sink and cleans up resources (e.g. the flushing + // goroutine). + Close() error +} + +// NewTempFileSink creates a temp file and returns a Sink that writes to this +// file. +func NewTempFileSink() (Sink, error) { + // Two other options to replace this function: + // 1. take filename as input. + // 2. export NewBufferedSink(). + tempFile, err := os.CreateTemp("/tmp", "grpcgo_binarylog_*.txt") + if err != nil { + return nil, fmt.Errorf("failed to create temp file: %v", err) + } + return iblog.NewBufferedSink(tempFile), nil +} diff --git a/call.go b/call.go index 9e20e4d385f9..a67a3db02eb4 100644 --- a/call.go +++ b/call.go @@ -26,7 +26,12 @@ import ( // received. This is typically called by generated code. // // All errors returned by Invoke are compatible with the status package. -func (cc *ClientConn) Invoke(ctx context.Context, method string, args, reply interface{}, opts ...CallOption) error { +func (cc *ClientConn) Invoke(ctx context.Context, method string, args, reply any, opts ...CallOption) error { + if err := cc.idlenessMgr.OnCallBegin(); err != nil { + return err + } + defer cc.idlenessMgr.OnCallEnd() + // allow interceptor to see all applicable call options, which means those // configured as defaults from dial option as well as per-call options opts = combine(cc.dopts.callOptions, opts) @@ -56,13 +61,13 @@ func combine(o1 []CallOption, o2 []CallOption) []CallOption { // received. This is typically called by generated code. // // DEPRECATED: Use ClientConn.Invoke instead. -func Invoke(ctx context.Context, method string, args, reply interface{}, cc *ClientConn, opts ...CallOption) error { +func Invoke(ctx context.Context, method string, args, reply any, cc *ClientConn, opts ...CallOption) error { return cc.Invoke(ctx, method, args, reply, opts...) } var unaryStreamDesc = &StreamDesc{ServerStreams: false, ClientStreams: false} -func invoke(ctx context.Context, method string, req, reply interface{}, cc *ClientConn, opts ...CallOption) error { +func invoke(ctx context.Context, method string, req, reply any, cc *ClientConn, opts ...CallOption) error { cs, err := newClientStream(ctx, unaryStreamDesc, cc, method, opts...) if err != nil { return err diff --git a/call_test.go b/call_test.go deleted file mode 100644 index 78760ba5297a..000000000000 --- a/call_test.go +++ /dev/null @@ -1,489 +0,0 @@ -/* - * - * Copyright 2014 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package grpc - -import ( - "context" - "fmt" - "io" - "math" - "net" - "strconv" - "strings" - "sync" - "testing" - "time" - - "google.golang.org/grpc/codes" - "google.golang.org/grpc/internal/transport" - "google.golang.org/grpc/status" -) - -var ( - expectedRequest = "ping" - expectedResponse = "pong" - weirdError = "format verbs: %v%s" - sizeLargeErr = 1024 * 1024 - canceled = 0 -) - -type testCodec struct { -} - -func (testCodec) Marshal(v interface{}) ([]byte, error) { - return []byte(*(v.(*string))), nil -} - -func (testCodec) Unmarshal(data []byte, v interface{}) error { - *(v.(*string)) = string(data) - return nil -} - -func (testCodec) String() string { - return "test" -} - -type testStreamHandler struct { - port string - t transport.ServerTransport -} - -func (h *testStreamHandler) handleStream(t *testing.T, s *transport.Stream) { - p := &parser{r: s} - for { - pf, req, err := p.recvMsg(math.MaxInt32) - if err == io.EOF { - break - } - if err != nil { - return - } - if pf != compressionNone { - t.Errorf("Received the mistaken message format %d, want %d", pf, compressionNone) - return - } - var v string - codec := testCodec{} - if err := codec.Unmarshal(req, &v); err != nil { - t.Errorf("Failed to unmarshal the received message: %v", err) - return - } - if v == "weird error" { - h.t.WriteStatus(s, status.New(codes.Internal, weirdError)) - return - } - if v == "canceled" { - canceled++ - h.t.WriteStatus(s, status.New(codes.Internal, "")) - return - } - if v == "port" { - h.t.WriteStatus(s, status.New(codes.Internal, h.port)) - return - } - - if v != expectedRequest { - h.t.WriteStatus(s, status.New(codes.Internal, strings.Repeat("A", sizeLargeErr))) - return - } - } - // send a response back to end the stream. - data, err := encode(testCodec{}, &expectedResponse) - if err != nil { - t.Errorf("Failed to encode the response: %v", err) - return - } - hdr, payload := msgHeader(data, nil) - h.t.Write(s, hdr, payload, &transport.Options{}) - h.t.WriteStatus(s, status.New(codes.OK, "")) -} - -type server struct { - lis net.Listener - port string - addr string - startedErr chan error // sent nil or an error after server starts - mu sync.Mutex - conns map[transport.ServerTransport]bool -} - -type ctxKey string - -func newTestServer() *server { - return &server{startedErr: make(chan error, 1)} -} - -// start starts server. Other goroutines should block on s.startedErr for further operations. -func (s *server) start(t *testing.T, port int, maxStreams uint32) { - var err error - if port == 0 { - s.lis, err = net.Listen("tcp", "localhost:0") - } else { - s.lis, err = net.Listen("tcp", "localhost:"+strconv.Itoa(port)) - } - if err != nil { - s.startedErr <- fmt.Errorf("failed to listen: %v", err) - return - } - s.addr = s.lis.Addr().String() - _, p, err := net.SplitHostPort(s.addr) - if err != nil { - s.startedErr <- fmt.Errorf("failed to parse listener address: %v", err) - return - } - s.port = p - s.conns = make(map[transport.ServerTransport]bool) - s.startedErr <- nil - for { - conn, err := s.lis.Accept() - if err != nil { - return - } - config := &transport.ServerConfig{ - MaxStreams: maxStreams, - } - st, err := transport.NewServerTransport("http2", conn, config) - if err != nil { - continue - } - s.mu.Lock() - if s.conns == nil { - s.mu.Unlock() - st.Close() - return - } - s.conns[st] = true - s.mu.Unlock() - h := &testStreamHandler{ - port: s.port, - t: st, - } - go st.HandleStreams(func(s *transport.Stream) { - go h.handleStream(t, s) - }, func(ctx context.Context, method string) context.Context { - return ctx - }) - } -} - -func (s *server) wait(t *testing.T, timeout time.Duration) { - select { - case err := <-s.startedErr: - if err != nil { - t.Fatal(err) - } - case <-time.After(timeout): - t.Fatalf("Timed out after %v waiting for server to be ready", timeout) - } -} - -func (s *server) stop() { - s.lis.Close() - s.mu.Lock() - for c := range s.conns { - c.Close() - } - s.conns = nil - s.mu.Unlock() -} - -func setUp(t *testing.T, port int, maxStreams uint32) (*server, *ClientConn) { - return setUpWithOptions(t, port, maxStreams) -} - -func setUpWithOptions(t *testing.T, port int, maxStreams uint32, dopts ...DialOption) (*server, *ClientConn) { - server := newTestServer() - go server.start(t, port, maxStreams) - server.wait(t, 2*time.Second) - addr := "localhost:" + server.port - dopts = append(dopts, WithBlock(), WithInsecure(), WithCodec(testCodec{})) - cc, err := Dial(addr, dopts...) - if err != nil { - t.Fatalf("Failed to create ClientConn: %v", err) - } - return server, cc -} - -func (s) TestUnaryClientInterceptor(t *testing.T) { - parentKey := ctxKey("parentKey") - - interceptor := func(ctx context.Context, method string, req, reply interface{}, cc *ClientConn, invoker UnaryInvoker, opts ...CallOption) error { - if ctx.Value(parentKey) == nil { - t.Fatalf("interceptor should have %v in context", parentKey) - } - return invoker(ctx, method, req, reply, cc, opts...) - } - - server, cc := setUpWithOptions(t, 0, math.MaxUint32, WithUnaryInterceptor(interceptor)) - defer func() { - cc.Close() - server.stop() - }() - - var reply string - ctx := context.Background() - parentCtx := context.WithValue(ctx, ctxKey("parentKey"), 0) - if err := cc.Invoke(parentCtx, "/foo/bar", &expectedRequest, &reply); err != nil || reply != expectedResponse { - t.Fatalf("grpc.Invoke(_, _, _, _, _) = %v, want ", err) - } -} - -func (s) TestChainUnaryClientInterceptor(t *testing.T) { - var ( - parentKey = ctxKey("parentKey") - firstIntKey = ctxKey("firstIntKey") - secondIntKey = ctxKey("secondIntKey") - ) - - firstInt := func(ctx context.Context, method string, req, reply interface{}, cc *ClientConn, invoker UnaryInvoker, opts ...CallOption) error { - if ctx.Value(parentKey) == nil { - t.Fatalf("first interceptor should have %v in context", parentKey) - } - if ctx.Value(firstIntKey) != nil { - t.Fatalf("first interceptor should not have %v in context", firstIntKey) - } - if ctx.Value(secondIntKey) != nil { - t.Fatalf("first interceptor should not have %v in context", secondIntKey) - } - firstCtx := context.WithValue(ctx, firstIntKey, 1) - err := invoker(firstCtx, method, req, reply, cc, opts...) - *(reply.(*string)) += "1" - return err - } - - secondInt := func(ctx context.Context, method string, req, reply interface{}, cc *ClientConn, invoker UnaryInvoker, opts ...CallOption) error { - if ctx.Value(parentKey) == nil { - t.Fatalf("second interceptor should have %v in context", parentKey) - } - if ctx.Value(firstIntKey) == nil { - t.Fatalf("second interceptor should have %v in context", firstIntKey) - } - if ctx.Value(secondIntKey) != nil { - t.Fatalf("second interceptor should not have %v in context", secondIntKey) - } - secondCtx := context.WithValue(ctx, secondIntKey, 2) - err := invoker(secondCtx, method, req, reply, cc, opts...) - *(reply.(*string)) += "2" - return err - } - - lastInt := func(ctx context.Context, method string, req, reply interface{}, cc *ClientConn, invoker UnaryInvoker, opts ...CallOption) error { - if ctx.Value(parentKey) == nil { - t.Fatalf("last interceptor should have %v in context", parentKey) - } - if ctx.Value(firstIntKey) == nil { - t.Fatalf("last interceptor should have %v in context", firstIntKey) - } - if ctx.Value(secondIntKey) == nil { - t.Fatalf("last interceptor should have %v in context", secondIntKey) - } - err := invoker(ctx, method, req, reply, cc, opts...) - *(reply.(*string)) += "3" - return err - } - - server, cc := setUpWithOptions(t, 0, math.MaxUint32, WithChainUnaryInterceptor(firstInt, secondInt, lastInt)) - defer func() { - cc.Close() - server.stop() - }() - - var reply string - ctx := context.Background() - parentCtx := context.WithValue(ctx, ctxKey("parentKey"), 0) - if err := cc.Invoke(parentCtx, "/foo/bar", &expectedRequest, &reply); err != nil || reply != expectedResponse+"321" { - t.Fatalf("grpc.Invoke(_, _, _, _, _) = %v, want ", err) - } -} - -func (s) TestChainOnBaseUnaryClientInterceptor(t *testing.T) { - var ( - parentKey = ctxKey("parentKey") - baseIntKey = ctxKey("baseIntKey") - ) - - baseInt := func(ctx context.Context, method string, req, reply interface{}, cc *ClientConn, invoker UnaryInvoker, opts ...CallOption) error { - if ctx.Value(parentKey) == nil { - t.Fatalf("base interceptor should have %v in context", parentKey) - } - if ctx.Value(baseIntKey) != nil { - t.Fatalf("base interceptor should not have %v in context", baseIntKey) - } - baseCtx := context.WithValue(ctx, baseIntKey, 1) - return invoker(baseCtx, method, req, reply, cc, opts...) - } - - chainInt := func(ctx context.Context, method string, req, reply interface{}, cc *ClientConn, invoker UnaryInvoker, opts ...CallOption) error { - if ctx.Value(parentKey) == nil { - t.Fatalf("chain interceptor should have %v in context", parentKey) - } - if ctx.Value(baseIntKey) == nil { - t.Fatalf("chain interceptor should have %v in context", baseIntKey) - } - return invoker(ctx, method, req, reply, cc, opts...) - } - - server, cc := setUpWithOptions(t, 0, math.MaxUint32, WithUnaryInterceptor(baseInt), WithChainUnaryInterceptor(chainInt)) - defer func() { - cc.Close() - server.stop() - }() - - var reply string - ctx := context.Background() - parentCtx := context.WithValue(ctx, ctxKey("parentKey"), 0) - if err := cc.Invoke(parentCtx, "/foo/bar", &expectedRequest, &reply); err != nil || reply != expectedResponse { - t.Fatalf("grpc.Invoke(_, _, _, _, _) = %v, want ", err) - } -} - -func (s) TestChainStreamClientInterceptor(t *testing.T) { - var ( - parentKey = ctxKey("parentKey") - firstIntKey = ctxKey("firstIntKey") - secondIntKey = ctxKey("secondIntKey") - ) - - firstInt := func(ctx context.Context, desc *StreamDesc, cc *ClientConn, method string, streamer Streamer, opts ...CallOption) (ClientStream, error) { - if ctx.Value(parentKey) == nil { - t.Fatalf("first interceptor should have %v in context", parentKey) - } - if ctx.Value(firstIntKey) != nil { - t.Fatalf("first interceptor should not have %v in context", firstIntKey) - } - if ctx.Value(secondIntKey) != nil { - t.Fatalf("first interceptor should not have %v in context", secondIntKey) - } - firstCtx := context.WithValue(ctx, firstIntKey, 1) - return streamer(firstCtx, desc, cc, method, opts...) - } - - secondInt := func(ctx context.Context, desc *StreamDesc, cc *ClientConn, method string, streamer Streamer, opts ...CallOption) (ClientStream, error) { - if ctx.Value(parentKey) == nil { - t.Fatalf("second interceptor should have %v in context", parentKey) - } - if ctx.Value(firstIntKey) == nil { - t.Fatalf("second interceptor should have %v in context", firstIntKey) - } - if ctx.Value(secondIntKey) != nil { - t.Fatalf("second interceptor should not have %v in context", secondIntKey) - } - secondCtx := context.WithValue(ctx, secondIntKey, 2) - return streamer(secondCtx, desc, cc, method, opts...) - } - - lastInt := func(ctx context.Context, desc *StreamDesc, cc *ClientConn, method string, streamer Streamer, opts ...CallOption) (ClientStream, error) { - if ctx.Value(parentKey) == nil { - t.Fatalf("last interceptor should have %v in context", parentKey) - } - if ctx.Value(firstIntKey) == nil { - t.Fatalf("last interceptor should have %v in context", firstIntKey) - } - if ctx.Value(secondIntKey) == nil { - t.Fatalf("last interceptor should have %v in context", secondIntKey) - } - return streamer(ctx, desc, cc, method, opts...) - } - - server, cc := setUpWithOptions(t, 0, math.MaxUint32, WithChainStreamInterceptor(firstInt, secondInt, lastInt)) - defer func() { - cc.Close() - server.stop() - }() - - ctx := context.Background() - parentCtx := context.WithValue(ctx, ctxKey("parentKey"), 0) - _, err := cc.NewStream(parentCtx, &StreamDesc{}, "/foo/bar") - if err != nil { - t.Fatalf("grpc.NewStream(_, _, _) = %v, want ", err) - } -} - -func (s) TestInvoke(t *testing.T) { - server, cc := setUp(t, 0, math.MaxUint32) - var reply string - if err := cc.Invoke(context.Background(), "/foo/bar", &expectedRequest, &reply); err != nil || reply != expectedResponse { - t.Fatalf("grpc.Invoke(_, _, _, _, _) = %v, want ", err) - } - cc.Close() - server.stop() -} - -func (s) TestInvokeLargeErr(t *testing.T) { - server, cc := setUp(t, 0, math.MaxUint32) - var reply string - req := "hello" - err := cc.Invoke(context.Background(), "/foo/bar", &req, &reply) - if _, ok := status.FromError(err); !ok { - t.Fatalf("grpc.Invoke(_, _, _, _, _) receives non rpc error.") - } - if status.Code(err) != codes.Internal || len(errorDesc(err)) != sizeLargeErr { - t.Fatalf("grpc.Invoke(_, _, _, _, _) = %v, want an error of code %d and desc size %d", err, codes.Internal, sizeLargeErr) - } - cc.Close() - server.stop() -} - -// TestInvokeErrorSpecialChars checks that error messages don't get mangled. -func (s) TestInvokeErrorSpecialChars(t *testing.T) { - server, cc := setUp(t, 0, math.MaxUint32) - var reply string - req := "weird error" - err := cc.Invoke(context.Background(), "/foo/bar", &req, &reply) - if _, ok := status.FromError(err); !ok { - t.Fatalf("grpc.Invoke(_, _, _, _, _) receives non rpc error.") - } - if got, want := errorDesc(err), weirdError; got != want { - t.Fatalf("grpc.Invoke(_, _, _, _, _) error = %q, want %q", got, want) - } - cc.Close() - server.stop() -} - -// TestInvokeCancel checks that an Invoke with a canceled context is not sent. -func (s) TestInvokeCancel(t *testing.T) { - server, cc := setUp(t, 0, math.MaxUint32) - var reply string - req := "canceled" - for i := 0; i < 100; i++ { - ctx, cancel := context.WithCancel(context.Background()) - cancel() - cc.Invoke(ctx, "/foo/bar", &req, &reply) - } - if canceled != 0 { - t.Fatalf("received %d of 100 canceled requests", canceled) - } - cc.Close() - server.stop() -} - -// TestInvokeCancelClosedNonFail checks that a canceled non-failfast RPC -// on a closed client will terminate. -func (s) TestInvokeCancelClosedNonFailFast(t *testing.T) { - server, cc := setUp(t, 0, math.MaxUint32) - var reply string - cc.Close() - req := "hello" - ctx, cancel := context.WithCancel(context.Background()) - cancel() - if err := cc.Invoke(ctx, "/foo/bar", &req, &reply, WaitForReady(true)); err == nil { - t.Fatalf("canceled invoke on closed connection should fail") - } - server.stop() -} diff --git a/channelz/channelz.go b/channelz/channelz.go new file mode 100644 index 000000000000..32b7fa5794e1 --- /dev/null +++ b/channelz/channelz.go @@ -0,0 +1,36 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package channelz exports internals of the channelz implementation as required +// by other gRPC packages. +// +// The implementation of the channelz spec as defined in +// https://github.com/grpc/proposal/blob/master/A14-channelz.md, is provided by +// the `internal/channelz` package. +// +// # Experimental +// +// Notice: All APIs in this package are experimental and may be removed in a +// later release. +package channelz + +import "google.golang.org/grpc/internal/channelz" + +// Identifier is an opaque identifier which uniquely identifies an entity in the +// channelz database. +type Identifier = channelz.Identifier diff --git a/channelz/grpc_channelz_v1/channelz.pb.go b/channelz/grpc_channelz_v1/channelz.pb.go index 34bfa5ab8f50..401bf697099d 100644 --- a/channelz/grpc_channelz_v1/channelz.pb.go +++ b/channelz/grpc_channelz_v1/channelz.pb.go @@ -1,32 +1,49 @@ +// Copyright 2018 The gRPC Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file defines an interface for exporting monitoring information +// out of gRPC servers. See the full design at +// https://github.com/grpc/proposal/blob/master/A14-channelz.md +// +// The canonical version of this proto can be found at +// https://github.com/grpc/grpc-proto/blob/master/grpc/channelz/v1/channelz.proto + // Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.31.0 +// protoc v4.22.0 // source: grpc/channelz/v1/channelz.proto package grpc_channelz_v1 import ( - context "context" - fmt "fmt" - proto "github.com/golang/protobuf/proto" - any "github.com/golang/protobuf/ptypes/any" - duration "github.com/golang/protobuf/ptypes/duration" - timestamp "github.com/golang/protobuf/ptypes/timestamp" - wrappers "github.com/golang/protobuf/ptypes/wrappers" - grpc "google.golang.org/grpc" - codes "google.golang.org/grpc/codes" - status "google.golang.org/grpc/status" - math "math" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + anypb "google.golang.org/protobuf/types/known/anypb" + durationpb "google.golang.org/protobuf/types/known/durationpb" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" + wrapperspb "google.golang.org/protobuf/types/known/wrapperspb" + reflect "reflect" + sync "sync" ) -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) type ChannelConnectivityState_State int32 @@ -39,30 +56,51 @@ const ( ChannelConnectivityState_SHUTDOWN ChannelConnectivityState_State = 5 ) -var ChannelConnectivityState_State_name = map[int32]string{ - 0: "UNKNOWN", - 1: "IDLE", - 2: "CONNECTING", - 3: "READY", - 4: "TRANSIENT_FAILURE", - 5: "SHUTDOWN", -} +// Enum value maps for ChannelConnectivityState_State. +var ( + ChannelConnectivityState_State_name = map[int32]string{ + 0: "UNKNOWN", + 1: "IDLE", + 2: "CONNECTING", + 3: "READY", + 4: "TRANSIENT_FAILURE", + 5: "SHUTDOWN", + } + ChannelConnectivityState_State_value = map[string]int32{ + "UNKNOWN": 0, + "IDLE": 1, + "CONNECTING": 2, + "READY": 3, + "TRANSIENT_FAILURE": 4, + "SHUTDOWN": 5, + } +) -var ChannelConnectivityState_State_value = map[string]int32{ - "UNKNOWN": 0, - "IDLE": 1, - "CONNECTING": 2, - "READY": 3, - "TRANSIENT_FAILURE": 4, - "SHUTDOWN": 5, +func (x ChannelConnectivityState_State) Enum() *ChannelConnectivityState_State { + p := new(ChannelConnectivityState_State) + *p = x + return p } func (x ChannelConnectivityState_State) String() string { - return proto.EnumName(ChannelConnectivityState_State_name, int32(x)) + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (ChannelConnectivityState_State) Descriptor() protoreflect.EnumDescriptor { + return file_grpc_channelz_v1_channelz_proto_enumTypes[0].Descriptor() +} + +func (ChannelConnectivityState_State) Type() protoreflect.EnumType { + return &file_grpc_channelz_v1_channelz_proto_enumTypes[0] +} + +func (x ChannelConnectivityState_State) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) } +// Deprecated: Use ChannelConnectivityState_State.Descriptor instead. func (ChannelConnectivityState_State) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_6ee37dfd35a8ab00, []int{2, 0} + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{2, 0} } // The supported severity levels of trace events. @@ -75,34 +113,59 @@ const ( ChannelTraceEvent_CT_ERROR ChannelTraceEvent_Severity = 3 ) -var ChannelTraceEvent_Severity_name = map[int32]string{ - 0: "CT_UNKNOWN", - 1: "CT_INFO", - 2: "CT_WARNING", - 3: "CT_ERROR", -} +// Enum value maps for ChannelTraceEvent_Severity. +var ( + ChannelTraceEvent_Severity_name = map[int32]string{ + 0: "CT_UNKNOWN", + 1: "CT_INFO", + 2: "CT_WARNING", + 3: "CT_ERROR", + } + ChannelTraceEvent_Severity_value = map[string]int32{ + "CT_UNKNOWN": 0, + "CT_INFO": 1, + "CT_WARNING": 2, + "CT_ERROR": 3, + } +) -var ChannelTraceEvent_Severity_value = map[string]int32{ - "CT_UNKNOWN": 0, - "CT_INFO": 1, - "CT_WARNING": 2, - "CT_ERROR": 3, +func (x ChannelTraceEvent_Severity) Enum() *ChannelTraceEvent_Severity { + p := new(ChannelTraceEvent_Severity) + *p = x + return p } func (x ChannelTraceEvent_Severity) String() string { - return proto.EnumName(ChannelTraceEvent_Severity_name, int32(x)) + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (ChannelTraceEvent_Severity) Descriptor() protoreflect.EnumDescriptor { + return file_grpc_channelz_v1_channelz_proto_enumTypes[1].Descriptor() +} + +func (ChannelTraceEvent_Severity) Type() protoreflect.EnumType { + return &file_grpc_channelz_v1_channelz_proto_enumTypes[1] +} + +func (x ChannelTraceEvent_Severity) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) } +// Deprecated: Use ChannelTraceEvent_Severity.Descriptor instead. func (ChannelTraceEvent_Severity) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_6ee37dfd35a8ab00, []int{4, 0} + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{4, 0} } // Channel is a logical grouping of channels, subchannels, and sockets. type Channel struct { - // The identifier for this channel. This should bet set. + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // The identifier for this channel. This should be set. Ref *ChannelRef `protobuf:"bytes,1,opt,name=ref,proto3" json:"ref,omitempty"` // Data specific to this channel. - Data *ChannelData `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"` + Data *ChannelData `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"` // At most one of 'channel_ref+subchannel_ref' and 'socket' is set. // There are no ordering guarantees on the order of channel refs. // There may not be cycles in the ref graph. // A channel ref may be present in more than one channel or subchannel. @@ -113,68 +176,72 @@ type Channel struct { // A sub channel ref may be present in more than one channel or subchannel. SubchannelRef []*SubchannelRef `protobuf:"bytes,4,rep,name=subchannel_ref,json=subchannelRef,proto3" json:"subchannel_ref,omitempty"` // There are no ordering guarantees on the order of sockets. - SocketRef []*SocketRef `protobuf:"bytes,5,rep,name=socket_ref,json=socketRef,proto3" json:"socket_ref,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + SocketRef []*SocketRef `protobuf:"bytes,5,rep,name=socket_ref,json=socketRef,proto3" json:"socket_ref,omitempty"` } -func (m *Channel) Reset() { *m = Channel{} } -func (m *Channel) String() string { return proto.CompactTextString(m) } -func (*Channel) ProtoMessage() {} -func (*Channel) Descriptor() ([]byte, []int) { - return fileDescriptor_6ee37dfd35a8ab00, []int{0} +func (x *Channel) Reset() { + *x = Channel{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *Channel) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Channel.Unmarshal(m, b) -} -func (m *Channel) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Channel.Marshal(b, m, deterministic) -} -func (m *Channel) XXX_Merge(src proto.Message) { - xxx_messageInfo_Channel.Merge(m, src) +func (x *Channel) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *Channel) XXX_Size() int { - return xxx_messageInfo_Channel.Size(m) -} -func (m *Channel) XXX_DiscardUnknown() { - xxx_messageInfo_Channel.DiscardUnknown(m) + +func (*Channel) ProtoMessage() {} + +func (x *Channel) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_Channel proto.InternalMessageInfo +// Deprecated: Use Channel.ProtoReflect.Descriptor instead. +func (*Channel) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{0} +} -func (m *Channel) GetRef() *ChannelRef { - if m != nil { - return m.Ref +func (x *Channel) GetRef() *ChannelRef { + if x != nil { + return x.Ref } return nil } -func (m *Channel) GetData() *ChannelData { - if m != nil { - return m.Data +func (x *Channel) GetData() *ChannelData { + if x != nil { + return x.Data } return nil } -func (m *Channel) GetChannelRef() []*ChannelRef { - if m != nil { - return m.ChannelRef +func (x *Channel) GetChannelRef() []*ChannelRef { + if x != nil { + return x.ChannelRef } return nil } -func (m *Channel) GetSubchannelRef() []*SubchannelRef { - if m != nil { - return m.SubchannelRef +func (x *Channel) GetSubchannelRef() []*SubchannelRef { + if x != nil { + return x.SubchannelRef } return nil } -func (m *Channel) GetSocketRef() []*SocketRef { - if m != nil { - return m.SocketRef +func (x *Channel) GetSocketRef() []*SocketRef { + if x != nil { + return x.SocketRef } return nil } @@ -182,10 +249,14 @@ func (m *Channel) GetSocketRef() []*SocketRef { // Subchannel is a logical grouping of channels, subchannels, and sockets. // A subchannel is load balanced over by it's ancestor type Subchannel struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // The identifier for this channel. Ref *SubchannelRef `protobuf:"bytes,1,opt,name=ref,proto3" json:"ref,omitempty"` // Data specific to this channel. - Data *ChannelData `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"` + Data *ChannelData `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"` // At most one of 'channel_ref+subchannel_ref' and 'socket' is set. // There are no ordering guarantees on the order of channel refs. // There may not be cycles in the ref graph. // A channel ref may be present in more than one channel or subchannel. @@ -196,68 +267,72 @@ type Subchannel struct { // A sub channel ref may be present in more than one channel or subchannel. SubchannelRef []*SubchannelRef `protobuf:"bytes,4,rep,name=subchannel_ref,json=subchannelRef,proto3" json:"subchannel_ref,omitempty"` // There are no ordering guarantees on the order of sockets. - SocketRef []*SocketRef `protobuf:"bytes,5,rep,name=socket_ref,json=socketRef,proto3" json:"socket_ref,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + SocketRef []*SocketRef `protobuf:"bytes,5,rep,name=socket_ref,json=socketRef,proto3" json:"socket_ref,omitempty"` } -func (m *Subchannel) Reset() { *m = Subchannel{} } -func (m *Subchannel) String() string { return proto.CompactTextString(m) } -func (*Subchannel) ProtoMessage() {} -func (*Subchannel) Descriptor() ([]byte, []int) { - return fileDescriptor_6ee37dfd35a8ab00, []int{1} +func (x *Subchannel) Reset() { + *x = Subchannel{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *Subchannel) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Subchannel.Unmarshal(m, b) +func (x *Subchannel) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *Subchannel) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Subchannel.Marshal(b, m, deterministic) -} -func (m *Subchannel) XXX_Merge(src proto.Message) { - xxx_messageInfo_Subchannel.Merge(m, src) -} -func (m *Subchannel) XXX_Size() int { - return xxx_messageInfo_Subchannel.Size(m) -} -func (m *Subchannel) XXX_DiscardUnknown() { - xxx_messageInfo_Subchannel.DiscardUnknown(m) + +func (*Subchannel) ProtoMessage() {} + +func (x *Subchannel) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_Subchannel proto.InternalMessageInfo +// Deprecated: Use Subchannel.ProtoReflect.Descriptor instead. +func (*Subchannel) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{1} +} -func (m *Subchannel) GetRef() *SubchannelRef { - if m != nil { - return m.Ref +func (x *Subchannel) GetRef() *SubchannelRef { + if x != nil { + return x.Ref } return nil } -func (m *Subchannel) GetData() *ChannelData { - if m != nil { - return m.Data +func (x *Subchannel) GetData() *ChannelData { + if x != nil { + return x.Data } return nil } -func (m *Subchannel) GetChannelRef() []*ChannelRef { - if m != nil { - return m.ChannelRef +func (x *Subchannel) GetChannelRef() []*ChannelRef { + if x != nil { + return x.ChannelRef } return nil } -func (m *Subchannel) GetSubchannelRef() []*SubchannelRef { - if m != nil { - return m.SubchannelRef +func (x *Subchannel) GetSubchannelRef() []*SubchannelRef { + if x != nil { + return x.SubchannelRef } return nil } -func (m *Subchannel) GetSocketRef() []*SocketRef { - if m != nil { - return m.SocketRef +func (x *Subchannel) GetSocketRef() []*SocketRef { + if x != nil { + return x.SocketRef } return nil } @@ -265,46 +340,58 @@ func (m *Subchannel) GetSocketRef() []*SocketRef { // These come from the specified states in this document: // https://github.com/grpc/grpc/blob/master/doc/connectivity-semantics-and-api.md type ChannelConnectivityState struct { - State ChannelConnectivityState_State `protobuf:"varint,1,opt,name=state,proto3,enum=grpc.channelz.v1.ChannelConnectivityState_State" json:"state,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields -func (m *ChannelConnectivityState) Reset() { *m = ChannelConnectivityState{} } -func (m *ChannelConnectivityState) String() string { return proto.CompactTextString(m) } -func (*ChannelConnectivityState) ProtoMessage() {} -func (*ChannelConnectivityState) Descriptor() ([]byte, []int) { - return fileDescriptor_6ee37dfd35a8ab00, []int{2} + State ChannelConnectivityState_State `protobuf:"varint,1,opt,name=state,proto3,enum=grpc.channelz.v1.ChannelConnectivityState_State" json:"state,omitempty"` } -func (m *ChannelConnectivityState) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ChannelConnectivityState.Unmarshal(m, b) -} -func (m *ChannelConnectivityState) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ChannelConnectivityState.Marshal(b, m, deterministic) -} -func (m *ChannelConnectivityState) XXX_Merge(src proto.Message) { - xxx_messageInfo_ChannelConnectivityState.Merge(m, src) +func (x *ChannelConnectivityState) Reset() { + *x = ChannelConnectivityState{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *ChannelConnectivityState) XXX_Size() int { - return xxx_messageInfo_ChannelConnectivityState.Size(m) + +func (x *ChannelConnectivityState) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *ChannelConnectivityState) XXX_DiscardUnknown() { - xxx_messageInfo_ChannelConnectivityState.DiscardUnknown(m) + +func (*ChannelConnectivityState) ProtoMessage() {} + +func (x *ChannelConnectivityState) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_ChannelConnectivityState proto.InternalMessageInfo +// Deprecated: Use ChannelConnectivityState.ProtoReflect.Descriptor instead. +func (*ChannelConnectivityState) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{2} +} -func (m *ChannelConnectivityState) GetState() ChannelConnectivityState_State { - if m != nil { - return m.State +func (x *ChannelConnectivityState) GetState() ChannelConnectivityState_State { + if x != nil { + return x.State } return ChannelConnectivityState_UNKNOWN } // Channel data is data related to a specific Channel or Subchannel. type ChannelData struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // The connectivity state of the channel or subchannel. Implementations // should always set this. State *ChannelConnectivityState `protobuf:"bytes,1,opt,name=state,proto3" json:"state,omitempty"` @@ -319,82 +406,86 @@ type ChannelData struct { // The number of calls that have completed with a non-OK status CallsFailed int64 `protobuf:"varint,6,opt,name=calls_failed,json=callsFailed,proto3" json:"calls_failed,omitempty"` // The last time a call was started on the channel. - LastCallStartedTimestamp *timestamp.Timestamp `protobuf:"bytes,7,opt,name=last_call_started_timestamp,json=lastCallStartedTimestamp,proto3" json:"last_call_started_timestamp,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + LastCallStartedTimestamp *timestamppb.Timestamp `protobuf:"bytes,7,opt,name=last_call_started_timestamp,json=lastCallStartedTimestamp,proto3" json:"last_call_started_timestamp,omitempty"` } -func (m *ChannelData) Reset() { *m = ChannelData{} } -func (m *ChannelData) String() string { return proto.CompactTextString(m) } -func (*ChannelData) ProtoMessage() {} -func (*ChannelData) Descriptor() ([]byte, []int) { - return fileDescriptor_6ee37dfd35a8ab00, []int{3} +func (x *ChannelData) Reset() { + *x = ChannelData{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *ChannelData) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ChannelData.Unmarshal(m, b) -} -func (m *ChannelData) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ChannelData.Marshal(b, m, deterministic) -} -func (m *ChannelData) XXX_Merge(src proto.Message) { - xxx_messageInfo_ChannelData.Merge(m, src) +func (x *ChannelData) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *ChannelData) XXX_Size() int { - return xxx_messageInfo_ChannelData.Size(m) -} -func (m *ChannelData) XXX_DiscardUnknown() { - xxx_messageInfo_ChannelData.DiscardUnknown(m) + +func (*ChannelData) ProtoMessage() {} + +func (x *ChannelData) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_ChannelData proto.InternalMessageInfo +// Deprecated: Use ChannelData.ProtoReflect.Descriptor instead. +func (*ChannelData) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{3} +} -func (m *ChannelData) GetState() *ChannelConnectivityState { - if m != nil { - return m.State +func (x *ChannelData) GetState() *ChannelConnectivityState { + if x != nil { + return x.State } return nil } -func (m *ChannelData) GetTarget() string { - if m != nil { - return m.Target +func (x *ChannelData) GetTarget() string { + if x != nil { + return x.Target } return "" } -func (m *ChannelData) GetTrace() *ChannelTrace { - if m != nil { - return m.Trace +func (x *ChannelData) GetTrace() *ChannelTrace { + if x != nil { + return x.Trace } return nil } -func (m *ChannelData) GetCallsStarted() int64 { - if m != nil { - return m.CallsStarted +func (x *ChannelData) GetCallsStarted() int64 { + if x != nil { + return x.CallsStarted } return 0 } -func (m *ChannelData) GetCallsSucceeded() int64 { - if m != nil { - return m.CallsSucceeded +func (x *ChannelData) GetCallsSucceeded() int64 { + if x != nil { + return x.CallsSucceeded } return 0 } -func (m *ChannelData) GetCallsFailed() int64 { - if m != nil { - return m.CallsFailed +func (x *ChannelData) GetCallsFailed() int64 { + if x != nil { + return x.CallsFailed } return 0 } -func (m *ChannelData) GetLastCallStartedTimestamp() *timestamp.Timestamp { - if m != nil { - return m.LastCallStartedTimestamp +func (x *ChannelData) GetLastCallStartedTimestamp() *timestamppb.Timestamp { + if x != nil { + return x.LastCallStartedTimestamp } return nil } @@ -402,88 +493,81 @@ func (m *ChannelData) GetLastCallStartedTimestamp() *timestamp.Timestamp { // A trace event is an interesting thing that happened to a channel or // subchannel, such as creation, address resolution, subchannel creation, etc. type ChannelTraceEvent struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // High level description of the event. Description string `protobuf:"bytes,1,opt,name=description,proto3" json:"description,omitempty"` // the severity of the trace event Severity ChannelTraceEvent_Severity `protobuf:"varint,2,opt,name=severity,proto3,enum=grpc.channelz.v1.ChannelTraceEvent_Severity" json:"severity,omitempty"` // When this event occurred. - Timestamp *timestamp.Timestamp `protobuf:"bytes,3,opt,name=timestamp,proto3" json:"timestamp,omitempty"` + Timestamp *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=timestamp,proto3" json:"timestamp,omitempty"` // ref of referenced channel or subchannel. // Optional, only present if this event refers to a child object. For example, // this field would be filled if this trace event was for a subchannel being // created. // - // Types that are valid to be assigned to ChildRef: + // Types that are assignable to ChildRef: + // // *ChannelTraceEvent_ChannelRef // *ChannelTraceEvent_SubchannelRef - ChildRef isChannelTraceEvent_ChildRef `protobuf_oneof:"child_ref"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + ChildRef isChannelTraceEvent_ChildRef `protobuf_oneof:"child_ref"` } -func (m *ChannelTraceEvent) Reset() { *m = ChannelTraceEvent{} } -func (m *ChannelTraceEvent) String() string { return proto.CompactTextString(m) } -func (*ChannelTraceEvent) ProtoMessage() {} -func (*ChannelTraceEvent) Descriptor() ([]byte, []int) { - return fileDescriptor_6ee37dfd35a8ab00, []int{4} +func (x *ChannelTraceEvent) Reset() { + *x = ChannelTraceEvent{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *ChannelTraceEvent) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ChannelTraceEvent.Unmarshal(m, b) -} -func (m *ChannelTraceEvent) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ChannelTraceEvent.Marshal(b, m, deterministic) -} -func (m *ChannelTraceEvent) XXX_Merge(src proto.Message) { - xxx_messageInfo_ChannelTraceEvent.Merge(m, src) -} -func (m *ChannelTraceEvent) XXX_Size() int { - return xxx_messageInfo_ChannelTraceEvent.Size(m) +func (x *ChannelTraceEvent) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *ChannelTraceEvent) XXX_DiscardUnknown() { - xxx_messageInfo_ChannelTraceEvent.DiscardUnknown(m) + +func (*ChannelTraceEvent) ProtoMessage() {} + +func (x *ChannelTraceEvent) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_ChannelTraceEvent proto.InternalMessageInfo +// Deprecated: Use ChannelTraceEvent.ProtoReflect.Descriptor instead. +func (*ChannelTraceEvent) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{4} +} -func (m *ChannelTraceEvent) GetDescription() string { - if m != nil { - return m.Description +func (x *ChannelTraceEvent) GetDescription() string { + if x != nil { + return x.Description } return "" } -func (m *ChannelTraceEvent) GetSeverity() ChannelTraceEvent_Severity { - if m != nil { - return m.Severity +func (x *ChannelTraceEvent) GetSeverity() ChannelTraceEvent_Severity { + if x != nil { + return x.Severity } return ChannelTraceEvent_CT_UNKNOWN } -func (m *ChannelTraceEvent) GetTimestamp() *timestamp.Timestamp { - if m != nil { - return m.Timestamp +func (x *ChannelTraceEvent) GetTimestamp() *timestamppb.Timestamp { + if x != nil { + return x.Timestamp } return nil } -type isChannelTraceEvent_ChildRef interface { - isChannelTraceEvent_ChildRef() -} - -type ChannelTraceEvent_ChannelRef struct { - ChannelRef *ChannelRef `protobuf:"bytes,4,opt,name=channel_ref,json=channelRef,proto3,oneof"` -} - -type ChannelTraceEvent_SubchannelRef struct { - SubchannelRef *SubchannelRef `protobuf:"bytes,5,opt,name=subchannel_ref,json=subchannelRef,proto3,oneof"` -} - -func (*ChannelTraceEvent_ChannelRef) isChannelTraceEvent_ChildRef() {} - -func (*ChannelTraceEvent_SubchannelRef) isChannelTraceEvent_ChildRef() {} - func (m *ChannelTraceEvent) GetChildRef() isChannelTraceEvent_ChildRef { if m != nil { return m.ChildRef @@ -491,285 +575,333 @@ func (m *ChannelTraceEvent) GetChildRef() isChannelTraceEvent_ChildRef { return nil } -func (m *ChannelTraceEvent) GetChannelRef() *ChannelRef { - if x, ok := m.GetChildRef().(*ChannelTraceEvent_ChannelRef); ok { +func (x *ChannelTraceEvent) GetChannelRef() *ChannelRef { + if x, ok := x.GetChildRef().(*ChannelTraceEvent_ChannelRef); ok { return x.ChannelRef } return nil } -func (m *ChannelTraceEvent) GetSubchannelRef() *SubchannelRef { - if x, ok := m.GetChildRef().(*ChannelTraceEvent_SubchannelRef); ok { +func (x *ChannelTraceEvent) GetSubchannelRef() *SubchannelRef { + if x, ok := x.GetChildRef().(*ChannelTraceEvent_SubchannelRef); ok { return x.SubchannelRef } return nil } -// XXX_OneofWrappers is for the internal use of the proto package. -func (*ChannelTraceEvent) XXX_OneofWrappers() []interface{} { - return []interface{}{ - (*ChannelTraceEvent_ChannelRef)(nil), - (*ChannelTraceEvent_SubchannelRef)(nil), - } +type isChannelTraceEvent_ChildRef interface { + isChannelTraceEvent_ChildRef() +} + +type ChannelTraceEvent_ChannelRef struct { + ChannelRef *ChannelRef `protobuf:"bytes,4,opt,name=channel_ref,json=channelRef,proto3,oneof"` +} + +type ChannelTraceEvent_SubchannelRef struct { + SubchannelRef *SubchannelRef `protobuf:"bytes,5,opt,name=subchannel_ref,json=subchannelRef,proto3,oneof"` } +func (*ChannelTraceEvent_ChannelRef) isChannelTraceEvent_ChildRef() {} + +func (*ChannelTraceEvent_SubchannelRef) isChannelTraceEvent_ChildRef() {} + // ChannelTrace represents the recent events that have occurred on the channel. type ChannelTrace struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // Number of events ever logged in this tracing object. This can differ from // events.size() because events can be overwritten or garbage collected by // implementations. NumEventsLogged int64 `protobuf:"varint,1,opt,name=num_events_logged,json=numEventsLogged,proto3" json:"num_events_logged,omitempty"` // Time that this channel was created. - CreationTimestamp *timestamp.Timestamp `protobuf:"bytes,2,opt,name=creation_timestamp,json=creationTimestamp,proto3" json:"creation_timestamp,omitempty"` + CreationTimestamp *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=creation_timestamp,json=creationTimestamp,proto3" json:"creation_timestamp,omitempty"` // List of events that have occurred on this channel. - Events []*ChannelTraceEvent `protobuf:"bytes,3,rep,name=events,proto3" json:"events,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Events []*ChannelTraceEvent `protobuf:"bytes,3,rep,name=events,proto3" json:"events,omitempty"` } -func (m *ChannelTrace) Reset() { *m = ChannelTrace{} } -func (m *ChannelTrace) String() string { return proto.CompactTextString(m) } -func (*ChannelTrace) ProtoMessage() {} -func (*ChannelTrace) Descriptor() ([]byte, []int) { - return fileDescriptor_6ee37dfd35a8ab00, []int{5} +func (x *ChannelTrace) Reset() { + *x = ChannelTrace{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *ChannelTrace) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ChannelTrace.Unmarshal(m, b) +func (x *ChannelTrace) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *ChannelTrace) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ChannelTrace.Marshal(b, m, deterministic) -} -func (m *ChannelTrace) XXX_Merge(src proto.Message) { - xxx_messageInfo_ChannelTrace.Merge(m, src) -} -func (m *ChannelTrace) XXX_Size() int { - return xxx_messageInfo_ChannelTrace.Size(m) -} -func (m *ChannelTrace) XXX_DiscardUnknown() { - xxx_messageInfo_ChannelTrace.DiscardUnknown(m) + +func (*ChannelTrace) ProtoMessage() {} + +func (x *ChannelTrace) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_ChannelTrace proto.InternalMessageInfo +// Deprecated: Use ChannelTrace.ProtoReflect.Descriptor instead. +func (*ChannelTrace) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{5} +} -func (m *ChannelTrace) GetNumEventsLogged() int64 { - if m != nil { - return m.NumEventsLogged +func (x *ChannelTrace) GetNumEventsLogged() int64 { + if x != nil { + return x.NumEventsLogged } return 0 } -func (m *ChannelTrace) GetCreationTimestamp() *timestamp.Timestamp { - if m != nil { - return m.CreationTimestamp +func (x *ChannelTrace) GetCreationTimestamp() *timestamppb.Timestamp { + if x != nil { + return x.CreationTimestamp } return nil } -func (m *ChannelTrace) GetEvents() []*ChannelTraceEvent { - if m != nil { - return m.Events +func (x *ChannelTrace) GetEvents() []*ChannelTraceEvent { + if x != nil { + return x.Events } return nil } // ChannelRef is a reference to a Channel. type ChannelRef struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // The globally unique id for this channel. Must be a positive number. ChannelId int64 `protobuf:"varint,1,opt,name=channel_id,json=channelId,proto3" json:"channel_id,omitempty"` // An optional name associated with the channel. - Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` } -func (m *ChannelRef) Reset() { *m = ChannelRef{} } -func (m *ChannelRef) String() string { return proto.CompactTextString(m) } -func (*ChannelRef) ProtoMessage() {} -func (*ChannelRef) Descriptor() ([]byte, []int) { - return fileDescriptor_6ee37dfd35a8ab00, []int{6} +func (x *ChannelRef) Reset() { + *x = ChannelRef{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *ChannelRef) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ChannelRef.Unmarshal(m, b) -} -func (m *ChannelRef) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ChannelRef.Marshal(b, m, deterministic) -} -func (m *ChannelRef) XXX_Merge(src proto.Message) { - xxx_messageInfo_ChannelRef.Merge(m, src) +func (x *ChannelRef) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *ChannelRef) XXX_Size() int { - return xxx_messageInfo_ChannelRef.Size(m) -} -func (m *ChannelRef) XXX_DiscardUnknown() { - xxx_messageInfo_ChannelRef.DiscardUnknown(m) + +func (*ChannelRef) ProtoMessage() {} + +func (x *ChannelRef) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_ChannelRef proto.InternalMessageInfo +// Deprecated: Use ChannelRef.ProtoReflect.Descriptor instead. +func (*ChannelRef) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{6} +} -func (m *ChannelRef) GetChannelId() int64 { - if m != nil { - return m.ChannelId +func (x *ChannelRef) GetChannelId() int64 { + if x != nil { + return x.ChannelId } return 0 } -func (m *ChannelRef) GetName() string { - if m != nil { - return m.Name +func (x *ChannelRef) GetName() string { + if x != nil { + return x.Name } return "" } // SubchannelRef is a reference to a Subchannel. type SubchannelRef struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // The globally unique id for this subchannel. Must be a positive number. SubchannelId int64 `protobuf:"varint,7,opt,name=subchannel_id,json=subchannelId,proto3" json:"subchannel_id,omitempty"` // An optional name associated with the subchannel. - Name string `protobuf:"bytes,8,opt,name=name,proto3" json:"name,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Name string `protobuf:"bytes,8,opt,name=name,proto3" json:"name,omitempty"` } -func (m *SubchannelRef) Reset() { *m = SubchannelRef{} } -func (m *SubchannelRef) String() string { return proto.CompactTextString(m) } -func (*SubchannelRef) ProtoMessage() {} -func (*SubchannelRef) Descriptor() ([]byte, []int) { - return fileDescriptor_6ee37dfd35a8ab00, []int{7} +func (x *SubchannelRef) Reset() { + *x = SubchannelRef{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *SubchannelRef) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_SubchannelRef.Unmarshal(m, b) -} -func (m *SubchannelRef) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_SubchannelRef.Marshal(b, m, deterministic) +func (x *SubchannelRef) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *SubchannelRef) XXX_Merge(src proto.Message) { - xxx_messageInfo_SubchannelRef.Merge(m, src) -} -func (m *SubchannelRef) XXX_Size() int { - return xxx_messageInfo_SubchannelRef.Size(m) -} -func (m *SubchannelRef) XXX_DiscardUnknown() { - xxx_messageInfo_SubchannelRef.DiscardUnknown(m) + +func (*SubchannelRef) ProtoMessage() {} + +func (x *SubchannelRef) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[7] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_SubchannelRef proto.InternalMessageInfo +// Deprecated: Use SubchannelRef.ProtoReflect.Descriptor instead. +func (*SubchannelRef) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{7} +} -func (m *SubchannelRef) GetSubchannelId() int64 { - if m != nil { - return m.SubchannelId +func (x *SubchannelRef) GetSubchannelId() int64 { + if x != nil { + return x.SubchannelId } return 0 } -func (m *SubchannelRef) GetName() string { - if m != nil { - return m.Name +func (x *SubchannelRef) GetName() string { + if x != nil { + return x.Name } return "" } // SocketRef is a reference to a Socket. type SocketRef struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // The globally unique id for this socket. Must be a positive number. SocketId int64 `protobuf:"varint,3,opt,name=socket_id,json=socketId,proto3" json:"socket_id,omitempty"` // An optional name associated with the socket. - Name string `protobuf:"bytes,4,opt,name=name,proto3" json:"name,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Name string `protobuf:"bytes,4,opt,name=name,proto3" json:"name,omitempty"` } -func (m *SocketRef) Reset() { *m = SocketRef{} } -func (m *SocketRef) String() string { return proto.CompactTextString(m) } -func (*SocketRef) ProtoMessage() {} -func (*SocketRef) Descriptor() ([]byte, []int) { - return fileDescriptor_6ee37dfd35a8ab00, []int{8} +func (x *SocketRef) Reset() { + *x = SocketRef{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *SocketRef) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_SocketRef.Unmarshal(m, b) +func (x *SocketRef) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *SocketRef) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_SocketRef.Marshal(b, m, deterministic) -} -func (m *SocketRef) XXX_Merge(src proto.Message) { - xxx_messageInfo_SocketRef.Merge(m, src) -} -func (m *SocketRef) XXX_Size() int { - return xxx_messageInfo_SocketRef.Size(m) -} -func (m *SocketRef) XXX_DiscardUnknown() { - xxx_messageInfo_SocketRef.DiscardUnknown(m) + +func (*SocketRef) ProtoMessage() {} + +func (x *SocketRef) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[8] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_SocketRef proto.InternalMessageInfo +// Deprecated: Use SocketRef.ProtoReflect.Descriptor instead. +func (*SocketRef) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{8} +} -func (m *SocketRef) GetSocketId() int64 { - if m != nil { - return m.SocketId +func (x *SocketRef) GetSocketId() int64 { + if x != nil { + return x.SocketId } return 0 } -func (m *SocketRef) GetName() string { - if m != nil { - return m.Name +func (x *SocketRef) GetName() string { + if x != nil { + return x.Name } return "" } // ServerRef is a reference to a Server. type ServerRef struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // A globally unique identifier for this server. Must be a positive number. ServerId int64 `protobuf:"varint,5,opt,name=server_id,json=serverId,proto3" json:"server_id,omitempty"` // An optional name associated with the server. - Name string `protobuf:"bytes,6,opt,name=name,proto3" json:"name,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Name string `protobuf:"bytes,6,opt,name=name,proto3" json:"name,omitempty"` } -func (m *ServerRef) Reset() { *m = ServerRef{} } -func (m *ServerRef) String() string { return proto.CompactTextString(m) } -func (*ServerRef) ProtoMessage() {} -func (*ServerRef) Descriptor() ([]byte, []int) { - return fileDescriptor_6ee37dfd35a8ab00, []int{9} +func (x *ServerRef) Reset() { + *x = ServerRef{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *ServerRef) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ServerRef.Unmarshal(m, b) -} -func (m *ServerRef) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ServerRef.Marshal(b, m, deterministic) -} -func (m *ServerRef) XXX_Merge(src proto.Message) { - xxx_messageInfo_ServerRef.Merge(m, src) -} -func (m *ServerRef) XXX_Size() int { - return xxx_messageInfo_ServerRef.Size(m) +func (x *ServerRef) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *ServerRef) XXX_DiscardUnknown() { - xxx_messageInfo_ServerRef.DiscardUnknown(m) + +func (*ServerRef) ProtoMessage() {} + +func (x *ServerRef) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[9] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_ServerRef proto.InternalMessageInfo +// Deprecated: Use ServerRef.ProtoReflect.Descriptor instead. +func (*ServerRef) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{9} +} -func (m *ServerRef) GetServerId() int64 { - if m != nil { - return m.ServerId +func (x *ServerRef) GetServerId() int64 { + if x != nil { + return x.ServerId } return 0 } -func (m *ServerRef) GetName() string { - if m != nil { - return m.Name +func (x *ServerRef) GetName() string { + if x != nil { + return x.Name } return "" } @@ -777,66 +909,78 @@ func (m *ServerRef) GetName() string { // Server represents a single server. There may be multiple servers in a single // program. type Server struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // The identifier for a Server. This should be set. Ref *ServerRef `protobuf:"bytes,1,opt,name=ref,proto3" json:"ref,omitempty"` // The associated data of the Server. Data *ServerData `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"` // The sockets that the server is listening on. There are no ordering // guarantees. This may be absent. - ListenSocket []*SocketRef `protobuf:"bytes,3,rep,name=listen_socket,json=listenSocket,proto3" json:"listen_socket,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + ListenSocket []*SocketRef `protobuf:"bytes,3,rep,name=listen_socket,json=listenSocket,proto3" json:"listen_socket,omitempty"` } -func (m *Server) Reset() { *m = Server{} } -func (m *Server) String() string { return proto.CompactTextString(m) } -func (*Server) ProtoMessage() {} -func (*Server) Descriptor() ([]byte, []int) { - return fileDescriptor_6ee37dfd35a8ab00, []int{10} +func (x *Server) Reset() { + *x = Server{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *Server) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Server.Unmarshal(m, b) -} -func (m *Server) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Server.Marshal(b, m, deterministic) -} -func (m *Server) XXX_Merge(src proto.Message) { - xxx_messageInfo_Server.Merge(m, src) +func (x *Server) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *Server) XXX_Size() int { - return xxx_messageInfo_Server.Size(m) -} -func (m *Server) XXX_DiscardUnknown() { - xxx_messageInfo_Server.DiscardUnknown(m) + +func (*Server) ProtoMessage() {} + +func (x *Server) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[10] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_Server proto.InternalMessageInfo +// Deprecated: Use Server.ProtoReflect.Descriptor instead. +func (*Server) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{10} +} -func (m *Server) GetRef() *ServerRef { - if m != nil { - return m.Ref +func (x *Server) GetRef() *ServerRef { + if x != nil { + return x.Ref } return nil } -func (m *Server) GetData() *ServerData { - if m != nil { - return m.Data +func (x *Server) GetData() *ServerData { + if x != nil { + return x.Data } return nil } -func (m *Server) GetListenSocket() []*SocketRef { - if m != nil { - return m.ListenSocket +func (x *Server) GetListenSocket() []*SocketRef { + if x != nil { + return x.ListenSocket } return nil } // ServerData is data for a specific Server. type ServerData struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // A trace of recent events on the server. May be absent. Trace *ChannelTrace `protobuf:"bytes,1,opt,name=trace,proto3" json:"trace,omitempty"` // The number of incoming calls started on the server @@ -846,74 +990,82 @@ type ServerData struct { // The number of incoming calls that have a completed with a non-OK status CallsFailed int64 `protobuf:"varint,4,opt,name=calls_failed,json=callsFailed,proto3" json:"calls_failed,omitempty"` // The last time a call was started on the server. - LastCallStartedTimestamp *timestamp.Timestamp `protobuf:"bytes,5,opt,name=last_call_started_timestamp,json=lastCallStartedTimestamp,proto3" json:"last_call_started_timestamp,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + LastCallStartedTimestamp *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=last_call_started_timestamp,json=lastCallStartedTimestamp,proto3" json:"last_call_started_timestamp,omitempty"` } -func (m *ServerData) Reset() { *m = ServerData{} } -func (m *ServerData) String() string { return proto.CompactTextString(m) } -func (*ServerData) ProtoMessage() {} -func (*ServerData) Descriptor() ([]byte, []int) { - return fileDescriptor_6ee37dfd35a8ab00, []int{11} +func (x *ServerData) Reset() { + *x = ServerData{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *ServerData) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ServerData.Unmarshal(m, b) +func (x *ServerData) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *ServerData) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ServerData.Marshal(b, m, deterministic) -} -func (m *ServerData) XXX_Merge(src proto.Message) { - xxx_messageInfo_ServerData.Merge(m, src) -} -func (m *ServerData) XXX_Size() int { - return xxx_messageInfo_ServerData.Size(m) -} -func (m *ServerData) XXX_DiscardUnknown() { - xxx_messageInfo_ServerData.DiscardUnknown(m) + +func (*ServerData) ProtoMessage() {} + +func (x *ServerData) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[11] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_ServerData proto.InternalMessageInfo +// Deprecated: Use ServerData.ProtoReflect.Descriptor instead. +func (*ServerData) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{11} +} -func (m *ServerData) GetTrace() *ChannelTrace { - if m != nil { - return m.Trace +func (x *ServerData) GetTrace() *ChannelTrace { + if x != nil { + return x.Trace } return nil } -func (m *ServerData) GetCallsStarted() int64 { - if m != nil { - return m.CallsStarted +func (x *ServerData) GetCallsStarted() int64 { + if x != nil { + return x.CallsStarted } return 0 } -func (m *ServerData) GetCallsSucceeded() int64 { - if m != nil { - return m.CallsSucceeded +func (x *ServerData) GetCallsSucceeded() int64 { + if x != nil { + return x.CallsSucceeded } return 0 } -func (m *ServerData) GetCallsFailed() int64 { - if m != nil { - return m.CallsFailed +func (x *ServerData) GetCallsFailed() int64 { + if x != nil { + return x.CallsFailed } return 0 } -func (m *ServerData) GetLastCallStartedTimestamp() *timestamp.Timestamp { - if m != nil { - return m.LastCallStartedTimestamp +func (x *ServerData) GetLastCallStartedTimestamp() *timestamppb.Timestamp { + if x != nil { + return x.LastCallStartedTimestamp } return nil } // Information about an actual connection. Pronounced "sock-ay". type Socket struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // The identifier for the Socket. Ref *SocketRef `protobuf:"bytes,1,opt,name=ref,proto3" json:"ref,omitempty"` // Data specific to this Socket. @@ -927,75 +1079,79 @@ type Socket struct { Security *Security `protobuf:"bytes,5,opt,name=security,proto3" json:"security,omitempty"` // Optional, represents the name of the remote endpoint, if different than // the original target name. - RemoteName string `protobuf:"bytes,6,opt,name=remote_name,json=remoteName,proto3" json:"remote_name,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + RemoteName string `protobuf:"bytes,6,opt,name=remote_name,json=remoteName,proto3" json:"remote_name,omitempty"` } -func (m *Socket) Reset() { *m = Socket{} } -func (m *Socket) String() string { return proto.CompactTextString(m) } -func (*Socket) ProtoMessage() {} -func (*Socket) Descriptor() ([]byte, []int) { - return fileDescriptor_6ee37dfd35a8ab00, []int{12} +func (x *Socket) Reset() { + *x = Socket{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *Socket) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Socket.Unmarshal(m, b) +func (x *Socket) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *Socket) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Socket.Marshal(b, m, deterministic) -} -func (m *Socket) XXX_Merge(src proto.Message) { - xxx_messageInfo_Socket.Merge(m, src) -} -func (m *Socket) XXX_Size() int { - return xxx_messageInfo_Socket.Size(m) -} -func (m *Socket) XXX_DiscardUnknown() { - xxx_messageInfo_Socket.DiscardUnknown(m) + +func (*Socket) ProtoMessage() {} + +func (x *Socket) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[12] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_Socket proto.InternalMessageInfo +// Deprecated: Use Socket.ProtoReflect.Descriptor instead. +func (*Socket) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{12} +} -func (m *Socket) GetRef() *SocketRef { - if m != nil { - return m.Ref +func (x *Socket) GetRef() *SocketRef { + if x != nil { + return x.Ref } return nil } -func (m *Socket) GetData() *SocketData { - if m != nil { - return m.Data +func (x *Socket) GetData() *SocketData { + if x != nil { + return x.Data } return nil } -func (m *Socket) GetLocal() *Address { - if m != nil { - return m.Local +func (x *Socket) GetLocal() *Address { + if x != nil { + return x.Local } return nil } -func (m *Socket) GetRemote() *Address { - if m != nil { - return m.Remote +func (x *Socket) GetRemote() *Address { + if x != nil { + return x.Remote } return nil } -func (m *Socket) GetSecurity() *Security { - if m != nil { - return m.Security +func (x *Socket) GetSecurity() *Security { + if x != nil { + return x.Security } return nil } -func (m *Socket) GetRemoteName() string { - if m != nil { - return m.RemoteName +func (x *Socket) GetRemoteName() string { + if x != nil { + return x.RemoteName } return "" } @@ -1004,6 +1160,10 @@ func (m *Socket) GetRemoteName() string { // are specific to the implementation, so there may be minor differences in // the semantics. (e.g. flow control windows) type SocketData struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // The number of streams that have been started. StreamsStarted int64 `protobuf:"varint,1,opt,name=streams_started,json=streamsStarted,proto3" json:"streams_started,omitempty"` // The number of streams that have ended successfully: @@ -1023,205 +1183,196 @@ type SocketData struct { KeepAlivesSent int64 `protobuf:"varint,6,opt,name=keep_alives_sent,json=keepAlivesSent,proto3" json:"keep_alives_sent,omitempty"` // The last time a stream was created by this endpoint. Usually unset for // servers. - LastLocalStreamCreatedTimestamp *timestamp.Timestamp `protobuf:"bytes,7,opt,name=last_local_stream_created_timestamp,json=lastLocalStreamCreatedTimestamp,proto3" json:"last_local_stream_created_timestamp,omitempty"` + LastLocalStreamCreatedTimestamp *timestamppb.Timestamp `protobuf:"bytes,7,opt,name=last_local_stream_created_timestamp,json=lastLocalStreamCreatedTimestamp,proto3" json:"last_local_stream_created_timestamp,omitempty"` // The last time a stream was created by the remote endpoint. Usually unset // for clients. - LastRemoteStreamCreatedTimestamp *timestamp.Timestamp `protobuf:"bytes,8,opt,name=last_remote_stream_created_timestamp,json=lastRemoteStreamCreatedTimestamp,proto3" json:"last_remote_stream_created_timestamp,omitempty"` + LastRemoteStreamCreatedTimestamp *timestamppb.Timestamp `protobuf:"bytes,8,opt,name=last_remote_stream_created_timestamp,json=lastRemoteStreamCreatedTimestamp,proto3" json:"last_remote_stream_created_timestamp,omitempty"` // The last time a message was sent by this endpoint. - LastMessageSentTimestamp *timestamp.Timestamp `protobuf:"bytes,9,opt,name=last_message_sent_timestamp,json=lastMessageSentTimestamp,proto3" json:"last_message_sent_timestamp,omitempty"` + LastMessageSentTimestamp *timestamppb.Timestamp `protobuf:"bytes,9,opt,name=last_message_sent_timestamp,json=lastMessageSentTimestamp,proto3" json:"last_message_sent_timestamp,omitempty"` // The last time a message was received by this endpoint. - LastMessageReceivedTimestamp *timestamp.Timestamp `protobuf:"bytes,10,opt,name=last_message_received_timestamp,json=lastMessageReceivedTimestamp,proto3" json:"last_message_received_timestamp,omitempty"` + LastMessageReceivedTimestamp *timestamppb.Timestamp `protobuf:"bytes,10,opt,name=last_message_received_timestamp,json=lastMessageReceivedTimestamp,proto3" json:"last_message_received_timestamp,omitempty"` // The amount of window, granted to the local endpoint by the remote endpoint. // This may be slightly out of date due to network latency. This does NOT // include stream level or TCP level flow control info. - LocalFlowControlWindow *wrappers.Int64Value `protobuf:"bytes,11,opt,name=local_flow_control_window,json=localFlowControlWindow,proto3" json:"local_flow_control_window,omitempty"` + LocalFlowControlWindow *wrapperspb.Int64Value `protobuf:"bytes,11,opt,name=local_flow_control_window,json=localFlowControlWindow,proto3" json:"local_flow_control_window,omitempty"` // The amount of window, granted to the remote endpoint by the local endpoint. // This may be slightly out of date due to network latency. This does NOT // include stream level or TCP level flow control info. - RemoteFlowControlWindow *wrappers.Int64Value `protobuf:"bytes,12,opt,name=remote_flow_control_window,json=remoteFlowControlWindow,proto3" json:"remote_flow_control_window,omitempty"` + RemoteFlowControlWindow *wrapperspb.Int64Value `protobuf:"bytes,12,opt,name=remote_flow_control_window,json=remoteFlowControlWindow,proto3" json:"remote_flow_control_window,omitempty"` // Socket options set on this socket. May be absent if 'summary' is set // on GetSocketRequest. - Option []*SocketOption `protobuf:"bytes,13,rep,name=option,proto3" json:"option,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Option []*SocketOption `protobuf:"bytes,13,rep,name=option,proto3" json:"option,omitempty"` } -func (m *SocketData) Reset() { *m = SocketData{} } -func (m *SocketData) String() string { return proto.CompactTextString(m) } -func (*SocketData) ProtoMessage() {} -func (*SocketData) Descriptor() ([]byte, []int) { - return fileDescriptor_6ee37dfd35a8ab00, []int{13} +func (x *SocketData) Reset() { + *x = SocketData{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *SocketData) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_SocketData.Unmarshal(m, b) -} -func (m *SocketData) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_SocketData.Marshal(b, m, deterministic) -} -func (m *SocketData) XXX_Merge(src proto.Message) { - xxx_messageInfo_SocketData.Merge(m, src) -} -func (m *SocketData) XXX_Size() int { - return xxx_messageInfo_SocketData.Size(m) +func (x *SocketData) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *SocketData) XXX_DiscardUnknown() { - xxx_messageInfo_SocketData.DiscardUnknown(m) + +func (*SocketData) ProtoMessage() {} + +func (x *SocketData) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[13] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_SocketData proto.InternalMessageInfo +// Deprecated: Use SocketData.ProtoReflect.Descriptor instead. +func (*SocketData) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{13} +} -func (m *SocketData) GetStreamsStarted() int64 { - if m != nil { - return m.StreamsStarted +func (x *SocketData) GetStreamsStarted() int64 { + if x != nil { + return x.StreamsStarted } return 0 } -func (m *SocketData) GetStreamsSucceeded() int64 { - if m != nil { - return m.StreamsSucceeded +func (x *SocketData) GetStreamsSucceeded() int64 { + if x != nil { + return x.StreamsSucceeded } return 0 } -func (m *SocketData) GetStreamsFailed() int64 { - if m != nil { - return m.StreamsFailed +func (x *SocketData) GetStreamsFailed() int64 { + if x != nil { + return x.StreamsFailed } return 0 } -func (m *SocketData) GetMessagesSent() int64 { - if m != nil { - return m.MessagesSent +func (x *SocketData) GetMessagesSent() int64 { + if x != nil { + return x.MessagesSent } return 0 } -func (m *SocketData) GetMessagesReceived() int64 { - if m != nil { - return m.MessagesReceived +func (x *SocketData) GetMessagesReceived() int64 { + if x != nil { + return x.MessagesReceived } return 0 } -func (m *SocketData) GetKeepAlivesSent() int64 { - if m != nil { - return m.KeepAlivesSent +func (x *SocketData) GetKeepAlivesSent() int64 { + if x != nil { + return x.KeepAlivesSent } return 0 } -func (m *SocketData) GetLastLocalStreamCreatedTimestamp() *timestamp.Timestamp { - if m != nil { - return m.LastLocalStreamCreatedTimestamp +func (x *SocketData) GetLastLocalStreamCreatedTimestamp() *timestamppb.Timestamp { + if x != nil { + return x.LastLocalStreamCreatedTimestamp } return nil } -func (m *SocketData) GetLastRemoteStreamCreatedTimestamp() *timestamp.Timestamp { - if m != nil { - return m.LastRemoteStreamCreatedTimestamp +func (x *SocketData) GetLastRemoteStreamCreatedTimestamp() *timestamppb.Timestamp { + if x != nil { + return x.LastRemoteStreamCreatedTimestamp } return nil } -func (m *SocketData) GetLastMessageSentTimestamp() *timestamp.Timestamp { - if m != nil { - return m.LastMessageSentTimestamp +func (x *SocketData) GetLastMessageSentTimestamp() *timestamppb.Timestamp { + if x != nil { + return x.LastMessageSentTimestamp } return nil } -func (m *SocketData) GetLastMessageReceivedTimestamp() *timestamp.Timestamp { - if m != nil { - return m.LastMessageReceivedTimestamp +func (x *SocketData) GetLastMessageReceivedTimestamp() *timestamppb.Timestamp { + if x != nil { + return x.LastMessageReceivedTimestamp } return nil } -func (m *SocketData) GetLocalFlowControlWindow() *wrappers.Int64Value { - if m != nil { - return m.LocalFlowControlWindow +func (x *SocketData) GetLocalFlowControlWindow() *wrapperspb.Int64Value { + if x != nil { + return x.LocalFlowControlWindow } return nil } -func (m *SocketData) GetRemoteFlowControlWindow() *wrappers.Int64Value { - if m != nil { - return m.RemoteFlowControlWindow +func (x *SocketData) GetRemoteFlowControlWindow() *wrapperspb.Int64Value { + if x != nil { + return x.RemoteFlowControlWindow } return nil } -func (m *SocketData) GetOption() []*SocketOption { - if m != nil { - return m.Option +func (x *SocketData) GetOption() []*SocketOption { + if x != nil { + return x.Option } return nil } // Address represents the address used to create the socket. type Address struct { - // Types that are valid to be assigned to Address: + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Types that are assignable to Address: + // // *Address_TcpipAddress // *Address_UdsAddress_ // *Address_OtherAddress_ - Address isAddress_Address `protobuf_oneof:"address"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Address isAddress_Address `protobuf_oneof:"address"` } -func (m *Address) Reset() { *m = Address{} } -func (m *Address) String() string { return proto.CompactTextString(m) } -func (*Address) ProtoMessage() {} -func (*Address) Descriptor() ([]byte, []int) { - return fileDescriptor_6ee37dfd35a8ab00, []int{14} -} - -func (m *Address) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Address.Unmarshal(m, b) -} -func (m *Address) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Address.Marshal(b, m, deterministic) -} -func (m *Address) XXX_Merge(src proto.Message) { - xxx_messageInfo_Address.Merge(m, src) -} -func (m *Address) XXX_Size() int { - return xxx_messageInfo_Address.Size(m) -} -func (m *Address) XXX_DiscardUnknown() { - xxx_messageInfo_Address.DiscardUnknown(m) +func (x *Address) Reset() { + *x = Address{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[14] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -var xxx_messageInfo_Address proto.InternalMessageInfo - -type isAddress_Address interface { - isAddress_Address() +func (x *Address) String() string { + return protoimpl.X.MessageStringOf(x) } -type Address_TcpipAddress struct { - TcpipAddress *Address_TcpIpAddress `protobuf:"bytes,1,opt,name=tcpip_address,json=tcpipAddress,proto3,oneof"` -} +func (*Address) ProtoMessage() {} -type Address_UdsAddress_ struct { - UdsAddress *Address_UdsAddress `protobuf:"bytes,2,opt,name=uds_address,json=udsAddress,proto3,oneof"` +func (x *Address) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[14] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -type Address_OtherAddress_ struct { - OtherAddress *Address_OtherAddress `protobuf:"bytes,3,opt,name=other_address,json=otherAddress,proto3,oneof"` +// Deprecated: Use Address.ProtoReflect.Descriptor instead. +func (*Address) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{14} } -func (*Address_TcpipAddress) isAddress_Address() {} - -func (*Address_UdsAddress_) isAddress_Address() {} - -func (*Address_OtherAddress_) isAddress_Address() {} - func (m *Address) GetAddress() isAddress_Address { if m != nil { return m.Address @@ -1229,228 +1380,94 @@ func (m *Address) GetAddress() isAddress_Address { return nil } -func (m *Address) GetTcpipAddress() *Address_TcpIpAddress { - if x, ok := m.GetAddress().(*Address_TcpipAddress); ok { +func (x *Address) GetTcpipAddress() *Address_TcpIpAddress { + if x, ok := x.GetAddress().(*Address_TcpipAddress); ok { return x.TcpipAddress } return nil } -func (m *Address) GetUdsAddress() *Address_UdsAddress { - if x, ok := m.GetAddress().(*Address_UdsAddress_); ok { +func (x *Address) GetUdsAddress() *Address_UdsAddress { + if x, ok := x.GetAddress().(*Address_UdsAddress_); ok { return x.UdsAddress } return nil } -func (m *Address) GetOtherAddress() *Address_OtherAddress { - if x, ok := m.GetAddress().(*Address_OtherAddress_); ok { +func (x *Address) GetOtherAddress() *Address_OtherAddress { + if x, ok := x.GetAddress().(*Address_OtherAddress_); ok { return x.OtherAddress } return nil } -// XXX_OneofWrappers is for the internal use of the proto package. -func (*Address) XXX_OneofWrappers() []interface{} { - return []interface{}{ - (*Address_TcpipAddress)(nil), - (*Address_UdsAddress_)(nil), - (*Address_OtherAddress_)(nil), - } -} - -type Address_TcpIpAddress struct { - // Either the IPv4 or IPv6 address in bytes. Will be either 4 bytes or 16 - // bytes in length. - IpAddress []byte `protobuf:"bytes,1,opt,name=ip_address,json=ipAddress,proto3" json:"ip_address,omitempty"` - // 0-64k, or -1 if not appropriate. - Port int32 `protobuf:"varint,2,opt,name=port,proto3" json:"port,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *Address_TcpIpAddress) Reset() { *m = Address_TcpIpAddress{} } -func (m *Address_TcpIpAddress) String() string { return proto.CompactTextString(m) } -func (*Address_TcpIpAddress) ProtoMessage() {} -func (*Address_TcpIpAddress) Descriptor() ([]byte, []int) { - return fileDescriptor_6ee37dfd35a8ab00, []int{14, 0} -} - -func (m *Address_TcpIpAddress) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Address_TcpIpAddress.Unmarshal(m, b) -} -func (m *Address_TcpIpAddress) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Address_TcpIpAddress.Marshal(b, m, deterministic) -} -func (m *Address_TcpIpAddress) XXX_Merge(src proto.Message) { - xxx_messageInfo_Address_TcpIpAddress.Merge(m, src) -} -func (m *Address_TcpIpAddress) XXX_Size() int { - return xxx_messageInfo_Address_TcpIpAddress.Size(m) -} -func (m *Address_TcpIpAddress) XXX_DiscardUnknown() { - xxx_messageInfo_Address_TcpIpAddress.DiscardUnknown(m) -} - -var xxx_messageInfo_Address_TcpIpAddress proto.InternalMessageInfo - -func (m *Address_TcpIpAddress) GetIpAddress() []byte { - if m != nil { - return m.IpAddress - } - return nil -} - -func (m *Address_TcpIpAddress) GetPort() int32 { - if m != nil { - return m.Port - } - return 0 -} - -// A Unix Domain Socket address. -type Address_UdsAddress struct { - Filename string `protobuf:"bytes,1,opt,name=filename,proto3" json:"filename,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *Address_UdsAddress) Reset() { *m = Address_UdsAddress{} } -func (m *Address_UdsAddress) String() string { return proto.CompactTextString(m) } -func (*Address_UdsAddress) ProtoMessage() {} -func (*Address_UdsAddress) Descriptor() ([]byte, []int) { - return fileDescriptor_6ee37dfd35a8ab00, []int{14, 1} -} - -func (m *Address_UdsAddress) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Address_UdsAddress.Unmarshal(m, b) -} -func (m *Address_UdsAddress) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Address_UdsAddress.Marshal(b, m, deterministic) -} -func (m *Address_UdsAddress) XXX_Merge(src proto.Message) { - xxx_messageInfo_Address_UdsAddress.Merge(m, src) -} -func (m *Address_UdsAddress) XXX_Size() int { - return xxx_messageInfo_Address_UdsAddress.Size(m) -} -func (m *Address_UdsAddress) XXX_DiscardUnknown() { - xxx_messageInfo_Address_UdsAddress.DiscardUnknown(m) -} - -var xxx_messageInfo_Address_UdsAddress proto.InternalMessageInfo - -func (m *Address_UdsAddress) GetFilename() string { - if m != nil { - return m.Filename - } - return "" +type isAddress_Address interface { + isAddress_Address() } -// An address type not included above. -type Address_OtherAddress struct { - // The human readable version of the value. This value should be set. - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - // The actual address message. - Value *any.Any `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` +type Address_TcpipAddress struct { + TcpipAddress *Address_TcpIpAddress `protobuf:"bytes,1,opt,name=tcpip_address,json=tcpipAddress,proto3,oneof"` } -func (m *Address_OtherAddress) Reset() { *m = Address_OtherAddress{} } -func (m *Address_OtherAddress) String() string { return proto.CompactTextString(m) } -func (*Address_OtherAddress) ProtoMessage() {} -func (*Address_OtherAddress) Descriptor() ([]byte, []int) { - return fileDescriptor_6ee37dfd35a8ab00, []int{14, 2} +type Address_UdsAddress_ struct { + UdsAddress *Address_UdsAddress `protobuf:"bytes,2,opt,name=uds_address,json=udsAddress,proto3,oneof"` } -func (m *Address_OtherAddress) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Address_OtherAddress.Unmarshal(m, b) -} -func (m *Address_OtherAddress) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Address_OtherAddress.Marshal(b, m, deterministic) -} -func (m *Address_OtherAddress) XXX_Merge(src proto.Message) { - xxx_messageInfo_Address_OtherAddress.Merge(m, src) -} -func (m *Address_OtherAddress) XXX_Size() int { - return xxx_messageInfo_Address_OtherAddress.Size(m) -} -func (m *Address_OtherAddress) XXX_DiscardUnknown() { - xxx_messageInfo_Address_OtherAddress.DiscardUnknown(m) +type Address_OtherAddress_ struct { + OtherAddress *Address_OtherAddress `protobuf:"bytes,3,opt,name=other_address,json=otherAddress,proto3,oneof"` } -var xxx_messageInfo_Address_OtherAddress proto.InternalMessageInfo +func (*Address_TcpipAddress) isAddress_Address() {} -func (m *Address_OtherAddress) GetName() string { - if m != nil { - return m.Name - } - return "" -} +func (*Address_UdsAddress_) isAddress_Address() {} -func (m *Address_OtherAddress) GetValue() *any.Any { - if m != nil { - return m.Value - } - return nil -} +func (*Address_OtherAddress_) isAddress_Address() {} // Security represents details about how secure the socket is. type Security struct { - // Types that are valid to be assigned to Model: + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Types that are assignable to Model: + // // *Security_Tls_ // *Security_Other - Model isSecurity_Model `protobuf_oneof:"model"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Model isSecurity_Model `protobuf_oneof:"model"` } -func (m *Security) Reset() { *m = Security{} } -func (m *Security) String() string { return proto.CompactTextString(m) } -func (*Security) ProtoMessage() {} -func (*Security) Descriptor() ([]byte, []int) { - return fileDescriptor_6ee37dfd35a8ab00, []int{15} +func (x *Security) Reset() { + *x = Security{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[15] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *Security) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Security.Unmarshal(m, b) -} -func (m *Security) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Security.Marshal(b, m, deterministic) -} -func (m *Security) XXX_Merge(src proto.Message) { - xxx_messageInfo_Security.Merge(m, src) -} -func (m *Security) XXX_Size() int { - return xxx_messageInfo_Security.Size(m) -} -func (m *Security) XXX_DiscardUnknown() { - xxx_messageInfo_Security.DiscardUnknown(m) +func (x *Security) String() string { + return protoimpl.X.MessageStringOf(x) } -var xxx_messageInfo_Security proto.InternalMessageInfo - -type isSecurity_Model interface { - isSecurity_Model() -} +func (*Security) ProtoMessage() {} -type Security_Tls_ struct { - Tls *Security_Tls `protobuf:"bytes,1,opt,name=tls,proto3,oneof"` +func (x *Security) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[15] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -type Security_Other struct { - Other *Security_OtherSecurity `protobuf:"bytes,2,opt,name=other,proto3,oneof"` +// Deprecated: Use Security.ProtoReflect.Descriptor instead. +func (*Security) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{15} } -func (*Security_Tls_) isSecurity_Model() {} - -func (*Security_Other) isSecurity_Model() {} - func (m *Security) GetModel() isSecurity_Model { if m != nil { return m.Model @@ -1458,178 +1475,43 @@ func (m *Security) GetModel() isSecurity_Model { return nil } -func (m *Security) GetTls() *Security_Tls { - if x, ok := m.GetModel().(*Security_Tls_); ok { +func (x *Security) GetTls() *Security_Tls { + if x, ok := x.GetModel().(*Security_Tls_); ok { return x.Tls } return nil } -func (m *Security) GetOther() *Security_OtherSecurity { - if x, ok := m.GetModel().(*Security_Other); ok { +func (x *Security) GetOther() *Security_OtherSecurity { + if x, ok := x.GetModel().(*Security_Other); ok { return x.Other } return nil } -// XXX_OneofWrappers is for the internal use of the proto package. -func (*Security) XXX_OneofWrappers() []interface{} { - return []interface{}{ - (*Security_Tls_)(nil), - (*Security_Other)(nil), - } -} - -type Security_Tls struct { - // Types that are valid to be assigned to CipherSuite: - // *Security_Tls_StandardName - // *Security_Tls_OtherName - CipherSuite isSecurity_Tls_CipherSuite `protobuf_oneof:"cipher_suite"` - // the certificate used by this endpoint. - LocalCertificate []byte `protobuf:"bytes,3,opt,name=local_certificate,json=localCertificate,proto3" json:"local_certificate,omitempty"` - // the certificate used by the remote endpoint. - RemoteCertificate []byte `protobuf:"bytes,4,opt,name=remote_certificate,json=remoteCertificate,proto3" json:"remote_certificate,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *Security_Tls) Reset() { *m = Security_Tls{} } -func (m *Security_Tls) String() string { return proto.CompactTextString(m) } -func (*Security_Tls) ProtoMessage() {} -func (*Security_Tls) Descriptor() ([]byte, []int) { - return fileDescriptor_6ee37dfd35a8ab00, []int{15, 0} -} - -func (m *Security_Tls) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Security_Tls.Unmarshal(m, b) -} -func (m *Security_Tls) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Security_Tls.Marshal(b, m, deterministic) -} -func (m *Security_Tls) XXX_Merge(src proto.Message) { - xxx_messageInfo_Security_Tls.Merge(m, src) -} -func (m *Security_Tls) XXX_Size() int { - return xxx_messageInfo_Security_Tls.Size(m) -} -func (m *Security_Tls) XXX_DiscardUnknown() { - xxx_messageInfo_Security_Tls.DiscardUnknown(m) -} - -var xxx_messageInfo_Security_Tls proto.InternalMessageInfo - -type isSecurity_Tls_CipherSuite interface { - isSecurity_Tls_CipherSuite() -} - -type Security_Tls_StandardName struct { - StandardName string `protobuf:"bytes,1,opt,name=standard_name,json=standardName,proto3,oneof"` -} - -type Security_Tls_OtherName struct { - OtherName string `protobuf:"bytes,2,opt,name=other_name,json=otherName,proto3,oneof"` -} - -func (*Security_Tls_StandardName) isSecurity_Tls_CipherSuite() {} - -func (*Security_Tls_OtherName) isSecurity_Tls_CipherSuite() {} - -func (m *Security_Tls) GetCipherSuite() isSecurity_Tls_CipherSuite { - if m != nil { - return m.CipherSuite - } - return nil -} - -func (m *Security_Tls) GetStandardName() string { - if x, ok := m.GetCipherSuite().(*Security_Tls_StandardName); ok { - return x.StandardName - } - return "" -} - -func (m *Security_Tls) GetOtherName() string { - if x, ok := m.GetCipherSuite().(*Security_Tls_OtherName); ok { - return x.OtherName - } - return "" -} - -func (m *Security_Tls) GetLocalCertificate() []byte { - if m != nil { - return m.LocalCertificate - } - return nil -} - -func (m *Security_Tls) GetRemoteCertificate() []byte { - if m != nil { - return m.RemoteCertificate - } - return nil -} - -// XXX_OneofWrappers is for the internal use of the proto package. -func (*Security_Tls) XXX_OneofWrappers() []interface{} { - return []interface{}{ - (*Security_Tls_StandardName)(nil), - (*Security_Tls_OtherName)(nil), - } -} - -type Security_OtherSecurity struct { - // The human readable version of the value. - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - // The actual security details message. - Value *any.Any `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` +type isSecurity_Model interface { + isSecurity_Model() } -func (m *Security_OtherSecurity) Reset() { *m = Security_OtherSecurity{} } -func (m *Security_OtherSecurity) String() string { return proto.CompactTextString(m) } -func (*Security_OtherSecurity) ProtoMessage() {} -func (*Security_OtherSecurity) Descriptor() ([]byte, []int) { - return fileDescriptor_6ee37dfd35a8ab00, []int{15, 1} +type Security_Tls_ struct { + Tls *Security_Tls `protobuf:"bytes,1,opt,name=tls,proto3,oneof"` } -func (m *Security_OtherSecurity) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Security_OtherSecurity.Unmarshal(m, b) -} -func (m *Security_OtherSecurity) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Security_OtherSecurity.Marshal(b, m, deterministic) -} -func (m *Security_OtherSecurity) XXX_Merge(src proto.Message) { - xxx_messageInfo_Security_OtherSecurity.Merge(m, src) -} -func (m *Security_OtherSecurity) XXX_Size() int { - return xxx_messageInfo_Security_OtherSecurity.Size(m) -} -func (m *Security_OtherSecurity) XXX_DiscardUnknown() { - xxx_messageInfo_Security_OtherSecurity.DiscardUnknown(m) +type Security_Other struct { + Other *Security_OtherSecurity `protobuf:"bytes,2,opt,name=other,proto3,oneof"` } -var xxx_messageInfo_Security_OtherSecurity proto.InternalMessageInfo - -func (m *Security_OtherSecurity) GetName() string { - if m != nil { - return m.Name - } - return "" -} +func (*Security_Tls_) isSecurity_Model() {} -func (m *Security_OtherSecurity) GetValue() *any.Any { - if m != nil { - return m.Value - } - return nil -} +func (*Security_Other) isSecurity_Model() {} // SocketOption represents socket options for a socket. Specifically, these // are the options returned by getsockopt(). type SocketOption struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // The full name of the socket option. Typically this will be the upper case // name, such as "SO_REUSEPORT". Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` @@ -1638,54 +1520,58 @@ type SocketOption struct { Value string `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` // Additional data associated with the socket option. At least one of value // or additional will be set. - Additional *any.Any `protobuf:"bytes,3,opt,name=additional,proto3" json:"additional,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Additional *anypb.Any `protobuf:"bytes,3,opt,name=additional,proto3" json:"additional,omitempty"` } -func (m *SocketOption) Reset() { *m = SocketOption{} } -func (m *SocketOption) String() string { return proto.CompactTextString(m) } -func (*SocketOption) ProtoMessage() {} -func (*SocketOption) Descriptor() ([]byte, []int) { - return fileDescriptor_6ee37dfd35a8ab00, []int{16} +func (x *SocketOption) Reset() { + *x = SocketOption{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[16] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *SocketOption) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_SocketOption.Unmarshal(m, b) -} -func (m *SocketOption) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_SocketOption.Marshal(b, m, deterministic) -} -func (m *SocketOption) XXX_Merge(src proto.Message) { - xxx_messageInfo_SocketOption.Merge(m, src) -} -func (m *SocketOption) XXX_Size() int { - return xxx_messageInfo_SocketOption.Size(m) +func (x *SocketOption) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *SocketOption) XXX_DiscardUnknown() { - xxx_messageInfo_SocketOption.DiscardUnknown(m) + +func (*SocketOption) ProtoMessage() {} + +func (x *SocketOption) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[16] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_SocketOption proto.InternalMessageInfo +// Deprecated: Use SocketOption.ProtoReflect.Descriptor instead. +func (*SocketOption) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{16} +} -func (m *SocketOption) GetName() string { - if m != nil { - return m.Name +func (x *SocketOption) GetName() string { + if x != nil { + return x.Name } return "" } -func (m *SocketOption) GetValue() string { - if m != nil { - return m.Value +func (x *SocketOption) GetValue() string { + if x != nil { + return x.Value } return "" } -func (m *SocketOption) GetAdditional() *any.Any { - if m != nil { - return m.Additional +func (x *SocketOption) GetAdditional() *anypb.Any { + if x != nil { + return x.Additional } return nil } @@ -1693,40 +1579,48 @@ func (m *SocketOption) GetAdditional() *any.Any { // For use with SocketOption's additional field. This is primarily used for // SO_RCVTIMEO and SO_SNDTIMEO type SocketOptionTimeout struct { - Duration *duration.Duration `protobuf:"bytes,1,opt,name=duration,proto3" json:"duration,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields -func (m *SocketOptionTimeout) Reset() { *m = SocketOptionTimeout{} } -func (m *SocketOptionTimeout) String() string { return proto.CompactTextString(m) } -func (*SocketOptionTimeout) ProtoMessage() {} -func (*SocketOptionTimeout) Descriptor() ([]byte, []int) { - return fileDescriptor_6ee37dfd35a8ab00, []int{17} + Duration *durationpb.Duration `protobuf:"bytes,1,opt,name=duration,proto3" json:"duration,omitempty"` } -func (m *SocketOptionTimeout) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_SocketOptionTimeout.Unmarshal(m, b) -} -func (m *SocketOptionTimeout) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_SocketOptionTimeout.Marshal(b, m, deterministic) -} -func (m *SocketOptionTimeout) XXX_Merge(src proto.Message) { - xxx_messageInfo_SocketOptionTimeout.Merge(m, src) +func (x *SocketOptionTimeout) Reset() { + *x = SocketOptionTimeout{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[17] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *SocketOptionTimeout) XXX_Size() int { - return xxx_messageInfo_SocketOptionTimeout.Size(m) + +func (x *SocketOptionTimeout) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *SocketOptionTimeout) XXX_DiscardUnknown() { - xxx_messageInfo_SocketOptionTimeout.DiscardUnknown(m) + +func (*SocketOptionTimeout) ProtoMessage() {} + +func (x *SocketOptionTimeout) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[17] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_SocketOptionTimeout proto.InternalMessageInfo +// Deprecated: Use SocketOptionTimeout.ProtoReflect.Descriptor instead. +func (*SocketOptionTimeout) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{17} +} -func (m *SocketOptionTimeout) GetDuration() *duration.Duration { - if m != nil { - return m.Duration +func (x *SocketOptionTimeout) GetDuration() *durationpb.Duration { + if x != nil { + return x.Duration } return nil } @@ -1734,50 +1628,58 @@ func (m *SocketOptionTimeout) GetDuration() *duration.Duration { // For use with SocketOption's additional field. This is primarily used for // SO_LINGER. type SocketOptionLinger struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // active maps to `struct linger.l_onoff` Active bool `protobuf:"varint,1,opt,name=active,proto3" json:"active,omitempty"` // duration maps to `struct linger.l_linger` - Duration *duration.Duration `protobuf:"bytes,2,opt,name=duration,proto3" json:"duration,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Duration *durationpb.Duration `protobuf:"bytes,2,opt,name=duration,proto3" json:"duration,omitempty"` } -func (m *SocketOptionLinger) Reset() { *m = SocketOptionLinger{} } -func (m *SocketOptionLinger) String() string { return proto.CompactTextString(m) } -func (*SocketOptionLinger) ProtoMessage() {} -func (*SocketOptionLinger) Descriptor() ([]byte, []int) { - return fileDescriptor_6ee37dfd35a8ab00, []int{18} +func (x *SocketOptionLinger) Reset() { + *x = SocketOptionLinger{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[18] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *SocketOptionLinger) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_SocketOptionLinger.Unmarshal(m, b) -} -func (m *SocketOptionLinger) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_SocketOptionLinger.Marshal(b, m, deterministic) +func (x *SocketOptionLinger) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *SocketOptionLinger) XXX_Merge(src proto.Message) { - xxx_messageInfo_SocketOptionLinger.Merge(m, src) -} -func (m *SocketOptionLinger) XXX_Size() int { - return xxx_messageInfo_SocketOptionLinger.Size(m) -} -func (m *SocketOptionLinger) XXX_DiscardUnknown() { - xxx_messageInfo_SocketOptionLinger.DiscardUnknown(m) + +func (*SocketOptionLinger) ProtoMessage() {} + +func (x *SocketOptionLinger) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[18] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_SocketOptionLinger proto.InternalMessageInfo +// Deprecated: Use SocketOptionLinger.ProtoReflect.Descriptor instead. +func (*SocketOptionLinger) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{18} +} -func (m *SocketOptionLinger) GetActive() bool { - if m != nil { - return m.Active +func (x *SocketOptionLinger) GetActive() bool { + if x != nil { + return x.Active } return false } -func (m *SocketOptionLinger) GetDuration() *duration.Duration { - if m != nil { - return m.Duration +func (x *SocketOptionLinger) GetDuration() *durationpb.Duration { + if x != nil { + return x.Duration } return nil } @@ -1785,269 +1687,281 @@ func (m *SocketOptionLinger) GetDuration() *duration.Duration { // For use with SocketOption's additional field. Tcp info for // SOL_TCP and TCP_INFO. type SocketOptionTcpInfo struct { - TcpiState uint32 `protobuf:"varint,1,opt,name=tcpi_state,json=tcpiState,proto3" json:"tcpi_state,omitempty"` - TcpiCaState uint32 `protobuf:"varint,2,opt,name=tcpi_ca_state,json=tcpiCaState,proto3" json:"tcpi_ca_state,omitempty"` - TcpiRetransmits uint32 `protobuf:"varint,3,opt,name=tcpi_retransmits,json=tcpiRetransmits,proto3" json:"tcpi_retransmits,omitempty"` - TcpiProbes uint32 `protobuf:"varint,4,opt,name=tcpi_probes,json=tcpiProbes,proto3" json:"tcpi_probes,omitempty"` - TcpiBackoff uint32 `protobuf:"varint,5,opt,name=tcpi_backoff,json=tcpiBackoff,proto3" json:"tcpi_backoff,omitempty"` - TcpiOptions uint32 `protobuf:"varint,6,opt,name=tcpi_options,json=tcpiOptions,proto3" json:"tcpi_options,omitempty"` - TcpiSndWscale uint32 `protobuf:"varint,7,opt,name=tcpi_snd_wscale,json=tcpiSndWscale,proto3" json:"tcpi_snd_wscale,omitempty"` - TcpiRcvWscale uint32 `protobuf:"varint,8,opt,name=tcpi_rcv_wscale,json=tcpiRcvWscale,proto3" json:"tcpi_rcv_wscale,omitempty"` - TcpiRto uint32 `protobuf:"varint,9,opt,name=tcpi_rto,json=tcpiRto,proto3" json:"tcpi_rto,omitempty"` - TcpiAto uint32 `protobuf:"varint,10,opt,name=tcpi_ato,json=tcpiAto,proto3" json:"tcpi_ato,omitempty"` - TcpiSndMss uint32 `protobuf:"varint,11,opt,name=tcpi_snd_mss,json=tcpiSndMss,proto3" json:"tcpi_snd_mss,omitempty"` - TcpiRcvMss uint32 `protobuf:"varint,12,opt,name=tcpi_rcv_mss,json=tcpiRcvMss,proto3" json:"tcpi_rcv_mss,omitempty"` - TcpiUnacked uint32 `protobuf:"varint,13,opt,name=tcpi_unacked,json=tcpiUnacked,proto3" json:"tcpi_unacked,omitempty"` - TcpiSacked uint32 `protobuf:"varint,14,opt,name=tcpi_sacked,json=tcpiSacked,proto3" json:"tcpi_sacked,omitempty"` - TcpiLost uint32 `protobuf:"varint,15,opt,name=tcpi_lost,json=tcpiLost,proto3" json:"tcpi_lost,omitempty"` - TcpiRetrans uint32 `protobuf:"varint,16,opt,name=tcpi_retrans,json=tcpiRetrans,proto3" json:"tcpi_retrans,omitempty"` - TcpiFackets uint32 `protobuf:"varint,17,opt,name=tcpi_fackets,json=tcpiFackets,proto3" json:"tcpi_fackets,omitempty"` - TcpiLastDataSent uint32 `protobuf:"varint,18,opt,name=tcpi_last_data_sent,json=tcpiLastDataSent,proto3" json:"tcpi_last_data_sent,omitempty"` - TcpiLastAckSent uint32 `protobuf:"varint,19,opt,name=tcpi_last_ack_sent,json=tcpiLastAckSent,proto3" json:"tcpi_last_ack_sent,omitempty"` - TcpiLastDataRecv uint32 `protobuf:"varint,20,opt,name=tcpi_last_data_recv,json=tcpiLastDataRecv,proto3" json:"tcpi_last_data_recv,omitempty"` - TcpiLastAckRecv uint32 `protobuf:"varint,21,opt,name=tcpi_last_ack_recv,json=tcpiLastAckRecv,proto3" json:"tcpi_last_ack_recv,omitempty"` - TcpiPmtu uint32 `protobuf:"varint,22,opt,name=tcpi_pmtu,json=tcpiPmtu,proto3" json:"tcpi_pmtu,omitempty"` - TcpiRcvSsthresh uint32 `protobuf:"varint,23,opt,name=tcpi_rcv_ssthresh,json=tcpiRcvSsthresh,proto3" json:"tcpi_rcv_ssthresh,omitempty"` - TcpiRtt uint32 `protobuf:"varint,24,opt,name=tcpi_rtt,json=tcpiRtt,proto3" json:"tcpi_rtt,omitempty"` - TcpiRttvar uint32 `protobuf:"varint,25,opt,name=tcpi_rttvar,json=tcpiRttvar,proto3" json:"tcpi_rttvar,omitempty"` - TcpiSndSsthresh uint32 `protobuf:"varint,26,opt,name=tcpi_snd_ssthresh,json=tcpiSndSsthresh,proto3" json:"tcpi_snd_ssthresh,omitempty"` - TcpiSndCwnd uint32 `protobuf:"varint,27,opt,name=tcpi_snd_cwnd,json=tcpiSndCwnd,proto3" json:"tcpi_snd_cwnd,omitempty"` - TcpiAdvmss uint32 `protobuf:"varint,28,opt,name=tcpi_advmss,json=tcpiAdvmss,proto3" json:"tcpi_advmss,omitempty"` - TcpiReordering uint32 `protobuf:"varint,29,opt,name=tcpi_reordering,json=tcpiReordering,proto3" json:"tcpi_reordering,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *SocketOptionTcpInfo) Reset() { *m = SocketOptionTcpInfo{} } -func (m *SocketOptionTcpInfo) String() string { return proto.CompactTextString(m) } -func (*SocketOptionTcpInfo) ProtoMessage() {} + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + TcpiState uint32 `protobuf:"varint,1,opt,name=tcpi_state,json=tcpiState,proto3" json:"tcpi_state,omitempty"` + TcpiCaState uint32 `protobuf:"varint,2,opt,name=tcpi_ca_state,json=tcpiCaState,proto3" json:"tcpi_ca_state,omitempty"` + TcpiRetransmits uint32 `protobuf:"varint,3,opt,name=tcpi_retransmits,json=tcpiRetransmits,proto3" json:"tcpi_retransmits,omitempty"` + TcpiProbes uint32 `protobuf:"varint,4,opt,name=tcpi_probes,json=tcpiProbes,proto3" json:"tcpi_probes,omitempty"` + TcpiBackoff uint32 `protobuf:"varint,5,opt,name=tcpi_backoff,json=tcpiBackoff,proto3" json:"tcpi_backoff,omitempty"` + TcpiOptions uint32 `protobuf:"varint,6,opt,name=tcpi_options,json=tcpiOptions,proto3" json:"tcpi_options,omitempty"` + TcpiSndWscale uint32 `protobuf:"varint,7,opt,name=tcpi_snd_wscale,json=tcpiSndWscale,proto3" json:"tcpi_snd_wscale,omitempty"` + TcpiRcvWscale uint32 `protobuf:"varint,8,opt,name=tcpi_rcv_wscale,json=tcpiRcvWscale,proto3" json:"tcpi_rcv_wscale,omitempty"` + TcpiRto uint32 `protobuf:"varint,9,opt,name=tcpi_rto,json=tcpiRto,proto3" json:"tcpi_rto,omitempty"` + TcpiAto uint32 `protobuf:"varint,10,opt,name=tcpi_ato,json=tcpiAto,proto3" json:"tcpi_ato,omitempty"` + TcpiSndMss uint32 `protobuf:"varint,11,opt,name=tcpi_snd_mss,json=tcpiSndMss,proto3" json:"tcpi_snd_mss,omitempty"` + TcpiRcvMss uint32 `protobuf:"varint,12,opt,name=tcpi_rcv_mss,json=tcpiRcvMss,proto3" json:"tcpi_rcv_mss,omitempty"` + TcpiUnacked uint32 `protobuf:"varint,13,opt,name=tcpi_unacked,json=tcpiUnacked,proto3" json:"tcpi_unacked,omitempty"` + TcpiSacked uint32 `protobuf:"varint,14,opt,name=tcpi_sacked,json=tcpiSacked,proto3" json:"tcpi_sacked,omitempty"` + TcpiLost uint32 `protobuf:"varint,15,opt,name=tcpi_lost,json=tcpiLost,proto3" json:"tcpi_lost,omitempty"` + TcpiRetrans uint32 `protobuf:"varint,16,opt,name=tcpi_retrans,json=tcpiRetrans,proto3" json:"tcpi_retrans,omitempty"` + TcpiFackets uint32 `protobuf:"varint,17,opt,name=tcpi_fackets,json=tcpiFackets,proto3" json:"tcpi_fackets,omitempty"` + TcpiLastDataSent uint32 `protobuf:"varint,18,opt,name=tcpi_last_data_sent,json=tcpiLastDataSent,proto3" json:"tcpi_last_data_sent,omitempty"` + TcpiLastAckSent uint32 `protobuf:"varint,19,opt,name=tcpi_last_ack_sent,json=tcpiLastAckSent,proto3" json:"tcpi_last_ack_sent,omitempty"` + TcpiLastDataRecv uint32 `protobuf:"varint,20,opt,name=tcpi_last_data_recv,json=tcpiLastDataRecv,proto3" json:"tcpi_last_data_recv,omitempty"` + TcpiLastAckRecv uint32 `protobuf:"varint,21,opt,name=tcpi_last_ack_recv,json=tcpiLastAckRecv,proto3" json:"tcpi_last_ack_recv,omitempty"` + TcpiPmtu uint32 `protobuf:"varint,22,opt,name=tcpi_pmtu,json=tcpiPmtu,proto3" json:"tcpi_pmtu,omitempty"` + TcpiRcvSsthresh uint32 `protobuf:"varint,23,opt,name=tcpi_rcv_ssthresh,json=tcpiRcvSsthresh,proto3" json:"tcpi_rcv_ssthresh,omitempty"` + TcpiRtt uint32 `protobuf:"varint,24,opt,name=tcpi_rtt,json=tcpiRtt,proto3" json:"tcpi_rtt,omitempty"` + TcpiRttvar uint32 `protobuf:"varint,25,opt,name=tcpi_rttvar,json=tcpiRttvar,proto3" json:"tcpi_rttvar,omitempty"` + TcpiSndSsthresh uint32 `protobuf:"varint,26,opt,name=tcpi_snd_ssthresh,json=tcpiSndSsthresh,proto3" json:"tcpi_snd_ssthresh,omitempty"` + TcpiSndCwnd uint32 `protobuf:"varint,27,opt,name=tcpi_snd_cwnd,json=tcpiSndCwnd,proto3" json:"tcpi_snd_cwnd,omitempty"` + TcpiAdvmss uint32 `protobuf:"varint,28,opt,name=tcpi_advmss,json=tcpiAdvmss,proto3" json:"tcpi_advmss,omitempty"` + TcpiReordering uint32 `protobuf:"varint,29,opt,name=tcpi_reordering,json=tcpiReordering,proto3" json:"tcpi_reordering,omitempty"` +} + +func (x *SocketOptionTcpInfo) Reset() { + *x = SocketOptionTcpInfo{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[19] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SocketOptionTcpInfo) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SocketOptionTcpInfo) ProtoMessage() {} + +func (x *SocketOptionTcpInfo) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[19] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SocketOptionTcpInfo.ProtoReflect.Descriptor instead. func (*SocketOptionTcpInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_6ee37dfd35a8ab00, []int{19} + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{19} } -func (m *SocketOptionTcpInfo) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_SocketOptionTcpInfo.Unmarshal(m, b) -} -func (m *SocketOptionTcpInfo) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_SocketOptionTcpInfo.Marshal(b, m, deterministic) -} -func (m *SocketOptionTcpInfo) XXX_Merge(src proto.Message) { - xxx_messageInfo_SocketOptionTcpInfo.Merge(m, src) -} -func (m *SocketOptionTcpInfo) XXX_Size() int { - return xxx_messageInfo_SocketOptionTcpInfo.Size(m) -} -func (m *SocketOptionTcpInfo) XXX_DiscardUnknown() { - xxx_messageInfo_SocketOptionTcpInfo.DiscardUnknown(m) -} - -var xxx_messageInfo_SocketOptionTcpInfo proto.InternalMessageInfo - -func (m *SocketOptionTcpInfo) GetTcpiState() uint32 { - if m != nil { - return m.TcpiState +func (x *SocketOptionTcpInfo) GetTcpiState() uint32 { + if x != nil { + return x.TcpiState } return 0 } -func (m *SocketOptionTcpInfo) GetTcpiCaState() uint32 { - if m != nil { - return m.TcpiCaState +func (x *SocketOptionTcpInfo) GetTcpiCaState() uint32 { + if x != nil { + return x.TcpiCaState } return 0 } -func (m *SocketOptionTcpInfo) GetTcpiRetransmits() uint32 { - if m != nil { - return m.TcpiRetransmits +func (x *SocketOptionTcpInfo) GetTcpiRetransmits() uint32 { + if x != nil { + return x.TcpiRetransmits } return 0 } -func (m *SocketOptionTcpInfo) GetTcpiProbes() uint32 { - if m != nil { - return m.TcpiProbes +func (x *SocketOptionTcpInfo) GetTcpiProbes() uint32 { + if x != nil { + return x.TcpiProbes } return 0 } -func (m *SocketOptionTcpInfo) GetTcpiBackoff() uint32 { - if m != nil { - return m.TcpiBackoff +func (x *SocketOptionTcpInfo) GetTcpiBackoff() uint32 { + if x != nil { + return x.TcpiBackoff } return 0 } -func (m *SocketOptionTcpInfo) GetTcpiOptions() uint32 { - if m != nil { - return m.TcpiOptions +func (x *SocketOptionTcpInfo) GetTcpiOptions() uint32 { + if x != nil { + return x.TcpiOptions } return 0 } -func (m *SocketOptionTcpInfo) GetTcpiSndWscale() uint32 { - if m != nil { - return m.TcpiSndWscale +func (x *SocketOptionTcpInfo) GetTcpiSndWscale() uint32 { + if x != nil { + return x.TcpiSndWscale } return 0 } -func (m *SocketOptionTcpInfo) GetTcpiRcvWscale() uint32 { - if m != nil { - return m.TcpiRcvWscale +func (x *SocketOptionTcpInfo) GetTcpiRcvWscale() uint32 { + if x != nil { + return x.TcpiRcvWscale } return 0 } -func (m *SocketOptionTcpInfo) GetTcpiRto() uint32 { - if m != nil { - return m.TcpiRto +func (x *SocketOptionTcpInfo) GetTcpiRto() uint32 { + if x != nil { + return x.TcpiRto } return 0 } -func (m *SocketOptionTcpInfo) GetTcpiAto() uint32 { - if m != nil { - return m.TcpiAto +func (x *SocketOptionTcpInfo) GetTcpiAto() uint32 { + if x != nil { + return x.TcpiAto } return 0 } -func (m *SocketOptionTcpInfo) GetTcpiSndMss() uint32 { - if m != nil { - return m.TcpiSndMss +func (x *SocketOptionTcpInfo) GetTcpiSndMss() uint32 { + if x != nil { + return x.TcpiSndMss } return 0 } -func (m *SocketOptionTcpInfo) GetTcpiRcvMss() uint32 { - if m != nil { - return m.TcpiRcvMss +func (x *SocketOptionTcpInfo) GetTcpiRcvMss() uint32 { + if x != nil { + return x.TcpiRcvMss } return 0 } -func (m *SocketOptionTcpInfo) GetTcpiUnacked() uint32 { - if m != nil { - return m.TcpiUnacked +func (x *SocketOptionTcpInfo) GetTcpiUnacked() uint32 { + if x != nil { + return x.TcpiUnacked } return 0 } -func (m *SocketOptionTcpInfo) GetTcpiSacked() uint32 { - if m != nil { - return m.TcpiSacked +func (x *SocketOptionTcpInfo) GetTcpiSacked() uint32 { + if x != nil { + return x.TcpiSacked } return 0 } -func (m *SocketOptionTcpInfo) GetTcpiLost() uint32 { - if m != nil { - return m.TcpiLost +func (x *SocketOptionTcpInfo) GetTcpiLost() uint32 { + if x != nil { + return x.TcpiLost } return 0 } -func (m *SocketOptionTcpInfo) GetTcpiRetrans() uint32 { - if m != nil { - return m.TcpiRetrans +func (x *SocketOptionTcpInfo) GetTcpiRetrans() uint32 { + if x != nil { + return x.TcpiRetrans } return 0 } -func (m *SocketOptionTcpInfo) GetTcpiFackets() uint32 { - if m != nil { - return m.TcpiFackets +func (x *SocketOptionTcpInfo) GetTcpiFackets() uint32 { + if x != nil { + return x.TcpiFackets } return 0 } -func (m *SocketOptionTcpInfo) GetTcpiLastDataSent() uint32 { - if m != nil { - return m.TcpiLastDataSent +func (x *SocketOptionTcpInfo) GetTcpiLastDataSent() uint32 { + if x != nil { + return x.TcpiLastDataSent } return 0 } -func (m *SocketOptionTcpInfo) GetTcpiLastAckSent() uint32 { - if m != nil { - return m.TcpiLastAckSent +func (x *SocketOptionTcpInfo) GetTcpiLastAckSent() uint32 { + if x != nil { + return x.TcpiLastAckSent } return 0 } -func (m *SocketOptionTcpInfo) GetTcpiLastDataRecv() uint32 { - if m != nil { - return m.TcpiLastDataRecv +func (x *SocketOptionTcpInfo) GetTcpiLastDataRecv() uint32 { + if x != nil { + return x.TcpiLastDataRecv } return 0 } -func (m *SocketOptionTcpInfo) GetTcpiLastAckRecv() uint32 { - if m != nil { - return m.TcpiLastAckRecv +func (x *SocketOptionTcpInfo) GetTcpiLastAckRecv() uint32 { + if x != nil { + return x.TcpiLastAckRecv } return 0 } -func (m *SocketOptionTcpInfo) GetTcpiPmtu() uint32 { - if m != nil { - return m.TcpiPmtu +func (x *SocketOptionTcpInfo) GetTcpiPmtu() uint32 { + if x != nil { + return x.TcpiPmtu } return 0 } -func (m *SocketOptionTcpInfo) GetTcpiRcvSsthresh() uint32 { - if m != nil { - return m.TcpiRcvSsthresh +func (x *SocketOptionTcpInfo) GetTcpiRcvSsthresh() uint32 { + if x != nil { + return x.TcpiRcvSsthresh } return 0 } -func (m *SocketOptionTcpInfo) GetTcpiRtt() uint32 { - if m != nil { - return m.TcpiRtt +func (x *SocketOptionTcpInfo) GetTcpiRtt() uint32 { + if x != nil { + return x.TcpiRtt } return 0 } -func (m *SocketOptionTcpInfo) GetTcpiRttvar() uint32 { - if m != nil { - return m.TcpiRttvar +func (x *SocketOptionTcpInfo) GetTcpiRttvar() uint32 { + if x != nil { + return x.TcpiRttvar } return 0 } -func (m *SocketOptionTcpInfo) GetTcpiSndSsthresh() uint32 { - if m != nil { - return m.TcpiSndSsthresh +func (x *SocketOptionTcpInfo) GetTcpiSndSsthresh() uint32 { + if x != nil { + return x.TcpiSndSsthresh } return 0 } -func (m *SocketOptionTcpInfo) GetTcpiSndCwnd() uint32 { - if m != nil { - return m.TcpiSndCwnd +func (x *SocketOptionTcpInfo) GetTcpiSndCwnd() uint32 { + if x != nil { + return x.TcpiSndCwnd } return 0 } -func (m *SocketOptionTcpInfo) GetTcpiAdvmss() uint32 { - if m != nil { - return m.TcpiAdvmss +func (x *SocketOptionTcpInfo) GetTcpiAdvmss() uint32 { + if x != nil { + return x.TcpiAdvmss } return 0 } -func (m *SocketOptionTcpInfo) GetTcpiReordering() uint32 { - if m != nil { - return m.TcpiReordering +func (x *SocketOptionTcpInfo) GetTcpiReordering() uint32 { + if x != nil { + return x.TcpiReordering } return 0 } type GetTopChannelsRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // start_channel_id indicates that only channels at or above this id should be // included in the results. // To request the first page, this should be set to 0. To request @@ -2057,52 +1971,60 @@ type GetTopChannelsRequest struct { // If non-zero, the server will return a page of results containing // at most this many items. If zero, the server will choose a // reasonable page size. Must never be negative. - MaxResults int64 `protobuf:"varint,2,opt,name=max_results,json=maxResults,proto3" json:"max_results,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + MaxResults int64 `protobuf:"varint,2,opt,name=max_results,json=maxResults,proto3" json:"max_results,omitempty"` } -func (m *GetTopChannelsRequest) Reset() { *m = GetTopChannelsRequest{} } -func (m *GetTopChannelsRequest) String() string { return proto.CompactTextString(m) } -func (*GetTopChannelsRequest) ProtoMessage() {} -func (*GetTopChannelsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_6ee37dfd35a8ab00, []int{20} +func (x *GetTopChannelsRequest) Reset() { + *x = GetTopChannelsRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[20] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *GetTopChannelsRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_GetTopChannelsRequest.Unmarshal(m, b) +func (x *GetTopChannelsRequest) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *GetTopChannelsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_GetTopChannelsRequest.Marshal(b, m, deterministic) -} -func (m *GetTopChannelsRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_GetTopChannelsRequest.Merge(m, src) -} -func (m *GetTopChannelsRequest) XXX_Size() int { - return xxx_messageInfo_GetTopChannelsRequest.Size(m) -} -func (m *GetTopChannelsRequest) XXX_DiscardUnknown() { - xxx_messageInfo_GetTopChannelsRequest.DiscardUnknown(m) + +func (*GetTopChannelsRequest) ProtoMessage() {} + +func (x *GetTopChannelsRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[20] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_GetTopChannelsRequest proto.InternalMessageInfo +// Deprecated: Use GetTopChannelsRequest.ProtoReflect.Descriptor instead. +func (*GetTopChannelsRequest) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{20} +} -func (m *GetTopChannelsRequest) GetStartChannelId() int64 { - if m != nil { - return m.StartChannelId +func (x *GetTopChannelsRequest) GetStartChannelId() int64 { + if x != nil { + return x.StartChannelId } return 0 } -func (m *GetTopChannelsRequest) GetMaxResults() int64 { - if m != nil { - return m.MaxResults +func (x *GetTopChannelsRequest) GetMaxResults() int64 { + if x != nil { + return x.MaxResults } return 0 } type GetTopChannelsResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // list of channels that the connection detail service knows about. Sorted in // ascending channel_id order. // Must contain at least 1 result, otherwise 'end' must be true. @@ -2110,52 +2032,60 @@ type GetTopChannelsResponse struct { // If set, indicates that the list of channels is the final list. Requesting // more channels can only return more if they are created after this RPC // completes. - End bool `protobuf:"varint,2,opt,name=end,proto3" json:"end,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + End bool `protobuf:"varint,2,opt,name=end,proto3" json:"end,omitempty"` } -func (m *GetTopChannelsResponse) Reset() { *m = GetTopChannelsResponse{} } -func (m *GetTopChannelsResponse) String() string { return proto.CompactTextString(m) } -func (*GetTopChannelsResponse) ProtoMessage() {} -func (*GetTopChannelsResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_6ee37dfd35a8ab00, []int{21} +func (x *GetTopChannelsResponse) Reset() { + *x = GetTopChannelsResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[21] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *GetTopChannelsResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_GetTopChannelsResponse.Unmarshal(m, b) -} -func (m *GetTopChannelsResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_GetTopChannelsResponse.Marshal(b, m, deterministic) -} -func (m *GetTopChannelsResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_GetTopChannelsResponse.Merge(m, src) +func (x *GetTopChannelsResponse) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *GetTopChannelsResponse) XXX_Size() int { - return xxx_messageInfo_GetTopChannelsResponse.Size(m) -} -func (m *GetTopChannelsResponse) XXX_DiscardUnknown() { - xxx_messageInfo_GetTopChannelsResponse.DiscardUnknown(m) + +func (*GetTopChannelsResponse) ProtoMessage() {} + +func (x *GetTopChannelsResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[21] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_GetTopChannelsResponse proto.InternalMessageInfo +// Deprecated: Use GetTopChannelsResponse.ProtoReflect.Descriptor instead. +func (*GetTopChannelsResponse) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{21} +} -func (m *GetTopChannelsResponse) GetChannel() []*Channel { - if m != nil { - return m.Channel +func (x *GetTopChannelsResponse) GetChannel() []*Channel { + if x != nil { + return x.Channel } return nil } -func (m *GetTopChannelsResponse) GetEnd() bool { - if m != nil { - return m.End +func (x *GetTopChannelsResponse) GetEnd() bool { + if x != nil { + return x.End } return false } type GetServersRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // start_server_id indicates that only servers at or above this id should be // included in the results. // To request the first page, this must be set to 0. To request @@ -2165,52 +2095,60 @@ type GetServersRequest struct { // If non-zero, the server will return a page of results containing // at most this many items. If zero, the server will choose a // reasonable page size. Must never be negative. - MaxResults int64 `protobuf:"varint,2,opt,name=max_results,json=maxResults,proto3" json:"max_results,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + MaxResults int64 `protobuf:"varint,2,opt,name=max_results,json=maxResults,proto3" json:"max_results,omitempty"` } -func (m *GetServersRequest) Reset() { *m = GetServersRequest{} } -func (m *GetServersRequest) String() string { return proto.CompactTextString(m) } -func (*GetServersRequest) ProtoMessage() {} -func (*GetServersRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_6ee37dfd35a8ab00, []int{22} +func (x *GetServersRequest) Reset() { + *x = GetServersRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[22] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *GetServersRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_GetServersRequest.Unmarshal(m, b) -} -func (m *GetServersRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_GetServersRequest.Marshal(b, m, deterministic) +func (x *GetServersRequest) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *GetServersRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_GetServersRequest.Merge(m, src) -} -func (m *GetServersRequest) XXX_Size() int { - return xxx_messageInfo_GetServersRequest.Size(m) -} -func (m *GetServersRequest) XXX_DiscardUnknown() { - xxx_messageInfo_GetServersRequest.DiscardUnknown(m) + +func (*GetServersRequest) ProtoMessage() {} + +func (x *GetServersRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[22] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_GetServersRequest proto.InternalMessageInfo +// Deprecated: Use GetServersRequest.ProtoReflect.Descriptor instead. +func (*GetServersRequest) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{22} +} -func (m *GetServersRequest) GetStartServerId() int64 { - if m != nil { - return m.StartServerId +func (x *GetServersRequest) GetStartServerId() int64 { + if x != nil { + return x.StartServerId } return 0 } -func (m *GetServersRequest) GetMaxResults() int64 { - if m != nil { - return m.MaxResults +func (x *GetServersRequest) GetMaxResults() int64 { + if x != nil { + return x.MaxResults } return 0 } type GetServersResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // list of servers that the connection detail service knows about. Sorted in // ascending server_id order. // Must contain at least 1 result, otherwise 'end' must be true. @@ -2218,133 +2156,157 @@ type GetServersResponse struct { // If set, indicates that the list of servers is the final list. Requesting // more servers will only return more if they are created after this RPC // completes. - End bool `protobuf:"varint,2,opt,name=end,proto3" json:"end,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + End bool `protobuf:"varint,2,opt,name=end,proto3" json:"end,omitempty"` } -func (m *GetServersResponse) Reset() { *m = GetServersResponse{} } -func (m *GetServersResponse) String() string { return proto.CompactTextString(m) } -func (*GetServersResponse) ProtoMessage() {} -func (*GetServersResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_6ee37dfd35a8ab00, []int{23} +func (x *GetServersResponse) Reset() { + *x = GetServersResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[23] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *GetServersResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_GetServersResponse.Unmarshal(m, b) -} -func (m *GetServersResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_GetServersResponse.Marshal(b, m, deterministic) -} -func (m *GetServersResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_GetServersResponse.Merge(m, src) -} -func (m *GetServersResponse) XXX_Size() int { - return xxx_messageInfo_GetServersResponse.Size(m) +func (x *GetServersResponse) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *GetServersResponse) XXX_DiscardUnknown() { - xxx_messageInfo_GetServersResponse.DiscardUnknown(m) + +func (*GetServersResponse) ProtoMessage() {} + +func (x *GetServersResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[23] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_GetServersResponse proto.InternalMessageInfo +// Deprecated: Use GetServersResponse.ProtoReflect.Descriptor instead. +func (*GetServersResponse) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{23} +} -func (m *GetServersResponse) GetServer() []*Server { - if m != nil { - return m.Server +func (x *GetServersResponse) GetServer() []*Server { + if x != nil { + return x.Server } return nil } -func (m *GetServersResponse) GetEnd() bool { - if m != nil { - return m.End +func (x *GetServersResponse) GetEnd() bool { + if x != nil { + return x.End } return false } type GetServerRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // server_id is the identifier of the specific server to get. - ServerId int64 `protobuf:"varint,1,opt,name=server_id,json=serverId,proto3" json:"server_id,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + ServerId int64 `protobuf:"varint,1,opt,name=server_id,json=serverId,proto3" json:"server_id,omitempty"` } -func (m *GetServerRequest) Reset() { *m = GetServerRequest{} } -func (m *GetServerRequest) String() string { return proto.CompactTextString(m) } -func (*GetServerRequest) ProtoMessage() {} -func (*GetServerRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_6ee37dfd35a8ab00, []int{24} +func (x *GetServerRequest) Reset() { + *x = GetServerRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[24] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *GetServerRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_GetServerRequest.Unmarshal(m, b) -} -func (m *GetServerRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_GetServerRequest.Marshal(b, m, deterministic) +func (x *GetServerRequest) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *GetServerRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_GetServerRequest.Merge(m, src) -} -func (m *GetServerRequest) XXX_Size() int { - return xxx_messageInfo_GetServerRequest.Size(m) + +func (*GetServerRequest) ProtoMessage() {} + +func (x *GetServerRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[24] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -func (m *GetServerRequest) XXX_DiscardUnknown() { - xxx_messageInfo_GetServerRequest.DiscardUnknown(m) + +// Deprecated: Use GetServerRequest.ProtoReflect.Descriptor instead. +func (*GetServerRequest) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{24} } -var xxx_messageInfo_GetServerRequest proto.InternalMessageInfo - -func (m *GetServerRequest) GetServerId() int64 { - if m != nil { - return m.ServerId +func (x *GetServerRequest) GetServerId() int64 { + if x != nil { + return x.ServerId } return 0 } type GetServerResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // The Server that corresponds to the requested server_id. This field // should be set. - Server *Server `protobuf:"bytes,1,opt,name=server,proto3" json:"server,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Server *Server `protobuf:"bytes,1,opt,name=server,proto3" json:"server,omitempty"` } -func (m *GetServerResponse) Reset() { *m = GetServerResponse{} } -func (m *GetServerResponse) String() string { return proto.CompactTextString(m) } -func (*GetServerResponse) ProtoMessage() {} -func (*GetServerResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_6ee37dfd35a8ab00, []int{25} +func (x *GetServerResponse) Reset() { + *x = GetServerResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[25] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *GetServerResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_GetServerResponse.Unmarshal(m, b) -} -func (m *GetServerResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_GetServerResponse.Marshal(b, m, deterministic) -} -func (m *GetServerResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_GetServerResponse.Merge(m, src) -} -func (m *GetServerResponse) XXX_Size() int { - return xxx_messageInfo_GetServerResponse.Size(m) +func (x *GetServerResponse) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *GetServerResponse) XXX_DiscardUnknown() { - xxx_messageInfo_GetServerResponse.DiscardUnknown(m) + +func (*GetServerResponse) ProtoMessage() {} + +func (x *GetServerResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[25] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_GetServerResponse proto.InternalMessageInfo +// Deprecated: Use GetServerResponse.ProtoReflect.Descriptor instead. +func (*GetServerResponse) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{25} +} -func (m *GetServerResponse) GetServer() *Server { - if m != nil { - return m.Server +func (x *GetServerResponse) GetServer() *Server { + if x != nil { + return x.Server } return nil } type GetServerSocketsRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + ServerId int64 `protobuf:"varint,1,opt,name=server_id,json=serverId,proto3" json:"server_id,omitempty"` // start_socket_id indicates that only sockets at or above this id should be // included in the results. @@ -2355,59 +2317,67 @@ type GetServerSocketsRequest struct { // If non-zero, the server will return a page of results containing // at most this many items. If zero, the server will choose a // reasonable page size. Must never be negative. - MaxResults int64 `protobuf:"varint,3,opt,name=max_results,json=maxResults,proto3" json:"max_results,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + MaxResults int64 `protobuf:"varint,3,opt,name=max_results,json=maxResults,proto3" json:"max_results,omitempty"` } -func (m *GetServerSocketsRequest) Reset() { *m = GetServerSocketsRequest{} } -func (m *GetServerSocketsRequest) String() string { return proto.CompactTextString(m) } -func (*GetServerSocketsRequest) ProtoMessage() {} -func (*GetServerSocketsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_6ee37dfd35a8ab00, []int{26} +func (x *GetServerSocketsRequest) Reset() { + *x = GetServerSocketsRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[26] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *GetServerSocketsRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_GetServerSocketsRequest.Unmarshal(m, b) -} -func (m *GetServerSocketsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_GetServerSocketsRequest.Marshal(b, m, deterministic) -} -func (m *GetServerSocketsRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_GetServerSocketsRequest.Merge(m, src) -} -func (m *GetServerSocketsRequest) XXX_Size() int { - return xxx_messageInfo_GetServerSocketsRequest.Size(m) +func (x *GetServerSocketsRequest) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *GetServerSocketsRequest) XXX_DiscardUnknown() { - xxx_messageInfo_GetServerSocketsRequest.DiscardUnknown(m) + +func (*GetServerSocketsRequest) ProtoMessage() {} + +func (x *GetServerSocketsRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[26] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_GetServerSocketsRequest proto.InternalMessageInfo +// Deprecated: Use GetServerSocketsRequest.ProtoReflect.Descriptor instead. +func (*GetServerSocketsRequest) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{26} +} -func (m *GetServerSocketsRequest) GetServerId() int64 { - if m != nil { - return m.ServerId +func (x *GetServerSocketsRequest) GetServerId() int64 { + if x != nil { + return x.ServerId } return 0 } -func (m *GetServerSocketsRequest) GetStartSocketId() int64 { - if m != nil { - return m.StartSocketId +func (x *GetServerSocketsRequest) GetStartSocketId() int64 { + if x != nil { + return x.StartSocketId } return 0 } -func (m *GetServerSocketsRequest) GetMaxResults() int64 { - if m != nil { - return m.MaxResults +func (x *GetServerSocketsRequest) GetMaxResults() int64 { + if x != nil { + return x.MaxResults } return 0 } type GetServerSocketsResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // list of socket refs that the connection detail service knows about. Sorted in // ascending socket_id order. // Must contain at least 1 result, otherwise 'end' must be true. @@ -2415,825 +2385,1842 @@ type GetServerSocketsResponse struct { // If set, indicates that the list of sockets is the final list. Requesting // more sockets will only return more if they are created after this RPC // completes. - End bool `protobuf:"varint,2,opt,name=end,proto3" json:"end,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + End bool `protobuf:"varint,2,opt,name=end,proto3" json:"end,omitempty"` } -func (m *GetServerSocketsResponse) Reset() { *m = GetServerSocketsResponse{} } -func (m *GetServerSocketsResponse) String() string { return proto.CompactTextString(m) } -func (*GetServerSocketsResponse) ProtoMessage() {} -func (*GetServerSocketsResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_6ee37dfd35a8ab00, []int{27} +func (x *GetServerSocketsResponse) Reset() { + *x = GetServerSocketsResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[27] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *GetServerSocketsResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_GetServerSocketsResponse.Unmarshal(m, b) -} -func (m *GetServerSocketsResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_GetServerSocketsResponse.Marshal(b, m, deterministic) +func (x *GetServerSocketsResponse) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *GetServerSocketsResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_GetServerSocketsResponse.Merge(m, src) -} -func (m *GetServerSocketsResponse) XXX_Size() int { - return xxx_messageInfo_GetServerSocketsResponse.Size(m) -} -func (m *GetServerSocketsResponse) XXX_DiscardUnknown() { - xxx_messageInfo_GetServerSocketsResponse.DiscardUnknown(m) + +func (*GetServerSocketsResponse) ProtoMessage() {} + +func (x *GetServerSocketsResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[27] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_GetServerSocketsResponse proto.InternalMessageInfo +// Deprecated: Use GetServerSocketsResponse.ProtoReflect.Descriptor instead. +func (*GetServerSocketsResponse) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{27} +} -func (m *GetServerSocketsResponse) GetSocketRef() []*SocketRef { - if m != nil { - return m.SocketRef +func (x *GetServerSocketsResponse) GetSocketRef() []*SocketRef { + if x != nil { + return x.SocketRef } return nil } -func (m *GetServerSocketsResponse) GetEnd() bool { - if m != nil { - return m.End +func (x *GetServerSocketsResponse) GetEnd() bool { + if x != nil { + return x.End } return false } type GetChannelRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // channel_id is the identifier of the specific channel to get. - ChannelId int64 `protobuf:"varint,1,opt,name=channel_id,json=channelId,proto3" json:"channel_id,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + ChannelId int64 `protobuf:"varint,1,opt,name=channel_id,json=channelId,proto3" json:"channel_id,omitempty"` } -func (m *GetChannelRequest) Reset() { *m = GetChannelRequest{} } -func (m *GetChannelRequest) String() string { return proto.CompactTextString(m) } -func (*GetChannelRequest) ProtoMessage() {} -func (*GetChannelRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_6ee37dfd35a8ab00, []int{28} +func (x *GetChannelRequest) Reset() { + *x = GetChannelRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[28] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *GetChannelRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_GetChannelRequest.Unmarshal(m, b) -} -func (m *GetChannelRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_GetChannelRequest.Marshal(b, m, deterministic) -} -func (m *GetChannelRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_GetChannelRequest.Merge(m, src) -} -func (m *GetChannelRequest) XXX_Size() int { - return xxx_messageInfo_GetChannelRequest.Size(m) +func (x *GetChannelRequest) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *GetChannelRequest) XXX_DiscardUnknown() { - xxx_messageInfo_GetChannelRequest.DiscardUnknown(m) + +func (*GetChannelRequest) ProtoMessage() {} + +func (x *GetChannelRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[28] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_GetChannelRequest proto.InternalMessageInfo +// Deprecated: Use GetChannelRequest.ProtoReflect.Descriptor instead. +func (*GetChannelRequest) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{28} +} -func (m *GetChannelRequest) GetChannelId() int64 { - if m != nil { - return m.ChannelId +func (x *GetChannelRequest) GetChannelId() int64 { + if x != nil { + return x.ChannelId } return 0 } type GetChannelResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // The Channel that corresponds to the requested channel_id. This field // should be set. - Channel *Channel `protobuf:"bytes,1,opt,name=channel,proto3" json:"channel,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Channel *Channel `protobuf:"bytes,1,opt,name=channel,proto3" json:"channel,omitempty"` } -func (m *GetChannelResponse) Reset() { *m = GetChannelResponse{} } -func (m *GetChannelResponse) String() string { return proto.CompactTextString(m) } -func (*GetChannelResponse) ProtoMessage() {} -func (*GetChannelResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_6ee37dfd35a8ab00, []int{29} +func (x *GetChannelResponse) Reset() { + *x = GetChannelResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[29] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *GetChannelResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_GetChannelResponse.Unmarshal(m, b) -} -func (m *GetChannelResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_GetChannelResponse.Marshal(b, m, deterministic) -} -func (m *GetChannelResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_GetChannelResponse.Merge(m, src) -} -func (m *GetChannelResponse) XXX_Size() int { - return xxx_messageInfo_GetChannelResponse.Size(m) +func (x *GetChannelResponse) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *GetChannelResponse) XXX_DiscardUnknown() { - xxx_messageInfo_GetChannelResponse.DiscardUnknown(m) + +func (*GetChannelResponse) ProtoMessage() {} + +func (x *GetChannelResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[29] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_GetChannelResponse proto.InternalMessageInfo +// Deprecated: Use GetChannelResponse.ProtoReflect.Descriptor instead. +func (*GetChannelResponse) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{29} +} -func (m *GetChannelResponse) GetChannel() *Channel { - if m != nil { - return m.Channel +func (x *GetChannelResponse) GetChannel() *Channel { + if x != nil { + return x.Channel } return nil } type GetSubchannelRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // subchannel_id is the identifier of the specific subchannel to get. - SubchannelId int64 `protobuf:"varint,1,opt,name=subchannel_id,json=subchannelId,proto3" json:"subchannel_id,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + SubchannelId int64 `protobuf:"varint,1,opt,name=subchannel_id,json=subchannelId,proto3" json:"subchannel_id,omitempty"` } -func (m *GetSubchannelRequest) Reset() { *m = GetSubchannelRequest{} } -func (m *GetSubchannelRequest) String() string { return proto.CompactTextString(m) } -func (*GetSubchannelRequest) ProtoMessage() {} -func (*GetSubchannelRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_6ee37dfd35a8ab00, []int{30} +func (x *GetSubchannelRequest) Reset() { + *x = GetSubchannelRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[30] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *GetSubchannelRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_GetSubchannelRequest.Unmarshal(m, b) -} -func (m *GetSubchannelRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_GetSubchannelRequest.Marshal(b, m, deterministic) -} -func (m *GetSubchannelRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_GetSubchannelRequest.Merge(m, src) -} -func (m *GetSubchannelRequest) XXX_Size() int { - return xxx_messageInfo_GetSubchannelRequest.Size(m) +func (x *GetSubchannelRequest) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *GetSubchannelRequest) XXX_DiscardUnknown() { - xxx_messageInfo_GetSubchannelRequest.DiscardUnknown(m) + +func (*GetSubchannelRequest) ProtoMessage() {} + +func (x *GetSubchannelRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[30] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_GetSubchannelRequest proto.InternalMessageInfo +// Deprecated: Use GetSubchannelRequest.ProtoReflect.Descriptor instead. +func (*GetSubchannelRequest) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{30} +} -func (m *GetSubchannelRequest) GetSubchannelId() int64 { - if m != nil { - return m.SubchannelId +func (x *GetSubchannelRequest) GetSubchannelId() int64 { + if x != nil { + return x.SubchannelId } return 0 } type GetSubchannelResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // The Subchannel that corresponds to the requested subchannel_id. This // field should be set. - Subchannel *Subchannel `protobuf:"bytes,1,opt,name=subchannel,proto3" json:"subchannel,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Subchannel *Subchannel `protobuf:"bytes,1,opt,name=subchannel,proto3" json:"subchannel,omitempty"` } -func (m *GetSubchannelResponse) Reset() { *m = GetSubchannelResponse{} } -func (m *GetSubchannelResponse) String() string { return proto.CompactTextString(m) } -func (*GetSubchannelResponse) ProtoMessage() {} -func (*GetSubchannelResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_6ee37dfd35a8ab00, []int{31} +func (x *GetSubchannelResponse) Reset() { + *x = GetSubchannelResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[31] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *GetSubchannelResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_GetSubchannelResponse.Unmarshal(m, b) -} -func (m *GetSubchannelResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_GetSubchannelResponse.Marshal(b, m, deterministic) -} -func (m *GetSubchannelResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_GetSubchannelResponse.Merge(m, src) -} -func (m *GetSubchannelResponse) XXX_Size() int { - return xxx_messageInfo_GetSubchannelResponse.Size(m) +func (x *GetSubchannelResponse) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *GetSubchannelResponse) XXX_DiscardUnknown() { - xxx_messageInfo_GetSubchannelResponse.DiscardUnknown(m) + +func (*GetSubchannelResponse) ProtoMessage() {} + +func (x *GetSubchannelResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[31] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_GetSubchannelResponse proto.InternalMessageInfo +// Deprecated: Use GetSubchannelResponse.ProtoReflect.Descriptor instead. +func (*GetSubchannelResponse) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{31} +} -func (m *GetSubchannelResponse) GetSubchannel() *Subchannel { - if m != nil { - return m.Subchannel +func (x *GetSubchannelResponse) GetSubchannel() *Subchannel { + if x != nil { + return x.Subchannel } return nil } type GetSocketRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // socket_id is the identifier of the specific socket to get. SocketId int64 `protobuf:"varint,1,opt,name=socket_id,json=socketId,proto3" json:"socket_id,omitempty"` // If true, the response will contain only high level information // that is inexpensive to obtain. Fields thay may be omitted are // documented. - Summary bool `protobuf:"varint,2,opt,name=summary,proto3" json:"summary,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Summary bool `protobuf:"varint,2,opt,name=summary,proto3" json:"summary,omitempty"` } -func (m *GetSocketRequest) Reset() { *m = GetSocketRequest{} } -func (m *GetSocketRequest) String() string { return proto.CompactTextString(m) } -func (*GetSocketRequest) ProtoMessage() {} -func (*GetSocketRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_6ee37dfd35a8ab00, []int{32} +func (x *GetSocketRequest) Reset() { + *x = GetSocketRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[32] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *GetSocketRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_GetSocketRequest.Unmarshal(m, b) -} -func (m *GetSocketRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_GetSocketRequest.Marshal(b, m, deterministic) -} -func (m *GetSocketRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_GetSocketRequest.Merge(m, src) -} -func (m *GetSocketRequest) XXX_Size() int { - return xxx_messageInfo_GetSocketRequest.Size(m) +func (x *GetSocketRequest) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *GetSocketRequest) XXX_DiscardUnknown() { - xxx_messageInfo_GetSocketRequest.DiscardUnknown(m) + +func (*GetSocketRequest) ProtoMessage() {} + +func (x *GetSocketRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[32] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_GetSocketRequest proto.InternalMessageInfo +// Deprecated: Use GetSocketRequest.ProtoReflect.Descriptor instead. +func (*GetSocketRequest) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{32} +} -func (m *GetSocketRequest) GetSocketId() int64 { - if m != nil { - return m.SocketId +func (x *GetSocketRequest) GetSocketId() int64 { + if x != nil { + return x.SocketId } return 0 } -func (m *GetSocketRequest) GetSummary() bool { - if m != nil { - return m.Summary +func (x *GetSocketRequest) GetSummary() bool { + if x != nil { + return x.Summary } return false } type GetSocketResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // The Socket that corresponds to the requested socket_id. This field // should be set. - Socket *Socket `protobuf:"bytes,1,opt,name=socket,proto3" json:"socket,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Socket *Socket `protobuf:"bytes,1,opt,name=socket,proto3" json:"socket,omitempty"` } -func (m *GetSocketResponse) Reset() { *m = GetSocketResponse{} } -func (m *GetSocketResponse) String() string { return proto.CompactTextString(m) } -func (*GetSocketResponse) ProtoMessage() {} -func (*GetSocketResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_6ee37dfd35a8ab00, []int{33} +func (x *GetSocketResponse) Reset() { + *x = GetSocketResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[33] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *GetSocketResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_GetSocketResponse.Unmarshal(m, b) -} -func (m *GetSocketResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_GetSocketResponse.Marshal(b, m, deterministic) -} -func (m *GetSocketResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_GetSocketResponse.Merge(m, src) +func (x *GetSocketResponse) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *GetSocketResponse) XXX_Size() int { - return xxx_messageInfo_GetSocketResponse.Size(m) -} -func (m *GetSocketResponse) XXX_DiscardUnknown() { - xxx_messageInfo_GetSocketResponse.DiscardUnknown(m) + +func (*GetSocketResponse) ProtoMessage() {} + +func (x *GetSocketResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[33] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_GetSocketResponse proto.InternalMessageInfo +// Deprecated: Use GetSocketResponse.ProtoReflect.Descriptor instead. +func (*GetSocketResponse) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{33} +} -func (m *GetSocketResponse) GetSocket() *Socket { - if m != nil { - return m.Socket +func (x *GetSocketResponse) GetSocket() *Socket { + if x != nil { + return x.Socket } return nil } -func init() { - proto.RegisterEnum("grpc.channelz.v1.ChannelConnectivityState_State", ChannelConnectivityState_State_name, ChannelConnectivityState_State_value) - proto.RegisterEnum("grpc.channelz.v1.ChannelTraceEvent_Severity", ChannelTraceEvent_Severity_name, ChannelTraceEvent_Severity_value) - proto.RegisterType((*Channel)(nil), "grpc.channelz.v1.Channel") - proto.RegisterType((*Subchannel)(nil), "grpc.channelz.v1.Subchannel") - proto.RegisterType((*ChannelConnectivityState)(nil), "grpc.channelz.v1.ChannelConnectivityState") - proto.RegisterType((*ChannelData)(nil), "grpc.channelz.v1.ChannelData") - proto.RegisterType((*ChannelTraceEvent)(nil), "grpc.channelz.v1.ChannelTraceEvent") - proto.RegisterType((*ChannelTrace)(nil), "grpc.channelz.v1.ChannelTrace") - proto.RegisterType((*ChannelRef)(nil), "grpc.channelz.v1.ChannelRef") - proto.RegisterType((*SubchannelRef)(nil), "grpc.channelz.v1.SubchannelRef") - proto.RegisterType((*SocketRef)(nil), "grpc.channelz.v1.SocketRef") - proto.RegisterType((*ServerRef)(nil), "grpc.channelz.v1.ServerRef") - proto.RegisterType((*Server)(nil), "grpc.channelz.v1.Server") - proto.RegisterType((*ServerData)(nil), "grpc.channelz.v1.ServerData") - proto.RegisterType((*Socket)(nil), "grpc.channelz.v1.Socket") - proto.RegisterType((*SocketData)(nil), "grpc.channelz.v1.SocketData") - proto.RegisterType((*Address)(nil), "grpc.channelz.v1.Address") - proto.RegisterType((*Address_TcpIpAddress)(nil), "grpc.channelz.v1.Address.TcpIpAddress") - proto.RegisterType((*Address_UdsAddress)(nil), "grpc.channelz.v1.Address.UdsAddress") - proto.RegisterType((*Address_OtherAddress)(nil), "grpc.channelz.v1.Address.OtherAddress") - proto.RegisterType((*Security)(nil), "grpc.channelz.v1.Security") - proto.RegisterType((*Security_Tls)(nil), "grpc.channelz.v1.Security.Tls") - proto.RegisterType((*Security_OtherSecurity)(nil), "grpc.channelz.v1.Security.OtherSecurity") - proto.RegisterType((*SocketOption)(nil), "grpc.channelz.v1.SocketOption") - proto.RegisterType((*SocketOptionTimeout)(nil), "grpc.channelz.v1.SocketOptionTimeout") - proto.RegisterType((*SocketOptionLinger)(nil), "grpc.channelz.v1.SocketOptionLinger") - proto.RegisterType((*SocketOptionTcpInfo)(nil), "grpc.channelz.v1.SocketOptionTcpInfo") - proto.RegisterType((*GetTopChannelsRequest)(nil), "grpc.channelz.v1.GetTopChannelsRequest") - proto.RegisterType((*GetTopChannelsResponse)(nil), "grpc.channelz.v1.GetTopChannelsResponse") - proto.RegisterType((*GetServersRequest)(nil), "grpc.channelz.v1.GetServersRequest") - proto.RegisterType((*GetServersResponse)(nil), "grpc.channelz.v1.GetServersResponse") - proto.RegisterType((*GetServerRequest)(nil), "grpc.channelz.v1.GetServerRequest") - proto.RegisterType((*GetServerResponse)(nil), "grpc.channelz.v1.GetServerResponse") - proto.RegisterType((*GetServerSocketsRequest)(nil), "grpc.channelz.v1.GetServerSocketsRequest") - proto.RegisterType((*GetServerSocketsResponse)(nil), "grpc.channelz.v1.GetServerSocketsResponse") - proto.RegisterType((*GetChannelRequest)(nil), "grpc.channelz.v1.GetChannelRequest") - proto.RegisterType((*GetChannelResponse)(nil), "grpc.channelz.v1.GetChannelResponse") - proto.RegisterType((*GetSubchannelRequest)(nil), "grpc.channelz.v1.GetSubchannelRequest") - proto.RegisterType((*GetSubchannelResponse)(nil), "grpc.channelz.v1.GetSubchannelResponse") - proto.RegisterType((*GetSocketRequest)(nil), "grpc.channelz.v1.GetSocketRequest") - proto.RegisterType((*GetSocketResponse)(nil), "grpc.channelz.v1.GetSocketResponse") -} - -func init() { proto.RegisterFile("grpc/channelz/v1/channelz.proto", fileDescriptor_6ee37dfd35a8ab00) } - -var fileDescriptor_6ee37dfd35a8ab00 = []byte{ - // 2584 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe4, 0x59, 0x4b, 0x6f, 0xdb, 0xd8, - 0xf5, 0xb7, 0xde, 0xd4, 0xd1, 0x23, 0xf2, 0x4d, 0x26, 0x43, 0x2b, 0x99, 0xb1, 0xff, 0xf4, 0x4c, - 0xc6, 0x93, 0xfc, 0x23, 0xc7, 0x9e, 0x34, 0x28, 0x3a, 0x2d, 0x3a, 0xb6, 0x62, 0xc7, 0x72, 0x1d, - 0x39, 0xa0, 0xe4, 0x49, 0xa6, 0x28, 0xca, 0xa1, 0xc9, 0x6b, 0x99, 0x35, 0x45, 0xaa, 0xbc, 0x57, - 0xf2, 0x24, 0x9b, 0x2e, 0xba, 0xef, 0xb2, 0x28, 0xfa, 0x01, 0xba, 0xe9, 0xa2, 0x40, 0x81, 0x02, - 0xed, 0xb6, 0xdf, 0xa6, 0xdf, 0xa2, 0xb8, 0x0f, 0x3e, 0xf4, 0xb2, 0x14, 0x64, 0xd9, 0x8d, 0x21, - 0x1e, 0xfe, 0xce, 0xef, 0x9c, 0x7b, 0x5e, 0xf7, 0xf2, 0x1a, 0xd6, 0x7b, 0xc1, 0xc0, 0xda, 0xb6, - 0x2e, 0x4d, 0xcf, 0xc3, 0xee, 0xbb, 0xed, 0xd1, 0x4e, 0xf4, 0xbb, 0x31, 0x08, 0x7c, 0xea, 0xa3, - 0x1a, 0x03, 0x34, 0x22, 0xe1, 0x68, 0xa7, 0xbe, 0xd6, 0xf3, 0xfd, 0x9e, 0x8b, 0xb7, 0xf9, 0xfb, - 0xf3, 0xe1, 0xc5, 0xb6, 0xe9, 0xbd, 0x15, 0xe0, 0xfa, 0xa7, 0x93, 0xaf, 0xec, 0x61, 0x60, 0x52, - 0xc7, 0xf7, 0xe4, 0xfb, 0xf5, 0xc9, 0xf7, 0xd4, 0xe9, 0x63, 0x42, 0xcd, 0xfe, 0x60, 0x1e, 0xc1, - 0x75, 0x60, 0x0e, 0x06, 0x38, 0x20, 0xe2, 0xbd, 0xf6, 0xb7, 0x34, 0x14, 0x9a, 0xc2, 0x17, 0xd4, - 0x80, 0x4c, 0x80, 0x2f, 0xd4, 0xd4, 0x46, 0x6a, 0xab, 0xb4, 0x7b, 0xbf, 0x31, 0xe9, 0x67, 0x43, - 0xe2, 0x74, 0x7c, 0xa1, 0x33, 0x20, 0xda, 0x81, 0xac, 0x6d, 0x52, 0x53, 0x4d, 0x73, 0x85, 0x4f, - 0xe6, 0x2a, 0x3c, 0x37, 0xa9, 0xa9, 0x73, 0x28, 0xfa, 0x19, 0x94, 0x24, 0xc0, 0x60, 0xa6, 0x32, - 0x1b, 0x99, 0x85, 0xa6, 0xc0, 0x8a, 0x7e, 0xa3, 0x43, 0xa8, 0x92, 0xe1, 0x79, 0x92, 0x21, 0xcb, - 0x19, 0xd6, 0xa7, 0x19, 0x3a, 0x11, 0x8e, 0x91, 0x54, 0x48, 0xf2, 0x11, 0xfd, 0x04, 0x80, 0xf8, - 0xd6, 0x15, 0xa6, 0x9c, 0x23, 0xc7, 0x39, 0xee, 0xcd, 0xe0, 0xe0, 0x18, 0xa6, 0x5f, 0x24, 0xe1, - 0x4f, 0xed, 0x1f, 0x69, 0x80, 0x98, 0x1c, 0xed, 0x24, 0x83, 0xb6, 0xd0, 0x8f, 0xff, 0xe1, 0xb8, - 0xfd, 0x3b, 0x05, 0xaa, 0x74, 0xaf, 0xe9, 0x7b, 0x1e, 0xb6, 0xa8, 0x33, 0x72, 0xe8, 0xdb, 0x0e, - 0x35, 0x29, 0x46, 0x87, 0x90, 0x23, 0xec, 0x07, 0x8f, 0x63, 0x75, 0xf7, 0xc9, 0xdc, 0x95, 0x4d, - 0xa9, 0x36, 0xf8, 0x5f, 0x5d, 0xa8, 0x6b, 0xbf, 0x86, 0x9c, 0x20, 0x2c, 0x41, 0xe1, 0xac, 0xfd, - 0x8b, 0xf6, 0xe9, 0xeb, 0x76, 0x6d, 0x05, 0x29, 0x90, 0x6d, 0x3d, 0x3f, 0x39, 0xa8, 0xa5, 0x50, - 0x15, 0xa0, 0x79, 0xda, 0x6e, 0x1f, 0x34, 0xbb, 0xad, 0xf6, 0x8b, 0x5a, 0x1a, 0x15, 0x21, 0xa7, - 0x1f, 0xec, 0x3d, 0xff, 0xae, 0x96, 0x41, 0x1f, 0xc1, 0x6a, 0x57, 0xdf, 0x6b, 0x77, 0x5a, 0x07, - 0xed, 0xae, 0x71, 0xb8, 0xd7, 0x3a, 0x39, 0xd3, 0x0f, 0x6a, 0x59, 0x54, 0x06, 0xa5, 0x73, 0x74, - 0xd6, 0x7d, 0xce, 0x98, 0x72, 0xda, 0x7f, 0xd2, 0x50, 0x4a, 0x64, 0x07, 0x7d, 0x93, 0xf4, 0xbb, - 0xb4, 0xfb, 0x70, 0x79, 0xbf, 0xa5, 0xc7, 0xe8, 0x2e, 0xe4, 0xa9, 0x19, 0xf4, 0x30, 0xe5, 0xe5, - 0x50, 0xd4, 0xe5, 0x13, 0x7a, 0x0a, 0x39, 0x1a, 0x98, 0x16, 0x56, 0x33, 0x9c, 0xf9, 0xd3, 0xb9, - 0xcc, 0x5d, 0x86, 0xd2, 0x05, 0x18, 0x6d, 0x42, 0xc5, 0x32, 0x5d, 0x97, 0x18, 0x84, 0x9a, 0x01, - 0xc5, 0xb6, 0x9a, 0xdd, 0x48, 0x6d, 0x65, 0xf4, 0x32, 0x17, 0x76, 0x84, 0x0c, 0x7d, 0x01, 0xb7, - 0x24, 0x68, 0x68, 0x59, 0x18, 0xdb, 0xd8, 0x56, 0x73, 0x1c, 0x56, 0x15, 0xb0, 0x50, 0x8a, 0xfe, - 0x0f, 0x84, 0xa2, 0x71, 0x61, 0x3a, 0x2e, 0xb6, 0xd5, 0x3c, 0x47, 0x95, 0xb8, 0xec, 0x90, 0x8b, - 0xd0, 0x77, 0x70, 0xcf, 0x35, 0x09, 0x35, 0x98, 0x2c, 0x34, 0x6a, 0x44, 0x43, 0x48, 0x2d, 0x70, - 0xe7, 0xeb, 0x0d, 0x31, 0x85, 0x1a, 0xe1, 0x14, 0x6a, 0x74, 0x43, 0x84, 0xae, 0x32, 0xf5, 0xa6, - 0xe9, 0xba, 0xd2, 0xbb, 0xe8, 0x8d, 0xf6, 0xa7, 0x0c, 0xac, 0x26, 0xd7, 0x78, 0x30, 0xc2, 0x1e, - 0x45, 0x1b, 0x50, 0xb2, 0x31, 0xb1, 0x02, 0x67, 0xc0, 0xc6, 0x20, 0x8f, 0x7b, 0x51, 0x4f, 0x8a, - 0xd0, 0x11, 0x28, 0x04, 0x8f, 0x70, 0xe0, 0xd0, 0xb7, 0x3c, 0xa6, 0xd5, 0xdd, 0xff, 0xbf, 0x39, - 0x78, 0x9c, 0xb8, 0xd1, 0x91, 0x3a, 0x7a, 0xa4, 0x8d, 0x7e, 0x0c, 0xc5, 0x78, 0x29, 0x99, 0x85, - 0x4b, 0x89, 0xc1, 0xe8, 0xe7, 0xe3, 0xfd, 0x9a, 0x5d, 0x3c, 0x52, 0x8f, 0x56, 0xc6, 0x3a, 0xf6, - 0x68, 0xaa, 0x63, 0x73, 0x4b, 0x4d, 0x98, 0xa3, 0x95, 0x89, 0x9e, 0xd5, 0x0e, 0x40, 0x09, 0x97, - 0xc6, 0xcb, 0xbf, 0x6b, 0xc4, 0x8d, 0x51, 0x82, 0x42, 0xb3, 0x6b, 0xb4, 0xda, 0x87, 0xa7, 0xb2, - 0x37, 0xba, 0xc6, 0xeb, 0x3d, 0xbd, 0x2d, 0x7a, 0xa3, 0x0c, 0x4a, 0xb3, 0x6b, 0x1c, 0xe8, 0xfa, - 0xa9, 0x5e, 0xcb, 0xec, 0x97, 0xa0, 0x68, 0x5d, 0x3a, 0xae, 0xcd, 0x7c, 0x61, 0xbd, 0x5c, 0x4e, - 0x46, 0x10, 0x3d, 0x84, 0x55, 0x6f, 0xd8, 0x37, 0x30, 0x8b, 0x24, 0x31, 0x5c, 0xbf, 0xd7, 0xc3, - 0x36, 0xcf, 0x4d, 0x46, 0xbf, 0xe5, 0x0d, 0xfb, 0x3c, 0xc2, 0xe4, 0x84, 0x8b, 0x51, 0x0b, 0x90, - 0x15, 0x60, 0xbe, 0x8b, 0x25, 0x2a, 0x25, 0xbd, 0x30, 0xbc, 0xab, 0xa1, 0x56, 0x24, 0x42, 0x5f, - 0x43, 0x5e, 0x98, 0x94, 0x13, 0x71, 0x73, 0x89, 0x44, 0xeb, 0x52, 0x45, 0xb3, 0x00, 0xe2, 0xf0, - 0xa3, 0x4f, 0x20, 0x0c, 0xbf, 0xe1, 0x84, 0xae, 0x17, 0xa5, 0xa4, 0x65, 0x23, 0x04, 0x59, 0xcf, - 0xec, 0x63, 0xd9, 0xa4, 0xfc, 0xf7, 0x71, 0x56, 0xc9, 0xd4, 0xb2, 0xc7, 0x59, 0x25, 0x5b, 0xcb, - 0x1d, 0x67, 0x95, 0x5c, 0x2d, 0x7f, 0x9c, 0x55, 0xf2, 0xb5, 0xc2, 0x71, 0x56, 0x29, 0xd4, 0x94, - 0xe3, 0xac, 0xa2, 0xd4, 0x8a, 0x9a, 0x0b, 0x95, 0xb1, 0xfc, 0xb0, 0x0e, 0x4d, 0x24, 0xd6, 0xb1, - 0x79, 0x8b, 0x64, 0xf4, 0x72, 0x2c, 0x4c, 0x58, 0x53, 0xc6, 0xac, 0xa5, 0x6a, 0xe9, 0xe3, 0xac, - 0x92, 0xae, 0x65, 0xe6, 0x59, 0xd6, 0xbe, 0x87, 0x62, 0x34, 0x7b, 0xd1, 0x3d, 0x90, 0xd3, 0x97, - 0x59, 0xc9, 0x70, 0x2b, 0x8a, 0x10, 0x24, 0x2c, 0x64, 0xe7, 0x5a, 0x98, 0xbd, 0x1e, 0x66, 0x01, - 0x07, 0x23, 0x1c, 0x84, 0x16, 0xf8, 0x03, 0xb3, 0x90, 0x93, 0x16, 0xb8, 0x20, 0x61, 0x21, 0xbf, - 0xd4, 0x1a, 0x62, 0x0b, 0x7f, 0x4d, 0x41, 0x5e, 0x98, 0x40, 0x8f, 0x93, 0x7b, 0xeb, 0xac, 0x7d, - 0x26, 0xf4, 0x44, 0xec, 0xab, 0x4f, 0xc6, 0xf6, 0xd5, 0xfb, 0xf3, 0xf0, 0x89, 0x6d, 0xf5, 0x1b, - 0xa8, 0xb8, 0x0e, 0xa1, 0xd8, 0x33, 0x44, 0x60, 0x64, 0x19, 0xdd, 0xb8, 0xa5, 0x95, 0x85, 0x86, - 0x10, 0x68, 0x7f, 0x60, 0xa7, 0x81, 0x88, 0x36, 0x9e, 0xda, 0xa9, 0x0f, 0x9a, 0xda, 0xe9, 0xe5, - 0xa6, 0x76, 0x66, 0xa9, 0xa9, 0x9d, 0x7d, 0xef, 0xa9, 0x9d, 0xfb, 0x80, 0xa9, 0xfd, 0x97, 0x34, - 0xe4, 0x45, 0x6c, 0x16, 0xa7, 0x2f, 0x8a, 0xe9, 0x92, 0xe9, 0xe3, 0xf8, 0x44, 0xfa, 0xb6, 0x21, - 0xe7, 0xfa, 0x96, 0xe9, 0xca, 0xd9, 0xbc, 0x36, 0xad, 0xb2, 0x67, 0xdb, 0x01, 0x26, 0x44, 0x17, - 0x38, 0xb4, 0x03, 0xf9, 0x00, 0xf7, 0x7d, 0x8a, 0xe5, 0x44, 0xbe, 0x41, 0x43, 0x02, 0xd1, 0x33, - 0xb6, 0x9b, 0x58, 0x43, 0xbe, 0x9b, 0x44, 0x71, 0x99, 0x2e, 0x2c, 0x81, 0xd0, 0x23, 0x2c, 0x5a, - 0x87, 0x92, 0x60, 0x30, 0x12, 0x5d, 0x00, 0x42, 0xd4, 0x36, 0xfb, 0x58, 0xfb, 0x7d, 0x01, 0x20, - 0x5e, 0x11, 0x4b, 0x2f, 0xa1, 0x01, 0x36, 0xfb, 0x71, 0x15, 0x88, 0x21, 0x54, 0x95, 0xe2, 0xb0, - 0x0e, 0x1e, 0xc1, 0x6a, 0x04, 0x8c, 0x2a, 0x41, 0x14, 0x4c, 0x2d, 0x84, 0x46, 0xb5, 0xf0, 0x39, - 0x84, 0xea, 0x61, 0x35, 0x88, 0x9a, 0xa9, 0x48, 0xa9, 0xac, 0x87, 0x4d, 0xa8, 0xf4, 0x31, 0x21, - 0x66, 0x0f, 0x13, 0x83, 0x60, 0x8f, 0x86, 0xc7, 0x86, 0x50, 0xd8, 0x61, 0x3b, 0xef, 0x23, 0x58, - 0x8d, 0x40, 0x01, 0xb6, 0xb0, 0x33, 0x8a, 0x0e, 0x0e, 0xb5, 0xf0, 0x85, 0x2e, 0xe5, 0x68, 0x0b, - 0x6a, 0x57, 0x18, 0x0f, 0x0c, 0xd3, 0x75, 0x46, 0x21, 0xa9, 0x38, 0x3e, 0x54, 0x99, 0x7c, 0x8f, - 0x8b, 0x39, 0xed, 0x25, 0x6c, 0xf2, 0x5a, 0xe4, 0x19, 0x32, 0x84, 0x5f, 0x06, 0x1f, 0xf5, 0xef, - 0x79, 0x92, 0x58, 0x67, 0x34, 0x27, 0x8c, 0xa5, 0xc3, 0x49, 0x9a, 0x82, 0x23, 0xde, 0x2d, 0x7e, - 0x03, 0x9f, 0x71, 0x4b, 0x32, 0x2f, 0x73, 0x4d, 0x29, 0x0b, 0x4d, 0x6d, 0x30, 0x1e, 0x9d, 0xd3, - 0xcc, 0xb1, 0x15, 0x76, 0x98, 0x0c, 0x0c, 0x0f, 0x40, 0xc2, 0x44, 0x71, 0xb9, 0x0e, 0x7b, 0x29, - 0xb4, 0x59, 0x9c, 0x62, 0x6a, 0x13, 0xd6, 0xc7, 0xa8, 0xc3, 0x5c, 0x24, 0xe8, 0x61, 0x21, 0xfd, - 0xfd, 0x04, 0x7d, 0x98, 0xb4, 0xd8, 0xc4, 0xb7, 0xb0, 0x26, 0xd2, 0x71, 0xe1, 0xfa, 0xd7, 0x86, - 0xe5, 0x7b, 0x34, 0xf0, 0x5d, 0xe3, 0xda, 0xf1, 0x6c, 0xff, 0x5a, 0x2d, 0x85, 0xfd, 0x3c, 0x41, - 0xde, 0xf2, 0xe8, 0xb3, 0xa7, 0xdf, 0x9a, 0xee, 0x10, 0xeb, 0x77, 0xb9, 0xf6, 0xa1, 0xeb, 0x5f, - 0x37, 0x85, 0xee, 0x6b, 0xae, 0x8a, 0xde, 0x40, 0x5d, 0x06, 0x7f, 0x16, 0x71, 0x79, 0x31, 0xf1, - 0xc7, 0x42, 0x7d, 0x9a, 0xf9, 0x19, 0xe4, 0x7d, 0x71, 0x22, 0xac, 0xf0, 0x11, 0xfe, 0xe9, 0xbc, - 0xf1, 0x71, 0xca, 0x51, 0xba, 0x44, 0x6b, 0xff, 0xcc, 0x40, 0x41, 0xb6, 0x3c, 0x7a, 0x09, 0x15, - 0x6a, 0x0d, 0x9c, 0x81, 0x61, 0x0a, 0x81, 0x9c, 0x5c, 0x0f, 0xe6, 0x0e, 0x89, 0x46, 0xd7, 0x1a, - 0xb4, 0x06, 0xf2, 0xe1, 0x68, 0x45, 0x2f, 0x73, 0xf5, 0x90, 0xee, 0x05, 0x94, 0x86, 0x36, 0x89, - 0xc8, 0xc4, 0x58, 0xfb, 0x6c, 0x3e, 0xd9, 0x99, 0x4d, 0x62, 0x2a, 0x18, 0x46, 0x4f, 0xcc, 0x2f, - 0x9f, 0x5e, 0xe2, 0x20, 0xa2, 0xca, 0x2c, 0xf2, 0xeb, 0x94, 0xc1, 0x13, 0x7e, 0xf9, 0x89, 0xe7, - 0xfa, 0x1e, 0x94, 0x93, 0x7e, 0xb3, 0x93, 0xcf, 0xc4, 0x9a, 0xcb, 0x7a, 0x31, 0x5e, 0x06, 0x82, - 0xec, 0xc0, 0x0f, 0xc4, 0xe7, 0x49, 0x4e, 0xe7, 0xbf, 0xeb, 0x5b, 0x00, 0xb1, 0xb7, 0xa8, 0x0e, - 0xca, 0x85, 0xe3, 0x62, 0x3e, 0xe7, 0xc4, 0x79, 0x3c, 0x7a, 0xae, 0xb7, 0xa1, 0x9c, 0x74, 0x26, - 0x3a, 0x15, 0xa4, 0xe2, 0x53, 0x01, 0x7a, 0x08, 0xb9, 0x11, 0xcb, 0xae, 0x0c, 0xd1, 0x9d, 0xa9, - 0x02, 0xd8, 0xf3, 0xde, 0xea, 0x02, 0xb2, 0x5f, 0x84, 0x82, 0xf4, 0x54, 0xfb, 0x63, 0x86, 0x9d, - 0x6c, 0xe5, 0xb8, 0xdd, 0x85, 0x0c, 0x75, 0xc9, 0xfc, 0x6d, 0x37, 0x04, 0x36, 0xba, 0x2e, 0x8b, - 0x08, 0x03, 0xb3, 0x8f, 0x37, 0x1e, 0x18, 0x69, 0x77, 0xeb, 0x06, 0x2d, 0xbe, 0x86, 0xf0, 0xe9, - 0x68, 0x45, 0x17, 0x8a, 0xf5, 0x7f, 0xa5, 0x20, 0xd3, 0x75, 0x09, 0xfa, 0x1c, 0x2a, 0x84, 0x9a, - 0x9e, 0x6d, 0x06, 0xb6, 0x11, 0x2f, 0x8f, 0x45, 0x3e, 0x14, 0xb3, 0x91, 0x8f, 0xd6, 0x01, 0x44, - 0x22, 0xe3, 0xa3, 0xe4, 0xd1, 0x8a, 0x5e, 0xe4, 0x32, 0x0e, 0x78, 0x04, 0xab, 0xa2, 0xef, 0x2c, - 0x1c, 0x50, 0xe7, 0xc2, 0xb1, 0xd8, 0xa7, 0x65, 0x86, 0x67, 0xa4, 0xc6, 0x5f, 0x34, 0x63, 0x39, - 0x7a, 0x0c, 0x48, 0x36, 0x53, 0x12, 0x9d, 0xe5, 0xe8, 0x55, 0xf1, 0x26, 0x01, 0xdf, 0xaf, 0x42, - 0xd9, 0x72, 0x06, 0xcc, 0x3a, 0x19, 0x3a, 0x14, 0xd7, 0x4f, 0xa1, 0x32, 0xb6, 0xaa, 0x0f, 0x4e, - 0x4d, 0x01, 0x72, 0x7d, 0xdf, 0xc6, 0xae, 0xe6, 0x41, 0x39, 0xd9, 0x6b, 0x33, 0x89, 0xef, 0x24, - 0x89, 0x8b, 0x92, 0x02, 0x3d, 0x05, 0x30, 0x6d, 0xdb, 0x61, 0x5a, 0xd1, 0xae, 0x3e, 0xdb, 0x66, - 0x02, 0xa7, 0x9d, 0xc0, 0xed, 0xa4, 0x3d, 0x36, 0xc6, 0xfc, 0x21, 0x45, 0x3f, 0x02, 0x25, 0xbc, - 0x2d, 0x93, 0x75, 0xb1, 0x36, 0x45, 0xf5, 0x5c, 0x02, 0xf4, 0x08, 0xaa, 0x59, 0x80, 0x92, 0x6c, - 0x27, 0x8e, 0xd7, 0xc3, 0x01, 0xfb, 0x4c, 0x37, 0xd9, 0xe7, 0xbb, 0x58, 0x85, 0xa2, 0xcb, 0xa7, - 0x31, 0x23, 0xe9, 0xe5, 0x8d, 0xfc, 0x5d, 0x99, 0xf0, 0xd9, 0x1a, 0xb4, 0xbc, 0x0b, 0x9f, 0xf5, - 0x22, 0x9b, 0x21, 0x46, 0x7c, 0xa9, 0x50, 0xd1, 0x8b, 0x4c, 0x22, 0x6e, 0x35, 0x34, 0x31, 0xa1, - 0x0c, 0xcb, 0x94, 0x88, 0x34, 0x47, 0x94, 0x98, 0xb0, 0x69, 0x0a, 0xcc, 0x97, 0x50, 0xe3, 0x98, - 0x00, 0xd3, 0xc0, 0xf4, 0x48, 0xdf, 0xa1, 0x62, 0x60, 0x54, 0xf4, 0x5b, 0x4c, 0xae, 0xc7, 0x62, - 0x76, 0x46, 0xe1, 0xd0, 0x41, 0xe0, 0x9f, 0x63, 0xc2, 0x4b, 0xa7, 0xa2, 0x73, 0x07, 0x5e, 0x71, - 0x09, 0x3b, 0x4a, 0x72, 0xc0, 0xb9, 0x69, 0x5d, 0xf9, 0x17, 0xe2, 0x1b, 0x54, 0x9a, 0xdb, 0x17, - 0xa2, 0x08, 0x22, 0xe6, 0x29, 0xe1, 0x9b, 0xbc, 0x84, 0x88, 0xa5, 0x11, 0xf4, 0x00, 0x6e, 0x89, - 0x45, 0x79, 0xb6, 0x71, 0x4d, 0x2c, 0xd3, 0xc5, 0x7c, 0x37, 0xaf, 0xe8, 0x7c, 0x31, 0x1d, 0xcf, - 0x7e, 0xcd, 0x85, 0x11, 0x2e, 0xb0, 0x46, 0x21, 0x4e, 0x89, 0x71, 0xba, 0x35, 0x92, 0xb8, 0x35, - 0x50, 0x04, 0x8e, 0xfa, 0x7c, 0x23, 0xad, 0xe8, 0x05, 0x0e, 0xa0, 0x7e, 0xf4, 0xca, 0xa4, 0x3e, - 0xdf, 0x04, 0xe5, 0xab, 0x3d, 0xea, 0xa3, 0x0d, 0xe9, 0x28, 0xf3, 0xa2, 0x4f, 0x08, 0xdf, 0xc6, - 0xe4, 0x6a, 0x3b, 0x9e, 0xfd, 0x92, 0x90, 0x08, 0xc1, 0xec, 0x33, 0x44, 0x39, 0x46, 0xe8, 0xd6, - 0x88, 0x21, 0xc2, 0xc5, 0x0e, 0x3d, 0xd3, 0xba, 0xc2, 0xb6, 0x5a, 0x89, 0x17, 0x7b, 0x26, 0x44, - 0x51, 0x4c, 0x89, 0x40, 0x54, 0x13, 0x56, 0x04, 0xe0, 0x1e, 0xf0, 0x84, 0x1a, 0xae, 0x4f, 0xa8, - 0x7a, 0x8b, 0xbf, 0xe6, 0x3e, 0x9f, 0xf8, 0x84, 0x46, 0x06, 0x64, 0xf2, 0xd4, 0x5a, 0x6c, 0x40, - 0x26, 0x2e, 0x82, 0x5c, 0x30, 0x3a, 0x4a, 0xd4, 0xd5, 0x18, 0x72, 0x28, 0x44, 0xe8, 0x31, 0xdc, - 0x16, 0x26, 0xd8, 0x31, 0x81, 0x9d, 0x94, 0xc5, 0xf9, 0x0b, 0x71, 0x24, 0xaf, 0x8e, 0x13, 0x93, - 0xf0, 0x63, 0xa7, 0x3c, 0xd8, 0xa1, 0x18, 0x6e, 0x5a, 0x57, 0x02, 0x7d, 0x3b, 0xae, 0x19, 0x86, - 0xde, 0xb3, 0xae, 0x38, 0x78, 0x9a, 0x3b, 0xc0, 0xd6, 0x48, 0xbd, 0x33, 0xcd, 0xad, 0x63, 0x6b, - 0x34, 0xcd, 0xcd, 0xd1, 0x1f, 0x4d, 0x71, 0x73, 0x70, 0x18, 0x9a, 0x41, 0x9f, 0x0e, 0xd5, 0xbb, - 0x71, 0x68, 0x5e, 0xf5, 0xe9, 0x10, 0x3d, 0x84, 0xd5, 0x28, 0x3b, 0x84, 0xd0, 0xcb, 0x00, 0x93, - 0x4b, 0xf5, 0xe3, 0x44, 0x61, 0x5b, 0xa3, 0x8e, 0x14, 0x27, 0x2a, 0x84, 0xaa, 0x6a, 0xb2, 0x42, - 0x68, 0x94, 0x9f, 0x80, 0xd2, 0x91, 0x19, 0xa8, 0x6b, 0x89, 0x1c, 0x73, 0x49, 0x64, 0x87, 0xd5, - 0x49, 0x64, 0xa7, 0x1e, 0xdb, 0xe9, 0x78, 0x76, 0x64, 0x27, 0xec, 0x47, 0x86, 0xb5, 0xae, 0x3d, - 0x5b, 0xbd, 0x17, 0x27, 0xa3, 0xe3, 0xd9, 0xcd, 0x6b, 0x2f, 0x2e, 0x08, 0xd3, 0x1e, 0xb1, 0xa2, - 0xba, 0x1f, 0x1b, 0xdc, 0xe3, 0x12, 0x76, 0xf2, 0x97, 0x39, 0xf7, 0x03, 0x1b, 0x07, 0x8e, 0xd7, - 0x53, 0x3f, 0xe1, 0xa0, 0xaa, 0x48, 0x7b, 0x28, 0xd5, 0xce, 0xe1, 0xa3, 0x17, 0x98, 0x76, 0xfd, - 0x81, 0xfc, 0x86, 0x24, 0x3a, 0xfe, 0xed, 0x10, 0x13, 0xca, 0x0e, 0xdb, 0xfc, 0x9b, 0xc1, 0x98, - 0xba, 0xc1, 0xa8, 0x72, 0x79, 0x33, 0xba, 0x58, 0x58, 0x87, 0x52, 0xdf, 0xfc, 0xc1, 0x08, 0x30, - 0x19, 0xba, 0x94, 0xc8, 0xcf, 0x06, 0xe8, 0x9b, 0x3f, 0xe8, 0x42, 0xa2, 0x19, 0x70, 0x77, 0xd2, - 0x06, 0x19, 0xf8, 0x1e, 0xc1, 0xe8, 0x2b, 0x28, 0x48, 0x7a, 0x35, 0xc5, 0x8f, 0x58, 0x6b, 0xf3, - 0xaf, 0xb3, 0x42, 0x24, 0xaa, 0x41, 0x06, 0x7b, 0xe2, 0xf3, 0x44, 0xd1, 0xd9, 0x4f, 0xed, 0x57, - 0xb0, 0xfa, 0x02, 0x53, 0xf1, 0xc9, 0x1c, 0x2d, 0xe0, 0x01, 0xfb, 0xf8, 0x61, 0x0b, 0x88, 0xaf, - 0x13, 0x52, 0xe1, 0x77, 0x8a, 0x19, 0x48, 0xf4, 0x32, 0xee, 0xbf, 0x01, 0x94, 0x64, 0x97, 0xae, - 0x3f, 0x81, 0xbc, 0x20, 0x96, 0x9e, 0xab, 0x73, 0xaf, 0x12, 0x24, 0x6e, 0x86, 0xdf, 0xdb, 0x50, - 0x8b, 0x98, 0x43, 0xb7, 0xc7, 0xee, 0x3f, 0x52, 0xe3, 0xf7, 0x1f, 0xda, 0x41, 0x62, 0xa1, 0x33, - 0x3d, 0x49, 0x2d, 0xe3, 0x89, 0xf6, 0x3b, 0xf8, 0x38, 0xa2, 0x11, 0x3b, 0x06, 0x59, 0xc6, 0x7c, - 0x22, 0xa4, 0xd1, 0x1d, 0x50, 0x3a, 0x19, 0xd2, 0xf0, 0x22, 0x68, 0x22, 0xa4, 0x99, 0xa9, 0x90, - 0x5e, 0x82, 0x3a, 0xed, 0x80, 0x5c, 0xce, 0xf8, 0xff, 0x03, 0x52, 0xef, 0xf3, 0xff, 0x80, 0x19, - 0x21, 0xde, 0xe5, 0x11, 0x8b, 0xee, 0xe4, 0xc4, 0x22, 0x6f, 0xbe, 0x97, 0xd3, 0x5a, 0x3c, 0xe1, - 0x91, 0xce, 0xac, 0x5a, 0x4d, 0x2d, 0x57, 0xab, 0xda, 0xd7, 0x70, 0x87, 0x2d, 0x34, 0x71, 0x5b, - 0x27, 0x3c, 0x98, 0xba, 0xb1, 0x4b, 0x4d, 0xdf, 0xd8, 0x69, 0x67, 0xbc, 0x37, 0x93, 0xca, 0xd2, - 0x95, 0x9f, 0x02, 0xc4, 0xc0, 0xf9, 0xff, 0x5b, 0x4b, 0x68, 0x26, 0xf0, 0x5a, 0x4b, 0x54, 0x9d, - 0x0c, 0x5a, 0x9c, 0xf6, 0x28, 0xa7, 0xa9, 0x89, 0x7b, 0x3d, 0x15, 0x0a, 0x64, 0xd8, 0xef, 0x9b, - 0xc1, 0x5b, 0x19, 0xd9, 0xf0, 0x31, 0xac, 0x47, 0x49, 0x95, 0xa8, 0x47, 0x71, 0xf3, 0x35, 0xbf, - 0x1e, 0x85, 0x86, 0xc4, 0xed, 0xfe, 0x39, 0x07, 0x8a, 0x0c, 0xdd, 0x3b, 0x64, 0x41, 0x75, 0x7c, - 0x5a, 0xa0, 0x2f, 0xa6, 0x09, 0x66, 0xce, 0xac, 0xfa, 0xd6, 0x62, 0xa0, 0xf4, 0xf1, 0x35, 0x40, - 0xdc, 0xd3, 0x68, 0x73, 0xa6, 0xde, 0xf8, 0x3c, 0xa9, 0x7f, 0x76, 0x33, 0x48, 0x12, 0x77, 0xa1, - 0x18, 0x49, 0x91, 0x76, 0x83, 0x4a, 0x48, 0xbb, 0x79, 0x23, 0x46, 0xb2, 0x3a, 0x89, 0x41, 0x21, - 0xfb, 0x05, 0x7d, 0x79, 0x83, 0xe2, 0x78, 0x53, 0xd7, 0x1f, 0x2e, 0x03, 0x1d, 0x8b, 0x4c, 0xf8, - 0xef, 0xdb, 0xd9, 0xde, 0x8d, 0xb7, 0xd3, 0x9c, 0xc8, 0x4c, 0xf6, 0xcf, 0xf7, 0x50, 0x19, 0xab, - 0x66, 0xf4, 0x60, 0xb6, 0x57, 0x93, 0xbd, 0x52, 0xff, 0x62, 0x21, 0x6e, 0x3c, 0xf6, 0xe2, 0xa2, - 0x70, 0x4e, 0xec, 0x93, 0x55, 0x3f, 0x2f, 0xf6, 0x63, 0xe5, 0xbc, 0xff, 0x06, 0x6e, 0x3b, 0xfe, - 0x14, 0x70, 0xbf, 0x12, 0x16, 0xec, 0x2b, 0x76, 0x24, 0x7f, 0x95, 0xfa, 0xe5, 0x13, 0x79, 0x44, - 0xef, 0xf9, 0xae, 0xe9, 0xf5, 0x1a, 0x7e, 0xd0, 0xdb, 0x1e, 0xff, 0xb7, 0x3d, 0x7b, 0x0a, 0x77, - 0xd3, 0x77, 0xc6, 0x68, 0xe7, 0x3c, 0xcf, 0x4f, 0xf3, 0x5f, 0xfd, 0x37, 0x00, 0x00, 0xff, 0xff, - 0x54, 0xae, 0x0b, 0x93, 0xdf, 0x1f, 0x00, 0x00, -} - -// Reference imports to suppress errors if they are not otherwise used. -var _ context.Context -var _ grpc.ClientConnInterface - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the grpc package it is being compiled against. -const _ = grpc.SupportPackageIsVersion6 - -// ChannelzClient is the client API for Channelz service. -// -// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. -type ChannelzClient interface { - // Gets all root channels (i.e. channels the application has directly - // created). This does not include subchannels nor non-top level channels. - GetTopChannels(ctx context.Context, in *GetTopChannelsRequest, opts ...grpc.CallOption) (*GetTopChannelsResponse, error) - // Gets all servers that exist in the process. - GetServers(ctx context.Context, in *GetServersRequest, opts ...grpc.CallOption) (*GetServersResponse, error) - // Returns a single Server, or else a NOT_FOUND code. - GetServer(ctx context.Context, in *GetServerRequest, opts ...grpc.CallOption) (*GetServerResponse, error) - // Gets all server sockets that exist in the process. - GetServerSockets(ctx context.Context, in *GetServerSocketsRequest, opts ...grpc.CallOption) (*GetServerSocketsResponse, error) - // Returns a single Channel, or else a NOT_FOUND code. - GetChannel(ctx context.Context, in *GetChannelRequest, opts ...grpc.CallOption) (*GetChannelResponse, error) - // Returns a single Subchannel, or else a NOT_FOUND code. - GetSubchannel(ctx context.Context, in *GetSubchannelRequest, opts ...grpc.CallOption) (*GetSubchannelResponse, error) - // Returns a single Socket or else a NOT_FOUND code. - GetSocket(ctx context.Context, in *GetSocketRequest, opts ...grpc.CallOption) (*GetSocketResponse, error) +type Address_TcpIpAddress struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Either the IPv4 or IPv6 address in bytes. Will be either 4 bytes or 16 + // bytes in length. + IpAddress []byte `protobuf:"bytes,1,opt,name=ip_address,json=ipAddress,proto3" json:"ip_address,omitempty"` + // 0-64k, or -1 if not appropriate. + Port int32 `protobuf:"varint,2,opt,name=port,proto3" json:"port,omitempty"` } -type channelzClient struct { - cc grpc.ClientConnInterface +func (x *Address_TcpIpAddress) Reset() { + *x = Address_TcpIpAddress{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[34] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func NewChannelzClient(cc grpc.ClientConnInterface) ChannelzClient { - return &channelzClient{cc} +func (x *Address_TcpIpAddress) String() string { + return protoimpl.X.MessageStringOf(x) } -func (c *channelzClient) GetTopChannels(ctx context.Context, in *GetTopChannelsRequest, opts ...grpc.CallOption) (*GetTopChannelsResponse, error) { - out := new(GetTopChannelsResponse) - err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetTopChannels", in, out, opts...) - if err != nil { - return nil, err +func (*Address_TcpIpAddress) ProtoMessage() {} + +func (x *Address_TcpIpAddress) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[34] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms } - return out, nil + return mi.MessageOf(x) } -func (c *channelzClient) GetServers(ctx context.Context, in *GetServersRequest, opts ...grpc.CallOption) (*GetServersResponse, error) { - out := new(GetServersResponse) - err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetServers", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil +// Deprecated: Use Address_TcpIpAddress.ProtoReflect.Descriptor instead. +func (*Address_TcpIpAddress) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{14, 0} } -func (c *channelzClient) GetServer(ctx context.Context, in *GetServerRequest, opts ...grpc.CallOption) (*GetServerResponse, error) { - out := new(GetServerResponse) - err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetServer", in, out, opts...) - if err != nil { - return nil, err +func (x *Address_TcpIpAddress) GetIpAddress() []byte { + if x != nil { + return x.IpAddress } - return out, nil + return nil } -func (c *channelzClient) GetServerSockets(ctx context.Context, in *GetServerSocketsRequest, opts ...grpc.CallOption) (*GetServerSocketsResponse, error) { - out := new(GetServerSocketsResponse) - err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetServerSockets", in, out, opts...) - if err != nil { - return nil, err +func (x *Address_TcpIpAddress) GetPort() int32 { + if x != nil { + return x.Port } - return out, nil + return 0 } -func (c *channelzClient) GetChannel(ctx context.Context, in *GetChannelRequest, opts ...grpc.CallOption) (*GetChannelResponse, error) { - out := new(GetChannelResponse) - err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetChannel", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil +// A Unix Domain Socket address. +type Address_UdsAddress struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Filename string `protobuf:"bytes,1,opt,name=filename,proto3" json:"filename,omitempty"` } -func (c *channelzClient) GetSubchannel(ctx context.Context, in *GetSubchannelRequest, opts ...grpc.CallOption) (*GetSubchannelResponse, error) { - out := new(GetSubchannelResponse) - err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetSubchannel", in, out, opts...) - if err != nil { - return nil, err +func (x *Address_UdsAddress) Reset() { + *x = Address_UdsAddress{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[35] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } - return out, nil } -func (c *channelzClient) GetSocket(ctx context.Context, in *GetSocketRequest, opts ...grpc.CallOption) (*GetSocketResponse, error) { - out := new(GetSocketResponse) - err := c.cc.Invoke(ctx, "/grpc.channelz.v1.Channelz/GetSocket", in, out, opts...) - if err != nil { - return nil, err +func (x *Address_UdsAddress) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Address_UdsAddress) ProtoMessage() {} + +func (x *Address_UdsAddress) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[35] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms } - return out, nil + return mi.MessageOf(x) } -// ChannelzServer is the server API for Channelz service. -type ChannelzServer interface { - // Gets all root channels (i.e. channels the application has directly - // created). This does not include subchannels nor non-top level channels. - GetTopChannels(context.Context, *GetTopChannelsRequest) (*GetTopChannelsResponse, error) - // Gets all servers that exist in the process. - GetServers(context.Context, *GetServersRequest) (*GetServersResponse, error) - // Returns a single Server, or else a NOT_FOUND code. - GetServer(context.Context, *GetServerRequest) (*GetServerResponse, error) - // Gets all server sockets that exist in the process. - GetServerSockets(context.Context, *GetServerSocketsRequest) (*GetServerSocketsResponse, error) - // Returns a single Channel, or else a NOT_FOUND code. - GetChannel(context.Context, *GetChannelRequest) (*GetChannelResponse, error) - // Returns a single Subchannel, or else a NOT_FOUND code. - GetSubchannel(context.Context, *GetSubchannelRequest) (*GetSubchannelResponse, error) - // Returns a single Socket or else a NOT_FOUND code. - GetSocket(context.Context, *GetSocketRequest) (*GetSocketResponse, error) +// Deprecated: Use Address_UdsAddress.ProtoReflect.Descriptor instead. +func (*Address_UdsAddress) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{14, 1} } -// UnimplementedChannelzServer can be embedded to have forward compatible implementations. -type UnimplementedChannelzServer struct { +func (x *Address_UdsAddress) GetFilename() string { + if x != nil { + return x.Filename + } + return "" } -func (*UnimplementedChannelzServer) GetTopChannels(ctx context.Context, req *GetTopChannelsRequest) (*GetTopChannelsResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method GetTopChannels not implemented") +// An address type not included above. +type Address_OtherAddress struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // The human readable version of the value. This value should be set. + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + // The actual address message. + Value *anypb.Any `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` } -func (*UnimplementedChannelzServer) GetServers(ctx context.Context, req *GetServersRequest) (*GetServersResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method GetServers not implemented") + +func (x *Address_OtherAddress) Reset() { + *x = Address_OtherAddress{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[36] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (*UnimplementedChannelzServer) GetServer(ctx context.Context, req *GetServerRequest) (*GetServerResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method GetServer not implemented") + +func (x *Address_OtherAddress) String() string { + return protoimpl.X.MessageStringOf(x) } -func (*UnimplementedChannelzServer) GetServerSockets(ctx context.Context, req *GetServerSocketsRequest) (*GetServerSocketsResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method GetServerSockets not implemented") + +func (*Address_OtherAddress) ProtoMessage() {} + +func (x *Address_OtherAddress) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[36] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -func (*UnimplementedChannelzServer) GetChannel(ctx context.Context, req *GetChannelRequest) (*GetChannelResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method GetChannel not implemented") + +// Deprecated: Use Address_OtherAddress.ProtoReflect.Descriptor instead. +func (*Address_OtherAddress) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{14, 2} } -func (*UnimplementedChannelzServer) GetSubchannel(ctx context.Context, req *GetSubchannelRequest) (*GetSubchannelResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method GetSubchannel not implemented") + +func (x *Address_OtherAddress) GetName() string { + if x != nil { + return x.Name + } + return "" } -func (*UnimplementedChannelzServer) GetSocket(ctx context.Context, req *GetSocketRequest) (*GetSocketResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method GetSocket not implemented") + +func (x *Address_OtherAddress) GetValue() *anypb.Any { + if x != nil { + return x.Value + } + return nil } -func RegisterChannelzServer(s *grpc.Server, srv ChannelzServer) { - s.RegisterService(&_Channelz_serviceDesc, srv) +type Security_Tls struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Types that are assignable to CipherSuite: + // + // *Security_Tls_StandardName + // *Security_Tls_OtherName + CipherSuite isSecurity_Tls_CipherSuite `protobuf_oneof:"cipher_suite"` + // the certificate used by this endpoint. + LocalCertificate []byte `protobuf:"bytes,3,opt,name=local_certificate,json=localCertificate,proto3" json:"local_certificate,omitempty"` + // the certificate used by the remote endpoint. + RemoteCertificate []byte `protobuf:"bytes,4,opt,name=remote_certificate,json=remoteCertificate,proto3" json:"remote_certificate,omitempty"` } -func _Channelz_GetTopChannels_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(GetTopChannelsRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(ChannelzServer).GetTopChannels(ctx, in) +func (x *Security_Tls) Reset() { + *x = Security_Tls{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[37] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/grpc.channelz.v1.Channelz/GetTopChannels", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(ChannelzServer).GetTopChannels(ctx, req.(*GetTopChannelsRequest)) - } - return interceptor(ctx, in, info, handler) } -func _Channelz_GetServers_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(GetServersRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(ChannelzServer).GetServers(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/grpc.channelz.v1.Channelz/GetServers", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(ChannelzServer).GetServers(ctx, req.(*GetServersRequest)) - } - return interceptor(ctx, in, info, handler) +func (x *Security_Tls) String() string { + return protoimpl.X.MessageStringOf(x) } -func _Channelz_GetServer_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(GetServerRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(ChannelzServer).GetServer(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/grpc.channelz.v1.Channelz/GetServer", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(ChannelzServer).GetServer(ctx, req.(*GetServerRequest)) +func (*Security_Tls) ProtoMessage() {} + +func (x *Security_Tls) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[37] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms } - return interceptor(ctx, in, info, handler) + return mi.MessageOf(x) } -func _Channelz_GetServerSockets_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(GetServerSocketsRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(ChannelzServer).GetServerSockets(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/grpc.channelz.v1.Channelz/GetServerSockets", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(ChannelzServer).GetServerSockets(ctx, req.(*GetServerSocketsRequest)) +// Deprecated: Use Security_Tls.ProtoReflect.Descriptor instead. +func (*Security_Tls) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{15, 0} +} + +func (m *Security_Tls) GetCipherSuite() isSecurity_Tls_CipherSuite { + if m != nil { + return m.CipherSuite } - return interceptor(ctx, in, info, handler) + return nil } -func _Channelz_GetChannel_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(GetChannelRequest) - if err := dec(in); err != nil { - return nil, err +func (x *Security_Tls) GetStandardName() string { + if x, ok := x.GetCipherSuite().(*Security_Tls_StandardName); ok { + return x.StandardName } - if interceptor == nil { - return srv.(ChannelzServer).GetChannel(ctx, in) + return "" +} + +func (x *Security_Tls) GetOtherName() string { + if x, ok := x.GetCipherSuite().(*Security_Tls_OtherName); ok { + return x.OtherName } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/grpc.channelz.v1.Channelz/GetChannel", + return "" +} + +func (x *Security_Tls) GetLocalCertificate() []byte { + if x != nil { + return x.LocalCertificate } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(ChannelzServer).GetChannel(ctx, req.(*GetChannelRequest)) + return nil +} + +func (x *Security_Tls) GetRemoteCertificate() []byte { + if x != nil { + return x.RemoteCertificate } - return interceptor(ctx, in, info, handler) + return nil +} + +type isSecurity_Tls_CipherSuite interface { + isSecurity_Tls_CipherSuite() +} + +type Security_Tls_StandardName struct { + // The cipher suite name in the RFC 4346 format: + // https://tools.ietf.org/html/rfc4346#appendix-C + StandardName string `protobuf:"bytes,1,opt,name=standard_name,json=standardName,proto3,oneof"` } -func _Channelz_GetSubchannel_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(GetSubchannelRequest) - if err := dec(in); err != nil { - return nil, err +type Security_Tls_OtherName struct { + // Some other way to describe the cipher suite if + // the RFC 4346 name is not available. + OtherName string `protobuf:"bytes,2,opt,name=other_name,json=otherName,proto3,oneof"` +} + +func (*Security_Tls_StandardName) isSecurity_Tls_CipherSuite() {} + +func (*Security_Tls_OtherName) isSecurity_Tls_CipherSuite() {} + +type Security_OtherSecurity struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // The human readable version of the value. + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + // The actual security details message. + Value *anypb.Any `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` +} + +func (x *Security_OtherSecurity) Reset() { + *x = Security_OtherSecurity{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[38] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } - if interceptor == nil { - return srv.(ChannelzServer).GetSubchannel(ctx, in) +} + +func (x *Security_OtherSecurity) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Security_OtherSecurity) ProtoMessage() {} + +func (x *Security_OtherSecurity) ProtoReflect() protoreflect.Message { + mi := &file_grpc_channelz_v1_channelz_proto_msgTypes[38] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/grpc.channelz.v1.Channelz/GetSubchannel", + return mi.MessageOf(x) +} + +// Deprecated: Use Security_OtherSecurity.ProtoReflect.Descriptor instead. +func (*Security_OtherSecurity) Descriptor() ([]byte, []int) { + return file_grpc_channelz_v1_channelz_proto_rawDescGZIP(), []int{15, 1} +} + +func (x *Security_OtherSecurity) GetName() string { + if x != nil { + return x.Name } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(ChannelzServer).GetSubchannel(ctx, req.(*GetSubchannelRequest)) + return "" +} + +func (x *Security_OtherSecurity) GetValue() *anypb.Any { + if x != nil { + return x.Value } - return interceptor(ctx, in, info, handler) + return nil } -func _Channelz_GetSocket_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(GetSocketRequest) - if err := dec(in); err != nil { - return nil, err +var File_grpc_channelz_v1_channelz_proto protoreflect.FileDescriptor + +var file_grpc_channelz_v1_channelz_proto_rawDesc = []byte{ + 0x0a, 0x1f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2f, + 0x76, 0x31, 0x2f, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x12, 0x10, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, + 0x2e, 0x76, 0x31, 0x1a, 0x19, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x61, 0x6e, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1e, + 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, + 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1f, + 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, + 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, + 0x1e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2f, 0x77, 0x72, 0x61, 0x70, 0x70, 0x65, 0x72, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, + 0xaf, 0x02, 0x0a, 0x07, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x12, 0x2e, 0x0a, 0x03, 0x72, + 0x65, 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, + 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x68, 0x61, 0x6e, + 0x6e, 0x65, 0x6c, 0x52, 0x65, 0x66, 0x52, 0x03, 0x72, 0x65, 0x66, 0x12, 0x31, 0x0a, 0x04, 0x64, + 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x68, 0x61, + 0x6e, 0x6e, 0x65, 0x6c, 0x44, 0x61, 0x74, 0x61, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x3d, + 0x0a, 0x0b, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x5f, 0x72, 0x65, 0x66, 0x18, 0x03, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, + 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x52, 0x65, + 0x66, 0x52, 0x0a, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x52, 0x65, 0x66, 0x12, 0x46, 0x0a, + 0x0e, 0x73, 0x75, 0x62, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x5f, 0x72, 0x65, 0x66, 0x18, + 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, + 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x75, 0x62, 0x63, 0x68, 0x61, 0x6e, + 0x6e, 0x65, 0x6c, 0x52, 0x65, 0x66, 0x52, 0x0d, 0x73, 0x75, 0x62, 0x63, 0x68, 0x61, 0x6e, 0x6e, + 0x65, 0x6c, 0x52, 0x65, 0x66, 0x12, 0x3a, 0x0a, 0x0a, 0x73, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x5f, + 0x72, 0x65, 0x66, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6f, 0x63, + 0x6b, 0x65, 0x74, 0x52, 0x65, 0x66, 0x52, 0x09, 0x73, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x65, + 0x66, 0x22, 0xb5, 0x02, 0x0a, 0x0a, 0x53, 0x75, 0x62, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, + 0x12, 0x31, 0x0a, 0x03, 0x72, 0x65, 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, + 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, + 0x2e, 0x53, 0x75, 0x62, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x52, 0x65, 0x66, 0x52, 0x03, + 0x72, 0x65, 0x66, 0x12, 0x31, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, + 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x44, 0x61, 0x74, 0x61, + 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x3d, 0x0a, 0x0b, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, + 0x6c, 0x5f, 0x72, 0x65, 0x66, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x72, + 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x43, + 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x52, 0x65, 0x66, 0x52, 0x0a, 0x63, 0x68, 0x61, 0x6e, 0x6e, + 0x65, 0x6c, 0x52, 0x65, 0x66, 0x12, 0x46, 0x0a, 0x0e, 0x73, 0x75, 0x62, 0x63, 0x68, 0x61, 0x6e, + 0x6e, 0x65, 0x6c, 0x5f, 0x72, 0x65, 0x66, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, + 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, + 0x2e, 0x53, 0x75, 0x62, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x52, 0x65, 0x66, 0x52, 0x0d, + 0x73, 0x75, 0x62, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x52, 0x65, 0x66, 0x12, 0x3a, 0x0a, + 0x0a, 0x73, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x5f, 0x72, 0x65, 0x66, 0x18, 0x05, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, + 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x65, 0x66, 0x52, 0x09, + 0x73, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x65, 0x66, 0x22, 0xc2, 0x01, 0x0a, 0x18, 0x43, 0x68, + 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x76, 0x69, 0x74, + 0x79, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x46, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x30, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, + 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, + 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x76, 0x69, 0x74, 0x79, 0x53, 0x74, 0x61, 0x74, + 0x65, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x22, 0x5e, + 0x0a, 0x05, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, + 0x57, 0x4e, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x49, 0x44, 0x4c, 0x45, 0x10, 0x01, 0x12, 0x0e, + 0x0a, 0x0a, 0x43, 0x4f, 0x4e, 0x4e, 0x45, 0x43, 0x54, 0x49, 0x4e, 0x47, 0x10, 0x02, 0x12, 0x09, + 0x0a, 0x05, 0x52, 0x45, 0x41, 0x44, 0x59, 0x10, 0x03, 0x12, 0x15, 0x0a, 0x11, 0x54, 0x52, 0x41, + 0x4e, 0x53, 0x49, 0x45, 0x4e, 0x54, 0x5f, 0x46, 0x41, 0x49, 0x4c, 0x55, 0x52, 0x45, 0x10, 0x04, + 0x12, 0x0c, 0x0a, 0x08, 0x53, 0x48, 0x55, 0x54, 0x44, 0x4f, 0x57, 0x4e, 0x10, 0x05, 0x22, 0xe9, + 0x02, 0x0a, 0x0b, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x44, 0x61, 0x74, 0x61, 0x12, 0x40, + 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, + 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, + 0x2e, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, + 0x76, 0x69, 0x74, 0x79, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, + 0x12, 0x16, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x34, 0x0a, 0x05, 0x74, 0x72, 0x61, 0x63, + 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, + 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x68, 0x61, 0x6e, 0x6e, + 0x65, 0x6c, 0x54, 0x72, 0x61, 0x63, 0x65, 0x52, 0x05, 0x74, 0x72, 0x61, 0x63, 0x65, 0x12, 0x23, + 0x0a, 0x0d, 0x63, 0x61, 0x6c, 0x6c, 0x73, 0x5f, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0c, 0x63, 0x61, 0x6c, 0x6c, 0x73, 0x53, 0x74, 0x61, 0x72, + 0x74, 0x65, 0x64, 0x12, 0x27, 0x0a, 0x0f, 0x63, 0x61, 0x6c, 0x6c, 0x73, 0x5f, 0x73, 0x75, 0x63, + 0x63, 0x65, 0x65, 0x64, 0x65, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0e, 0x63, 0x61, + 0x6c, 0x6c, 0x73, 0x53, 0x75, 0x63, 0x63, 0x65, 0x65, 0x64, 0x65, 0x64, 0x12, 0x21, 0x0a, 0x0c, + 0x63, 0x61, 0x6c, 0x6c, 0x73, 0x5f, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x18, 0x06, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x0b, 0x63, 0x61, 0x6c, 0x6c, 0x73, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x12, + 0x59, 0x0a, 0x1b, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x63, 0x61, 0x6c, 0x6c, 0x5f, 0x73, 0x74, 0x61, + 0x72, 0x74, 0x65, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x07, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, + 0x52, 0x18, 0x6c, 0x61, 0x73, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x53, 0x74, 0x61, 0x72, 0x74, 0x65, + 0x64, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x22, 0x98, 0x03, 0x0a, 0x11, 0x43, + 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x54, 0x72, 0x61, 0x63, 0x65, 0x45, 0x76, 0x65, 0x6e, 0x74, + 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, + 0x6f, 0x6e, 0x12, 0x48, 0x0a, 0x08, 0x73, 0x65, 0x76, 0x65, 0x72, 0x69, 0x74, 0x79, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, + 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x54, + 0x72, 0x61, 0x63, 0x65, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x2e, 0x53, 0x65, 0x76, 0x65, 0x72, 0x69, + 0x74, 0x79, 0x52, 0x08, 0x73, 0x65, 0x76, 0x65, 0x72, 0x69, 0x74, 0x79, 0x12, 0x38, 0x0a, 0x09, + 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x74, 0x69, 0x6d, + 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x3f, 0x0a, 0x0b, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, + 0x6c, 0x5f, 0x72, 0x65, 0x66, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x72, + 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x43, + 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x52, 0x65, 0x66, 0x48, 0x00, 0x52, 0x0a, 0x63, 0x68, 0x61, + 0x6e, 0x6e, 0x65, 0x6c, 0x52, 0x65, 0x66, 0x12, 0x48, 0x0a, 0x0e, 0x73, 0x75, 0x62, 0x63, 0x68, + 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x5f, 0x72, 0x65, 0x66, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x1f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, + 0x76, 0x31, 0x2e, 0x53, 0x75, 0x62, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x52, 0x65, 0x66, + 0x48, 0x00, 0x52, 0x0d, 0x73, 0x75, 0x62, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x52, 0x65, + 0x66, 0x22, 0x45, 0x0a, 0x08, 0x53, 0x65, 0x76, 0x65, 0x72, 0x69, 0x74, 0x79, 0x12, 0x0e, 0x0a, + 0x0a, 0x43, 0x54, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x0b, 0x0a, + 0x07, 0x43, 0x54, 0x5f, 0x49, 0x4e, 0x46, 0x4f, 0x10, 0x01, 0x12, 0x0e, 0x0a, 0x0a, 0x43, 0x54, + 0x5f, 0x57, 0x41, 0x52, 0x4e, 0x49, 0x4e, 0x47, 0x10, 0x02, 0x12, 0x0c, 0x0a, 0x08, 0x43, 0x54, + 0x5f, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x03, 0x42, 0x0b, 0x0a, 0x09, 0x63, 0x68, 0x69, 0x6c, + 0x64, 0x5f, 0x72, 0x65, 0x66, 0x22, 0xc2, 0x01, 0x0a, 0x0c, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, + 0x6c, 0x54, 0x72, 0x61, 0x63, 0x65, 0x12, 0x2a, 0x0a, 0x11, 0x6e, 0x75, 0x6d, 0x5f, 0x65, 0x76, + 0x65, 0x6e, 0x74, 0x73, 0x5f, 0x6c, 0x6f, 0x67, 0x67, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x03, 0x52, 0x0f, 0x6e, 0x75, 0x6d, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x4c, 0x6f, 0x67, 0x67, + 0x65, 0x64, 0x12, 0x49, 0x0a, 0x12, 0x63, 0x72, 0x65, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, + 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, + 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x11, 0x63, 0x72, 0x65, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x3b, 0x0a, + 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x23, 0x2e, + 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, + 0x2e, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x54, 0x72, 0x61, 0x63, 0x65, 0x45, 0x76, 0x65, + 0x6e, 0x74, 0x52, 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x22, 0x63, 0x0a, 0x0a, 0x43, 0x68, + 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x52, 0x65, 0x66, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x68, 0x61, 0x6e, + 0x6e, 0x65, 0x6c, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x63, 0x68, + 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x4a, 0x04, 0x08, 0x03, 0x10, + 0x04, 0x4a, 0x04, 0x08, 0x04, 0x10, 0x05, 0x4a, 0x04, 0x08, 0x05, 0x10, 0x06, 0x4a, 0x04, 0x08, + 0x06, 0x10, 0x07, 0x4a, 0x04, 0x08, 0x07, 0x10, 0x08, 0x4a, 0x04, 0x08, 0x08, 0x10, 0x09, 0x22, + 0x6c, 0x0a, 0x0d, 0x53, 0x75, 0x62, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x52, 0x65, 0x66, + 0x12, 0x23, 0x0a, 0x0d, 0x73, 0x75, 0x62, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x5f, 0x69, + 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0c, 0x73, 0x75, 0x62, 0x63, 0x68, 0x61, 0x6e, + 0x6e, 0x65, 0x6c, 0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x08, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x4a, 0x04, 0x08, 0x01, 0x10, 0x02, 0x4a, + 0x04, 0x08, 0x02, 0x10, 0x03, 0x4a, 0x04, 0x08, 0x03, 0x10, 0x04, 0x4a, 0x04, 0x08, 0x04, 0x10, + 0x05, 0x4a, 0x04, 0x08, 0x05, 0x10, 0x06, 0x4a, 0x04, 0x08, 0x06, 0x10, 0x07, 0x22, 0x60, 0x0a, + 0x09, 0x53, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x65, 0x66, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x6f, + 0x63, 0x6b, 0x65, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x73, + 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x4a, 0x04, 0x08, 0x01, 0x10, + 0x02, 0x4a, 0x04, 0x08, 0x02, 0x10, 0x03, 0x4a, 0x04, 0x08, 0x05, 0x10, 0x06, 0x4a, 0x04, 0x08, + 0x06, 0x10, 0x07, 0x4a, 0x04, 0x08, 0x07, 0x10, 0x08, 0x4a, 0x04, 0x08, 0x08, 0x10, 0x09, 0x22, + 0x60, 0x0a, 0x09, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x65, 0x66, 0x12, 0x1b, 0x0a, 0x09, + 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, + 0x08, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, + 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x4a, 0x04, 0x08, + 0x01, 0x10, 0x02, 0x4a, 0x04, 0x08, 0x02, 0x10, 0x03, 0x4a, 0x04, 0x08, 0x03, 0x10, 0x04, 0x4a, + 0x04, 0x08, 0x04, 0x10, 0x05, 0x4a, 0x04, 0x08, 0x07, 0x10, 0x08, 0x4a, 0x04, 0x08, 0x08, 0x10, + 0x09, 0x22, 0xab, 0x01, 0x0a, 0x06, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x2d, 0x0a, 0x03, + 0x72, 0x65, 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x72, + 0x76, 0x65, 0x72, 0x52, 0x65, 0x66, 0x52, 0x03, 0x72, 0x65, 0x66, 0x12, 0x30, 0x0a, 0x04, 0x64, + 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x72, + 0x76, 0x65, 0x72, 0x44, 0x61, 0x74, 0x61, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x40, 0x0a, + 0x0d, 0x6c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x5f, 0x73, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x18, 0x03, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, + 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x65, + 0x66, 0x52, 0x0c, 0x6c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x53, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x22, + 0x8e, 0x02, 0x0a, 0x0a, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x44, 0x61, 0x74, 0x61, 0x12, 0x34, + 0x0a, 0x05, 0x74, 0x72, 0x61, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, + 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, + 0x2e, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x54, 0x72, 0x61, 0x63, 0x65, 0x52, 0x05, 0x74, + 0x72, 0x61, 0x63, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x63, 0x61, 0x6c, 0x6c, 0x73, 0x5f, 0x73, 0x74, + 0x61, 0x72, 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0c, 0x63, 0x61, 0x6c, + 0x6c, 0x73, 0x53, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x12, 0x27, 0x0a, 0x0f, 0x63, 0x61, 0x6c, + 0x6c, 0x73, 0x5f, 0x73, 0x75, 0x63, 0x63, 0x65, 0x65, 0x64, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x0e, 0x63, 0x61, 0x6c, 0x6c, 0x73, 0x53, 0x75, 0x63, 0x63, 0x65, 0x65, 0x64, + 0x65, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x61, 0x6c, 0x6c, 0x73, 0x5f, 0x66, 0x61, 0x69, 0x6c, + 0x65, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x63, 0x61, 0x6c, 0x6c, 0x73, 0x46, + 0x61, 0x69, 0x6c, 0x65, 0x64, 0x12, 0x59, 0x0a, 0x1b, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x63, 0x61, + 0x6c, 0x6c, 0x5f, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x73, + 0x74, 0x61, 0x6d, 0x70, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, + 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x18, 0x6c, 0x61, 0x73, 0x74, 0x43, 0x61, 0x6c, 0x6c, + 0x53, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, + 0x22, 0xa6, 0x02, 0x0a, 0x06, 0x53, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x2d, 0x0a, 0x03, 0x72, + 0x65, 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, + 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6f, 0x63, 0x6b, + 0x65, 0x74, 0x52, 0x65, 0x66, 0x52, 0x03, 0x72, 0x65, 0x66, 0x12, 0x30, 0x0a, 0x04, 0x64, 0x61, + 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, + 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6f, 0x63, 0x6b, + 0x65, 0x74, 0x44, 0x61, 0x74, 0x61, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x2f, 0x0a, 0x05, + 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x72, + 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x41, + 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x52, 0x05, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x12, 0x31, 0x0a, + 0x06, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, + 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, + 0x2e, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x52, 0x06, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, + 0x12, 0x36, 0x0a, 0x08, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x18, 0x05, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, + 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x52, 0x08, + 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x12, 0x1f, 0x0a, 0x0b, 0x72, 0x65, 0x6d, 0x6f, + 0x74, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x72, + 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x83, 0x07, 0x0a, 0x0a, 0x53, 0x6f, + 0x63, 0x6b, 0x65, 0x74, 0x44, 0x61, 0x74, 0x61, 0x12, 0x27, 0x0a, 0x0f, 0x73, 0x74, 0x72, 0x65, + 0x61, 0x6d, 0x73, 0x5f, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x03, 0x52, 0x0e, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x73, 0x53, 0x74, 0x61, 0x72, 0x74, 0x65, + 0x64, 0x12, 0x2b, 0x0a, 0x11, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x73, 0x5f, 0x73, 0x75, 0x63, + 0x63, 0x65, 0x65, 0x64, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x10, 0x73, 0x74, + 0x72, 0x65, 0x61, 0x6d, 0x73, 0x53, 0x75, 0x63, 0x63, 0x65, 0x65, 0x64, 0x65, 0x64, 0x12, 0x25, + 0x0a, 0x0e, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x73, 0x5f, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x73, 0x46, + 0x61, 0x69, 0x6c, 0x65, 0x64, 0x12, 0x23, 0x0a, 0x0d, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, + 0x73, 0x5f, 0x73, 0x65, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0c, 0x6d, 0x65, + 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x53, 0x65, 0x6e, 0x74, 0x12, 0x2b, 0x0a, 0x11, 0x6d, 0x65, + 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x5f, 0x72, 0x65, 0x63, 0x65, 0x69, 0x76, 0x65, 0x64, 0x18, + 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x10, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x52, + 0x65, 0x63, 0x65, 0x69, 0x76, 0x65, 0x64, 0x12, 0x28, 0x0a, 0x10, 0x6b, 0x65, 0x65, 0x70, 0x5f, + 0x61, 0x6c, 0x69, 0x76, 0x65, 0x73, 0x5f, 0x73, 0x65, 0x6e, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, + 0x03, 0x52, 0x0e, 0x6b, 0x65, 0x65, 0x70, 0x41, 0x6c, 0x69, 0x76, 0x65, 0x73, 0x53, 0x65, 0x6e, + 0x74, 0x12, 0x68, 0x0a, 0x23, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x5f, + 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x5f, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x74, + 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, + 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x1f, 0x6c, 0x61, 0x73, 0x74, + 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, 0x72, 0x65, 0x61, 0x74, + 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x6a, 0x0a, 0x24, 0x6c, + 0x61, 0x73, 0x74, 0x5f, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x73, 0x74, 0x72, 0x65, 0x61, + 0x6d, 0x5f, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, + 0x61, 0x6d, 0x70, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, + 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x20, 0x6c, 0x61, 0x73, 0x74, 0x52, 0x65, 0x6d, 0x6f, 0x74, + 0x65, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x54, 0x69, + 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x59, 0x0a, 0x1b, 0x6c, 0x61, 0x73, 0x74, 0x5f, + 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x5f, 0x73, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, + 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, + 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x18, 0x6c, 0x61, 0x73, 0x74, 0x4d, 0x65, + 0x73, 0x73, 0x61, 0x67, 0x65, 0x53, 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, + 0x6d, 0x70, 0x12, 0x61, 0x0a, 0x1f, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, + 0x67, 0x65, 0x5f, 0x72, 0x65, 0x63, 0x65, 0x69, 0x76, 0x65, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, + 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, + 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x1c, 0x6c, 0x61, 0x73, 0x74, 0x4d, 0x65, 0x73, + 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x63, 0x65, 0x69, 0x76, 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, + 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x56, 0x0a, 0x19, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x66, + 0x6c, 0x6f, 0x77, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x5f, 0x77, 0x69, 0x6e, 0x64, + 0x6f, 0x77, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x49, 0x6e, 0x74, 0x36, 0x34, + 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x16, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x46, 0x6c, 0x6f, 0x77, + 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x12, 0x58, 0x0a, + 0x1a, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x66, 0x6c, 0x6f, 0x77, 0x5f, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x5f, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x18, 0x0c, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x49, 0x6e, 0x74, 0x36, 0x34, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x17, + 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x46, 0x6c, 0x6f, 0x77, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, + 0x6c, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x12, 0x36, 0x0a, 0x06, 0x6f, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x18, 0x0d, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, + 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6f, 0x63, 0x6b, 0x65, + 0x74, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x06, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x22, + 0xb8, 0x03, 0x0a, 0x07, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x4d, 0x0a, 0x0d, 0x74, + 0x63, 0x70, 0x69, 0x70, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, + 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x2e, 0x54, 0x63, + 0x70, 0x49, 0x70, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x48, 0x00, 0x52, 0x0c, 0x74, 0x63, + 0x70, 0x69, 0x70, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x47, 0x0a, 0x0b, 0x75, 0x64, + 0x73, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x24, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, + 0x76, 0x31, 0x2e, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x2e, 0x55, 0x64, 0x73, 0x41, 0x64, + 0x64, 0x72, 0x65, 0x73, 0x73, 0x48, 0x00, 0x52, 0x0a, 0x75, 0x64, 0x73, 0x41, 0x64, 0x64, 0x72, + 0x65, 0x73, 0x73, 0x12, 0x4d, 0x0a, 0x0d, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x5f, 0x61, 0x64, 0x64, + 0x72, 0x65, 0x73, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x67, 0x72, 0x70, + 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x64, + 0x64, 0x72, 0x65, 0x73, 0x73, 0x2e, 0x4f, 0x74, 0x68, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x65, + 0x73, 0x73, 0x48, 0x00, 0x52, 0x0c, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x65, + 0x73, 0x73, 0x1a, 0x41, 0x0a, 0x0c, 0x54, 0x63, 0x70, 0x49, 0x70, 0x41, 0x64, 0x64, 0x72, 0x65, + 0x73, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x69, 0x70, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x69, 0x70, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, + 0x73, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, + 0x04, 0x70, 0x6f, 0x72, 0x74, 0x1a, 0x28, 0x0a, 0x0a, 0x55, 0x64, 0x73, 0x41, 0x64, 0x64, 0x72, + 0x65, 0x73, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x1a, + 0x4e, 0x0a, 0x0c, 0x4f, 0x74, 0x68, 0x65, 0x72, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, + 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, + 0x61, 0x6d, 0x65, 0x12, 0x2a, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x41, 0x6e, 0x79, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x42, + 0x09, 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x22, 0x96, 0x03, 0x0a, 0x08, 0x53, + 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x12, 0x32, 0x0a, 0x03, 0x74, 0x6c, 0x73, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, + 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, + 0x2e, 0x54, 0x6c, 0x73, 0x48, 0x00, 0x52, 0x03, 0x74, 0x6c, 0x73, 0x12, 0x40, 0x0a, 0x05, 0x6f, + 0x74, 0x68, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x67, 0x72, 0x70, + 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, + 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2e, 0x4f, 0x74, 0x68, 0x65, 0x72, 0x53, 0x65, 0x63, 0x75, + 0x72, 0x69, 0x74, 0x79, 0x48, 0x00, 0x52, 0x05, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x1a, 0xb9, 0x01, + 0x0a, 0x03, 0x54, 0x6c, 0x73, 0x12, 0x25, 0x0a, 0x0d, 0x73, 0x74, 0x61, 0x6e, 0x64, 0x61, 0x72, + 0x64, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0c, + 0x73, 0x74, 0x61, 0x6e, 0x64, 0x61, 0x72, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1f, 0x0a, 0x0a, + 0x6f, 0x74, 0x68, 0x65, 0x72, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x48, 0x00, 0x52, 0x09, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2b, 0x0a, + 0x11, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x63, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, + 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x10, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x43, + 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x12, 0x2d, 0x0a, 0x12, 0x72, 0x65, + 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x63, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x11, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x43, 0x65, + 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x42, 0x0e, 0x0a, 0x0c, 0x63, 0x69, 0x70, + 0x68, 0x65, 0x72, 0x5f, 0x73, 0x75, 0x69, 0x74, 0x65, 0x1a, 0x4f, 0x0a, 0x0d, 0x4f, 0x74, 0x68, + 0x65, 0x72, 0x53, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, + 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x2a, + 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, + 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, + 0x41, 0x6e, 0x79, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x42, 0x07, 0x0a, 0x05, 0x6d, 0x6f, + 0x64, 0x65, 0x6c, 0x22, 0x6e, 0x0a, 0x0c, 0x53, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x4f, 0x70, 0x74, + 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x34, 0x0a, + 0x0a, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x14, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x41, 0x6e, 0x79, 0x52, 0x0a, 0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6f, + 0x6e, 0x61, 0x6c, 0x22, 0x4c, 0x0a, 0x13, 0x53, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x4f, 0x70, 0x74, + 0x69, 0x6f, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x12, 0x35, 0x0a, 0x08, 0x64, 0x75, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, + 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x22, 0x63, 0x0a, 0x12, 0x53, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x4f, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x4c, 0x69, 0x6e, 0x67, 0x65, 0x72, 0x12, 0x16, 0x0a, 0x06, 0x61, 0x63, 0x74, 0x69, 0x76, + 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x12, + 0x35, 0x0a, 0x08, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x08, 0x64, 0x75, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0xb2, 0x08, 0x0a, 0x13, 0x53, 0x6f, 0x63, 0x6b, 0x65, + 0x74, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x63, 0x70, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x1d, + 0x0a, 0x0a, 0x74, 0x63, 0x70, 0x69, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0d, 0x52, 0x09, 0x74, 0x63, 0x70, 0x69, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x22, 0x0a, + 0x0d, 0x74, 0x63, 0x70, 0x69, 0x5f, 0x63, 0x61, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x74, 0x63, 0x70, 0x69, 0x43, 0x61, 0x53, 0x74, 0x61, 0x74, + 0x65, 0x12, 0x29, 0x0a, 0x10, 0x74, 0x63, 0x70, 0x69, 0x5f, 0x72, 0x65, 0x74, 0x72, 0x61, 0x6e, + 0x73, 0x6d, 0x69, 0x74, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0f, 0x74, 0x63, 0x70, + 0x69, 0x52, 0x65, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x6d, 0x69, 0x74, 0x73, 0x12, 0x1f, 0x0a, 0x0b, + 0x74, 0x63, 0x70, 0x69, 0x5f, 0x70, 0x72, 0x6f, 0x62, 0x65, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x0d, 0x52, 0x0a, 0x74, 0x63, 0x70, 0x69, 0x50, 0x72, 0x6f, 0x62, 0x65, 0x73, 0x12, 0x21, 0x0a, + 0x0c, 0x74, 0x63, 0x70, 0x69, 0x5f, 0x62, 0x61, 0x63, 0x6b, 0x6f, 0x66, 0x66, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x74, 0x63, 0x70, 0x69, 0x42, 0x61, 0x63, 0x6b, 0x6f, 0x66, 0x66, + 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x63, 0x70, 0x69, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x18, 0x06, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x74, 0x63, 0x70, 0x69, 0x4f, 0x70, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x12, 0x26, 0x0a, 0x0f, 0x74, 0x63, 0x70, 0x69, 0x5f, 0x73, 0x6e, 0x64, 0x5f, + 0x77, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0d, 0x74, 0x63, + 0x70, 0x69, 0x53, 0x6e, 0x64, 0x57, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x12, 0x26, 0x0a, 0x0f, 0x74, + 0x63, 0x70, 0x69, 0x5f, 0x72, 0x63, 0x76, 0x5f, 0x77, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x18, 0x08, + 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0d, 0x74, 0x63, 0x70, 0x69, 0x52, 0x63, 0x76, 0x57, 0x73, 0x63, + 0x61, 0x6c, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x74, 0x63, 0x70, 0x69, 0x5f, 0x72, 0x74, 0x6f, 0x18, + 0x09, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x07, 0x74, 0x63, 0x70, 0x69, 0x52, 0x74, 0x6f, 0x12, 0x19, + 0x0a, 0x08, 0x74, 0x63, 0x70, 0x69, 0x5f, 0x61, 0x74, 0x6f, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0d, + 0x52, 0x07, 0x74, 0x63, 0x70, 0x69, 0x41, 0x74, 0x6f, 0x12, 0x20, 0x0a, 0x0c, 0x74, 0x63, 0x70, + 0x69, 0x5f, 0x73, 0x6e, 0x64, 0x5f, 0x6d, 0x73, 0x73, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0d, 0x52, + 0x0a, 0x74, 0x63, 0x70, 0x69, 0x53, 0x6e, 0x64, 0x4d, 0x73, 0x73, 0x12, 0x20, 0x0a, 0x0c, 0x74, + 0x63, 0x70, 0x69, 0x5f, 0x72, 0x63, 0x76, 0x5f, 0x6d, 0x73, 0x73, 0x18, 0x0c, 0x20, 0x01, 0x28, + 0x0d, 0x52, 0x0a, 0x74, 0x63, 0x70, 0x69, 0x52, 0x63, 0x76, 0x4d, 0x73, 0x73, 0x12, 0x21, 0x0a, + 0x0c, 0x74, 0x63, 0x70, 0x69, 0x5f, 0x75, 0x6e, 0x61, 0x63, 0x6b, 0x65, 0x64, 0x18, 0x0d, 0x20, + 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x74, 0x63, 0x70, 0x69, 0x55, 0x6e, 0x61, 0x63, 0x6b, 0x65, 0x64, + 0x12, 0x1f, 0x0a, 0x0b, 0x74, 0x63, 0x70, 0x69, 0x5f, 0x73, 0x61, 0x63, 0x6b, 0x65, 0x64, 0x18, + 0x0e, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0a, 0x74, 0x63, 0x70, 0x69, 0x53, 0x61, 0x63, 0x6b, 0x65, + 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x63, 0x70, 0x69, 0x5f, 0x6c, 0x6f, 0x73, 0x74, 0x18, 0x0f, + 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x74, 0x63, 0x70, 0x69, 0x4c, 0x6f, 0x73, 0x74, 0x12, 0x21, + 0x0a, 0x0c, 0x74, 0x63, 0x70, 0x69, 0x5f, 0x72, 0x65, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x18, 0x10, + 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x74, 0x63, 0x70, 0x69, 0x52, 0x65, 0x74, 0x72, 0x61, 0x6e, + 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x63, 0x70, 0x69, 0x5f, 0x66, 0x61, 0x63, 0x6b, 0x65, 0x74, + 0x73, 0x18, 0x11, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x74, 0x63, 0x70, 0x69, 0x46, 0x61, 0x63, + 0x6b, 0x65, 0x74, 0x73, 0x12, 0x2d, 0x0a, 0x13, 0x74, 0x63, 0x70, 0x69, 0x5f, 0x6c, 0x61, 0x73, + 0x74, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x73, 0x65, 0x6e, 0x74, 0x18, 0x12, 0x20, 0x01, 0x28, + 0x0d, 0x52, 0x10, 0x74, 0x63, 0x70, 0x69, 0x4c, 0x61, 0x73, 0x74, 0x44, 0x61, 0x74, 0x61, 0x53, + 0x65, 0x6e, 0x74, 0x12, 0x2b, 0x0a, 0x12, 0x74, 0x63, 0x70, 0x69, 0x5f, 0x6c, 0x61, 0x73, 0x74, + 0x5f, 0x61, 0x63, 0x6b, 0x5f, 0x73, 0x65, 0x6e, 0x74, 0x18, 0x13, 0x20, 0x01, 0x28, 0x0d, 0x52, + 0x0f, 0x74, 0x63, 0x70, 0x69, 0x4c, 0x61, 0x73, 0x74, 0x41, 0x63, 0x6b, 0x53, 0x65, 0x6e, 0x74, + 0x12, 0x2d, 0x0a, 0x13, 0x74, 0x63, 0x70, 0x69, 0x5f, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x64, 0x61, + 0x74, 0x61, 0x5f, 0x72, 0x65, 0x63, 0x76, 0x18, 0x14, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x10, 0x74, + 0x63, 0x70, 0x69, 0x4c, 0x61, 0x73, 0x74, 0x44, 0x61, 0x74, 0x61, 0x52, 0x65, 0x63, 0x76, 0x12, + 0x2b, 0x0a, 0x12, 0x74, 0x63, 0x70, 0x69, 0x5f, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x61, 0x63, 0x6b, + 0x5f, 0x72, 0x65, 0x63, 0x76, 0x18, 0x15, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0f, 0x74, 0x63, 0x70, + 0x69, 0x4c, 0x61, 0x73, 0x74, 0x41, 0x63, 0x6b, 0x52, 0x65, 0x63, 0x76, 0x12, 0x1b, 0x0a, 0x09, + 0x74, 0x63, 0x70, 0x69, 0x5f, 0x70, 0x6d, 0x74, 0x75, 0x18, 0x16, 0x20, 0x01, 0x28, 0x0d, 0x52, + 0x08, 0x74, 0x63, 0x70, 0x69, 0x50, 0x6d, 0x74, 0x75, 0x12, 0x2a, 0x0a, 0x11, 0x74, 0x63, 0x70, + 0x69, 0x5f, 0x72, 0x63, 0x76, 0x5f, 0x73, 0x73, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x18, 0x17, + 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0f, 0x74, 0x63, 0x70, 0x69, 0x52, 0x63, 0x76, 0x53, 0x73, 0x74, + 0x68, 0x72, 0x65, 0x73, 0x68, 0x12, 0x19, 0x0a, 0x08, 0x74, 0x63, 0x70, 0x69, 0x5f, 0x72, 0x74, + 0x74, 0x18, 0x18, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x07, 0x74, 0x63, 0x70, 0x69, 0x52, 0x74, 0x74, + 0x12, 0x1f, 0x0a, 0x0b, 0x74, 0x63, 0x70, 0x69, 0x5f, 0x72, 0x74, 0x74, 0x76, 0x61, 0x72, 0x18, + 0x19, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0a, 0x74, 0x63, 0x70, 0x69, 0x52, 0x74, 0x74, 0x76, 0x61, + 0x72, 0x12, 0x2a, 0x0a, 0x11, 0x74, 0x63, 0x70, 0x69, 0x5f, 0x73, 0x6e, 0x64, 0x5f, 0x73, 0x73, + 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x18, 0x1a, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0f, 0x74, 0x63, + 0x70, 0x69, 0x53, 0x6e, 0x64, 0x53, 0x73, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x12, 0x22, 0x0a, + 0x0d, 0x74, 0x63, 0x70, 0x69, 0x5f, 0x73, 0x6e, 0x64, 0x5f, 0x63, 0x77, 0x6e, 0x64, 0x18, 0x1b, + 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x74, 0x63, 0x70, 0x69, 0x53, 0x6e, 0x64, 0x43, 0x77, 0x6e, + 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x74, 0x63, 0x70, 0x69, 0x5f, 0x61, 0x64, 0x76, 0x6d, 0x73, 0x73, + 0x18, 0x1c, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0a, 0x74, 0x63, 0x70, 0x69, 0x41, 0x64, 0x76, 0x6d, + 0x73, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x74, 0x63, 0x70, 0x69, 0x5f, 0x72, 0x65, 0x6f, 0x72, 0x64, + 0x65, 0x72, 0x69, 0x6e, 0x67, 0x18, 0x1d, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0e, 0x74, 0x63, 0x70, + 0x69, 0x52, 0x65, 0x6f, 0x72, 0x64, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x22, 0x62, 0x0a, 0x15, 0x47, + 0x65, 0x74, 0x54, 0x6f, 0x70, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x73, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x12, 0x28, 0x0a, 0x10, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x63, 0x68, + 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0e, + 0x73, 0x74, 0x61, 0x72, 0x74, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x49, 0x64, 0x12, 0x1f, + 0x0a, 0x0b, 0x6d, 0x61, 0x78, 0x5f, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x03, 0x52, 0x0a, 0x6d, 0x61, 0x78, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x22, + 0x5f, 0x0a, 0x16, 0x47, 0x65, 0x74, 0x54, 0x6f, 0x70, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x33, 0x0a, 0x07, 0x63, 0x68, 0x61, + 0x6e, 0x6e, 0x65, 0x6c, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x72, 0x70, + 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x68, + 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x12, 0x10, + 0x0a, 0x03, 0x65, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x03, 0x65, 0x6e, 0x64, + 0x22, 0x5c, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x26, 0x0a, 0x0f, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x73, + 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, + 0x73, 0x74, 0x61, 0x72, 0x74, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x49, 0x64, 0x12, 0x1f, 0x0a, + 0x0b, 0x6d, 0x61, 0x78, 0x5f, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x0a, 0x6d, 0x61, 0x78, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x22, 0x58, + 0x0a, 0x12, 0x47, 0x65, 0x74, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x30, 0x0a, 0x06, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x18, 0x01, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, + 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x06, + 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x10, 0x0a, 0x03, 0x65, 0x6e, 0x64, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x03, 0x65, 0x6e, 0x64, 0x22, 0x2f, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x53, + 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, + 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, + 0x08, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x49, 0x64, 0x22, 0x45, 0x0a, 0x11, 0x47, 0x65, 0x74, + 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x30, + 0x0a, 0x06, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, + 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, + 0x31, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x06, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, + 0x22, 0x7f, 0x0a, 0x17, 0x47, 0x65, 0x74, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x6f, 0x63, + 0x6b, 0x65, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x73, + 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, + 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x49, 0x64, 0x12, 0x26, 0x0a, 0x0f, 0x73, 0x74, 0x61, 0x72, + 0x74, 0x5f, 0x73, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x03, 0x52, 0x0d, 0x73, 0x74, 0x61, 0x72, 0x74, 0x53, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x49, 0x64, + 0x12, 0x1f, 0x0a, 0x0b, 0x6d, 0x61, 0x78, 0x5f, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x6d, 0x61, 0x78, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, + 0x73, 0x22, 0x68, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x6f, + 0x63, 0x6b, 0x65, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3a, 0x0a, + 0x0a, 0x73, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x5f, 0x72, 0x65, 0x66, 0x18, 0x01, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, + 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x65, 0x66, 0x52, 0x09, + 0x73, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x65, 0x66, 0x12, 0x10, 0x0a, 0x03, 0x65, 0x6e, 0x64, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x03, 0x65, 0x6e, 0x64, 0x22, 0x32, 0x0a, 0x11, 0x47, + 0x65, 0x74, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x5f, 0x69, 0x64, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x49, 0x64, 0x22, + 0x49, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x33, 0x0a, 0x07, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, + 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, + 0x6c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x22, 0x3b, 0x0a, 0x14, 0x47, 0x65, + 0x74, 0x53, 0x75, 0x62, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x73, 0x75, 0x62, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, + 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0c, 0x73, 0x75, 0x62, 0x63, 0x68, + 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x49, 0x64, 0x22, 0x55, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x53, 0x75, + 0x62, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x3c, 0x0a, 0x0a, 0x73, 0x75, 0x62, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, + 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x75, 0x62, 0x63, 0x68, 0x61, 0x6e, 0x6e, + 0x65, 0x6c, 0x52, 0x0a, 0x73, 0x75, 0x62, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x22, 0x49, + 0x0a, 0x10, 0x47, 0x65, 0x74, 0x53, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x5f, 0x69, 0x64, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x73, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x49, 0x64, 0x12, + 0x18, 0x0a, 0x07, 0x73, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x07, 0x73, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x22, 0x45, 0x0a, 0x11, 0x47, 0x65, 0x74, + 0x53, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x30, + 0x0a, 0x06, 0x73, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, + 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, + 0x31, 0x2e, 0x53, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x06, 0x73, 0x6f, 0x63, 0x6b, 0x65, 0x74, + 0x32, 0x9a, 0x05, 0x0a, 0x08, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x12, 0x63, 0x0a, + 0x0e, 0x47, 0x65, 0x74, 0x54, 0x6f, 0x70, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x73, 0x12, + 0x27, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, + 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x54, 0x6f, 0x70, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, + 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, + 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x54, + 0x6f, 0x70, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x57, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, + 0x12, 0x23, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, + 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, + 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x65, 0x72, 0x76, + 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x09, 0x47, + 0x65, 0x74, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x22, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, + 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x53, + 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x67, + 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, + 0x47, 0x65, 0x74, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x69, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x6f, + 0x63, 0x6b, 0x65, 0x74, 0x73, 0x12, 0x29, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, + 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x65, 0x72, 0x76, + 0x65, 0x72, 0x53, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x2a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, + 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x6f, 0x63, + 0x6b, 0x65, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x57, 0x0a, 0x0a, + 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x12, 0x23, 0x2e, 0x67, 0x72, 0x70, + 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, + 0x74, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x24, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, + 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x60, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x53, 0x75, 0x62, 0x63, + 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x12, 0x26, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, + 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x75, 0x62, + 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, + 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, + 0x31, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x75, 0x62, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x53, 0x6f, + 0x63, 0x6b, 0x65, 0x74, 0x12, 0x22, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, + 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x53, 0x6f, 0x63, 0x6b, 0x65, + 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, + 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x53, + 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x58, 0x0a, + 0x13, 0x69, 0x6f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, + 0x7a, 0x2e, 0x76, 0x31, 0x42, 0x0d, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x50, 0x72, + 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x30, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67, 0x6f, + 0x6c, 0x61, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x63, 0x68, + 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x7a, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x63, 0x68, 0x61, 0x6e, + 0x6e, 0x65, 0x6c, 0x7a, 0x5f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_grpc_channelz_v1_channelz_proto_rawDescOnce sync.Once + file_grpc_channelz_v1_channelz_proto_rawDescData = file_grpc_channelz_v1_channelz_proto_rawDesc +) + +func file_grpc_channelz_v1_channelz_proto_rawDescGZIP() []byte { + file_grpc_channelz_v1_channelz_proto_rawDescOnce.Do(func() { + file_grpc_channelz_v1_channelz_proto_rawDescData = protoimpl.X.CompressGZIP(file_grpc_channelz_v1_channelz_proto_rawDescData) + }) + return file_grpc_channelz_v1_channelz_proto_rawDescData +} + +var file_grpc_channelz_v1_channelz_proto_enumTypes = make([]protoimpl.EnumInfo, 2) +var file_grpc_channelz_v1_channelz_proto_msgTypes = make([]protoimpl.MessageInfo, 39) +var file_grpc_channelz_v1_channelz_proto_goTypes = []interface{}{ + (ChannelConnectivityState_State)(0), // 0: grpc.channelz.v1.ChannelConnectivityState.State + (ChannelTraceEvent_Severity)(0), // 1: grpc.channelz.v1.ChannelTraceEvent.Severity + (*Channel)(nil), // 2: grpc.channelz.v1.Channel + (*Subchannel)(nil), // 3: grpc.channelz.v1.Subchannel + (*ChannelConnectivityState)(nil), // 4: grpc.channelz.v1.ChannelConnectivityState + (*ChannelData)(nil), // 5: grpc.channelz.v1.ChannelData + (*ChannelTraceEvent)(nil), // 6: grpc.channelz.v1.ChannelTraceEvent + (*ChannelTrace)(nil), // 7: grpc.channelz.v1.ChannelTrace + (*ChannelRef)(nil), // 8: grpc.channelz.v1.ChannelRef + (*SubchannelRef)(nil), // 9: grpc.channelz.v1.SubchannelRef + (*SocketRef)(nil), // 10: grpc.channelz.v1.SocketRef + (*ServerRef)(nil), // 11: grpc.channelz.v1.ServerRef + (*Server)(nil), // 12: grpc.channelz.v1.Server + (*ServerData)(nil), // 13: grpc.channelz.v1.ServerData + (*Socket)(nil), // 14: grpc.channelz.v1.Socket + (*SocketData)(nil), // 15: grpc.channelz.v1.SocketData + (*Address)(nil), // 16: grpc.channelz.v1.Address + (*Security)(nil), // 17: grpc.channelz.v1.Security + (*SocketOption)(nil), // 18: grpc.channelz.v1.SocketOption + (*SocketOptionTimeout)(nil), // 19: grpc.channelz.v1.SocketOptionTimeout + (*SocketOptionLinger)(nil), // 20: grpc.channelz.v1.SocketOptionLinger + (*SocketOptionTcpInfo)(nil), // 21: grpc.channelz.v1.SocketOptionTcpInfo + (*GetTopChannelsRequest)(nil), // 22: grpc.channelz.v1.GetTopChannelsRequest + (*GetTopChannelsResponse)(nil), // 23: grpc.channelz.v1.GetTopChannelsResponse + (*GetServersRequest)(nil), // 24: grpc.channelz.v1.GetServersRequest + (*GetServersResponse)(nil), // 25: grpc.channelz.v1.GetServersResponse + (*GetServerRequest)(nil), // 26: grpc.channelz.v1.GetServerRequest + (*GetServerResponse)(nil), // 27: grpc.channelz.v1.GetServerResponse + (*GetServerSocketsRequest)(nil), // 28: grpc.channelz.v1.GetServerSocketsRequest + (*GetServerSocketsResponse)(nil), // 29: grpc.channelz.v1.GetServerSocketsResponse + (*GetChannelRequest)(nil), // 30: grpc.channelz.v1.GetChannelRequest + (*GetChannelResponse)(nil), // 31: grpc.channelz.v1.GetChannelResponse + (*GetSubchannelRequest)(nil), // 32: grpc.channelz.v1.GetSubchannelRequest + (*GetSubchannelResponse)(nil), // 33: grpc.channelz.v1.GetSubchannelResponse + (*GetSocketRequest)(nil), // 34: grpc.channelz.v1.GetSocketRequest + (*GetSocketResponse)(nil), // 35: grpc.channelz.v1.GetSocketResponse + (*Address_TcpIpAddress)(nil), // 36: grpc.channelz.v1.Address.TcpIpAddress + (*Address_UdsAddress)(nil), // 37: grpc.channelz.v1.Address.UdsAddress + (*Address_OtherAddress)(nil), // 38: grpc.channelz.v1.Address.OtherAddress + (*Security_Tls)(nil), // 39: grpc.channelz.v1.Security.Tls + (*Security_OtherSecurity)(nil), // 40: grpc.channelz.v1.Security.OtherSecurity + (*timestamppb.Timestamp)(nil), // 41: google.protobuf.Timestamp + (*wrapperspb.Int64Value)(nil), // 42: google.protobuf.Int64Value + (*anypb.Any)(nil), // 43: google.protobuf.Any + (*durationpb.Duration)(nil), // 44: google.protobuf.Duration +} +var file_grpc_channelz_v1_channelz_proto_depIdxs = []int32{ + 8, // 0: grpc.channelz.v1.Channel.ref:type_name -> grpc.channelz.v1.ChannelRef + 5, // 1: grpc.channelz.v1.Channel.data:type_name -> grpc.channelz.v1.ChannelData + 8, // 2: grpc.channelz.v1.Channel.channel_ref:type_name -> grpc.channelz.v1.ChannelRef + 9, // 3: grpc.channelz.v1.Channel.subchannel_ref:type_name -> grpc.channelz.v1.SubchannelRef + 10, // 4: grpc.channelz.v1.Channel.socket_ref:type_name -> grpc.channelz.v1.SocketRef + 9, // 5: grpc.channelz.v1.Subchannel.ref:type_name -> grpc.channelz.v1.SubchannelRef + 5, // 6: grpc.channelz.v1.Subchannel.data:type_name -> grpc.channelz.v1.ChannelData + 8, // 7: grpc.channelz.v1.Subchannel.channel_ref:type_name -> grpc.channelz.v1.ChannelRef + 9, // 8: grpc.channelz.v1.Subchannel.subchannel_ref:type_name -> grpc.channelz.v1.SubchannelRef + 10, // 9: grpc.channelz.v1.Subchannel.socket_ref:type_name -> grpc.channelz.v1.SocketRef + 0, // 10: grpc.channelz.v1.ChannelConnectivityState.state:type_name -> grpc.channelz.v1.ChannelConnectivityState.State + 4, // 11: grpc.channelz.v1.ChannelData.state:type_name -> grpc.channelz.v1.ChannelConnectivityState + 7, // 12: grpc.channelz.v1.ChannelData.trace:type_name -> grpc.channelz.v1.ChannelTrace + 41, // 13: grpc.channelz.v1.ChannelData.last_call_started_timestamp:type_name -> google.protobuf.Timestamp + 1, // 14: grpc.channelz.v1.ChannelTraceEvent.severity:type_name -> grpc.channelz.v1.ChannelTraceEvent.Severity + 41, // 15: grpc.channelz.v1.ChannelTraceEvent.timestamp:type_name -> google.protobuf.Timestamp + 8, // 16: grpc.channelz.v1.ChannelTraceEvent.channel_ref:type_name -> grpc.channelz.v1.ChannelRef + 9, // 17: grpc.channelz.v1.ChannelTraceEvent.subchannel_ref:type_name -> grpc.channelz.v1.SubchannelRef + 41, // 18: grpc.channelz.v1.ChannelTrace.creation_timestamp:type_name -> google.protobuf.Timestamp + 6, // 19: grpc.channelz.v1.ChannelTrace.events:type_name -> grpc.channelz.v1.ChannelTraceEvent + 11, // 20: grpc.channelz.v1.Server.ref:type_name -> grpc.channelz.v1.ServerRef + 13, // 21: grpc.channelz.v1.Server.data:type_name -> grpc.channelz.v1.ServerData + 10, // 22: grpc.channelz.v1.Server.listen_socket:type_name -> grpc.channelz.v1.SocketRef + 7, // 23: grpc.channelz.v1.ServerData.trace:type_name -> grpc.channelz.v1.ChannelTrace + 41, // 24: grpc.channelz.v1.ServerData.last_call_started_timestamp:type_name -> google.protobuf.Timestamp + 10, // 25: grpc.channelz.v1.Socket.ref:type_name -> grpc.channelz.v1.SocketRef + 15, // 26: grpc.channelz.v1.Socket.data:type_name -> grpc.channelz.v1.SocketData + 16, // 27: grpc.channelz.v1.Socket.local:type_name -> grpc.channelz.v1.Address + 16, // 28: grpc.channelz.v1.Socket.remote:type_name -> grpc.channelz.v1.Address + 17, // 29: grpc.channelz.v1.Socket.security:type_name -> grpc.channelz.v1.Security + 41, // 30: grpc.channelz.v1.SocketData.last_local_stream_created_timestamp:type_name -> google.protobuf.Timestamp + 41, // 31: grpc.channelz.v1.SocketData.last_remote_stream_created_timestamp:type_name -> google.protobuf.Timestamp + 41, // 32: grpc.channelz.v1.SocketData.last_message_sent_timestamp:type_name -> google.protobuf.Timestamp + 41, // 33: grpc.channelz.v1.SocketData.last_message_received_timestamp:type_name -> google.protobuf.Timestamp + 42, // 34: grpc.channelz.v1.SocketData.local_flow_control_window:type_name -> google.protobuf.Int64Value + 42, // 35: grpc.channelz.v1.SocketData.remote_flow_control_window:type_name -> google.protobuf.Int64Value + 18, // 36: grpc.channelz.v1.SocketData.option:type_name -> grpc.channelz.v1.SocketOption + 36, // 37: grpc.channelz.v1.Address.tcpip_address:type_name -> grpc.channelz.v1.Address.TcpIpAddress + 37, // 38: grpc.channelz.v1.Address.uds_address:type_name -> grpc.channelz.v1.Address.UdsAddress + 38, // 39: grpc.channelz.v1.Address.other_address:type_name -> grpc.channelz.v1.Address.OtherAddress + 39, // 40: grpc.channelz.v1.Security.tls:type_name -> grpc.channelz.v1.Security.Tls + 40, // 41: grpc.channelz.v1.Security.other:type_name -> grpc.channelz.v1.Security.OtherSecurity + 43, // 42: grpc.channelz.v1.SocketOption.additional:type_name -> google.protobuf.Any + 44, // 43: grpc.channelz.v1.SocketOptionTimeout.duration:type_name -> google.protobuf.Duration + 44, // 44: grpc.channelz.v1.SocketOptionLinger.duration:type_name -> google.protobuf.Duration + 2, // 45: grpc.channelz.v1.GetTopChannelsResponse.channel:type_name -> grpc.channelz.v1.Channel + 12, // 46: grpc.channelz.v1.GetServersResponse.server:type_name -> grpc.channelz.v1.Server + 12, // 47: grpc.channelz.v1.GetServerResponse.server:type_name -> grpc.channelz.v1.Server + 10, // 48: grpc.channelz.v1.GetServerSocketsResponse.socket_ref:type_name -> grpc.channelz.v1.SocketRef + 2, // 49: grpc.channelz.v1.GetChannelResponse.channel:type_name -> grpc.channelz.v1.Channel + 3, // 50: grpc.channelz.v1.GetSubchannelResponse.subchannel:type_name -> grpc.channelz.v1.Subchannel + 14, // 51: grpc.channelz.v1.GetSocketResponse.socket:type_name -> grpc.channelz.v1.Socket + 43, // 52: grpc.channelz.v1.Address.OtherAddress.value:type_name -> google.protobuf.Any + 43, // 53: grpc.channelz.v1.Security.OtherSecurity.value:type_name -> google.protobuf.Any + 22, // 54: grpc.channelz.v1.Channelz.GetTopChannels:input_type -> grpc.channelz.v1.GetTopChannelsRequest + 24, // 55: grpc.channelz.v1.Channelz.GetServers:input_type -> grpc.channelz.v1.GetServersRequest + 26, // 56: grpc.channelz.v1.Channelz.GetServer:input_type -> grpc.channelz.v1.GetServerRequest + 28, // 57: grpc.channelz.v1.Channelz.GetServerSockets:input_type -> grpc.channelz.v1.GetServerSocketsRequest + 30, // 58: grpc.channelz.v1.Channelz.GetChannel:input_type -> grpc.channelz.v1.GetChannelRequest + 32, // 59: grpc.channelz.v1.Channelz.GetSubchannel:input_type -> grpc.channelz.v1.GetSubchannelRequest + 34, // 60: grpc.channelz.v1.Channelz.GetSocket:input_type -> grpc.channelz.v1.GetSocketRequest + 23, // 61: grpc.channelz.v1.Channelz.GetTopChannels:output_type -> grpc.channelz.v1.GetTopChannelsResponse + 25, // 62: grpc.channelz.v1.Channelz.GetServers:output_type -> grpc.channelz.v1.GetServersResponse + 27, // 63: grpc.channelz.v1.Channelz.GetServer:output_type -> grpc.channelz.v1.GetServerResponse + 29, // 64: grpc.channelz.v1.Channelz.GetServerSockets:output_type -> grpc.channelz.v1.GetServerSocketsResponse + 31, // 65: grpc.channelz.v1.Channelz.GetChannel:output_type -> grpc.channelz.v1.GetChannelResponse + 33, // 66: grpc.channelz.v1.Channelz.GetSubchannel:output_type -> grpc.channelz.v1.GetSubchannelResponse + 35, // 67: grpc.channelz.v1.Channelz.GetSocket:output_type -> grpc.channelz.v1.GetSocketResponse + 61, // [61:68] is the sub-list for method output_type + 54, // [54:61] is the sub-list for method input_type + 54, // [54:54] is the sub-list for extension type_name + 54, // [54:54] is the sub-list for extension extendee + 0, // [0:54] is the sub-list for field type_name +} + +func init() { file_grpc_channelz_v1_channelz_proto_init() } +func file_grpc_channelz_v1_channelz_proto_init() { + if File_grpc_channelz_v1_channelz_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_grpc_channelz_v1_channelz_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Channel); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_channelz_v1_channelz_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Subchannel); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_channelz_v1_channelz_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ChannelConnectivityState); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_channelz_v1_channelz_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ChannelData); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_channelz_v1_channelz_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ChannelTraceEvent); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_channelz_v1_channelz_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ChannelTrace); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_channelz_v1_channelz_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ChannelRef); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_channelz_v1_channelz_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SubchannelRef); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_channelz_v1_channelz_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SocketRef); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_channelz_v1_channelz_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ServerRef); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_channelz_v1_channelz_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Server); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_channelz_v1_channelz_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ServerData); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_channelz_v1_channelz_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Socket); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_channelz_v1_channelz_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SocketData); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_channelz_v1_channelz_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Address); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_channelz_v1_channelz_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Security); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_channelz_v1_channelz_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SocketOption); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_channelz_v1_channelz_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SocketOptionTimeout); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_channelz_v1_channelz_proto_msgTypes[18].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SocketOptionLinger); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_channelz_v1_channelz_proto_msgTypes[19].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SocketOptionTcpInfo); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_channelz_v1_channelz_proto_msgTypes[20].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetTopChannelsRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_channelz_v1_channelz_proto_msgTypes[21].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetTopChannelsResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_channelz_v1_channelz_proto_msgTypes[22].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetServersRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_channelz_v1_channelz_proto_msgTypes[23].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetServersResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_channelz_v1_channelz_proto_msgTypes[24].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetServerRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_channelz_v1_channelz_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetServerResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_channelz_v1_channelz_proto_msgTypes[26].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetServerSocketsRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_channelz_v1_channelz_proto_msgTypes[27].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetServerSocketsResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_channelz_v1_channelz_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetChannelRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_channelz_v1_channelz_proto_msgTypes[29].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetChannelResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_channelz_v1_channelz_proto_msgTypes[30].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetSubchannelRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_channelz_v1_channelz_proto_msgTypes[31].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetSubchannelResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_channelz_v1_channelz_proto_msgTypes[32].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetSocketRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_channelz_v1_channelz_proto_msgTypes[33].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetSocketResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_channelz_v1_channelz_proto_msgTypes[34].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Address_TcpIpAddress); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_channelz_v1_channelz_proto_msgTypes[35].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Address_UdsAddress); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_channelz_v1_channelz_proto_msgTypes[36].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Address_OtherAddress); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_channelz_v1_channelz_proto_msgTypes[37].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Security_Tls); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_channelz_v1_channelz_proto_msgTypes[38].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Security_OtherSecurity); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + file_grpc_channelz_v1_channelz_proto_msgTypes[4].OneofWrappers = []interface{}{ + (*ChannelTraceEvent_ChannelRef)(nil), + (*ChannelTraceEvent_SubchannelRef)(nil), } - if interceptor == nil { - return srv.(ChannelzServer).GetSocket(ctx, in) + file_grpc_channelz_v1_channelz_proto_msgTypes[14].OneofWrappers = []interface{}{ + (*Address_TcpipAddress)(nil), + (*Address_UdsAddress_)(nil), + (*Address_OtherAddress_)(nil), } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/grpc.channelz.v1.Channelz/GetSocket", + file_grpc_channelz_v1_channelz_proto_msgTypes[15].OneofWrappers = []interface{}{ + (*Security_Tls_)(nil), + (*Security_Other)(nil), } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(ChannelzServer).GetSocket(ctx, req.(*GetSocketRequest)) + file_grpc_channelz_v1_channelz_proto_msgTypes[37].OneofWrappers = []interface{}{ + (*Security_Tls_StandardName)(nil), + (*Security_Tls_OtherName)(nil), } - return interceptor(ctx, in, info, handler) -} - -var _Channelz_serviceDesc = grpc.ServiceDesc{ - ServiceName: "grpc.channelz.v1.Channelz", - HandlerType: (*ChannelzServer)(nil), - Methods: []grpc.MethodDesc{ - { - MethodName: "GetTopChannels", - Handler: _Channelz_GetTopChannels_Handler, - }, - { - MethodName: "GetServers", - Handler: _Channelz_GetServers_Handler, - }, - { - MethodName: "GetServer", - Handler: _Channelz_GetServer_Handler, - }, - { - MethodName: "GetServerSockets", - Handler: _Channelz_GetServerSockets_Handler, - }, - { - MethodName: "GetChannel", - Handler: _Channelz_GetChannel_Handler, - }, - { - MethodName: "GetSubchannel", - Handler: _Channelz_GetSubchannel_Handler, - }, - { - MethodName: "GetSocket", - Handler: _Channelz_GetSocket_Handler, + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_grpc_channelz_v1_channelz_proto_rawDesc, + NumEnums: 2, + NumMessages: 39, + NumExtensions: 0, + NumServices: 1, }, - }, - Streams: []grpc.StreamDesc{}, - Metadata: "grpc/channelz/v1/channelz.proto", + GoTypes: file_grpc_channelz_v1_channelz_proto_goTypes, + DependencyIndexes: file_grpc_channelz_v1_channelz_proto_depIdxs, + EnumInfos: file_grpc_channelz_v1_channelz_proto_enumTypes, + MessageInfos: file_grpc_channelz_v1_channelz_proto_msgTypes, + }.Build() + File_grpc_channelz_v1_channelz_proto = out.File + file_grpc_channelz_v1_channelz_proto_rawDesc = nil + file_grpc_channelz_v1_channelz_proto_goTypes = nil + file_grpc_channelz_v1_channelz_proto_depIdxs = nil } diff --git a/channelz/grpc_channelz_v1/channelz_grpc.pb.go b/channelz/grpc_channelz_v1/channelz_grpc.pb.go index e4815814853e..070f787ca527 100644 --- a/channelz/grpc_channelz_v1/channelz_grpc.pb.go +++ b/channelz/grpc_channelz_v1/channelz_grpc.pb.go @@ -1,4 +1,29 @@ +// Copyright 2018 The gRPC Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file defines an interface for exporting monitoring information +// out of gRPC servers. See the full design at +// https://github.com/grpc/proposal/blob/master/A14-channelz.md +// +// The canonical version of this proto can be found at +// https://github.com/grpc/grpc-proto/blob/master/grpc/channelz/v1/channelz.proto + // Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.3.0 +// - protoc v4.22.0 +// source: grpc/channelz/v1/channelz.proto package grpc_channelz_v1 @@ -11,276 +36,331 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. const _ = grpc.SupportPackageIsVersion7 -// ChannelzService is the service API for Channelz service. -// Fields should be assigned to their respective handler implementations only before -// RegisterChannelzService is called. Any unassigned fields will result in the -// handler for that method returning an Unimplemented error. -type ChannelzService struct { +const ( + Channelz_GetTopChannels_FullMethodName = "/grpc.channelz.v1.Channelz/GetTopChannels" + Channelz_GetServers_FullMethodName = "/grpc.channelz.v1.Channelz/GetServers" + Channelz_GetServer_FullMethodName = "/grpc.channelz.v1.Channelz/GetServer" + Channelz_GetServerSockets_FullMethodName = "/grpc.channelz.v1.Channelz/GetServerSockets" + Channelz_GetChannel_FullMethodName = "/grpc.channelz.v1.Channelz/GetChannel" + Channelz_GetSubchannel_FullMethodName = "/grpc.channelz.v1.Channelz/GetSubchannel" + Channelz_GetSocket_FullMethodName = "/grpc.channelz.v1.Channelz/GetSocket" +) + +// ChannelzClient is the client API for Channelz service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type ChannelzClient interface { // Gets all root channels (i.e. channels the application has directly // created). This does not include subchannels nor non-top level channels. - GetTopChannels func(context.Context, *GetTopChannelsRequest) (*GetTopChannelsResponse, error) + GetTopChannels(ctx context.Context, in *GetTopChannelsRequest, opts ...grpc.CallOption) (*GetTopChannelsResponse, error) // Gets all servers that exist in the process. - GetServers func(context.Context, *GetServersRequest) (*GetServersResponse, error) + GetServers(ctx context.Context, in *GetServersRequest, opts ...grpc.CallOption) (*GetServersResponse, error) // Returns a single Server, or else a NOT_FOUND code. - GetServer func(context.Context, *GetServerRequest) (*GetServerResponse, error) + GetServer(ctx context.Context, in *GetServerRequest, opts ...grpc.CallOption) (*GetServerResponse, error) // Gets all server sockets that exist in the process. - GetServerSockets func(context.Context, *GetServerSocketsRequest) (*GetServerSocketsResponse, error) + GetServerSockets(ctx context.Context, in *GetServerSocketsRequest, opts ...grpc.CallOption) (*GetServerSocketsResponse, error) // Returns a single Channel, or else a NOT_FOUND code. - GetChannel func(context.Context, *GetChannelRequest) (*GetChannelResponse, error) + GetChannel(ctx context.Context, in *GetChannelRequest, opts ...grpc.CallOption) (*GetChannelResponse, error) // Returns a single Subchannel, or else a NOT_FOUND code. - GetSubchannel func(context.Context, *GetSubchannelRequest) (*GetSubchannelResponse, error) + GetSubchannel(ctx context.Context, in *GetSubchannelRequest, opts ...grpc.CallOption) (*GetSubchannelResponse, error) // Returns a single Socket or else a NOT_FOUND code. - GetSocket func(context.Context, *GetSocketRequest) (*GetSocketResponse, error) + GetSocket(ctx context.Context, in *GetSocketRequest, opts ...grpc.CallOption) (*GetSocketResponse, error) +} + +type channelzClient struct { + cc grpc.ClientConnInterface +} + +func NewChannelzClient(cc grpc.ClientConnInterface) ChannelzClient { + return &channelzClient{cc} } -func (s *ChannelzService) getTopChannels(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - if s.GetTopChannels == nil { - return nil, status.Errorf(codes.Unimplemented, "method GetTopChannels not implemented") +func (c *channelzClient) GetTopChannels(ctx context.Context, in *GetTopChannelsRequest, opts ...grpc.CallOption) (*GetTopChannelsResponse, error) { + out := new(GetTopChannelsResponse) + err := c.cc.Invoke(ctx, Channelz_GetTopChannels_FullMethodName, in, out, opts...) + if err != nil { + return nil, err } + return out, nil +} + +func (c *channelzClient) GetServers(ctx context.Context, in *GetServersRequest, opts ...grpc.CallOption) (*GetServersResponse, error) { + out := new(GetServersResponse) + err := c.cc.Invoke(ctx, Channelz_GetServers_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *channelzClient) GetServer(ctx context.Context, in *GetServerRequest, opts ...grpc.CallOption) (*GetServerResponse, error) { + out := new(GetServerResponse) + err := c.cc.Invoke(ctx, Channelz_GetServer_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *channelzClient) GetServerSockets(ctx context.Context, in *GetServerSocketsRequest, opts ...grpc.CallOption) (*GetServerSocketsResponse, error) { + out := new(GetServerSocketsResponse) + err := c.cc.Invoke(ctx, Channelz_GetServerSockets_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *channelzClient) GetChannel(ctx context.Context, in *GetChannelRequest, opts ...grpc.CallOption) (*GetChannelResponse, error) { + out := new(GetChannelResponse) + err := c.cc.Invoke(ctx, Channelz_GetChannel_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *channelzClient) GetSubchannel(ctx context.Context, in *GetSubchannelRequest, opts ...grpc.CallOption) (*GetSubchannelResponse, error) { + out := new(GetSubchannelResponse) + err := c.cc.Invoke(ctx, Channelz_GetSubchannel_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *channelzClient) GetSocket(ctx context.Context, in *GetSocketRequest, opts ...grpc.CallOption) (*GetSocketResponse, error) { + out := new(GetSocketResponse) + err := c.cc.Invoke(ctx, Channelz_GetSocket_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// ChannelzServer is the server API for Channelz service. +// All implementations should embed UnimplementedChannelzServer +// for forward compatibility +type ChannelzServer interface { + // Gets all root channels (i.e. channels the application has directly + // created). This does not include subchannels nor non-top level channels. + GetTopChannels(context.Context, *GetTopChannelsRequest) (*GetTopChannelsResponse, error) + // Gets all servers that exist in the process. + GetServers(context.Context, *GetServersRequest) (*GetServersResponse, error) + // Returns a single Server, or else a NOT_FOUND code. + GetServer(context.Context, *GetServerRequest) (*GetServerResponse, error) + // Gets all server sockets that exist in the process. + GetServerSockets(context.Context, *GetServerSocketsRequest) (*GetServerSocketsResponse, error) + // Returns a single Channel, or else a NOT_FOUND code. + GetChannel(context.Context, *GetChannelRequest) (*GetChannelResponse, error) + // Returns a single Subchannel, or else a NOT_FOUND code. + GetSubchannel(context.Context, *GetSubchannelRequest) (*GetSubchannelResponse, error) + // Returns a single Socket or else a NOT_FOUND code. + GetSocket(context.Context, *GetSocketRequest) (*GetSocketResponse, error) +} + +// UnimplementedChannelzServer should be embedded to have forward compatible implementations. +type UnimplementedChannelzServer struct { +} + +func (UnimplementedChannelzServer) GetTopChannels(context.Context, *GetTopChannelsRequest) (*GetTopChannelsResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetTopChannels not implemented") +} +func (UnimplementedChannelzServer) GetServers(context.Context, *GetServersRequest) (*GetServersResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetServers not implemented") +} +func (UnimplementedChannelzServer) GetServer(context.Context, *GetServerRequest) (*GetServerResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetServer not implemented") +} +func (UnimplementedChannelzServer) GetServerSockets(context.Context, *GetServerSocketsRequest) (*GetServerSocketsResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetServerSockets not implemented") +} +func (UnimplementedChannelzServer) GetChannel(context.Context, *GetChannelRequest) (*GetChannelResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetChannel not implemented") +} +func (UnimplementedChannelzServer) GetSubchannel(context.Context, *GetSubchannelRequest) (*GetSubchannelResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetSubchannel not implemented") +} +func (UnimplementedChannelzServer) GetSocket(context.Context, *GetSocketRequest) (*GetSocketResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetSocket not implemented") +} + +// UnsafeChannelzServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to ChannelzServer will +// result in compilation errors. +type UnsafeChannelzServer interface { + mustEmbedUnimplementedChannelzServer() +} + +func RegisterChannelzServer(s grpc.ServiceRegistrar, srv ChannelzServer) { + s.RegisterService(&Channelz_ServiceDesc, srv) +} + +func _Channelz_GetTopChannels_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(GetTopChannelsRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { - return s.GetTopChannels(ctx, in) + return srv.(ChannelzServer).GetTopChannels(ctx, in) } info := &grpc.UnaryServerInfo{ - Server: s, - FullMethod: "/grpc.channelz.v1.Channelz/GetTopChannels", + Server: srv, + FullMethod: Channelz_GetTopChannels_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return s.GetTopChannels(ctx, req.(*GetTopChannelsRequest)) + return srv.(ChannelzServer).GetTopChannels(ctx, req.(*GetTopChannelsRequest)) } return interceptor(ctx, in, info, handler) } -func (s *ChannelzService) getServers(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - if s.GetServers == nil { - return nil, status.Errorf(codes.Unimplemented, "method GetServers not implemented") - } + +func _Channelz_GetServers_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(GetServersRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { - return s.GetServers(ctx, in) + return srv.(ChannelzServer).GetServers(ctx, in) } info := &grpc.UnaryServerInfo{ - Server: s, - FullMethod: "/grpc.channelz.v1.Channelz/GetServers", + Server: srv, + FullMethod: Channelz_GetServers_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return s.GetServers(ctx, req.(*GetServersRequest)) + return srv.(ChannelzServer).GetServers(ctx, req.(*GetServersRequest)) } return interceptor(ctx, in, info, handler) } -func (s *ChannelzService) getServer(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - if s.GetServer == nil { - return nil, status.Errorf(codes.Unimplemented, "method GetServer not implemented") - } + +func _Channelz_GetServer_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(GetServerRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { - return s.GetServer(ctx, in) + return srv.(ChannelzServer).GetServer(ctx, in) } info := &grpc.UnaryServerInfo{ - Server: s, - FullMethod: "/grpc.channelz.v1.Channelz/GetServer", + Server: srv, + FullMethod: Channelz_GetServer_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return s.GetServer(ctx, req.(*GetServerRequest)) + return srv.(ChannelzServer).GetServer(ctx, req.(*GetServerRequest)) } return interceptor(ctx, in, info, handler) } -func (s *ChannelzService) getServerSockets(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - if s.GetServerSockets == nil { - return nil, status.Errorf(codes.Unimplemented, "method GetServerSockets not implemented") - } + +func _Channelz_GetServerSockets_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(GetServerSocketsRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { - return s.GetServerSockets(ctx, in) + return srv.(ChannelzServer).GetServerSockets(ctx, in) } info := &grpc.UnaryServerInfo{ - Server: s, - FullMethod: "/grpc.channelz.v1.Channelz/GetServerSockets", + Server: srv, + FullMethod: Channelz_GetServerSockets_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return s.GetServerSockets(ctx, req.(*GetServerSocketsRequest)) + return srv.(ChannelzServer).GetServerSockets(ctx, req.(*GetServerSocketsRequest)) } return interceptor(ctx, in, info, handler) } -func (s *ChannelzService) getChannel(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - if s.GetChannel == nil { - return nil, status.Errorf(codes.Unimplemented, "method GetChannel not implemented") - } + +func _Channelz_GetChannel_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(GetChannelRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { - return s.GetChannel(ctx, in) + return srv.(ChannelzServer).GetChannel(ctx, in) } info := &grpc.UnaryServerInfo{ - Server: s, - FullMethod: "/grpc.channelz.v1.Channelz/GetChannel", + Server: srv, + FullMethod: Channelz_GetChannel_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return s.GetChannel(ctx, req.(*GetChannelRequest)) + return srv.(ChannelzServer).GetChannel(ctx, req.(*GetChannelRequest)) } return interceptor(ctx, in, info, handler) } -func (s *ChannelzService) getSubchannel(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - if s.GetSubchannel == nil { - return nil, status.Errorf(codes.Unimplemented, "method GetSubchannel not implemented") - } + +func _Channelz_GetSubchannel_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(GetSubchannelRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { - return s.GetSubchannel(ctx, in) + return srv.(ChannelzServer).GetSubchannel(ctx, in) } info := &grpc.UnaryServerInfo{ - Server: s, - FullMethod: "/grpc.channelz.v1.Channelz/GetSubchannel", + Server: srv, + FullMethod: Channelz_GetSubchannel_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return s.GetSubchannel(ctx, req.(*GetSubchannelRequest)) + return srv.(ChannelzServer).GetSubchannel(ctx, req.(*GetSubchannelRequest)) } return interceptor(ctx, in, info, handler) } -func (s *ChannelzService) getSocket(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - if s.GetSocket == nil { - return nil, status.Errorf(codes.Unimplemented, "method GetSocket not implemented") - } + +func _Channelz_GetSocket_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(GetSocketRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { - return s.GetSocket(ctx, in) + return srv.(ChannelzServer).GetSocket(ctx, in) } info := &grpc.UnaryServerInfo{ - Server: s, - FullMethod: "/grpc.channelz.v1.Channelz/GetSocket", + Server: srv, + FullMethod: Channelz_GetSocket_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return s.GetSocket(ctx, req.(*GetSocketRequest)) + return srv.(ChannelzServer).GetSocket(ctx, req.(*GetSocketRequest)) } return interceptor(ctx, in, info, handler) } -// RegisterChannelzService registers a service implementation with a gRPC server. -func RegisterChannelzService(s grpc.ServiceRegistrar, srv *ChannelzService) { - sd := grpc.ServiceDesc{ - ServiceName: "grpc.channelz.v1.Channelz", - Methods: []grpc.MethodDesc{ - { - MethodName: "GetTopChannels", - Handler: srv.getTopChannels, - }, - { - MethodName: "GetServers", - Handler: srv.getServers, - }, - { - MethodName: "GetServer", - Handler: srv.getServer, - }, - { - MethodName: "GetServerSockets", - Handler: srv.getServerSockets, - }, - { - MethodName: "GetChannel", - Handler: srv.getChannel, - }, - { - MethodName: "GetSubchannel", - Handler: srv.getSubchannel, - }, - { - MethodName: "GetSocket", - Handler: srv.getSocket, - }, +// Channelz_ServiceDesc is the grpc.ServiceDesc for Channelz service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var Channelz_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.channelz.v1.Channelz", + HandlerType: (*ChannelzServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "GetTopChannels", + Handler: _Channelz_GetTopChannels_Handler, }, - Streams: []grpc.StreamDesc{}, - Metadata: "grpc/channelz/v1/channelz.proto", - } - - s.RegisterService(&sd, nil) -} - -// NewChannelzService creates a new ChannelzService containing the -// implemented methods of the Channelz service in s. Any unimplemented -// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client. -// This includes situations where the method handler is misspelled or has the wrong -// signature. For this reason, this function should be used with great care and -// is not recommended to be used by most users. -func NewChannelzService(s interface{}) *ChannelzService { - ns := &ChannelzService{} - if h, ok := s.(interface { - GetTopChannels(context.Context, *GetTopChannelsRequest) (*GetTopChannelsResponse, error) - }); ok { - ns.GetTopChannels = h.GetTopChannels - } - if h, ok := s.(interface { - GetServers(context.Context, *GetServersRequest) (*GetServersResponse, error) - }); ok { - ns.GetServers = h.GetServers - } - if h, ok := s.(interface { - GetServer(context.Context, *GetServerRequest) (*GetServerResponse, error) - }); ok { - ns.GetServer = h.GetServer - } - if h, ok := s.(interface { - GetServerSockets(context.Context, *GetServerSocketsRequest) (*GetServerSocketsResponse, error) - }); ok { - ns.GetServerSockets = h.GetServerSockets - } - if h, ok := s.(interface { - GetChannel(context.Context, *GetChannelRequest) (*GetChannelResponse, error) - }); ok { - ns.GetChannel = h.GetChannel - } - if h, ok := s.(interface { - GetSubchannel(context.Context, *GetSubchannelRequest) (*GetSubchannelResponse, error) - }); ok { - ns.GetSubchannel = h.GetSubchannel - } - if h, ok := s.(interface { - GetSocket(context.Context, *GetSocketRequest) (*GetSocketResponse, error) - }); ok { - ns.GetSocket = h.GetSocket - } - return ns -} - -// UnstableChannelzService is the service API for Channelz service. -// New methods may be added to this interface if they are added to the service -// definition, which is not a backward-compatible change. For this reason, -// use of this type is not recommended. -type UnstableChannelzService interface { - // Gets all root channels (i.e. channels the application has directly - // created). This does not include subchannels nor non-top level channels. - GetTopChannels(context.Context, *GetTopChannelsRequest) (*GetTopChannelsResponse, error) - // Gets all servers that exist in the process. - GetServers(context.Context, *GetServersRequest) (*GetServersResponse, error) - // Returns a single Server, or else a NOT_FOUND code. - GetServer(context.Context, *GetServerRequest) (*GetServerResponse, error) - // Gets all server sockets that exist in the process. - GetServerSockets(context.Context, *GetServerSocketsRequest) (*GetServerSocketsResponse, error) - // Returns a single Channel, or else a NOT_FOUND code. - GetChannel(context.Context, *GetChannelRequest) (*GetChannelResponse, error) - // Returns a single Subchannel, or else a NOT_FOUND code. - GetSubchannel(context.Context, *GetSubchannelRequest) (*GetSubchannelResponse, error) - // Returns a single Socket or else a NOT_FOUND code. - GetSocket(context.Context, *GetSocketRequest) (*GetSocketResponse, error) + { + MethodName: "GetServers", + Handler: _Channelz_GetServers_Handler, + }, + { + MethodName: "GetServer", + Handler: _Channelz_GetServer_Handler, + }, + { + MethodName: "GetServerSockets", + Handler: _Channelz_GetServerSockets_Handler, + }, + { + MethodName: "GetChannel", + Handler: _Channelz_GetChannel_Handler, + }, + { + MethodName: "GetSubchannel", + Handler: _Channelz_GetSubchannel_Handler, + }, + { + MethodName: "GetSocket", + Handler: _Channelz_GetSocket_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "grpc/channelz/v1/channelz.proto", } diff --git a/channelz/service/func_linux.go b/channelz/service/func_linux.go index ce38a921b974..0873603c8520 100644 --- a/channelz/service/func_linux.go +++ b/channelz/service/func_linux.go @@ -25,6 +25,7 @@ import ( durpb "github.com/golang/protobuf/ptypes/duration" channelzpb "google.golang.org/grpc/channelz/grpc_channelz_v1" "google.golang.org/grpc/internal/channelz" + "google.golang.org/protobuf/types/known/anypb" ) func convertToPtypesDuration(sec int64, usec int64) *durpb.Duration { @@ -34,7 +35,7 @@ func convertToPtypesDuration(sec int64, usec int64) *durpb.Duration { func sockoptToProto(skopts *channelz.SocketOptionData) []*channelzpb.SocketOption { var opts []*channelzpb.SocketOption if skopts.Linger != nil { - additional, err := ptypes.MarshalAny(&channelzpb.SocketOptionLinger{ + additional, err := anypb.New(&channelzpb.SocketOptionLinger{ Active: skopts.Linger.Onoff != 0, Duration: convertToPtypesDuration(int64(skopts.Linger.Linger), 0), }) @@ -43,10 +44,12 @@ func sockoptToProto(skopts *channelz.SocketOptionData) []*channelzpb.SocketOptio Name: "SO_LINGER", Additional: additional, }) + } else { + logger.Warningf("Failed to marshal socket options linger %+v: %v", skopts.Linger, err) } } if skopts.RecvTimeout != nil { - additional, err := ptypes.MarshalAny(&channelzpb.SocketOptionTimeout{ + additional, err := anypb.New(&channelzpb.SocketOptionTimeout{ Duration: convertToPtypesDuration(int64(skopts.RecvTimeout.Sec), int64(skopts.RecvTimeout.Usec)), }) if err == nil { @@ -54,10 +57,12 @@ func sockoptToProto(skopts *channelz.SocketOptionData) []*channelzpb.SocketOptio Name: "SO_RCVTIMEO", Additional: additional, }) + } else { + logger.Warningf("Failed to marshal socket options receive timeout %+v: %v", skopts.RecvTimeout, err) } } if skopts.SendTimeout != nil { - additional, err := ptypes.MarshalAny(&channelzpb.SocketOptionTimeout{ + additional, err := anypb.New(&channelzpb.SocketOptionTimeout{ Duration: convertToPtypesDuration(int64(skopts.SendTimeout.Sec), int64(skopts.SendTimeout.Usec)), }) if err == nil { @@ -65,10 +70,12 @@ func sockoptToProto(skopts *channelz.SocketOptionData) []*channelzpb.SocketOptio Name: "SO_SNDTIMEO", Additional: additional, }) + } else { + logger.Warningf("Failed to marshal socket options send timeout %+v: %v", skopts.SendTimeout, err) } } if skopts.TCPInfo != nil { - additional, err := ptypes.MarshalAny(&channelzpb.SocketOptionTcpInfo{ + additional, err := anypb.New(&channelzpb.SocketOptionTcpInfo{ TcpiState: uint32(skopts.TCPInfo.State), TcpiCaState: uint32(skopts.TCPInfo.Ca_state), TcpiRetransmits: uint32(skopts.TCPInfo.Retransmits), @@ -104,6 +111,8 @@ func sockoptToProto(skopts *channelz.SocketOptionData) []*channelzpb.SocketOptio Name: "TCP_INFO", Additional: additional, }) + } else { + logger.Warningf("Failed to marshal socket options TCP info %+v: %v", skopts.TCPInfo, err) } } return opts diff --git a/channelz/service/func_nonlinux.go b/channelz/service/func_nonlinux.go index eb53334ed0d1..473495d6655e 100644 --- a/channelz/service/func_nonlinux.go +++ b/channelz/service/func_nonlinux.go @@ -1,4 +1,5 @@ -// +build !linux appengine +//go:build !linux +// +build !linux /* * diff --git a/channelz/service/service.go b/channelz/service/service.go index 4d175fef823d..ae19ed3792ea 100644 --- a/channelz/service/service.go +++ b/channelz/service/service.go @@ -25,15 +25,18 @@ import ( "github.com/golang/protobuf/ptypes" wrpb "github.com/golang/protobuf/ptypes/wrappers" - "google.golang.org/grpc" channelzgrpc "google.golang.org/grpc/channelz/grpc_channelz_v1" channelzpb "google.golang.org/grpc/channelz/grpc_channelz_v1" + + "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/connectivity" "google.golang.org/grpc/credentials" "google.golang.org/grpc/grpclog" "google.golang.org/grpc/internal/channelz" "google.golang.org/grpc/status" + "google.golang.org/protobuf/protoadapt" + "google.golang.org/protobuf/types/known/anypb" ) func init() { @@ -43,7 +46,11 @@ func init() { var logger = grpclog.Component("channelz") // RegisterChannelzServiceToServer registers the channelz service to the given server. -func RegisterChannelzServiceToServer(s *grpc.Server) { +// +// Note: it is preferred to use the admin API +// (https://pkg.go.dev/google.golang.org/grpc/admin#Register) instead to +// register Channelz and other administrative services. +func RegisterChannelzServiceToServer(s grpc.ServiceRegistrar) { channelzgrpc.RegisterChannelzServer(s, newCZServer()) } @@ -78,7 +85,7 @@ func channelTraceToProto(ct *channelz.ChannelTrace) *channelzpb.ChannelTrace { if ts, err := ptypes.TimestampProto(ct.CreationTime); err == nil { pbt.CreationTimestamp = ts } - var events []*channelzpb.ChannelTraceEvent + events := make([]*channelzpb.ChannelTraceEvent, 0, len(ct.Events)) for _, e := range ct.Events { cte := &channelzpb.ChannelTraceEvent{ Description: e.Desc, @@ -183,7 +190,7 @@ func securityToProto(se credentials.ChannelzSecurityValue) *channelzpb.Security otherSecurity := &channelzpb.Security_OtherSecurity{ Name: v.Name, } - if anyval, err := ptypes.MarshalAny(v.Value); err == nil { + if anyval, err := anypb.New(protoadapt.MessageV2Of(v.Value)); err == nil { otherSecurity.Value = anyval } return &channelzpb.Security{Model: &channelzpb.Security_Other{Other: otherSecurity}} diff --git a/channelz/service/service_sktopt_test.go b/channelz/service/service_sktopt_test.go index e2d024f83652..1da38aa7fbf3 100644 --- a/channelz/service/service_sktopt_test.go +++ b/channelz/service/service_sktopt_test.go @@ -1,3 +1,4 @@ +//go:build linux && (386 || amd64) // +build linux // +build 386 amd64 @@ -27,15 +28,17 @@ package service import ( "context" - "reflect" "strconv" "testing" "github.com/golang/protobuf/ptypes" - durpb "github.com/golang/protobuf/ptypes/duration" + "github.com/google/go-cmp/cmp" "golang.org/x/sys/unix" - channelzpb "google.golang.org/grpc/channelz/grpc_channelz_v1" "google.golang.org/grpc/internal/channelz" + "google.golang.org/protobuf/testing/protocmp" + + durpb "github.com/golang/protobuf/ptypes/duration" + channelzpb "google.golang.org/grpc/channelz/grpc_channelz_v1" ) func init() { @@ -125,8 +128,6 @@ func protoToSocketOption(skopts []*channelzpb.SocketOption) *channelz.SocketOpti } func (s) TestGetSocketOptions(t *testing.T) { - czCleanup := channelz.NewChannelzStorage() - defer cleanupWrapper(czCleanup, t) ss := []*dummySocket{ { socketOptions: &channelz.SocketOptionData{ @@ -138,18 +139,27 @@ func (s) TestGetSocketOptions(t *testing.T) { }, } svr := newCZServer() - ids := make([]int64, len(ss)) + ids := make([]*channelz.Identifier, len(ss)) svrID := channelz.RegisterServer(&dummyServer{}, "") defer channelz.RemoveEntry(svrID) for i, s := range ss { - ids[i] = channelz.RegisterNormalSocket(s, svrID, strconv.Itoa(i)) + ids[i], _ = channelz.RegisterNormalSocket(s, svrID, strconv.Itoa(i)) defer channelz.RemoveEntry(ids[i]) } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() for i, s := range ss { - resp, _ := svr.GetSocket(context.Background(), &channelzpb.GetSocketRequest{SocketId: ids[i]}) - metrics := resp.GetSocket() - if !reflect.DeepEqual(metrics.GetRef(), &channelzpb.SocketRef{SocketId: ids[i], Name: strconv.Itoa(i)}) || !reflect.DeepEqual(socketProtoToStruct(metrics), s) { - t.Fatalf("resp.GetSocket() want: metrics.GetRef() = %#v and %#v, got: metrics.GetRef() = %#v and %#v", &channelzpb.SocketRef{SocketId: ids[i], Name: strconv.Itoa(i)}, s, metrics.GetRef(), socketProtoToStruct(metrics)) + resp, _ := svr.GetSocket(ctx, &channelzpb.GetSocketRequest{SocketId: ids[i].Int()}) + got, want := resp.GetSocket().GetRef(), &channelzpb.SocketRef{SocketId: ids[i].Int(), Name: strconv.Itoa(i)} + if !cmp.Equal(got, want, protocmp.Transform()) { + t.Fatalf("resp.GetSocket() returned metrics.GetRef() = %#v, want %#v", got, want) + } + socket, err := socketProtoToStruct(resp.GetSocket()) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(s, socket, protocmp.Transform(), cmp.AllowUnexported(dummySocket{})); diff != "" { + t.Fatalf("unexpected socket, diff (-want +got):\n%s", diff) } } } diff --git a/channelz/service/service_test.go b/channelz/service/service_test.go index 41f4f31a5b94..38b1f7dda7d8 100644 --- a/channelz/service/service_test.go +++ b/channelz/service/service_test.go @@ -22,18 +22,21 @@ import ( "context" "fmt" "net" - "reflect" "strconv" + "strings" "testing" "time" "github.com/golang/protobuf/proto" "github.com/golang/protobuf/ptypes" - channelzpb "google.golang.org/grpc/channelz/grpc_channelz_v1" + "github.com/google/go-cmp/cmp" "google.golang.org/grpc/connectivity" "google.golang.org/grpc/credentials" "google.golang.org/grpc/internal/channelz" "google.golang.org/grpc/internal/grpctest" + "google.golang.org/protobuf/testing/protocmp" + + channelzpb "google.golang.org/grpc/channelz/grpc_channelz_v1" ) func init() { @@ -48,12 +51,6 @@ func Test(t *testing.T) { grpctest.RunSubTests(t, s{}) } -func cleanupWrapper(cleanup func() error, t *testing.T) { - if err := cleanup(); err != nil { - t.Error(err) - } -} - type protoToSocketOptFunc func([]*channelzpb.SocketOption) *channelz.SocketOptionData // protoToSocketOpt is used in function socketProtoToStruct to extract socket option @@ -61,13 +58,7 @@ type protoToSocketOptFunc func([]*channelzpb.SocketOption) *channelz.SocketOptio // It is only defined under linux environment on x86 architecture. var protoToSocketOpt protoToSocketOptFunc -// emptyTime is used for detecting unset value of time.Time type. -// For go1.7 and earlier, ptypes.Timestamp will fill in the loc field of time.Time -// with &utcLoc. However zero value of a time.Time type value loc field is nil. -// This behavior will make reflect.DeepEqual fail upon unset time.Time field, -// and cause false positive fatal error. -// TODO: Go1.7 is no longer supported - does this need a change? -var emptyTime time.Time +const defaultTestTimeout = 10 * time.Second type dummyChannel struct { state connectivity.State @@ -147,7 +138,7 @@ func (d *dummySocket) ChannelzMetric() *channelz.SocketInternalMetric { } } -func channelProtoToStruct(c *channelzpb.Channel) *dummyChannel { +func channelProtoToStruct(c *channelzpb.Channel) (*dummyChannel, error) { dc := &dummyChannel{} pdata := c.GetData() switch pdata.GetState().GetState() { @@ -168,29 +159,27 @@ func channelProtoToStruct(c *channelzpb.Channel) *dummyChannel { dc.callsStarted = pdata.CallsStarted dc.callsSucceeded = pdata.CallsSucceeded dc.callsFailed = pdata.CallsFailed - if t, err := ptypes.Timestamp(pdata.GetLastCallStartedTimestamp()); err == nil { - if !t.Equal(emptyTime) { - dc.lastCallStartedTimestamp = t - } + if err := pdata.GetLastCallStartedTimestamp().CheckValid(); err != nil { + return nil, err } - return dc + dc.lastCallStartedTimestamp = pdata.GetLastCallStartedTimestamp().AsTime() + return dc, nil } -func serverProtoToStruct(s *channelzpb.Server) *dummyServer { +func serverProtoToStruct(s *channelzpb.Server) (*dummyServer, error) { ds := &dummyServer{} pdata := s.GetData() ds.callsStarted = pdata.CallsStarted ds.callsSucceeded = pdata.CallsSucceeded ds.callsFailed = pdata.CallsFailed - if t, err := ptypes.Timestamp(pdata.GetLastCallStartedTimestamp()); err == nil { - if !t.Equal(emptyTime) { - ds.lastCallStartedTimestamp = t - } + if err := pdata.GetLastCallStartedTimestamp().CheckValid(); err != nil { + return nil, err } - return ds + ds.lastCallStartedTimestamp = pdata.GetLastCallStartedTimestamp().AsTime() + return ds, nil } -func socketProtoToStruct(s *channelzpb.Socket) *dummySocket { +func socketProtoToStruct(s *channelzpb.Socket) (*dummySocket, error) { ds := &dummySocket{} pdata := s.GetData() ds.streamsStarted = pdata.GetStreamsStarted() @@ -199,26 +188,22 @@ func socketProtoToStruct(s *channelzpb.Socket) *dummySocket { ds.messagesSent = pdata.GetMessagesSent() ds.messagesReceived = pdata.GetMessagesReceived() ds.keepAlivesSent = pdata.GetKeepAlivesSent() - if t, err := ptypes.Timestamp(pdata.GetLastLocalStreamCreatedTimestamp()); err == nil { - if !t.Equal(emptyTime) { - ds.lastLocalStreamCreatedTimestamp = t - } + if err := pdata.GetLastLocalStreamCreatedTimestamp().CheckValid(); err != nil { + return nil, err } - if t, err := ptypes.Timestamp(pdata.GetLastRemoteStreamCreatedTimestamp()); err == nil { - if !t.Equal(emptyTime) { - ds.lastRemoteStreamCreatedTimestamp = t - } + ds.lastLocalStreamCreatedTimestamp = pdata.GetLastLocalStreamCreatedTimestamp().AsTime() + if err := pdata.GetLastRemoteStreamCreatedTimestamp().CheckValid(); err != nil { + return nil, err } - if t, err := ptypes.Timestamp(pdata.GetLastMessageSentTimestamp()); err == nil { - if !t.Equal(emptyTime) { - ds.lastMessageSentTimestamp = t - } + ds.lastRemoteStreamCreatedTimestamp = pdata.GetLastRemoteStreamCreatedTimestamp().AsTime() + if err := pdata.GetLastMessageSentTimestamp().CheckValid(); err != nil { + return nil, err } - if t, err := ptypes.Timestamp(pdata.GetLastMessageReceivedTimestamp()); err == nil { - if !t.Equal(emptyTime) { - ds.lastMessageReceivedTimestamp = t - } + ds.lastMessageSentTimestamp = pdata.GetLastMessageSentTimestamp().AsTime() + if err := pdata.GetLastMessageReceivedTimestamp().CheckValid(); err != nil { + return nil, err } + ds.lastMessageReceivedTimestamp = pdata.GetLastMessageReceivedTimestamp().AsTime() if v := pdata.GetLocalFlowControlWindow(); v != nil { ds.localFlowControlWindow = v.Value } @@ -238,7 +223,7 @@ func socketProtoToStruct(s *channelzpb.Socket) *dummySocket { ds.remoteAddr = protoToAddr(remote) } ds.remoteName = s.GetRemoteName() - return ds + return ds, nil } func protoToSecurity(protoSecurity *channelzpb.Security) credentials.ChannelzSecurityValue { @@ -320,27 +305,32 @@ func (s) TestGetTopChannels(t *testing.T) { }, {}, } - czCleanup := channelz.NewChannelzStorage() - defer cleanupWrapper(czCleanup, t) + for _, c := range tcs { - id := channelz.RegisterChannel(c, 0, "") + id := channelz.RegisterChannel(c, nil, "") defer channelz.RemoveEntry(id) } s := newCZServer() - resp, _ := s.GetTopChannels(context.Background(), &channelzpb.GetTopChannelsRequest{StartChannelId: 0}) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + resp, _ := s.GetTopChannels(ctx, &channelzpb.GetTopChannelsRequest{StartChannelId: 0}) if !resp.GetEnd() { t.Fatalf("resp.GetEnd() want true, got %v", resp.GetEnd()) } for i, c := range resp.GetChannel() { - if !reflect.DeepEqual(channelProtoToStruct(c), tcs[i]) { - t.Fatalf("dummyChannel: %d, want: %#v, got: %#v", i, tcs[i], channelProtoToStruct(c)) + channel, err := channelProtoToStruct(c) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(tcs[i], channel, protocmp.Transform(), cmp.AllowUnexported(dummyChannel{})); diff != "" { + t.Fatalf("unexpected channel, diff (-want +got):\n%s", diff) } } for i := 0; i < 50; i++ { - id := channelz.RegisterChannel(tcs[0], 0, "") + id := channelz.RegisterChannel(tcs[0], nil, "") defer channelz.RemoveEntry(id) } - resp, _ = s.GetTopChannels(context.Background(), &channelzpb.GetTopChannelsRequest{StartChannelId: 0}) + resp, _ = s.GetTopChannels(ctx, &channelzpb.GetTopChannelsRequest{StartChannelId: 0}) if resp.GetEnd() { t.Fatalf("resp.GetEnd() want false, got %v", resp.GetEnd()) } @@ -367,64 +357,69 @@ func (s) TestGetServers(t *testing.T) { lastCallStartedTimestamp: time.Now().UTC(), }, } - czCleanup := channelz.NewChannelzStorage() - defer cleanupWrapper(czCleanup, t) + for _, s := range ss { id := channelz.RegisterServer(s, "") defer channelz.RemoveEntry(id) } svr := newCZServer() - resp, _ := svr.GetServers(context.Background(), &channelzpb.GetServersRequest{StartServerId: 0}) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + resp, _ := svr.GetServers(ctx, &channelzpb.GetServersRequest{StartServerId: 0}) if !resp.GetEnd() { t.Fatalf("resp.GetEnd() want true, got %v", resp.GetEnd()) } for i, s := range resp.GetServer() { - if !reflect.DeepEqual(serverProtoToStruct(s), ss[i]) { - t.Fatalf("dummyServer: %d, want: %#v, got: %#v", i, ss[i], serverProtoToStruct(s)) + server, err := serverProtoToStruct(s) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(ss[i], server, protocmp.Transform(), cmp.AllowUnexported(dummyServer{})); diff != "" { + t.Fatalf("unexpected server, diff (-want +got):\n%s", diff) } } for i := 0; i < 50; i++ { id := channelz.RegisterServer(ss[0], "") defer channelz.RemoveEntry(id) } - resp, _ = svr.GetServers(context.Background(), &channelzpb.GetServersRequest{StartServerId: 0}) + resp, _ = svr.GetServers(ctx, &channelzpb.GetServersRequest{StartServerId: 0}) if resp.GetEnd() { t.Fatalf("resp.GetEnd() want false, got %v", resp.GetEnd()) } } func (s) TestGetServerSockets(t *testing.T) { - czCleanup := channelz.NewChannelzStorage() - defer cleanupWrapper(czCleanup, t) svrID := channelz.RegisterServer(&dummyServer{}, "") defer channelz.RemoveEntry(svrID) refNames := []string{"listen socket 1", "normal socket 1", "normal socket 2"} - ids := make([]int64, 3) - ids[0] = channelz.RegisterListenSocket(&dummySocket{}, svrID, refNames[0]) - ids[1] = channelz.RegisterNormalSocket(&dummySocket{}, svrID, refNames[1]) - ids[2] = channelz.RegisterNormalSocket(&dummySocket{}, svrID, refNames[2]) + ids := make([]*channelz.Identifier, 3) + ids[0], _ = channelz.RegisterListenSocket(&dummySocket{}, svrID, refNames[0]) + ids[1], _ = channelz.RegisterNormalSocket(&dummySocket{}, svrID, refNames[1]) + ids[2], _ = channelz.RegisterNormalSocket(&dummySocket{}, svrID, refNames[2]) for _, id := range ids { defer channelz.RemoveEntry(id) } svr := newCZServer() - resp, _ := svr.GetServerSockets(context.Background(), &channelzpb.GetServerSocketsRequest{ServerId: svrID, StartSocketId: 0}) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + resp, _ := svr.GetServerSockets(ctx, &channelzpb.GetServerSocketsRequest{ServerId: svrID.Int(), StartSocketId: 0}) if !resp.GetEnd() { t.Fatalf("resp.GetEnd() want: true, got: %v", resp.GetEnd()) } // GetServerSockets only return normal sockets. want := map[int64]string{ - ids[1]: refNames[1], - ids[2]: refNames[2], + ids[1].Int(): refNames[1], + ids[2].Int(): refNames[2], } - if !reflect.DeepEqual(convertSocketRefSliceToMap(resp.GetSocketRef()), want) { + if !cmp.Equal(convertSocketRefSliceToMap(resp.GetSocketRef()), want) { t.Fatalf("GetServerSockets want: %#v, got: %#v", want, resp.GetSocketRef()) } for i := 0; i < 50; i++ { - id := channelz.RegisterNormalSocket(&dummySocket{}, svrID, "") + id, _ := channelz.RegisterNormalSocket(&dummySocket{}, svrID, "") defer channelz.RemoveEntry(id) } - resp, _ = svr.GetServerSockets(context.Background(), &channelzpb.GetServerSocketsRequest{ServerId: svrID, StartSocketId: 0}) + resp, _ = svr.GetServerSockets(ctx, &channelzpb.GetServerSocketsRequest{ServerId: svrID.Int(), StartSocketId: 0}) if resp.GetEnd() { t.Fatalf("resp.GetEnd() want false, got %v", resp.GetEnd()) } @@ -433,94 +428,102 @@ func (s) TestGetServerSockets(t *testing.T) { // This test makes a GetServerSockets with a non-zero start ID, and expect only // sockets with ID >= the given start ID. func (s) TestGetServerSocketsNonZeroStartID(t *testing.T) { - czCleanup := channelz.NewChannelzStorage() - defer cleanupWrapper(czCleanup, t) svrID := channelz.RegisterServer(&dummyServer{}, "") defer channelz.RemoveEntry(svrID) refNames := []string{"listen socket 1", "normal socket 1", "normal socket 2"} - ids := make([]int64, 3) - ids[0] = channelz.RegisterListenSocket(&dummySocket{}, svrID, refNames[0]) - ids[1] = channelz.RegisterNormalSocket(&dummySocket{}, svrID, refNames[1]) - ids[2] = channelz.RegisterNormalSocket(&dummySocket{}, svrID, refNames[2]) + ids := make([]*channelz.Identifier, 3) + ids[0], _ = channelz.RegisterListenSocket(&dummySocket{}, svrID, refNames[0]) + ids[1], _ = channelz.RegisterNormalSocket(&dummySocket{}, svrID, refNames[1]) + ids[2], _ = channelz.RegisterNormalSocket(&dummySocket{}, svrID, refNames[2]) for _, id := range ids { defer channelz.RemoveEntry(id) } svr := newCZServer() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() // Make GetServerSockets with startID = ids[1]+1, so socket-1 won't be // included in the response. - resp, _ := svr.GetServerSockets(context.Background(), &channelzpb.GetServerSocketsRequest{ServerId: svrID, StartSocketId: ids[1] + 1}) + resp, _ := svr.GetServerSockets(ctx, &channelzpb.GetServerSocketsRequest{ServerId: svrID.Int(), StartSocketId: ids[1].Int() + 1}) if !resp.GetEnd() { t.Fatalf("resp.GetEnd() want: true, got: %v", resp.GetEnd()) } // GetServerSockets only return normal socket-2, socket-1 should be // filtered by start ID. want := map[int64]string{ - ids[2]: refNames[2], + ids[2].Int(): refNames[2], } - if !reflect.DeepEqual(convertSocketRefSliceToMap(resp.GetSocketRef()), want) { + if !cmp.Equal(convertSocketRefSliceToMap(resp.GetSocketRef()), want) { t.Fatalf("GetServerSockets want: %#v, got: %#v", want, resp.GetSocketRef()) } } func (s) TestGetChannel(t *testing.T) { - czCleanup := channelz.NewChannelzStorage() - defer cleanupWrapper(czCleanup, t) refNames := []string{"top channel 1", "nested channel 1", "sub channel 2", "nested channel 3"} - ids := make([]int64, 4) - ids[0] = channelz.RegisterChannel(&dummyChannel{}, 0, refNames[0]) + ids := make([]*channelz.Identifier, 4) + ids[0] = channelz.RegisterChannel(&dummyChannel{}, nil, refNames[0]) channelz.AddTraceEvent(logger, ids[0], 0, &channelz.TraceEventDesc{ Desc: "Channel Created", - Severity: channelz.CtINFO, + Severity: channelz.CtInfo, }) + ids[1] = channelz.RegisterChannel(&dummyChannel{}, ids[0], refNames[1]) channelz.AddTraceEvent(logger, ids[1], 0, &channelz.TraceEventDesc{ Desc: "Channel Created", - Severity: channelz.CtINFO, + Severity: channelz.CtInfo, Parent: &channelz.TraceEventDesc{ - Desc: fmt.Sprintf("Nested Channel(id:%d) created", ids[1]), - Severity: channelz.CtINFO, + Desc: fmt.Sprintf("Nested Channel(id:%d) created", ids[1].Int()), + Severity: channelz.CtInfo, }, }) - ids[2] = channelz.RegisterSubChannel(&dummyChannel{}, ids[0], refNames[2]) + var err error + ids[2], err = channelz.RegisterSubChannel(&dummyChannel{}, ids[0], refNames[2]) + if err != nil { + t.Fatalf("channelz.RegisterSubChannel() failed: %v", err) + } channelz.AddTraceEvent(logger, ids[2], 0, &channelz.TraceEventDesc{ Desc: "SubChannel Created", - Severity: channelz.CtINFO, + Severity: channelz.CtInfo, Parent: &channelz.TraceEventDesc{ - Desc: fmt.Sprintf("SubChannel(id:%d) created", ids[2]), - Severity: channelz.CtINFO, + Desc: fmt.Sprintf("SubChannel(id:%d) created", ids[2].Int()), + Severity: channelz.CtInfo, }, }) + ids[3] = channelz.RegisterChannel(&dummyChannel{}, ids[1], refNames[3]) channelz.AddTraceEvent(logger, ids[3], 0, &channelz.TraceEventDesc{ Desc: "Channel Created", - Severity: channelz.CtINFO, + Severity: channelz.CtInfo, Parent: &channelz.TraceEventDesc{ - Desc: fmt.Sprintf("Nested Channel(id:%d) created", ids[3]), - Severity: channelz.CtINFO, + Desc: fmt.Sprintf("Nested Channel(id:%d) created", ids[3].Int()), + Severity: channelz.CtInfo, }, }) channelz.AddTraceEvent(logger, ids[0], 0, &channelz.TraceEventDesc{ Desc: fmt.Sprintf("Channel Connectivity change to %v", connectivity.Ready), - Severity: channelz.CtINFO, + Severity: channelz.CtInfo, }) channelz.AddTraceEvent(logger, ids[0], 0, &channelz.TraceEventDesc{ Desc: "Resolver returns an empty address list", Severity: channelz.CtWarning, }) + for _, id := range ids { defer channelz.RemoveEntry(id) } + svr := newCZServer() - resp, _ := svr.GetChannel(context.Background(), &channelzpb.GetChannelRequest{ChannelId: ids[0]}) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + resp, _ := svr.GetChannel(ctx, &channelzpb.GetChannelRequest{ChannelId: ids[0].Int()}) metrics := resp.GetChannel() subChans := metrics.GetSubchannelRef() - if len(subChans) != 1 || subChans[0].GetName() != refNames[2] || subChans[0].GetSubchannelId() != ids[2] { - t.Fatalf("metrics.GetSubChannelRef() want %#v, got %#v", []*channelzpb.SubchannelRef{{SubchannelId: ids[2], Name: refNames[2]}}, subChans) + if len(subChans) != 1 || subChans[0].GetName() != refNames[2] || subChans[0].GetSubchannelId() != ids[2].Int() { + t.Fatalf("metrics.GetSubChannelRef() want %#v, got %#v", []*channelzpb.SubchannelRef{{SubchannelId: ids[2].Int(), Name: refNames[2]}}, subChans) } nestedChans := metrics.GetChannelRef() - if len(nestedChans) != 1 || nestedChans[0].GetName() != refNames[1] || nestedChans[0].GetChannelId() != ids[1] { - t.Fatalf("metrics.GetChannelRef() want %#v, got %#v", []*channelzpb.ChannelRef{{ChannelId: ids[1], Name: refNames[1]}}, nestedChans) + if len(nestedChans) != 1 || nestedChans[0].GetName() != refNames[1] || nestedChans[0].GetChannelId() != ids[1].Int() { + t.Fatalf("metrics.GetChannelRef() want %#v, got %#v", []*channelzpb.ChannelRef{{ChannelId: ids[1].Int(), Name: refNames[1]}}, nestedChans) } trace := metrics.GetData().GetTrace() want := []struct { @@ -530,14 +533,14 @@ func (s) TestGetChannel(t *testing.T) { childRef string }{ {desc: "Channel Created", severity: channelzpb.ChannelTraceEvent_CT_INFO}, - {desc: fmt.Sprintf("Nested Channel(id:%d) created", ids[1]), severity: channelzpb.ChannelTraceEvent_CT_INFO, childID: ids[1], childRef: refNames[1]}, - {desc: fmt.Sprintf("SubChannel(id:%d) created", ids[2]), severity: channelzpb.ChannelTraceEvent_CT_INFO, childID: ids[2], childRef: refNames[2]}, + {desc: fmt.Sprintf("Nested Channel(id:%d) created", ids[1].Int()), severity: channelzpb.ChannelTraceEvent_CT_INFO, childID: ids[1].Int(), childRef: refNames[1]}, + {desc: fmt.Sprintf("SubChannel(id:%d) created", ids[2].Int()), severity: channelzpb.ChannelTraceEvent_CT_INFO, childID: ids[2].Int(), childRef: refNames[2]}, {desc: fmt.Sprintf("Channel Connectivity change to %v", connectivity.Ready), severity: channelzpb.ChannelTraceEvent_CT_INFO}, {desc: "Resolver returns an empty address list", severity: channelzpb.ChannelTraceEvent_CT_WARNING}, } for i, e := range trace.Events { - if e.GetDescription() != want[i].desc { + if !strings.Contains(e.GetDescription(), want[i].desc) { t.Fatalf("trace: GetDescription want %#v, got %#v", want[i].desc, e.GetDescription()) } if e.GetSeverity() != want[i].severity { @@ -552,11 +555,11 @@ func (s) TestGetChannel(t *testing.T) { } } } - resp, _ = svr.GetChannel(context.Background(), &channelzpb.GetChannelRequest{ChannelId: ids[1]}) + resp, _ = svr.GetChannel(ctx, &channelzpb.GetChannelRequest{ChannelId: ids[1].Int()}) metrics = resp.GetChannel() nestedChans = metrics.GetChannelRef() - if len(nestedChans) != 1 || nestedChans[0].GetName() != refNames[3] || nestedChans[0].GetChannelId() != ids[3] { - t.Fatalf("metrics.GetChannelRef() want %#v, got %#v", []*channelzpb.ChannelRef{{ChannelId: ids[3], Name: refNames[3]}}, nestedChans) + if len(nestedChans) != 1 || nestedChans[0].GetName() != refNames[3] || nestedChans[0].GetChannelId() != ids[3].Int() { + t.Fatalf("metrics.GetChannelRef() want %#v, got %#v", []*channelzpb.ChannelRef{{ChannelId: ids[3].Int(), Name: refNames[3]}}, nestedChans) } } @@ -566,45 +569,50 @@ func (s) TestGetSubChannel(t *testing.T) { subchanConnectivityChange = fmt.Sprintf("Subchannel Connectivity change to %v", connectivity.Ready) subChanPickNewAddress = fmt.Sprintf("Subchannel picks a new address %q to connect", "0.0.0.0") ) - czCleanup := channelz.NewChannelzStorage() - defer cleanupWrapper(czCleanup, t) + refNames := []string{"top channel 1", "sub channel 1", "socket 1", "socket 2"} - ids := make([]int64, 4) - ids[0] = channelz.RegisterChannel(&dummyChannel{}, 0, refNames[0]) + ids := make([]*channelz.Identifier, 4) + ids[0] = channelz.RegisterChannel(&dummyChannel{}, nil, refNames[0]) channelz.AddTraceEvent(logger, ids[0], 0, &channelz.TraceEventDesc{ Desc: "Channel Created", - Severity: channelz.CtINFO, + Severity: channelz.CtInfo, }) - ids[1] = channelz.RegisterSubChannel(&dummyChannel{}, ids[0], refNames[1]) + var err error + ids[1], err = channelz.RegisterSubChannel(&dummyChannel{}, ids[0], refNames[1]) + if err != nil { + t.Fatalf("channelz.RegisterSubChannel() failed: %v", err) + } channelz.AddTraceEvent(logger, ids[1], 0, &channelz.TraceEventDesc{ Desc: subchanCreated, - Severity: channelz.CtINFO, + Severity: channelz.CtInfo, Parent: &channelz.TraceEventDesc{ - Desc: fmt.Sprintf("Nested Channel(id:%d) created", ids[0]), - Severity: channelz.CtINFO, + Desc: fmt.Sprintf("Nested Channel(id:%d) created", ids[0].Int()), + Severity: channelz.CtInfo, }, }) - ids[2] = channelz.RegisterNormalSocket(&dummySocket{}, ids[1], refNames[2]) - ids[3] = channelz.RegisterNormalSocket(&dummySocket{}, ids[1], refNames[3]) + ids[2], _ = channelz.RegisterNormalSocket(&dummySocket{}, ids[1], refNames[2]) + ids[3], _ = channelz.RegisterNormalSocket(&dummySocket{}, ids[1], refNames[3]) channelz.AddTraceEvent(logger, ids[1], 0, &channelz.TraceEventDesc{ Desc: subchanConnectivityChange, - Severity: channelz.CtINFO, + Severity: channelz.CtInfo, }) channelz.AddTraceEvent(logger, ids[1], 0, &channelz.TraceEventDesc{ Desc: subChanPickNewAddress, - Severity: channelz.CtINFO, + Severity: channelz.CtInfo, }) for _, id := range ids { defer channelz.RemoveEntry(id) } svr := newCZServer() - resp, _ := svr.GetSubchannel(context.Background(), &channelzpb.GetSubchannelRequest{SubchannelId: ids[1]}) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + resp, _ := svr.GetSubchannel(ctx, &channelzpb.GetSubchannelRequest{SubchannelId: ids[1].Int()}) metrics := resp.GetSubchannel() want := map[int64]string{ - ids[2]: refNames[2], - ids[3]: refNames[3], + ids[2].Int(): refNames[2], + ids[3].Int(): refNames[3], } - if !reflect.DeepEqual(convertSocketRefSliceToMap(metrics.GetSocketRef()), want) { + if !cmp.Equal(convertSocketRefSliceToMap(metrics.GetSocketRef()), want) { t.Fatalf("metrics.GetSocketRef() want %#v: got: %#v", want, metrics.GetSocketRef()) } @@ -638,8 +646,6 @@ func (s) TestGetSubChannel(t *testing.T) { } func (s) TestGetSocket(t *testing.T) { - czCleanup := channelz.NewChannelzStorage() - defer cleanupWrapper(czCleanup, t) ss := []*dummySocket{ { streamsStarted: 10, @@ -712,18 +718,27 @@ func (s) TestGetSocket(t *testing.T) { }, } svr := newCZServer() - ids := make([]int64, len(ss)) + ids := make([]*channelz.Identifier, len(ss)) svrID := channelz.RegisterServer(&dummyServer{}, "") defer channelz.RemoveEntry(svrID) for i, s := range ss { - ids[i] = channelz.RegisterNormalSocket(s, svrID, strconv.Itoa(i)) + ids[i], _ = channelz.RegisterNormalSocket(s, svrID, strconv.Itoa(i)) defer channelz.RemoveEntry(ids[i]) } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() for i, s := range ss { - resp, _ := svr.GetSocket(context.Background(), &channelzpb.GetSocketRequest{SocketId: ids[i]}) - metrics := resp.GetSocket() - if !reflect.DeepEqual(metrics.GetRef(), &channelzpb.SocketRef{SocketId: ids[i], Name: strconv.Itoa(i)}) || !reflect.DeepEqual(socketProtoToStruct(metrics), s) { - t.Fatalf("resp.GetSocket() want: metrics.GetRef() = %#v and %#v, got: metrics.GetRef() = %#v and %#v", &channelzpb.SocketRef{SocketId: ids[i], Name: strconv.Itoa(i)}, s, metrics.GetRef(), socketProtoToStruct(metrics)) + resp, _ := svr.GetSocket(ctx, &channelzpb.GetSocketRequest{SocketId: ids[i].Int()}) + got, want := resp.GetSocket().GetRef(), &channelzpb.SocketRef{SocketId: ids[i].Int(), Name: strconv.Itoa(i)} + if !cmp.Equal(got, want, protocmp.Transform()) { + t.Fatalf("resp.GetSocket() returned metrics.GetRef() = %#v, want %#v", got, want) + } + socket, err := socketProtoToStruct(resp.GetSocket()) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(s, socket, protocmp.Transform(), cmp.AllowUnexported(dummySocket{})); diff != "" { + t.Fatalf("unexpected socket, diff (-want +got):\n%s", diff) } } } diff --git a/channelz/service/util_sktopt_386_test.go b/channelz/service/util_sktopt_386_test.go index d9c981271361..3ba3dc96e7c6 100644 --- a/channelz/service/util_sktopt_386_test.go +++ b/channelz/service/util_sktopt_386_test.go @@ -1,3 +1,4 @@ +//go:build 386 && linux // +build 386,linux /* diff --git a/channelz/service/util_sktopt_amd64_test.go b/channelz/service/util_sktopt_amd64_test.go index 0ff06d128330..124d7b758199 100644 --- a/channelz/service/util_sktopt_amd64_test.go +++ b/channelz/service/util_sktopt_amd64_test.go @@ -1,3 +1,4 @@ +//go:build amd64 && linux // +build amd64,linux /* diff --git a/clientconn.go b/clientconn.go index ae5ce4947e2e..d53d91d5d9f3 100644 --- a/clientconn.go +++ b/clientconn.go @@ -23,8 +23,7 @@ import ( "errors" "fmt" "math" - "net" - "reflect" + "net/url" "strings" "sync" "sync/atomic" @@ -35,10 +34,13 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/connectivity" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/internal" "google.golang.org/grpc/internal/backoff" "google.golang.org/grpc/internal/channelz" "google.golang.org/grpc/internal/grpcsync" - "google.golang.org/grpc/internal/grpcutil" + "google.golang.org/grpc/internal/idle" + "google.golang.org/grpc/internal/pretty" + iresolver "google.golang.org/grpc/internal/resolver" "google.golang.org/grpc/internal/transport" "google.golang.org/grpc/keepalive" "google.golang.org/grpc/resolver" @@ -48,13 +50,12 @@ import ( _ "google.golang.org/grpc/balancer/roundrobin" // To register roundrobin. _ "google.golang.org/grpc/internal/resolver/dns" // To register dns resolver. _ "google.golang.org/grpc/internal/resolver/passthrough" // To register passthrough resolver. + _ "google.golang.org/grpc/internal/resolver/unix" // To register unix resolver. ) const ( // minimum time to give a connection to complete minConnectTimeout = 20 * time.Second - // must match grpclbName in grpclb/grpclb.go - grpclbName = "grpclb" ) var ( @@ -68,6 +69,9 @@ var ( errConnDrain = errors.New("grpc: the connection is drained") // errConnClosing indicates that the connection is closing. errConnClosing = errors.New("grpc: the connection is closing") + // errConnIdling indicates the the connection is being closed as the channel + // is moving to an idle mode due to inactivity. + errConnIdling = errors.New("grpc: the connection is closing due to channel idleness") // invalidDefaultServiceConfigErrPrefix is used to prefix the json parsing error for the default // service config. invalidDefaultServiceConfigErrPrefix = "grpc: the provided default service config is invalid" @@ -78,17 +82,17 @@ var ( // errNoTransportSecurity indicates that there is no transport security // being set for ClientConn. Users should either set one or explicitly // call WithInsecure DialOption to disable security. - errNoTransportSecurity = errors.New("grpc: no transport security set (use grpc.WithInsecure() explicitly or set credentials)") + errNoTransportSecurity = errors.New("grpc: no transport security set (use grpc.WithTransportCredentials(insecure.NewCredentials()) explicitly or set credentials)") // errTransportCredsAndBundle indicates that creds bundle is used together // with other individual Transport Credentials. errTransportCredsAndBundle = errors.New("grpc: credentials.Bundle may not be used with individual TransportCredentials") - // errTransportCredentialsMissing indicates that users want to transmit security - // information (e.g., OAuth2 token) which requires secure connection on an insecure - // connection. + // errNoTransportCredsInBundle indicated that the configured creds bundle + // returned a transport credentials which was nil. + errNoTransportCredsInBundle = errors.New("grpc: credentials.Bundle must return non-nil transport credentials") + // errTransportCredentialsMissing indicates that users want to transmit + // security information (e.g., OAuth2 token) which requires secure + // connection on an insecure connection. errTransportCredentialsMissing = errors.New("grpc: the credentials require transport level security (use grpc.WithTransportCredentials() to set)") - // errCredentialsConflict indicates that grpc.WithTransportCredentials() - // and grpc.WithInsecure() are both called for a connection. - errCredentialsConflict = errors.New("grpc: transport credentials are set for an insecure connection (grpc.WithTransportCredentials() and grpc.WithInsecure() are both called)") ) const ( @@ -104,6 +108,17 @@ func Dial(target string, opts ...DialOption) (*ClientConn, error) { return DialContext(context.Background(), target, opts...) } +type defaultConfigSelector struct { + sc *ServiceConfig +} + +func (dcs *defaultConfigSelector) SelectConfig(rpcInfo iresolver.RPCInfo) (*iresolver.RPCConfig, error) { + return &iresolver.RPCConfig{ + Context: rpcInfo.Context, + MethodConfig: getMethodConfig(dcs.sc, rpcInfo.Method), + }, nil +} + // DialContext creates a client connection to the given target. By default, it's // a non-blocking dial (the function won't wait for connections to be // established, and connecting happens in the background). To make it a blocking @@ -122,16 +137,42 @@ func Dial(target string, opts ...DialOption) (*ClientConn, error) { // e.g. to use dns resolver, a "dns:///" prefix should be applied to the target. func DialContext(ctx context.Context, target string, opts ...DialOption) (conn *ClientConn, err error) { cc := &ClientConn{ - target: target, - csMgr: &connectivityStateManager{}, - conns: make(map[*addrConn]struct{}), - dopts: defaultDialOptions(), - blockingpicker: newPickerWrapper(), - czData: new(channelzData), - firstResolveEvent: grpcsync.NewEvent(), + target: target, + conns: make(map[*addrConn]struct{}), + dopts: defaultDialOptions(), + czData: new(channelzData), } + + // We start the channel off in idle mode, but kick it out of idle at the end + // of this method, instead of waiting for the first RPC. Other gRPC + // implementations do wait for the first RPC to kick the channel out of + // idle. But doing so would be a major behavior change for our users who are + // used to seeing the channel active after Dial. + // + // Taking this approach of kicking it out of idle at the end of this method + // allows us to share the code between channel creation and exiting idle + // mode. This will also make it easy for us to switch to starting the + // channel off in idle, if at all we ever get to do that. + cc.idlenessState = ccIdlenessStateIdle + cc.retryThrottler.Store((*retryThrottler)(nil)) + cc.safeConfigSelector.UpdateConfigSelector(&defaultConfigSelector{nil}) cc.ctx, cc.cancel = context.WithCancel(context.Background()) + cc.exitIdleCond = sync.NewCond(&cc.mu) + + disableGlobalOpts := false + for _, opt := range opts { + if _, ok := opt.(*disableGlobalDialOptions); ok { + disableGlobalOpts = true + break + } + } + + if !disableGlobalOpts { + for _, opt := range globalDialOptions { + opt.apply(&cc.dopts) + } + } for _, opt := range opts { opt.apply(&cc.dopts) @@ -146,40 +187,13 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn * } }() - if channelz.IsOn() { - if cc.dopts.channelzParentID != 0 { - cc.channelzID = channelz.RegisterChannel(&channelzChannel{cc}, cc.dopts.channelzParentID, target) - channelz.AddTraceEvent(logger, cc.channelzID, 0, &channelz.TraceEventDesc{ - Desc: "Channel Created", - Severity: channelz.CtINFO, - Parent: &channelz.TraceEventDesc{ - Desc: fmt.Sprintf("Nested Channel(id:%d) created", cc.channelzID), - Severity: channelz.CtINFO, - }, - }) - } else { - cc.channelzID = channelz.RegisterChannel(&channelzChannel{cc}, 0, target) - channelz.Info(logger, cc.channelzID, "Channel Created") - } - cc.csMgr.channelzID = cc.channelzID - } + // Register ClientConn with channelz. + cc.channelzRegistration(target) - if !cc.dopts.insecure { - if cc.dopts.copts.TransportCredentials == nil && cc.dopts.copts.CredsBundle == nil { - return nil, errNoTransportSecurity - } - if cc.dopts.copts.TransportCredentials != nil && cc.dopts.copts.CredsBundle != nil { - return nil, errTransportCredsAndBundle - } - } else { - if cc.dopts.copts.TransportCredentials != nil || cc.dopts.copts.CredsBundle != nil { - return nil, errCredentialsConflict - } - for _, cd := range cc.dopts.copts.PerRPCCredentials { - if cd.RequireTransportSecurity() { - return nil, errTransportCredentialsMissing - } - } + cc.csMgr = newConnectivityStateManager(cc.ctx, cc.channelzID) + + if err := cc.validateTransportCredentials(); err != nil { + return nil, err } if cc.dopts.defaultServiceConfigRawJSON != nil { @@ -191,16 +205,6 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn * } cc.mkp = cc.dopts.copts.KeepaliveParams - if cc.dopts.copts.Dialer == nil { - cc.dopts.copts.Dialer = func(ctx context.Context, addr string) (net.Conn, error) { - network, addr := parseDialTarget(addr) - return (&net.Dialer{}).DialContext(ctx, network, addr) - } - if cc.dopts.withProxy { - cc.dopts.copts.Dialer = newProxyDialer(cc.dopts.copts.Dialer) - } - } - if cc.dopts.copts.UserAgent != "" { cc.dopts.copts.UserAgent += " " + grpcUA } else { @@ -227,61 +231,25 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn * } }() - scSet := false - if cc.dopts.scChan != nil { - // Try to get an initial service config. - select { - case sc, ok := <-cc.dopts.scChan: - if ok { - cc.sc = &sc - scSet = true - } - default: - } - } if cc.dopts.bs == nil { cc.dopts.bs = backoff.DefaultExponential } // Determine the resolver to use. - cc.parsedTarget = grpcutil.ParseTarget(cc.target) - unixScheme := strings.HasPrefix(cc.target, "unix:") - channelz.Infof(logger, cc.channelzID, "parsed scheme: %q", cc.parsedTarget.Scheme) - resolverBuilder := cc.getResolver(cc.parsedTarget.Scheme) - if resolverBuilder == nil { - // If resolver builder is still nil, the parsed target's scheme is - // not registered. Fallback to default resolver and set Endpoint to - // the original target. - channelz.Infof(logger, cc.channelzID, "scheme %q not registered, fallback to default scheme", cc.parsedTarget.Scheme) - cc.parsedTarget = resolver.Target{ - Scheme: resolver.GetDefaultScheme(), - Endpoint: target, - } - resolverBuilder = cc.getResolver(cc.parsedTarget.Scheme) - if resolverBuilder == nil { - return nil, fmt.Errorf("could not get resolver for default scheme: %q", cc.parsedTarget.Scheme) - } + if err := cc.parseTargetAndFindResolver(); err != nil { + return nil, err } - - creds := cc.dopts.copts.TransportCredentials - if creds != nil && creds.Info().ServerName != "" { - cc.authority = creds.Info().ServerName - } else if cc.dopts.insecure && cc.dopts.authority != "" { - cc.authority = cc.dopts.authority - } else if unixScheme { - cc.authority = "localhost" - } else { - // Use endpoint from "scheme://authority/endpoint" as the default - // authority for ClientConn. - cc.authority = cc.parsedTarget.Endpoint + if err = cc.determineAuthority(); err != nil { + return nil, err } - if cc.dopts.scChan != nil && !scSet { + if cc.dopts.scChan != nil { // Blocking wait for the initial service config. select { case sc, ok := <-cc.dopts.scChan: if ok { cc.sc = &sc + cc.safeConfigSelector.UpdateConfigSelector(&defaultConfigSelector{&sc}) } case <-ctx.Done(): return nil, ctx.Err() @@ -291,54 +259,234 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn * go cc.scWatcher() } + // This creates the name resolver, load balancer, blocking picker etc. + if err := cc.exitIdleMode(); err != nil { + return nil, err + } + + // Configure idleness support with configured idle timeout or default idle + // timeout duration. Idleness can be explicitly disabled by the user, by + // setting the dial option to 0. + cc.idlenessMgr = idle.NewManager(idle.ManagerOptions{Enforcer: (*idler)(cc), Timeout: cc.dopts.idleTimeout, Logger: logger}) + + // Return early for non-blocking dials. + if !cc.dopts.block { + return cc, nil + } + + // A blocking dial blocks until the clientConn is ready. + for { + s := cc.GetState() + if s == connectivity.Idle { + cc.Connect() + } + if s == connectivity.Ready { + return cc, nil + } else if cc.dopts.copts.FailOnNonTempDialError && s == connectivity.TransientFailure { + if err = cc.connectionError(); err != nil { + terr, ok := err.(interface { + Temporary() bool + }) + if ok && !terr.Temporary() { + return nil, err + } + } + } + if !cc.WaitForStateChange(ctx, s) { + // ctx got timeout or canceled. + if err = cc.connectionError(); err != nil && cc.dopts.returnLastError { + return nil, err + } + return nil, ctx.Err() + } + } +} + +// addTraceEvent is a helper method to add a trace event on the channel. If the +// channel is a nested one, the same event is also added on the parent channel. +func (cc *ClientConn) addTraceEvent(msg string) { + ted := &channelz.TraceEventDesc{ + Desc: fmt.Sprintf("Channel %s", msg), + Severity: channelz.CtInfo, + } + if cc.dopts.channelzParentID != nil { + ted.Parent = &channelz.TraceEventDesc{ + Desc: fmt.Sprintf("Nested channel(id:%d) %s", cc.channelzID.Int(), msg), + Severity: channelz.CtInfo, + } + } + channelz.AddTraceEvent(logger, cc.channelzID, 0, ted) +} + +type idler ClientConn + +func (i *idler) EnterIdleMode() error { + return (*ClientConn)(i).enterIdleMode() +} + +func (i *idler) ExitIdleMode() error { + return (*ClientConn)(i).exitIdleMode() +} + +// exitIdleMode moves the channel out of idle mode by recreating the name +// resolver and load balancer. +func (cc *ClientConn) exitIdleMode() error { + cc.mu.Lock() + if cc.conns == nil { + cc.mu.Unlock() + return errConnClosing + } + if cc.idlenessState != ccIdlenessStateIdle { + cc.mu.Unlock() + channelz.Infof(logger, cc.channelzID, "ClientConn asked to exit idle mode, current mode is %v", cc.idlenessState) + return nil + } + + defer func() { + // When Close() and exitIdleMode() race against each other, one of the + // following two can happen: + // - Close() wins the race and runs first. exitIdleMode() runs after, and + // sees that the ClientConn is already closed and hence returns early. + // - exitIdleMode() wins the race and runs first and recreates the balancer + // and releases the lock before recreating the resolver. If Close() runs + // in this window, it will wait for exitIdleMode to complete. + // + // We achieve this synchronization using the below condition variable. + cc.mu.Lock() + cc.idlenessState = ccIdlenessStateActive + cc.exitIdleCond.Signal() + cc.mu.Unlock() + }() + + cc.idlenessState = ccIdlenessStateExitingIdle + exitedIdle := false + if cc.blockingpicker == nil { + cc.blockingpicker = newPickerWrapper(cc.dopts.copts.StatsHandlers) + } else { + cc.blockingpicker.exitIdleMode() + exitedIdle = true + } + var credsClone credentials.TransportCredentials if creds := cc.dopts.copts.TransportCredentials; creds != nil { credsClone = creds.Clone() } - cc.balancerBuildOpts = balancer.BuildOptions{ - DialCreds: credsClone, - CredsBundle: cc.dopts.copts.CredsBundle, - Dialer: cc.dopts.copts.Dialer, - ChannelzParentID: cc.channelzID, - Target: cc.parsedTarget, + if cc.balancerWrapper == nil { + cc.balancerWrapper = newCCBalancerWrapper(cc, balancer.BuildOptions{ + DialCreds: credsClone, + CredsBundle: cc.dopts.copts.CredsBundle, + Dialer: cc.dopts.copts.Dialer, + Authority: cc.authority, + CustomUserAgent: cc.dopts.copts.UserAgent, + ChannelzParentID: cc.channelzID, + Target: cc.parsedTarget, + }) + } else { + cc.balancerWrapper.exitIdleMode() } + cc.firstResolveEvent = grpcsync.NewEvent() + cc.mu.Unlock() - // Build the resolver. - rWrapper, err := newCCResolverWrapper(cc, resolverBuilder) - if err != nil { - return nil, fmt.Errorf("failed to build resolver: %v", err) + // This needs to be called without cc.mu because this builds a new resolver + // which might update state or report error inline which needs to be handled + // by cc.updateResolverState() which also grabs cc.mu. + if err := cc.initResolverWrapper(credsClone); err != nil { + return err + } + + if exitedIdle { + cc.addTraceEvent("exiting idle mode") } + return nil +} + +// enterIdleMode puts the channel in idle mode, and as part of it shuts down the +// name resolver, load balancer and any subchannels. +func (cc *ClientConn) enterIdleMode() error { cc.mu.Lock() - cc.resolverWrapper = rWrapper + if cc.conns == nil { + cc.mu.Unlock() + return ErrClientConnClosing + } + if cc.idlenessState != ccIdlenessStateActive { + channelz.Errorf(logger, cc.channelzID, "ClientConn asked to enter idle mode, current mode is %v", cc.idlenessState) + cc.mu.Unlock() + return nil + } + + // cc.conns == nil is a proxy for the ClientConn being closed. So, instead + // of setting it to nil here, we recreate the map. This also means that we + // don't have to do this when exiting idle mode. + conns := cc.conns + cc.conns = make(map[*addrConn]struct{}) + + // TODO: Currently, we close the resolver wrapper upon entering idle mode + // and create a new one upon exiting idle mode. This means that the + // `cc.resolverWrapper` field would be overwritten everytime we exit idle + // mode. While this means that we need to hold `cc.mu` when accessing + // `cc.resolverWrapper`, it makes the code simpler in the wrapper. We should + // try to do the same for the balancer and picker wrappers too. + cc.resolverWrapper.close() + cc.blockingpicker.enterIdleMode() + cc.balancerWrapper.enterIdleMode() + cc.csMgr.updateState(connectivity.Idle) + cc.idlenessState = ccIdlenessStateIdle cc.mu.Unlock() - // A blocking dial blocks until the clientConn is ready. - if cc.dopts.block { - for { - s := cc.GetState() - if s == connectivity.Ready { - break - } else if cc.dopts.copts.FailOnNonTempDialError && s == connectivity.TransientFailure { - if err = cc.connectionError(); err != nil { - terr, ok := err.(interface { - Temporary() bool - }) - if ok && !terr.Temporary() { - return nil, err - } - } - } - if !cc.WaitForStateChange(ctx, s) { - // ctx got timeout or canceled. - if err = cc.connectionError(); err != nil && cc.dopts.returnLastError { - return nil, err - } - return nil, ctx.Err() + go func() { + cc.addTraceEvent("entering idle mode") + for ac := range conns { + ac.tearDown(errConnIdling) + } + }() + return nil +} + +// validateTransportCredentials performs a series of checks on the configured +// transport credentials. It returns a non-nil error if any of these conditions +// are met: +// - no transport creds and no creds bundle is configured +// - both transport creds and creds bundle are configured +// - creds bundle is configured, but it lacks a transport credentials +// - insecure transport creds configured alongside call creds that require +// transport level security +// +// If none of the above conditions are met, the configured credentials are +// deemed valid and a nil error is returned. +func (cc *ClientConn) validateTransportCredentials() error { + if cc.dopts.copts.TransportCredentials == nil && cc.dopts.copts.CredsBundle == nil { + return errNoTransportSecurity + } + if cc.dopts.copts.TransportCredentials != nil && cc.dopts.copts.CredsBundle != nil { + return errTransportCredsAndBundle + } + if cc.dopts.copts.CredsBundle != nil && cc.dopts.copts.CredsBundle.TransportCredentials() == nil { + return errNoTransportCredsInBundle + } + transportCreds := cc.dopts.copts.TransportCredentials + if transportCreds == nil { + transportCreds = cc.dopts.copts.CredsBundle.TransportCredentials() + } + if transportCreds.Info().SecurityProtocol == "insecure" { + for _, cd := range cc.dopts.copts.PerRPCCredentials { + if cd.RequireTransportSecurity() { + return errTransportCredentialsMissing } } } + return nil +} - return cc, nil +// channelzRegistration registers the newly created ClientConn with channelz and +// stores the returned identifier in `cc.channelzID` and `cc.csMgr.channelzID`. +// A channelz trace event is emitted for ClientConn creation. If the newly +// created ClientConn is a nested one, i.e a valid parent ClientConn ID is +// specified via a dial option, the trace event is also added to the parent. +// +// Doesn't grab cc.mu as this method is expected to be called only at Dial time. +func (cc *ClientConn) channelzRegistration(target string) { + cc.channelzID = channelz.RegisterChannel(&channelzChannel{cc}, cc.dopts.channelzParentID, target) + cc.addTraceEvent("created") } // chainUnaryClientInterceptors chains all unary client interceptors into one. @@ -355,7 +503,7 @@ func chainUnaryClientInterceptors(cc *ClientConn) { } else if len(interceptors) == 1 { chainedInt = interceptors[0] } else { - chainedInt = func(ctx context.Context, method string, req, reply interface{}, cc *ClientConn, invoker UnaryInvoker, opts ...CallOption) error { + chainedInt = func(ctx context.Context, method string, req, reply any, cc *ClientConn, invoker UnaryInvoker, opts ...CallOption) error { return interceptors[0](ctx, method, req, reply, cc, getChainUnaryInvoker(interceptors, 0, invoker), opts...) } } @@ -367,7 +515,7 @@ func getChainUnaryInvoker(interceptors []UnaryClientInterceptor, curr int, final if curr == len(interceptors)-1 { return finalInvoker } - return func(ctx context.Context, method string, req, reply interface{}, cc *ClientConn, opts ...CallOption) error { + return func(ctx context.Context, method string, req, reply any, cc *ClientConn, opts ...CallOption) error { return interceptors[curr+1](ctx, method, req, reply, cc, getChainUnaryInvoker(interceptors, curr+1, finalInvoker), opts...) } } @@ -403,13 +551,27 @@ func getChainStreamer(interceptors []StreamClientInterceptor, curr int, finalStr } } +// newConnectivityStateManager creates an connectivityStateManager with +// the specified id. +func newConnectivityStateManager(ctx context.Context, id *channelz.Identifier) *connectivityStateManager { + return &connectivityStateManager{ + channelzID: id, + pubSub: grpcsync.NewPubSub(ctx), + } +} + // connectivityStateManager keeps the connectivity.State of ClientConn. // This struct will eventually be exported so the balancers can access it. +// +// TODO: If possible, get rid of the `connectivityStateManager` type, and +// provide this functionality using the `PubSub`, to avoid keeping track of +// the connectivity state at two places. type connectivityStateManager struct { mu sync.Mutex state connectivity.State notifyChan chan struct{} - channelzID int64 + channelzID *channelz.Identifier + pubSub *grpcsync.PubSub } // updateState updates the connectivity.State of ClientConn. @@ -425,6 +587,8 @@ func (csm *connectivityStateManager) updateState(state connectivity.State) { return } csm.state = state + csm.pubSub.Publish(state) + channelz.Infof(logger, csm.channelzID, "Channel Connectivity change to %v", state) if csm.notifyChan != nil { // There are other goroutines waiting on this channel. @@ -454,7 +618,7 @@ func (csm *connectivityStateManager) getNotifyChan() <-chan struct{} { type ClientConnInterface interface { // Invoke performs a unary RPC and returns after the response is received // into reply. - Invoke(ctx context.Context, method string, args interface{}, reply interface{}, opts ...CallOption) error + Invoke(ctx context.Context, method string, args any, reply any, opts ...CallOption) error // NewStream begins a streaming RPC. NewStream(ctx context.Context, desc *StreamDesc, method string, opts ...CallOption) (ClientStream, error) } @@ -475,40 +639,83 @@ var _ ClientConnInterface = (*ClientConn)(nil) // handshakes. It also handles errors on established connections by // re-resolving the name and reconnecting. type ClientConn struct { - ctx context.Context - cancel context.CancelFunc - - target string - parsedTarget resolver.Target - authority string - dopts dialOptions - csMgr *connectivityStateManager - - balancerBuildOpts balancer.BuildOptions - blockingpicker *pickerWrapper - - mu sync.RWMutex - resolverWrapper *ccResolverWrapper - sc *ServiceConfig - conns map[*addrConn]struct{} - // Keepalive parameter can be updated if a GoAway is received. - mkp keepalive.ClientParameters - curBalancerName string - balancerWrapper *ccBalancerWrapper - retryThrottler atomic.Value - + ctx context.Context // Initialized using the background context at dial time. + cancel context.CancelFunc // Cancelled on close. + + // The following are initialized at dial time, and are read-only after that. + target string // User's dial target. + parsedTarget resolver.Target // See parseTargetAndFindResolver(). + authority string // See determineAuthority(). + dopts dialOptions // Default and user specified dial options. + channelzID *channelz.Identifier // Channelz identifier for the channel. + resolverBuilder resolver.Builder // See parseTargetAndFindResolver(). + balancerWrapper *ccBalancerWrapper // Uses gracefulswitch.balancer underneath. + idlenessMgr idle.Manager + + // The following provide their own synchronization, and therefore don't + // require cc.mu to be held to access them. + csMgr *connectivityStateManager + blockingpicker *pickerWrapper + safeConfigSelector iresolver.SafeConfigSelector + czData *channelzData + retryThrottler atomic.Value // Updated from service config. + + // firstResolveEvent is used to track whether the name resolver sent us at + // least one update. RPCs block on this event. firstResolveEvent *grpcsync.Event - channelzID int64 // channelz unique identification number - czData *channelzData + // mu protects the following fields. + // TODO: split mu so the same mutex isn't used for everything. + mu sync.RWMutex + resolverWrapper *ccResolverWrapper // Initialized in Dial; cleared in Close. + sc *ServiceConfig // Latest service config received from the resolver. + conns map[*addrConn]struct{} // Set to nil on close. + mkp keepalive.ClientParameters // May be updated upon receipt of a GoAway. + idlenessState ccIdlenessState // Tracks idleness state of the channel. + exitIdleCond *sync.Cond // Signalled when channel exits idle. lceMu sync.Mutex // protects lastConnectionError lastConnectionError error } +// ccIdlenessState tracks the idleness state of the channel. +// +// Channels start off in `active` and move to `idle` after a period of +// inactivity. When moving back to `active` upon an incoming RPC, they +// transition through `exiting_idle`. This state is useful for synchronization +// with Close(). +// +// This state tracking is mostly for self-protection. The idlenessManager is +// expected to keep track of the state as well, and is expected not to call into +// the ClientConn unnecessarily. +type ccIdlenessState int8 + +const ( + ccIdlenessStateActive ccIdlenessState = iota + ccIdlenessStateIdle + ccIdlenessStateExitingIdle +) + +func (s ccIdlenessState) String() string { + switch s { + case ccIdlenessStateActive: + return "active" + case ccIdlenessStateIdle: + return "idle" + case ccIdlenessStateExitingIdle: + return "exitingIdle" + default: + return "unknown" + } +} + // WaitForStateChange waits until the connectivity.State of ClientConn changes from sourceState or // ctx expires. A true value is returned in former case and false in latter. -// This is an EXPERIMENTAL API. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. func (cc *ClientConn) WaitForStateChange(ctx context.Context, sourceState connectivity.State) bool { ch := cc.csMgr.getNotifyChan() if cc.csMgr.getState() != sourceState { @@ -523,11 +730,30 @@ func (cc *ClientConn) WaitForStateChange(ctx context.Context, sourceState connec } // GetState returns the connectivity.State of ClientConn. -// This is an EXPERIMENTAL API. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a later +// release. func (cc *ClientConn) GetState() connectivity.State { return cc.csMgr.getState() } +// Connect causes all subchannels in the ClientConn to attempt to connect if +// the channel is idle. Does not wait for the connection attempts to begin +// before returning. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a later +// release. +func (cc *ClientConn) Connect() { + cc.exitIdleMode() + // If the ClientConn was not in idle mode, we need to call ExitIdle on the + // LB policy so that connections can be created. + cc.balancerWrapper.exitIdleMode() +} + func (cc *ClientConn) scWatcher() { for { select { @@ -539,6 +765,7 @@ func (cc *ClientConn) scWatcher() { // TODO: load balance policy runtime change is ignored. // We may revisit this decision in the future. cc.sc = &sc + cc.safeConfigSelector.UpdateConfigSelector(&defaultConfigSelector{&sc}) cc.mu.Unlock() case <-cc.ctx.Done(): return @@ -573,17 +800,21 @@ func init() { panic(fmt.Sprintf("impossible error parsing empty service config: %v", cfg.Err)) } emptyServiceConfig = cfg.Config.(*ServiceConfig) + + internal.SubscribeToConnectivityStateChanges = func(cc *ClientConn, s grpcsync.Subscriber) func() { + return cc.csMgr.pubSub.Subscribe(s) + } } func (cc *ClientConn) maybeApplyDefaultServiceConfig(addrs []resolver.Address) { if cc.sc != nil { - cc.applyServiceConfigAndBalancer(cc.sc, addrs) + cc.applyServiceConfigAndBalancer(cc.sc, nil, addrs) return } if cc.dopts.defaultServiceConfig != nil { - cc.applyServiceConfigAndBalancer(cc.dopts.defaultServiceConfig, addrs) + cc.applyServiceConfigAndBalancer(cc.dopts.defaultServiceConfig, &defaultConfigSelector{cc.dopts.defaultServiceConfig}, addrs) } else { - cc.applyServiceConfigAndBalancer(emptyServiceConfig, addrs) + cc.applyServiceConfigAndBalancer(emptyServiceConfig, &defaultConfigSelector{emptyServiceConfig}, addrs) } } @@ -604,9 +835,7 @@ func (cc *ClientConn) updateResolverState(s resolver.State, err error) error { // with the new addresses. cc.maybeApplyDefaultServiceConfig(nil) - if cc.balancerWrapper != nil { - cc.balancerWrapper.resolverError(err) - } + cc.balancerWrapper.resolverError(err) // No addresses are valid with err set; return early. cc.mu.Unlock() @@ -614,24 +843,30 @@ func (cc *ClientConn) updateResolverState(s resolver.State, err error) error { } var ret error - if cc.dopts.disableServiceConfig || s.ServiceConfig == nil { + if cc.dopts.disableServiceConfig { + channelz.Infof(logger, cc.channelzID, "ignoring service config from resolver (%v) and applying the default because service config is disabled", s.ServiceConfig) + cc.maybeApplyDefaultServiceConfig(s.Addresses) + } else if s.ServiceConfig == nil { cc.maybeApplyDefaultServiceConfig(s.Addresses) // TODO: do we need to apply a failing LB policy if there is no // default, per the error handling design? } else { if sc, ok := s.ServiceConfig.Config.(*ServiceConfig); s.ServiceConfig.Err == nil && ok { - cc.applyServiceConfigAndBalancer(sc, s.Addresses) + configSelector := iresolver.GetConfigSelector(s) + if configSelector != nil { + if len(s.ServiceConfig.Config.(*ServiceConfig).Methods) != 0 { + channelz.Infof(logger, cc.channelzID, "method configs in service config will be ignored due to presence of config selector") + } + } else { + configSelector = &defaultConfigSelector{sc} + } + cc.applyServiceConfigAndBalancer(sc, configSelector, s.Addresses) } else { ret = balancer.ErrBadResolverState - if cc.balancerWrapper == nil { - var err error - if s.ServiceConfig.Err != nil { - err = status.Errorf(codes.Unavailable, "error parsing service config: %v", s.ServiceConfig.Err) - } else { - err = status.Errorf(codes.Unavailable, "illegal service config type: %T", s.ServiceConfig.Config) - } - cc.blockingpicker.updatePicker(base.NewErrPicker(err)) - cc.csMgr.updateState(connectivity.TransientFailure) + if cc.sc == nil { + // Apply the failing LB only if we haven't received valid service config + // from the name resolver in the past. + cc.applyFailingLB(s.ServiceConfig) cc.mu.Unlock() return ret } @@ -639,24 +874,12 @@ func (cc *ClientConn) updateResolverState(s resolver.State, err error) error { } var balCfg serviceconfig.LoadBalancingConfig - if cc.dopts.balancerBuilder == nil && cc.sc != nil && cc.sc.lbConfig != nil { + if cc.sc != nil && cc.sc.lbConfig != nil { balCfg = cc.sc.lbConfig.cfg } - - cbn := cc.curBalancerName bw := cc.balancerWrapper cc.mu.Unlock() - if cbn != grpclbName { - // Filter any grpclb addresses since we don't have the grpclb balancer. - for i := 0; i < len(s.Addresses); { - if s.Addresses[i].Type == resolver.GRPCLB { - copy(s.Addresses[i:], s.Addresses[i+1:]) - s.Addresses = s.Addresses[:len(s.Addresses)-1] - continue - } - i++ - } - } + uccsErr := bw.updateClientConnState(&balancer.ClientConnState{ResolverState: s, BalancerConfig: balCfg}) if ret == nil { ret = uccsErr // prefer ErrBadResolver state since any other error is @@ -665,51 +888,42 @@ func (cc *ClientConn) updateResolverState(s resolver.State, err error) error { return ret } -// switchBalancer starts the switching from current balancer to the balancer -// with the given name. -// -// It will NOT send the current address list to the new balancer. If needed, -// caller of this function should send address list to the new balancer after -// this function returns. +// applyFailingLB is akin to configuring an LB policy on the channel which +// always fails RPCs. Here, an actual LB policy is not configured, but an always +// erroring picker is configured, which returns errors with information about +// what was invalid in the received service config. A config selector with no +// service config is configured, and the connectivity state of the channel is +// set to TransientFailure. // // Caller must hold cc.mu. -func (cc *ClientConn) switchBalancer(name string) { - if strings.EqualFold(cc.curBalancerName, name) { - return - } - - channelz.Infof(logger, cc.channelzID, "ClientConn switching balancer to %q", name) - if cc.dopts.balancerBuilder != nil { - channelz.Info(logger, cc.channelzID, "ignoring balancer switching: Balancer DialOption used instead") - return - } - if cc.balancerWrapper != nil { - cc.balancerWrapper.close() - } - - builder := balancer.Get(name) - if builder == nil { - channelz.Warningf(logger, cc.channelzID, "Channel switches to new LB policy %q due to fallback from invalid balancer name", PickFirstBalancerName) - channelz.Infof(logger, cc.channelzID, "failed to get balancer builder for: %v, using pick_first instead", name) - builder = newPickfirstBuilder() +func (cc *ClientConn) applyFailingLB(sc *serviceconfig.ParseResult) { + var err error + if sc.Err != nil { + err = status.Errorf(codes.Unavailable, "error parsing service config: %v", sc.Err) } else { - channelz.Infof(logger, cc.channelzID, "Channel switches to new LB policy %q", name) + err = status.Errorf(codes.Unavailable, "illegal service config type: %T", sc.Config) } - - cc.curBalancerName = builder.Name() - cc.balancerWrapper = newCCBalancerWrapper(cc, builder, cc.balancerBuildOpts) + cc.safeConfigSelector.UpdateConfigSelector(&defaultConfigSelector{nil}) + cc.blockingpicker.updatePicker(base.NewErrPicker(err)) + cc.csMgr.updateState(connectivity.TransientFailure) } func (cc *ClientConn) handleSubConnStateChange(sc balancer.SubConn, s connectivity.State, err error) { - cc.mu.Lock() - if cc.conns == nil { - cc.mu.Unlock() - return - } - // TODO(bar switching) send updates to all balancer wrappers when balancer - // gracefully switching is supported. - cc.balancerWrapper.handleSubConnStateChange(sc, s, err) - cc.mu.Unlock() + cc.balancerWrapper.updateSubConnState(sc, s, err) +} + +// Makes a copy of the input addresses slice and clears out the balancer +// attributes field. Addresses are passed during subconn creation and address +// update operations. In both cases, we will clear the balancer attributes by +// calling this function, and therefore we will be able to use the Equal method +// provided by the resolver.Address type for comparison. +func copyAddressesWithoutBalancerAttributes(in []resolver.Address) []resolver.Address { + out := make([]resolver.Address, len(in)) + for i := range in { + out[i] = in[i] + out[i].BalancerAttributes = nil + } + return out } // newAddrConn creates an addrConn for addrs and adds it to cc.conns. @@ -719,32 +933,36 @@ func (cc *ClientConn) newAddrConn(addrs []resolver.Address, opts balancer.NewSub ac := &addrConn{ state: connectivity.Idle, cc: cc, - addrs: addrs, + addrs: copyAddressesWithoutBalancerAttributes(addrs), scopts: opts, dopts: cc.dopts, czData: new(channelzData), resetBackoff: make(chan struct{}), + stateChan: make(chan struct{}), } ac.ctx, ac.cancel = context.WithCancel(cc.ctx) // Track ac in cc. This needs to be done before any getTransport(...) is called. cc.mu.Lock() + defer cc.mu.Unlock() if cc.conns == nil { - cc.mu.Unlock() return nil, ErrClientConnClosing } - if channelz.IsOn() { - ac.channelzID = channelz.RegisterSubChannel(ac, cc.channelzID, "") - channelz.AddTraceEvent(logger, ac.channelzID, 0, &channelz.TraceEventDesc{ - Desc: "Subchannel Created", - Severity: channelz.CtINFO, - Parent: &channelz.TraceEventDesc{ - Desc: fmt.Sprintf("Subchannel(id:%d) created", ac.channelzID), - Severity: channelz.CtINFO, - }, - }) - } + + var err error + ac.channelzID, err = channelz.RegisterSubChannel(ac, cc.channelzID, "") + if err != nil { + return nil, err + } + channelz.AddTraceEvent(logger, ac.channelzID, 0, &channelz.TraceEventDesc{ + Desc: "Subchannel created", + Severity: channelz.CtInfo, + Parent: &channelz.TraceEventDesc{ + Desc: fmt.Sprintf("Subchannel(id:%d) created", ac.channelzID.Int()), + Severity: channelz.CtInfo, + }, + }) + cc.conns[ac] = struct{}{} - cc.mu.Unlock() return ac, nil } @@ -773,7 +991,11 @@ func (cc *ClientConn) channelzMetric() *channelz.ChannelInternalMetric { } // Target returns the target string of the ClientConn. -// This is an EXPERIMENTAL API. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. func (cc *ClientConn) Target() string { return cc.target } @@ -797,67 +1019,128 @@ func (cc *ClientConn) incrCallsFailed() { func (ac *addrConn) connect() error { ac.mu.Lock() if ac.state == connectivity.Shutdown { + if logger.V(2) { + logger.Infof("connect called on shutdown addrConn; ignoring.") + } ac.mu.Unlock() return errConnClosing } if ac.state != connectivity.Idle { + if logger.V(2) { + logger.Infof("connect called on addrConn in non-idle state (%v); ignoring.", ac.state) + } ac.mu.Unlock() return nil } - // Update connectivity state within the lock to prevent subsequent or - // concurrent calls from resetting the transport more than once. - ac.updateConnectivityState(connectivity.Connecting, nil) ac.mu.Unlock() - // Start a goroutine connecting to the server asynchronously. - go ac.resetTransport() + ac.resetTransport() return nil } -// tryUpdateAddrs tries to update ac.addrs with the new addresses list. -// -// If ac is Connecting, it returns false. The caller should tear down the ac and -// create a new one. Note that the backoff will be reset when this happens. -// -// If ac is TransientFailure, it updates ac.addrs and returns true. The updated -// addresses will be picked up by retry in the next iteration after backoff. -// -// If ac is Shutdown or Idle, it updates ac.addrs and returns true. -// -// If ac is Ready, it checks whether current connected address of ac is in the -// new addrs list. -// - If true, it updates ac.addrs and returns true. The ac will keep using -// the existing connection. -// - If false, it does nothing and returns false. -func (ac *addrConn) tryUpdateAddrs(addrs []resolver.Address) bool { +func equalAddresses(a, b []resolver.Address) bool { + if len(a) != len(b) { + return false + } + for i, v := range a { + if !v.Equal(b[i]) { + return false + } + } + return true +} + +// updateAddrs updates ac.addrs with the new addresses list and handles active +// connections or connection attempts. +func (ac *addrConn) updateAddrs(addrs []resolver.Address) { ac.mu.Lock() - defer ac.mu.Unlock() - channelz.Infof(logger, ac.channelzID, "addrConn: tryUpdateAddrs curAddr: %v, addrs: %v", ac.curAddr, addrs) + channelz.Infof(logger, ac.channelzID, "addrConn: updateAddrs curAddr: %v, addrs: %v", pretty.ToJSON(ac.curAddr), pretty.ToJSON(addrs)) + + addrs = copyAddressesWithoutBalancerAttributes(addrs) + if equalAddresses(ac.addrs, addrs) { + ac.mu.Unlock() + return + } + + ac.addrs = addrs + if ac.state == connectivity.Shutdown || ac.state == connectivity.TransientFailure || ac.state == connectivity.Idle { - ac.addrs = addrs - return true + // We were not connecting, so do nothing but update the addresses. + ac.mu.Unlock() + return } - if ac.state == connectivity.Connecting { - return false + if ac.state == connectivity.Ready { + // Try to find the connected address. + for _, a := range addrs { + a.ServerName = ac.cc.getServerName(a) + if a.Equal(ac.curAddr) { + // We are connected to a valid address, so do nothing but + // update the addresses. + ac.mu.Unlock() + return + } + } } - // ac.state is Ready, try to find the connected address. - var curAddrFound bool - for _, a := range addrs { - if reflect.DeepEqual(ac.curAddr, a) { - curAddrFound = true - break - } + // We are either connected to the wrong address or currently connecting. + // Stop the current iteration and restart. + + ac.cancel() + ac.ctx, ac.cancel = context.WithCancel(ac.cc.ctx) + + // We have to defer here because GracefulClose => Close => onClose, which + // requires locking ac.mu. + if ac.transport != nil { + defer ac.transport.GracefulClose() + ac.transport = nil } - channelz.Infof(logger, ac.channelzID, "addrConn: tryUpdateAddrs curAddrFound: %v", curAddrFound) - if curAddrFound { - ac.addrs = addrs + + if len(addrs) == 0 { + ac.updateConnectivityState(connectivity.Idle, nil) } - return curAddrFound + ac.mu.Unlock() + + // Since we were connecting/connected, we should start a new connection + // attempt. + go ac.resetTransport() +} + +// getServerName determines the serverName to be used in the connection +// handshake. The default value for the serverName is the authority on the +// ClientConn, which either comes from the user's dial target or through an +// authority override specified using the WithAuthority dial option. Name +// resolvers can specify a per-address override for the serverName through the +// resolver.Address.ServerName field which is used only if the WithAuthority +// dial option was not used. The rationale is that per-address authority +// overrides specified by the name resolver can represent a security risk, while +// an override specified by the user is more dependable since they probably know +// what they are doing. +func (cc *ClientConn) getServerName(addr resolver.Address) string { + if cc.dopts.authority != "" { + return cc.dopts.authority + } + if addr.ServerName != "" { + return addr.ServerName + } + return cc.authority +} + +func getMethodConfig(sc *ServiceConfig, method string) MethodConfig { + if sc == nil { + return MethodConfig{} + } + if m, ok := sc.Methods[method]; ok { + return m + } + i := strings.LastIndex(method, "/") + if m, ok := sc.Methods[method[:i+1]]; ok { + return m + } + return sc.Methods[""] } // GetMethodConfig gets the method config of the input method. @@ -872,17 +1155,7 @@ func (cc *ClientConn) GetMethodConfig(method string) MethodConfig { // TODO: Avoid the locking here. cc.mu.RLock() defer cc.mu.RUnlock() - if cc.sc == nil { - return MethodConfig{} - } - if m, ok := cc.sc.Methods[method]; ok { - return m - } - i := strings.LastIndex(method, "/") - if m, ok := cc.sc.Methods[method[:i+1]]; ok { - return m - } - return cc.sc.Methods[""] + return getMethodConfig(cc.sc, method) } func (cc *ClientConn) healthCheckConfig() *healthCheckConfig { @@ -894,23 +1167,22 @@ func (cc *ClientConn) healthCheckConfig() *healthCheckConfig { return cc.sc.healthCheckConfig } -func (cc *ClientConn) getTransport(ctx context.Context, failfast bool, method string) (transport.ClientTransport, func(balancer.DoneInfo), error) { - t, done, err := cc.blockingpicker.pick(ctx, failfast, balancer.PickInfo{ +func (cc *ClientConn) getTransport(ctx context.Context, failfast bool, method string) (transport.ClientTransport, balancer.PickResult, error) { + return cc.blockingpicker.pick(ctx, failfast, balancer.PickInfo{ Ctx: ctx, FullMethodName: method, }) - if err != nil { - return nil, nil, toRPCErr(err) - } - return t, done, nil } -func (cc *ClientConn) applyServiceConfigAndBalancer(sc *ServiceConfig, addrs []resolver.Address) { +func (cc *ClientConn) applyServiceConfigAndBalancer(sc *ServiceConfig, configSelector iresolver.ConfigSelector, addrs []resolver.Address) { if sc == nil { // should never reach here. return } cc.sc = sc + if configSelector != nil { + cc.safeConfigSelector.UpdateConfigSelector(configSelector) + } if cc.sc.retryThrottling != nil { newThrottler := &retryThrottler{ @@ -924,35 +1196,16 @@ func (cc *ClientConn) applyServiceConfigAndBalancer(sc *ServiceConfig, addrs []r cc.retryThrottler.Store((*retryThrottler)(nil)) } - if cc.dopts.balancerBuilder == nil { - // Only look at balancer types and switch balancer if balancer dial - // option is not set. - var newBalancerName string - if cc.sc != nil && cc.sc.lbConfig != nil { - newBalancerName = cc.sc.lbConfig.name - } else { - var isGRPCLB bool - for _, a := range addrs { - if a.Type == resolver.GRPCLB { - isGRPCLB = true - break - } - } - if isGRPCLB { - newBalancerName = grpclbName - } else if cc.sc != nil && cc.sc.LB != nil { - newBalancerName = *cc.sc.LB - } else { - newBalancerName = PickFirstBalancerName - } - } - cc.switchBalancer(newBalancerName) - } else if cc.balancerWrapper == nil { - // Balancer dial option was set, and this is the first time handling - // resolved addresses. Build a balancer with dopts.balancerBuilder. - cc.curBalancerName = cc.dopts.balancerBuilder.Name() - cc.balancerWrapper = newCCBalancerWrapper(cc, cc.dopts.balancerBuilder, cc.balancerBuildOpts) + var newBalancerName string + if cc.sc == nil || (cc.sc.lbConfig == nil && cc.sc.LB == nil) { + // No service config or no LB policy specified in config. + newBalancerName = PickFirstBalancerName + } else if cc.sc.lbConfig != nil { + newBalancerName = cc.sc.lbConfig.name + } else { // cc.sc.LB != nil + newBalancerName = *cc.sc.LB } + cc.balancerWrapper.switchTo(newBalancerName) } func (cc *ClientConn) resolveNow(o resolver.ResolveNowOptions) { @@ -974,7 +1227,10 @@ func (cc *ClientConn) resolveNow(o resolver.ResolveNowOptions) { // However, if a previously unavailable network becomes available, this may be // used to trigger an immediate reconnect. // -// This API is EXPERIMENTAL. +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. func (cc *ClientConn) ResetConnectBackoff() { cc.mu.Lock() conns := cc.conns @@ -986,51 +1242,55 @@ func (cc *ClientConn) ResetConnectBackoff() { // Close tears down the ClientConn and all underlying connections. func (cc *ClientConn) Close() error { - defer cc.cancel() + defer func() { + cc.cancel() + <-cc.csMgr.pubSub.Done() + }() cc.mu.Lock() if cc.conns == nil { cc.mu.Unlock() return ErrClientConnClosing } + + for cc.idlenessState == ccIdlenessStateExitingIdle { + cc.exitIdleCond.Wait() + } + conns := cc.conns cc.conns = nil cc.csMgr.updateState(connectivity.Shutdown) + pWrapper := cc.blockingpicker rWrapper := cc.resolverWrapper - cc.resolverWrapper = nil bWrapper := cc.balancerWrapper - cc.balancerWrapper = nil + idlenessMgr := cc.idlenessMgr cc.mu.Unlock() - cc.blockingpicker.close() - - if rWrapper != nil { - rWrapper.close() + // The order of closing matters here since the balancer wrapper assumes the + // picker is closed before it is closed. + if pWrapper != nil { + pWrapper.close() } if bWrapper != nil { bWrapper.close() } + if rWrapper != nil { + rWrapper.close() + } + if idlenessMgr != nil { + idlenessMgr.Close() + } for ac := range conns { ac.tearDown(ErrClientConnClosing) } - if channelz.IsOn() { - ted := &channelz.TraceEventDesc{ - Desc: "Channel Deleted", - Severity: channelz.CtINFO, - } - if cc.dopts.channelzParentID != 0 { - ted.Parent = &channelz.TraceEventDesc{ - Desc: fmt.Sprintf("Nested channel(id:%d) deleted", cc.channelzID), - Severity: channelz.CtINFO, - } - } - channelz.AddTraceEvent(logger, cc.channelzID, 0, ted) - // TraceEvent needs to be called before RemoveEntry, as TraceEvent may add trace reference to - // the entity being deleted, and thus prevent it from being deleted right away. - channelz.RemoveEntry(cc.channelzID) - } + cc.addTraceEvent("deleted") + // TraceEvent needs to be called before RemoveEntry, as TraceEvent may add + // trace reference to the entity being deleted, and thus prevent it from being + // deleted right away. + channelz.RemoveEntry(cc.channelzID) + return nil } @@ -1055,12 +1315,13 @@ type addrConn struct { addrs []resolver.Address // All addresses that the resolver resolved to. // Use updateConnectivityState for updating addrConn's connectivity state. - state connectivity.State + state connectivity.State + stateChan chan struct{} // closed and recreated on every state change. backoffIdx int // Needs to be stateful for resetConnectBackoff. resetBackoff chan struct{} - channelzID int64 // channelz unique identification number. + channelzID *channelz.Identifier czData *channelzData } @@ -1069,8 +1330,15 @@ func (ac *addrConn) updateConnectivityState(s connectivity.State, lastErr error) if ac.state == s { return } + // When changing states, reset the state change channel. + close(ac.stateChan) + ac.stateChan = make(chan struct{}) ac.state = s - channelz.Infof(logger, ac.channelzID, "Subchannel Connectivity change to %v", s) + if lastErr == nil { + channelz.Infof(logger, ac.channelzID, "Subchannel Connectivity change to %v", s) + } else { + channelz.Infof(logger, ac.channelzID, "Subchannel Connectivity change to %v, last error: %s", s, lastErr) + } ac.cc.handleSubConnStateChange(ac.acbw, s, lastErr) } @@ -1089,113 +1357,88 @@ func (ac *addrConn) adjustParams(r transport.GoAwayReason) { } func (ac *addrConn) resetTransport() { - for i := 0; ; i++ { - if i > 0 { - ac.cc.resolveNow(resolver.ResolveNowOptions{}) - } + ac.mu.Lock() + acCtx := ac.ctx + if acCtx.Err() != nil { + ac.mu.Unlock() + return + } + + addrs := ac.addrs + backoffFor := ac.dopts.bs.Backoff(ac.backoffIdx) + // This will be the duration that dial gets to finish. + dialDuration := minConnectTimeout + if ac.dopts.minConnectTimeout != nil { + dialDuration = ac.dopts.minConnectTimeout() + } + + if dialDuration < backoffFor { + // Give dial more time as we keep failing to connect. + dialDuration = backoffFor + } + // We can potentially spend all the time trying the first address, and + // if the server accepts the connection and then hangs, the following + // addresses will never be tried. + // + // The spec doesn't mention what should be done for multiple addresses. + // https://github.com/grpc/grpc/blob/master/doc/connection-backoff.md#proposed-backoff-algorithm + connectDeadline := time.Now().Add(dialDuration) + + ac.updateConnectivityState(connectivity.Connecting, nil) + ac.mu.Unlock() + if err := ac.tryAllAddrs(acCtx, addrs, connectDeadline); err != nil { + ac.cc.resolveNow(resolver.ResolveNowOptions{}) ac.mu.Lock() - if ac.state == connectivity.Shutdown { + if acCtx.Err() != nil { + // addrConn was torn down. ac.mu.Unlock() return } + // After exhausting all addresses, the addrConn enters + // TRANSIENT_FAILURE. + ac.updateConnectivityState(connectivity.TransientFailure, err) - addrs := ac.addrs - backoffFor := ac.dopts.bs.Backoff(ac.backoffIdx) - // This will be the duration that dial gets to finish. - dialDuration := minConnectTimeout - if ac.dopts.minConnectTimeout != nil { - dialDuration = ac.dopts.minConnectTimeout() - } - - if dialDuration < backoffFor { - // Give dial more time as we keep failing to connect. - dialDuration = backoffFor - } - // We can potentially spend all the time trying the first address, and - // if the server accepts the connection and then hangs, the following - // addresses will never be tried. - // - // The spec doesn't mention what should be done for multiple addresses. - // https://github.com/grpc/grpc/blob/master/doc/connection-backoff.md#proposed-backoff-algorithm - connectDeadline := time.Now().Add(dialDuration) - - ac.updateConnectivityState(connectivity.Connecting, nil) - ac.transport = nil + // Backoff. + b := ac.resetBackoff ac.mu.Unlock() - newTr, addr, reconnect, err := ac.tryAllAddrs(addrs, connectDeadline) - if err != nil { - // After exhausting all addresses, the addrConn enters - // TRANSIENT_FAILURE. + timer := time.NewTimer(backoffFor) + select { + case <-timer.C: ac.mu.Lock() - if ac.state == connectivity.Shutdown { - ac.mu.Unlock() - return - } - ac.updateConnectivityState(connectivity.TransientFailure, err) - - // Backoff. - b := ac.resetBackoff + ac.backoffIdx++ ac.mu.Unlock() - - timer := time.NewTimer(backoffFor) - select { - case <-timer.C: - ac.mu.Lock() - ac.backoffIdx++ - ac.mu.Unlock() - case <-b: - timer.Stop() - case <-ac.ctx.Done(): - timer.Stop() - return - } - continue + case <-b: + timer.Stop() + case <-acCtx.Done(): + timer.Stop() + return } ac.mu.Lock() - if ac.state == connectivity.Shutdown { - ac.mu.Unlock() - newTr.Close() - return + if acCtx.Err() == nil { + ac.updateConnectivityState(connectivity.Idle, err) } - ac.curAddr = addr - ac.transport = newTr - ac.backoffIdx = 0 - - hctx, hcancel := context.WithCancel(ac.ctx) - ac.startHealthCheck(hctx) ac.mu.Unlock() - - // Block until the created transport is down. And when this happens, - // we restart from the top of the addr list. - <-reconnect.Done() - hcancel() - // restart connecting - the top of the loop will set state to - // CONNECTING. This is against the current connectivity semantics doc, - // however it allows for graceful behavior for RPCs not yet dispatched - // - unfortunate timing would otherwise lead to the RPC failing even - // though the TRANSIENT_FAILURE state (called for by the doc) would be - // instantaneous. - // - // Ideally we should transition to Idle here and block until there is - // RPC activity that leads to the balancer requesting a reconnect of - // the associated SubConn. + return } + // Success; reset backoff. + ac.mu.Lock() + ac.backoffIdx = 0 + ac.mu.Unlock() } -// tryAllAddrs tries to creates a connection to the addresses, and stop when at the -// first successful one. It returns the transport, the address and a Event in -// the successful case. The Event fires when the returned transport disconnects. -func (ac *addrConn) tryAllAddrs(addrs []resolver.Address, connectDeadline time.Time) (transport.ClientTransport, resolver.Address, *grpcsync.Event, error) { +// tryAllAddrs tries to creates a connection to the addresses, and stop when at +// the first successful one. It returns an error if no address was successfully +// connected, or updates ac appropriately with the new transport. +func (ac *addrConn) tryAllAddrs(ctx context.Context, addrs []resolver.Address, connectDeadline time.Time) error { var firstConnErr error for _, addr := range addrs { - ac.mu.Lock() - if ac.state == connectivity.Shutdown { - ac.mu.Unlock() - return nil, resolver.Address{}, nil, errConnClosing + if ctx.Err() != nil { + return errConnClosing } + ac.mu.Lock() ac.cc.mu.RLock() ac.dopts.copts.KeepaliveParams = ac.cc.mkp @@ -1209,9 +1452,9 @@ func (ac *addrConn) tryAllAddrs(addrs []resolver.Address, connectDeadline time.T channelz.Infof(logger, ac.channelzID, "Subchannel picks a new address %q to connect", addr.Addr) - newTr, reconnect, err := ac.createTransport(addr, copts, connectDeadline) + err := ac.createTransport(ctx, addr, copts, connectDeadline) if err == nil { - return newTr, addr, reconnect, nil + return nil } if firstConnErr == nil { firstConnErr = err @@ -1220,86 +1463,90 @@ func (ac *addrConn) tryAllAddrs(addrs []resolver.Address, connectDeadline time.T } // Couldn't connect to any address. - return nil, resolver.Address{}, nil, firstConnErr + return firstConnErr } -// createTransport creates a connection to addr. It returns the transport and a -// Event in the successful case. The Event fires when the returned transport -// disconnects. -func (ac *addrConn) createTransport(addr resolver.Address, copts transport.ConnectOptions, connectDeadline time.Time) (transport.ClientTransport, *grpcsync.Event, error) { - prefaceReceived := make(chan struct{}) - onCloseCalled := make(chan struct{}) - reconnect := grpcsync.NewEvent() +// createTransport creates a connection to addr. It returns an error if the +// address was not successfully connected, or updates ac appropriately with the +// new transport. +func (ac *addrConn) createTransport(ctx context.Context, addr resolver.Address, copts transport.ConnectOptions, connectDeadline time.Time) error { + addr.ServerName = ac.cc.getServerName(addr) + hctx, hcancel := context.WithCancel(ctx) - // addr.ServerName takes precedent over ClientConn authority, if present. - if addr.ServerName == "" { - addr.ServerName = ac.cc.authority - } - - once := sync.Once{} - onGoAway := func(r transport.GoAwayReason) { + onClose := func(r transport.GoAwayReason) { ac.mu.Lock() + defer ac.mu.Unlock() + // adjust params based on GoAwayReason ac.adjustParams(r) - once.Do(func() { - if ac.state == connectivity.Ready { - // Prevent this SubConn from being used for new RPCs by setting its - // state to Connecting. - // - // TODO: this should be Idle when grpc-go properly supports it. - ac.updateConnectivityState(connectivity.Connecting, nil) - } - }) - ac.mu.Unlock() - reconnect.Fire() - } - - onClose := func() { - ac.mu.Lock() - once.Do(func() { - if ac.state == connectivity.Ready { - // Prevent this SubConn from being used for new RPCs by setting its - // state to Connecting. - // - // TODO: this should be Idle when grpc-go properly supports it. - ac.updateConnectivityState(connectivity.Connecting, nil) - } - }) - ac.mu.Unlock() - close(onCloseCalled) - reconnect.Fire() - } - - onPrefaceReceipt := func() { - close(prefaceReceived) + if ctx.Err() != nil { + // Already shut down or connection attempt canceled. tearDown() or + // updateAddrs() already cleared the transport and canceled hctx + // via ac.ctx, and we expected this connection to be closed, so do + // nothing here. + return + } + hcancel() + if ac.transport == nil { + // We're still connecting to this address, which could error. Do + // not update the connectivity state or resolve; these will happen + // at the end of the tryAllAddrs connection loop in the event of an + // error. + return + } + ac.transport = nil + // Refresh the name resolver on any connection loss. + ac.cc.resolveNow(resolver.ResolveNowOptions{}) + // Always go idle and wait for the LB policy to initiate a new + // connection attempt. + ac.updateConnectivityState(connectivity.Idle, nil) } - connectCtx, cancel := context.WithDeadline(ac.ctx, connectDeadline) + connectCtx, cancel := context.WithDeadline(ctx, connectDeadline) defer cancel() - if channelz.IsOn() { - copts.ChannelzParentID = ac.channelzID - } + copts.ChannelzParentID = ac.channelzID - newTr, err := transport.NewClientTransport(connectCtx, ac.cc.ctx, addr, copts, onPrefaceReceipt, onGoAway, onClose) + newTr, err := transport.NewClientTransport(connectCtx, ac.cc.ctx, addr, copts, onClose) if err != nil { + if logger.V(2) { + logger.Infof("Creating new client transport to %q: %v", addr, err) + } // newTr is either nil, or closed. - channelz.Warningf(logger, ac.channelzID, "grpc: addrConn.createTransport failed to connect to %v. Err: %v. Reconnecting...", addr, err) - return nil, nil, err + hcancel() + channelz.Warningf(logger, ac.channelzID, "grpc: addrConn.createTransport failed to connect to %s. Err: %v", addr, err) + return err } - select { - case <-time.After(time.Until(connectDeadline)): - // We didn't get the preface in time. - newTr.Close() - channelz.Warningf(logger, ac.channelzID, "grpc: addrConn.createTransport failed to connect to %v: didn't receive server preface in time. Reconnecting...", addr) - return nil, nil, errors.New("timed out waiting for server handshake") - case <-prefaceReceived: - // We got the preface - huzzah! things are good. - case <-onCloseCalled: - // The transport has already closed - noop. - return nil, nil, errors.New("connection closed") - // TODO(deklerk) this should bail on ac.ctx.Done(). Add a test and fix. + ac.mu.Lock() + defer ac.mu.Unlock() + if ctx.Err() != nil { + // This can happen if the subConn was removed while in `Connecting` + // state. tearDown() would have set the state to `Shutdown`, but + // would not have closed the transport since ac.transport would not + // have been set at that point. + // + // We run this in a goroutine because newTr.Close() calls onClose() + // inline, which requires locking ac.mu. + // + // The error we pass to Close() is immaterial since there are no open + // streams at this point, so no trailers with error details will be sent + // out. We just need to pass a non-nil error. + // + // This can also happen when updateAddrs is called during a connection + // attempt. + go newTr.Close(transport.ErrConnClosing) + return nil + } + if hctx.Err() != nil { + // onClose was already called for this connection, but the connection + // was successfully established first. Consider it a success and set + // the new state to Idle. + ac.updateConnectivityState(connectivity.Idle, nil) + return nil } - return newTr, reconnect, nil + ac.curAddr = addr + ac.transport = newTr + ac.startHealthCheck(hctx) // Will set state to READY if appropriate. + return nil } // startHealthCheck starts the health checking stream (RPC) to watch the health @@ -1345,7 +1592,7 @@ func (ac *addrConn) startHealthCheck(ctx context.Context) { // Set up the health check helper functions. currentTr := ac.transport - newStream := func(method string) (interface{}, error) { + newStream := func(method string) (any, error) { ac.mu.Lock() if ac.transport != currentTr { ac.mu.Unlock() @@ -1369,7 +1616,7 @@ func (ac *addrConn) startHealthCheck(ctx context.Context) { if status.Code(err) == codes.Unimplemented { channelz.Error(logger, ac.channelzID, "Subchannel health check is unimplemented at server side, thus health check is disabled") } else { - channelz.Errorf(logger, ac.channelzID, "HealthCheckFunc exits with unexpected error %v", err) + channelz.Errorf(logger, ac.channelzID, "Health checking failed: %v", err) } } }() @@ -1383,33 +1630,43 @@ func (ac *addrConn) resetConnectBackoff() { ac.mu.Unlock() } -// getReadyTransport returns the transport if ac's state is READY. -// Otherwise it returns nil, false. -// If ac's state is IDLE, it will trigger ac to connect. -func (ac *addrConn) getReadyTransport() (transport.ClientTransport, bool) { +// getReadyTransport returns the transport if ac's state is READY or nil if not. +func (ac *addrConn) getReadyTransport() transport.ClientTransport { ac.mu.Lock() - if ac.state == connectivity.Ready && ac.transport != nil { - t := ac.transport - ac.mu.Unlock() - return t, true - } - var idle bool - if ac.state == connectivity.Idle { - idle = true + defer ac.mu.Unlock() + if ac.state == connectivity.Ready { + return ac.transport } - ac.mu.Unlock() - // Trigger idle ac to connect. - if idle { - ac.connect() + return nil +} + +// getTransport waits until the addrconn is ready and returns the transport. +// If the context expires first, returns an appropriate status. If the +// addrConn is stopped first, returns an Unavailable status error. +func (ac *addrConn) getTransport(ctx context.Context) (transport.ClientTransport, error) { + for ctx.Err() == nil { + ac.mu.Lock() + t, state, sc := ac.transport, ac.state, ac.stateChan + ac.mu.Unlock() + if state == connectivity.Ready { + return t, nil + } + if state == connectivity.Shutdown { + return nil, status.Errorf(codes.Unavailable, "SubConn shutting down") + } + + select { + case <-ctx.Done(): + case <-sc: + } } - return nil, false + return nil, status.FromContextError(ctx.Err()).Err() } // tearDown starts to tear down the addrConn. -// TODO(zhaoq): Make this synchronous to avoid unbounded memory consumption in -// some edge cases (e.g., the caller opens and closes many addrConn's in a -// tight loop. -// tearDown doesn't remove ac from ac.cc.conns. +// +// Note that tearDown doesn't remove ac from ac.cc.conns, so the addrConn struct +// will leak. In most cases, call cc.removeAddrConn() instead. func (ac *addrConn) tearDown(err error) { ac.mu.Lock() if ac.state == connectivity.Shutdown { @@ -1433,19 +1690,18 @@ func (ac *addrConn) tearDown(err error) { curTr.GracefulClose() ac.mu.Lock() } - if channelz.IsOn() { - channelz.AddTraceEvent(logger, ac.channelzID, 0, &channelz.TraceEventDesc{ - Desc: "Subchannel Deleted", - Severity: channelz.CtINFO, - Parent: &channelz.TraceEventDesc{ - Desc: fmt.Sprintf("Subchanel(id:%d) deleted", ac.channelzID), - Severity: channelz.CtINFO, - }, - }) - // TraceEvent needs to be called before RemoveEntry, as TraceEvent may add trace reference to - // the entity being deleted, and thus prevent it from being deleted right away. - channelz.RemoveEntry(ac.channelzID) - } + channelz.AddTraceEvent(logger, ac.channelzID, 0, &channelz.TraceEventDesc{ + Desc: "Subchannel deleted", + Severity: channelz.CtInfo, + Parent: &channelz.TraceEventDesc{ + Desc: fmt.Sprintf("Subchannel(id:%d) deleted", ac.channelzID.Int()), + Severity: channelz.CtInfo, + }, + }) + // TraceEvent needs to be called before RemoveEntry, as TraceEvent may add + // trace reference to the entity being deleted, and thus prevent it from + // being deleted right away. + channelz.RemoveEntry(ac.channelzID) ac.mu.Unlock() } @@ -1534,6 +1790,9 @@ func (c *channelzChannel) ChannelzMetric() *channelz.ChannelInternalMetric { // referenced by users. var ErrClientConnTimeout = errors.New("grpc: timed out when dialing") +// getResolver finds the scheme in the cc's resolvers or the global registry. +// scheme should always be lowercase (typically by virtue of url.Parse() +// performing proper RFC3986 behavior). func (cc *ClientConn) getResolver(scheme string) resolver.Builder { for _, rb := range cc.dopts.resolvers { if scheme == rb.Scheme() { @@ -1554,3 +1813,206 @@ func (cc *ClientConn) connectionError() error { defer cc.lceMu.Unlock() return cc.lastConnectionError } + +// parseTargetAndFindResolver parses the user's dial target and stores the +// parsed target in `cc.parsedTarget`. +// +// The resolver to use is determined based on the scheme in the parsed target +// and the same is stored in `cc.resolverBuilder`. +// +// Doesn't grab cc.mu as this method is expected to be called only at Dial time. +func (cc *ClientConn) parseTargetAndFindResolver() error { + channelz.Infof(logger, cc.channelzID, "original dial target is: %q", cc.target) + + var rb resolver.Builder + parsedTarget, err := parseTarget(cc.target) + if err != nil { + channelz.Infof(logger, cc.channelzID, "dial target %q parse failed: %v", cc.target, err) + } else { + channelz.Infof(logger, cc.channelzID, "parsed dial target is: %+v", parsedTarget) + rb = cc.getResolver(parsedTarget.URL.Scheme) + if rb != nil { + cc.parsedTarget = parsedTarget + cc.resolverBuilder = rb + return nil + } + } + + // We are here because the user's dial target did not contain a scheme or + // specified an unregistered scheme. We should fallback to the default + // scheme, except when a custom dialer is specified in which case, we should + // always use passthrough scheme. + defScheme := resolver.GetDefaultScheme() + channelz.Infof(logger, cc.channelzID, "fallback to scheme %q", defScheme) + canonicalTarget := defScheme + ":///" + cc.target + + parsedTarget, err = parseTarget(canonicalTarget) + if err != nil { + channelz.Infof(logger, cc.channelzID, "dial target %q parse failed: %v", canonicalTarget, err) + return err + } + channelz.Infof(logger, cc.channelzID, "parsed dial target is: %+v", parsedTarget) + rb = cc.getResolver(parsedTarget.URL.Scheme) + if rb == nil { + return fmt.Errorf("could not get resolver for default scheme: %q", parsedTarget.URL.Scheme) + } + cc.parsedTarget = parsedTarget + cc.resolverBuilder = rb + return nil +} + +// parseTarget uses RFC 3986 semantics to parse the given target into a +// resolver.Target struct containing url. Query params are stripped from the +// endpoint. +func parseTarget(target string) (resolver.Target, error) { + u, err := url.Parse(target) + if err != nil { + return resolver.Target{}, err + } + + return resolver.Target{URL: *u}, nil +} + +func encodeAuthority(authority string) string { + const upperhex = "0123456789ABCDEF" + + // Return for characters that must be escaped as per + // Valid chars are mentioned here: + // https://datatracker.ietf.org/doc/html/rfc3986#section-3.2 + shouldEscape := func(c byte) bool { + // Alphanum are always allowed. + if 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || '0' <= c && c <= '9' { + return false + } + switch c { + case '-', '_', '.', '~': // Unreserved characters + return false + case '!', '$', '&', '\'', '(', ')', '*', '+', ',', ';', '=': // Subdelim characters + return false + case ':', '[', ']', '@': // Authority related delimeters + return false + } + // Everything else must be escaped. + return true + } + + hexCount := 0 + for i := 0; i < len(authority); i++ { + c := authority[i] + if shouldEscape(c) { + hexCount++ + } + } + + if hexCount == 0 { + return authority + } + + required := len(authority) + 2*hexCount + t := make([]byte, required) + + j := 0 + // This logic is a barebones version of escape in the go net/url library. + for i := 0; i < len(authority); i++ { + switch c := authority[i]; { + case shouldEscape(c): + t[j] = '%' + t[j+1] = upperhex[c>>4] + t[j+2] = upperhex[c&15] + j += 3 + default: + t[j] = authority[i] + j++ + } + } + return string(t) +} + +// Determine channel authority. The order of precedence is as follows: +// - user specified authority override using `WithAuthority` dial option +// - creds' notion of server name for the authentication handshake +// - endpoint from dial target of the form "scheme://[authority]/endpoint" +// +// Stores the determined authority in `cc.authority`. +// +// Returns a non-nil error if the authority returned by the transport +// credentials do not match the authority configured through the dial option. +// +// Doesn't grab cc.mu as this method is expected to be called only at Dial time. +func (cc *ClientConn) determineAuthority() error { + dopts := cc.dopts + // Historically, we had two options for users to specify the serverName or + // authority for a channel. One was through the transport credentials + // (either in its constructor, or through the OverrideServerName() method). + // The other option (for cases where WithInsecure() dial option was used) + // was to use the WithAuthority() dial option. + // + // A few things have changed since: + // - `insecure` package with an implementation of the `TransportCredentials` + // interface for the insecure case + // - WithAuthority() dial option support for secure credentials + authorityFromCreds := "" + if creds := dopts.copts.TransportCredentials; creds != nil && creds.Info().ServerName != "" { + authorityFromCreds = creds.Info().ServerName + } + authorityFromDialOption := dopts.authority + if (authorityFromCreds != "" && authorityFromDialOption != "") && authorityFromCreds != authorityFromDialOption { + return fmt.Errorf("ClientConn's authority from transport creds %q and dial option %q don't match", authorityFromCreds, authorityFromDialOption) + } + + endpoint := cc.parsedTarget.Endpoint() + target := cc.target + switch { + case authorityFromDialOption != "": + cc.authority = authorityFromDialOption + case authorityFromCreds != "": + cc.authority = authorityFromCreds + case strings.HasPrefix(target, "unix:") || strings.HasPrefix(target, "unix-abstract:"): + // TODO: remove when the unix resolver implements optional interface to + // return channel authority. + cc.authority = "localhost" + case strings.HasPrefix(endpoint, ":"): + cc.authority = "localhost" + endpoint + default: + // TODO: Define an optional interface on the resolver builder to return + // the channel authority given the user's dial target. For resolvers + // which don't implement this interface, we will use the endpoint from + // "scheme://authority/endpoint" as the default authority. + // Escape the endpoint to handle use cases where the endpoint + // might not be a valid authority by default. + // For example an endpoint which has multiple paths like + // 'a/b/c', which is not a valid authority by default. + cc.authority = encodeAuthority(endpoint) + } + channelz.Infof(logger, cc.channelzID, "Channel authority set to %q", cc.authority) + return nil +} + +// initResolverWrapper creates a ccResolverWrapper, which builds the name +// resolver. This method grabs the lock to assign the newly built resolver +// wrapper to the cc.resolverWrapper field. +func (cc *ClientConn) initResolverWrapper(creds credentials.TransportCredentials) error { + rw, err := newCCResolverWrapper(cc, ccResolverWrapperOpts{ + target: cc.parsedTarget, + builder: cc.resolverBuilder, + bOpts: resolver.BuildOptions{ + DisableServiceConfig: cc.dopts.disableServiceConfig, + DialCreds: creds, + CredsBundle: cc.dopts.copts.CredsBundle, + Dialer: cc.dopts.copts.Dialer, + }, + channelzID: cc.channelzID, + }) + if err != nil { + return fmt.Errorf("failed to build resolver: %v", err) + } + // Resolver implementations may report state update or error inline when + // built (or right after), and this is handled in cc.updateResolverState. + // Also, an error from the resolver might lead to a re-resolution request + // from the balancer, which is handled in resolveNow() where + // `cc.resolverWrapper` is accessed. Hence, we need to hold the lock here. + cc.mu.Lock() + cc.resolverWrapper = rw + cc.mu.Unlock() + return nil +} diff --git a/clientconn_authority_test.go b/clientconn_authority_test.go new file mode 100644 index 000000000000..3efb2ae8571e --- /dev/null +++ b/clientconn_authority_test.go @@ -0,0 +1,135 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "context" + "net" + "testing" + + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/testdata" +) + +func (s) TestClientConnAuthority(t *testing.T) { + serverNameOverride := "over.write.server.name" + creds, err := credentials.NewClientTLSFromFile(testdata.Path("x509/server_ca_cert.pem"), serverNameOverride) + if err != nil { + t.Fatalf("credentials.NewClientTLSFromFile(_, %q) failed: %v", err, serverNameOverride) + } + + tests := []struct { + name string + target string + opts []DialOption + wantAuthority string + }{ + { + name: "default", + target: "Non-Existent.Server:8080", + opts: []DialOption{WithTransportCredentials(insecure.NewCredentials())}, + wantAuthority: "Non-Existent.Server:8080", + }, + { + name: "override-via-creds", + target: "Non-Existent.Server:8080", + opts: []DialOption{WithTransportCredentials(creds)}, + wantAuthority: serverNameOverride, + }, + { + name: "override-via-WithAuthority", + target: "Non-Existent.Server:8080", + opts: []DialOption{WithTransportCredentials(insecure.NewCredentials()), WithAuthority("authority-override")}, + wantAuthority: "authority-override", + }, + { + name: "override-via-creds-and-WithAuthority", + target: "Non-Existent.Server:8080", + opts: []DialOption{WithTransportCredentials(creds), WithAuthority(serverNameOverride)}, + wantAuthority: serverNameOverride, + }, + { + name: "unix relative", + target: "unix:sock.sock", + opts: []DialOption{WithTransportCredentials(insecure.NewCredentials())}, + wantAuthority: "localhost", + }, + { + name: "unix relative with custom dialer", + target: "unix:sock.sock", + opts: []DialOption{WithTransportCredentials(insecure.NewCredentials()), WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, "", addr) + })}, + wantAuthority: "localhost", + }, + { + name: "unix absolute", + target: "unix:/sock.sock", + opts: []DialOption{WithTransportCredentials(insecure.NewCredentials())}, + wantAuthority: "localhost", + }, + { + name: "unix absolute with custom dialer", + target: "unix:///sock.sock", + opts: []DialOption{WithTransportCredentials(insecure.NewCredentials()), WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, "", addr) + })}, + wantAuthority: "localhost", + }, + { + name: "localhost colon port", + target: "localhost:50051", + opts: []DialOption{WithTransportCredentials(insecure.NewCredentials())}, + wantAuthority: "localhost:50051", + }, + { + name: "colon port", + target: ":50051", + opts: []DialOption{WithTransportCredentials(insecure.NewCredentials())}, + wantAuthority: "localhost:50051", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cc, err := Dial(test.target, test.opts...) + if err != nil { + t.Fatalf("Dial(%q) failed: %v", test.target, err) + } + defer cc.Close() + if cc.authority != test.wantAuthority { + t.Fatalf("cc.authority = %q, want %q", cc.authority, test.wantAuthority) + } + }) + } +} + +func (s) TestClientConnAuthority_CredsAndDialOptionMismatch(t *testing.T) { + serverNameOverride := "over.write.server.name" + creds, err := credentials.NewClientTLSFromFile(testdata.Path("x509/server_ca_cert.pem"), serverNameOverride) + if err != nil { + t.Fatalf("credentials.NewClientTLSFromFile(_, %q) failed: %v", err, serverNameOverride) + } + opts := []DialOption{WithTransportCredentials(creds), WithAuthority("authority-override")} + if cc, err := Dial("Non-Existent.Server:8000", opts...); err == nil { + cc.Close() + t.Fatal("grpc.Dial() succeeded when expected to fail") + } +} diff --git a/clientconn_parsed_target_test.go b/clientconn_parsed_target_test.go new file mode 100644 index 000000000000..1ff46aaf08c7 --- /dev/null +++ b/clientconn_parsed_target_test.go @@ -0,0 +1,183 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "context" + "errors" + "fmt" + "net" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/testutils" + + "google.golang.org/grpc/resolver" +) + +func (s) TestParsedTarget_Success_WithoutCustomDialer(t *testing.T) { + defScheme := resolver.GetDefaultScheme() + tests := []struct { + target string + wantParsed resolver.Target + }{ + // No scheme is specified. + {target: "://a/b", wantParsed: resolver.Target{URL: *testutils.MustParseURL(fmt.Sprintf("%s:///%s", defScheme, "://a/b"))}}, + {target: "a//b", wantParsed: resolver.Target{URL: *testutils.MustParseURL(fmt.Sprintf("%s:///%s", defScheme, "a//b"))}}, + + // An unregistered scheme is specified. + {target: "a:///", wantParsed: resolver.Target{URL: *testutils.MustParseURL(fmt.Sprintf("%s:///%s", defScheme, "a:///"))}}, + {target: "a:b", wantParsed: resolver.Target{URL: *testutils.MustParseURL(fmt.Sprintf("%s:///%s", defScheme, "a:b"))}}, + + // A registered scheme is specified. + {target: "dns://a.server.com/google.com", wantParsed: resolver.Target{URL: *testutils.MustParseURL("dns://a.server.com/google.com")}}, + {target: "unix-abstract:/ a///://::!@#$%25^&*()b", wantParsed: resolver.Target{URL: *testutils.MustParseURL("unix-abstract:/ a///://::!@#$%25^&*()b")}}, + {target: "unix-abstract:passthrough:abc", wantParsed: resolver.Target{URL: *testutils.MustParseURL("unix-abstract:passthrough:abc")}}, + {target: "passthrough:///unix:///a/b/c", wantParsed: resolver.Target{URL: *testutils.MustParseURL("passthrough:///unix:///a/b/c")}}, + + // Cases for `scheme:absolute-path`. + {target: "dns:/a/b/c", wantParsed: resolver.Target{URL: *testutils.MustParseURL("dns:/a/b/c")}}, + {target: "unregistered:/a/b/c", wantParsed: resolver.Target{URL: *testutils.MustParseURL(fmt.Sprintf("%s:///%s", defScheme, "unregistered:/a/b/c"))}}, + } + + for _, test := range tests { + t.Run(test.target, func(t *testing.T) { + cc, err := Dial(test.target, WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Dial(%q) failed: %v", test.target, err) + } + defer cc.Close() + + if !cmp.Equal(cc.parsedTarget, test.wantParsed) { + t.Errorf("cc.parsedTarget for dial target %q = %+v, want %+v", test.target, cc.parsedTarget, test.wantParsed) + } + }) + } +} + +func (s) TestParsedTarget_Failure_WithoutCustomDialer(t *testing.T) { + targets := []string{ + "", + "unix://a/b/c", + "unix://authority", + "unix-abstract://authority/a/b/c", + "unix-abstract://authority", + } + + for _, target := range targets { + t.Run(target, func(t *testing.T) { + if cc, err := Dial(target, WithTransportCredentials(insecure.NewCredentials())); err == nil { + defer cc.Close() + t.Fatalf("Dial(%q) succeeded cc.parsedTarget = %+v, expected to fail", target, cc.parsedTarget) + } + }) + } +} + +func (s) TestParsedTarget_WithCustomDialer(t *testing.T) { + defScheme := resolver.GetDefaultScheme() + tests := []struct { + target string + wantParsed resolver.Target + wantDialerAddress string + }{ + // unix:[local_path], unix:[/absolute], and unix://[/absolute] have + // different behaviors with a custom dialer. + { + target: "unix:a/b/c", + wantParsed: resolver.Target{URL: *testutils.MustParseURL("unix:a/b/c")}, + wantDialerAddress: "unix:a/b/c", + }, + { + target: "unix:/a/b/c", + wantParsed: resolver.Target{URL: *testutils.MustParseURL("unix:/a/b/c")}, + wantDialerAddress: "unix:///a/b/c", + }, + { + target: "unix:///a/b/c", + wantParsed: resolver.Target{URL: *testutils.MustParseURL("unix:///a/b/c")}, + wantDialerAddress: "unix:///a/b/c", + }, + { + target: "dns:///127.0.0.1:50051", + wantParsed: resolver.Target{URL: *testutils.MustParseURL("dns:///127.0.0.1:50051")}, + wantDialerAddress: "127.0.0.1:50051", + }, + { + target: ":///127.0.0.1:50051", + wantParsed: resolver.Target{URL: *testutils.MustParseURL(fmt.Sprintf("%s:///%s", defScheme, ":///127.0.0.1:50051"))}, + wantDialerAddress: ":///127.0.0.1:50051", + }, + { + target: "dns://authority/127.0.0.1:50051", + wantParsed: resolver.Target{URL: *testutils.MustParseURL("dns://authority/127.0.0.1:50051")}, + wantDialerAddress: "127.0.0.1:50051", + }, + { + target: "://authority/127.0.0.1:50051", + wantParsed: resolver.Target{URL: *testutils.MustParseURL(fmt.Sprintf("%s:///%s", defScheme, "://authority/127.0.0.1:50051"))}, + wantDialerAddress: "://authority/127.0.0.1:50051", + }, + { + target: "/unix/socket/address", + wantParsed: resolver.Target{URL: *testutils.MustParseURL(fmt.Sprintf("%s:///%s", defScheme, "/unix/socket/address"))}, + wantDialerAddress: "/unix/socket/address", + }, + { + target: "", + wantParsed: resolver.Target{URL: *testutils.MustParseURL(fmt.Sprintf("%s:///%s", defScheme, ""))}, + wantDialerAddress: "", + }, + { + target: "passthrough://a.server.com/google.com", + wantParsed: resolver.Target{URL: *testutils.MustParseURL("passthrough://a.server.com/google.com")}, + wantDialerAddress: "google.com", + }, + } + + for _, test := range tests { + t.Run(test.target, func(t *testing.T) { + addrCh := make(chan string, 1) + dialer := func(ctx context.Context, address string) (net.Conn, error) { + addrCh <- address + return nil, errors.New("dialer error") + } + + cc, err := Dial(test.target, WithTransportCredentials(insecure.NewCredentials()), WithContextDialer(dialer)) + if err != nil { + t.Fatalf("Dial(%q) failed: %v", test.target, err) + } + defer cc.Close() + + select { + case addr := <-addrCh: + if addr != test.wantDialerAddress { + t.Fatalf("address in custom dialer is %q, want %q", addr, test.wantDialerAddress) + } + case <-time.After(time.Second): + t.Fatal("timeout when waiting for custom dialer to be invoked") + } + if !cmp.Equal(cc.parsedTarget, test.wantParsed) { + t.Errorf("cc.parsedTarget for dial target %q = %+v, want %+v", test.target, cc.parsedTarget, test.wantParsed) + } + }) + } +} diff --git a/clientconn_test.go b/clientconn_test.go index 6c61666b7efb..e614045bee01 100644 --- a/clientconn_test.go +++ b/clientconn_test.go @@ -25,22 +25,47 @@ import ( "math" "net" "strings" + "sync" "sync/atomic" "testing" "time" "golang.org/x/net/http2" "google.golang.org/grpc/backoff" + "google.golang.org/grpc/balancer" "google.golang.org/grpc/connectivity" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" internalbackoff "google.golang.org/grpc/internal/backoff" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/grpctest" "google.golang.org/grpc/internal/transport" "google.golang.org/grpc/keepalive" "google.golang.org/grpc/resolver" "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/serviceconfig" "google.golang.org/grpc/testdata" ) +const ( + defaultTestTimeout = 10 * time.Second + stateRecordingBalancerName = "state_recording_balancer" +) + +var testBalancerBuilder = newStateRecordingBalancerBuilder() + +func init() { + balancer.Register(testBalancerBuilder) +} + +func parseCfg(r *manual.Resolver, s string) *serviceconfig.ParseResult { + scpr := r.CC.ParseServiceConfig(s) + if scpr.Err != nil { + panic(fmt.Sprintf("Error parsing config %q: %v", s, scpr.Err)) + } + return scpr +} + func (s) TestDialWithTimeout(t *testing.T) { lis, err := net.Listen("tcp", "localhost:0") if err != nil { @@ -68,7 +93,7 @@ func (s) TestDialWithTimeout(t *testing.T) { r := manual.NewBuilderWithScheme("whatever") r.InitialState(resolver.State{Addresses: []resolver.Address{lisAddr}}) - client, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithResolvers(r), WithTimeout(5*time.Second)) + client, err := Dial(r.Scheme()+":///test.server", WithTransportCredentials(insecure.NewCredentials()), WithResolvers(r), WithTimeout(5*time.Second)) close(dialDone) if err != nil { t.Fatalf("Dial failed. Err: %v", err) @@ -120,7 +145,7 @@ func (s) TestDialWithMultipleBackendsNotSendingServerPreface(t *testing.T) { r := manual.NewBuilderWithScheme("whatever") r.InitialState(resolver.State{Addresses: []resolver.Address{lis1Addr, lis2Addr}}) - client, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithResolvers(r)) + client, err := Dial(r.Scheme()+":///test.server", WithTransportCredentials(insecure.NewCredentials()), WithResolvers(r)) if err != nil { t.Fatalf("Dial failed. Err: %v", err) } @@ -170,7 +195,7 @@ func (s) TestDialWaitsForServerSettings(t *testing.T) { }() ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - client, err := DialContext(ctx, lis.Addr().String(), WithInsecure(), WithBlock()) + client, err := DialContext(ctx, lis.Addr().String(), WithTransportCredentials(insecure.NewCredentials()), WithBlock()) close(dialDone) if err != nil { t.Fatalf("Error while dialing. Err: %v", err) @@ -208,16 +233,18 @@ func (s) TestDialWaitsForServerSettingsAndFails(t *testing.T) { defer cancel() client, err := DialContext(ctx, lis.Addr().String(), - WithInsecure(), + WithTransportCredentials(insecure.NewCredentials()), WithReturnConnectionError(), - withBackoff(noBackoff{}), - withMinConnectDeadline(func() time.Duration { return time.Second / 4 })) + WithConnectParams(ConnectParams{ + Backoff: backoff.Config{}, + MinConnectTimeout: 250 * time.Millisecond, + })) lis.Close() if err == nil { client.Close() t.Fatalf("Unexpected success (err=nil) while dialing") } - expectedMsg := "server handshake" + expectedMsg := "server preface" if !strings.Contains(err.Error(), context.DeadlineExceeded.Error()) || !strings.Contains(err.Error(), expectedMsg) { t.Fatalf("DialContext(_) = %v; want a message that includes both %q and %q", err, context.DeadlineExceeded.Error(), expectedMsg) } @@ -285,10 +312,13 @@ func (s) TestCloseConnectionWhenServerPrefaceNotReceived(t *testing.T) { break } }() - client, err := Dial(lis.Addr().String(), WithInsecure(), withMinConnectDeadline(func() time.Duration { return time.Millisecond * 500 })) + client, err := Dial(lis.Addr().String(), WithTransportCredentials(insecure.NewCredentials()), withMinConnectDeadline(func() time.Duration { return time.Millisecond * 500 })) if err != nil { t.Fatalf("Error while dialing. Err: %v", err) } + + go stayConnected(client) + // wait for connection to be accepted on the server. timer := time.NewTimer(time.Second * 10) select { @@ -306,14 +336,12 @@ func (s) TestCloseConnectionWhenServerPrefaceNotReceived(t *testing.T) { func (s) TestBackoffWhenNoServerPrefaceReceived(t *testing.T) { lis, err := net.Listen("tcp", "localhost:0") if err != nil { - t.Fatalf("Error while listening. Err: %v", err) + t.Fatalf("Unexpected error from net.Listen(%q, %q): %v", "tcp", "localhost:0", err) } defer lis.Close() done := make(chan struct{}) go func() { // Launch the server. - defer func() { - close(done) - }() + defer close(done) conn, err := lis.Accept() // Accept the connection only to close it immediately. if err != nil { t.Errorf("Error while accepting. Err: %v", err) @@ -340,17 +368,30 @@ func (s) TestBackoffWhenNoServerPrefaceReceived(t *testing.T) { prevAt = meow } }() - client, err := Dial(lis.Addr().String(), WithInsecure()) + bc := backoff.Config{ + BaseDelay: 200 * time.Millisecond, + Multiplier: 2.0, + Jitter: 0, + MaxDelay: 120 * time.Second, + } + cp := ConnectParams{ + Backoff: bc, + MinConnectTimeout: 1 * time.Second, + } + cc, err := Dial(lis.Addr().String(), WithTransportCredentials(insecure.NewCredentials()), WithConnectParams(cp)) if err != nil { - t.Fatalf("Error while dialing. Err: %v", err) + t.Fatalf("Unexpected error from Dial(%v) = %v", lis.Addr(), err) } - defer client.Close() + defer cc.Close() + go stayConnected(cc) <-done - } func (s) TestWithTimeout(t *testing.T) { - conn, err := Dial("passthrough:///Non-Existent.Server:80", WithTimeout(time.Millisecond), WithBlock(), WithInsecure()) + conn, err := Dial("passthrough:///Non-Existent.Server:80", + WithTimeout(time.Millisecond), + WithBlock(), + WithTransportCredentials(insecure.NewCredentials())) if err == nil { conn.Close() } @@ -375,62 +416,6 @@ func (s) TestWithTransportCredentialsTLS(t *testing.T) { } } -func (s) TestDefaultAuthority(t *testing.T) { - target := "Non-Existent.Server:8080" - conn, err := Dial(target, WithInsecure()) - if err != nil { - t.Fatalf("Dial(_, _) = _, %v, want _, ", err) - } - defer conn.Close() - if conn.authority != target { - t.Fatalf("%v.authority = %v, want %v", conn, conn.authority, target) - } -} - -func (s) TestTLSServerNameOverwrite(t *testing.T) { - overwriteServerName := "over.write.server.name" - creds, err := credentials.NewClientTLSFromFile(testdata.Path("x509/server_ca_cert.pem"), overwriteServerName) - if err != nil { - t.Fatalf("Failed to create credentials %v", err) - } - conn, err := Dial("passthrough:///Non-Existent.Server:80", WithTransportCredentials(creds)) - if err != nil { - t.Fatalf("Dial(_, _) = _, %v, want _, ", err) - } - defer conn.Close() - if conn.authority != overwriteServerName { - t.Fatalf("%v.authority = %v, want %v", conn, conn.authority, overwriteServerName) - } -} - -func (s) TestWithAuthority(t *testing.T) { - overwriteServerName := "over.write.server.name" - conn, err := Dial("passthrough:///Non-Existent.Server:80", WithInsecure(), WithAuthority(overwriteServerName)) - if err != nil { - t.Fatalf("Dial(_, _) = _, %v, want _, ", err) - } - defer conn.Close() - if conn.authority != overwriteServerName { - t.Fatalf("%v.authority = %v, want %v", conn, conn.authority, overwriteServerName) - } -} - -func (s) TestWithAuthorityAndTLS(t *testing.T) { - overwriteServerName := "over.write.server.name" - creds, err := credentials.NewClientTLSFromFile(testdata.Path("x509/server_ca_cert.pem"), overwriteServerName) - if err != nil { - t.Fatalf("Failed to create credentials %v", err) - } - conn, err := Dial("passthrough:///Non-Existent.Server:80", WithTransportCredentials(creds), WithAuthority("no.effect.authority")) - if err != nil { - t.Fatalf("Dial(_, _) = _, %v, want _, ", err) - } - defer conn.Close() - if conn.authority != overwriteServerName { - t.Fatalf("%v.authority = %v, want %v", conn, conn.authority, overwriteServerName) - } -} - // When creating a transport configured with n addresses, only calculate the // backoff once per "round" of attempts instead of once per address (n times // per "round" of attempts). @@ -493,8 +478,7 @@ func (s) TestDial_OneBackoffPerRetryGroup(t *testing.T) { {Addr: lis2.Addr().String()}, }}) client, err := DialContext(ctx, "whatever:///this-gets-overwritten", - WithInsecure(), - WithBalancerName(stateRecordingBalancerName), + WithTransportCredentials(insecure.NewCredentials()), WithResolvers(rb), withMinConnectDeadline(getMinConnectTimeout)) if err != nil { @@ -520,7 +504,7 @@ func (s) TestDial_OneBackoffPerRetryGroup(t *testing.T) { func (s) TestDialContextCancel(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - if _, err := DialContext(ctx, "Non-Existent.Server:80", WithBlock(), WithInsecure()); err != context.Canceled { + if _, err := DialContext(ctx, "Non-Existent.Server:80", WithBlock(), WithTransportCredentials(insecure.NewCredentials())); err != context.Canceled { t.Fatalf("DialContext(%v, _) = _, %v, want _, %v", ctx, err, context.Canceled) } } @@ -538,36 +522,59 @@ func (s) TestDialContextFailFast(t *testing.T) { return nil, failErr } - _, err := DialContext(ctx, "Non-Existent.Server:80", WithBlock(), WithInsecure(), WithDialer(dialer), FailOnNonTempDialError(true)) + _, err := DialContext(ctx, "Non-Existent.Server:80", WithBlock(), WithTransportCredentials(insecure.NewCredentials()), WithDialer(dialer), FailOnNonTempDialError(true)) if terr, ok := err.(transport.ConnectionError); !ok || terr.Origin() != failErr { t.Fatalf("DialContext() = _, %v, want _, %v", err, failErr) } } // securePerRPCCredentials always requires transport security. -type securePerRPCCredentials struct{} - -func (c securePerRPCCredentials) GetRequestMetadata(ctx context.Context, uri ...string) (map[string]string, error) { - return nil, nil +type securePerRPCCredentials struct { + credentials.PerRPCCredentials } func (c securePerRPCCredentials) RequireTransportSecurity() bool { return true } +type fakeBundleCreds struct { + credentials.Bundle + transportCreds credentials.TransportCredentials +} + +func (b *fakeBundleCreds) TransportCredentials() credentials.TransportCredentials { + return b.transportCreds +} + func (s) TestCredentialsMisuse(t *testing.T) { - tlsCreds, err := credentials.NewClientTLSFromFile(testdata.Path("x509/server_ca_cert.pem"), "x.test.example.com") + // Use of no transport creds and no creds bundle must fail. + if _, err := Dial("passthrough:///Non-Existent.Server:80"); err != errNoTransportSecurity { + t.Fatalf("Dial(_, _) = _, %v, want _, %v", err, errNoTransportSecurity) + } + + // Use of both transport creds and creds bundle must fail. + creds, err := credentials.NewClientTLSFromFile(testdata.Path("x509/server_ca_cert.pem"), "x.test.example.com") if err != nil { t.Fatalf("Failed to create authenticator %v", err) } - // Two conflicting credential configurations - if _, err := Dial("passthrough:///Non-Existent.Server:80", WithTransportCredentials(tlsCreds), WithBlock(), WithInsecure()); err != errCredentialsConflict { - t.Fatalf("Dial(_, _) = _, %v, want _, %v", err, errCredentialsConflict) + dopts := []DialOption{ + WithTransportCredentials(creds), + WithCredentialsBundle(&fakeBundleCreds{transportCreds: creds}), } - // security info on insecure connection - if _, err := Dial("passthrough:///Non-Existent.Server:80", WithPerRPCCredentials(securePerRPCCredentials{}), WithBlock(), WithInsecure()); err != errTransportCredentialsMissing { + if _, err := Dial("passthrough:///Non-Existent.Server:80", dopts...); err != errTransportCredsAndBundle { + t.Fatalf("Dial(_, _) = _, %v, want _, %v", err, errTransportCredsAndBundle) + } + + // Use of perRPC creds requiring transport security over an insecure + // transport must fail. + if _, err := Dial("passthrough:///Non-Existent.Server:80", WithPerRPCCredentials(securePerRPCCredentials{}), WithTransportCredentials(insecure.NewCredentials())); err != errTransportCredentialsMissing { t.Fatalf("Dial(_, _) = _, %v, want _, %v", err, errTransportCredentialsMissing) } + + // Use of a creds bundle with nil transport credentials must fail. + if _, err := Dial("passthrough:///Non-Existent.Server:80", WithCredentialsBundle(&fakeBundleCreds{})); err != errNoTransportCredsInBundle { + t.Fatalf("Dial(_, _) = _, %v, want _, %v", err, errTransportCredsAndBundle) + } } func (s) TestWithBackoffConfigDefault(t *testing.T) { @@ -604,7 +611,7 @@ func (s) TestWithConnectParams(t *testing.T) { } func testBackoffConfigSet(t *testing.T, wantBackoff internalbackoff.Exponential, opts ...DialOption) { - opts = append(opts, WithInsecure()) + opts = append(opts, WithTransportCredentials(insecure.NewCredentials())) conn, err := Dial("passthrough:///foo:80", opts...) if err != nil { t.Fatalf("unexpected error dialing connection: %v", err) @@ -628,7 +635,7 @@ func testBackoffConfigSet(t *testing.T, wantBackoff internalbackoff.Exponential, func (s) TestConnectParamsWithMinConnectTimeout(t *testing.T) { // Default value specified for minConnectTimeout in the spec is 20 seconds. mct := 1 * time.Minute - conn, err := Dial("passthrough:///foo:80", WithInsecure(), WithConnectParams(ConnectParams{MinConnectTimeout: mct})) + conn, err := Dial("passthrough:///foo:80", WithTransportCredentials(insecure.NewCredentials()), WithConnectParams(ConnectParams{MinConnectTimeout: mct})) if err != nil { t.Fatalf("unexpected error dialing connection: %v", err) } @@ -642,7 +649,7 @@ func (s) TestConnectParamsWithMinConnectTimeout(t *testing.T) { func (s) TestResolverServiceConfigBeforeAddressNotPanic(t *testing.T) { r := manual.NewBuilderWithScheme("whatever") - cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithResolvers(r)) + cc, err := Dial(r.Scheme()+":///test.server", WithTransportCredentials(insecure.NewCredentials()), WithResolvers(r)) if err != nil { t.Fatalf("failed to dial: %v", err) } @@ -659,7 +666,7 @@ func (s) TestResolverServiceConfigWhileClosingNotPanic(t *testing.T) { for i := 0; i < 10; i++ { // Run this multiple times to make sure it doesn't panic. r := manual.NewBuilderWithScheme(fmt.Sprintf("whatever-%d", i)) - cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithResolvers(r)) + cc, err := Dial(r.Scheme()+":///test.server", WithTransportCredentials(insecure.NewCredentials()), WithResolvers(r)) if err != nil { t.Fatalf("failed to dial: %v", err) } @@ -672,7 +679,7 @@ func (s) TestResolverServiceConfigWhileClosingNotPanic(t *testing.T) { func (s) TestResolverEmptyUpdateNotPanic(t *testing.T) { r := manual.NewBuilderWithScheme("whatever") - cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithResolvers(r)) + cc, err := Dial(r.Scheme()+":///test.server", WithTransportCredentials(insecure.NewCredentials()), WithResolvers(r)) if err != nil { t.Fatalf("failed to dial: %v", err) } @@ -685,12 +692,15 @@ func (s) TestResolverEmptyUpdateNotPanic(t *testing.T) { } func (s) TestClientUpdatesParamsAfterGoAway(t *testing.T) { + grpctest.TLogger.ExpectError("Client received GoAway with error code ENHANCE_YOUR_CALM and debug data equal to ASCII \"too_many_pings\"") + lis, err := net.Listen("tcp", "localhost:0") if err != nil { t.Fatalf("Failed to listen. Err: %v", err) } defer lis.Close() - connected := make(chan struct{}) + connected := grpcsync.NewEvent() + defer connected.Fire() go func() { conn, err := lis.Accept() if err != nil { @@ -712,7 +722,7 @@ func (s) TestClientUpdatesParamsAfterGoAway(t *testing.T) { t.Errorf("error writing settings: %v", err) return } - <-connected + <-connected.Done() if err := f.WriteGoAway(0, http2.ErrCodeEnhanceYourCalm, []byte("too_many_pings")); err != nil { t.Errorf("error writing GOAWAY: %v", err) return @@ -721,7 +731,7 @@ func (s) TestClientUpdatesParamsAfterGoAway(t *testing.T) { addr := lis.Addr().String() ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - cc, err := DialContext(ctx, addr, WithBlock(), WithInsecure(), WithKeepaliveParams(keepalive.ClientParameters{ + cc, err := DialContext(ctx, addr, WithBlock(), WithTransportCredentials(insecure.NewCredentials()), WithKeepaliveParams(keepalive.ClientParameters{ Time: 10 * time.Second, Timeout: 100 * time.Millisecond, PermitWithoutStream: true, @@ -730,28 +740,27 @@ func (s) TestClientUpdatesParamsAfterGoAway(t *testing.T) { t.Fatalf("Dial(%s, _) = _, %v, want _, ", addr, err) } defer cc.Close() - close(connected) + connected.Fire() for { time.Sleep(10 * time.Millisecond) cc.mu.RLock() v := cc.mkp.Time + cc.mu.RUnlock() if v == 20*time.Second { // Success - cc.mu.RUnlock() return } if ctx.Err() != nil { // Timeout t.Fatalf("cc.dopts.copts.Keepalive.Time = %v , want 20s", v) } - cc.mu.RUnlock() } } func (s) TestDisableServiceConfigOption(t *testing.T) { r := manual.NewBuilderWithScheme("whatever") addr := r.Scheme() + ":///non.existent" - cc, err := Dial(addr, WithInsecure(), WithResolvers(r), WithDisableServiceConfig()) + cc, err := Dial(addr, WithTransportCredentials(insecure.NewCredentials()), WithResolvers(r), WithDisableServiceConfig()) if err != nil { t.Fatalf("Dial(%s, _) = _, %v, want _, ", addr, err) } @@ -778,7 +787,7 @@ func (s) TestDisableServiceConfigOption(t *testing.T) { func (s) TestMethodConfigDefaultService(t *testing.T) { addr := "nonexist:///non.existent" - cc, err := Dial(addr, WithInsecure(), WithDefaultServiceConfig(`{ + cc, err := Dial(addr, WithTransportCredentials(insecure.NewCredentials()), WithDefaultServiceConfig(`{ "methodConfig": [{ "name": [ { @@ -801,7 +810,7 @@ func (s) TestMethodConfigDefaultService(t *testing.T) { func (s) TestGetClientConnTarget(t *testing.T) { addr := "nonexist:///non.existent" - cc, err := Dial(addr, WithInsecure()) + cc, err := Dial(addr, WithTransportCredentials(insecure.NewCredentials())) if err != nil { t.Fatalf("Dial(%s, _) = _, %v, want _, ", addr, err) } @@ -827,11 +836,12 @@ func (s) TestResetConnectBackoff(t *testing.T) { dials <- struct{}{} return nil, errors.New("failed to fake dial") } - cc, err := Dial("any", WithInsecure(), WithDialer(dialer), withBackoff(backoffForever{})) + cc, err := Dial("any", WithTransportCredentials(insecure.NewCredentials()), WithDialer(dialer), withBackoff(backoffForever{})) if err != nil { t.Fatalf("Dial() = _, %v; want _, nil", err) } defer cc.Close() + go stayConnected(cc) select { case <-dials: case <-time.NewTimer(10 * time.Second).C: @@ -855,21 +865,26 @@ func (s) TestResetConnectBackoff(t *testing.T) { func (s) TestBackoffCancel(t *testing.T) { dialStrCh := make(chan string) - cc, err := Dial("any", WithInsecure(), WithDialer(func(t string, _ time.Duration) (net.Conn, error) { + cc, err := Dial("any", WithTransportCredentials(insecure.NewCredentials()), WithDialer(func(t string, _ time.Duration) (net.Conn, error) { dialStrCh <- t return nil, fmt.Errorf("test dialer, always error") })) if err != nil { t.Fatalf("Failed to create ClientConn: %v", err) } - <-dialStrCh - cc.Close() - // Should not leak. May need -count 5000 to exercise. + defer cc.Close() + + select { + case <-time.After(defaultTestTimeout): + t.Fatal("Timeout when waiting for custom dialer to be invoked during Dial") + case <-dialStrCh: + } } -// UpdateAddresses should cause the next reconnect to begin from the top of the -// list if the connection is not READY. -func (s) TestUpdateAddresses_RetryFromFirstAddr(t *testing.T) { +// TestUpdateAddresses_NoopIfCalledWithSameAddresses tests that UpdateAddresses +// should be noop if UpdateAddresses is called with the same list of addresses, +// even when the SubConn is in Connecting and doesn't have a current address. +func (s) TestUpdateAddresses_NoopIfCalledWithSameAddresses(t *testing.T) { lis1, err := net.Listen("tcp", "localhost:0") if err != nil { t.Fatalf("Error while listening. Err: %v", err) @@ -889,12 +904,15 @@ func (s) TestUpdateAddresses_RetryFromFirstAddr(t *testing.T) { defer lis3.Close() closeServer2 := make(chan struct{}) + exitCh := make(chan struct{}) server1ContactedFirstTime := make(chan struct{}) server1ContactedSecondTime := make(chan struct{}) server2ContactedFirstTime := make(chan struct{}) server2ContactedSecondTime := make(chan struct{}) server3Contacted := make(chan struct{}) + defer close(exitCh) + // Launch server 1. go func() { // First, let's allow the initial connection to go READY. We need to do @@ -918,12 +936,18 @@ func (s) TestUpdateAddresses_RetryFromFirstAddr(t *testing.T) { // until balancer is built to process the addresses. stateNotifications := testBalancerBuilder.nextStateNotifier() // Wait for the transport to become ready. - for s := range stateNotifications { - if s == connectivity.Ready { - break + for { + select { + case st := <-stateNotifications: + if st == connectivity.Ready { + goto ready + } + case <-exitCh: + return } } + ready: // Once it's ready, curAddress has been set. So let's close this // connection prompting the first reconnect cycle. conn1.Close() @@ -977,15 +1001,18 @@ func (s) TestUpdateAddresses_RetryFromFirstAddr(t *testing.T) { rb.InitialState(resolver.State{Addresses: addrsList}) client, err := Dial("whatever:///this-gets-overwritten", - WithInsecure(), + WithTransportCredentials(insecure.NewCredentials()), WithResolvers(rb), - withBackoff(noBackoff{}), - WithBalancerName(stateRecordingBalancerName), - withMinConnectDeadline(func() time.Duration { return time.Hour })) + WithConnectParams(ConnectParams{ + Backoff: backoff.Config{}, + MinConnectTimeout: time.Hour, + }), + WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, stateRecordingBalancerName))) if err != nil { t.Fatal(err) } defer client.Close() + go stayConnected(client) timeout := time.After(5 * time.Second) @@ -1011,19 +1038,20 @@ func (s) TestUpdateAddresses_RetryFromFirstAddr(t *testing.T) { } client.mu.Unlock() + // Call UpdateAddresses with the same list of addresses, it should be a noop + // (even when the SubConn is Connecting, and doesn't have a curAddr). ac.acbw.UpdateAddresses(addrsList) // We've called tryUpdateAddrs - now let's make server2 close the - // connection and check that it goes back to server1 instead of continuing - // to server3 or trying server2 again. + // connection and check that it continues to server3. close(closeServer2) select { case <-server1ContactedSecondTime: + t.Fatal("server1 was contacted a second time, but it should have continued to server 3") case <-server2ContactedSecondTime: - t.Fatal("server2 was contacted a second time, but it after tryUpdateAddrs it should have re-started the list and tried server1") + t.Fatal("server2 was contacted a second time, but it should have continued to server 3") case <-server3Contacted: - t.Fatal("server3 was contacted, but after tryUpdateAddrs it should have re-started the list and tried server1") case <-timeout: t.Fatal("timed out waiting for any server to be contacted after tryUpdateAddrs") } @@ -1064,14 +1092,14 @@ func verifyWaitForReadyEqualsTrue(cc *ClientConn) bool { } func testInvalidDefaultServiceConfig(t *testing.T) { - _, err := Dial("fake.com", WithInsecure(), WithDefaultServiceConfig("")) + _, err := Dial("fake.com", WithTransportCredentials(insecure.NewCredentials()), WithDefaultServiceConfig("")) if !strings.Contains(err.Error(), invalidDefaultServiceConfigErrPrefix) { t.Fatalf("Dial got err: %v, want err contains: %v", err, invalidDefaultServiceConfigErrPrefix) } } func testDefaultServiceConfigWhenResolverServiceConfigDisabled(t *testing.T, r *manual.Resolver, addr string, js string) { - cc, err := Dial(addr, WithInsecure(), WithDisableServiceConfig(), WithResolvers(r), WithDefaultServiceConfig(js)) + cc, err := Dial(addr, WithTransportCredentials(insecure.NewCredentials()), WithDisableServiceConfig(), WithResolvers(r), WithDefaultServiceConfig(js)) if err != nil { t.Fatalf("Dial(%s, _) = _, %v, want _, ", addr, err) } @@ -1087,7 +1115,7 @@ func testDefaultServiceConfigWhenResolverServiceConfigDisabled(t *testing.T, r * } func testDefaultServiceConfigWhenResolverDoesNotReturnServiceConfig(t *testing.T, r *manual.Resolver, addr string, js string) { - cc, err := Dial(addr, WithInsecure(), WithResolvers(r), WithDefaultServiceConfig(js)) + cc, err := Dial(addr, WithTransportCredentials(insecure.NewCredentials()), WithResolvers(r), WithDefaultServiceConfig(js)) if err != nil { t.Fatalf("Dial(%s, _) = _, %v, want _, ", addr, err) } @@ -1101,7 +1129,7 @@ func testDefaultServiceConfigWhenResolverDoesNotReturnServiceConfig(t *testing.T } func testDefaultServiceConfigWhenResolverReturnInvalidServiceConfig(t *testing.T, r *manual.Resolver, addr string, js string) { - cc, err := Dial(addr, WithInsecure(), WithResolvers(r), WithDefaultServiceConfig(js)) + cc, err := Dial(addr, WithTransportCredentials(insecure.NewCredentials()), WithResolvers(r), WithDefaultServiceConfig(js)) if err != nil { t.Fatalf("Dial(%s, _) = _, %v, want _, ", addr, err) } @@ -1113,3 +1141,127 @@ func testDefaultServiceConfigWhenResolverReturnInvalidServiceConfig(t *testing.T t.Fatal("default service config failed to be applied after 1s") } } + +type stateRecordingBalancer struct { + balancer.Balancer +} + +func (b *stateRecordingBalancer) UpdateSubConnState(sc balancer.SubConn, s balancer.SubConnState) { + panic(fmt.Sprintf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, s)) +} + +func (b *stateRecordingBalancer) Close() { + b.Balancer.Close() +} + +type stateRecordingBalancerBuilder struct { + mu sync.Mutex + notifier chan connectivity.State // The notifier used in the last Balancer. +} + +func newStateRecordingBalancerBuilder() *stateRecordingBalancerBuilder { + return &stateRecordingBalancerBuilder{} +} + +func (b *stateRecordingBalancerBuilder) Name() string { + return stateRecordingBalancerName +} + +func (b *stateRecordingBalancerBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { + stateNotifications := make(chan connectivity.State, 10) + b.mu.Lock() + b.notifier = stateNotifications + b.mu.Unlock() + return &stateRecordingBalancer{ + Balancer: balancer.Get("pick_first").Build(&stateRecordingCCWrapper{cc, stateNotifications}, opts), + } +} + +func (b *stateRecordingBalancerBuilder) nextStateNotifier() <-chan connectivity.State { + b.mu.Lock() + defer b.mu.Unlock() + ret := b.notifier + b.notifier = nil + return ret +} + +type stateRecordingCCWrapper struct { + balancer.ClientConn + notifier chan<- connectivity.State +} + +func (ccw *stateRecordingCCWrapper) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) { + oldListener := opts.StateListener + opts.StateListener = func(s balancer.SubConnState) { + ccw.notifier <- s.ConnectivityState + oldListener(s) + } + return ccw.ClientConn.NewSubConn(addrs, opts) +} + +// Keep reading until something causes the connection to die (EOF, server +// closed, etc). Useful as a tool for mindlessly keeping the connection +// healthy, since the client will error if things like client prefaces are not +// accepted in a timely fashion. +func keepReading(conn net.Conn) { + buf := make([]byte, 1024) + for _, err := conn.Read(buf); err == nil; _, err = conn.Read(buf) { + } +} + +// stayConnected makes cc stay connected by repeatedly calling cc.Connect() +// until the state becomes Shutdown or until 10 seconds elapses. +func stayConnected(cc *ClientConn) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + for { + state := cc.GetState() + switch state { + case connectivity.Idle: + cc.Connect() + case connectivity.Shutdown: + return + } + if !cc.WaitForStateChange(ctx, state) { + return + } + } +} + +func (s) TestURLAuthorityEscape(t *testing.T) { + tests := []struct { + name string + authority string + want string + }{ + { + name: "ipv6_authority", + authority: "[::1]", + want: "[::1]", + }, + { + name: "with_user_and_host", + authority: "userinfo@host:10001", + want: "userinfo@host:10001", + }, + { + name: "with_multiple_slashes", + authority: "projects/123/network/abc/service", + want: "projects%2F123%2Fnetwork%2Fabc%2Fservice", + }, + { + name: "all_possible_allowed_chars", + authority: "abc123-._~!$&'()*+,;=@:[]", + want: "abc123-._~!$&'()*+,;=@:[]", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got, want := encodeAuthority(test.authority), test.want; got != want { + t.Errorf("encodeAuthority(%s) = %s, want %s", test.authority, got, test.want) + } + }) + } +} diff --git a/cmd/protoc-gen-go-grpc/README.md b/cmd/protoc-gen-go-grpc/README.md index 6d047f1e514c..a2d4d010212a 100644 --- a/cmd/protoc-gen-go-grpc/README.md +++ b/cmd/protoc-gen-go-grpc/README.md @@ -4,140 +4,18 @@ This tool generates Go language bindings of `service`s in protobuf definition files for gRPC. For usage information, please see our [quick start guide](https://grpc.io/docs/languages/go/quickstart/). -## Service implementation and registration +## Future-proofing services -**NOTE:** service registration has changed from the previous version of the -code generator. Please read this section carefully if you are migrating. +By default, to register services using the methods generated by this tool, the +service implementations must embed the corresponding +`UnimplementedServer` for future compatibility. This is a behavior +change from the grpc code generator previously included with `protoc-gen-go`. +To restore this behavior, set the option `require_unimplemented_servers=false`. +E.g.: -To register your service handlers with a gRPC server, first implement the -methods as either functions or methods on a struct. Examples: - -```go -// As a function: - -func unaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) { - // Echo.UnaryEcho implementation -} - -// As a struct + method: - -type myEchoService struct { - // ...fields used by this service... -} - -func (s *myEchoService) UnaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) { - // Echo.UnaryEcho implementation -} ``` - -Then create an instance of the generated `Service` struct type and initialize -the handlers which have been implemented: - -```go -func main() { - // ... - - // As a function: - echoService := pb.EchoService{ - UnaryEcho: unaryEcho, - // etc - } - - // As a struct+method: - mes := &myEchoService{...} - echoService := pb.EchoService{ - UnaryEcho: mes.UnaryEcho, - // etc - } - - // ... - } -``` - -Finally, pass this `Service` instance to the generated `Register` function: - -```go - pb.RegisterEchoService(grpcServer, echoService) -``` - -### Migration from legacy version - -#### Updating existing code - -The previous version of protoc-gen-go-grpc used a different method to register -services. In that version, it was only possible to register a service -implementation that was a complete implementation of the service. It was also -possible to embed an `Unimplemented` implementation of the service, which was -also generated and returned an UNIMPLEMENTED status for all methods. To make -it easier to migrate from the previous version, two new symbols are generated: -`NewService` and `UnstableService`. - -`NewService` allows for easy wrapping of a service implementation into -an instance of the new generated `Service` struct type. *This has drawbacks, -however: `NewService` accepts any parameter, and does not panic or -error if any methods are missing, are misspelled, or have the wrong signature.* -These methods will return an UNIMPLEMENTED status if called by a client. - -`UnstableService` allows for asserting that a type properly implements -all methods of a service. It is generated with the name `Unstable` to denote -that it will be extended whenever new methods are added to the service -definition. This kind of change to the service definition is considered -backward-compatible in protobuf, but is not backward-compatible in Go, because -adding methods to an interface invalidates any existing implementations. *Use -of this symbol could result in future compilation errors.* - -To convert your existing code using the previous code generator, please refer -to the following example: - -```go -type myEchoService{ - // ...fields used by this service... -} -// ... method handler implementation ... - - -// Optional; not recommended: to guarantee myEchoService fully implements -// EchoService: -var _ pb.UnstableEchoService = &myEchoService{} - -func main() { - // ... - - // OLD: - pb.RegisterEchoServer(grpcServer, &myEchoService{}) - - // NEW: - pb.RegisterEchoService(grpcServer, pb.NewEchoService(&myEchoService{})) - - - // Optional: to gracefully detect missing methods: - if _, ok := &myEchoService{}.(pb.UnstableEchoService); !ok { - fmt.Println("myEchoService does not implement all methods of EchoService.") - } - - - // ... -} -``` - -#### Updating generated code to support both legacy and new code - -`protoc-gen-go-grpc` supports a flag, `migration_mode`, which enables it to be -run in tandem with the previous tool (`protoc-gen-go` with the grpc plugin). -It can be used as follows: - -```sh -go install github.com/golang/protobuf/protoc-gen-go - -# Example 1: with OPTS set to common options for protoc-gen-go and -# protoc-gen-go-grpc -protoc --go_out=${OPTS},plugins=grpc:. --go-grpc_out=${OPTS},migration_mode=true:. *.proto - -# Example 2: if no special options are needed -protoc --go_out=plugins=grpc:. --go-grpc_out=migration_mode=true:. *.proto + protoc --go-grpc_out=. --go-grpc_opt=require_unimplemented_servers=false[,other options...] \ ``` -This is recommended for temporary use only to ease migration from the legacy -version. The `RegisterServer` and `Server` symbols it -produced are not backward compatible in the presence of newly added methods to -a service. \ No newline at end of file +Note that this is not recommended, and the option is only provided to restore +backward compatibility with previously-generated code. diff --git a/cmd/protoc-gen-go-grpc/go.mod b/cmd/protoc-gen-go-grpc/go.mod index d0cfd8ebf56f..92cf6c762689 100644 --- a/cmd/protoc-gen-go-grpc/go.mod +++ b/cmd/protoc-gen-go-grpc/go.mod @@ -1,5 +1,5 @@ module google.golang.org/grpc/cmd/protoc-gen-go-grpc -go 1.9 +go 1.19 -require google.golang.org/protobuf v1.23.0 +require google.golang.org/protobuf v1.31.0 diff --git a/cmd/protoc-gen-go-grpc/go.sum b/cmd/protoc-gen-go-grpc/go.sum index 92baf2631b73..9ea5597b8346 100644 --- a/cmd/protoc-gen-go-grpc/go.sum +++ b/cmd/protoc-gen-go-grpc/go.sum @@ -1,18 +1,8 @@ -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= diff --git a/cmd/protoc-gen-go-grpc/grpc.go b/cmd/protoc-gen-go-grpc/grpc.go index 47503bdc0f42..9e15d2d8daf3 100644 --- a/cmd/protoc-gen-go-grpc/grpc.go +++ b/cmd/protoc-gen-go-grpc/grpc.go @@ -24,7 +24,7 @@ import ( "strings" "google.golang.org/protobuf/compiler/protogen" - + "google.golang.org/protobuf/reflect/protoreflect" "google.golang.org/protobuf/types/descriptorpb" ) @@ -35,6 +35,94 @@ const ( statusPackage = protogen.GoImportPath("google.golang.org/grpc/status") ) +type serviceGenerateHelperInterface interface { + formatFullMethodSymbol(service *protogen.Service, method *protogen.Method) string + genFullMethods(g *protogen.GeneratedFile, service *protogen.Service) + generateClientStruct(g *protogen.GeneratedFile, clientName string) + generateNewClientDefinitions(g *protogen.GeneratedFile, service *protogen.Service, clientName string) + generateUnimplementedServerType(gen *protogen.Plugin, file *protogen.File, g *protogen.GeneratedFile, service *protogen.Service) + generateServerFunctions(gen *protogen.Plugin, file *protogen.File, g *protogen.GeneratedFile, service *protogen.Service, serverType string, serviceDescVar string) + formatHandlerFuncName(service *protogen.Service, hname string) string +} + +type serviceGenerateHelper struct{} + +func (serviceGenerateHelper) formatFullMethodSymbol(service *protogen.Service, method *protogen.Method) string { + return fmt.Sprintf("%s_%s_FullMethodName", service.GoName, method.GoName) +} + +func (serviceGenerateHelper) genFullMethods(g *protogen.GeneratedFile, service *protogen.Service) { + g.P("const (") + for _, method := range service.Methods { + fmSymbol := helper.formatFullMethodSymbol(service, method) + fmName := fmt.Sprintf("/%s/%s", service.Desc.FullName(), method.Desc.Name()) + g.P(fmSymbol, ` = "`, fmName, `"`) + } + g.P(")") + g.P() +} + +func (serviceGenerateHelper) generateClientStruct(g *protogen.GeneratedFile, clientName string) { + g.P("type ", unexport(clientName), " struct {") + g.P("cc ", grpcPackage.Ident("ClientConnInterface")) + g.P("}") + g.P() +} + +func (serviceGenerateHelper) generateNewClientDefinitions(g *protogen.GeneratedFile, service *protogen.Service, clientName string) { + g.P("return &", unexport(clientName), "{cc}") +} + +func (serviceGenerateHelper) generateUnimplementedServerType(gen *protogen.Plugin, file *protogen.File, g *protogen.GeneratedFile, service *protogen.Service) { + serverType := service.GoName + "Server" + mustOrShould := "must" + if !*requireUnimplemented { + mustOrShould = "should" + } + // Server Unimplemented struct for forward compatibility. + g.P("// Unimplemented", serverType, " ", mustOrShould, " be embedded to have forward compatible implementations.") + g.P("type Unimplemented", serverType, " struct {") + g.P("}") + g.P() + for _, method := range service.Methods { + nilArg := "" + if !method.Desc.IsStreamingClient() && !method.Desc.IsStreamingServer() { + nilArg = "nil," + } + g.P("func (Unimplemented", serverType, ") ", serverSignature(g, method), "{") + g.P("return ", nilArg, statusPackage.Ident("Errorf"), "(", codesPackage.Ident("Unimplemented"), `, "method `, method.GoName, ` not implemented")`) + g.P("}") + } + if *requireUnimplemented { + g.P("func (Unimplemented", serverType, ") mustEmbedUnimplemented", serverType, "() {}") + } + g.P() +} + +func (serviceGenerateHelper) generateServerFunctions(gen *protogen.Plugin, file *protogen.File, g *protogen.GeneratedFile, service *protogen.Service, serverType string, serviceDescVar string) { + // Server handler implementations. + handlerNames := make([]string, 0, len(service.Methods)) + for _, method := range service.Methods { + hname := genServerMethod(gen, file, g, method, func(hname string) string { + return hname + }) + handlerNames = append(handlerNames, hname) + } + genServiceDesc(file, g, serviceDescVar, serverType, service, handlerNames) +} + +func (serviceGenerateHelper) formatHandlerFuncName(service *protogen.Service, hname string) string { + return hname +} + +var helper serviceGenerateHelperInterface = serviceGenerateHelper{} + +// FileDescriptorProto.package field number +const fileDescriptorProtoPackageFieldNumber = 2 + +// FileDescriptorProto.syntax field number +const fileDescriptorProtoSyntaxFieldNumber = 12 + // generateFile generates a _grpc.pb.go file containing gRPC service definitions. func generateFile(gen *protogen.Plugin, file *protogen.File) *protogen.GeneratedFile { if len(file.Services) == 0 { @@ -42,14 +130,38 @@ func generateFile(gen *protogen.Plugin, file *protogen.File) *protogen.Generated } filename := file.GeneratedFilenamePrefix + "_grpc.pb.go" g := gen.NewGeneratedFile(filename, file.GoImportPath) + // Attach all comments associated with the syntax field. + genLeadingComments(g, file.Desc.SourceLocations().ByPath(protoreflect.SourcePath{fileDescriptorProtoSyntaxFieldNumber})) g.P("// Code generated by protoc-gen-go-grpc. DO NOT EDIT.") + g.P("// versions:") + g.P("// - protoc-gen-go-grpc v", version) + g.P("// - protoc ", protocVersion(gen)) + if file.Proto.GetOptions().GetDeprecated() { + g.P("// ", file.Desc.Path(), " is a deprecated file.") + } else { + g.P("// source: ", file.Desc.Path()) + } g.P() + // Attach all comments associated with the package field. + genLeadingComments(g, file.Desc.SourceLocations().ByPath(protoreflect.SourcePath{fileDescriptorProtoPackageFieldNumber})) g.P("package ", file.GoPackageName) g.P() generateFileContent(gen, file, g) return g } +func protocVersion(gen *protogen.Plugin) string { + v := gen.Request.GetCompilerVersion() + if v == nil { + return "(unknown)" + } + var suffix string + if s := v.GetSuffix(); s != "" { + suffix = "-" + s + } + return fmt.Sprintf("v%d.%d.%d%s", v.GetMajor(), v.GetMinor(), v.GetPatch(), suffix) +} + // generateFileContent generates the gRPC service definitions, excluding the package statement. func generateFileContent(gen *protogen.Plugin, file *protogen.File, g *protogen.GeneratedFile) { if len(file.Services) == 0 { @@ -58,26 +170,25 @@ func generateFileContent(gen *protogen.Plugin, file *protogen.File, g *protogen. g.P("// This is a compile-time assertion to ensure that this generated file") g.P("// is compatible with the grpc package it is being compiled against.") - g.P("const _ = ", grpcPackage.Ident("SupportPackageIsVersion7")) + g.P("// Requires gRPC-Go v1.32.0 or later.") + g.P("const _ = ", grpcPackage.Ident("SupportPackageIsVersion7")) // When changing, update version number above. g.P() for _, service := range file.Services { - genClient(gen, file, g, service) genService(gen, file, g, service) - genUnstableServiceInterface(gen, file, g, service) } } -func genClient(gen *protogen.Plugin, file *protogen.File, g *protogen.GeneratedFile, service *protogen.Service) { - if *migrationMode { - return - } +func genService(gen *protogen.Plugin, file *protogen.File, g *protogen.GeneratedFile, service *protogen.Service) { + // Full methods constants. + helper.genFullMethods(g, service) + + // Client interface. clientName := service.GoName + "Client" g.P("// ", clientName, " is the client API for ", service.GoName, " service.") g.P("//") g.P("// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.") - // Client interface. if service.Desc.Options().(*descriptorpb.ServiceOptions).GetDeprecated() { g.P("//") g.P(deprecationComment) @@ -96,24 +207,83 @@ func genClient(gen *protogen.Plugin, file *protogen.File, g *protogen.GeneratedF g.P() // Client structure. - g.P("type ", unexport(clientName), " struct {") - g.P("cc ", grpcPackage.Ident("ClientConnInterface")) - g.P("}") - g.P() + helper.generateClientStruct(g, clientName) // NewClient factory. if service.Desc.Options().(*descriptorpb.ServiceOptions).GetDeprecated() { g.P(deprecationComment) } g.P("func New", clientName, " (cc ", grpcPackage.Ident("ClientConnInterface"), ") ", clientName, " {") - g.P("return &", unexport(clientName), "{cc}") + helper.generateNewClientDefinitions(g, service, clientName) g.P("}") g.P() + var methodIndex, streamIndex int // Client method implementations. for _, method := range service.Methods { - genClientMethod(gen, g, method) + if !method.Desc.IsStreamingServer() && !method.Desc.IsStreamingClient() { + // Unary RPC method + genClientMethod(gen, file, g, method, methodIndex) + methodIndex++ + } else { + // Streaming RPC method + genClientMethod(gen, file, g, method, streamIndex) + streamIndex++ + } + } + + mustOrShould := "must" + if !*requireUnimplemented { + mustOrShould = "should" } + + // Server interface. + serverType := service.GoName + "Server" + g.P("// ", serverType, " is the server API for ", service.GoName, " service.") + g.P("// All implementations ", mustOrShould, " embed Unimplemented", serverType) + g.P("// for forward compatibility") + if service.Desc.Options().(*descriptorpb.ServiceOptions).GetDeprecated() { + g.P("//") + g.P(deprecationComment) + } + g.Annotate(serverType, service.Location) + g.P("type ", serverType, " interface {") + for _, method := range service.Methods { + g.Annotate(serverType+"."+method.GoName, method.Location) + if method.Desc.Options().(*descriptorpb.MethodOptions).GetDeprecated() { + g.P(deprecationComment) + } + g.P(method.Comments.Leading, + serverSignature(g, method)) + } + if *requireUnimplemented { + g.P("mustEmbedUnimplemented", serverType, "()") + } + g.P("}") + g.P() + + // Server Unimplemented struct for forward compatibility. + helper.generateUnimplementedServerType(gen, file, g, service) + + // Unsafe Server interface to opt-out of forward compatibility. + g.P("// Unsafe", serverType, " may be embedded to opt out of forward compatibility for this service.") + g.P("// Use of this interface is not recommended, as added methods to ", serverType, " will") + g.P("// result in compilation errors.") + g.P("type Unsafe", serverType, " interface {") + g.P("mustEmbedUnimplemented", serverType, "()") + g.P("}") + + // Server registration. + if service.Desc.Options().(*descriptorpb.ServiceOptions).GetDeprecated() { + g.P(deprecationComment) + } + serviceDescVar := service.GoName + "_ServiceDesc" + g.P("func Register", service.GoName, "Server(s ", grpcPackage.Ident("ServiceRegistrar"), ", srv ", serverType, ") {") + g.P("s.RegisterService(&", serviceDescVar, `, srv)`) + g.P("}") + g.P() + + helper.generateServerFunctions(gen, file, g, service, serverType, serviceDescVar) } func clientSignature(g *protogen.GeneratedFile, method *protogen.Method) string { @@ -131,29 +301,17 @@ func clientSignature(g *protogen.GeneratedFile, method *protogen.Method) string return s } -func genClientMethod(gen *protogen.Plugin, g *protogen.GeneratedFile, method *protogen.Method) { +func genClientMethod(gen *protogen.Plugin, file *protogen.File, g *protogen.GeneratedFile, method *protogen.Method, index int) { service := method.Parent - sname := fmt.Sprintf("/%s/%s", service.Desc.FullName(), method.Desc.Name()) + fmSymbol := helper.formatFullMethodSymbol(service, method) if method.Desc.Options().(*descriptorpb.MethodOptions).GetDeprecated() { g.P(deprecationComment) } - - streamDescName := unexport(service.GoName) + method.GoName + "StreamDesc" - g.P("var ", streamDescName, " = &", grpcPackage.Ident("StreamDesc"), "{") - g.P("StreamName: ", strconv.Quote(string(method.Desc.Name())), ",") - if method.Desc.IsStreamingServer() { - g.P("ServerStreams: true,") - } - if method.Desc.IsStreamingClient() { - g.P("ClientStreams: true,") - } - g.P("}") - g.P("func (c *", unexport(service.GoName), "Client) ", clientSignature(g, method), "{") if !method.Desc.IsStreamingServer() && !method.Desc.IsStreamingClient() { g.P("out := new(", method.Output.GoIdent, ")") - g.P(`err := c.cc.Invoke(ctx, "`, sname, `", in, out, opts...)`) + g.P(`err := c.cc.Invoke(ctx, `, fmSymbol, `, in, out, opts...)`) g.P("if err != nil { return nil, err }") g.P("return out, nil") g.P("}") @@ -161,8 +319,8 @@ func genClientMethod(gen *protogen.Plugin, g *protogen.GeneratedFile, method *pr return } streamType := unexport(service.GoName) + method.GoName + "Client" - - g.P(`stream, err := c.cc.NewStream(ctx, `, streamDescName, `, "`, sname, `", opts...)`) + serviceDescVar := service.GoName + "_ServiceDesc" + g.P("stream, err := c.cc.NewStream(ctx, &", serviceDescVar, ".Streams[", index, `], `, fmSymbol, `, opts...)`) g.P("if err != nil { return nil, err }") g.P("x := &", streamType, "{stream}") if !method.Desc.IsStreamingClient() { @@ -222,77 +380,49 @@ func genClientMethod(gen *protogen.Plugin, g *protogen.GeneratedFile, method *pr } } -func genService(gen *protogen.Plugin, file *protogen.File, g *protogen.GeneratedFile, service *protogen.Service) { - // Server struct. - serviceType := service.GoName + "Service" - g.P("// ", serviceType, " is the service API for ", service.GoName, " service.") - g.P("// Fields should be assigned to their respective handler implementations only before") - g.P("// Register", serviceType, " is called. Any unassigned fields will result in the") - g.P("// handler for that method returning an Unimplemented error.") - if service.Desc.Options().(*descriptorpb.ServiceOptions).GetDeprecated() { - g.P("//") - g.P(deprecationComment) - } - g.Annotate(serviceType, service.Location) - g.P("type ", serviceType, " struct {") - for _, method := range service.Methods { - if method.Desc.Options().(*descriptorpb.MethodOptions).GetDeprecated() { - g.P(deprecationComment) - } - g.Annotate(serviceType+"."+method.GoName, method.Location) - g.P(method.Comments.Leading, - handlerSignature(g, method)) +func serverSignature(g *protogen.GeneratedFile, method *protogen.Method) string { + var reqArgs []string + ret := "error" + if !method.Desc.IsStreamingClient() && !method.Desc.IsStreamingServer() { + reqArgs = append(reqArgs, g.QualifiedGoIdent(contextPackage.Ident("Context"))) + ret = "(*" + g.QualifiedGoIdent(method.Output.GoIdent) + ", error)" } - g.P("}") - g.P() - - // Method handler implementations. - for _, method := range service.Methods { - genMethodHandler(gen, g, method) + if !method.Desc.IsStreamingClient() { + reqArgs = append(reqArgs, "*"+g.QualifiedGoIdent(method.Input.GoIdent)) } - - // Stream interfaces and implementations. - for _, method := range service.Methods { - genServerStreamTypes(gen, g, method) + if method.Desc.IsStreamingClient() || method.Desc.IsStreamingServer() { + reqArgs = append(reqArgs, method.Parent.GoName+"_"+method.GoName+"Server") } - - // Service registration. - genRegisterFunction(gen, file, g, service) - - // Short-cut service constructor. - genServiceConstructor(gen, g, service) + return method.GoName + "(" + strings.Join(reqArgs, ", ") + ") " + ret } -func genRegisterFunction(gen *protogen.Plugin, file *protogen.File, g *protogen.GeneratedFile, service *protogen.Service) { - g.P("// Register", service.GoName, "Service registers a service implementation with a gRPC server.") - if service.Desc.Options().(*descriptorpb.ServiceOptions).GetDeprecated() { - g.P("//") - g.P(deprecationComment) - } - g.P("func Register", service.GoName, "Service(s ", grpcPackage.Ident("ServiceRegistrar"), ", srv *", service.GoName, "Service) {") - +func genServiceDesc(file *protogen.File, g *protogen.GeneratedFile, serviceDescVar string, serverType string, service *protogen.Service, handlerNames []string) { // Service descriptor. - g.P("sd := ", grpcPackage.Ident("ServiceDesc"), " {") + g.P("// ", serviceDescVar, " is the ", grpcPackage.Ident("ServiceDesc"), " for ", service.GoName, " service.") + g.P("// It's only intended for direct use with ", grpcPackage.Ident("RegisterService"), ",") + g.P("// and not to be introspected or modified (even as a copy)") + g.P("var ", serviceDescVar, " = ", grpcPackage.Ident("ServiceDesc"), " {") g.P("ServiceName: ", strconv.Quote(string(service.Desc.FullName())), ",") + g.P("HandlerType: (*", serverType, ")(nil),") g.P("Methods: []", grpcPackage.Ident("MethodDesc"), "{") - for _, method := range service.Methods { + for i, method := range service.Methods { if method.Desc.IsStreamingClient() || method.Desc.IsStreamingServer() { continue } g.P("{") g.P("MethodName: ", strconv.Quote(string(method.Desc.Name())), ",") - g.P("Handler: srv.", unexport(method.GoName), ",") + g.P("Handler: ", handlerNames[i], ",") g.P("},") } g.P("},") g.P("Streams: []", grpcPackage.Ident("StreamDesc"), "{") - for _, method := range service.Methods { + for i, method := range service.Methods { if !method.Desc.IsStreamingClient() && !method.Desc.IsStreamingServer() { continue } g.P("{") g.P("StreamName: ", strconv.Quote(string(method.Desc.Name())), ",") - g.P("Handler: srv.", unexport(method.GoName), ",") + g.P("Handler: ", handlerNames[i], ",") if method.Desc.IsStreamingServer() { g.P("ServerStreams: true,") } @@ -305,154 +435,42 @@ func genRegisterFunction(gen *protogen.Plugin, file *protogen.File, g *protogen. g.P("Metadata: \"", file.Desc.Path(), "\",") g.P("}") g.P() - - g.P("s.RegisterService(&sd, nil)") - g.P("}") - g.P() } -func genServiceConstructor(gen *protogen.Plugin, g *protogen.GeneratedFile, service *protogen.Service) { - g.P("// New", service.GoName, "Service creates a new ", service.GoName, "Service containing the") - g.P("// implemented methods of the ", service.GoName, " service in s. Any unimplemented") - g.P("// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client.") - g.P("// This includes situations where the method handler is misspelled or has the wrong") - g.P("// signature. For this reason, this function should be used with great care and") - g.P("// is not recommended to be used by most users.") - g.P("func New", service.GoName, "Service(s interface{}) *", service.GoName, "Service {") - g.P("ns := &", service.GoName, "Service{}") - for _, method := range service.Methods { - g.P("if h, ok := s.(interface {", methodSignature(g, method), "}); ok {") - g.P("ns.", method.GoName, " = h.", method.GoName) - g.P("}") - } - g.P("return ns") - g.P("}") - g.P() -} - -func genUnstableServiceInterface(gen *protogen.Plugin, file *protogen.File, g *protogen.GeneratedFile, service *protogen.Service) { - // Service interface. - serviceType := service.GoName + "Service" - g.P("// Unstable", serviceType, " is the service API for ", service.GoName, " service.") - g.P("// New methods may be added to this interface if they are added to the service") - g.P("// definition, which is not a backward-compatible change. For this reason, ") - g.P("// use of this type is not recommended.") - if service.Desc.Options().(*descriptorpb.ServiceOptions).GetDeprecated() { - g.P("//") - g.P(deprecationComment) - } - g.Annotate("Unstable"+serviceType, service.Location) - g.P("type Unstable", serviceType, " interface {") - for _, method := range service.Methods { - g.Annotate("Unstable"+serviceType+"."+method.GoName, method.Location) - if method.Desc.Options().(*descriptorpb.MethodOptions).GetDeprecated() { - g.P(deprecationComment) - } - g.P(method.Comments.Leading, - methodSignature(g, method)) - } - g.P("}") - g.P() -} - -func methodSignature(g *protogen.GeneratedFile, method *protogen.Method) string { - var reqArgs []string - ret := "error" - if !method.Desc.IsStreamingClient() && !method.Desc.IsStreamingServer() { - reqArgs = append(reqArgs, g.QualifiedGoIdent(contextPackage.Ident("Context"))) - ret = "(*" + g.QualifiedGoIdent(method.Output.GoIdent) + ", error)" - } - if !method.Desc.IsStreamingClient() { - reqArgs = append(reqArgs, "*"+g.QualifiedGoIdent(method.Input.GoIdent)) - } - if method.Desc.IsStreamingClient() || method.Desc.IsStreamingServer() { - reqArgs = append(reqArgs, method.Parent.GoName+"_"+method.GoName+"Server") - } - return method.GoName + "(" + strings.Join(reqArgs, ", ") + ") " + ret -} - -func handlerSignature(g *protogen.GeneratedFile, method *protogen.Method) string { - var reqArgs []string - ret := "error" - if !method.Desc.IsStreamingClient() && !method.Desc.IsStreamingServer() { - reqArgs = append(reqArgs, g.QualifiedGoIdent(contextPackage.Ident("Context"))) - ret = "(*" + g.QualifiedGoIdent(method.Output.GoIdent) + ", error)" - } - if !method.Desc.IsStreamingClient() { - reqArgs = append(reqArgs, "*"+g.QualifiedGoIdent(method.Input.GoIdent)) - } - if method.Desc.IsStreamingClient() || method.Desc.IsStreamingServer() { - reqArgs = append(reqArgs, method.Parent.GoName+"_"+method.GoName+"Server") - } - return method.GoName + " func(" + strings.Join(reqArgs, ", ") + ") " + ret -} - -func unaryHandlerSignature(g *protogen.GeneratedFile) string { - return "(_ interface{}, ctx " + g.QualifiedGoIdent(contextPackage.Ident("Context")) + - ", dec func(interface{}) error, interceptor " + g.QualifiedGoIdent(grpcPackage.Ident("UnaryServerInterceptor")) + ") (interface{}, error)" -} - -func streamHandlerSignature(g *protogen.GeneratedFile) string { - return "(_ interface{}, stream " + g.QualifiedGoIdent(grpcPackage.Ident("ServerStream")) + ") error" -} - -func genMethodHandler(gen *protogen.Plugin, g *protogen.GeneratedFile, method *protogen.Method) { +func genServerMethod(gen *protogen.Plugin, file *protogen.File, g *protogen.GeneratedFile, method *protogen.Method, hnameFuncNameFormatter func(string) string) string { service := method.Parent + hname := fmt.Sprintf("_%s_%s_Handler", service.GoName, method.GoName) - nilArg := "" - signature := streamHandlerSignature(g) - if !method.Desc.IsStreamingClient() && !method.Desc.IsStreamingServer() { - nilArg = "nil," - signature = unaryHandlerSignature(g) - } - g.P("func (s *", service.GoName, "Service) ", unexport(method.GoName), signature, " {") - - g.P("if s.", method.GoName, " == nil {") - g.P("return ", nilArg, statusPackage.Ident("Errorf"), "(", codesPackage.Ident("Unimplemented"), `, "method `, method.GoName, ` not implemented")`) - g.P("}") - genHandlerBody(gen, g, method) - - g.P("}") -} - -func genHandlerBody(gen *protogen.Plugin, g *protogen.GeneratedFile, method *protogen.Method) { - service := method.Parent if !method.Desc.IsStreamingClient() && !method.Desc.IsStreamingServer() { + g.P("func ", hnameFuncNameFormatter(hname), "(srv interface{}, ctx ", contextPackage.Ident("Context"), ", dec func(interface{}) error, interceptor ", grpcPackage.Ident("UnaryServerInterceptor"), ") (interface{}, error) {") g.P("in := new(", method.Input.GoIdent, ")") g.P("if err := dec(in); err != nil { return nil, err }") - g.P("if interceptor == nil { return s.", method.GoName, "(ctx, in) }") + g.P("if interceptor == nil { return srv.(", service.GoName, "Server).", method.GoName, "(ctx, in) }") g.P("info := &", grpcPackage.Ident("UnaryServerInfo"), "{") - g.P("Server: s,") - g.P("FullMethod: ", strconv.Quote(fmt.Sprintf("/%s/%s", service.Desc.FullName(), method.GoName)), ",") + g.P("Server: srv,") + fmSymbol := helper.formatFullMethodSymbol(service, method) + g.P("FullMethod: ", fmSymbol, ",") g.P("}") g.P("handler := func(ctx ", contextPackage.Ident("Context"), ", req interface{}) (interface{}, error) {") - g.P("return s.", method.GoName, "(ctx, req.(*", method.Input.GoIdent, "))") + g.P("return srv.(", service.GoName, "Server).", method.GoName, "(ctx, req.(*", method.Input.GoIdent, "))") g.P("}") g.P("return interceptor(ctx, in, info, handler)") - return + g.P("}") + g.P() + return hname } streamType := unexport(service.GoName) + method.GoName + "Server" + g.P("func ", hnameFuncNameFormatter(hname), "(srv interface{}, stream ", grpcPackage.Ident("ServerStream"), ") error {") if !method.Desc.IsStreamingClient() { - // Server-streaming g.P("m := new(", method.Input.GoIdent, ")") g.P("if err := stream.RecvMsg(m); err != nil { return err }") - g.P("return s.", method.GoName, "(m, &", streamType, "{stream})") + g.P("return srv.(", service.GoName, "Server).", method.GoName, "(m, &", streamType, "{stream})") } else { - // Bidi-streaming - g.P("return s.", method.GoName, "(&", streamType, "{stream})") + g.P("return srv.(", service.GoName, "Server).", method.GoName, "(&", streamType, "{stream})") } -} + g.P("}") + g.P() -func genServerStreamTypes(gen *protogen.Plugin, g *protogen.GeneratedFile, method *protogen.Method) { - if *migrationMode { - return - } - if !method.Desc.IsStreamingClient() && !method.Desc.IsStreamingServer() { - // Unary method - return - } - service := method.Parent - streamType := unexport(service.GoName) + method.GoName + "Server" genSend := method.Desc.IsStreamingServer() genSendAndClose := !method.Desc.IsStreamingServer() genRecv := method.Desc.IsStreamingClient() @@ -498,7 +516,18 @@ func genServerStreamTypes(gen *protogen.Plugin, g *protogen.GeneratedFile, metho g.P() } - return + return hname +} + +func genLeadingComments(g *protogen.GeneratedFile, loc protoreflect.SourceLocation) { + for _, s := range loc.LeadingDetachedComments { + g.P(protogen.Comments(s)) + g.P() + } + if s := loc.LeadingComments; s != "" { + g.P(protogen.Comments(s)) + g.P() + } } const deprecationComment = "// Deprecated: Do not use." diff --git a/cmd/protoc-gen-go-grpc/main.go b/cmd/protoc-gen-go-grpc/main.go index 0d7e76c64c4e..340eaf3ee7bf 100644 --- a/cmd/protoc-gen-go-grpc/main.go +++ b/cmd/protoc-gen-go-grpc/main.go @@ -19,29 +19,42 @@ // protoc-gen-go-grpc is a plugin for the Google protocol buffer compiler to // generate Go code. Install it by building this program and making it // accessible within your PATH with the name: +// // protoc-gen-go-grpc // // The 'go-grpc' suffix becomes part of the argument for the protocol compiler, // such that it can be invoked as: +// // protoc --go-grpc_out=. path/to/file.proto // // This generates Go service definitions for the protocol buffer defined by // file.proto. With that input, the output will be written to: +// // path/to/file_grpc.pb.go package main import ( "flag" + "fmt" "google.golang.org/protobuf/compiler/protogen" "google.golang.org/protobuf/types/pluginpb" ) -var migrationMode *bool +const version = "1.3.0" + +var requireUnimplemented *bool func main() { + showVersion := flag.Bool("version", false, "print the version and exit") + flag.Parse() + if *showVersion { + fmt.Printf("protoc-gen-go-grpc %v\n", version) + return + } + var flags flag.FlagSet - migrationMode = flags.Bool("migration_mode", false, "set to generate new symbols only; requires symbols produced by legacy protoc-gen-go") + requireUnimplemented = flags.Bool("require_unimplemented_servers", true, "set to false to match legacy behavior") protogen.Options{ ParamFunc: flags.Set, diff --git a/codec.go b/codec.go index 129776547811..411e3dfd47cc 100644 --- a/codec.go +++ b/codec.go @@ -27,8 +27,8 @@ import ( // omits the name/string, which vary between the two and are not needed for // anything besides the registry in the encoding package. type baseCodec interface { - Marshal(v interface{}) ([]byte, error) - Unmarshal(data []byte, v interface{}) error + Marshal(v any) ([]byte, error) + Unmarshal(data []byte, v any) error } var _ baseCodec = Codec(nil) @@ -41,9 +41,9 @@ var _ baseCodec = encoding.Codec(nil) // Deprecated: use encoding.Codec instead. type Codec interface { // Marshal returns the wire format of v. - Marshal(v interface{}) ([]byte, error) + Marshal(v any) ([]byte, error) // Unmarshal parses the wire format into v. - Unmarshal(data []byte, v interface{}) error + Unmarshal(data []byte, v any) error // String returns the name of the Codec implementation. This is unused by // gRPC. String() string diff --git a/codes/code_string.go b/codes/code_string.go index 0b206a57822a..934fac2b090a 100644 --- a/codes/code_string.go +++ b/codes/code_string.go @@ -18,7 +18,15 @@ package codes -import "strconv" +import ( + "strconv" + + "google.golang.org/grpc/internal" +) + +func init() { + internal.CanonicalString = canonicalString +} func (c Code) String() string { switch c { @@ -60,3 +68,44 @@ func (c Code) String() string { return "Code(" + strconv.FormatInt(int64(c), 10) + ")" } } + +func canonicalString(c Code) string { + switch c { + case OK: + return "OK" + case Canceled: + return "CANCELLED" + case Unknown: + return "UNKNOWN" + case InvalidArgument: + return "INVALID_ARGUMENT" + case DeadlineExceeded: + return "DEADLINE_EXCEEDED" + case NotFound: + return "NOT_FOUND" + case AlreadyExists: + return "ALREADY_EXISTS" + case PermissionDenied: + return "PERMISSION_DENIED" + case ResourceExhausted: + return "RESOURCE_EXHAUSTED" + case FailedPrecondition: + return "FAILED_PRECONDITION" + case Aborted: + return "ABORTED" + case OutOfRange: + return "OUT_OF_RANGE" + case Unimplemented: + return "UNIMPLEMENTED" + case Internal: + return "INTERNAL" + case Unavailable: + return "UNAVAILABLE" + case DataLoss: + return "DATA_LOSS" + case Unauthenticated: + return "UNAUTHENTICATED" + default: + return "CODE(" + strconv.FormatInt(int64(c), 10) + ")" + } +} diff --git a/connectivity/connectivity.go b/connectivity/connectivity.go index 0a8d682ac6af..4a89926422bc 100644 --- a/connectivity/connectivity.go +++ b/connectivity/connectivity.go @@ -18,12 +18,9 @@ // Package connectivity defines connectivity semantics. // For details, see https://github.com/grpc/grpc/blob/master/doc/connectivity-semantics-and-api.md. -// All APIs in this package are experimental. package connectivity import ( - "context" - "google.golang.org/grpc/grpclog" ) @@ -47,7 +44,7 @@ func (s State) String() string { return "SHUTDOWN" default: logger.Errorf("unknown connectivity state: %d", s) - return "Invalid-State" + return "INVALID_STATE" } } @@ -64,12 +61,34 @@ const ( Shutdown ) -// Reporter reports the connectivity states. -type Reporter interface { - // CurrentState returns the current state of the reporter. - CurrentState() State - // WaitForStateChange blocks until the reporter's state is different from the given state, - // and returns true. - // It returns false if <-ctx.Done() can proceed (ctx got timeout or got canceled). - WaitForStateChange(context.Context, State) bool +// ServingMode indicates the current mode of operation of the server. +// +// Only xDS enabled gRPC servers currently report their serving mode. +type ServingMode int + +const ( + // ServingModeStarting indicates that the server is starting up. + ServingModeStarting ServingMode = iota + // ServingModeServing indicates that the server contains all required + // configuration and is serving RPCs. + ServingModeServing + // ServingModeNotServing indicates that the server is not accepting new + // connections. Existing connections will be closed gracefully, allowing + // in-progress RPCs to complete. A server enters this mode when it does not + // contain the required configuration to serve RPCs. + ServingModeNotServing +) + +func (s ServingMode) String() string { + switch s { + case ServingModeStarting: + return "STARTING" + case ServingModeServing: + return "SERVING" + case ServingModeNotServing: + return "NOT_SERVING" + default: + logger.Errorf("unknown serving mode: %d", s) + return "INVALID_MODE" + } } diff --git a/credentials/alts/alts.go b/credentials/alts/alts.go index 729c4b43b5fc..579adf210c48 100644 --- a/credentials/alts/alts.go +++ b/credentials/alts/alts.go @@ -37,6 +37,7 @@ import ( "google.golang.org/grpc/credentials/alts/internal/handshaker/service" altspb "google.golang.org/grpc/credentials/alts/internal/proto/grpc_gcp" "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal/googlecloud" ) const ( @@ -54,6 +55,7 @@ const ( ) var ( + vmOnGCP bool once sync.Once maxRPCVersion = &altspb.RpcProtocolVersions_Version{ Major: protocolVersionMaxMajor, @@ -149,9 +151,8 @@ func NewServerCreds(opts *ServerOptions) credentials.TransportCredentials { func newALTS(side core.Side, accounts []string, hsAddress string) credentials.TransportCredentials { once.Do(func() { - vmOnGCP = isRunningOnGCP() + vmOnGCP = googlecloud.OnGCE() }) - if hsAddress == "" { hsAddress = hypervisorHandshakerServiceAddress } diff --git a/credentials/alts/alts_test.go b/credentials/alts/alts_test.go index 61666696942f..20062fe77539 100644 --- a/credentials/alts/alts_test.go +++ b/credentials/alts/alts_test.go @@ -1,3 +1,6 @@ +//go:build linux || windows +// +build linux windows + /* * * Copyright 2018 gRPC authors. @@ -19,18 +22,44 @@ package alts import ( + "context" "reflect" + "sync" "testing" + "time" "github.com/golang/protobuf/proto" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/alts/internal/handshaker" + "google.golang.org/grpc/credentials/alts/internal/handshaker/service" + altsgrpc "google.golang.org/grpc/credentials/alts/internal/proto/grpc_gcp" altspb "google.golang.org/grpc/credentials/alts/internal/proto/grpc_gcp" + "google.golang.org/grpc/credentials/alts/internal/testutil" "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/testutils" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + "google.golang.org/grpc/status" +) + +const ( + defaultTestLongTimeout = 10 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond ) type s struct { grpctest.Tester } +func init() { + // The vmOnGCP global variable MUST be forced to true. Otherwise, if + // this test is run anywhere except on a GCP VM, then an ALTS handshake + // will immediately fail. + once.Do(func() {}) + vmOnGCP = true +} + func Test(t *testing.T) { grpctest.RunSubTests(t, s{}) } @@ -284,6 +313,61 @@ func (s) TestCheckRPCVersions(t *testing.T) { } } +// TestFullHandshake performs a full ALTS handshake between a test client and +// server, where both client and server offload to a local, fake handshaker +// service. +func (s) TestFullHandshake(t *testing.T) { + // Start the fake handshaker service and the server. + var wait sync.WaitGroup + defer wait.Wait() + stopHandshaker, handshakerAddress := startFakeHandshakerService(t, &wait) + defer stopHandshaker() + stopServer, serverAddress := startServer(t, handshakerAddress, &wait) + defer stopServer() + + // Ping the server, authenticating with ALTS. + establishAltsConnection(t, handshakerAddress, serverAddress) + + // Close open connections to the fake handshaker service. + if err := service.CloseForTesting(); err != nil { + t.Errorf("service.CloseForTesting() failed: %v", err) + } +} + +// TestConcurrentHandshakes performs a several, concurrent ALTS handshakes +// between a test client and server, where both client and server offload to a +// local, fake handshaker service. +func (s) TestConcurrentHandshakes(t *testing.T) { + // Set the max number of concurrent handshakes to 3, so that we can + // test the handshaker behavior when handshakes are queued by + // performing more than 3 concurrent handshakes (specifically, 10). + handshaker.ResetConcurrentHandshakeSemaphoreForTesting(3) + + // Start the fake handshaker service and the server. + var wait sync.WaitGroup + defer wait.Wait() + stopHandshaker, handshakerAddress := startFakeHandshakerService(t, &wait) + defer stopHandshaker() + stopServer, serverAddress := startServer(t, handshakerAddress, &wait) + defer stopServer() + + // Ping the server, authenticating with ALTS. + var waitForConnections sync.WaitGroup + for i := 0; i < 10; i++ { + waitForConnections.Add(1) + go func() { + establishAltsConnection(t, handshakerAddress, serverAddress) + waitForConnections.Done() + }() + } + waitForConnections.Wait() + + // Close open connections to the fake handshaker service. + if err := service.CloseForTesting(); err != nil { + t.Errorf("service.CloseForTesting() failed: %v", err) + } +} + func version(major, minor uint32) *altspb.RpcProtocolVersions_Version { return &altspb.RpcProtocolVersions_Version{ Major: major, @@ -297,3 +381,72 @@ func versions(minMajor, minMinor, maxMajor, maxMinor uint32) *altspb.RpcProtocol MaxRpcVersion: version(maxMajor, maxMinor), } } + +func establishAltsConnection(t *testing.T, handshakerAddress, serverAddress string) { + clientCreds := NewClientCreds(&ClientOptions{HandshakerServiceAddress: handshakerAddress}) + conn, err := grpc.Dial(serverAddress, grpc.WithTransportCredentials(clientCreds)) + if err != nil { + t.Fatalf("grpc.Dial(%v) failed: %v", serverAddress, err) + } + defer conn.Close() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestLongTimeout) + defer cancel() + c := testgrpc.NewTestServiceClient(conn) + for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) { + _, err = c.UnaryCall(ctx, &testpb.SimpleRequest{}) + if err == nil { + break + } + if code := status.Code(err); code == codes.Unavailable { + // The server is not ready yet. Try again. + continue + } + t.Fatalf("c.UnaryCall() failed: %v", err) + } +} + +func startFakeHandshakerService(t *testing.T, wait *sync.WaitGroup) (stop func(), address string) { + listener, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("LocalTCPListener() failed: %v", err) + } + s := grpc.NewServer() + altsgrpc.RegisterHandshakerServiceServer(s, &testutil.FakeHandshaker{}) + wait.Add(1) + go func() { + defer wait.Done() + if err := s.Serve(listener); err != nil { + t.Errorf("failed to serve: %v", err) + } + }() + return func() { s.Stop() }, listener.Addr().String() +} + +func startServer(t *testing.T, handshakerServiceAddress string, wait *sync.WaitGroup) (stop func(), address string) { + listener, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("LocalTCPListener() failed: %v", err) + } + serverOpts := &ServerOptions{HandshakerServiceAddress: handshakerServiceAddress} + creds := NewServerCreds(serverOpts) + s := grpc.NewServer(grpc.Creds(creds)) + testgrpc.RegisterTestServiceServer(s, &testServer{}) + wait.Add(1) + go func() { + defer wait.Done() + if err := s.Serve(listener); err != nil { + t.Errorf("s.Serve(%v) failed: %v", listener, err) + } + }() + return func() { s.Stop() }, listener.Addr().String() +} + +type testServer struct { + testgrpc.UnimplementedTestServiceServer +} + +func (s *testServer) UnaryCall(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{ + Payload: &testpb.Payload{}, + }, nil +} diff --git a/credentials/alts/internal/conn/record_test.go b/credentials/alts/internal/conn/record_test.go index 59d4f41e9e1c..0b4177a581ed 100644 --- a/credentials/alts/internal/conn/record_test.go +++ b/credentials/alts/internal/conn/record_test.go @@ -40,11 +40,15 @@ func Test(t *testing.T) { grpctest.RunSubTests(t, s{}) } +const ( + rekeyRecordProtocol = "ALTSRP_GCM_AES128_REKEY" +) + var ( - nextProtocols = []string{"ALTSRP_GCM_AES128"} + recordProtocols = []string{rekeyRecordProtocol} altsRecordFuncs = map[string]ALTSRecordFunc{ // ALTS handshaker protocols. - "ALTSRP_GCM_AES128": func(s core.Side, keyData []byte) (ALTSRecordCrypto, error) { + rekeyRecordProtocol: func(s core.Side, keyData []byte) (ALTSRecordCrypto, error) { return NewAES128GCM(s, keyData) }, } @@ -77,7 +81,7 @@ func (c *testConn) Close() error { return nil } -func newTestALTSRecordConn(in, out *bytes.Buffer, side core.Side, np string, protected []byte) *conn { +func newTestALTSRecordConn(in, out *bytes.Buffer, side core.Side, rp string, protected []byte) *conn { key := []byte{ // 16 arbitrary bytes. 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0xd2, 0x4c, 0xce, 0x4f, 0x49} @@ -85,23 +89,23 @@ func newTestALTSRecordConn(in, out *bytes.Buffer, side core.Side, np string, pro in: in, out: out, } - c, err := NewConn(&tc, side, np, key, protected) + c, err := NewConn(&tc, side, rp, key, protected) if err != nil { panic(fmt.Sprintf("Unexpected error creating test ALTS record connection: %v", err)) } return c.(*conn) } -func newConnPair(np string, clientProtected []byte, serverProtected []byte) (client, server *conn) { +func newConnPair(rp string, clientProtected []byte, serverProtected []byte) (client, server *conn) { clientBuf := new(bytes.Buffer) serverBuf := new(bytes.Buffer) - clientConn := newTestALTSRecordConn(clientBuf, serverBuf, core.ClientSide, np, clientProtected) - serverConn := newTestALTSRecordConn(serverBuf, clientBuf, core.ServerSide, np, serverProtected) + clientConn := newTestALTSRecordConn(clientBuf, serverBuf, core.ClientSide, rp, clientProtected) + serverConn := newTestALTSRecordConn(serverBuf, clientBuf, core.ServerSide, rp, serverProtected) return clientConn, serverConn } -func testPingPong(t *testing.T, np string) { - clientConn, serverConn := newConnPair(np, nil, nil) +func testPingPong(t *testing.T, rp string) { + clientConn, serverConn := newConnPair(rp, nil, nil) clientMsg := []byte("Client Message") if n, err := clientConn.Write(clientMsg); n != len(clientMsg) || err != nil { t.Fatalf("Client Write() = %v, %v; want %v, ", n, err, len(clientMsg)) @@ -128,13 +132,13 @@ func testPingPong(t *testing.T, np string) { } func (s) TestPingPong(t *testing.T) { - for _, np := range nextProtocols { - testPingPong(t, np) + for _, rp := range recordProtocols { + testPingPong(t, rp) } } -func testSmallReadBuffer(t *testing.T, np string) { - clientConn, serverConn := newConnPair(np, nil, nil) +func testSmallReadBuffer(t *testing.T, rp string) { + clientConn, serverConn := newConnPair(rp, nil, nil) msg := []byte("Very Important Message") if n, err := clientConn.Write(msg); err != nil { t.Fatalf("Write() = %v, %v; want %v, ", n, err, len(msg)) @@ -155,13 +159,13 @@ func testSmallReadBuffer(t *testing.T, np string) { } func (s) TestSmallReadBuffer(t *testing.T) { - for _, np := range nextProtocols { - testSmallReadBuffer(t, np) + for _, rp := range recordProtocols { + testSmallReadBuffer(t, rp) } } -func testLargeMsg(t *testing.T, np string) { - clientConn, serverConn := newConnPair(np, nil, nil) +func testLargeMsg(t *testing.T, rp string) { + clientConn, serverConn := newConnPair(rp, nil, nil) // msgLen is such that the length in the framing is larger than the // default size of one frame. msgLen := altsRecordDefaultLength - msgTypeFieldSize - clientConn.crypto.EncryptionOverhead() + 1 @@ -179,12 +183,12 @@ func testLargeMsg(t *testing.T, np string) { } func (s) TestLargeMsg(t *testing.T) { - for _, np := range nextProtocols { - testLargeMsg(t, np) + for _, rp := range recordProtocols { + testLargeMsg(t, rp) } } -func testIncorrectMsgType(t *testing.T, np string) { +func testIncorrectMsgType(t *testing.T, rp string) { // framedMsg is an empty ciphertext with correct framing but wrong // message type. framedMsg := make([]byte, MsgLenFieldSize+msgTypeFieldSize) @@ -193,7 +197,7 @@ func testIncorrectMsgType(t *testing.T, np string) { binary.LittleEndian.PutUint32(framedMsg[MsgLenFieldSize:], wrongMsgType) in := bytes.NewBuffer(framedMsg) - c := newTestALTSRecordConn(in, nil, core.ClientSide, np, nil) + c := newTestALTSRecordConn(in, nil, core.ClientSide, rp, nil) b := make([]byte, 1) if n, err := c.Read(b); n != 0 || err == nil { t.Fatalf("Read() = , want %v", fmt.Errorf("received frame with incorrect message type %v", wrongMsgType)) @@ -201,22 +205,22 @@ func testIncorrectMsgType(t *testing.T, np string) { } func (s) TestIncorrectMsgType(t *testing.T) { - for _, np := range nextProtocols { - testIncorrectMsgType(t, np) + for _, rp := range recordProtocols { + testIncorrectMsgType(t, rp) } } -func testFrameTooLarge(t *testing.T, np string) { +func testFrameTooLarge(t *testing.T, rp string) { buf := new(bytes.Buffer) - clientConn := newTestALTSRecordConn(nil, buf, core.ClientSide, np, nil) - serverConn := newTestALTSRecordConn(buf, nil, core.ServerSide, np, nil) + clientConn := newTestALTSRecordConn(nil, buf, core.ClientSide, rp, nil) + serverConn := newTestALTSRecordConn(buf, nil, core.ServerSide, rp, nil) // payloadLen is such that the length in the framing is larger than // allowed in one frame. payloadLen := altsRecordLengthLimit - msgTypeFieldSize - clientConn.crypto.EncryptionOverhead() + 1 payload := make([]byte, payloadLen) c, err := clientConn.crypto.Encrypt(nil, payload) if err != nil { - t.Fatalf(fmt.Sprintf("Error encrypting message: %v", err)) + t.Fatalf("Error encrypting message: %v", err) } msgLen := msgTypeFieldSize + len(c) framedMsg := make([]byte, MsgLenFieldSize+msgLen) @@ -225,7 +229,7 @@ func testFrameTooLarge(t *testing.T, np string) { binary.LittleEndian.PutUint32(msg[:msgTypeFieldSize], altsRecordMsgType) copy(msg[msgTypeFieldSize:], c) if _, err = buf.Write(framedMsg); err != nil { - t.Fatal(fmt.Sprintf("Unexpected error writing to buffer: %v", err)) + t.Fatalf("Unexpected error writing to buffer: %v", err) } b := make([]byte, 1) if n, err := serverConn.Read(b); n != 0 || err == nil { @@ -234,15 +238,15 @@ func testFrameTooLarge(t *testing.T, np string) { } func (s) TestFrameTooLarge(t *testing.T) { - for _, np := range nextProtocols { - testFrameTooLarge(t, np) + for _, rp := range recordProtocols { + testFrameTooLarge(t, rp) } } -func testWriteLargeData(t *testing.T, np string) { +func testWriteLargeData(t *testing.T, rp string) { // Test sending and receiving messages larger than the maximum write // buffer size. - clientConn, serverConn := newConnPair(np, nil, nil) + clientConn, serverConn := newConnPair(rp, nil, nil) // Message size is intentionally chosen to not be multiple of // payloadLengthLimtit. msgSize := altsWriteBufferMaxSize + (100 * 1024) @@ -277,25 +281,25 @@ func testWriteLargeData(t *testing.T, np string) { } func (s) TestWriteLargeData(t *testing.T) { - for _, np := range nextProtocols { - testWriteLargeData(t, np) + for _, rp := range recordProtocols { + testWriteLargeData(t, rp) } } -func testProtectedBuffer(t *testing.T, np string) { +func testProtectedBuffer(t *testing.T, rp string) { key := []byte{ // 16 arbitrary bytes. 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0xd2, 0x4c, 0xce, 0x4f, 0x49} // Encrypt a message to be passed to NewConn as a client-side protected // buffer. - newCrypto := protocols[np] + newCrypto := protocols[rp] if newCrypto == nil { - t.Fatalf("Unknown next protocol %q", np) + t.Fatalf("Unknown record protocol %q", rp) } crypto, err := newCrypto(core.ClientSide, key) if err != nil { - t.Fatalf("Failed to create a crypter for protocol %q: %v", np, err) + t.Fatalf("Failed to create a crypter for protocol %q: %v", rp, err) } msg := []byte("Client Protected Message") encryptedMsg, err := crypto.Encrypt(nil, msg) @@ -307,7 +311,7 @@ func testProtectedBuffer(t *testing.T, np string) { binary.LittleEndian.PutUint32(protectedMsg[4:], altsRecordMsgType) protectedMsg = append(protectedMsg, encryptedMsg...) - _, serverConn := newConnPair(np, nil, protectedMsg) + _, serverConn := newConnPair(rp, nil, protectedMsg) rcvClientMsg := make([]byte, len(msg)) if n, err := serverConn.Read(rcvClientMsg); n != len(rcvClientMsg) || err != nil { t.Fatalf("Server Read() = %v, %v; want %v, ", n, err, len(rcvClientMsg)) @@ -318,7 +322,7 @@ func testProtectedBuffer(t *testing.T, np string) { } func (s) TestProtectedBuffer(t *testing.T) { - for _, np := range nextProtocols { - testProtectedBuffer(t, np) + for _, rp := range recordProtocols { + testProtectedBuffer(t, rp) } } diff --git a/credentials/alts/internal/handshaker/handshaker.go b/credentials/alts/internal/handshaker/handshaker.go index 8bc7ceee0aff..0854e7af6518 100644 --- a/credentials/alts/internal/handshaker/handshaker.go +++ b/credentials/alts/internal/handshaker/handshaker.go @@ -25,8 +25,8 @@ import ( "fmt" "io" "net" - "sync" + "golang.org/x/sync/semaphore" grpc "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" @@ -35,15 +35,13 @@ import ( "google.golang.org/grpc/credentials/alts/internal/conn" altsgrpc "google.golang.org/grpc/credentials/alts/internal/proto/grpc_gcp" altspb "google.golang.org/grpc/credentials/alts/internal/proto/grpc_gcp" + "google.golang.org/grpc/internal/envconfig" ) const ( // The maximum byte size of receive frames. frameLimit = 64 * 1024 // 64 KB rekeyRecordProtocolName = "ALTSRP_GCM_AES128_REKEY" - // maxPendingHandshakes represents the maximum number of concurrent - // handshakes. - maxPendingHandshakes = 100 ) var ( @@ -59,9 +57,9 @@ var ( return conn.NewAES128GCMRekey(s, keyData) }, } - // control number of concurrent created (but not closed) handshakers. - mu sync.Mutex - concurrentHandshakes = int64(0) + // control number of concurrent created (but not closed) handshakes. + clientHandshakes = semaphore.NewWeighted(int64(envconfig.ALTSMaxConcurrentHandshakes)) + serverHandshakes = semaphore.NewWeighted(int64(envconfig.ALTSMaxConcurrentHandshakes)) // errDropped occurs when maxPendingHandshakes is reached. errDropped = errors.New("maximum number of concurrent ALTS handshakes is reached") // errOutOfBound occurs when the handshake service returns a consumed @@ -77,30 +75,6 @@ func init() { } } -func acquire() bool { - mu.Lock() - // If we need n to be configurable, we can pass it as an argument. - n := int64(1) - success := maxPendingHandshakes-concurrentHandshakes >= n - if success { - concurrentHandshakes += n - } - mu.Unlock() - return success -} - -func release() { - mu.Lock() - // If we need n to be configurable, we can pass it as an argument. - n := int64(1) - concurrentHandshakes -= n - if concurrentHandshakes < 0 { - mu.Unlock() - panic("bad release") - } - mu.Unlock() -} - // ClientHandshakerOptions contains the client handshaker options that can // provided by the caller. type ClientHandshakerOptions struct { @@ -134,11 +108,7 @@ func DefaultServerHandshakerOptions() *ServerHandshakerOptions { return &ServerHandshakerOptions{} } -// TODO: add support for future local and remote endpoint in both client options -// and server options (server options struct does not exist now. When -// caller can provide endpoints, it should be created. - -// altsHandshaker is used to complete a ALTS handshaking between client and +// altsHandshaker is used to complete an ALTS handshake between client and // server. This handshaker talks to the ALTS handshaker service in the metadata // server. type altsHandshaker struct { @@ -146,6 +116,8 @@ type altsHandshaker struct { stream altsgrpc.HandshakerService_DoHandshakeClient // the connection to the peer. conn net.Conn + // a virtual connection to the ALTS handshaker service. + clientConn *grpc.ClientConn // client handshake options. clientOpts *ClientHandshakerOptions // server handshake options. @@ -154,50 +126,54 @@ type altsHandshaker struct { side core.Side } -// NewClientHandshaker creates a ALTS handshaker for GCP which contains an RPC -// stub created using the passed conn and used to talk to the ALTS Handshaker +// NewClientHandshaker creates a core.Handshaker that performs a client-side +// ALTS handshake by acting as a proxy between the peer and the ALTS handshaker // service in the metadata server. func NewClientHandshaker(ctx context.Context, conn *grpc.ClientConn, c net.Conn, opts *ClientHandshakerOptions) (core.Handshaker, error) { - stream, err := altsgrpc.NewHandshakerServiceClient(conn).DoHandshake(ctx, grpc.WaitForReady(true)) - if err != nil { - return nil, err - } return &altsHandshaker{ - stream: stream, + stream: nil, conn: c, + clientConn: conn, clientOpts: opts, side: core.ClientSide, }, nil } -// NewServerHandshaker creates a ALTS handshaker for GCP which contains an RPC -// stub created using the passed conn and used to talk to the ALTS Handshaker +// NewServerHandshaker creates a core.Handshaker that performs a server-side +// ALTS handshake by acting as a proxy between the peer and the ALTS handshaker // service in the metadata server. func NewServerHandshaker(ctx context.Context, conn *grpc.ClientConn, c net.Conn, opts *ServerHandshakerOptions) (core.Handshaker, error) { - stream, err := altsgrpc.NewHandshakerServiceClient(conn).DoHandshake(ctx, grpc.WaitForReady(true)) - if err != nil { - return nil, err - } return &altsHandshaker{ - stream: stream, + stream: nil, conn: c, + clientConn: conn, serverOpts: opts, side: core.ServerSide, }, nil } -// ClientHandshake starts and completes a client ALTS handshaking for GCP. Once +// ClientHandshake starts and completes a client ALTS handshake for GCP. Once // done, ClientHandshake returns a secure connection. func (h *altsHandshaker) ClientHandshake(ctx context.Context) (net.Conn, credentials.AuthInfo, error) { - if !acquire() { + if !clientHandshakes.TryAcquire(1) { return nil, nil, errDropped } - defer release() + defer clientHandshakes.Release(1) if h.side != core.ClientSide { return nil, nil, errors.New("only handshakers created using NewClientHandshaker can perform a client handshaker") } + // TODO(matthewstevenson88): Change unit tests to use public APIs so + // that h.stream can unconditionally be set based on h.clientConn. + if h.stream == nil { + stream, err := altsgrpc.NewHandshakerServiceClient(h.clientConn).DoHandshake(ctx) + if err != nil { + return nil, nil, fmt.Errorf("failed to establish stream to ALTS handshaker service: %v", err) + } + h.stream = stream + } + // Create target identities from service account list. targetIdentities := make([]*altspb.Identity, 0, len(h.clientOpts.TargetServiceAccounts)) for _, account := range h.clientOpts.TargetServiceAccounts { @@ -229,18 +205,28 @@ func (h *altsHandshaker) ClientHandshake(ctx context.Context) (net.Conn, credent return conn, authInfo, nil } -// ServerHandshake starts and completes a server ALTS handshaking for GCP. Once +// ServerHandshake starts and completes a server ALTS handshake for GCP. Once // done, ServerHandshake returns a secure connection. func (h *altsHandshaker) ServerHandshake(ctx context.Context) (net.Conn, credentials.AuthInfo, error) { - if !acquire() { + if !serverHandshakes.TryAcquire(1) { return nil, nil, errDropped } - defer release() + defer serverHandshakes.Release(1) if h.side != core.ServerSide { return nil, nil, errors.New("only handshakers created using NewServerHandshaker can perform a server handshaker") } + // TODO(matthewstevenson88): Change unit tests to use public APIs so + // that h.stream can unconditionally be set based on h.clientConn. + if h.stream == nil { + stream, err := altsgrpc.NewHandshakerServiceClient(h.clientConn).DoHandshake(ctx) + if err != nil { + return nil, nil, fmt.Errorf("failed to establish stream to ALTS handshaker service: %v", err) + } + h.stream = stream + } + p := make([]byte, frameLimit) n, err := h.conn.Read(p) if err != nil { @@ -248,8 +234,6 @@ func (h *altsHandshaker) ServerHandshake(ctx context.Context) (net.Conn, credent } // Prepare server parameters. - // TODO: currently only ALTS parameters are provided. Might need to use - // more options in the future. params := make(map[int32]*altspb.ServerHandshakeParameters) params[int32(altspb.HandshakeProtocol_ALTS)] = &altspb.ServerHandshakeParameters{ RecordProtocols: recordProtocols, @@ -371,5 +355,14 @@ func (h *altsHandshaker) processUntilDone(resp *altspb.HandshakerResp, extra []b // Close terminates the Handshaker. It should be called when the caller obtains // the secure connection. func (h *altsHandshaker) Close() { - h.stream.CloseSend() + if h.stream != nil { + h.stream.CloseSend() + } +} + +// ResetConcurrentHandshakeSemaphoreForTesting resets the handshake semaphores +// to allow numberOfAllowedHandshakes concurrent handshakes each. +func ResetConcurrentHandshakeSemaphoreForTesting(numberOfAllowedHandshakes int64) { + clientHandshakes = semaphore.NewWeighted(numberOfAllowedHandshakes) + serverHandshakes = semaphore.NewWeighted(numberOfAllowedHandshakes) } diff --git a/credentials/alts/internal/handshaker/handshaker_test.go b/credentials/alts/internal/handshaker/handshaker_test.go index 9214f647c853..40d66161c7b6 100644 --- a/credentials/alts/internal/handshaker/handshaker_test.go +++ b/credentials/alts/internal/handshaker/handshaker_test.go @@ -21,13 +21,17 @@ package handshaker import ( "bytes" "context" + "errors" "testing" "time" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" grpc "google.golang.org/grpc" core "google.golang.org/grpc/credentials/alts/internal" altspb "google.golang.org/grpc/credentials/alts/internal/proto/grpc_gcp" "google.golang.org/grpc/credentials/alts/internal/testutil" + "google.golang.org/grpc/internal/envconfig" "google.golang.org/grpc/internal/grpctest" ) @@ -56,6 +60,8 @@ var ( } ) +const defaultTestTimeout = 10 * time.Second + // testRPCStream mimics a altspb.HandshakerService_DoHandshakeClient object. type testRPCStream struct { grpc.ClientStream @@ -129,10 +135,14 @@ func (s) TestClientHandshake(t *testing.T) { numberOfHandshakes int }{ {0 * time.Millisecond, 1}, - {100 * time.Millisecond, 10 * maxPendingHandshakes}, + {100 * time.Millisecond, 10 * int(envconfig.ALTSMaxConcurrentHandshakes)}, } { errc := make(chan error) stat.Reset() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + for i := 0; i < testCase.numberOfHandshakes; i++ { stream := &testRPCStream{ t: t, @@ -155,9 +165,10 @@ func (s) TestClientHandshake(t *testing.T) { side: core.ClientSide, } go func() { - _, context, err := chs.ClientHandshake(context.Background()) + _, context, err := chs.ClientHandshake(ctx) if err == nil && context == nil { - panic("expected non-nil ALTS context") + errc <- errors.New("expected non-nil ALTS context") + return } errc <- err chs.Close() @@ -172,8 +183,8 @@ func (s) TestClientHandshake(t *testing.T) { } // Ensure that there are no concurrent calls more than the limit. - if stat.MaxConcurrentCalls > maxPendingHandshakes { - t.Errorf("Observed %d concurrent handshakes; want <= %d", stat.MaxConcurrentCalls, maxPendingHandshakes) + if stat.MaxConcurrentCalls > int(envconfig.ALTSMaxConcurrentHandshakes) { + t.Errorf("Observed %d concurrent handshakes; want <= %d", stat.MaxConcurrentCalls, envconfig.ALTSMaxConcurrentHandshakes) } } } @@ -184,10 +195,14 @@ func (s) TestServerHandshake(t *testing.T) { numberOfHandshakes int }{ {0 * time.Millisecond, 1}, - {100 * time.Millisecond, 10 * maxPendingHandshakes}, + {100 * time.Millisecond, 10 * int(envconfig.ALTSMaxConcurrentHandshakes)}, } { errc := make(chan error) stat.Reset() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + for i := 0; i < testCase.numberOfHandshakes; i++ { stream := &testRPCStream{ t: t, @@ -207,9 +222,10 @@ func (s) TestServerHandshake(t *testing.T) { side: core.ServerSide, } go func() { - _, context, err := shs.ServerHandshake(context.Background()) + _, context, err := shs.ServerHandshake(ctx) if err == nil && context == nil { - panic("expected non-nil ALTS context") + errc <- errors.New("expected non-nil ALTS context") + return } errc <- err shs.Close() @@ -224,8 +240,8 @@ func (s) TestServerHandshake(t *testing.T) { } // Ensure that there are no concurrent calls more than the limit. - if stat.MaxConcurrentCalls > maxPendingHandshakes { - t.Errorf("Observed %d concurrent handshakes; want <= %d", stat.MaxConcurrentCalls, maxPendingHandshakes) + if stat.MaxConcurrentCalls > int(envconfig.ALTSMaxConcurrentHandshakes) { + t.Errorf("Observed %d concurrent handshakes; want <= %d", stat.MaxConcurrentCalls, envconfig.ALTSMaxConcurrentHandshakes) } } } @@ -258,7 +274,10 @@ func (s) TestPeerNotResponding(t *testing.T) { }, side: core.ClientSide, } - _, context, err := chs.ClientHandshake(context.Background()) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + _, context, err := chs.ClientHandshake(ctx) chs.Close() if context != nil { t.Error("expected non-nil ALTS context") @@ -267,3 +286,67 @@ func (s) TestPeerNotResponding(t *testing.T) { t.Errorf("ClientHandshake() = %v, want %v", got, want) } } + +func (s) TestNewClientHandshaker(t *testing.T) { + conn := testutil.NewTestConn(nil, nil) + clientConn := &grpc.ClientConn{} + opts := &ClientHandshakerOptions{} + hs, err := NewClientHandshaker(context.Background(), clientConn, conn, opts) + if err != nil { + t.Errorf("NewClientHandshaker returned unexpected error: %v", err) + } + expectedHs := &altsHandshaker{ + stream: nil, + conn: conn, + clientConn: clientConn, + clientOpts: opts, + serverOpts: nil, + side: core.ClientSide, + } + cmpOpts := []cmp.Option{ + cmp.AllowUnexported(altsHandshaker{}), + cmpopts.IgnoreFields(altsHandshaker{}, "conn", "clientConn"), + } + if got, want := hs.(*altsHandshaker), expectedHs; !cmp.Equal(got, want, cmpOpts...) { + t.Errorf("NewClientHandshaker() returned unexpected handshaker: got: %v, want: %v", got, want) + } + if hs.(*altsHandshaker).stream != nil { + t.Errorf("NewClientHandshaker() returned handshaker with non-nil stream") + } + if hs.(*altsHandshaker).clientConn != clientConn { + t.Errorf("NewClientHandshaker() returned handshaker with unexpected clientConn") + } + hs.Close() +} + +func (s) TestNewServerHandshaker(t *testing.T) { + conn := testutil.NewTestConn(nil, nil) + clientConn := &grpc.ClientConn{} + opts := &ServerHandshakerOptions{} + hs, err := NewServerHandshaker(context.Background(), clientConn, conn, opts) + if err != nil { + t.Errorf("NewServerHandshaker returned unexpected error: %v", err) + } + expectedHs := &altsHandshaker{ + stream: nil, + conn: conn, + clientConn: clientConn, + clientOpts: nil, + serverOpts: opts, + side: core.ServerSide, + } + cmpOpts := []cmp.Option{ + cmp.AllowUnexported(altsHandshaker{}), + cmpopts.IgnoreFields(altsHandshaker{}, "conn", "clientConn"), + } + if got, want := hs.(*altsHandshaker), expectedHs; !cmp.Equal(got, want, cmpOpts...) { + t.Errorf("NewServerHandshaker() returned unexpected handshaker: got: %v, want: %v", got, want) + } + if hs.(*altsHandshaker).stream != nil { + t.Errorf("NewServerHandshaker() returned handshaker with non-nil stream") + } + if hs.(*altsHandshaker).clientConn != clientConn { + t.Errorf("NewServerHandshaker() returned handshaker with unexpected clientConn") + } + hs.Close() +} diff --git a/credentials/alts/internal/handshaker/service/service.go b/credentials/alts/internal/handshaker/service/service.go index 77d759cd956f..e1cdafb980cd 100644 --- a/credentials/alts/internal/handshaker/service/service.go +++ b/credentials/alts/internal/handshaker/service/service.go @@ -24,6 +24,7 @@ import ( "sync" grpc "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" ) var ( @@ -49,7 +50,7 @@ func Dial(hsAddress string) (*grpc.ClientConn, error) { // Create a new connection to the handshaker service. Note that // this connection stays open until the application is closed. var err error - hsConn, err = hsDialer(hsAddress, grpc.WithInsecure()) + hsConn, err = hsDialer(hsAddress, grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { return nil, err } @@ -57,3 +58,21 @@ func Dial(hsAddress string) (*grpc.ClientConn, error) { } return hsConn, nil } + +// CloseForTesting closes all open connections to the handshaker service. +// +// For testing purposes only. +func CloseForTesting() error { + for _, hsConn := range hsConnMap { + if hsConn == nil { + continue + } + if err := hsConn.Close(); err != nil { + return err + } + } + + // Reset the connection map. + hsConnMap = make(map[string]*grpc.ClientConn) + return nil +} diff --git a/credentials/alts/internal/proto/grpc_gcp/altscontext.pb.go b/credentials/alts/internal/proto/grpc_gcp/altscontext.pb.go index 38c4832dfd4b..c7cf1810a196 100644 --- a/credentials/alts/internal/proto/grpc_gcp/altscontext.pb.go +++ b/credentials/alts/internal/proto/grpc_gcp/altscontext.pb.go @@ -1,26 +1,47 @@ +// Copyright 2018 The gRPC Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The canonical version of this proto can be found at +// https://github.com/grpc/grpc-proto/blob/master/grpc/gcp/altscontext.proto + // Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.31.0 +// protoc v4.22.0 // source: grpc/gcp/altscontext.proto package grpc_gcp import ( - fmt "fmt" - proto "github.com/golang/protobuf/proto" - math "math" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" ) -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) type AltsContext struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // The application protocol negotiated for this connection. ApplicationProtocol string `protobuf:"bytes,1,opt,name=application_protocol,json=applicationProtocol,proto3" json:"application_protocol,omitempty"` // The record protocol negotiated for this connection. @@ -34,119 +55,205 @@ type AltsContext struct { // The RPC protocol versions supported by the peer. PeerRpcVersions *RpcProtocolVersions `protobuf:"bytes,6,opt,name=peer_rpc_versions,json=peerRpcVersions,proto3" json:"peer_rpc_versions,omitempty"` // Additional attributes of the peer. - PeerAttributes map[string]string `protobuf:"bytes,7,rep,name=peer_attributes,json=peerAttributes,proto3" json:"peer_attributes,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + PeerAttributes map[string]string `protobuf:"bytes,7,rep,name=peer_attributes,json=peerAttributes,proto3" json:"peer_attributes,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` } -func (m *AltsContext) Reset() { *m = AltsContext{} } -func (m *AltsContext) String() string { return proto.CompactTextString(m) } -func (*AltsContext) ProtoMessage() {} -func (*AltsContext) Descriptor() ([]byte, []int) { - return fileDescriptor_6647a41e53a575a3, []int{0} +func (x *AltsContext) Reset() { + *x = AltsContext{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_gcp_altscontext_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *AltsContext) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_AltsContext.Unmarshal(m, b) +func (x *AltsContext) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *AltsContext) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_AltsContext.Marshal(b, m, deterministic) -} -func (m *AltsContext) XXX_Merge(src proto.Message) { - xxx_messageInfo_AltsContext.Merge(m, src) -} -func (m *AltsContext) XXX_Size() int { - return xxx_messageInfo_AltsContext.Size(m) -} -func (m *AltsContext) XXX_DiscardUnknown() { - xxx_messageInfo_AltsContext.DiscardUnknown(m) + +func (*AltsContext) ProtoMessage() {} + +func (x *AltsContext) ProtoReflect() protoreflect.Message { + mi := &file_grpc_gcp_altscontext_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_AltsContext proto.InternalMessageInfo +// Deprecated: Use AltsContext.ProtoReflect.Descriptor instead. +func (*AltsContext) Descriptor() ([]byte, []int) { + return file_grpc_gcp_altscontext_proto_rawDescGZIP(), []int{0} +} -func (m *AltsContext) GetApplicationProtocol() string { - if m != nil { - return m.ApplicationProtocol +func (x *AltsContext) GetApplicationProtocol() string { + if x != nil { + return x.ApplicationProtocol } return "" } -func (m *AltsContext) GetRecordProtocol() string { - if m != nil { - return m.RecordProtocol +func (x *AltsContext) GetRecordProtocol() string { + if x != nil { + return x.RecordProtocol } return "" } -func (m *AltsContext) GetSecurityLevel() SecurityLevel { - if m != nil { - return m.SecurityLevel +func (x *AltsContext) GetSecurityLevel() SecurityLevel { + if x != nil { + return x.SecurityLevel } return SecurityLevel_SECURITY_NONE } -func (m *AltsContext) GetPeerServiceAccount() string { - if m != nil { - return m.PeerServiceAccount +func (x *AltsContext) GetPeerServiceAccount() string { + if x != nil { + return x.PeerServiceAccount } return "" } -func (m *AltsContext) GetLocalServiceAccount() string { - if m != nil { - return m.LocalServiceAccount +func (x *AltsContext) GetLocalServiceAccount() string { + if x != nil { + return x.LocalServiceAccount } return "" } -func (m *AltsContext) GetPeerRpcVersions() *RpcProtocolVersions { - if m != nil { - return m.PeerRpcVersions +func (x *AltsContext) GetPeerRpcVersions() *RpcProtocolVersions { + if x != nil { + return x.PeerRpcVersions } return nil } -func (m *AltsContext) GetPeerAttributes() map[string]string { - if m != nil { - return m.PeerAttributes +func (x *AltsContext) GetPeerAttributes() map[string]string { + if x != nil { + return x.PeerAttributes } return nil } -func init() { - proto.RegisterType((*AltsContext)(nil), "grpc.gcp.AltsContext") - proto.RegisterMapType((map[string]string)(nil), "grpc.gcp.AltsContext.PeerAttributesEntry") -} - -func init() { proto.RegisterFile("grpc/gcp/altscontext.proto", fileDescriptor_6647a41e53a575a3) } - -var fileDescriptor_6647a41e53a575a3 = []byte{ - // 411 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x92, 0x4d, 0x6f, 0x13, 0x31, - 0x10, 0x86, 0xb5, 0x0d, 0x2d, 0xe0, 0x88, 0xb4, 0xb8, 0xa9, 0x58, 0x45, 0x42, 0x8a, 0xb8, 0xb0, - 0x5c, 0x76, 0x21, 0x5c, 0x10, 0x07, 0x50, 0x8a, 0x38, 0x20, 0x71, 0x88, 0xb6, 0x12, 0x07, 0x2e, - 0x2b, 0x77, 0x3a, 0xb2, 0x2c, 0x5c, 0x8f, 0x35, 0x76, 0x22, 0xf2, 0xb3, 0xf9, 0x07, 0x68, 0xed, - 0xcd, 0x07, 0x1f, 0xb7, 0x9d, 0x79, 0x9f, 0x19, 0xbf, 0xb3, 0x33, 0x62, 0xa6, 0xd9, 0x43, 0xa3, - 0xc1, 0x37, 0xca, 0xc6, 0x00, 0xe4, 0x22, 0xfe, 0x8c, 0xb5, 0x67, 0x8a, 0x24, 0x1f, 0xf5, 0x5a, - 0xad, 0xc1, 0xcf, 0xaa, 0x3d, 0x15, 0x59, 0xb9, 0xe0, 0x89, 0x63, 0x17, 0x10, 0xd6, 0x6c, 0xe2, - 0xb6, 0x03, 0xba, 0xbf, 0x27, 0x97, 0x6b, 0x5e, 0xfc, 0x1a, 0x89, 0xf1, 0xd2, 0xc6, 0xf0, 0x29, - 0x77, 0x92, 0x6f, 0xc4, 0x54, 0x79, 0x6f, 0x0d, 0xa8, 0x68, 0xc8, 0x75, 0x09, 0x02, 0xb2, 0x65, - 0x31, 0x2f, 0xaa, 0xc7, 0xed, 0xe5, 0x91, 0xb6, 0x1a, 0x24, 0xf9, 0x52, 0x9c, 0x33, 0x02, 0xf1, - 0xdd, 0x81, 0x3e, 0x49, 0xf4, 0x24, 0xa7, 0xf7, 0xe0, 0x07, 0x31, 0xd9, 0x9b, 0xb0, 0xb8, 0x41, - 0x5b, 0x8e, 0xe6, 0x45, 0x35, 0x59, 0x3c, 0xab, 0x77, 0xc6, 0xeb, 0x9b, 0x41, 0xff, 0xda, 0xcb, - 0xed, 0x93, 0x70, 0x1c, 0xca, 0xd7, 0x62, 0xea, 0x11, 0xb9, 0x0b, 0xc8, 0x1b, 0x03, 0xd8, 0x29, - 0x00, 0x5a, 0xbb, 0x58, 0x3e, 0x48, 0xaf, 0xc9, 0x5e, 0xbb, 0xc9, 0xd2, 0x32, 0x2b, 0x72, 0x21, - 0xae, 0x2c, 0x81, 0xb2, 0xff, 0x94, 0x9c, 0xe6, 0x71, 0x92, 0xf8, 0x57, 0xcd, 0x17, 0xf1, 0x34, - 0xbd, 0xc2, 0x1e, 0xba, 0x0d, 0x72, 0x30, 0xe4, 0x42, 0x79, 0x36, 0x2f, 0xaa, 0xf1, 0xe2, 0xf9, - 0xc1, 0x68, 0xeb, 0x61, 0x37, 0xd7, 0xb7, 0x01, 0x6a, 0xcf, 0xfb, 0xba, 0xd6, 0xc3, 0x2e, 0x21, - 0x5b, 0x91, 0x52, 0x9d, 0x8a, 0x91, 0xcd, 0xed, 0x3a, 0x62, 0x28, 0x1f, 0xce, 0x47, 0xd5, 0x78, - 0xf1, 0xea, 0xd0, 0xe8, 0xe8, 0xe7, 0xd7, 0x2b, 0x44, 0x5e, 0xee, 0xd9, 0xcf, 0x2e, 0xf2, 0xb6, - 0x9d, 0xf8, 0x3f, 0x92, 0xb3, 0xa5, 0xb8, 0xfc, 0x0f, 0x26, 0x2f, 0xc4, 0xe8, 0x07, 0x6e, 0x87, - 0x35, 0xf5, 0x9f, 0x72, 0x2a, 0x4e, 0x37, 0xca, 0xae, 0x71, 0x58, 0x46, 0x0e, 0xde, 0x9f, 0xbc, - 0x2b, 0xae, 0xad, 0xb8, 0x32, 0x94, 0x1d, 0xf4, 0x47, 0x54, 0x1b, 0x17, 0x91, 0x9d, 0xb2, 0xd7, - 0x17, 0x47, 0x66, 0xd2, 0x74, 0xab, 0xe2, 0xfb, 0x47, 0x4d, 0xa4, 0x2d, 0xd6, 0x9a, 0xac, 0x72, - 0xba, 0x26, 0xd6, 0x4d, 0x3a, 0x2e, 0x60, 0xbc, 0x43, 0x17, 0x8d, 0xb2, 0x21, 0x9d, 0x62, 0xb3, - 0xeb, 0xd2, 0xa4, 0x2b, 0x48, 0x50, 0xa7, 0xc1, 0xdf, 0x9e, 0xa5, 0xf8, 0xed, 0xef, 0x00, 0x00, - 0x00, 0xff, 0xff, 0x9b, 0x8c, 0xe4, 0x6a, 0xba, 0x02, 0x00, 0x00, +var File_grpc_gcp_altscontext_proto protoreflect.FileDescriptor + +var file_grpc_gcp_altscontext_proto_rawDesc = []byte{ + 0x0a, 0x1a, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x67, 0x63, 0x70, 0x2f, 0x61, 0x6c, 0x74, 0x73, 0x63, + 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x08, 0x67, 0x72, + 0x70, 0x63, 0x2e, 0x67, 0x63, 0x70, 0x1a, 0x28, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x67, 0x63, 0x70, + 0x2f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x6f, 0x72, 0x74, 0x5f, 0x73, 0x65, 0x63, 0x75, 0x72, + 0x69, 0x74, 0x79, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x22, 0xf1, 0x03, 0x0a, 0x0b, 0x41, 0x6c, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, + 0x12, 0x31, 0x0a, 0x14, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x13, + 0x61, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x74, 0x6f, + 0x63, 0x6f, 0x6c, 0x12, 0x27, 0x0a, 0x0f, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x5f, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x72, 0x65, + 0x63, 0x6f, 0x72, 0x64, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x3e, 0x0a, 0x0e, + 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x5f, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x0e, 0x32, 0x17, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x63, 0x70, 0x2e, + 0x53, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x0d, 0x73, + 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x30, 0x0a, 0x14, + 0x70, 0x65, 0x65, 0x72, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x61, 0x63, 0x63, + 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x70, 0x65, 0x65, 0x72, + 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x41, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x32, + 0x0a, 0x15, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, + 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x13, 0x6c, + 0x6f, 0x63, 0x61, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x41, 0x63, 0x63, 0x6f, 0x75, + 0x6e, 0x74, 0x12, 0x49, 0x0a, 0x11, 0x70, 0x65, 0x65, 0x72, 0x5f, 0x72, 0x70, 0x63, 0x5f, 0x76, + 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, + 0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x63, 0x70, 0x2e, 0x52, 0x70, 0x63, 0x50, 0x72, 0x6f, 0x74, + 0x6f, 0x63, 0x6f, 0x6c, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x0f, 0x70, 0x65, + 0x65, 0x72, 0x52, 0x70, 0x63, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x52, 0x0a, + 0x0f, 0x70, 0x65, 0x65, 0x72, 0x5f, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, + 0x18, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x63, + 0x70, 0x2e, 0x41, 0x6c, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x2e, 0x50, 0x65, + 0x65, 0x72, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x52, 0x0e, 0x70, 0x65, 0x65, 0x72, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, + 0x73, 0x1a, 0x41, 0x0a, 0x13, 0x50, 0x65, 0x65, 0x72, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, + 0x74, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x3a, 0x02, 0x38, 0x01, 0x42, 0x6c, 0x0a, 0x15, 0x69, 0x6f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, + 0x61, 0x6c, 0x74, 0x73, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x42, 0x10, 0x41, + 0x6c, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, + 0x01, 0x5a, 0x3f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, + 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, + 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2f, 0x61, 0x6c, 0x74, 0x73, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, + 0x6e, 0x61, 0x6c, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x67, + 0x63, 0x70, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_grpc_gcp_altscontext_proto_rawDescOnce sync.Once + file_grpc_gcp_altscontext_proto_rawDescData = file_grpc_gcp_altscontext_proto_rawDesc +) + +func file_grpc_gcp_altscontext_proto_rawDescGZIP() []byte { + file_grpc_gcp_altscontext_proto_rawDescOnce.Do(func() { + file_grpc_gcp_altscontext_proto_rawDescData = protoimpl.X.CompressGZIP(file_grpc_gcp_altscontext_proto_rawDescData) + }) + return file_grpc_gcp_altscontext_proto_rawDescData +} + +var file_grpc_gcp_altscontext_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_grpc_gcp_altscontext_proto_goTypes = []interface{}{ + (*AltsContext)(nil), // 0: grpc.gcp.AltsContext + nil, // 1: grpc.gcp.AltsContext.PeerAttributesEntry + (SecurityLevel)(0), // 2: grpc.gcp.SecurityLevel + (*RpcProtocolVersions)(nil), // 3: grpc.gcp.RpcProtocolVersions +} +var file_grpc_gcp_altscontext_proto_depIdxs = []int32{ + 2, // 0: grpc.gcp.AltsContext.security_level:type_name -> grpc.gcp.SecurityLevel + 3, // 1: grpc.gcp.AltsContext.peer_rpc_versions:type_name -> grpc.gcp.RpcProtocolVersions + 1, // 2: grpc.gcp.AltsContext.peer_attributes:type_name -> grpc.gcp.AltsContext.PeerAttributesEntry + 3, // [3:3] is the sub-list for method output_type + 3, // [3:3] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name +} + +func init() { file_grpc_gcp_altscontext_proto_init() } +func file_grpc_gcp_altscontext_proto_init() { + if File_grpc_gcp_altscontext_proto != nil { + return + } + file_grpc_gcp_transport_security_common_proto_init() + if !protoimpl.UnsafeEnabled { + file_grpc_gcp_altscontext_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*AltsContext); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_grpc_gcp_altscontext_proto_rawDesc, + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_grpc_gcp_altscontext_proto_goTypes, + DependencyIndexes: file_grpc_gcp_altscontext_proto_depIdxs, + MessageInfos: file_grpc_gcp_altscontext_proto_msgTypes, + }.Build() + File_grpc_gcp_altscontext_proto = out.File + file_grpc_gcp_altscontext_proto_rawDesc = nil + file_grpc_gcp_altscontext_proto_goTypes = nil + file_grpc_gcp_altscontext_proto_depIdxs = nil } diff --git a/credentials/alts/internal/proto/grpc_gcp/handshaker.pb.go b/credentials/alts/internal/proto/grpc_gcp/handshaker.pb.go index a2060de402bc..81d0f1140847 100644 --- a/credentials/alts/internal/proto/grpc_gcp/handshaker.pb.go +++ b/credentials/alts/internal/proto/grpc_gcp/handshaker.pb.go @@ -1,28 +1,41 @@ +// Copyright 2018 The gRPC Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The canonical version of this proto can be found at +// https://github.com/grpc/grpc-proto/blob/master/grpc/gcp/handshaker.proto + // Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.31.0 +// protoc v4.22.0 // source: grpc/gcp/handshaker.proto package grpc_gcp import ( - context "context" - fmt "fmt" - proto "github.com/golang/protobuf/proto" - grpc "google.golang.org/grpc" - codes "google.golang.org/grpc/codes" - status "google.golang.org/grpc/status" - math "math" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" ) -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) type HandshakeProtocol int32 @@ -35,24 +48,45 @@ const ( HandshakeProtocol_ALTS HandshakeProtocol = 2 ) -var HandshakeProtocol_name = map[int32]string{ - 0: "HANDSHAKE_PROTOCOL_UNSPECIFIED", - 1: "TLS", - 2: "ALTS", -} +// Enum value maps for HandshakeProtocol. +var ( + HandshakeProtocol_name = map[int32]string{ + 0: "HANDSHAKE_PROTOCOL_UNSPECIFIED", + 1: "TLS", + 2: "ALTS", + } + HandshakeProtocol_value = map[string]int32{ + "HANDSHAKE_PROTOCOL_UNSPECIFIED": 0, + "TLS": 1, + "ALTS": 2, + } +) -var HandshakeProtocol_value = map[string]int32{ - "HANDSHAKE_PROTOCOL_UNSPECIFIED": 0, - "TLS": 1, - "ALTS": 2, +func (x HandshakeProtocol) Enum() *HandshakeProtocol { + p := new(HandshakeProtocol) + *p = x + return p } func (x HandshakeProtocol) String() string { - return proto.EnumName(HandshakeProtocol_name, int32(x)) + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (HandshakeProtocol) Descriptor() protoreflect.EnumDescriptor { + return file_grpc_gcp_handshaker_proto_enumTypes[0].Descriptor() +} + +func (HandshakeProtocol) Type() protoreflect.EnumType { + return &file_grpc_gcp_handshaker_proto_enumTypes[0] +} + +func (x HandshakeProtocol) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) } +// Deprecated: Use HandshakeProtocol.Descriptor instead. func (HandshakeProtocol) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_54c074f40c7c7e99, []int{0} + return file_grpc_gcp_handshaker_proto_rawDescGZIP(), []int{0} } type NetworkProtocol int32 @@ -63,138 +97,160 @@ const ( NetworkProtocol_UDP NetworkProtocol = 2 ) -var NetworkProtocol_name = map[int32]string{ - 0: "NETWORK_PROTOCOL_UNSPECIFIED", - 1: "TCP", - 2: "UDP", -} +// Enum value maps for NetworkProtocol. +var ( + NetworkProtocol_name = map[int32]string{ + 0: "NETWORK_PROTOCOL_UNSPECIFIED", + 1: "TCP", + 2: "UDP", + } + NetworkProtocol_value = map[string]int32{ + "NETWORK_PROTOCOL_UNSPECIFIED": 0, + "TCP": 1, + "UDP": 2, + } +) -var NetworkProtocol_value = map[string]int32{ - "NETWORK_PROTOCOL_UNSPECIFIED": 0, - "TCP": 1, - "UDP": 2, +func (x NetworkProtocol) Enum() *NetworkProtocol { + p := new(NetworkProtocol) + *p = x + return p } func (x NetworkProtocol) String() string { - return proto.EnumName(NetworkProtocol_name, int32(x)) + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (NetworkProtocol) Descriptor() protoreflect.EnumDescriptor { + return file_grpc_gcp_handshaker_proto_enumTypes[1].Descriptor() +} + +func (NetworkProtocol) Type() protoreflect.EnumType { + return &file_grpc_gcp_handshaker_proto_enumTypes[1] +} + +func (x NetworkProtocol) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) } +// Deprecated: Use NetworkProtocol.Descriptor instead. func (NetworkProtocol) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_54c074f40c7c7e99, []int{1} + return file_grpc_gcp_handshaker_proto_rawDescGZIP(), []int{1} } type Endpoint struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // IP address. It should contain an IPv4 or IPv6 string literal, e.g. // "192.168.0.1" or "2001:db8::1". IpAddress string `protobuf:"bytes,1,opt,name=ip_address,json=ipAddress,proto3" json:"ip_address,omitempty"` // Port number. Port int32 `protobuf:"varint,2,opt,name=port,proto3" json:"port,omitempty"` // Network protocol (e.g., TCP, UDP) associated with this endpoint. - Protocol NetworkProtocol `protobuf:"varint,3,opt,name=protocol,proto3,enum=grpc.gcp.NetworkProtocol" json:"protocol,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Protocol NetworkProtocol `protobuf:"varint,3,opt,name=protocol,proto3,enum=grpc.gcp.NetworkProtocol" json:"protocol,omitempty"` } -func (m *Endpoint) Reset() { *m = Endpoint{} } -func (m *Endpoint) String() string { return proto.CompactTextString(m) } -func (*Endpoint) ProtoMessage() {} -func (*Endpoint) Descriptor() ([]byte, []int) { - return fileDescriptor_54c074f40c7c7e99, []int{0} +func (x *Endpoint) Reset() { + *x = Endpoint{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_gcp_handshaker_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *Endpoint) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Endpoint.Unmarshal(m, b) -} -func (m *Endpoint) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Endpoint.Marshal(b, m, deterministic) +func (x *Endpoint) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *Endpoint) XXX_Merge(src proto.Message) { - xxx_messageInfo_Endpoint.Merge(m, src) -} -func (m *Endpoint) XXX_Size() int { - return xxx_messageInfo_Endpoint.Size(m) -} -func (m *Endpoint) XXX_DiscardUnknown() { - xxx_messageInfo_Endpoint.DiscardUnknown(m) + +func (*Endpoint) ProtoMessage() {} + +func (x *Endpoint) ProtoReflect() protoreflect.Message { + mi := &file_grpc_gcp_handshaker_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_Endpoint proto.InternalMessageInfo +// Deprecated: Use Endpoint.ProtoReflect.Descriptor instead. +func (*Endpoint) Descriptor() ([]byte, []int) { + return file_grpc_gcp_handshaker_proto_rawDescGZIP(), []int{0} +} -func (m *Endpoint) GetIpAddress() string { - if m != nil { - return m.IpAddress +func (x *Endpoint) GetIpAddress() string { + if x != nil { + return x.IpAddress } return "" } -func (m *Endpoint) GetPort() int32 { - if m != nil { - return m.Port +func (x *Endpoint) GetPort() int32 { + if x != nil { + return x.Port } return 0 } -func (m *Endpoint) GetProtocol() NetworkProtocol { - if m != nil { - return m.Protocol +func (x *Endpoint) GetProtocol() NetworkProtocol { + if x != nil { + return x.Protocol } return NetworkProtocol_NETWORK_PROTOCOL_UNSPECIFIED } type Identity struct { - // Types that are valid to be assigned to IdentityOneof: + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Types that are assignable to IdentityOneof: + // // *Identity_ServiceAccount // *Identity_Hostname IdentityOneof isIdentity_IdentityOneof `protobuf_oneof:"identity_oneof"` // Additional attributes of the identity. - Attributes map[string]string `protobuf:"bytes,3,rep,name=attributes,proto3" json:"attributes,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Attributes map[string]string `protobuf:"bytes,3,rep,name=attributes,proto3" json:"attributes,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` } -func (m *Identity) Reset() { *m = Identity{} } -func (m *Identity) String() string { return proto.CompactTextString(m) } -func (*Identity) ProtoMessage() {} -func (*Identity) Descriptor() ([]byte, []int) { - return fileDescriptor_54c074f40c7c7e99, []int{1} +func (x *Identity) Reset() { + *x = Identity{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_gcp_handshaker_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *Identity) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Identity.Unmarshal(m, b) -} -func (m *Identity) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Identity.Marshal(b, m, deterministic) -} -func (m *Identity) XXX_Merge(src proto.Message) { - xxx_messageInfo_Identity.Merge(m, src) -} -func (m *Identity) XXX_Size() int { - return xxx_messageInfo_Identity.Size(m) -} -func (m *Identity) XXX_DiscardUnknown() { - xxx_messageInfo_Identity.DiscardUnknown(m) +func (x *Identity) String() string { + return protoimpl.X.MessageStringOf(x) } -var xxx_messageInfo_Identity proto.InternalMessageInfo +func (*Identity) ProtoMessage() {} -type isIdentity_IdentityOneof interface { - isIdentity_IdentityOneof() -} - -type Identity_ServiceAccount struct { - ServiceAccount string `protobuf:"bytes,1,opt,name=service_account,json=serviceAccount,proto3,oneof"` +func (x *Identity) ProtoReflect() protoreflect.Message { + mi := &file_grpc_gcp_handshaker_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -type Identity_Hostname struct { - Hostname string `protobuf:"bytes,2,opt,name=hostname,proto3,oneof"` +// Deprecated: Use Identity.ProtoReflect.Descriptor instead. +func (*Identity) Descriptor() ([]byte, []int) { + return file_grpc_gcp_handshaker_proto_rawDescGZIP(), []int{1} } -func (*Identity_ServiceAccount) isIdentity_IdentityOneof() {} - -func (*Identity_Hostname) isIdentity_IdentityOneof() {} - func (m *Identity) GetIdentityOneof() isIdentity_IdentityOneof { if m != nil { return m.IdentityOneof @@ -202,36 +258,50 @@ func (m *Identity) GetIdentityOneof() isIdentity_IdentityOneof { return nil } -func (m *Identity) GetServiceAccount() string { - if x, ok := m.GetIdentityOneof().(*Identity_ServiceAccount); ok { +func (x *Identity) GetServiceAccount() string { + if x, ok := x.GetIdentityOneof().(*Identity_ServiceAccount); ok { return x.ServiceAccount } return "" } -func (m *Identity) GetHostname() string { - if x, ok := m.GetIdentityOneof().(*Identity_Hostname); ok { +func (x *Identity) GetHostname() string { + if x, ok := x.GetIdentityOneof().(*Identity_Hostname); ok { return x.Hostname } return "" } -func (m *Identity) GetAttributes() map[string]string { - if m != nil { - return m.Attributes +func (x *Identity) GetAttributes() map[string]string { + if x != nil { + return x.Attributes } return nil } -// XXX_OneofWrappers is for the internal use of the proto package. -func (*Identity) XXX_OneofWrappers() []interface{} { - return []interface{}{ - (*Identity_ServiceAccount)(nil), - (*Identity_Hostname)(nil), - } +type isIdentity_IdentityOneof interface { + isIdentity_IdentityOneof() +} + +type Identity_ServiceAccount struct { + // Service account of a connection endpoint. + ServiceAccount string `protobuf:"bytes,1,opt,name=service_account,json=serviceAccount,proto3,oneof"` } +type Identity_Hostname struct { + // Hostname of a connection endpoint. + Hostname string `protobuf:"bytes,2,opt,name=hostname,proto3,oneof"` +} + +func (*Identity_ServiceAccount) isIdentity_IdentityOneof() {} + +func (*Identity_Hostname) isIdentity_IdentityOneof() {} + type StartClientHandshakeReq struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // Handshake security protocol requested by the client. HandshakeSecurityProtocol HandshakeProtocol `protobuf:"varint,1,opt,name=handshake_security_protocol,json=handshakeSecurityProtocol,proto3,enum=grpc.gcp.HandshakeProtocol" json:"handshake_security_protocol,omitempty"` // The application protocols supported by the client, e.g., "h2" (for http2), @@ -260,159 +330,175 @@ type StartClientHandshakeReq struct { // (Optional) RPC protocol versions supported by the client. RpcVersions *RpcProtocolVersions `protobuf:"bytes,9,opt,name=rpc_versions,json=rpcVersions,proto3" json:"rpc_versions,omitempty"` // (Optional) Maximum frame size supported by the client. - MaxFrameSize uint32 `protobuf:"varint,10,opt,name=max_frame_size,json=maxFrameSize,proto3" json:"max_frame_size,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + MaxFrameSize uint32 `protobuf:"varint,10,opt,name=max_frame_size,json=maxFrameSize,proto3" json:"max_frame_size,omitempty"` } -func (m *StartClientHandshakeReq) Reset() { *m = StartClientHandshakeReq{} } -func (m *StartClientHandshakeReq) String() string { return proto.CompactTextString(m) } -func (*StartClientHandshakeReq) ProtoMessage() {} -func (*StartClientHandshakeReq) Descriptor() ([]byte, []int) { - return fileDescriptor_54c074f40c7c7e99, []int{2} +func (x *StartClientHandshakeReq) Reset() { + *x = StartClientHandshakeReq{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_gcp_handshaker_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *StartClientHandshakeReq) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_StartClientHandshakeReq.Unmarshal(m, b) -} -func (m *StartClientHandshakeReq) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_StartClientHandshakeReq.Marshal(b, m, deterministic) +func (x *StartClientHandshakeReq) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *StartClientHandshakeReq) XXX_Merge(src proto.Message) { - xxx_messageInfo_StartClientHandshakeReq.Merge(m, src) -} -func (m *StartClientHandshakeReq) XXX_Size() int { - return xxx_messageInfo_StartClientHandshakeReq.Size(m) -} -func (m *StartClientHandshakeReq) XXX_DiscardUnknown() { - xxx_messageInfo_StartClientHandshakeReq.DiscardUnknown(m) + +func (*StartClientHandshakeReq) ProtoMessage() {} + +func (x *StartClientHandshakeReq) ProtoReflect() protoreflect.Message { + mi := &file_grpc_gcp_handshaker_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_StartClientHandshakeReq proto.InternalMessageInfo +// Deprecated: Use StartClientHandshakeReq.ProtoReflect.Descriptor instead. +func (*StartClientHandshakeReq) Descriptor() ([]byte, []int) { + return file_grpc_gcp_handshaker_proto_rawDescGZIP(), []int{2} +} -func (m *StartClientHandshakeReq) GetHandshakeSecurityProtocol() HandshakeProtocol { - if m != nil { - return m.HandshakeSecurityProtocol +func (x *StartClientHandshakeReq) GetHandshakeSecurityProtocol() HandshakeProtocol { + if x != nil { + return x.HandshakeSecurityProtocol } return HandshakeProtocol_HANDSHAKE_PROTOCOL_UNSPECIFIED } -func (m *StartClientHandshakeReq) GetApplicationProtocols() []string { - if m != nil { - return m.ApplicationProtocols +func (x *StartClientHandshakeReq) GetApplicationProtocols() []string { + if x != nil { + return x.ApplicationProtocols } return nil } -func (m *StartClientHandshakeReq) GetRecordProtocols() []string { - if m != nil { - return m.RecordProtocols +func (x *StartClientHandshakeReq) GetRecordProtocols() []string { + if x != nil { + return x.RecordProtocols } return nil } -func (m *StartClientHandshakeReq) GetTargetIdentities() []*Identity { - if m != nil { - return m.TargetIdentities +func (x *StartClientHandshakeReq) GetTargetIdentities() []*Identity { + if x != nil { + return x.TargetIdentities } return nil } -func (m *StartClientHandshakeReq) GetLocalIdentity() *Identity { - if m != nil { - return m.LocalIdentity +func (x *StartClientHandshakeReq) GetLocalIdentity() *Identity { + if x != nil { + return x.LocalIdentity } return nil } -func (m *StartClientHandshakeReq) GetLocalEndpoint() *Endpoint { - if m != nil { - return m.LocalEndpoint +func (x *StartClientHandshakeReq) GetLocalEndpoint() *Endpoint { + if x != nil { + return x.LocalEndpoint } return nil } -func (m *StartClientHandshakeReq) GetRemoteEndpoint() *Endpoint { - if m != nil { - return m.RemoteEndpoint +func (x *StartClientHandshakeReq) GetRemoteEndpoint() *Endpoint { + if x != nil { + return x.RemoteEndpoint } return nil } -func (m *StartClientHandshakeReq) GetTargetName() string { - if m != nil { - return m.TargetName +func (x *StartClientHandshakeReq) GetTargetName() string { + if x != nil { + return x.TargetName } return "" } -func (m *StartClientHandshakeReq) GetRpcVersions() *RpcProtocolVersions { - if m != nil { - return m.RpcVersions +func (x *StartClientHandshakeReq) GetRpcVersions() *RpcProtocolVersions { + if x != nil { + return x.RpcVersions } return nil } -func (m *StartClientHandshakeReq) GetMaxFrameSize() uint32 { - if m != nil { - return m.MaxFrameSize +func (x *StartClientHandshakeReq) GetMaxFrameSize() uint32 { + if x != nil { + return x.MaxFrameSize } return 0 } type ServerHandshakeParameters struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // The record protocols supported by the server, e.g., // "ALTSRP_GCM_AES128". RecordProtocols []string `protobuf:"bytes,1,rep,name=record_protocols,json=recordProtocols,proto3" json:"record_protocols,omitempty"` // (Optional) A list of local identities supported by the server, if // specified. Otherwise, the handshaker chooses a default local identity. - LocalIdentities []*Identity `protobuf:"bytes,2,rep,name=local_identities,json=localIdentities,proto3" json:"local_identities,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + LocalIdentities []*Identity `protobuf:"bytes,2,rep,name=local_identities,json=localIdentities,proto3" json:"local_identities,omitempty"` } -func (m *ServerHandshakeParameters) Reset() { *m = ServerHandshakeParameters{} } -func (m *ServerHandshakeParameters) String() string { return proto.CompactTextString(m) } -func (*ServerHandshakeParameters) ProtoMessage() {} -func (*ServerHandshakeParameters) Descriptor() ([]byte, []int) { - return fileDescriptor_54c074f40c7c7e99, []int{3} +func (x *ServerHandshakeParameters) Reset() { + *x = ServerHandshakeParameters{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_gcp_handshaker_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *ServerHandshakeParameters) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ServerHandshakeParameters.Unmarshal(m, b) -} -func (m *ServerHandshakeParameters) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ServerHandshakeParameters.Marshal(b, m, deterministic) -} -func (m *ServerHandshakeParameters) XXX_Merge(src proto.Message) { - xxx_messageInfo_ServerHandshakeParameters.Merge(m, src) +func (x *ServerHandshakeParameters) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *ServerHandshakeParameters) XXX_Size() int { - return xxx_messageInfo_ServerHandshakeParameters.Size(m) -} -func (m *ServerHandshakeParameters) XXX_DiscardUnknown() { - xxx_messageInfo_ServerHandshakeParameters.DiscardUnknown(m) + +func (*ServerHandshakeParameters) ProtoMessage() {} + +func (x *ServerHandshakeParameters) ProtoReflect() protoreflect.Message { + mi := &file_grpc_gcp_handshaker_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_ServerHandshakeParameters proto.InternalMessageInfo +// Deprecated: Use ServerHandshakeParameters.ProtoReflect.Descriptor instead. +func (*ServerHandshakeParameters) Descriptor() ([]byte, []int) { + return file_grpc_gcp_handshaker_proto_rawDescGZIP(), []int{3} +} -func (m *ServerHandshakeParameters) GetRecordProtocols() []string { - if m != nil { - return m.RecordProtocols +func (x *ServerHandshakeParameters) GetRecordProtocols() []string { + if x != nil { + return x.RecordProtocols } return nil } -func (m *ServerHandshakeParameters) GetLocalIdentities() []*Identity { - if m != nil { - return m.LocalIdentities +func (x *ServerHandshakeParameters) GetLocalIdentities() []*Identity { + if x != nil { + return x.LocalIdentities } return nil } type StartServerHandshakeReq struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // The application protocols supported by the server, e.g., "h2" (for http2), // "grpc". ApplicationProtocols []string `protobuf:"bytes,1,rep,name=application_protocols,json=applicationProtocols,proto3" json:"application_protocols,omitempty"` @@ -434,186 +520,185 @@ type StartServerHandshakeReq struct { // (Optional) RPC protocol versions supported by the server. RpcVersions *RpcProtocolVersions `protobuf:"bytes,6,opt,name=rpc_versions,json=rpcVersions,proto3" json:"rpc_versions,omitempty"` // (Optional) Maximum frame size supported by the server. - MaxFrameSize uint32 `protobuf:"varint,7,opt,name=max_frame_size,json=maxFrameSize,proto3" json:"max_frame_size,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + MaxFrameSize uint32 `protobuf:"varint,7,opt,name=max_frame_size,json=maxFrameSize,proto3" json:"max_frame_size,omitempty"` } -func (m *StartServerHandshakeReq) Reset() { *m = StartServerHandshakeReq{} } -func (m *StartServerHandshakeReq) String() string { return proto.CompactTextString(m) } -func (*StartServerHandshakeReq) ProtoMessage() {} -func (*StartServerHandshakeReq) Descriptor() ([]byte, []int) { - return fileDescriptor_54c074f40c7c7e99, []int{4} +func (x *StartServerHandshakeReq) Reset() { + *x = StartServerHandshakeReq{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_gcp_handshaker_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *StartServerHandshakeReq) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_StartServerHandshakeReq.Unmarshal(m, b) -} -func (m *StartServerHandshakeReq) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_StartServerHandshakeReq.Marshal(b, m, deterministic) -} -func (m *StartServerHandshakeReq) XXX_Merge(src proto.Message) { - xxx_messageInfo_StartServerHandshakeReq.Merge(m, src) -} -func (m *StartServerHandshakeReq) XXX_Size() int { - return xxx_messageInfo_StartServerHandshakeReq.Size(m) +func (x *StartServerHandshakeReq) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *StartServerHandshakeReq) XXX_DiscardUnknown() { - xxx_messageInfo_StartServerHandshakeReq.DiscardUnknown(m) + +func (*StartServerHandshakeReq) ProtoMessage() {} + +func (x *StartServerHandshakeReq) ProtoReflect() protoreflect.Message { + mi := &file_grpc_gcp_handshaker_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_StartServerHandshakeReq proto.InternalMessageInfo +// Deprecated: Use StartServerHandshakeReq.ProtoReflect.Descriptor instead. +func (*StartServerHandshakeReq) Descriptor() ([]byte, []int) { + return file_grpc_gcp_handshaker_proto_rawDescGZIP(), []int{4} +} -func (m *StartServerHandshakeReq) GetApplicationProtocols() []string { - if m != nil { - return m.ApplicationProtocols +func (x *StartServerHandshakeReq) GetApplicationProtocols() []string { + if x != nil { + return x.ApplicationProtocols } return nil } -func (m *StartServerHandshakeReq) GetHandshakeParameters() map[int32]*ServerHandshakeParameters { - if m != nil { - return m.HandshakeParameters +func (x *StartServerHandshakeReq) GetHandshakeParameters() map[int32]*ServerHandshakeParameters { + if x != nil { + return x.HandshakeParameters } return nil } -func (m *StartServerHandshakeReq) GetInBytes() []byte { - if m != nil { - return m.InBytes +func (x *StartServerHandshakeReq) GetInBytes() []byte { + if x != nil { + return x.InBytes } return nil } -func (m *StartServerHandshakeReq) GetLocalEndpoint() *Endpoint { - if m != nil { - return m.LocalEndpoint +func (x *StartServerHandshakeReq) GetLocalEndpoint() *Endpoint { + if x != nil { + return x.LocalEndpoint } return nil } -func (m *StartServerHandshakeReq) GetRemoteEndpoint() *Endpoint { - if m != nil { - return m.RemoteEndpoint +func (x *StartServerHandshakeReq) GetRemoteEndpoint() *Endpoint { + if x != nil { + return x.RemoteEndpoint } return nil } -func (m *StartServerHandshakeReq) GetRpcVersions() *RpcProtocolVersions { - if m != nil { - return m.RpcVersions +func (x *StartServerHandshakeReq) GetRpcVersions() *RpcProtocolVersions { + if x != nil { + return x.RpcVersions } return nil } -func (m *StartServerHandshakeReq) GetMaxFrameSize() uint32 { - if m != nil { - return m.MaxFrameSize +func (x *StartServerHandshakeReq) GetMaxFrameSize() uint32 { + if x != nil { + return x.MaxFrameSize } return 0 } type NextHandshakeMessageReq struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // Bytes in out_frames returned from the peer's HandshakerResp. It is possible // that the peer's out_frames are split into multiple NextHandshakerMessageReq // messages. - InBytes []byte `protobuf:"bytes,1,opt,name=in_bytes,json=inBytes,proto3" json:"in_bytes,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + InBytes []byte `protobuf:"bytes,1,opt,name=in_bytes,json=inBytes,proto3" json:"in_bytes,omitempty"` } -func (m *NextHandshakeMessageReq) Reset() { *m = NextHandshakeMessageReq{} } -func (m *NextHandshakeMessageReq) String() string { return proto.CompactTextString(m) } -func (*NextHandshakeMessageReq) ProtoMessage() {} -func (*NextHandshakeMessageReq) Descriptor() ([]byte, []int) { - return fileDescriptor_54c074f40c7c7e99, []int{5} +func (x *NextHandshakeMessageReq) Reset() { + *x = NextHandshakeMessageReq{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_gcp_handshaker_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *NextHandshakeMessageReq) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_NextHandshakeMessageReq.Unmarshal(m, b) -} -func (m *NextHandshakeMessageReq) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_NextHandshakeMessageReq.Marshal(b, m, deterministic) -} -func (m *NextHandshakeMessageReq) XXX_Merge(src proto.Message) { - xxx_messageInfo_NextHandshakeMessageReq.Merge(m, src) +func (x *NextHandshakeMessageReq) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *NextHandshakeMessageReq) XXX_Size() int { - return xxx_messageInfo_NextHandshakeMessageReq.Size(m) -} -func (m *NextHandshakeMessageReq) XXX_DiscardUnknown() { - xxx_messageInfo_NextHandshakeMessageReq.DiscardUnknown(m) + +func (*NextHandshakeMessageReq) ProtoMessage() {} + +func (x *NextHandshakeMessageReq) ProtoReflect() protoreflect.Message { + mi := &file_grpc_gcp_handshaker_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_NextHandshakeMessageReq proto.InternalMessageInfo +// Deprecated: Use NextHandshakeMessageReq.ProtoReflect.Descriptor instead. +func (*NextHandshakeMessageReq) Descriptor() ([]byte, []int) { + return file_grpc_gcp_handshaker_proto_rawDescGZIP(), []int{5} +} -func (m *NextHandshakeMessageReq) GetInBytes() []byte { - if m != nil { - return m.InBytes +func (x *NextHandshakeMessageReq) GetInBytes() []byte { + if x != nil { + return x.InBytes } return nil } type HandshakerReq struct { - // Types that are valid to be assigned to ReqOneof: + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Types that are assignable to ReqOneof: + // // *HandshakerReq_ClientStart // *HandshakerReq_ServerStart // *HandshakerReq_Next - ReqOneof isHandshakerReq_ReqOneof `protobuf_oneof:"req_oneof"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + ReqOneof isHandshakerReq_ReqOneof `protobuf_oneof:"req_oneof"` } -func (m *HandshakerReq) Reset() { *m = HandshakerReq{} } -func (m *HandshakerReq) String() string { return proto.CompactTextString(m) } -func (*HandshakerReq) ProtoMessage() {} -func (*HandshakerReq) Descriptor() ([]byte, []int) { - return fileDescriptor_54c074f40c7c7e99, []int{6} -} - -func (m *HandshakerReq) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_HandshakerReq.Unmarshal(m, b) -} -func (m *HandshakerReq) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_HandshakerReq.Marshal(b, m, deterministic) -} -func (m *HandshakerReq) XXX_Merge(src proto.Message) { - xxx_messageInfo_HandshakerReq.Merge(m, src) -} -func (m *HandshakerReq) XXX_Size() int { - return xxx_messageInfo_HandshakerReq.Size(m) -} -func (m *HandshakerReq) XXX_DiscardUnknown() { - xxx_messageInfo_HandshakerReq.DiscardUnknown(m) +func (x *HandshakerReq) Reset() { + *x = HandshakerReq{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_gcp_handshaker_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -var xxx_messageInfo_HandshakerReq proto.InternalMessageInfo - -type isHandshakerReq_ReqOneof interface { - isHandshakerReq_ReqOneof() +func (x *HandshakerReq) String() string { + return protoimpl.X.MessageStringOf(x) } -type HandshakerReq_ClientStart struct { - ClientStart *StartClientHandshakeReq `protobuf:"bytes,1,opt,name=client_start,json=clientStart,proto3,oneof"` -} +func (*HandshakerReq) ProtoMessage() {} -type HandshakerReq_ServerStart struct { - ServerStart *StartServerHandshakeReq `protobuf:"bytes,2,opt,name=server_start,json=serverStart,proto3,oneof"` +func (x *HandshakerReq) ProtoReflect() protoreflect.Message { + mi := &file_grpc_gcp_handshaker_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -type HandshakerReq_Next struct { - Next *NextHandshakeMessageReq `protobuf:"bytes,3,opt,name=next,proto3,oneof"` +// Deprecated: Use HandshakerReq.ProtoReflect.Descriptor instead. +func (*HandshakerReq) Descriptor() ([]byte, []int) { + return file_grpc_gcp_handshaker_proto_rawDescGZIP(), []int{6} } -func (*HandshakerReq_ClientStart) isHandshakerReq_ReqOneof() {} - -func (*HandshakerReq_ServerStart) isHandshakerReq_ReqOneof() {} - -func (*HandshakerReq_Next) isHandshakerReq_ReqOneof() {} - func (m *HandshakerReq) GetReqOneof() isHandshakerReq_ReqOneof { if m != nil { return m.ReqOneof @@ -621,37 +706,57 @@ func (m *HandshakerReq) GetReqOneof() isHandshakerReq_ReqOneof { return nil } -func (m *HandshakerReq) GetClientStart() *StartClientHandshakeReq { - if x, ok := m.GetReqOneof().(*HandshakerReq_ClientStart); ok { +func (x *HandshakerReq) GetClientStart() *StartClientHandshakeReq { + if x, ok := x.GetReqOneof().(*HandshakerReq_ClientStart); ok { return x.ClientStart } return nil } -func (m *HandshakerReq) GetServerStart() *StartServerHandshakeReq { - if x, ok := m.GetReqOneof().(*HandshakerReq_ServerStart); ok { +func (x *HandshakerReq) GetServerStart() *StartServerHandshakeReq { + if x, ok := x.GetReqOneof().(*HandshakerReq_ServerStart); ok { return x.ServerStart } return nil } -func (m *HandshakerReq) GetNext() *NextHandshakeMessageReq { - if x, ok := m.GetReqOneof().(*HandshakerReq_Next); ok { +func (x *HandshakerReq) GetNext() *NextHandshakeMessageReq { + if x, ok := x.GetReqOneof().(*HandshakerReq_Next); ok { return x.Next } return nil } -// XXX_OneofWrappers is for the internal use of the proto package. -func (*HandshakerReq) XXX_OneofWrappers() []interface{} { - return []interface{}{ - (*HandshakerReq_ClientStart)(nil), - (*HandshakerReq_ServerStart)(nil), - (*HandshakerReq_Next)(nil), - } +type isHandshakerReq_ReqOneof interface { + isHandshakerReq_ReqOneof() } +type HandshakerReq_ClientStart struct { + // The start client handshake request message. + ClientStart *StartClientHandshakeReq `protobuf:"bytes,1,opt,name=client_start,json=clientStart,proto3,oneof"` +} + +type HandshakerReq_ServerStart struct { + // The start server handshake request message. + ServerStart *StartServerHandshakeReq `protobuf:"bytes,2,opt,name=server_start,json=serverStart,proto3,oneof"` +} + +type HandshakerReq_Next struct { + // The next handshake request message. + Next *NextHandshakeMessageReq `protobuf:"bytes,3,opt,name=next,proto3,oneof"` +} + +func (*HandshakerReq_ClientStart) isHandshakerReq_ReqOneof() {} + +func (*HandshakerReq_ServerStart) isHandshakerReq_ReqOneof() {} + +func (*HandshakerReq_Next) isHandshakerReq_ReqOneof() {} + type HandshakerResult struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // The application protocol negotiated for this connection. ApplicationProtocol string `protobuf:"bytes,1,opt,name=application_protocol,json=applicationProtocol,proto3" json:"application_protocol,omitempty"` // The record protocol negotiated for this connection. @@ -671,143 +776,159 @@ type HandshakerResult struct { // The RPC protocol versions supported by the peer. PeerRpcVersions *RpcProtocolVersions `protobuf:"bytes,7,opt,name=peer_rpc_versions,json=peerRpcVersions,proto3" json:"peer_rpc_versions,omitempty"` // The maximum frame size of the peer. - MaxFrameSize uint32 `protobuf:"varint,8,opt,name=max_frame_size,json=maxFrameSize,proto3" json:"max_frame_size,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + MaxFrameSize uint32 `protobuf:"varint,8,opt,name=max_frame_size,json=maxFrameSize,proto3" json:"max_frame_size,omitempty"` } -func (m *HandshakerResult) Reset() { *m = HandshakerResult{} } -func (m *HandshakerResult) String() string { return proto.CompactTextString(m) } -func (*HandshakerResult) ProtoMessage() {} -func (*HandshakerResult) Descriptor() ([]byte, []int) { - return fileDescriptor_54c074f40c7c7e99, []int{7} +func (x *HandshakerResult) Reset() { + *x = HandshakerResult{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_gcp_handshaker_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *HandshakerResult) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_HandshakerResult.Unmarshal(m, b) -} -func (m *HandshakerResult) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_HandshakerResult.Marshal(b, m, deterministic) -} -func (m *HandshakerResult) XXX_Merge(src proto.Message) { - xxx_messageInfo_HandshakerResult.Merge(m, src) +func (x *HandshakerResult) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *HandshakerResult) XXX_Size() int { - return xxx_messageInfo_HandshakerResult.Size(m) -} -func (m *HandshakerResult) XXX_DiscardUnknown() { - xxx_messageInfo_HandshakerResult.DiscardUnknown(m) + +func (*HandshakerResult) ProtoMessage() {} + +func (x *HandshakerResult) ProtoReflect() protoreflect.Message { + mi := &file_grpc_gcp_handshaker_proto_msgTypes[7] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_HandshakerResult proto.InternalMessageInfo +// Deprecated: Use HandshakerResult.ProtoReflect.Descriptor instead. +func (*HandshakerResult) Descriptor() ([]byte, []int) { + return file_grpc_gcp_handshaker_proto_rawDescGZIP(), []int{7} +} -func (m *HandshakerResult) GetApplicationProtocol() string { - if m != nil { - return m.ApplicationProtocol +func (x *HandshakerResult) GetApplicationProtocol() string { + if x != nil { + return x.ApplicationProtocol } return "" } -func (m *HandshakerResult) GetRecordProtocol() string { - if m != nil { - return m.RecordProtocol +func (x *HandshakerResult) GetRecordProtocol() string { + if x != nil { + return x.RecordProtocol } return "" } -func (m *HandshakerResult) GetKeyData() []byte { - if m != nil { - return m.KeyData +func (x *HandshakerResult) GetKeyData() []byte { + if x != nil { + return x.KeyData } return nil } -func (m *HandshakerResult) GetPeerIdentity() *Identity { - if m != nil { - return m.PeerIdentity +func (x *HandshakerResult) GetPeerIdentity() *Identity { + if x != nil { + return x.PeerIdentity } return nil } -func (m *HandshakerResult) GetLocalIdentity() *Identity { - if m != nil { - return m.LocalIdentity +func (x *HandshakerResult) GetLocalIdentity() *Identity { + if x != nil { + return x.LocalIdentity } return nil } -func (m *HandshakerResult) GetKeepChannelOpen() bool { - if m != nil { - return m.KeepChannelOpen +func (x *HandshakerResult) GetKeepChannelOpen() bool { + if x != nil { + return x.KeepChannelOpen } return false } -func (m *HandshakerResult) GetPeerRpcVersions() *RpcProtocolVersions { - if m != nil { - return m.PeerRpcVersions +func (x *HandshakerResult) GetPeerRpcVersions() *RpcProtocolVersions { + if x != nil { + return x.PeerRpcVersions } return nil } -func (m *HandshakerResult) GetMaxFrameSize() uint32 { - if m != nil { - return m.MaxFrameSize +func (x *HandshakerResult) GetMaxFrameSize() uint32 { + if x != nil { + return x.MaxFrameSize } return 0 } type HandshakerStatus struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // The status code. This could be the gRPC status code. Code uint32 `protobuf:"varint,1,opt,name=code,proto3" json:"code,omitempty"` // The status details. - Details string `protobuf:"bytes,2,opt,name=details,proto3" json:"details,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Details string `protobuf:"bytes,2,opt,name=details,proto3" json:"details,omitempty"` } -func (m *HandshakerStatus) Reset() { *m = HandshakerStatus{} } -func (m *HandshakerStatus) String() string { return proto.CompactTextString(m) } -func (*HandshakerStatus) ProtoMessage() {} -func (*HandshakerStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_54c074f40c7c7e99, []int{8} +func (x *HandshakerStatus) Reset() { + *x = HandshakerStatus{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_gcp_handshaker_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *HandshakerStatus) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_HandshakerStatus.Unmarshal(m, b) +func (x *HandshakerStatus) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *HandshakerStatus) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_HandshakerStatus.Marshal(b, m, deterministic) -} -func (m *HandshakerStatus) XXX_Merge(src proto.Message) { - xxx_messageInfo_HandshakerStatus.Merge(m, src) -} -func (m *HandshakerStatus) XXX_Size() int { - return xxx_messageInfo_HandshakerStatus.Size(m) -} -func (m *HandshakerStatus) XXX_DiscardUnknown() { - xxx_messageInfo_HandshakerStatus.DiscardUnknown(m) + +func (*HandshakerStatus) ProtoMessage() {} + +func (x *HandshakerStatus) ProtoReflect() protoreflect.Message { + mi := &file_grpc_gcp_handshaker_proto_msgTypes[8] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_HandshakerStatus proto.InternalMessageInfo +// Deprecated: Use HandshakerStatus.ProtoReflect.Descriptor instead. +func (*HandshakerStatus) Descriptor() ([]byte, []int) { + return file_grpc_gcp_handshaker_proto_rawDescGZIP(), []int{8} +} -func (m *HandshakerStatus) GetCode() uint32 { - if m != nil { - return m.Code +func (x *HandshakerStatus) GetCode() uint32 { + if x != nil { + return x.Code } return 0 } -func (m *HandshakerStatus) GetDetails() string { - if m != nil { - return m.Details +func (x *HandshakerStatus) GetDetails() string { + if x != nil { + return x.Details } return "" } type HandshakerResp struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // Frames to be given to the peer for the NextHandshakeMessageReq. May be // empty if no out_frames have to be sent to the peer or if in_bytes in the // HandshakerReq are incomplete. All the non-empty out frames must be sent to @@ -822,284 +943,481 @@ type HandshakerResp struct { // to frames that needs to be forwarded to the peer. Result *HandshakerResult `protobuf:"bytes,3,opt,name=result,proto3" json:"result,omitempty"` // Status of the handshaker. - Status *HandshakerStatus `protobuf:"bytes,4,opt,name=status,proto3" json:"status,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Status *HandshakerStatus `protobuf:"bytes,4,opt,name=status,proto3" json:"status,omitempty"` } -func (m *HandshakerResp) Reset() { *m = HandshakerResp{} } -func (m *HandshakerResp) String() string { return proto.CompactTextString(m) } -func (*HandshakerResp) ProtoMessage() {} -func (*HandshakerResp) Descriptor() ([]byte, []int) { - return fileDescriptor_54c074f40c7c7e99, []int{9} +func (x *HandshakerResp) Reset() { + *x = HandshakerResp{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_gcp_handshaker_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *HandshakerResp) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_HandshakerResp.Unmarshal(m, b) +func (x *HandshakerResp) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *HandshakerResp) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_HandshakerResp.Marshal(b, m, deterministic) -} -func (m *HandshakerResp) XXX_Merge(src proto.Message) { - xxx_messageInfo_HandshakerResp.Merge(m, src) -} -func (m *HandshakerResp) XXX_Size() int { - return xxx_messageInfo_HandshakerResp.Size(m) -} -func (m *HandshakerResp) XXX_DiscardUnknown() { - xxx_messageInfo_HandshakerResp.DiscardUnknown(m) + +func (*HandshakerResp) ProtoMessage() {} + +func (x *HandshakerResp) ProtoReflect() protoreflect.Message { + mi := &file_grpc_gcp_handshaker_proto_msgTypes[9] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_HandshakerResp proto.InternalMessageInfo +// Deprecated: Use HandshakerResp.ProtoReflect.Descriptor instead. +func (*HandshakerResp) Descriptor() ([]byte, []int) { + return file_grpc_gcp_handshaker_proto_rawDescGZIP(), []int{9} +} -func (m *HandshakerResp) GetOutFrames() []byte { - if m != nil { - return m.OutFrames +func (x *HandshakerResp) GetOutFrames() []byte { + if x != nil { + return x.OutFrames } return nil } -func (m *HandshakerResp) GetBytesConsumed() uint32 { - if m != nil { - return m.BytesConsumed +func (x *HandshakerResp) GetBytesConsumed() uint32 { + if x != nil { + return x.BytesConsumed } return 0 } -func (m *HandshakerResp) GetResult() *HandshakerResult { - if m != nil { - return m.Result +func (x *HandshakerResp) GetResult() *HandshakerResult { + if x != nil { + return x.Result } return nil } -func (m *HandshakerResp) GetStatus() *HandshakerStatus { - if m != nil { - return m.Status +func (x *HandshakerResp) GetStatus() *HandshakerStatus { + if x != nil { + return x.Status } return nil } -func init() { - proto.RegisterEnum("grpc.gcp.HandshakeProtocol", HandshakeProtocol_name, HandshakeProtocol_value) - proto.RegisterEnum("grpc.gcp.NetworkProtocol", NetworkProtocol_name, NetworkProtocol_value) - proto.RegisterType((*Endpoint)(nil), "grpc.gcp.Endpoint") - proto.RegisterType((*Identity)(nil), "grpc.gcp.Identity") - proto.RegisterMapType((map[string]string)(nil), "grpc.gcp.Identity.AttributesEntry") - proto.RegisterType((*StartClientHandshakeReq)(nil), "grpc.gcp.StartClientHandshakeReq") - proto.RegisterType((*ServerHandshakeParameters)(nil), "grpc.gcp.ServerHandshakeParameters") - proto.RegisterType((*StartServerHandshakeReq)(nil), "grpc.gcp.StartServerHandshakeReq") - proto.RegisterMapType((map[int32]*ServerHandshakeParameters)(nil), "grpc.gcp.StartServerHandshakeReq.HandshakeParametersEntry") - proto.RegisterType((*NextHandshakeMessageReq)(nil), "grpc.gcp.NextHandshakeMessageReq") - proto.RegisterType((*HandshakerReq)(nil), "grpc.gcp.HandshakerReq") - proto.RegisterType((*HandshakerResult)(nil), "grpc.gcp.HandshakerResult") - proto.RegisterType((*HandshakerStatus)(nil), "grpc.gcp.HandshakerStatus") - proto.RegisterType((*HandshakerResp)(nil), "grpc.gcp.HandshakerResp") -} - -func init() { proto.RegisterFile("grpc/gcp/handshaker.proto", fileDescriptor_54c074f40c7c7e99) } - -var fileDescriptor_54c074f40c7c7e99 = []byte{ - // 1203 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xdd, 0x6e, 0x1b, 0x45, - 0x14, 0xce, 0xda, 0x4e, 0xe2, 0x1c, 0xc7, 0x3f, 0x99, 0xa6, 0xea, 0x26, 0x6d, 0xc1, 0x18, 0x10, - 0x6e, 0x2f, 0x6c, 0x70, 0x41, 0xa5, 0x45, 0x55, 0x6b, 0x3b, 0x8e, 0x1c, 0x5a, 0x1c, 0x6b, 0x9d, - 0x82, 0x44, 0x2f, 0x56, 0xd3, 0xf5, 0xd4, 0x59, 0x79, 0x3d, 0xb3, 0x9d, 0x19, 0x87, 0xb8, 0xf7, - 0xbc, 0x04, 0xf7, 0xbc, 0x06, 0x2f, 0xc1, 0x33, 0x20, 0xf1, 0x18, 0x68, 0x67, 0x7f, 0x6d, 0xaf, - 0xab, 0x22, 0xb8, 0xdb, 0x39, 0xf3, 0x7d, 0x67, 0xce, 0x9c, 0xf3, 0x9d, 0xb3, 0x03, 0x47, 0x13, - 0xee, 0x5a, 0xcd, 0x89, 0xe5, 0x36, 0x2f, 0x31, 0x1d, 0x8b, 0x4b, 0x3c, 0x25, 0xbc, 0xe1, 0x72, - 0x26, 0x19, 0xca, 0x7b, 0x5b, 0x8d, 0x89, 0xe5, 0x1e, 0xd7, 0x23, 0x90, 0xe4, 0x98, 0x0a, 0x97, - 0x71, 0x69, 0x0a, 0x62, 0xcd, 0xb9, 0x2d, 0x17, 0xa6, 0xc5, 0x66, 0x33, 0x46, 0x7d, 0x4e, 0x4d, - 0x42, 0xbe, 0x47, 0xc7, 0x2e, 0xb3, 0xa9, 0x44, 0x77, 0x01, 0x6c, 0xd7, 0xc4, 0xe3, 0x31, 0x27, - 0x42, 0xe8, 0x5a, 0x55, 0xab, 0xef, 0x19, 0x7b, 0xb6, 0xdb, 0xf6, 0x0d, 0x08, 0x41, 0xce, 0x73, - 0xa4, 0x67, 0xaa, 0x5a, 0x7d, 0xdb, 0x50, 0xdf, 0xe8, 0x1b, 0xc8, 0x2b, 0x3f, 0x16, 0x73, 0xf4, - 0x6c, 0x55, 0xab, 0x97, 0x5a, 0x47, 0x8d, 0x30, 0x8a, 0xc6, 0x80, 0xc8, 0x5f, 0x18, 0x9f, 0x0e, - 0x03, 0x80, 0x11, 0x41, 0x6b, 0x7f, 0x6b, 0x90, 0x3f, 0x1b, 0x13, 0x2a, 0x6d, 0xb9, 0x40, 0xf7, - 0xa0, 0x2c, 0x08, 0xbf, 0xb2, 0x2d, 0x62, 0x62, 0xcb, 0x62, 0x73, 0x2a, 0xfd, 0xb3, 0xfb, 0x5b, - 0x46, 0x29, 0xd8, 0x68, 0xfb, 0x76, 0x74, 0x07, 0xf2, 0x97, 0x4c, 0x48, 0x8a, 0x67, 0x44, 0x85, - 0xe1, 0x61, 0x22, 0x0b, 0xea, 0x00, 0x60, 0x29, 0xb9, 0xfd, 0x7a, 0x2e, 0x89, 0xd0, 0xb3, 0xd5, - 0x6c, 0xbd, 0xd0, 0xaa, 0xc5, 0xe1, 0x84, 0x07, 0x36, 0xda, 0x11, 0xa8, 0x47, 0x25, 0x5f, 0x18, - 0x09, 0xd6, 0xf1, 0x13, 0x28, 0xaf, 0x6c, 0xa3, 0x0a, 0x64, 0xa7, 0x64, 0x11, 0xe4, 0xc3, 0xfb, - 0x44, 0x87, 0xb0, 0x7d, 0x85, 0x9d, 0x79, 0x10, 0x83, 0xe1, 0x2f, 0x1e, 0x67, 0xbe, 0xd5, 0x3a, - 0x15, 0x28, 0xd9, 0xc1, 0x31, 0x26, 0xa3, 0x84, 0xbd, 0xa9, 0xfd, 0x99, 0x83, 0x5b, 0x23, 0x89, - 0xb9, 0xec, 0x3a, 0x36, 0xa1, 0xb2, 0x1f, 0x16, 0xcd, 0x20, 0x6f, 0xd1, 0x2b, 0xb8, 0x1d, 0x15, - 0x31, 0xae, 0x4f, 0x94, 0x50, 0x4d, 0x25, 0xf4, 0x76, 0x7c, 0x83, 0x88, 0x1c, 0xa5, 0xf4, 0x28, - 0xe2, 0x8f, 0x02, 0x7a, 0xb8, 0x85, 0x1e, 0xc0, 0x4d, 0xec, 0xba, 0x8e, 0x6d, 0x61, 0x69, 0x33, - 0x1a, 0x79, 0x15, 0x7a, 0xa6, 0x9a, 0xad, 0xef, 0x19, 0x87, 0x89, 0xcd, 0x90, 0x23, 0xd0, 0x3d, - 0xa8, 0x70, 0x62, 0x31, 0x3e, 0x4e, 0xe0, 0xb3, 0x0a, 0x5f, 0xf6, 0xed, 0x31, 0xf4, 0x29, 0x1c, - 0x48, 0xcc, 0x27, 0x44, 0x9a, 0xc1, 0x8d, 0x6d, 0x22, 0xf4, 0x9c, 0x4a, 0x3a, 0x5a, 0x4f, 0xba, - 0x51, 0xf1, 0xc1, 0x67, 0x11, 0x16, 0x3d, 0x82, 0x92, 0xc3, 0x2c, 0xec, 0x84, 0xfc, 0x85, 0xbe, - 0x5d, 0xd5, 0x36, 0xb0, 0x8b, 0x0a, 0x19, 0x49, 0x26, 0xa2, 0x92, 0x40, 0xbb, 0xfa, 0xce, 0x2a, - 0x35, 0x54, 0x75, 0x40, 0x8d, 0x44, 0xfe, 0x1d, 0x94, 0x39, 0x99, 0x31, 0x49, 0x62, 0xee, 0xee, - 0x46, 0x6e, 0xc9, 0x87, 0x46, 0xe4, 0x8f, 0xa1, 0x10, 0xdc, 0x59, 0x49, 0x30, 0xaf, 0xca, 0x0f, - 0xbe, 0x69, 0xe0, 0x49, 0xf0, 0x19, 0xec, 0x73, 0xd7, 0x32, 0xaf, 0x08, 0x17, 0x36, 0xa3, 0x42, - 0xdf, 0x53, 0xae, 0xef, 0xc6, 0xae, 0x0d, 0xd7, 0x0a, 0x53, 0xf8, 0x63, 0x00, 0x32, 0x0a, 0xdc, - 0xb5, 0xc2, 0x05, 0xfa, 0x0c, 0x4a, 0x33, 0x7c, 0x6d, 0xbe, 0xe1, 0x78, 0x46, 0x4c, 0x61, 0xbf, - 0x23, 0x3a, 0x54, 0xb5, 0x7a, 0xd1, 0xd8, 0x9f, 0xe1, 0xeb, 0x53, 0xcf, 0x38, 0xb2, 0xdf, 0x91, - 0xda, 0xaf, 0x1a, 0x1c, 0x8d, 0x08, 0xbf, 0x22, 0x3c, 0xd6, 0x04, 0xf6, 0x76, 0x25, 0xe1, 0xe9, - 0x55, 0xd4, 0xd2, 0xab, 0xf8, 0x04, 0x2a, 0x4b, 0x45, 0xf0, 0x8a, 0x98, 0xd9, 0x58, 0xc4, 0x72, - 0xb2, 0x0c, 0x36, 0x11, 0xb5, 0xdf, 0x43, 0x75, 0xaf, 0x04, 0xe3, 0xa9, 0x7b, 0xa3, 0x00, 0xb5, - 0xf7, 0x08, 0x70, 0x06, 0x87, 0x71, 0x4b, 0xb8, 0xd1, 0x95, 0x82, 0x98, 0x1e, 0xc7, 0x31, 0x6d, - 0x38, 0xb5, 0x91, 0x92, 0x0f, 0xbf, 0xcb, 0x6f, 0x5c, 0xa6, 0x64, 0xea, 0x08, 0xf2, 0x36, 0x35, - 0x5f, 0x2f, 0xfc, 0x81, 0xa1, 0xd5, 0xf7, 0x8d, 0x5d, 0x9b, 0x76, 0xbc, 0x65, 0x8a, 0xc6, 0x72, - 0xff, 0x41, 0x63, 0xdb, 0x1f, 0xac, 0xb1, 0x55, 0x09, 0xed, 0xfc, 0x0f, 0x12, 0xda, 0x5d, 0x97, - 0xd0, 0xf1, 0x14, 0xf4, 0x4d, 0xb9, 0x4a, 0x8e, 0xbc, 0x6d, 0x7f, 0xe4, 0x3d, 0x4a, 0x8e, 0xbc, - 0x42, 0xeb, 0xd3, 0x44, 0x21, 0x36, 0xc9, 0x30, 0x31, 0x17, 0x6b, 0x5f, 0xc3, 0xad, 0x01, 0xb9, - 0x8e, 0xa7, 0xdf, 0x0f, 0x44, 0x08, 0x3c, 0x51, 0x32, 0x49, 0x96, 0x40, 0x5b, 0x2a, 0x41, 0xed, - 0x2f, 0x0d, 0x8a, 0x11, 0x85, 0x7b, 0xe0, 0x53, 0xd8, 0xb7, 0xd4, 0x1c, 0x35, 0x85, 0x57, 0x7f, - 0x45, 0x28, 0xb4, 0x3e, 0x59, 0x91, 0xc5, 0xfa, 0xa8, 0xed, 0x6f, 0x19, 0x05, 0x9f, 0xa8, 0x00, - 0x9e, 0x1f, 0xa1, 0xe2, 0x0e, 0xfc, 0x64, 0x52, 0xfd, 0xac, 0xcb, 0xcb, 0xf3, 0xe3, 0x13, 0x7d, - 0x3f, 0x0f, 0x21, 0x47, 0xc9, 0xb5, 0x54, 0xda, 0x59, 0xe2, 0x6f, 0xb8, 0x6d, 0x7f, 0xcb, 0x50, - 0x84, 0x4e, 0x01, 0xf6, 0x38, 0x79, 0x1b, 0xfc, 0x23, 0x7e, 0xcb, 0x42, 0x25, 0x79, 0x4f, 0x31, - 0x77, 0x24, 0xfa, 0x0a, 0x0e, 0xd3, 0xda, 0x27, 0xf8, 0x0f, 0xdd, 0x48, 0xe9, 0x1e, 0xf4, 0x05, - 0x94, 0x57, 0xfa, 0x3e, 0xf8, 0x43, 0x95, 0x96, 0xdb, 0xde, 0xcb, 0xf9, 0x94, 0x2c, 0xcc, 0x31, - 0x96, 0x38, 0x94, 0xfd, 0x94, 0x2c, 0x4e, 0xb0, 0xc4, 0xe8, 0x21, 0x14, 0x5d, 0x42, 0x78, 0x3c, - 0x94, 0x73, 0x1b, 0x87, 0xf2, 0xbe, 0x07, 0x5c, 0x9f, 0xc9, 0xff, 0x7e, 0x9c, 0xdf, 0x87, 0x83, - 0x29, 0x21, 0xae, 0x69, 0x5d, 0x62, 0x4a, 0x89, 0x63, 0x32, 0x97, 0x50, 0xa5, 0xfb, 0xbc, 0x51, - 0xf6, 0x36, 0xba, 0xbe, 0xfd, 0xdc, 0x25, 0x14, 0x9d, 0xc1, 0x81, 0x8a, 0x6f, 0xa9, 0x47, 0x76, - 0x3f, 0xa4, 0x47, 0xca, 0x1e, 0xcf, 0x78, 0x6f, 0x9f, 0xe4, 0x53, 0x46, 0xed, 0xb3, 0x64, 0x6d, - 0x46, 0x12, 0xcb, 0xb9, 0x7a, 0x0a, 0x59, 0x6c, 0x4c, 0x54, 0x2d, 0x8a, 0x86, 0xfa, 0x46, 0x3a, - 0xec, 0x8e, 0x89, 0xc4, 0xb6, 0xfa, 0xc3, 0x7a, 0x49, 0x0f, 0x97, 0xb5, 0x3f, 0x34, 0x28, 0x2d, - 0x95, 0xd7, 0xf5, 0x9e, 0x5a, 0x6c, 0x2e, 0xfd, 0xa3, 0x43, 0xd9, 0xef, 0xb1, 0xb9, 0x54, 0xc7, - 0x0a, 0xf4, 0x39, 0x94, 0x54, 0x43, 0x98, 0x16, 0xa3, 0x62, 0x3e, 0x23, 0x63, 0xe5, 0xb2, 0x68, - 0x14, 0x95, 0xb5, 0x1b, 0x18, 0x51, 0x0b, 0x76, 0xb8, 0x12, 0x4b, 0xa0, 0xbf, 0xe3, 0x94, 0xa7, - 0x42, 0x20, 0x27, 0x23, 0x40, 0x7a, 0x1c, 0xa1, 0x2e, 0x11, 0x14, 0x36, 0x95, 0xe3, 0x5f, 0xd3, - 0x08, 0x90, 0xf7, 0xbf, 0x87, 0x83, 0xb5, 0xa7, 0x07, 0xaa, 0xc1, 0x47, 0xfd, 0xf6, 0xe0, 0x64, - 0xd4, 0x6f, 0x3f, 0xef, 0x99, 0x43, 0xe3, 0xfc, 0xe2, 0xbc, 0x7b, 0xfe, 0xc2, 0x7c, 0x39, 0x18, - 0x0d, 0x7b, 0xdd, 0xb3, 0xd3, 0xb3, 0xde, 0x49, 0x65, 0x0b, 0xed, 0x42, 0xf6, 0xe2, 0xc5, 0xa8, - 0xa2, 0xa1, 0x3c, 0xe4, 0xda, 0x2f, 0x2e, 0x46, 0x95, 0xcc, 0xfd, 0x1e, 0x94, 0x57, 0xde, 0x85, - 0xa8, 0x0a, 0x77, 0x06, 0xbd, 0x8b, 0x9f, 0xce, 0x8d, 0xe7, 0xef, 0xf3, 0xd3, 0x1d, 0x56, 0x34, - 0xef, 0xe3, 0xe5, 0xc9, 0xb0, 0x92, 0x69, 0xbd, 0x4a, 0x84, 0xc4, 0x47, 0xfe, 0x2b, 0x11, 0x9d, - 0x42, 0xe1, 0x84, 0x45, 0x66, 0x74, 0x2b, 0x3d, 0x1d, 0x6f, 0x8f, 0xf5, 0x0d, 0x79, 0x72, 0x6b, - 0x5b, 0x75, 0xed, 0x4b, 0xad, 0x33, 0x85, 0x9b, 0x36, 0xf3, 0x31, 0xd8, 0x91, 0xa2, 0x61, 0x53, - 0x49, 0x38, 0xc5, 0x4e, 0xa7, 0x1c, 0xc3, 0x55, 0xf4, 0x43, 0xed, 0xe7, 0xa7, 0x13, 0xc6, 0x26, - 0x0e, 0x69, 0x4c, 0x98, 0x83, 0xe9, 0xa4, 0xc1, 0xf8, 0xa4, 0xa9, 0x1e, 0xdf, 0x16, 0x27, 0x4a, - 0xde, 0xd8, 0x11, 0x4d, 0xcf, 0x49, 0x33, 0x74, 0xd2, 0x54, 0xbd, 0xa9, 0x40, 0xe6, 0xc4, 0x72, - 0x5f, 0xef, 0xa8, 0xf5, 0x83, 0x7f, 0x02, 0x00, 0x00, 0xff, 0xff, 0xc1, 0xf9, 0x9d, 0xf2, 0xd9, - 0x0b, 0x00, 0x00, -} - -// Reference imports to suppress errors if they are not otherwise used. -var _ context.Context -var _ grpc.ClientConnInterface - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the grpc package it is being compiled against. -const _ = grpc.SupportPackageIsVersion6 - -// HandshakerServiceClient is the client API for HandshakerService service. -// -// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. -type HandshakerServiceClient interface { - // Handshaker service accepts a stream of handshaker request, returning a - // stream of handshaker response. Client is expected to send exactly one - // message with either client_start or server_start followed by one or more - // messages with next. Each time client sends a request, the handshaker - // service expects to respond. Client does not have to wait for service's - // response before sending next request. - DoHandshake(ctx context.Context, opts ...grpc.CallOption) (HandshakerService_DoHandshakeClient, error) -} - -type handshakerServiceClient struct { - cc grpc.ClientConnInterface -} - -func NewHandshakerServiceClient(cc grpc.ClientConnInterface) HandshakerServiceClient { - return &handshakerServiceClient{cc} -} +var File_grpc_gcp_handshaker_proto protoreflect.FileDescriptor + +var file_grpc_gcp_handshaker_proto_rawDesc = []byte{ + 0x0a, 0x19, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x67, 0x63, 0x70, 0x2f, 0x68, 0x61, 0x6e, 0x64, 0x73, + 0x68, 0x61, 0x6b, 0x65, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x08, 0x67, 0x72, 0x70, + 0x63, 0x2e, 0x67, 0x63, 0x70, 0x1a, 0x28, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x67, 0x63, 0x70, 0x2f, + 0x74, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x6f, 0x72, 0x74, 0x5f, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, + 0x74, 0x79, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, + 0x74, 0x0a, 0x08, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x69, + 0x70, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x09, 0x69, 0x70, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, + 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x35, + 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, + 0x32, 0x19, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x63, 0x70, 0x2e, 0x4e, 0x65, 0x74, 0x77, + 0x6f, 0x72, 0x6b, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x52, 0x08, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x22, 0xe8, 0x01, 0x0a, 0x08, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, + 0x74, 0x79, 0x12, 0x29, 0x0a, 0x0f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x61, 0x63, + 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0e, 0x73, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x41, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x1c, 0x0a, + 0x08, 0x68, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, + 0x00, 0x52, 0x08, 0x68, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x42, 0x0a, 0x0a, 0x61, + 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x22, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x63, 0x70, 0x2e, 0x49, 0x64, 0x65, 0x6e, 0x74, + 0x69, 0x74, 0x79, 0x2e, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x52, 0x0a, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x1a, + 0x3d, 0x0a, 0x0f, 0x41, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x42, 0x10, + 0x0a, 0x0e, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x5f, 0x6f, 0x6e, 0x65, 0x6f, 0x66, + 0x22, 0xd3, 0x04, 0x0a, 0x17, 0x53, 0x74, 0x61, 0x72, 0x74, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, + 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x52, 0x65, 0x71, 0x12, 0x5b, 0x0a, 0x1b, + 0x68, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x5f, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, + 0x74, 0x79, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0e, 0x32, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x63, 0x70, 0x2e, 0x48, 0x61, 0x6e, + 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x52, 0x19, + 0x68, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x53, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, + 0x79, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x33, 0x0a, 0x15, 0x61, 0x70, 0x70, + 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, + 0x6c, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x14, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x63, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x73, 0x12, 0x29, + 0x0a, 0x10, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, + 0x6c, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0f, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, + 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x73, 0x12, 0x3f, 0x0a, 0x11, 0x74, 0x61, 0x72, + 0x67, 0x65, 0x74, 0x5f, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x18, 0x04, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x63, 0x70, 0x2e, + 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x10, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, + 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x12, 0x39, 0x0a, 0x0e, 0x6c, 0x6f, + 0x63, 0x61, 0x6c, 0x5f, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x18, 0x05, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x63, 0x70, 0x2e, 0x49, 0x64, + 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x0d, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x49, 0x64, 0x65, + 0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x39, 0x0a, 0x0e, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x65, + 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, + 0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x63, 0x70, 0x2e, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, + 0x74, 0x52, 0x0d, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, + 0x12, 0x3b, 0x0a, 0x0f, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, + 0x69, 0x6e, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x67, 0x63, 0x70, 0x2e, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x52, 0x0e, 0x72, + 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x1f, 0x0a, + 0x0b, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x08, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0a, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x40, + 0x0a, 0x0c, 0x72, 0x70, 0x63, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x09, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x63, 0x70, 0x2e, + 0x52, 0x70, 0x63, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x56, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x73, 0x52, 0x0b, 0x72, 0x70, 0x63, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, + 0x12, 0x24, 0x0a, 0x0e, 0x6d, 0x61, 0x78, 0x5f, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x5f, 0x73, 0x69, + 0x7a, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0c, 0x6d, 0x61, 0x78, 0x46, 0x72, 0x61, + 0x6d, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x22, 0x85, 0x01, 0x0a, 0x19, 0x53, 0x65, 0x72, 0x76, 0x65, + 0x72, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, + 0x74, 0x65, 0x72, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x5f, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0f, + 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x73, 0x12, + 0x3d, 0x0a, 0x10, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, + 0x69, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x67, 0x63, 0x70, 0x2e, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x0f, 0x6c, + 0x6f, 0x63, 0x61, 0x6c, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x22, 0xa5, + 0x04, 0x0a, 0x17, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x48, 0x61, + 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x52, 0x65, 0x71, 0x12, 0x33, 0x0a, 0x15, 0x61, 0x70, + 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, + 0x6f, 0x6c, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x14, 0x61, 0x70, 0x70, 0x6c, 0x69, + 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x73, 0x12, + 0x6d, 0x0a, 0x14, 0x68, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x5f, 0x70, 0x61, 0x72, + 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3a, 0x2e, + 0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x63, 0x70, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x65, + 0x72, 0x76, 0x65, 0x72, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x52, 0x65, 0x71, + 0x2e, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, + 0x74, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x13, 0x68, 0x61, 0x6e, 0x64, 0x73, + 0x68, 0x61, 0x6b, 0x65, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x12, 0x19, + 0x0a, 0x08, 0x69, 0x6e, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, + 0x52, 0x07, 0x69, 0x6e, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, 0x39, 0x0a, 0x0e, 0x6c, 0x6f, 0x63, + 0x61, 0x6c, 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x12, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x63, 0x70, 0x2e, 0x45, 0x6e, 0x64, + 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x52, 0x0d, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x45, 0x6e, 0x64, 0x70, + 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x3b, 0x0a, 0x0f, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x65, + 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, + 0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x63, 0x70, 0x2e, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, + 0x74, 0x52, 0x0e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, + 0x74, 0x12, 0x40, 0x0a, 0x0c, 0x72, 0x70, 0x63, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, + 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, + 0x63, 0x70, 0x2e, 0x52, 0x70, 0x63, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x56, 0x65, + 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x0b, 0x72, 0x70, 0x63, 0x56, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x73, 0x12, 0x24, 0x0a, 0x0e, 0x6d, 0x61, 0x78, 0x5f, 0x66, 0x72, 0x61, 0x6d, 0x65, + 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0c, 0x6d, 0x61, 0x78, + 0x46, 0x72, 0x61, 0x6d, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x1a, 0x6b, 0x0a, 0x18, 0x48, 0x61, 0x6e, + 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x05, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x39, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x63, + 0x70, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, + 0x65, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x52, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x34, 0x0a, 0x17, 0x4e, 0x65, 0x78, 0x74, 0x48, 0x61, + 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, + 0x71, 0x12, 0x19, 0x0a, 0x08, 0x69, 0x6e, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0c, 0x52, 0x07, 0x69, 0x6e, 0x42, 0x79, 0x74, 0x65, 0x73, 0x22, 0xe5, 0x01, 0x0a, + 0x0d, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x71, 0x12, 0x46, + 0x0a, 0x0c, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x73, 0x74, 0x61, 0x72, 0x74, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x63, 0x70, 0x2e, + 0x53, 0x74, 0x61, 0x72, 0x74, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x48, 0x61, 0x6e, 0x64, 0x73, + 0x68, 0x61, 0x6b, 0x65, 0x52, 0x65, 0x71, 0x48, 0x00, 0x52, 0x0b, 0x63, 0x6c, 0x69, 0x65, 0x6e, + 0x74, 0x53, 0x74, 0x61, 0x72, 0x74, 0x12, 0x46, 0x0a, 0x0c, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, + 0x5f, 0x73, 0x74, 0x61, 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x67, + 0x72, 0x70, 0x63, 0x2e, 0x67, 0x63, 0x70, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x53, 0x65, 0x72, + 0x76, 0x65, 0x72, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x52, 0x65, 0x71, 0x48, + 0x00, 0x52, 0x0b, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x61, 0x72, 0x74, 0x12, 0x37, + 0x0a, 0x04, 0x6e, 0x65, 0x78, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x67, + 0x72, 0x70, 0x63, 0x2e, 0x67, 0x63, 0x70, 0x2e, 0x4e, 0x65, 0x78, 0x74, 0x48, 0x61, 0x6e, 0x64, + 0x73, 0x68, 0x61, 0x6b, 0x65, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x71, 0x48, + 0x00, 0x52, 0x04, 0x6e, 0x65, 0x78, 0x74, 0x42, 0x0b, 0x0a, 0x09, 0x72, 0x65, 0x71, 0x5f, 0x6f, + 0x6e, 0x65, 0x6f, 0x66, 0x22, 0x9a, 0x03, 0x0a, 0x10, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, + 0x6b, 0x65, 0x72, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x31, 0x0a, 0x14, 0x61, 0x70, 0x70, + 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, + 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x13, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x27, 0x0a, 0x0f, + 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x50, 0x72, 0x6f, + 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x19, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x5f, 0x64, 0x61, 0x74, + 0x61, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x6b, 0x65, 0x79, 0x44, 0x61, 0x74, 0x61, + 0x12, 0x37, 0x0a, 0x0d, 0x70, 0x65, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, + 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, + 0x63, 0x70, 0x2e, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x0c, 0x70, 0x65, 0x65, + 0x72, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x39, 0x0a, 0x0e, 0x6c, 0x6f, 0x63, + 0x61, 0x6c, 0x5f, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x12, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x63, 0x70, 0x2e, 0x49, 0x64, 0x65, + 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x0d, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x49, 0x64, 0x65, 0x6e, + 0x74, 0x69, 0x74, 0x79, 0x12, 0x2a, 0x0a, 0x11, 0x6b, 0x65, 0x65, 0x70, 0x5f, 0x63, 0x68, 0x61, + 0x6e, 0x6e, 0x65, 0x6c, 0x5f, 0x6f, 0x70, 0x65, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x0f, 0x6b, 0x65, 0x65, 0x70, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x4f, 0x70, 0x65, 0x6e, + 0x12, 0x49, 0x0a, 0x11, 0x70, 0x65, 0x65, 0x72, 0x5f, 0x72, 0x70, 0x63, 0x5f, 0x76, 0x65, 0x72, + 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x67, 0x72, + 0x70, 0x63, 0x2e, 0x67, 0x63, 0x70, 0x2e, 0x52, 0x70, 0x63, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, + 0x6f, 0x6c, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x0f, 0x70, 0x65, 0x65, 0x72, + 0x52, 0x70, 0x63, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x24, 0x0a, 0x0e, 0x6d, + 0x61, 0x78, 0x5f, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x08, 0x20, + 0x01, 0x28, 0x0d, 0x52, 0x0c, 0x6d, 0x61, 0x78, 0x46, 0x72, 0x61, 0x6d, 0x65, 0x53, 0x69, 0x7a, + 0x65, 0x22, 0x40, 0x0a, 0x10, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x72, 0x53, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0d, 0x52, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x64, 0x65, 0x74, + 0x61, 0x69, 0x6c, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x64, 0x65, 0x74, 0x61, + 0x69, 0x6c, 0x73, 0x22, 0xbe, 0x01, 0x0a, 0x0e, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, + 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x12, 0x1d, 0x0a, 0x0a, 0x6f, 0x75, 0x74, 0x5f, 0x66, 0x72, + 0x61, 0x6d, 0x65, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x6f, 0x75, 0x74, 0x46, + 0x72, 0x61, 0x6d, 0x65, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x62, 0x79, 0x74, 0x65, 0x73, 0x5f, 0x63, + 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0d, 0x62, + 0x79, 0x74, 0x65, 0x73, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x64, 0x12, 0x32, 0x0a, 0x06, + 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, + 0x72, 0x70, 0x63, 0x2e, 0x67, 0x63, 0x70, 0x2e, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, + 0x65, 0x72, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, + 0x12, 0x32, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x63, 0x70, 0x2e, 0x48, 0x61, 0x6e, 0x64, + 0x73, 0x68, 0x61, 0x6b, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x2a, 0x4a, 0x0a, 0x11, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, + 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x22, 0x0a, 0x1e, 0x48, 0x41, 0x4e, + 0x44, 0x53, 0x48, 0x41, 0x4b, 0x45, 0x5f, 0x50, 0x52, 0x4f, 0x54, 0x4f, 0x43, 0x4f, 0x4c, 0x5f, + 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x07, 0x0a, + 0x03, 0x54, 0x4c, 0x53, 0x10, 0x01, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4c, 0x54, 0x53, 0x10, 0x02, + 0x2a, 0x45, 0x0a, 0x0f, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x50, 0x72, 0x6f, 0x74, 0x6f, + 0x63, 0x6f, 0x6c, 0x12, 0x20, 0x0a, 0x1c, 0x4e, 0x45, 0x54, 0x57, 0x4f, 0x52, 0x4b, 0x5f, 0x50, + 0x52, 0x4f, 0x54, 0x4f, 0x43, 0x4f, 0x4c, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, + 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x54, 0x43, 0x50, 0x10, 0x01, 0x12, 0x07, + 0x0a, 0x03, 0x55, 0x44, 0x50, 0x10, 0x02, 0x32, 0x5b, 0x0a, 0x11, 0x48, 0x61, 0x6e, 0x64, 0x73, + 0x68, 0x61, 0x6b, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x46, 0x0a, 0x0b, + 0x44, 0x6f, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x12, 0x17, 0x2e, 0x67, 0x72, + 0x70, 0x63, 0x2e, 0x67, 0x63, 0x70, 0x2e, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, + 0x72, 0x52, 0x65, 0x71, 0x1a, 0x18, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x63, 0x70, 0x2e, + 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x22, 0x00, + 0x28, 0x01, 0x30, 0x01, 0x42, 0x6b, 0x0a, 0x15, 0x69, 0x6f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, + 0x61, 0x6c, 0x74, 0x73, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x42, 0x0f, 0x48, + 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, + 0x5a, 0x3f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e, + 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, + 0x69, 0x61, 0x6c, 0x73, 0x2f, 0x61, 0x6c, 0x74, 0x73, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, + 0x61, 0x6c, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x67, 0x63, + 0x70, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_grpc_gcp_handshaker_proto_rawDescOnce sync.Once + file_grpc_gcp_handshaker_proto_rawDescData = file_grpc_gcp_handshaker_proto_rawDesc +) -func (c *handshakerServiceClient) DoHandshake(ctx context.Context, opts ...grpc.CallOption) (HandshakerService_DoHandshakeClient, error) { - stream, err := c.cc.NewStream(ctx, &_HandshakerService_serviceDesc.Streams[0], "/grpc.gcp.HandshakerService/DoHandshake", opts...) - if err != nil { - return nil, err +func file_grpc_gcp_handshaker_proto_rawDescGZIP() []byte { + file_grpc_gcp_handshaker_proto_rawDescOnce.Do(func() { + file_grpc_gcp_handshaker_proto_rawDescData = protoimpl.X.CompressGZIP(file_grpc_gcp_handshaker_proto_rawDescData) + }) + return file_grpc_gcp_handshaker_proto_rawDescData +} + +var file_grpc_gcp_handshaker_proto_enumTypes = make([]protoimpl.EnumInfo, 2) +var file_grpc_gcp_handshaker_proto_msgTypes = make([]protoimpl.MessageInfo, 12) +var file_grpc_gcp_handshaker_proto_goTypes = []interface{}{ + (HandshakeProtocol)(0), // 0: grpc.gcp.HandshakeProtocol + (NetworkProtocol)(0), // 1: grpc.gcp.NetworkProtocol + (*Endpoint)(nil), // 2: grpc.gcp.Endpoint + (*Identity)(nil), // 3: grpc.gcp.Identity + (*StartClientHandshakeReq)(nil), // 4: grpc.gcp.StartClientHandshakeReq + (*ServerHandshakeParameters)(nil), // 5: grpc.gcp.ServerHandshakeParameters + (*StartServerHandshakeReq)(nil), // 6: grpc.gcp.StartServerHandshakeReq + (*NextHandshakeMessageReq)(nil), // 7: grpc.gcp.NextHandshakeMessageReq + (*HandshakerReq)(nil), // 8: grpc.gcp.HandshakerReq + (*HandshakerResult)(nil), // 9: grpc.gcp.HandshakerResult + (*HandshakerStatus)(nil), // 10: grpc.gcp.HandshakerStatus + (*HandshakerResp)(nil), // 11: grpc.gcp.HandshakerResp + nil, // 12: grpc.gcp.Identity.AttributesEntry + nil, // 13: grpc.gcp.StartServerHandshakeReq.HandshakeParametersEntry + (*RpcProtocolVersions)(nil), // 14: grpc.gcp.RpcProtocolVersions +} +var file_grpc_gcp_handshaker_proto_depIdxs = []int32{ + 1, // 0: grpc.gcp.Endpoint.protocol:type_name -> grpc.gcp.NetworkProtocol + 12, // 1: grpc.gcp.Identity.attributes:type_name -> grpc.gcp.Identity.AttributesEntry + 0, // 2: grpc.gcp.StartClientHandshakeReq.handshake_security_protocol:type_name -> grpc.gcp.HandshakeProtocol + 3, // 3: grpc.gcp.StartClientHandshakeReq.target_identities:type_name -> grpc.gcp.Identity + 3, // 4: grpc.gcp.StartClientHandshakeReq.local_identity:type_name -> grpc.gcp.Identity + 2, // 5: grpc.gcp.StartClientHandshakeReq.local_endpoint:type_name -> grpc.gcp.Endpoint + 2, // 6: grpc.gcp.StartClientHandshakeReq.remote_endpoint:type_name -> grpc.gcp.Endpoint + 14, // 7: grpc.gcp.StartClientHandshakeReq.rpc_versions:type_name -> grpc.gcp.RpcProtocolVersions + 3, // 8: grpc.gcp.ServerHandshakeParameters.local_identities:type_name -> grpc.gcp.Identity + 13, // 9: grpc.gcp.StartServerHandshakeReq.handshake_parameters:type_name -> grpc.gcp.StartServerHandshakeReq.HandshakeParametersEntry + 2, // 10: grpc.gcp.StartServerHandshakeReq.local_endpoint:type_name -> grpc.gcp.Endpoint + 2, // 11: grpc.gcp.StartServerHandshakeReq.remote_endpoint:type_name -> grpc.gcp.Endpoint + 14, // 12: grpc.gcp.StartServerHandshakeReq.rpc_versions:type_name -> grpc.gcp.RpcProtocolVersions + 4, // 13: grpc.gcp.HandshakerReq.client_start:type_name -> grpc.gcp.StartClientHandshakeReq + 6, // 14: grpc.gcp.HandshakerReq.server_start:type_name -> grpc.gcp.StartServerHandshakeReq + 7, // 15: grpc.gcp.HandshakerReq.next:type_name -> grpc.gcp.NextHandshakeMessageReq + 3, // 16: grpc.gcp.HandshakerResult.peer_identity:type_name -> grpc.gcp.Identity + 3, // 17: grpc.gcp.HandshakerResult.local_identity:type_name -> grpc.gcp.Identity + 14, // 18: grpc.gcp.HandshakerResult.peer_rpc_versions:type_name -> grpc.gcp.RpcProtocolVersions + 9, // 19: grpc.gcp.HandshakerResp.result:type_name -> grpc.gcp.HandshakerResult + 10, // 20: grpc.gcp.HandshakerResp.status:type_name -> grpc.gcp.HandshakerStatus + 5, // 21: grpc.gcp.StartServerHandshakeReq.HandshakeParametersEntry.value:type_name -> grpc.gcp.ServerHandshakeParameters + 8, // 22: grpc.gcp.HandshakerService.DoHandshake:input_type -> grpc.gcp.HandshakerReq + 11, // 23: grpc.gcp.HandshakerService.DoHandshake:output_type -> grpc.gcp.HandshakerResp + 23, // [23:24] is the sub-list for method output_type + 22, // [22:23] is the sub-list for method input_type + 22, // [22:22] is the sub-list for extension type_name + 22, // [22:22] is the sub-list for extension extendee + 0, // [0:22] is the sub-list for field type_name +} + +func init() { file_grpc_gcp_handshaker_proto_init() } +func file_grpc_gcp_handshaker_proto_init() { + if File_grpc_gcp_handshaker_proto != nil { + return } - x := &handshakerServiceDoHandshakeClient{stream} - return x, nil -} - -type HandshakerService_DoHandshakeClient interface { - Send(*HandshakerReq) error - Recv() (*HandshakerResp, error) - grpc.ClientStream -} - -type handshakerServiceDoHandshakeClient struct { - grpc.ClientStream -} - -func (x *handshakerServiceDoHandshakeClient) Send(m *HandshakerReq) error { - return x.ClientStream.SendMsg(m) -} - -func (x *handshakerServiceDoHandshakeClient) Recv() (*HandshakerResp, error) { - m := new(HandshakerResp) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err + file_grpc_gcp_transport_security_common_proto_init() + if !protoimpl.UnsafeEnabled { + file_grpc_gcp_handshaker_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Endpoint); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_gcp_handshaker_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Identity); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_gcp_handshaker_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*StartClientHandshakeReq); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_gcp_handshaker_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ServerHandshakeParameters); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_gcp_handshaker_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*StartServerHandshakeReq); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_gcp_handshaker_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*NextHandshakeMessageReq); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_gcp_handshaker_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*HandshakerReq); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_gcp_handshaker_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*HandshakerResult); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_gcp_handshaker_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*HandshakerStatus); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_gcp_handshaker_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*HandshakerResp); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } - return m, nil -} - -// HandshakerServiceServer is the server API for HandshakerService service. -type HandshakerServiceServer interface { - // Handshaker service accepts a stream of handshaker request, returning a - // stream of handshaker response. Client is expected to send exactly one - // message with either client_start or server_start followed by one or more - // messages with next. Each time client sends a request, the handshaker - // service expects to respond. Client does not have to wait for service's - // response before sending next request. - DoHandshake(HandshakerService_DoHandshakeServer) error -} - -// UnimplementedHandshakerServiceServer can be embedded to have forward compatible implementations. -type UnimplementedHandshakerServiceServer struct { -} - -func (*UnimplementedHandshakerServiceServer) DoHandshake(srv HandshakerService_DoHandshakeServer) error { - return status.Errorf(codes.Unimplemented, "method DoHandshake not implemented") -} - -func RegisterHandshakerServiceServer(s *grpc.Server, srv HandshakerServiceServer) { - s.RegisterService(&_HandshakerService_serviceDesc, srv) -} - -func _HandshakerService_DoHandshake_Handler(srv interface{}, stream grpc.ServerStream) error { - return srv.(HandshakerServiceServer).DoHandshake(&handshakerServiceDoHandshakeServer{stream}) -} - -type HandshakerService_DoHandshakeServer interface { - Send(*HandshakerResp) error - Recv() (*HandshakerReq, error) - grpc.ServerStream -} - -type handshakerServiceDoHandshakeServer struct { - grpc.ServerStream -} - -func (x *handshakerServiceDoHandshakeServer) Send(m *HandshakerResp) error { - return x.ServerStream.SendMsg(m) -} - -func (x *handshakerServiceDoHandshakeServer) Recv() (*HandshakerReq, error) { - m := new(HandshakerReq) - if err := x.ServerStream.RecvMsg(m); err != nil { - return nil, err + file_grpc_gcp_handshaker_proto_msgTypes[1].OneofWrappers = []interface{}{ + (*Identity_ServiceAccount)(nil), + (*Identity_Hostname)(nil), } - return m, nil -} - -var _HandshakerService_serviceDesc = grpc.ServiceDesc{ - ServiceName: "grpc.gcp.HandshakerService", - HandlerType: (*HandshakerServiceServer)(nil), - Methods: []grpc.MethodDesc{}, - Streams: []grpc.StreamDesc{ - { - StreamName: "DoHandshake", - Handler: _HandshakerService_DoHandshake_Handler, - ServerStreams: true, - ClientStreams: true, + file_grpc_gcp_handshaker_proto_msgTypes[6].OneofWrappers = []interface{}{ + (*HandshakerReq_ClientStart)(nil), + (*HandshakerReq_ServerStart)(nil), + (*HandshakerReq_Next)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_grpc_gcp_handshaker_proto_rawDesc, + NumEnums: 2, + NumMessages: 12, + NumExtensions: 0, + NumServices: 1, }, - }, - Metadata: "grpc/gcp/handshaker.proto", + GoTypes: file_grpc_gcp_handshaker_proto_goTypes, + DependencyIndexes: file_grpc_gcp_handshaker_proto_depIdxs, + EnumInfos: file_grpc_gcp_handshaker_proto_enumTypes, + MessageInfos: file_grpc_gcp_handshaker_proto_msgTypes, + }.Build() + File_grpc_gcp_handshaker_proto = out.File + file_grpc_gcp_handshaker_proto_rawDesc = nil + file_grpc_gcp_handshaker_proto_goTypes = nil + file_grpc_gcp_handshaker_proto_depIdxs = nil } diff --git a/credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go b/credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go index bf7ff0a7bb76..39ecccf878ee 100644 --- a/credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go +++ b/credentials/alts/internal/proto/grpc_gcp/handshaker_grpc.pb.go @@ -1,8 +1,30 @@ +// Copyright 2018 The gRPC Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The canonical version of this proto can be found at +// https://github.com/grpc/grpc-proto/blob/master/grpc/gcp/handshaker.proto + // Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.3.0 +// - protoc v4.22.0 +// source: grpc/gcp/handshaker.proto package grpc_gcp import ( + context "context" grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" @@ -10,69 +32,69 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. const _ = grpc.SupportPackageIsVersion7 -// HandshakerServiceService is the service API for HandshakerService service. -// Fields should be assigned to their respective handler implementations only before -// RegisterHandshakerServiceService is called. Any unassigned fields will result in the -// handler for that method returning an Unimplemented error. -type HandshakerServiceService struct { +const ( + HandshakerService_DoHandshake_FullMethodName = "/grpc.gcp.HandshakerService/DoHandshake" +) + +// HandshakerServiceClient is the client API for HandshakerService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type HandshakerServiceClient interface { // Handshaker service accepts a stream of handshaker request, returning a // stream of handshaker response. Client is expected to send exactly one // message with either client_start or server_start followed by one or more // messages with next. Each time client sends a request, the handshaker // service expects to respond. Client does not have to wait for service's // response before sending next request. - DoHandshake func(HandshakerService_DoHandshakeServer) error + DoHandshake(ctx context.Context, opts ...grpc.CallOption) (HandshakerService_DoHandshakeClient, error) } -func (s *HandshakerServiceService) doHandshake(_ interface{}, stream grpc.ServerStream) error { - if s.DoHandshake == nil { - return status.Errorf(codes.Unimplemented, "method DoHandshake not implemented") - } - return s.DoHandshake(&handshakerServiceDoHandshakeServer{stream}) -} - -// RegisterHandshakerServiceService registers a service implementation with a gRPC server. -func RegisterHandshakerServiceService(s grpc.ServiceRegistrar, srv *HandshakerServiceService) { - sd := grpc.ServiceDesc{ - ServiceName: "grpc.gcp.HandshakerService", - Methods: []grpc.MethodDesc{}, - Streams: []grpc.StreamDesc{ - { - StreamName: "DoHandshake", - Handler: srv.doHandshake, - ServerStreams: true, - ClientStreams: true, - }, - }, - Metadata: "grpc/gcp/handshaker.proto", +type handshakerServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewHandshakerServiceClient(cc grpc.ClientConnInterface) HandshakerServiceClient { + return &handshakerServiceClient{cc} +} + +func (c *handshakerServiceClient) DoHandshake(ctx context.Context, opts ...grpc.CallOption) (HandshakerService_DoHandshakeClient, error) { + stream, err := c.cc.NewStream(ctx, &HandshakerService_ServiceDesc.Streams[0], HandshakerService_DoHandshake_FullMethodName, opts...) + if err != nil { + return nil, err } + x := &handshakerServiceDoHandshakeClient{stream} + return x, nil +} + +type HandshakerService_DoHandshakeClient interface { + Send(*HandshakerReq) error + Recv() (*HandshakerResp, error) + grpc.ClientStream +} + +type handshakerServiceDoHandshakeClient struct { + grpc.ClientStream +} - s.RegisterService(&sd, nil) -} - -// NewHandshakerServiceService creates a new HandshakerServiceService containing the -// implemented methods of the HandshakerService service in s. Any unimplemented -// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client. -// This includes situations where the method handler is misspelled or has the wrong -// signature. For this reason, this function should be used with great care and -// is not recommended to be used by most users. -func NewHandshakerServiceService(s interface{}) *HandshakerServiceService { - ns := &HandshakerServiceService{} - if h, ok := s.(interface { - DoHandshake(HandshakerService_DoHandshakeServer) error - }); ok { - ns.DoHandshake = h.DoHandshake +func (x *handshakerServiceDoHandshakeClient) Send(m *HandshakerReq) error { + return x.ClientStream.SendMsg(m) +} + +func (x *handshakerServiceDoHandshakeClient) Recv() (*HandshakerResp, error) { + m := new(HandshakerResp) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err } - return ns + return m, nil } -// UnstableHandshakerServiceService is the service API for HandshakerService service. -// New methods may be added to this interface if they are added to the service -// definition, which is not a backward-compatible change. For this reason, -// use of this type is not recommended. -type UnstableHandshakerServiceService interface { +// HandshakerServiceServer is the server API for HandshakerService service. +// All implementations must embed UnimplementedHandshakerServiceServer +// for forward compatibility +type HandshakerServiceServer interface { // Handshaker service accepts a stream of handshaker request, returning a // stream of handshaker response. Client is expected to send exactly one // message with either client_start or server_start followed by one or more @@ -80,4 +102,69 @@ type UnstableHandshakerServiceService interface { // service expects to respond. Client does not have to wait for service's // response before sending next request. DoHandshake(HandshakerService_DoHandshakeServer) error + mustEmbedUnimplementedHandshakerServiceServer() +} + +// UnimplementedHandshakerServiceServer must be embedded to have forward compatible implementations. +type UnimplementedHandshakerServiceServer struct { +} + +func (UnimplementedHandshakerServiceServer) DoHandshake(HandshakerService_DoHandshakeServer) error { + return status.Errorf(codes.Unimplemented, "method DoHandshake not implemented") +} +func (UnimplementedHandshakerServiceServer) mustEmbedUnimplementedHandshakerServiceServer() {} + +// UnsafeHandshakerServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to HandshakerServiceServer will +// result in compilation errors. +type UnsafeHandshakerServiceServer interface { + mustEmbedUnimplementedHandshakerServiceServer() +} + +func RegisterHandshakerServiceServer(s grpc.ServiceRegistrar, srv HandshakerServiceServer) { + s.RegisterService(&HandshakerService_ServiceDesc, srv) +} + +func _HandshakerService_DoHandshake_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(HandshakerServiceServer).DoHandshake(&handshakerServiceDoHandshakeServer{stream}) +} + +type HandshakerService_DoHandshakeServer interface { + Send(*HandshakerResp) error + Recv() (*HandshakerReq, error) + grpc.ServerStream +} + +type handshakerServiceDoHandshakeServer struct { + grpc.ServerStream +} + +func (x *handshakerServiceDoHandshakeServer) Send(m *HandshakerResp) error { + return x.ServerStream.SendMsg(m) +} + +func (x *handshakerServiceDoHandshakeServer) Recv() (*HandshakerReq, error) { + m := new(HandshakerReq) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +// HandshakerService_ServiceDesc is the grpc.ServiceDesc for HandshakerService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var HandshakerService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.gcp.HandshakerService", + HandlerType: (*HandshakerServiceServer)(nil), + Methods: []grpc.MethodDesc{}, + Streams: []grpc.StreamDesc{ + { + StreamName: "DoHandshake", + Handler: _HandshakerService_DoHandshake_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "grpc/gcp/handshaker.proto", } diff --git a/credentials/alts/internal/proto/grpc_gcp/transport_security_common.pb.go b/credentials/alts/internal/proto/grpc_gcp/transport_security_common.pb.go index 992805165db6..69f0947582da 100644 --- a/credentials/alts/internal/proto/grpc_gcp/transport_security_common.pb.go +++ b/credentials/alts/internal/proto/grpc_gcp/transport_security_common.pb.go @@ -1,24 +1,41 @@ +// Copyright 2018 The gRPC Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The canonical version of this proto can be found at +// https://github.com/grpc/grpc-proto/blob/master/grpc/gcp/transport_security_common.proto + // Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.31.0 +// protoc v4.22.0 // source: grpc/gcp/transport_security_common.proto package grpc_gcp import ( - fmt "fmt" - proto "github.com/golang/protobuf/proto" - math "math" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" ) -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) // The security level of the created channel. The list is sorted in increasing // level of security. This order must always be maintained. @@ -30,155 +47,275 @@ const ( SecurityLevel_INTEGRITY_AND_PRIVACY SecurityLevel = 2 ) -var SecurityLevel_name = map[int32]string{ - 0: "SECURITY_NONE", - 1: "INTEGRITY_ONLY", - 2: "INTEGRITY_AND_PRIVACY", -} +// Enum value maps for SecurityLevel. +var ( + SecurityLevel_name = map[int32]string{ + 0: "SECURITY_NONE", + 1: "INTEGRITY_ONLY", + 2: "INTEGRITY_AND_PRIVACY", + } + SecurityLevel_value = map[string]int32{ + "SECURITY_NONE": 0, + "INTEGRITY_ONLY": 1, + "INTEGRITY_AND_PRIVACY": 2, + } +) -var SecurityLevel_value = map[string]int32{ - "SECURITY_NONE": 0, - "INTEGRITY_ONLY": 1, - "INTEGRITY_AND_PRIVACY": 2, +func (x SecurityLevel) Enum() *SecurityLevel { + p := new(SecurityLevel) + *p = x + return p } func (x SecurityLevel) String() string { - return proto.EnumName(SecurityLevel_name, int32(x)) + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (SecurityLevel) Descriptor() protoreflect.EnumDescriptor { + return file_grpc_gcp_transport_security_common_proto_enumTypes[0].Descriptor() +} + +func (SecurityLevel) Type() protoreflect.EnumType { + return &file_grpc_gcp_transport_security_common_proto_enumTypes[0] } +func (x SecurityLevel) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use SecurityLevel.Descriptor instead. func (SecurityLevel) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_b97e31e3cc23582a, []int{0} + return file_grpc_gcp_transport_security_common_proto_rawDescGZIP(), []int{0} } // Max and min supported RPC protocol versions. type RpcProtocolVersions struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // Maximum supported RPC version. MaxRpcVersion *RpcProtocolVersions_Version `protobuf:"bytes,1,opt,name=max_rpc_version,json=maxRpcVersion,proto3" json:"max_rpc_version,omitempty"` // Minimum supported RPC version. - MinRpcVersion *RpcProtocolVersions_Version `protobuf:"bytes,2,opt,name=min_rpc_version,json=minRpcVersion,proto3" json:"min_rpc_version,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + MinRpcVersion *RpcProtocolVersions_Version `protobuf:"bytes,2,opt,name=min_rpc_version,json=minRpcVersion,proto3" json:"min_rpc_version,omitempty"` } -func (m *RpcProtocolVersions) Reset() { *m = RpcProtocolVersions{} } -func (m *RpcProtocolVersions) String() string { return proto.CompactTextString(m) } -func (*RpcProtocolVersions) ProtoMessage() {} -func (*RpcProtocolVersions) Descriptor() ([]byte, []int) { - return fileDescriptor_b97e31e3cc23582a, []int{0} +func (x *RpcProtocolVersions) Reset() { + *x = RpcProtocolVersions{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_gcp_transport_security_common_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *RpcProtocolVersions) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_RpcProtocolVersions.Unmarshal(m, b) -} -func (m *RpcProtocolVersions) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_RpcProtocolVersions.Marshal(b, m, deterministic) +func (x *RpcProtocolVersions) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *RpcProtocolVersions) XXX_Merge(src proto.Message) { - xxx_messageInfo_RpcProtocolVersions.Merge(m, src) -} -func (m *RpcProtocolVersions) XXX_Size() int { - return xxx_messageInfo_RpcProtocolVersions.Size(m) -} -func (m *RpcProtocolVersions) XXX_DiscardUnknown() { - xxx_messageInfo_RpcProtocolVersions.DiscardUnknown(m) + +func (*RpcProtocolVersions) ProtoMessage() {} + +func (x *RpcProtocolVersions) ProtoReflect() protoreflect.Message { + mi := &file_grpc_gcp_transport_security_common_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_RpcProtocolVersions proto.InternalMessageInfo +// Deprecated: Use RpcProtocolVersions.ProtoReflect.Descriptor instead. +func (*RpcProtocolVersions) Descriptor() ([]byte, []int) { + return file_grpc_gcp_transport_security_common_proto_rawDescGZIP(), []int{0} +} -func (m *RpcProtocolVersions) GetMaxRpcVersion() *RpcProtocolVersions_Version { - if m != nil { - return m.MaxRpcVersion +func (x *RpcProtocolVersions) GetMaxRpcVersion() *RpcProtocolVersions_Version { + if x != nil { + return x.MaxRpcVersion } return nil } -func (m *RpcProtocolVersions) GetMinRpcVersion() *RpcProtocolVersions_Version { - if m != nil { - return m.MinRpcVersion +func (x *RpcProtocolVersions) GetMinRpcVersion() *RpcProtocolVersions_Version { + if x != nil { + return x.MinRpcVersion } return nil } // RPC version contains a major version and a minor version. type RpcProtocolVersions_Version struct { - Major uint32 `protobuf:"varint,1,opt,name=major,proto3" json:"major,omitempty"` - Minor uint32 `protobuf:"varint,2,opt,name=minor,proto3" json:"minor,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields -func (m *RpcProtocolVersions_Version) Reset() { *m = RpcProtocolVersions_Version{} } -func (m *RpcProtocolVersions_Version) String() string { return proto.CompactTextString(m) } -func (*RpcProtocolVersions_Version) ProtoMessage() {} -func (*RpcProtocolVersions_Version) Descriptor() ([]byte, []int) { - return fileDescriptor_b97e31e3cc23582a, []int{0, 0} + Major uint32 `protobuf:"varint,1,opt,name=major,proto3" json:"major,omitempty"` + Minor uint32 `protobuf:"varint,2,opt,name=minor,proto3" json:"minor,omitempty"` } -func (m *RpcProtocolVersions_Version) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_RpcProtocolVersions_Version.Unmarshal(m, b) -} -func (m *RpcProtocolVersions_Version) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_RpcProtocolVersions_Version.Marshal(b, m, deterministic) -} -func (m *RpcProtocolVersions_Version) XXX_Merge(src proto.Message) { - xxx_messageInfo_RpcProtocolVersions_Version.Merge(m, src) +func (x *RpcProtocolVersions_Version) Reset() { + *x = RpcProtocolVersions_Version{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_gcp_transport_security_common_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *RpcProtocolVersions_Version) XXX_Size() int { - return xxx_messageInfo_RpcProtocolVersions_Version.Size(m) + +func (x *RpcProtocolVersions_Version) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *RpcProtocolVersions_Version) XXX_DiscardUnknown() { - xxx_messageInfo_RpcProtocolVersions_Version.DiscardUnknown(m) + +func (*RpcProtocolVersions_Version) ProtoMessage() {} + +func (x *RpcProtocolVersions_Version) ProtoReflect() protoreflect.Message { + mi := &file_grpc_gcp_transport_security_common_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_RpcProtocolVersions_Version proto.InternalMessageInfo +// Deprecated: Use RpcProtocolVersions_Version.ProtoReflect.Descriptor instead. +func (*RpcProtocolVersions_Version) Descriptor() ([]byte, []int) { + return file_grpc_gcp_transport_security_common_proto_rawDescGZIP(), []int{0, 0} +} -func (m *RpcProtocolVersions_Version) GetMajor() uint32 { - if m != nil { - return m.Major +func (x *RpcProtocolVersions_Version) GetMajor() uint32 { + if x != nil { + return x.Major } return 0 } -func (m *RpcProtocolVersions_Version) GetMinor() uint32 { - if m != nil { - return m.Minor +func (x *RpcProtocolVersions_Version) GetMinor() uint32 { + if x != nil { + return x.Minor } return 0 } -func init() { - proto.RegisterEnum("grpc.gcp.SecurityLevel", SecurityLevel_name, SecurityLevel_value) - proto.RegisterType((*RpcProtocolVersions)(nil), "grpc.gcp.RpcProtocolVersions") - proto.RegisterType((*RpcProtocolVersions_Version)(nil), "grpc.gcp.RpcProtocolVersions.Version") -} - -func init() { - proto.RegisterFile("grpc/gcp/transport_security_common.proto", fileDescriptor_b97e31e3cc23582a) -} - -var fileDescriptor_b97e31e3cc23582a = []byte{ - // 323 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x91, 0x41, 0x4b, 0x3b, 0x31, - 0x10, 0xc5, 0xff, 0x5b, 0xf8, 0xab, 0x44, 0x56, 0xeb, 0x6a, 0x41, 0xc5, 0x83, 0x08, 0x42, 0xf1, - 0x90, 0x05, 0xc5, 0xb3, 0xb4, 0xb5, 0x48, 0xa1, 0x6e, 0xeb, 0xb6, 0x16, 0xea, 0x25, 0xc4, 0x18, - 0x42, 0x24, 0x9b, 0x09, 0xb3, 0xb1, 0xd4, 0xaf, 0xec, 0xa7, 0x90, 0x4d, 0xbb, 0x14, 0xc1, 0x8b, - 0xb7, 0xbc, 0xc7, 0xcc, 0x6f, 0x32, 0xf3, 0x48, 0x5b, 0xa1, 0x13, 0xa9, 0x12, 0x2e, 0xf5, 0xc8, - 0x6d, 0xe9, 0x00, 0x3d, 0x2b, 0xa5, 0xf8, 0x40, 0xed, 0x3f, 0x99, 0x80, 0xa2, 0x00, 0x4b, 0x1d, - 0x82, 0x87, 0x64, 0xa7, 0xaa, 0xa4, 0x4a, 0xb8, 0x8b, 0xaf, 0x88, 0x1c, 0xe6, 0x4e, 0x8c, 0x2b, - 0x5b, 0x80, 0x99, 0x49, 0x2c, 0x35, 0xd8, 0x32, 0x79, 0x24, 0xfb, 0x05, 0x5f, 0x32, 0x74, 0x82, - 0x2d, 0x56, 0xde, 0x71, 0x74, 0x1e, 0xb5, 0x77, 0xaf, 0x2f, 0x69, 0xdd, 0x4b, 0x7f, 0xe9, 0xa3, - 0xeb, 0x47, 0x1e, 0x17, 0x7c, 0x99, 0x3b, 0xb1, 0x96, 0x01, 0xa7, 0xed, 0x0f, 0x5c, 0xe3, 0x6f, - 0x38, 0x6d, 0x37, 0xb8, 0xd3, 0x5b, 0xb2, 0x5d, 0x93, 0x8f, 0xc8, 0xff, 0x82, 0xbf, 0x03, 0x86, - 0xef, 0xc5, 0xf9, 0x4a, 0x04, 0x57, 0x5b, 0xc0, 0x30, 0xa5, 0x72, 0x2b, 0x71, 0xf5, 0x44, 0xe2, - 0xc9, 0xfa, 0x1e, 0x43, 0xb9, 0x90, 0x26, 0x39, 0x20, 0xf1, 0xa4, 0xdf, 0x7b, 0xce, 0x07, 0xd3, - 0x39, 0xcb, 0x46, 0x59, 0xbf, 0xf9, 0x2f, 0x49, 0xc8, 0xde, 0x20, 0x9b, 0xf6, 0x1f, 0x82, 0x37, - 0xca, 0x86, 0xf3, 0x66, 0x94, 0x9c, 0x90, 0xd6, 0xc6, 0xeb, 0x64, 0xf7, 0x6c, 0x9c, 0x0f, 0x66, - 0x9d, 0xde, 0xbc, 0xd9, 0xe8, 0x2e, 0x49, 0x4b, 0xc3, 0x6a, 0x07, 0x6e, 0x7c, 0x49, 0xb5, 0xf5, - 0x12, 0x2d, 0x37, 0xdd, 0xb3, 0x69, 0x9d, 0x41, 0x3d, 0xb2, 0x17, 0x12, 0x08, 0x2b, 0x8e, 0xa3, - 0x97, 0x3b, 0x05, 0xa0, 0x8c, 0xa4, 0x0a, 0x0c, 0xb7, 0x8a, 0x02, 0xaa, 0x34, 0xc4, 0x27, 0x50, - 0xbe, 0x49, 0xeb, 0x35, 0x37, 0x65, 0x5a, 0x11, 0xd3, 0x9a, 0x98, 0x86, 0xe8, 0x42, 0x11, 0x53, - 0xc2, 0xbd, 0x6e, 0x05, 0x7d, 0xf3, 0x1d, 0x00, 0x00, 0xff, 0xff, 0x31, 0x14, 0xb4, 0x11, 0xf6, - 0x01, 0x00, 0x00, +var File_grpc_gcp_transport_security_common_proto protoreflect.FileDescriptor + +var file_grpc_gcp_transport_security_common_proto_rawDesc = []byte{ + 0x0a, 0x28, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x67, 0x63, 0x70, 0x2f, 0x74, 0x72, 0x61, 0x6e, 0x73, + 0x70, 0x6f, 0x72, 0x74, 0x5f, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x5f, 0x63, 0x6f, + 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x08, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x67, 0x63, 0x70, 0x22, 0xea, 0x01, 0x0a, 0x13, 0x52, 0x70, 0x63, 0x50, 0x72, 0x6f, 0x74, + 0x6f, 0x63, 0x6f, 0x6c, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x4d, 0x0a, 0x0f, + 0x6d, 0x61, 0x78, 0x5f, 0x72, 0x70, 0x63, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x63, 0x70, + 0x2e, 0x52, 0x70, 0x63, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x56, 0x65, 0x72, 0x73, + 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x0d, 0x6d, 0x61, + 0x78, 0x52, 0x70, 0x63, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x4d, 0x0a, 0x0f, 0x6d, + 0x69, 0x6e, 0x5f, 0x72, 0x70, 0x63, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x63, 0x70, 0x2e, + 0x52, 0x70, 0x63, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x56, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x73, 0x2e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x0d, 0x6d, 0x69, 0x6e, + 0x52, 0x70, 0x63, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x1a, 0x35, 0x0a, 0x07, 0x56, 0x65, + 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x6d, 0x61, 0x6a, 0x6f, 0x72, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x6d, 0x61, 0x6a, 0x6f, 0x72, 0x12, 0x14, 0x0a, 0x05, 0x6d, + 0x69, 0x6e, 0x6f, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x6d, 0x69, 0x6e, 0x6f, + 0x72, 0x2a, 0x51, 0x0a, 0x0d, 0x53, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x4c, 0x65, 0x76, + 0x65, 0x6c, 0x12, 0x11, 0x0a, 0x0d, 0x53, 0x45, 0x43, 0x55, 0x52, 0x49, 0x54, 0x59, 0x5f, 0x4e, + 0x4f, 0x4e, 0x45, 0x10, 0x00, 0x12, 0x12, 0x0a, 0x0e, 0x49, 0x4e, 0x54, 0x45, 0x47, 0x52, 0x49, + 0x54, 0x59, 0x5f, 0x4f, 0x4e, 0x4c, 0x59, 0x10, 0x01, 0x12, 0x19, 0x0a, 0x15, 0x49, 0x4e, 0x54, + 0x45, 0x47, 0x52, 0x49, 0x54, 0x59, 0x5f, 0x41, 0x4e, 0x44, 0x5f, 0x50, 0x52, 0x49, 0x56, 0x41, + 0x43, 0x59, 0x10, 0x02, 0x42, 0x78, 0x0a, 0x15, 0x69, 0x6f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, + 0x61, 0x6c, 0x74, 0x73, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x42, 0x1c, 0x54, + 0x72, 0x61, 0x6e, 0x73, 0x70, 0x6f, 0x72, 0x74, 0x53, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, + 0x43, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x3f, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, + 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, + 0x73, 0x2f, 0x61, 0x6c, 0x74, 0x73, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x67, 0x63, 0x70, 0x62, 0x06, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_grpc_gcp_transport_security_common_proto_rawDescOnce sync.Once + file_grpc_gcp_transport_security_common_proto_rawDescData = file_grpc_gcp_transport_security_common_proto_rawDesc +) + +func file_grpc_gcp_transport_security_common_proto_rawDescGZIP() []byte { + file_grpc_gcp_transport_security_common_proto_rawDescOnce.Do(func() { + file_grpc_gcp_transport_security_common_proto_rawDescData = protoimpl.X.CompressGZIP(file_grpc_gcp_transport_security_common_proto_rawDescData) + }) + return file_grpc_gcp_transport_security_common_proto_rawDescData +} + +var file_grpc_gcp_transport_security_common_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_grpc_gcp_transport_security_common_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_grpc_gcp_transport_security_common_proto_goTypes = []interface{}{ + (SecurityLevel)(0), // 0: grpc.gcp.SecurityLevel + (*RpcProtocolVersions)(nil), // 1: grpc.gcp.RpcProtocolVersions + (*RpcProtocolVersions_Version)(nil), // 2: grpc.gcp.RpcProtocolVersions.Version +} +var file_grpc_gcp_transport_security_common_proto_depIdxs = []int32{ + 2, // 0: grpc.gcp.RpcProtocolVersions.max_rpc_version:type_name -> grpc.gcp.RpcProtocolVersions.Version + 2, // 1: grpc.gcp.RpcProtocolVersions.min_rpc_version:type_name -> grpc.gcp.RpcProtocolVersions.Version + 2, // [2:2] is the sub-list for method output_type + 2, // [2:2] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name +} + +func init() { file_grpc_gcp_transport_security_common_proto_init() } +func file_grpc_gcp_transport_security_common_proto_init() { + if File_grpc_gcp_transport_security_common_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_grpc_gcp_transport_security_common_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RpcProtocolVersions); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_gcp_transport_security_common_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RpcProtocolVersions_Version); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_grpc_gcp_transport_security_common_proto_rawDesc, + NumEnums: 1, + NumMessages: 2, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_grpc_gcp_transport_security_common_proto_goTypes, + DependencyIndexes: file_grpc_gcp_transport_security_common_proto_depIdxs, + EnumInfos: file_grpc_gcp_transport_security_common_proto_enumTypes, + MessageInfos: file_grpc_gcp_transport_security_common_proto_msgTypes, + }.Build() + File_grpc_gcp_transport_security_common_proto = out.File + file_grpc_gcp_transport_security_common_proto_rawDesc = nil + file_grpc_gcp_transport_security_common_proto_goTypes = nil + file_grpc_gcp_transport_security_common_proto_depIdxs = nil } diff --git a/credentials/alts/internal/testutil/testutil.go b/credentials/alts/internal/testutil/testutil.go index e114719d5a88..cdc88c8f9da0 100644 --- a/credentials/alts/internal/testutil/testutil.go +++ b/credentials/alts/internal/testutil/testutil.go @@ -22,11 +22,15 @@ package testutil import ( "bytes" "encoding/binary" + "fmt" "io" "net" "sync" + "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials/alts/internal/conn" + altsgrpc "google.golang.org/grpc/credentials/alts/internal/proto/grpc_gcp" + altspb "google.golang.org/grpc/credentials/alts/internal/proto/grpc_gcp" ) // Stats is used to collect statistics about concurrent handshake calls. @@ -123,3 +127,178 @@ func MakeFrame(pl string) []byte { copy(f[conn.MsgLenFieldSize:], []byte(pl)) return f } + +// FakeHandshaker is a fake implementation of the ALTS handshaker service. +type FakeHandshaker struct { + altsgrpc.HandshakerServiceServer +} + +// DoHandshake performs a fake ALTS handshake. +func (h *FakeHandshaker) DoHandshake(stream altsgrpc.HandshakerService_DoHandshakeServer) error { + var isAssistingClient bool + var handshakeFramesReceivedSoFar []byte + for { + req, err := stream.Recv() + if err != nil { + if err == io.EOF { + return nil + } + return fmt.Errorf("stream recv failure: %v", err) + } + var resp *altspb.HandshakerResp + switch req := req.ReqOneof.(type) { + case *altspb.HandshakerReq_ClientStart: + isAssistingClient = true + resp, err = h.processStartClient(req.ClientStart) + if err != nil { + return fmt.Errorf("processStartClient failure: %v", err) + } + case *altspb.HandshakerReq_ServerStart: + // If we have received the full ClientInit, send the ServerInit and + // ServerFinished. Otherwise, wait for more bytes to arrive from the client. + isAssistingClient = false + handshakeFramesReceivedSoFar = append(handshakeFramesReceivedSoFar, req.ServerStart.InBytes...) + sendHandshakeFrame := bytes.Equal(handshakeFramesReceivedSoFar, []byte("ClientInit")) + resp, err = h.processServerStart(req.ServerStart, sendHandshakeFrame) + if err != nil { + return fmt.Errorf("processServerStart failure: %v", err) + } + case *altspb.HandshakerReq_Next: + // If we have received all handshake frames, send the handshake result. + // Otherwise, wait for more bytes to arrive from the peer. + oldHandshakesBytes := len(handshakeFramesReceivedSoFar) + handshakeFramesReceivedSoFar = append(handshakeFramesReceivedSoFar, req.Next.InBytes...) + isHandshakeComplete := false + if isAssistingClient { + isHandshakeComplete = bytes.HasPrefix(handshakeFramesReceivedSoFar, []byte("ServerInitServerFinished")) + } else { + isHandshakeComplete = bytes.HasPrefix(handshakeFramesReceivedSoFar, []byte("ClientInitClientFinished")) + } + if !isHandshakeComplete { + resp = &altspb.HandshakerResp{ + BytesConsumed: uint32(len(handshakeFramesReceivedSoFar) - oldHandshakesBytes), + Status: &altspb.HandshakerStatus{ + Code: uint32(codes.OK), + }, + } + break + } + resp, err = h.getHandshakeResult(isAssistingClient) + if err != nil { + return fmt.Errorf("getHandshakeResult failure: %v", err) + } + default: + return fmt.Errorf("handshake request has unexpected type: %v", req) + } + + if err = stream.Send(resp); err != nil { + return fmt.Errorf("stream send failure: %v", err) + } + } +} + +func (h *FakeHandshaker) processStartClient(req *altspb.StartClientHandshakeReq) (*altspb.HandshakerResp, error) { + if req.HandshakeSecurityProtocol != altspb.HandshakeProtocol_ALTS { + return nil, fmt.Errorf("unexpected handshake security protocol: %v", req.HandshakeSecurityProtocol) + } + if len(req.ApplicationProtocols) != 1 || req.ApplicationProtocols[0] != "grpc" { + return nil, fmt.Errorf("unexpected application protocols: %v", req.ApplicationProtocols) + } + if len(req.RecordProtocols) != 1 || req.RecordProtocols[0] != "ALTSRP_GCM_AES128_REKEY" { + return nil, fmt.Errorf("unexpected record protocols: %v", req.RecordProtocols) + } + return &altspb.HandshakerResp{ + OutFrames: []byte("ClientInit"), + BytesConsumed: 0, + Status: &altspb.HandshakerStatus{ + Code: uint32(codes.OK), + }, + }, nil +} + +func (h *FakeHandshaker) processServerStart(req *altspb.StartServerHandshakeReq, sendHandshakeFrame bool) (*altspb.HandshakerResp, error) { + if len(req.ApplicationProtocols) != 1 || req.ApplicationProtocols[0] != "grpc" { + return nil, fmt.Errorf("unexpected application protocols: %v", req.ApplicationProtocols) + } + parameters, ok := req.GetHandshakeParameters()[int32(altspb.HandshakeProtocol_ALTS)] + if !ok { + return nil, fmt.Errorf("missing ALTS handshake parameters") + } + if len(parameters.RecordProtocols) != 1 || parameters.RecordProtocols[0] != "ALTSRP_GCM_AES128_REKEY" { + return nil, fmt.Errorf("unexpected record protocols: %v", parameters.RecordProtocols) + } + if sendHandshakeFrame { + return &altspb.HandshakerResp{ + OutFrames: []byte("ServerInitServerFinished"), + BytesConsumed: uint32(len(req.InBytes)), + Status: &altspb.HandshakerStatus{ + Code: uint32(codes.OK), + }, + }, nil + } + return &altspb.HandshakerResp{ + OutFrames: []byte("ServerInitServerFinished"), + BytesConsumed: 10, + Status: &altspb.HandshakerStatus{ + Code: uint32(codes.OK), + }, + }, nil +} + +func (h *FakeHandshaker) getHandshakeResult(isAssistingClient bool) (*altspb.HandshakerResp, error) { + if isAssistingClient { + return &altspb.HandshakerResp{ + OutFrames: []byte("ClientFinished"), + BytesConsumed: 24, + Result: &altspb.HandshakerResult{ + ApplicationProtocol: "grpc", + RecordProtocol: "ALTSRP_GCM_AES128_REKEY", + KeyData: []byte("negotiated-key-data-for-altsrp-gcm-aes128-rekey"), + PeerIdentity: &altspb.Identity{ + IdentityOneof: &altspb.Identity_ServiceAccount{ + ServiceAccount: "server@bar.com", + }, + }, + PeerRpcVersions: &altspb.RpcProtocolVersions{ + MaxRpcVersion: &altspb.RpcProtocolVersions_Version{ + Minor: 1, + Major: 2, + }, + MinRpcVersion: &altspb.RpcProtocolVersions_Version{ + Minor: 1, + Major: 2, + }, + }, + }, + Status: &altspb.HandshakerStatus{ + Code: uint32(codes.OK), + }, + }, nil + } + return &altspb.HandshakerResp{ + BytesConsumed: 14, + Result: &altspb.HandshakerResult{ + ApplicationProtocol: "grpc", + RecordProtocol: "ALTSRP_GCM_AES128_REKEY", + KeyData: []byte("negotiated-key-data-for-altsrp-gcm-aes128-rekey"), + PeerIdentity: &altspb.Identity{ + IdentityOneof: &altspb.Identity_ServiceAccount{ + ServiceAccount: "client@baz.com", + }, + }, + PeerRpcVersions: &altspb.RpcProtocolVersions{ + MaxRpcVersion: &altspb.RpcProtocolVersions_Version{ + Minor: 1, + Major: 2, + }, + MinRpcVersion: &altspb.RpcProtocolVersions_Version{ + Minor: 1, + Major: 2, + }, + }, + }, + Status: &altspb.HandshakerStatus{ + Code: uint32(codes.OK), + }, + }, nil +} diff --git a/credentials/alts/utils.go b/credentials/alts/utils.go index e46280ad5f58..cbfd056cfb14 100644 --- a/credentials/alts/utils.go +++ b/credentials/alts/utils.go @@ -21,14 +21,6 @@ package alts import ( "context" "errors" - "fmt" - "io" - "io/ioutil" - "log" - "os" - "os/exec" - "regexp" - "runtime" "strings" "google.golang.org/grpc/codes" @@ -36,92 +28,6 @@ import ( "google.golang.org/grpc/status" ) -const ( - linuxProductNameFile = "/sys/class/dmi/id/product_name" - windowsCheckCommand = "powershell.exe" - windowsCheckCommandArgs = "Get-WmiObject -Class Win32_BIOS" - powershellOutputFilter = "Manufacturer" - windowsManufacturerRegex = ":(.*)" -) - -type platformError string - -func (k platformError) Error() string { - return fmt.Sprintf("%s is not supported", string(k)) -} - -var ( - // The following two variables will be reassigned in tests. - runningOS = runtime.GOOS - manufacturerReader = func() (io.Reader, error) { - switch runningOS { - case "linux": - return os.Open(linuxProductNameFile) - case "windows": - cmd := exec.Command(windowsCheckCommand, windowsCheckCommandArgs) - out, err := cmd.Output() - if err != nil { - return nil, err - } - - for _, line := range strings.Split(strings.TrimSuffix(string(out), "\n"), "\n") { - if strings.HasPrefix(line, powershellOutputFilter) { - re := regexp.MustCompile(windowsManufacturerRegex) - name := re.FindString(line) - name = strings.TrimLeft(name, ":") - return strings.NewReader(name), nil - } - } - - return nil, errors.New("cannot determine the machine's manufacturer") - default: - return nil, platformError(runningOS) - } - } - vmOnGCP bool -) - -// isRunningOnGCP checks whether the local system, without doing a network request is -// running on GCP. -func isRunningOnGCP() bool { - manufacturer, err := readManufacturer() - if os.IsNotExist(err) { - return false - } - if err != nil { - log.Fatalf("failure to read manufacturer information: %v", err) - } - name := string(manufacturer) - switch runningOS { - case "linux": - name = strings.TrimSpace(name) - return name == "Google" || name == "Google Compute Engine" - case "windows": - name = strings.Replace(name, " ", "", -1) - name = strings.Replace(name, "\n", "", -1) - name = strings.Replace(name, "\r", "", -1) - return name == "Google" - default: - log.Fatal(platformError(runningOS)) - } - return false -} - -func readManufacturer() ([]byte, error) { - reader, err := manufacturerReader() - if err != nil { - return nil, err - } - if reader == nil { - return nil, errors.New("got nil reader") - } - manufacturer, err := ioutil.ReadAll(reader) - if err != nil { - return nil, fmt.Errorf("failed reading %v: %v", linuxProductNameFile, err) - } - return manufacturer, nil -} - // AuthInfoFromContext extracts the alts.AuthInfo object from the given context, // if it exists. This API should be used by gRPC server RPC handlers to get // information about the communicating peer. For client-side, use grpc.Peer() @@ -152,12 +58,13 @@ func AuthInfoFromPeer(p *peer.Peer) (AuthInfo, error) { func ClientAuthorizationCheck(ctx context.Context, expectedServiceAccounts []string) error { authInfo, err := AuthInfoFromContext(ctx) if err != nil { - return status.Newf(codes.PermissionDenied, "The context is not an ALTS-compatible context: %v", err).Err() + return status.Errorf(codes.PermissionDenied, "The context is not an ALTS-compatible context: %v", err) } + peer := authInfo.PeerServiceAccount() for _, sa := range expectedServiceAccounts { - if authInfo.PeerServiceAccount() == sa { + if strings.EqualFold(peer, sa) { return nil } } - return status.Newf(codes.PermissionDenied, "Client %v is not authorized", authInfo.PeerServiceAccount()).Err() + return status.Errorf(codes.PermissionDenied, "Client %v is not authorized", peer) } diff --git a/credentials/alts/utils_test.go b/credentials/alts/utils_test.go index f0ab2377c953..531cdfce6e3a 100644 --- a/credentials/alts/utils_test.go +++ b/credentials/alts/utils_test.go @@ -1,3 +1,6 @@ +//go:build linux || windows +// +build linux windows + /* * * Copyright 2018 gRPC authors. @@ -20,10 +23,9 @@ package alts import ( "context" - "io" - "os" "strings" "testing" + "time" "google.golang.org/grpc/codes" altspb "google.golang.org/grpc/credentials/alts/internal/proto/grpc_gcp" @@ -35,71 +37,13 @@ const ( testServiceAccount1 = "service_account1" testServiceAccount2 = "service_account2" testServiceAccount3 = "service_account3" -) - -func setupManufacturerReader(testOS string, reader func() (io.Reader, error)) func() { - tmpOS := runningOS - tmpReader := manufacturerReader - - // Set test OS and reader function. - runningOS = testOS - manufacturerReader = reader - return func() { - runningOS = tmpOS - manufacturerReader = tmpReader - } - -} - -func setup(testOS string, testReader io.Reader) func() { - reader := func() (io.Reader, error) { - return testReader, nil - } - return setupManufacturerReader(testOS, reader) -} - -func setupError(testOS string, err error) func() { - reader := func() (io.Reader, error) { - return nil, err - } - return setupManufacturerReader(testOS, reader) -} - -func (s) TestIsRunningOnGCP(t *testing.T) { - for _, tc := range []struct { - description string - testOS string - testReader io.Reader - out bool - }{ - // Linux tests. - {"linux: not a GCP platform", "linux", strings.NewReader("not GCP"), false}, - {"Linux: GCP platform (Google)", "linux", strings.NewReader("Google"), true}, - {"Linux: GCP platform (Google Compute Engine)", "linux", strings.NewReader("Google Compute Engine"), true}, - {"Linux: GCP platform (Google Compute Engine) with extra spaces", "linux", strings.NewReader(" Google Compute Engine "), true}, - // Windows tests. - {"windows: not a GCP platform", "windows", strings.NewReader("not GCP"), false}, - {"windows: GCP platform (Google)", "windows", strings.NewReader("Google"), true}, - {"windows: GCP platform (Google) with extra spaces", "windows", strings.NewReader(" Google "), true}, - } { - reverseFunc := setup(tc.testOS, tc.testReader) - if got, want := isRunningOnGCP(), tc.out; got != want { - t.Errorf("%v: isRunningOnGCP()=%v, want %v", tc.description, got, want) - } - reverseFunc() - } -} -func (s) TestIsRunningOnGCPNoProductNameFile(t *testing.T) { - reverseFunc := setupError("linux", os.ErrNotExist) - if isRunningOnGCP() { - t.Errorf("ErrNotExist: isRunningOnGCP()=true, want false") - } - reverseFunc() -} + defaultTestTimeout = 10 * time.Second +) func (s) TestAuthInfoFromContext(t *testing.T) { - ctx := context.Background() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() altsAuthInfo := &fakeALTSAuthInfo{} p := &peer.Peer{ AuthInfo: altsAuthInfo, @@ -156,7 +100,8 @@ func (s) TestAuthInfoFromPeer(t *testing.T) { } func (s) TestClientAuthorizationCheck(t *testing.T) { - ctx := context.Background() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() altsAuthInfo := &fakeALTSAuthInfo{testServiceAccount1} p := &peer.Peer{ AuthInfo: altsAuthInfo, @@ -175,6 +120,13 @@ func (s) TestClientAuthorizationCheck(t *testing.T) { true, codes.OK, // err is nil, code is OK. }, + { + "working case (case ignored)", + peer.NewContext(ctx, p), + []string{strings.ToUpper(testServiceAccount1), testServiceAccount2}, + true, + codes.OK, // err is nil, code is OK. + }, { "context does not have AuthInfo", ctx, diff --git a/credentials/credentials.go b/credentials/credentials.go index 02766443ae74..5feac3aa0e41 100644 --- a/credentials/credentials.go +++ b/credentials/credentials.go @@ -30,22 +30,22 @@ import ( "github.com/golang/protobuf/proto" "google.golang.org/grpc/attributes" - "google.golang.org/grpc/internal" + icredentials "google.golang.org/grpc/internal/credentials" ) // PerRPCCredentials defines the common interface for the credentials which need to // attach security information to every RPC (e.g., oauth2). type PerRPCCredentials interface { - // GetRequestMetadata gets the current request metadata, refreshing - // tokens if required. This should be called by the transport layer on - // each request, and the data should be populated in headers or other - // context. If a status code is returned, it will be used as the status - // for the RPC. uri is the URI of the entry point for the request. - // When supported by the underlying implementation, ctx can be used for - // timeout and cancellation. Additionally, RequestInfo data will be - // available via ctx to this call. - // TODO(zhaoq): Define the set of the qualified keys instead of leaving - // it as an arbitrary string. + // GetRequestMetadata gets the current request metadata, refreshing tokens + // if required. This should be called by the transport layer on each + // request, and the data should be populated in headers or other + // context. If a status code is returned, it will be used as the status for + // the RPC (restricted to an allowable set of codes as defined by gRFC + // A54). uri is the URI of the entry point for the request. When supported + // by the underlying implementation, ctx can be used for timeout and + // cancellation. Additionally, RequestInfo data will be available via ctx + // to this call. TODO(zhaoq): Define the set of the qualified keys instead + // of leaving it as an arbitrary string. GetRequestMetadata(ctx context.Context, uri ...string) (map[string]string, error) // RequireTransportSecurity indicates whether the credentials requires // transport security. @@ -58,9 +58,9 @@ type PerRPCCredentials interface { type SecurityLevel int const ( - // Invalid indicates an invalid security level. + // InvalidSecurityLevel indicates an invalid security level. // The zero SecurityLevel value is invalid for backward compatibility. - Invalid SecurityLevel = iota + InvalidSecurityLevel SecurityLevel = iota // NoSecurity indicates a connection is insecure. NoSecurity // IntegrityOnly indicates a connection only provides integrity protection. @@ -92,7 +92,7 @@ type CommonAuthInfo struct { } // GetCommonAuthInfo returns the pointer to CommonAuthInfo struct. -func (c *CommonAuthInfo) GetCommonAuthInfo() *CommonAuthInfo { +func (c CommonAuthInfo) GetCommonAuthInfo() CommonAuthInfo { return c } @@ -140,6 +140,11 @@ type TransportCredentials interface { // Additionally, ClientHandshakeInfo data will be available via the context // passed to this call. // + // The second argument to this method is the `:authority` header value used + // while creating new streams on this connection after authentication + // succeeds. Implementations must use this as the server name during the + // authentication handshake. + // // If the returned net.Conn is closed, it MUST close the net.Conn provided. ClientHandshake(context.Context, string, net.Conn) (net.Conn, AuthInfo, error) // ServerHandshake does the authentication handshake for servers. It returns @@ -153,9 +158,13 @@ type TransportCredentials interface { Info() ProtocolInfo // Clone makes a copy of this TransportCredentials. Clone() TransportCredentials - // OverrideServerName overrides the server name used to verify the hostname on the returned certificates from the server. - // gRPC internals also use it to override the virtual hosting name if it is set. - // It must be called before dialing. Currently, this is only used by grpclb. + // OverrideServerName specifies the value used for the following: + // - verifying the hostname on the returned certificates + // - as SNI in the client's handshake to support virtual hosting + // - as the value for `:authority` header at stream creation time + // + // Deprecated: use grpc.WithAuthority instead. Will be supported + // throughout 1.x. OverrideServerName(string) error } @@ -169,8 +178,18 @@ type TransportCredentials interface { // // This API is experimental. type Bundle interface { + // TransportCredentials returns the transport credentials from the Bundle. + // + // Implementations must return non-nil transport credentials. If transport + // security is not needed by the Bundle, implementations may choose to + // return insecure.NewCredentials(). TransportCredentials() TransportCredentials + + // PerRPCCredentials returns the per-RPC credentials from the Bundle. + // + // May be nil if per-RPC credentials are not needed. PerRPCCredentials() PerRPCCredentials + // NewWithMode should make a copy of Bundle, and switch mode. Modifying the // existing Bundle may cause races. // @@ -188,15 +207,12 @@ type RequestInfo struct { AuthInfo AuthInfo } -// requestInfoKey is a struct to be used as the key when attaching a RequestInfo to a context object. -type requestInfoKey struct{} - // RequestInfoFromContext extracts the RequestInfo from the context if it exists. // // This API is experimental. func RequestInfoFromContext(ctx context.Context) (ri RequestInfo, ok bool) { - ri, ok = ctx.Value(requestInfoKey{}).(RequestInfo) - return + ri, ok = icredentials.RequestInfoFromContext(ctx).(RequestInfo) + return ri, ok } // ClientHandshakeInfo holds data to be passed to ClientHandshake. This makes @@ -211,16 +227,12 @@ type ClientHandshakeInfo struct { Attributes *attributes.Attributes } -// clientHandshakeInfoKey is a struct used as the key to store -// ClientHandshakeInfo in a context. -type clientHandshakeInfoKey struct{} - // ClientHandshakeInfoFromContext returns the ClientHandshakeInfo struct stored // in ctx. // // This API is experimental. func ClientHandshakeInfoFromContext(ctx context.Context) ClientHandshakeInfo { - chi, _ := ctx.Value(clientHandshakeInfoKey{}).(ClientHandshakeInfo) + chi, _ := icredentials.ClientHandshakeInfoFromContext(ctx).(ClientHandshakeInfo) return chi } @@ -229,17 +241,16 @@ func ClientHandshakeInfoFromContext(ctx context.Context) ClientHandshakeInfo { // or 3) CommonAuthInfo.SecurityLevel has an invalid zero value. For 2) and 3), it is for the purpose of backward-compatibility. // // This API is experimental. -func CheckSecurityLevel(ctx context.Context, level SecurityLevel) error { +func CheckSecurityLevel(ai AuthInfo, level SecurityLevel) error { type internalInfo interface { - GetCommonAuthInfo() *CommonAuthInfo + GetCommonAuthInfo() CommonAuthInfo } - ri, _ := RequestInfoFromContext(ctx) - if ri.AuthInfo == nil { - return errors.New("unable to obtain SecurityLevel from context") + if ai == nil { + return errors.New("AuthInfo is nil") } - if ci, ok := ri.AuthInfo.(internalInfo); ok { + if ci, ok := ai.(internalInfo); ok { // CommonAuthInfo.SecurityLevel has an invalid value. - if ci.GetCommonAuthInfo().SecurityLevel == Invalid { + if ci.GetCommonAuthInfo().SecurityLevel == InvalidSecurityLevel { return nil } if ci.GetCommonAuthInfo().SecurityLevel < level { @@ -250,15 +261,6 @@ func CheckSecurityLevel(ctx context.Context, level SecurityLevel) error { return nil } -func init() { - internal.NewRequestInfoContext = func(ctx context.Context, ri RequestInfo) context.Context { - return context.WithValue(ctx, requestInfoKey{}, ri) - } - internal.NewClientHandshakeInfoContext = func(ctx context.Context, chi ClientHandshakeInfo) context.Context { - return context.WithValue(ctx, clientHandshakeInfoKey{}, chi) - } -} - // ChannelzSecurityInfo defines the interface that security protocols should implement // in order to provide security info to channelz. // diff --git a/credentials/credentials_test.go b/credentials/credentials_test.go index ea0cf5819ff8..08eb9c430ffc 100644 --- a/credentials/credentials_test.go +++ b/credentials/credentials_test.go @@ -24,12 +24,14 @@ import ( "net" "strings" "testing" + "time" - "google.golang.org/grpc/internal" "google.golang.org/grpc/internal/grpctest" "google.golang.org/grpc/testdata" ) +const defaultTestTimeout = 10 * time.Second + type s struct { grpctest.Tester } @@ -54,15 +56,6 @@ func (ta testAuthInfo) AuthType() string { return "testAuthInfo" } -func createTestContext(s SecurityLevel) context.Context { - auth := &testAuthInfo{CommonAuthInfo: CommonAuthInfo{SecurityLevel: s}} - ri := RequestInfo{ - Method: "testInfo", - AuthInfo: auth, - } - return internal.NewRequestInfoContext.(func(context.Context, RequestInfo) context.Context)(context.Background(), ri) -} - func (s) TestCheckSecurityLevel(t *testing.T) { testCases := []struct { authLevel SecurityLevel @@ -85,18 +78,18 @@ func (s) TestCheckSecurityLevel(t *testing.T) { want: true, }, { - authLevel: Invalid, + authLevel: InvalidSecurityLevel, testLevel: IntegrityOnly, want: true, }, { - authLevel: Invalid, + authLevel: InvalidSecurityLevel, testLevel: PrivacyAndIntegrity, want: true, }, } for _, tc := range testCases { - err := CheckSecurityLevel(createTestContext(tc.authLevel), tc.testLevel) + err := CheckSecurityLevel(testAuthInfo{CommonAuthInfo: CommonAuthInfo{SecurityLevel: tc.authLevel}}, tc.testLevel) if tc.want && (err != nil) { t.Fatalf("CheckSeurityLevel(%s, %s) returned failure but want success", tc.authLevel.String(), tc.testLevel.String()) } else if !tc.want && (err == nil) { @@ -107,13 +100,7 @@ func (s) TestCheckSecurityLevel(t *testing.T) { } func (s) TestCheckSecurityLevelNoGetCommonAuthInfoMethod(t *testing.T) { - auth := &testAuthInfoNoGetCommonAuthInfoMethod{} - ri := RequestInfo{ - Method: "testInfo", - AuthInfo: auth, - } - ctxWithRequestInfo := internal.NewRequestInfoContext.(func(context.Context, RequestInfo) context.Context)(context.Background(), ri) - if err := CheckSecurityLevel(ctxWithRequestInfo, PrivacyAndIntegrity); err != nil { + if err := CheckSecurityLevel(testAuthInfoNoGetCommonAuthInfoMethod{}, PrivacyAndIntegrity); err != nil { t.Fatalf("CheckSeurityLevel() returned failure but want success") } } @@ -296,7 +283,9 @@ func gRPCServerHandshake(conn net.Conn) (AuthInfo, error) { // Client handshake implementation in gRPC. func gRPCClientHandshake(conn net.Conn, lisAddr string) (AuthInfo, error) { clientTLS := NewTLS(&tls.Config{InsecureSkipVerify: true}) - _, authInfo, err := clientTLS.ClientHandshake(context.Background(), lisAddr, conn) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + _, authInfo, err := clientTLS.ClientHandshake(ctx, lisAddr, conn) if err != nil { return nil, err } diff --git a/credentials/google/google.go b/credentials/google/google.go index 7f3e240e475b..fbdf7dc2997a 100644 --- a/credentials/google/google.go +++ b/credentials/google/google.go @@ -35,57 +35,63 @@ const tokenRequestTimeout = 30 * time.Second var logger = grpclog.Component("credentials") -// NewDefaultCredentials returns a credentials bundle that is configured to work -// with google services. +// DefaultCredentialsOptions constructs options to build DefaultCredentials. +type DefaultCredentialsOptions struct { + // PerRPCCreds is a per RPC credentials that is passed to a bundle. + PerRPCCreds credentials.PerRPCCredentials +} + +// NewDefaultCredentialsWithOptions returns a credentials bundle that is +// configured to work with google services. // // This API is experimental. -func NewDefaultCredentials() credentials.Bundle { - c := &creds{ - newPerRPCCreds: func() credentials.PerRPCCredentials { - ctx, cancel := context.WithTimeout(context.Background(), tokenRequestTimeout) - defer cancel() - perRPCCreds, err := oauth.NewApplicationDefault(ctx) - if err != nil { - logger.Warningf("google default creds: failed to create application oauth: %v", err) - } - return perRPCCreds - }, +func NewDefaultCredentialsWithOptions(opts DefaultCredentialsOptions) credentials.Bundle { + if opts.PerRPCCreds == nil { + ctx, cancel := context.WithTimeout(context.Background(), tokenRequestTimeout) + defer cancel() + var err error + opts.PerRPCCreds, err = newADC(ctx) + if err != nil { + logger.Warningf("NewDefaultCredentialsWithOptions: failed to create application oauth: %v", err) + } } + c := &creds{opts: opts} bundle, err := c.NewWithMode(internal.CredsBundleModeFallback) if err != nil { - logger.Warningf("google default creds: failed to create new creds: %v", err) + logger.Warningf("NewDefaultCredentialsWithOptions: failed to create new creds: %v", err) } return bundle } +// NewDefaultCredentials returns a credentials bundle that is configured to work +// with google services. +// +// This API is experimental. +func NewDefaultCredentials() credentials.Bundle { + return NewDefaultCredentialsWithOptions(DefaultCredentialsOptions{}) +} + // NewComputeEngineCredentials returns a credentials bundle that is configured to work // with google services. This API must only be used when running on GCE. Authentication configured // by this API represents the GCE VM's default service account. // // This API is experimental. func NewComputeEngineCredentials() credentials.Bundle { - c := &creds{ - newPerRPCCreds: func() credentials.PerRPCCredentials { - return oauth.NewComputeEngine() - }, - } - bundle, err := c.NewWithMode(internal.CredsBundleModeFallback) - if err != nil { - logger.Warningf("compute engine creds: failed to create new creds: %v", err) - } - return bundle + return NewDefaultCredentialsWithOptions(DefaultCredentialsOptions{ + PerRPCCreds: oauth.NewComputeEngine(), + }) } // creds implements credentials.Bundle. type creds struct { + opts DefaultCredentialsOptions + // Supported modes are defined in internal/internal.go. mode string - // The transport credentials associated with this bundle. + // The active transport credentials associated with this bundle. transportCreds credentials.TransportCredentials - // The per RPC credentials associated with this bundle. + // The active per RPC credentials associated with this bundle. perRPCCreds credentials.PerRPCCredentials - // Creates new per RPC credentials - newPerRPCCreds func() credentials.PerRPCCredentials } func (c *creds) TransportCredentials() credentials.TransportCredentials { @@ -99,28 +105,40 @@ func (c *creds) PerRPCCredentials() credentials.PerRPCCredentials { return c.perRPCCreds } +var ( + newTLS = func() credentials.TransportCredentials { + return credentials.NewTLS(nil) + } + newALTS = func() credentials.TransportCredentials { + return alts.NewClientCreds(alts.DefaultClientOptions()) + } + newADC = func(ctx context.Context) (credentials.PerRPCCredentials, error) { + return oauth.NewApplicationDefault(ctx) + } +) + // NewWithMode should make a copy of Bundle, and switch mode. Modifying the // existing Bundle may cause races. func (c *creds) NewWithMode(mode string) (credentials.Bundle, error) { newCreds := &creds{ - mode: mode, - newPerRPCCreds: c.newPerRPCCreds, + opts: c.opts, + mode: mode, } // Create transport credentials. switch mode { case internal.CredsBundleModeFallback: - newCreds.transportCreds = credentials.NewTLS(nil) + newCreds.transportCreds = newClusterTransportCreds(newTLS(), newALTS()) case internal.CredsBundleModeBackendFromBalancer, internal.CredsBundleModeBalancer: // Only the clients can use google default credentials, so we only need // to create new ALTS client creds here. - newCreds.transportCreds = alts.NewClientCreds(alts.DefaultClientOptions()) + newCreds.transportCreds = newALTS() default: return nil, fmt.Errorf("unsupported mode: %v", mode) } if mode == internal.CredsBundleModeFallback || mode == internal.CredsBundleModeBackendFromBalancer { - newCreds.perRPCCreds = newCreds.newPerRPCCreds() + newCreds.perRPCCreds = newCreds.opts.PerRPCCreds } return newCreds, nil diff --git a/credentials/google/google_test.go b/credentials/google/google_test.go new file mode 100644 index 000000000000..1809d545d0ec --- /dev/null +++ b/credentials/google/google_test.go @@ -0,0 +1,163 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package google + +import ( + "context" + "net" + "testing" + + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/internal" + icredentials "google.golang.org/grpc/internal/credentials" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/resolver" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +type testCreds struct { + credentials.TransportCredentials + typ string +} + +func (c *testCreds) ClientHandshake(ctx context.Context, authority string, rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) { + return nil, &testAuthInfo{typ: c.typ}, nil +} + +func (c *testCreds) ServerHandshake(conn net.Conn) (net.Conn, credentials.AuthInfo, error) { + return nil, &testAuthInfo{typ: c.typ}, nil +} + +type testAuthInfo struct { + typ string +} + +func (t *testAuthInfo) AuthType() string { + return t.typ +} + +var ( + testTLS = &testCreds{typ: "tls"} + testALTS = &testCreds{typ: "alts"} +) + +func overrideNewCredsFuncs() func() { + origNewTLS := newTLS + newTLS = func() credentials.TransportCredentials { + return testTLS + } + origNewALTS := newALTS + newALTS = func() credentials.TransportCredentials { + return testALTS + } + origNewADC := newADC + newADC = func(context.Context) (credentials.PerRPCCredentials, error) { + // We do not use perRPC creds in this test. It is safe to return nil here. + return nil, nil + } + + return func() { + newTLS = origNewTLS + newALTS = origNewALTS + newADC = origNewADC + } +} + +// TestClientHandshakeBasedOnClusterName that by default (without switching +// modes), ClientHandshake does either tls or alts base on the cluster name in +// attributes. +func (s) TestClientHandshakeBasedOnClusterName(t *testing.T) { + defer overrideNewCredsFuncs()() + for bundleTyp, tc := range map[string]credentials.Bundle{ + "defaultCredsWithOptions": NewDefaultCredentialsWithOptions(DefaultCredentialsOptions{}), + "defaultCreds": NewDefaultCredentials(), + "computeCreds": NewComputeEngineCredentials(), + } { + tests := []struct { + name string + ctx context.Context + wantTyp string + }{ + { + name: "no cluster name", + ctx: context.Background(), + wantTyp: "tls", + }, + { + name: "with non-CFE cluster name", + ctx: icredentials.NewClientHandshakeInfoContext(context.Background(), credentials.ClientHandshakeInfo{ + Attributes: internal.SetXDSHandshakeClusterName(resolver.Address{}, "lalala").Attributes, + }), + // non-CFE backends should use alts. + wantTyp: "alts", + }, + { + name: "with CFE cluster name", + ctx: icredentials.NewClientHandshakeInfoContext(context.Background(), credentials.ClientHandshakeInfo{ + Attributes: internal.SetXDSHandshakeClusterName(resolver.Address{}, "google_cfe_bigtable.googleapis.com").Attributes, + }), + // CFE should use tls. + wantTyp: "tls", + }, + { + name: "with xdstp CFE cluster name", + ctx: icredentials.NewClientHandshakeInfoContext(context.Background(), credentials.ClientHandshakeInfo{ + Attributes: internal.SetXDSHandshakeClusterName(resolver.Address{}, "xdstp://traffic-director-c2p.xds.googleapis.com/envoy.config.cluster.v3.Cluster/google_cfe_bigtable.googleapis.com").Attributes, + }), + // CFE should use tls. + wantTyp: "tls", + }, + { + name: "with xdstp non-CFE cluster name", + ctx: icredentials.NewClientHandshakeInfoContext(context.Background(), credentials.ClientHandshakeInfo{ + Attributes: internal.SetXDSHandshakeClusterName(resolver.Address{}, "xdstp://other.com/envoy.config.cluster.v3.Cluster/google_cfe_bigtable.googleapis.com").Attributes, + }), + // non-CFE should use atls. + wantTyp: "alts", + }, + } + for _, tt := range tests { + t.Run(bundleTyp+" "+tt.name, func(t *testing.T) { + _, info, err := tc.TransportCredentials().ClientHandshake(tt.ctx, "", nil) + if err != nil { + t.Fatalf("ClientHandshake failed: %v", err) + } + if gotType := info.AuthType(); gotType != tt.wantTyp { + t.Fatalf("unexpected authtype: %v, want: %v", gotType, tt.wantTyp) + } + + _, infoServer, err := tc.TransportCredentials().ServerHandshake(nil) + if err != nil { + t.Fatalf("ClientHandshake failed: %v", err) + } + // ServerHandshake should always do TLS. + if gotType := infoServer.AuthType(); gotType != "tls" { + t.Fatalf("unexpected server authtype: %v, want: %v", gotType, "tls") + } + }) + } + } +} diff --git a/credentials/google/xds.go b/credentials/google/xds.go new file mode 100644 index 000000000000..2c5c8b9eee13 --- /dev/null +++ b/credentials/google/xds.go @@ -0,0 +1,128 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package google + +import ( + "context" + "net" + "net/url" + "strings" + + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/internal" +) + +const cfeClusterNamePrefix = "google_cfe_" +const cfeClusterResourceNamePrefix = "/envoy.config.cluster.v3.Cluster/google_cfe_" +const cfeClusterAuthorityName = "traffic-director-c2p.xds.googleapis.com" + +// clusterTransportCreds is a combo of TLS + ALTS. +// +// On the client, ClientHandshake picks TLS or ALTS based on address attributes. +// - if attributes has cluster name +// - if cluster name has prefix "google_cfe_", or +// "xdstp://traffic-director-c2p.xds.googleapis.com/envoy.config.cluster.v3.Cluster/google_cfe_", +// use TLS +// - otherwise, use ALTS +// +// - else, do TLS +// +// On the server, ServerHandshake always does TLS. +type clusterTransportCreds struct { + tls credentials.TransportCredentials + alts credentials.TransportCredentials +} + +func newClusterTransportCreds(tls, alts credentials.TransportCredentials) *clusterTransportCreds { + return &clusterTransportCreds{ + tls: tls, + alts: alts, + } +} + +// clusterName returns the xDS cluster name stored in the attributes in the +// context. +func clusterName(ctx context.Context) string { + chi := credentials.ClientHandshakeInfoFromContext(ctx) + if chi.Attributes == nil { + return "" + } + cluster, _ := internal.GetXDSHandshakeClusterName(chi.Attributes) + return cluster +} + +// isDirectPathCluster returns true if the cluster in the context is a +// directpath cluster, meaning ALTS should be used. +func isDirectPathCluster(ctx context.Context) bool { + cluster := clusterName(ctx) + if cluster == "" { + // No cluster; not xDS; use TLS. + return false + } + if strings.HasPrefix(cluster, cfeClusterNamePrefix) { + // xDS cluster prefixed by "google_cfe_"; use TLS. + return false + } + if !strings.HasPrefix(cluster, "xdstp:") { + // Other xDS cluster name; use ALTS. + return true + } + u, err := url.Parse(cluster) + if err != nil { + // Shouldn't happen, but assume ALTS. + return true + } + // If authority AND path match our CFE checks, use TLS; otherwise use ALTS. + return u.Host != cfeClusterAuthorityName || !strings.HasPrefix(u.Path, cfeClusterResourceNamePrefix) +} + +func (c *clusterTransportCreds) ClientHandshake(ctx context.Context, authority string, rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) { + if isDirectPathCluster(ctx) { + // If attributes have cluster name, and cluster name is not cfe, it's a + // backend address, use ALTS. + return c.alts.ClientHandshake(ctx, authority, rawConn) + } + return c.tls.ClientHandshake(ctx, authority, rawConn) +} + +func (c *clusterTransportCreds) ServerHandshake(conn net.Conn) (net.Conn, credentials.AuthInfo, error) { + return c.tls.ServerHandshake(conn) +} + +func (c *clusterTransportCreds) Info() credentials.ProtocolInfo { + // TODO: this always returns tls.Info now, because we don't have a cluster + // name to check when this method is called. This method doesn't affect + // anything important now. We may want to revisit this if it becomes more + // important later. + return c.tls.Info() +} + +func (c *clusterTransportCreds) Clone() credentials.TransportCredentials { + return &clusterTransportCreds{ + tls: c.tls.Clone(), + alts: c.alts.Clone(), + } +} + +func (c *clusterTransportCreds) OverrideServerName(s string) error { + if err := c.tls.OverrideServerName(s); err != nil { + return err + } + return c.alts.OverrideServerName(s) +} diff --git a/credentials/google/xds_test.go b/credentials/google/xds_test.go new file mode 100644 index 000000000000..8aeba396a518 --- /dev/null +++ b/credentials/google/xds_test.go @@ -0,0 +1,58 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package google + +import ( + "context" + "testing" + + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/internal" + icredentials "google.golang.org/grpc/internal/credentials" + "google.golang.org/grpc/resolver" +) + +func (s) TestIsDirectPathCluster(t *testing.T) { + c := func(cluster string) context.Context { + return icredentials.NewClientHandshakeInfoContext(context.Background(), credentials.ClientHandshakeInfo{ + Attributes: internal.SetXDSHandshakeClusterName(resolver.Address{}, cluster).Attributes, + }) + } + + testCases := []struct { + name string + ctx context.Context + want bool + }{ + {"not an xDS cluster", context.Background(), false}, + {"cfe", c("google_cfe_bigtable.googleapis.com"), false}, + {"non-cfe", c("google_bigtable.googleapis.com"), true}, + {"starts with xdstp but not cfe format", c("xdstp:google_cfe_bigtable.googleapis.com"), true}, + {"no authority", c("xdstp:///envoy.config.cluster.v3.Cluster/google_cfe_"), true}, + {"wrong authority", c("xdstp://foo.bar/envoy.config.cluster.v3.Cluster/google_cfe_"), true}, + {"xdstp CFE", c("xdstp://traffic-director-c2p.xds.googleapis.com/envoy.config.cluster.v3.Cluster/google_cfe_"), false}, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + if got := isDirectPathCluster(tc.ctx); got != tc.want { + t.Errorf("isDirectPathCluster(_) = %v; want %v", got, tc.want) + } + }) + } +} diff --git a/credentials/insecure/insecure.go b/credentials/insecure/insecure.go new file mode 100644 index 000000000000..82bee1443bfe --- /dev/null +++ b/credentials/insecure/insecure.go @@ -0,0 +1,98 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package insecure provides an implementation of the +// credentials.TransportCredentials interface which disables transport security. +package insecure + +import ( + "context" + "net" + + "google.golang.org/grpc/credentials" +) + +// NewCredentials returns a credentials which disables transport security. +// +// Note that using this credentials with per-RPC credentials which require +// transport security is incompatible and will cause grpc.Dial() to fail. +func NewCredentials() credentials.TransportCredentials { + return insecureTC{} +} + +// insecureTC implements the insecure transport credentials. The handshake +// methods simply return the passed in net.Conn and set the security level to +// NoSecurity. +type insecureTC struct{} + +func (insecureTC) ClientHandshake(ctx context.Context, _ string, conn net.Conn) (net.Conn, credentials.AuthInfo, error) { + return conn, info{credentials.CommonAuthInfo{SecurityLevel: credentials.NoSecurity}}, nil +} + +func (insecureTC) ServerHandshake(conn net.Conn) (net.Conn, credentials.AuthInfo, error) { + return conn, info{credentials.CommonAuthInfo{SecurityLevel: credentials.NoSecurity}}, nil +} + +func (insecureTC) Info() credentials.ProtocolInfo { + return credentials.ProtocolInfo{SecurityProtocol: "insecure"} +} + +func (insecureTC) Clone() credentials.TransportCredentials { + return insecureTC{} +} + +func (insecureTC) OverrideServerName(string) error { + return nil +} + +// info contains the auth information for an insecure connection. +// It implements the AuthInfo interface. +type info struct { + credentials.CommonAuthInfo +} + +// AuthType returns the type of info as a string. +func (info) AuthType() string { + return "insecure" +} + +// insecureBundle implements an insecure bundle. +// An insecure bundle provides a thin wrapper around insecureTC to support +// the credentials.Bundle interface. +type insecureBundle struct{} + +// NewBundle returns a bundle with disabled transport security and no per rpc credential. +func NewBundle() credentials.Bundle { + return insecureBundle{} +} + +// NewWithMode returns a new insecure Bundle. The mode is ignored. +func (insecureBundle) NewWithMode(string) (credentials.Bundle, error) { + return insecureBundle{}, nil +} + +// PerRPCCredentials returns an nil implementation as insecure +// bundle does not support a per rpc credential. +func (insecureBundle) PerRPCCredentials() credentials.PerRPCCredentials { + return nil +} + +// TransportCredentials returns the underlying insecure transport credential. +func (insecureBundle) TransportCredentials() credentials.TransportCredentials { + return NewCredentials() +} diff --git a/credentials/local/local.go b/credentials/local/local.go index 23de34cf8a18..d5a3a8596056 100644 --- a/credentials/local/local.go +++ b/credentials/local/local.go @@ -23,7 +23,10 @@ // reported. If local credentials is not used in local connections // (local TCP or UDS), it will fail. // -// This package is EXPERIMENTAL. +// # Experimental +// +// Notice: This package is EXPERIMENTAL and may be changed or removed in a +// later release. package local import ( @@ -35,14 +38,14 @@ import ( "google.golang.org/grpc/credentials" ) -// Info contains the auth information for a local connection. +// info contains the auth information for a local connection. // It implements the AuthInfo interface. -type Info struct { +type info struct { credentials.CommonAuthInfo } -// AuthType returns the type of Info as a string. -func (Info) AuthType() string { +// AuthType returns the type of info as a string. +func (info) AuthType() string { return "local" } @@ -67,7 +70,7 @@ func getSecurityLevel(network, addr string) (credentials.SecurityLevel, error) { return credentials.PrivacyAndIntegrity, nil // Not a local connection and should fail default: - return credentials.Invalid, fmt.Errorf("local credentials rejected connection to non-local address %q", addr) + return credentials.InvalidSecurityLevel, fmt.Errorf("local credentials rejected connection to non-local address %q", addr) } } @@ -76,7 +79,7 @@ func (*localTC) ClientHandshake(ctx context.Context, authority string, conn net. if err != nil { return nil, nil, err } - return conn, Info{credentials.CommonAuthInfo{SecurityLevel: secLevel}}, nil + return conn, info{credentials.CommonAuthInfo{SecurityLevel: secLevel}}, nil } func (*localTC) ServerHandshake(conn net.Conn) (net.Conn, credentials.AuthInfo, error) { @@ -84,7 +87,7 @@ func (*localTC) ServerHandshake(conn net.Conn) (net.Conn, credentials.AuthInfo, if err != nil { return nil, nil, err } - return conn, Info{credentials.CommonAuthInfo{SecurityLevel: secLevel}}, nil + return conn, info{credentials.CommonAuthInfo{SecurityLevel: secLevel}}, nil } // NewCredentials returns a local credential implementing credentials.TransportCredentials. diff --git a/credentials/local/local_test.go b/credentials/local/local_test.go index 3c65010e8b2a..47f8dbb4ec85 100644 --- a/credentials/local/local_test.go +++ b/credentials/local/local_test.go @@ -31,6 +31,8 @@ import ( "google.golang.org/grpc/internal/grpctest" ) +const defaultTestTimeout = 10 * time.Second + type s struct { grpctest.Tester } @@ -63,7 +65,7 @@ func (s) TestGetSecurityLevel(t *testing.T) { { testNetwork: "tcp", testAddr: "192.168.0.1:10000", - want: credentials.Invalid, + want: credentials.InvalidSecurityLevel, }, } for _, tc := range testCases { @@ -76,6 +78,15 @@ func (s) TestGetSecurityLevel(t *testing.T) { type serverHandshake func(net.Conn) (credentials.AuthInfo, error) +func getSecurityLevelFromAuthInfo(ai credentials.AuthInfo) credentials.SecurityLevel { + if c, ok := ai.(interface { + GetCommonAuthInfo() credentials.CommonAuthInfo + }); ok { + return c.GetCommonAuthInfo().SecurityLevel + } + return credentials.InvalidSecurityLevel +} + // Server local handshake implementation. func serverLocalHandshake(conn net.Conn) (credentials.AuthInfo, error) { cred := NewCredentials() @@ -89,7 +100,10 @@ func serverLocalHandshake(conn net.Conn) (credentials.AuthInfo, error) { // Client local handshake implementation. func clientLocalHandshake(conn net.Conn, lisAddr string) (credentials.AuthInfo, error) { cred := NewCredentials() - _, authInfo, err := cred.ClientHandshake(context.Background(), lisAddr, conn) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + _, authInfo, err := cred.ClientHandshake(ctx, lisAddr, conn) if err != nil { return nil, err } @@ -117,11 +131,13 @@ func serverHandle(hs serverHandshake, done chan testServerHandleResult, lis net. serverRawConn, err := lis.Accept() if err != nil { done <- testServerHandleResult{authInfo: nil, err: fmt.Errorf("Server failed to accept connection. Error: %v", err)} + return } serverAuthInfo, err := hs(serverRawConn) if err != nil { serverRawConn.Close() done <- testServerHandleResult{authInfo: nil, err: fmt.Errorf("Server failed while handshake. Error: %v", err)} + return } done <- testServerHandleResult{authInfo: serverAuthInfo, err: nil} } @@ -135,21 +151,26 @@ func serverAndClientHandshake(lis net.Listener) (credentials.SecurityLevel, erro defer lis.Close() clientAuthInfo, err := clientHandle(clientLocalHandshake, lis.Addr().Network(), lis.Addr().String()) if err != nil { - return credentials.Invalid, fmt.Errorf("Error at client-side: %v", err) + return credentials.InvalidSecurityLevel, fmt.Errorf("Error at client-side: %v", err) } select { case <-timer.C: - return credentials.Invalid, fmt.Errorf("Test didn't finish in time") + return credentials.InvalidSecurityLevel, fmt.Errorf("Test didn't finish in time") case serverHandleResult := <-done: if serverHandleResult.err != nil { - return credentials.Invalid, fmt.Errorf("Error at server-side: %v", serverHandleResult.err) + return credentials.InvalidSecurityLevel, fmt.Errorf("Error at server-side: %v", serverHandleResult.err) + } + clientSecLevel := getSecurityLevelFromAuthInfo(clientAuthInfo) + serverSecLevel := getSecurityLevelFromAuthInfo(serverHandleResult.authInfo) + + if clientSecLevel == credentials.InvalidSecurityLevel { + return credentials.InvalidSecurityLevel, fmt.Errorf("Error at client-side: client's AuthInfo does not implement GetCommonAuthInfo()") + } + if serverSecLevel == credentials.InvalidSecurityLevel { + return credentials.InvalidSecurityLevel, fmt.Errorf("Error at server-side: server's AuthInfo does not implement GetCommonAuthInfo()") } - clientLocal, _ := clientAuthInfo.(Info) - serverLocal, _ := serverHandleResult.authInfo.(Info) - clientSecLevel := clientLocal.CommonAuthInfo.SecurityLevel - serverSecLevel := serverLocal.CommonAuthInfo.SecurityLevel if clientSecLevel != serverSecLevel { - return credentials.Invalid, fmt.Errorf("client's AuthInfo contains %s but server's AuthInfo contains %s", clientSecLevel.String(), serverSecLevel.String()) + return credentials.InvalidSecurityLevel, fmt.Errorf("client's AuthInfo contains %s but server's AuthInfo contains %s", clientSecLevel.String(), serverSecLevel.String()) } return clientSecLevel, nil } diff --git a/credentials/oauth/oauth.go b/credentials/oauth/oauth.go index 6657055d6609..d475cbc0894c 100644 --- a/credentials/oauth/oauth.go +++ b/credentials/oauth/oauth.go @@ -22,7 +22,8 @@ package oauth import ( "context" "fmt" - "io/ioutil" + "net/url" + "os" "sync" "golang.org/x/oauth2" @@ -42,7 +43,8 @@ func (ts TokenSource) GetRequestMetadata(ctx context.Context, uri ...string) (ma if err != nil { return nil, err } - if err = credentials.CheckSecurityLevel(ctx, credentials.PrivacyAndIntegrity); err != nil { + ri, _ := credentials.RequestInfoFromContext(ctx) + if err = credentials.CheckSecurityLevel(ri.AuthInfo, credentials.PrivacyAndIntegrity); err != nil { return nil, fmt.Errorf("unable to transfer TokenSource PerRPCCredentials: %v", err) } return map[string]string{ @@ -55,13 +57,23 @@ func (ts TokenSource) RequireTransportSecurity() bool { return true } +// removeServiceNameFromJWTURI removes RPC service name from URI. +func removeServiceNameFromJWTURI(uri string) (string, error) { + parsed, err := url.Parse(uri) + if err != nil { + return "", err + } + parsed.Path = "/" + return parsed.String(), nil +} + type jwtAccess struct { jsonKey []byte } // NewJWTAccessFromFile creates PerRPCCredentials from the given keyFile. func NewJWTAccessFromFile(keyFile string) (credentials.PerRPCCredentials, error) { - jsonKey, err := ioutil.ReadFile(keyFile) + jsonKey, err := os.ReadFile(keyFile) if err != nil { return nil, fmt.Errorf("credentials: failed to read the service account key file: %v", err) } @@ -74,9 +86,15 @@ func NewJWTAccessFromKey(jsonKey []byte) (credentials.PerRPCCredentials, error) } func (j jwtAccess) GetRequestMetadata(ctx context.Context, uri ...string) (map[string]string, error) { + // Remove RPC service name from URI that will be used as audience + // in a self-signed JWT token. It follows https://google.aip.dev/auth/4111. + aud, err := removeServiceNameFromJWTURI(uri[0]) + if err != nil { + return nil, err + } // TODO: the returned TokenSource is reusable. Store it in a sync.Map, with // uri as the key, to avoid recreating for every RPC. - ts, err := google.JWTAccessTokenSourceFromJSON(j.jsonKey, uri[0]) + ts, err := google.JWTAccessTokenSourceFromJSON(j.jsonKey, aud) if err != nil { return nil, err } @@ -84,7 +102,8 @@ func (j jwtAccess) GetRequestMetadata(ctx context.Context, uri ...string) (map[s if err != nil { return nil, err } - if err = credentials.CheckSecurityLevel(ctx, credentials.PrivacyAndIntegrity); err != nil { + ri, _ := credentials.RequestInfoFromContext(ctx) + if err = credentials.CheckSecurityLevel(ri.AuthInfo, credentials.PrivacyAndIntegrity); err != nil { return nil, fmt.Errorf("unable to transfer jwtAccess PerRPCCredentials: %v", err) } return map[string]string{ @@ -102,12 +121,15 @@ type oauthAccess struct { } // NewOauthAccess constructs the PerRPCCredentials using a given token. +// +// Deprecated: use oauth.TokenSource instead. func NewOauthAccess(token *oauth2.Token) credentials.PerRPCCredentials { return oauthAccess{token: *token} } func (oa oauthAccess) GetRequestMetadata(ctx context.Context, uri ...string) (map[string]string, error) { - if err := credentials.CheckSecurityLevel(ctx, credentials.PrivacyAndIntegrity); err != nil { + ri, _ := credentials.RequestInfoFromContext(ctx) + if err := credentials.CheckSecurityLevel(ri.AuthInfo, credentials.PrivacyAndIntegrity); err != nil { return nil, fmt.Errorf("unable to transfer oauthAccess PerRPCCredentials: %v", err) } return map[string]string{ @@ -144,7 +166,8 @@ func (s *serviceAccount) GetRequestMetadata(ctx context.Context, uri ...string) return nil, err } } - if err := credentials.CheckSecurityLevel(ctx, credentials.PrivacyAndIntegrity); err != nil { + ri, _ := credentials.RequestInfoFromContext(ctx) + if err := credentials.CheckSecurityLevel(ri.AuthInfo, credentials.PrivacyAndIntegrity); err != nil { return nil, fmt.Errorf("unable to transfer serviceAccount PerRPCCredentials: %v", err) } return map[string]string{ @@ -169,7 +192,7 @@ func NewServiceAccountFromKey(jsonKey []byte, scope ...string) (credentials.PerR // NewServiceAccountFromFile constructs the PerRPCCredentials using the JSON key file // of a Google Developers service account. func NewServiceAccountFromFile(keyFile string, scope ...string) (credentials.PerRPCCredentials, error) { - jsonKey, err := ioutil.ReadFile(keyFile) + jsonKey, err := os.ReadFile(keyFile) if err != nil { return nil, fmt.Errorf("credentials: failed to read the service account key file: %v", err) } diff --git a/credentials/oauth/oauth_test.go b/credentials/oauth/oauth_test.go new file mode 100644 index 000000000000..7e62ecb36c12 --- /dev/null +++ b/credentials/oauth/oauth_test.go @@ -0,0 +1,60 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package oauth + +import ( + "strings" + "testing" +) + +func checkErrorMsg(err error, msg string) bool { + if err == nil && msg == "" { + return true + } else if err != nil { + return strings.Contains(err.Error(), msg) + } + return false +} + +func TestRemoveServiceNameFromJWTURI(t *testing.T) { + tests := []struct { + name string + uri string + wantedURI string + wantedErrMsg string + }{ + { + name: "invalid URI", + uri: "ht tp://foo.com", + wantedErrMsg: "first path segment in URL cannot contain colon", + }, + { + name: "valid URI", + uri: "https://foo.com/go/", + wantedURI: "https://foo.com/", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got, err := removeServiceNameFromJWTURI(tt.uri); got != tt.wantedURI || !checkErrorMsg(err, tt.wantedErrMsg) { + t.Errorf("RemoveServiceNameFromJWTURI() = %s, %v, want %s, %v", got, err, tt.wantedURI, tt.wantedErrMsg) + } + }) + } +} diff --git a/credentials/sts/sts.go b/credentials/sts/sts.go index a97b20a9eb72..0110201a98f3 100644 --- a/credentials/sts/sts.go +++ b/credentials/sts/sts.go @@ -1,5 +1,3 @@ -// +build go1.13 - /* * * Copyright 2020 gRPC authors. @@ -21,7 +19,7 @@ // Package sts implements call credentials using STS (Security Token Service) as // defined in https://tools.ietf.org/html/rfc8693. // -// Experimental +// # Experimental // // Notice: All APIs in this package are experimental and may be changed or // removed in a later release. @@ -35,9 +33,10 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" + "io" "net/http" "net/url" + "os" "sync" "time" @@ -60,8 +59,8 @@ const ( var ( loadSystemCertPool = x509.SystemCertPool makeHTTPDoer = makeHTTPClient - readSubjectTokenFrom = ioutil.ReadFile - readActorTokenFrom = ioutil.ReadFile + readSubjectTokenFrom = os.ReadFile + readActorTokenFrom = os.ReadFile logger = grpclog.Component("credentials") ) @@ -107,10 +106,14 @@ type Options struct { // ActorTokenType is an identifier, as described in // https://tools.ietf.org/html/rfc8693#section-3, that indicates the type of - // the the security token in the "actor_token_path" parameter. + // the security token in the "actor_token_path" parameter. ActorTokenType string // Optional. } +func (o Options) String() string { + return fmt.Sprintf("%s:%s:%s:%s:%s:%s:%s:%s:%s", o.TokenExchangeServiceURI, o.Resource, o.Audience, o.Scope, o.RequestedTokenType, o.SubjectTokenPath, o.SubjectTokenType, o.ActorTokenPath, o.ActorTokenType) +} + // NewCredentials returns a new PerRPCCredentials implementation, configured // using opts, which performs token exchange using STS. func NewCredentials(opts Options) (credentials.PerRPCCredentials, error) { @@ -147,7 +150,8 @@ type callCreds struct { // GetRequestMetadata returns the cached accessToken, if available and valid, or // fetches a new one by performing an STS token exchange. func (c *callCreds) GetRequestMetadata(ctx context.Context, _ ...string) (map[string]string, error) { - if err := credentials.CheckSecurityLevel(ctx, credentials.PrivacyAndIntegrity); err != nil { + ri, _ := credentials.RequestInfoFromContext(ctx) + if err := credentials.CheckSecurityLevel(ri.AuthInfo, credentials.PrivacyAndIntegrity); err != nil { return nil, fmt.Errorf("unable to transfer STS PerRPCCredentials: %v", err) } @@ -213,7 +217,7 @@ func validateOptions(opts Options) error { return err } if u.Scheme != "http" && u.Scheme != "https" { - return fmt.Errorf("scheme is not supported: %s. Only http(s) is supported", u.Scheme) + return fmt.Errorf("scheme is not supported: %q. Only http(s) is supported", u.Scheme) } if opts.SubjectTokenPath == "" { @@ -242,12 +246,12 @@ func (c *callCreds) cachedMetadata() map[string]string { // constructRequest creates the STS request body in JSON based on the provided // options. -// - Contents of the subjectToken are read from the file specified in -// options. If we encounter an error here, we bail out. -// - Contents of the actorToken are read from the file specified in options. -// If we encounter an error here, we ignore this field because this is -// optional. -// - Most of the other fields in the request come directly from options. +// - Contents of the subjectToken are read from the file specified in +// options. If we encounter an error here, we bail out. +// - Contents of the actorToken are read from the file specified in options. +// If we encounter an error here, we ignore this field because this is +// optional. +// - Most of the other fields in the request come directly from options. // // A new HTTP request is created by calling http.NewRequestWithContext() and // passing the provided context, thereby enforcing any timeouts specified in @@ -303,7 +307,7 @@ func sendRequest(client httpDoer, req *http.Request) ([]byte, error) { // When the http.Client returns a non-nil error, it is the // responsibility of the caller to read the response body till an EOF is // encountered and to close it. - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) resp.Body.Close() if err != nil { return nil, err diff --git a/credentials/sts/sts_test.go b/credentials/sts/sts_test.go index 641bad5820bb..70bfa8b046f3 100644 --- a/credentials/sts/sts_test.go +++ b/credentials/sts/sts_test.go @@ -1,5 +1,3 @@ -// +build go1.13 - /* * * Copyright 2020 gRPC authors. @@ -27,7 +25,7 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" + "io" "net/http" "net/http/httputil" "strings" @@ -37,24 +35,26 @@ import ( "github.com/google/go-cmp/cmp" "google.golang.org/grpc/credentials" - "google.golang.org/grpc/internal" + icredentials "google.golang.org/grpc/internal/credentials" "google.golang.org/grpc/internal/grpctest" "google.golang.org/grpc/internal/testutils" ) const ( - requestedTokenType = "urn:ietf:params:oauth:token-type:access-token" - actorTokenPath = "/var/run/secrets/token.jwt" - actorTokenType = "urn:ietf:params:oauth:token-type:refresh_token" - actorTokenContents = "actorToken.jwt.contents" - accessTokenContents = "access_token" - subjectTokenPath = "/var/run/secrets/token.jwt" - subjectTokenType = "urn:ietf:params:oauth:token-type:id_token" - subjectTokenContents = "subjectToken.jwt.contents" - serviceURI = "http://localhost" - exampleResource = "https://backend.example.com/api" - exampleAudience = "example-backend-service" - testScope = "https://www.googleapis.com/auth/monitoring" + requestedTokenType = "urn:ietf:params:oauth:token-type:access-token" + actorTokenPath = "/var/run/secrets/token.jwt" + actorTokenType = "urn:ietf:params:oauth:token-type:refresh_token" + actorTokenContents = "actorToken.jwt.contents" + accessTokenContents = "access_token" + subjectTokenPath = "/var/run/secrets/token.jwt" + subjectTokenType = "urn:ietf:params:oauth:token-type:id_token" + subjectTokenContents = "subjectToken.jwt.contents" + serviceURI = "http://localhost" + exampleResource = "https://backend.example.com/api" + exampleAudience = "example-backend-service" + testScope = "https://www.googleapis.com/auth/monitoring" + defaultTestTimeout = 1 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond ) var ( @@ -102,7 +102,7 @@ func createTestContext(ctx context.Context, s credentials.SecurityLevel) context Method: "testInfo", AuthInfo: auth, } - return internal.NewRequestInfoContext.(func(context.Context, credentials.RequestInfo) context.Context)(ctx, ri) + return icredentials.NewRequestInfoContext(ctx, ri) } // errReader implements the io.Reader interface and returns an error from the @@ -114,7 +114,7 @@ func (r errReader) Read(b []byte) (n int, err error) { } // We need a function to construct the response instead of simply declaring it -// as a variable since the the response body will be consumed by the +// as a variable since the response body will be consumed by the // credentials, and therefore we will need a new one everytime. func makeGoodResponse() *http.Response { respJSON, _ := json.Marshal(responseParameters{ @@ -123,7 +123,7 @@ func makeGoodResponse() *http.Response { TokenType: "Bearer", ExpiresIn: 3600, }) - respBody := ioutil.NopCloser(bytes.NewReader(respJSON)) + respBody := io.NopCloser(bytes.NewReader(respJSON)) return &http.Response{ Status: "200 OK", StatusCode: http.StatusOK, @@ -131,31 +131,13 @@ func makeGoodResponse() *http.Response { } } -// fakeHTTPDoer helps mock out the http.Client.Do calls made by the credentials -// code under test. It makes the http.Request made by the credentials available -// through a channel, and makes it possible to inject various responses. -type fakeHTTPDoer struct { - reqCh *testutils.Channel - respCh *testutils.Channel - err error -} - -func (fc *fakeHTTPDoer) Do(req *http.Request) (*http.Response, error) { - fc.reqCh.Send(req) - val, err := fc.respCh.Receive() - if err != nil { - return nil, err - } - return val.(*http.Response), fc.err -} - // Overrides the http.Client with a fakeClient which sends a good response. -func overrideHTTPClientGood() (*fakeHTTPDoer, func()) { - fc := &fakeHTTPDoer{ - reqCh: testutils.NewChannel(), - respCh: testutils.NewChannel(), +func overrideHTTPClientGood() (*testutils.FakeHTTPClient, func()) { + fc := &testutils.FakeHTTPClient{ + ReqChan: testutils.NewChannel(), + RespChan: testutils.NewChannel(), } - fc.respCh.Send(makeGoodResponse()) + fc.RespChan.Send(makeGoodResponse()) origMakeHTTPDoer := makeHTTPDoer makeHTTPDoer = func(_ *x509.CertPool) httpDoer { return fc } @@ -163,7 +145,7 @@ func overrideHTTPClientGood() (*fakeHTTPDoer, func()) { } // Overrides the http.Client with the provided fakeClient. -func overrideHTTPClient(fc *fakeHTTPDoer) func() { +func overrideHTTPClient(fc *testutils.FakeHTTPClient) func() { origMakeHTTPDoer := makeHTTPDoer makeHTTPDoer = func(_ *x509.CertPool) httpDoer { return fc } return func() { makeHTTPDoer = origMakeHTTPDoer } @@ -239,8 +221,11 @@ func compareRequest(gotRequest *http.Request, wantReqParams *requestParameters) // expected goodRequest. This is expected to be called in a separate goroutine // by the tests. So, any errors encountered are pushed to an error channel // which is monitored by the test. -func receiveAndCompareRequest(reqCh *testutils.Channel, errCh chan error) { - val, err := reqCh.Receive() +func receiveAndCompareRequest(ReqChan *testutils.Channel, errCh chan error) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + val, err := ReqChan.Receive(ctx) if err != nil { errCh <- err return @@ -266,9 +251,12 @@ func (s) TestGetRequestMetadataSuccess(t *testing.T) { } errCh := make(chan error, 1) - go receiveAndCompareRequest(fc.reqCh, errCh) + go receiveAndCompareRequest(fc.ReqChan, errCh) - gotMetadata, err := creds.GetRequestMetadata(createTestContext(context.Background(), credentials.PrivacyAndIntegrity), "") + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + gotMetadata, err := creds.GetRequestMetadata(createTestContext(ctx, credentials.PrivacyAndIntegrity), "") if err != nil { t.Fatalf("creds.GetRequestMetadata() = %v", err) } @@ -283,7 +271,7 @@ func (s) TestGetRequestMetadataSuccess(t *testing.T) { // from the cache. This will fail if the credentials tries to send a fresh // request here since we have not configured our fakeClient to return any // response on retries. - gotMetadata, err = creds.GetRequestMetadata(createTestContext(context.Background(), credentials.PrivacyAndIntegrity), "") + gotMetadata, err = creds.GetRequestMetadata(createTestContext(ctx, credentials.PrivacyAndIntegrity), "") if err != nil { t.Fatalf("creds.GetRequestMetadata() = %v", err) } @@ -303,7 +291,9 @@ func (s) TestGetRequestMetadataBadSecurityLevel(t *testing.T) { t.Fatalf("NewCredentials(%v) = %v", goodOptions, err) } - gotMetadata, err := creds.GetRequestMetadata(createTestContext(context.Background(), credentials.IntegrityOnly), "") + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + gotMetadata, err := creds.GetRequestMetadata(createTestContext(ctx, credentials.IntegrityOnly), "") if err == nil { t.Fatalf("creds.GetRequestMetadata() succeeded with metadata %v, expected to fail", gotMetadata) } @@ -315,9 +305,9 @@ func (s) TestGetRequestMetadataBadSecurityLevel(t *testing.T) { func (s) TestGetRequestMetadataCacheExpiry(t *testing.T) { const expiresInSecs = 1 defer overrideSubjectTokenGood()() - fc := &fakeHTTPDoer{ - reqCh: testutils.NewChannel(), - respCh: testutils.NewChannel(), + fc := &testutils.FakeHTTPClient{ + ReqChan: testutils.NewChannel(), + RespChan: testutils.NewChannel(), } defer overrideHTTPClient(fc)() @@ -332,7 +322,7 @@ func (s) TestGetRequestMetadataCacheExpiry(t *testing.T) { // out a fresh request. for i := 0; i < 2; i++ { errCh := make(chan error, 1) - go receiveAndCompareRequest(fc.reqCh, errCh) + go receiveAndCompareRequest(fc.ReqChan, errCh) respJSON, _ := json.Marshal(responseParameters{ AccessToken: accessTokenContents, @@ -340,15 +330,17 @@ func (s) TestGetRequestMetadataCacheExpiry(t *testing.T) { TokenType: "Bearer", ExpiresIn: expiresInSecs, }) - respBody := ioutil.NopCloser(bytes.NewReader(respJSON)) + respBody := io.NopCloser(bytes.NewReader(respJSON)) resp := &http.Response{ Status: "200 OK", StatusCode: http.StatusOK, Body: respBody, } - fc.respCh.Send(resp) + fc.RespChan.Send(resp) - gotMetadata, err := creds.GetRequestMetadata(createTestContext(context.Background(), credentials.PrivacyAndIntegrity), "") + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + gotMetadata, err := creds.GetRequestMetadata(createTestContext(ctx, credentials.PrivacyAndIntegrity), "") if err != nil { t.Fatalf("creds.GetRequestMetadata() = %v", err) } @@ -374,7 +366,7 @@ func (s) TestGetRequestMetadataBadResponses(t *testing.T) { response: &http.Response{ Status: "200 OK", StatusCode: http.StatusOK, - Body: ioutil.NopCloser(strings.NewReader("not JSON")), + Body: io.NopCloser(strings.NewReader("not JSON")), }, }, { @@ -382,18 +374,20 @@ func (s) TestGetRequestMetadataBadResponses(t *testing.T) { response: &http.Response{ Status: "200 OK", StatusCode: http.StatusOK, - Body: ioutil.NopCloser(strings.NewReader("{}")), + Body: io.NopCloser(strings.NewReader("{}")), }, }, } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() for _, test := range tests { t.Run(test.name, func(t *testing.T) { defer overrideSubjectTokenGood()() - fc := &fakeHTTPDoer{ - reqCh: testutils.NewChannel(), - respCh: testutils.NewChannel(), + fc := &testutils.FakeHTTPClient{ + ReqChan: testutils.NewChannel(), + RespChan: testutils.NewChannel(), } defer overrideHTTPClient(fc)() @@ -403,10 +397,10 @@ func (s) TestGetRequestMetadataBadResponses(t *testing.T) { } errCh := make(chan error, 1) - go receiveAndCompareRequest(fc.reqCh, errCh) + go receiveAndCompareRequest(fc.ReqChan, errCh) - fc.respCh.Send(test.response) - if _, err := creds.GetRequestMetadata(createTestContext(context.Background(), credentials.PrivacyAndIntegrity), ""); err == nil { + fc.RespChan.Send(test.response) + if _, err := creds.GetRequestMetadata(createTestContext(ctx, credentials.PrivacyAndIntegrity), ""); err == nil { t.Fatal("creds.GetRequestMetadata() succeeded when expected to fail") } if err := <-errCh; err != nil { @@ -430,14 +424,18 @@ func (s) TestGetRequestMetadataBadSubjectTokenRead(t *testing.T) { errCh := make(chan error, 1) go func() { - if _, err := fc.reqCh.Receive(); err != testutils.ErrRecvTimeout { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer cancel() + if _, err := fc.ReqChan.Receive(ctx); err != context.DeadlineExceeded { errCh <- err return } errCh <- nil }() - if _, err := creds.GetRequestMetadata(createTestContext(context.Background(), credentials.PrivacyAndIntegrity), ""); err == nil { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := creds.GetRequestMetadata(createTestContext(ctx, credentials.PrivacyAndIntegrity), ""); err == nil { t.Fatal("creds.GetRequestMetadata() succeeded when expected to fail") } if err := <-errCh; err != nil { @@ -615,6 +613,9 @@ func (s) TestConstructRequest(t *testing.T) { }, }, } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() for _, test := range tests { t.Run(test.name, func(t *testing.T) { if test.subjectTokenReadErr { @@ -629,7 +630,7 @@ func (s) TestConstructRequest(t *testing.T) { defer overrideActorTokenGood()() } - gotRequest, err := constructRequest(context.Background(), test.opts) + gotRequest, err := constructRequest(ctx, test.opts) if (err != nil) != test.wantErr { t.Fatalf("constructRequest(%v) = %v, wantErr: %v", test.opts, err, test.wantErr) } @@ -645,7 +646,9 @@ func (s) TestConstructRequest(t *testing.T) { func (s) TestSendRequest(t *testing.T) { defer overrideSubjectTokenGood()() - req, err := constructRequest(context.Background(), goodOptions) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + req, err := constructRequest(ctx, goodOptions) if err != nil { t.Fatal(err) } @@ -666,7 +669,7 @@ func (s) TestSendRequest(t *testing.T) { resp: &http.Response{ Status: "200 OK", StatusCode: http.StatusOK, - Body: ioutil.NopCloser(errReader{}), + Body: io.NopCloser(errReader{}), }, wantErr: true, }, @@ -675,7 +678,7 @@ func (s) TestSendRequest(t *testing.T) { resp: &http.Response{ Status: "400 BadRequest", StatusCode: http.StatusBadRequest, - Body: ioutil.NopCloser(strings.NewReader("")), + Body: io.NopCloser(strings.NewReader("")), }, wantErr: true, }, @@ -687,12 +690,12 @@ func (s) TestSendRequest(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - client := &fakeHTTPDoer{ - reqCh: testutils.NewChannel(), - respCh: testutils.NewChannel(), - err: test.respErr, + client := &testutils.FakeHTTPClient{ + ReqChan: testutils.NewChannel(), + RespChan: testutils.NewChannel(), + Err: test.respErr, } - client.respCh.Send(test.resp) + client.RespChan.Send(test.resp) _, err := sendRequest(client, req) if (err != nil) != test.wantErr { t.Errorf("sendRequest(%v) = %v, wantErr: %v", req, err, test.wantErr) diff --git a/credentials/tls.go b/credentials/tls.go index 48384f5050e7..877b7cd21af7 100644 --- a/credentials/tls.go +++ b/credentials/tls.go @@ -23,9 +23,9 @@ import ( "crypto/tls" "crypto/x509" "fmt" - "io/ioutil" "net" "net/url" + "os" credinternal "google.golang.org/grpc/internal/credentials" ) @@ -166,7 +166,7 @@ func NewClientTLSFromCert(cp *x509.CertPool, serverNameOverride string) Transpor // it will override the virtual host name of authority (e.g. :authority header // field) in requests. func NewClientTLSFromFile(certFile, serverNameOverride string) (TransportCredentials, error) { - b, err := ioutil.ReadFile(certFile) + b, err := os.ReadFile(certFile) if err != nil { return nil, err } @@ -195,7 +195,10 @@ func NewServerTLSFromFile(certFile, keyFile string) (TransportCredentials, error // TLSChannelzSecurityValue defines the struct that TLS protocol should return // from GetSecurityValue(), containing security info like cipher and certificate used. // -// This API is EXPERIMENTAL. +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. type TLSChannelzSecurityValue struct { ChannelzSecurityValue StandardName string @@ -227,4 +230,7 @@ var cipherSuiteLookup = map[uint16]string{ tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256: "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256", tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305: "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305", tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305: "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305", + tls.TLS_AES_128_GCM_SHA256: "TLS_AES_128_GCM_SHA256", + tls.TLS_AES_256_GCM_SHA384: "TLS_AES_256_GCM_SHA384", + tls.TLS_CHACHA20_POLY1305_SHA256: "TLS_CHACHA20_POLY1305_SHA256", } diff --git a/credentials/tls/certprovider/distributor.go b/credentials/tls/certprovider/distributor.go index fdb38a663fe8..11fae92adace 100644 --- a/credentials/tls/certprovider/distributor.go +++ b/credentials/tls/certprovider/distributor.go @@ -31,11 +31,11 @@ import ( // // Provider implementations which choose to use a Distributor should do the // following: -// - create a new Distributor using the NewDistributor() function. -// - invoke the Set() method whenever they have new key material or errors to -// report. -// - delegate to the distributor when handing calls to KeyMaterial(). -// - invoke the Stop() method when they are done using the distributor. +// - create a new Distributor using the NewDistributor() function. +// - invoke the Set() method whenever they have new key material or errors to +// report. +// - delegate to the distributor when handing calls to KeyMaterial(). +// - invoke the Stop() method when they are done using the distributor. type Distributor struct { // mu protects the underlying key material. mu sync.Mutex diff --git a/credentials/tls/certprovider/distributor_test.go b/credentials/tls/certprovider/distributor_test.go index bec00e919bcf..48d51375616f 100644 --- a/credentials/tls/certprovider/distributor_test.go +++ b/credentials/tls/certprovider/distributor_test.go @@ -1,5 +1,3 @@ -// +build go1.13 - /* * * Copyright 2020 gRPC authors. diff --git a/credentials/tls/certprovider/meshca/internal/meshca_experimental/config.pb.go b/credentials/tls/certprovider/meshca/internal/meshca_experimental/config.pb.go deleted file mode 100644 index 936dfb6ed4bf..000000000000 --- a/credentials/tls/certprovider/meshca/internal/meshca_experimental/config.pb.go +++ /dev/null @@ -1,198 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. -// source: grpc/tls/provider/meshca/experimental/config.proto - -// NOTE: This proto will very likely move to a different namespace and a -// different git repo in the future. - -package meshca_experimental - -import ( - fmt "fmt" - v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" - proto "github.com/golang/protobuf/proto" - duration "github.com/golang/protobuf/ptypes/duration" - math "math" -) - -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package - -// Type of key to be embedded in CSRs sent to the MeshCA. -type GoogleMeshCaConfig_KeyType int32 - -const ( - GoogleMeshCaConfig_KEY_TYPE_UNKNOWN GoogleMeshCaConfig_KeyType = 0 - GoogleMeshCaConfig_KEY_TYPE_RSA GoogleMeshCaConfig_KeyType = 1 -) - -var GoogleMeshCaConfig_KeyType_name = map[int32]string{ - 0: "KEY_TYPE_UNKNOWN", - 1: "KEY_TYPE_RSA", -} - -var GoogleMeshCaConfig_KeyType_value = map[string]int32{ - "KEY_TYPE_UNKNOWN": 0, - "KEY_TYPE_RSA": 1, -} - -func (x GoogleMeshCaConfig_KeyType) String() string { - return proto.EnumName(GoogleMeshCaConfig_KeyType_name, int32(x)) -} - -func (GoogleMeshCaConfig_KeyType) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_2f29f6dba7c86653, []int{0, 0} -} - -// GoogleMeshCaConfig contains all configuration parameters required by the -// MeshCA CertificateProvider plugin implementation. -type GoogleMeshCaConfig struct { - // GoogleMeshCA server endpoint to get CSRs signed via the *CreateCertificate* - // unary call. This must have :ref:`api_type - // ` :ref:`GRPC - // `. - // STS based call credentials need to be supplied in :ref:`call_credentials - // `. - // - // If :ref:`timeout envoy_api_field_config.core.v3.GrpcService.timeout` is - // left unspecified, a default value of 10s will be used. - Server *v3.ApiConfigSource `protobuf:"bytes,1,opt,name=server,proto3" json:"server,omitempty"` - // Certificate lifetime to request in CSRs sent to the MeshCA. - // - // A default value of 24h will be used if left unspecified. - CertificateLifetime *duration.Duration `protobuf:"bytes,2,opt,name=certificate_lifetime,json=certificateLifetime,proto3" json:"certificate_lifetime,omitempty"` - // How long before certificate expiration should the certificate be renewed. - // - // A default value of 12h will be used if left unspecified. - RenewalGracePeriod *duration.Duration `protobuf:"bytes,3,opt,name=renewal_grace_period,json=renewalGracePeriod,proto3" json:"renewal_grace_period,omitempty"` - // Type of key. - // - // RSA keys will be used if left unspecified. - KeyType GoogleMeshCaConfig_KeyType `protobuf:"varint,4,opt,name=key_type,json=keyType,proto3,enum=grpc.tls.provider.meshca.experimental.GoogleMeshCaConfig_KeyType" json:"key_type,omitempty"` - // Size of the key in bits. - // - // 2048 bit keys will be used if left unspecified. - KeySize uint32 `protobuf:"varint,5,opt,name=key_size,json=keySize,proto3" json:"key_size,omitempty"` - // GCE location (region/zone) where the workload is located. - // - // GCE/GKE Metadata Server will be contacted if left unspecified. - Location string `protobuf:"bytes,6,opt,name=location,proto3" json:"location,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *GoogleMeshCaConfig) Reset() { *m = GoogleMeshCaConfig{} } -func (m *GoogleMeshCaConfig) String() string { return proto.CompactTextString(m) } -func (*GoogleMeshCaConfig) ProtoMessage() {} -func (*GoogleMeshCaConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_2f29f6dba7c86653, []int{0} -} - -func (m *GoogleMeshCaConfig) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_GoogleMeshCaConfig.Unmarshal(m, b) -} -func (m *GoogleMeshCaConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_GoogleMeshCaConfig.Marshal(b, m, deterministic) -} -func (m *GoogleMeshCaConfig) XXX_Merge(src proto.Message) { - xxx_messageInfo_GoogleMeshCaConfig.Merge(m, src) -} -func (m *GoogleMeshCaConfig) XXX_Size() int { - return xxx_messageInfo_GoogleMeshCaConfig.Size(m) -} -func (m *GoogleMeshCaConfig) XXX_DiscardUnknown() { - xxx_messageInfo_GoogleMeshCaConfig.DiscardUnknown(m) -} - -var xxx_messageInfo_GoogleMeshCaConfig proto.InternalMessageInfo - -func (m *GoogleMeshCaConfig) GetServer() *v3.ApiConfigSource { - if m != nil { - return m.Server - } - return nil -} - -func (m *GoogleMeshCaConfig) GetCertificateLifetime() *duration.Duration { - if m != nil { - return m.CertificateLifetime - } - return nil -} - -func (m *GoogleMeshCaConfig) GetRenewalGracePeriod() *duration.Duration { - if m != nil { - return m.RenewalGracePeriod - } - return nil -} - -func (m *GoogleMeshCaConfig) GetKeyType() GoogleMeshCaConfig_KeyType { - if m != nil { - return m.KeyType - } - return GoogleMeshCaConfig_KEY_TYPE_UNKNOWN -} - -func (m *GoogleMeshCaConfig) GetKeySize() uint32 { - if m != nil { - return m.KeySize - } - return 0 -} - -func (m *GoogleMeshCaConfig) GetLocation() string { - if m != nil { - return m.Location - } - return "" -} - -func init() { - proto.RegisterEnum("grpc.tls.provider.meshca.experimental.GoogleMeshCaConfig_KeyType", GoogleMeshCaConfig_KeyType_name, GoogleMeshCaConfig_KeyType_value) - proto.RegisterType((*GoogleMeshCaConfig)(nil), "grpc.tls.provider.meshca.experimental.GoogleMeshCaConfig") -} - -func init() { - proto.RegisterFile("grpc/tls/provider/meshca/experimental/config.proto", fileDescriptor_2f29f6dba7c86653) -} - -var fileDescriptor_2f29f6dba7c86653 = []byte{ - // 439 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x92, 0x4f, 0x6f, 0xd3, 0x30, - 0x18, 0xc6, 0x09, 0x83, 0x6e, 0x98, 0x3f, 0x2a, 0xa6, 0x87, 0xac, 0x07, 0x54, 0x4d, 0x9a, 0x94, - 0x93, 0x2d, 0xda, 0x33, 0x87, 0x6e, 0x4c, 0x3b, 0x74, 0x94, 0x2a, 0x1d, 0x9a, 0x86, 0x90, 0x22, - 0xcf, 0x7d, 0x9b, 0x59, 0x73, 0xed, 0xe8, 0x8d, 0x1b, 0xc8, 0x3e, 0x09, 0x9f, 0x84, 0xcf, 0x87, - 0xe2, 0x78, 0x53, 0x04, 0x07, 0x7a, 0xcb, 0x1b, 0xfb, 0xf7, 0x3c, 0x8f, 0x1e, 0xbf, 0x64, 0x9c, - 0x63, 0x21, 0xb9, 0xd3, 0x25, 0x2f, 0xd0, 0x56, 0x6a, 0x05, 0xc8, 0x37, 0x50, 0xde, 0x4a, 0xc1, - 0xe1, 0x67, 0x01, 0xa8, 0x36, 0x60, 0x9c, 0xd0, 0x5c, 0x5a, 0xb3, 0x56, 0x39, 0x2b, 0xd0, 0x3a, - 0x4b, 0x8f, 0x1b, 0x86, 0x39, 0x5d, 0xb2, 0x07, 0x86, 0xb5, 0x0c, 0xeb, 0x32, 0xc3, 0x04, 0x4c, - 0x65, 0xeb, 0x80, 0x72, 0x69, 0x11, 0x78, 0x35, 0x09, 0x63, 0x56, 0xda, 0x2d, 0x4a, 0x68, 0x05, - 0x87, 0xef, 0x73, 0x6b, 0x73, 0x0d, 0xdc, 0x4f, 0x37, 0xdb, 0x35, 0x5f, 0x6d, 0x51, 0x38, 0x65, - 0x4d, 0x7b, 0x7e, 0xf4, 0x7b, 0x8f, 0xd0, 0x73, 0x7f, 0xe5, 0x33, 0x94, 0xb7, 0xa7, 0xe2, 0xd4, - 0x6b, 0xd0, 0x8f, 0xa4, 0x57, 0x02, 0x56, 0x80, 0x71, 0x34, 0x8a, 0x92, 0x97, 0xe3, 0x63, 0xe6, - 0x1d, 0x59, 0x08, 0xdb, 0x38, 0xb2, 0x6a, 0xc2, 0xa6, 0x85, 0x6a, 0x81, 0xa5, 0xf7, 0x4c, 0x03, - 0x44, 0x2f, 0xc8, 0x40, 0x02, 0x3a, 0xb5, 0x56, 0x52, 0x38, 0xc8, 0xb4, 0x5a, 0x83, 0x53, 0x1b, - 0x88, 0x9f, 0x7a, 0xb1, 0x43, 0xd6, 0x86, 0x62, 0x0f, 0xa1, 0xd8, 0xa7, 0x10, 0x2a, 0x7d, 0xd7, - 0xc1, 0x2e, 0x02, 0x45, 0x67, 0x64, 0x80, 0x60, 0xe0, 0x87, 0xd0, 0x59, 0x8e, 0x42, 0x42, 0xd6, - 0x34, 0x61, 0x57, 0xf1, 0xde, 0xff, 0xd4, 0x68, 0xc0, 0xce, 0x1b, 0x6a, 0xe1, 0x21, 0xfa, 0x9d, - 0x1c, 0xdc, 0x41, 0x9d, 0xb9, 0xba, 0x80, 0xf8, 0xd9, 0x28, 0x4a, 0xde, 0x8c, 0xa7, 0x6c, 0xa7, - 0xd2, 0xd9, 0xbf, 0x35, 0xb1, 0x19, 0xd4, 0x97, 0x75, 0x01, 0xe9, 0xfe, 0x5d, 0xfb, 0x41, 0x0f, - 0x5b, 0xf5, 0x52, 0xdd, 0x43, 0xfc, 0x7c, 0x14, 0x25, 0xaf, 0xfd, 0xd1, 0x52, 0xdd, 0x03, 0x1d, - 0x92, 0x03, 0x6d, 0xa5, 0x0f, 0x16, 0xf7, 0x46, 0x51, 0xf2, 0x22, 0x7d, 0x9c, 0x8f, 0x3e, 0x90, - 0xfd, 0x20, 0x45, 0x07, 0xa4, 0x3f, 0x3b, 0xbb, 0xce, 0x2e, 0xaf, 0x17, 0x67, 0xd9, 0xd7, 0xf9, - 0x6c, 0xfe, 0xe5, 0x6a, 0xde, 0x7f, 0x42, 0xfb, 0xe4, 0xd5, 0xe3, 0xdf, 0x74, 0x39, 0xed, 0x47, - 0x27, 0xbf, 0x22, 0x92, 0x28, 0xbb, 0x5b, 0xf4, 0x93, 0xb7, 0xdd, 0xd4, 0x8b, 0xa6, 0xa7, 0x45, - 0xf4, 0xed, 0x2a, 0xf4, 0x96, 0x5b, 0x2d, 0x4c, 0xce, 0x2c, 0xe6, 0xdc, 0x6f, 0xac, 0x44, 0x58, - 0x81, 0x71, 0x4a, 0xe8, 0xd2, 0x6f, 0x6f, 0xf3, 0x24, 0x7f, 0x6f, 0xb0, 0x32, 0x0e, 0xd0, 0x08, - 0x1d, 0xe6, 0xac, 0xeb, 0x76, 0xd3, 0xf3, 0x2f, 0x31, 0xf9, 0x13, 0x00, 0x00, 0xff, 0xff, 0x72, - 0x44, 0x68, 0xc5, 0x01, 0x03, 0x00, 0x00, -} diff --git a/credentials/tls/certprovider/meshca/internal/v1/meshca.pb.go b/credentials/tls/certprovider/meshca/internal/v1/meshca.pb.go deleted file mode 100644 index b09de93beac6..000000000000 --- a/credentials/tls/certprovider/meshca/internal/v1/meshca.pb.go +++ /dev/null @@ -1,243 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. -// source: istio/google/security/meshca/v1/meshca.proto - -package google_security_meshca_v1 - -import ( - context "context" - fmt "fmt" - proto "github.com/golang/protobuf/proto" - duration "github.com/golang/protobuf/ptypes/duration" - grpc "google.golang.org/grpc" - codes "google.golang.org/grpc/codes" - status "google.golang.org/grpc/status" - math "math" -) - -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package - -// Certificate request message. -type MeshCertificateRequest struct { - // The request ID must be a valid UUID with the exception that zero UUID is - // not supported (00000000-0000-0000-0000-000000000000). - RequestId string `protobuf:"bytes,1,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"` - // PEM-encoded certificate request. - Csr string `protobuf:"bytes,2,opt,name=csr,proto3" json:"csr,omitempty"` - // Optional: requested certificate validity period. - Validity *duration.Duration `protobuf:"bytes,3,opt,name=validity,proto3" json:"validity,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *MeshCertificateRequest) Reset() { *m = MeshCertificateRequest{} } -func (m *MeshCertificateRequest) String() string { return proto.CompactTextString(m) } -func (*MeshCertificateRequest) ProtoMessage() {} -func (*MeshCertificateRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_f72841047b94fe5e, []int{0} -} - -func (m *MeshCertificateRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_MeshCertificateRequest.Unmarshal(m, b) -} -func (m *MeshCertificateRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_MeshCertificateRequest.Marshal(b, m, deterministic) -} -func (m *MeshCertificateRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_MeshCertificateRequest.Merge(m, src) -} -func (m *MeshCertificateRequest) XXX_Size() int { - return xxx_messageInfo_MeshCertificateRequest.Size(m) -} -func (m *MeshCertificateRequest) XXX_DiscardUnknown() { - xxx_messageInfo_MeshCertificateRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_MeshCertificateRequest proto.InternalMessageInfo - -func (m *MeshCertificateRequest) GetRequestId() string { - if m != nil { - return m.RequestId - } - return "" -} - -func (m *MeshCertificateRequest) GetCsr() string { - if m != nil { - return m.Csr - } - return "" -} - -func (m *MeshCertificateRequest) GetValidity() *duration.Duration { - if m != nil { - return m.Validity - } - return nil -} - -// Certificate response message. -type MeshCertificateResponse struct { - // PEM-encoded certificate chain. - // Leaf cert is element '0'. Root cert is element 'n'. - CertChain []string `protobuf:"bytes,1,rep,name=cert_chain,json=certChain,proto3" json:"cert_chain,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *MeshCertificateResponse) Reset() { *m = MeshCertificateResponse{} } -func (m *MeshCertificateResponse) String() string { return proto.CompactTextString(m) } -func (*MeshCertificateResponse) ProtoMessage() {} -func (*MeshCertificateResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_f72841047b94fe5e, []int{1} -} - -func (m *MeshCertificateResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_MeshCertificateResponse.Unmarshal(m, b) -} -func (m *MeshCertificateResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_MeshCertificateResponse.Marshal(b, m, deterministic) -} -func (m *MeshCertificateResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_MeshCertificateResponse.Merge(m, src) -} -func (m *MeshCertificateResponse) XXX_Size() int { - return xxx_messageInfo_MeshCertificateResponse.Size(m) -} -func (m *MeshCertificateResponse) XXX_DiscardUnknown() { - xxx_messageInfo_MeshCertificateResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_MeshCertificateResponse proto.InternalMessageInfo - -func (m *MeshCertificateResponse) GetCertChain() []string { - if m != nil { - return m.CertChain - } - return nil -} - -func init() { - proto.RegisterType((*MeshCertificateRequest)(nil), "google.security.meshca.v1.MeshCertificateRequest") - proto.RegisterType((*MeshCertificateResponse)(nil), "google.security.meshca.v1.MeshCertificateResponse") -} - -func init() { - proto.RegisterFile("istio/google/security/meshca/v1/meshca.proto", fileDescriptor_f72841047b94fe5e) -} - -var fileDescriptor_f72841047b94fe5e = []byte{ - // 284 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x90, 0x41, 0x4b, 0xc3, 0x30, - 0x1c, 0xc5, 0x8d, 0x05, 0x71, 0xd9, 0x45, 0x73, 0xd0, 0x6e, 0x30, 0x29, 0x3d, 0xed, 0x20, 0x29, - 0xad, 0x08, 0x9e, 0x57, 0x2f, 0x1e, 0x84, 0x51, 0x3f, 0xc0, 0xc8, 0xd2, 0xff, 0xd6, 0x3f, 0x6c, - 0xcd, 0x4c, 0xd2, 0xc2, 0xc0, 0x83, 0x9f, 0xc2, 0xcf, 0x2b, 0x69, 0x33, 0x11, 0xe6, 0x0e, 0xde, - 0x1e, 0x2f, 0xef, 0xe5, 0xff, 0xe3, 0xd1, 0x7b, 0x34, 0x16, 0x55, 0xb2, 0x56, 0x6a, 0xbd, 0x81, - 0xc4, 0x80, 0x6c, 0x34, 0xda, 0x7d, 0xb2, 0x05, 0x53, 0x49, 0x91, 0xb4, 0xa9, 0x57, 0x7c, 0xa7, - 0x95, 0x55, 0x6c, 0xd4, 0xe7, 0xf8, 0x21, 0xc7, 0xfd, 0x6b, 0x9b, 0x8e, 0xef, 0xfc, 0x17, 0x5d, - 0x70, 0xd9, 0xac, 0x92, 0xb2, 0xd1, 0xc2, 0xa2, 0xaa, 0xfb, 0x6a, 0xfc, 0x49, 0xe8, 0xcd, 0x2b, - 0x98, 0x2a, 0x07, 0x6d, 0x71, 0x85, 0x52, 0x58, 0x28, 0xe0, 0xbd, 0x01, 0x63, 0xd9, 0x84, 0x52, - 0xdd, 0xcb, 0x05, 0x96, 0x21, 0x89, 0xc8, 0x74, 0x50, 0x0c, 0xbc, 0xf3, 0x52, 0xb2, 0x2b, 0x1a, - 0x48, 0xa3, 0xc3, 0xf3, 0xce, 0x77, 0x92, 0x3d, 0xd2, 0xcb, 0x56, 0x6c, 0xb0, 0x44, 0xbb, 0x0f, - 0x83, 0x88, 0x4c, 0x87, 0xd9, 0x88, 0x7b, 0xb2, 0xc3, 0x79, 0xfe, 0xec, 0xcf, 0x17, 0x3f, 0xd1, - 0xf8, 0x89, 0xde, 0x1e, 0x11, 0x98, 0x9d, 0xaa, 0x0d, 0x38, 0x04, 0x09, 0xda, 0x2e, 0x64, 0x25, - 0xb0, 0x0e, 0x49, 0x14, 0x38, 0x04, 0xe7, 0xe4, 0xce, 0xc8, 0xbe, 0x8e, 0xe1, 0xdf, 0x40, 0xb7, - 0x28, 0x81, 0x7d, 0xd0, 0xeb, 0x5c, 0x83, 0xb0, 0xf0, 0xeb, 0x8d, 0xa5, 0xfc, 0xe4, 0x50, 0xfc, - 0xef, 0x11, 0xc6, 0xd9, 0x7f, 0x2a, 0x3d, 0x75, 0x7c, 0x36, 0xe3, 0x74, 0x22, 0xd5, 0xf6, 0x74, - 0x75, 0x36, 0xec, 0xba, 0x62, 0xee, 0x66, 0x99, 0x93, 0xe5, 0x45, 0xb7, 0xcf, 0xc3, 0x77, 0x00, - 0x00, 0x00, 0xff, 0xff, 0xb7, 0x0d, 0xfd, 0xff, 0xf7, 0x01, 0x00, 0x00, -} - -// Reference imports to suppress errors if they are not otherwise used. -var _ context.Context -var _ grpc.ClientConnInterface - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the grpc package it is being compiled against. -const _ = grpc.SupportPackageIsVersion6 - -// MeshCertificateServiceClient is the client API for MeshCertificateService service. -// -// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. -type MeshCertificateServiceClient interface { - // Using provided CSR, returns a signed certificate that represents a GCP - // service account identity. - CreateCertificate(ctx context.Context, in *MeshCertificateRequest, opts ...grpc.CallOption) (*MeshCertificateResponse, error) -} - -type meshCertificateServiceClient struct { - cc grpc.ClientConnInterface -} - -func NewMeshCertificateServiceClient(cc grpc.ClientConnInterface) MeshCertificateServiceClient { - return &meshCertificateServiceClient{cc} -} - -func (c *meshCertificateServiceClient) CreateCertificate(ctx context.Context, in *MeshCertificateRequest, opts ...grpc.CallOption) (*MeshCertificateResponse, error) { - out := new(MeshCertificateResponse) - err := c.cc.Invoke(ctx, "/google.security.meshca.v1.MeshCertificateService/CreateCertificate", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -// MeshCertificateServiceServer is the server API for MeshCertificateService service. -type MeshCertificateServiceServer interface { - // Using provided CSR, returns a signed certificate that represents a GCP - // service account identity. - CreateCertificate(context.Context, *MeshCertificateRequest) (*MeshCertificateResponse, error) -} - -// UnimplementedMeshCertificateServiceServer can be embedded to have forward compatible implementations. -type UnimplementedMeshCertificateServiceServer struct { -} - -func (*UnimplementedMeshCertificateServiceServer) CreateCertificate(ctx context.Context, req *MeshCertificateRequest) (*MeshCertificateResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method CreateCertificate not implemented") -} - -func RegisterMeshCertificateServiceServer(s *grpc.Server, srv MeshCertificateServiceServer) { - s.RegisterService(&_MeshCertificateService_serviceDesc, srv) -} - -func _MeshCertificateService_CreateCertificate_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(MeshCertificateRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(MeshCertificateServiceServer).CreateCertificate(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/google.security.meshca.v1.MeshCertificateService/CreateCertificate", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(MeshCertificateServiceServer).CreateCertificate(ctx, req.(*MeshCertificateRequest)) - } - return interceptor(ctx, in, info, handler) -} - -var _MeshCertificateService_serviceDesc = grpc.ServiceDesc{ - ServiceName: "google.security.meshca.v1.MeshCertificateService", - HandlerType: (*MeshCertificateServiceServer)(nil), - Methods: []grpc.MethodDesc{ - { - MethodName: "CreateCertificate", - Handler: _MeshCertificateService_CreateCertificate_Handler, - }, - }, - Streams: []grpc.StreamDesc{}, - Metadata: "istio/google/security/meshca/v1/meshca.proto", -} diff --git a/credentials/tls/certprovider/meshca/internal/v1/meshca_grpc.pb.go b/credentials/tls/certprovider/meshca/internal/v1/meshca_grpc.pb.go deleted file mode 100644 index a063519dab8c..000000000000 --- a/credentials/tls/certprovider/meshca/internal/v1/meshca_grpc.pb.go +++ /dev/null @@ -1,88 +0,0 @@ -// Code generated by protoc-gen-go-grpc. DO NOT EDIT. - -package google_security_meshca_v1 - -import ( - context "context" - grpc "google.golang.org/grpc" - codes "google.golang.org/grpc/codes" - status "google.golang.org/grpc/status" -) - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the grpc package it is being compiled against. -const _ = grpc.SupportPackageIsVersion7 - -// MeshCertificateServiceService is the service API for MeshCertificateService service. -// Fields should be assigned to their respective handler implementations only before -// RegisterMeshCertificateServiceService is called. Any unassigned fields will result in the -// handler for that method returning an Unimplemented error. -type MeshCertificateServiceService struct { - // Using provided CSR, returns a signed certificate that represents a GCP - // service account identity. - CreateCertificate func(context.Context, *MeshCertificateRequest) (*MeshCertificateResponse, error) -} - -func (s *MeshCertificateServiceService) createCertificate(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - if s.CreateCertificate == nil { - return nil, status.Errorf(codes.Unimplemented, "method CreateCertificate not implemented") - } - in := new(MeshCertificateRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return s.CreateCertificate(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: s, - FullMethod: "/google.security.meshca.v1.MeshCertificateService/CreateCertificate", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return s.CreateCertificate(ctx, req.(*MeshCertificateRequest)) - } - return interceptor(ctx, in, info, handler) -} - -// RegisterMeshCertificateServiceService registers a service implementation with a gRPC server. -func RegisterMeshCertificateServiceService(s grpc.ServiceRegistrar, srv *MeshCertificateServiceService) { - sd := grpc.ServiceDesc{ - ServiceName: "google.security.meshca.v1.MeshCertificateService", - Methods: []grpc.MethodDesc{ - { - MethodName: "CreateCertificate", - Handler: srv.createCertificate, - }, - }, - Streams: []grpc.StreamDesc{}, - Metadata: "istio/google/security/meshca/v1/meshca.proto", - } - - s.RegisterService(&sd, nil) -} - -// NewMeshCertificateServiceService creates a new MeshCertificateServiceService containing the -// implemented methods of the MeshCertificateService service in s. Any unimplemented -// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client. -// This includes situations where the method handler is misspelled or has the wrong -// signature. For this reason, this function should be used with great care and -// is not recommended to be used by most users. -func NewMeshCertificateServiceService(s interface{}) *MeshCertificateServiceService { - ns := &MeshCertificateServiceService{} - if h, ok := s.(interface { - CreateCertificate(context.Context, *MeshCertificateRequest) (*MeshCertificateResponse, error) - }); ok { - ns.CreateCertificate = h.CreateCertificate - } - return ns -} - -// UnstableMeshCertificateServiceService is the service API for MeshCertificateService service. -// New methods may be added to this interface if they are added to the service -// definition, which is not a backward-compatible change. For this reason, -// use of this type is not recommended. -type UnstableMeshCertificateServiceService interface { - // Using provided CSR, returns a signed certificate that represents a GCP - // service account identity. - CreateCertificate(context.Context, *MeshCertificateRequest) (*MeshCertificateResponse, error) -} diff --git a/credentials/tls/certprovider/pemfile/builder.go b/credentials/tls/certprovider/pemfile/builder.go new file mode 100644 index 000000000000..8d8e2d4a0f5a --- /dev/null +++ b/credentials/tls/certprovider/pemfile/builder.go @@ -0,0 +1,96 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package pemfile + +import ( + "encoding/json" + "fmt" + "time" + + "google.golang.org/grpc/credentials/tls/certprovider" + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/types/known/durationpb" +) + +const ( + pluginName = "file_watcher" + defaultRefreshInterval = 10 * time.Minute +) + +func init() { + certprovider.Register(&pluginBuilder{}) +} + +type pluginBuilder struct{} + +func (p *pluginBuilder) ParseConfig(c any) (*certprovider.BuildableConfig, error) { + data, ok := c.(json.RawMessage) + if !ok { + return nil, fmt.Errorf("meshca: unsupported config type: %T", c) + } + opts, err := pluginConfigFromJSON(data) + if err != nil { + return nil, err + } + return certprovider.NewBuildableConfig(pluginName, opts.canonical(), func(certprovider.BuildOptions) certprovider.Provider { + return newProvider(opts) + }), nil +} + +func (p *pluginBuilder) Name() string { + return pluginName +} + +func pluginConfigFromJSON(jd json.RawMessage) (Options, error) { + // The only difference between this anonymous struct and the Options struct + // is that the refresh_interval is represented here as a duration proto, + // while in the latter a time.Duration is used. + cfg := &struct { + CertificateFile string `json:"certificate_file,omitempty"` + PrivateKeyFile string `json:"private_key_file,omitempty"` + CACertificateFile string `json:"ca_certificate_file,omitempty"` + RefreshInterval json.RawMessage `json:"refresh_interval,omitempty"` + }{} + if err := json.Unmarshal(jd, cfg); err != nil { + return Options{}, fmt.Errorf("pemfile: json.Unmarshal(%s) failed: %v", string(jd), err) + } + + opts := Options{ + CertFile: cfg.CertificateFile, + KeyFile: cfg.PrivateKeyFile, + RootFile: cfg.CACertificateFile, + // Refresh interval is the only field in the configuration for which we + // support a default value. We cannot possibly have valid defaults for + // file paths to watch. Also, it is valid to specify an empty path for + // some of those fields if the user does not want to watch them. + RefreshDuration: defaultRefreshInterval, + } + if cfg.RefreshInterval != nil { + dur := &durationpb.Duration{} + if err := protojson.Unmarshal(cfg.RefreshInterval, dur); err != nil { + return Options{}, fmt.Errorf("pemfile: protojson.Unmarshal(%+v) failed: %v", cfg.RefreshInterval, err) + } + opts.RefreshDuration = dur.AsDuration() + } + + if err := opts.validate(); err != nil { + return Options{}, err + } + return opts, nil +} diff --git a/credentials/tls/certprovider/pemfile/builder_test.go b/credentials/tls/certprovider/pemfile/builder_test.go new file mode 100644 index 000000000000..dfd5ee899233 --- /dev/null +++ b/credentials/tls/certprovider/pemfile/builder_test.go @@ -0,0 +1,130 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package pemfile + +import ( + "encoding/json" + "testing" +) + +func TestParseConfig(t *testing.T) { + tests := []struct { + desc string + input any + wantOutput string + wantErr bool + }{ + { + desc: "non JSON input", + input: new(int), + wantErr: true, + }, + { + desc: "invalid JSON", + input: json.RawMessage(`bad bad json`), + wantErr: true, + }, + { + desc: "JSON input does not match expected", + input: json.RawMessage(`["foo": "bar"]`), + wantErr: true, + }, + { + desc: "no credential files", + input: json.RawMessage(`{}`), + wantErr: true, + }, + { + desc: "only cert file", + input: json.RawMessage(` + { + "certificate_file": "/a/b/cert.pem" + }`), + wantErr: true, + }, + { + desc: "only key file", + input: json.RawMessage(` + { + "private_key_file": "/a/b/key.pem" + }`), + wantErr: true, + }, + { + desc: "cert and key in different directories", + input: json.RawMessage(` + { + "certificate_file": "/b/a/cert.pem", + "private_key_file": "/a/b/key.pem" + }`), + wantErr: true, + }, + { + desc: "bad refresh duration", + input: json.RawMessage(` + { + "certificate_file": "/a/b/cert.pem", + "private_key_file": "/a/b/key.pem", + "ca_certificate_file": "/a/b/ca.pem", + "refresh_interval": "duration" + }`), + wantErr: true, + }, + { + desc: "good config with default refresh interval", + input: json.RawMessage(` + { + "certificate_file": "/a/b/cert.pem", + "private_key_file": "/a/b/key.pem", + "ca_certificate_file": "/a/b/ca.pem" + }`), + wantOutput: "file_watcher:/a/b/cert.pem:/a/b/key.pem:/a/b/ca.pem:10m0s", + }, + { + desc: "good config", + input: json.RawMessage(` + { + "certificate_file": "/a/b/cert.pem", + "private_key_file": "/a/b/key.pem", + "ca_certificate_file": "/a/b/ca.pem", + "refresh_interval": "200s" + }`), + wantOutput: "file_watcher:/a/b/cert.pem:/a/b/key.pem:/a/b/ca.pem:3m20s", + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + builder := &pluginBuilder{} + + bc, err := builder.ParseConfig(test.input) + if (err != nil) != test.wantErr { + t.Fatalf("ParseConfig(%+v) failed: %v", test.input, err) + } + if test.wantErr { + return + } + + gotConfig := bc.String() + if gotConfig != test.wantOutput { + t.Fatalf("ParseConfig(%v) = %s, want %s", test.input, gotConfig, test.wantOutput) + } + }) + } +} diff --git a/credentials/tls/certprovider/pemfile/watcher.go b/credentials/tls/certprovider/pemfile/watcher.go new file mode 100644 index 000000000000..7ed5c53ba404 --- /dev/null +++ b/credentials/tls/certprovider/pemfile/watcher.go @@ -0,0 +1,260 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package pemfile provides a file watching certificate provider plugin +// implementation which works for files with PEM contents. +// +// # Experimental +// +// Notice: All APIs in this package are experimental and may be removed in a +// later release. +package pemfile + +import ( + "bytes" + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "os" + "path/filepath" + "time" + + "google.golang.org/grpc/credentials/tls/certprovider" + "google.golang.org/grpc/grpclog" +) + +const defaultCertRefreshDuration = 1 * time.Hour + +var ( + // For overriding from unit tests. + newDistributor = func() distributor { return certprovider.NewDistributor() } + + logger = grpclog.Component("pemfile") +) + +// Options configures a certificate provider plugin that watches a specified set +// of files that contain certificates and keys in PEM format. +type Options struct { + // CertFile is the file that holds the identity certificate. + // Optional. If this is set, KeyFile must also be set. + CertFile string + // KeyFile is the file that holds identity private key. + // Optional. If this is set, CertFile must also be set. + KeyFile string + // RootFile is the file that holds trusted root certificate(s). + // Optional. + RootFile string + // RefreshDuration is the amount of time the plugin waits before checking + // for updates in the specified files. + // Optional. If not set, a default value (1 hour) will be used. + RefreshDuration time.Duration +} + +func (o Options) canonical() []byte { + return []byte(fmt.Sprintf("%s:%s:%s:%s", o.CertFile, o.KeyFile, o.RootFile, o.RefreshDuration)) +} + +func (o Options) validate() error { + if o.CertFile == "" && o.KeyFile == "" && o.RootFile == "" { + return fmt.Errorf("pemfile: at least one credential file needs to be specified") + } + if keySpecified, certSpecified := o.KeyFile != "", o.CertFile != ""; keySpecified != certSpecified { + return fmt.Errorf("pemfile: private key file and identity cert file should be both specified or not specified") + } + // C-core has a limitation that they cannot verify that a certificate file + // matches a key file. So, the only way to get around this is to make sure + // that both files are in the same directory and that they do an atomic + // read. Even though Java/Go do not have this limitation, we want the + // overall plugin behavior to be consistent across languages. + if certDir, keyDir := filepath.Dir(o.CertFile), filepath.Dir(o.KeyFile); certDir != keyDir { + return errors.New("pemfile: certificate and key file must be in the same directory") + } + return nil +} + +// NewProvider returns a new certificate provider plugin that is configured to +// watch the PEM files specified in the passed in options. +func NewProvider(o Options) (certprovider.Provider, error) { + if err := o.validate(); err != nil { + return nil, err + } + return newProvider(o), nil +} + +// newProvider is used to create a new certificate provider plugin after +// validating the options, and hence does not return an error. +func newProvider(o Options) certprovider.Provider { + if o.RefreshDuration == 0 { + o.RefreshDuration = defaultCertRefreshDuration + } + + provider := &watcher{opts: o} + if o.CertFile != "" && o.KeyFile != "" { + provider.identityDistributor = newDistributor() + } + if o.RootFile != "" { + provider.rootDistributor = newDistributor() + } + + ctx, cancel := context.WithCancel(context.Background()) + provider.cancel = cancel + go provider.run(ctx) + return provider +} + +// watcher is a certificate provider plugin that implements the +// certprovider.Provider interface. It watches a set of certificate and key +// files and provides the most up-to-date key material for consumption by +// credentials implementation. +type watcher struct { + identityDistributor distributor + rootDistributor distributor + opts Options + certFileContents []byte + keyFileContents []byte + rootFileContents []byte + cancel context.CancelFunc +} + +// distributor wraps the methods on certprovider.Distributor which are used by +// the plugin. This is very useful in tests which need to know exactly when the +// plugin updates its key material. +type distributor interface { + KeyMaterial(ctx context.Context) (*certprovider.KeyMaterial, error) + Set(km *certprovider.KeyMaterial, err error) + Stop() +} + +// updateIdentityDistributor checks if the cert/key files that the plugin is +// watching have changed, and if so, reads the new contents and updates the +// identityDistributor with the new key material. +// +// Skips updates when file reading or parsing fails. +// TODO(easwars): Retry with limit (on the number of retries or the amount of +// time) upon failures. +func (w *watcher) updateIdentityDistributor() { + if w.identityDistributor == nil { + return + } + + certFileContents, err := os.ReadFile(w.opts.CertFile) + if err != nil { + logger.Warningf("certFile (%s) read failed: %v", w.opts.CertFile, err) + return + } + keyFileContents, err := os.ReadFile(w.opts.KeyFile) + if err != nil { + logger.Warningf("keyFile (%s) read failed: %v", w.opts.KeyFile, err) + return + } + // If the file contents have not changed, skip updating the distributor. + if bytes.Equal(w.certFileContents, certFileContents) && bytes.Equal(w.keyFileContents, keyFileContents) { + return + } + + cert, err := tls.X509KeyPair(certFileContents, keyFileContents) + if err != nil { + logger.Warningf("tls.X509KeyPair(%q, %q) failed: %v", certFileContents, keyFileContents, err) + return + } + w.certFileContents = certFileContents + w.keyFileContents = keyFileContents + w.identityDistributor.Set(&certprovider.KeyMaterial{Certs: []tls.Certificate{cert}}, nil) +} + +// updateRootDistributor checks if the root cert file that the plugin is +// watching hs changed, and if so, updates the rootDistributor with the new key +// material. +// +// Skips updates when root cert reading or parsing fails. +// TODO(easwars): Retry with limit (on the number of retries or the amount of +// time) upon failures. +func (w *watcher) updateRootDistributor() { + if w.rootDistributor == nil { + return + } + + rootFileContents, err := os.ReadFile(w.opts.RootFile) + if err != nil { + logger.Warningf("rootFile (%s) read failed: %v", w.opts.RootFile, err) + return + } + trustPool := x509.NewCertPool() + if !trustPool.AppendCertsFromPEM(rootFileContents) { + logger.Warning("failed to parse root certificate") + return + } + // If the file contents have not changed, skip updating the distributor. + if bytes.Equal(w.rootFileContents, rootFileContents) { + return + } + + w.rootFileContents = rootFileContents + w.rootDistributor.Set(&certprovider.KeyMaterial{Roots: trustPool}, nil) +} + +// run is a long running goroutine which watches the configured files for +// changes, and pushes new key material into the appropriate distributors which +// is returned from calls to KeyMaterial(). +func (w *watcher) run(ctx context.Context) { + ticker := time.NewTicker(w.opts.RefreshDuration) + for { + w.updateIdentityDistributor() + w.updateRootDistributor() + select { + case <-ctx.Done(): + ticker.Stop() + if w.identityDistributor != nil { + w.identityDistributor.Stop() + } + if w.rootDistributor != nil { + w.rootDistributor.Stop() + } + return + case <-ticker.C: + } + } +} + +// KeyMaterial returns the key material sourced by the watcher. +// Callers are expected to use the returned value as read-only. +func (w *watcher) KeyMaterial(ctx context.Context) (*certprovider.KeyMaterial, error) { + km := &certprovider.KeyMaterial{} + if w.identityDistributor != nil { + identityKM, err := w.identityDistributor.KeyMaterial(ctx) + if err != nil { + return nil, err + } + km.Certs = identityKM.Certs + } + if w.rootDistributor != nil { + rootKM, err := w.rootDistributor.KeyMaterial(ctx) + if err != nil { + return nil, err + } + km.Roots = rootKM.Roots + } + return km, nil +} + +// Close cleans up resources allocated by the watcher. +func (w *watcher) Close() { + w.cancel() +} diff --git a/credentials/tls/certprovider/pemfile/watcher_test.go b/credentials/tls/certprovider/pemfile/watcher_test.go new file mode 100644 index 000000000000..521f762d3a41 --- /dev/null +++ b/credentials/tls/certprovider/pemfile/watcher_test.go @@ -0,0 +1,444 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package pemfile + +import ( + "context" + "fmt" + "os" + "path" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "google.golang.org/grpc/credentials/tls/certprovider" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/testdata" +) + +const ( + // These are the names of files inside temporary directories, which the + // plugin is asked to watch. + certFile = "cert.pem" + keyFile = "key.pem" + rootFile = "ca.pem" + + defaultTestRefreshDuration = 100 * time.Millisecond + defaultTestTimeout = 5 * time.Second +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func compareKeyMaterial(got, want *certprovider.KeyMaterial) error { + if len(got.Certs) != len(want.Certs) { + return fmt.Errorf("keyMaterial certs = %+v, want %+v", got, want) + } + for i := 0; i < len(got.Certs); i++ { + if !got.Certs[i].Leaf.Equal(want.Certs[i].Leaf) { + return fmt.Errorf("keyMaterial certs = %+v, want %+v", got, want) + } + } + + // x509.CertPool contains only unexported fields some of which contain other + // unexported fields. So usage of cmp.AllowUnexported() or + // cmpopts.IgnoreUnexported() does not help us much here. Also, the standard + // library does not provide a way to compare CertPool values. Comparing the + // subjects field of the certs in the CertPool seems like a reasonable + // approach. + if gotR, wantR := got.Roots.Subjects(), want.Roots.Subjects(); !cmp.Equal(gotR, wantR, cmpopts.EquateEmpty()) { + return fmt.Errorf("keyMaterial roots = %v, want %v", gotR, wantR) + } + return nil +} + +// TestNewProvider tests the NewProvider() function with different inputs. +func (s) TestNewProvider(t *testing.T) { + tests := []struct { + desc string + options Options + wantError bool + }{ + { + desc: "No credential files specified", + options: Options{}, + wantError: true, + }, + { + desc: "Only identity cert is specified", + options: Options{ + CertFile: testdata.Path("x509/client1_cert.pem"), + }, + wantError: true, + }, + { + desc: "Only identity key is specified", + options: Options{ + KeyFile: testdata.Path("x509/client1_key.pem"), + }, + wantError: true, + }, + { + desc: "Identity cert/key pair is specified", + options: Options{ + KeyFile: testdata.Path("x509/client1_key.pem"), + CertFile: testdata.Path("x509/client1_cert.pem"), + }, + }, + { + desc: "Only root certs are specified", + options: Options{ + RootFile: testdata.Path("x509/client_ca_cert.pem"), + }, + }, + { + desc: "Everything is specified", + options: Options{ + KeyFile: testdata.Path("x509/client1_key.pem"), + CertFile: testdata.Path("x509/client1_cert.pem"), + RootFile: testdata.Path("x509/client_ca_cert.pem"), + }, + wantError: false, + }, + } + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + provider, err := NewProvider(test.options) + if (err != nil) != test.wantError { + t.Fatalf("NewProvider(%v) = %v, want %v", test.options, err, test.wantError) + } + if err != nil { + return + } + provider.Close() + }) + } +} + +// wrappedDistributor wraps a distributor and pushes on a channel whenever new +// key material is pushed to the distributor. +type wrappedDistributor struct { + *certprovider.Distributor + distCh *testutils.Channel +} + +func newWrappedDistributor(distCh *testutils.Channel) *wrappedDistributor { + return &wrappedDistributor{ + distCh: distCh, + Distributor: certprovider.NewDistributor(), + } +} + +func (wd *wrappedDistributor) Set(km *certprovider.KeyMaterial, err error) { + wd.Distributor.Set(km, err) + wd.distCh.Send(nil) +} + +func createTmpFile(t *testing.T, src, dst string) { + t.Helper() + + data, err := os.ReadFile(src) + if err != nil { + t.Fatalf("os.ReadFile(%q) failed: %v", src, err) + } + if err := os.WriteFile(dst, data, os.ModePerm); err != nil { + t.Fatalf("os.WriteFile(%q) failed: %v", dst, err) + } + t.Logf("Wrote file at: %s", dst) + t.Logf("%s", string(data)) +} + +// createTempDirWithFiles creates a temporary directory under the system default +// tempDir with the given dirSuffix. It also reads from certSrc, keySrc and +// rootSrc files are creates appropriate files under the newly create tempDir. +// Returns the name of the created tempDir. +func createTmpDirWithFiles(t *testing.T, dirSuffix, certSrc, keySrc, rootSrc string) string { + t.Helper() + + // Create a temp directory. Passing an empty string for the first argument + // uses the system temp directory. + dir, err := os.MkdirTemp("", dirSuffix) + if err != nil { + t.Fatalf("os.MkdirTemp() failed: %v", err) + } + t.Logf("Using tmpdir: %s", dir) + + createTmpFile(t, testdata.Path(certSrc), path.Join(dir, certFile)) + createTmpFile(t, testdata.Path(keySrc), path.Join(dir, keyFile)) + createTmpFile(t, testdata.Path(rootSrc), path.Join(dir, rootFile)) + return dir +} + +// initializeProvider performs setup steps common to all tests (except the one +// which uses symlinks). +func initializeProvider(t *testing.T, testName string) (string, certprovider.Provider, *testutils.Channel, func()) { + t.Helper() + + // Override the newDistributor to one which pushes on a channel that we + // can block on. + origDistributorFunc := newDistributor + distCh := testutils.NewChannel() + d := newWrappedDistributor(distCh) + newDistributor = func() distributor { return d } + + // Create a new provider to watch the files in tmpdir. + dir := createTmpDirWithFiles(t, testName+"*", "x509/client1_cert.pem", "x509/client1_key.pem", "x509/client_ca_cert.pem") + opts := Options{ + CertFile: path.Join(dir, certFile), + KeyFile: path.Join(dir, keyFile), + RootFile: path.Join(dir, rootFile), + RefreshDuration: defaultTestRefreshDuration, + } + prov, err := NewProvider(opts) + if err != nil { + t.Fatalf("NewProvider(%+v) failed: %v", opts, err) + } + + // Make sure the provider picks up the files and pushes the key material on + // to the distributors. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + for i := 0; i < 2; i++ { + // Since we have root and identity certs, we need to make sure the + // update is pushed on both of them. + if _, err := distCh.Receive(ctx); err != nil { + t.Fatalf("timeout waiting for provider to read files and push key material to distributor: %v", err) + } + } + + return dir, prov, distCh, func() { + newDistributor = origDistributorFunc + prov.Close() + } +} + +// TestProvider_NoUpdate tests the case where a file watcher plugin is created +// successfully, and the underlying files do not change. Verifies that the +// plugin does not push new updates to the distributor in this case. +func (s) TestProvider_NoUpdate(t *testing.T) { + _, prov, distCh, cancel := initializeProvider(t, "no_update") + defer cancel() + + // Make sure the provider is healthy and returns key material. + ctx, cc := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cc() + if _, err := prov.KeyMaterial(ctx); err != nil { + t.Fatalf("provider.KeyMaterial() failed: %v", err) + } + + // Files haven't change. Make sure no updates are pushed by the provider. + sCtx, sc := context.WithTimeout(context.Background(), 2*defaultTestRefreshDuration) + defer sc() + if _, err := distCh.Receive(sCtx); err == nil { + t.Fatal("new key material pushed to distributor when underlying files did not change") + } +} + +// TestProvider_UpdateSuccess tests the case where a file watcher plugin is +// created successfully and the underlying files change. Verifies that the +// changes are picked up by the provider. +func (s) TestProvider_UpdateSuccess(t *testing.T) { + dir, prov, distCh, cancel := initializeProvider(t, "update_success") + defer cancel() + + // Make sure the provider is healthy and returns key material. + ctx, cc := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cc() + km1, err := prov.KeyMaterial(ctx) + if err != nil { + t.Fatalf("provider.KeyMaterial() failed: %v", err) + } + + // Change only the root file. + createTmpFile(t, testdata.Path("x509/server_ca_cert.pem"), path.Join(dir, rootFile)) + if _, err := distCh.Receive(ctx); err != nil { + t.Fatal("timeout waiting for new key material to be pushed to the distributor") + } + + // Make sure update is picked up. + km2, err := prov.KeyMaterial(ctx) + if err != nil { + t.Fatalf("provider.KeyMaterial() failed: %v", err) + } + if err := compareKeyMaterial(km1, km2); err == nil { + t.Fatal("expected provider to return new key material after update to underlying file") + } + + // Change only cert/key files. + createTmpFile(t, testdata.Path("x509/client2_cert.pem"), path.Join(dir, certFile)) + createTmpFile(t, testdata.Path("x509/client2_key.pem"), path.Join(dir, keyFile)) + if _, err := distCh.Receive(ctx); err != nil { + t.Fatal("timeout waiting for new key material to be pushed to the distributor") + } + + // Make sure update is picked up. + km3, err := prov.KeyMaterial(ctx) + if err != nil { + t.Fatalf("provider.KeyMaterial() failed: %v", err) + } + if err := compareKeyMaterial(km2, km3); err == nil { + t.Fatal("expected provider to return new key material after update to underlying file") + } +} + +// TestProvider_UpdateSuccessWithSymlink tests the case where a file watcher +// plugin is created successfully to watch files through a symlink and the +// symlink is updates to point to new files. Verifies that the changes are +// picked up by the provider. +func (s) TestProvider_UpdateSuccessWithSymlink(t *testing.T) { + // Override the newDistributor to one which pushes on a channel that we + // can block on. + origDistributorFunc := newDistributor + distCh := testutils.NewChannel() + d := newWrappedDistributor(distCh) + newDistributor = func() distributor { return d } + defer func() { newDistributor = origDistributorFunc }() + + // Create two tempDirs with different files. + dir1 := createTmpDirWithFiles(t, "update_with_symlink1_*", "x509/client1_cert.pem", "x509/client1_key.pem", "x509/client_ca_cert.pem") + dir2 := createTmpDirWithFiles(t, "update_with_symlink2_*", "x509/server1_cert.pem", "x509/server1_key.pem", "x509/server_ca_cert.pem") + + // Create a symlink under a new tempdir, and make it point to dir1. + tmpdir, err := os.MkdirTemp("", "test_symlink_*") + if err != nil { + t.Fatalf("os.MkdirTemp() failed: %v", err) + } + symLinkName := path.Join(tmpdir, "test_symlink") + if err := os.Symlink(dir1, symLinkName); err != nil { + t.Fatalf("failed to create symlink to %q: %v", dir1, err) + } + + // Create a provider which watches the files pointed to by the symlink. + opts := Options{ + CertFile: path.Join(symLinkName, certFile), + KeyFile: path.Join(symLinkName, keyFile), + RootFile: path.Join(symLinkName, rootFile), + RefreshDuration: defaultTestRefreshDuration, + } + prov, err := NewProvider(opts) + if err != nil { + t.Fatalf("NewProvider(%+v) failed: %v", opts, err) + } + defer prov.Close() + + // Make sure the provider picks up the files and pushes the key material on + // to the distributors. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + for i := 0; i < 2; i++ { + // Since we have root and identity certs, we need to make sure the + // update is pushed on both of them. + if _, err := distCh.Receive(ctx); err != nil { + t.Fatalf("timeout waiting for provider to read files and push key material to distributor: %v", err) + } + } + km1, err := prov.KeyMaterial(ctx) + if err != nil { + t.Fatalf("provider.KeyMaterial() failed: %v", err) + } + + // Update the symlink to point to dir2. + symLinkTmpName := path.Join(tmpdir, "test_symlink.tmp") + if err := os.Symlink(dir2, symLinkTmpName); err != nil { + t.Fatalf("failed to create symlink to %q: %v", dir2, err) + } + if err := os.Rename(symLinkTmpName, symLinkName); err != nil { + t.Fatalf("failed to update symlink: %v", err) + } + + // Make sure the provider picks up the new files and pushes the key material + // on to the distributors. + for i := 0; i < 2; i++ { + // Since we have root and identity certs, we need to make sure the + // update is pushed on both of them. + if _, err := distCh.Receive(ctx); err != nil { + t.Fatalf("timeout waiting for provider to read files and push key material to distributor: %v", err) + } + } + km2, err := prov.KeyMaterial(ctx) + if err != nil { + t.Fatalf("provider.KeyMaterial() failed: %v", err) + } + + if err := compareKeyMaterial(km1, km2); err == nil { + t.Fatal("expected provider to return new key material after symlink update") + } +} + +// TestProvider_UpdateFailure_ThenSuccess tests the case where updating cert/key +// files fail. Verifies that the failed update does not push anything on the +// distributor. Then the update succeeds, and the test verifies that the key +// material is updated. +func (s) TestProvider_UpdateFailure_ThenSuccess(t *testing.T) { + dir, prov, distCh, cancel := initializeProvider(t, "update_failure") + defer cancel() + + // Make sure the provider is healthy and returns key material. + ctx, cc := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cc() + km1, err := prov.KeyMaterial(ctx) + if err != nil { + t.Fatalf("provider.KeyMaterial() failed: %v", err) + } + + // Update only the cert file. The key file is left unchanged. This should + // lead to these two files being not compatible with each other. This + // simulates the case where the watching goroutine might catch the files in + // the midst of an update. + createTmpFile(t, testdata.Path("x509/server1_cert.pem"), path.Join(dir, certFile)) + + // Since the last update left the files in an incompatible state, the update + // should not be picked up by our provider. + sCtx, sc := context.WithTimeout(context.Background(), 2*defaultTestRefreshDuration) + defer sc() + if _, err := distCh.Receive(sCtx); err == nil { + t.Fatal("new key material pushed to distributor when underlying files did not change") + } + + // The provider should return key material corresponding to the old state. + km2, err := prov.KeyMaterial(ctx) + if err != nil { + t.Fatalf("provider.KeyMaterial() failed: %v", err) + } + if err := compareKeyMaterial(km1, km2); err != nil { + t.Fatalf("expected provider to not update key material: %v", err) + } + + // Update the key file to match the cert file. + createTmpFile(t, testdata.Path("x509/server1_key.pem"), path.Join(dir, keyFile)) + + // Make sure update is picked up. + if _, err := distCh.Receive(ctx); err != nil { + t.Fatal("timeout waiting for new key material to be pushed to the distributor") + } + km3, err := prov.KeyMaterial(ctx) + if err != nil { + t.Fatalf("provider.KeyMaterial() failed: %v", err) + } + if err := compareKeyMaterial(km2, km3); err == nil { + t.Fatal("expected provider to return new key material after update to underlying file") + } +} diff --git a/credentials/tls/certprovider/provider.go b/credentials/tls/certprovider/provider.go index 204e55686e91..07ba05b10746 100644 --- a/credentials/tls/certprovider/provider.go +++ b/credentials/tls/certprovider/provider.go @@ -18,7 +18,7 @@ // Package certprovider defines APIs for Certificate Providers in gRPC. // -// Experimental +// # Experimental // // Notice: All APIs in this package are experimental and may be removed in a // later release. @@ -29,8 +29,14 @@ import ( "crypto/tls" "crypto/x509" "errors" + + "google.golang.org/grpc/internal" ) +func init() { + internal.GetCertificateProviderBuilder = getBuilder +} + var ( // errProviderClosed is returned by Distributor.KeyMaterial when it is // closed. @@ -58,28 +64,14 @@ func getBuilder(name string) Builder { // Builder creates a Provider. type Builder interface { - // Build creates a new Provider and initializes it with the given config and - // options combination. - Build(StableConfig, Options) Provider - - // ParseConfig converts config input in a format specific to individual - // implementations and returns an implementation of the StableConfig - // interface. - // Equivalent configurations must return StableConfig types whose - // Canonical() method returns the same output. - ParseConfig(interface{}) (StableConfig, error) + // ParseConfig parses the given config, which is in a format specific to individual + // implementations, and returns a BuildableConfig on success. + ParseConfig(any) (*BuildableConfig, error) // Name returns the name of providers built by this builder. Name() string } -// StableConfig wraps the method to return a stable Provider configuration. -type StableConfig interface { - // Canonical returns Provider config as an arbitrary byte slice. - // Equivalent configurations must return the same output. - Canonical() []byte -} - // Provider makes it possible to keep channel credential implementations up to // date with secrets that they rely on to secure communications on the // underlying channel. @@ -104,8 +96,8 @@ type KeyMaterial struct { Roots *x509.CertPool } -// Options contains configuration knobs passed to a Provider at creation time. -type Options struct { +// BuildOptions contains parameters passed to a Provider at build time. +type BuildOptions struct { // CertName holds the certificate name, whose key material is of interest to // the caller. CertName string diff --git a/credentials/tls/certprovider/store.go b/credentials/tls/certprovider/store.go index 990e6ab06c34..5c72f192cc33 100644 --- a/credentials/tls/certprovider/store.go +++ b/credentials/tls/certprovider/store.go @@ -31,15 +31,15 @@ var provStore = &store{ // storeKey acts as the key to the map of providers maintained by the store. A // combination of provider name and configuration is used to uniquely identify // every provider instance in the store. Go maps need to be indexed by -// comparable types, so the provider configuration is converted from -// `interface{}` to string using the ParseConfig method while creating this key. +// comparable types, so the provider configuration is converted from `any` to +// `string` using the ParseConfig method while creating this key. type storeKey struct { // name of the certificate provider. name string // configuration of the certificate provider in string form. config string // opts contains the certificate name and other keyMaterial options. - opts Options + opts BuildOptions } // wrappedProvider wraps a provider instance with a reference count. @@ -59,35 +59,55 @@ type store struct { providers map[storeKey]*wrappedProvider } -// GetProvider returns a provider instance from which keyMaterial can be read. -// -// name is the registered name of the provider, config is the provider-specific -// configuration, opts contains extra information that controls the keyMaterial -// returned by the provider. -// -// Implementations of the Builder interface should clearly document the type of -// configuration accepted by them. -// -// If a provider exists for passed arguments, its reference count is incremented -// before returning. If no provider exists for the passed arguments, a new one -// is created using the registered builder. If no registered builder is found, -// or the provider configuration is rejected by it, a non-nil error is returned. -func GetProvider(name string, config interface{}, opts Options) (Provider, error) { - provStore.mu.Lock() - defer provStore.mu.Unlock() +// Close overrides the Close method of the embedded provider. It releases the +// reference held by the caller on the underlying provider and if the +// provider's reference count reaches zero, it is removed from the store, and +// its Close method is also invoked. +func (wp *wrappedProvider) Close() { + ps := wp.store + ps.mu.Lock() + defer ps.mu.Unlock() - builder := getBuilder(name) - if builder == nil { - return nil, fmt.Errorf("no registered builder for provider name: %s", name) + wp.refCount-- + if wp.refCount == 0 { + wp.Provider.Close() + delete(ps.providers, wp.storeKey) } - stableConfig, err := builder.ParseConfig(config) - if err != nil { - return nil, err +} + +// BuildableConfig wraps parsed provider configuration and functionality to +// instantiate provider instances. +type BuildableConfig struct { + name string + config []byte + starter func(BuildOptions) Provider + pStore *store +} + +// NewBuildableConfig creates a new BuildableConfig with the given arguments. +// Provider implementations are expected to invoke this function after parsing +// the given configuration as part of their ParseConfig() method. +// Equivalent configurations are expected to invoke this function with the same +// config argument. +func NewBuildableConfig(name string, config []byte, starter func(BuildOptions) Provider) *BuildableConfig { + return &BuildableConfig{ + name: name, + config: config, + starter: starter, + pStore: provStore, } +} + +// Build kicks off a provider instance with the wrapped configuration. Multiple +// invocations of this method with the same opts will result in provider +// instances being reused. +func (bc *BuildableConfig) Build(opts BuildOptions) (Provider, error) { + provStore.mu.Lock() + defer provStore.mu.Unlock() sk := storeKey{ - name: name, - config: string(stableConfig.Canonical()), + name: bc.name, + config: string(bc.config), opts: opts, } if wp, ok := provStore.providers[sk]; ok { @@ -95,9 +115,9 @@ func GetProvider(name string, config interface{}, opts Options) (Provider, error return wp, nil } - provider := builder.Build(stableConfig, opts) + provider := bc.starter(opts) if provider == nil { - return nil, fmt.Errorf("certprovider.Build(%v) failed", sk) + return nil, fmt.Errorf("provider(%q, %q).Build(%v) failed", sk.name, sk.config, opts) } wp := &wrappedProvider{ Provider: provider, @@ -109,18 +129,28 @@ func GetProvider(name string, config interface{}, opts Options) (Provider, error return wp, nil } -// Close overrides the Close method of the embedded provider. It releases the -// reference held by the caller on the underlying provider and if the -// provider's reference count reaches zero, it is removed from the store, and -// its Close method is also invoked. -func (wp *wrappedProvider) Close() { - ps := wp.store - ps.mu.Lock() - defer ps.mu.Unlock() +// String returns the provider name and config as a colon separated string. +func (bc *BuildableConfig) String() string { + return fmt.Sprintf("%s:%s", bc.name, string(bc.config)) +} - wp.refCount-- - if wp.refCount == 0 { - wp.Provider.Close() - delete(ps.providers, wp.storeKey) +// ParseConfig is a convenience function to create a BuildableConfig given a +// provider name and configuration. Returns an error if there is no registered +// builder for the given name or if the config parsing fails. +func ParseConfig(name string, config any) (*BuildableConfig, error) { + parser := getBuilder(name) + if parser == nil { + return nil, fmt.Errorf("no certificate provider builder found for %q", name) + } + return parser.ParseConfig(config) +} + +// GetProvider is a convenience function to create a provider given the name, +// config and build options. +func GetProvider(name string, config any, opts BuildOptions) (Provider, error) { + bc, err := ParseConfig(name, config) + if err != nil { + return nil, err } + return bc.Build(opts) } diff --git a/credentials/tls/certprovider/store_test.go b/credentials/tls/certprovider/store_test.go index a116148945eb..dce61912977b 100644 --- a/credentials/tls/certprovider/store_test.go +++ b/credentials/tls/certprovider/store_test.go @@ -1,5 +1,3 @@ -// +build go1.13 - /* * * Copyright 2020 gRPC authors. @@ -26,21 +24,23 @@ import ( "crypto/x509" "errors" "fmt" - "io/ioutil" - "reflect" + "os" "testing" "time" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "google.golang.org/grpc/internal/grpctest" "google.golang.org/grpc/internal/testutils" "google.golang.org/grpc/testdata" ) const ( - fakeProvider1Name = "fake-certificate-provider-1" - fakeProvider2Name = "fake-certificate-provider-2" - fakeConfig = "my fake config" - defaultTestTimeout = 1 * time.Second + fakeProvider1Name = "fake-certificate-provider-1" + fakeProvider2Name = "fake-certificate-provider-2" + fakeConfig = "my fake config" + defaultTestTimeout = 5 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond ) var fpb1, fpb2 *fakeProviderBuilder @@ -73,36 +73,36 @@ type fakeProviderBuilder struct { providerChan *testutils.Channel } -func (b *fakeProviderBuilder) Build(StableConfig, Options) Provider { - p := &fakeProvider{Distributor: NewDistributor()} - b.providerChan.Send(p) - return p -} - -func (b *fakeProviderBuilder) ParseConfig(config interface{}) (StableConfig, error) { +func (b *fakeProviderBuilder) ParseConfig(config any) (*BuildableConfig, error) { s, ok := config.(string) if !ok { return nil, fmt.Errorf("providerBuilder %s received config of type %T, want string", b.name, config) } - return &fakeStableConfig{config: s}, nil + return NewBuildableConfig(b.name, []byte(s), func(BuildOptions) Provider { + fp := &fakeProvider{ + Distributor: NewDistributor(), + config: s, + } + b.providerChan.Send(fp) + return fp + }), nil } func (b *fakeProviderBuilder) Name() string { return b.name } -type fakeStableConfig struct { - config string -} - -func (c *fakeStableConfig) Canonical() []byte { - return []byte(c.config) -} - // fakeProvider is an implementation of the Provider interface which provides a // method for tests to invoke to push new key materials. type fakeProvider struct { *Distributor + config string +} + +func (p *fakeProvider) Start(BuildOptions) Provider { + // This is practically a no-op since this provider doesn't do any work which + // needs to be started at this point. + return p } // newKeyMaterial allows tests to push new key material to the fake provider @@ -126,7 +126,7 @@ func loadKeyMaterials(t *testing.T, cert, key, ca string) *KeyMaterial { t.Fatalf("Failed to load keyPair: %v", err) } - pemData, err := ioutil.ReadFile(testdata.Path(ca)) + pemData, err := os.ReadFile(testdata.Path(ca)) if err != nil { t.Fatal(err) } @@ -153,33 +153,47 @@ func readAndVerifyKeyMaterial(ctx context.Context, kmr kmReader, wantKM *KeyMate } func compareKeyMaterial(got, want *KeyMaterial) error { - // TODO(easwars): Remove all references to reflect.DeepEqual and use - // cmp.Equal instead. Currently, the later panics because x509.Certificate - // type defines an Equal method, but does not check for nil. This has been - // fixed in - // https://github.com/golang/go/commit/89865f8ba64ccb27f439cce6daaa37c9aa38f351, - // but this is only available starting go1.14. So, once we remove support - // for go1.13, we can make the switch. - if !reflect.DeepEqual(got, want) { - return fmt.Errorf("provider.KeyMaterial() = %+v, want %+v", got, want) + if len(got.Certs) != len(want.Certs) { + return fmt.Errorf("keyMaterial certs = %+v, want %+v", got, want) + } + for i := 0; i < len(got.Certs); i++ { + if !got.Certs[i].Leaf.Equal(want.Certs[i].Leaf) { + return fmt.Errorf("keyMaterial certs = %+v, want %+v", got, want) + } + } + + // x509.CertPool contains only unexported fields some of which contain other + // unexported fields. So usage of cmp.AllowUnexported() or + // cmpopts.IgnoreUnexported() does not help us much here. Also, the standard + // library does not provide a way to compare CertPool values. Comparing the + // subjects field of the certs in the CertPool seems like a reasonable + // approach. + if gotR, wantR := got.Roots.Subjects(), want.Roots.Subjects(); !cmp.Equal(gotR, wantR, cmpopts.EquateEmpty()) { + return fmt.Errorf("keyMaterial roots = %v, want %v", gotR, wantR) } return nil } +func createProvider(t *testing.T, name, config string, opts BuildOptions) Provider { + t.Helper() + prov, err := GetProvider(name, config, opts) + if err != nil { + t.Fatalf("GetProvider(%s, %s, %v) failed: %v", name, config, opts, err) + } + return prov +} + // TestStoreSingleProvider creates a single provider through the store and calls // methods on them. func (s) TestStoreSingleProvider(t *testing.T) { - // Create a Provider through the store. - kmOpts := Options{CertName: "default"} - prov, err := GetProvider(fakeProvider1Name, fakeConfig, kmOpts) - if err != nil { - t.Fatalf("GetProvider(%s, %s, %v) failed: %v", fakeProvider1Name, fakeConfig, kmOpts, err) - } + prov := createProvider(t, fakeProvider1Name, fakeConfig, BuildOptions{CertName: "default"}) defer prov.Close() // Our fakeProviderBuilder pushes newly created providers on a channel. Grab // the fake provider from that channel. - p, err := fpb1.providerChan.Receive() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + p, err := fpb1.providerChan.Receive(ctx) if err != nil { t.Fatalf("Timeout when expecting certProvider %q to be created", fakeProvider1Name) } @@ -188,9 +202,9 @@ func (s) TestStoreSingleProvider(t *testing.T) { // Attempt to read from key material from the Provider returned by the // store. This will fail because we have not pushed any key material into // our fake provider. - ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) - defer cancel() - if err := readAndVerifyKeyMaterial(ctx, prov, nil); !errors.Is(err, context.DeadlineExceeded) { + sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + if err := readAndVerifyKeyMaterial(sCtx, prov, nil); !errors.Is(err, context.DeadlineExceeded) { t.Fatal(err) } @@ -198,8 +212,6 @@ func (s) TestStoreSingleProvider(t *testing.T) { // and attempt to read from the Provider returned by the store. testKM1 := loadKeyMaterials(t, "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem") fakeProv.newKeyMaterial(testKM1, nil) - ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout) - defer cancel() if err := readAndVerifyKeyMaterial(ctx, prov, testKM1); err != nil { t.Fatal(err) } @@ -208,8 +220,6 @@ func (s) TestStoreSingleProvider(t *testing.T) { // updated key material. testKM2 := loadKeyMaterials(t, "x509/server2_cert.pem", "x509/server2_key.pem", "x509/client_ca_cert.pem") fakeProv.newKeyMaterial(testKM2, nil) - ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout) - defer cancel() if err := readAndVerifyKeyMaterial(ctx, prov, testKM2); err != nil { t.Fatal(err) } @@ -222,33 +232,38 @@ func (s) TestStoreSingleProvider(t *testing.T) { func (s) TestStoreSingleProviderSameConfigDifferentOpts(t *testing.T) { // Create three readers on the same fake provider. Two of these readers use // certName `foo`, while the third one uses certName `bar`. - optsFoo := Options{CertName: "foo"} - optsBar := Options{CertName: "bar"} - provFoo1, err := GetProvider(fakeProvider1Name, fakeConfig, optsFoo) - if err != nil { - t.Fatalf("GetProvider(%s, %s, %v) failed: %v", fakeProvider1Name, fakeConfig, optsFoo, err) - } - defer provFoo1.Close() - provFoo2, err := GetProvider(fakeProvider1Name, fakeConfig, optsFoo) - if err != nil { - t.Fatalf("GetProvider(%s, %s, %v) failed: %v", fakeProvider1Name, fakeConfig, optsFoo, err) - } - defer provFoo2.Close() + optsFoo := BuildOptions{CertName: "foo"} + provFoo1 := createProvider(t, fakeProvider1Name, fakeConfig, optsFoo) + provFoo2 := createProvider(t, fakeProvider1Name, fakeConfig, optsFoo) + defer func() { + provFoo1.Close() + provFoo2.Close() + }() + // Our fakeProviderBuilder pushes newly created providers on a channel. // Grab the fake provider for optsFoo. - p, err := fpb1.providerChan.Receive() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + p, err := fpb1.providerChan.Receive(ctx) if err != nil { t.Fatalf("Timeout when expecting certProvider %q to be created", fakeProvider1Name) } fakeProvFoo := p.(*fakeProvider) - provBar1, err := GetProvider(fakeProvider1Name, fakeConfig, optsBar) - if err != nil { - t.Fatalf("GetProvider(%s, %s, %v) failed: %v", fakeProvider1Name, fakeConfig, optsBar, err) + // Make sure only provider was created by the builder so far. The store + // should be able to share the providers. + sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + if _, err := fpb1.providerChan.Receive(sCtx); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("A second provider created when expected to be shared by the store") } + + optsBar := BuildOptions{CertName: "bar"} + provBar1 := createProvider(t, fakeProvider1Name, fakeConfig, optsBar) defer provBar1.Close() + // Grab the fake provider for optsBar. - p, err = fpb1.providerChan.Receive() + p, err = fpb1.providerChan.Receive(ctx) if err != nil { t.Fatalf("Timeout when expecting certProvider %q to be created", fakeProvider1Name) } @@ -258,15 +273,15 @@ func (s) TestStoreSingleProviderSameConfigDifferentOpts(t *testing.T) { // appropriate key material and the bar provider times out. fooKM := loadKeyMaterials(t, "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem") fakeProvFoo.newKeyMaterial(fooKM, nil) - ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) - defer cancel() if err := readAndVerifyKeyMaterial(ctx, provFoo1, fooKM); err != nil { t.Fatal(err) } if err := readAndVerifyKeyMaterial(ctx, provFoo2, fooKM); err != nil { t.Fatal(err) } - if err := readAndVerifyKeyMaterial(ctx, provBar1, nil); !errors.Is(err, context.DeadlineExceeded) { + sCtx, sCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + if err := readAndVerifyKeyMaterial(sCtx, provBar1, nil); !errors.Is(err, context.DeadlineExceeded) { t.Fatal(err) } @@ -274,8 +289,6 @@ func (s) TestStoreSingleProviderSameConfigDifferentOpts(t *testing.T) { // appropriate key material. barKM := loadKeyMaterials(t, "x509/server2_cert.pem", "x509/server2_key.pem", "x509/client_ca_cert.pem") fakeProvBar.newKeyMaterial(barKM, nil) - ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout) - defer cancel() if err := readAndVerifyKeyMaterial(ctx, provBar1, barKM); err != nil { t.Fatal(err) } @@ -292,29 +305,26 @@ func (s) TestStoreSingleProviderSameConfigDifferentOpts(t *testing.T) { // would take place. func (s) TestStoreSingleProviderDifferentConfigs(t *testing.T) { // Create two providers of the same type, but with different configs. - opts := Options{CertName: "foo"} + opts := BuildOptions{CertName: "foo"} cfg1 := fakeConfig + "1111" cfg2 := fakeConfig + "2222" - prov1, err := GetProvider(fakeProvider1Name, cfg1, opts) - if err != nil { - t.Fatalf("GetProvider(%s, %s, %v) failed: %v", fakeProvider1Name, cfg1, opts, err) - } + + prov1 := createProvider(t, fakeProvider1Name, cfg1, opts) defer prov1.Close() // Our fakeProviderBuilder pushes newly created providers on a channel. Grab // the fake provider from that channel. - p1, err := fpb1.providerChan.Receive() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + p1, err := fpb1.providerChan.Receive(ctx) if err != nil { t.Fatalf("Timeout when expecting certProvider %q to be created", fakeProvider1Name) } fakeProv1 := p1.(*fakeProvider) - prov2, err := GetProvider(fakeProvider1Name, cfg2, opts) - if err != nil { - t.Fatalf("GetProvider(%s, %s, %v) failed: %v", fakeProvider1Name, cfg2, opts, err) - } + prov2 := createProvider(t, fakeProvider1Name, cfg2, opts) defer prov2.Close() // Grab the second provider from the channel. - p2, err := fpb1.providerChan.Receive() + p2, err := fpb1.providerChan.Receive(ctx) if err != nil { t.Fatalf("Timeout when expecting certProvider %q to be created", fakeProvider1Name) } @@ -325,8 +335,6 @@ func (s) TestStoreSingleProviderDifferentConfigs(t *testing.T) { km1 := loadKeyMaterials(t, "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem") fakeProv1.newKeyMaterial(km1, nil) fakeProv2.newKeyMaterial(km1, nil) - ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) - defer cancel() if err := readAndVerifyKeyMaterial(ctx, prov1, km1); err != nil { t.Fatal(err) } @@ -334,13 +342,11 @@ func (s) TestStoreSingleProviderDifferentConfigs(t *testing.T) { t.Fatal(err) } - // Push new key material into only one of the fake providers and and verify + // Push new key material into only one of the fake providers and verify // that the providers returned by the store return the appropriate key // material. km2 := loadKeyMaterials(t, "x509/server2_cert.pem", "x509/server2_key.pem", "x509/client_ca_cert.pem") fakeProv2.newKeyMaterial(km2, nil) - ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout) - defer cancel() if err := readAndVerifyKeyMaterial(ctx, prov1, km1); err != nil { t.Fatal(err) } @@ -358,27 +364,23 @@ func (s) TestStoreSingleProviderDifferentConfigs(t *testing.T) { // TestStoreMultipleProviders creates providers of different types and makes // sure closing of one does not affect the other. func (s) TestStoreMultipleProviders(t *testing.T) { - opts := Options{CertName: "foo"} - prov1, err := GetProvider(fakeProvider1Name, fakeConfig, opts) - if err != nil { - t.Fatalf("GetProvider(%s, %s, %v) failed: %v", fakeProvider1Name, fakeConfig, opts, err) - } + opts := BuildOptions{CertName: "foo"} + prov1 := createProvider(t, fakeProvider1Name, fakeConfig, opts) defer prov1.Close() // Our fakeProviderBuilder pushes newly created providers on a channel. Grab // the fake provider from that channel. - p1, err := fpb1.providerChan.Receive() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + p1, err := fpb1.providerChan.Receive(ctx) if err != nil { t.Fatalf("Timeout when expecting certProvider %q to be created", fakeProvider1Name) } fakeProv1 := p1.(*fakeProvider) - prov2, err := GetProvider(fakeProvider2Name, fakeConfig, opts) - if err != nil { - t.Fatalf("GetProvider(%s, %s, %v) failed: %v", fakeProvider1Name, fakeConfig, opts, err) - } + prov2 := createProvider(t, fakeProvider2Name, fakeConfig, opts) defer prov2.Close() // Grab the second provider from the channel. - p2, err := fpb2.providerChan.Receive() + p2, err := fpb2.providerChan.Receive(ctx) if err != nil { t.Fatalf("Timeout when expecting certProvider %q to be created", fakeProvider2Name) } @@ -390,8 +392,6 @@ func (s) TestStoreMultipleProviders(t *testing.T) { fakeProv1.newKeyMaterial(km1, nil) km2 := loadKeyMaterials(t, "x509/server2_cert.pem", "x509/server2_key.pem", "x509/client_ca_cert.pem") fakeProv2.newKeyMaterial(km2, nil) - ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) - defer cancel() if err := readAndVerifyKeyMaterial(ctx, prov1, km1); err != nil { t.Fatal(err) } diff --git a/credentials/xds/xds.go b/credentials/xds/xds.go new file mode 100644 index 000000000000..d232e6786746 --- /dev/null +++ b/credentials/xds/xds.go @@ -0,0 +1,279 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package xds provides a transport credentials implementation where the +// security configuration is pushed by a management server using xDS APIs. +package xds + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "net" + "time" + + "google.golang.org/grpc/credentials" + credinternal "google.golang.org/grpc/internal/credentials" + xdsinternal "google.golang.org/grpc/internal/credentials/xds" +) + +// ClientOptions contains parameters to configure a new client-side xDS +// credentials implementation. +type ClientOptions struct { + // FallbackCreds specifies the fallback credentials to be used when either + // the `xds` scheme is not used in the user's dial target or when the + // management server does not return any security configuration. Attempts to + // create client credentials without fallback credentials will fail. + FallbackCreds credentials.TransportCredentials +} + +// NewClientCredentials returns a new client-side transport credentials +// implementation which uses xDS APIs to fetch its security configuration. +func NewClientCredentials(opts ClientOptions) (credentials.TransportCredentials, error) { + if opts.FallbackCreds == nil { + return nil, errors.New("missing fallback credentials") + } + return &credsImpl{ + isClient: true, + fallback: opts.FallbackCreds, + }, nil +} + +// ServerOptions contains parameters to configure a new server-side xDS +// credentials implementation. +type ServerOptions struct { + // FallbackCreds specifies the fallback credentials to be used when the + // management server does not return any security configuration. Attempts to + // create server credentials without fallback credentials will fail. + FallbackCreds credentials.TransportCredentials +} + +// NewServerCredentials returns a new server-side transport credentials +// implementation which uses xDS APIs to fetch its security configuration. +func NewServerCredentials(opts ServerOptions) (credentials.TransportCredentials, error) { + if opts.FallbackCreds == nil { + return nil, errors.New("missing fallback credentials") + } + return &credsImpl{ + isClient: false, + fallback: opts.FallbackCreds, + }, nil +} + +// credsImpl is an implementation of the credentials.TransportCredentials +// interface which uses xDS APIs to fetch its security configuration. +type credsImpl struct { + isClient bool + fallback credentials.TransportCredentials +} + +// ClientHandshake performs the TLS handshake on the client-side. +// +// It looks for the presence of a HandshakeInfo value in the passed in context +// (added using a call to NewContextWithHandshakeInfo()), and retrieves identity +// and root certificates from there. It also retrieves a list of acceptable SANs +// and uses a custom verification function to validate the certificate presented +// by the peer. It uses fallback credentials if no HandshakeInfo is present in +// the passed in context. +func (c *credsImpl) ClientHandshake(ctx context.Context, authority string, rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) { + if !c.isClient { + return nil, nil, errors.New("ClientHandshake() is not supported for server credentials") + } + + // The CDS balancer constructs a new HandshakeInfo using a call to + // NewHandshakeInfo(), and then adds it to the attributes field of the + // resolver.Address when handling calls to NewSubConn(). The transport layer + // takes care of shipping these attributes in the context to this handshake + // function. We first read the credentials.ClientHandshakeInfo type from the + // context, which contains the attributes added by the CDS balancer. We then + // read the HandshakeInfo from the attributes to get to the actual data that + // we need here for the handshake. + chi := credentials.ClientHandshakeInfoFromContext(ctx) + // If there are no attributes in the received context or the attributes does + // not contain a HandshakeInfo, it could either mean that the user did not + // specify an `xds` scheme in their dial target or that the xDS server did + // not provide any security configuration. In both of these cases, we use + // the fallback credentials specified by the user. + if chi.Attributes == nil { + return c.fallback.ClientHandshake(ctx, authority, rawConn) + } + hi := xdsinternal.GetHandshakeInfo(chi.Attributes) + if hi.UseFallbackCreds() { + return c.fallback.ClientHandshake(ctx, authority, rawConn) + } + + // We build the tls.Config with the following values + // 1. Root certificate as returned by the root provider. + // 2. Identity certificate as returned by the identity provider. This may be + // empty on the client side, if the client is not doing mTLS. + // 3. InsecureSkipVerify to true. Certificates used in Mesh environments + // usually contains the identity of the workload presenting the + // certificate as a SAN (instead of a hostname in the CommonName field). + // This means that normal certificate verification as done by the + // standard library will fail. + // 4. Key usage to match whether client/server usage. + // 5. A `VerifyPeerCertificate` function which performs normal peer + // cert verification using configured roots, and the custom SAN checks. + cfg, err := hi.ClientSideTLSConfig(ctx) + if err != nil { + return nil, nil, err + } + cfg.VerifyPeerCertificate = func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error { + // Parse all raw certificates presented by the peer. + var certs []*x509.Certificate + for _, rc := range rawCerts { + cert, err := x509.ParseCertificate(rc) + if err != nil { + return err + } + certs = append(certs, cert) + } + + // Build the intermediates list and verify that the leaf certificate + // is signed by one of the root certificates. + intermediates := x509.NewCertPool() + for _, cert := range certs[1:] { + intermediates.AddCert(cert) + } + opts := x509.VerifyOptions{ + Roots: cfg.RootCAs, + Intermediates: intermediates, + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + if _, err := certs[0].Verify(opts); err != nil { + return err + } + // The SANs sent by the MeshCA are encoded as SPIFFE IDs. We need to + // only look at the SANs on the leaf cert. + if cert := certs[0]; !hi.MatchingSANExists(cert) { + // TODO: Print the complete certificate once the x509 package + // supports a String() method on the Certificate type. + return fmt.Errorf("xds: received SANs {DNSNames: %v, EmailAddresses: %v, IPAddresses: %v, URIs: %v} do not match any of the accepted SANs", cert.DNSNames, cert.EmailAddresses, cert.IPAddresses, cert.URIs) + } + return nil + } + + // Perform the TLS handshake with the tls.Config that we have. We run the + // actual Handshake() function in a goroutine because we need to respect the + // deadline specified on the passed in context, and we need a way to cancel + // the handshake if the context is cancelled. + conn := tls.Client(rawConn, cfg) + errCh := make(chan error, 1) + go func() { + errCh <- conn.Handshake() + close(errCh) + }() + select { + case err := <-errCh: + if err != nil { + conn.Close() + return nil, nil, err + } + case <-ctx.Done(): + conn.Close() + return nil, nil, ctx.Err() + } + info := credentials.TLSInfo{ + State: conn.ConnectionState(), + CommonAuthInfo: credentials.CommonAuthInfo{ + SecurityLevel: credentials.PrivacyAndIntegrity, + }, + SPIFFEID: credinternal.SPIFFEIDFromState(conn.ConnectionState()), + } + return credinternal.WrapSyscallConn(rawConn, conn), info, nil +} + +// ServerHandshake performs the TLS handshake on the server-side. +func (c *credsImpl) ServerHandshake(rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) { + if c.isClient { + return nil, nil, errors.New("ServerHandshake is not supported for client credentials") + } + + // An xds-enabled gRPC server wraps the underlying raw net.Conn in a type + // that provides a way to retrieve `HandshakeInfo`, which contains the + // certificate providers to be used during the handshake. If the net.Conn + // passed to this function does not implement this interface, or if the + // `HandshakeInfo` does not contain the information we are looking for, we + // delegate the handshake to the fallback credentials. + hiConn, ok := rawConn.(interface { + XDSHandshakeInfo() (*xdsinternal.HandshakeInfo, error) + }) + if !ok { + return c.fallback.ServerHandshake(rawConn) + } + hi, err := hiConn.XDSHandshakeInfo() + if err != nil { + return nil, nil, err + } + if hi.UseFallbackCreds() { + return c.fallback.ServerHandshake(rawConn) + } + + // An xds-enabled gRPC server is expected to wrap the underlying raw + // net.Conn in a type which provides a way to retrieve the deadline set on + // it. If we cannot retrieve the deadline here, we fail (by setting deadline + // to time.Now()), instead of using a default deadline and possibly taking + // longer to eventually fail. + deadline := time.Now() + if dConn, ok := rawConn.(interface{ GetDeadline() time.Time }); ok { + deadline = dConn.GetDeadline() + } + ctx, cancel := context.WithDeadline(context.Background(), deadline) + defer cancel() + cfg, err := hi.ServerSideTLSConfig(ctx) + if err != nil { + return nil, nil, err + } + + conn := tls.Server(rawConn, cfg) + if err := conn.Handshake(); err != nil { + conn.Close() + return nil, nil, err + } + info := credentials.TLSInfo{ + State: conn.ConnectionState(), + CommonAuthInfo: credentials.CommonAuthInfo{ + SecurityLevel: credentials.PrivacyAndIntegrity, + }, + } + info.SPIFFEID = credinternal.SPIFFEIDFromState(conn.ConnectionState()) + return credinternal.WrapSyscallConn(rawConn, conn), info, nil +} + +// Info provides the ProtocolInfo of this TransportCredentials. +func (c *credsImpl) Info() credentials.ProtocolInfo { + return credentials.ProtocolInfo{SecurityProtocol: "tls"} +} + +// Clone makes a copy of this TransportCredentials. +func (c *credsImpl) Clone() credentials.TransportCredentials { + clone := *c + return &clone +} + +func (c *credsImpl) OverrideServerName(_ string) error { + return errors.New("serverName for peer validation must be configured as a list of acceptable SANs") +} + +// UsesXDS returns true if c uses xDS to fetch security configuration +// used at handshake time, and false otherwise. +func (c *credsImpl) UsesXDS() bool { + return true +} diff --git a/credentials/xds/xds_client_test.go b/credentials/xds/xds_client_test.go new file mode 100644 index 000000000000..2fd2e21cdd73 --- /dev/null +++ b/credentials/xds/xds_client_test.go @@ -0,0 +1,591 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "net" + "os" + "strings" + "testing" + "time" + + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/tls/certprovider" + icredentials "google.golang.org/grpc/internal/credentials" + xdsinternal "google.golang.org/grpc/internal/credentials/xds" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/xds/matcher" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/testdata" +) + +const ( + defaultTestTimeout = 1 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond + defaultTestCertSAN = "abc.test.example.com" + authority = "authority" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// Helper function to create a real TLS client credentials which is used as +// fallback credentials from multiple tests. +func makeFallbackClientCreds(t *testing.T) credentials.TransportCredentials { + creds, err := credentials.NewClientTLSFromFile(testdata.Path("x509/server_ca_cert.pem"), "x.test.example.com") + if err != nil { + t.Fatal(err) + } + return creds +} + +// testServer is a no-op server which listens on a local TCP port for incoming +// connections, and performs a manual TLS handshake on the received raw +// connection using a user specified handshake function. It then makes the +// result of the handshake operation available through a channel for tests to +// inspect. Tests should stop the testServer as part of their cleanup. +type testServer struct { + lis net.Listener + address string // Listening address of the test server. + handshakeFunc testHandshakeFunc // Test specified handshake function. + hsResult *testutils.Channel // Channel to deliver handshake results. +} + +// handshakeResult wraps the result of the handshake operation on the test +// server. It consists of TLS connection state and an error, if the handshake +// failed. This result is delivered on the `hsResult` channel on the testServer. +type handshakeResult struct { + connState tls.ConnectionState + err error +} + +// Configurable handshake function for the testServer. Tests can set this to +// simulate different conditions like handshake success, failure, timeout etc. +type testHandshakeFunc func(net.Conn) handshakeResult + +// newTestServerWithHandshakeFunc starts a new testServer which listens for +// connections on a local TCP port, and uses the provided custom handshake +// function to perform TLS handshake. +func newTestServerWithHandshakeFunc(f testHandshakeFunc) *testServer { + ts := &testServer{ + handshakeFunc: f, + hsResult: testutils.NewChannel(), + } + ts.start() + return ts +} + +// starts actually starts listening on a local TCP port, and spawns a goroutine +// to handle new connections. +func (ts *testServer) start() error { + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + return err + } + ts.lis = lis + ts.address = lis.Addr().String() + go ts.handleConn() + return nil +} + +// handleconn accepts a new raw connection, and invokes the test provided +// handshake function to perform TLS handshake, and returns the result on the +// `hsResult` channel. +func (ts *testServer) handleConn() { + for { + rawConn, err := ts.lis.Accept() + if err != nil { + // Once the listeners closed, Accept() will return with an error. + return + } + hsr := ts.handshakeFunc(rawConn) + ts.hsResult.Send(hsr) + } +} + +// stop closes the associated listener which causes the connection handling +// goroutine to exit. +func (ts *testServer) stop() { + ts.lis.Close() +} + +// A handshake function which simulates a successful handshake without client +// authentication (server does not request for client certificate during the +// handshake here). +func testServerTLSHandshake(rawConn net.Conn) handshakeResult { + cert, err := tls.LoadX509KeyPair(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem")) + if err != nil { + return handshakeResult{err: err} + } + cfg := &tls.Config{Certificates: []tls.Certificate{cert}} + conn := tls.Server(rawConn, cfg) + if err := conn.Handshake(); err != nil { + return handshakeResult{err: err} + } + return handshakeResult{connState: conn.ConnectionState()} +} + +// A handshake function which simulates a successful handshake with mutual +// authentication. +func testServerMutualTLSHandshake(rawConn net.Conn) handshakeResult { + cert, err := tls.LoadX509KeyPair(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem")) + if err != nil { + return handshakeResult{err: err} + } + pemData, err := os.ReadFile(testdata.Path("x509/client_ca_cert.pem")) + if err != nil { + return handshakeResult{err: err} + } + roots := x509.NewCertPool() + roots.AppendCertsFromPEM(pemData) + cfg := &tls.Config{ + Certificates: []tls.Certificate{cert}, + ClientCAs: roots, + } + conn := tls.Server(rawConn, cfg) + if err := conn.Handshake(); err != nil { + return handshakeResult{err: err} + } + return handshakeResult{connState: conn.ConnectionState()} +} + +// fakeProvider is an implementation of the certprovider.Provider interface +// which returns the configured key material and error in calls to +// KeyMaterial(). +type fakeProvider struct { + km *certprovider.KeyMaterial + err error +} + +func (f *fakeProvider) KeyMaterial(ctx context.Context) (*certprovider.KeyMaterial, error) { + return f.km, f.err +} + +func (f *fakeProvider) Close() {} + +// makeIdentityProvider creates a new instance of the fakeProvider returning the +// identity key material specified in the provider file paths. +func makeIdentityProvider(t *testing.T, certPath, keyPath string) certprovider.Provider { + t.Helper() + cert, err := tls.LoadX509KeyPair(testdata.Path(certPath), testdata.Path(keyPath)) + if err != nil { + t.Fatal(err) + } + return &fakeProvider{km: &certprovider.KeyMaterial{Certs: []tls.Certificate{cert}}} +} + +// makeRootProvider creates a new instance of the fakeProvider returning the +// root key material specified in the provider file paths. +func makeRootProvider(t *testing.T, caPath string) *fakeProvider { + pemData, err := os.ReadFile(testdata.Path(caPath)) + if err != nil { + t.Fatal(err) + } + roots := x509.NewCertPool() + roots.AppendCertsFromPEM(pemData) + return &fakeProvider{km: &certprovider.KeyMaterial{Roots: roots}} +} + +// newTestContextWithHandshakeInfo returns a copy of parent with HandshakeInfo +// context value added to it. +func newTestContextWithHandshakeInfo(parent context.Context, root, identity certprovider.Provider, sanExactMatch string) context.Context { + // Creating the HandshakeInfo and adding it to the attributes is very + // similar to what the CDS balancer would do when it intercepts calls to + // NewSubConn(). + info := xdsinternal.NewHandshakeInfo(root, identity) + if sanExactMatch != "" { + info.SetSANMatchers([]matcher.StringMatcher{matcher.StringMatcherForTesting(newStringP(sanExactMatch), nil, nil, nil, nil, false)}) + } + addr := xdsinternal.SetHandshakeInfo(resolver.Address{}, info) + + // Moving the attributes from the resolver.Address to the context passed to + // the handshaker is done in the transport layer. Since we directly call the + // handshaker in these tests, we need to do the same here. + return icredentials.NewClientHandshakeInfoContext(parent, credentials.ClientHandshakeInfo{Attributes: addr.Attributes}) +} + +// compareAuthInfo compares the AuthInfo received on the client side after a +// successful handshake with the authInfo available on the testServer. +func compareAuthInfo(ctx context.Context, ts *testServer, ai credentials.AuthInfo) error { + if ai.AuthType() != "tls" { + return fmt.Errorf("ClientHandshake returned authType %q, want %q", ai.AuthType(), "tls") + } + info, ok := ai.(credentials.TLSInfo) + if !ok { + return fmt.Errorf("ClientHandshake returned authInfo of type %T, want %T", ai, credentials.TLSInfo{}) + } + gotState := info.State + + // Read the handshake result from the testServer which contains the TLS + // connection state and compare it with the one received on the client-side. + val, err := ts.hsResult.Receive(ctx) + if err != nil { + return fmt.Errorf("testServer failed to return handshake result: %v", err) + } + hsr := val.(handshakeResult) + if hsr.err != nil { + return fmt.Errorf("testServer handshake failure: %v", hsr.err) + } + // AuthInfo contains a variety of information. We only verify a subset here. + // This is the same subset which is verified in TLS credentials tests. + if err := compareConnState(gotState, hsr.connState); err != nil { + return err + } + return nil +} + +func compareConnState(got, want tls.ConnectionState) error { + switch { + case got.Version != want.Version: + return fmt.Errorf("TLS.ConnectionState got Version: %v, want: %v", got.Version, want.Version) + case got.HandshakeComplete != want.HandshakeComplete: + return fmt.Errorf("TLS.ConnectionState got HandshakeComplete: %v, want: %v", got.HandshakeComplete, want.HandshakeComplete) + case got.CipherSuite != want.CipherSuite: + return fmt.Errorf("TLS.ConnectionState got CipherSuite: %v, want: %v", got.CipherSuite, want.CipherSuite) + case got.NegotiatedProtocol != want.NegotiatedProtocol: + return fmt.Errorf("TLS.ConnectionState got NegotiatedProtocol: %v, want: %v", got.NegotiatedProtocol, want.NegotiatedProtocol) + } + return nil +} + +// TestClientCredsWithoutFallback verifies that the call to +// NewClientCredentials() fails when no fallback is specified. +func (s) TestClientCredsWithoutFallback(t *testing.T) { + if _, err := NewClientCredentials(ClientOptions{}); err == nil { + t.Fatal("NewClientCredentials() succeeded without specifying fallback") + } +} + +// TestClientCredsInvalidHandshakeInfo verifies scenarios where the passed in +// HandshakeInfo is invalid because it does not contain the expected certificate +// providers. +func (s) TestClientCredsInvalidHandshakeInfo(t *testing.T) { + opts := ClientOptions{FallbackCreds: makeFallbackClientCreds(t)} + creds, err := NewClientCredentials(opts) + if err != nil { + t.Fatalf("NewClientCredentials(%v) failed: %v", opts, err) + } + + pCtx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctx := newTestContextWithHandshakeInfo(pCtx, nil, &fakeProvider{}, "") + if _, _, err := creds.ClientHandshake(ctx, authority, nil); err == nil { + t.Fatal("ClientHandshake succeeded without root certificate provider in HandshakeInfo") + } +} + +// TestClientCredsProviderFailure verifies the cases where an expected +// certificate provider is missing in the HandshakeInfo value in the context. +func (s) TestClientCredsProviderFailure(t *testing.T) { + opts := ClientOptions{FallbackCreds: makeFallbackClientCreds(t)} + creds, err := NewClientCredentials(opts) + if err != nil { + t.Fatalf("NewClientCredentials(%v) failed: %v", opts, err) + } + + tests := []struct { + desc string + rootProvider certprovider.Provider + identityProvider certprovider.Provider + wantErr string + }{ + { + desc: "erroring root provider", + rootProvider: &fakeProvider{err: errors.New("root provider error")}, + wantErr: "root provider error", + }, + { + desc: "erroring identity provider", + rootProvider: &fakeProvider{km: &certprovider.KeyMaterial{}}, + identityProvider: &fakeProvider{err: errors.New("identity provider error")}, + wantErr: "identity provider error", + }, + } + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctx = newTestContextWithHandshakeInfo(ctx, test.rootProvider, test.identityProvider, "") + if _, _, err := creds.ClientHandshake(ctx, authority, nil); err == nil || !strings.Contains(err.Error(), test.wantErr) { + t.Fatalf("ClientHandshake() returned error: %q, wantErr: %q", err, test.wantErr) + } + }) + } +} + +// TestClientCredsSuccess verifies successful client handshake cases. +func (s) TestClientCredsSuccess(t *testing.T) { + tests := []struct { + desc string + handshakeFunc testHandshakeFunc + handshakeInfoCtx func(ctx context.Context) context.Context + }{ + { + desc: "fallback", + handshakeFunc: testServerTLSHandshake, + handshakeInfoCtx: func(ctx context.Context) context.Context { + // Since we don't add a HandshakeInfo to the context, the + // ClientHandshake() method will delegate to the fallback. + return ctx + }, + }, + { + desc: "TLS", + handshakeFunc: testServerTLSHandshake, + handshakeInfoCtx: func(ctx context.Context) context.Context { + return newTestContextWithHandshakeInfo(ctx, makeRootProvider(t, "x509/server_ca_cert.pem"), nil, defaultTestCertSAN) + }, + }, + { + desc: "mTLS", + handshakeFunc: testServerMutualTLSHandshake, + handshakeInfoCtx: func(ctx context.Context) context.Context { + return newTestContextWithHandshakeInfo(ctx, makeRootProvider(t, "x509/server_ca_cert.pem"), makeIdentityProvider(t, "x509/server1_cert.pem", "x509/server1_key.pem"), defaultTestCertSAN) + }, + }, + { + desc: "mTLS with no acceptedSANs specified", + handshakeFunc: testServerMutualTLSHandshake, + handshakeInfoCtx: func(ctx context.Context) context.Context { + return newTestContextWithHandshakeInfo(ctx, makeRootProvider(t, "x509/server_ca_cert.pem"), makeIdentityProvider(t, "x509/server1_cert.pem", "x509/server1_key.pem"), "") + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + ts := newTestServerWithHandshakeFunc(test.handshakeFunc) + defer ts.stop() + + opts := ClientOptions{FallbackCreds: makeFallbackClientCreds(t)} + creds, err := NewClientCredentials(opts) + if err != nil { + t.Fatalf("NewClientCredentials(%v) failed: %v", opts, err) + } + + conn, err := net.Dial("tcp", ts.address) + if err != nil { + t.Fatalf("net.Dial(%s) failed: %v", ts.address, err) + } + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + _, ai, err := creds.ClientHandshake(test.handshakeInfoCtx(ctx), authority, conn) + if err != nil { + t.Fatalf("ClientHandshake() returned failed: %q", err) + } + if err := compareAuthInfo(ctx, ts, ai); err != nil { + t.Fatal(err) + } + }) + } +} + +func (s) TestClientCredsHandshakeTimeout(t *testing.T) { + clientDone := make(chan struct{}) + // A handshake function which simulates a handshake timeout from the + // server-side by simply blocking on the client-side handshake to timeout + // and not writing any handshake data. + hErr := errors.New("server handshake error") + ts := newTestServerWithHandshakeFunc(func(rawConn net.Conn) handshakeResult { + <-clientDone + return handshakeResult{err: hErr} + }) + defer ts.stop() + + opts := ClientOptions{FallbackCreds: makeFallbackClientCreds(t)} + creds, err := NewClientCredentials(opts) + if err != nil { + t.Fatalf("NewClientCredentials(%v) failed: %v", opts, err) + } + + conn, err := net.Dial("tcp", ts.address) + if err != nil { + t.Fatalf("net.Dial(%s) failed: %v", ts.address, err) + } + defer conn.Close() + + sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + ctx := newTestContextWithHandshakeInfo(sCtx, makeRootProvider(t, "x509/server_ca_cert.pem"), nil, defaultTestCertSAN) + if _, _, err := creds.ClientHandshake(ctx, authority, conn); err == nil { + t.Fatal("ClientHandshake() succeeded when expected to timeout") + } + close(clientDone) + + // Read the handshake result from the testServer and make sure the expected + // error is returned. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + val, err := ts.hsResult.Receive(ctx) + if err != nil { + t.Fatalf("testServer failed to return handshake result: %v", err) + } + hsr := val.(handshakeResult) + if hsr.err != hErr { + t.Fatalf("testServer handshake returned error: %v, want: %v", hsr.err, hErr) + } +} + +// TestClientCredsHandshakeFailure verifies different handshake failure cases. +func (s) TestClientCredsHandshakeFailure(t *testing.T) { + tests := []struct { + desc string + handshakeFunc testHandshakeFunc + rootProvider certprovider.Provider + san string + wantErr string + }{ + { + desc: "cert validation failure", + handshakeFunc: testServerTLSHandshake, + rootProvider: makeRootProvider(t, "x509/client_ca_cert.pem"), + san: defaultTestCertSAN, + wantErr: "x509: certificate signed by unknown authority", + }, + { + desc: "SAN mismatch", + handshakeFunc: testServerTLSHandshake, + rootProvider: makeRootProvider(t, "x509/server_ca_cert.pem"), + san: "bad-san", + wantErr: "do not match any of the accepted SANs", + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + ts := newTestServerWithHandshakeFunc(test.handshakeFunc) + defer ts.stop() + + opts := ClientOptions{FallbackCreds: makeFallbackClientCreds(t)} + creds, err := NewClientCredentials(opts) + if err != nil { + t.Fatalf("NewClientCredentials(%v) failed: %v", opts, err) + } + + conn, err := net.Dial("tcp", ts.address) + if err != nil { + t.Fatalf("net.Dial(%s) failed: %v", ts.address, err) + } + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctx = newTestContextWithHandshakeInfo(ctx, test.rootProvider, nil, test.san) + if _, _, err := creds.ClientHandshake(ctx, authority, conn); err == nil || !strings.Contains(err.Error(), test.wantErr) { + t.Fatalf("ClientHandshake() returned %q, wantErr %q", err, test.wantErr) + } + }) + } +} + +// TestClientCredsProviderSwitch verifies the case where the first attempt of +// ClientHandshake fails because of a handshake failure. Then we update the +// certificate provider and the second attempt succeeds. This is an +// approximation of the flow of events when the control plane specifies new +// security config which results in new certificate providers being used. +func (s) TestClientCredsProviderSwitch(t *testing.T) { + ts := newTestServerWithHandshakeFunc(testServerTLSHandshake) + defer ts.stop() + + opts := ClientOptions{FallbackCreds: makeFallbackClientCreds(t)} + creds, err := NewClientCredentials(opts) + if err != nil { + t.Fatalf("NewClientCredentials(%v) failed: %v", opts, err) + } + + conn, err := net.Dial("tcp", ts.address) + if err != nil { + t.Fatalf("net.Dial(%s) failed: %v", ts.address, err) + } + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + // Create a root provider which will fail the handshake because it does not + // use the correct trust roots. + root1 := makeRootProvider(t, "x509/client_ca_cert.pem") + handshakeInfo := xdsinternal.NewHandshakeInfo(root1, nil) + handshakeInfo.SetSANMatchers([]matcher.StringMatcher{matcher.StringMatcherForTesting(newStringP(defaultTestCertSAN), nil, nil, nil, nil, false)}) + + // We need to repeat most of what newTestContextWithHandshakeInfo() does + // here because we need access to the underlying HandshakeInfo so that we + // can update it before the next call to ClientHandshake(). + addr := xdsinternal.SetHandshakeInfo(resolver.Address{}, handshakeInfo) + ctx = icredentials.NewClientHandshakeInfoContext(ctx, credentials.ClientHandshakeInfo{Attributes: addr.Attributes}) + if _, _, err := creds.ClientHandshake(ctx, authority, conn); err == nil { + t.Fatal("ClientHandshake() succeeded when expected to fail") + } + // Drain the result channel on the test server so that we can inspect the + // result for the next handshake. + _, err = ts.hsResult.Receive(ctx) + if err != nil { + t.Errorf("testServer failed to return handshake result: %v", err) + } + + conn, err = net.Dial("tcp", ts.address) + if err != nil { + t.Fatalf("net.Dial(%s) failed: %v", ts.address, err) + } + defer conn.Close() + + // Create a new root provider which uses the correct trust roots. And update + // the HandshakeInfo with the new provider. + root2 := makeRootProvider(t, "x509/server_ca_cert.pem") + handshakeInfo.SetRootCertProvider(root2) + _, ai, err := creds.ClientHandshake(ctx, authority, conn) + if err != nil { + t.Fatalf("ClientHandshake() returned failed: %q", err) + } + if err := compareAuthInfo(ctx, ts, ai); err != nil { + t.Fatal(err) + } +} + +// TestClientClone verifies the Clone() method on client credentials. +func (s) TestClientClone(t *testing.T) { + opts := ClientOptions{FallbackCreds: makeFallbackClientCreds(t)} + orig, err := NewClientCredentials(opts) + if err != nil { + t.Fatalf("NewClientCredentials(%v) failed: %v", opts, err) + } + + // The credsImpl does not have any exported fields, and it does not make + // sense to use any cmp options to look deep into. So, all we make sure here + // is that the cloned object points to a different location in memory. + if clone := orig.Clone(); clone == orig { + t.Fatal("return value from Clone() doesn't point to new credentials instance") + } +} + +func newStringP(s string) *string { + return &s +} diff --git a/credentials/xds/xds_server_test.go b/credentials/xds/xds_server_test.go new file mode 100644 index 000000000000..bc32a04e69a1 --- /dev/null +++ b/credentials/xds/xds_server_test.go @@ -0,0 +1,546 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "net" + "os" + "strings" + "testing" + "time" + + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/tls/certprovider" + xdsinternal "google.golang.org/grpc/internal/credentials/xds" + "google.golang.org/grpc/testdata" +) + +func makeClientTLSConfig(t *testing.T, mTLS bool) *tls.Config { + t.Helper() + + pemData, err := os.ReadFile(testdata.Path("x509/server_ca_cert.pem")) + if err != nil { + t.Fatal(err) + } + roots := x509.NewCertPool() + roots.AppendCertsFromPEM(pemData) + + var certs []tls.Certificate + if mTLS { + cert, err := tls.LoadX509KeyPair(testdata.Path("x509/client1_cert.pem"), testdata.Path("x509/client1_key.pem")) + if err != nil { + t.Fatal(err) + } + certs = append(certs, cert) + } + + return &tls.Config{ + Certificates: certs, + RootCAs: roots, + ServerName: "*.test.example.com", + // Setting this to true completely turns off the certificate validation + // on the client side. So, the client side handshake always seems to + // succeed. But if we want to turn this ON, we will need to generate + // certificates which work with localhost, or supply a custom + // verification function. So, the server credentials tests will rely + // solely on the success/failure of the server-side handshake. + InsecureSkipVerify: true, + } +} + +// Helper function to create a real TLS server credentials which is used as +// fallback credentials from multiple tests. +func makeFallbackServerCreds(t *testing.T) credentials.TransportCredentials { + t.Helper() + + creds, err := credentials.NewServerTLSFromFile(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem")) + if err != nil { + t.Fatal(err) + } + return creds +} + +type errorCreds struct { + credentials.TransportCredentials +} + +// TestServerCredsWithoutFallback verifies that the call to +// NewServerCredentials() fails when no fallback is specified. +func (s) TestServerCredsWithoutFallback(t *testing.T) { + if _, err := NewServerCredentials(ServerOptions{}); err == nil { + t.Fatal("NewServerCredentials() succeeded without specifying fallback") + } +} + +type wrapperConn struct { + net.Conn + xdsHI *xdsinternal.HandshakeInfo + deadline time.Time + handshakeInfoErr error +} + +func (wc *wrapperConn) XDSHandshakeInfo() (*xdsinternal.HandshakeInfo, error) { + return wc.xdsHI, wc.handshakeInfoErr +} + +func (wc *wrapperConn) GetDeadline() time.Time { + return wc.deadline +} + +func newWrappedConn(conn net.Conn, xdsHI *xdsinternal.HandshakeInfo, deadline time.Time) *wrapperConn { + return &wrapperConn{Conn: conn, xdsHI: xdsHI, deadline: deadline} +} + +// TestServerCredsInvalidHandshakeInfo verifies scenarios where the passed in +// HandshakeInfo is invalid because it does not contain the expected certificate +// providers. +func (s) TestServerCredsInvalidHandshakeInfo(t *testing.T) { + opts := ServerOptions{FallbackCreds: &errorCreds{}} + creds, err := NewServerCredentials(opts) + if err != nil { + t.Fatalf("NewServerCredentials(%v) failed: %v", opts, err) + } + + info := xdsinternal.NewHandshakeInfo(&fakeProvider{}, nil) + conn := newWrappedConn(nil, info, time.Time{}) + if _, _, err := creds.ServerHandshake(conn); err == nil { + t.Fatal("ServerHandshake succeeded without identity certificate provider in HandshakeInfo") + } +} + +// TestServerCredsProviderFailure verifies the cases where an expected +// certificate provider is missing in the HandshakeInfo value in the context. +func (s) TestServerCredsProviderFailure(t *testing.T) { + opts := ServerOptions{FallbackCreds: &errorCreds{}} + creds, err := NewServerCredentials(opts) + if err != nil { + t.Fatalf("NewServerCredentials(%v) failed: %v", opts, err) + } + + tests := []struct { + desc string + rootProvider certprovider.Provider + identityProvider certprovider.Provider + wantErr string + }{ + { + desc: "erroring identity provider", + identityProvider: &fakeProvider{err: errors.New("identity provider error")}, + wantErr: "identity provider error", + }, + { + desc: "erroring root provider", + identityProvider: &fakeProvider{km: &certprovider.KeyMaterial{}}, + rootProvider: &fakeProvider{err: errors.New("root provider error")}, + wantErr: "root provider error", + }, + } + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + info := xdsinternal.NewHandshakeInfo(test.rootProvider, test.identityProvider) + conn := newWrappedConn(nil, info, time.Time{}) + if _, _, err := creds.ServerHandshake(conn); err == nil || !strings.Contains(err.Error(), test.wantErr) { + t.Fatalf("ServerHandshake() returned error: %q, wantErr: %q", err, test.wantErr) + } + }) + } +} + +// TestServerCredsHandshake_XDSHandshakeInfoError verifies the case where the +// call to XDSHandshakeInfo() from the ServerHandshake() method returns an +// error, and the test verifies that the ServerHandshake() fails with the +// expected error. +func (s) TestServerCredsHandshake_XDSHandshakeInfoError(t *testing.T) { + opts := ServerOptions{FallbackCreds: &errorCreds{}} + creds, err := NewServerCredentials(opts) + if err != nil { + t.Fatalf("NewServerCredentials(%v) failed: %v", opts, err) + } + + // Create a test server which uses the xDS server credentials created above + // to perform TLS handshake on incoming connections. + ts := newTestServerWithHandshakeFunc(func(rawConn net.Conn) handshakeResult { + // Create a wrapped conn which returns a nil HandshakeInfo and a non-nil error. + conn := newWrappedConn(rawConn, nil, time.Now().Add(defaultTestTimeout)) + hiErr := errors.New("xdsHandshakeInfo error") + conn.handshakeInfoErr = hiErr + + // Invoke the ServerHandshake() method on the xDS credentials and verify + // that the error returned by the XDSHandshakeInfo() method on the + // wrapped conn is returned here. + _, _, err := creds.ServerHandshake(conn) + if !errors.Is(err, hiErr) { + return handshakeResult{err: fmt.Errorf("ServerHandshake() returned err: %v, wantErr: %v", err, hiErr)} + } + return handshakeResult{} + }) + defer ts.stop() + + // Dial the test server, but don't trigger the TLS handshake. This will + // cause ServerHandshake() to fail. + rawConn, err := net.Dial("tcp", ts.address) + if err != nil { + t.Fatalf("net.Dial(%s) failed: %v", ts.address, err) + } + defer rawConn.Close() + + // Read handshake result from the testServer which will return an error if + // the handshake succeeded. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + val, err := ts.hsResult.Receive(ctx) + if err != nil { + t.Fatalf("testServer failed to return handshake result: %v", err) + } + hsr := val.(handshakeResult) + if hsr.err != nil { + t.Fatalf("testServer handshake failure: %v", hsr.err) + } +} + +// TestServerCredsHandshakeTimeout verifies the case where the client does not +// send required handshake data before the deadline set on the net.Conn passed +// to ServerHandshake(). +func (s) TestServerCredsHandshakeTimeout(t *testing.T) { + opts := ServerOptions{FallbackCreds: &errorCreds{}} + creds, err := NewServerCredentials(opts) + if err != nil { + t.Fatalf("NewServerCredentials(%v) failed: %v", opts, err) + } + + // Create a test server which uses the xDS server credentials created above + // to perform TLS handshake on incoming connections. + ts := newTestServerWithHandshakeFunc(func(rawConn net.Conn) handshakeResult { + hi := xdsinternal.NewHandshakeInfo(makeRootProvider(t, "x509/client_ca_cert.pem"), makeIdentityProvider(t, "x509/server2_cert.pem", "x509/server2_key.pem")) + hi.SetRequireClientCert(true) + + // Create a wrapped conn which can return the HandshakeInfo created + // above with a very small deadline. + d := time.Now().Add(defaultTestShortTimeout) + rawConn.SetDeadline(d) + conn := newWrappedConn(rawConn, hi, d) + + // ServerHandshake() on the xDS credentials is expected to fail. + if _, _, err := creds.ServerHandshake(conn); err == nil { + return handshakeResult{err: errors.New("ServerHandshake() succeeded when expected to timeout")} + } + return handshakeResult{} + }) + defer ts.stop() + + // Dial the test server, but don't trigger the TLS handshake. This will + // cause ServerHandshake() to fail. + rawConn, err := net.Dial("tcp", ts.address) + if err != nil { + t.Fatalf("net.Dial(%s) failed: %v", ts.address, err) + } + defer rawConn.Close() + + // Read handshake result from the testServer and expect a failure result. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + val, err := ts.hsResult.Receive(ctx) + if err != nil { + t.Fatalf("testServer failed to return handshake result: %v", err) + } + hsr := val.(handshakeResult) + if hsr.err != nil { + t.Fatalf("testServer handshake failure: %v", hsr.err) + } +} + +// TestServerCredsHandshakeFailure verifies the case where the server-side +// credentials uses a root certificate which does not match the certificate +// presented by the client, and hence the handshake must fail. +func (s) TestServerCredsHandshakeFailure(t *testing.T) { + opts := ServerOptions{FallbackCreds: &errorCreds{}} + creds, err := NewServerCredentials(opts) + if err != nil { + t.Fatalf("NewServerCredentials(%v) failed: %v", opts, err) + } + + // Create a test server which uses the xDS server credentials created above + // to perform TLS handshake on incoming connections. + ts := newTestServerWithHandshakeFunc(func(rawConn net.Conn) handshakeResult { + // Create a HandshakeInfo which has a root provider which does not match + // the certificate sent by the client. + hi := xdsinternal.NewHandshakeInfo(makeRootProvider(t, "x509/server_ca_cert.pem"), makeIdentityProvider(t, "x509/client2_cert.pem", "x509/client2_key.pem")) + hi.SetRequireClientCert(true) + + // Create a wrapped conn which can return the HandshakeInfo and + // configured deadline to the xDS credentials' ServerHandshake() + // method. + conn := newWrappedConn(rawConn, hi, time.Now().Add(defaultTestTimeout)) + + // ServerHandshake() on the xDS credentials is expected to fail. + if _, _, err := creds.ServerHandshake(conn); err == nil { + return handshakeResult{err: errors.New("ServerHandshake() succeeded when expected to fail")} + } + return handshakeResult{} + }) + defer ts.stop() + + // Dial the test server, and trigger the TLS handshake. + rawConn, err := net.Dial("tcp", ts.address) + if err != nil { + t.Fatalf("net.Dial(%s) failed: %v", ts.address, err) + } + defer rawConn.Close() + tlsConn := tls.Client(rawConn, makeClientTLSConfig(t, true)) + tlsConn.SetDeadline(time.Now().Add(defaultTestTimeout)) + if err := tlsConn.Handshake(); err != nil { + t.Fatal(err) + } + + // Read handshake result from the testServer which will return an error if + // the handshake succeeded. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + val, err := ts.hsResult.Receive(ctx) + if err != nil { + t.Fatalf("testServer failed to return handshake result: %v", err) + } + hsr := val.(handshakeResult) + if hsr.err != nil { + t.Fatalf("testServer handshake failure: %v", hsr.err) + } +} + +// TestServerCredsHandshakeSuccess verifies success handshake cases. +func (s) TestServerCredsHandshakeSuccess(t *testing.T) { + tests := []struct { + desc string + fallbackCreds credentials.TransportCredentials + rootProvider certprovider.Provider + identityProvider certprovider.Provider + requireClientCert bool + }{ + { + desc: "fallback", + fallbackCreds: makeFallbackServerCreds(t), + }, + { + desc: "TLS", + fallbackCreds: &errorCreds{}, + identityProvider: makeIdentityProvider(t, "x509/server2_cert.pem", "x509/server2_key.pem"), + }, + { + desc: "mTLS", + fallbackCreds: &errorCreds{}, + identityProvider: makeIdentityProvider(t, "x509/server2_cert.pem", "x509/server2_key.pem"), + rootProvider: makeRootProvider(t, "x509/client_ca_cert.pem"), + requireClientCert: true, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + // Create an xDS server credentials. + opts := ServerOptions{FallbackCreds: test.fallbackCreds} + creds, err := NewServerCredentials(opts) + if err != nil { + t.Fatalf("NewServerCredentials(%v) failed: %v", opts, err) + } + + // Create a test server which uses the xDS server credentials + // created above to perform TLS handshake on incoming connections. + ts := newTestServerWithHandshakeFunc(func(rawConn net.Conn) handshakeResult { + // Create a HandshakeInfo with information from the test table. + hi := xdsinternal.NewHandshakeInfo(test.rootProvider, test.identityProvider) + hi.SetRequireClientCert(test.requireClientCert) + + // Create a wrapped conn which can return the HandshakeInfo and + // configured deadline to the xDS credentials' ServerHandshake() + // method. + conn := newWrappedConn(rawConn, hi, time.Now().Add(defaultTestTimeout)) + + // Invoke the ServerHandshake() method on the xDS credentials + // and make some sanity checks before pushing the result for + // inspection by the main test body. + _, ai, err := creds.ServerHandshake(conn) + if err != nil { + return handshakeResult{err: fmt.Errorf("ServerHandshake() failed: %v", err)} + } + if ai.AuthType() != "tls" { + return handshakeResult{err: fmt.Errorf("ServerHandshake returned authType %q, want %q", ai.AuthType(), "tls")} + } + info, ok := ai.(credentials.TLSInfo) + if !ok { + return handshakeResult{err: fmt.Errorf("ServerHandshake returned authInfo of type %T, want %T", ai, credentials.TLSInfo{})} + } + return handshakeResult{connState: info.State} + }) + defer ts.stop() + + // Dial the test server, and trigger the TLS handshake. + rawConn, err := net.Dial("tcp", ts.address) + if err != nil { + t.Fatalf("net.Dial(%s) failed: %v", ts.address, err) + } + defer rawConn.Close() + tlsConn := tls.Client(rawConn, makeClientTLSConfig(t, test.requireClientCert)) + tlsConn.SetDeadline(time.Now().Add(defaultTestTimeout)) + if err := tlsConn.Handshake(); err != nil { + t.Fatal(err) + } + + // Read the handshake result from the testServer which contains the + // TLS connection state on the server-side and compare it with the + // one received on the client-side. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + val, err := ts.hsResult.Receive(ctx) + if err != nil { + t.Fatalf("testServer failed to return handshake result: %v", err) + } + hsr := val.(handshakeResult) + if hsr.err != nil { + t.Fatalf("testServer handshake failure: %v", hsr.err) + } + + // AuthInfo contains a variety of information. We only verify a + // subset here. This is the same subset which is verified in TLS + // credentials tests. + if err := compareConnState(tlsConn.ConnectionState(), hsr.connState); err != nil { + t.Fatal(err) + } + }) + } +} + +func (s) TestServerCredsProviderSwitch(t *testing.T) { + opts := ServerOptions{FallbackCreds: &errorCreds{}} + creds, err := NewServerCredentials(opts) + if err != nil { + t.Fatalf("NewServerCredentials(%v) failed: %v", opts, err) + } + + // The first time the handshake function is invoked, it returns a + // HandshakeInfo which is expected to fail. Further invocations return a + // HandshakeInfo which is expected to succeed. + cnt := 0 + // Create a test server which uses the xDS server credentials created above + // to perform TLS handshake on incoming connections. + ts := newTestServerWithHandshakeFunc(func(rawConn net.Conn) handshakeResult { + cnt++ + var hi *xdsinternal.HandshakeInfo + if cnt == 1 { + // Create a HandshakeInfo which has a root provider which does not match + // the certificate sent by the client. + hi = xdsinternal.NewHandshakeInfo(makeRootProvider(t, "x509/server_ca_cert.pem"), makeIdentityProvider(t, "x509/client2_cert.pem", "x509/client2_key.pem")) + hi.SetRequireClientCert(true) + + // Create a wrapped conn which can return the HandshakeInfo and + // configured deadline to the xDS credentials' ServerHandshake() + // method. + conn := newWrappedConn(rawConn, hi, time.Now().Add(defaultTestTimeout)) + + // ServerHandshake() on the xDS credentials is expected to fail. + if _, _, err := creds.ServerHandshake(conn); err == nil { + return handshakeResult{err: errors.New("ServerHandshake() succeeded when expected to fail")} + } + return handshakeResult{} + } + + hi = xdsinternal.NewHandshakeInfo(makeRootProvider(t, "x509/client_ca_cert.pem"), makeIdentityProvider(t, "x509/server1_cert.pem", "x509/server1_key.pem")) + hi.SetRequireClientCert(true) + + // Create a wrapped conn which can return the HandshakeInfo and + // configured deadline to the xDS credentials' ServerHandshake() + // method. + conn := newWrappedConn(rawConn, hi, time.Now().Add(defaultTestTimeout)) + + // Invoke the ServerHandshake() method on the xDS credentials + // and make some sanity checks before pushing the result for + // inspection by the main test body. + _, ai, err := creds.ServerHandshake(conn) + if err != nil { + return handshakeResult{err: fmt.Errorf("ServerHandshake() failed: %v", err)} + } + if ai.AuthType() != "tls" { + return handshakeResult{err: fmt.Errorf("ServerHandshake returned authType %q, want %q", ai.AuthType(), "tls")} + } + info, ok := ai.(credentials.TLSInfo) + if !ok { + return handshakeResult{err: fmt.Errorf("ServerHandshake returned authInfo of type %T, want %T", ai, credentials.TLSInfo{})} + } + return handshakeResult{connState: info.State} + }) + defer ts.stop() + + for i := 0; i < 5; i++ { + // Dial the test server, and trigger the TLS handshake. + rawConn, err := net.Dial("tcp", ts.address) + if err != nil { + t.Fatalf("net.Dial(%s) failed: %v", ts.address, err) + } + defer rawConn.Close() + tlsConn := tls.Client(rawConn, makeClientTLSConfig(t, true)) + tlsConn.SetDeadline(time.Now().Add(defaultTestTimeout)) + if err := tlsConn.Handshake(); err != nil { + t.Fatal(err) + } + + // Read the handshake result from the testServer which contains the + // TLS connection state on the server-side and compare it with the + // one received on the client-side. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + val, err := ts.hsResult.Receive(ctx) + if err != nil { + t.Fatalf("testServer failed to return handshake result: %v", err) + } + hsr := val.(handshakeResult) + if hsr.err != nil { + t.Fatalf("testServer handshake failure: %v", hsr.err) + } + if i == 0 { + // We expect the first handshake to fail. So, we skip checks which + // compare connection state. + continue + } + // AuthInfo contains a variety of information. We only verify a + // subset here. This is the same subset which is verified in TLS + // credentials tests. + if err := compareConnState(tlsConn.ConnectionState(), hsr.connState); err != nil { + t.Fatal(err) + } + } +} + +// TestServerClone verifies the Clone() method on client credentials. +func (s) TestServerClone(t *testing.T) { + opts := ServerOptions{FallbackCreds: makeFallbackServerCreds(t)} + orig, err := NewServerCredentials(opts) + if err != nil { + t.Fatalf("NewServerCredentials(%v) failed: %v", opts, err) + } + + // The credsImpl does not have any exported fields, and it does not make + // sense to use any cmp options to look deep into. So, all we make sure here + // is that the cloned object points to a different location in memory. + if clone := orig.Clone(); clone == orig { + t.Fatal("return value from Clone() doesn't point to new credentials instance") + } +} diff --git a/default_dial_option_server_option_test.go b/default_dial_option_server_option_test.go new file mode 100644 index 000000000000..4af5c0b8772c --- /dev/null +++ b/default_dial_option_server_option_test.go @@ -0,0 +1,134 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "fmt" + "strings" + "testing" + + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" +) + +func (s) TestAddGlobalDialOptions(t *testing.T) { + // Ensure the Dial fails without credentials + if _, err := Dial("fake"); err == nil { + t.Fatalf("Dialing without a credential did not fail") + } else { + if !strings.Contains(err.Error(), "no transport security set") { + t.Fatalf("Dialing failed with unexpected error: %v", err) + } + } + + // Set and check the DialOptions + opts := []DialOption{WithTransportCredentials(insecure.NewCredentials()), WithTransportCredentials(insecure.NewCredentials()), WithTransportCredentials(insecure.NewCredentials())} + internal.AddGlobalDialOptions.(func(opt ...DialOption))(opts...) + for i, opt := range opts { + if globalDialOptions[i] != opt { + t.Fatalf("Unexpected global dial option at index %d: %v != %v", i, globalDialOptions[i], opt) + } + } + + // Ensure the Dial passes with the extra dial options + if cc, err := Dial("fake"); err != nil { + t.Fatalf("Dialing with insecure credential failed: %v", err) + } else { + cc.Close() + } + + internal.ClearGlobalDialOptions() + if len(globalDialOptions) != 0 { + t.Fatalf("Unexpected len of globalDialOptions: %d != 0", len(globalDialOptions)) + } +} + +// TestDisableGlobalOptions tests dialing with the disableGlobalDialOptions dial +// option. Dialing with this set should not pick up global options. +func (s) TestDisableGlobalOptions(t *testing.T) { + // Set transport credentials as a global option. + internal.AddGlobalDialOptions.(func(opt ...DialOption))(WithTransportCredentials(insecure.NewCredentials())) + // Dial with the disable global options dial option. This dial should fail + // due to the global dial options with credentials not being picked up due + // to global options being disabled. + noTSecStr := "no transport security set" + if _, err := Dial("fake", internal.DisableGlobalDialOptions.(func() DialOption)()); !strings.Contains(fmt.Sprint(err), noTSecStr) { + t.Fatalf("Dialing received unexpected error: %v, want error containing \"%v\"", err, noTSecStr) + } + internal.ClearGlobalDialOptions() +} + +func (s) TestAddGlobalServerOptions(t *testing.T) { + const maxRecvSize = 998765 + // Set and check the ServerOptions + opts := []ServerOption{Creds(insecure.NewCredentials()), MaxRecvMsgSize(maxRecvSize)} + internal.AddGlobalServerOptions.(func(opt ...ServerOption))(opts...) + for i, opt := range opts { + if globalServerOptions[i] != opt { + t.Fatalf("Unexpected global server option at index %d: %v != %v", i, globalServerOptions[i], opt) + } + } + + // Ensure the extra server options applies to new servers + s := NewServer() + if s.opts.maxReceiveMessageSize != maxRecvSize { + t.Fatalf("Unexpected s.opts.maxReceiveMessageSize: %d != %d", s.opts.maxReceiveMessageSize, maxRecvSize) + } + + internal.ClearGlobalServerOptions() + if len(globalServerOptions) != 0 { + t.Fatalf("Unexpected len of globalServerOptions: %d != 0", len(globalServerOptions)) + } +} + +// TestJoinDialOption tests the join dial option. It configures a joined dial +// option with three individual dial options, and verifies that all three are +// successfully applied. +func (s) TestJoinDialOption(t *testing.T) { + const maxRecvSize = 998765 + const initialWindowSize = 100 + jdo := newJoinDialOption(WithTransportCredentials(insecure.NewCredentials()), WithReadBufferSize(maxRecvSize), WithInitialWindowSize(initialWindowSize)) + cc, err := Dial("fake", jdo) + if err != nil { + t.Fatalf("Dialing with insecure credentials failed: %v", err) + } + defer cc.Close() + if cc.dopts.copts.ReadBufferSize != maxRecvSize { + t.Fatalf("Unexpected cc.dopts.copts.ReadBufferSize: %d != %d", cc.dopts.copts.ReadBufferSize, maxRecvSize) + } + if cc.dopts.copts.InitialWindowSize != initialWindowSize { + t.Fatalf("Unexpected cc.dopts.copts.InitialWindowSize: %d != %d", cc.dopts.copts.InitialWindowSize, initialWindowSize) + } +} + +// TestJoinDialOption tests the join server option. It configures a joined +// server option with three individual server options, and verifies that all +// three are successfully applied. +func (s) TestJoinServerOption(t *testing.T) { + const maxRecvSize = 998765 + const initialWindowSize = 100 + jso := newJoinServerOption(Creds(insecure.NewCredentials()), MaxRecvMsgSize(maxRecvSize), InitialWindowSize(initialWindowSize)) + s := NewServer(jso) + if s.opts.maxReceiveMessageSize != maxRecvSize { + t.Fatalf("Unexpected s.opts.maxReceiveMessageSize: %d != %d", s.opts.maxReceiveMessageSize, maxRecvSize) + } + if s.opts.initialWindowSize != initialWindowSize { + t.Fatalf("Unexpected s.opts.initialWindowSize: %d != %d", s.opts.initialWindowSize, initialWindowSize) + } +} diff --git a/dialoptions.go b/dialoptions.go index decb4c5ee891..1fd0d5c127f4 100644 --- a/dialoptions.go +++ b/dialoptions.go @@ -20,22 +20,34 @@ package grpc import ( "context" - "fmt" "net" "time" "google.golang.org/grpc/backoff" - "google.golang.org/grpc/balancer" + "google.golang.org/grpc/channelz" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/internal" internalbackoff "google.golang.org/grpc/internal/backoff" - "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/binarylog" "google.golang.org/grpc/internal/transport" "google.golang.org/grpc/keepalive" "google.golang.org/grpc/resolver" "google.golang.org/grpc/stats" ) +func init() { + internal.AddGlobalDialOptions = func(opt ...DialOption) { + globalDialOptions = append(globalDialOptions, opt...) + } + internal.ClearGlobalDialOptions = func() { + globalDialOptions = nil + } + internal.WithBinaryLogger = withBinaryLogger + internal.JoinDialOptions = newJoinDialOption + internal.DisableGlobalDialOptions = newDisableGlobalDialOptions +} + // dialOptions configure a Dial call. dialOptions are set by the DialOption // values passed to Dial. type dialOptions struct { @@ -45,20 +57,18 @@ type dialOptions struct { chainUnaryInts []UnaryClientInterceptor chainStreamInts []StreamClientInterceptor - cp Compressor - dc Decompressor - bs internalbackoff.Strategy - block bool - returnLastError bool - insecure bool - timeout time.Duration - scChan <-chan ServiceConfig - authority string - copts transport.ConnectOptions - callOptions []CallOption - // This is used by WithBalancerName dial option. - balancerBuilder balancer.Builder - channelzParentID int64 + cp Compressor + dc Decompressor + bs internalbackoff.Strategy + block bool + returnLastError bool + timeout time.Duration + scChan <-chan ServiceConfig + authority string + binaryLogger binarylog.Logger + copts transport.ConnectOptions + callOptions []CallOption + channelzParentID *channelz.Identifier disableServiceConfig bool disableRetry bool disableHealthCheck bool @@ -66,12 +76,9 @@ type dialOptions struct { minConnectTimeout func() time.Duration defaultServiceConfig *ServiceConfig // defaultServiceConfig is parsed from defaultServiceConfigRawJSON. defaultServiceConfigRawJSON *string - // This is used by ccResolverWrapper to backoff between successive calls to - // resolver.ResolveNow(). The user will have no need to configure this, but - // we need to be able to configure this in tests. - resolveNowBackoff func(int) time.Duration - resolvers []resolver.Builder - withProxy bool + resolvers []resolver.Builder + idleTimeout time.Duration + recvBufferPool SharedBufferPool } // DialOption configures how we set up the connection. @@ -79,14 +86,29 @@ type DialOption interface { apply(*dialOptions) } +var globalDialOptions []DialOption + // EmptyDialOption does not alter the dial configuration. It can be embedded in // another structure to build custom dial options. // -// This API is EXPERIMENTAL. +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. type EmptyDialOption struct{} func (EmptyDialOption) apply(*dialOptions) {} +type disableGlobalDialOptions struct{} + +func (disableGlobalDialOptions) apply(*dialOptions) {} + +// newDisableGlobalDialOptions returns a DialOption that prevents the ClientConn +// from applying the global DialOptions (set via AddGlobalDialOptions). +func newDisableGlobalDialOptions() DialOption { + return &disableGlobalDialOptions{} +} + // funcDialOption wraps a function that modifies dialOptions into an // implementation of the DialOption interface. type funcDialOption struct { @@ -103,13 +125,42 @@ func newFuncDialOption(f func(*dialOptions)) *funcDialOption { } } +type joinDialOption struct { + opts []DialOption +} + +func (jdo *joinDialOption) apply(do *dialOptions) { + for _, opt := range jdo.opts { + opt.apply(do) + } +} + +func newJoinDialOption(opts ...DialOption) DialOption { + return &joinDialOption{opts: opts} +} + +// WithSharedWriteBuffer allows reusing per-connection transport write buffer. +// If this option is set to true every connection will release the buffer after +// flushing the data on the wire. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. +func WithSharedWriteBuffer(val bool) DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.copts.SharedWriteBuffer = val + }) +} + // WithWriteBufferSize determines how much data can be batched before doing a // write on the wire. The corresponding memory allocation for this buffer will // be twice the size to keep syscalls low. The default value for this buffer is // 32KB. // -// Zero will disable the write buffer such that each write will be on underlying -// connection. Note: A Send call may not directly translate to a write. +// Zero or negative values will disable the write buffer such that each write +// will be on underlying connection. Note: A Send call may not directly +// translate to a write. func WithWriteBufferSize(s int) DialOption { return newFuncDialOption(func(o *dialOptions) { o.copts.WriteBufferSize = s @@ -119,8 +170,9 @@ func WithWriteBufferSize(s int) DialOption { // WithReadBufferSize lets you set the size of read buffer, this determines how // much data can be read at most for each read syscall. // -// The default value for this buffer is 32KB. Zero will disable read buffer for -// a connection so data framer can access the underlying conn directly. +// The default value for this buffer is 32KB. Zero or negative values will +// disable read buffer for a connection so data framer can access the +// underlying conn directly. func WithReadBufferSize(s int) DialOption { return newFuncDialOption(func(o *dialOptions) { o.copts.ReadBufferSize = s @@ -198,25 +250,6 @@ func WithDecompressor(dc Decompressor) DialOption { }) } -// WithBalancerName sets the balancer that the ClientConn will be initialized -// with. Balancer registered with balancerName will be used. This function -// panics if no balancer was registered by balancerName. -// -// The balancer cannot be overridden by balancer option specified by service -// config. -// -// Deprecated: use WithDefaultServiceConfig and WithDisableServiceConfig -// instead. Will be removed in a future 1.x release. -func WithBalancerName(balancerName string) DialOption { - builder := balancer.Get(balancerName) - if builder == nil { - panic(fmt.Sprintf("grpc.WithBalancerName: no balancer is registered for name %v", balancerName)) - } - return newFuncDialOption(func(o *dialOptions) { - o.balancerBuilder = builder - }) -} - // WithServiceConfig returns a DialOption which has a channel to read the // service configuration. // @@ -230,15 +263,14 @@ func WithServiceConfig(c <-chan ServiceConfig) DialOption { }) } -// WithConnectParams configures the dialer to use the provided ConnectParams. +// WithConnectParams configures the ClientConn to use the provided ConnectParams +// for creating and maintaining connections to servers. // // The backoff configuration specified as part of the ConnectParams overrides // all defaults specified in // https://github.com/grpc/grpc/blob/master/doc/connection-backoff.md. Consider // using the backoff.DefaultConfig as a base, in cases where you want to // override only a subset of the backoff configuration. -// -// This API is EXPERIMENTAL. func WithConnectParams(p ConnectParams) DialOption { return newFuncDialOption(func(o *dialOptions) { o.bs = internalbackoff.Exponential{Config: p.Backoff} @@ -276,9 +308,12 @@ func withBackoff(bs internalbackoff.Strategy) DialOption { }) } -// WithBlock returns a DialOption which makes caller of Dial blocks until the +// WithBlock returns a DialOption which makes callers of Dial block until the // underlying connection is up. Without this, Dial returns immediately and // connecting the server happens in background. +// +// Use of this feature is not recommended. For more information, please see: +// https://github.com/grpc/grpc-go/blob/master/Documentation/anti-patterns.md func WithBlock() DialOption { return newFuncDialOption(func(o *dialOptions) { o.block = true @@ -290,7 +325,13 @@ func WithBlock() DialOption { // the context.DeadlineExceeded error. // Implies WithBlock() // -// This API is EXPERIMENTAL. +// Use of this feature is not recommended. For more information, please see: +// https://github.com/grpc/grpc-go/blob/master/Documentation/anti-patterns.md +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. func WithReturnConnectionError() DialOption { return newFuncDialOption(func(o *dialOptions) { o.block = true @@ -299,21 +340,30 @@ func WithReturnConnectionError() DialOption { } // WithInsecure returns a DialOption which disables transport security for this -// ClientConn. Note that transport security is required unless WithInsecure is -// set. +// ClientConn. Under the hood, it uses insecure.NewCredentials(). +// +// Note that using this DialOption with per-RPC credentials (through +// WithCredentialsBundle or WithPerRPCCredentials) which require transport +// security is incompatible and will cause grpc.Dial() to fail. +// +// Deprecated: use WithTransportCredentials and insecure.NewCredentials() +// instead. Will be supported throughout 1.x. func WithInsecure() DialOption { return newFuncDialOption(func(o *dialOptions) { - o.insecure = true + o.copts.TransportCredentials = insecure.NewCredentials() }) } // WithNoProxy returns a DialOption which disables the use of proxies for this // ClientConn. This is ignored if WithDialer or WithContextDialer are used. // -// This API is EXPERIMENTAL. +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. func WithNoProxy() DialOption { return newFuncDialOption(func(o *dialOptions) { - o.withProxy = false + o.copts.UseProxy = false }) } @@ -338,7 +388,10 @@ func WithPerRPCCredentials(creds credentials.PerRPCCredentials) DialOption { // the ClientConn.WithCreds. This should not be used together with // WithTransportCredentials. // -// This API is experimental. +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. func WithCredentialsBundle(b credentials.Bundle) DialOption { return newFuncDialOption(func(o *dialOptions) { o.copts.CredsBundle = b @@ -391,7 +444,21 @@ func WithDialer(f func(string, time.Duration) (net.Conn, error)) DialOption { // all the RPCs and underlying network connections in this ClientConn. func WithStatsHandler(h stats.Handler) DialOption { return newFuncDialOption(func(o *dialOptions) { - o.copts.StatsHandler = h + if h == nil { + logger.Error("ignoring nil parameter in grpc.WithStatsHandler ClientOption") + // Do not allow a nil stats handler, which would otherwise cause + // panics. + return + } + o.copts.StatsHandlers = append(o.copts.StatsHandlers, h) + }) +} + +// withBinaryLogger returns a DialOption that specifies the binary logger for +// this ClientConn. +func withBinaryLogger(bl binarylog.Logger) DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.binaryLogger = bl }) } @@ -403,7 +470,13 @@ func WithStatsHandler(h stats.Handler) DialOption { // FailOnNonTempDialError only affects the initial dial, and does not do // anything useful unless you are also using WithBlock(). // -// This is an EXPERIMENTAL API. +// Use of this feature is not recommended. For more information, please see: +// https://github.com/grpc/grpc-go/blob/master/Documentation/anti-patterns.md +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. func FailOnNonTempDialError(f bool) DialOption { return newFuncDialOption(func(o *dialOptions) { o.copts.FailOnNonTempDialError = f @@ -469,8 +542,7 @@ func WithChainStreamInterceptor(interceptors ...StreamClientInterceptor) DialOpt } // WithAuthority returns a DialOption that specifies the value to be used as the -// :authority pseudo-header. This value only works with WithInsecure and has no -// effect if TransportCredentials are present. +// :authority pseudo-header and as the server name in authentication handshake. func WithAuthority(a string) DialOption { return newFuncDialOption(func(o *dialOptions) { o.authority = a @@ -481,8 +553,11 @@ func WithAuthority(a string) DialOption { // current ClientConn's parent. This function is used in nested channel creation // (e.g. grpclb dial). // -// This API is EXPERIMENTAL. -func WithChannelzParentID(id int64) DialOption { +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. +func WithChannelzParentID(id *channelz.Identifier) DialOption { return newFuncDialOption(func(o *dialOptions) { o.channelzParentID = id }) @@ -503,11 +578,16 @@ func WithDisableServiceConfig() DialOption { // WithDefaultServiceConfig returns a DialOption that configures the default // service config, which will be used in cases where: // -// 1. WithDisableServiceConfig is also used. -// 2. Resolver does not return a service config or if the resolver returns an -// invalid service config. +// 1. WithDisableServiceConfig is also used, or +// +// 2. The name resolver does not provide a service config or provides an +// invalid service config. // -// This API is EXPERIMENTAL. +// The parameter s is the JSON representation of the default service config. +// For more information about service configs, see: +// https://github.com/grpc/grpc/blob/master/doc/service_config.md +// For a simple example of usage, see: +// examples/features/load_balancing/client/main.go func WithDefaultServiceConfig(s string) DialOption { return newFuncDialOption(func(o *dialOptions) { o.defaultServiceConfigRawJSON = &s @@ -518,12 +598,6 @@ func WithDefaultServiceConfig(s string) DialOption { // service config enables them. This does not impact transparent retries, which // will happen automatically if no data is written to the wire or if the RPC is // unprocessed by the remote server. -// -// Retry support is currently disabled by default, but will be enabled by -// default in the future. Until then, it may be enabled by setting the -// environment variable "GRPC_GO_RETRY" to "on". -// -// This API is EXPERIMENTAL. func WithDisableRetry() DialOption { return newFuncDialOption(func(o *dialOptions) { o.disableRetry = true @@ -541,7 +615,10 @@ func WithMaxHeaderListSize(s uint32) DialOption { // WithDisableHealthCheck disables the LB channel health checking for all // SubConns of this ClientConn. // -// This API is EXPERIMENTAL. +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. func WithDisableHealthCheck() DialOption { return newFuncDialOption(func(o *dialOptions) { o.disableHealthCheck = true @@ -560,14 +637,13 @@ func withHealthCheckFunc(f internal.HealthChecker) DialOption { func defaultDialOptions() dialOptions { return dialOptions{ - disableRetry: !envconfig.Retry, healthCheckFunc: internal.HealthCheckFunc, copts: transport.ConnectOptions{ WriteBufferSize: defaultWriteBufSize, ReadBufferSize: defaultReadBufSize, + UseProxy: true, }, - resolveNowBackoff: internalbackoff.DefaultExponential.Backoff, - withProxy: true, + recvBufferPool: nopBufferPool{}, } } @@ -582,24 +658,58 @@ func withMinConnectDeadline(f func() time.Duration) DialOption { }) } -// withResolveNowBackoff specifies the function that clientconn uses to backoff -// between successive calls to resolver.ResolveNow(). -// -// For testing purpose only. -func withResolveNowBackoff(f func(int) time.Duration) DialOption { - return newFuncDialOption(func(o *dialOptions) { - o.resolveNowBackoff = f - }) -} - // WithResolvers allows a list of resolver implementations to be registered // locally with the ClientConn without needing to be globally registered via // resolver.Register. They will be matched against the scheme used for the // current Dial only, and will take precedence over the global registry. // -// This API is EXPERIMENTAL. +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. func WithResolvers(rs ...resolver.Builder) DialOption { return newFuncDialOption(func(o *dialOptions) { o.resolvers = append(o.resolvers, rs...) }) } + +// WithIdleTimeout returns a DialOption that configures an idle timeout for the +// channel. If the channel is idle for the configured timeout, i.e there are no +// ongoing RPCs and no new RPCs are initiated, the channel will enter idle mode +// and as a result the name resolver and load balancer will be shut down. The +// channel will exit idle mode when the Connect() method is called or when an +// RPC is initiated. +// +// By default this feature is disabled, which can also be explicitly configured +// by passing zero to this function. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. +func WithIdleTimeout(d time.Duration) DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.idleTimeout = d + }) +} + +// WithRecvBufferPool returns a DialOption that configures the ClientConn +// to use the provided shared buffer pool for parsing incoming messages. Depending +// on the application's workload, this could result in reduced memory allocation. +// +// If you are unsure about how to implement a memory pool but want to utilize one, +// begin with grpc.NewSharedBufferPool. +// +// Note: The shared buffer pool feature will not be active if any of the following +// options are used: WithStatsHandler, EnableTracing, or binary logging. In such +// cases, the shared buffer pool will be ignored. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. +func WithRecvBufferPool(bufferPool SharedBufferPool) DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.recvBufferPool = bufferPool + }) +} diff --git a/encoding/encoding.go b/encoding/encoding.go index 195e8448b646..69d5580b6adf 100644 --- a/encoding/encoding.go +++ b/encoding/encoding.go @@ -19,12 +19,17 @@ // Package encoding defines the interface for the compressor and codec, and // functions to register and retrieve compressors and codecs. // -// This package is EXPERIMENTAL. +// # Experimental +// +// Notice: This package is EXPERIMENTAL and may be changed or removed in a +// later release. package encoding import ( "io" "strings" + + "google.golang.org/grpc/internal/grpcutil" ) // Identity specifies the optional encoding for uncompressed streams. @@ -46,10 +51,15 @@ type Compressor interface { // coding header. The result must be static; the result cannot change // between calls. Name() string - // EXPERIMENTAL: if a Compressor implements + // If a Compressor implements // DecompressedSize(compressedBytes []byte) int, gRPC will call it // to determine the size of the buffer allocated for the result of decompression. // Return -1 to indicate unknown size. + // + // Experimental + // + // Notice: This API is EXPERIMENTAL and may be changed or removed in a + // later release. } var registeredCompressor = make(map[string]Compressor) @@ -65,6 +75,9 @@ var registeredCompressor = make(map[string]Compressor) // registered with the same name, the one registered last will take effect. func RegisterCompressor(c Compressor) { registeredCompressor[c.Name()] = c + if !grpcutil.IsCompressorNameRegistered(c.Name()) { + grpcutil.RegisteredCompressorNames = append(grpcutil.RegisteredCompressorNames, c.Name()) + } } // GetCompressor returns Compressor for the given compressor name. @@ -77,9 +90,9 @@ func GetCompressor(name string) Compressor { // methods can be called from concurrent goroutines. type Codec interface { // Marshal returns the wire format of v. - Marshal(v interface{}) ([]byte, error) + Marshal(v any) ([]byte, error) // Unmarshal parses the wire format into v. - Unmarshal(data []byte, v interface{}) error + Unmarshal(data []byte, v any) error // Name returns the name of the Codec implementation. The returned string // will be used as part of content type in transmission. The result must be // static; the result cannot change between calls. @@ -100,7 +113,7 @@ var registeredCodecs = make(map[string]Codec) // more details. // // NOTE: this function must only be called during initialization time (i.e. in -// an init() function), and is not thread-safe. If multiple Compressors are +// an init() function), and is not thread-safe. If multiple Codecs are // registered with the same name, the one registered last will take effect. func RegisterCodec(codec Codec) { if codec == nil { diff --git a/encoding/encoding_test.go b/encoding/encoding_test.go new file mode 100644 index 000000000000..38c31dcdddcc --- /dev/null +++ b/encoding/encoding_test.go @@ -0,0 +1,55 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package encoding + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/internal/grpcutil" +) + +type mockNamedCompressor struct { + Compressor +} + +func (mockNamedCompressor) Name() string { + return "mock-compressor" +} + +func TestDuplicateCompressorRegister(t *testing.T) { + defer func(m map[string]Compressor) { registeredCompressor = m }(registeredCompressor) + defer func(c []string) { grpcutil.RegisteredCompressorNames = c }(grpcutil.RegisteredCompressorNames) + registeredCompressor = map[string]Compressor{} + grpcutil.RegisteredCompressorNames = []string{} + + RegisterCompressor(&mockNamedCompressor{}) + + // Register another instance of the same compressor. + mc := &mockNamedCompressor{} + RegisterCompressor(mc) + if got := registeredCompressor["mock-compressor"]; got != mc { + t.Fatalf("Unexpected compressor, got: %+v, want:%+v", got, mc) + } + + wantNames := []string{"mock-compressor"} + if !cmp.Equal(wantNames, grpcutil.RegisteredCompressorNames) { + t.Fatalf("Unexpected compressor names, got: %+v, want:%+v", grpcutil.RegisteredCompressorNames, wantNames) + } +} diff --git a/encoding/gzip/gzip.go b/encoding/gzip/gzip.go index 5f2991a3b377..6306e8bb0f0a 100644 --- a/encoding/gzip/gzip.go +++ b/encoding/gzip/gzip.go @@ -18,7 +18,11 @@ // Package gzip implements and registers the gzip compressor // during the initialization. -// This package is EXPERIMENTAL. +// +// # Experimental +// +// Notice: This package is EXPERIMENTAL and may be changed or removed in a +// later release. package gzip import ( @@ -26,7 +30,6 @@ import ( "encoding/binary" "fmt" "io" - "io/ioutil" "sync" "google.golang.org/grpc/encoding" @@ -37,8 +40,8 @@ const Name = "gzip" func init() { c := &compressor{} - c.poolCompressor.New = func() interface{} { - return &writer{Writer: gzip.NewWriter(ioutil.Discard), pool: &c.poolCompressor} + c.poolCompressor.New = func() any { + return &writer{Writer: gzip.NewWriter(io.Discard), pool: &c.poolCompressor} } encoding.RegisterCompressor(c) } @@ -58,8 +61,8 @@ func SetLevel(level int) error { return fmt.Errorf("grpc: invalid gzip compression level: %d", level) } c := encoding.GetCompressor(Name).(*compressor) - c.poolCompressor.New = func() interface{} { - w, err := gzip.NewWriterLevel(ioutil.Discard, level) + c.poolCompressor.New = func() any { + w, err := gzip.NewWriterLevel(io.Discard, level) if err != nil { panic(err) } diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index 66b97a6f692a..0ee3d3bae973 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -21,8 +21,7 @@ package proto import ( - "math" - "sync" + "fmt" "github.com/golang/protobuf/proto" "google.golang.org/grpc/encoding" @@ -38,73 +37,22 @@ func init() { // codec is a Codec implementation with protobuf. It is the default codec for gRPC. type codec struct{} -type cachedProtoBuffer struct { - lastMarshaledSize uint32 - proto.Buffer -} - -func capToMaxInt32(val int) uint32 { - if val > math.MaxInt32 { - return uint32(math.MaxInt32) - } - return uint32(val) -} - -func marshal(v interface{}, cb *cachedProtoBuffer) ([]byte, error) { - protoMsg := v.(proto.Message) - newSlice := make([]byte, 0, cb.lastMarshaledSize) - - cb.SetBuf(newSlice) - cb.Reset() - if err := cb.Marshal(protoMsg); err != nil { - return nil, err +func (codec) Marshal(v any) ([]byte, error) { + vv, ok := v.(proto.Message) + if !ok { + return nil, fmt.Errorf("failed to marshal, message is %T, want proto.Message", v) } - out := cb.Bytes() - cb.lastMarshaledSize = capToMaxInt32(len(out)) - return out, nil + return proto.Marshal(vv) } -func (codec) Marshal(v interface{}) ([]byte, error) { - if pm, ok := v.(proto.Marshaler); ok { - // object can marshal itself, no need for buffer - return pm.Marshal() +func (codec) Unmarshal(data []byte, v any) error { + vv, ok := v.(proto.Message) + if !ok { + return fmt.Errorf("failed to unmarshal, message is %T, want proto.Message", v) } - - cb := protoBufferPool.Get().(*cachedProtoBuffer) - out, err := marshal(v, cb) - - // put back buffer and lose the ref to the slice - cb.SetBuf(nil) - protoBufferPool.Put(cb) - return out, err -} - -func (codec) Unmarshal(data []byte, v interface{}) error { - protoMsg := v.(proto.Message) - protoMsg.Reset() - - if pu, ok := protoMsg.(proto.Unmarshaler); ok { - // object can unmarshal itself, no need for buffer - return pu.Unmarshal(data) - } - - cb := protoBufferPool.Get().(*cachedProtoBuffer) - cb.SetBuf(data) - err := cb.Unmarshal(protoMsg) - cb.SetBuf(nil) - protoBufferPool.Put(cb) - return err + return proto.Unmarshal(data, vv) } func (codec) Name() string { return Name } - -var protoBufferPool = &sync.Pool{ - New: func() interface{} { - return &cachedProtoBuffer{ - Buffer: proto.Buffer{}, - lastMarshaledSize: 16, - } - }, -} diff --git a/encoding/proto/proto_test.go b/encoding/proto/proto_test.go index e56015bfe63e..8421ea7a3896 100644 --- a/encoding/proto/proto_test.go +++ b/encoding/proto/proto_test.go @@ -106,11 +106,11 @@ func (s) TestStaggeredMarshalAndUnmarshalUsingSamePool(t *testing.T) { var err error if m1, err = codec1.Marshal(&proto1); err != nil { - t.Errorf("codec.Marshal(%v) failed", proto1) + t.Errorf("codec.Marshal(%s) failed", &proto1) } if m2, err = codec2.Marshal(&proto2); err != nil { - t.Errorf("codec.Marshal(%v) failed", proto2) + t.Errorf("codec.Marshal(%s) failed", &proto2) } if err = codec1.Unmarshal(m1, &proto1); err != nil { diff --git a/examples/README.md b/examples/README.md index bb2138f26ffb..2a5c88cd1cbe 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,29 +1,10 @@ -# gRPC Hello World +# Examples -Follow these setup to run the [quick start][] example: +The following examples are provided to help users get started with gRPC-Go. +They are arranged as follows: - 1. Get the code: +* `helloworld` - a simple example showing a basic client and server +* `routeguide` - a more complicated example showing different types of streaming RPCs +* `features` - a collection of examples, each focused on a single gRPC feature - ```console - $ go get google.golang.org/grpc/examples/helloworld/greeter_client - $ go get google.golang.org/grpc/examples/helloworld/greeter_server - ``` - - 2. Run the server: - - ```console - $ $(go env GOPATH)/bin/greeter_server & - ``` - - 3. Run the client: - - ```console - $ $(go env GOPATH)/bin/greeter_client - Greeting: Hello world - ``` - -For more details (including instructions for making a small change to the -example code) or if you're having trouble running this example, see [Quick -Start][]. - -[quick start]: https://grpc.io/docs/languages/go/quickstart +`data` is a directory containing data used by the examples, e.g. TLS certificates. diff --git a/examples/data/x509/README.md b/examples/data/x509/README.md new file mode 100644 index 000000000000..3b9a05dac364 --- /dev/null +++ b/examples/data/x509/README.md @@ -0,0 +1,6 @@ +This directory contains x509 certificates and associated private keys used in +examples. + +How were these test certs/keys generated ? +------------------------------------------ +Run `./create.sh` diff --git a/examples/data/x509/ca_cert.pem b/examples/data/x509/ca_cert.pem index eee033e8cb05..868a01eb92f9 100644 --- a/examples/data/x509/ca_cert.pem +++ b/examples/data/x509/ca_cert.pem @@ -1,34 +1,34 @@ -----BEGIN CERTIFICATE----- -MIIF6jCCA9KgAwIBAgIJAKnJpgBC9CHNMA0GCSqGSIb3DQEBCwUAMFAxCzAJBgNV +MIIF6jCCA9KgAwIBAgIJANQvyb7tgLDkMA0GCSqGSIb3DQEBCwUAMFAxCzAJBgNV BAYTAlVTMQswCQYDVQQIDAJDQTEMMAoGA1UEBwwDU1ZMMQ0wCwYDVQQKDARnUlBD -MRcwFQYDVQQDDA50ZXN0LXNlcnZlcl9jYTAeFw0yMDA4MDQwMTU5NTdaFw0zMDA4 -MDIwMTU5NTdaMFAxCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEMMAoGA1UEBwwD +MRcwFQYDVQQDDA50ZXN0LXNlcnZlcl9jYTAeFw0yMjAzMTgyMTQ0NTZaFw0zMjAz +MTUyMTQ0NTZaMFAxCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEMMAoGA1UEBwwD U1ZMMQ0wCwYDVQQKDARnUlBDMRcwFQYDVQQDDA50ZXN0LXNlcnZlcl9jYTCCAiIw -DQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAMZFKSUi+PlQ6z/aTz1Jp9lqrFAY -38cEIzpxS9ktQiWvLoYICImXRFhCH/h+WjmiyV8zYHcbft63BTUwgXJFuE0cxsJY -mqOUYL2wTD5PzgoN0B9KVgKyyi0SQ6WH9+D2ZvYAolHb1l6pYuxxk1bQL2OA80Cc -K659UioynIQtJ52NRqGRDI2EYsC9XRuhfddnDu/RwBaiv3ix84R3VAqcgRyOeGwH -cX2e+aX0m6ULnsiyPXG9y9wQi956CGGZimInV63S+sU3Mc6PuUt8rwFlmSXCZ/07 -D8No5ljNUo6Vt2BpAMQzSz+SU4PUFE7Vxbq4ypI+2ZbkI80YjDwF52/pMauqZFIP -Kjw0b2yyWD/F4hLmR7Rx9d8EFWRLZm2VYSVMiQTwANpb+uL7+kH8UE3QF7tryH8K -G65mMh18XiERgSAWgs5Z8j/B1W5bl17PVx2Ii1dYp0IquyAVjCIKRrFituvoXXZj -FHHpb/aUDpW0SYrT5dmDhAAGFkYfMTFd4EOj6bWepZtRRjPeIHR9B2yx8U0tFSMf -tuHCj95l2izJDUfKhVIkigpbRrElI2QqXAPIyIOqcdzlgtI6DIanCd/CwsfdyaEs -7AnW2mFWarbkxpw92RdGxYy6WXbdM+2EdY+cWKys06upINcnG2zvkCflAE39fg9F -BVCJC71oO3laXnf7AgMBAAGjgcYwgcMwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4E -FgQUBuToaw2a+AV/vfbooJn3yzwA3lMwgYAGA1UdIwR5MHeAFAbk6GsNmvgFf732 -6KCZ98s8AN5ToVSkUjBQMQswCQYDVQQGEwJVUzELMAkGA1UECAwCQ0ExDDAKBgNV +DQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBANGmhBQQ5f3n4UhgJLsXHh3CE3ej +Ox36ob+Hnny9Gb/OquA4FMKjTTaSrhKIQapqlCLODai50XKSRBJcgsvsqWk9UdL2 +3zf7CzAPmg5CmzpWWwgpKPTuK5W+gLA1+uMKecBdH5gqSswQ3TD1fMfnJuq9mNfC +GsMkplaqS5VATNFPVnqS7us3OXKEITmBaQP4wOpGP1PgqX7K08aZEeAyQJaTS5um +4MNlBLYa/nQ9Wca0Uk5tzoNjE6mWH7bTuwdoZgOIwKFmBbmsC9y/HzwV/zRsL8Yp ++7FwfIYuZ5j8gBNqSFQjDFkm6Q7RcQ/lyHHj9YduOgTciIFVgx+j8aZvFqH127h8 +WIb7Jppy0DEDJE1hRP6iV2uVoaUxhXWrCWLBUU+naLix7SJ8rqw8gHwRNWfM/Lwg +I3rGXdw5WIHVQcuxevN6qVSZeWVYAlAgfxjKtM5cKZyM+W80CSdVKEku1XA0sq6h +jaiJdo6hpm8BLIB2k7LWafc5MASst7XULk4uDC/OYcEz3+C3Ryn1qBltr1gA3+5K +ANuhjYCZH4P0pX08I1MpeVP6h8XhbBPEZg2txbVGlnDXEFoJN9Eg5iEKRBo/HKhf +lP84ljtBSmCnsF6K/y3vnRiu+BVNP5KMq179DNqEy7tSygzgY41m3pSFojdvA59N +JWJoy9/NZzdlU4nzAgMBAAGjgcYwgcMwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4E +FgQUW5AMXXg/zPSaLHwSO/7LwoBeZYUwgYAGA1UdIwR5MHeAFFuQDF14P8z0mix8 +Ejv+y8KAXmWFoVSkUjBQMQswCQYDVQQGEwJVUzELMAkGA1UECAwCQ0ExDDAKBgNV BAcMA1NWTDENMAsGA1UECgwEZ1JQQzEXMBUGA1UEAwwOdGVzdC1zZXJ2ZXJfY2GC -CQCpyaYAQvQhzTAOBgNVHQ8BAf8EBAMCAgQwDQYJKoZIhvcNAQELBQADggIBALUz -P2SiZAXZDwCH8kzHbLqsqacSM81bUSuG153t3fhwZU8hzXgQqifFububLkrLaRCj -VvtIS3XsbHmKYD1TBOOCZy5zE2KdpWYW47LmogBqUllKCSD099UHFB2YUepK9Zci -oxYJMhNWIhkoJ/NJMp70A8PZtxUvZafeUQl6xueo1yPbfQubg0lG9Pp2xkmTypSv -WJkpRyX8GSJYFoFFYdNcvICVw7E/Zg+PGXe8gjpAGWW8KxxaohPsdLid6f3KauJM -UCi/WQECzIpNzxQDSqnGeoqbZp+2y6mhgECQ3mG/K75n0fX0aV88DNwTd1o0xOpv -lHJo8VD9mvwnapbm/Bc7NWIzCjL8fo0IviRkmAuoz525eBy6NsUCf1f432auvNbg -OUaGGrY6Kse9sF8Tsc8XMoT9AfGQaR8Ay7oJHjaCZccvuxpB2n//L1UAjMRPYd2y -XAiSN2xz7WauUh4+v48lKbWa+dwn1G0pa6ZGB7IGBUbgva8Fi3iqVh3UZoz+0PFM -qVLG2SzhfMTMHg0kF+rI4eOcEKc1j3A83DmTTPZDz3APn53weJLJhKzrgQiI1JRW -boAJ4VFQF6zjxeecCIIiekH6saYKnol2yL6ksm0jyHoFejkrHWrzoRAwIhTf9avj -G7QS5fiSQk4PXCX42J5aS/zISy85RT120bkBjV/P +CQDUL8m+7YCw5DAOBgNVHQ8BAf8EBAMCAgQwDQYJKoZIhvcNAQELBQADggIBAKTh +Ofg4WospSN7Gg/q3bQqfSMT5XTFC7cj0j3cWDZBnmqb0HAFPmzHT+w3kBVNCyx1r +iatOhaZRH7RA0vacZQT5pD2MGU48/zFfwBV/qHENQWuRLD2WOOEU3cjjoINBclfP +im7ml/xgz0ACOgUyf+/2hkS7VLq4p9QQVGf2TQt65DZA9mUylZTdsBf4AfEg7IXv +gaYpq6tYmNi7fXDzR/LT+fPd4ejQARy9U7uVhecyH9zTUMzm2Fr/p7HhydSXNwhF +JUfPWw7XYO0lyA+8PxUSAKXOfsT44WNtHAeRm/Gkmn8inBdedFia/+M67k45b/wY +RF11QzvaMR33jmrdZWxCc0Xjg8oZIP7T9MfGFULEGCpB3NY4YjnRrid/JZ/edhPR +2iOiEiek4qAaxeIne3CR2dqCM+n+FV1zCs4n3S0os4+kknnS5aNR5wZpqpZfG0Co +FyWE+dE51cGcub1wT1oi5Xrxg/iRteCfd33Ky668FYKA/tHHdqkVfBflATU6iOtw +dIzvFJk1H1mUwpJrH/aNOHzVCQ5KSpcc+kXcOQPafTHFB6zMVJ6O+Vm7SrqiSENM +2b1fBKxHIsxOtwrKuzbRhU5+eAICqwMd6gcIpT/JSR1r+UfHVcrXalbeazmT2DS5 +CFOeinj4WQvtPYOdbYsWg8Y9zGN4L9zH6GovM1wD -----END CERTIFICATE----- diff --git a/examples/data/x509/ca_key.pem b/examples/data/x509/ca_key.pem new file mode 100644 index 000000000000..4dccea1be392 --- /dev/null +++ b/examples/data/x509/ca_key.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDRpoQUEOX95+FI +YCS7Fx4dwhN3ozsd+qG/h558vRm/zqrgOBTCo002kq4SiEGqapQizg2oudFykkQS +XILL7KlpPVHS9t83+wswD5oOQps6VlsIKSj07iuVvoCwNfrjCnnAXR+YKkrMEN0w +9XzH5ybqvZjXwhrDJKZWqkuVQEzRT1Z6ku7rNzlyhCE5gWkD+MDqRj9T4Kl+ytPG +mRHgMkCWk0ubpuDDZQS2Gv50PVnGtFJObc6DYxOplh+207sHaGYDiMChZgW5rAvc +vx88Ff80bC/GKfuxcHyGLmeY/IATakhUIwxZJukO0XEP5chx4/WHbjoE3IiBVYMf +o/Gmbxah9du4fFiG+yaactAxAyRNYUT+oldrlaGlMYV1qwliwVFPp2i4se0ifK6s +PIB8ETVnzPy8ICN6xl3cOViB1UHLsXrzeqlUmXllWAJQIH8YyrTOXCmcjPlvNAkn +VShJLtVwNLKuoY2oiXaOoaZvASyAdpOy1mn3OTAErLe11C5OLgwvzmHBM9/gt0cp +9agZba9YAN/uSgDboY2AmR+D9KV9PCNTKXlT+ofF4WwTxGYNrcW1RpZw1xBaCTfR +IOYhCkQaPxyoX5T/OJY7QUpgp7Beiv8t750YrvgVTT+SjKte/QzahMu7UsoM4GON +Zt6UhaI3bwOfTSViaMvfzWc3ZVOJ8wIDAQABAoICAQCxi7A9AhaUUWRzE6DnpGtH +zk0IO39cIx4KAsNQZiDBVDdXzYafUwaX2d57KVNbDAlJ9HCS3FKpEX9+gUPviQvr +aRe7boCZewv9dqkDvJqS7AEJxzm9O1pD5WI8WGqRDhUPuI2CIwbXDM0VokA7VuGZ +WFlxFxvs+UO5D10VF7A2blcRVQ/quQj4lzc/6P1TdL2DaVxGH3PLQd/ZR1ZhJI2Y +N0OHnOqp7wnvYqrtK+u0oI83hjym/ifvrYhMH8E7Q8lo4s4noSvmEvK0zlKYYxSO +g7RtwK47lcSPKgtn/yZDyvVX85qIgbBLcUmrqfB3qxMKz2lpJo6f4Rg7mm6SgW+K +zxYnGNCTPfiyPKiufM3rQPfJ4giqQ1XDKiZEKUJBo4mzzV6LcAoDaEqhHBlySpi3 +Z38I0rmAT62PRJ1sMkQl6j1Ben9TpwTzJmLX1sEO1Jsabsk8rRdV+ni5oRRUdW4H ++ratyQ8pmegLYyhAZqkD7FzKBLdznLmWXVTcBQkRoD5lQkCP2OF78TdL4twNvoTH +X4kQ3cNysWFXsm+yf4jSCHl4BEtGA2jOU690T0trtMf13aI3wEULmcBgc2ix+tch +wX79hwBYcjGGDfTMb39r/DrcgWMVFXawru78QFoN9vVxznit9LrOERBm6zN2ok4X +E1kD4YZGr8dxUHax0or4CQKCAQEA7W1Sxeqc0gV0ANQf3eCsFNjvT97z/RSzzUYF +wCe4rpzQ9ZNsY2UYMYmEzUuRBuQxYKCNTWot3hu+6OPMCp4pLuu2l8ha/wCM2TkY +6hceduvXkdUNUG1xZNSR8waw4PTXNeoOD30+GB4OpHdjzsF5pEzx853/Qo/ERJFx +A+aZZJy/Sfw82KTseYTniWYjH4iYUbC8TVLfRjPw6V2VcF78pYkdAQenGglqw/sI +4a3FhJspN9xV/PoPbb7PjBJFHUt7ZRQt+D3WPuhLSjyPxwV+3u2OsQ1/J/sxcih6 +rW2g+OJYrK4YkOqX9tLRB39RjO4H6Eiv5eUAw/+vHHufKRu1HwKCAQEA4gzxZNzm +r1X/5GAwwyBJ4eQUHFvEQsC2L4GTJnNNAvmJzSIWnmxGfFLhfJSabnlCMYelMhKS +Ntxokk5ItOhxlUbA1CucEtQgehJwREpUljlk7cii5MLZEkz11QxIVoAhGlq3svFG +B/gwYWNVWl2CXcK2o6BBD9sIgzgp7qhmdJej16h8YkWn7HibKs+OBcdCu+ri7wU+ +VdLpdhN3uqo1b1tO58Gv+40vuQE3ZKDdMy55V30+0qEqg6dXvDQ9nwYFkw6C31Ad +Wpa9ZB0A0HNSou1xTWyl/hDie6dlN84RHGX8on4sjgPrb8A8WVis+R2abvh9ApZA +fRZ3H/ZYXB1crQKCAQBgjgEHc+3qi0UtwRZkiSXyJHbOKIFY/r5QUJWuG3lDqYph +FF8T3N0F6EMVqhGEl/Bst14/iVq15Nqyo1ErUD63UiyjdVtsMLEW9d1n9ZbyDd9Q +8y/C8X8X3kqsZqAwG+IZjuHA8tH5xN93iwYP4yaw5onO5QYV75mFuRAY4gKnpAc2 +81lbUVbJ5H60pdDK1iX7ssAhQf6C8kSa4vAPDtH4D9a3wID4WbQNl115Sc31q5QL +n5NomdkEbIDDGfr5euTnqlk3hw5F7voPaqmd6mI6Dqnk3vRDMihdoJCjTt4T2Rju +wK5E4OKEAh/3yJNFmNemY0kFWSgCjUyNbMjBUv9JAoIBAQCYS9QO+m1JUA2ZVd1E +eWqNkFakTIdL2f5kv03ep+wIxwq6c+79SUGr3UMh5hStvXCFYjhAJhbwc0rY13lQ +uRJdWk/sIn2CifxfgjC1MccPdxeyxGxK56PMGqG9qgrKjITA9sGxA7EFCYe+9We5 +/Coq9VaLoxpyjkWL8rj9m+N7RfcTAubaZseeIBuamj+7UOZ7KOM/2i6HMBQugys1 +Thu2LLRanDnups6yPEmPuHmPVA5YjX9X9VFpZcNMf33MuAflbe9qeNVuBQUQgCHe +TvQr5QFjAoJLTCDq4nrlQCZzFZtB9vQZsjZbEg8WuxG+vN0hSrUemxBTtmEH3bbm +SLn5AoIBABGxznQFXXlF3eLIZqLvItDMSTpFp8YPk8GQWPT2V3pNNjvK/j7eg+tn +VouXv5LjyLTzWLKnPjIU4t+qwu6R9nohZ62OjGl6lssVdjPnf4R6UKzRa0iIZtH4 +BlGncnAbzb6TJuLX7dNwICoUCGyvk9tdnThH1FY3ZAEhOi1G8LEh7aBrj9/vUZ2d +S5jzZ7kLh04AB8OP1MXM3sZE7VlIxUtT/NLlwC8zRsg84pAjg3U7PygIDYQDzCRB +4yIvDziTPqDB/vdCKt7/Xary5Xj4NwqcPCRf6HvdHYCVeW7V+mWcMKZgodQARQhv +qQCK9iiN08MAFNia/0/Bj4D7XKurNRY= +-----END PRIVATE KEY----- diff --git a/examples/data/x509/client_ca_cert.pem b/examples/data/x509/client_ca_cert.pem new file mode 100644 index 000000000000..62a0ce0545a6 --- /dev/null +++ b/examples/data/x509/client_ca_cert.pem @@ -0,0 +1,34 @@ +-----BEGIN CERTIFICATE----- +MIIF6jCCA9KgAwIBAgIJAOhoXtjjP6JdMA0GCSqGSIb3DQEBCwUAMFAxCzAJBgNV +BAYTAlVTMQswCQYDVQQIDAJDQTEMMAoGA1UEBwwDU1ZMMQ0wCwYDVQQKDARnUlBD +MRcwFQYDVQQDDA50ZXN0LWNsaWVudF9jYTAeFw0yMjAzMTgyMTQ0NThaFw0zMjAz +MTUyMTQ0NThaMFAxCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEMMAoGA1UEBwwD +U1ZMMQ0wCwYDVQQKDARnUlBDMRcwFQYDVQQDDA50ZXN0LWNsaWVudF9jYTCCAiIw +DQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAO7fTqeU+8OfKMwXABNF90+RYL4X +YS4ULx4rpf14Ntp1SF6o3itCSM3jJfHzexj2Pm16aL+OQll8ODtvTadqVSMndMCn +UN/jVjxiMmjkSNKpwUGG69CsQzCKoueKBCEy/CZSopQae6Wxn7mqTAzhFlh3idNL +J+12UtdqDxnPDsiG2XBET3UrKyJeBxMgRyPi/g4wHfhH9oJ97jkdacUlLko8l22s +ZiMSSwwOlWxtTY5t0FbHu08ufP4eYTqC0LL3z1Fon4v+4BqUyK7BT3dISwPBmSd1 +uTD7Wbaa/QmfU6Y18dkNlK00GUAcKWgPfLcm7EH/AAz5XkqozVR3z5FLBYFTxVrA +Ly/Gu5HLx/uwoYWeYRWBOSkqvdgf9PT57imO4fOi1CTQuq/1LAdaxGkm7yXaz0YP +ySTiT6PvcLWFEbjrbufxdBrF4/ZsQz5vdJiKq2IQmCIKONJOFHWqgoF4AA7Ze1cl +mrK0eLzUlG1WmSy5mpjByRanahQWYvK1s0tc8IwMRRJY4DS6Dp99EVyteKZP/jc0 +x+ILet2ThDhjY3AxtkzlejyylABgl2AyGoGzZzbaf1q/0LfM6SfYBSVZK3TFR3Kt +8lQnG0tztoM+bnM/JZ8UZ61s16jJVxWzlZ+rx8rCpIvh3Cnl52DGo6oA4Kt60uDP +3iiTLGNYqEyHmzgnAgMBAAGjgcYwgcMwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4E +FgQUdOqNqaSjcn7BRN3fLs4eTIp1W9MwgYAGA1UdIwR5MHeAFHTqjamko3J+wUTd +3y7OHkyKdVvToVSkUjBQMQswCQYDVQQGEwJVUzELMAkGA1UECAwCQ0ExDDAKBgNV +BAcMA1NWTDENMAsGA1UECgwEZ1JQQzEXMBUGA1UEAwwOdGVzdC1jbGllbnRfY2GC +CQDoaF7Y4z+iXTAOBgNVHQ8BAf8EBAMCAgQwDQYJKoZIhvcNAQELBQADggIBAOnH +CrwiJd51oBic5PwjQBhQcUtGOfR1BJe/PACpLXTf1Fbo8bLT5GxZLATlw9+EVO9P +JhhH+oiUuvA7dE2SRiZXpY7faqtDgvVfssyCrvACkM7pcP9A5kM4LiunX7dpY2xp +naJAqDV5Av1mOohHuVEZHqV6xQSREQFW2IusfpCsPP+P+RPKM2o571e6oz5RGbuP +dQ39QycBTK8ezccxaDaH614peAnBi4Q1GuxzgNmXq2FPDcf7F1QcWMrW3jUI8npi +Q9rXRwrqUYP7Yzz+dIziGdpOfZd7x/MyCXuqRdFdA+bulGM2Es5lvtguPOFhcWp0 +3hzLJ+yolxyqxnNNdaU0r+TDbgxOBjw0VxahuhzFDeZsP6Civzp+Y6MRdvofNXBm +IBD4uqmQtUUyE2uoznXvZkXaSc+0VIGgs04AMS9irBC2oVEGDp0AbelcIhdgToam +/NTuOmxgadwDuEn3TIFYkzx84J81kL8g0HQ1N09nSXChkSVb+XlxC+Wosxoazydr +M4FOvaa1V4vnmIdA2aF1nWTzJNcc9FC23zTmQkV2YJ1IKNmxGd3xBZzUtUBu5OgZ +vPXECtUjRcraNuXeL6gSX0qBaaVkcdxhp8CpI8k6Qb+mgOaq/ixrVEKtczBVXjHD +pO6QmwMZtqR8JsStbMCYXa2owt4k8F3yMlIKE6qX +-----END CERTIFICATE----- diff --git a/examples/data/x509/client_ca_key.pem b/examples/data/x509/client_ca_key.pem new file mode 100644 index 000000000000..77065d5cc8a9 --- /dev/null +++ b/examples/data/x509/client_ca_key.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDu306nlPvDnyjM +FwATRfdPkWC+F2EuFC8eK6X9eDbadUheqN4rQkjN4yXx83sY9j5temi/jkJZfDg7 +b02nalUjJ3TAp1Df41Y8YjJo5EjSqcFBhuvQrEMwiqLnigQhMvwmUqKUGnulsZ+5 +qkwM4RZYd4nTSyftdlLXag8Zzw7IhtlwRE91KysiXgcTIEcj4v4OMB34R/aCfe45 +HWnFJS5KPJdtrGYjEksMDpVsbU2ObdBWx7tPLnz+HmE6gtCy989RaJ+L/uAalMiu +wU93SEsDwZkndbkw+1m2mv0Jn1OmNfHZDZStNBlAHCloD3y3JuxB/wAM+V5KqM1U +d8+RSwWBU8VawC8vxruRy8f7sKGFnmEVgTkpKr3YH/T0+e4pjuHzotQk0Lqv9SwH +WsRpJu8l2s9GD8kk4k+j73C1hRG4627n8XQaxeP2bEM+b3SYiqtiEJgiCjjSThR1 +qoKBeAAO2XtXJZqytHi81JRtVpksuZqYwckWp2oUFmLytbNLXPCMDEUSWOA0ug6f +fRFcrXimT/43NMfiC3rdk4Q4Y2NwMbZM5Xo8spQAYJdgMhqBs2c22n9av9C3zOkn +2AUlWSt0xUdyrfJUJxtLc7aDPm5zPyWfFGetbNeoyVcVs5Wfq8fKwqSL4dwp5edg +xqOqAOCretLgz94okyxjWKhMh5s4JwIDAQABAoICAAmMq9xPPHFpn3vpP3uFxIlN +yoxO6veonumZ3Rzw/WBmZ+pA3gDkuXxhpFaz4SvyTDScPCvMSCLDsIvPu08CFT0+ +ipBZIAaTVBM96b3/wlmJp8wy1KKXAGikYjbXcarSGvp9OzqohGDvZO9LO5cYOIh4 +3u2vh30ayd0KxGfHu1OQ8IhocrTAcQ0CrU26cJ2iqX1vtwMB/XziA/AMmPnkrqER +IwyjY8HrLUziGF8pT3xuL3IIshhMR3rxQ/nO2QEOnx8mC5rRKaxmXk9+MusV3Mnd +p33IWwr2QXPnZk5ILFPsvCptPJBgENJbTdx3IglAaRmKVDowjfB2Jx9FWur4ENQy ++yCzf0ygRoXnugtwE48/L7P8mlqZlZsxQbUUjXEPtht8rtM4CR5b0v7PHXiLh1oM +igfy1RDAQAZQRGIlWCOeV2soiyKLnCGyAaVXcM2ksDkYOSH4ObE4KwF1Ph87lNaG +ywolsPvQD0ygymXcuStrYHWamTp8qRjNvZBcThs3SaKN+lxXxPng2tBPUwU0S6nj +e0pjWco74elBk+fjjd0wNolKjUD7FhRXlWiXz9BgcCjRD9TLoVk8mp9cFL7OLzJc +735JmNKP8C5Qs91Ugo6Z9tWQQTdGHZe9ElUY0fWP0bs+4iBaadl63R26tchLncZE +LnYsi2AjDdV908cEkAiBAoIBAQD6LbGeyFHZA42nuSw/NFsMVldqU6QwmADQI3Tw +JEdw2thS8VIX2c8aeJkVL++dNmSPcqs4NqhzgJSm9o1xNqGZovAPK/B3NmLl1kzG +JPwSr8QwNxmKwUlbt1K48qIV0JmetOgRG/ll5ux2CxgWHzwgRwtvpbnxDa7Gf7BA +UfH7AfZJ3iV+HlJSxr9XxNgFoNEtpP9sqbOgt10f5JJlIELCTa38iMBojAGxlzyj +7DGYY/diQDr+6mRNnv2pY57dOnmdvN1w+p1W7saaeRCeltva/G+5n5AWMFl5qBjT +LDktBE+okH5wapkUsZzZTByTgFXdBC2wY2qBrOexBAyS8/F3AoIBAQD0bkNBc1ya +KYmWlCsVSUZxUGSOp9g7ZdzlB/1G523s3PltXSphsC4mACs7ZAs5OAO/bu05kurp +dOqEAxsC05IxD2/gGoarC6QfTum9CMNoKrvtczA7Gl+6D5djum17lULY6YSBO75J +L0FQK6nCVGfAbBRAqhiFi+9kXvNThuqjgoiCNwQYxaG8aovoAKTFdkzQjDw2tUgM +jqCM6ifOBJIRolFq2CBom8nB+wpsI1naFLaOdg0Luz/Ds03gD9nWa6a4XIowKCml +Tek1Q+S2hZoTgfOlKRbCcM1KyoaI9LKI/pbKmpNyyrADw/kZKevfsKnYwMpHlaTR +NSuQ2VJKuxrRAoIBAQCBQ3bQ+eQAYyugC7dm+OBKYZpNH+ZoDUHuSUO0iKo5D3pS +cMnf9PRjUwiVv+zoqCARVkhNhUBIXZlxI1c1teqNfXjX/fYDQqCa7L1Ca/2qkhKm +bvHNlc0XjIM7eHJzHxMgw4xcur2D/2sSGu1ZEM56RvsLtu96M32opnUk5rJG5V6i +EBwDLBuRFYvsB5MuZUdvdB9dv9lGIzgEsI9LnP2hc42APBBedGizn9b/Q5zkhlJd ++53/9I/a41lhWk3NNNd9vwYTyAnfzwPi8Ma7imsSnPgFSwKh1F2G1GnvQpxQPDgE +epQ59XofDR5j0EW7mMXEqtIIn3V6hyI3fkYY795FAoIBAQCsx7x26YsN1krRzA7g +TxmiQ8exJ2gsJIcOxqT8l98WTeVqry6kOxuD9R6aLs/YNIZBrbG2vuma+PBFPMS9 +LLzsPRNCAL4s7l+nWerTmvw2B+8rm/796Fi+dwL2lfOKJipIllj52TdbGDI874Bi +Q7PLSxrN0u7eh9pCwvORmY8G4eCI20bkE9+OBmq7JqlSg5ss19RAf8hcR/2pXmOg +t45hNLIEqp3OFEF8A26MnjiHdZjN/xidsFEUjwx/U/USIqqJK7Dq9ZjqprYw1rs3 +Yh1VqMiHeRIDhCU5twt+iCojuILy2G1d+XSOVNsiNIXtaz3EYBMcouUMlV8kVtpa +xQPhAoIBAEr8U7ZaAxN2Ptgb6B8M1CVNE6q7S1VuX+T8xkciadW2aRjJ3PufFfsk +Zo12fP9K/NeOPTIz0dQB6Gy/CKzDLb8NnJCJnCUUaO8E45C2L9r6qvIJpXWHp3vo +neGO49y/5st7suOZkWU2B6ZGwNWH90296mfSKcUNxSRMaHCotPdVDyvOgLC24ZWR +6teRaxB2sVZYqmoz+4+G8SOK40bHJKf1kwujbrS3OqzDzEeC/STtqYZWPW03MFkk +MBPQvwCWMJINv4zz4YrnOaA9COc1/fTXCG5kKYyalPD8VKxi1usas1pZwIqZkuwm +D6kBMuZ4gkKW24IYzXzOni0/BOnpOfM= +-----END PRIVATE KEY----- diff --git a/examples/data/x509/client_cert.pem b/examples/data/x509/client_cert.pem new file mode 100644 index 000000000000..e35b94b1f27a --- /dev/null +++ b/examples/data/x509/client_cert.pem @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFcTCCA1mgAwIBAgICA+gwDQYJKoZIhvcNAQELBQAwUDELMAkGA1UEBhMCVVMx +CzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTALBgNVBAoMBGdSUEMxFzAVBgNV +BAMMDnRlc3QtY2xpZW50X2NhMB4XDTIyMDMxODIxNDQ1OVoXDTMyMDMxNTIxNDQ1 +OVowTjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTAL +BgNVBAoMBGdSUEMxFTATBgNVBAMMDHRlc3QtY2xpZW50MTCCAiIwDQYJKoZIhvcN +AQEBBQADggIPADCCAgoCggIBAL2ec6a93OYIioefCs3KRz752E5VfJPyVuxalBMc +7Dx84NsdwpbUyDT6fO7ePYM8IvYAsLc5coLCP1HKGGRmYm423WZf8Kn93BDl0XcN +4bgtW9ZrekvYcXqSzygz3ifdQeZljZrqW43dkkYR2vWc+uJXs+vrRVZyUSLLbe97 +9zUbWbOfHBc1jK1vTUakl08VhllYbO0m0SYZIni0sioItVdVWTz9XE2COavLqwwL +MIq8N7JXEdYJC49JWfdzvqZYTxOn5FSTCWen7/mcZmuLYPwUCkSu05M5T2o1ygkd +ohA+/X9yjToPJ7NO509lKHWo7+sp9if6jZsiOU45/t84pD6juVZSZ20/A9i6hjtj +C0SqYk2iQEtRp+lT6yYa5ffeNllFUGtM+xq2are2n93PnXwMTUlYGuTtkyRPG717 +ZtQjKQuwfdJNoNbJl2cfQpmtLdm4Jzrg5cWiiFro+aqnZxIfUEEDkIBaUjYmwMkS +Qq+S32L4f4u7rtbnzdo/jVwq0wpSjTGQJEab+v2wZpDhVbQblTyI30A+TvBIzLil +09OX49/teZCp05kOJy0V/yXdQtPwlQGXdsCUmD6dnGav17fB1witXDdG+4SNoyF/ +PN+8wtlMQ8fWvLdxLsd/Rq6CEZQV9mBhrQxXUmFFDhd0O6wfxR/lVFxIWg70Fz7P ++z7tAgMBAAGjVzBVMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFG0psrHrGny8ziVm +RtulG3f9ROrhMA4GA1UdDwEB/wQEAwIF4DAWBgNVHSUBAf8EDDAKBggrBgEFBQcD +AjANBgkqhkiG9w0BAQsFAAOCAgEAtr1dzSQswIOlEGlLtoAwkL7ys/gP2fcdh7Jl +ggiPs266yzZFyGGdd2GKo6tcjdBNjfnO8T5h8eLzj7QlzKPqA/l0BgAW7s7WX9QF +wCivw1DHE815ujlQNo3yve38pd2/I0hdf9GtQLGyOirYpwW5YcHvpmLezrW6J3UU +CWIfYhqO6bSs+HCLkvQdsCG1TpveWYXfC9aXHjw+ZGOjBMEt6AgdWctwzTjQfZub +VjZosBC3ZkDjkA9LTqKP5f8XSWt89J4JCYkiFRiJuYYiNYcZpb0Ug93XjEHIHXMG +N/cD9fCB2HovoVu8YnezpSrqEhqEikHSq80fwbf+NaT0CEbPMx3UMzt8d8gwUiwE +nzzf/o4uOwoofNWfka0J1VPY1AtjUDvz44LyVhp4uvkEJEK1WQ46mM68H/EOUmpd +fHANEbV8HLq2iOjR78n5+MCHRcX7duScp5wT0ajfDg41VrhvV/u7YctFj8ynQJg5 +cqbH+GgTrEfAFFm5mZH1SGqNPyxr1eQFWXMRGE7R/NoyQo2uqrSRmz6JFXlnWtxF +YmLhnOdQaytcpiYN2YVyC/rLK3l3Tbh4u5axvlZP/hi+nQluiZzkH97iUqXcBU/9 +jYNohnJzXMHTIZM8FQY+9uGw9ErdDo7FmX5Xkp4TzEz9k10m1fnt0njSEzITtqpg +MoO9n00= +-----END CERTIFICATE----- diff --git a/examples/data/x509/client_key.pem b/examples/data/x509/client_key.pem new file mode 100644 index 000000000000..d9c4bae3bbb5 --- /dev/null +++ b/examples/data/x509/client_key.pem @@ -0,0 +1,51 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIJKAIBAAKCAgEAvZ5zpr3c5giKh58KzcpHPvnYTlV8k/JW7FqUExzsPHzg2x3C +ltTINPp87t49gzwi9gCwtzlygsI/UcoYZGZibjbdZl/wqf3cEOXRdw3huC1b1mt6 +S9hxepLPKDPeJ91B5mWNmupbjd2SRhHa9Zz64lez6+tFVnJRIstt73v3NRtZs58c +FzWMrW9NRqSXTxWGWVhs7SbRJhkieLSyKgi1V1VZPP1cTYI5q8urDAswirw3slcR +1gkLj0lZ93O+plhPE6fkVJMJZ6fv+Zxma4tg/BQKRK7TkzlPajXKCR2iED79f3KN +Og8ns07nT2Uodajv6yn2J/qNmyI5Tjn+3zikPqO5VlJnbT8D2LqGO2MLRKpiTaJA +S1Gn6VPrJhrl9942WUVQa0z7GrZqt7af3c+dfAxNSVga5O2TJE8bvXtm1CMpC7B9 +0k2g1smXZx9Cma0t2bgnOuDlxaKIWuj5qqdnEh9QQQOQgFpSNibAyRJCr5LfYvh/ +i7uu1ufN2j+NXCrTClKNMZAkRpv6/bBmkOFVtBuVPIjfQD5O8EjMuKXT05fj3+15 +kKnTmQ4nLRX/Jd1C0/CVAZd2wJSYPp2cZq/Xt8HXCK1cN0b7hI2jIX8837zC2UxD +x9a8t3Eux39GroIRlBX2YGGtDFdSYUUOF3Q7rB/FH+VUXEhaDvQXPs/7Pu0CAwEA +AQKCAgAtlwQ9adbLo/ASrYV+dwzsMkv0gY9DTvfhOeHyOnj+DhRN+njHpP9B5ZvW +Hq7xd6r8NKxIUVKb57Irqwh0Uz2FPEG9FIIbjQK1OVxEYJ0NmDJFem/b/n1CODwA +cYAPW541k+MZBRHgKQ67NB3OAeE8PFPw/A8euruRPxH+i3KjXSETE8VAO0rIhEMz +Ie2TQRydLKp71mJg45grJ17Sxmc7STT8efoQVKgjCwPkEGiqYpiNk2uhZ2lVGRC9 +cyG6gu74TdyTDQss1e7Xt+fUIZ2+3d6eJt6NvjC+25Ho4SwO9eYjF1qnQ++KqATr +TOoOaADPLLaXZCFZ1D+s9Dq4Vrj+QGk8Fajotj4gBpUtc0JxtvYM9EhlW7DpchYm +Cxe8vmEi/54YErXKawTUXYBB8IeDzwtvi3v3ktmH8BsGJ6Y3RXDI9KIG/6IE5Xeu +hkPCJnB0e3G2nlaffNSrVknxF+z74DB3T2kj0zC/4H4/hHo4W5D/pswcGWlhREWG +E7ViXJjBRkc5tpS9HfNdZ2wHiccioDIdGSHGqGMF4rLCUE2n+zc4m6pvvNCjN5KB +S4+zps50Gqtbp3DH2h1YLtkzuzvDhgpMPyJ1qZsdgelRSi2IaE5oekuBGP2WeXFw +DLI/cijc13cCacH+kpllQL//zBP8mMGmussWGgrVXdm9ZqD+rQKCAQEA6OG+s8sa +QZJ8W1nukcaS5rSvJBeZO6neCd6EB4oew5UGJsSz+x4RtJ7aJhdTGtyCXqiR2uFw +SBYdTcOgNbBUXg39vWAv+k2lmxiMGuLnAcNcGYyDLXr1SUJwe4Be984WNFdqzY0z +LCd9NvutWWX0Xd1VBdhlDuu3eBenzPBKIxTk3N2gLvzYxC/62e29Trsm7Sur11ut +Jay/CRdomjaqIiZ8q8qgdSU+pPe2DZYzUOutySJhLUegrrgWvPS/i8FHf7AGRgki +wpFn3gy5zCsFzr6n/TzJ5zQvlz+PcbUHHb06U1cnT45fkFNAJJvBYa4vi/tRx92E +Bi8d4bn40fUo3wKCAQEA0HFDHzhRxN/RbzBkymGlgfrsKcBdaAzgClo5uAXr8sdi +efsgBFo228I5lK6ywfzOfD/UxGB6ucdkZb/tRLtoK0OqOGiNx2Q1yazRVbuhrBrR +Y7DDbh7164o/MAYqPGxTMUxzXia7WBtNm00Tv9pDsw+NTzbrk7OxkLZWbjQEj99T +A9pcqXYA1RJtD/6io/43/oVscWPdRrbrNrJz+27Bsau20MBheVmX5sLTO2iWKTN4 +/ofrvOv0ru0I3ACHiLIaQFXs4snQjlhJm5MJ6kuZVdYKAzyNE+YOPnAxoiQAlHau +E1aV8ON7jmjhwxa2QICCwVcUNmwXU4UztGyGZ5a1swKCAQAi90Ia3LPkhIoHbUlU +uev0l8x0LtbjDm44LSDFwQc9dnKl/4LGgY1HAVLfxUDFF7a7X7QGmTKyoB9mPakg +ZolEVfVzKa4Kdv4We2kN4GOu8BYz/9TyTzPk/ATHhk68BkVvNnDizACS8JrsVn2A +nr5CGalaZ1NFGj9B2MtpCesXuVtjjiMu6ufhDRMtBXUXDSKbGaODglBNB9LnGoyq +GusQlZbCdHoDHMR7IHZFM/ggfkJpoK/WjJqjoSBI3raj1TFXCqbmfRiq/goKXP7I +mO0WTaoLa8Uk4cEDhJeVCwk2feL0AHH2j/npQZav6HLwp6ab7fApgikAhLKH4dRq +MdUhAoIBAQC7svJVf7qqRT3sGTD5yXpnlJPreOzj0IxC5kKJgtOYuJDl9Qw8vxwd +QkXlrHcOFl++JSCsgZCiEHpI4c6AER5Zr0HuL8BUJ9oDtJqA0EhimXeqhLdHR5v9 +sWz7CuInrQgxIX3V75zOVy/IRF0fayWBbeS6y2LRi4O/I2KrNC5TfC/eDVlZxAg1 +1rTdLVg5wqebi3w+k0Xj8r3WcFXeuTq0ikNCsapUwyf1RcU+/wwRJ+exlKXkZrnc +d1h9/AAQSQk4m+eHxWIHfFs0O/E2yULXt7kmdvU3UPfMo+0d67uV9VUF1veIhuBx +OeLqcV5GsTKNdaOe6jELJayMsRlK2LzfAoIBAEoWFSUdf3ruvj+ONju0TDtdvvTb ++i+3ttqMK/duYM2TlD3Lvqyx3kNxlMTAArfvnwtKVSw0ZIGSPc/5KHnxldcdALgT +4Ub1YesUv5585thMw1EWyXAPognLhfTEVSLYKcMPoBNCv7FvAT3Mk5SZPReRkbT9 +oqDAzg7r+0+pjD9LmnIXfCxfbSV6zcBFF8/iGAmzh3CanDqVkUds1+Ia8018cfDS +KW5PQAEnJC/BZAI7SQsxH0J9M7NYxJRN0bua5Be0N+uuYSOa+d9yecugfmvga6jf +9nEcohJShacCSkQvIXlq5Uy/WBb6sbiTmHjjW14FG25B0rrQUjmFAUiYceI= +-----END RSA PRIVATE KEY----- diff --git a/examples/data/x509/create.sh b/examples/data/x509/create.sh new file mode 100755 index 000000000000..2b5aa5cffa07 --- /dev/null +++ b/examples/data/x509/create.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +# Create the server CA certs. +openssl req -x509 \ + -newkey rsa:4096 \ + -nodes \ + -days 3650 \ + -keyout ca_key.pem \ + -out ca_cert.pem \ + -subj /C=US/ST=CA/L=SVL/O=gRPC/CN=test-server_ca/ \ + -config ./openssl.cnf \ + -extensions test_ca \ + -sha256 + +# Create the client CA certs. +openssl req -x509 \ + -newkey rsa:4096 \ + -nodes \ + -days 3650 \ + -keyout client_ca_key.pem \ + -out client_ca_cert.pem \ + -subj /C=US/ST=CA/L=SVL/O=gRPC/CN=test-client_ca/ \ + -config ./openssl.cnf \ + -extensions test_ca \ + -sha256 + +# Generate a server cert. +openssl genrsa -out server_key.pem 4096 +openssl req -new \ + -key server_key.pem \ + -days 3650 \ + -out server_csr.pem \ + -subj /C=US/ST=CA/L=SVL/O=gRPC/CN=test-server1/ \ + -config ./openssl.cnf \ + -reqexts test_server +openssl x509 -req \ + -in server_csr.pem \ + -CAkey ca_key.pem \ + -CA ca_cert.pem \ + -days 3650 \ + -set_serial 1000 \ + -out server_cert.pem \ + -extfile ./openssl.cnf \ + -extensions test_server \ + -sha256 +openssl verify -verbose -CAfile ca_cert.pem server_cert.pem + +# Generate a client cert. +openssl genrsa -out client_key.pem 4096 +openssl req -new \ + -key client_key.pem \ + -days 3650 \ + -out client_csr.pem \ + -subj /C=US/ST=CA/L=SVL/O=gRPC/CN=test-client1/ \ + -config ./openssl.cnf \ + -reqexts test_client +openssl x509 -req \ + -in client_csr.pem \ + -CAkey client_ca_key.pem \ + -CA client_ca_cert.pem \ + -days 3650 \ + -set_serial 1000 \ + -out client_cert.pem \ + -extfile ./openssl.cnf \ + -extensions test_client \ + -sha256 +openssl verify -verbose -CAfile client_ca_cert.pem client_cert.pem + +rm *_csr.pem diff --git a/examples/data/x509/openssl.cnf b/examples/data/x509/openssl.cnf new file mode 100644 index 000000000000..d1034214e1d3 --- /dev/null +++ b/examples/data/x509/openssl.cnf @@ -0,0 +1,28 @@ +[req] +distinguished_name = req_distinguished_name +attributes = req_attributes + +[req_distinguished_name] + +[req_attributes] + +[test_ca] +basicConstraints = critical,CA:TRUE +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid:always,issuer:always +keyUsage = critical,keyCertSign + +[test_server] +basicConstraints = critical,CA:FALSE +subjectKeyIdentifier = hash +keyUsage = critical,digitalSignature,keyEncipherment,keyAgreement +subjectAltName = @server_alt_names + +[server_alt_names] +DNS.1 = *.test.example.com + +[test_client] +basicConstraints = critical,CA:FALSE +subjectKeyIdentifier = hash +keyUsage = critical,nonRepudiation,digitalSignature,keyEncipherment +extendedKeyUsage = critical,clientAuth diff --git a/examples/data/x509/server_cert.pem b/examples/data/x509/server_cert.pem index 3e48a52fd108..f1a374008342 100644 --- a/examples/data/x509/server_cert.pem +++ b/examples/data/x509/server_cert.pem @@ -1,32 +1,32 @@ -----BEGIN CERTIFICATE----- -MIIFeDCCA2CgAwIBAgICA+gwDQYJKoZIhvcNAQEFBQAwUDELMAkGA1UEBhMCVVMx +MIIFeDCCA2CgAwIBAgICA+gwDQYJKoZIhvcNAQELBQAwUDELMAkGA1UEBhMCVVMx CzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTALBgNVBAoMBGdSUEMxFzAVBgNV -BAMMDnRlc3Qtc2VydmVyX2NhMB4XDTIwMDgwNDAxNTk1OFoXDTMwMDgwMjAxNTk1 +BAMMDnRlc3Qtc2VydmVyX2NhMB4XDTIyMDMxODIxNDQ1OFoXDTMyMDMxNTIxNDQ1 OFowTjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTAL BgNVBAoMBGdSUEMxFTATBgNVBAMMDHRlc3Qtc2VydmVyMTCCAiIwDQYJKoZIhvcN -AQEBBQADggIPADCCAgoCggIBAKonkszKvSg1IUvpfW3PAeDPLgLrXboOWJCXv3RD -5q6vf29+IBCaljSJmU6T7SplokUML5ZkY6adjX6awG+LH3tOMg9zvXpHuSPRpFUk -2oLFtaWuzJ+NC5HIM0wWDvdZ6KQsiPFbNxk2Rhkk+QKsiiptZy2yf/AbDY0sVieZ -BJZJ+os+BdFIk7+XUgDutPdSAutTANhrGycYa4iYAfDGQApz3sndSSsM2KVc0w5F -gW6w2UBC4ggc1ZaWdbVtkYo+0dCsrl1J7WUNsz8v8mjGsvm9eFuJjKFBiDhCF+xg -4Xzu1Wz7zV97994la/xMImQR4QDdky9IgKcJMVUGua6U0GE5lmt2wnd3aAI228Vm -6SnK7kKvnD8vRUyM9ByeRoMlrAuYb0AjnVBr/MTFbOaii6w2v3RjU0j6YFzp8+67 -ihOW9nkb1ayqSXD3T4QUD0p75Ne7/zz1r2amIh9pmSJlugLexVDpb86vXg9RnXjb -Zn2HTEkXsL5eHUIlQzuhK+gdmj+MLGf/Yzp3fdaJsA0cJfMjj5Ubb2gR4VwzrHy9 -AD2Kjjzs06pTtpULChwpr9IBTLEsZfw/4uW4II4pfe6Rwn4bGHFifjx0+3svlsSo -jdHcXEMHvdRPhWGUZ0rne+IK6Qxgb3OMZu7a04vV0RqvgovxM6hre3e0UzBJG45Y -qlQjAgMBAAGjXjBcMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFFL5HUzehgKNfgdz -4nuw5fru5OTPMA4GA1UdDwEB/wQEAwIDqDAdBgNVHREEFjAUghIqLnRlc3QuZXhh -bXBsZS5jb20wDQYJKoZIhvcNAQEFBQADggIBAHMPYTF4StfSx9869EoitlEi7Oz2 -YTOForDbsY9i0VnIamhIi9CpjekAGLo8SVojeAk7UV3ayiu0hEMAHJWbicgWTwWM -JvZWWfrIk/2WYyBWWTa711DuW26cvtbSebFzXsovNeTqMICiTeYbvOAK826UdH/o -OqNiHL+UO5xR1Xmqa2hKmLSl5J1n+zgm94l6SROzc9c5YDzn03U+8dlhoyXCwlTv -JRprOD+lupccxcKj5Tfh9/G6PjKsgxW+DZ+rvQV5f/l7c4m/bBrgS8tru4t2Xip0 -NhQW4qHnL0wXdTjaOG/1liLppjcp7SsP+vKF4shUvp+P8NQuAswBp/QtqUse5EYl -EUARWrjEpV4OHSKThkMackMg5E32keiOvQE6iICxtU+m2V+C3xXM3G2cGlDDx5Ob -tan0c9fZXoygrN2mc94GPogfwFGxwivajvvJIs/bsB3RkcIuLbi2UB76Wwoq+ZvH -15xxNZI1rpaDhjEuqwbSGPMPVpFtF5VERgYQ9LaDgj7yorwSQ1YLY8R1y0vSiAR2 -2YeOaBH1ZLPF9v9os1iK4TIC8XQfPv7ll2WdDwfbe2ux5GVbDBD4bPhP9s3F4a+f -oPhikWsUY4eN5CfS76x6xL0L60TL1AlWLlwuubTxpvNhv3GSyxjfunjcGiXDml20 -6S80qO4hepxzzjol +AQEBBQADggIPADCCAgoCggIBAL5GBWw+qfXyelelYL/RDA/Fk4GA8DlcBQgBOjBa +XCVDMAJj63sN+ubKBtphWe6Y9SWLJa2mt8a/ZTQZm2R5FPSp9rwdr04UQgmL11wh +DCmO+wkRUeTYwsqcidEHRwOxoctyO+lwgYw983T/fp83qtNS4bw+1kJwrLtFdgok +Kd9UGIugs8BTFqE/7CxFRXTYsNy/gj0pp411Dtgknl1UefPdjco2Qon8f3Dm5iDf +AyUM1oL8+fnRQj/r6P3XC4AOiBsF3duxiBzUp87YgmwDOaa8paKOx2UNLA/eP/aP +Uhd7HkygqOX+tc3H8dvYONo6lhwQD1JqyG6IOOWe2uf5YXKK2TphPPRnCW4QIED4 +PuXYHjIvGYA4Kf0Wmb2hPk6bxJidNoLp9lsJyqGfk3QnT5PRJVgO0mlzo/UsZo77 +5j+yq87yLe5OL2HrZd1KTfg7SKOtMJ9N6tm2Hw2jwypKz+x2jlEZOgXHmYb5aUaI ++4xG+9fqc8x3ScoHQGNujF3qHO5SxnXkufNUSVbWbv1Ble8peiKyG6AFQvtcs7KG +pEoFztGSlaABwSvxO8J3aJPAEok4OI5IAGJNy92XaBMLtyt270FC8JtUnL+JEubV +t8tY5cCcGK7EtRHb47mM0K8HEq+IU2nAq6/29Ka0IZlkb5fPoWzQAZEIVKgLNHt4 +96g9AgMBAAGjXjBcMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFNx36JXsCIzVWCOw +1ETtaxlN79XrMA4GA1UdDwEB/wQEAwIDqDAdBgNVHREEFjAUghIqLnRlc3QuZXhh +bXBsZS5jb20wDQYJKoZIhvcNAQELBQADggIBAAEEZln7lsS/HIysNPJktc0Gdu3n +X1BcA3wXh95YTugcxSSeLLx2SykXnwX+cJncc1OKbboO9DA5mZ+huCesGIOKeUkg +azQZL6FAdw9PQKdqKg3RgSQ4XhK990fPcmmBhSXY24jNNhRHxGw5lGBrD6X2SdW3 +m66yYzn9hMXL4yrweGO7OC4bdyISDrJiP+St/xeCoIcXP2s07dE6jl2VorJCWn4J +SxKfDhPPohZKl6dL9npkmPcpz2zRAYpo4tsVdAAQDBRui44Vvm1eBPUo7EH2UOEh +/3JtTeDUpldM8fDaKE0kTa1Ttxzs2e0Jm3M4/FMOxqSesyJldw54F4+4m24e/iQU +gceArYMFVFTipgrLfUuRvRxx/7D7V92pqTyuD3T78+KdTqrlxvCTOqSHhFE05jWD +RdynS6Ev/1QZLlnWgMwhQAnjhc1NKkso+namF1ZmHH9owiTRBlWDMNcHMDReaELd +QmFUvutHUpjidt1z+G6lzbP0XB5w+0vW4BsT0FqaYsFbK5ftryj1/K0VctrSd/ke +GI0vxrErAyLG2B8bdK88u2w7DCuXjAOp+CeA7HUmk93TsPEAhrxQ6lR51IC6LcK0 +gACSdnQDPGtkoRX00DTvdcOpzmkSgaGr/mXTqp2lR9IuZIhwKbhS3lDKsAZ/hinB +yaBwLiXfcvZrZOwy -----END CERTIFICATE----- diff --git a/examples/data/x509/server_key.pem b/examples/data/x509/server_key.pem index e71ad0ac9753..1c778db7c491 100644 --- a/examples/data/x509/server_key.pem +++ b/examples/data/x509/server_key.pem @@ -1,51 +1,51 @@ -----BEGIN RSA PRIVATE KEY----- -MIIJKQIBAAKCAgEAqieSzMq9KDUhS+l9bc8B4M8uAutdug5YkJe/dEPmrq9/b34g -EJqWNImZTpPtKmWiRQwvlmRjpp2NfprAb4sfe04yD3O9eke5I9GkVSTagsW1pa7M -n40LkcgzTBYO91nopCyI8Vs3GTZGGST5AqyKKm1nLbJ/8BsNjSxWJ5kElkn6iz4F -0UiTv5dSAO6091IC61MA2GsbJxhriJgB8MZACnPeyd1JKwzYpVzTDkWBbrDZQELi -CBzVlpZ1tW2Rij7R0KyuXUntZQ2zPy/yaMay+b14W4mMoUGIOEIX7GDhfO7VbPvN -X3v33iVr/EwiZBHhAN2TL0iApwkxVQa5rpTQYTmWa3bCd3doAjbbxWbpKcruQq+c -Py9FTIz0HJ5GgyWsC5hvQCOdUGv8xMVs5qKLrDa/dGNTSPpgXOnz7ruKE5b2eRvV -rKpJcPdPhBQPSnvk17v/PPWvZqYiH2mZImW6At7FUOlvzq9eD1GdeNtmfYdMSRew -vl4dQiVDO6Er6B2aP4wsZ/9jOnd91omwDRwl8yOPlRtvaBHhXDOsfL0APYqOPOzT -qlO2lQsKHCmv0gFMsSxl/D/i5bggjil97pHCfhsYcWJ+PHT7ey+WxKiN0dxcQwe9 -1E+FYZRnSud74grpDGBvc4xm7trTi9XRGq+Ci/EzqGt7d7RTMEkbjliqVCMCAwEA -AQKCAgEAjU6UEVMFSBDnd/2OVtUlQCeOlIoWql8jmeEL9Gg3eTbx5AugYWmf+D2V -fbZHrX/+BM2b74+rWkFZspyd14R4PpSv6jk6UASkcmS1zqfud8/tjIzgDli6FPVn -9HYVM8IM+9qoV5hi56M1D8iuq1PS4m081Kx6p1IwLN93JSdksdL6KQz3E9jsKp5m -UbPrwcDv/7JM723zfMJA+40Rf32EzalwicAl9YSTnrC57g428VAY+88Pm6EmmAqX -8nXt+hs1b9EYdQziA5wfEgiljfIFzHVXMN3IVlrv35iz+XBzkqddw0ZSRkvTiz8U -sNAhd22JqIhapVfWz+FIgM43Ag9ABUMNWoQlaT0+2KlhkL+cZ6J1nfpMTBEIatz0 -A/l4TGcvdDhREODrS5jrxwJNx/LMRENtFFnRzAPzX4RdkFvi8SOioAWRBvs1TZFo -ZLq2bzDOzDjs+EPQVx0SmjZEiBRhI6nC8Way00IdQi3T546r6qTKfPmXgjl5/fVO -J4adGVbEUnI/7+fqL2N82WVr+Le585EFP/6IL5FO++sAIGDqAOzEQhyRaLhmnz+D -GboeS/Tac9XdymFbrEvEMB4EFS3nsZHTeahfiqVd/SuXFDTHZ6kiqXweuhfsP1uW -7tGlnqtn+3zmLO6XRENPVvmjn7DhU255yjiKFdUqkajcoOYyWPECggEBANuYk+sr -UTScvJoh/VRHuqd9NkVVIoqfoTN61x6V1OuNNcmjMWsOIsH+n4SifLlUW6xCKaSK -8x8RJYfE9bnObv/NqM4DMhuaNd52bPKFi8IBbHSZpuRE/UEyJhMDpoto04H1GXx4 -1S49tndiNxQOv1/VojB4BH7kapY0yp30drK1CrocGN+YOUddxI9lOQpgt2AyoXVk -ehdyamK4uzQmkMyyGQljrV5EQbmyPCqZ1l/d0MJ9DixOBxnPDR9Ov9qrG4Dy6S/k -cH8PythqHTGTdlXgsBJaWEl2PyQupo3OhfiCV+79B9uxPfKvk5CIMVbnYxKgu+ly -RKSTSX+GHVgNwicCggEBAMZcwQIAA+I39sTRg/Vn/MxmUBAu3h2+oJcuZ3FQh4v5 -SL80BWEsooK9Oe4MzxyWkU+8FieFu5G6iXaSx8f3Wv6j90IzA3g6Xr9M5xBm5qUN -IqzF+hUZuKAEMY1NcPlFTa2NlrkT8JdfQvJ+D5QrcBIMFmg9cKG5x9yD7MfHTJkf -ztMDFOwP3n7ahKRBowfe7/unAEFf6hYFtYjV+bqMDmBFVmk2CIVtjFgO9BNBQ/LB -zGcnwo2VigWBIjRDF5BgV0v+2g0PZGaxJ362RigZjzJojx3gYj6kaZYX8yb6ttGo -RPGt1A9woz6m0G0fLLMlce1dpbBAna14UVY7AEVt56UCggEAVvii/Oz3CINbHyB/ -GLYf8t3gdK03NPfr/FuWf4KQBYqz1txPYjsDARo7S2ifRTdn51186LIvgApmdtNH -DwP3alClnpIdclktJKJ6m8LQi1HNBpEkTBwWwY9/DODRQT2PJ1VPdsDUja/baIT5 -k3QTz3zo85FVFnyYyky2QsDjkfup9/PQ1h2P8fftNW29naKYff0PfVMCF+80u0y2 -t/zeNHQE/nb/3unhrg4tTiIHiYhsedrVli6BGXOrms6xpYVHK1cJi/JJq8kxaWz9 -ivkAURrgISSu+sleUJI5XMiCvt3AveJxDk2wX0Gyi/eksuqJjoMiaV7cWOIMpfkT -/h/U2QKCAQAFirvduXBiVpvvXccpCRG4CDe+bADKpfPIpYRAVzaiQ4GzzdlEoMGd -k3nV28fBjbdbme6ohgT6ilKi3HD2dkO1j5Et6Uz0g/T3tUdTXvycqeRJHXLiOgi9 -d8CGqR456KTF74nBe/whzoiJS9pVkm0cI/hQSz8lVZJu58SqxDewo4HcxV5FRiA6 -PRKtoCPU6Xac+kp4iRx6JwiuXQQQIS+ZovZKFDdiuu/L2gcZrp4eXym9zA+UcxQb -GUOCYEl9QCPQPLuM19w/Pj3TPXZyUlx81Q0Cka1NALzuc5bYhPKsot3iPrAJCmWV -L4XtNozCKI6pSg+CABwnp4/mL9nPFsX9AoIBAQDHiDhG9jtBdgtAEog6oL2Z98qR -u5+nONtLQ61I5R22eZYOgWfxnz08fTtpaHaVWNLNzF0ApyxjxD+zkFHcMJDUuHkR -O0yxUbCaof7u8EFtq8P9ux4xjtCnZW+9da0Y07zBrcXTsHYnAOiqNbtvVYd6RPiW -AaE61hgvj1c9/BQh2lUcroQx+yJI8uAAQrfYtXzm90rb6qk6rWy4li2ybMjB+LmP -cIQIXIUzdwE5uhBnwIre74cIZRXFJBqFY01+mT8ShPUWJkpOe0Fojrkl633TUuNf -9thZ++Fjvs4s7alFH5Hc7Ulk4v/O1+owdjqERd8zlu7+568C9s50CGwFnH0d +MIIJKQIBAAKCAgEAvkYFbD6p9fJ6V6Vgv9EMD8WTgYDwOVwFCAE6MFpcJUMwAmPr +ew365soG2mFZ7pj1JYslraa3xr9lNBmbZHkU9Kn2vB2vThRCCYvXXCEMKY77CRFR +5NjCypyJ0QdHA7Ghy3I76XCBjD3zdP9+nzeq01LhvD7WQnCsu0V2CiQp31QYi6Cz +wFMWoT/sLEVFdNiw3L+CPSmnjXUO2CSeXVR5892NyjZCifx/cObmIN8DJQzWgvz5 ++dFCP+vo/dcLgA6IGwXd27GIHNSnztiCbAM5pryloo7HZQ0sD94/9o9SF3seTKCo +5f61zcfx29g42jqWHBAPUmrIbog45Z7a5/lhcorZOmE89GcJbhAgQPg+5dgeMi8Z +gDgp/RaZvaE+TpvEmJ02gun2WwnKoZ+TdCdPk9ElWA7SaXOj9SxmjvvmP7KrzvIt +7k4vYetl3UpN+DtIo60wn03q2bYfDaPDKkrP7HaOURk6BceZhvlpRoj7jEb71+pz +zHdJygdAY26MXeoc7lLGdeS581RJVtZu/UGV7yl6IrIboAVC+1yzsoakSgXO0ZKV +oAHBK/E7wndok8ASiTg4jkgAYk3L3ZdoEwu3K3bvQULwm1Scv4kS5tW3y1jlwJwY +rsS1EdvjuYzQrwcSr4hTacCrr/b0prQhmWRvl8+hbNABkQhUqAs0e3j3qD0CAwEA +AQKCAgBnR3CoGbd9hZl8u4qxc5IdeXwgflFmgRlGCAyCtHlxzG9hzMTD7Ymz/hMM +NG1xQltGfqn8AROd8MPJLOEY/1QtnZgM8fv24K4bqmlCW7nTUQXYHSubkUDiY2e3 +K0ETszaETMRSaLwY2IOujQQ4/ilePY3D9UOtmqVXnVN+G7USwP31xEvtZ+xPqHfU +a+FQlFIj8FuMQXDuKozdK7s+I51yjl7pVNx3M7QlH1/olcSKNta1EQXK4RgZxD6a +kkBuyPR93ohXOJ0OMSvI7eKVKIcBh0JM4z0+D5FMJ7IGbjL8Bdsjcs1a0g/y28Xf +NBVf9w8Fun3mmYmj3ZMsqDZgVg/bAfP2z7O9kMzbuqmjelOz8HXxTm/+GIHuseMx +b/nDZgB0ZN+FhATv/onshJcjr2L3SJYzEWqjYiqaCQo5qtib+/kxh6SHPhAY2o8l +zzMhKFsJMhmwW91FXqeDS9FTlcRXtYH1EJxNGa01GpyVa6plvvFTGBNkEUJnVuEp +ULohJw0NJQYQOz5omYaQVJ49lpzVhwLEolgSlIBiM3s9nSDvVBYu+bB1ovw5OTIJ +Wlc9cBrYmdxYdAj5n6JzIC1wixgxrFw1jBm8cL/2FQYtR7daZabTMyZj5vAUqjxr +OV+uvkSFcIyBs1ty9TnnKC3yd5Ma+5chR5u7JPc1lSSor6AwQQKCAQEA4d5XrCq5 +EikGII/unhkVZsh9xmILp/4PRKc+fV7TFEpGyn8HFCBToZk6nXv99roUBdeZFobw +gDuZqBa4ougm2zgBbhdQXGaW4yZdChJlSs9yY7OAVvnG9gjuHGmWsLhvmhaeXSr2 +auxVGRaltr3r8hP9eHhloDM6qdSSAQpsdeTBQD8Ep3//aL/BLqGcF0gLrZLPwo0+ +cku8jQoVXSSOW1+YSaXRGxueuIR8lldU4I3yp2DO++DGLsOZoGFT/+ZXc2B4nE1h +o1hCWt6RKw0q2rCkZ+i6SiPGsVgb9xn6W8wHFIPA/0sOwOdtbKqKd0xwn5DnX+vt +d8shlRRUDF7HDQKCAQEA16gR/2n59HZiQQhHU9BCvGFi4nxlsuij+nqDx9fUerDU +fK79NaOuraWNkCqz+2lqfu5o3e3XNFHlVsj98SyfmTdMZ8Fj19awqN20nCOmfRkk +/MDuEzRzvNlOYBa0PpMkKJn2sahEiXGNVI4g3cGip1c5wJ1HL3jF61io4F/auBLP +grLtw8CoTqc6VpJUvsWFjopTmNdAze8WMf3vK6AKu7PKkXH7mFQZusacpO/E61Ud +euiG9BYDIIkrnWIQdLpODgliLZzPNcJDTKTFJAfIzr3WQvUaFc1+tHyX3XhpicvP +J4zyNfHd2dZMK1csXQJvFSnPgXpy531Wca0riAYZ8QKCAQEAhaVEBxE4dLBlebrw +nAeHjEuxcELvVrWTXzH+XbxP9T+F56eGDriaA5JhBnIpcWXlFxfc82FgyN97KeRX +17y50Riwb+3HlQT23u0CPEVqPfvFWY0KsWwV99qM2a74hRR8pJYhmksjh1zTdYbb +AugZxiFh53iF2Wa2nWq0AX2jc5apalRfcqTgAaEEs4zYiUYN8uRdnmZovsRliqae +wYAx44sK1vkQY5PSNKff+C0wgbY8ECHOF2eGnIEMU8ODKnWm5RP+Ca4Xyckdahsr +lmeyJbhDb2BbaicFGEZkNa/fXZW50r+q4OQOlMHbE2NNjw1hzmi1HyLAXhOJiWZ/ +3NnvuQKCAQEAg04a/zeocBcwhcYjn717FLX6/kmdpkwNo3G7EQ+xmK5YAj6Nf35U +2fel9PR7N4WcyQIiKZYp5PpEOA4SyChSWHiZ9caDIyTd1UOAN11hfmOz6I0Tp+/U +1FQ/azQHtN3kMzBjSxJYAJN56NTM4BiJD3iFemiIsjfH0h7eXBcg1djmLf8B06FX +GOSrGZDpNmqPghVpBvNwyrJbAj9Jw3cjcdvrZ5lOBhaWv+kz8Rzn+h2N4Ir5uF46 +szGxs5bEzD2vTs6Zz4ndhC7uyRi9y81Nj8t4TLZtln7TOdNup/Mr1zGXxM4Fn6DP +YlYfdHgUU+Eqf2lApeZHVfkzi+1TRvPoEQKCAQAELU/d33TNwQ/Ylo2VhwAscY3s +hv31O4tpu5koHHjOo3RDPzjuEfwy006u8NVAoj97LrU2n+XTIlnXf14TKuKWQ+8q +ajIVNj+ZAbD3djCmYXbIEL+u6aL4K1ENdjo6DNTGgPMfISE79WrmGBIKtB//uMqy +fGTUSPeo+R5WmTGN29YxAnRE/jtwOgAcicACTc0e9nghHj3c2raI0IazY5XFP0/h +LszTNUQzWx6DjWsbB+Ymuhu4fHZTYftCrIMpjmjC9pkNggeJnkxylQz/pwO73uWg +ycDgJhRyaVhM8sJXiBk+OC/ySP2Lxo60aPa514LEYJKQxUCukCTXth/6p0Qo -----END RSA PRIVATE KEY----- diff --git a/examples/examples_test.sh b/examples/examples_test.sh index 85e2958bff8e..bead4d0dcbe1 100755 --- a/examples/examples_test.sh +++ b/examples/examples_test.sh @@ -20,6 +20,9 @@ set +e export TMPDIR=$(mktemp -d) trap "rm -rf ${TMPDIR}" EXIT +export SERVER_PORT=50051 +export UNIX_ADDR=abstract-unix-socket + clean () { for i in {1..10}; do jobs -p | xargs -n1 pkill -P @@ -49,45 +52,97 @@ EXAMPLES=( "helloworld" "route_guide" "features/authentication" + "features/authz" + "features/cancellation" "features/compression" "features/deadline" "features/encryption/TLS" - "features/errors" + "features/error_details" + "features/error_handling" "features/interceptor" "features/load_balancing" "features/metadata" + "features/metadata_interceptor" "features/multiplex" "features/name_resolving" + "features/orca" + "features/retry" + "features/unix_abstract" +) + +declare -A SERVER_ARGS=( + ["features/unix_abstract"]="-addr $UNIX_ADDR" + ["default"]="-port $SERVER_PORT" +) + +declare -A CLIENT_ARGS=( + ["features/unix_abstract"]="-addr $UNIX_ADDR" + ["features/orca"]="-test=true" + ["default"]="-addr localhost:$SERVER_PORT" ) +declare -A SERVER_WAIT_COMMAND=( + ["features/unix_abstract"]="lsof -U | grep $UNIX_ADDR" + ["default"]="lsof -i :$SERVER_PORT | grep $SERVER_PORT" +) + +wait_for_server () { + example=$1 + wait_command=${SERVER_WAIT_COMMAND[$example]:-${SERVER_WAIT_COMMAND["default"]}} + echo "$(tput setaf 4) waiting for server to start $(tput sgr 0)" + for i in {1..10}; do + eval "$wait_command" 2>&1 &>/dev/null + if [ $? -eq 0 ]; then + pass "server started" + return + fi + sleep 1 + done + fail "cannot determine if server started" +} + declare -A EXPECTED_SERVER_OUTPUT=( ["helloworld"]="Received: world" ["route_guide"]="" ["features/authentication"]="server starting on port 50051..." + ["features/authz"]="unary echoing message \"hello world\"" + ["features/cancellation"]="server: error receiving from stream: rpc error: code = Canceled desc = context canceled" ["features/compression"]="UnaryEcho called with message \"compress\"" ["features/deadline"]="" ["features/encryption/TLS"]="" - ["features/errors"]="" + ["features/error_details"]="" + ["features/error_handling"]="" ["features/interceptor"]="unary echoing message \"hello world\"" ["features/load_balancing"]="serving on :50051" - ["features/metadata"]="message:\"this is examples/metadata\" , sending echo" + ["features/metadata"]="message:\"this is examples/metadata\", sending echo" + ["features/metadata_interceptor"]="key1 from metadata: " ["features/multiplex"]=":50051" ["features/name_resolving"]="serving on localhost:50051" + ["features/orca"]="Server listening" + ["features/retry"]="request succeeded count: 4" + ["features/unix_abstract"]="serving on @abstract-unix-socket" ) declare -A EXPECTED_CLIENT_OUTPUT=( ["helloworld"]="Greeting: Hello world" - ["route_guide"]="location:" + ["route_guide"]="Feature: name: \"\", point:(416851321, -742674555)" ["features/authentication"]="UnaryEcho: hello world" + ["features/authz"]="UnaryEcho: hello world" + ["features/cancellation"]="cancelling context" ["features/compression"]="UnaryEcho call returned \"compress\", " ["features/deadline"]="wanted = DeadlineExceeded, got = DeadlineExceeded" ["features/encryption/TLS"]="UnaryEcho: hello world" - ["features/errors"]="Greeting: Hello world" + ["features/error_details"]="Greeting: Hello world" + ["features/error_handling"]="Received error" ["features/interceptor"]="UnaryEcho: hello world" ["features/load_balancing"]="calling helloworld.Greeter/SayHello with pick_first" ["features/metadata"]="this is examples/metadata" + ["features/metadata_interceptor"]="BidiStreaming Echo: hello world" ["features/multiplex"]="Greeting: Hello multiplex" ["features/name_resolving"]="calling helloworld.Greeter/SayHello to \"example:///resolver.example.grpc.io\"" + ["features/orca"]="Per-call load report received: map\[db_queries:10\]" + ["features/retry"]="UnaryEcho reply: message:\"Try and Success\"" + ["features/unix_abstract"]="calling echo.Echo/UnaryEcho to unix-abstract:abstract-unix-socket" ) cd ./examples @@ -102,6 +157,13 @@ for example in ${EXAMPLES[@]}; do pass "successfully built server" fi + # Start server + SERVER_LOG="$(mktemp)" + server_args=${SERVER_ARGS[$example]:-${SERVER_ARGS["default"]}} + go run ./$example/*server/*.go $server_args &> $SERVER_LOG & + + wait_for_server $example + # Build client if ! go build -o /dev/null ./${example}/*client/*.go; then fail "failed to build client" @@ -109,12 +171,10 @@ for example in ${EXAMPLES[@]}; do pass "successfully built client" fi - # Start server - SERVER_LOG="$(mktemp)" - go run ./$example/*server/*.go &> $SERVER_LOG & - + # Start client CLIENT_LOG="$(mktemp)" - if ! timeout 20 go run ${example}/*client/*.go &> $CLIENT_LOG; then + client_args=${CLIENT_ARGS[$example]:-${CLIENT_ARGS["default"]}} + if ! timeout 20 go run ${example}/*client/*.go $client_args &> $CLIENT_LOG; then fail "client failed to communicate with server got server log: $(cat $SERVER_LOG) diff --git a/examples/features/authentication/README.md b/examples/features/authentication/README.md index 0ba3f9469fc5..57028b8795d3 100644 --- a/examples/features/authentication/README.md +++ b/examples/features/authentication/README.md @@ -29,9 +29,9 @@ https://godoc.org/google.golang.org/grpc/credentials/oauth for details. #### Client -On client side, users should first get a valid oauth token, and then call -[`credentials.NewOauthAccess`](https://godoc.org/google.golang.org/grpc/credentials/oauth#NewOauthAccess) -to initialize a `credentials.PerRPCCredentials` with it. Next, if user wants to +On client side, users should first get a valid oauth token, and then initialize a +[`oauth.TokenSource`](https://godoc.org/google.golang.org/grpc/credentials/oauth#TokenSource) +which implements `credentials.PerRPCCredentials`. Next, if user wants to apply a single OAuth token for all RPC calls on the same connection, then configure grpc `Dial` with `DialOption` [`WithPerRPCCredentials`](https://godoc.org/google.golang.org/grpc#WithPerRPCCredentials). diff --git a/examples/features/authentication/client/main.go b/examples/features/authentication/client/main.go index 0c5c9d948e37..a189b4be8cfa 100644 --- a/examples/features/authentication/client/main.go +++ b/examples/features/authentication/client/main.go @@ -50,7 +50,7 @@ func main() { flag.Parse() // Set up the credentials for the connection. - perRPC := oauth.NewOauthAccess(fetchToken()) + perRPC := oauth.TokenSource{TokenSource: oauth2.StaticTokenSource(fetchToken())} creds, err := credentials.NewClientTLSFromFile(data.Path("x509/ca_cert.pem"), "x.test.example.com") if err != nil { log.Fatalf("failed to load credentials: %v", err) @@ -61,12 +61,11 @@ func main() { // itself. // See: https://godoc.org/google.golang.org/grpc#PerRPCCredentials grpc.WithPerRPCCredentials(perRPC), - // oauth.NewOauthAccess requires the configuration of transport + // oauth.TokenSource requires the configuration of transport // credentials. grpc.WithTransportCredentials(creds), } - opts = append(opts, grpc.WithBlock()) conn, err := grpc.Dial(*addr, opts...) if err != nil { log.Fatalf("did not connect: %v", err) diff --git a/examples/features/authentication/server/main.go b/examples/features/authentication/server/main.go index 505e8cb09847..1b3b5e8ad202 100644 --- a/examples/features/authentication/server/main.go +++ b/examples/features/authentication/server/main.go @@ -63,7 +63,7 @@ func main() { grpc.Creds(credentials.NewServerTLSFromCert(&cert)), } s := grpc.NewServer(opts...) - pb.RegisterEchoService(s, &pb.EchoService{UnaryEcho: unaryEcho}) + pb.RegisterEchoServer(s, &ecServer{}) lis, err := net.Listen("tcp", fmt.Sprintf(":%d", *port)) if err != nil { log.Fatalf("failed to listen: %v", err) @@ -73,7 +73,11 @@ func main() { } } -func unaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) { +type ecServer struct { + pb.UnimplementedEchoServer +} + +func (s *ecServer) UnaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) { return &pb.EchoResponse{Message: req.Message}, nil } @@ -93,7 +97,7 @@ func valid(authorization []string) bool { // the token is missing or invalid, the interceptor blocks execution of the // handler and returns an error. Otherwise, the interceptor invokes the unary // handler. -func ensureValidToken(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { +func ensureValidToken(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { md, ok := metadata.FromIncomingContext(ctx) if !ok { return nil, errMissingMetadata diff --git a/examples/features/authz/README.md b/examples/features/authz/README.md new file mode 100644 index 000000000000..498beb367f1e --- /dev/null +++ b/examples/features/authz/README.md @@ -0,0 +1,40 @@ +# RBAC authorization + +This example uses the `StaticInterceptor` from the `google.golang.org/grpc/authz` +package. It uses a header based RBAC policy to match each gRPC method to a +required role. For simplicity, the context is injected with mock metadata which +includes the required roles, but this should be fetched from an appropriate +service based on the authenticated context. + +## Try it + +Server requires the following roles on an authenticated user to authorize usage +of these methods: + +- `UnaryEcho` requires the role `UNARY_ECHO:W` +- `BidirectionalStreamingEcho` requires the role `STREAM_ECHO:RW` + +Upon receiving a request, the server first checks that a token was supplied, +decodes it and checks that a secret is correctly set (hardcoded to `super-secret` +for simplicity, this should use a proper ID provider in production). + +If the above is successful, it uses the username in the token to set appropriate +roles (hardcoded to the 2 required roles above if the username matches `super-user` +for simplicity, these roles should be supplied externally as well). + +Start the server with: + +``` +go run server/main.go +``` + +The client implementation shows how using a valid token (setting username and +secret) with each of the endpoints will return successfully. It also exemplifies +how using a bad token will result in `codes.PermissionDenied` being returned +from the service. + +Start the client with: + +``` +go run client/main.go +``` diff --git a/examples/features/authz/client/main.go b/examples/features/authz/client/main.go new file mode 100644 index 000000000000..2654314e5e11 --- /dev/null +++ b/examples/features/authz/client/main.go @@ -0,0 +1,140 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary client is an example client. +package main + +import ( + "context" + "flag" + "fmt" + "io" + "log" + "time" + + "golang.org/x/oauth2" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/oauth" + "google.golang.org/grpc/examples/data" + "google.golang.org/grpc/examples/features/authz/token" + ecpb "google.golang.org/grpc/examples/features/proto/echo" + "google.golang.org/grpc/status" +) + +var addr = flag.String("addr", "localhost:50051", "the address to connect to") + +func callUnaryEcho(ctx context.Context, client ecpb.EchoClient, message string, opts ...grpc.CallOption) error { + resp, err := client.UnaryEcho(ctx, &ecpb.EchoRequest{Message: message}, opts...) + if err != nil { + return status.Errorf(status.Code(err), "UnaryEcho RPC failed: %v", err) + } + fmt.Println("UnaryEcho: ", resp.Message) + return nil +} + +func callBidiStreamingEcho(ctx context.Context, client ecpb.EchoClient, opts ...grpc.CallOption) error { + c, err := client.BidirectionalStreamingEcho(ctx, opts...) + if err != nil { + return status.Errorf(status.Code(err), "BidirectionalStreamingEcho RPC failed: %v", err) + } + for i := 0; i < 5; i++ { + err := c.Send(&ecpb.EchoRequest{Message: fmt.Sprintf("Request %d", i+1)}) + if err == io.EOF { + // Bidi streaming RPC errors happen and make Send return io.EOF, + // not the RPC error itself. Call Recv to determine the error. + break + } + if err != nil { + // Some local errors are reported this way, e.g. errors serializing + // the request message. + return status.Errorf(status.Code(err), "sending StreamingEcho message: %v", err) + } + } + c.CloseSend() + for { + resp, err := c.Recv() + if err == io.EOF { + break + } + if err != nil { + return status.Errorf(status.Code(err), "receiving StreamingEcho message: %v", err) + } + fmt.Println("BidiStreaming Echo: ", resp.Message) + } + return nil +} + +func newCredentialsCallOption(t token.Token) grpc.CallOption { + tokenBase64, err := t.Encode() + if err != nil { + log.Fatalf("encoding token: %v", err) + } + oath2Token := oauth2.Token{AccessToken: tokenBase64} + return grpc.PerRPCCredentials(oauth.TokenSource{TokenSource: oauth2.StaticTokenSource(&oath2Token)}) +} + +func main() { + flag.Parse() + + // Create tls based credential. + creds, err := credentials.NewClientTLSFromFile(data.Path("x509/ca_cert.pem"), "x.test.example.com") + if err != nil { + log.Fatalf("failed to load credentials: %v", err) + } + // Set up a connection to the server. + conn, err := grpc.Dial(*addr, grpc.WithTransportCredentials(creds)) + if err != nil { + log.Fatalf("grpc.Dial(%q): %v", *addr, err) + } + defer conn.Close() + + // Make an echo client and send RPCs. + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + client := ecpb.NewEchoClient(conn) + + // Make RPCs as an authorized user and expect them to succeed. + authorisedUserTokenCallOption := newCredentialsCallOption(token.Token{Username: "super-user", Secret: "super-secret"}) + if err := callUnaryEcho(ctx, client, "hello world", authorisedUserTokenCallOption); err != nil { + log.Fatalf("Unary RPC by authorized user failed: %v", err) + } + if err := callBidiStreamingEcho(ctx, client, authorisedUserTokenCallOption); err != nil { + log.Fatalf("Bidirectional RPC by authorized user failed: %v", err) + } + + // Make RPCs as an unauthorized user and expect them to fail with status code PermissionDenied. + unauthorisedUserTokenCallOption := newCredentialsCallOption(token.Token{Username: "bad-actor", Secret: "super-secret"}) + if err := callUnaryEcho(ctx, client, "hello world", unauthorisedUserTokenCallOption); err != nil { + switch c := status.Code(err); c { + case codes.PermissionDenied: + log.Printf("Unary RPC by unauthorized user failed as expected: %v", err) + default: + log.Fatalf("Unary RPC by unauthorized user failed unexpectedly: %v, %v", c, err) + } + } + if err := callBidiStreamingEcho(ctx, client, unauthorisedUserTokenCallOption); err != nil { + switch c := status.Code(err); c { + case codes.PermissionDenied: + log.Printf("Bidirectional RPC by unauthorized user failed as expected: %v", err) + default: + log.Fatalf("Bidirectional RPC by unauthorized user failed unexpectedly: %v", err) + } + } +} diff --git a/examples/features/authz/server/main.go b/examples/features/authz/server/main.go new file mode 100644 index 000000000000..e66a5406de7b --- /dev/null +++ b/examples/features/authz/server/main.go @@ -0,0 +1,215 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary server is an example server. +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "io" + "log" + "net" + "strings" + + "google.golang.org/grpc" + "google.golang.org/grpc/authz" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/examples/data" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + + "google.golang.org/grpc/examples/features/authz/token" + pb "google.golang.org/grpc/examples/features/proto/echo" +) + +const ( + unaryEchoWriterRole = "UNARY_ECHO:W" + streamEchoReadWriterRole = "STREAM_ECHO:RW" + authzPolicy = ` + { + "name": "authz", + "allow_rules": [ + { + "name": "allow_UnaryEcho", + "request": { + "paths": ["/grpc.examples.echo.Echo/UnaryEcho"], + "headers": [ + { + "key": "UNARY_ECHO:W", + "values": ["true"] + } + ] + } + }, + { + "name": "allow_BidirectionalStreamingEcho", + "request": { + "paths": ["/grpc.examples.echo.Echo/BidirectionalStreamingEcho"], + "headers": [ + { + "key": "STREAM_ECHO:RW", + "values": ["true"] + } + ] + } + } + ], + "deny_rules": [] + } + ` +) + +var ( + port = flag.Int("port", 50051, "the port to serve on") + + errMissingMetadata = status.Errorf(codes.InvalidArgument, "missing metadata") +) + +func newContextWithRoles(ctx context.Context, username string) context.Context { + md := metadata.MD{} + if username == "super-user" { + md.Set(unaryEchoWriterRole, "true") + md.Set(streamEchoReadWriterRole, "true") + } + return metadata.NewIncomingContext(ctx, md) +} + +type server struct { + pb.UnimplementedEchoServer +} + +func (s *server) UnaryEcho(ctx context.Context, in *pb.EchoRequest) (*pb.EchoResponse, error) { + fmt.Printf("unary echoing message %q\n", in.Message) + return &pb.EchoResponse{Message: in.Message}, nil +} + +func (s *server) BidirectionalStreamingEcho(stream pb.Echo_BidirectionalStreamingEchoServer) error { + for { + in, err := stream.Recv() + if err != nil { + if err == io.EOF { + return nil + } + fmt.Printf("Receiving message from stream: %v\n", err) + return err + } + fmt.Printf("bidi echoing message %q\n", in.Message) + stream.Send(&pb.EchoResponse{Message: in.Message}) + } +} + +// isAuthenticated validates the authorization. +func isAuthenticated(authorization []string) (username string, err error) { + if len(authorization) < 1 { + return "", errors.New("received empty authorization token from client") + } + tokenBase64 := strings.TrimPrefix(authorization[0], "Bearer ") + // Perform the token validation here. For the sake of this example, the code + // here forgoes any of the usual OAuth2 token validation and instead checks + // for a token matching an arbitrary string. + var token token.Token + err = token.Decode(tokenBase64) + if err != nil { + return "", fmt.Errorf("base64 decoding of received token %q: %v", tokenBase64, err) + } + if token.Secret != "super-secret" { + return "", fmt.Errorf("received token %q does not match expected %q", token.Secret, "super-secret") + } + return token.Username, nil +} + +// authUnaryInterceptor looks up the authorization header from the incoming RPC context, +// retrieves the username from it and creates a new context with the username before invoking +// the provided handler. +func authUnaryInterceptor(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return nil, errMissingMetadata + } + username, err := isAuthenticated(md["authorization"]) + if err != nil { + return nil, status.Error(codes.Unauthenticated, err.Error()) + } + return handler(newContextWithRoles(ctx, username), req) +} + +// wrappedStream wraps a grpc.ServerStream associated with an incoming RPC, and +// a custom context containing the username derived from the authorization header +// specified in the incoming RPC metadata +type wrappedStream struct { + grpc.ServerStream + ctx context.Context +} + +func (w *wrappedStream) Context() context.Context { + return w.ctx +} + +func newWrappedStream(ctx context.Context, s grpc.ServerStream) grpc.ServerStream { + return &wrappedStream{s, ctx} +} + +// authStreamInterceptor looks up the authorization header from the incoming RPC context, +// retrieves the username from it and creates a new context with the username before invoking +// the provided handler. +func authStreamInterceptor(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + md, ok := metadata.FromIncomingContext(ss.Context()) + if !ok { + return errMissingMetadata + } + username, err := isAuthenticated(md["authorization"]) + if err != nil { + return status.Error(codes.Unauthenticated, err.Error()) + } + return handler(srv, newWrappedStream(newContextWithRoles(ss.Context(), username), ss)) +} + +func main() { + flag.Parse() + + lis, err := net.Listen("tcp", fmt.Sprintf(":%d", *port)) + if err != nil { + log.Fatalf("Listening on local port %q: %v", *port, err) + } + + // Create tls based credential. + creds, err := credentials.NewServerTLSFromFile(data.Path("x509/server_cert.pem"), data.Path("x509/server_key.pem")) + if err != nil { + log.Fatalf("Loading credentials: %v", err) + } + + // Create an authorization interceptor using a static policy. + staticInteceptor, err := authz.NewStatic(authzPolicy) + if err != nil { + log.Fatalf("Creating a static authz interceptor: %v", err) + } + unaryInts := grpc.ChainUnaryInterceptor(authUnaryInterceptor, staticInteceptor.UnaryInterceptor) + streamInts := grpc.ChainStreamInterceptor(authStreamInterceptor, staticInteceptor.StreamInterceptor) + s := grpc.NewServer(grpc.Creds(creds), unaryInts, streamInts) + + // Register EchoServer on the server. + pb.RegisterEchoServer(s, &server{}) + + if err := s.Serve(lis); err != nil { + log.Fatalf("Serving Echo service on local port: %v", err) + } +} diff --git a/examples/features/authz/token/token.go b/examples/features/authz/token/token.go new file mode 100644 index 000000000000..4899f3b08658 --- /dev/null +++ b/examples/features/authz/token/token.go @@ -0,0 +1,55 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package token implements an example of authorization token encoding/decoding +// that can be used in RPC headers. +package token + +import ( + "encoding/base64" + "encoding/json" +) + +// Token is a mock authorization token sent by the client as part of the RPC headers, +// and used by the server for authorization against a predefined policy. +type Token struct { + // Secret is used by the server to authenticate the user + Secret string `json:"secret"` + // Username is used by the server to assign roles in the metadata for authorization + Username string `json:"username"` +} + +// Encode returns a base64 encoded version of the JSON representation of token. +func (t *Token) Encode() (string, error) { + barr, err := json.Marshal(t) + if err != nil { + return "", err + } + s := base64.StdEncoding.EncodeToString(barr) + return s, nil +} + +// Decode updates the internals of Token using the passed in base64 +// encoded version of the JSON representation of token. +func (t *Token) Decode(s string) error { + barr, err := base64.StdEncoding.DecodeString(s) + if err != nil { + return err + } + return json.Unmarshal(barr, t) +} diff --git a/examples/features/cancellation/client/main.go b/examples/features/cancellation/client/main.go index 58bd4b6f0180..248619f7a617 100644 --- a/examples/features/cancellation/client/main.go +++ b/examples/features/cancellation/client/main.go @@ -28,6 +28,7 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" pb "google.golang.org/grpc/examples/features/proto/echo" "google.golang.org/grpc/status" ) @@ -55,7 +56,7 @@ func main() { flag.Parse() // Set up a connection to the server. - conn, err := grpc.Dial(*addr, grpc.WithInsecure()) + conn, err := grpc.Dial(*addr, grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { log.Fatalf("did not connect: %v", err) } diff --git a/examples/features/cancellation/server/main.go b/examples/features/cancellation/server/main.go index 1896828f2c99..520286bf193e 100644 --- a/examples/features/cancellation/server/main.go +++ b/examples/features/cancellation/server/main.go @@ -33,7 +33,11 @@ import ( var port = flag.Int("port", 50051, "the port to serve on") -func bidirectionalStreamingEcho(stream pb.Echo_BidirectionalStreamingEchoServer) error { +type server struct { + pb.UnimplementedEchoServer +} + +func (s *server) BidirectionalStreamingEcho(stream pb.Echo_BidirectionalStreamingEchoServer) error { for { in, err := stream.Recv() if err != nil { @@ -57,6 +61,6 @@ func main() { } fmt.Printf("server listening at port %v\n", lis.Addr()) s := grpc.NewServer() - pb.RegisterEchoService(s, &pb.EchoService{BidirectionalStreamingEcho: bidirectionalStreamingEcho}) + pb.RegisterEchoServer(s, &server{}) s.Serve(lis) } diff --git a/examples/features/compression/client/main.go b/examples/features/compression/client/main.go index df6d825a3ee5..24c3bbac1089 100644 --- a/examples/features/compression/client/main.go +++ b/examples/features/compression/client/main.go @@ -27,6 +27,7 @@ import ( "time" "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/encoding/gzip" // Install the gzip compressor pb "google.golang.org/grpc/examples/features/proto/echo" ) @@ -37,7 +38,7 @@ func main() { flag.Parse() // Set up a connection to the server. - conn, err := grpc.Dial(*addr, grpc.WithInsecure(), grpc.WithBlock()) + conn, err := grpc.Dial(*addr, grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { log.Fatalf("did not connect: %v", err) } diff --git a/examples/features/compression/server/main.go b/examples/features/compression/server/main.go index 442d196085c9..e495cc89106b 100644 --- a/examples/features/compression/server/main.go +++ b/examples/features/compression/server/main.go @@ -34,7 +34,11 @@ import ( var port = flag.Int("port", 50051, "the port to serve on") -func unaryEcho(ctx context.Context, in *pb.EchoRequest) (*pb.EchoResponse, error) { +type server struct { + pb.UnimplementedEchoServer +} + +func (s *server) UnaryEcho(ctx context.Context, in *pb.EchoRequest) (*pb.EchoResponse, error) { fmt.Printf("UnaryEcho called with message %q\n", in.GetMessage()) return &pb.EchoResponse{Message: in.Message}, nil } @@ -49,6 +53,6 @@ func main() { fmt.Printf("server listening at %v\n", lis.Addr()) s := grpc.NewServer() - pb.RegisterEchoService(s, &pb.EchoService{UnaryEcho: unaryEcho}) + pb.RegisterEchoServer(s, &server{}) s.Serve(lis) } diff --git a/examples/features/deadline/client/main.go b/examples/features/deadline/client/main.go index 026ce96f429a..8a4e3a2d26cc 100644 --- a/examples/features/deadline/client/main.go +++ b/examples/features/deadline/client/main.go @@ -28,6 +28,7 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" pb "google.golang.org/grpc/examples/features/proto/echo" "google.golang.org/grpc/status" ) @@ -72,7 +73,7 @@ func streamingCall(c pb.EchoClient, requestID int, message string, want codes.Co func main() { flag.Parse() - conn, err := grpc.Dial(*addr, grpc.WithInsecure(), grpc.WithBlock()) + conn, err := grpc.Dial(*addr, grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { log.Fatalf("did not connect: %v", err) } diff --git a/examples/features/deadline/server/main.go b/examples/features/deadline/server/main.go index f5a033db3d9c..ce3fc61679fc 100644 --- a/examples/features/deadline/server/main.go +++ b/examples/features/deadline/server/main.go @@ -31,6 +31,7 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/status" pb "google.golang.org/grpc/examples/features/proto/echo" @@ -40,6 +41,7 @@ var port = flag.Int("port", 50052, "port number") // server is used to implement EchoServer. type server struct { + pb.UnimplementedEchoServer client pb.EchoClient cc *grpc.ClientConn } @@ -93,7 +95,7 @@ func (s *server) Close() { func newEchoServer() *server { target := fmt.Sprintf("localhost:%v", *port) - cc, err := grpc.Dial(target, grpc.WithInsecure()) + cc, err := grpc.Dial(target, grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { log.Fatalf("did not connect: %v", err) } @@ -111,12 +113,9 @@ func main() { echoServer := newEchoServer() defer echoServer.Close() - grpcServer := grpc.NewServer() - pb.RegisterEchoService(grpcServer, &pb.EchoService{ - UnaryEcho: echoServer.UnaryEcho, - BidirectionalStreamingEcho: echoServer.BidirectionalStreamingEcho, - }) + grpcServer := grpc.NewServer() + pb.RegisterEchoServer(grpcServer, echoServer) if err := grpcServer.Serve(lis); err != nil { log.Fatalf("failed to serve: %v", err) diff --git a/examples/features/debugging/client/main.go b/examples/features/debugging/client/main.go index faf6b5d5fa2f..09acfa8112e5 100644 --- a/examples/features/debugging/client/main.go +++ b/examples/features/debugging/client/main.go @@ -21,13 +21,14 @@ package main import ( "context" + "flag" "log" "net" - "os" "time" "google.golang.org/grpc" "google.golang.org/grpc/channelz/service" + "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/resolver" "google.golang.org/grpc/resolver/manual" @@ -38,9 +39,15 @@ const ( defaultName = "world" ) +var ( + addr = flag.String("addr", "localhost:50051", "the address to connect to") + name = flag.String("name", defaultName, "Name to greet") +) + func main() { + flag.Parse() /***** Set up the server serving channelz service. *****/ - lis, err := net.Listen("tcp", ":50052") + lis, err := net.Listen("tcp", *addr) if err != nil { log.Fatalf("failed to listen: %v", err) } @@ -53,7 +60,7 @@ func main() { /***** Initialize manual resolver and Dial *****/ r := manual.NewBuilderWithScheme("whatever") // Set up a connection to the server. - conn, err := grpc.Dial(r.Scheme()+":///test.server", grpc.WithInsecure(), grpc.WithResolvers(r), grpc.WithDefaultServiceConfig(`{"loadBalancingPolicy":"round_robin"}`)) + conn, err := grpc.Dial(r.Scheme()+":///test.server", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r), grpc.WithDefaultServiceConfig(`{"loadBalancingPolicy":"round_robin"}`)) if err != nil { log.Fatalf("did not connect: %v", err) } @@ -64,17 +71,13 @@ func main() { c := pb.NewGreeterClient(conn) // Contact the server and print out its response. - name := defaultName - if len(os.Args) > 1 { - name = os.Args[1] - } /***** Make 100 SayHello RPCs *****/ for i := 0; i < 100; i++ { // Setting a 150ms timeout on the RPC. ctx, cancel := context.WithTimeout(context.Background(), 150*time.Millisecond) defer cancel() - r, err := c.SayHello(ctx, &pb.HelloRequest{Name: name}) + r, err := c.SayHello(ctx, &pb.HelloRequest{Name: *name}) if err != nil { log.Printf("could not greet: %v", err) } else { diff --git a/examples/features/debugging/server/main.go b/examples/features/debugging/server/main.go index 839212c99fbc..397cb0c781e6 100644 --- a/examples/features/debugging/server/main.go +++ b/examples/features/debugging/server/main.go @@ -36,13 +36,23 @@ var ( ports = []string{":10001", ":10002", ":10003"} ) -// sayHello implements helloworld.GreeterServer.SayHello -func sayHello(ctx context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) { +// server is used to implement helloworld.GreeterServer. +type server struct { + pb.UnimplementedGreeterServer +} + +// SayHello implements helloworld.GreeterServer +func (s *server) SayHello(ctx context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) { return &pb.HelloReply{Message: "Hello " + in.Name}, nil } -// sayHelloSlow implements helloworld.GreeterServer.SayHello -func sayHelloSlow(ctx context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) { +// slow server is used to simulate a server that has a variable delay in its response. +type slowServer struct { + pb.UnimplementedGreeterServer +} + +// SayHello implements helloworld.GreeterServer +func (s *slowServer) SayHello(ctx context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) { // Delay 100ms ~ 200ms before replying time.Sleep(time.Duration(100+grpcrand.Intn(100)) * time.Millisecond) return &pb.HelloReply{Message: "Hello " + in.Name}, nil @@ -60,7 +70,7 @@ func main() { go s.Serve(lis) defer s.Stop() - /***** Start three GreeterServers(with one of them to be the slow server). *****/ + /***** Start three GreeterServers(with one of them to be the slowServer). *****/ for i := 0; i < 3; i++ { lis, err := net.Listen("tcp", ports[i]) if err != nil { @@ -69,9 +79,9 @@ func main() { defer lis.Close() s := grpc.NewServer() if i == 2 { - pb.RegisterGreeterService(s, &pb.GreeterService{SayHello: sayHelloSlow}) + pb.RegisterGreeterServer(s, &slowServer{}) } else { - pb.RegisterGreeterService(s, &pb.GreeterService{SayHello: sayHello}) + pb.RegisterGreeterServer(s, &server{}) } go s.Serve(lis) } diff --git a/examples/features/encryption/ALTS/client/main.go b/examples/features/encryption/ALTS/client/main.go index e2654f5865ff..aa090807ba34 100644 --- a/examples/features/encryption/ALTS/client/main.go +++ b/examples/features/encryption/ALTS/client/main.go @@ -50,7 +50,7 @@ func main() { altsTC := alts.NewClientCreds(alts.DefaultClientOptions()) // Set up a connection to the server. - conn, err := grpc.Dial(*addr, grpc.WithTransportCredentials(altsTC), grpc.WithBlock()) + conn, err := grpc.Dial(*addr, grpc.WithTransportCredentials(altsTC)) if err != nil { log.Fatalf("did not connect: %v", err) } diff --git a/examples/features/encryption/ALTS/server/main.go b/examples/features/encryption/ALTS/server/main.go index a72ccd11e7fe..87bedd810f47 100644 --- a/examples/features/encryption/ALTS/server/main.go +++ b/examples/features/encryption/ALTS/server/main.go @@ -34,7 +34,11 @@ import ( var port = flag.Int("port", 50051, "the port to serve on") -func unaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) { +type ecServer struct { + pb.UnimplementedEchoServer +} + +func (s *ecServer) UnaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) { return &pb.EchoResponse{Message: req.Message}, nil } @@ -50,8 +54,8 @@ func main() { s := grpc.NewServer(grpc.Creds(altsTC)) - // Register EchoService on the server. - pb.RegisterEchoService(s, &pb.EchoService{UnaryEcho: unaryEcho}) + // Register EchoServer on the server. + pb.RegisterEchoServer(s, &ecServer{}) if err := s.Serve(lis); err != nil { log.Fatalf("failed to serve: %v", err) diff --git a/examples/features/encryption/README.md b/examples/features/encryption/README.md index a00188d66a2d..e4ce22933230 100644 --- a/examples/features/encryption/README.md +++ b/examples/features/encryption/README.md @@ -42,8 +42,8 @@ configure TLS and create the server credential using On client side, we provide the path to the "ca_cert.pem" to configure TLS and create the client credential using [`credentials.NewClientTLSFromFile`](https://godoc.org/google.golang.org/grpc/credentials#NewClientTLSFromFile). -Note that we override the server name with "x.test.youtube.com", as the server -certificate is valid for *.test.youtube.com but not localhost. It is solely for +Note that we override the server name with "x.test.example.com", as the server +certificate is valid for *.test.example.com but not localhost. It is solely for the convenience of making an example. Once the credentials have been created at both sides, we can start the server @@ -84,4 +84,23 @@ Next, same as TLS, start the server with the server credential and let client dial to server with the client credential. Finally, make an RPC to test the secure connection based upon ALTS is -successfully up. \ No newline at end of file +successfully up. + +### mTLS + +In mutual TLS (mTLS), the client and the server authenticate each other. gRPC +allows users to configure mutual TLS at the connection level. + +In normal TLS, the server is only concerned with presenting the server +certificate for clients to verify. In mutual TLS, the server also loads in a +list of trusted CA files for verifying client presented certificates with. +This is done via setting +[`tls.Config.ClientCAs`](https://pkg.go.dev/crypto/tls#Config.ClientCAs) +to the list of trusted CA files, +and setting [`tls.config.ClientAuth`](https://pkg.go.dev/crypto/tls#Config.ClientAuth) +to [`tls.RequireAndVerifyClientCert`](https://pkg.go.dev/crypto/tls#RequireAndVerifyClientCert). + +In normal TLS, the client is only concerned with authenticating the server by +using one or more trusted CA file. In mutual TLS, the client also presents its +client certificate to the server for authentication. This is done via setting +[`tls.Config.Certificates`](https://pkg.go.dev/crypto/tls#Config.Certificates). diff --git a/examples/features/encryption/TLS/client/main.go b/examples/features/encryption/TLS/client/main.go index 718196b1bb41..4f78ccca0366 100644 --- a/examples/features/encryption/TLS/client/main.go +++ b/examples/features/encryption/TLS/client/main.go @@ -54,7 +54,7 @@ func main() { } // Set up a connection to the server. - conn, err := grpc.Dial(*addr, grpc.WithTransportCredentials(creds), grpc.WithBlock()) + conn, err := grpc.Dial(*addr, grpc.WithTransportCredentials(creds)) if err != nil { log.Fatalf("did not connect: %v", err) } diff --git a/examples/features/encryption/TLS/server/main.go b/examples/features/encryption/TLS/server/main.go index d5ea6ccfcaf3..81bf1f3acc3e 100644 --- a/examples/features/encryption/TLS/server/main.go +++ b/examples/features/encryption/TLS/server/main.go @@ -35,7 +35,11 @@ import ( var port = flag.Int("port", 50051, "the port to serve on") -func unaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) { +type ecServer struct { + pb.UnimplementedEchoServer +} + +func (s *ecServer) UnaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) { return &pb.EchoResponse{Message: req.Message}, nil } @@ -56,7 +60,7 @@ func main() { s := grpc.NewServer(grpc.Creds(creds)) // Register EchoServer on the server. - pb.RegisterEchoService(s, &pb.EchoService{UnaryEcho: unaryEcho}) + pb.RegisterEchoServer(s, &ecServer{}) if err := s.Serve(lis); err != nil { log.Fatalf("failed to serve: %v", err) diff --git a/examples/features/encryption/mTLS/client/main.go b/examples/features/encryption/mTLS/client/main.go new file mode 100644 index 000000000000..4bc1ef06defa --- /dev/null +++ b/examples/features/encryption/mTLS/client/main.go @@ -0,0 +1,81 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary client is an example client which connects to the server using mTLS. +package main + +import ( + "context" + "crypto/tls" + "crypto/x509" + "flag" + "fmt" + "log" + "os" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/examples/data" + ecpb "google.golang.org/grpc/examples/features/proto/echo" +) + +var addr = flag.String("addr", "localhost:50051", "the address to connect to") + +func callUnaryEcho(client ecpb.EchoClient, message string) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + resp, err := client.UnaryEcho(ctx, &ecpb.EchoRequest{Message: message}) + if err != nil { + log.Fatalf("client.UnaryEcho(_) = _, %v: ", err) + } + fmt.Println("UnaryEcho: ", resp.Message) +} + +func main() { + flag.Parse() + + cert, err := tls.LoadX509KeyPair(data.Path("x509/client_cert.pem"), data.Path("x509/client_key.pem")) + if err != nil { + log.Fatalf("failed to load client cert: %v", err) + } + + ca := x509.NewCertPool() + caFilePath := data.Path("x509/ca_cert.pem") + caBytes, err := os.ReadFile(caFilePath) + if err != nil { + log.Fatalf("failed to read ca cert %q: %v", caFilePath, err) + } + if ok := ca.AppendCertsFromPEM(caBytes); !ok { + log.Fatalf("failed to parse %q", caFilePath) + } + + tlsConfig := &tls.Config{ + ServerName: "x.test.example.com", + Certificates: []tls.Certificate{cert}, + RootCAs: ca, + } + + conn, err := grpc.Dial(*addr, grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig))) + if err != nil { + log.Fatalf("did not connect: %v", err) + } + defer conn.Close() + + callUnaryEcho(ecpb.NewEchoClient(conn), "hello world") +} diff --git a/examples/features/encryption/mTLS/server/main.go b/examples/features/encryption/mTLS/server/main.go new file mode 100644 index 000000000000..edd6829dcb91 --- /dev/null +++ b/examples/features/encryption/mTLS/server/main.go @@ -0,0 +1,82 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary server is an example server which authenticates clients using mTLS. +package main + +import ( + "context" + "crypto/tls" + "crypto/x509" + "flag" + "fmt" + "log" + "net" + "os" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/examples/data" + pb "google.golang.org/grpc/examples/features/proto/echo" +) + +var port = flag.Int("port", 50051, "the port to serve on") + +type ecServer struct { + pb.UnimplementedEchoServer +} + +func (s *ecServer) UnaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) { + return &pb.EchoResponse{Message: req.Message}, nil +} + +func main() { + flag.Parse() + log.Printf("server starting on port %d...\n", *port) + + cert, err := tls.LoadX509KeyPair(data.Path("x509/server_cert.pem"), data.Path("x509/server_key.pem")) + if err != nil { + log.Fatalf("failed to load key pair: %s", err) + } + + ca := x509.NewCertPool() + caFilePath := data.Path("x509/client_ca_cert.pem") + caBytes, err := os.ReadFile(caFilePath) + if err != nil { + log.Fatalf("failed to read ca cert %q: %v", caFilePath, err) + } + if ok := ca.AppendCertsFromPEM(caBytes); !ok { + log.Fatalf("failed to parse %q", caFilePath) + } + + tlsConfig := &tls.Config{ + ClientAuth: tls.RequireAndVerifyClientCert, + Certificates: []tls.Certificate{cert}, + ClientCAs: ca, + } + + s := grpc.NewServer(grpc.Creds(credentials.NewTLS(tlsConfig))) + pb.RegisterEchoServer(s, &ecServer{}) + lis, err := net.Listen("tcp", fmt.Sprintf("localhost:%d", *port)) + if err != nil { + log.Fatalf("failed to listen: %v", err) + } + if err := s.Serve(lis); err != nil { + log.Fatalf("failed to serve: %v", err) + } +} diff --git a/examples/features/errors/README.md b/examples/features/error_details/README.md similarity index 100% rename from examples/features/errors/README.md rename to examples/features/error_details/README.md diff --git a/examples/features/errors/client/main.go b/examples/features/error_details/client/main.go similarity index 92% rename from examples/features/errors/client/main.go rename to examples/features/error_details/client/main.go index b87fb48a9bbd..7f905f82bef3 100644 --- a/examples/features/errors/client/main.go +++ b/examples/features/error_details/client/main.go @@ -28,6 +28,7 @@ import ( epb "google.golang.org/genproto/googleapis/rpc/errdetails" "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" pb "google.golang.org/grpc/examples/helloworld/helloworld" "google.golang.org/grpc/status" ) @@ -38,7 +39,7 @@ func main() { flag.Parse() // Set up a connection to the server. - conn, err := grpc.Dial(*addr, grpc.WithInsecure(), grpc.WithBlock()) + conn, err := grpc.Dial(*addr, grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { log.Fatalf("did not connect: %v", err) } diff --git a/examples/features/errors/server/main.go b/examples/features/error_details/server/main.go similarity index 94% rename from examples/features/errors/server/main.go rename to examples/features/error_details/server/main.go index c6c06a2bf2cd..dc337741ad02 100644 --- a/examples/features/errors/server/main.go +++ b/examples/features/error_details/server/main.go @@ -39,6 +39,7 @@ var port = flag.Int("port", 50052, "port number") // server is used to implement helloworld.GreeterServer. type server struct { + pb.UnimplementedGreeterServer mu sync.Mutex count map[string]int } @@ -77,8 +78,7 @@ func main() { } s := grpc.NewServer() - hw := &server{count: make(map[string]int)} - pb.RegisterGreeterService(s, &pb.GreeterService{SayHello: hw.SayHello}) + pb.RegisterGreeterServer(s, &server{count: make(map[string]int)}) if err := s.Serve(lis); err != nil { log.Fatalf("failed to serve: %v", err) } diff --git a/examples/features/error_handling/README.md b/examples/features/error_handling/README.md new file mode 100644 index 000000000000..c6c4ba2c2e2d --- /dev/null +++ b/examples/features/error_handling/README.md @@ -0,0 +1,22 @@ +# Description + +This example demonstrates basic RPC error handling in gRPC. + +# Run the sample code + +Run the server, which returns an error if the RPC request's `Name` field is +empty. + +```sh +$ go run ./server/main.go +``` + +Then run the client in another terminal, which does two requests: one with an +empty Name field and one with it populated with the current username provided by +os/user. + +```sh +$ go run ./client/main.go +``` + +It should print the status codes it received from the server. diff --git a/examples/features/error_handling/client/main.go b/examples/features/error_handling/client/main.go new file mode 100644 index 000000000000..bd4ec0a1d33f --- /dev/null +++ b/examples/features/error_handling/client/main.go @@ -0,0 +1,70 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary client is an example client. +package main + +import ( + "context" + "flag" + "log" + "os/user" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + pb "google.golang.org/grpc/examples/helloworld/helloworld" + "google.golang.org/grpc/status" +) + +var addr = flag.String("addr", "localhost:50052", "the address to connect to") + +func main() { + flag.Parse() + + name := "unknown" + if u, err := user.Current(); err == nil && u.Username != "" { + name = u.Username + } + + // Set up a connection to the server. + conn, err := grpc.Dial(*addr, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + log.Fatalf("Failed to connect: %v", err) + } + defer conn.Close() + c := pb.NewGreeterClient(conn) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + for _, reqName := range []string{"", name} { + log.Printf("Calling SayHello with Name:%q", reqName) + r, err := c.SayHello(ctx, &pb.HelloRequest{Name: reqName}) + if err != nil { + if status.Code(err) != codes.InvalidArgument { + log.Printf("Received unexpected error: %v", err) + continue + } + log.Printf("Received error: %v", err) + continue + } + log.Printf("Received response: %s", r.Message) + } +} diff --git a/examples/features/error_handling/server/main.go b/examples/features/error_handling/server/main.go new file mode 100644 index 000000000000..4471c560add9 --- /dev/null +++ b/examples/features/error_handling/server/main.go @@ -0,0 +1,65 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary server is an example server. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "net" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + pb "google.golang.org/grpc/examples/helloworld/helloworld" +) + +var port = flag.Int("port", 50052, "port number") + +// server is used to implement helloworld.GreeterServer. +type server struct { + pb.UnimplementedGreeterServer +} + +// SayHello implements helloworld.GreeterServer. +func (s *server) SayHello(ctx context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) { + if in.Name == "" { + return nil, status.Errorf(codes.InvalidArgument, "request missing required field: Name") + } + return &pb.HelloReply{Message: "Hello " + in.Name}, nil +} + +func main() { + flag.Parse() + + address := fmt.Sprintf(":%v", *port) + lis, err := net.Listen("tcp", address) + if err != nil { + log.Fatalf("failed to listen: %v", err) + } + + s := grpc.NewServer() + pb.RegisterGreeterServer(s, &server{}) + if err := s.Serve(lis); err != nil { + log.Fatalf("failed to serve: %v", err) + } +} diff --git a/examples/features/health/client/main.go b/examples/features/health/client/main.go index 9cbc03f90a47..63b4717b5257 100644 --- a/examples/features/health/client/main.go +++ b/examples/features/health/client/main.go @@ -27,6 +27,7 @@ import ( "time" "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" pb "google.golang.org/grpc/examples/features/proto/echo" _ "google.golang.org/grpc/health" "google.golang.org/grpc/resolver" @@ -65,7 +66,7 @@ func main() { address := fmt.Sprintf("%s:///unused", r.Scheme()) options := []grpc.DialOption{ - grpc.WithInsecure(), + grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithBlock(), grpc.WithResolvers(r), grpc.WithDefaultServiceConfig(serviceConfig), @@ -73,7 +74,7 @@ func main() { conn, err := grpc.Dial(address, options...) if err != nil { - log.Fatalf("did not connect %v", err) + log.Fatalf("grpc.Dial(%q): %v", address, err) } defer conn.Close() diff --git a/examples/features/health/server/main.go b/examples/features/health/server/main.go index 670518da9d08..65039b38d5be 100644 --- a/examples/features/health/server/main.go +++ b/examples/features/health/server/main.go @@ -30,6 +30,7 @@ import ( "google.golang.org/grpc" pb "google.golang.org/grpc/examples/features/proto/echo" "google.golang.org/grpc/health" + healthgrpc "google.golang.org/grpc/health/grpc_health_v1" healthpb "google.golang.org/grpc/health/grpc_health_v1" ) @@ -40,12 +41,18 @@ var ( system = "" // empty string represents the health of the system ) -func unaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) { +type echoServer struct { + pb.UnimplementedEchoServer +} + +func (e *echoServer) UnaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) { return &pb.EchoResponse{ Message: fmt.Sprintf("hello from localhost:%d", *port), }, nil } +var _ pb.EchoServer = &echoServer{} + func main() { flag.Parse() @@ -56,8 +63,8 @@ func main() { s := grpc.NewServer() healthcheck := health.NewServer() - healthpb.RegisterHealthServer(s, healthcheck) - pb.RegisterEchoService(s, &pb.EchoService{UnaryEcho: unaryEcho}) + healthgrpc.RegisterHealthServer(s, healthcheck) + pb.RegisterEchoServer(s, &echoServer{}) go func() { // asynchronously inspect dependencies and toggle serving status as needed diff --git a/examples/features/interceptor/README.md b/examples/features/interceptor/README.md index 1e74b2881cc7..0f1908be590a 100644 --- a/examples/features/interceptor/README.md +++ b/examples/features/interceptor/README.md @@ -76,7 +76,7 @@ user to operate on. In the example, we define a new struct `wrappedStream`, which is embedded with a `ClientStream`. Then, we implement (overload) the `SendMsg` and `RecvMsg` -methods on `wrappedStream` to intercepts these two operations on the embedded +methods on `wrappedStream` to intercept these two operations on the embedded `ClientStream`. In the example, we log the message type info and time info for interception purpose. @@ -113,6 +113,6 @@ handler StreamHandler) error`. Refer to client-side stream interceptor section for detailed implementation explanation. -To install the unary interceptor for a Server, configure `NewServer` with +To install the stream interceptor for a Server, configure `NewServer` with `ServerOption` [`StreamInterceptor`](https://godoc.org/google.golang.org/grpc#StreamInterceptor). diff --git a/examples/features/interceptor/client/main.go b/examples/features/interceptor/client/main.go index 0c2015169d17..4bfa9be29a75 100644 --- a/examples/features/interceptor/client/main.go +++ b/examples/features/interceptor/client/main.go @@ -40,12 +40,12 @@ var addr = flag.String("addr", "localhost:50051", "the address to connect to") const fallbackToken = "some-secret-token" // logger is to mock a sophisticated logging system. To simplify the example, we just print out the content. -func logger(format string, a ...interface{}) { +func logger(format string, a ...any) { fmt.Printf("LOG:\t"+format+"\n", a...) } // unaryInterceptor is an example unary interceptor. -func unaryInterceptor(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { +func unaryInterceptor(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { var credsConfigured bool for _, o := range opts { _, ok := o.(grpc.PerRPCCredsCallOption) @@ -55,9 +55,9 @@ func unaryInterceptor(ctx context.Context, method string, req, reply interface{} } } if !credsConfigured { - opts = append(opts, grpc.PerRPCCredentials(oauth.NewOauthAccess(&oauth2.Token{ - AccessToken: fallbackToken, - }))) + opts = append(opts, grpc.PerRPCCredentials(oauth.TokenSource{ + TokenSource: oauth2.StaticTokenSource(&oauth2.Token{AccessToken: fallbackToken}), + })) } start := time.Now() err := invoker(ctx, method, req, reply, cc, opts...) @@ -72,12 +72,12 @@ type wrappedStream struct { grpc.ClientStream } -func (w *wrappedStream) RecvMsg(m interface{}) error { +func (w *wrappedStream) RecvMsg(m any) error { logger("Receive a message (Type: %T) at %v", m, time.Now().Format(time.RFC3339)) return w.ClientStream.RecvMsg(m) } -func (w *wrappedStream) SendMsg(m interface{}) error { +func (w *wrappedStream) SendMsg(m any) error { logger("Send a message (Type: %T) at %v", m, time.Now().Format(time.RFC3339)) return w.ClientStream.SendMsg(m) } @@ -97,9 +97,9 @@ func streamInterceptor(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.Clie } } if !credsConfigured { - opts = append(opts, grpc.PerRPCCredentials(oauth.NewOauthAccess(&oauth2.Token{ - AccessToken: fallbackToken, - }))) + opts = append(opts, grpc.PerRPCCredentials(oauth.TokenSource{ + TokenSource: oauth2.StaticTokenSource(&oauth2.Token{AccessToken: fallbackToken}), + })) } s, err := streamer(ctx, desc, cc, method, opts...) if err != nil { @@ -153,7 +153,7 @@ func main() { } // Set up a connection to the server. - conn, err := grpc.Dial(*addr, grpc.WithTransportCredentials(creds), grpc.WithUnaryInterceptor(unaryInterceptor), grpc.WithStreamInterceptor(streamInterceptor), grpc.WithBlock()) + conn, err := grpc.Dial(*addr, grpc.WithTransportCredentials(creds), grpc.WithUnaryInterceptor(unaryInterceptor), grpc.WithStreamInterceptor(streamInterceptor)) if err != nil { log.Fatalf("did not connect: %v", err) } diff --git a/examples/features/interceptor/server/main.go b/examples/features/interceptor/server/main.go index 5ca5a8432517..57d520acdbd3 100644 --- a/examples/features/interceptor/server/main.go +++ b/examples/features/interceptor/server/main.go @@ -47,16 +47,20 @@ var ( ) // logger is to mock a sophisticated logging system. To simplify the example, we just print out the content. -func logger(format string, a ...interface{}) { +func logger(format string, a ...any) { fmt.Printf("LOG:\t"+format+"\n", a...) } -func unaryEcho(ctx context.Context, in *pb.EchoRequest) (*pb.EchoResponse, error) { +type server struct { + pb.UnimplementedEchoServer +} + +func (s *server) UnaryEcho(ctx context.Context, in *pb.EchoRequest) (*pb.EchoResponse, error) { fmt.Printf("unary echoing message %q\n", in.Message) return &pb.EchoResponse{Message: in.Message}, nil } -func bidirectionalStreamingEcho(stream pb.Echo_BidirectionalStreamingEchoServer) error { +func (s *server) BidirectionalStreamingEcho(stream pb.Echo_BidirectionalStreamingEchoServer) error { for { in, err := stream.Recv() if err != nil { @@ -83,7 +87,7 @@ func valid(authorization []string) bool { return token == "some-secret-token" } -func unaryInterceptor(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { +func unaryInterceptor(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { // authentication (token verification) md, ok := metadata.FromIncomingContext(ctx) if !ok { @@ -94,7 +98,7 @@ func unaryInterceptor(ctx context.Context, req interface{}, info *grpc.UnaryServ } m, err := handler(ctx, req) if err != nil { - logger("RPC failed with error %v", err) + logger("RPC failed with error: %v", err) } return m, err } @@ -105,12 +109,12 @@ type wrappedStream struct { grpc.ServerStream } -func (w *wrappedStream) RecvMsg(m interface{}) error { +func (w *wrappedStream) RecvMsg(m any) error { logger("Receive a message (Type: %T) at %s", m, time.Now().Format(time.RFC3339)) return w.ServerStream.RecvMsg(m) } -func (w *wrappedStream) SendMsg(m interface{}) error { +func (w *wrappedStream) SendMsg(m any) error { logger("Send a message (Type: %T) at %v", m, time.Now().Format(time.RFC3339)) return w.ServerStream.SendMsg(m) } @@ -119,7 +123,7 @@ func newWrappedStream(s grpc.ServerStream) grpc.ServerStream { return &wrappedStream{s} } -func streamInterceptor(srv interface{}, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { +func streamInterceptor(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { // authentication (token verification) md, ok := metadata.FromIncomingContext(ss.Context()) if !ok { @@ -131,7 +135,7 @@ func streamInterceptor(srv interface{}, ss grpc.ServerStream, info *grpc.StreamS err := handler(srv, newWrappedStream(ss)) if err != nil { - logger("RPC failed with error %v", err) + logger("RPC failed with error: %v", err) } return err } @@ -152,11 +156,8 @@ func main() { s := grpc.NewServer(grpc.Creds(creds), grpc.UnaryInterceptor(unaryInterceptor), grpc.StreamInterceptor(streamInterceptor)) - // Register EchoService on the server. - pb.RegisterEchoService(s, &pb.EchoService{ - UnaryEcho: unaryEcho, - BidirectionalStreamingEcho: bidirectionalStreamingEcho, - }) + // Register EchoServer on the server. + pb.RegisterEchoServer(s, &server{}) if err := s.Serve(lis); err != nil { log.Fatalf("failed to serve: %v", err) diff --git a/examples/features/keepalive/client/main.go b/examples/features/keepalive/client/main.go index a8cfbc5c4541..feb9b664bf4e 100644 --- a/examples/features/keepalive/client/main.go +++ b/examples/features/keepalive/client/main.go @@ -27,6 +27,7 @@ import ( "time" "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" pb "google.golang.org/grpc/examples/features/proto/echo" "google.golang.org/grpc/keepalive" ) @@ -42,7 +43,7 @@ var kacp = keepalive.ClientParameters{ func main() { flag.Parse() - conn, err := grpc.Dial(*addr, grpc.WithInsecure(), grpc.WithKeepaliveParams(kacp)) + conn, err := grpc.Dial(*addr, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithKeepaliveParams(kacp)) if err != nil { log.Fatalf("did not connect: %v", err) } diff --git a/examples/features/keepalive/server/main.go b/examples/features/keepalive/server/main.go index e3ec8ae0679a..beaa8f710885 100644 --- a/examples/features/keepalive/server/main.go +++ b/examples/features/keepalive/server/main.go @@ -48,7 +48,12 @@ var kasp = keepalive.ServerParameters{ Timeout: 1 * time.Second, // Wait 1 second for the ping ack before assuming the connection is dead } -func unaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) { +// server implements EchoServer. +type server struct { + pb.UnimplementedEchoServer +} + +func (s *server) UnaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) { return &pb.EchoResponse{Message: req.Message}, nil } @@ -62,7 +67,7 @@ func main() { } s := grpc.NewServer(grpc.KeepaliveEnforcementPolicy(kaep), grpc.KeepaliveParams(kasp)) - pb.RegisterEchoService(s, &pb.EchoService{UnaryEcho: unaryEcho}) + pb.RegisterEchoServer(s, &server{}) if err := s.Serve(lis); err != nil { log.Fatalf("failed to serve: %v", err) diff --git a/examples/features/load_balancing/client/main.go b/examples/features/load_balancing/client/main.go index 1578df16671b..6e3d1fc86fe3 100644 --- a/examples/features/load_balancing/client/main.go +++ b/examples/features/load_balancing/client/main.go @@ -26,6 +26,7 @@ import ( "time" "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" ecpb "google.golang.org/grpc/examples/features/proto/echo" "google.golang.org/grpc/resolver" ) @@ -55,11 +56,10 @@ func makeRPCs(cc *grpc.ClientConn, n int) { } func main() { - // "pick_first" is the default, so there's no need to set the load balancer. + // "pick_first" is the default, so there's no need to set the load balancing policy. pickfirstConn, err := grpc.Dial( fmt.Sprintf("%s:///%s", exampleScheme, exampleServiceName), - grpc.WithInsecure(), - grpc.WithBlock(), + grpc.WithTransportCredentials(insecure.NewCredentials()), ) if err != nil { log.Fatalf("did not connect: %v", err) @@ -74,9 +74,8 @@ func main() { // Make another ClientConn with round_robin policy. roundrobinConn, err := grpc.Dial( fmt.Sprintf("%s:///%s", exampleScheme, exampleServiceName), - grpc.WithDefaultServiceConfig(`{"loadBalancingPolicy":"round_robin"}`), // This sets the initial balancing policy. - grpc.WithInsecure(), - grpc.WithBlock(), + grpc.WithDefaultServiceConfig(`{"loadBalancingConfig": [{"round_robin":{}}]}`), // This sets the initial balancing policy. + grpc.WithTransportCredentials(insecure.NewCredentials()), ) if err != nil { log.Fatalf("did not connect: %v", err) @@ -112,7 +111,7 @@ type exampleResolver struct { } func (r *exampleResolver) start() { - addrStrs := r.addrsStore[r.target.Endpoint] + addrStrs := r.addrsStore[r.target.Endpoint()] addrs := make([]resolver.Address, len(addrStrs)) for i, s := range addrStrs { addrs[i] = resolver.Address{Addr: s} diff --git a/examples/features/load_balancing/server/main.go b/examples/features/load_balancing/server/main.go index 680895fe50f6..9d179579ed41 100644 --- a/examples/features/load_balancing/server/main.go +++ b/examples/features/load_balancing/server/main.go @@ -36,6 +36,7 @@ var ( ) type ecServer struct { + pb.UnimplementedEchoServer addr string } @@ -49,8 +50,7 @@ func startServer(addr string) { log.Fatalf("failed to listen: %v", err) } s := grpc.NewServer() - e := &ecServer{addr: addr} - pb.RegisterEchoService(s, &pb.EchoService{UnaryEcho: e.UnaryEcho}) + pb.RegisterEchoServer(s, &ecServer{addr: addr}) log.Printf("serving on %s\n", addr) if err := s.Serve(lis); err != nil { log.Fatalf("failed to serve: %v", err) diff --git a/examples/features/metadata/client/main.go b/examples/features/metadata/client/main.go index 715fb6f5acbd..97e7fd40cf45 100644 --- a/examples/features/metadata/client/main.go +++ b/examples/features/metadata/client/main.go @@ -28,6 +28,7 @@ import ( "time" "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" pb "google.golang.org/grpc/examples/features/proto/echo" "google.golang.org/grpc/metadata" ) @@ -286,7 +287,7 @@ const message = "this is examples/metadata" func main() { flag.Parse() // Set up a connection to the server. - conn, err := grpc.Dial(*addr, grpc.WithInsecure(), grpc.WithBlock()) + conn, err := grpc.Dial(*addr, grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { log.Fatalf("did not connect: %v", err) } diff --git a/examples/features/metadata/server/main.go b/examples/features/metadata/server/main.go index af12a3a65d17..cda3b4978697 100644 --- a/examples/features/metadata/server/main.go +++ b/examples/features/metadata/server/main.go @@ -44,7 +44,11 @@ const ( streamingCount = 10 ) -func unaryEcho(ctx context.Context, in *pb.EchoRequest) (*pb.EchoResponse, error) { +type server struct { + pb.UnimplementedEchoServer +} + +func (s *server) UnaryEcho(ctx context.Context, in *pb.EchoRequest) (*pb.EchoResponse, error) { fmt.Printf("--- UnaryEcho ---\n") // Create trailer in defer to record function return time. defer func() { @@ -73,7 +77,7 @@ func unaryEcho(ctx context.Context, in *pb.EchoRequest) (*pb.EchoResponse, error return &pb.EchoResponse{Message: in.Message}, nil } -func serverStreamingEcho(in *pb.EchoRequest, stream pb.Echo_ServerStreamingEchoServer) error { +func (s *server) ServerStreamingEcho(in *pb.EchoRequest, stream pb.Echo_ServerStreamingEchoServer) error { fmt.Printf("--- ServerStreamingEcho ---\n") // Create trailer in defer to record function return time. defer func() { @@ -110,7 +114,7 @@ func serverStreamingEcho(in *pb.EchoRequest, stream pb.Echo_ServerStreamingEchoS return nil } -func clientStreamingEcho(stream pb.Echo_ClientStreamingEchoServer) error { +func (s *server) ClientStreamingEcho(stream pb.Echo_ClientStreamingEchoServer) error { fmt.Printf("--- ClientStreamingEcho ---\n") // Create trailer in defer to record function return time. defer func() { @@ -150,7 +154,7 @@ func clientStreamingEcho(stream pb.Echo_ClientStreamingEchoServer) error { } } -func bidirectionalStreamingEcho(stream pb.Echo_BidirectionalStreamingEchoServer) error { +func (s *server) BidirectionalStreamingEcho(stream pb.Echo_BidirectionalStreamingEchoServer) error { fmt.Printf("--- BidirectionalStreamingEcho ---\n") // Create trailer in defer to record function return time. defer func() { @@ -201,11 +205,6 @@ func main() { fmt.Printf("server listening at %v\n", lis.Addr()) s := grpc.NewServer() - pb.RegisterEchoService(s, &pb.EchoService{ - UnaryEcho: unaryEcho, - ServerStreamingEcho: serverStreamingEcho, - ClientStreamingEcho: clientStreamingEcho, - BidirectionalStreamingEcho: bidirectionalStreamingEcho, - }) + pb.RegisterEchoServer(s, &server{}) s.Serve(lis) } diff --git a/examples/features/metadata_interceptor/README.md b/examples/features/metadata_interceptor/README.md new file mode 100644 index 000000000000..93a6925d79ef --- /dev/null +++ b/examples/features/metadata_interceptor/README.md @@ -0,0 +1,70 @@ +# Metadata interceptor example + +This example shows how to update metadata from unary and streaming interceptors on the server. +Please see +[grpc-metadata.md](https://github.com/grpc/grpc-go/blob/master/Documentation/grpc-metadata.md) +for more information. + +## Try it + +``` +go run server/main.go +``` + +``` +go run client/main.go +``` + +## Explanation + +#### Unary interceptor + +The interceptor can read existing metadata from the RPC context passed to it. +Since Go contexts are immutable, the interceptor will have to create a new context +with updated metadata and pass it to the provided handler. + +```go +func SomeInterceptor(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { + // Get the incoming metadata from the RPC context, and add a new + // key-value pair to it. + md, ok := metadata.FromIncomingContext(ctx) + md.Append("key1", "value1") + + // Create a context with the new metadata and pass it to handler. + ctx = metadata.NewIncomingContext(ctx, md) + return handler(ctx, req) +} +``` + +#### Streaming interceptor + +`grpc.ServerStream` does not provide a way to modify its RPC context. The streaming +interceptor therefore needs to implement the `grpc.ServerStream` interface and return +a context with updated metadata. + +The easiest way to do this would be to create a type which embeds the `grpc.ServerStream` +interface and overrides only the `Context()` method to return a context with updated +metadata. The streaming interceptor would then pass this wrapped stream to the provided handler. + +```go +type wrappedStream struct { + grpc.ServerStream + ctx context.Context +} + +func (s *wrappedStream) Context() context.Context { + return s.ctx +} + +func SomeStreamInterceptor(srv interface{}, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + // Get the incoming metadata from the RPC context, and add a new + // key-value pair to it. + md, ok := metadata.FromIncomingContext(ctx) + md.Append("key1", "value1") + + // Create a context with the new metadata and pass it to handler. + ctx = metadata.NewIncomingContext(ctx, md) + + return handler(srv, &wrappedStream{ss, ctx}) +} +``` \ No newline at end of file diff --git a/examples/features/metadata_interceptor/client/main.go b/examples/features/metadata_interceptor/client/main.go new file mode 100644 index 000000000000..5e1bebec12ae --- /dev/null +++ b/examples/features/metadata_interceptor/client/main.go @@ -0,0 +1,86 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary client is an example client. +package main + +import ( + "context" + "flag" + "fmt" + "io" + "log" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + + pb "google.golang.org/grpc/examples/features/proto/echo" +) + +var addr = flag.String("addr", "localhost:50051", "the address to connect to") + +func callUnaryEcho(ctx context.Context, client pb.EchoClient) { + resp, err := client.UnaryEcho(ctx, &pb.EchoRequest{Message: "hello world"}) + if err != nil { + log.Fatalf("UnaryEcho: %v", err) + } + fmt.Println("UnaryEcho: ", resp.Message) +} + +func callBidiStreamingEcho(ctx context.Context, client pb.EchoClient) { + c, err := client.BidirectionalStreamingEcho(ctx) + if err != nil { + log.Fatalf("BidiStreamingEcho: %v", err) + } + + if err := c.Send(&pb.EchoRequest{Message: "hello world"}); err != nil { + log.Fatalf("Sending echo request: %v", err) + } + c.CloseSend() + + for { + resp, err := c.Recv() + if err == io.EOF { + break + } + if err != nil { + log.Fatalf("Receiving echo response: %v", err) + } + fmt.Println("BidiStreaming Echo: ", resp.Message) + } +} + +func main() { + flag.Parse() + + conn, err := grpc.Dial(*addr, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + log.Fatalf("grpc.Dial(%q): %v", *addr, err) + } + defer conn.Close() + + ec := pb.NewEchoClient(conn) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + callUnaryEcho(ctx, ec) + + callBidiStreamingEcho(ctx, ec) +} diff --git a/examples/features/metadata_interceptor/server/main.go b/examples/features/metadata_interceptor/server/main.go new file mode 100644 index 000000000000..1d8f1d3262f8 --- /dev/null +++ b/examples/features/metadata_interceptor/server/main.go @@ -0,0 +1,140 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary server is an example server. +package main + +import ( + "context" + "flag" + "fmt" + "io" + "log" + "net" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + + pb "google.golang.org/grpc/examples/features/proto/echo" +) + +var port = flag.Int("port", 50051, "the port to serve on") + +var errMissingMetadata = status.Errorf(codes.InvalidArgument, "no incoming metadata in rpc context") + +type server struct { + pb.UnimplementedEchoServer +} + +func unaryInterceptor(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return nil, errMissingMetadata + } + + md.Append("key1", "value1") + ctx = metadata.NewIncomingContext(ctx, md) + + return handler(ctx, req) +} + +func (s *server) UnaryEcho(ctx context.Context, in *pb.EchoRequest) (*pb.EchoResponse, error) { + fmt.Printf("--- UnaryEcho ---\n") + + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return nil, status.Errorf(codes.Internal, "UnaryEcho: missing incoming metadata in rpc context") + } + + // Read and print metadata added by the interceptor. + if v, ok := md["key1"]; ok { + fmt.Printf("key1 from metadata: \n") + for i, e := range v { + fmt.Printf(" %d. %s\n", i, e) + } + } + + return &pb.EchoResponse{Message: in.Message}, nil +} + +type wrappedStream struct { + grpc.ServerStream + ctx context.Context +} + +func (s *wrappedStream) Context() context.Context { + return s.ctx +} + +func streamInterceptor(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + md, ok := metadata.FromIncomingContext(ss.Context()) + if !ok { + return errMissingMetadata + } + + md.Append("key1", "value1") + ctx := metadata.NewIncomingContext(ss.Context(), md) + + return handler(srv, &wrappedStream{ss, ctx}) +} + +func (s *server) BidirectionalStreamingEcho(stream pb.Echo_BidirectionalStreamingEchoServer) error { + fmt.Printf("--- BidirectionalStreamingEcho ---\n") + + md, ok := metadata.FromIncomingContext(stream.Context()) + if !ok { + return status.Errorf(codes.Internal, "BidirectionalStreamingEcho: missing incoming metadata in rpc context") + } + + // Read and print metadata added by the interceptor. + if v, ok := md["key1"]; ok { + fmt.Printf("key1 from metadata: \n") + for i, e := range v { + fmt.Printf(" %d. %s\n", i, e) + } + } + + // Read requests and send responses. + for { + in, err := stream.Recv() + if err == io.EOF { + return nil + } + if err != nil { + return err + } + if err = stream.Send(&pb.EchoResponse{Message: in.Message}); err != nil { + return err + } + } +} + +func main() { + flag.Parse() + lis, err := net.Listen("tcp", fmt.Sprintf(":%d", *port)) + if err != nil { + log.Fatalf("net.Listen() failed: %v", err) + } + fmt.Printf("Server listening at %v\n", lis.Addr()) + + s := grpc.NewServer(grpc.UnaryInterceptor(unaryInterceptor), grpc.StreamInterceptor(streamInterceptor)) + pb.RegisterEchoServer(s, &server{}) + s.Serve(lis) +} diff --git a/examples/features/multiplex/client/main.go b/examples/features/multiplex/client/main.go index 72d6cd56775b..3cd85240a335 100644 --- a/examples/features/multiplex/client/main.go +++ b/examples/features/multiplex/client/main.go @@ -27,6 +27,7 @@ import ( "time" "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" ecpb "google.golang.org/grpc/examples/features/proto/echo" hwpb "google.golang.org/grpc/examples/helloworld/helloworld" ) @@ -58,7 +59,7 @@ func callUnaryEcho(client ecpb.EchoClient, message string) { func main() { flag.Parse() // Set up a connection to the server. - conn, err := grpc.Dial(*addr, grpc.WithInsecure(), grpc.WithBlock()) + conn, err := grpc.Dial(*addr, grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { log.Fatalf("did not connect: %v", err) } diff --git a/examples/features/multiplex/server/main.go b/examples/features/multiplex/server/main.go index d8b13305da25..18da09adda3f 100644 --- a/examples/features/multiplex/server/main.go +++ b/examples/features/multiplex/server/main.go @@ -34,13 +34,21 @@ import ( var port = flag.Int("port", 50051, "the port to serve on") -// sayHello implements helloworld.GreeterServer.SayHello -func sayHello(ctx context.Context, in *hwpb.HelloRequest) (*hwpb.HelloReply, error) { +// hwServer is used to implement helloworld.GreeterServer. +type hwServer struct { + hwpb.UnimplementedGreeterServer +} + +// SayHello implements helloworld.GreeterServer +func (s *hwServer) SayHello(ctx context.Context, in *hwpb.HelloRequest) (*hwpb.HelloReply, error) { return &hwpb.HelloReply{Message: "Hello " + in.Name}, nil } -// unaryEcho implements echo.Echo.UnaryEcho -func unaryEcho(ctx context.Context, req *ecpb.EchoRequest) (*ecpb.EchoResponse, error) { +type ecServer struct { + ecpb.UnimplementedEchoServer +} + +func (s *ecServer) UnaryEcho(ctx context.Context, req *ecpb.EchoRequest) (*ecpb.EchoResponse, error) { return &ecpb.EchoResponse{Message: req.Message}, nil } @@ -55,10 +63,10 @@ func main() { s := grpc.NewServer() // Register Greeter on the server. - hwpb.RegisterGreeterService(s, &hwpb.GreeterService{SayHello: sayHello}) + hwpb.RegisterGreeterServer(s, &hwServer{}) - // Register Echo on the same server. - ecpb.RegisterEchoService(s, &ecpb.EchoService{UnaryEcho: unaryEcho}) + // Register RouteGuide on the same server. + ecpb.RegisterEchoServer(s, &ecServer{}) if err := s.Serve(lis); err != nil { log.Fatalf("failed to serve: %v", err) diff --git a/examples/features/name_resolving/client/main.go b/examples/features/name_resolving/client/main.go index 1c56dcce15df..2766611ba795 100644 --- a/examples/features/name_resolving/client/main.go +++ b/examples/features/name_resolving/client/main.go @@ -26,6 +26,7 @@ import ( "time" "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" ecpb "google.golang.org/grpc/examples/features/proto/echo" "google.golang.org/grpc/resolver" ) @@ -57,8 +58,7 @@ func makeRPCs(cc *grpc.ClientConn, n int) { func main() { passthroughConn, err := grpc.Dial( fmt.Sprintf("passthrough:///%s", backendAddr), // Dial to "passthrough:///localhost:50051" - grpc.WithInsecure(), - grpc.WithBlock(), + grpc.WithTransportCredentials(insecure.NewCredentials()), ) if err != nil { log.Fatalf("did not connect: %v", err) @@ -72,8 +72,7 @@ func main() { exampleConn, err := grpc.Dial( fmt.Sprintf("%s:///%s", exampleScheme, exampleServiceName), // Dial to "example:///resolver.example.grpc.io" - grpc.WithInsecure(), - grpc.WithBlock(), + grpc.WithTransportCredentials(insecure.NewCredentials()), ) if err != nil { log.Fatalf("did not connect: %v", err) @@ -120,7 +119,7 @@ type exampleResolver struct { } func (r *exampleResolver) start() { - addrStrs := r.addrsStore[r.target.Endpoint] + addrStrs := r.addrsStore[r.target.Endpoint()] addrs := make([]resolver.Address, len(addrStrs)) for i, s := range addrStrs { addrs[i] = resolver.Address{Addr: s} diff --git a/examples/features/name_resolving/server/main.go b/examples/features/name_resolving/server/main.go index 77d35eb2adac..1977f44d0c5e 100644 --- a/examples/features/name_resolving/server/main.go +++ b/examples/features/name_resolving/server/main.go @@ -32,8 +32,13 @@ import ( const addr = "localhost:50051" -func unaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) { - return &pb.EchoResponse{Message: fmt.Sprintf("%s (from %s)", req.Message, addr)}, nil +type ecServer struct { + pb.UnimplementedEchoServer + addr string +} + +func (s *ecServer) UnaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) { + return &pb.EchoResponse{Message: fmt.Sprintf("%s (from %s)", req.Message, s.addr)}, nil } func main() { @@ -42,7 +47,7 @@ func main() { log.Fatalf("failed to listen: %v", err) } s := grpc.NewServer() - pb.RegisterEchoService(s, &pb.EchoService{UnaryEcho: unaryEcho}) + pb.RegisterEchoServer(s, &ecServer{addr: addr}) log.Printf("serving on %s\n", addr) if err := s.Serve(lis); err != nil { log.Fatalf("failed to serve: %v", err) diff --git a/examples/features/observability/README.md b/examples/features/observability/README.md new file mode 100644 index 000000000000..f2aa52101069 --- /dev/null +++ b/examples/features/observability/README.md @@ -0,0 +1,3 @@ +This example is the Hello World example instrumented for logs, metrics, and tracing. + +Please refer to Microservices Observability user guide for setup. diff --git a/examples/features/observability/client/clientConfig.json b/examples/features/observability/client/clientConfig.json new file mode 100644 index 000000000000..b98ae25e1b77 --- /dev/null +++ b/examples/features/observability/client/clientConfig.json @@ -0,0 +1,17 @@ +{ + "cloud_monitoring": {}, + "cloud_trace": { + "sampling_rate": 1.0 + }, + "cloud_logging": { + "client_rpc_events": [{ + "methods": ["*"] + }], + "server_rpc_events": [{ + "methods": ["*"] + }] + }, + "labels": { + "environment" : "example-client" + } +} diff --git a/examples/features/observability/client/main.go b/examples/features/observability/client/main.go new file mode 100644 index 000000000000..4c1d994a30dd --- /dev/null +++ b/examples/features/observability/client/main.go @@ -0,0 +1,70 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package main implements a client for Greeter service. +package main + +import ( + "context" + "flag" + "log" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + pb "google.golang.org/grpc/examples/helloworld/helloworld" + "google.golang.org/grpc/gcp/observability" +) + +const ( + defaultName = "world" +) + +var ( + addr = flag.String("addr", "localhost:50051", "the address to connect to") + name = flag.String("name", defaultName, "Name to greet") +) + +func main() { + // Turn on global telemetry for the whole binary. If a configuration is + // specified, any created gRPC Client Conn's or Servers will emit telemetry + // data according the the configuration. + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + err := observability.Start(ctx) + if err != nil { + log.Fatalf("observability.Start() failed: %v", err) + } + defer observability.End() + + flag.Parse() + // Set up a connection to the server. + conn, err := grpc.Dial(*addr, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + log.Fatalf("did not connect: %v", err) + } + defer conn.Close() + c := pb.NewGreeterClient(conn) + + // Contact the server and print out its response. + r, err := c.SayHello(ctx, &pb.HelloRequest{Name: *name}) + if err != nil { + log.Fatalf("could not greet: %v", err) + } + log.Printf("Greeting: %s", r.GetMessage()) +} diff --git a/examples/features/observability/go.mod b/examples/features/observability/go.mod new file mode 100644 index 000000000000..5d7f9ad53ac8 --- /dev/null +++ b/examples/features/observability/go.mod @@ -0,0 +1,43 @@ +module google.golang.org/grpc/examples/features/observability + +go 1.19 + +require ( + google.golang.org/grpc v1.54.0 + google.golang.org/grpc/examples v0.0.0-20230323213306-0fdfd40215dc + google.golang.org/grpc/gcp/observability v1.0.0 +) + +require ( + cloud.google.com/go v0.110.0 // indirect + cloud.google.com/go/compute v1.19.0 // indirect + cloud.google.com/go/compute/metadata v0.2.3 // indirect + cloud.google.com/go/logging v1.7.0 // indirect + cloud.google.com/go/longrunning v0.4.1 // indirect + cloud.google.com/go/monitoring v1.13.0 // indirect + cloud.google.com/go/trace v1.9.0 // indirect + contrib.go.opencensus.io/exporter/stackdriver v0.13.12 // indirect + github.com/aws/aws-sdk-go v1.44.162 // indirect + github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/go-cmp v0.5.9 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect + github.com/googleapis/gax-go/v2 v2.7.1 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/prometheus/prometheus v2.5.0+incompatible // indirect + go.opencensus.io v0.24.0 // indirect + golang.org/x/net v0.8.0 // indirect + golang.org/x/oauth2 v0.6.0 // indirect + golang.org/x/sync v0.1.0 // indirect + golang.org/x/sys v0.6.0 // indirect + golang.org/x/text v0.8.0 // indirect + google.golang.org/api v0.114.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect + google.golang.org/grpc/stats/opencensus v1.0.0 // indirect + google.golang.org/protobuf v1.30.0 // indirect +) diff --git a/examples/features/observability/go.sum b/examples/features/observability/go.sum new file mode 100644 index 000000000000..d28df85cd149 --- /dev/null +++ b/examples/features/observability/go.sum @@ -0,0 +1,644 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= +cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= +cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= +cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= +cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= +cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= +cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= +cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys= +cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/compute v1.19.0 h1:+9zda3WGgW1ZSTlVppLCYFIr48Pa35q1uG2N1itbCEQ= +cloud.google.com/go/compute v1.19.0/go.mod h1:rikpw2y+UMidAe9tISo04EHNOIf42RLYF/q8Bs93scU= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/iam v0.13.0 h1:+CmB+K0J/33d0zSQ9SlFWUeCCEn5XJA0ZMZ3pHE9u8k= +cloud.google.com/go/logging v1.7.0 h1:CJYxlNNNNAMkHp9em/YEXcfJg+rPDg7YfwoRpMU+t5I= +cloud.google.com/go/logging v1.7.0/go.mod h1:3xjP2CjkM3ZkO73aj4ASA5wRPGGCRrPIAeNqVNkzY8M= +cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM= +cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo= +cloud.google.com/go/monitoring v1.1.0/go.mod h1:L81pzz7HKn14QCMaCs6NTQkdBnE87TElyanS95vIcl4= +cloud.google.com/go/monitoring v1.13.0 h1:2qsrgXGVoRXpP7otZ14eE1I568zAa92sJSDPyOJvwjM= +cloud.google.com/go/monitoring v1.13.0/go.mod h1:k2yMBAB1H9JT/QETjNkgdCGD9bPF712XiLTVr+cBrpw= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/trace v1.0.0/go.mod h1:4iErSByzxkyHWzzlAj63/Gmjz0NH1ASqhJguHpGcr6A= +cloud.google.com/go/trace v1.9.0 h1:olxC0QHC59zgJVALtgqfD9tGk0lfeCP5/AGXL3Px/no= +cloud.google.com/go/trace v1.9.0/go.mod h1:lOQqpE5IaWY0Ixg7/r2SjixMuc6lfTFeO4QGM4dQWOk= +contrib.go.opencensus.io/exporter/stackdriver v0.13.12 h1:bjBKzIf7/TAkxd7L2utGaLM78bmUWlCval5K9UeElbY= +contrib.go.opencensus.io/exporter/stackdriver v0.13.12/go.mod h1:mmxnWlrvrFdpiOHOhxBaVi1rkc0WOqhgfknj4Yg0SeQ= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/aws/aws-sdk-go v1.37.0/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= +github.com/aws/aws-sdk-go v1.44.162 h1:hKAd+X+/BLxVMzH+4zKxbQcQQGrk2UhFX0OTu1Mhon8= +github.com/aws/aws-sdk-go v1.44.162/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g= +github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k= +github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= +github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= +github.com/googleapis/gax-go/v2 v2.7.1 h1:gF4c0zjUP2H/s/hEGyLA3I0fA2ZWjzYiONAD6cvPr8A= +github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/prometheus v2.5.0+incompatible h1:7QPitgO2kOFG8ecuRn9O/4L9+10He72rVRJvMXrE9Hg= +github.com/prometheus/prometheus v2.5.0+incompatible/go.mod h1:oAIUtOny2rjMX0OWN5vPR5/q/twIROJvdqnQKDdil/s= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.6.0 h1:Lh8GPgSKBfWSwFvtuWOfeI3aAAnbXTSutYxJiOJFgIw= +golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210917161153-d61c044b1678/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= +google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= +google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= +google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= +google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= +google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= +google.golang.org/api v0.58.0/go.mod h1:cAbP2FsxoGVNwtgNAmmn3y5G1TWAiVYRmg4yku3lv+E= +google.golang.org/api v0.59.0/go.mod h1:sT2boj7M9YJxZzgeZqXogmhfmRWDtPzT31xkieUbuZU= +google.golang.org/api v0.114.0 h1:1xQPji6cO2E2vLiI+C/XiFAnsn1WV3mjaEwGLhi3grE= +google.golang.org/api v0.114.0/go.mod h1:ifYI2ZsFK6/uGddGfAD5BMxlnkBqCmqHSDUVi45N5Yg= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= +google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210917145530-b395a37504d4/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210921142501-181ce0d877f6/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211008145708-270636b82663/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211018162055-cf77aa76bad2/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54 h1:9NWlQfY2ePejTmfwUH1OWwmznFa+0kKcHGPDvcPza9M= +google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54/go.mod h1:zqTuNwFlFRsw5zIts5VnzLQxSRqh+CGOTVMlYbY0Eyk= +google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9 h1:m8v1xLLLzMe1m5P+gCTF8nJB9epwZQUBERm20Oy1poQ= +google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag= +google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= +google.golang.org/grpc/examples v0.0.0-20230323213306-0fdfd40215dc h1:H58v4RmBwciuKpwU6NFUn3w2hPZNL78HedaJUitCdpI= +google.golang.org/grpc/examples v0.0.0-20230323213306-0fdfd40215dc/go.mod h1:EXfxRt8PpWkTFBAXaWXB0Xgb1S/FFBXvFRry0nr2bHQ= +google.golang.org/grpc/gcp/observability v1.0.0 h1:YkGqlAFEFM69+GDI8MnuSV4RTvBWkx4AKealZ+yGizY= +google.golang.org/grpc/gcp/observability v1.0.0/go.mod h1:SmWxljYyQOJWPALwV6WhM3PdbH7sQsrCYIzlRy2PY00= +google.golang.org/grpc/stats/opencensus v1.0.0 h1:evSYcRZaSToQp+borzWE52+03joezZeXcKJvZDfkUJA= +google.golang.org/grpc/stats/opencensus v1.0.0/go.mod h1:FhdkeYvN43wLYUnapVuRJJ9JXkNwe403iLUW2LKSnjs= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/examples/features/observability/server/main.go b/examples/features/observability/server/main.go new file mode 100644 index 000000000000..0aae0699342d --- /dev/null +++ b/examples/features/observability/server/main.go @@ -0,0 +1,89 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package main implements a server for Greeter service. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "net" + "os" + "os/signal" + "syscall" + "time" + + "google.golang.org/grpc" + pb "google.golang.org/grpc/examples/helloworld/helloworld" + "google.golang.org/grpc/gcp/observability" +) + +var ( + port = flag.Int("port", 50051, "The server port") +) + +// server is used to implement helloworld.GreeterServer. +type server struct { + pb.UnimplementedGreeterServer +} + +// SayHello implements helloworld.GreeterServer +func (s *server) SayHello(ctx context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) { + log.Printf("Received: %v", in.GetName()) + return &pb.HelloReply{Message: "Hello " + in.GetName()}, nil +} + +func main() { + // Turn on global telemetry for the whole binary. If a configuration is + // specified, any created gRPC Client Conn's or Servers will emit telemetry + // data according the the configuration. + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + err := observability.Start(ctx) + if err != nil { + log.Fatalf("observability.Start() failed: %v", err) + } + defer observability.End() + + flag.Parse() + lis, err := net.Listen("tcp", fmt.Sprintf(":%d", *port)) + if err != nil { + log.Fatalf("failed to listen: %v", err) + } + s := grpc.NewServer() + pb.RegisterGreeterServer(s, &server{}) + log.Printf("server listening at %v", lis.Addr()) + + // This server can potentially be terminated by an external signal from the + // Operating System. The following catches those signals and calls s.Stop(). + // This causes the s.Serve() call to return and run main()'s defers, + // including the observability.End() call that ensures any pending + // observability data is sent to Cloud Operations. + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt, syscall.SIGTERM) + go func() { + <-c + s.Stop() + }() + + if err := s.Serve(lis); err != nil { + log.Fatalf("failed to serve: %v", err) + } +} diff --git a/examples/features/observability/server/serverConfig.json b/examples/features/observability/server/serverConfig.json new file mode 100644 index 000000000000..a2bf7f6b6e74 --- /dev/null +++ b/examples/features/observability/server/serverConfig.json @@ -0,0 +1,17 @@ +{ + "cloud_monitoring": {}, + "cloud_trace": { + "sampling_rate": 1.0 + }, + "cloud_logging": { + "client_rpc_events": [{ + "methods": ["*"] + }], + "server_rpc_events": [{ + "methods": ["*"] + }] + }, + "labels": { + "environment" : "example-server" + } +} diff --git a/examples/features/orca/README.md b/examples/features/orca/README.md new file mode 100644 index 000000000000..ef99aa255ba5 --- /dev/null +++ b/examples/features/orca/README.md @@ -0,0 +1,48 @@ +# ORCA Load Reporting + +ORCA is a protocol for reporting load between servers and clients. This +example shows how to implement this from both the client and server side. For +more details, please see [gRFC +A51](https://github.com/grpc/proposal/blob/master/A51-custom-backend-metrics.md) + +## Try it + +``` +go run server/main.go +``` + +``` +go run client/main.go +``` + +## Explanation + +gRPC ORCA support provides two different ways to report load data to clients +from servers: out-of-band and per-RPC. Out-of-band metrics are reported +regularly at some interval on a stream, while per-RPC metrics are reported +along with the trailers at the end of a call. Both of these mechanisms are +optional and work independently. + +The full ORCA API documentation is available here: +https://pkg.go.dev/google.golang.org/grpc/orca + +### Out-of-band Metrics + +The server registers an ORCA service that is used for out-of-band metrics. It +does this by using `orca.Register()` and then setting metrics on the returned +`orca.Service` using its methods. + +The client receives out-of-band metrics via the LB policy. It receives +callbacks to a listener by registering the listener on a `SubConn` via +`orca.RegisterOOBListener`. + +### Per-RPC Metrics + +The server is set up to report query cost metrics in its RPC handler. For +per-RPC metrics to be reported, the gRPC server must be created with the +`orca.CallMetricsServerOption()` option, and metrics are set by calling methods +on the returned `orca.CallMetricRecorder` from +`orca.CallMetricRecorderFromContext()`. + +The client performs one RPC per second. Per-RPC metrics are available for each +call via the `Done()` callback returned from the LB policy's picker. diff --git a/examples/features/orca/client/main.go b/examples/features/orca/client/main.go new file mode 100644 index 000000000000..40ab5a33ed42 --- /dev/null +++ b/examples/features/orca/client/main.go @@ -0,0 +1,157 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary client is an example client. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/orca" + + v3orcapb "github.com/cncf/xds/go/xds/data/orca/v3" + pb "google.golang.org/grpc/examples/features/proto/echo" +) + +var addr = flag.String("addr", "localhost:50051", "the address to connect to") +var test = flag.Bool("test", false, "if set, only 1 RPC is performed before exiting") + +func main() { + flag.Parse() + + // Set up a connection to the server. Configure to use our custom LB + // policy which will receive all the ORCA load reports. + conn, err := grpc.Dial(*addr, + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithDefaultServiceConfig(`{"loadBalancingConfig": [{"orca_example":{}}]}`), + ) + if err != nil { + log.Fatalf("did not connect: %v", err) + } + defer conn.Close() + + c := pb.NewEchoClient(conn) + + // Perform RPCs once per second. + ticker := time.NewTicker(time.Second) + for range ticker.C { + func() { + // Use an anonymous function to ensure context cancelation via defer. + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if _, err := c.UnaryEcho(ctx, &pb.EchoRequest{Message: "test echo message"}); err != nil { + log.Fatalf("Error from UnaryEcho call: %v", err) + } + }() + if *test { + return + } + } + +} + +// Register an ORCA load balancing policy to receive per-call metrics and +// out-of-band metrics. +func init() { + balancer.Register(orcaLBBuilder{}) +} + +type orcaLBBuilder struct{} + +func (orcaLBBuilder) Name() string { return "orca_example" } +func (orcaLBBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { + return &orcaLB{cc: cc} +} + +// orcaLB is an incomplete LB policy designed to show basic ORCA load reporting +// functionality. It collects per-call metrics in the `Done` callback returned +// by its picker, and it collects out-of-band metrics by registering a listener +// when its SubConn is created. It does not follow general LB policy best +// practices and makes assumptions about the simple test environment it is +// designed to run within. +type orcaLB struct { + cc balancer.ClientConn +} + +func (o *orcaLB) UpdateClientConnState(ccs balancer.ClientConnState) error { + // We assume only one update, ever, containing exactly one address, given + // the use of the "passthrough" (default) name resolver. + + addrs := ccs.ResolverState.Addresses + if len(addrs) != 1 { + return fmt.Errorf("orcaLB: expected 1 address; received: %v", addrs) + } + + // Create one SubConn for the address and connect it. + var sc balancer.SubConn + sc, err := o.cc.NewSubConn(addrs, balancer.NewSubConnOptions{ + StateListener: func(scs balancer.SubConnState) { + if scs.ConnectivityState == connectivity.Ready { + o.cc.UpdateState(balancer.State{ConnectivityState: connectivity.Ready, Picker: &picker{sc}}) + } + }, + }) + if err != nil { + return fmt.Errorf("orcaLB: error creating SubConn: %v", err) + } + sc.Connect() + + // Register a simple ORCA OOB listener on the SubConn. We request a 1 + // second report interval, but in this example the server indicated the + // minimum interval it will allow is 3 seconds, so reports will only be + // sent that often. + orca.RegisterOOBListener(sc, orcaLis{}, orca.OOBListenerOptions{ReportInterval: time.Second}) + + return nil +} + +func (o *orcaLB) ResolverError(error) {} + +// TODO: unused; remove when no longer required. +func (o *orcaLB) UpdateSubConnState(sc balancer.SubConn, scs balancer.SubConnState) {} + +func (o *orcaLB) Close() {} + +type picker struct { + sc balancer.SubConn +} + +func (p *picker) Pick(info balancer.PickInfo) (balancer.PickResult, error) { + return balancer.PickResult{ + SubConn: p.sc, + Done: func(di balancer.DoneInfo) { + fmt.Println("Per-call load report received:", di.ServerLoad.(*v3orcapb.OrcaLoadReport).GetRequestCost()) + }, + }, nil +} + +// orcaLis is the out-of-band load report listener that we pass to +// orca.RegisterOOBListener to receive periodic load report information. +type orcaLis struct{} + +func (orcaLis) OnLoadReport(lr *v3orcapb.OrcaLoadReport) { + fmt.Println("Out-of-band load report received:", lr) +} diff --git a/examples/features/orca/server/main.go b/examples/features/orca/server/main.go new file mode 100644 index 000000000000..e52d5d06eebf --- /dev/null +++ b/examples/features/orca/server/main.go @@ -0,0 +1,96 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary server is an example server. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "net" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/orca" + "google.golang.org/grpc/status" + + pb "google.golang.org/grpc/examples/features/proto/echo" +) + +var port = flag.Int("port", 50051, "the port to serve on") + +type server struct { + pb.UnimplementedEchoServer +} + +func (s *server) UnaryEcho(ctx context.Context, in *pb.EchoRequest) (*pb.EchoResponse, error) { + // Report a sample cost for this query. + cmr := orca.CallMetricsRecorderFromContext(ctx) + if cmr == nil { + return nil, status.Errorf(codes.Internal, "unable to retrieve call metrics recorder (missing ORCA ServerOption?)") + } + cmr.SetRequestCost("db_queries", 10) + + return &pb.EchoResponse{Message: in.Message}, nil +} + +func main() { + flag.Parse() + + lis, err := net.Listen("tcp", fmt.Sprintf("localhost:%d", *port)) + if err != nil { + log.Fatalf("Failed to listen: %v", err) + } + fmt.Printf("Server listening at %v\n", lis.Addr()) + + // Create the gRPC server with the orca.CallMetricsServerOption() option, + // which will enable per-call metric recording. No ServerMetricsProvider + // is given here because the out-of-band reporting is enabled separately. + s := grpc.NewServer(orca.CallMetricsServerOption(nil)) + pb.RegisterEchoServer(s, &server{}) + + // Register the orca service for out-of-band metric reporting, and set the + // minimum reporting interval to 3 seconds. Note that, by default, the + // minimum interval must be at least 30 seconds, but 3 seconds is set via + // an internal-only option for illustration purposes only. + smr := orca.NewServerMetricsRecorder() + opts := orca.ServiceOptions{ + MinReportingInterval: 3 * time.Second, + ServerMetricsProvider: smr, + } + internal.ORCAAllowAnyMinReportingInterval.(func(so *orca.ServiceOptions))(&opts) + if err := orca.Register(s, opts); err != nil { + log.Fatalf("Failed to register ORCA service: %v", err) + } + + // Simulate CPU utilization reporting. + go func() { + for { + smr.SetCPUUtilization(.5) + time.Sleep(2 * time.Second) + smr.SetCPUUtilization(.9) + time.Sleep(2 * time.Second) + } + }() + + s.Serve(lis) +} diff --git a/examples/features/profiling/README.md b/examples/features/profiling/README.md deleted file mode 100644 index 8c30a3dbcac4..000000000000 --- a/examples/features/profiling/README.md +++ /dev/null @@ -1,261 +0,0 @@ -# gRPC-Go Profiling - -- Author(s): adtac -- Status: Experimental -- Availability: gRPC-Go >= 1.27 -- Last updated: December 17, 2019 - -gRPC-Go has built-in profiling that can be used to generate a detailed timeline -of the lifecycle of an RPC request. This can be done on the client-side and the -server-side. This directory contains an example client-server implementation -with profiling enabled and some example commands you can run to remotely manage -profiling. - -Typically, there are three logically separate parts involved in integrating -profiling into your application: - -1. Register the `Profiling` service: this requires a simple code change in your - application. -1. Enable profiling when required: profiling is disabled by default and must be - enabled remotely or at server initialization. -1. Download and process profiling data: once your application has collected - enough profiling data, you must use a bundled command-line application to - download your data and process it to generate human-friendly visualization. - -## Registering the `Profiling` Service - -### Server-Side - -Typically, you would create and register a server like so (some Go is shortened -in the interest of brevity; please see the `server` subdirectory for a full -implementation): - -```go -import ( - "google.golang.org/grpc" - profsvc "google.golang.org/grpc/profiling/service" - pb "google.golang.org/grpc/examples/features/proto/echo" -) - -type server struct{} - -func main() error { - s := grpc.NewServer() - pb.RegisterEchoService(s, &pb.EchoService{...}) - - // Include this to register a profiling-specific service within your server. - if err := profsvc.Init(&profsvc.ProfilingConfig{Server: s}); err != nil { - fmt.Printf("error calling profsvc.Init: %v\n", err) - return - } - - lis, _ := net.Listen("tcp", address) - s.Serve(lis) -} -``` - -To register your server for profiling, simply call the `profsvc.Init` method -as shown above. The passed `ProfilingConfig` parameter must set the `Server` -field to a server that is being served on a TCP address. - -### Client-Side - -To register profiling on the client-side, you must create a server to expose -your profiling data in order for it to be retrievable. To do this, it is -recommended that you create a dummy, dedicated server with no service other -than profiling's. See the `client` directory for an example client. - -## Enabling/Disabling Profiling - -Once profiling is baked into your server (unless otherwise specified, from here -on, the word "server" will be used to refer to a `grpc.Server`, not the -server/client distinction from the previous subsection), you need to enable -profiling. There are three ways to do this -- at initialization, remotely -post-initialization, or programmatically within Go. - -### Enabling Profiling at Initialization - -To force profiling to start measuring data right from the first RPC, set the -`Enabled` attribute of the `ProfilingConfig` struct to `true` when you are -initializing profiling. - -```go - // Set Enabled: true to turn profiling on at initialization time. - profsvc.Init(&profsvc.ProfilingConfig{ - Server: s, - Enabled: true, - }) -``` - -### Enabling/Disabling Remotely - -Alternatively, you can enable/disable profiling any time after server -initialization by using a bundled command-line tool designed for remote -profiling management. Assuming `example.com:50051` is the address of the server -that you would like to enable profiling in, do the following: - -```bash -$ go run google.golang.org/grpc/profiling/cmd \ - -address example.com:50051 \ - -enable-profiling -``` - -Similarly, running the command with `-disable-profiling` can be used to disable -profiling remotely. - - -### Enabling/Disabling Within Go - -In addition to the remote service that is exposed, you may enable/disable -profiling within your application in Go: - -```go -import ( - "google.golang.org/grpc/profiling" -) - -func setProfiling(enable bool) { - profiling.Enable(true) -} -``` - -The `profiling.Enable` function can be safely accessed and called concurrently. - -## Downloading and Processing Profiling Data - -Once your server has collected enough profiling data, you may want to download -that data and perform some analysis on the retrieved data. The aforementioned -command-line application within gRPC comes bundled with support for both -operations. - -To retrieve profiling data from a remote server, run the following command: - -```bash -$ go run google.golang.org/grpc/profiling/cmd \ - -address example.com:50051 \ - -retrieve-snapshot \ - -snapshot /path/to/snapshot -``` - -You must provide a path to `-snapshot` that can be written to. This file will -store the retrieved data in a raw and binary form. - -To process this data into a human-consumable such as -[Catapult's trace-viewer format](https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/preview): - -```bash -$ go run google.golang.org/grpc/profiling/cmd \ - -snapshot /path/to/snapshot \ - -stream-stats-catapult-json /path/to/json -``` - -This would read the data stored in `/path/to/snapshot` and process it to -generate a JSON format that is understood by Chromium's -[Catapult project](https://chromium.googlesource.com/catapult). -The Catapult project comes with a utility called -[trace-viewer](https://chromium.googlesource.com/catapult/+/HEAD/tracing/README.md), -which can be used to generate human-readable visualizations: - -```bash -$ git clone https://chromium.googlesource.com/catapult /path/to/catapult -$ /path/to/catapult/tracing/bin/trace2html /path/to/json --output=/path/to/html -``` - -When the generated `/path/to/html` file is opened with a browser, you will be -presented with a detailed visualization of the lifecycle of all RPC requests. -To learn more about trace-viewer and how to navigate the generated HTML, see -[this](https://chromium.googlesource.com/catapult/+/HEAD/tracing/README.md). - -## Frequently Asked Questions - -##### I have multiple `grpc.Server`s in my application. Can I register profiling with just one of them? - -You may not call `profsvc.Init` more than once -- all calls except for the -first one will return an error. As a corollary, it is also not possible to -register or enable/disable profiling for just one `grpc.Server` or operation. -That is, you can enable/disable profiling globally for all gRPC operations or -none at all. - -##### Is `google.golang.org/grpc/profiling/cmd` the canonical implementation of a client that can talk to the profiling service? - -No, the command-line tool is simply provided as a reference implementation and -as a convenience. You are free to write your own tool as long as it can -communicate using the underlying protocol buffers. - -##### Is Catapult's `trace-viewer` the only option that is supported? - -Currently, yes. However, support for other (or better) visualization tools is -welcome. - -##### What is the impact of profiling on application performance? - -When turned off, profiling has virtually no impact on the performance (QPS, -latency, memory footprint) of your application. However, when turned on, expect -a 5-10% throughput/latency penalty and double the memory footprint. - -Profiling is mostly used by gRPC-Go devs. However, if you foresee using -profiling in production machines, because of the negligible impact of profiling -when turned off, you may want to register/initialize your applications with -profiling (but leave it turned off). This will be useful in the off-chance you -want to debug an application later -- in such an event, you can simply remotely -toggle profiling using the `go run` command previously described to enable -profiling data collection. Once you're confident that enough profiling data has -been measured, you can turn it off again and retrieve the data for -post-processing (see previous section). - -##### How many RPCs worth of data is stored by profiling? I'd like to restrict the memory footprint of gRPC's profiling framework to a fixed amount. - -By default, at any given time, the last 214 RPCs worth of data is -stored by profiling. Newly generated profiling data overwrites older data. Note -that the internal data structure is not strictly LIFO in order to be performant -(but is approximately LIFO). All profiling data is timestamped anyway, so -a LIFO property is unnecessary. - -This number is configurable. When registering your server with profiling, you -may specify the number of samples that should be stored, like so: - -```go - // Setting StreamStatsSize: 1024 will make profiling store the last 1024 - // RPCs' data (if profiling is enabled, of course). - profsvc.Init(&profsvc.ProfilingConfig{ - Server: s, - StreamStatsSize: 1024, - }) -``` - -As an estimate, a typical unary RPC is expected produce ~2-3 KiB of profiling -data in memory. This may be useful in estimating how many RPCs worth of data -you can afford depending on your memory capacity. For more complex RPCs such as -streaming RPCs, each RPC will consume more data. The amount of memory consumed -by profiling is mostly independent of the size of messages your application -handles. - -##### The generated visualization is flat and has no flows/arrows. How do I distinguish between different RPCs? - -Unfortunately, there isn't any way to do this without some changes to the way -your application is compiled. This is because gRPC's profiling relies on the -Goroutine ID to uniquely identify different components. - -To enable this, first apply the following patch to your Go runtime installation -directory: - -```diff -diff --git a/src/runtime/runtime2.go b/src/runtime/runtime2.go ---- a/src/runtime/runtime2.go -+++ b/src/runtime/runtime2.go -@@ -392,6 +392,10 @@ type stack struct { - hi uintptr - } - -+func Goid() int64 { -+ return getg().goid -+} -+ - type g struct { - // Stack parameters. - // stack describes the actual stack memory: [stack.lo, stack.hi). -``` - -Then, recompile your application with `-tags grpcgoid` to generate a new -binary. This binary should produce profiling data that is much nicer when -visualized. diff --git a/examples/features/profiling/client/main.go b/examples/features/profiling/client/main.go deleted file mode 100644 index d899e47dadb0..000000000000 --- a/examples/features/profiling/client/main.go +++ /dev/null @@ -1,89 +0,0 @@ -/* - * - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -// Binary client is an example client. -package main - -import ( - "context" - "flag" - "fmt" - "log" - "net" - "time" - - "google.golang.org/grpc" - pb "google.golang.org/grpc/examples/features/proto/echo" - profsvc "google.golang.org/grpc/profiling/service" -) - -var addr = flag.String("addr", "localhost:50051", "the address to connect to") -var profilingPort = flag.Int("profilingPort", 50052, "port to expose the profiling service on") - -func setupClientProfiling() error { - lis, err := net.Listen("tcp", fmt.Sprintf(":%d", *profilingPort)) - if err != nil { - log.Printf("failed to listen: %v\n", err) - return err - } - fmt.Printf("server listening at %v\n", lis.Addr()) - - s := grpc.NewServer() - - // Register this grpc.Server with profiling. - pc := &profsvc.ProfilingConfig{ - Server: s, - Enabled: true, - StreamStatsSize: 1024, - } - if err = profsvc.Init(pc); err != nil { - fmt.Printf("error calling profsvc.Init: %v\n", err) - return err - } - - go s.Serve(lis) - return nil -} - -func main() { - flag.Parse() - - if err := setupClientProfiling(); err != nil { - log.Fatalf("error setting up profiling: %v\n", err) - } - - // Set up a connection to the server. - conn, err := grpc.Dial(*addr, grpc.WithInsecure(), grpc.WithBlock()) - if err != nil { - log.Fatalf("did not connect: %v", err) - } - defer conn.Close() - - c := pb.NewEchoClient(conn) - - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - res, err := c.UnaryEcho(ctx, &pb.EchoRequest{Message: "hello, profiling"}) - fmt.Printf("UnaryEcho call returned %q, %v\n", res.GetMessage(), err) - if err != nil { - log.Fatalf("error calling UnaryEcho: %v", err) - } - - log.Printf("sleeping for 30 seconds with exposed profiling service on :%d\n", *profilingPort) - time.Sleep(30 * time.Second) -} diff --git a/examples/features/profiling/server/main.go b/examples/features/profiling/server/main.go deleted file mode 100644 index aa0bf11b2403..000000000000 --- a/examples/features/profiling/server/main.go +++ /dev/null @@ -1,65 +0,0 @@ -/* - * - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -// Binary server is an example server. -package main - -import ( - "context" - "flag" - "fmt" - "log" - "net" - - "google.golang.org/grpc" - pb "google.golang.org/grpc/examples/features/proto/echo" - profsvc "google.golang.org/grpc/profiling/service" -) - -var port = flag.Int("port", 50051, "the port to serve on") - -func unaryEcho(ctx context.Context, in *pb.EchoRequest) (*pb.EchoResponse, error) { - fmt.Printf("UnaryEcho called with message %q\n", in.GetMessage()) - return &pb.EchoResponse{Message: in.Message}, nil -} - -func main() { - flag.Parse() - - lis, err := net.Listen("tcp", fmt.Sprintf(":%d", *port)) - if err != nil { - log.Fatalf("failed to listen: %v", err) - } - fmt.Printf("server listening at %v\n", lis.Addr()) - - s := grpc.NewServer() - pb.RegisterEchoService(s, &pb.EchoService{UnaryEcho: unaryEcho}) - - // Register your grpc.Server with profiling. - pc := &profsvc.ProfilingConfig{ - Server: s, - Enabled: true, - StreamStatsSize: 1024, - } - if err = profsvc.Init(pc); err != nil { - fmt.Printf("error calling profsvc.Init: %v\n", err) - return - } - - s.Serve(lis) -} diff --git a/examples/features/proto/echo/echo.pb.go b/examples/features/proto/echo/echo.pb.go index 4048b43c0c2c..00e84bbf6b84 100644 --- a/examples/features/proto/echo/echo.pb.go +++ b/examples/features/proto/echo/echo.pb.go @@ -1,129 +1,260 @@ +// +// +// Copyright 2018 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + // Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.31.0 +// protoc v4.22.0 // source: examples/features/proto/echo/echo.proto package echo import ( - fmt "fmt" - proto "github.com/golang/protobuf/proto" - math "math" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" ) -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) // EchoRequest is the request for echo. type EchoRequest struct { - Message string `protobuf:"bytes,1,opt,name=message,proto3" json:"message,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields -func (m *EchoRequest) Reset() { *m = EchoRequest{} } -func (m *EchoRequest) String() string { return proto.CompactTextString(m) } -func (*EchoRequest) ProtoMessage() {} -func (*EchoRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_2fd1d686b7b805dc, []int{0} + Message string `protobuf:"bytes,1,opt,name=message,proto3" json:"message,omitempty"` } -func (m *EchoRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_EchoRequest.Unmarshal(m, b) -} -func (m *EchoRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_EchoRequest.Marshal(b, m, deterministic) -} -func (m *EchoRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_EchoRequest.Merge(m, src) +func (x *EchoRequest) Reset() { + *x = EchoRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_examples_features_proto_echo_echo_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *EchoRequest) XXX_Size() int { - return xxx_messageInfo_EchoRequest.Size(m) + +func (x *EchoRequest) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *EchoRequest) XXX_DiscardUnknown() { - xxx_messageInfo_EchoRequest.DiscardUnknown(m) + +func (*EchoRequest) ProtoMessage() {} + +func (x *EchoRequest) ProtoReflect() protoreflect.Message { + mi := &file_examples_features_proto_echo_echo_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_EchoRequest proto.InternalMessageInfo +// Deprecated: Use EchoRequest.ProtoReflect.Descriptor instead. +func (*EchoRequest) Descriptor() ([]byte, []int) { + return file_examples_features_proto_echo_echo_proto_rawDescGZIP(), []int{0} +} -func (m *EchoRequest) GetMessage() string { - if m != nil { - return m.Message +func (x *EchoRequest) GetMessage() string { + if x != nil { + return x.Message } return "" } // EchoResponse is the response for echo. type EchoResponse struct { - Message string `protobuf:"bytes,1,opt,name=message,proto3" json:"message,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Message string `protobuf:"bytes,1,opt,name=message,proto3" json:"message,omitempty"` } -func (m *EchoResponse) Reset() { *m = EchoResponse{} } -func (m *EchoResponse) String() string { return proto.CompactTextString(m) } -func (*EchoResponse) ProtoMessage() {} -func (*EchoResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_2fd1d686b7b805dc, []int{1} +func (x *EchoResponse) Reset() { + *x = EchoResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_examples_features_proto_echo_echo_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *EchoResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_EchoResponse.Unmarshal(m, b) +func (x *EchoResponse) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *EchoResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_EchoResponse.Marshal(b, m, deterministic) + +func (*EchoResponse) ProtoMessage() {} + +func (x *EchoResponse) ProtoReflect() protoreflect.Message { + mi := &file_examples_features_proto_echo_echo_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -func (m *EchoResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_EchoResponse.Merge(m, src) + +// Deprecated: Use EchoResponse.ProtoReflect.Descriptor instead. +func (*EchoResponse) Descriptor() ([]byte, []int) { + return file_examples_features_proto_echo_echo_proto_rawDescGZIP(), []int{1} } -func (m *EchoResponse) XXX_Size() int { - return xxx_messageInfo_EchoResponse.Size(m) + +func (x *EchoResponse) GetMessage() string { + if x != nil { + return x.Message + } + return "" } -func (m *EchoResponse) XXX_DiscardUnknown() { - xxx_messageInfo_EchoResponse.DiscardUnknown(m) + +var File_examples_features_proto_echo_echo_proto protoreflect.FileDescriptor + +var file_examples_features_proto_echo_echo_proto_rawDesc = []byte{ + 0x0a, 0x27, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73, 0x2f, 0x66, 0x65, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x65, 0x63, 0x68, 0x6f, 0x2f, 0x65, + 0x63, 0x68, 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x12, 0x67, 0x72, 0x70, 0x63, 0x2e, + 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73, 0x2e, 0x65, 0x63, 0x68, 0x6f, 0x22, 0x27, 0x0a, + 0x0b, 0x45, 0x63, 0x68, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, + 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, + 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x28, 0x0a, 0x0c, 0x45, 0x63, 0x68, 0x6f, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, + 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, + 0x32, 0xfb, 0x02, 0x0a, 0x04, 0x45, 0x63, 0x68, 0x6f, 0x12, 0x50, 0x0a, 0x09, 0x55, 0x6e, 0x61, + 0x72, 0x79, 0x45, 0x63, 0x68, 0x6f, 0x12, 0x1f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x65, 0x78, + 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73, 0x2e, 0x65, 0x63, 0x68, 0x6f, 0x2e, 0x45, 0x63, 0x68, 0x6f, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x65, + 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73, 0x2e, 0x65, 0x63, 0x68, 0x6f, 0x2e, 0x45, 0x63, 0x68, + 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5c, 0x0a, 0x13, 0x53, + 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x45, 0x63, + 0x68, 0x6f, 0x12, 0x1f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, + 0x65, 0x73, 0x2e, 0x65, 0x63, 0x68, 0x6f, 0x2e, 0x45, 0x63, 0x68, 0x6f, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x65, 0x78, 0x61, 0x6d, 0x70, + 0x6c, 0x65, 0x73, 0x2e, 0x65, 0x63, 0x68, 0x6f, 0x2e, 0x45, 0x63, 0x68, 0x6f, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x5c, 0x0a, 0x13, 0x43, 0x6c, 0x69, + 0x65, 0x6e, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x45, 0x63, 0x68, 0x6f, + 0x12, 0x1f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73, + 0x2e, 0x65, 0x63, 0x68, 0x6f, 0x2e, 0x45, 0x63, 0x68, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x20, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, + 0x73, 0x2e, 0x65, 0x63, 0x68, 0x6f, 0x2e, 0x45, 0x63, 0x68, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x28, 0x01, 0x12, 0x65, 0x0a, 0x1a, 0x42, 0x69, 0x64, 0x69, 0x72, + 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, + 0x67, 0x45, 0x63, 0x68, 0x6f, 0x12, 0x1f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x65, 0x78, 0x61, + 0x6d, 0x70, 0x6c, 0x65, 0x73, 0x2e, 0x65, 0x63, 0x68, 0x6f, 0x2e, 0x45, 0x63, 0x68, 0x6f, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x65, 0x78, + 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73, 0x2e, 0x65, 0x63, 0x68, 0x6f, 0x2e, 0x45, 0x63, 0x68, 0x6f, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x42, 0x35, + 0x5a, 0x33, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e, + 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, + 0x73, 0x2f, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x2f, 0x65, 0x63, 0x68, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } -var xxx_messageInfo_EchoResponse proto.InternalMessageInfo +var ( + file_examples_features_proto_echo_echo_proto_rawDescOnce sync.Once + file_examples_features_proto_echo_echo_proto_rawDescData = file_examples_features_proto_echo_echo_proto_rawDesc +) -func (m *EchoResponse) GetMessage() string { - if m != nil { - return m.Message - } - return "" +func file_examples_features_proto_echo_echo_proto_rawDescGZIP() []byte { + file_examples_features_proto_echo_echo_proto_rawDescOnce.Do(func() { + file_examples_features_proto_echo_echo_proto_rawDescData = protoimpl.X.CompressGZIP(file_examples_features_proto_echo_echo_proto_rawDescData) + }) + return file_examples_features_proto_echo_echo_proto_rawDescData +} + +var file_examples_features_proto_echo_echo_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_examples_features_proto_echo_echo_proto_goTypes = []interface{}{ + (*EchoRequest)(nil), // 0: grpc.examples.echo.EchoRequest + (*EchoResponse)(nil), // 1: grpc.examples.echo.EchoResponse +} +var file_examples_features_proto_echo_echo_proto_depIdxs = []int32{ + 0, // 0: grpc.examples.echo.Echo.UnaryEcho:input_type -> grpc.examples.echo.EchoRequest + 0, // 1: grpc.examples.echo.Echo.ServerStreamingEcho:input_type -> grpc.examples.echo.EchoRequest + 0, // 2: grpc.examples.echo.Echo.ClientStreamingEcho:input_type -> grpc.examples.echo.EchoRequest + 0, // 3: grpc.examples.echo.Echo.BidirectionalStreamingEcho:input_type -> grpc.examples.echo.EchoRequest + 1, // 4: grpc.examples.echo.Echo.UnaryEcho:output_type -> grpc.examples.echo.EchoResponse + 1, // 5: grpc.examples.echo.Echo.ServerStreamingEcho:output_type -> grpc.examples.echo.EchoResponse + 1, // 6: grpc.examples.echo.Echo.ClientStreamingEcho:output_type -> grpc.examples.echo.EchoResponse + 1, // 7: grpc.examples.echo.Echo.BidirectionalStreamingEcho:output_type -> grpc.examples.echo.EchoResponse + 4, // [4:8] is the sub-list for method output_type + 0, // [0:4] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name } -func init() { - proto.RegisterType((*EchoRequest)(nil), "grpc.examples.echo.EchoRequest") - proto.RegisterType((*EchoResponse)(nil), "grpc.examples.echo.EchoResponse") -} - -func init() { - proto.RegisterFile("examples/features/proto/echo/echo.proto", fileDescriptor_2fd1d686b7b805dc) -} - -var fileDescriptor_2fd1d686b7b805dc = []byte{ - // 236 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x92, 0xb1, 0x4b, 0x03, 0x31, - 0x14, 0xc6, 0x8d, 0x88, 0xd2, 0xe8, 0x14, 0x97, 0xd2, 0xc5, 0x72, 0x4b, 0x6f, 0x4a, 0x8a, 0xc5, - 0x7f, 0xa0, 0xe2, 0x2e, 0x2d, 0x2e, 0xe2, 0x12, 0xcf, 0xcf, 0x34, 0x90, 0xcb, 0x3b, 0x5f, 0x52, - 0xd1, 0xbf, 0xdd, 0x45, 0x72, 0x47, 0x41, 0x90, 0x3a, 0xd5, 0x25, 0xe4, 0xe3, 0xfd, 0xde, 0xf7, - 0x5b, 0x9e, 0x9c, 0xe1, 0xc3, 0xb6, 0x5d, 0x40, 0x32, 0xaf, 0xb0, 0x79, 0xcb, 0x48, 0xa6, 0x63, - 0xca, 0x64, 0xd0, 0x6c, 0x86, 0x47, 0xf7, 0x59, 0x29, 0xc7, 0x5d, 0xa3, 0x77, 0xb4, 0x2e, 0x93, - 0x6a, 0x26, 0xcf, 0xef, 0x9a, 0x0d, 0xad, 0xf0, 0xb6, 0x45, 0xca, 0x6a, 0x2c, 0xcf, 0x5a, 0xa4, - 0x64, 0x1d, 0xc6, 0x62, 0x2a, 0xea, 0xd1, 0x6a, 0x17, 0xab, 0x5a, 0x5e, 0x0c, 0x60, 0xea, 0x28, - 0x26, 0xec, 0x27, 0xaf, 0xbf, 0x8e, 0xe5, 0x49, 0x41, 0xd5, 0xbd, 0x1c, 0x3d, 0x44, 0xcb, 0x9f, - 0x7d, 0xb8, 0xd2, 0xbf, 0xed, 0xfa, 0x87, 0x7a, 0x32, 0xdd, 0x0f, 0x0c, 0xca, 0xea, 0x48, 0x3d, - 0xc9, 0xcb, 0x35, 0xf8, 0x1d, 0xbc, 0xce, 0x0c, 0xdb, 0xfa, 0xe8, 0x0e, 0xd6, 0x3d, 0x17, 0xa5, - 0xfd, 0x36, 0x78, 0xc4, 0x7c, 0xf8, 0xf6, 0x5a, 0x28, 0xc8, 0xc9, 0xd2, 0xbf, 0x78, 0x46, 0x93, - 0x3d, 0x45, 0x1b, 0xfe, 0x43, 0x32, 0x17, 0xcb, 0x9b, 0xc7, 0x85, 0x23, 0x72, 0x01, 0xda, 0x51, - 0xb0, 0xd1, 0x69, 0x62, 0x67, 0xca, 0xaa, 0xf9, 0xeb, 0x4c, 0x9e, 0x4f, 0xfb, 0xff, 0xe2, 0x3b, - 0x00, 0x00, 0xff, 0xff, 0xf7, 0x79, 0x87, 0xf0, 0x4d, 0x02, 0x00, 0x00, +func init() { file_examples_features_proto_echo_echo_proto_init() } +func file_examples_features_proto_echo_echo_proto_init() { + if File_examples_features_proto_echo_echo_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_examples_features_proto_echo_echo_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*EchoRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_examples_features_proto_echo_echo_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*EchoResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_examples_features_proto_echo_echo_proto_rawDesc, + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_examples_features_proto_echo_echo_proto_goTypes, + DependencyIndexes: file_examples_features_proto_echo_echo_proto_depIdxs, + MessageInfos: file_examples_features_proto_echo_echo_proto_msgTypes, + }.Build() + File_examples_features_proto_echo_echo_proto = out.File + file_examples_features_proto_echo_echo_proto_rawDesc = nil + file_examples_features_proto_echo_echo_proto_goTypes = nil + file_examples_features_proto_echo_echo_proto_depIdxs = nil } diff --git a/examples/features/proto/echo/echo_grpc.pb.go b/examples/features/proto/echo/echo_grpc.pb.go index a3597048577a..7efd51403fb9 100644 --- a/examples/features/proto/echo/echo_grpc.pb.go +++ b/examples/features/proto/echo/echo_grpc.pb.go @@ -1,4 +1,25 @@ +// +// +// Copyright 2018 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + // Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.3.0 +// - protoc v4.22.0 +// source: examples/features/proto/echo/echo.proto package echo @@ -11,8 +32,16 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. const _ = grpc.SupportPackageIsVersion7 +const ( + Echo_UnaryEcho_FullMethodName = "/grpc.examples.echo.Echo/UnaryEcho" + Echo_ServerStreamingEcho_FullMethodName = "/grpc.examples.echo.Echo/ServerStreamingEcho" + Echo_ClientStreamingEcho_FullMethodName = "/grpc.examples.echo.Echo/ClientStreamingEcho" + Echo_BidirectionalStreamingEcho_FullMethodName = "/grpc.examples.echo.Echo/BidirectionalStreamingEcho" +) + // EchoClient is the client API for Echo service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. @@ -35,26 +64,17 @@ func NewEchoClient(cc grpc.ClientConnInterface) EchoClient { return &echoClient{cc} } -var echoUnaryEchoStreamDesc = &grpc.StreamDesc{ - StreamName: "UnaryEcho", -} - func (c *echoClient) UnaryEcho(ctx context.Context, in *EchoRequest, opts ...grpc.CallOption) (*EchoResponse, error) { out := new(EchoResponse) - err := c.cc.Invoke(ctx, "/grpc.examples.echo.Echo/UnaryEcho", in, out, opts...) + err := c.cc.Invoke(ctx, Echo_UnaryEcho_FullMethodName, in, out, opts...) if err != nil { return nil, err } return out, nil } -var echoServerStreamingEchoStreamDesc = &grpc.StreamDesc{ - StreamName: "ServerStreamingEcho", - ServerStreams: true, -} - func (c *echoClient) ServerStreamingEcho(ctx context.Context, in *EchoRequest, opts ...grpc.CallOption) (Echo_ServerStreamingEchoClient, error) { - stream, err := c.cc.NewStream(ctx, echoServerStreamingEchoStreamDesc, "/grpc.examples.echo.Echo/ServerStreamingEcho", opts...) + stream, err := c.cc.NewStream(ctx, &Echo_ServiceDesc.Streams[0], Echo_ServerStreamingEcho_FullMethodName, opts...) if err != nil { return nil, err } @@ -85,13 +105,8 @@ func (x *echoServerStreamingEchoClient) Recv() (*EchoResponse, error) { return m, nil } -var echoClientStreamingEchoStreamDesc = &grpc.StreamDesc{ - StreamName: "ClientStreamingEcho", - ClientStreams: true, -} - func (c *echoClient) ClientStreamingEcho(ctx context.Context, opts ...grpc.CallOption) (Echo_ClientStreamingEchoClient, error) { - stream, err := c.cc.NewStream(ctx, echoClientStreamingEchoStreamDesc, "/grpc.examples.echo.Echo/ClientStreamingEcho", opts...) + stream, err := c.cc.NewStream(ctx, &Echo_ServiceDesc.Streams[1], Echo_ClientStreamingEcho_FullMethodName, opts...) if err != nil { return nil, err } @@ -124,14 +139,8 @@ func (x *echoClientStreamingEchoClient) CloseAndRecv() (*EchoResponse, error) { return m, nil } -var echoBidirectionalStreamingEchoStreamDesc = &grpc.StreamDesc{ - StreamName: "BidirectionalStreamingEcho", - ServerStreams: true, - ClientStreams: true, -} - func (c *echoClient) BidirectionalStreamingEcho(ctx context.Context, opts ...grpc.CallOption) (Echo_BidirectionalStreamingEchoClient, error) { - stream, err := c.cc.NewStream(ctx, echoBidirectionalStreamingEchoStreamDesc, "/grpc.examples.echo.Echo/BidirectionalStreamingEcho", opts...) + stream, err := c.cc.NewStream(ctx, &Echo_ServiceDesc.Streams[2], Echo_BidirectionalStreamingEcho_FullMethodName, opts...) if err != nil { return nil, err } @@ -161,62 +170,74 @@ func (x *echoBidirectionalStreamingEchoClient) Recv() (*EchoResponse, error) { return m, nil } -// EchoService is the service API for Echo service. -// Fields should be assigned to their respective handler implementations only before -// RegisterEchoService is called. Any unassigned fields will result in the -// handler for that method returning an Unimplemented error. -type EchoService struct { +// EchoServer is the server API for Echo service. +// All implementations must embed UnimplementedEchoServer +// for forward compatibility +type EchoServer interface { // UnaryEcho is unary echo. - UnaryEcho func(context.Context, *EchoRequest) (*EchoResponse, error) + UnaryEcho(context.Context, *EchoRequest) (*EchoResponse, error) // ServerStreamingEcho is server side streaming. - ServerStreamingEcho func(*EchoRequest, Echo_ServerStreamingEchoServer) error + ServerStreamingEcho(*EchoRequest, Echo_ServerStreamingEchoServer) error // ClientStreamingEcho is client side streaming. - ClientStreamingEcho func(Echo_ClientStreamingEchoServer) error + ClientStreamingEcho(Echo_ClientStreamingEchoServer) error // BidirectionalStreamingEcho is bidi streaming. - BidirectionalStreamingEcho func(Echo_BidirectionalStreamingEchoServer) error + BidirectionalStreamingEcho(Echo_BidirectionalStreamingEchoServer) error + mustEmbedUnimplementedEchoServer() } -func (s *EchoService) unaryEcho(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - if s.UnaryEcho == nil { - return nil, status.Errorf(codes.Unimplemented, "method UnaryEcho not implemented") - } +// UnimplementedEchoServer must be embedded to have forward compatible implementations. +type UnimplementedEchoServer struct { +} + +func (UnimplementedEchoServer) UnaryEcho(context.Context, *EchoRequest) (*EchoResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method UnaryEcho not implemented") +} +func (UnimplementedEchoServer) ServerStreamingEcho(*EchoRequest, Echo_ServerStreamingEchoServer) error { + return status.Errorf(codes.Unimplemented, "method ServerStreamingEcho not implemented") +} +func (UnimplementedEchoServer) ClientStreamingEcho(Echo_ClientStreamingEchoServer) error { + return status.Errorf(codes.Unimplemented, "method ClientStreamingEcho not implemented") +} +func (UnimplementedEchoServer) BidirectionalStreamingEcho(Echo_BidirectionalStreamingEchoServer) error { + return status.Errorf(codes.Unimplemented, "method BidirectionalStreamingEcho not implemented") +} +func (UnimplementedEchoServer) mustEmbedUnimplementedEchoServer() {} + +// UnsafeEchoServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to EchoServer will +// result in compilation errors. +type UnsafeEchoServer interface { + mustEmbedUnimplementedEchoServer() +} + +func RegisterEchoServer(s grpc.ServiceRegistrar, srv EchoServer) { + s.RegisterService(&Echo_ServiceDesc, srv) +} + +func _Echo_UnaryEcho_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(EchoRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { - return s.UnaryEcho(ctx, in) + return srv.(EchoServer).UnaryEcho(ctx, in) } info := &grpc.UnaryServerInfo{ - Server: s, - FullMethod: "/grpc.examples.echo.Echo/UnaryEcho", + Server: srv, + FullMethod: Echo_UnaryEcho_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return s.UnaryEcho(ctx, req.(*EchoRequest)) + return srv.(EchoServer).UnaryEcho(ctx, req.(*EchoRequest)) } return interceptor(ctx, in, info, handler) } -func (s *EchoService) serverStreamingEcho(_ interface{}, stream grpc.ServerStream) error { - if s.ServerStreamingEcho == nil { - return status.Errorf(codes.Unimplemented, "method ServerStreamingEcho not implemented") - } + +func _Echo_ServerStreamingEcho_Handler(srv interface{}, stream grpc.ServerStream) error { m := new(EchoRequest) if err := stream.RecvMsg(m); err != nil { return err } - return s.ServerStreamingEcho(m, &echoServerStreamingEchoServer{stream}) -} -func (s *EchoService) clientStreamingEcho(_ interface{}, stream grpc.ServerStream) error { - if s.ClientStreamingEcho == nil { - return status.Errorf(codes.Unimplemented, "method ClientStreamingEcho not implemented") - } - return s.ClientStreamingEcho(&echoClientStreamingEchoServer{stream}) -} -func (s *EchoService) bidirectionalStreamingEcho(_ interface{}, stream grpc.ServerStream) error { - if s.BidirectionalStreamingEcho == nil { - return status.Errorf(codes.Unimplemented, "method BidirectionalStreamingEcho not implemented") - } - return s.BidirectionalStreamingEcho(&echoBidirectionalStreamingEchoServer{stream}) + return srv.(EchoServer).ServerStreamingEcho(m, &echoServerStreamingEchoServer{stream}) } type Echo_ServerStreamingEchoServer interface { @@ -232,6 +253,10 @@ func (x *echoServerStreamingEchoServer) Send(m *EchoResponse) error { return x.ServerStream.SendMsg(m) } +func _Echo_ClientStreamingEcho_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(EchoServer).ClientStreamingEcho(&echoClientStreamingEchoServer{stream}) +} + type Echo_ClientStreamingEchoServer interface { SendAndClose(*EchoResponse) error Recv() (*EchoRequest, error) @@ -254,6 +279,10 @@ func (x *echoClientStreamingEchoServer) Recv() (*EchoRequest, error) { return m, nil } +func _Echo_BidirectionalStreamingEcho_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(EchoServer).BidirectionalStreamingEcho(&echoBidirectionalStreamingEchoServer{stream}) +} + type Echo_BidirectionalStreamingEchoServer interface { Send(*EchoResponse) error Recv() (*EchoRequest, error) @@ -276,82 +305,35 @@ func (x *echoBidirectionalStreamingEchoServer) Recv() (*EchoRequest, error) { return m, nil } -// RegisterEchoService registers a service implementation with a gRPC server. -func RegisterEchoService(s grpc.ServiceRegistrar, srv *EchoService) { - sd := grpc.ServiceDesc{ - ServiceName: "grpc.examples.echo.Echo", - Methods: []grpc.MethodDesc{ - { - MethodName: "UnaryEcho", - Handler: srv.unaryEcho, - }, +// Echo_ServiceDesc is the grpc.ServiceDesc for Echo service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var Echo_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.examples.echo.Echo", + HandlerType: (*EchoServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "UnaryEcho", + Handler: _Echo_UnaryEcho_Handler, }, - Streams: []grpc.StreamDesc{ - { - StreamName: "ServerStreamingEcho", - Handler: srv.serverStreamingEcho, - ServerStreams: true, - }, - { - StreamName: "ClientStreamingEcho", - Handler: srv.clientStreamingEcho, - ClientStreams: true, - }, - { - StreamName: "BidirectionalStreamingEcho", - Handler: srv.bidirectionalStreamingEcho, - ServerStreams: true, - ClientStreams: true, - }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "ServerStreamingEcho", + Handler: _Echo_ServerStreamingEcho_Handler, + ServerStreams: true, }, - Metadata: "examples/features/proto/echo/echo.proto", - } - - s.RegisterService(&sd, nil) -} - -// NewEchoService creates a new EchoService containing the -// implemented methods of the Echo service in s. Any unimplemented -// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client. -// This includes situations where the method handler is misspelled or has the wrong -// signature. For this reason, this function should be used with great care and -// is not recommended to be used by most users. -func NewEchoService(s interface{}) *EchoService { - ns := &EchoService{} - if h, ok := s.(interface { - UnaryEcho(context.Context, *EchoRequest) (*EchoResponse, error) - }); ok { - ns.UnaryEcho = h.UnaryEcho - } - if h, ok := s.(interface { - ServerStreamingEcho(*EchoRequest, Echo_ServerStreamingEchoServer) error - }); ok { - ns.ServerStreamingEcho = h.ServerStreamingEcho - } - if h, ok := s.(interface { - ClientStreamingEcho(Echo_ClientStreamingEchoServer) error - }); ok { - ns.ClientStreamingEcho = h.ClientStreamingEcho - } - if h, ok := s.(interface { - BidirectionalStreamingEcho(Echo_BidirectionalStreamingEchoServer) error - }); ok { - ns.BidirectionalStreamingEcho = h.BidirectionalStreamingEcho - } - return ns -} - -// UnstableEchoService is the service API for Echo service. -// New methods may be added to this interface if they are added to the service -// definition, which is not a backward-compatible change. For this reason, -// use of this type is not recommended. -type UnstableEchoService interface { - // UnaryEcho is unary echo. - UnaryEcho(context.Context, *EchoRequest) (*EchoResponse, error) - // ServerStreamingEcho is server side streaming. - ServerStreamingEcho(*EchoRequest, Echo_ServerStreamingEchoServer) error - // ClientStreamingEcho is client side streaming. - ClientStreamingEcho(Echo_ClientStreamingEchoServer) error - // BidirectionalStreamingEcho is bidi streaming. - BidirectionalStreamingEcho(Echo_BidirectionalStreamingEchoServer) error + { + StreamName: "ClientStreamingEcho", + Handler: _Echo_ClientStreamingEcho_Handler, + ClientStreams: true, + }, + { + StreamName: "BidirectionalStreamingEcho", + Handler: _Echo_BidirectionalStreamingEcho_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "examples/features/proto/echo/echo.proto", } diff --git a/examples/features/reflection/server/main.go b/examples/features/reflection/server/main.go index 1501c01f33aa..569273dfdd3e 100644 --- a/examples/features/reflection/server/main.go +++ b/examples/features/reflection/server/main.go @@ -35,13 +35,21 @@ import ( var port = flag.Int("port", 50051, "the port to serve on") -// sayHello implements helloworld.GreeterServer.SayHello -func sayHello(ctx context.Context, in *hwpb.HelloRequest) (*hwpb.HelloReply, error) { +// hwServer is used to implement helloworld.GreeterServer. +type hwServer struct { + hwpb.UnimplementedGreeterServer +} + +// SayHello implements helloworld.GreeterServer +func (s *hwServer) SayHello(ctx context.Context, in *hwpb.HelloRequest) (*hwpb.HelloReply, error) { return &hwpb.HelloReply{Message: "Hello " + in.Name}, nil } -// unaryEcho implements echo.Echo.UnaryEcho -func unaryEcho(ctx context.Context, req *ecpb.EchoRequest) (*ecpb.EchoResponse, error) { +type ecServer struct { + ecpb.UnimplementedEchoServer +} + +func (s *ecServer) UnaryEcho(ctx context.Context, req *ecpb.EchoRequest) (*ecpb.EchoResponse, error) { return &ecpb.EchoResponse{Message: req.Message}, nil } @@ -56,10 +64,10 @@ func main() { s := grpc.NewServer() // Register Greeter on the server. - hwpb.RegisterGreeterService(s, &hwpb.GreeterService{SayHello: sayHello}) + hwpb.RegisterGreeterServer(s, &hwServer{}) // Register RouteGuide on the same server. - ecpb.RegisterEchoService(s, &ecpb.EchoService{UnaryEcho: unaryEcho}) + ecpb.RegisterEchoServer(s, &ecServer{}) // Register reflection service on gRPC server. reflection.Register(s) diff --git a/examples/features/retry/README.md b/examples/features/retry/README.md index f56d438adc2b..e39a1c71704d 100644 --- a/examples/features/retry/README.md +++ b/examples/features/retry/README.md @@ -18,11 +18,10 @@ First start the server: go run server/main.go ``` -Then run the client. Note that when running the client, `GRPC_GO_RETRY=on` must be set in -your environment: +Then run the client: ```bash -GRPC_GO_RETRY=on go run client/main.go +go run client/main.go ``` ## Usage @@ -62,5 +61,5 @@ To use the above service config, pass it with `grpc.WithDefaultServiceConfig` to `grpc.Dial`. ```go -conn, err := grpc.Dial(ctx,grpc.WithInsecure(), grpc.WithDefaultServiceConfig(retryPolicy)) +conn, err := grpc.Dial(ctx,grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultServiceConfig(retryPolicy)) ``` diff --git a/examples/features/retry/client/main.go b/examples/features/retry/client/main.go index 73147cfe0a27..3b9b80e24ba7 100644 --- a/examples/features/retry/client/main.go +++ b/examples/features/retry/client/main.go @@ -26,6 +26,7 @@ import ( "time" "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" pb "google.golang.org/grpc/examples/features/proto/echo" ) @@ -48,7 +49,7 @@ var ( // use grpc.WithDefaultServiceConfig() to set service config func retryDial() (*grpc.ClientConn, error) { - return grpc.Dial(*addr, grpc.WithInsecure(), grpc.WithDefaultServiceConfig(retryPolicy)) + return grpc.Dial(*addr, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultServiceConfig(retryPolicy)) } func main() { diff --git a/examples/features/retry/server/main.go b/examples/features/retry/server/main.go index 08dd40c93f1c..fdec2f052e27 100644 --- a/examples/features/retry/server/main.go +++ b/examples/features/retry/server/main.go @@ -37,6 +37,7 @@ import ( var port = flag.Int("port", 50052, "port number") type failingServer struct { + pb.UnimplementedEchoServer mu sync.Mutex reqCounter uint @@ -85,7 +86,7 @@ func main() { reqModulo: 4, } - pb.RegisterEchoService(s, &pb.EchoService{UnaryEcho: failingservice.UnaryEcho}) + pb.RegisterEchoServer(s, failingservice) if err := s.Serve(lis); err != nil { log.Fatalf("failed to serve: %v", err) } diff --git a/examples/features/stats_monitoring/README.md b/examples/features/stats_monitoring/README.md new file mode 100644 index 000000000000..079b6b4f1ee6 --- /dev/null +++ b/examples/features/stats_monitoring/README.md @@ -0,0 +1,58 @@ +# Stats Monitoring Handler + +This example demonstrates the use of the [`stats`](https://pkg.go.dev/google.golang.org/grpc/stats) package for reporting various +network and RPC stats. +_Note that all fields are READ-ONLY and the APIs of the `stats` package are +experimental_. + +## Try it + +``` +go run server/main.go +``` + +``` +go run client/main.go +``` + +## Explanation + +gRPC provides a mechanism to hook on to various events (phases) of the +request-response network cycle through the [`stats.Handler`](https://pkg.go.dev/google.golang.org/grpc/stats#Handler) interface. To access +these events, a concrete type that implements `stats.Handler` should be passed to +`grpc.WithStatsHandler()` on the client side and `grpc.StatsHandler()` on the +server side. + +The `HandleRPC(context.Context, RPCStats)` method on `stats.Handler` is called +multiple times during a request-response cycle, and various event stats are +passed to its `RPCStats` parameter (an interface). The concrete types that +implement this interface are: `*stats.Begin`, `*stats.InHeader`, `*stats.InPayload`, +`*stats.InTrailer`, `*stats.OutHeader`, `*stats.OutPayload`, `*stats.OutTrailer`, and +`*stats.End`. The order of these events differs on client and server. + +Similarly, the `HandleConn(context.Context, ConnStats)` method on `stats.Handler` +is called twice, once at the beginning of the connection with `*stats.ConnBegin` +and once at the end with `*stats.ConnEnd`. + +The [`stats.Handler`](https://pkg.go.dev/google.golang.org/grpc/stats#Handler) interface also provides +`TagRPC(context.Context, *RPCTagInfo) context.Context` and +`TagConn(context.Context, *ConnTagInfo) context.Context` methods. These methods +are mainly used to attach network related information to the given context. + +The `TagRPC(context.Context, *RPCTagInfo) context.Context` method returns a +context from which the context used for the rest lifetime of the RPC will be +derived. This behavior is consistent between the gRPC client and server. + +The context returned from +`TagConn(context.Context, *ConnTagInfo) context.Context` has varied lifespan: + +- In the gRPC client: + The context used for the rest lifetime of the RPC will NOT be derived from + this context. Hence the information attached to this context can only be + consumed by `HandleConn(context.Context, ConnStats)` method. +- In the gRPC server: + The context used for the rest lifetime of the RPC will be derived from + this context. + +NOTE: The [stats](https://pkg.go.dev/google.golang.org/grpc/stats) package should only be used for network monitoring purposes, +and not as an alternative to [interceptors](https://github.com/grpc/grpc-go/blob/master/examples/features/metadata). diff --git a/examples/features/stats_monitoring/client/main.go b/examples/features/stats_monitoring/client/main.go new file mode 100644 index 000000000000..0fb820d11c63 --- /dev/null +++ b/examples/features/stats_monitoring/client/main.go @@ -0,0 +1,60 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary client is an example client to illustrate the use of the stats handler. +package main + +import ( + "context" + "flag" + "log" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + + echogrpc "google.golang.org/grpc/examples/features/proto/echo" + echopb "google.golang.org/grpc/examples/features/proto/echo" + "google.golang.org/grpc/examples/features/stats_monitoring/statshandler" +) + +var addr = flag.String("addr", "localhost:50051", "the address to connect to") + +func main() { + flag.Parse() + opts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithStatsHandler(statshandler.New()), + } + conn, err := grpc.Dial(*addr, opts...) + if err != nil { + log.Fatalf("failed to connect to server %q: %v", *addr, err) + } + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + c := echogrpc.NewEchoClient(conn) + + resp, err := c.UnaryEcho(ctx, &echopb.EchoRequest{Message: "stats handler demo"}) + if err != nil { + log.Fatalf("unexpected error from UnaryEcho: %v", err) + } + log.Printf("RPC response: %s", resp.Message) +} diff --git a/examples/features/stats_monitoring/server/main.go b/examples/features/stats_monitoring/server/main.go new file mode 100644 index 000000000000..a460522c29db --- /dev/null +++ b/examples/features/stats_monitoring/server/main.go @@ -0,0 +1,62 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary server is an example server to illustrate the use of the stats handler. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "net" + "time" + + "google.golang.org/grpc" + + echogrpc "google.golang.org/grpc/examples/features/proto/echo" + echopb "google.golang.org/grpc/examples/features/proto/echo" + "google.golang.org/grpc/examples/features/stats_monitoring/statshandler" +) + +var port = flag.Int("port", 50051, "the port to serve on") + +type server struct { + echogrpc.UnimplementedEchoServer +} + +func (s *server) UnaryEcho(ctx context.Context, req *echopb.EchoRequest) (*echopb.EchoResponse, error) { + time.Sleep(2 * time.Second) + return &echopb.EchoResponse{Message: req.Message}, nil +} + +func main() { + flag.Parse() + lis, err := net.Listen("tcp", fmt.Sprintf(":%d", *port)) + if err != nil { + log.Fatalf("failed to listen on port %d: %v", *port, err) + } + log.Printf("server listening at %v\n", lis.Addr()) + + s := grpc.NewServer(grpc.StatsHandler(statshandler.New())) + echogrpc.RegisterEchoServer(s, &server{}) + + if err := s.Serve(lis); err != nil { + log.Fatalf("failed to serve: %v", err) + } +} diff --git a/examples/features/stats_monitoring/statshandler/handler.go b/examples/features/stats_monitoring/statshandler/handler.go new file mode 100644 index 000000000000..85688b8c3856 --- /dev/null +++ b/examples/features/stats_monitoring/statshandler/handler.go @@ -0,0 +1,93 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package statshandler is an example pkg to illustrate the use of the stats handler. +package statshandler + +import ( + "context" + "log" + "net" + "path/filepath" + + "google.golang.org/grpc/stats" +) + +// Handler implements [stats.Handler](https://pkg.go.dev/google.golang.org/grpc/stats#Handler) interface. +type Handler struct{} + +type connStatCtxKey struct{} + +// TagConn can attach some information to the given context. +// The context used in HandleConn for this connection will be derived from the context returned. +// In the gRPC client: +// The context used in HandleRPC for RPCs on this connection will be the user's context and NOT derived from the context returned here. +// In the gRPC server: +// The context used in HandleRPC for RPCs on this connection will be derived from the context returned here. +func (st *Handler) TagConn(ctx context.Context, stat *stats.ConnTagInfo) context.Context { + log.Printf("[TagConn] [%T]: %+[1]v", stat) + return context.WithValue(ctx, connStatCtxKey{}, stat) +} + +// HandleConn processes the Conn stats. +func (st *Handler) HandleConn(ctx context.Context, stat stats.ConnStats) { + var rAddr net.Addr + if s, ok := ctx.Value(connStatCtxKey{}).(*stats.ConnTagInfo); ok { + rAddr = s.RemoteAddr + } + + if stat.IsClient() { + log.Printf("[server addr: %s] [HandleConn] [%T]: %+[2]v", rAddr, stat) + } else { + log.Printf("[client addr: %s] [HandleConn] [%T]: %+[2]v", rAddr, stat) + } +} + +type rpcStatCtxKey struct{} + +// TagRPC can attach some information to the given context. +// The context used for the rest lifetime of the RPC will be derived from the returned context. +func (st *Handler) TagRPC(ctx context.Context, stat *stats.RPCTagInfo) context.Context { + log.Printf("[TagRPC] [%T]: %+[1]v", stat) + return context.WithValue(ctx, rpcStatCtxKey{}, stat) +} + +// HandleRPC processes the RPC stats. Note: All stat fields are read-only. +func (st *Handler) HandleRPC(ctx context.Context, stat stats.RPCStats) { + var sMethod string + if s, ok := ctx.Value(rpcStatCtxKey{}).(*stats.RPCTagInfo); ok { + sMethod = filepath.Base(s.FullMethodName) + } + + var cAddr net.Addr + // for gRPC clients, key connStatCtxKey{} will not be present in HandleRPC's context. + if s, ok := ctx.Value(connStatCtxKey{}).(*stats.ConnTagInfo); ok { + cAddr = s.RemoteAddr + } + + if stat.IsClient() { + log.Printf("[server method: %s] [HandleRPC] [%T]: %+[2]v", sMethod, stat) + } else { + log.Printf("[client addr: %s] [HandleRPC] [%T]: %+[2]v", cAddr, stat) + } +} + +// New returns a new implementation of [stats.Handler](https://pkg.go.dev/google.golang.org/grpc/stats#Handler) interface. +func New() *Handler { + return &Handler{} +} diff --git a/examples/features/unix_abstract/README.md b/examples/features/unix_abstract/README.md new file mode 100644 index 000000000000..32b3bd5f262c --- /dev/null +++ b/examples/features/unix_abstract/README.md @@ -0,0 +1,29 @@ +# Unix abstract sockets + +This examples shows how to start a gRPC server listening on a unix abstract +socket and how to get a gRPC client to connect to it. + +## What is a unix abstract socket + +An abstract socket address is distinguished from a regular unix socket by the +fact that the first byte of the address is a null byte ('\0'). The address has +no connection with filesystem path names. + +## Try it + +``` +go run server/main.go +``` + +``` +go run client/main.go +``` + +## Explanation + +The gRPC server in this example listens on an address starting with a null byte +and the network is `unix`. The client uses the `unix-abstract` scheme with the +endpoint set to the abstract unix socket address without the null byte. The +`unix` resolver takes care of adding the null byte on the client. See +https://github.com/grpc/grpc/blob/master/doc/naming.md for the more details. + diff --git a/examples/features/unix_abstract/client/main.go b/examples/features/unix_abstract/client/main.go new file mode 100644 index 000000000000..3564e7e82fee --- /dev/null +++ b/examples/features/unix_abstract/client/main.go @@ -0,0 +1,75 @@ +//go:build linux +// +build linux + +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary client is an example client which dials a server on an abstract unix +// socket. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + ecpb "google.golang.org/grpc/examples/features/proto/echo" +) + +var ( + // A dial target of `unix:@abstract-unix-socket` should also work fine for + // this example because of golang conventions (net.Dial behavior). But we do + // not recommend this since we explicitly added the `unix-abstract` scheme + // for cross-language compatibility. + addr = flag.String("addr", "abstract-unix-socket", "The unix abstract socket address") +) + +func callUnaryEcho(c ecpb.EchoClient, message string) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + r, err := c.UnaryEcho(ctx, &ecpb.EchoRequest{Message: message}) + if err != nil { + log.Fatalf("could not greet: %v", err) + } + fmt.Println(r.Message) +} + +func makeRPCs(cc *grpc.ClientConn, n int) { + hwc := ecpb.NewEchoClient(cc) + for i := 0; i < n; i++ { + callUnaryEcho(hwc, "this is examples/unix_abstract") + } +} + +func main() { + flag.Parse() + sockAddr := fmt.Sprintf("unix-abstract:%v", *addr) + cc, err := grpc.Dial(sockAddr, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + log.Fatalf("grpc.Dial(%q) failed: %v", sockAddr, err) + } + defer cc.Close() + + fmt.Printf("--- calling echo.Echo/UnaryEcho to %s\n", sockAddr) + makeRPCs(cc, 10) + fmt.Println() +} diff --git a/examples/features/unix_abstract/server/main.go b/examples/features/unix_abstract/server/main.go new file mode 100644 index 000000000000..7013466b4917 --- /dev/null +++ b/examples/features/unix_abstract/server/main.go @@ -0,0 +1,65 @@ +//go:build linux +// +build linux + +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary server is an example server listening for gRPC connections on an +// abstract unix socket. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "net" + + "google.golang.org/grpc" + + pb "google.golang.org/grpc/examples/features/proto/echo" +) + +var ( + addr = flag.String("addr", "abstract-unix-socket", "The unix abstract socket address") +) + +type ecServer struct { + pb.UnimplementedEchoServer + addr string +} + +func (s *ecServer) UnaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) { + return &pb.EchoResponse{Message: fmt.Sprintf("%s (from %s)", req.Message, s.addr)}, nil +} + +func main() { + flag.Parse() + netw := "unix" + socketAddr := fmt.Sprintf("@%v", *addr) + lis, err := net.Listen(netw, socketAddr) + if err != nil { + log.Fatalf("net.Listen(%q, %q) failed: %v", netw, socketAddr, err) + } + s := grpc.NewServer() + pb.RegisterEchoServer(s, &ecServer{addr: socketAddr}) + log.Printf("serving on %s\n", lis.Addr().String()) + if err := s.Serve(lis); err != nil { + log.Fatalf("failed to serve: %v", err) + } +} diff --git a/examples/features/wait_for_ready/main.go b/examples/features/wait_for_ready/main.go index f3669e986576..96524a81da32 100644 --- a/examples/features/wait_for_ready/main.go +++ b/examples/features/wait_for_ready/main.go @@ -29,12 +29,18 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/status" pb "google.golang.org/grpc/examples/features/proto/echo" ) -func unaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) { +// server is used to implement EchoServer. +type server struct { + pb.UnimplementedEchoServer +} + +func (s *server) UnaryEcho(ctx context.Context, req *pb.EchoRequest) (*pb.EchoResponse, error) { return &pb.EchoResponse{Message: req.Message}, nil } @@ -45,7 +51,7 @@ func serve() { log.Fatalf("failed to listen: %v", err) } s := grpc.NewServer() - pb.RegisterEchoService(s, &pb.EchoService{UnaryEcho: unaryEcho}) + pb.RegisterEchoServer(s, &server{}) if err := s.Serve(lis); err != nil { log.Fatalf("failed to serve: %v", err) @@ -53,7 +59,7 @@ func serve() { } func main() { - conn, err := grpc.Dial("localhost:50053", grpc.WithInsecure()) + conn, err := grpc.Dial("localhost:50053", grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { log.Fatalf("did not connect: %v", err) } diff --git a/examples/features/xds/client/main.go b/examples/features/xds/client/main.go index b1daa1cae9c8..97918faa2245 100644 --- a/examples/features/xds/client/main.go +++ b/examples/features/xds/client/main.go @@ -16,78 +16,56 @@ * */ -// Package main implements a client for Greeter service. +// Binary main implements a client for Greeter service using gRPC's client-side +// support for xDS APIs. package main import ( "context" "flag" - "fmt" "log" + "strings" "time" "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + xdscreds "google.golang.org/grpc/credentials/xds" pb "google.golang.org/grpc/examples/helloworld/helloworld" _ "google.golang.org/grpc/xds" // To install the xds resolvers and balancers. ) -const ( - defaultTarget = "localhost:50051" - defaultName = "world" +var ( + target = flag.String("target", "xds:///localhost:50051", "uri of the Greeter Server, e.g. 'xds:///helloworld-service:8080'") + name = flag.String("name", "world", "name you wished to be greeted by the server") + xdsCreds = flag.Bool("xds_creds", false, "whether the server should use xDS APIs to receive security configuration") ) -var help = flag.Bool("help", false, "Print usage information") - -func init() { - flag.Usage = func() { - fmt.Fprintf(flag.CommandLine.Output(), ` -Usage: client [name [target]] - - name - The name you wish to be greeted by. Defaults to %q - target - The URI of the server, e.g. "xds:///helloworld-service". Defaults to %q -`, defaultName, defaultTarget) - - flag.PrintDefaults() - } -} - func main() { flag.Parse() - if *help { - flag.Usage() - return - } - args := flag.Args() - - if len(args) > 2 { - flag.Usage() - return - } - name := defaultName - if len(args) > 0 { - name = args[0] + if !strings.HasPrefix(*target, "xds:///") { + log.Fatalf("-target must use a URI with scheme set to 'xds'") } - target := defaultTarget - if len(args) > 1 { - target = args[1] + creds := insecure.NewCredentials() + if *xdsCreds { + log.Println("Using xDS credentials...") + var err error + if creds, err = xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()}); err != nil { + log.Fatalf("failed to create client-side xDS credentials: %v", err) + } } - - // Set up a connection to the server. - conn, err := grpc.Dial(target, grpc.WithInsecure()) + conn, err := grpc.Dial(*target, grpc.WithTransportCredentials(creds)) if err != nil { - log.Fatalf("did not connect: %v", err) + log.Fatalf("grpc.Dial(%s) failed: %v", *target, err) } defer conn.Close() - c := pb.NewGreeterClient(conn) ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - r, err := c.SayHello(ctx, &pb.HelloRequest{Name: name}) + c := pb.NewGreeterClient(conn) + r, err := c.SayHello(ctx, &pb.HelloRequest{Name: *name}) if err != nil { log.Fatalf("could not greet: %v", err) } diff --git a/examples/features/xds/server/main.go b/examples/features/xds/server/main.go index 96cd6e468cf5..83ddac13264f 100644 --- a/examples/features/xds/server/main.go +++ b/examples/features/xds/server/main.go @@ -16,7 +16,8 @@ * */ -// Package main starts Greeter service that will response with the hostname. +// Binary server demonstrated gRPC's support for xDS APIs on the server-side. It +// exposes the Greeter service that will response with the hostname. package main import ( @@ -27,34 +28,30 @@ import ( "math/rand" "net" "os" - "strconv" "time" "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + xdscreds "google.golang.org/grpc/credentials/xds" pb "google.golang.org/grpc/examples/helloworld/helloworld" "google.golang.org/grpc/health" + healthgrpc "google.golang.org/grpc/health/grpc_health_v1" healthpb "google.golang.org/grpc/health/grpc_health_v1" - "google.golang.org/grpc/reflection" + "google.golang.org/grpc/xds" ) -var help = flag.Bool("help", false, "Print usage information") - -const ( - defaultPort = 50051 +var ( + port = flag.Int("port", 50051, "the port to serve Greeter service requests on. Health service will be served on `port+1`") + xdsCreds = flag.Bool("xds_creds", false, "whether the server should use xDS APIs to receive security configuration") ) -// server is used to implement helloworld.GreeterServer. +// server implements helloworld.GreeterServer interface. type server struct { + pb.UnimplementedGreeterServer serverName string } -func newServer(serverName string) *server { - return &server{ - serverName: serverName, - } -} - -// SayHello implements helloworld.GreeterServer +// SayHello implements helloworld.GreeterServer interface. func (s *server) SayHello(ctx context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) { log.Printf("Received: %v", in.GetName()) return &pb.HelloReply{Message: "Hello " + in.GetName() + ", from " + s.serverName}, nil @@ -70,66 +67,43 @@ func determineHostname() string { return hostname } -func init() { - flag.Usage = func() { - fmt.Fprintf(flag.CommandLine.Output(), ` -Usage: server [port [hostname]] - - port - The listen port. Defaults to %d - hostname - The name clients will see in greet responses. Defaults to the machine's hostname -`, defaultPort) - - flag.PrintDefaults() - } -} - func main() { flag.Parse() - if *help { - flag.Usage() - return - } - args := flag.Args() - if len(args) > 2 { - flag.Usage() - return + greeterPort := fmt.Sprintf(":%d", *port) + greeterLis, err := net.Listen("tcp4", greeterPort) + if err != nil { + log.Fatalf("net.Listen(tcp4, %q) failed: %v", greeterPort, err) } - port := defaultPort - if len(args) > 0 { + creds := insecure.NewCredentials() + if *xdsCreds { + log.Println("Using xDS credentials...") var err error - port, err = strconv.Atoi(args[0]) - if err != nil { - log.Printf("Invalid port number: %v", err) - flag.Usage() - return + if creds, err = xdscreds.NewServerCredentials(xdscreds.ServerOptions{FallbackCreds: insecure.NewCredentials()}); err != nil { + log.Fatalf("failed to create server-side xDS credentials: %v", err) } } - var hostname string - if len(args) > 1 { - hostname = args[1] - } - if hostname == "" { - hostname = determineHostname() + greeterServer, err := xds.NewGRPCServer(grpc.Creds(creds)) + if err != nil { + log.Fatalf("Failed to create an xDS enabled gRPC server: %v", err) } + pb.RegisterGreeterServer(greeterServer, &server{serverName: determineHostname()}) - lis, err := net.Listen("tcp", fmt.Sprintf("localhost:%d", port)) + healthPort := fmt.Sprintf(":%d", *port+1) + healthLis, err := net.Listen("tcp4", healthPort) if err != nil { - log.Fatalf("failed to listen: %v", err) + log.Fatalf("net.Listen(tcp4, %q) failed: %v", healthPort, err) } - s := grpc.NewServer() - hw := newServer(hostname) - pb.RegisterGreeterService(s, &pb.GreeterService{SayHello: hw.SayHello}) - - reflection.Register(s) + grpcServer := grpc.NewServer() healthServer := health.NewServer() healthServer.SetServingStatus("", healthpb.HealthCheckResponse_SERVING) - healthpb.RegisterHealthServer(s, healthServer) + healthgrpc.RegisterHealthServer(grpcServer, healthServer) - log.Printf("serving on %s, hostname %s", lis.Addr(), hostname) - s.Serve(lis) + log.Printf("Serving GreeterService on %s and HealthService on %s", greeterLis.Addr().String(), healthLis.Addr().String()) + go func() { + greeterServer.Serve(greeterLis) + }() + grpcServer.Serve(healthLis) } diff --git a/examples/go.mod b/examples/go.mod index d4e2b7c34a0f..e4b2103e0b34 100644 --- a/examples/go.mod +++ b/examples/go.mod @@ -1,13 +1,31 @@ module google.golang.org/grpc/examples -go 1.11 +go 1.19 require ( - cloud.google.com/go v0.63.0 // indirect - github.com/golang/protobuf v1.4.2 - golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d - google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98 - google.golang.org/grpc v1.31.0 + github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 + github.com/golang/protobuf v1.5.3 + golang.org/x/oauth2 v0.10.0 + google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 + google.golang.org/grpc v1.56.2 + google.golang.org/protobuf v1.31.0 +) + +require ( + cloud.google.com/go/compute v1.21.0 // indirect + cloud.google.com/go/compute/metadata v0.2.3 // indirect + github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe // indirect + github.com/envoyproxy/go-control-plane v0.11.1 // indirect + github.com/envoyproxy/protoc-gen-validate v1.0.2 // indirect + golang.org/x/net v0.12.0 // indirect + golang.org/x/sync v0.3.0 // indirect + golang.org/x/sys v0.10.0 // indirect + golang.org/x/text v0.11.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 // indirect ) replace google.golang.org/grpc => ../ diff --git a/examples/go.sum b/examples/go.sum index 1ef62013c1a0..85c3e3a46bfe 100644 --- a/examples/go.sum +++ b/examples/go.sum @@ -2,6 +2,7 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= @@ -11,52 +12,799 @@ cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bP cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.63.0 h1:A+DfAZQ/eWca7gvu42CS6FNSDX4R8cghF+XfWLn4R6g= -cloud.google.com/go v0.63.0/go.mod h1:GmezbQc7T2snqkEXWfZ0sy0VfkB/ivI2DdtJL2DEmlg= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= +cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= +cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= +cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= +cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= +cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= +cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= +cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= +cloud.google.com/go v0.100.1/go.mod h1:fs4QogzfH5n2pBXBP9vRiU+eCny7lD2vmFZy79Iuw1U= +cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= +cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc= +cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34hIU= +cloud.google.com/go v0.104.0/go.mod h1:OO6xxXdJyvuJPcEPBLN9BJPD+jep5G1+2U5B5gkRYtA= +cloud.google.com/go v0.105.0/go.mod h1:PrLgOJNe5nfE9UMxKxgXj4mD3voiP+YQ6gdt6KMFOKM= +cloud.google.com/go v0.107.0/go.mod h1:wpc2eNrD7hXUTy8EKS10jkxpZBjASrORK7goS+3YX2I= +cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= +cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVquxiw= +cloud.google.com/go v0.110.4/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= +cloud.google.com/go/accessapproval v1.4.0/go.mod h1:zybIuC3KpDOvotz59lFe5qxRZx6C75OtwbisN56xYB4= +cloud.google.com/go/accessapproval v1.5.0/go.mod h1:HFy3tuiGvMdcd/u+Cu5b9NkO1pEICJ46IR82PoUdplw= +cloud.google.com/go/accessapproval v1.6.0/go.mod h1:R0EiYnwV5fsRFiKZkPHr6mwyk2wxUJ30nL4j2pcFY2E= +cloud.google.com/go/accessapproval v1.7.1/go.mod h1:JYczztsHRMK7NTXb6Xw+dwbs/WnOJxbo/2mTI+Kgg68= +cloud.google.com/go/accesscontextmanager v1.3.0/go.mod h1:TgCBehyr5gNMz7ZaH9xubp+CE8dkrszb4oK9CWyvD4o= +cloud.google.com/go/accesscontextmanager v1.4.0/go.mod h1:/Kjh7BBu/Gh83sv+K60vN9QE5NJcd80sU33vIe2IFPE= +cloud.google.com/go/accesscontextmanager v1.6.0/go.mod h1:8XCvZWfYw3K/ji0iVnp+6pu7huxoQTLmxAbVjbloTtM= +cloud.google.com/go/accesscontextmanager v1.7.0/go.mod h1:CEGLewx8dwa33aDAZQujl7Dx+uYhS0eay198wB/VumQ= +cloud.google.com/go/accesscontextmanager v1.8.0/go.mod h1:uI+AI/r1oyWK99NN8cQ3UK76AMelMzgZCvJfsi2c+ps= +cloud.google.com/go/accesscontextmanager v1.8.1/go.mod h1:JFJHfvuaTC+++1iL1coPiG1eu5D24db2wXCDWDjIrxo= +cloud.google.com/go/aiplatform v1.22.0/go.mod h1:ig5Nct50bZlzV6NvKaTwmplLLddFx0YReh9WfTO5jKw= +cloud.google.com/go/aiplatform v1.24.0/go.mod h1:67UUvRBKG6GTayHKV8DBv2RtR1t93YRu5B1P3x99mYY= +cloud.google.com/go/aiplatform v1.27.0/go.mod h1:Bvxqtl40l0WImSb04d0hXFU7gDOiq9jQmorivIiWcKg= +cloud.google.com/go/aiplatform v1.35.0/go.mod h1:7MFT/vCaOyZT/4IIFfxH4ErVg/4ku6lKv3w0+tFTgXQ= +cloud.google.com/go/aiplatform v1.36.1/go.mod h1:WTm12vJRPARNvJ+v6P52RDHCNe4AhvjcIZ/9/RRHy/k= +cloud.google.com/go/aiplatform v1.37.0/go.mod h1:IU2Cv29Lv9oCn/9LkFiiuKfwrRTq+QQMbW+hPCxJGZw= +cloud.google.com/go/aiplatform v1.45.0/go.mod h1:Iu2Q7sC7QGhXUeOhAj/oCK9a+ULz1O4AotZiqjQ8MYA= +cloud.google.com/go/analytics v0.11.0/go.mod h1:DjEWCu41bVbYcKyvlws9Er60YE4a//bK6mnhWvQeFNI= +cloud.google.com/go/analytics v0.12.0/go.mod h1:gkfj9h6XRf9+TS4bmuhPEShsh3hH8PAZzm/41OOhQd4= +cloud.google.com/go/analytics v0.17.0/go.mod h1:WXFa3WSym4IZ+JiKmavYdJwGG/CvpqiqczmL59bTD9M= +cloud.google.com/go/analytics v0.18.0/go.mod h1:ZkeHGQlcIPkw0R/GW+boWHhCOR43xz9RN/jn7WcqfIE= +cloud.google.com/go/analytics v0.19.0/go.mod h1:k8liqf5/HCnOUkbawNtrWWc+UAzyDlW89doe8TtoDsE= +cloud.google.com/go/analytics v0.21.2/go.mod h1:U8dcUtmDmjrmUTnnnRnI4m6zKn/yaA5N9RlEkYFHpQo= +cloud.google.com/go/apigateway v1.3.0/go.mod h1:89Z8Bhpmxu6AmUxuVRg/ECRGReEdiP3vQtk4Z1J9rJk= +cloud.google.com/go/apigateway v1.4.0/go.mod h1:pHVY9MKGaH9PQ3pJ4YLzoj6U5FUDeDFBllIz7WmzJoc= +cloud.google.com/go/apigateway v1.5.0/go.mod h1:GpnZR3Q4rR7LVu5951qfXPJCHquZt02jf7xQx7kpqN8= +cloud.google.com/go/apigateway v1.6.1/go.mod h1:ufAS3wpbRjqfZrzpvLC2oh0MFlpRJm2E/ts25yyqmXA= +cloud.google.com/go/apigeeconnect v1.3.0/go.mod h1:G/AwXFAKo0gIXkPTVfZDd2qA1TxBXJ3MgMRBQkIi9jc= +cloud.google.com/go/apigeeconnect v1.4.0/go.mod h1:kV4NwOKqjvt2JYR0AoIWo2QGfoRtn/pkS3QlHp0Ni04= +cloud.google.com/go/apigeeconnect v1.5.0/go.mod h1:KFaCqvBRU6idyhSNyn3vlHXc8VMDJdRmwDF6JyFRqZ8= +cloud.google.com/go/apigeeconnect v1.6.1/go.mod h1:C4awq7x0JpLtrlQCr8AzVIzAaYgngRqWf9S5Uhg+wWs= +cloud.google.com/go/apigeeregistry v0.4.0/go.mod h1:EUG4PGcsZvxOXAdyEghIdXwAEi/4MEaoqLMLDMIwKXY= +cloud.google.com/go/apigeeregistry v0.5.0/go.mod h1:YR5+s0BVNZfVOUkMa5pAR2xGd0A473vA5M7j247o1wM= +cloud.google.com/go/apigeeregistry v0.6.0/go.mod h1:BFNzW7yQVLZ3yj0TKcwzb8n25CFBri51GVGOEUcgQsc= +cloud.google.com/go/apigeeregistry v0.7.1/go.mod h1:1XgyjZye4Mqtw7T9TsY4NW10U7BojBvG4RMD+vRDrIw= +cloud.google.com/go/apikeys v0.4.0/go.mod h1:XATS/yqZbaBK0HOssf+ALHp8jAlNHUgyfprvNcBIszU= +cloud.google.com/go/apikeys v0.5.0/go.mod h1:5aQfwY4D+ewMMWScd3hm2en3hCj+BROlyrt3ytS7KLI= +cloud.google.com/go/apikeys v0.6.0/go.mod h1:kbpXu5upyiAlGkKrJgQl8A0rKNNJ7dQ377pdroRSSi8= +cloud.google.com/go/appengine v1.4.0/go.mod h1:CS2NhuBuDXM9f+qscZ6V86m1MIIqPj3WC/UoEuR1Sno= +cloud.google.com/go/appengine v1.5.0/go.mod h1:TfasSozdkFI0zeoxW3PTBLiNqRmzraodCWatWI9Dmak= +cloud.google.com/go/appengine v1.6.0/go.mod h1:hg6i0J/BD2cKmDJbaFSYHFyZkgBEfQrDg/X0V5fJn84= +cloud.google.com/go/appengine v1.7.0/go.mod h1:eZqpbHFCqRGa2aCdope7eC0SWLV1j0neb/QnMJVWx6A= +cloud.google.com/go/appengine v1.7.1/go.mod h1:IHLToyb/3fKutRysUlFO0BPt5j7RiQ45nrzEJmKTo6E= +cloud.google.com/go/appengine v1.8.1/go.mod h1:6NJXGLVhZCN9aQ/AEDvmfzKEfoYBlfB80/BHiKVputY= +cloud.google.com/go/area120 v0.5.0/go.mod h1:DE/n4mp+iqVyvxHN41Vf1CR602GiHQjFPusMFW6bGR4= +cloud.google.com/go/area120 v0.6.0/go.mod h1:39yFJqWVgm0UZqWTOdqkLhjoC7uFfgXRC8g/ZegeAh0= +cloud.google.com/go/area120 v0.7.0/go.mod h1:a3+8EUD1SX5RUcCs3MY5YasiO1z6yLiNLRiFrykbynY= +cloud.google.com/go/area120 v0.7.1/go.mod h1:j84i4E1RboTWjKtZVWXPqvK5VHQFJRF2c1Nm69pWm9k= +cloud.google.com/go/area120 v0.8.1/go.mod h1:BVfZpGpB7KFVNxPiQBuHkX6Ed0rS51xIgmGyjrAfzsg= +cloud.google.com/go/artifactregistry v1.6.0/go.mod h1:IYt0oBPSAGYj/kprzsBjZ/4LnG/zOcHyFHjWPCi6SAQ= +cloud.google.com/go/artifactregistry v1.7.0/go.mod h1:mqTOFOnGZx8EtSqK/ZWcsm/4U8B77rbcLP6ruDU2Ixk= +cloud.google.com/go/artifactregistry v1.8.0/go.mod h1:w3GQXkJX8hiKN0v+at4b0qotwijQbYUqF2GWkZzAhC0= +cloud.google.com/go/artifactregistry v1.9.0/go.mod h1:2K2RqvA2CYvAeARHRkLDhMDJ3OXy26h3XW+3/Jh2uYc= +cloud.google.com/go/artifactregistry v1.11.1/go.mod h1:lLYghw+Itq9SONbCa1YWBoWs1nOucMH0pwXN1rOBZFI= +cloud.google.com/go/artifactregistry v1.11.2/go.mod h1:nLZns771ZGAwVLzTX/7Al6R9ehma4WUEhZGWV6CeQNQ= +cloud.google.com/go/artifactregistry v1.12.0/go.mod h1:o6P3MIvtzTOnmvGagO9v/rOjjA0HmhJ+/6KAXrmYDCI= +cloud.google.com/go/artifactregistry v1.13.0/go.mod h1:uy/LNfoOIivepGhooAUpL1i30Hgee3Cu0l4VTWHUC08= +cloud.google.com/go/artifactregistry v1.14.1/go.mod h1:nxVdG19jTaSTu7yA7+VbWL346r3rIdkZ142BSQqhn5E= +cloud.google.com/go/asset v1.5.0/go.mod h1:5mfs8UvcM5wHhqtSv8J1CtxxaQq3AdBxxQi2jGW/K4o= +cloud.google.com/go/asset v1.7.0/go.mod h1:YbENsRK4+xTiL+Ofoj5Ckf+O17kJtgp3Y3nn4uzZz5s= +cloud.google.com/go/asset v1.8.0/go.mod h1:mUNGKhiqIdbr8X7KNayoYvyc4HbbFO9URsjbytpUaW0= +cloud.google.com/go/asset v1.9.0/go.mod h1:83MOE6jEJBMqFKadM9NLRcs80Gdw76qGuHn8m3h8oHQ= +cloud.google.com/go/asset v1.10.0/go.mod h1:pLz7uokL80qKhzKr4xXGvBQXnzHn5evJAEAtZiIb0wY= +cloud.google.com/go/asset v1.11.1/go.mod h1:fSwLhbRvC9p9CXQHJ3BgFeQNM4c9x10lqlrdEUYXlJo= +cloud.google.com/go/asset v1.12.0/go.mod h1:h9/sFOa4eDIyKmH6QMpm4eUK3pDojWnUhTgJlk762Hg= +cloud.google.com/go/asset v1.13.0/go.mod h1:WQAMyYek/b7NBpYq/K4KJWcRqzoalEsxz/t/dTk4THw= +cloud.google.com/go/asset v1.14.1/go.mod h1:4bEJ3dnHCqWCDbWJ/6Vn7GVI9LerSi7Rfdi03hd+WTQ= +cloud.google.com/go/assuredworkloads v1.5.0/go.mod h1:n8HOZ6pff6re5KYfBXcFvSViQjDwxFkAkmUFffJRbbY= +cloud.google.com/go/assuredworkloads v1.6.0/go.mod h1:yo2YOk37Yc89Rsd5QMVECvjaMKymF9OP+QXWlKXUkXw= +cloud.google.com/go/assuredworkloads v1.7.0/go.mod h1:z/736/oNmtGAyU47reJgGN+KVoYoxeLBoj4XkKYscNI= +cloud.google.com/go/assuredworkloads v1.8.0/go.mod h1:AsX2cqyNCOvEQC8RMPnoc0yEarXQk6WEKkxYfL6kGIo= +cloud.google.com/go/assuredworkloads v1.9.0/go.mod h1:kFuI1P78bplYtT77Tb1hi0FMxM0vVpRC7VVoJC3ZoT0= +cloud.google.com/go/assuredworkloads v1.10.0/go.mod h1:kwdUQuXcedVdsIaKgKTp9t0UJkE5+PAVNhdQm4ZVq2E= +cloud.google.com/go/assuredworkloads v1.11.1/go.mod h1:+F04I52Pgn5nmPG36CWFtxmav6+7Q+c5QyJoL18Lry0= +cloud.google.com/go/automl v1.5.0/go.mod h1:34EjfoFGMZ5sgJ9EoLsRtdPSNZLcfflJR39VbVNS2M0= +cloud.google.com/go/automl v1.6.0/go.mod h1:ugf8a6Fx+zP0D59WLhqgTDsQI9w07o64uf/Is3Nh5p8= +cloud.google.com/go/automl v1.7.0/go.mod h1:RL9MYCCsJEOmt0Wf3z9uzG0a7adTT1fe+aObgSpkCt8= +cloud.google.com/go/automl v1.8.0/go.mod h1:xWx7G/aPEe/NP+qzYXktoBSDfjO+vnKMGgsApGJJquM= +cloud.google.com/go/automl v1.12.0/go.mod h1:tWDcHDp86aMIuHmyvjuKeeHEGq76lD7ZqfGLN6B0NuU= +cloud.google.com/go/automl v1.13.1/go.mod h1:1aowgAHWYZU27MybSCFiukPO7xnyawv7pt3zK4bheQE= +cloud.google.com/go/baremetalsolution v0.3.0/go.mod h1:XOrocE+pvK1xFfleEnShBlNAXf+j5blPPxrhjKgnIFc= +cloud.google.com/go/baremetalsolution v0.4.0/go.mod h1:BymplhAadOO/eBa7KewQ0Ppg4A4Wplbn+PsFKRLo0uI= +cloud.google.com/go/baremetalsolution v0.5.0/go.mod h1:dXGxEkmR9BMwxhzBhV0AioD0ULBmuLZI8CdwalUxuss= +cloud.google.com/go/batch v0.3.0/go.mod h1:TR18ZoAekj1GuirsUsR1ZTKN3FC/4UDnScjT8NXImFE= +cloud.google.com/go/batch v0.4.0/go.mod h1:WZkHnP43R/QCGQsZ+0JyG4i79ranE2u8xvjq/9+STPE= +cloud.google.com/go/batch v0.7.0/go.mod h1:vLZN95s6teRUqRQ4s3RLDsH8PvboqBK+rn1oevL159g= +cloud.google.com/go/beyondcorp v0.2.0/go.mod h1:TB7Bd+EEtcw9PCPQhCJtJGjk/7TC6ckmnSFS+xwTfm4= +cloud.google.com/go/beyondcorp v0.3.0/go.mod h1:E5U5lcrcXMsCuoDNyGrpyTm/hn7ne941Jz2vmksAxW8= +cloud.google.com/go/beyondcorp v0.4.0/go.mod h1:3ApA0mbhHx6YImmuubf5pyW8srKnCEPON32/5hj+RmM= +cloud.google.com/go/beyondcorp v0.5.0/go.mod h1:uFqj9X+dSfrheVp7ssLTaRHd2EHqSL4QZmH4e8WXGGU= +cloud.google.com/go/beyondcorp v0.6.1/go.mod h1:YhxDWw946SCbmcWo3fAhw3V4XZMSpQ/VYfcKGAEU8/4= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/bigquery v1.42.0/go.mod h1:8dRTJxhtG+vwBKzE5OseQn/hiydoQN3EedCaOdYmxRA= +cloud.google.com/go/bigquery v1.43.0/go.mod h1:ZMQcXHsl+xmU1z36G2jNGZmKp9zNY5BUua5wDgmNCfw= +cloud.google.com/go/bigquery v1.44.0/go.mod h1:0Y33VqXTEsbamHJvJHdFmtqHvMIY28aK1+dFsvaChGc= +cloud.google.com/go/bigquery v1.47.0/go.mod h1:sA9XOgy0A8vQK9+MWhEQTY6Tix87M/ZurWFIxmF9I/E= +cloud.google.com/go/bigquery v1.48.0/go.mod h1:QAwSz+ipNgfL5jxiaK7weyOhzdoAy1zFm0Nf1fysJac= +cloud.google.com/go/bigquery v1.49.0/go.mod h1:Sv8hMmTFFYBlt/ftw2uN6dFdQPzBlREY9yBh7Oy7/4Q= +cloud.google.com/go/bigquery v1.50.0/go.mod h1:YrleYEh2pSEbgTBZYMJ5SuSr0ML3ypjRB1zgf7pvQLU= +cloud.google.com/go/bigquery v1.52.0/go.mod h1:3b/iXjRQGU4nKa87cXeg6/gogLjO8C6PmuM8i5Bi/u4= +cloud.google.com/go/billing v1.4.0/go.mod h1:g9IdKBEFlItS8bTtlrZdVLWSSdSyFUZKXNS02zKMOZY= +cloud.google.com/go/billing v1.5.0/go.mod h1:mztb1tBc3QekhjSgmpf/CV4LzWXLzCArwpLmP2Gm88s= +cloud.google.com/go/billing v1.6.0/go.mod h1:WoXzguj+BeHXPbKfNWkqVtDdzORazmCjraY+vrxcyvI= +cloud.google.com/go/billing v1.7.0/go.mod h1:q457N3Hbj9lYwwRbnlD7vUpyjq6u5U1RAOArInEiD5Y= +cloud.google.com/go/billing v1.12.0/go.mod h1:yKrZio/eu+okO/2McZEbch17O5CB5NpZhhXG6Z766ss= +cloud.google.com/go/billing v1.13.0/go.mod h1:7kB2W9Xf98hP9Sr12KfECgfGclsH3CQR0R08tnRlRbc= +cloud.google.com/go/billing v1.16.0/go.mod h1:y8vx09JSSJG02k5QxbycNRrN7FGZB6F3CAcgum7jvGA= +cloud.google.com/go/binaryauthorization v1.1.0/go.mod h1:xwnoWu3Y84jbuHa0zd526MJYmtnVXn0syOjaJgy4+dM= +cloud.google.com/go/binaryauthorization v1.2.0/go.mod h1:86WKkJHtRcv5ViNABtYMhhNWRrD1Vpi//uKEy7aYEfI= +cloud.google.com/go/binaryauthorization v1.3.0/go.mod h1:lRZbKgjDIIQvzYQS1p99A7/U1JqvqeZg0wiI5tp6tg0= +cloud.google.com/go/binaryauthorization v1.4.0/go.mod h1:tsSPQrBd77VLplV70GUhBf/Zm3FsKmgSqgm4UmiDItk= +cloud.google.com/go/binaryauthorization v1.5.0/go.mod h1:OSe4OU1nN/VswXKRBmciKpo9LulY41gch5c68htf3/Q= +cloud.google.com/go/binaryauthorization v1.6.1/go.mod h1:TKt4pa8xhowwffiBmbrbcxijJRZED4zrqnwZ1lKH51U= +cloud.google.com/go/certificatemanager v1.3.0/go.mod h1:n6twGDvcUBFu9uBgt4eYvvf3sQ6My8jADcOVwHmzadg= +cloud.google.com/go/certificatemanager v1.4.0/go.mod h1:vowpercVFyqs8ABSmrdV+GiFf2H/ch3KyudYQEMM590= +cloud.google.com/go/certificatemanager v1.6.0/go.mod h1:3Hh64rCKjRAX8dXgRAyOcY5vQ/fE1sh8o+Mdd6KPgY8= +cloud.google.com/go/certificatemanager v1.7.1/go.mod h1:iW8J3nG6SaRYImIa+wXQ0g8IgoofDFRp5UMzaNk1UqI= +cloud.google.com/go/channel v1.8.0/go.mod h1:W5SwCXDJsq/rg3tn3oG0LOxpAo6IMxNa09ngphpSlnk= +cloud.google.com/go/channel v1.9.0/go.mod h1:jcu05W0my9Vx4mt3/rEHpfxc9eKi9XwsdDL8yBMbKUk= +cloud.google.com/go/channel v1.11.0/go.mod h1:IdtI0uWGqhEeatSB62VOoJ8FSUhJ9/+iGkJVqp74CGE= +cloud.google.com/go/channel v1.12.0/go.mod h1:VkxCGKASi4Cq7TbXxlaBezonAYpp1GCnKMY6tnMQnLU= +cloud.google.com/go/channel v1.16.0/go.mod h1:eN/q1PFSl5gyu0dYdmxNXscY/4Fi7ABmeHCJNf/oHmc= +cloud.google.com/go/cloudbuild v1.3.0/go.mod h1:WequR4ULxlqvMsjDEEEFnOG5ZSRSgWOywXYDb1vPE6U= +cloud.google.com/go/cloudbuild v1.4.0/go.mod h1:5Qwa40LHiOXmz3386FrjrYM93rM/hdRr7b53sySrTqA= +cloud.google.com/go/cloudbuild v1.6.0/go.mod h1:UIbc/w9QCbH12xX+ezUsgblrWv+Cv4Tw83GiSMHOn9M= +cloud.google.com/go/cloudbuild v1.7.0/go.mod h1:zb5tWh2XI6lR9zQmsm1VRA+7OCuve5d8S+zJUul8KTg= +cloud.google.com/go/cloudbuild v1.9.0/go.mod h1:qK1d7s4QlO0VwfYn5YuClDGg2hfmLZEb4wQGAbIgL1s= +cloud.google.com/go/cloudbuild v1.10.1/go.mod h1:lyJg7v97SUIPq4RC2sGsz/9tNczhyv2AjML/ci4ulzU= +cloud.google.com/go/clouddms v1.3.0/go.mod h1:oK6XsCDdW4Ib3jCCBugx+gVjevp2TMXFtgxvPSee3OM= +cloud.google.com/go/clouddms v1.4.0/go.mod h1:Eh7sUGCC+aKry14O1NRljhjyrr0NFC0G2cjwX0cByRk= +cloud.google.com/go/clouddms v1.5.0/go.mod h1:QSxQnhikCLUw13iAbffF2CZxAER3xDGNHjsTAkQJcQA= +cloud.google.com/go/clouddms v1.6.1/go.mod h1:Ygo1vL52Ov4TBZQquhz5fiw2CQ58gvu+PlS6PVXCpZI= +cloud.google.com/go/cloudtasks v1.5.0/go.mod h1:fD92REy1x5woxkKEkLdvavGnPJGEn8Uic9nWuLzqCpY= +cloud.google.com/go/cloudtasks v1.6.0/go.mod h1:C6Io+sxuke9/KNRkbQpihnW93SWDU3uXt92nu85HkYI= +cloud.google.com/go/cloudtasks v1.7.0/go.mod h1:ImsfdYWwlWNJbdgPIIGJWC+gemEGTBK/SunNQQNCAb4= +cloud.google.com/go/cloudtasks v1.8.0/go.mod h1:gQXUIwCSOI4yPVK7DgTVFiiP0ZW/eQkydWzwVMdHxrI= +cloud.google.com/go/cloudtasks v1.9.0/go.mod h1:w+EyLsVkLWHcOaqNEyvcKAsWp9p29dL6uL9Nst1cI7Y= +cloud.google.com/go/cloudtasks v1.10.0/go.mod h1:NDSoTLkZ3+vExFEWu2UJV1arUyzVDAiZtdWcsUyNwBs= +cloud.google.com/go/cloudtasks v1.11.1/go.mod h1:a9udmnou9KO2iulGscKR0qBYjreuX8oHwpmFsKspEvM= +cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= +cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= +cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= +cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s= +cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= +cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= +cloud.google.com/go/compute v1.10.0/go.mod h1:ER5CLbMxl90o2jtNbGSbtfOpQKR0t15FOtRsugnLrlU= +cloud.google.com/go/compute v1.12.0/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= +cloud.google.com/go/compute v1.12.1/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= +cloud.google.com/go/compute v1.13.0/go.mod h1:5aPTS0cUNMIc1CE546K+Th6weJUNQErARyZtRXDJ8GE= +cloud.google.com/go/compute v1.14.0/go.mod h1:YfLtxrj9sU4Yxv+sXzZkyPjEyPBZfXHUvjxega5vAdo= +cloud.google.com/go/compute v1.18.0/go.mod h1:1X7yHxec2Ga+Ss6jPyjxRxpu2uu7PLgsOVXvgU0yacs= +cloud.google.com/go/compute v1.19.0/go.mod h1:rikpw2y+UMidAe9tISo04EHNOIf42RLYF/q8Bs93scU= +cloud.google.com/go/compute v1.19.3/go.mod h1:qxvISKp/gYnXkSAD1ppcSOveRAmzxicEv/JlizULFrI= +cloud.google.com/go/compute v1.20.1/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= +cloud.google.com/go/compute v1.21.0 h1:JNBsyXVoOoNJtTQcnEY5uYpZIbeCTYIeDe0Xh1bySMk= +cloud.google.com/go/compute v1.21.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= +cloud.google.com/go/compute/metadata v0.1.0/go.mod h1:Z1VN+bulIf6bt4P/C37K4DyZYZEXYonfTBHHFPO/4UU= +cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= +cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +cloud.google.com/go/contactcenterinsights v1.3.0/go.mod h1:Eu2oemoePuEFc/xKFPjbTuPSj0fYJcPls9TFlPNnHHY= +cloud.google.com/go/contactcenterinsights v1.4.0/go.mod h1:L2YzkGbPsv+vMQMCADxJoT9YiTTnSEd6fEvCeHTYVck= +cloud.google.com/go/contactcenterinsights v1.6.0/go.mod h1:IIDlT6CLcDoyv79kDv8iWxMSTZhLxSCofVV5W6YFM/w= +cloud.google.com/go/contactcenterinsights v1.9.1/go.mod h1:bsg/R7zGLYMVxFFzfh9ooLTruLRCG9fnzhH9KznHhbM= +cloud.google.com/go/container v1.6.0/go.mod h1:Xazp7GjJSeUYo688S+6J5V+n/t+G5sKBTFkKNudGRxg= +cloud.google.com/go/container v1.7.0/go.mod h1:Dp5AHtmothHGX3DwwIHPgq45Y8KmNsgN3amoYfxVkLo= +cloud.google.com/go/container v1.13.1/go.mod h1:6wgbMPeQRw9rSnKBCAJXnds3Pzj03C4JHamr8asWKy4= +cloud.google.com/go/container v1.14.0/go.mod h1:3AoJMPhHfLDxLvrlVWaK57IXzaPnLaZq63WX59aQBfM= +cloud.google.com/go/container v1.15.0/go.mod h1:ft+9S0WGjAyjDggg5S06DXj+fHJICWg8L7isCQe9pQA= +cloud.google.com/go/container v1.22.1/go.mod h1:lTNExE2R7f+DLbAN+rJiKTisauFCaoDq6NURZ83eVH4= +cloud.google.com/go/containeranalysis v0.5.1/go.mod h1:1D92jd8gRR/c0fGMlymRgxWD3Qw9C1ff6/T7mLgVL8I= +cloud.google.com/go/containeranalysis v0.6.0/go.mod h1:HEJoiEIu+lEXM+k7+qLCci0h33lX3ZqoYFdmPcoO7s4= +cloud.google.com/go/containeranalysis v0.7.0/go.mod h1:9aUL+/vZ55P2CXfuZjS4UjQ9AgXoSw8Ts6lemfmxBxI= +cloud.google.com/go/containeranalysis v0.9.0/go.mod h1:orbOANbwk5Ejoom+s+DUCTTJ7IBdBQJDcSylAx/on9s= +cloud.google.com/go/containeranalysis v0.10.1/go.mod h1:Ya2jiILITMY68ZLPaogjmOMNkwsDrWBSTyBubGXO7j0= +cloud.google.com/go/datacatalog v1.3.0/go.mod h1:g9svFY6tuR+j+hrTw3J2dNcmI0dzmSiyOzm8kpLq0a0= +cloud.google.com/go/datacatalog v1.5.0/go.mod h1:M7GPLNQeLfWqeIm3iuiruhPzkt65+Bx8dAKvScX8jvs= +cloud.google.com/go/datacatalog v1.6.0/go.mod h1:+aEyF8JKg+uXcIdAmmaMUmZ3q1b/lKLtXCmXdnc0lbc= +cloud.google.com/go/datacatalog v1.7.0/go.mod h1:9mEl4AuDYWw81UGc41HonIHH7/sn52H0/tc8f8ZbZIE= +cloud.google.com/go/datacatalog v1.8.0/go.mod h1:KYuoVOv9BM8EYz/4eMFxrr4DUKhGIOXxZoKYF5wdISM= +cloud.google.com/go/datacatalog v1.8.1/go.mod h1:RJ58z4rMp3gvETA465Vg+ag8BGgBdnRPEMMSTr5Uv+M= +cloud.google.com/go/datacatalog v1.12.0/go.mod h1:CWae8rFkfp6LzLumKOnmVh4+Zle4A3NXLzVJ1d1mRm0= +cloud.google.com/go/datacatalog v1.13.0/go.mod h1:E4Rj9a5ZtAxcQJlEBTLgMTphfP11/lNaAshpoBgemX8= +cloud.google.com/go/datacatalog v1.14.0/go.mod h1:h0PrGtlihoutNMp/uvwhawLQ9+c63Kz65UFqh49Yo+E= +cloud.google.com/go/datacatalog v1.14.1/go.mod h1:d2CevwTG4yedZilwe+v3E3ZBDRMobQfSG/a6cCCN5R4= +cloud.google.com/go/dataflow v0.6.0/go.mod h1:9QwV89cGoxjjSR9/r7eFDqqjtvbKxAK2BaYU6PVk9UM= +cloud.google.com/go/dataflow v0.7.0/go.mod h1:PX526vb4ijFMesO1o202EaUmouZKBpjHsTlCtB4parQ= +cloud.google.com/go/dataflow v0.8.0/go.mod h1:Rcf5YgTKPtQyYz8bLYhFoIV/vP39eL7fWNcSOyFfLJE= +cloud.google.com/go/dataflow v0.9.1/go.mod h1:Wp7s32QjYuQDWqJPFFlnBKhkAtiFpMTdg00qGbnIHVw= +cloud.google.com/go/dataform v0.3.0/go.mod h1:cj8uNliRlHpa6L3yVhDOBrUXH+BPAO1+KFMQQNSThKo= +cloud.google.com/go/dataform v0.4.0/go.mod h1:fwV6Y4Ty2yIFL89huYlEkwUPtS7YZinZbzzj5S9FzCE= +cloud.google.com/go/dataform v0.5.0/go.mod h1:GFUYRe8IBa2hcomWplodVmUx/iTL0FrsauObOM3Ipr0= +cloud.google.com/go/dataform v0.6.0/go.mod h1:QPflImQy33e29VuapFdf19oPbE4aYTJxr31OAPV+ulA= +cloud.google.com/go/dataform v0.7.0/go.mod h1:7NulqnVozfHvWUBpMDfKMUESr+85aJsC/2O0o3jWPDE= +cloud.google.com/go/dataform v0.8.1/go.mod h1:3BhPSiw8xmppbgzeBbmDvmSWlwouuJkXsXsb8UBih9M= +cloud.google.com/go/datafusion v1.4.0/go.mod h1:1Zb6VN+W6ALo85cXnM1IKiPw+yQMKMhB9TsTSRDo/38= +cloud.google.com/go/datafusion v1.5.0/go.mod h1:Kz+l1FGHB0J+4XF2fud96WMmRiq/wj8N9u007vyXZ2w= +cloud.google.com/go/datafusion v1.6.0/go.mod h1:WBsMF8F1RhSXvVM8rCV3AeyWVxcC2xY6vith3iw3S+8= +cloud.google.com/go/datafusion v1.7.1/go.mod h1:KpoTBbFmoToDExJUso/fcCiguGDk7MEzOWXUsJo0wsI= +cloud.google.com/go/datalabeling v0.5.0/go.mod h1:TGcJ0G2NzcsXSE/97yWjIZO0bXj0KbVlINXMG9ud42I= +cloud.google.com/go/datalabeling v0.6.0/go.mod h1:WqdISuk/+WIGeMkpw/1q7bK/tFEZxsrFJOJdY2bXvTQ= +cloud.google.com/go/datalabeling v0.7.0/go.mod h1:WPQb1y08RJbmpM3ww0CSUAGweL0SxByuW2E+FU+wXcM= +cloud.google.com/go/datalabeling v0.8.1/go.mod h1:XS62LBSVPbYR54GfYQsPXZjTW8UxCK2fkDciSrpRFdY= +cloud.google.com/go/dataplex v1.3.0/go.mod h1:hQuRtDg+fCiFgC8j0zV222HvzFQdRd+SVX8gdmFcZzA= +cloud.google.com/go/dataplex v1.4.0/go.mod h1:X51GfLXEMVJ6UN47ESVqvlsRplbLhcsAt0kZCCKsU0A= +cloud.google.com/go/dataplex v1.5.2/go.mod h1:cVMgQHsmfRoI5KFYq4JtIBEUbYwc3c7tXmIDhRmNNVQ= +cloud.google.com/go/dataplex v1.6.0/go.mod h1:bMsomC/aEJOSpHXdFKFGQ1b0TDPIeL28nJObeO1ppRs= +cloud.google.com/go/dataplex v1.8.1/go.mod h1:7TyrDT6BCdI8/38Uvp0/ZxBslOslP2X2MPDucliyvSE= +cloud.google.com/go/dataproc v1.7.0/go.mod h1:CKAlMjII9H90RXaMpSxQ8EU6dQx6iAYNPcYPOkSbi8s= +cloud.google.com/go/dataproc v1.8.0/go.mod h1:5OW+zNAH0pMpw14JVrPONsxMQYMBqJuzORhIBfBn9uI= +cloud.google.com/go/dataproc v1.12.0/go.mod h1:zrF3aX0uV3ikkMz6z4uBbIKyhRITnxvr4i3IjKsKrw4= +cloud.google.com/go/dataqna v0.5.0/go.mod h1:90Hyk596ft3zUQ8NkFfvICSIfHFh1Bc7C4cK3vbhkeo= +cloud.google.com/go/dataqna v0.6.0/go.mod h1:1lqNpM7rqNLVgWBJyk5NF6Uen2PHym0jtVJonplVsDA= +cloud.google.com/go/dataqna v0.7.0/go.mod h1:Lx9OcIIeqCrw1a6KdO3/5KMP1wAmTc0slZWwP12Qq3c= +cloud.google.com/go/dataqna v0.8.1/go.mod h1:zxZM0Bl6liMePWsHA8RMGAfmTG34vJMapbHAxQ5+WA8= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/datastore v1.10.0/go.mod h1:PC5UzAmDEkAmkfaknstTYbNpgE49HAgW2J1gcgUfmdM= +cloud.google.com/go/datastore v1.11.0/go.mod h1:TvGxBIHCS50u8jzG+AW/ppf87v1of8nwzFNgEZU1D3c= +cloud.google.com/go/datastore v1.12.0/go.mod h1:KjdB88W897MRITkvWWJrg2OUtrR5XVj1EoLgSp6/N70= +cloud.google.com/go/datastore v1.12.1/go.mod h1:KjdB88W897MRITkvWWJrg2OUtrR5XVj1EoLgSp6/N70= +cloud.google.com/go/datastream v1.2.0/go.mod h1:i/uTP8/fZwgATHS/XFu0TcNUhuA0twZxxQ3EyCUQMwo= +cloud.google.com/go/datastream v1.3.0/go.mod h1:cqlOX8xlyYF/uxhiKn6Hbv6WjwPPuI9W2M9SAXwaLLQ= +cloud.google.com/go/datastream v1.4.0/go.mod h1:h9dpzScPhDTs5noEMQVWP8Wx8AFBRyS0s8KWPx/9r0g= +cloud.google.com/go/datastream v1.5.0/go.mod h1:6TZMMNPwjUqZHBKPQ1wwXpb0d5VDVPl2/XoS5yi88q4= +cloud.google.com/go/datastream v1.6.0/go.mod h1:6LQSuswqLa7S4rPAOZFVjHIG3wJIjZcZrw8JDEDJuIs= +cloud.google.com/go/datastream v1.7.0/go.mod h1:uxVRMm2elUSPuh65IbZpzJNMbuzkcvu5CjMqVIUHrww= +cloud.google.com/go/datastream v1.9.1/go.mod h1:hqnmr8kdUBmrnk65k5wNRoHSCYksvpdZIcZIEl8h43Q= +cloud.google.com/go/deploy v1.4.0/go.mod h1:5Xghikd4VrmMLNaF6FiRFDlHb59VM59YoDQnOUdsH/c= +cloud.google.com/go/deploy v1.5.0/go.mod h1:ffgdD0B89tToyW/U/D2eL0jN2+IEV/3EMuXHA0l4r+s= +cloud.google.com/go/deploy v1.6.0/go.mod h1:f9PTHehG/DjCom3QH0cntOVRm93uGBDt2vKzAPwpXQI= +cloud.google.com/go/deploy v1.8.0/go.mod h1:z3myEJnA/2wnB4sgjqdMfgxCA0EqC3RBTNcVPs93mtQ= +cloud.google.com/go/deploy v1.11.0/go.mod h1:tKuSUV5pXbn67KiubiUNUejqLs4f5cxxiCNCeyl0F2g= +cloud.google.com/go/dialogflow v1.15.0/go.mod h1:HbHDWs33WOGJgn6rfzBW1Kv807BE3O1+xGbn59zZWI4= +cloud.google.com/go/dialogflow v1.16.1/go.mod h1:po6LlzGfK+smoSmTBnbkIZY2w8ffjz/RcGSS+sh1el0= +cloud.google.com/go/dialogflow v1.17.0/go.mod h1:YNP09C/kXA1aZdBgC/VtXX74G/TKn7XVCcVumTflA+8= +cloud.google.com/go/dialogflow v1.18.0/go.mod h1:trO7Zu5YdyEuR+BhSNOqJezyFQ3aUzz0njv7sMx/iek= +cloud.google.com/go/dialogflow v1.19.0/go.mod h1:JVmlG1TwykZDtxtTXujec4tQ+D8SBFMoosgy+6Gn0s0= +cloud.google.com/go/dialogflow v1.29.0/go.mod h1:b+2bzMe+k1s9V+F2jbJwpHPzrnIyHihAdRFMtn2WXuM= +cloud.google.com/go/dialogflow v1.31.0/go.mod h1:cuoUccuL1Z+HADhyIA7dci3N5zUssgpBJmCzI6fNRB4= +cloud.google.com/go/dialogflow v1.32.0/go.mod h1:jG9TRJl8CKrDhMEcvfcfFkkpp8ZhgPz3sBGmAUYJ2qE= +cloud.google.com/go/dialogflow v1.38.0/go.mod h1:L7jnH+JL2mtmdChzAIcXQHXMvQkE3U4hTaNltEuxXn4= +cloud.google.com/go/dlp v1.6.0/go.mod h1:9eyB2xIhpU0sVwUixfBubDoRwP+GjeUoxxeueZmqvmM= +cloud.google.com/go/dlp v1.7.0/go.mod h1:68ak9vCiMBjbasxeVD17hVPxDEck+ExiHavX8kiHG+Q= +cloud.google.com/go/dlp v1.9.0/go.mod h1:qdgmqgTyReTz5/YNSSuueR8pl7hO0o9bQ39ZhtgkWp4= +cloud.google.com/go/dlp v1.10.1/go.mod h1:IM8BWz1iJd8njcNcG0+Kyd9OPnqnRNkDV8j42VT5KOI= +cloud.google.com/go/documentai v1.7.0/go.mod h1:lJvftZB5NRiFSX4moiye1SMxHx0Bc3x1+p9e/RfXYiU= +cloud.google.com/go/documentai v1.8.0/go.mod h1:xGHNEB7CtsnySCNrCFdCyyMz44RhFEEX2Q7UD0c5IhU= +cloud.google.com/go/documentai v1.9.0/go.mod h1:FS5485S8R00U10GhgBC0aNGrJxBP8ZVpEeJ7PQDZd6k= +cloud.google.com/go/documentai v1.10.0/go.mod h1:vod47hKQIPeCfN2QS/jULIvQTugbmdc0ZvxxfQY1bg4= +cloud.google.com/go/documentai v1.16.0/go.mod h1:o0o0DLTEZ+YnJZ+J4wNfTxmDVyrkzFvttBXXtYRMHkM= +cloud.google.com/go/documentai v1.18.0/go.mod h1:F6CK6iUH8J81FehpskRmhLq/3VlwQvb7TvwOceQ2tbs= +cloud.google.com/go/documentai v1.20.0/go.mod h1:yJkInoMcK0qNAEdRnqY/D5asy73tnPe88I1YTZT+a8E= +cloud.google.com/go/domains v0.6.0/go.mod h1:T9Rz3GasrpYk6mEGHh4rymIhjlnIuB4ofT1wTxDeT4Y= +cloud.google.com/go/domains v0.7.0/go.mod h1:PtZeqS1xjnXuRPKE/88Iru/LdfoRyEHYA9nFQf4UKpg= +cloud.google.com/go/domains v0.8.0/go.mod h1:M9i3MMDzGFXsydri9/vW+EWz9sWb4I6WyHqdlAk0idE= +cloud.google.com/go/domains v0.9.1/go.mod h1:aOp1c0MbejQQ2Pjf1iJvnVyT+z6R6s8pX66KaCSDYfE= +cloud.google.com/go/edgecontainer v0.1.0/go.mod h1:WgkZ9tp10bFxqO8BLPqv2LlfmQF1X8lZqwW4r1BTajk= +cloud.google.com/go/edgecontainer v0.2.0/go.mod h1:RTmLijy+lGpQ7BXuTDa4C4ssxyXT34NIuHIgKuP4s5w= +cloud.google.com/go/edgecontainer v0.3.0/go.mod h1:FLDpP4nykgwwIfcLt6zInhprzw0lEi2P1fjO6Ie0qbc= +cloud.google.com/go/edgecontainer v1.0.0/go.mod h1:cttArqZpBB2q58W/upSG++ooo6EsblxDIolxa3jSjbY= +cloud.google.com/go/edgecontainer v1.1.1/go.mod h1:O5bYcS//7MELQZs3+7mabRqoWQhXCzenBu0R8bz2rwk= +cloud.google.com/go/errorreporting v0.3.0/go.mod h1:xsP2yaAp+OAW4OIm60An2bbLpqIhKXdWR/tawvl7QzU= +cloud.google.com/go/essentialcontacts v1.3.0/go.mod h1:r+OnHa5jfj90qIfZDO/VztSFqbQan7HV75p8sA+mdGI= +cloud.google.com/go/essentialcontacts v1.4.0/go.mod h1:8tRldvHYsmnBCHdFpvU+GL75oWiBKl80BiqlFh9tp+8= +cloud.google.com/go/essentialcontacts v1.5.0/go.mod h1:ay29Z4zODTuwliK7SnX8E86aUF2CTzdNtvv42niCX0M= +cloud.google.com/go/essentialcontacts v1.6.2/go.mod h1:T2tB6tX+TRak7i88Fb2N9Ok3PvY3UNbUsMag9/BARh4= +cloud.google.com/go/eventarc v1.7.0/go.mod h1:6ctpF3zTnaQCxUjHUdcfgcA1A2T309+omHZth7gDfmc= +cloud.google.com/go/eventarc v1.8.0/go.mod h1:imbzxkyAU4ubfsaKYdQg04WS1NvncblHEup4kvF+4gw= +cloud.google.com/go/eventarc v1.10.0/go.mod h1:u3R35tmZ9HvswGRBnF48IlYgYeBcPUCjkr4BTdem2Kw= +cloud.google.com/go/eventarc v1.11.0/go.mod h1:PyUjsUKPWoRBCHeOxZd/lbOOjahV41icXyUY5kSTvVY= +cloud.google.com/go/eventarc v1.12.1/go.mod h1:mAFCW6lukH5+IZjkvrEss+jmt2kOdYlN8aMx3sRJiAI= +cloud.google.com/go/filestore v1.3.0/go.mod h1:+qbvHGvXU1HaKX2nD0WEPo92TP/8AQuCVEBXNY9z0+w= +cloud.google.com/go/filestore v1.4.0/go.mod h1:PaG5oDfo9r224f8OYXURtAsY+Fbyq/bLYoINEK8XQAI= +cloud.google.com/go/filestore v1.5.0/go.mod h1:FqBXDWBp4YLHqRnVGveOkHDf8svj9r5+mUDLupOWEDs= +cloud.google.com/go/filestore v1.6.0/go.mod h1:di5unNuss/qfZTw2U9nhFqo8/ZDSc466dre85Kydllg= +cloud.google.com/go/filestore v1.7.1/go.mod h1:y10jsorq40JJnjR/lQ8AfFbbcGlw3g+Dp8oN7i7FjV4= +cloud.google.com/go/firestore v1.9.0/go.mod h1:HMkjKHNTtRyZNiMzu7YAsLr9K3X2udY2AMwDaMEQiiE= +cloud.google.com/go/firestore v1.11.0/go.mod h1:b38dKhgzlmNNGTNZZwe7ZRFEuRab1Hay3/DBsIGKKy4= +cloud.google.com/go/functions v1.6.0/go.mod h1:3H1UA3qiIPRWD7PeZKLvHZ9SaQhR26XIJcC0A5GbvAk= +cloud.google.com/go/functions v1.7.0/go.mod h1:+d+QBcWM+RsrgZfV9xo6KfA1GlzJfxcfZcRPEhDDfzg= +cloud.google.com/go/functions v1.8.0/go.mod h1:RTZ4/HsQjIqIYP9a9YPbU+QFoQsAlYgrwOXJWHn1POY= +cloud.google.com/go/functions v1.9.0/go.mod h1:Y+Dz8yGguzO3PpIjhLTbnqV1CWmgQ5UwtlpzoyquQ08= +cloud.google.com/go/functions v1.10.0/go.mod h1:0D3hEOe3DbEvCXtYOZHQZmD+SzYsi1YbI7dGvHfldXw= +cloud.google.com/go/functions v1.12.0/go.mod h1:AXWGrF3e2C/5ehvwYo/GH6O5s09tOPksiKhz+hH8WkA= +cloud.google.com/go/functions v1.13.0/go.mod h1:EU4O007sQm6Ef/PwRsI8N2umygGqPBS/IZQKBQBcJ3c= +cloud.google.com/go/functions v1.15.1/go.mod h1:P5yNWUTkyU+LvW/S9O6V+V423VZooALQlqoXdoPz5AE= +cloud.google.com/go/gaming v1.5.0/go.mod h1:ol7rGcxP/qHTRQE/RO4bxkXq+Fix0j6D4LFPzYTIrDM= +cloud.google.com/go/gaming v1.6.0/go.mod h1:YMU1GEvA39Qt3zWGyAVA9bpYz/yAhTvaQ1t2sK4KPUA= +cloud.google.com/go/gaming v1.7.0/go.mod h1:LrB8U7MHdGgFG851iHAfqUdLcKBdQ55hzXy9xBJz0+w= +cloud.google.com/go/gaming v1.8.0/go.mod h1:xAqjS8b7jAVW0KFYeRUxngo9My3f33kFmua++Pi+ggM= +cloud.google.com/go/gaming v1.9.0/go.mod h1:Fc7kEmCObylSWLO334NcO+O9QMDyz+TKC4v1D7X+Bc0= +cloud.google.com/go/gaming v1.10.1/go.mod h1:XQQvtfP8Rb9Rxnxm5wFVpAp9zCQkJi2bLIb7iHGwB3s= +cloud.google.com/go/gkebackup v0.2.0/go.mod h1:XKvv/4LfG829/B8B7xRkk8zRrOEbKtEam6yNfuQNH60= +cloud.google.com/go/gkebackup v0.3.0/go.mod h1:n/E671i1aOQvUxT541aTkCwExO/bTer2HDlj4TsBRAo= +cloud.google.com/go/gkebackup v0.4.0/go.mod h1:byAyBGUwYGEEww7xsbnUTBHIYcOPy/PgUWUtOeRm9Vg= +cloud.google.com/go/gkeconnect v0.5.0/go.mod h1:c5lsNAg5EwAy7fkqX/+goqFsU1Da/jQFqArp+wGNr/o= +cloud.google.com/go/gkeconnect v0.6.0/go.mod h1:Mln67KyU/sHJEBY8kFZ0xTeyPtzbq9StAVvEULYK16A= +cloud.google.com/go/gkeconnect v0.7.0/go.mod h1:SNfmVqPkaEi3bF/B3CNZOAYPYdg7sU+obZ+QTky2Myw= +cloud.google.com/go/gkeconnect v0.8.1/go.mod h1:KWiK1g9sDLZqhxB2xEuPV8V9NYzrqTUmQR9shJHpOZw= +cloud.google.com/go/gkehub v0.9.0/go.mod h1:WYHN6WG8w9bXU0hqNxt8rm5uxnk8IH+lPY9J2TV7BK0= +cloud.google.com/go/gkehub v0.10.0/go.mod h1:UIPwxI0DsrpsVoWpLB0stwKCP+WFVG9+y977wO+hBH0= +cloud.google.com/go/gkehub v0.11.0/go.mod h1:JOWHlmN+GHyIbuWQPl47/C2RFhnFKH38jH9Ascu3n0E= +cloud.google.com/go/gkehub v0.12.0/go.mod h1:djiIwwzTTBrF5NaXCGv3mf7klpEMcST17VBTVVDcuaw= +cloud.google.com/go/gkehub v0.14.1/go.mod h1:VEXKIJZ2avzrbd7u+zeMtW00Y8ddk/4V9511C9CQGTY= +cloud.google.com/go/gkemulticloud v0.3.0/go.mod h1:7orzy7O0S+5kq95e4Hpn7RysVA7dPs8W/GgfUtsPbrA= +cloud.google.com/go/gkemulticloud v0.4.0/go.mod h1:E9gxVBnseLWCk24ch+P9+B2CoDFJZTyIgLKSalC7tuI= +cloud.google.com/go/gkemulticloud v0.5.0/go.mod h1:W0JDkiyi3Tqh0TJr//y19wyb1yf8llHVto2Htf2Ja3Y= +cloud.google.com/go/gkemulticloud v0.6.1/go.mod h1:kbZ3HKyTsiwqKX7Yw56+wUGwwNZViRnxWK2DVknXWfw= +cloud.google.com/go/grafeas v0.2.0/go.mod h1:KhxgtF2hb0P191HlY5besjYm6MqTSTj3LSI+M+ByZHc= +cloud.google.com/go/grafeas v0.3.0/go.mod h1:P7hgN24EyONOTMyeJH6DxG4zD7fwiYa5Q6GUgyFSOU8= +cloud.google.com/go/gsuiteaddons v1.3.0/go.mod h1:EUNK/J1lZEZO8yPtykKxLXI6JSVN2rg9bN8SXOa0bgM= +cloud.google.com/go/gsuiteaddons v1.4.0/go.mod h1:rZK5I8hht7u7HxFQcFei0+AtfS9uSushomRlg+3ua1o= +cloud.google.com/go/gsuiteaddons v1.5.0/go.mod h1:TFCClYLd64Eaa12sFVmUyG62tk4mdIsI7pAnSXRkcFo= +cloud.google.com/go/gsuiteaddons v1.6.1/go.mod h1:CodrdOqRZcLp5WOwejHWYBjZvfY0kOphkAKpF/3qdZY= +cloud.google.com/go/iam v0.1.0/go.mod h1:vcUNEa0pEm0qRVpmWepWaFMIAI8/hjB9mO8rNCJtF6c= +cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= +cloud.google.com/go/iam v0.5.0/go.mod h1:wPU9Vt0P4UmCux7mqtRu6jcpPAb74cP1fh50J3QpkUc= +cloud.google.com/go/iam v0.6.0/go.mod h1:+1AH33ueBne5MzYccyMHtEKqLE4/kJOibtffMHDMFMc= +cloud.google.com/go/iam v0.7.0/go.mod h1:H5Br8wRaDGNc8XP3keLc4unfUUZeyH3Sfl9XpQEYOeg= +cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGESjkE= +cloud.google.com/go/iam v0.11.0/go.mod h1:9PiLDanza5D+oWFZiH1uG+RnRCfEGKoyl6yo4cgWZGY= +cloud.google.com/go/iam v0.12.0/go.mod h1:knyHGviacl11zrtZUoDuYpDgLjvr28sLQaG0YB2GYAY= +cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= +cloud.google.com/go/iam v1.0.1/go.mod h1:yR3tmSL8BcZB4bxByRv2jkSIahVmCtfKZwLYGBalRE8= +cloud.google.com/go/iam v1.1.0/go.mod h1:nxdHjaKfCr7fNYx/HJMM8LgiMugmveWlkatear5gVyk= +cloud.google.com/go/iam v1.1.1/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU= +cloud.google.com/go/iap v1.4.0/go.mod h1:RGFwRJdihTINIe4wZ2iCP0zF/qu18ZwyKxrhMhygBEc= +cloud.google.com/go/iap v1.5.0/go.mod h1:UH/CGgKd4KyohZL5Pt0jSKE4m3FR51qg6FKQ/z/Ix9A= +cloud.google.com/go/iap v1.6.0/go.mod h1:NSuvI9C/j7UdjGjIde7t7HBz+QTwBcapPE07+sSRcLk= +cloud.google.com/go/iap v1.7.0/go.mod h1:beqQx56T9O1G1yNPph+spKpNibDlYIiIixiqsQXxLIo= +cloud.google.com/go/iap v1.7.1/go.mod h1:WapEwPc7ZxGt2jFGB/C/bm+hP0Y6NXzOYGjpPnmMS74= +cloud.google.com/go/iap v1.8.1/go.mod h1:sJCbeqg3mvWLqjZNsI6dfAtbbV1DL2Rl7e1mTyXYREQ= +cloud.google.com/go/ids v1.1.0/go.mod h1:WIuwCaYVOzHIj2OhN9HAwvW+DBdmUAdcWlFxRl+KubM= +cloud.google.com/go/ids v1.2.0/go.mod h1:5WXvp4n25S0rA/mQWAg1YEEBBq6/s+7ml1RDCW1IrcY= +cloud.google.com/go/ids v1.3.0/go.mod h1:JBdTYwANikFKaDP6LtW5JAi4gubs57SVNQjemdt6xV4= +cloud.google.com/go/ids v1.4.1/go.mod h1:np41ed8YMU8zOgv53MMMoCntLTn2lF+SUzlM+O3u/jw= +cloud.google.com/go/iot v1.3.0/go.mod h1:r7RGh2B61+B8oz0AGE+J72AhA0G7tdXItODWsaA2oLs= +cloud.google.com/go/iot v1.4.0/go.mod h1:dIDxPOn0UvNDUMD8Ger7FIaTuvMkj+aGk94RPP0iV+g= +cloud.google.com/go/iot v1.5.0/go.mod h1:mpz5259PDl3XJthEmh9+ap0affn/MqNSP4My77Qql9o= +cloud.google.com/go/iot v1.6.0/go.mod h1:IqdAsmE2cTYYNO1Fvjfzo9po179rAtJeVGUvkLN3rLE= +cloud.google.com/go/iot v1.7.1/go.mod h1:46Mgw7ev1k9KqK1ao0ayW9h0lI+3hxeanz+L1zmbbbk= +cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxsnnOA= +cloud.google.com/go/kms v1.5.0/go.mod h1:QJS2YY0eJGBg3mnDfuaCyLauWwBJiHRboYxJ++1xJNg= +cloud.google.com/go/kms v1.6.0/go.mod h1:Jjy850yySiasBUDi6KFUwUv2n1+o7QZFyuUJg6OgjA0= +cloud.google.com/go/kms v1.8.0/go.mod h1:4xFEhYFqvW+4VMELtZyxomGSYtSQKzM178ylFW4jMAg= +cloud.google.com/go/kms v1.9.0/go.mod h1:qb1tPTgfF9RQP8e1wq4cLFErVuTJv7UsSC915J8dh3w= +cloud.google.com/go/kms v1.10.0/go.mod h1:ng3KTUtQQU9bPX3+QGLsflZIHlkbn8amFAMY63m8d24= +cloud.google.com/go/kms v1.10.1/go.mod h1:rIWk/TryCkR59GMC3YtHtXeLzd634lBbKenvyySAyYI= +cloud.google.com/go/kms v1.11.0/go.mod h1:hwdiYC0xjnWsKQQCQQmIQnS9asjYVSK6jtXm+zFqXLM= +cloud.google.com/go/kms v1.12.1/go.mod h1:c9J991h5DTl+kg7gi3MYomh12YEENGrf48ee/N/2CDM= +cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic= +cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= +cloud.google.com/go/language v1.7.0/go.mod h1:DJ6dYN/W+SQOjF8e1hLQXMF21AkH2w9wiPzPCJa2MIE= +cloud.google.com/go/language v1.8.0/go.mod h1:qYPVHf7SPoNNiCL2Dr0FfEFNil1qi3pQEyygwpgVKB8= +cloud.google.com/go/language v1.9.0/go.mod h1:Ns15WooPM5Ad/5no/0n81yUetis74g3zrbeJBE+ptUY= +cloud.google.com/go/language v1.10.1/go.mod h1:CPp94nsdVNiQEt1CNjF5WkTcisLiHPyIbMhvR8H2AW0= +cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8= +cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6t/iPhY2Tyfu08= +cloud.google.com/go/lifesciences v0.8.0/go.mod h1:lFxiEOMqII6XggGbOnKiyZ7IBwoIqA84ClvoezaA/bo= +cloud.google.com/go/lifesciences v0.9.1/go.mod h1:hACAOd1fFbCGLr/+weUKRAJas82Y4vrL3O5326N//Wc= +cloud.google.com/go/logging v1.6.1/go.mod h1:5ZO0mHHbvm8gEmeEUHrmDlTDSu5imF6MUP9OfilNXBw= +cloud.google.com/go/logging v1.7.0/go.mod h1:3xjP2CjkM3ZkO73aj4ASA5wRPGGCRrPIAeNqVNkzY8M= +cloud.google.com/go/longrunning v0.1.1/go.mod h1:UUFxuDWkv22EuY93jjmDMFT5GPQKeFVJBIF6QlTqdsE= +cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI0YSmW3iIlLJc= +cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo= +cloud.google.com/go/longrunning v0.4.2/go.mod h1:OHrnaYyLUV6oqwh0xiS7e5sLQhP1m0QU9R+WhGDMgIQ= +cloud.google.com/go/longrunning v0.5.0/go.mod h1:0JNuqRShmscVAhIACGtskSAWtqtOoPkwP0YF1oVEchc= +cloud.google.com/go/longrunning v0.5.1/go.mod h1:spvimkwdz6SPWKEt/XBij79E9fiTkHSQl/fRUUQJYJc= +cloud.google.com/go/managedidentities v1.3.0/go.mod h1:UzlW3cBOiPrzucO5qWkNkh0w33KFtBJU281hacNvsdE= +cloud.google.com/go/managedidentities v1.4.0/go.mod h1:NWSBYbEMgqmbZsLIyKvxrYbtqOsxY1ZrGM+9RgDqInM= +cloud.google.com/go/managedidentities v1.5.0/go.mod h1:+dWcZ0JlUmpuxpIDfyP5pP5y0bLdRwOS4Lp7gMni/LA= +cloud.google.com/go/managedidentities v1.6.1/go.mod h1:h/irGhTN2SkZ64F43tfGPMbHnypMbu4RB3yl8YcuEak= +cloud.google.com/go/maps v0.1.0/go.mod h1:BQM97WGyfw9FWEmQMpZ5T6cpovXXSd1cGmFma94eubI= +cloud.google.com/go/maps v0.6.0/go.mod h1:o6DAMMfb+aINHz/p/jbcY+mYeXBoZoxTfdSQ8VAJaCw= +cloud.google.com/go/maps v0.7.0/go.mod h1:3GnvVl3cqeSvgMcpRlQidXsPYuDGQ8naBis7MVzpXsY= +cloud.google.com/go/mediatranslation v0.5.0/go.mod h1:jGPUhGTybqsPQn91pNXw0xVHfuJ3leR1wj37oU3y1f4= +cloud.google.com/go/mediatranslation v0.6.0/go.mod h1:hHdBCTYNigsBxshbznuIMFNe5QXEowAuNmmC7h8pu5w= +cloud.google.com/go/mediatranslation v0.7.0/go.mod h1:LCnB/gZr90ONOIQLgSXagp8XUW1ODs2UmUMvcgMfI2I= +cloud.google.com/go/mediatranslation v0.8.1/go.mod h1:L/7hBdEYbYHQJhX2sldtTO5SZZ1C1vkapubj0T2aGig= +cloud.google.com/go/memcache v1.4.0/go.mod h1:rTOfiGZtJX1AaFUrOgsMHX5kAzaTQ8azHiuDoTPzNsE= +cloud.google.com/go/memcache v1.5.0/go.mod h1:dk3fCK7dVo0cUU2c36jKb4VqKPS22BTkf81Xq617aWM= +cloud.google.com/go/memcache v1.6.0/go.mod h1:XS5xB0eQZdHtTuTF9Hf8eJkKtR3pVRCcvJwtm68T3rA= +cloud.google.com/go/memcache v1.7.0/go.mod h1:ywMKfjWhNtkQTxrWxCkCFkoPjLHPW6A7WOTVI8xy3LY= +cloud.google.com/go/memcache v1.9.0/go.mod h1:8oEyzXCu+zo9RzlEaEjHl4KkgjlNDaXbCQeQWlzNFJM= +cloud.google.com/go/memcache v1.10.1/go.mod h1:47YRQIarv4I3QS5+hoETgKO40InqzLP6kpNLvyXuyaA= +cloud.google.com/go/metastore v1.5.0/go.mod h1:2ZNrDcQwghfdtCwJ33nM0+GrBGlVuh8rakL3vdPY3XY= +cloud.google.com/go/metastore v1.6.0/go.mod h1:6cyQTls8CWXzk45G55x57DVQ9gWg7RiH65+YgPsNh9s= +cloud.google.com/go/metastore v1.7.0/go.mod h1:s45D0B4IlsINu87/AsWiEVYbLaIMeUSoxlKKDqBGFS8= +cloud.google.com/go/metastore v1.8.0/go.mod h1:zHiMc4ZUpBiM7twCIFQmJ9JMEkDSyZS9U12uf7wHqSI= +cloud.google.com/go/metastore v1.10.0/go.mod h1:fPEnH3g4JJAk+gMRnrAnoqyv2lpUCqJPWOodSaf45Eo= +cloud.google.com/go/metastore v1.11.1/go.mod h1:uZuSo80U3Wd4zi6C22ZZliOUJ3XeM/MlYi/z5OAOWRA= +cloud.google.com/go/monitoring v1.7.0/go.mod h1:HpYse6kkGo//7p6sT0wsIC6IBDET0RhIsnmlA53dvEk= +cloud.google.com/go/monitoring v1.8.0/go.mod h1:E7PtoMJ1kQXWxPjB6mv2fhC5/15jInuulFdYYtlcvT4= +cloud.google.com/go/monitoring v1.12.0/go.mod h1:yx8Jj2fZNEkL/GYZyTLS4ZtZEZN8WtDEiEqG4kLK50w= +cloud.google.com/go/monitoring v1.13.0/go.mod h1:k2yMBAB1H9JT/QETjNkgdCGD9bPF712XiLTVr+cBrpw= +cloud.google.com/go/monitoring v1.15.1/go.mod h1:lADlSAlFdbqQuwwpaImhsJXu1QSdd3ojypXrFSMr2rM= +cloud.google.com/go/networkconnectivity v1.4.0/go.mod h1:nOl7YL8odKyAOtzNX73/M5/mGZgqqMeryi6UPZTk/rA= +cloud.google.com/go/networkconnectivity v1.5.0/go.mod h1:3GzqJx7uhtlM3kln0+x5wyFvuVH1pIBJjhCpjzSt75o= +cloud.google.com/go/networkconnectivity v1.6.0/go.mod h1:OJOoEXW+0LAxHh89nXd64uGG+FbQoeH8DtxCHVOMlaM= +cloud.google.com/go/networkconnectivity v1.7.0/go.mod h1:RMuSbkdbPwNMQjB5HBWD5MpTBnNm39iAVpC3TmsExt8= +cloud.google.com/go/networkconnectivity v1.10.0/go.mod h1:UP4O4sWXJG13AqrTdQCD9TnLGEbtNRqjuaaA7bNjF5E= +cloud.google.com/go/networkconnectivity v1.11.0/go.mod h1:iWmDD4QF16VCDLXUqvyspJjIEtBR/4zq5hwnY2X3scM= +cloud.google.com/go/networkconnectivity v1.12.1/go.mod h1:PelxSWYM7Sh9/guf8CFhi6vIqf19Ir/sbfZRUwXh92E= +cloud.google.com/go/networkmanagement v1.4.0/go.mod h1:Q9mdLLRn60AsOrPc8rs8iNV6OHXaGcDdsIQe1ohekq8= +cloud.google.com/go/networkmanagement v1.5.0/go.mod h1:ZnOeZ/evzUdUsnvRt792H0uYEnHQEMaz+REhhzJRcf4= +cloud.google.com/go/networkmanagement v1.6.0/go.mod h1:5pKPqyXjB/sgtvB5xqOemumoQNB7y95Q7S+4rjSOPYY= +cloud.google.com/go/networkmanagement v1.8.0/go.mod h1:Ho/BUGmtyEqrttTgWEe7m+8vDdK74ibQc+Be0q7Fof0= +cloud.google.com/go/networksecurity v0.5.0/go.mod h1:xS6fOCoqpVC5zx15Z/MqkfDwH4+m/61A3ODiDV1xmiQ= +cloud.google.com/go/networksecurity v0.6.0/go.mod h1:Q5fjhTr9WMI5mbpRYEbiexTzROf7ZbDzvzCrNl14nyU= +cloud.google.com/go/networksecurity v0.7.0/go.mod h1:mAnzoxx/8TBSyXEeESMy9OOYwo1v+gZ5eMRnsT5bC8k= +cloud.google.com/go/networksecurity v0.8.0/go.mod h1:B78DkqsxFG5zRSVuwYFRZ9Xz8IcQ5iECsNrPn74hKHU= +cloud.google.com/go/networksecurity v0.9.1/go.mod h1:MCMdxOKQ30wsBI1eI659f9kEp4wuuAueoC9AJKSPWZQ= +cloud.google.com/go/notebooks v1.2.0/go.mod h1:9+wtppMfVPUeJ8fIWPOq1UnATHISkGXGqTkxeieQ6UY= +cloud.google.com/go/notebooks v1.3.0/go.mod h1:bFR5lj07DtCPC7YAAJ//vHskFBxA5JzYlH68kXVdk34= +cloud.google.com/go/notebooks v1.4.0/go.mod h1:4QPMngcwmgb6uw7Po99B2xv5ufVoIQ7nOGDyL4P8AgA= +cloud.google.com/go/notebooks v1.5.0/go.mod h1:q8mwhnP9aR8Hpfnrc5iN5IBhrXUy8S2vuYs+kBJ/gu0= +cloud.google.com/go/notebooks v1.7.0/go.mod h1:PVlaDGfJgj1fl1S3dUwhFMXFgfYGhYQt2164xOMONmE= +cloud.google.com/go/notebooks v1.8.0/go.mod h1:Lq6dYKOYOWUCTvw5t2q1gp1lAp0zxAxRycayS0iJcqQ= +cloud.google.com/go/notebooks v1.9.1/go.mod h1:zqG9/gk05JrzgBt4ghLzEepPHNwE5jgPcHZRKhlC1A8= +cloud.google.com/go/optimization v1.1.0/go.mod h1:5po+wfvX5AQlPznyVEZjGJTMr4+CAkJf2XSTQOOl9l4= +cloud.google.com/go/optimization v1.2.0/go.mod h1:Lr7SOHdRDENsh+WXVmQhQTrzdu9ybg0NecjHidBq6xs= +cloud.google.com/go/optimization v1.3.1/go.mod h1:IvUSefKiwd1a5p0RgHDbWCIbDFgKuEdB+fPPuP0IDLI= +cloud.google.com/go/optimization v1.4.1/go.mod h1:j64vZQP7h9bO49m2rVaTVoNM0vEBEN5eKPUPbZyXOrk= +cloud.google.com/go/orchestration v1.3.0/go.mod h1:Sj5tq/JpWiB//X/q3Ngwdl5K7B7Y0KZ7bfv0wL6fqVA= +cloud.google.com/go/orchestration v1.4.0/go.mod h1:6W5NLFWs2TlniBphAViZEVhrXRSMgUGDfW7vrWKvsBk= +cloud.google.com/go/orchestration v1.6.0/go.mod h1:M62Bevp7pkxStDfFfTuCOaXgaaqRAga1yKyoMtEoWPQ= +cloud.google.com/go/orchestration v1.8.1/go.mod h1:4sluRF3wgbYVRqz7zJ1/EUNc90TTprliq9477fGobD8= +cloud.google.com/go/orgpolicy v1.4.0/go.mod h1:xrSLIV4RePWmP9P3tBl8S93lTmlAxjm06NSm2UTmKvE= +cloud.google.com/go/orgpolicy v1.5.0/go.mod h1:hZEc5q3wzwXJaKrsx5+Ewg0u1LxJ51nNFlext7Tanwc= +cloud.google.com/go/orgpolicy v1.10.0/go.mod h1:w1fo8b7rRqlXlIJbVhOMPrwVljyuW5mqssvBtU18ONc= +cloud.google.com/go/orgpolicy v1.11.0/go.mod h1:2RK748+FtVvnfuynxBzdnyu7sygtoZa1za/0ZfpOs1M= +cloud.google.com/go/orgpolicy v1.11.1/go.mod h1:8+E3jQcpZJQliP+zaFfayC2Pg5bmhuLK755wKhIIUCE= +cloud.google.com/go/osconfig v1.7.0/go.mod h1:oVHeCeZELfJP7XLxcBGTMBvRO+1nQ5tFG9VQTmYS2Fs= +cloud.google.com/go/osconfig v1.8.0/go.mod h1:EQqZLu5w5XA7eKizepumcvWx+m8mJUhEwiPqWiZeEdg= +cloud.google.com/go/osconfig v1.9.0/go.mod h1:Yx+IeIZJ3bdWmzbQU4fxNl8xsZ4amB+dygAwFPlvnNo= +cloud.google.com/go/osconfig v1.10.0/go.mod h1:uMhCzqC5I8zfD9zDEAfvgVhDS8oIjySWh+l4WK6GnWw= +cloud.google.com/go/osconfig v1.11.0/go.mod h1:aDICxrur2ogRd9zY5ytBLV89KEgT2MKB2L/n6x1ooPw= +cloud.google.com/go/osconfig v1.12.0/go.mod h1:8f/PaYzoS3JMVfdfTubkowZYGmAhUCjjwnjqWI7NVBc= +cloud.google.com/go/osconfig v1.12.1/go.mod h1:4CjBxND0gswz2gfYRCUoUzCm9zCABp91EeTtWXyz0tE= +cloud.google.com/go/oslogin v1.4.0/go.mod h1:YdgMXWRaElXz/lDk1Na6Fh5orF7gvmJ0FGLIs9LId4E= +cloud.google.com/go/oslogin v1.5.0/go.mod h1:D260Qj11W2qx/HVF29zBg+0fd6YCSjSqLUkY/qEenQU= +cloud.google.com/go/oslogin v1.6.0/go.mod h1:zOJ1O3+dTU8WPlGEkFSh7qeHPPSoxrcMbbK1Nm2iX70= +cloud.google.com/go/oslogin v1.7.0/go.mod h1:e04SN0xO1UNJ1M5GP0vzVBFicIe4O53FOfcixIqTyXo= +cloud.google.com/go/oslogin v1.9.0/go.mod h1:HNavntnH8nzrn8JCTT5fj18FuJLFJc4NaZJtBnQtKFs= +cloud.google.com/go/oslogin v1.10.1/go.mod h1:x692z7yAue5nE7CsSnoG0aaMbNoRJRXO4sn73R+ZqAs= +cloud.google.com/go/phishingprotection v0.5.0/go.mod h1:Y3HZknsK9bc9dMi+oE8Bim0lczMU6hrX0UpADuMefr0= +cloud.google.com/go/phishingprotection v0.6.0/go.mod h1:9Y3LBLgy0kDTcYET8ZH3bq/7qni15yVUoAxiFxnlSUA= +cloud.google.com/go/phishingprotection v0.7.0/go.mod h1:8qJI4QKHoda/sb/7/YmMQ2omRLSLYSu9bU0EKCNI+Lk= +cloud.google.com/go/phishingprotection v0.8.1/go.mod h1:AxonW7GovcA8qdEk13NfHq9hNx5KPtfxXNeUxTDxB6I= +cloud.google.com/go/policytroubleshooter v1.3.0/go.mod h1:qy0+VwANja+kKrjlQuOzmlvscn4RNsAc0e15GGqfMxg= +cloud.google.com/go/policytroubleshooter v1.4.0/go.mod h1:DZT4BcRw3QoO8ota9xw/LKtPa8lKeCByYeKTIf/vxdE= +cloud.google.com/go/policytroubleshooter v1.5.0/go.mod h1:Rz1WfV+1oIpPdN2VvvuboLVRsB1Hclg3CKQ53j9l8vw= +cloud.google.com/go/policytroubleshooter v1.6.0/go.mod h1:zYqaPTsmfvpjm5ULxAyD/lINQxJ0DDsnWOP/GZ7xzBc= +cloud.google.com/go/policytroubleshooter v1.7.1/go.mod h1:0NaT5v3Ag1M7U5r0GfDCpUFkWd9YqpubBWsQlhanRv0= +cloud.google.com/go/privatecatalog v0.5.0/go.mod h1:XgosMUvvPyxDjAVNDYxJ7wBW8//hLDDYmnsNcMGq1K0= +cloud.google.com/go/privatecatalog v0.6.0/go.mod h1:i/fbkZR0hLN29eEWiiwue8Pb+GforiEIBnV9yrRUOKI= +cloud.google.com/go/privatecatalog v0.7.0/go.mod h1:2s5ssIFO69F5csTXcwBP7NPFTZvps26xGzvQ2PQaBYg= +cloud.google.com/go/privatecatalog v0.8.0/go.mod h1:nQ6pfaegeDAq/Q5lrfCQzQLhubPiZhSaNhIgfJlnIXs= +cloud.google.com/go/privatecatalog v0.9.1/go.mod h1:0XlDXW2unJXdf9zFz968Hp35gl/bhF4twwpXZAW50JA= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/pubsub v1.26.0/go.mod h1:QgBH3U/jdJy/ftjPhTkyXNj543Tin1pRYcdcPRnFIRI= +cloud.google.com/go/pubsub v1.27.1/go.mod h1:hQN39ymbV9geqBnfQq6Xf63yNhUAhv9CZhzp5O6qsW0= +cloud.google.com/go/pubsub v1.28.0/go.mod h1:vuXFpwaVoIPQMGXqRyUQigu/AX1S3IWugR9xznmcXX8= +cloud.google.com/go/pubsub v1.30.0/go.mod h1:qWi1OPS0B+b5L+Sg6Gmc9zD1Y+HaM0MdUr7LsupY1P4= +cloud.google.com/go/pubsub v1.32.0/go.mod h1:f+w71I33OMyxf9VpMVcZbnG5KSUkCOUHYpFd5U1GdRc= +cloud.google.com/go/pubsublite v1.5.0/go.mod h1:xapqNQ1CuLfGi23Yda/9l4bBCKz/wC3KIJ5gKcxveZg= +cloud.google.com/go/pubsublite v1.6.0/go.mod h1:1eFCS0U11xlOuMFV/0iBqw3zP12kddMeCbj/F3FSj9k= +cloud.google.com/go/pubsublite v1.7.0/go.mod h1:8hVMwRXfDfvGm3fahVbtDbiLePT3gpoiJYJY+vxWxVM= +cloud.google.com/go/pubsublite v1.8.1/go.mod h1:fOLdU4f5xldK4RGJrBMm+J7zMWNj/k4PxwEZXy39QS0= +cloud.google.com/go/recaptchaenterprise v1.3.1/go.mod h1:OdD+q+y4XGeAlxRaMn1Y7/GveP6zmq76byL6tjPE7d4= +cloud.google.com/go/recaptchaenterprise/v2 v2.1.0/go.mod h1:w9yVqajwroDNTfGuhmOjPDN//rZGySaf6PtFVcSCa7o= +cloud.google.com/go/recaptchaenterprise/v2 v2.2.0/go.mod h1:/Zu5jisWGeERrd5HnlS3EUGb/D335f9k51B/FVil0jk= +cloud.google.com/go/recaptchaenterprise/v2 v2.3.0/go.mod h1:O9LwGCjrhGHBQET5CA7dd5NwwNQUErSgEDit1DLNTdo= +cloud.google.com/go/recaptchaenterprise/v2 v2.4.0/go.mod h1:Am3LHfOuBstrLrNCBrlI5sbwx9LBg3te2N6hGvHn2mE= +cloud.google.com/go/recaptchaenterprise/v2 v2.5.0/go.mod h1:O8LzcHXN3rz0j+LBC91jrwI3R+1ZSZEWrfL7XHgNo9U= +cloud.google.com/go/recaptchaenterprise/v2 v2.6.0/go.mod h1:RPauz9jeLtB3JVzg6nCbe12qNoaa8pXc4d/YukAmcnA= +cloud.google.com/go/recaptchaenterprise/v2 v2.7.0/go.mod h1:19wVj/fs5RtYtynAPJdDTb69oW0vNHYDBTbB4NvMD9c= +cloud.google.com/go/recaptchaenterprise/v2 v2.7.2/go.mod h1:kR0KjsJS7Jt1YSyWFkseQ756D45kaYNTlDPPaRAvDBU= +cloud.google.com/go/recommendationengine v0.5.0/go.mod h1:E5756pJcVFeVgaQv3WNpImkFP8a+RptV6dDLGPILjvg= +cloud.google.com/go/recommendationengine v0.6.0/go.mod h1:08mq2umu9oIqc7tDy8sx+MNJdLG0fUi3vaSVbztHgJ4= +cloud.google.com/go/recommendationengine v0.7.0/go.mod h1:1reUcE3GIu6MeBz/h5xZJqNLuuVjNg1lmWMPyjatzac= +cloud.google.com/go/recommendationengine v0.8.1/go.mod h1:MrZihWwtFYWDzE6Hz5nKcNz3gLizXVIDI/o3G1DLcrE= +cloud.google.com/go/recommender v1.5.0/go.mod h1:jdoeiBIVrJe9gQjwd759ecLJbxCDED4A6p+mqoqDvTg= +cloud.google.com/go/recommender v1.6.0/go.mod h1:+yETpm25mcoiECKh9DEScGzIRyDKpZ0cEhWGo+8bo+c= +cloud.google.com/go/recommender v1.7.0/go.mod h1:XLHs/W+T8olwlGOgfQenXBTbIseGclClff6lhFVe9Bs= +cloud.google.com/go/recommender v1.8.0/go.mod h1:PkjXrTT05BFKwxaUxQmtIlrtj0kph108r02ZZQ5FE70= +cloud.google.com/go/recommender v1.9.0/go.mod h1:PnSsnZY7q+VL1uax2JWkt/UegHssxjUVVCrX52CuEmQ= +cloud.google.com/go/recommender v1.10.1/go.mod h1:XFvrE4Suqn5Cq0Lf+mCP6oBHD/yRMA8XxP5sb7Q7gpA= +cloud.google.com/go/redis v1.7.0/go.mod h1:V3x5Jq1jzUcg+UNsRvdmsfuFnit1cfe3Z/PGyq/lm4Y= +cloud.google.com/go/redis v1.8.0/go.mod h1:Fm2szCDavWzBk2cDKxrkmWBqoCiL1+Ctwq7EyqBCA/A= +cloud.google.com/go/redis v1.9.0/go.mod h1:HMYQuajvb2D0LvMgZmLDZW8V5aOC/WxstZHiy4g8OiA= +cloud.google.com/go/redis v1.10.0/go.mod h1:ThJf3mMBQtW18JzGgh41/Wld6vnDDc/F/F35UolRZPM= +cloud.google.com/go/redis v1.11.0/go.mod h1:/X6eicana+BWcUda5PpwZC48o37SiFVTFSs0fWAJ7uQ= +cloud.google.com/go/redis v1.13.1/go.mod h1:VP7DGLpE91M6bcsDdMuyCm2hIpB6Vp2hI090Mfd1tcg= +cloud.google.com/go/resourcemanager v1.3.0/go.mod h1:bAtrTjZQFJkiWTPDb1WBjzvc6/kifjj4QBYuKCCoqKA= +cloud.google.com/go/resourcemanager v1.4.0/go.mod h1:MwxuzkumyTX7/a3n37gmsT3py7LIXwrShilPh3P1tR0= +cloud.google.com/go/resourcemanager v1.5.0/go.mod h1:eQoXNAiAvCf5PXxWxXjhKQoTMaUSNrEfg+6qdf/wots= +cloud.google.com/go/resourcemanager v1.6.0/go.mod h1:YcpXGRs8fDzcUl1Xw8uOVmI8JEadvhRIkoXXUNVYcVo= +cloud.google.com/go/resourcemanager v1.7.0/go.mod h1:HlD3m6+bwhzj9XCouqmeiGuni95NTrExfhoSrkC/3EI= +cloud.google.com/go/resourcemanager v1.9.1/go.mod h1:dVCuosgrh1tINZ/RwBufr8lULmWGOkPS8gL5gqyjdT8= +cloud.google.com/go/resourcesettings v1.3.0/go.mod h1:lzew8VfESA5DQ8gdlHwMrqZs1S9V87v3oCnKCWoOuQU= +cloud.google.com/go/resourcesettings v1.4.0/go.mod h1:ldiH9IJpcrlC3VSuCGvjR5of/ezRrOxFtpJoJo5SmXg= +cloud.google.com/go/resourcesettings v1.5.0/go.mod h1:+xJF7QSG6undsQDfsCJyqWXyBwUoJLhetkRMDRnIoXA= +cloud.google.com/go/resourcesettings v1.6.1/go.mod h1:M7mk9PIZrC5Fgsu1kZJci6mpgN8o0IUzVx3eJU3y4Jw= +cloud.google.com/go/retail v1.8.0/go.mod h1:QblKS8waDmNUhghY2TI9O3JLlFk8jybHeV4BF19FrE4= +cloud.google.com/go/retail v1.9.0/go.mod h1:g6jb6mKuCS1QKnH/dpu7isX253absFl6iE92nHwlBUY= +cloud.google.com/go/retail v1.10.0/go.mod h1:2gDk9HsL4HMS4oZwz6daui2/jmKvqShXKQuB2RZ+cCc= +cloud.google.com/go/retail v1.11.0/go.mod h1:MBLk1NaWPmh6iVFSz9MeKG/Psyd7TAgm6y/9L2B4x9Y= +cloud.google.com/go/retail v1.12.0/go.mod h1:UMkelN/0Z8XvKymXFbD4EhFJlYKRx1FGhQkVPU5kF14= +cloud.google.com/go/retail v1.14.1/go.mod h1:y3Wv3Vr2k54dLNIrCzenyKG8g8dhvhncT2NcNjb/6gE= +cloud.google.com/go/run v0.2.0/go.mod h1:CNtKsTA1sDcnqqIFR3Pb5Tq0usWxJJvsWOCPldRU3Do= +cloud.google.com/go/run v0.3.0/go.mod h1:TuyY1+taHxTjrD0ZFk2iAR+xyOXEA0ztb7U3UNA0zBo= +cloud.google.com/go/run v0.8.0/go.mod h1:VniEnuBwqjigv0A7ONfQUaEItaiCRVujlMqerPPiktM= +cloud.google.com/go/run v0.9.0/go.mod h1:Wwu+/vvg8Y+JUApMwEDfVfhetv30hCG4ZwDR/IXl2Qg= +cloud.google.com/go/scheduler v1.4.0/go.mod h1:drcJBmxF3aqZJRhmkHQ9b3uSSpQoltBPGPxGAWROx6s= +cloud.google.com/go/scheduler v1.5.0/go.mod h1:ri073ym49NW3AfT6DZi21vLZrG07GXr5p3H1KxN5QlI= +cloud.google.com/go/scheduler v1.6.0/go.mod h1:SgeKVM7MIwPn3BqtcBntpLyrIJftQISRrYB5ZtT+KOk= +cloud.google.com/go/scheduler v1.7.0/go.mod h1:jyCiBqWW956uBjjPMMuX09n3x37mtyPJegEWKxRsn44= +cloud.google.com/go/scheduler v1.8.0/go.mod h1:TCET+Y5Gp1YgHT8py4nlg2Sew8nUHMqcpousDgXJVQc= +cloud.google.com/go/scheduler v1.9.0/go.mod h1:yexg5t+KSmqu+njTIh3b7oYPheFtBWGcbVUYF1GGMIc= +cloud.google.com/go/scheduler v1.10.1/go.mod h1:R63Ldltd47Bs4gnhQkmNDse5w8gBRrhObZ54PxgR2Oo= +cloud.google.com/go/secretmanager v1.6.0/go.mod h1:awVa/OXF6IiyaU1wQ34inzQNc4ISIDIrId8qE5QGgKA= +cloud.google.com/go/secretmanager v1.8.0/go.mod h1:hnVgi/bN5MYHd3Gt0SPuTPPp5ENina1/LxM+2W9U9J4= +cloud.google.com/go/secretmanager v1.9.0/go.mod h1:b71qH2l1yHmWQHt9LC80akm86mX8AL6X1MA01dW8ht4= +cloud.google.com/go/secretmanager v1.10.0/go.mod h1:MfnrdvKMPNra9aZtQFvBcvRU54hbPD8/HayQdlUgJpU= +cloud.google.com/go/secretmanager v1.11.1/go.mod h1:znq9JlXgTNdBeQk9TBW/FnR/W4uChEKGeqQWAJ8SXFw= +cloud.google.com/go/security v1.5.0/go.mod h1:lgxGdyOKKjHL4YG3/YwIL2zLqMFCKs0UbQwgyZmfJl4= +cloud.google.com/go/security v1.7.0/go.mod h1:mZklORHl6Bg7CNnnjLH//0UlAlaXqiG7Lb9PsPXLfD0= +cloud.google.com/go/security v1.8.0/go.mod h1:hAQOwgmaHhztFhiQ41CjDODdWP0+AE1B3sX4OFlq+GU= +cloud.google.com/go/security v1.9.0/go.mod h1:6Ta1bO8LXI89nZnmnsZGp9lVoVWXqsVbIq/t9dzI+2Q= +cloud.google.com/go/security v1.10.0/go.mod h1:QtOMZByJVlibUT2h9afNDWRZ1G96gVywH8T5GUSb9IA= +cloud.google.com/go/security v1.12.0/go.mod h1:rV6EhrpbNHrrxqlvW0BWAIawFWq3X90SduMJdFwtLB8= +cloud.google.com/go/security v1.13.0/go.mod h1:Q1Nvxl1PAgmeW0y3HTt54JYIvUdtcpYKVfIB8AOMZ+0= +cloud.google.com/go/security v1.15.1/go.mod h1:MvTnnbsWnehoizHi09zoiZob0iCHVcL4AUBj76h9fXA= +cloud.google.com/go/securitycenter v1.13.0/go.mod h1:cv5qNAqjY84FCN6Y9z28WlkKXyWsgLO832YiWwkCWcU= +cloud.google.com/go/securitycenter v1.14.0/go.mod h1:gZLAhtyKv85n52XYWt6RmeBdydyxfPeTrpToDPw4Auc= +cloud.google.com/go/securitycenter v1.15.0/go.mod h1:PeKJ0t8MoFmmXLXWm41JidyzI3PJjd8sXWaVqg43WWk= +cloud.google.com/go/securitycenter v1.16.0/go.mod h1:Q9GMaLQFUD+5ZTabrbujNWLtSLZIZF7SAR0wWECrjdk= +cloud.google.com/go/securitycenter v1.18.1/go.mod h1:0/25gAzCM/9OL9vVx4ChPeM/+DlfGQJDwBy/UC8AKK0= +cloud.google.com/go/securitycenter v1.19.0/go.mod h1:LVLmSg8ZkkyaNy4u7HCIshAngSQ8EcIRREP3xBnyfag= +cloud.google.com/go/securitycenter v1.23.0/go.mod h1:8pwQ4n+Y9WCWM278R8W3nF65QtY172h4S8aXyI9/hsQ= +cloud.google.com/go/servicecontrol v1.4.0/go.mod h1:o0hUSJ1TXJAmi/7fLJAedOovnujSEvjKCAFNXPQ1RaU= +cloud.google.com/go/servicecontrol v1.5.0/go.mod h1:qM0CnXHhyqKVuiZnGKrIurvVImCs8gmqWsDoqe9sU1s= +cloud.google.com/go/servicecontrol v1.10.0/go.mod h1:pQvyvSRh7YzUF2efw7H87V92mxU8FnFDawMClGCNuAA= +cloud.google.com/go/servicecontrol v1.11.0/go.mod h1:kFmTzYzTUIuZs0ycVqRHNaNhgR+UMUpw9n02l/pY+mc= +cloud.google.com/go/servicecontrol v1.11.1/go.mod h1:aSnNNlwEFBY+PWGQ2DoM0JJ/QUXqV5/ZD9DOLB7SnUk= +cloud.google.com/go/servicedirectory v1.4.0/go.mod h1:gH1MUaZCgtP7qQiI+F+A+OpeKF/HQWgtAddhTbhL2bs= +cloud.google.com/go/servicedirectory v1.5.0/go.mod h1:QMKFL0NUySbpZJ1UZs3oFAmdvVxhhxB6eJ/Vlp73dfg= +cloud.google.com/go/servicedirectory v1.6.0/go.mod h1:pUlbnWsLH9c13yGkxCmfumWEPjsRs1RlmJ4pqiNjVL4= +cloud.google.com/go/servicedirectory v1.7.0/go.mod h1:5p/U5oyvgYGYejufvxhgwjL8UVXjkuw7q5XcG10wx1U= +cloud.google.com/go/servicedirectory v1.8.0/go.mod h1:srXodfhY1GFIPvltunswqXpVxFPpZjf8nkKQT7XcXaY= +cloud.google.com/go/servicedirectory v1.9.0/go.mod h1:29je5JjiygNYlmsGz8k6o+OZ8vd4f//bQLtvzkPPT/s= +cloud.google.com/go/servicedirectory v1.10.1/go.mod h1:Xv0YVH8s4pVOwfM/1eMTl0XJ6bzIOSLDt8f8eLaGOxQ= +cloud.google.com/go/servicemanagement v1.4.0/go.mod h1:d8t8MDbezI7Z2R1O/wu8oTggo3BI2GKYbdG4y/SJTco= +cloud.google.com/go/servicemanagement v1.5.0/go.mod h1:XGaCRe57kfqu4+lRxaFEAuqmjzF0r+gWHjWqKqBvKFo= +cloud.google.com/go/servicemanagement v1.6.0/go.mod h1:aWns7EeeCOtGEX4OvZUWCCJONRZeFKiptqKf1D0l/Jc= +cloud.google.com/go/servicemanagement v1.8.0/go.mod h1:MSS2TDlIEQD/fzsSGfCdJItQveu9NXnUniTrq/L8LK4= +cloud.google.com/go/serviceusage v1.3.0/go.mod h1:Hya1cozXM4SeSKTAgGXgj97GlqUvF5JaoXacR1JTP/E= +cloud.google.com/go/serviceusage v1.4.0/go.mod h1:SB4yxXSaYVuUBYUml6qklyONXNLt83U0Rb+CXyhjEeU= +cloud.google.com/go/serviceusage v1.5.0/go.mod h1:w8U1JvqUqwJNPEOTQjrMHkw3IaIFLoLsPLvsE3xueec= +cloud.google.com/go/serviceusage v1.6.0/go.mod h1:R5wwQcbOWsyuOfbP9tGdAnCAc6B9DRwPG1xtWMDeuPA= +cloud.google.com/go/shell v1.3.0/go.mod h1:VZ9HmRjZBsjLGXusm7K5Q5lzzByZmJHf1d0IWHEN5X4= +cloud.google.com/go/shell v1.4.0/go.mod h1:HDxPzZf3GkDdhExzD/gs8Grqk+dmYcEjGShZgYa9URw= +cloud.google.com/go/shell v1.6.0/go.mod h1:oHO8QACS90luWgxP3N9iZVuEiSF84zNyLytb+qE2f9A= +cloud.google.com/go/shell v1.7.1/go.mod h1:u1RaM+huXFaTojTbW4g9P5emOrrmLE69KrxqQahKn4g= +cloud.google.com/go/spanner v1.41.0/go.mod h1:MLYDBJR/dY4Wt7ZaMIQ7rXOTLjYrmxLE/5ve9vFfWos= +cloud.google.com/go/spanner v1.44.0/go.mod h1:G8XIgYdOK+Fbcpbs7p2fiprDw4CaZX63whnSMLVBxjk= +cloud.google.com/go/spanner v1.45.0/go.mod h1:FIws5LowYz8YAE1J8fOS7DJup8ff7xJeetWEo5REA2M= +cloud.google.com/go/spanner v1.47.0/go.mod h1:IXsJwVW2j4UKs0eYDqodab6HgGuA1bViSqW4uH9lfUI= +cloud.google.com/go/speech v1.6.0/go.mod h1:79tcr4FHCimOp56lwC01xnt/WPJZc4v3gzyT7FoBkCM= +cloud.google.com/go/speech v1.7.0/go.mod h1:KptqL+BAQIhMsj1kOP2la5DSEEerPDuOP/2mmkhHhZQ= +cloud.google.com/go/speech v1.8.0/go.mod h1:9bYIl1/tjsAnMgKGHKmBZzXKEkGgtU+MpdDPTE9f7y0= +cloud.google.com/go/speech v1.9.0/go.mod h1:xQ0jTcmnRFFM2RfX/U+rk6FQNUF6DQlydUSyoooSpco= +cloud.google.com/go/speech v1.14.1/go.mod h1:gEosVRPJ9waG7zqqnsHpYTOoAS4KouMRLDFMekpJ0J0= +cloud.google.com/go/speech v1.15.0/go.mod h1:y6oH7GhqCaZANH7+Oe0BhgIogsNInLlz542tg3VqeYI= +cloud.google.com/go/speech v1.17.1/go.mod h1:8rVNzU43tQvxDaGvqOhpDqgkJTFowBpDvCJ14kGlJYo= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= +cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= +cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc= +cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= +cloud.google.com/go/storage v1.28.1/go.mod h1:Qnisd4CqDdo6BGs2AD5LLnEsmSQ80wQ5ogcBBKhU86Y= +cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4= +cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7BiccwkR7+P7gN8E= +cloud.google.com/go/storagetransfer v1.5.0/go.mod h1:dxNzUopWy7RQevYFHewchb29POFv3/AaBgnhqzqiK0w= +cloud.google.com/go/storagetransfer v1.6.0/go.mod h1:y77xm4CQV/ZhFZH75PLEXY0ROiS7Gh6pSKrM8dJyg6I= +cloud.google.com/go/storagetransfer v1.7.0/go.mod h1:8Giuj1QNb1kfLAiWM1bN6dHzfdlDAVC9rv9abHot2W4= +cloud.google.com/go/storagetransfer v1.8.0/go.mod h1:JpegsHHU1eXg7lMHkvf+KE5XDJ7EQu0GwNJbbVGanEw= +cloud.google.com/go/storagetransfer v1.10.0/go.mod h1:DM4sTlSmGiNczmV6iZyceIh2dbs+7z2Ayg6YAiQlYfA= +cloud.google.com/go/talent v1.1.0/go.mod h1:Vl4pt9jiHKvOgF9KoZo6Kob9oV4lwd/ZD5Cto54zDRw= +cloud.google.com/go/talent v1.2.0/go.mod h1:MoNF9bhFQbiJ6eFD3uSsg0uBALw4n4gaCaEjBw9zo8g= +cloud.google.com/go/talent v1.3.0/go.mod h1:CmcxwJ/PKfRgd1pBjQgU6W3YBwiewmUzQYH5HHmSCmM= +cloud.google.com/go/talent v1.4.0/go.mod h1:ezFtAgVuRf8jRsvyE6EwmbTK5LKciD4KVnHuDEFmOOA= +cloud.google.com/go/talent v1.5.0/go.mod h1:G+ODMj9bsasAEJkQSzO2uHQWXHHXUomArjWQQYkqK6c= +cloud.google.com/go/talent v1.6.2/go.mod h1:CbGvmKCG61mkdjcqTcLOkb2ZN1SrQI8MDyma2l7VD24= +cloud.google.com/go/texttospeech v1.4.0/go.mod h1:FX8HQHA6sEpJ7rCMSfXuzBcysDAuWusNNNvN9FELDd8= +cloud.google.com/go/texttospeech v1.5.0/go.mod h1:oKPLhR4n4ZdQqWKURdwxMy0uiTS1xU161C8W57Wkea4= +cloud.google.com/go/texttospeech v1.6.0/go.mod h1:YmwmFT8pj1aBblQOI3TfKmwibnsfvhIBzPXcW4EBovc= +cloud.google.com/go/texttospeech v1.7.1/go.mod h1:m7QfG5IXxeneGqTapXNxv2ItxP/FS0hCZBwXYqucgSk= +cloud.google.com/go/tpu v1.3.0/go.mod h1:aJIManG0o20tfDQlRIej44FcwGGl/cD0oiRyMKG19IQ= +cloud.google.com/go/tpu v1.4.0/go.mod h1:mjZaX8p0VBgllCzF6wcU2ovUXN9TONFLd7iz227X2Xg= +cloud.google.com/go/tpu v1.5.0/go.mod h1:8zVo1rYDFuW2l4yZVY0R0fb/v44xLh3llq7RuV61fPM= +cloud.google.com/go/tpu v1.6.1/go.mod h1:sOdcHVIgDEEOKuqUoi6Fq53MKHJAtOwtz0GuKsWSH3E= +cloud.google.com/go/trace v1.3.0/go.mod h1:FFUE83d9Ca57C+K8rDl/Ih8LwOzWIV1krKgxg6N0G28= +cloud.google.com/go/trace v1.4.0/go.mod h1:UG0v8UBqzusp+z63o7FK74SdFE+AXpCLdFb1rshXG+Y= +cloud.google.com/go/trace v1.8.0/go.mod h1:zH7vcsbAhklH8hWFig58HvxcxyQbaIqMarMg9hn5ECA= +cloud.google.com/go/trace v1.9.0/go.mod h1:lOQqpE5IaWY0Ixg7/r2SjixMuc6lfTFeO4QGM4dQWOk= +cloud.google.com/go/trace v1.10.1/go.mod h1:gbtL94KE5AJLH3y+WVpfWILmqgc6dXcqgNXdOPAQTYk= +cloud.google.com/go/translate v1.3.0/go.mod h1:gzMUwRjvOqj5i69y/LYLd8RrNQk+hOmIXTi9+nb3Djs= +cloud.google.com/go/translate v1.4.0/go.mod h1:06Dn/ppvLD6WvA5Rhdp029IX2Mi3Mn7fpMRLPvXT5Wg= +cloud.google.com/go/translate v1.5.0/go.mod h1:29YDSYveqqpA1CQFD7NQuP49xymq17RXNaUDdc0mNu0= +cloud.google.com/go/translate v1.6.0/go.mod h1:lMGRudH1pu7I3n3PETiOB2507gf3HnfLV8qlkHZEyos= +cloud.google.com/go/translate v1.7.0/go.mod h1:lMGRudH1pu7I3n3PETiOB2507gf3HnfLV8qlkHZEyos= +cloud.google.com/go/translate v1.8.1/go.mod h1:d1ZH5aaOA0CNhWeXeC8ujd4tdCFw8XoNWRljklu5RHs= +cloud.google.com/go/video v1.8.0/go.mod h1:sTzKFc0bUSByE8Yoh8X0mn8bMymItVGPfTuUBUyRgxk= +cloud.google.com/go/video v1.9.0/go.mod h1:0RhNKFRF5v92f8dQt0yhaHrEuH95m068JYOvLZYnJSw= +cloud.google.com/go/video v1.12.0/go.mod h1:MLQew95eTuaNDEGriQdcYn0dTwf9oWiA4uYebxM5kdg= +cloud.google.com/go/video v1.13.0/go.mod h1:ulzkYlYgCp15N2AokzKjy7MQ9ejuynOJdf1tR5lGthk= +cloud.google.com/go/video v1.14.0/go.mod h1:SkgaXwT+lIIAKqWAJfktHT/RbgjSuY6DobxEp0C5yTQ= +cloud.google.com/go/video v1.15.0/go.mod h1:SkgaXwT+lIIAKqWAJfktHT/RbgjSuY6DobxEp0C5yTQ= +cloud.google.com/go/video v1.17.1/go.mod h1:9qmqPqw/Ib2tLqaeHgtakU+l5TcJxCJbhFXM7UJjVzU= +cloud.google.com/go/videointelligence v1.6.0/go.mod h1:w0DIDlVRKtwPCn/C4iwZIJdvC69yInhW0cfi+p546uU= +cloud.google.com/go/videointelligence v1.7.0/go.mod h1:k8pI/1wAhjznARtVT9U1llUaFNPh7muw8QyOUpavru4= +cloud.google.com/go/videointelligence v1.8.0/go.mod h1:dIcCn4gVDdS7yte/w+koiXn5dWVplOZkE+xwG9FgK+M= +cloud.google.com/go/videointelligence v1.9.0/go.mod h1:29lVRMPDYHikk3v8EdPSaL8Ku+eMzDljjuvRs105XoU= +cloud.google.com/go/videointelligence v1.10.0/go.mod h1:LHZngX1liVtUhZvi2uNS0VQuOzNi2TkY1OakiuoUOjU= +cloud.google.com/go/videointelligence v1.11.1/go.mod h1:76xn/8InyQHarjTWsBR058SmlPCwQjgcvoW0aZykOvo= +cloud.google.com/go/vision v1.2.0/go.mod h1:SmNwgObm5DpFBme2xpyOyasvBc1aPdjvMk2bBk0tKD0= +cloud.google.com/go/vision/v2 v2.2.0/go.mod h1:uCdV4PpN1S0jyCyq8sIM42v2Y6zOLkZs+4R9LrGYwFo= +cloud.google.com/go/vision/v2 v2.3.0/go.mod h1:UO61abBx9QRMFkNBbf1D8B1LXdS2cGiiCRx0vSpZoUo= +cloud.google.com/go/vision/v2 v2.4.0/go.mod h1:VtI579ll9RpVTrdKdkMzckdnwMyX2JILb+MhPqRbPsY= +cloud.google.com/go/vision/v2 v2.5.0/go.mod h1:MmaezXOOE+IWa+cS7OhRRLK2cNv1ZL98zhqFFZaaH2E= +cloud.google.com/go/vision/v2 v2.6.0/go.mod h1:158Hes0MvOS9Z/bDMSFpjwsUrZ5fPrdwuyyvKSGAGMY= +cloud.google.com/go/vision/v2 v2.7.0/go.mod h1:H89VysHy21avemp6xcf9b9JvZHVehWbET0uT/bcuY/0= +cloud.google.com/go/vision/v2 v2.7.2/go.mod h1:jKa8oSYBWhYiXarHPvP4USxYANYUEdEsQrloLjrSwJU= +cloud.google.com/go/vmmigration v1.2.0/go.mod h1:IRf0o7myyWFSmVR1ItrBSFLFD/rJkfDCUTO4vLlJvsE= +cloud.google.com/go/vmmigration v1.3.0/go.mod h1:oGJ6ZgGPQOFdjHuocGcLqX4lc98YQ7Ygq8YQwHh9A7g= +cloud.google.com/go/vmmigration v1.5.0/go.mod h1:E4YQ8q7/4W9gobHjQg4JJSgXXSgY21nA5r8swQV+Xxc= +cloud.google.com/go/vmmigration v1.6.0/go.mod h1:bopQ/g4z+8qXzichC7GW1w2MjbErL54rk3/C843CjfY= +cloud.google.com/go/vmmigration v1.7.1/go.mod h1:WD+5z7a/IpZ5bKK//YmT9E047AD+rjycCAvyMxGJbro= +cloud.google.com/go/vmwareengine v0.1.0/go.mod h1:RsdNEf/8UDvKllXhMz5J40XxDrNJNN4sagiox+OI208= +cloud.google.com/go/vmwareengine v0.2.2/go.mod h1:sKdctNJxb3KLZkE/6Oui94iw/xs9PRNC2wnNLXsHvH8= +cloud.google.com/go/vmwareengine v0.3.0/go.mod h1:wvoyMvNWdIzxMYSpH/R7y2h5h3WFkx6d+1TIsP39WGY= +cloud.google.com/go/vmwareengine v0.4.1/go.mod h1:Px64x+BvjPZwWuc4HdmVhoygcXqEkGHXoa7uyfTgSI0= +cloud.google.com/go/vpcaccess v1.4.0/go.mod h1:aQHVbTWDYUR1EbTApSVvMq1EnT57ppDmQzZ3imqIk4w= +cloud.google.com/go/vpcaccess v1.5.0/go.mod h1:drmg4HLk9NkZpGfCmZ3Tz0Bwnm2+DKqViEpeEpOq0m8= +cloud.google.com/go/vpcaccess v1.6.0/go.mod h1:wX2ILaNhe7TlVa4vC5xce1bCnqE3AeH27RV31lnmZes= +cloud.google.com/go/vpcaccess v1.7.1/go.mod h1:FogoD46/ZU+JUBX9D606X21EnxiszYi2tArQwLY4SXs= +cloud.google.com/go/webrisk v1.4.0/go.mod h1:Hn8X6Zr+ziE2aNd8SliSDWpEnSS1u4R9+xXZmFiHmGE= +cloud.google.com/go/webrisk v1.5.0/go.mod h1:iPG6fr52Tv7sGk0H6qUFzmL3HHZev1htXuWDEEsqMTg= +cloud.google.com/go/webrisk v1.6.0/go.mod h1:65sW9V9rOosnc9ZY7A7jsy1zoHS5W9IAXv6dGqhMQMc= +cloud.google.com/go/webrisk v1.7.0/go.mod h1:mVMHgEYH0r337nmt1JyLthzMr6YxwN1aAIEc2fTcq7A= +cloud.google.com/go/webrisk v1.8.0/go.mod h1:oJPDuamzHXgUc+b8SiHRcVInZQuybnvEW72PqTc7sSg= +cloud.google.com/go/webrisk v1.9.1/go.mod h1:4GCmXKcOa2BZcZPn6DCEvE7HypmEJcJkr4mtM+sqYPc= +cloud.google.com/go/websecurityscanner v1.3.0/go.mod h1:uImdKm2wyeXQevQJXeh8Uun/Ym1VqworNDlBXQevGMo= +cloud.google.com/go/websecurityscanner v1.4.0/go.mod h1:ebit/Fp0a+FWu5j4JOmJEV8S8CzdTkAS77oDsiSqYWQ= +cloud.google.com/go/websecurityscanner v1.5.0/go.mod h1:Y6xdCPy81yi0SQnDY1xdNTNpfY1oAgXUlcfN3B3eSng= +cloud.google.com/go/websecurityscanner v1.6.1/go.mod h1:Njgaw3rttgRHXzwCB8kgCYqv5/rGpFCsBOvPbYgszpg= +cloud.google.com/go/workflows v1.6.0/go.mod h1:6t9F5h/unJz41YqfBmqSASJSXccBLtD1Vwf+KmJENM0= +cloud.google.com/go/workflows v1.7.0/go.mod h1:JhSrZuVZWuiDfKEFxU0/F1PQjmpnpcoISEXH2bcHC3M= +cloud.google.com/go/workflows v1.8.0/go.mod h1:ysGhmEajwZxGn1OhGOGKsTXc5PyxOc0vfKf5Af+to4M= +cloud.google.com/go/workflows v1.9.0/go.mod h1:ZGkj1aFIOd9c8Gerkjjq7OW7I5+l6cSvT3ujaO/WwSA= +cloud.google.com/go/workflows v1.10.0/go.mod h1:fZ8LmRmZQWacon9UCX1r/g/DfAXx5VcPALq2CxzdePw= +cloud.google.com/go/workflows v1.11.1/go.mod h1:Z+t10G1wF7h8LgdY/EmRcQY8ptBD/nvofaL6FqlET6g= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zumjgTw83q2ge/PI+yyw8= +git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3pc= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= +github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm/4RlzPXRlREEwqTHAN3T56Bv2ITsFT3gY= +github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19/go.mod h1:T13YZdzov6OU0A1+RfKZiZN9ca6VeKdBdyDV+BY97Tk= +github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= +github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b/go.mod h1:1KcenG0jGWcpt8ov532z81sp/kMMUG485J2InIOyADM= +github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/apache/arrow/go/v10 v10.0.1/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0= +github.com/apache/arrow/go/v11 v11.0.0/go.mod h1:Eg5OsL5H+e299f7u5ssuXsuHQVEGC4xei5aX110hRiI= +github.com/apache/arrow/go/v12 v12.0.0/go.mod h1:d+tV/eHZZ7Dz7RPrFKtPK02tpr+c9/PEd/zm8mDS9Vg= +github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= +github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= +github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g= +github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f h1:WBZRG4aNOuI15bLRrCgN8fCq8E5Xuty6jGbmSNEvSsU= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe h1:QQ3GSy+MqSHxm/d8nCtnAiZdYFd45cYZPs8vOOIYKfk= +github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20230310173818-32f1caf87195/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20230428030218-4003588d1b74/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= +github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/envoyproxy/go-control-plane v0.9.4 h1:rEvIZUSZ3fx39WIi3JkQqQBitGwpELBIYWeBVh6wn+E= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/protoc-gen-validate v0.1.0 h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= +github.com/envoyproxy/go-control-plane v0.10.3/go.mod h1:fJJn/j26vwOu972OllsvAgJJM//w9BV6Fxbg2LuVd34= +github.com/envoyproxy/go-control-plane v0.11.0/go.mod h1:VnHyVMpzcLvCFt9yUz1UnCwHLhwx1WguiVDV7pTG/tI= +github.com/envoyproxy/go-control-plane v0.11.1 h1:wSUXTlLfiAQRWs2F+p+EKOY9rUyis1MyGqJ2DIk5HpM= +github.com/envoyproxy/go-control-plane v0.11.1/go.mod h1:uhMcXKCQMEJHiAb0w+YGefQLaTEw+YhGluxZkrTmD0g= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v0.6.7/go.mod h1:dyJXwwfPK2VSqiB9Klm1J6romD608Ba7Hij42vrOBCo= +github.com/envoyproxy/protoc-gen-validate v0.9.1/go.mod h1:OKNgG7TCp5pF4d6XftA0++PMirau2/yoOwVac3AbF2w= +github.com/envoyproxy/protoc-gen-validate v0.10.0/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss= +github.com/envoyproxy/protoc-gen-validate v1.0.1/go.mod h1:0vj8bNkYbSTNS2PIyH87KZaeN4x9zpL9Qt8fQC7d+vs= +github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA= +github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= +github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-fonts/dejavu v0.1.0/go.mod h1:4Wt4I4OU2Nq9asgDCteaAaWZOV24E+0/Pwo0gppep4g= +github.com/go-fonts/latin-modern v0.2.0/go.mod h1:rQVLdDMK+mK1xscDwsqM5J8U2jrRa3T0ecnM9pNujks= +github.com/go-fonts/liberation v0.1.1/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY= +github.com/go-fonts/liberation v0.2.0/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY= +github.com/go-fonts/stix v0.1.0/go.mod h1:w/c1f0ldAUlJmLBvlbkvVXLAD+tAMqobIIQpmnUIzUY= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2CSIqUrmQPqA0gdRIlnLEY0gK5JGjh37zN5U= +github.com/go-latex/latex v0.0.0-20210823091927-c0d11ff05a81/go.mod h1:SX0U8uGpxhq9o2S/CELCSUxEWWAuoCUcVCQWv7G2OCk= +github.com/go-pdf/fpdf v0.5.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= +github.com/go-pdf/fpdf v0.6.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= +github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -67,24 +815,41 @@ github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0 h1:oOuy+ugB+P/kBdUnG5QaMXSIyJ1q38wWSojYCb3z5VQ= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/flatbuffers v2.0.8+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1 h1:JFrFEBb2xKufg6XkJsJr+WbKb4FQlURi5RUcBveYu9k= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -92,48 +857,177 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/s2a-go v0.1.0/go.mod h1:OJpEgntRZo8ugHpF9hkoLJbS5dSI20XZeXJ9JVywLlM= +github.com/google/s2a-go v0.1.3/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= +github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= +github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= +github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= +github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= +github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= +github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= +github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= +github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= +github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= +github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo= +github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY= +github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= +github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= +github.com/googleapis/gax-go/v2 v2.8.0/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= +github.com/googleapis/gax-go/v2 v2.10.0/go.mod h1:4UOEnMCrxsSqQ940WnTiD6qJ63le2ev3xfyagutxiPw= +github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI= +github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= +github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU0GSB0f8Nhgmxx0VIRUvaC0w= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= +github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lyft/protoc-gen-star v0.6.0/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= +github.com/lyft/protoc-gen-star v0.6.1/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= +github.com/lyft/protoc-gen-star/v2 v2.0.1/go.mod h1:RcCdONR2ScXaYnQC5tUzxzlpA3WVYF7/opLeUgcQs/o= +github.com/lyft/protoc-gen-star/v2 v2.0.3/go.mod h1:amey7yeodaJhXSbf/TlLvWiqQfLOSpEk//mLlc+axEk= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= +github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2dXMnm1mY= +github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= +github.com/phpdave11/gofpdi v1.0.13/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= +github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= +github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w= +github.com/ruudk/golang-pdf417 v0.0.0-20201230142125-a7e3863a1245/go.mod h1:pQAZKsJ8yyVxGRWYNEm9oFB8ieLgKFnamEyDmSA0BRk= +github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= +github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= +github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= +go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= +golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= +golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= +golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3/go.mod h1:NOZ3BPKG0ec/BKJQgnvsSFpcKLM5xXVWnvZS97DWHgE= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20220827204233-334a2380cb91/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= +golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20190910094157-69e4b8554b2a/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200430140353-33d19683fad8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200618115811-c13761719519/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20201208152932-35266b937fa6/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20210216034530-4410531fe030/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20210607152325-775e3b0c77b9/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= +golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= +golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= +golang.org/x/image v0.0.0-20220302094943-723b81ca9867/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -143,6 +1037,8 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -151,10 +1047,20 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -175,15 +1081,74 @@ golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.0.0-20221012135044-0b7e1fb9d458/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= +golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= +golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= +golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= +golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.0.0-20221006150949-b44042a4b9c1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= +golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= +golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= +golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= +golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= +golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -191,7 +1156,17 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -204,6 +1179,7 @@ golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -215,19 +1191,100 @@ golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642 h1:B6caxRw+hozq68X2MY7jEpZh/cr4/aHLv9xU8Kkadrw= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210304124612-50617c2ba197/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220829200755-d48e67d00261/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= +golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= +golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -240,6 +1297,7 @@ golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -265,13 +1323,44 @@ golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200806022845-90696ccdc692/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= +golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= +golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= +golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= +gonum.org/v1/gonum v0.9.3/go.mod h1:TZumC3NeyVQskjXqmyWt4S3bINhy7B4eYwW69EbyX+0= +gonum.org/v1/gonum v0.11.0/go.mod h1:fSG4YDCxxUZQJ7rKsQrj0gMOg00Il0Z96/qMA4bVQhA= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= +gonum.org/v1/plot v0.9.0/go.mod h1:3Pcqqmp6RHvJI72kgb8fThyUnav364FOsdDo2aGW5lY= +gonum.org/v1/plot v0.10.1/go.mod h1:VZW5OlhkL1mysU9vaqNHnsy86inf6Ot+jB3r+BczCEo= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -288,19 +1377,64 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= +google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= +google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= +google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= +google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= +google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= +google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= +google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= +google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g= +google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA= +google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8= +google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= +google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= +google.golang.org/api v0.77.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= +google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw= +google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg= +google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o= +google.golang.org/api v0.85.0/go.mod h1:AqZf8Ep9uZ2pyTvgL+x0D3Zt0eoT9b5E8fmzfu6FO2g= +google.golang.org/api v0.90.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= +google.golang.org/api v0.93.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= +google.golang.org/api v0.95.0/go.mod h1:eADj+UBuxkh5zlrSntJghuNeg8HwQ1w5lTKkuqaETEI= +google.golang.org/api v0.96.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= +google.golang.org/api v0.97.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= +google.golang.org/api v0.98.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= +google.golang.org/api v0.99.0/go.mod h1:1YOf74vkVndF7pG6hIHuINsM7eWwpVTAfNMNiL91A08= +google.golang.org/api v0.100.0/go.mod h1:ZE3Z2+ZOr87Rx7dqFsdRQkRBk36kDtp/h+QpHbB7a70= +google.golang.org/api v0.102.0/go.mod h1:3VFl6/fzoA+qNuS1N1/VfXY4LjoXN/wzeIp7TweWwGo= +google.golang.org/api v0.103.0/go.mod h1:hGtW6nK1AC+d9si/UBhw8Xli+QMOf6xyNAyJw4qU9w0= +google.golang.org/api v0.106.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= +google.golang.org/api v0.107.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= +google.golang.org/api v0.108.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= +google.golang.org/api v0.110.0/go.mod h1:7FC4Vvx1Mooxh8C5HWjzZHcavuS2f6pmJpZx60ca7iI= +google.golang.org/api v0.111.0/go.mod h1:qtFHvU9mhgTJegR31csQ+rwxyUTHOKFqCKWp1J0fdw0= +google.golang.org/api v0.114.0/go.mod h1:ifYI2ZsFK6/uGddGfAD5BMxlnkBqCmqHSDUVi45N5Yg= +google.golang.org/api v0.118.0/go.mod h1:76TtD3vkgmZ66zZzp72bUUklpmQmKlhh6sYtIjYK+5E= +google.golang.org/api v0.122.0/go.mod h1:gcitW0lvnyWjSp9nKxAbdHKIZ6vF4aajGueeslZOyms= +google.golang.org/api v0.124.0/go.mod h1:xu2HQurE5gi/3t1aFCvhPD781p0a3p11sdunTJ2BlP4= +google.golang.org/api v0.125.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw= +google.golang.org/api v0.126.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= @@ -318,30 +1452,168 @@ google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98 h1:LCO0fg4kb6WwkXQXRQQgUYsFeFb5taTX5WAx5O/Vt28= -google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210329143202-679c6ae281ee/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= +google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= +google.golang.org/genproto v0.0.0-20220329172620-7be39ac1afc7/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220628213854-d9e0b6570c03/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220722212130-b98a9ff5e252/go.mod h1:GkXuJDJ6aQ7lnJcRF+SJVgFdQhypqgl3LB1C9vabdRE= +google.golang.org/genproto v0.0.0-20220801145646-83ce21fca29f/go.mod h1:iHe1svFLAZg9VWz891+QbRMwUv9O/1Ww+/mngYeThbc= +google.golang.org/genproto v0.0.0-20220815135757-37a418bb8959/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220817144833-d7fd3f11b9b1/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220829144015-23454907ede3/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220829175752-36a9c930ecbf/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220913154956-18f8339a66a5/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220914142337-ca0e39ece12f/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220915135415-7fd63a7952de/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220916172020-2692e8806bfa/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220919141832-68c03719ef51/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220920201722-2b89144ce006/go.mod h1:ht8XFiar2npT/g4vkk7O0WYS1sHOHbdujxbEp7CJWbw= +google.golang.org/genproto v0.0.0-20220926165614-551eb538f295/go.mod h1:woMGP53BroOrRY3xTxlbr8Y3eB/nzAvvFM83q7kG2OI= +google.golang.org/genproto v0.0.0-20220926220553-6981cbe3cfce/go.mod h1:woMGP53BroOrRY3xTxlbr8Y3eB/nzAvvFM83q7kG2OI= +google.golang.org/genproto v0.0.0-20221010155953-15ba04fc1c0e/go.mod h1:3526vdqwhZAwq4wsRUaVG555sVgsNmIjRtO7t/JH29U= +google.golang.org/genproto v0.0.0-20221014173430-6e2ab493f96b/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= +google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= +google.golang.org/genproto v0.0.0-20221024153911-1573dae28c9c/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= +google.golang.org/genproto v0.0.0-20221024183307-1bc688fe9f3e/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= +google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c/go.mod h1:CGI5F/G+E5bKwmfYo09AXuVN4dD894kIKUFmVbP2/Fo= +google.golang.org/genproto v0.0.0-20221109142239-94d6d90a7d66/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221114212237-e4508ebdbee1/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221117204609-8f9c96812029/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221201204527-e3fa12d562f3/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd/go.mod h1:cTsE614GARnxrLsqKREzmNYJACSWWpAWdNMwnD7c2BE= +google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230112194545-e10362b5ecf9/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230113154510-dbe35b8444a5/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230123190316-2c411cf9d197/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230124163310-31e0e69b6fc2/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230125152338-dcaf20b6aeaa/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230127162408-596548ed4efa/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230209215440-0dfe4f8abfcc/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230216225411-c8e22ba71e44/go.mod h1:8B0gmkoRebU8ukX6HP+4wrVQUY1+6PkQ44BSyIlflHA= +google.golang.org/genproto v0.0.0-20230222225845-10f96fb3dbec/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw= +google.golang.org/genproto v0.0.0-20230223222841-637eb2293923/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw= +google.golang.org/genproto v0.0.0-20230303212802-e74f57abe488/go.mod h1:TvhZT5f700eVlTNwND1xoEZQeWTB2RY/65kplwl/bFA= +google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= +google.golang.org/genproto v0.0.0-20230320184635-7606e756e683/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= +google.golang.org/genproto v0.0.0-20230323212658-478b75c54725/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/genproto v0.0.0-20230330154414-c0448cd141ea/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= +google.golang.org/genproto v0.0.0-20230525234025-438c736192d0/go.mod h1:9ExIQyXL5hZrHzQceCwuSYwZZ5QZBazOcprJ5rgs3lY= +google.golang.org/genproto v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:zqTuNwFlFRsw5zIts5VnzLQxSRqh+CGOTVMlYbY0Eyk= +google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= +google.golang.org/genproto v0.0.0-20230629202037-9506855d4529/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= +google.golang.org/genproto v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:O9kGHb51iE/nOGvQaDUuadVYqovW56s5emA88lQnj6Y= +google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 h1:Z0hjGZePRE0ZBWotvtrwxFNrNE9CUAGtplaDK5NNI/g= +google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98/go.mod h1:S7mY02OqCJTD0E1OiQy1F72PWFB4bZJ87cAtLPYgDR0= +google.golang.org/genproto/googleapis/api v0.0.0-20230525234020-1aefcd67740a/go.mod h1:ts19tUU+Z0ZShN1y3aPyq2+O3d5FUNNgT6FtOzmrNn8= +google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto/googleapis/api v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto/googleapis/api v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:mPBs5jNgx2GuQGvFwUvVKqtn6HsUw9nP64BedgvqEsQ= +google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 h1:FmF5cCW94Ij59cfpoLiwTgodWmm60eEV0CjlsVg2fuw= +google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= +google.golang.org/genproto/googleapis/bytestream v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:ylj+BE99M198VPbBh6A8d9n3w8fChvyLK3wwBOjXBFA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234015-3fc162c6f38a/go.mod h1:xURIpW9ES5+/GZhnV6beoEtxQrnkRGIfP5VQG2tCBLc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:8mL13HKkDa+IuJ8yruA3ci0q+0vsUz4m//+ottjwS5o= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 h1:bVf09lpb+OJbByTj913DRJioFFAjf/ZGxEz7MajTp2U= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0 h1:qdOKuR/EIArgaWNjetjgTzgVTAZ+S/WXVrq9HW9zimw= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -349,6 +1621,59 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= +lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= +lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= +modernc.org/cc/v3 v3.36.0/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= +modernc.org/cc/v3 v3.36.2/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= +modernc.org/cc/v3 v3.36.3/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= +modernc.org/cc/v3 v3.37.0/go.mod h1:vtL+3mdHx/wcj3iEGz84rQa8vEqR6XM84v5Lcvfph20= +modernc.org/cc/v3 v3.40.0/go.mod h1:/bTg4dnWkSXowUO6ssQKnOV0yMVxDYNIsIrzqTFDGH0= +modernc.org/ccgo/v3 v3.0.0-20220428102840-41399a37e894/go.mod h1:eI31LL8EwEBKPpNpA4bU1/i+sKOwOrQy8D87zWUcRZc= +modernc.org/ccgo/v3 v3.0.0-20220430103911-bc99d88307be/go.mod h1:bwdAnOoaIt8Ax9YdWGjxWsdkPcZyRPHqrOvJxaKAKGw= +modernc.org/ccgo/v3 v3.0.0-20220904174949-82d86e1b6d56/go.mod h1:YSXjPL62P2AMSxBphRHPn7IkzhVHqkvOnRKAKh+W6ZI= +modernc.org/ccgo/v3 v3.16.4/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ= +modernc.org/ccgo/v3 v3.16.6/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ= +modernc.org/ccgo/v3 v3.16.8/go.mod h1:zNjwkizS+fIFDrDjIAgBSCLkWbJuHF+ar3QRn+Z9aws= +modernc.org/ccgo/v3 v3.16.9/go.mod h1:zNMzC9A9xeNUepy6KuZBbugn3c0Mc9TeiJO4lgvkJDo= +modernc.org/ccgo/v3 v3.16.13-0.20221017192402-261537637ce8/go.mod h1:fUB3Vn0nVPReA+7IG7yZDfjv1TMWjhQP8gCxrFAtL5g= +modernc.org/ccgo/v3 v3.16.13/go.mod h1:2Quk+5YgpImhPjv2Qsob1DnZ/4som1lJTodubIcoUkY= +modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ= +modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM= +modernc.org/libc v0.0.0-20220428101251-2d5f3daf273b/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA= +modernc.org/libc v1.16.0/go.mod h1:N4LD6DBE9cf+Dzf9buBlzVJndKr/iJHG97vGLHYnb5A= +modernc.org/libc v1.16.1/go.mod h1:JjJE0eu4yeK7tab2n4S1w8tlWd9MxXLRzheaRnAKymU= +modernc.org/libc v1.16.17/go.mod h1:hYIV5VZczAmGZAnG15Vdngn5HSF5cSkbvfz2B7GRuVU= +modernc.org/libc v1.16.19/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA= +modernc.org/libc v1.17.0/go.mod h1:XsgLldpP4aWlPlsjqKRdHPqCxCjISdHfM/yeWC5GyW0= +modernc.org/libc v1.17.1/go.mod h1:FZ23b+8LjxZs7XtFMbSzL/EhPxNbfZbErxEHc7cbD9s= +modernc.org/libc v1.17.4/go.mod h1:WNg2ZH56rDEwdropAJeZPQkXmDwh+JCA1s/htl6r2fA= +modernc.org/libc v1.18.0/go.mod h1:vj6zehR5bfc98ipowQOM2nIDUZnVew/wNC/2tOGS+q0= +modernc.org/libc v1.20.3/go.mod h1:ZRfIaEkgrYgZDl6pa4W39HgN5G/yDW+NRmNKZBDFrk0= +modernc.org/libc v1.21.4/go.mod h1:przBsL5RDOZajTVslkugzLBj1evTue36jEomFQOoYuI= +modernc.org/libc v1.22.2/go.mod h1:uvQavJ1pZ0hIoC/jfqNoMLURIMhKzINIWypNM17puug= +modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/mathutil v1.4.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/memory v1.1.1/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw= +modernc.org/memory v1.2.0/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw= +modernc.org/memory v1.2.1/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= +modernc.org/memory v1.3.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= +modernc.org/memory v1.4.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= +modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= +modernc.org/opt v0.1.1/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/sqlite v1.18.1/go.mod h1:6ho+Gow7oX5V+OiOQ6Tr4xeqbx13UZ6t+Fw9IRUG4d4= +modernc.org/sqlite v1.18.2/go.mod h1:kvrTLEWgxUcHa2GfHBQtanR1H9ht3hTJNtKpzH9k1u0= +modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw= +modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw= +modernc.org/tcl v1.13.1/go.mod h1:XOLfOwzhkljL4itZkK6T72ckMgvj0BDsnKNdZVUOecw= +modernc.org/tcl v1.13.2/go.mod h1:7CLiGIPo1M8Rv1Mitpv5akc2+8fxUd2y2UzC/MfMzy0= +modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +modernc.org/token v1.0.1/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +modernc.org/z v1.5.1/go.mod h1:eWFB510QWW5Th9YGZT81s+LwvaAs3Q2yr4sP0rmLkv8= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/examples/gotutorial.md b/examples/gotutorial.md index 2862b0a28b98..86993562e909 100644 --- a/examples/gotutorial.md +++ b/examples/gotutorial.md @@ -268,13 +268,7 @@ if err != nil { log.Fatalf("failed to listen: %v", err) } grpcServer := grpc.NewServer() -rgs := &routeGuideServer{} -pb.RegisterRouteGuideService(grpcServer, pb.RouteGuideService{ - GetFeature: rgs.GetFeature, - ListFeatures: rgs.ListFeatures, - RecordRoute: rgs.RecordRoute, - RouteChat: rgs.RouteChat, -}) +pb.RegisterRouteGuideServer(grpcServer, &routeGuideServer{}) ... // determine whether to use TLS grpcServer.Serve(lis) ``` diff --git a/examples/helloworld/README.md b/examples/helloworld/README.md new file mode 100644 index 000000000000..bb2138f26ffb --- /dev/null +++ b/examples/helloworld/README.md @@ -0,0 +1,29 @@ +# gRPC Hello World + +Follow these setup to run the [quick start][] example: + + 1. Get the code: + + ```console + $ go get google.golang.org/grpc/examples/helloworld/greeter_client + $ go get google.golang.org/grpc/examples/helloworld/greeter_server + ``` + + 2. Run the server: + + ```console + $ $(go env GOPATH)/bin/greeter_server & + ``` + + 3. Run the client: + + ```console + $ $(go env GOPATH)/bin/greeter_client + Greeting: Hello world + ``` + +For more details (including instructions for making a small change to the +example code) or if you're having trouble running this example, see [Quick +Start][]. + +[quick start]: https://grpc.io/docs/languages/go/quickstart diff --git a/examples/helloworld/greeter_client/main.go b/examples/helloworld/greeter_client/main.go index 0ca4cbaa344c..452906937dde 100644 --- a/examples/helloworld/greeter_client/main.go +++ b/examples/helloworld/greeter_client/main.go @@ -21,22 +21,28 @@ package main import ( "context" + "flag" "log" - "os" "time" "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" pb "google.golang.org/grpc/examples/helloworld/helloworld" ) const ( - address = "localhost:50051" defaultName = "world" ) +var ( + addr = flag.String("addr", "localhost:50051", "the address to connect to") + name = flag.String("name", defaultName, "Name to greet") +) + func main() { + flag.Parse() // Set up a connection to the server. - conn, err := grpc.Dial(address, grpc.WithInsecure(), grpc.WithBlock()) + conn, err := grpc.Dial(*addr, grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { log.Fatalf("did not connect: %v", err) } @@ -44,13 +50,9 @@ func main() { c := pb.NewGreeterClient(conn) // Contact the server and print out its response. - name := defaultName - if len(os.Args) > 1 { - name = os.Args[1] - } ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() - r, err := c.SayHello(ctx, &pb.HelloRequest{Name: name}) + r, err := c.SayHello(ctx, &pb.HelloRequest{Name: *name}) if err != nil { log.Fatalf("could not greet: %v", err) } diff --git a/examples/helloworld/greeter_server/main.go b/examples/helloworld/greeter_server/main.go index 4406f42011e4..7a62a9b9ff25 100644 --- a/examples/helloworld/greeter_server/main.go +++ b/examples/helloworld/greeter_server/main.go @@ -21,6 +21,8 @@ package main import ( "context" + "flag" + "fmt" "log" "net" @@ -28,23 +30,30 @@ import ( pb "google.golang.org/grpc/examples/helloworld/helloworld" ) -const ( - port = ":50051" +var ( + port = flag.Int("port", 50051, "The server port") ) -// sayHello implements helloworld.GreeterServer.SayHello -func sayHello(ctx context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) { +// server is used to implement helloworld.GreeterServer. +type server struct { + pb.UnimplementedGreeterServer +} + +// SayHello implements helloworld.GreeterServer +func (s *server) SayHello(ctx context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) { log.Printf("Received: %v", in.GetName()) return &pb.HelloReply{Message: "Hello " + in.GetName()}, nil } func main() { - lis, err := net.Listen("tcp", port) + flag.Parse() + lis, err := net.Listen("tcp", fmt.Sprintf(":%d", *port)) if err != nil { log.Fatalf("failed to listen: %v", err) } s := grpc.NewServer() - pb.RegisterGreeterService(s, &pb.GreeterService{SayHello: sayHello}) + pb.RegisterGreeterServer(s, &server{}) + log.Printf("server listening at %v", lis.Addr()) if err := s.Serve(lis); err != nil { log.Fatalf("failed to serve: %v", err) } diff --git a/examples/helloworld/helloworld/helloworld.pb.go b/examples/helloworld/helloworld/helloworld.pb.go index 531c792a1e7c..92299693d1c2 100644 --- a/examples/helloworld/helloworld/helloworld.pb.go +++ b/examples/helloworld/helloworld/helloworld.pb.go @@ -1,127 +1,235 @@ +// Copyright 2015 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.31.0 +// protoc v4.22.0 // source: examples/helloworld/helloworld/helloworld.proto package helloworld import ( - fmt "fmt" - proto "github.com/golang/protobuf/proto" - math "math" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" ) -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) // The request message containing the user's name. type HelloRequest struct { - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields -func (m *HelloRequest) Reset() { *m = HelloRequest{} } -func (m *HelloRequest) String() string { return proto.CompactTextString(m) } -func (*HelloRequest) ProtoMessage() {} -func (*HelloRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_b83ea99a5323a2c7, []int{0} + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` } -func (m *HelloRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_HelloRequest.Unmarshal(m, b) -} -func (m *HelloRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_HelloRequest.Marshal(b, m, deterministic) -} -func (m *HelloRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_HelloRequest.Merge(m, src) +func (x *HelloRequest) Reset() { + *x = HelloRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_examples_helloworld_helloworld_helloworld_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *HelloRequest) XXX_Size() int { - return xxx_messageInfo_HelloRequest.Size(m) + +func (x *HelloRequest) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *HelloRequest) XXX_DiscardUnknown() { - xxx_messageInfo_HelloRequest.DiscardUnknown(m) + +func (*HelloRequest) ProtoMessage() {} + +func (x *HelloRequest) ProtoReflect() protoreflect.Message { + mi := &file_examples_helloworld_helloworld_helloworld_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_HelloRequest proto.InternalMessageInfo +// Deprecated: Use HelloRequest.ProtoReflect.Descriptor instead. +func (*HelloRequest) Descriptor() ([]byte, []int) { + return file_examples_helloworld_helloworld_helloworld_proto_rawDescGZIP(), []int{0} +} -func (m *HelloRequest) GetName() string { - if m != nil { - return m.Name +func (x *HelloRequest) GetName() string { + if x != nil { + return x.Name } return "" } // The response message containing the greetings type HelloReply struct { - Message string `protobuf:"bytes,1,opt,name=message,proto3" json:"message,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields -func (m *HelloReply) Reset() { *m = HelloReply{} } -func (m *HelloReply) String() string { return proto.CompactTextString(m) } -func (*HelloReply) ProtoMessage() {} -func (*HelloReply) Descriptor() ([]byte, []int) { - return fileDescriptor_b83ea99a5323a2c7, []int{1} + Message string `protobuf:"bytes,1,opt,name=message,proto3" json:"message,omitempty"` } -func (m *HelloReply) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_HelloReply.Unmarshal(m, b) -} -func (m *HelloReply) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_HelloReply.Marshal(b, m, deterministic) -} -func (m *HelloReply) XXX_Merge(src proto.Message) { - xxx_messageInfo_HelloReply.Merge(m, src) +func (x *HelloReply) Reset() { + *x = HelloReply{} + if protoimpl.UnsafeEnabled { + mi := &file_examples_helloworld_helloworld_helloworld_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *HelloReply) XXX_Size() int { - return xxx_messageInfo_HelloReply.Size(m) + +func (x *HelloReply) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *HelloReply) XXX_DiscardUnknown() { - xxx_messageInfo_HelloReply.DiscardUnknown(m) + +func (*HelloReply) ProtoMessage() {} + +func (x *HelloReply) ProtoReflect() protoreflect.Message { + mi := &file_examples_helloworld_helloworld_helloworld_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_HelloReply proto.InternalMessageInfo +// Deprecated: Use HelloReply.ProtoReflect.Descriptor instead. +func (*HelloReply) Descriptor() ([]byte, []int) { + return file_examples_helloworld_helloworld_helloworld_proto_rawDescGZIP(), []int{1} +} -func (m *HelloReply) GetMessage() string { - if m != nil { - return m.Message +func (x *HelloReply) GetMessage() string { + if x != nil { + return x.Message } return "" } -func init() { - proto.RegisterType((*HelloRequest)(nil), "helloworld.HelloRequest") - proto.RegisterType((*HelloReply)(nil), "helloworld.HelloReply") +var File_examples_helloworld_helloworld_helloworld_proto protoreflect.FileDescriptor + +var file_examples_helloworld_helloworld_helloworld_proto_rawDesc = []byte{ + 0x0a, 0x2f, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73, 0x2f, 0x68, 0x65, 0x6c, 0x6c, 0x6f, + 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x2f, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x77, 0x6f, 0x72, 0x6c, 0x64, + 0x2f, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x12, 0x0a, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x22, 0x22, 0x0a, + 0x0c, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, + 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, + 0x65, 0x22, 0x26, 0x0a, 0x0a, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, + 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x32, 0x49, 0x0a, 0x07, 0x47, 0x72, 0x65, + 0x65, 0x74, 0x65, 0x72, 0x12, 0x3e, 0x0a, 0x08, 0x53, 0x61, 0x79, 0x48, 0x65, 0x6c, 0x6c, 0x6f, + 0x12, 0x18, 0x2e, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x2e, 0x48, 0x65, + 0x6c, 0x6c, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x68, 0x65, 0x6c, + 0x6c, 0x6f, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x2e, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x52, 0x65, 0x70, + 0x6c, 0x79, 0x22, 0x00, 0x42, 0x67, 0x0a, 0x1b, 0x69, 0x6f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, + 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73, 0x2e, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x77, 0x6f, + 0x72, 0x6c, 0x64, 0x42, 0x0f, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x57, 0x6f, 0x72, 0x6c, 0x64, 0x50, + 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x35, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67, + 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x65, + 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73, 0x2f, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x77, 0x6f, 0x72, + 0x6c, 0x64, 0x2f, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x62, 0x06, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_examples_helloworld_helloworld_helloworld_proto_rawDescOnce sync.Once + file_examples_helloworld_helloworld_helloworld_proto_rawDescData = file_examples_helloworld_helloworld_helloworld_proto_rawDesc +) + +func file_examples_helloworld_helloworld_helloworld_proto_rawDescGZIP() []byte { + file_examples_helloworld_helloworld_helloworld_proto_rawDescOnce.Do(func() { + file_examples_helloworld_helloworld_helloworld_proto_rawDescData = protoimpl.X.CompressGZIP(file_examples_helloworld_helloworld_helloworld_proto_rawDescData) + }) + return file_examples_helloworld_helloworld_helloworld_proto_rawDescData } -func init() { - proto.RegisterFile("examples/helloworld/helloworld/helloworld.proto", fileDescriptor_b83ea99a5323a2c7) +var file_examples_helloworld_helloworld_helloworld_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_examples_helloworld_helloworld_helloworld_proto_goTypes = []interface{}{ + (*HelloRequest)(nil), // 0: helloworld.HelloRequest + (*HelloReply)(nil), // 1: helloworld.HelloReply +} +var file_examples_helloworld_helloworld_helloworld_proto_depIdxs = []int32{ + 0, // 0: helloworld.Greeter.SayHello:input_type -> helloworld.HelloRequest + 1, // 1: helloworld.Greeter.SayHello:output_type -> helloworld.HelloReply + 1, // [1:2] is the sub-list for method output_type + 0, // [0:1] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name } -var fileDescriptor_b83ea99a5323a2c7 = []byte{ - // 205 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xd2, 0x4f, 0xad, 0x48, 0xcc, - 0x2d, 0xc8, 0x49, 0x2d, 0xd6, 0xcf, 0x48, 0xcd, 0xc9, 0xc9, 0x2f, 0xcf, 0x2f, 0xca, 0x49, 0xc1, - 0xce, 0xd4, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0x42, 0x88, 0x28, 0x29, 0x71, 0xf1, 0x78, - 0x80, 0x78, 0x41, 0xa9, 0x85, 0xa5, 0xa9, 0xc5, 0x25, 0x42, 0x42, 0x5c, 0x2c, 0x79, 0x89, 0xb9, - 0xa9, 0x12, 0x8c, 0x0a, 0x8c, 0x1a, 0x9c, 0x41, 0x60, 0xb6, 0x92, 0x1a, 0x17, 0x17, 0x54, 0x4d, - 0x41, 0x4e, 0xa5, 0x90, 0x04, 0x17, 0x7b, 0x6e, 0x6a, 0x71, 0x71, 0x62, 0x3a, 0x4c, 0x11, 0x8c, - 0x6b, 0xe4, 0xc9, 0xc5, 0xee, 0x5e, 0x94, 0x9a, 0x5a, 0x92, 0x5a, 0x24, 0x64, 0xc7, 0xc5, 0x11, - 0x9c, 0x58, 0x09, 0xd6, 0x25, 0x24, 0xa1, 0x87, 0xe4, 0x02, 0x64, 0xcb, 0xa4, 0xc4, 0xb0, 0xc8, - 0x14, 0xe4, 0x54, 0x2a, 0x31, 0x38, 0xa5, 0x73, 0x49, 0x67, 0xe6, 0xeb, 0xa5, 0x17, 0x15, 0x24, - 0xeb, 0xc1, 0x7c, 0x87, 0xa4, 0xd6, 0x89, 0x1f, 0xac, 0x38, 0x1c, 0xc4, 0x0e, 0x00, 0x79, 0x29, - 0x80, 0x31, 0xca, 0x34, 0x3d, 0x3f, 0x3f, 0x3d, 0x27, 0x55, 0x2f, 0x3d, 0x3f, 0x27, 0x31, 0x2f, - 0x5d, 0x2f, 0xbf, 0x28, 0x5d, 0x1f, 0xa4, 0x9d, 0x40, 0xe0, 0x24, 0xb1, 0x81, 0x83, 0xc4, 0x18, - 0x10, 0x00, 0x00, 0xff, 0xff, 0x46, 0xfe, 0x45, 0x5c, 0x45, 0x01, 0x00, 0x00, +func init() { file_examples_helloworld_helloworld_helloworld_proto_init() } +func file_examples_helloworld_helloworld_helloworld_proto_init() { + if File_examples_helloworld_helloworld_helloworld_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_examples_helloworld_helloworld_helloworld_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*HelloRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_examples_helloworld_helloworld_helloworld_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*HelloReply); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_examples_helloworld_helloworld_helloworld_proto_rawDesc, + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_examples_helloworld_helloworld_helloworld_proto_goTypes, + DependencyIndexes: file_examples_helloworld_helloworld_helloworld_proto_depIdxs, + MessageInfos: file_examples_helloworld_helloworld_helloworld_proto_msgTypes, + }.Build() + File_examples_helloworld_helloworld_helloworld_proto = out.File + file_examples_helloworld_helloworld_helloworld_proto_rawDesc = nil + file_examples_helloworld_helloworld_helloworld_proto_goTypes = nil + file_examples_helloworld_helloworld_helloworld_proto_depIdxs = nil } diff --git a/examples/helloworld/helloworld/helloworld_grpc.pb.go b/examples/helloworld/helloworld/helloworld_grpc.pb.go index b45a600cead2..55e4f31df3ca 100644 --- a/examples/helloworld/helloworld/helloworld_grpc.pb.go +++ b/examples/helloworld/helloworld/helloworld_grpc.pb.go @@ -1,4 +1,22 @@ +// Copyright 2015 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.3.0 +// - protoc v4.22.0 +// source: examples/helloworld/helloworld/helloworld.proto package helloworld @@ -11,8 +29,13 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. const _ = grpc.SupportPackageIsVersion7 +const ( + Greeter_SayHello_FullMethodName = "/helloworld.Greeter/SayHello" +) + // GreeterClient is the client API for Greeter service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. @@ -29,87 +52,74 @@ func NewGreeterClient(cc grpc.ClientConnInterface) GreeterClient { return &greeterClient{cc} } -var greeterSayHelloStreamDesc = &grpc.StreamDesc{ - StreamName: "SayHello", -} - func (c *greeterClient) SayHello(ctx context.Context, in *HelloRequest, opts ...grpc.CallOption) (*HelloReply, error) { out := new(HelloReply) - err := c.cc.Invoke(ctx, "/helloworld.Greeter/SayHello", in, out, opts...) + err := c.cc.Invoke(ctx, Greeter_SayHello_FullMethodName, in, out, opts...) if err != nil { return nil, err } return out, nil } -// GreeterService is the service API for Greeter service. -// Fields should be assigned to their respective handler implementations only before -// RegisterGreeterService is called. Any unassigned fields will result in the -// handler for that method returning an Unimplemented error. -type GreeterService struct { +// GreeterServer is the server API for Greeter service. +// All implementations must embed UnimplementedGreeterServer +// for forward compatibility +type GreeterServer interface { // Sends a greeting - SayHello func(context.Context, *HelloRequest) (*HelloReply, error) + SayHello(context.Context, *HelloRequest) (*HelloReply, error) + mustEmbedUnimplementedGreeterServer() } -func (s *GreeterService) sayHello(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - if s.SayHello == nil { - return nil, status.Errorf(codes.Unimplemented, "method SayHello not implemented") - } +// UnimplementedGreeterServer must be embedded to have forward compatible implementations. +type UnimplementedGreeterServer struct { +} + +func (UnimplementedGreeterServer) SayHello(context.Context, *HelloRequest) (*HelloReply, error) { + return nil, status.Errorf(codes.Unimplemented, "method SayHello not implemented") +} +func (UnimplementedGreeterServer) mustEmbedUnimplementedGreeterServer() {} + +// UnsafeGreeterServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to GreeterServer will +// result in compilation errors. +type UnsafeGreeterServer interface { + mustEmbedUnimplementedGreeterServer() +} + +func RegisterGreeterServer(s grpc.ServiceRegistrar, srv GreeterServer) { + s.RegisterService(&Greeter_ServiceDesc, srv) +} + +func _Greeter_SayHello_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(HelloRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { - return s.SayHello(ctx, in) + return srv.(GreeterServer).SayHello(ctx, in) } info := &grpc.UnaryServerInfo{ - Server: s, - FullMethod: "/helloworld.Greeter/SayHello", + Server: srv, + FullMethod: Greeter_SayHello_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return s.SayHello(ctx, req.(*HelloRequest)) + return srv.(GreeterServer).SayHello(ctx, req.(*HelloRequest)) } return interceptor(ctx, in, info, handler) } -// RegisterGreeterService registers a service implementation with a gRPC server. -func RegisterGreeterService(s grpc.ServiceRegistrar, srv *GreeterService) { - sd := grpc.ServiceDesc{ - ServiceName: "helloworld.Greeter", - Methods: []grpc.MethodDesc{ - { - MethodName: "SayHello", - Handler: srv.sayHello, - }, +// Greeter_ServiceDesc is the grpc.ServiceDesc for Greeter service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var Greeter_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "helloworld.Greeter", + HandlerType: (*GreeterServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "SayHello", + Handler: _Greeter_SayHello_Handler, }, - Streams: []grpc.StreamDesc{}, - Metadata: "examples/helloworld/helloworld/helloworld.proto", - } - - s.RegisterService(&sd, nil) -} - -// NewGreeterService creates a new GreeterService containing the -// implemented methods of the Greeter service in s. Any unimplemented -// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client. -// This includes situations where the method handler is misspelled or has the wrong -// signature. For this reason, this function should be used with great care and -// is not recommended to be used by most users. -func NewGreeterService(s interface{}) *GreeterService { - ns := &GreeterService{} - if h, ok := s.(interface { - SayHello(context.Context, *HelloRequest) (*HelloReply, error) - }); ok { - ns.SayHello = h.SayHello - } - return ns -} - -// UnstableGreeterService is the service API for Greeter service. -// New methods may be added to this interface if they are added to the service -// definition, which is not a backward-compatible change. For this reason, -// use of this type is not recommended. -type UnstableGreeterService interface { - // Sends a greeting - SayHello(context.Context, *HelloRequest) (*HelloReply, error) + }, + Streams: []grpc.StreamDesc{}, + Metadata: "examples/helloworld/helloworld/helloworld.proto", } diff --git a/examples/route_guide/README.md b/examples/route_guide/README.md index ddec3a0bb5b8..29343b1c6b59 100644 --- a/examples/route_guide/README.md +++ b/examples/route_guide/README.md @@ -4,11 +4,11 @@ perform unary, client streaming, server streaming and full duplex RPCs. Please refer to [gRPC Basics: Go](https://grpc.io/docs/tutorials/basic/go.html) for more information. -See the definition of the route guide service in routeguide/route_guide.proto. +See the definition of the route guide service in `routeguide/route_guide.proto`. # Run the sample code -To compile and run the server, assuming you are in the root of the route_guide -folder, i.e., .../examples/route_guide/, simply: +To compile and run the server, assuming you are in the root of the `route_guide` +folder, i.e., `.../examples/route_guide/`, simply: ```sh $ go run server/server.go diff --git a/examples/route_guide/client/client.go b/examples/route_guide/client/client.go index 3e9d4e1183cc..d027d2d6d42b 100644 --- a/examples/route_guide/client/client.go +++ b/examples/route_guide/client/client.go @@ -32,6 +32,7 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/examples/data" pb "google.golang.org/grpc/examples/route_guide/routeguide" ) @@ -39,8 +40,8 @@ import ( var ( tls = flag.Bool("tls", false, "Connection uses TLS if true, else plain TCP") caFile = flag.String("ca_file", "", "The file containing the CA root cert file") - serverAddr = flag.String("server_addr", "localhost:10000", "The server address in the format of host:port") - serverHostOverride = flag.String("server_host_override", "x.test.youtube.com", "The server name used to verify the hostname returned by the TLS handshake") + serverAddr = flag.String("addr", "localhost:50051", "The server address in the format of host:port") + serverHostOverride = flag.String("server_host_override", "x.test.example.com", "The server name used to verify the hostname returned by the TLS handshake") ) // printFeature gets the feature for the given point. @@ -50,7 +51,7 @@ func printFeature(client pb.RouteGuideClient, point *pb.Point) { defer cancel() feature, err := client.GetFeature(ctx, point) if err != nil { - log.Fatalf("%v.GetFeatures(_) = _, %v: ", client, err) + log.Fatalf("client.GetFeature failed: %v", err) } log.Println(feature) } @@ -62,7 +63,7 @@ func printFeatures(client pb.RouteGuideClient, rect *pb.Rectangle) { defer cancel() stream, err := client.ListFeatures(ctx, rect) if err != nil { - log.Fatalf("%v.ListFeatures(_) = _, %v", client, err) + log.Fatalf("client.ListFeatures failed: %v", err) } for { feature, err := stream.Recv() @@ -70,9 +71,10 @@ func printFeatures(client pb.RouteGuideClient, rect *pb.Rectangle) { break } if err != nil { - log.Fatalf("%v.ListFeatures(_) = _, %v", client, err) + log.Fatalf("client.ListFeatures failed: %v", err) } - log.Println(feature) + log.Printf("Feature: name: %q, point:(%v, %v)", feature.GetName(), + feature.GetLocation().GetLatitude(), feature.GetLocation().GetLongitude()) } } @@ -90,16 +92,16 @@ func runRecordRoute(client pb.RouteGuideClient) { defer cancel() stream, err := client.RecordRoute(ctx) if err != nil { - log.Fatalf("%v.RecordRoute(_) = _, %v", client, err) + log.Fatalf("client.RecordRoute failed: %v", err) } for _, point := range points { if err := stream.Send(point); err != nil { - log.Fatalf("%v.Send(%v) = %v", stream, point, err) + log.Fatalf("client.RecordRoute: stream.Send(%v) failed: %v", point, err) } } reply, err := stream.CloseAndRecv() if err != nil { - log.Fatalf("%v.CloseAndRecv() got error %v, want %v", stream, err, nil) + log.Fatalf("client.RecordRoute failed: %v", err) } log.Printf("Route summary: %v", reply) } @@ -118,7 +120,7 @@ func runRouteChat(client pb.RouteGuideClient) { defer cancel() stream, err := client.RouteChat(ctx) if err != nil { - log.Fatalf("%v.RouteChat(_) = _, %v", client, err) + log.Fatalf("client.RouteChat failed: %v", err) } waitc := make(chan struct{}) go func() { @@ -130,14 +132,14 @@ func runRouteChat(client pb.RouteGuideClient) { return } if err != nil { - log.Fatalf("Failed to receive a note : %v", err) + log.Fatalf("client.RouteChat failed: %v", err) } log.Printf("Got message %s at point(%d, %d)", in.Message, in.Location.Latitude, in.Location.Longitude) } }() for _, note := range notes { if err := stream.Send(note); err != nil { - log.Fatalf("Failed to send a note: %v", err) + log.Fatalf("client.RouteChat: stream.Send(%v) failed: %v", note, err) } } stream.CloseSend() @@ -159,14 +161,13 @@ func main() { } creds, err := credentials.NewClientTLSFromFile(*caFile, *serverHostOverride) if err != nil { - log.Fatalf("Failed to create TLS credentials %v", err) + log.Fatalf("Failed to create TLS credentials: %v", err) } opts = append(opts, grpc.WithTransportCredentials(creds)) } else { - opts = append(opts, grpc.WithInsecure()) + opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) } - opts = append(opts, grpc.WithBlock()) conn, err := grpc.Dial(*serverAddr, opts...) if err != nil { log.Fatalf("fail to dial: %v", err) diff --git a/examples/route_guide/routeguide/route_guide.pb.go b/examples/route_guide/routeguide/route_guide.pb.go index 78c856645a7b..0b215c8cd92d 100644 --- a/examples/route_guide/routeguide/route_guide.pb.go +++ b/examples/route_guide/routeguide/route_guide.pb.go @@ -1,72 +1,94 @@ +// Copyright 2015 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.31.0 +// protoc v4.22.0 // source: examples/route_guide/routeguide/route_guide.proto package routeguide import ( - fmt "fmt" - proto "github.com/golang/protobuf/proto" - math "math" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" ) -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) // Points are represented as latitude-longitude pairs in the E7 representation // (degrees multiplied by 10**7 and rounded to the nearest integer). // Latitudes should be in the range +/- 90 degrees and longitude should be in // the range +/- 180 degrees (inclusive). type Point struct { - Latitude int32 `protobuf:"varint,1,opt,name=latitude,proto3" json:"latitude,omitempty"` - Longitude int32 `protobuf:"varint,2,opt,name=longitude,proto3" json:"longitude,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields -func (m *Point) Reset() { *m = Point{} } -func (m *Point) String() string { return proto.CompactTextString(m) } -func (*Point) ProtoMessage() {} -func (*Point) Descriptor() ([]byte, []int) { - return fileDescriptor_af806a20656386f8, []int{0} + Latitude int32 `protobuf:"varint,1,opt,name=latitude,proto3" json:"latitude,omitempty"` + Longitude int32 `protobuf:"varint,2,opt,name=longitude,proto3" json:"longitude,omitempty"` } -func (m *Point) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Point.Unmarshal(m, b) -} -func (m *Point) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Point.Marshal(b, m, deterministic) -} -func (m *Point) XXX_Merge(src proto.Message) { - xxx_messageInfo_Point.Merge(m, src) +func (x *Point) Reset() { + *x = Point{} + if protoimpl.UnsafeEnabled { + mi := &file_examples_route_guide_routeguide_route_guide_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *Point) XXX_Size() int { - return xxx_messageInfo_Point.Size(m) + +func (x *Point) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *Point) XXX_DiscardUnknown() { - xxx_messageInfo_Point.DiscardUnknown(m) + +func (*Point) ProtoMessage() {} + +func (x *Point) ProtoReflect() protoreflect.Message { + mi := &file_examples_route_guide_routeguide_route_guide_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_Point proto.InternalMessageInfo +// Deprecated: Use Point.ProtoReflect.Descriptor instead. +func (*Point) Descriptor() ([]byte, []int) { + return file_examples_route_guide_routeguide_route_guide_proto_rawDescGZIP(), []int{0} +} -func (m *Point) GetLatitude() int32 { - if m != nil { - return m.Latitude +func (x *Point) GetLatitude() int32 { + if x != nil { + return x.Latitude } return 0 } -func (m *Point) GetLongitude() int32 { - if m != nil { - return m.Longitude +func (x *Point) GetLongitude() int32 { + if x != nil { + return x.Longitude } return 0 } @@ -74,50 +96,58 @@ func (m *Point) GetLongitude() int32 { // A latitude-longitude rectangle, represented as two diagonally opposite // points "lo" and "hi". type Rectangle struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // One corner of the rectangle. Lo *Point `protobuf:"bytes,1,opt,name=lo,proto3" json:"lo,omitempty"` // The other corner of the rectangle. - Hi *Point `protobuf:"bytes,2,opt,name=hi,proto3" json:"hi,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Hi *Point `protobuf:"bytes,2,opt,name=hi,proto3" json:"hi,omitempty"` } -func (m *Rectangle) Reset() { *m = Rectangle{} } -func (m *Rectangle) String() string { return proto.CompactTextString(m) } -func (*Rectangle) ProtoMessage() {} -func (*Rectangle) Descriptor() ([]byte, []int) { - return fileDescriptor_af806a20656386f8, []int{1} +func (x *Rectangle) Reset() { + *x = Rectangle{} + if protoimpl.UnsafeEnabled { + mi := &file_examples_route_guide_routeguide_route_guide_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *Rectangle) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Rectangle.Unmarshal(m, b) -} -func (m *Rectangle) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Rectangle.Marshal(b, m, deterministic) +func (x *Rectangle) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *Rectangle) XXX_Merge(src proto.Message) { - xxx_messageInfo_Rectangle.Merge(m, src) -} -func (m *Rectangle) XXX_Size() int { - return xxx_messageInfo_Rectangle.Size(m) -} -func (m *Rectangle) XXX_DiscardUnknown() { - xxx_messageInfo_Rectangle.DiscardUnknown(m) + +func (*Rectangle) ProtoMessage() {} + +func (x *Rectangle) ProtoReflect() protoreflect.Message { + mi := &file_examples_route_guide_routeguide_route_guide_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_Rectangle proto.InternalMessageInfo +// Deprecated: Use Rectangle.ProtoReflect.Descriptor instead. +func (*Rectangle) Descriptor() ([]byte, []int) { + return file_examples_route_guide_routeguide_route_guide_proto_rawDescGZIP(), []int{1} +} -func (m *Rectangle) GetLo() *Point { - if m != nil { - return m.Lo +func (x *Rectangle) GetLo() *Point { + if x != nil { + return x.Lo } return nil } -func (m *Rectangle) GetHi() *Point { - if m != nil { - return m.Hi +func (x *Rectangle) GetHi() *Point { + if x != nil { + return x.Hi } return nil } @@ -126,100 +156,116 @@ func (m *Rectangle) GetHi() *Point { // // If a feature could not be named, the name is empty. type Feature struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // The name of the feature. Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` // The point where the feature is detected. - Location *Point `protobuf:"bytes,2,opt,name=location,proto3" json:"location,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Location *Point `protobuf:"bytes,2,opt,name=location,proto3" json:"location,omitempty"` } -func (m *Feature) Reset() { *m = Feature{} } -func (m *Feature) String() string { return proto.CompactTextString(m) } -func (*Feature) ProtoMessage() {} -func (*Feature) Descriptor() ([]byte, []int) { - return fileDescriptor_af806a20656386f8, []int{2} +func (x *Feature) Reset() { + *x = Feature{} + if protoimpl.UnsafeEnabled { + mi := &file_examples_route_guide_routeguide_route_guide_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *Feature) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Feature.Unmarshal(m, b) -} -func (m *Feature) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Feature.Marshal(b, m, deterministic) -} -func (m *Feature) XXX_Merge(src proto.Message) { - xxx_messageInfo_Feature.Merge(m, src) +func (x *Feature) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *Feature) XXX_Size() int { - return xxx_messageInfo_Feature.Size(m) -} -func (m *Feature) XXX_DiscardUnknown() { - xxx_messageInfo_Feature.DiscardUnknown(m) + +func (*Feature) ProtoMessage() {} + +func (x *Feature) ProtoReflect() protoreflect.Message { + mi := &file_examples_route_guide_routeguide_route_guide_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_Feature proto.InternalMessageInfo +// Deprecated: Use Feature.ProtoReflect.Descriptor instead. +func (*Feature) Descriptor() ([]byte, []int) { + return file_examples_route_guide_routeguide_route_guide_proto_rawDescGZIP(), []int{2} +} -func (m *Feature) GetName() string { - if m != nil { - return m.Name +func (x *Feature) GetName() string { + if x != nil { + return x.Name } return "" } -func (m *Feature) GetLocation() *Point { - if m != nil { - return m.Location +func (x *Feature) GetLocation() *Point { + if x != nil { + return x.Location } return nil } // A RouteNote is a message sent while at a given point. type RouteNote struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // The location from which the message is sent. Location *Point `protobuf:"bytes,1,opt,name=location,proto3" json:"location,omitempty"` // The message to be sent. - Message string `protobuf:"bytes,2,opt,name=message,proto3" json:"message,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Message string `protobuf:"bytes,2,opt,name=message,proto3" json:"message,omitempty"` } -func (m *RouteNote) Reset() { *m = RouteNote{} } -func (m *RouteNote) String() string { return proto.CompactTextString(m) } -func (*RouteNote) ProtoMessage() {} -func (*RouteNote) Descriptor() ([]byte, []int) { - return fileDescriptor_af806a20656386f8, []int{3} +func (x *RouteNote) Reset() { + *x = RouteNote{} + if protoimpl.UnsafeEnabled { + mi := &file_examples_route_guide_routeguide_route_guide_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *RouteNote) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_RouteNote.Unmarshal(m, b) -} -func (m *RouteNote) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_RouteNote.Marshal(b, m, deterministic) -} -func (m *RouteNote) XXX_Merge(src proto.Message) { - xxx_messageInfo_RouteNote.Merge(m, src) -} -func (m *RouteNote) XXX_Size() int { - return xxx_messageInfo_RouteNote.Size(m) +func (x *RouteNote) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *RouteNote) XXX_DiscardUnknown() { - xxx_messageInfo_RouteNote.DiscardUnknown(m) + +func (*RouteNote) ProtoMessage() {} + +func (x *RouteNote) ProtoReflect() protoreflect.Message { + mi := &file_examples_route_guide_routeguide_route_guide_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_RouteNote proto.InternalMessageInfo +// Deprecated: Use RouteNote.ProtoReflect.Descriptor instead. +func (*RouteNote) Descriptor() ([]byte, []int) { + return file_examples_route_guide_routeguide_route_guide_proto_rawDescGZIP(), []int{3} +} -func (m *RouteNote) GetLocation() *Point { - if m != nil { - return m.Location +func (x *RouteNote) GetLocation() *Point { + if x != nil { + return x.Location } return nil } -func (m *RouteNote) GetMessage() string { - if m != nil { - return m.Message +func (x *RouteNote) GetMessage() string { + if x != nil { + return x.Message } return "" } @@ -230,6 +276,10 @@ func (m *RouteNote) GetMessage() string { // detected features, and the total distance covered as the cumulative sum of // the distance between each point. type RouteSummary struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // The number of points received. PointCount int32 `protobuf:"varint,1,opt,name=point_count,json=pointCount,proto3" json:"point_count,omitempty"` // The number of known features passed while traversing the route. @@ -237,105 +287,254 @@ type RouteSummary struct { // The distance covered in metres. Distance int32 `protobuf:"varint,3,opt,name=distance,proto3" json:"distance,omitempty"` // The duration of the traversal in seconds. - ElapsedTime int32 `protobuf:"varint,4,opt,name=elapsed_time,json=elapsedTime,proto3" json:"elapsed_time,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + ElapsedTime int32 `protobuf:"varint,4,opt,name=elapsed_time,json=elapsedTime,proto3" json:"elapsed_time,omitempty"` } -func (m *RouteSummary) Reset() { *m = RouteSummary{} } -func (m *RouteSummary) String() string { return proto.CompactTextString(m) } -func (*RouteSummary) ProtoMessage() {} -func (*RouteSummary) Descriptor() ([]byte, []int) { - return fileDescriptor_af806a20656386f8, []int{4} +func (x *RouteSummary) Reset() { + *x = RouteSummary{} + if protoimpl.UnsafeEnabled { + mi := &file_examples_route_guide_routeguide_route_guide_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *RouteSummary) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_RouteSummary.Unmarshal(m, b) +func (x *RouteSummary) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *RouteSummary) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_RouteSummary.Marshal(b, m, deterministic) -} -func (m *RouteSummary) XXX_Merge(src proto.Message) { - xxx_messageInfo_RouteSummary.Merge(m, src) -} -func (m *RouteSummary) XXX_Size() int { - return xxx_messageInfo_RouteSummary.Size(m) -} -func (m *RouteSummary) XXX_DiscardUnknown() { - xxx_messageInfo_RouteSummary.DiscardUnknown(m) + +func (*RouteSummary) ProtoMessage() {} + +func (x *RouteSummary) ProtoReflect() protoreflect.Message { + mi := &file_examples_route_guide_routeguide_route_guide_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_RouteSummary proto.InternalMessageInfo +// Deprecated: Use RouteSummary.ProtoReflect.Descriptor instead. +func (*RouteSummary) Descriptor() ([]byte, []int) { + return file_examples_route_guide_routeguide_route_guide_proto_rawDescGZIP(), []int{4} +} -func (m *RouteSummary) GetPointCount() int32 { - if m != nil { - return m.PointCount +func (x *RouteSummary) GetPointCount() int32 { + if x != nil { + return x.PointCount } return 0 } -func (m *RouteSummary) GetFeatureCount() int32 { - if m != nil { - return m.FeatureCount +func (x *RouteSummary) GetFeatureCount() int32 { + if x != nil { + return x.FeatureCount } return 0 } -func (m *RouteSummary) GetDistance() int32 { - if m != nil { - return m.Distance +func (x *RouteSummary) GetDistance() int32 { + if x != nil { + return x.Distance } return 0 } -func (m *RouteSummary) GetElapsedTime() int32 { - if m != nil { - return m.ElapsedTime +func (x *RouteSummary) GetElapsedTime() int32 { + if x != nil { + return x.ElapsedTime } return 0 } -func init() { - proto.RegisterType((*Point)(nil), "routeguide.Point") - proto.RegisterType((*Rectangle)(nil), "routeguide.Rectangle") - proto.RegisterType((*Feature)(nil), "routeguide.Feature") - proto.RegisterType((*RouteNote)(nil), "routeguide.RouteNote") - proto.RegisterType((*RouteSummary)(nil), "routeguide.RouteSummary") -} - -func init() { - proto.RegisterFile("examples/route_guide/routeguide/route_guide.proto", fileDescriptor_af806a20656386f8) -} - -var fileDescriptor_af806a20656386f8 = []byte{ - // 434 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x53, 0xcd, 0x6e, 0xd3, 0x40, - 0x10, 0xae, 0x43, 0x4b, 0xe3, 0x49, 0x10, 0x62, 0x10, 0x92, 0x15, 0x90, 0xa0, 0xe6, 0xd2, 0x0b, - 0x4e, 0x29, 0x52, 0x8f, 0x45, 0xb4, 0x12, 0xbd, 0x54, 0x28, 0x98, 0x9e, 0xb8, 0x44, 0x8b, 0x3d, - 0x6c, 0x56, 0x5a, 0x7b, 0x2c, 0x7b, 0x2d, 0xc1, 0x03, 0xf0, 0x04, 0xbc, 0x30, 0xda, 0x5d, 0x3b, - 0x76, 0x69, 0xaa, 0xde, 0x66, 0xbe, 0xf9, 0xbe, 0xf9, 0xd5, 0xc0, 0x7b, 0xfa, 0x25, 0x8a, 0x4a, - 0x53, 0xb3, 0xac, 0xb9, 0x35, 0xb4, 0x96, 0xad, 0xca, 0xc9, 0xdb, 0x23, 0xd3, 0xc3, 0x49, 0x55, - 0xb3, 0x61, 0x84, 0x21, 0x1a, 0x7f, 0x82, 0x83, 0x15, 0xab, 0xd2, 0xe0, 0x02, 0xa6, 0x5a, 0x18, - 0x65, 0xda, 0x9c, 0xa2, 0xe0, 0x4d, 0x70, 0x7c, 0x90, 0x6e, 0x7d, 0x7c, 0x05, 0xa1, 0xe6, 0x52, - 0xfa, 0xe0, 0xc4, 0x05, 0x07, 0x20, 0xfe, 0x0a, 0x61, 0x4a, 0x99, 0x11, 0xa5, 0xd4, 0x84, 0x47, - 0x30, 0xd1, 0xec, 0x12, 0xcc, 0x4e, 0x9f, 0x25, 0x43, 0xa1, 0xc4, 0x55, 0x49, 0x27, 0x9a, 0x2d, - 0x65, 0xa3, 0x5c, 0x9a, 0xdd, 0x94, 0x8d, 0x8a, 0xaf, 0xe1, 0xf0, 0x33, 0x09, 0xd3, 0xd6, 0x84, - 0x08, 0xfb, 0xa5, 0x28, 0x7c, 0x4f, 0x61, 0xea, 0x6c, 0x7c, 0x07, 0x53, 0xcd, 0x99, 0x30, 0x8a, - 0xcb, 0xfb, 0xf3, 0x6c, 0x29, 0xf1, 0x0d, 0x84, 0xa9, 0x8d, 0x7e, 0x61, 0x73, 0x5b, 0x1b, 0x3c, - 0xa8, 0xc5, 0x08, 0x0e, 0x0b, 0x6a, 0x1a, 0x21, 0xfd, 0xe0, 0x61, 0xda, 0xbb, 0xf1, 0xdf, 0x00, - 0xe6, 0x2e, 0xed, 0xb7, 0xb6, 0x28, 0x44, 0xfd, 0x1b, 0x5f, 0xc3, 0xac, 0xb2, 0xea, 0x75, 0xc6, - 0x6d, 0x69, 0xba, 0x25, 0x82, 0x83, 0x2e, 0x2d, 0x82, 0x6f, 0xe1, 0xc9, 0x4f, 0x3f, 0x55, 0x47, - 0xf1, 0xab, 0x9c, 0x77, 0xa0, 0x27, 0x2d, 0x60, 0x9a, 0xab, 0xc6, 0x88, 0x32, 0xa3, 0xe8, 0x91, - 0xbf, 0x43, 0xef, 0xe3, 0x11, 0xcc, 0x49, 0x8b, 0xaa, 0xa1, 0x7c, 0x6d, 0x54, 0x41, 0xd1, 0xbe, - 0x8b, 0xcf, 0x3a, 0xec, 0x46, 0x15, 0x74, 0xfa, 0x67, 0x02, 0xe0, 0xba, 0xba, 0xb2, 0xe3, 0xe0, - 0x19, 0xc0, 0x15, 0x99, 0x7e, 0x97, 0x77, 0x27, 0x5d, 0x3c, 0x1f, 0x43, 0x1d, 0x2f, 0xde, 0xc3, - 0x73, 0x98, 0x5f, 0xab, 0xa6, 0x17, 0x36, 0xf8, 0x62, 0x4c, 0xdb, 0x5e, 0xfb, 0x1e, 0xf5, 0x49, - 0x80, 0xe7, 0x30, 0x4b, 0x29, 0xe3, 0x3a, 0x77, 0xbd, 0xec, 0x2a, 0x1c, 0xdd, 0xca, 0x38, 0xda, - 0x63, 0xbc, 0x77, 0x1c, 0xe0, 0xc7, 0xee, 0x64, 0x97, 0x1b, 0x61, 0xfe, 0x2b, 0xde, 0x5f, 0x72, - 0xb1, 0x1b, 0xb6, 0xf2, 0x93, 0xe0, 0x62, 0x03, 0x2f, 0x15, 0x27, 0xb2, 0xae, 0xb2, 0xa4, 0x7f, - 0x90, 0x11, 0xfd, 0xe2, 0xe9, 0xb0, 0xa3, 0x95, 0xfd, 0x89, 0x55, 0xf0, 0xfd, 0x4c, 0x32, 0x4b, - 0x4d, 0x89, 0x64, 0x2d, 0x4a, 0x99, 0x70, 0x2d, 0x97, 0x56, 0xbe, 0x7c, 0xe0, 0xbf, 0x7e, 0x3c, - 0x76, 0x4f, 0xf5, 0xe1, 0x5f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x28, 0xef, 0x54, 0xdd, 0x89, 0x03, - 0x00, 0x00, +var File_examples_route_guide_routeguide_route_guide_proto protoreflect.FileDescriptor + +var file_examples_route_guide_routeguide_route_guide_proto_rawDesc = []byte{ + 0x0a, 0x31, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73, 0x2f, 0x72, 0x6f, 0x75, 0x74, 0x65, + 0x5f, 0x67, 0x75, 0x69, 0x64, 0x65, 0x2f, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x67, 0x75, 0x69, 0x64, + 0x65, 0x2f, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x5f, 0x67, 0x75, 0x69, 0x64, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x12, 0x0a, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x67, 0x75, 0x69, 0x64, 0x65, 0x22, + 0x41, 0x0a, 0x05, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x6c, 0x61, 0x74, 0x69, + 0x74, 0x75, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x6c, 0x61, 0x74, 0x69, + 0x74, 0x75, 0x64, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x6c, 0x6f, 0x6e, 0x67, 0x69, 0x74, 0x75, 0x64, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, 0x6c, 0x6f, 0x6e, 0x67, 0x69, 0x74, 0x75, + 0x64, 0x65, 0x22, 0x51, 0x0a, 0x09, 0x52, 0x65, 0x63, 0x74, 0x61, 0x6e, 0x67, 0x6c, 0x65, 0x12, + 0x21, 0x0a, 0x02, 0x6c, 0x6f, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x72, 0x6f, + 0x75, 0x74, 0x65, 0x67, 0x75, 0x69, 0x64, 0x65, 0x2e, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x52, 0x02, + 0x6c, 0x6f, 0x12, 0x21, 0x0a, 0x02, 0x68, 0x69, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, + 0x2e, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x67, 0x75, 0x69, 0x64, 0x65, 0x2e, 0x50, 0x6f, 0x69, 0x6e, + 0x74, 0x52, 0x02, 0x68, 0x69, 0x22, 0x4c, 0x0a, 0x07, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, + 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x2d, 0x0a, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x67, 0x75, + 0x69, 0x64, 0x65, 0x2e, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x52, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x22, 0x54, 0x0a, 0x09, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x4e, 0x6f, 0x74, 0x65, + 0x12, 0x2d, 0x0a, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x67, 0x75, 0x69, 0x64, 0x65, 0x2e, + 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x52, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, + 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x93, 0x01, 0x0a, 0x0c, 0x52, 0x6f, + 0x75, 0x74, 0x65, 0x53, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x12, 0x1f, 0x0a, 0x0b, 0x70, 0x6f, + 0x69, 0x6e, 0x74, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, + 0x0a, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x23, 0x0a, 0x0d, 0x66, + 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x05, 0x52, 0x0c, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, + 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x69, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x05, 0x52, 0x08, 0x64, 0x69, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x12, 0x21, 0x0a, 0x0c, + 0x65, 0x6c, 0x61, 0x70, 0x73, 0x65, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x05, 0x52, 0x0b, 0x65, 0x6c, 0x61, 0x70, 0x73, 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x32, + 0x85, 0x02, 0x0a, 0x0a, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x47, 0x75, 0x69, 0x64, 0x65, 0x12, 0x36, + 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x12, 0x11, 0x2e, 0x72, + 0x6f, 0x75, 0x74, 0x65, 0x67, 0x75, 0x69, 0x64, 0x65, 0x2e, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x1a, + 0x13, 0x2e, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x67, 0x75, 0x69, 0x64, 0x65, 0x2e, 0x46, 0x65, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x22, 0x00, 0x12, 0x3e, 0x0a, 0x0c, 0x4c, 0x69, 0x73, 0x74, 0x46, 0x65, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, 0x15, 0x2e, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x67, 0x75, + 0x69, 0x64, 0x65, 0x2e, 0x52, 0x65, 0x63, 0x74, 0x61, 0x6e, 0x67, 0x6c, 0x65, 0x1a, 0x13, 0x2e, + 0x72, 0x6f, 0x75, 0x74, 0x65, 0x67, 0x75, 0x69, 0x64, 0x65, 0x2e, 0x46, 0x65, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x3e, 0x0a, 0x0b, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, + 0x52, 0x6f, 0x75, 0x74, 0x65, 0x12, 0x11, 0x2e, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x67, 0x75, 0x69, + 0x64, 0x65, 0x2e, 0x50, 0x6f, 0x69, 0x6e, 0x74, 0x1a, 0x18, 0x2e, 0x72, 0x6f, 0x75, 0x74, 0x65, + 0x67, 0x75, 0x69, 0x64, 0x65, 0x2e, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x53, 0x75, 0x6d, 0x6d, 0x61, + 0x72, 0x79, 0x22, 0x00, 0x28, 0x01, 0x12, 0x3f, 0x0a, 0x09, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x43, + 0x68, 0x61, 0x74, 0x12, 0x15, 0x2e, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x67, 0x75, 0x69, 0x64, 0x65, + 0x2e, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x4e, 0x6f, 0x74, 0x65, 0x1a, 0x15, 0x2e, 0x72, 0x6f, 0x75, + 0x74, 0x65, 0x67, 0x75, 0x69, 0x64, 0x65, 0x2e, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x4e, 0x6f, 0x74, + 0x65, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x42, 0x68, 0x0a, 0x1b, 0x69, 0x6f, 0x2e, 0x67, 0x72, + 0x70, 0x63, 0x2e, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73, 0x2e, 0x72, 0x6f, 0x75, 0x74, + 0x65, 0x67, 0x75, 0x69, 0x64, 0x65, 0x42, 0x0f, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x47, 0x75, 0x69, + 0x64, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x36, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x65, 0x2e, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x72, 0x70, + 0x63, 0x2f, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x73, 0x2f, 0x72, 0x6f, 0x75, 0x74, 0x65, + 0x5f, 0x67, 0x75, 0x69, 0x64, 0x65, 0x2f, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x67, 0x75, 0x69, 0x64, + 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_examples_route_guide_routeguide_route_guide_proto_rawDescOnce sync.Once + file_examples_route_guide_routeguide_route_guide_proto_rawDescData = file_examples_route_guide_routeguide_route_guide_proto_rawDesc +) + +func file_examples_route_guide_routeguide_route_guide_proto_rawDescGZIP() []byte { + file_examples_route_guide_routeguide_route_guide_proto_rawDescOnce.Do(func() { + file_examples_route_guide_routeguide_route_guide_proto_rawDescData = protoimpl.X.CompressGZIP(file_examples_route_guide_routeguide_route_guide_proto_rawDescData) + }) + return file_examples_route_guide_routeguide_route_guide_proto_rawDescData +} + +var file_examples_route_guide_routeguide_route_guide_proto_msgTypes = make([]protoimpl.MessageInfo, 5) +var file_examples_route_guide_routeguide_route_guide_proto_goTypes = []interface{}{ + (*Point)(nil), // 0: routeguide.Point + (*Rectangle)(nil), // 1: routeguide.Rectangle + (*Feature)(nil), // 2: routeguide.Feature + (*RouteNote)(nil), // 3: routeguide.RouteNote + (*RouteSummary)(nil), // 4: routeguide.RouteSummary +} +var file_examples_route_guide_routeguide_route_guide_proto_depIdxs = []int32{ + 0, // 0: routeguide.Rectangle.lo:type_name -> routeguide.Point + 0, // 1: routeguide.Rectangle.hi:type_name -> routeguide.Point + 0, // 2: routeguide.Feature.location:type_name -> routeguide.Point + 0, // 3: routeguide.RouteNote.location:type_name -> routeguide.Point + 0, // 4: routeguide.RouteGuide.GetFeature:input_type -> routeguide.Point + 1, // 5: routeguide.RouteGuide.ListFeatures:input_type -> routeguide.Rectangle + 0, // 6: routeguide.RouteGuide.RecordRoute:input_type -> routeguide.Point + 3, // 7: routeguide.RouteGuide.RouteChat:input_type -> routeguide.RouteNote + 2, // 8: routeguide.RouteGuide.GetFeature:output_type -> routeguide.Feature + 2, // 9: routeguide.RouteGuide.ListFeatures:output_type -> routeguide.Feature + 4, // 10: routeguide.RouteGuide.RecordRoute:output_type -> routeguide.RouteSummary + 3, // 11: routeguide.RouteGuide.RouteChat:output_type -> routeguide.RouteNote + 8, // [8:12] is the sub-list for method output_type + 4, // [4:8] is the sub-list for method input_type + 4, // [4:4] is the sub-list for extension type_name + 4, // [4:4] is the sub-list for extension extendee + 0, // [0:4] is the sub-list for field type_name +} + +func init() { file_examples_route_guide_routeguide_route_guide_proto_init() } +func file_examples_route_guide_routeguide_route_guide_proto_init() { + if File_examples_route_guide_routeguide_route_guide_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_examples_route_guide_routeguide_route_guide_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Point); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_examples_route_guide_routeguide_route_guide_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Rectangle); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_examples_route_guide_routeguide_route_guide_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Feature); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_examples_route_guide_routeguide_route_guide_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RouteNote); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_examples_route_guide_routeguide_route_guide_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RouteSummary); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_examples_route_guide_routeguide_route_guide_proto_rawDesc, + NumEnums: 0, + NumMessages: 5, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_examples_route_guide_routeguide_route_guide_proto_goTypes, + DependencyIndexes: file_examples_route_guide_routeguide_route_guide_proto_depIdxs, + MessageInfos: file_examples_route_guide_routeguide_route_guide_proto_msgTypes, + }.Build() + File_examples_route_guide_routeguide_route_guide_proto = out.File + file_examples_route_guide_routeguide_route_guide_proto_rawDesc = nil + file_examples_route_guide_routeguide_route_guide_proto_goTypes = nil + file_examples_route_guide_routeguide_route_guide_proto_depIdxs = nil } diff --git a/examples/route_guide/routeguide/route_guide_grpc.pb.go b/examples/route_guide/routeguide/route_guide_grpc.pb.go index f9c59a7f1d0f..08012c0f4bcf 100644 --- a/examples/route_guide/routeguide/route_guide_grpc.pb.go +++ b/examples/route_guide/routeguide/route_guide_grpc.pb.go @@ -1,4 +1,22 @@ +// Copyright 2015 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.3.0 +// - protoc v4.22.0 +// source: examples/route_guide/routeguide/route_guide.proto package routeguide @@ -11,8 +29,16 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. const _ = grpc.SupportPackageIsVersion7 +const ( + RouteGuide_GetFeature_FullMethodName = "/routeguide.RouteGuide/GetFeature" + RouteGuide_ListFeatures_FullMethodName = "/routeguide.RouteGuide/ListFeatures" + RouteGuide_RecordRoute_FullMethodName = "/routeguide.RouteGuide/RecordRoute" + RouteGuide_RouteChat_FullMethodName = "/routeguide.RouteGuide/RouteChat" +) + // RouteGuideClient is the client API for RouteGuide service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. @@ -51,26 +77,17 @@ func NewRouteGuideClient(cc grpc.ClientConnInterface) RouteGuideClient { return &routeGuideClient{cc} } -var routeGuideGetFeatureStreamDesc = &grpc.StreamDesc{ - StreamName: "GetFeature", -} - func (c *routeGuideClient) GetFeature(ctx context.Context, in *Point, opts ...grpc.CallOption) (*Feature, error) { out := new(Feature) - err := c.cc.Invoke(ctx, "/routeguide.RouteGuide/GetFeature", in, out, opts...) + err := c.cc.Invoke(ctx, RouteGuide_GetFeature_FullMethodName, in, out, opts...) if err != nil { return nil, err } return out, nil } -var routeGuideListFeaturesStreamDesc = &grpc.StreamDesc{ - StreamName: "ListFeatures", - ServerStreams: true, -} - func (c *routeGuideClient) ListFeatures(ctx context.Context, in *Rectangle, opts ...grpc.CallOption) (RouteGuide_ListFeaturesClient, error) { - stream, err := c.cc.NewStream(ctx, routeGuideListFeaturesStreamDesc, "/routeguide.RouteGuide/ListFeatures", opts...) + stream, err := c.cc.NewStream(ctx, &RouteGuide_ServiceDesc.Streams[0], RouteGuide_ListFeatures_FullMethodName, opts...) if err != nil { return nil, err } @@ -101,13 +118,8 @@ func (x *routeGuideListFeaturesClient) Recv() (*Feature, error) { return m, nil } -var routeGuideRecordRouteStreamDesc = &grpc.StreamDesc{ - StreamName: "RecordRoute", - ClientStreams: true, -} - func (c *routeGuideClient) RecordRoute(ctx context.Context, opts ...grpc.CallOption) (RouteGuide_RecordRouteClient, error) { - stream, err := c.cc.NewStream(ctx, routeGuideRecordRouteStreamDesc, "/routeguide.RouteGuide/RecordRoute", opts...) + stream, err := c.cc.NewStream(ctx, &RouteGuide_ServiceDesc.Streams[1], RouteGuide_RecordRoute_FullMethodName, opts...) if err != nil { return nil, err } @@ -140,14 +152,8 @@ func (x *routeGuideRecordRouteClient) CloseAndRecv() (*RouteSummary, error) { return m, nil } -var routeGuideRouteChatStreamDesc = &grpc.StreamDesc{ - StreamName: "RouteChat", - ServerStreams: true, - ClientStreams: true, -} - func (c *routeGuideClient) RouteChat(ctx context.Context, opts ...grpc.CallOption) (RouteGuide_RouteChatClient, error) { - stream, err := c.cc.NewStream(ctx, routeGuideRouteChatStreamDesc, "/routeguide.RouteGuide/RouteChat", opts...) + stream, err := c.cc.NewStream(ctx, &RouteGuide_ServiceDesc.Streams[2], RouteGuide_RouteChat_FullMethodName, opts...) if err != nil { return nil, err } @@ -177,78 +183,90 @@ func (x *routeGuideRouteChatClient) Recv() (*RouteNote, error) { return m, nil } -// RouteGuideService is the service API for RouteGuide service. -// Fields should be assigned to their respective handler implementations only before -// RegisterRouteGuideService is called. Any unassigned fields will result in the -// handler for that method returning an Unimplemented error. -type RouteGuideService struct { +// RouteGuideServer is the server API for RouteGuide service. +// All implementations must embed UnimplementedRouteGuideServer +// for forward compatibility +type RouteGuideServer interface { // A simple RPC. // // Obtains the feature at a given position. // // A feature with an empty name is returned if there's no feature at the given // position. - GetFeature func(context.Context, *Point) (*Feature, error) + GetFeature(context.Context, *Point) (*Feature, error) // A server-to-client streaming RPC. // // Obtains the Features available within the given Rectangle. Results are // streamed rather than returned at once (e.g. in a response message with a // repeated field), as the rectangle may cover a large area and contain a // huge number of features. - ListFeatures func(*Rectangle, RouteGuide_ListFeaturesServer) error + ListFeatures(*Rectangle, RouteGuide_ListFeaturesServer) error // A client-to-server streaming RPC. // // Accepts a stream of Points on a route being traversed, returning a // RouteSummary when traversal is completed. - RecordRoute func(RouteGuide_RecordRouteServer) error + RecordRoute(RouteGuide_RecordRouteServer) error // A Bidirectional streaming RPC. // // Accepts a stream of RouteNotes sent while a route is being traversed, // while receiving other RouteNotes (e.g. from other users). - RouteChat func(RouteGuide_RouteChatServer) error + RouteChat(RouteGuide_RouteChatServer) error + mustEmbedUnimplementedRouteGuideServer() } -func (s *RouteGuideService) getFeature(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - if s.GetFeature == nil { - return nil, status.Errorf(codes.Unimplemented, "method GetFeature not implemented") - } +// UnimplementedRouteGuideServer must be embedded to have forward compatible implementations. +type UnimplementedRouteGuideServer struct { +} + +func (UnimplementedRouteGuideServer) GetFeature(context.Context, *Point) (*Feature, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetFeature not implemented") +} +func (UnimplementedRouteGuideServer) ListFeatures(*Rectangle, RouteGuide_ListFeaturesServer) error { + return status.Errorf(codes.Unimplemented, "method ListFeatures not implemented") +} +func (UnimplementedRouteGuideServer) RecordRoute(RouteGuide_RecordRouteServer) error { + return status.Errorf(codes.Unimplemented, "method RecordRoute not implemented") +} +func (UnimplementedRouteGuideServer) RouteChat(RouteGuide_RouteChatServer) error { + return status.Errorf(codes.Unimplemented, "method RouteChat not implemented") +} +func (UnimplementedRouteGuideServer) mustEmbedUnimplementedRouteGuideServer() {} + +// UnsafeRouteGuideServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to RouteGuideServer will +// result in compilation errors. +type UnsafeRouteGuideServer interface { + mustEmbedUnimplementedRouteGuideServer() +} + +func RegisterRouteGuideServer(s grpc.ServiceRegistrar, srv RouteGuideServer) { + s.RegisterService(&RouteGuide_ServiceDesc, srv) +} + +func _RouteGuide_GetFeature_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(Point) if err := dec(in); err != nil { return nil, err } if interceptor == nil { - return s.GetFeature(ctx, in) + return srv.(RouteGuideServer).GetFeature(ctx, in) } info := &grpc.UnaryServerInfo{ - Server: s, - FullMethod: "/routeguide.RouteGuide/GetFeature", + Server: srv, + FullMethod: RouteGuide_GetFeature_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return s.GetFeature(ctx, req.(*Point)) + return srv.(RouteGuideServer).GetFeature(ctx, req.(*Point)) } return interceptor(ctx, in, info, handler) } -func (s *RouteGuideService) listFeatures(_ interface{}, stream grpc.ServerStream) error { - if s.ListFeatures == nil { - return status.Errorf(codes.Unimplemented, "method ListFeatures not implemented") - } + +func _RouteGuide_ListFeatures_Handler(srv interface{}, stream grpc.ServerStream) error { m := new(Rectangle) if err := stream.RecvMsg(m); err != nil { return err } - return s.ListFeatures(m, &routeGuideListFeaturesServer{stream}) -} -func (s *RouteGuideService) recordRoute(_ interface{}, stream grpc.ServerStream) error { - if s.RecordRoute == nil { - return status.Errorf(codes.Unimplemented, "method RecordRoute not implemented") - } - return s.RecordRoute(&routeGuideRecordRouteServer{stream}) -} -func (s *RouteGuideService) routeChat(_ interface{}, stream grpc.ServerStream) error { - if s.RouteChat == nil { - return status.Errorf(codes.Unimplemented, "method RouteChat not implemented") - } - return s.RouteChat(&routeGuideRouteChatServer{stream}) + return srv.(RouteGuideServer).ListFeatures(m, &routeGuideListFeaturesServer{stream}) } type RouteGuide_ListFeaturesServer interface { @@ -264,6 +282,10 @@ func (x *routeGuideListFeaturesServer) Send(m *Feature) error { return x.ServerStream.SendMsg(m) } +func _RouteGuide_RecordRoute_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(RouteGuideServer).RecordRoute(&routeGuideRecordRouteServer{stream}) +} + type RouteGuide_RecordRouteServer interface { SendAndClose(*RouteSummary) error Recv() (*Point, error) @@ -286,6 +308,10 @@ func (x *routeGuideRecordRouteServer) Recv() (*Point, error) { return m, nil } +func _RouteGuide_RouteChat_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(RouteGuideServer).RouteChat(&routeGuideRouteChatServer{stream}) +} + type RouteGuide_RouteChatServer interface { Send(*RouteNote) error Recv() (*RouteNote, error) @@ -308,98 +334,35 @@ func (x *routeGuideRouteChatServer) Recv() (*RouteNote, error) { return m, nil } -// RegisterRouteGuideService registers a service implementation with a gRPC server. -func RegisterRouteGuideService(s grpc.ServiceRegistrar, srv *RouteGuideService) { - sd := grpc.ServiceDesc{ - ServiceName: "routeguide.RouteGuide", - Methods: []grpc.MethodDesc{ - { - MethodName: "GetFeature", - Handler: srv.getFeature, - }, +// RouteGuide_ServiceDesc is the grpc.ServiceDesc for RouteGuide service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var RouteGuide_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "routeguide.RouteGuide", + HandlerType: (*RouteGuideServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "GetFeature", + Handler: _RouteGuide_GetFeature_Handler, }, - Streams: []grpc.StreamDesc{ - { - StreamName: "ListFeatures", - Handler: srv.listFeatures, - ServerStreams: true, - }, - { - StreamName: "RecordRoute", - Handler: srv.recordRoute, - ClientStreams: true, - }, - { - StreamName: "RouteChat", - Handler: srv.routeChat, - ServerStreams: true, - ClientStreams: true, - }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "ListFeatures", + Handler: _RouteGuide_ListFeatures_Handler, + ServerStreams: true, }, - Metadata: "examples/route_guide/routeguide/route_guide.proto", - } - - s.RegisterService(&sd, nil) -} - -// NewRouteGuideService creates a new RouteGuideService containing the -// implemented methods of the RouteGuide service in s. Any unimplemented -// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client. -// This includes situations where the method handler is misspelled or has the wrong -// signature. For this reason, this function should be used with great care and -// is not recommended to be used by most users. -func NewRouteGuideService(s interface{}) *RouteGuideService { - ns := &RouteGuideService{} - if h, ok := s.(interface { - GetFeature(context.Context, *Point) (*Feature, error) - }); ok { - ns.GetFeature = h.GetFeature - } - if h, ok := s.(interface { - ListFeatures(*Rectangle, RouteGuide_ListFeaturesServer) error - }); ok { - ns.ListFeatures = h.ListFeatures - } - if h, ok := s.(interface { - RecordRoute(RouteGuide_RecordRouteServer) error - }); ok { - ns.RecordRoute = h.RecordRoute - } - if h, ok := s.(interface { - RouteChat(RouteGuide_RouteChatServer) error - }); ok { - ns.RouteChat = h.RouteChat - } - return ns -} - -// UnstableRouteGuideService is the service API for RouteGuide service. -// New methods may be added to this interface if they are added to the service -// definition, which is not a backward-compatible change. For this reason, -// use of this type is not recommended. -type UnstableRouteGuideService interface { - // A simple RPC. - // - // Obtains the feature at a given position. - // - // A feature with an empty name is returned if there's no feature at the given - // position. - GetFeature(context.Context, *Point) (*Feature, error) - // A server-to-client streaming RPC. - // - // Obtains the Features available within the given Rectangle. Results are - // streamed rather than returned at once (e.g. in a response message with a - // repeated field), as the rectangle may cover a large area and contain a - // huge number of features. - ListFeatures(*Rectangle, RouteGuide_ListFeaturesServer) error - // A client-to-server streaming RPC. - // - // Accepts a stream of Points on a route being traversed, returning a - // RouteSummary when traversal is completed. - RecordRoute(RouteGuide_RecordRouteServer) error - // A Bidirectional streaming RPC. - // - // Accepts a stream of RouteNotes sent while a route is being traversed, - // while receiving other RouteNotes (e.g. from other users). - RouteChat(RouteGuide_RouteChatServer) error + { + StreamName: "RecordRoute", + Handler: _RouteGuide_RecordRoute_Handler, + ClientStreams: true, + }, + { + StreamName: "RouteChat", + Handler: _RouteGuide_RouteChat_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "examples/route_guide/routeguide/route_guide.proto", } diff --git a/examples/route_guide/server/server.go b/examples/route_guide/server/server.go index 7c959dde06b1..44b2f963516b 100644 --- a/examples/route_guide/server/server.go +++ b/examples/route_guide/server/server.go @@ -28,10 +28,10 @@ import ( "flag" "fmt" "io" - "io/ioutil" "log" "math" "net" + "os" "sync" "time" @@ -50,10 +50,11 @@ var ( certFile = flag.String("cert_file", "", "The TLS cert file") keyFile = flag.String("key_file", "", "The TLS key file") jsonDBFile = flag.String("json_db_file", "", "A json file containing a list of features") - port = flag.Int("port", 10000, "The server port") + port = flag.Int("port", 50051, "The server port") ) type routeGuideServer struct { + pb.UnimplementedRouteGuideServer savedFeatures []*pb.Feature // read-only after initialized mu sync.Mutex // protects routeNotes @@ -154,7 +155,7 @@ func (s *routeGuideServer) loadFeatures(filePath string) { var data []byte if filePath != "" { var err error - data, err = ioutil.ReadFile(filePath) + data, err = os.ReadFile(filePath) if err != nil { log.Fatalf("Failed to load default features: %v", err) } @@ -232,12 +233,12 @@ func main() { } creds, err := credentials.NewServerTLSFromFile(*certFile, *keyFile) if err != nil { - log.Fatalf("Failed to generate credentials %v", err) + log.Fatalf("Failed to generate credentials: %v", err) } opts = []grpc.ServerOption{grpc.Creds(creds)} } grpcServer := grpc.NewServer(opts...) - pb.RegisterRouteGuideService(grpcServer, pb.NewRouteGuideService(newServer())) + pb.RegisterRouteGuideServer(grpcServer, newServer()) grpcServer.Serve(lis) } diff --git a/gcp/observability/config.go b/gcp/observability/config.go new file mode 100644 index 000000000000..ae7ea8b6983c --- /dev/null +++ b/gcp/observability/config.go @@ -0,0 +1,273 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package observability + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "strings" + + gcplogging "cloud.google.com/go/logging" + "golang.org/x/oauth2/google" + "google.golang.org/grpc/internal/envconfig" +) + +const envProjectID = "GOOGLE_CLOUD_PROJECT" + +// fetchDefaultProjectID fetches the default GCP project id from environment. +func fetchDefaultProjectID(ctx context.Context) string { + // Step 1: Check ENV var + if s := os.Getenv(envProjectID); s != "" { + logger.Infof("Found project ID from env %v: %v", envProjectID, s) + return s + } + // Step 2: Check default credential + credentials, err := google.FindDefaultCredentials(ctx, gcplogging.WriteScope) + if err != nil { + logger.Infof("Failed to locate Google Default Credential: %v", err) + return "" + } + if credentials.ProjectID == "" { + logger.Infof("Failed to find project ID in default credential: %v", err) + return "" + } + logger.Infof("Found project ID from Google Default Credential: %v", credentials.ProjectID) + return credentials.ProjectID +} + +// validateMethodString validates whether the string passed in is a valid +// pattern. +func validateMethodString(method string) error { + if strings.HasPrefix(method, "/") { + return errors.New("cannot have a leading slash") + } + serviceMethod := strings.Split(method, "/") + if len(serviceMethod) != 2 { + return errors.New("/ must come in between service and method, only one /") + } + if serviceMethod[1] == "" { + return errors.New("method name must be non empty") + } + if serviceMethod[0] == "*" { + return errors.New("cannot have service wildcard * i.e. (*/m)") + } + return nil +} + +func validateLogEventMethod(methods []string, exclude bool) error { + for _, method := range methods { + if method == "*" { + if exclude { + return errors.New("cannot have exclude and a '*' wildcard") + } + continue + } + if err := validateMethodString(method); err != nil { + return fmt.Errorf("invalid method string: %v, err: %v", method, err) + } + } + return nil +} + +func validateLoggingEvents(config *config) error { + if config.CloudLogging == nil { + return nil + } + for _, clientRPCEvent := range config.CloudLogging.ClientRPCEvents { + if err := validateLogEventMethod(clientRPCEvent.Methods, clientRPCEvent.Exclude); err != nil { + return fmt.Errorf("error in clientRPCEvent method: %v", err) + } + } + for _, serverRPCEvent := range config.CloudLogging.ServerRPCEvents { + if err := validateLogEventMethod(serverRPCEvent.Methods, serverRPCEvent.Exclude); err != nil { + return fmt.Errorf("error in serverRPCEvent method: %v", err) + } + } + return nil +} + +// unmarshalAndVerifyConfig unmarshals a json string representing an +// observability config into its internal go format, and also verifies the +// configuration's fields for validity. +func unmarshalAndVerifyConfig(rawJSON json.RawMessage) (*config, error) { + var config config + if err := json.Unmarshal(rawJSON, &config); err != nil { + return nil, fmt.Errorf("error parsing observability config: %v", err) + } + if err := validateLoggingEvents(&config); err != nil { + return nil, fmt.Errorf("error parsing observability config: %v", err) + } + if config.CloudTrace != nil && (config.CloudTrace.SamplingRate > 1 || config.CloudTrace.SamplingRate < 0) { + return nil, fmt.Errorf("error parsing observability config: invalid cloud trace sampling rate %v", config.CloudTrace.SamplingRate) + } + logger.Infof("Parsed ObservabilityConfig: %+v", &config) + return &config, nil +} + +func parseObservabilityConfig() (*config, error) { + if f := envconfig.ObservabilityConfigFile; f != "" { + if envconfig.ObservabilityConfig != "" { + logger.Warning("Ignoring GRPC_GCP_OBSERVABILITY_CONFIG and using GRPC_GCP_OBSERVABILITY_CONFIG_FILE contents.") + } + content, err := os.ReadFile(f) + if err != nil { + return nil, fmt.Errorf("error reading observability configuration file %q: %v", f, err) + } + return unmarshalAndVerifyConfig(content) + } else if envconfig.ObservabilityConfig != "" { + return unmarshalAndVerifyConfig([]byte(envconfig.ObservabilityConfig)) + } + // If the ENV var doesn't exist, do nothing + return nil, nil +} + +func ensureProjectIDInObservabilityConfig(ctx context.Context, config *config) error { + if config.ProjectID == "" { + // Try to fetch the GCP project id + projectID := fetchDefaultProjectID(ctx) + if projectID == "" { + return fmt.Errorf("empty destination project ID") + } + config.ProjectID = projectID + } + return nil +} + +type clientRPCEvents struct { + // Methods is a list of strings which can select a group of methods. By + // default, the list is empty, matching no methods. + // + // The value of the method is in the form of /. + // + // "*" is accepted as a wildcard for: + // 1. The method name. If the value is /*, it matches all + // methods in the specified service. + // 2. The whole value of the field which matches any /. + // It’s not supported when Exclude is true. + // 3. The * wildcard cannot be used on the service name independently, + // */ is not supported. + // + // The service name, when specified, must be the fully qualified service + // name, including the package name. + // + // Examples: + // 1."goo.Foo/Bar" selects only the method "Bar" from service "goo.Foo", + // here “goo” is the package name. + // 2."goo.Foo/*" selects all methods from service "goo.Foo" + // 3. "*" selects all methods from all services. + Methods []string `json:"methods,omitempty"` + // Exclude represents whether the methods denoted by Methods should be + // excluded from logging. The default value is false, meaning the methods + // denoted by Methods are included in the logging. If Exclude is true, the + // wildcard `*` cannot be used as value of an entry in Methods. + Exclude bool `json:"exclude,omitempty"` + // MaxMetadataBytes is the maximum number of bytes of each header to log. If + // the size of the metadata is greater than the defined limit, content past + // the limit will be truncated. The default value is 0. + MaxMetadataBytes int `json:"max_metadata_bytes"` + // MaxMessageBytes is the maximum number of bytes of each message to log. If + // the size of the message is greater than the defined limit, content past + // the limit will be truncated. The default value is 0. + MaxMessageBytes int `json:"max_message_bytes"` +} + +type serverRPCEvents struct { + // Methods is a list of strings which can select a group of methods. By + // default, the list is empty, matching no methods. + // + // The value of the method is in the form of /. + // + // "*" is accepted as a wildcard for: + // 1. The method name. If the value is /*, it matches all + // methods in the specified service. + // 2. The whole value of the field which matches any /. + // It’s not supported when Exclude is true. + // 3. The * wildcard cannot be used on the service name independently, + // */ is not supported. + // + // The service name, when specified, must be the fully qualified service + // name, including the package name. + // + // Examples: + // 1."goo.Foo/Bar" selects only the method "Bar" from service "goo.Foo", + // here “goo” is the package name. + // 2."goo.Foo/*" selects all methods from service "goo.Foo" + // 3. "*" selects all methods from all services. + Methods []string `json:"methods,omitempty"` + // Exclude represents whether the methods denoted by Methods should be + // excluded from logging. The default value is false, meaning the methods + // denoted by Methods are included in the logging. If Exclude is true, the + // wildcard `*` cannot be used as value of an entry in Methods. + Exclude bool `json:"exclude,omitempty"` + // MaxMetadataBytes is the maximum number of bytes of each header to log. If + // the size of the metadata is greater than the defined limit, content past + // the limit will be truncated. The default value is 0. + MaxMetadataBytes int `json:"max_metadata_bytes"` + // MaxMessageBytes is the maximum number of bytes of each message to log. If + // the size of the message is greater than the defined limit, content past + // the limit will be truncated. The default value is 0. + MaxMessageBytes int `json:"max_message_bytes"` +} + +type cloudLogging struct { + // ClientRPCEvents represents the configuration for outgoing RPC's from the + // binary. The client_rpc_events configs are evaluated in text order, the + // first one matched is used. If an RPC doesn't match an entry, it will + // continue on to the next entry in the list. + ClientRPCEvents []clientRPCEvents `json:"client_rpc_events,omitempty"` + + // ServerRPCEvents represents the configuration for incoming RPC's to the + // binary. The server_rpc_events configs are evaluated in text order, the + // first one matched is used. If an RPC doesn't match an entry, it will + // continue on to the next entry in the list. + ServerRPCEvents []serverRPCEvents `json:"server_rpc_events,omitempty"` +} + +type cloudMonitoring struct{} + +type cloudTrace struct { + // SamplingRate is the global setting that controls the probability of a RPC + // being traced. For example, 0.05 means there is a 5% chance for a RPC to + // be traced, 1.0 means trace every call, 0 means don’t start new traces. By + // default, the sampling_rate is 0. + SamplingRate float64 `json:"sampling_rate,omitempty"` +} + +type config struct { + // ProjectID is the destination GCP project identifier for uploading log + // entries. If empty, the gRPC Observability plugin will attempt to fetch + // the project_id from the GCP environment variables, or from the default + // credentials. If not found, the observability init functions will return + // an error. + ProjectID string `json:"project_id,omitempty"` + // CloudLogging defines the logging options. If not present, logging is disabled. + CloudLogging *cloudLogging `json:"cloud_logging,omitempty"` + // CloudMonitoring determines whether or not metrics are enabled based on + // whether it is present or not. If present, monitoring will be enabled, if + // not present, monitoring is disabled. + CloudMonitoring *cloudMonitoring `json:"cloud_monitoring,omitempty"` + // CloudTrace defines the tracing options. When present, tracing is enabled + // with default configurations. When absent, the tracing is disabled. + CloudTrace *cloudTrace `json:"cloud_trace,omitempty"` + // Labels are applied to cloud logging, monitoring, and trace. + Labels map[string]string `json:"labels,omitempty"` +} diff --git a/gcp/observability/exporting.go b/gcp/observability/exporting.go new file mode 100644 index 000000000000..3c27b3533e04 --- /dev/null +++ b/gcp/observability/exporting.go @@ -0,0 +1,108 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package observability + +import ( + "context" + "fmt" + "time" + + "google.golang.org/api/option" + "google.golang.org/grpc" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/stats/opencensus" + + gcplogging "cloud.google.com/go/logging" +) + +// cOptsDisableLogTrace are client options for the go client libraries which are +// used to configure connections to GCP exporting backends. These disable global +// dial and server options set by this module, which configure logging, metrics, +// and tracing on all created grpc.ClientConn's and grpc.Server's. These options +// turn on only metrics, and also disable the client libraries behavior of +// plumbing in the older opencensus instrumentation code. +var cOptsDisableLogTrace = []option.ClientOption{ + option.WithTelemetryDisabled(), + option.WithGRPCDialOption(internal.DisableGlobalDialOptions.(func() grpc.DialOption)()), + option.WithGRPCDialOption(opencensus.DialOption(opencensus.TraceOptions{ + DisableTrace: true, + })), +} + +// loggingExporter is the interface of logging exporter for gRPC Observability. +// In future, we might expose this to allow users provide custom exporters. But +// now, it exists for testing purposes. +type loggingExporter interface { + // EmitGrpcLogRecord writes a gRPC LogRecord to cache without blocking. + EmitGcpLoggingEntry(entry gcplogging.Entry) + // Close flushes all pending data and closes the exporter. + Close() error +} + +type cloudLoggingExporter struct { + projectID string + client *gcplogging.Client + logger *gcplogging.Logger +} + +func newCloudLoggingExporter(ctx context.Context, config *config) (loggingExporter, error) { + c, err := gcplogging.NewClient(ctx, fmt.Sprintf("projects/%v", config.ProjectID), cOptsDisableLogTrace...) + if err != nil { + return nil, fmt.Errorf("failed to create cloudLoggingExporter: %v", err) + } + defer logger.Infof("Successfully created cloudLoggingExporter") + if len(config.Labels) != 0 { + logger.Infof("Adding labels: %+v", config.Labels) + } + return &cloudLoggingExporter{ + projectID: config.ProjectID, + client: c, + logger: c.Logger("microservices.googleapis.com/observability/grpc", gcplogging.CommonLabels(config.Labels), gcplogging.BufferedByteLimit(1024*1024*50), gcplogging.DelayThreshold(time.Second*10)), + }, nil +} + +func (cle *cloudLoggingExporter) EmitGcpLoggingEntry(entry gcplogging.Entry) { + cle.logger.Log(entry) + if logger.V(2) { + logger.Infof("Uploading event to CloudLogging: %+v", entry) + } +} + +func (cle *cloudLoggingExporter) Close() error { + var errFlush, errClose error + if cle.logger != nil { + errFlush = cle.logger.Flush() + } + if cle.client != nil { + errClose = cle.client.Close() + } + if errFlush != nil && errClose != nil { + return fmt.Errorf("failed to close exporter. Flush failed: %v; Close failed: %v", errFlush, errClose) + } + if errFlush != nil { + return errFlush + } + if errClose != nil { + return errClose + } + cle.logger = nil + cle.client = nil + logger.Infof("Closed CloudLogging exporter") + return nil +} diff --git a/gcp/observability/go.mod b/gcp/observability/go.mod new file mode 100644 index 000000000000..a3d7ed89e12c --- /dev/null +++ b/gcp/observability/go.mod @@ -0,0 +1,45 @@ +module google.golang.org/grpc/gcp/observability + +go 1.19 + +require ( + cloud.google.com/go/logging v1.7.0 + contrib.go.opencensus.io/exporter/stackdriver v0.13.12 + github.com/google/go-cmp v0.5.9 + github.com/google/uuid v1.3.0 + go.opencensus.io v0.24.0 + golang.org/x/oauth2 v0.10.0 + google.golang.org/api v0.126.0 + google.golang.org/grpc v1.56.2 + google.golang.org/grpc/stats/opencensus v1.0.0 +) + +require ( + cloud.google.com/go v0.110.4 // indirect + cloud.google.com/go/compute v1.21.0 // indirect + cloud.google.com/go/compute/metadata v0.2.3 // indirect + cloud.google.com/go/longrunning v0.5.1 // indirect + cloud.google.com/go/monitoring v1.15.1 // indirect + cloud.google.com/go/trace v1.10.1 // indirect + github.com/aws/aws-sdk-go v1.44.162 // indirect + github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/s2a-go v0.1.4 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect + github.com/googleapis/gax-go/v2 v2.11.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/prometheus/prometheus v2.5.0+incompatible // indirect + golang.org/x/crypto v0.11.0 // indirect + golang.org/x/net v0.12.0 // indirect + golang.org/x/sync v0.3.0 // indirect + golang.org/x/sys v0.10.0 // indirect + golang.org/x/text v0.11.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect + google.golang.org/protobuf v1.31.0 // indirect +) + +replace google.golang.org/grpc => ../.. diff --git a/gcp/observability/go.sum b/gcp/observability/go.sum new file mode 100644 index 000000000000..117638a6394c --- /dev/null +++ b/gcp/observability/go.sum @@ -0,0 +1,1714 @@ +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= +cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= +cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= +cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= +cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= +cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= +cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= +cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= +cloud.google.com/go v0.100.1/go.mod h1:fs4QogzfH5n2pBXBP9vRiU+eCny7lD2vmFZy79Iuw1U= +cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= +cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc= +cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34hIU= +cloud.google.com/go v0.104.0/go.mod h1:OO6xxXdJyvuJPcEPBLN9BJPD+jep5G1+2U5B5gkRYtA= +cloud.google.com/go v0.105.0/go.mod h1:PrLgOJNe5nfE9UMxKxgXj4mD3voiP+YQ6gdt6KMFOKM= +cloud.google.com/go v0.107.0/go.mod h1:wpc2eNrD7hXUTy8EKS10jkxpZBjASrORK7goS+3YX2I= +cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= +cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVquxiw= +cloud.google.com/go v0.110.4 h1:1JYyxKMN9hd5dR2MYTPWkGUgcoxVVhg0LKNKEo0qvmk= +cloud.google.com/go v0.110.4/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= +cloud.google.com/go/accessapproval v1.4.0/go.mod h1:zybIuC3KpDOvotz59lFe5qxRZx6C75OtwbisN56xYB4= +cloud.google.com/go/accessapproval v1.5.0/go.mod h1:HFy3tuiGvMdcd/u+Cu5b9NkO1pEICJ46IR82PoUdplw= +cloud.google.com/go/accessapproval v1.6.0/go.mod h1:R0EiYnwV5fsRFiKZkPHr6mwyk2wxUJ30nL4j2pcFY2E= +cloud.google.com/go/accessapproval v1.7.1/go.mod h1:JYczztsHRMK7NTXb6Xw+dwbs/WnOJxbo/2mTI+Kgg68= +cloud.google.com/go/accesscontextmanager v1.3.0/go.mod h1:TgCBehyr5gNMz7ZaH9xubp+CE8dkrszb4oK9CWyvD4o= +cloud.google.com/go/accesscontextmanager v1.4.0/go.mod h1:/Kjh7BBu/Gh83sv+K60vN9QE5NJcd80sU33vIe2IFPE= +cloud.google.com/go/accesscontextmanager v1.6.0/go.mod h1:8XCvZWfYw3K/ji0iVnp+6pu7huxoQTLmxAbVjbloTtM= +cloud.google.com/go/accesscontextmanager v1.7.0/go.mod h1:CEGLewx8dwa33aDAZQujl7Dx+uYhS0eay198wB/VumQ= +cloud.google.com/go/accesscontextmanager v1.8.0/go.mod h1:uI+AI/r1oyWK99NN8cQ3UK76AMelMzgZCvJfsi2c+ps= +cloud.google.com/go/accesscontextmanager v1.8.1/go.mod h1:JFJHfvuaTC+++1iL1coPiG1eu5D24db2wXCDWDjIrxo= +cloud.google.com/go/aiplatform v1.22.0/go.mod h1:ig5Nct50bZlzV6NvKaTwmplLLddFx0YReh9WfTO5jKw= +cloud.google.com/go/aiplatform v1.24.0/go.mod h1:67UUvRBKG6GTayHKV8DBv2RtR1t93YRu5B1P3x99mYY= +cloud.google.com/go/aiplatform v1.27.0/go.mod h1:Bvxqtl40l0WImSb04d0hXFU7gDOiq9jQmorivIiWcKg= +cloud.google.com/go/aiplatform v1.35.0/go.mod h1:7MFT/vCaOyZT/4IIFfxH4ErVg/4ku6lKv3w0+tFTgXQ= +cloud.google.com/go/aiplatform v1.36.1/go.mod h1:WTm12vJRPARNvJ+v6P52RDHCNe4AhvjcIZ/9/RRHy/k= +cloud.google.com/go/aiplatform v1.37.0/go.mod h1:IU2Cv29Lv9oCn/9LkFiiuKfwrRTq+QQMbW+hPCxJGZw= +cloud.google.com/go/aiplatform v1.45.0/go.mod h1:Iu2Q7sC7QGhXUeOhAj/oCK9a+ULz1O4AotZiqjQ8MYA= +cloud.google.com/go/analytics v0.11.0/go.mod h1:DjEWCu41bVbYcKyvlws9Er60YE4a//bK6mnhWvQeFNI= +cloud.google.com/go/analytics v0.12.0/go.mod h1:gkfj9h6XRf9+TS4bmuhPEShsh3hH8PAZzm/41OOhQd4= +cloud.google.com/go/analytics v0.17.0/go.mod h1:WXFa3WSym4IZ+JiKmavYdJwGG/CvpqiqczmL59bTD9M= +cloud.google.com/go/analytics v0.18.0/go.mod h1:ZkeHGQlcIPkw0R/GW+boWHhCOR43xz9RN/jn7WcqfIE= +cloud.google.com/go/analytics v0.19.0/go.mod h1:k8liqf5/HCnOUkbawNtrWWc+UAzyDlW89doe8TtoDsE= +cloud.google.com/go/analytics v0.21.2/go.mod h1:U8dcUtmDmjrmUTnnnRnI4m6zKn/yaA5N9RlEkYFHpQo= +cloud.google.com/go/apigateway v1.3.0/go.mod h1:89Z8Bhpmxu6AmUxuVRg/ECRGReEdiP3vQtk4Z1J9rJk= +cloud.google.com/go/apigateway v1.4.0/go.mod h1:pHVY9MKGaH9PQ3pJ4YLzoj6U5FUDeDFBllIz7WmzJoc= +cloud.google.com/go/apigateway v1.5.0/go.mod h1:GpnZR3Q4rR7LVu5951qfXPJCHquZt02jf7xQx7kpqN8= +cloud.google.com/go/apigateway v1.6.1/go.mod h1:ufAS3wpbRjqfZrzpvLC2oh0MFlpRJm2E/ts25yyqmXA= +cloud.google.com/go/apigeeconnect v1.3.0/go.mod h1:G/AwXFAKo0gIXkPTVfZDd2qA1TxBXJ3MgMRBQkIi9jc= +cloud.google.com/go/apigeeconnect v1.4.0/go.mod h1:kV4NwOKqjvt2JYR0AoIWo2QGfoRtn/pkS3QlHp0Ni04= +cloud.google.com/go/apigeeconnect v1.5.0/go.mod h1:KFaCqvBRU6idyhSNyn3vlHXc8VMDJdRmwDF6JyFRqZ8= +cloud.google.com/go/apigeeconnect v1.6.1/go.mod h1:C4awq7x0JpLtrlQCr8AzVIzAaYgngRqWf9S5Uhg+wWs= +cloud.google.com/go/apigeeregistry v0.4.0/go.mod h1:EUG4PGcsZvxOXAdyEghIdXwAEi/4MEaoqLMLDMIwKXY= +cloud.google.com/go/apigeeregistry v0.5.0/go.mod h1:YR5+s0BVNZfVOUkMa5pAR2xGd0A473vA5M7j247o1wM= +cloud.google.com/go/apigeeregistry v0.6.0/go.mod h1:BFNzW7yQVLZ3yj0TKcwzb8n25CFBri51GVGOEUcgQsc= +cloud.google.com/go/apigeeregistry v0.7.1/go.mod h1:1XgyjZye4Mqtw7T9TsY4NW10U7BojBvG4RMD+vRDrIw= +cloud.google.com/go/apikeys v0.4.0/go.mod h1:XATS/yqZbaBK0HOssf+ALHp8jAlNHUgyfprvNcBIszU= +cloud.google.com/go/apikeys v0.5.0/go.mod h1:5aQfwY4D+ewMMWScd3hm2en3hCj+BROlyrt3ytS7KLI= +cloud.google.com/go/apikeys v0.6.0/go.mod h1:kbpXu5upyiAlGkKrJgQl8A0rKNNJ7dQ377pdroRSSi8= +cloud.google.com/go/appengine v1.4.0/go.mod h1:CS2NhuBuDXM9f+qscZ6V86m1MIIqPj3WC/UoEuR1Sno= +cloud.google.com/go/appengine v1.5.0/go.mod h1:TfasSozdkFI0zeoxW3PTBLiNqRmzraodCWatWI9Dmak= +cloud.google.com/go/appengine v1.6.0/go.mod h1:hg6i0J/BD2cKmDJbaFSYHFyZkgBEfQrDg/X0V5fJn84= +cloud.google.com/go/appengine v1.7.0/go.mod h1:eZqpbHFCqRGa2aCdope7eC0SWLV1j0neb/QnMJVWx6A= +cloud.google.com/go/appengine v1.7.1/go.mod h1:IHLToyb/3fKutRysUlFO0BPt5j7RiQ45nrzEJmKTo6E= +cloud.google.com/go/appengine v1.8.1/go.mod h1:6NJXGLVhZCN9aQ/AEDvmfzKEfoYBlfB80/BHiKVputY= +cloud.google.com/go/area120 v0.5.0/go.mod h1:DE/n4mp+iqVyvxHN41Vf1CR602GiHQjFPusMFW6bGR4= +cloud.google.com/go/area120 v0.6.0/go.mod h1:39yFJqWVgm0UZqWTOdqkLhjoC7uFfgXRC8g/ZegeAh0= +cloud.google.com/go/area120 v0.7.0/go.mod h1:a3+8EUD1SX5RUcCs3MY5YasiO1z6yLiNLRiFrykbynY= +cloud.google.com/go/area120 v0.7.1/go.mod h1:j84i4E1RboTWjKtZVWXPqvK5VHQFJRF2c1Nm69pWm9k= +cloud.google.com/go/area120 v0.8.1/go.mod h1:BVfZpGpB7KFVNxPiQBuHkX6Ed0rS51xIgmGyjrAfzsg= +cloud.google.com/go/artifactregistry v1.6.0/go.mod h1:IYt0oBPSAGYj/kprzsBjZ/4LnG/zOcHyFHjWPCi6SAQ= +cloud.google.com/go/artifactregistry v1.7.0/go.mod h1:mqTOFOnGZx8EtSqK/ZWcsm/4U8B77rbcLP6ruDU2Ixk= +cloud.google.com/go/artifactregistry v1.8.0/go.mod h1:w3GQXkJX8hiKN0v+at4b0qotwijQbYUqF2GWkZzAhC0= +cloud.google.com/go/artifactregistry v1.9.0/go.mod h1:2K2RqvA2CYvAeARHRkLDhMDJ3OXy26h3XW+3/Jh2uYc= +cloud.google.com/go/artifactregistry v1.11.1/go.mod h1:lLYghw+Itq9SONbCa1YWBoWs1nOucMH0pwXN1rOBZFI= +cloud.google.com/go/artifactregistry v1.11.2/go.mod h1:nLZns771ZGAwVLzTX/7Al6R9ehma4WUEhZGWV6CeQNQ= +cloud.google.com/go/artifactregistry v1.12.0/go.mod h1:o6P3MIvtzTOnmvGagO9v/rOjjA0HmhJ+/6KAXrmYDCI= +cloud.google.com/go/artifactregistry v1.13.0/go.mod h1:uy/LNfoOIivepGhooAUpL1i30Hgee3Cu0l4VTWHUC08= +cloud.google.com/go/artifactregistry v1.14.1/go.mod h1:nxVdG19jTaSTu7yA7+VbWL346r3rIdkZ142BSQqhn5E= +cloud.google.com/go/asset v1.5.0/go.mod h1:5mfs8UvcM5wHhqtSv8J1CtxxaQq3AdBxxQi2jGW/K4o= +cloud.google.com/go/asset v1.7.0/go.mod h1:YbENsRK4+xTiL+Ofoj5Ckf+O17kJtgp3Y3nn4uzZz5s= +cloud.google.com/go/asset v1.8.0/go.mod h1:mUNGKhiqIdbr8X7KNayoYvyc4HbbFO9URsjbytpUaW0= +cloud.google.com/go/asset v1.9.0/go.mod h1:83MOE6jEJBMqFKadM9NLRcs80Gdw76qGuHn8m3h8oHQ= +cloud.google.com/go/asset v1.10.0/go.mod h1:pLz7uokL80qKhzKr4xXGvBQXnzHn5evJAEAtZiIb0wY= +cloud.google.com/go/asset v1.11.1/go.mod h1:fSwLhbRvC9p9CXQHJ3BgFeQNM4c9x10lqlrdEUYXlJo= +cloud.google.com/go/asset v1.12.0/go.mod h1:h9/sFOa4eDIyKmH6QMpm4eUK3pDojWnUhTgJlk762Hg= +cloud.google.com/go/asset v1.13.0/go.mod h1:WQAMyYek/b7NBpYq/K4KJWcRqzoalEsxz/t/dTk4THw= +cloud.google.com/go/asset v1.14.1/go.mod h1:4bEJ3dnHCqWCDbWJ/6Vn7GVI9LerSi7Rfdi03hd+WTQ= +cloud.google.com/go/assuredworkloads v1.5.0/go.mod h1:n8HOZ6pff6re5KYfBXcFvSViQjDwxFkAkmUFffJRbbY= +cloud.google.com/go/assuredworkloads v1.6.0/go.mod h1:yo2YOk37Yc89Rsd5QMVECvjaMKymF9OP+QXWlKXUkXw= +cloud.google.com/go/assuredworkloads v1.7.0/go.mod h1:z/736/oNmtGAyU47reJgGN+KVoYoxeLBoj4XkKYscNI= +cloud.google.com/go/assuredworkloads v1.8.0/go.mod h1:AsX2cqyNCOvEQC8RMPnoc0yEarXQk6WEKkxYfL6kGIo= +cloud.google.com/go/assuredworkloads v1.9.0/go.mod h1:kFuI1P78bplYtT77Tb1hi0FMxM0vVpRC7VVoJC3ZoT0= +cloud.google.com/go/assuredworkloads v1.10.0/go.mod h1:kwdUQuXcedVdsIaKgKTp9t0UJkE5+PAVNhdQm4ZVq2E= +cloud.google.com/go/assuredworkloads v1.11.1/go.mod h1:+F04I52Pgn5nmPG36CWFtxmav6+7Q+c5QyJoL18Lry0= +cloud.google.com/go/automl v1.5.0/go.mod h1:34EjfoFGMZ5sgJ9EoLsRtdPSNZLcfflJR39VbVNS2M0= +cloud.google.com/go/automl v1.6.0/go.mod h1:ugf8a6Fx+zP0D59WLhqgTDsQI9w07o64uf/Is3Nh5p8= +cloud.google.com/go/automl v1.7.0/go.mod h1:RL9MYCCsJEOmt0Wf3z9uzG0a7adTT1fe+aObgSpkCt8= +cloud.google.com/go/automl v1.8.0/go.mod h1:xWx7G/aPEe/NP+qzYXktoBSDfjO+vnKMGgsApGJJquM= +cloud.google.com/go/automl v1.12.0/go.mod h1:tWDcHDp86aMIuHmyvjuKeeHEGq76lD7ZqfGLN6B0NuU= +cloud.google.com/go/automl v1.13.1/go.mod h1:1aowgAHWYZU27MybSCFiukPO7xnyawv7pt3zK4bheQE= +cloud.google.com/go/baremetalsolution v0.3.0/go.mod h1:XOrocE+pvK1xFfleEnShBlNAXf+j5blPPxrhjKgnIFc= +cloud.google.com/go/baremetalsolution v0.4.0/go.mod h1:BymplhAadOO/eBa7KewQ0Ppg4A4Wplbn+PsFKRLo0uI= +cloud.google.com/go/baremetalsolution v0.5.0/go.mod h1:dXGxEkmR9BMwxhzBhV0AioD0ULBmuLZI8CdwalUxuss= +cloud.google.com/go/batch v0.3.0/go.mod h1:TR18ZoAekj1GuirsUsR1ZTKN3FC/4UDnScjT8NXImFE= +cloud.google.com/go/batch v0.4.0/go.mod h1:WZkHnP43R/QCGQsZ+0JyG4i79ranE2u8xvjq/9+STPE= +cloud.google.com/go/batch v0.7.0/go.mod h1:vLZN95s6teRUqRQ4s3RLDsH8PvboqBK+rn1oevL159g= +cloud.google.com/go/beyondcorp v0.2.0/go.mod h1:TB7Bd+EEtcw9PCPQhCJtJGjk/7TC6ckmnSFS+xwTfm4= +cloud.google.com/go/beyondcorp v0.3.0/go.mod h1:E5U5lcrcXMsCuoDNyGrpyTm/hn7ne941Jz2vmksAxW8= +cloud.google.com/go/beyondcorp v0.4.0/go.mod h1:3ApA0mbhHx6YImmuubf5pyW8srKnCEPON32/5hj+RmM= +cloud.google.com/go/beyondcorp v0.5.0/go.mod h1:uFqj9X+dSfrheVp7ssLTaRHd2EHqSL4QZmH4e8WXGGU= +cloud.google.com/go/beyondcorp v0.6.1/go.mod h1:YhxDWw946SCbmcWo3fAhw3V4XZMSpQ/VYfcKGAEU8/4= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/bigquery v1.42.0/go.mod h1:8dRTJxhtG+vwBKzE5OseQn/hiydoQN3EedCaOdYmxRA= +cloud.google.com/go/bigquery v1.43.0/go.mod h1:ZMQcXHsl+xmU1z36G2jNGZmKp9zNY5BUua5wDgmNCfw= +cloud.google.com/go/bigquery v1.44.0/go.mod h1:0Y33VqXTEsbamHJvJHdFmtqHvMIY28aK1+dFsvaChGc= +cloud.google.com/go/bigquery v1.47.0/go.mod h1:sA9XOgy0A8vQK9+MWhEQTY6Tix87M/ZurWFIxmF9I/E= +cloud.google.com/go/bigquery v1.48.0/go.mod h1:QAwSz+ipNgfL5jxiaK7weyOhzdoAy1zFm0Nf1fysJac= +cloud.google.com/go/bigquery v1.49.0/go.mod h1:Sv8hMmTFFYBlt/ftw2uN6dFdQPzBlREY9yBh7Oy7/4Q= +cloud.google.com/go/bigquery v1.50.0/go.mod h1:YrleYEh2pSEbgTBZYMJ5SuSr0ML3ypjRB1zgf7pvQLU= +cloud.google.com/go/bigquery v1.52.0/go.mod h1:3b/iXjRQGU4nKa87cXeg6/gogLjO8C6PmuM8i5Bi/u4= +cloud.google.com/go/billing v1.4.0/go.mod h1:g9IdKBEFlItS8bTtlrZdVLWSSdSyFUZKXNS02zKMOZY= +cloud.google.com/go/billing v1.5.0/go.mod h1:mztb1tBc3QekhjSgmpf/CV4LzWXLzCArwpLmP2Gm88s= +cloud.google.com/go/billing v1.6.0/go.mod h1:WoXzguj+BeHXPbKfNWkqVtDdzORazmCjraY+vrxcyvI= +cloud.google.com/go/billing v1.7.0/go.mod h1:q457N3Hbj9lYwwRbnlD7vUpyjq6u5U1RAOArInEiD5Y= +cloud.google.com/go/billing v1.12.0/go.mod h1:yKrZio/eu+okO/2McZEbch17O5CB5NpZhhXG6Z766ss= +cloud.google.com/go/billing v1.13.0/go.mod h1:7kB2W9Xf98hP9Sr12KfECgfGclsH3CQR0R08tnRlRbc= +cloud.google.com/go/billing v1.16.0/go.mod h1:y8vx09JSSJG02k5QxbycNRrN7FGZB6F3CAcgum7jvGA= +cloud.google.com/go/binaryauthorization v1.1.0/go.mod h1:xwnoWu3Y84jbuHa0zd526MJYmtnVXn0syOjaJgy4+dM= +cloud.google.com/go/binaryauthorization v1.2.0/go.mod h1:86WKkJHtRcv5ViNABtYMhhNWRrD1Vpi//uKEy7aYEfI= +cloud.google.com/go/binaryauthorization v1.3.0/go.mod h1:lRZbKgjDIIQvzYQS1p99A7/U1JqvqeZg0wiI5tp6tg0= +cloud.google.com/go/binaryauthorization v1.4.0/go.mod h1:tsSPQrBd77VLplV70GUhBf/Zm3FsKmgSqgm4UmiDItk= +cloud.google.com/go/binaryauthorization v1.5.0/go.mod h1:OSe4OU1nN/VswXKRBmciKpo9LulY41gch5c68htf3/Q= +cloud.google.com/go/binaryauthorization v1.6.1/go.mod h1:TKt4pa8xhowwffiBmbrbcxijJRZED4zrqnwZ1lKH51U= +cloud.google.com/go/certificatemanager v1.3.0/go.mod h1:n6twGDvcUBFu9uBgt4eYvvf3sQ6My8jADcOVwHmzadg= +cloud.google.com/go/certificatemanager v1.4.0/go.mod h1:vowpercVFyqs8ABSmrdV+GiFf2H/ch3KyudYQEMM590= +cloud.google.com/go/certificatemanager v1.6.0/go.mod h1:3Hh64rCKjRAX8dXgRAyOcY5vQ/fE1sh8o+Mdd6KPgY8= +cloud.google.com/go/certificatemanager v1.7.1/go.mod h1:iW8J3nG6SaRYImIa+wXQ0g8IgoofDFRp5UMzaNk1UqI= +cloud.google.com/go/channel v1.8.0/go.mod h1:W5SwCXDJsq/rg3tn3oG0LOxpAo6IMxNa09ngphpSlnk= +cloud.google.com/go/channel v1.9.0/go.mod h1:jcu05W0my9Vx4mt3/rEHpfxc9eKi9XwsdDL8yBMbKUk= +cloud.google.com/go/channel v1.11.0/go.mod h1:IdtI0uWGqhEeatSB62VOoJ8FSUhJ9/+iGkJVqp74CGE= +cloud.google.com/go/channel v1.12.0/go.mod h1:VkxCGKASi4Cq7TbXxlaBezonAYpp1GCnKMY6tnMQnLU= +cloud.google.com/go/channel v1.16.0/go.mod h1:eN/q1PFSl5gyu0dYdmxNXscY/4Fi7ABmeHCJNf/oHmc= +cloud.google.com/go/cloudbuild v1.3.0/go.mod h1:WequR4ULxlqvMsjDEEEFnOG5ZSRSgWOywXYDb1vPE6U= +cloud.google.com/go/cloudbuild v1.4.0/go.mod h1:5Qwa40LHiOXmz3386FrjrYM93rM/hdRr7b53sySrTqA= +cloud.google.com/go/cloudbuild v1.6.0/go.mod h1:UIbc/w9QCbH12xX+ezUsgblrWv+Cv4Tw83GiSMHOn9M= +cloud.google.com/go/cloudbuild v1.7.0/go.mod h1:zb5tWh2XI6lR9zQmsm1VRA+7OCuve5d8S+zJUul8KTg= +cloud.google.com/go/cloudbuild v1.9.0/go.mod h1:qK1d7s4QlO0VwfYn5YuClDGg2hfmLZEb4wQGAbIgL1s= +cloud.google.com/go/cloudbuild v1.10.1/go.mod h1:lyJg7v97SUIPq4RC2sGsz/9tNczhyv2AjML/ci4ulzU= +cloud.google.com/go/clouddms v1.3.0/go.mod h1:oK6XsCDdW4Ib3jCCBugx+gVjevp2TMXFtgxvPSee3OM= +cloud.google.com/go/clouddms v1.4.0/go.mod h1:Eh7sUGCC+aKry14O1NRljhjyrr0NFC0G2cjwX0cByRk= +cloud.google.com/go/clouddms v1.5.0/go.mod h1:QSxQnhikCLUw13iAbffF2CZxAER3xDGNHjsTAkQJcQA= +cloud.google.com/go/clouddms v1.6.1/go.mod h1:Ygo1vL52Ov4TBZQquhz5fiw2CQ58gvu+PlS6PVXCpZI= +cloud.google.com/go/cloudtasks v1.5.0/go.mod h1:fD92REy1x5woxkKEkLdvavGnPJGEn8Uic9nWuLzqCpY= +cloud.google.com/go/cloudtasks v1.6.0/go.mod h1:C6Io+sxuke9/KNRkbQpihnW93SWDU3uXt92nu85HkYI= +cloud.google.com/go/cloudtasks v1.7.0/go.mod h1:ImsfdYWwlWNJbdgPIIGJWC+gemEGTBK/SunNQQNCAb4= +cloud.google.com/go/cloudtasks v1.8.0/go.mod h1:gQXUIwCSOI4yPVK7DgTVFiiP0ZW/eQkydWzwVMdHxrI= +cloud.google.com/go/cloudtasks v1.9.0/go.mod h1:w+EyLsVkLWHcOaqNEyvcKAsWp9p29dL6uL9Nst1cI7Y= +cloud.google.com/go/cloudtasks v1.10.0/go.mod h1:NDSoTLkZ3+vExFEWu2UJV1arUyzVDAiZtdWcsUyNwBs= +cloud.google.com/go/cloudtasks v1.11.1/go.mod h1:a9udmnou9KO2iulGscKR0qBYjreuX8oHwpmFsKspEvM= +cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= +cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= +cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= +cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s= +cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= +cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= +cloud.google.com/go/compute v1.10.0/go.mod h1:ER5CLbMxl90o2jtNbGSbtfOpQKR0t15FOtRsugnLrlU= +cloud.google.com/go/compute v1.12.0/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= +cloud.google.com/go/compute v1.12.1/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= +cloud.google.com/go/compute v1.13.0/go.mod h1:5aPTS0cUNMIc1CE546K+Th6weJUNQErARyZtRXDJ8GE= +cloud.google.com/go/compute v1.14.0/go.mod h1:YfLtxrj9sU4Yxv+sXzZkyPjEyPBZfXHUvjxega5vAdo= +cloud.google.com/go/compute v1.18.0/go.mod h1:1X7yHxec2Ga+Ss6jPyjxRxpu2uu7PLgsOVXvgU0yacs= +cloud.google.com/go/compute v1.19.0/go.mod h1:rikpw2y+UMidAe9tISo04EHNOIf42RLYF/q8Bs93scU= +cloud.google.com/go/compute v1.19.3/go.mod h1:qxvISKp/gYnXkSAD1ppcSOveRAmzxicEv/JlizULFrI= +cloud.google.com/go/compute v1.20.1/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= +cloud.google.com/go/compute v1.21.0 h1:JNBsyXVoOoNJtTQcnEY5uYpZIbeCTYIeDe0Xh1bySMk= +cloud.google.com/go/compute v1.21.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= +cloud.google.com/go/compute/metadata v0.1.0/go.mod h1:Z1VN+bulIf6bt4P/C37K4DyZYZEXYonfTBHHFPO/4UU= +cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= +cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +cloud.google.com/go/contactcenterinsights v1.3.0/go.mod h1:Eu2oemoePuEFc/xKFPjbTuPSj0fYJcPls9TFlPNnHHY= +cloud.google.com/go/contactcenterinsights v1.4.0/go.mod h1:L2YzkGbPsv+vMQMCADxJoT9YiTTnSEd6fEvCeHTYVck= +cloud.google.com/go/contactcenterinsights v1.6.0/go.mod h1:IIDlT6CLcDoyv79kDv8iWxMSTZhLxSCofVV5W6YFM/w= +cloud.google.com/go/contactcenterinsights v1.9.1/go.mod h1:bsg/R7zGLYMVxFFzfh9ooLTruLRCG9fnzhH9KznHhbM= +cloud.google.com/go/container v1.6.0/go.mod h1:Xazp7GjJSeUYo688S+6J5V+n/t+G5sKBTFkKNudGRxg= +cloud.google.com/go/container v1.7.0/go.mod h1:Dp5AHtmothHGX3DwwIHPgq45Y8KmNsgN3amoYfxVkLo= +cloud.google.com/go/container v1.13.1/go.mod h1:6wgbMPeQRw9rSnKBCAJXnds3Pzj03C4JHamr8asWKy4= +cloud.google.com/go/container v1.14.0/go.mod h1:3AoJMPhHfLDxLvrlVWaK57IXzaPnLaZq63WX59aQBfM= +cloud.google.com/go/container v1.15.0/go.mod h1:ft+9S0WGjAyjDggg5S06DXj+fHJICWg8L7isCQe9pQA= +cloud.google.com/go/container v1.22.1/go.mod h1:lTNExE2R7f+DLbAN+rJiKTisauFCaoDq6NURZ83eVH4= +cloud.google.com/go/containeranalysis v0.5.1/go.mod h1:1D92jd8gRR/c0fGMlymRgxWD3Qw9C1ff6/T7mLgVL8I= +cloud.google.com/go/containeranalysis v0.6.0/go.mod h1:HEJoiEIu+lEXM+k7+qLCci0h33lX3ZqoYFdmPcoO7s4= +cloud.google.com/go/containeranalysis v0.7.0/go.mod h1:9aUL+/vZ55P2CXfuZjS4UjQ9AgXoSw8Ts6lemfmxBxI= +cloud.google.com/go/containeranalysis v0.9.0/go.mod h1:orbOANbwk5Ejoom+s+DUCTTJ7IBdBQJDcSylAx/on9s= +cloud.google.com/go/containeranalysis v0.10.1/go.mod h1:Ya2jiILITMY68ZLPaogjmOMNkwsDrWBSTyBubGXO7j0= +cloud.google.com/go/datacatalog v1.3.0/go.mod h1:g9svFY6tuR+j+hrTw3J2dNcmI0dzmSiyOzm8kpLq0a0= +cloud.google.com/go/datacatalog v1.5.0/go.mod h1:M7GPLNQeLfWqeIm3iuiruhPzkt65+Bx8dAKvScX8jvs= +cloud.google.com/go/datacatalog v1.6.0/go.mod h1:+aEyF8JKg+uXcIdAmmaMUmZ3q1b/lKLtXCmXdnc0lbc= +cloud.google.com/go/datacatalog v1.7.0/go.mod h1:9mEl4AuDYWw81UGc41HonIHH7/sn52H0/tc8f8ZbZIE= +cloud.google.com/go/datacatalog v1.8.0/go.mod h1:KYuoVOv9BM8EYz/4eMFxrr4DUKhGIOXxZoKYF5wdISM= +cloud.google.com/go/datacatalog v1.8.1/go.mod h1:RJ58z4rMp3gvETA465Vg+ag8BGgBdnRPEMMSTr5Uv+M= +cloud.google.com/go/datacatalog v1.12.0/go.mod h1:CWae8rFkfp6LzLumKOnmVh4+Zle4A3NXLzVJ1d1mRm0= +cloud.google.com/go/datacatalog v1.13.0/go.mod h1:E4Rj9a5ZtAxcQJlEBTLgMTphfP11/lNaAshpoBgemX8= +cloud.google.com/go/datacatalog v1.14.0/go.mod h1:h0PrGtlihoutNMp/uvwhawLQ9+c63Kz65UFqh49Yo+E= +cloud.google.com/go/datacatalog v1.14.1/go.mod h1:d2CevwTG4yedZilwe+v3E3ZBDRMobQfSG/a6cCCN5R4= +cloud.google.com/go/dataflow v0.6.0/go.mod h1:9QwV89cGoxjjSR9/r7eFDqqjtvbKxAK2BaYU6PVk9UM= +cloud.google.com/go/dataflow v0.7.0/go.mod h1:PX526vb4ijFMesO1o202EaUmouZKBpjHsTlCtB4parQ= +cloud.google.com/go/dataflow v0.8.0/go.mod h1:Rcf5YgTKPtQyYz8bLYhFoIV/vP39eL7fWNcSOyFfLJE= +cloud.google.com/go/dataflow v0.9.1/go.mod h1:Wp7s32QjYuQDWqJPFFlnBKhkAtiFpMTdg00qGbnIHVw= +cloud.google.com/go/dataform v0.3.0/go.mod h1:cj8uNliRlHpa6L3yVhDOBrUXH+BPAO1+KFMQQNSThKo= +cloud.google.com/go/dataform v0.4.0/go.mod h1:fwV6Y4Ty2yIFL89huYlEkwUPtS7YZinZbzzj5S9FzCE= +cloud.google.com/go/dataform v0.5.0/go.mod h1:GFUYRe8IBa2hcomWplodVmUx/iTL0FrsauObOM3Ipr0= +cloud.google.com/go/dataform v0.6.0/go.mod h1:QPflImQy33e29VuapFdf19oPbE4aYTJxr31OAPV+ulA= +cloud.google.com/go/dataform v0.7.0/go.mod h1:7NulqnVozfHvWUBpMDfKMUESr+85aJsC/2O0o3jWPDE= +cloud.google.com/go/dataform v0.8.1/go.mod h1:3BhPSiw8xmppbgzeBbmDvmSWlwouuJkXsXsb8UBih9M= +cloud.google.com/go/datafusion v1.4.0/go.mod h1:1Zb6VN+W6ALo85cXnM1IKiPw+yQMKMhB9TsTSRDo/38= +cloud.google.com/go/datafusion v1.5.0/go.mod h1:Kz+l1FGHB0J+4XF2fud96WMmRiq/wj8N9u007vyXZ2w= +cloud.google.com/go/datafusion v1.6.0/go.mod h1:WBsMF8F1RhSXvVM8rCV3AeyWVxcC2xY6vith3iw3S+8= +cloud.google.com/go/datafusion v1.7.1/go.mod h1:KpoTBbFmoToDExJUso/fcCiguGDk7MEzOWXUsJo0wsI= +cloud.google.com/go/datalabeling v0.5.0/go.mod h1:TGcJ0G2NzcsXSE/97yWjIZO0bXj0KbVlINXMG9ud42I= +cloud.google.com/go/datalabeling v0.6.0/go.mod h1:WqdISuk/+WIGeMkpw/1q7bK/tFEZxsrFJOJdY2bXvTQ= +cloud.google.com/go/datalabeling v0.7.0/go.mod h1:WPQb1y08RJbmpM3ww0CSUAGweL0SxByuW2E+FU+wXcM= +cloud.google.com/go/datalabeling v0.8.1/go.mod h1:XS62LBSVPbYR54GfYQsPXZjTW8UxCK2fkDciSrpRFdY= +cloud.google.com/go/dataplex v1.3.0/go.mod h1:hQuRtDg+fCiFgC8j0zV222HvzFQdRd+SVX8gdmFcZzA= +cloud.google.com/go/dataplex v1.4.0/go.mod h1:X51GfLXEMVJ6UN47ESVqvlsRplbLhcsAt0kZCCKsU0A= +cloud.google.com/go/dataplex v1.5.2/go.mod h1:cVMgQHsmfRoI5KFYq4JtIBEUbYwc3c7tXmIDhRmNNVQ= +cloud.google.com/go/dataplex v1.6.0/go.mod h1:bMsomC/aEJOSpHXdFKFGQ1b0TDPIeL28nJObeO1ppRs= +cloud.google.com/go/dataplex v1.8.1/go.mod h1:7TyrDT6BCdI8/38Uvp0/ZxBslOslP2X2MPDucliyvSE= +cloud.google.com/go/dataproc v1.7.0/go.mod h1:CKAlMjII9H90RXaMpSxQ8EU6dQx6iAYNPcYPOkSbi8s= +cloud.google.com/go/dataproc v1.8.0/go.mod h1:5OW+zNAH0pMpw14JVrPONsxMQYMBqJuzORhIBfBn9uI= +cloud.google.com/go/dataproc v1.12.0/go.mod h1:zrF3aX0uV3ikkMz6z4uBbIKyhRITnxvr4i3IjKsKrw4= +cloud.google.com/go/dataqna v0.5.0/go.mod h1:90Hyk596ft3zUQ8NkFfvICSIfHFh1Bc7C4cK3vbhkeo= +cloud.google.com/go/dataqna v0.6.0/go.mod h1:1lqNpM7rqNLVgWBJyk5NF6Uen2PHym0jtVJonplVsDA= +cloud.google.com/go/dataqna v0.7.0/go.mod h1:Lx9OcIIeqCrw1a6KdO3/5KMP1wAmTc0slZWwP12Qq3c= +cloud.google.com/go/dataqna v0.8.1/go.mod h1:zxZM0Bl6liMePWsHA8RMGAfmTG34vJMapbHAxQ5+WA8= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/datastore v1.10.0/go.mod h1:PC5UzAmDEkAmkfaknstTYbNpgE49HAgW2J1gcgUfmdM= +cloud.google.com/go/datastore v1.11.0/go.mod h1:TvGxBIHCS50u8jzG+AW/ppf87v1of8nwzFNgEZU1D3c= +cloud.google.com/go/datastore v1.12.0/go.mod h1:KjdB88W897MRITkvWWJrg2OUtrR5XVj1EoLgSp6/N70= +cloud.google.com/go/datastore v1.12.1/go.mod h1:KjdB88W897MRITkvWWJrg2OUtrR5XVj1EoLgSp6/N70= +cloud.google.com/go/datastream v1.2.0/go.mod h1:i/uTP8/fZwgATHS/XFu0TcNUhuA0twZxxQ3EyCUQMwo= +cloud.google.com/go/datastream v1.3.0/go.mod h1:cqlOX8xlyYF/uxhiKn6Hbv6WjwPPuI9W2M9SAXwaLLQ= +cloud.google.com/go/datastream v1.4.0/go.mod h1:h9dpzScPhDTs5noEMQVWP8Wx8AFBRyS0s8KWPx/9r0g= +cloud.google.com/go/datastream v1.5.0/go.mod h1:6TZMMNPwjUqZHBKPQ1wwXpb0d5VDVPl2/XoS5yi88q4= +cloud.google.com/go/datastream v1.6.0/go.mod h1:6LQSuswqLa7S4rPAOZFVjHIG3wJIjZcZrw8JDEDJuIs= +cloud.google.com/go/datastream v1.7.0/go.mod h1:uxVRMm2elUSPuh65IbZpzJNMbuzkcvu5CjMqVIUHrww= +cloud.google.com/go/datastream v1.9.1/go.mod h1:hqnmr8kdUBmrnk65k5wNRoHSCYksvpdZIcZIEl8h43Q= +cloud.google.com/go/deploy v1.4.0/go.mod h1:5Xghikd4VrmMLNaF6FiRFDlHb59VM59YoDQnOUdsH/c= +cloud.google.com/go/deploy v1.5.0/go.mod h1:ffgdD0B89tToyW/U/D2eL0jN2+IEV/3EMuXHA0l4r+s= +cloud.google.com/go/deploy v1.6.0/go.mod h1:f9PTHehG/DjCom3QH0cntOVRm93uGBDt2vKzAPwpXQI= +cloud.google.com/go/deploy v1.8.0/go.mod h1:z3myEJnA/2wnB4sgjqdMfgxCA0EqC3RBTNcVPs93mtQ= +cloud.google.com/go/deploy v1.11.0/go.mod h1:tKuSUV5pXbn67KiubiUNUejqLs4f5cxxiCNCeyl0F2g= +cloud.google.com/go/dialogflow v1.15.0/go.mod h1:HbHDWs33WOGJgn6rfzBW1Kv807BE3O1+xGbn59zZWI4= +cloud.google.com/go/dialogflow v1.16.1/go.mod h1:po6LlzGfK+smoSmTBnbkIZY2w8ffjz/RcGSS+sh1el0= +cloud.google.com/go/dialogflow v1.17.0/go.mod h1:YNP09C/kXA1aZdBgC/VtXX74G/TKn7XVCcVumTflA+8= +cloud.google.com/go/dialogflow v1.18.0/go.mod h1:trO7Zu5YdyEuR+BhSNOqJezyFQ3aUzz0njv7sMx/iek= +cloud.google.com/go/dialogflow v1.19.0/go.mod h1:JVmlG1TwykZDtxtTXujec4tQ+D8SBFMoosgy+6Gn0s0= +cloud.google.com/go/dialogflow v1.29.0/go.mod h1:b+2bzMe+k1s9V+F2jbJwpHPzrnIyHihAdRFMtn2WXuM= +cloud.google.com/go/dialogflow v1.31.0/go.mod h1:cuoUccuL1Z+HADhyIA7dci3N5zUssgpBJmCzI6fNRB4= +cloud.google.com/go/dialogflow v1.32.0/go.mod h1:jG9TRJl8CKrDhMEcvfcfFkkpp8ZhgPz3sBGmAUYJ2qE= +cloud.google.com/go/dialogflow v1.38.0/go.mod h1:L7jnH+JL2mtmdChzAIcXQHXMvQkE3U4hTaNltEuxXn4= +cloud.google.com/go/dlp v1.6.0/go.mod h1:9eyB2xIhpU0sVwUixfBubDoRwP+GjeUoxxeueZmqvmM= +cloud.google.com/go/dlp v1.7.0/go.mod h1:68ak9vCiMBjbasxeVD17hVPxDEck+ExiHavX8kiHG+Q= +cloud.google.com/go/dlp v1.9.0/go.mod h1:qdgmqgTyReTz5/YNSSuueR8pl7hO0o9bQ39ZhtgkWp4= +cloud.google.com/go/dlp v1.10.1/go.mod h1:IM8BWz1iJd8njcNcG0+Kyd9OPnqnRNkDV8j42VT5KOI= +cloud.google.com/go/documentai v1.7.0/go.mod h1:lJvftZB5NRiFSX4moiye1SMxHx0Bc3x1+p9e/RfXYiU= +cloud.google.com/go/documentai v1.8.0/go.mod h1:xGHNEB7CtsnySCNrCFdCyyMz44RhFEEX2Q7UD0c5IhU= +cloud.google.com/go/documentai v1.9.0/go.mod h1:FS5485S8R00U10GhgBC0aNGrJxBP8ZVpEeJ7PQDZd6k= +cloud.google.com/go/documentai v1.10.0/go.mod h1:vod47hKQIPeCfN2QS/jULIvQTugbmdc0ZvxxfQY1bg4= +cloud.google.com/go/documentai v1.16.0/go.mod h1:o0o0DLTEZ+YnJZ+J4wNfTxmDVyrkzFvttBXXtYRMHkM= +cloud.google.com/go/documentai v1.18.0/go.mod h1:F6CK6iUH8J81FehpskRmhLq/3VlwQvb7TvwOceQ2tbs= +cloud.google.com/go/documentai v1.20.0/go.mod h1:yJkInoMcK0qNAEdRnqY/D5asy73tnPe88I1YTZT+a8E= +cloud.google.com/go/domains v0.6.0/go.mod h1:T9Rz3GasrpYk6mEGHh4rymIhjlnIuB4ofT1wTxDeT4Y= +cloud.google.com/go/domains v0.7.0/go.mod h1:PtZeqS1xjnXuRPKE/88Iru/LdfoRyEHYA9nFQf4UKpg= +cloud.google.com/go/domains v0.8.0/go.mod h1:M9i3MMDzGFXsydri9/vW+EWz9sWb4I6WyHqdlAk0idE= +cloud.google.com/go/domains v0.9.1/go.mod h1:aOp1c0MbejQQ2Pjf1iJvnVyT+z6R6s8pX66KaCSDYfE= +cloud.google.com/go/edgecontainer v0.1.0/go.mod h1:WgkZ9tp10bFxqO8BLPqv2LlfmQF1X8lZqwW4r1BTajk= +cloud.google.com/go/edgecontainer v0.2.0/go.mod h1:RTmLijy+lGpQ7BXuTDa4C4ssxyXT34NIuHIgKuP4s5w= +cloud.google.com/go/edgecontainer v0.3.0/go.mod h1:FLDpP4nykgwwIfcLt6zInhprzw0lEi2P1fjO6Ie0qbc= +cloud.google.com/go/edgecontainer v1.0.0/go.mod h1:cttArqZpBB2q58W/upSG++ooo6EsblxDIolxa3jSjbY= +cloud.google.com/go/edgecontainer v1.1.1/go.mod h1:O5bYcS//7MELQZs3+7mabRqoWQhXCzenBu0R8bz2rwk= +cloud.google.com/go/errorreporting v0.3.0/go.mod h1:xsP2yaAp+OAW4OIm60An2bbLpqIhKXdWR/tawvl7QzU= +cloud.google.com/go/essentialcontacts v1.3.0/go.mod h1:r+OnHa5jfj90qIfZDO/VztSFqbQan7HV75p8sA+mdGI= +cloud.google.com/go/essentialcontacts v1.4.0/go.mod h1:8tRldvHYsmnBCHdFpvU+GL75oWiBKl80BiqlFh9tp+8= +cloud.google.com/go/essentialcontacts v1.5.0/go.mod h1:ay29Z4zODTuwliK7SnX8E86aUF2CTzdNtvv42niCX0M= +cloud.google.com/go/essentialcontacts v1.6.2/go.mod h1:T2tB6tX+TRak7i88Fb2N9Ok3PvY3UNbUsMag9/BARh4= +cloud.google.com/go/eventarc v1.7.0/go.mod h1:6ctpF3zTnaQCxUjHUdcfgcA1A2T309+omHZth7gDfmc= +cloud.google.com/go/eventarc v1.8.0/go.mod h1:imbzxkyAU4ubfsaKYdQg04WS1NvncblHEup4kvF+4gw= +cloud.google.com/go/eventarc v1.10.0/go.mod h1:u3R35tmZ9HvswGRBnF48IlYgYeBcPUCjkr4BTdem2Kw= +cloud.google.com/go/eventarc v1.11.0/go.mod h1:PyUjsUKPWoRBCHeOxZd/lbOOjahV41icXyUY5kSTvVY= +cloud.google.com/go/eventarc v1.12.1/go.mod h1:mAFCW6lukH5+IZjkvrEss+jmt2kOdYlN8aMx3sRJiAI= +cloud.google.com/go/filestore v1.3.0/go.mod h1:+qbvHGvXU1HaKX2nD0WEPo92TP/8AQuCVEBXNY9z0+w= +cloud.google.com/go/filestore v1.4.0/go.mod h1:PaG5oDfo9r224f8OYXURtAsY+Fbyq/bLYoINEK8XQAI= +cloud.google.com/go/filestore v1.5.0/go.mod h1:FqBXDWBp4YLHqRnVGveOkHDf8svj9r5+mUDLupOWEDs= +cloud.google.com/go/filestore v1.6.0/go.mod h1:di5unNuss/qfZTw2U9nhFqo8/ZDSc466dre85Kydllg= +cloud.google.com/go/filestore v1.7.1/go.mod h1:y10jsorq40JJnjR/lQ8AfFbbcGlw3g+Dp8oN7i7FjV4= +cloud.google.com/go/firestore v1.9.0/go.mod h1:HMkjKHNTtRyZNiMzu7YAsLr9K3X2udY2AMwDaMEQiiE= +cloud.google.com/go/firestore v1.11.0/go.mod h1:b38dKhgzlmNNGTNZZwe7ZRFEuRab1Hay3/DBsIGKKy4= +cloud.google.com/go/functions v1.6.0/go.mod h1:3H1UA3qiIPRWD7PeZKLvHZ9SaQhR26XIJcC0A5GbvAk= +cloud.google.com/go/functions v1.7.0/go.mod h1:+d+QBcWM+RsrgZfV9xo6KfA1GlzJfxcfZcRPEhDDfzg= +cloud.google.com/go/functions v1.8.0/go.mod h1:RTZ4/HsQjIqIYP9a9YPbU+QFoQsAlYgrwOXJWHn1POY= +cloud.google.com/go/functions v1.9.0/go.mod h1:Y+Dz8yGguzO3PpIjhLTbnqV1CWmgQ5UwtlpzoyquQ08= +cloud.google.com/go/functions v1.10.0/go.mod h1:0D3hEOe3DbEvCXtYOZHQZmD+SzYsi1YbI7dGvHfldXw= +cloud.google.com/go/functions v1.12.0/go.mod h1:AXWGrF3e2C/5ehvwYo/GH6O5s09tOPksiKhz+hH8WkA= +cloud.google.com/go/functions v1.13.0/go.mod h1:EU4O007sQm6Ef/PwRsI8N2umygGqPBS/IZQKBQBcJ3c= +cloud.google.com/go/functions v1.15.1/go.mod h1:P5yNWUTkyU+LvW/S9O6V+V423VZooALQlqoXdoPz5AE= +cloud.google.com/go/gaming v1.5.0/go.mod h1:ol7rGcxP/qHTRQE/RO4bxkXq+Fix0j6D4LFPzYTIrDM= +cloud.google.com/go/gaming v1.6.0/go.mod h1:YMU1GEvA39Qt3zWGyAVA9bpYz/yAhTvaQ1t2sK4KPUA= +cloud.google.com/go/gaming v1.7.0/go.mod h1:LrB8U7MHdGgFG851iHAfqUdLcKBdQ55hzXy9xBJz0+w= +cloud.google.com/go/gaming v1.8.0/go.mod h1:xAqjS8b7jAVW0KFYeRUxngo9My3f33kFmua++Pi+ggM= +cloud.google.com/go/gaming v1.9.0/go.mod h1:Fc7kEmCObylSWLO334NcO+O9QMDyz+TKC4v1D7X+Bc0= +cloud.google.com/go/gaming v1.10.1/go.mod h1:XQQvtfP8Rb9Rxnxm5wFVpAp9zCQkJi2bLIb7iHGwB3s= +cloud.google.com/go/gkebackup v0.2.0/go.mod h1:XKvv/4LfG829/B8B7xRkk8zRrOEbKtEam6yNfuQNH60= +cloud.google.com/go/gkebackup v0.3.0/go.mod h1:n/E671i1aOQvUxT541aTkCwExO/bTer2HDlj4TsBRAo= +cloud.google.com/go/gkebackup v0.4.0/go.mod h1:byAyBGUwYGEEww7xsbnUTBHIYcOPy/PgUWUtOeRm9Vg= +cloud.google.com/go/gkeconnect v0.5.0/go.mod h1:c5lsNAg5EwAy7fkqX/+goqFsU1Da/jQFqArp+wGNr/o= +cloud.google.com/go/gkeconnect v0.6.0/go.mod h1:Mln67KyU/sHJEBY8kFZ0xTeyPtzbq9StAVvEULYK16A= +cloud.google.com/go/gkeconnect v0.7.0/go.mod h1:SNfmVqPkaEi3bF/B3CNZOAYPYdg7sU+obZ+QTky2Myw= +cloud.google.com/go/gkeconnect v0.8.1/go.mod h1:KWiK1g9sDLZqhxB2xEuPV8V9NYzrqTUmQR9shJHpOZw= +cloud.google.com/go/gkehub v0.9.0/go.mod h1:WYHN6WG8w9bXU0hqNxt8rm5uxnk8IH+lPY9J2TV7BK0= +cloud.google.com/go/gkehub v0.10.0/go.mod h1:UIPwxI0DsrpsVoWpLB0stwKCP+WFVG9+y977wO+hBH0= +cloud.google.com/go/gkehub v0.11.0/go.mod h1:JOWHlmN+GHyIbuWQPl47/C2RFhnFKH38jH9Ascu3n0E= +cloud.google.com/go/gkehub v0.12.0/go.mod h1:djiIwwzTTBrF5NaXCGv3mf7klpEMcST17VBTVVDcuaw= +cloud.google.com/go/gkehub v0.14.1/go.mod h1:VEXKIJZ2avzrbd7u+zeMtW00Y8ddk/4V9511C9CQGTY= +cloud.google.com/go/gkemulticloud v0.3.0/go.mod h1:7orzy7O0S+5kq95e4Hpn7RysVA7dPs8W/GgfUtsPbrA= +cloud.google.com/go/gkemulticloud v0.4.0/go.mod h1:E9gxVBnseLWCk24ch+P9+B2CoDFJZTyIgLKSalC7tuI= +cloud.google.com/go/gkemulticloud v0.5.0/go.mod h1:W0JDkiyi3Tqh0TJr//y19wyb1yf8llHVto2Htf2Ja3Y= +cloud.google.com/go/gkemulticloud v0.6.1/go.mod h1:kbZ3HKyTsiwqKX7Yw56+wUGwwNZViRnxWK2DVknXWfw= +cloud.google.com/go/grafeas v0.2.0/go.mod h1:KhxgtF2hb0P191HlY5besjYm6MqTSTj3LSI+M+ByZHc= +cloud.google.com/go/grafeas v0.3.0/go.mod h1:P7hgN24EyONOTMyeJH6DxG4zD7fwiYa5Q6GUgyFSOU8= +cloud.google.com/go/gsuiteaddons v1.3.0/go.mod h1:EUNK/J1lZEZO8yPtykKxLXI6JSVN2rg9bN8SXOa0bgM= +cloud.google.com/go/gsuiteaddons v1.4.0/go.mod h1:rZK5I8hht7u7HxFQcFei0+AtfS9uSushomRlg+3ua1o= +cloud.google.com/go/gsuiteaddons v1.5.0/go.mod h1:TFCClYLd64Eaa12sFVmUyG62tk4mdIsI7pAnSXRkcFo= +cloud.google.com/go/gsuiteaddons v1.6.1/go.mod h1:CodrdOqRZcLp5WOwejHWYBjZvfY0kOphkAKpF/3qdZY= +cloud.google.com/go/iam v0.1.0/go.mod h1:vcUNEa0pEm0qRVpmWepWaFMIAI8/hjB9mO8rNCJtF6c= +cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= +cloud.google.com/go/iam v0.5.0/go.mod h1:wPU9Vt0P4UmCux7mqtRu6jcpPAb74cP1fh50J3QpkUc= +cloud.google.com/go/iam v0.6.0/go.mod h1:+1AH33ueBne5MzYccyMHtEKqLE4/kJOibtffMHDMFMc= +cloud.google.com/go/iam v0.7.0/go.mod h1:H5Br8wRaDGNc8XP3keLc4unfUUZeyH3Sfl9XpQEYOeg= +cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGESjkE= +cloud.google.com/go/iam v0.11.0/go.mod h1:9PiLDanza5D+oWFZiH1uG+RnRCfEGKoyl6yo4cgWZGY= +cloud.google.com/go/iam v0.12.0/go.mod h1:knyHGviacl11zrtZUoDuYpDgLjvr28sLQaG0YB2GYAY= +cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= +cloud.google.com/go/iam v1.0.1/go.mod h1:yR3tmSL8BcZB4bxByRv2jkSIahVmCtfKZwLYGBalRE8= +cloud.google.com/go/iam v1.1.0/go.mod h1:nxdHjaKfCr7fNYx/HJMM8LgiMugmveWlkatear5gVyk= +cloud.google.com/go/iam v1.1.1 h1:lW7fzj15aVIXYHREOqjRBV9PsH0Z6u8Y46a1YGvQP4Y= +cloud.google.com/go/iam v1.1.1/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU= +cloud.google.com/go/iap v1.4.0/go.mod h1:RGFwRJdihTINIe4wZ2iCP0zF/qu18ZwyKxrhMhygBEc= +cloud.google.com/go/iap v1.5.0/go.mod h1:UH/CGgKd4KyohZL5Pt0jSKE4m3FR51qg6FKQ/z/Ix9A= +cloud.google.com/go/iap v1.6.0/go.mod h1:NSuvI9C/j7UdjGjIde7t7HBz+QTwBcapPE07+sSRcLk= +cloud.google.com/go/iap v1.7.0/go.mod h1:beqQx56T9O1G1yNPph+spKpNibDlYIiIixiqsQXxLIo= +cloud.google.com/go/iap v1.7.1/go.mod h1:WapEwPc7ZxGt2jFGB/C/bm+hP0Y6NXzOYGjpPnmMS74= +cloud.google.com/go/iap v1.8.1/go.mod h1:sJCbeqg3mvWLqjZNsI6dfAtbbV1DL2Rl7e1mTyXYREQ= +cloud.google.com/go/ids v1.1.0/go.mod h1:WIuwCaYVOzHIj2OhN9HAwvW+DBdmUAdcWlFxRl+KubM= +cloud.google.com/go/ids v1.2.0/go.mod h1:5WXvp4n25S0rA/mQWAg1YEEBBq6/s+7ml1RDCW1IrcY= +cloud.google.com/go/ids v1.3.0/go.mod h1:JBdTYwANikFKaDP6LtW5JAi4gubs57SVNQjemdt6xV4= +cloud.google.com/go/ids v1.4.1/go.mod h1:np41ed8YMU8zOgv53MMMoCntLTn2lF+SUzlM+O3u/jw= +cloud.google.com/go/iot v1.3.0/go.mod h1:r7RGh2B61+B8oz0AGE+J72AhA0G7tdXItODWsaA2oLs= +cloud.google.com/go/iot v1.4.0/go.mod h1:dIDxPOn0UvNDUMD8Ger7FIaTuvMkj+aGk94RPP0iV+g= +cloud.google.com/go/iot v1.5.0/go.mod h1:mpz5259PDl3XJthEmh9+ap0affn/MqNSP4My77Qql9o= +cloud.google.com/go/iot v1.6.0/go.mod h1:IqdAsmE2cTYYNO1Fvjfzo9po179rAtJeVGUvkLN3rLE= +cloud.google.com/go/iot v1.7.1/go.mod h1:46Mgw7ev1k9KqK1ao0ayW9h0lI+3hxeanz+L1zmbbbk= +cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxsnnOA= +cloud.google.com/go/kms v1.5.0/go.mod h1:QJS2YY0eJGBg3mnDfuaCyLauWwBJiHRboYxJ++1xJNg= +cloud.google.com/go/kms v1.6.0/go.mod h1:Jjy850yySiasBUDi6KFUwUv2n1+o7QZFyuUJg6OgjA0= +cloud.google.com/go/kms v1.8.0/go.mod h1:4xFEhYFqvW+4VMELtZyxomGSYtSQKzM178ylFW4jMAg= +cloud.google.com/go/kms v1.9.0/go.mod h1:qb1tPTgfF9RQP8e1wq4cLFErVuTJv7UsSC915J8dh3w= +cloud.google.com/go/kms v1.10.0/go.mod h1:ng3KTUtQQU9bPX3+QGLsflZIHlkbn8amFAMY63m8d24= +cloud.google.com/go/kms v1.10.1/go.mod h1:rIWk/TryCkR59GMC3YtHtXeLzd634lBbKenvyySAyYI= +cloud.google.com/go/kms v1.11.0/go.mod h1:hwdiYC0xjnWsKQQCQQmIQnS9asjYVSK6jtXm+zFqXLM= +cloud.google.com/go/kms v1.12.1/go.mod h1:c9J991h5DTl+kg7gi3MYomh12YEENGrf48ee/N/2CDM= +cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic= +cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= +cloud.google.com/go/language v1.7.0/go.mod h1:DJ6dYN/W+SQOjF8e1hLQXMF21AkH2w9wiPzPCJa2MIE= +cloud.google.com/go/language v1.8.0/go.mod h1:qYPVHf7SPoNNiCL2Dr0FfEFNil1qi3pQEyygwpgVKB8= +cloud.google.com/go/language v1.9.0/go.mod h1:Ns15WooPM5Ad/5no/0n81yUetis74g3zrbeJBE+ptUY= +cloud.google.com/go/language v1.10.1/go.mod h1:CPp94nsdVNiQEt1CNjF5WkTcisLiHPyIbMhvR8H2AW0= +cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8= +cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6t/iPhY2Tyfu08= +cloud.google.com/go/lifesciences v0.8.0/go.mod h1:lFxiEOMqII6XggGbOnKiyZ7IBwoIqA84ClvoezaA/bo= +cloud.google.com/go/lifesciences v0.9.1/go.mod h1:hACAOd1fFbCGLr/+weUKRAJas82Y4vrL3O5326N//Wc= +cloud.google.com/go/logging v1.6.1/go.mod h1:5ZO0mHHbvm8gEmeEUHrmDlTDSu5imF6MUP9OfilNXBw= +cloud.google.com/go/logging v1.7.0 h1:CJYxlNNNNAMkHp9em/YEXcfJg+rPDg7YfwoRpMU+t5I= +cloud.google.com/go/logging v1.7.0/go.mod h1:3xjP2CjkM3ZkO73aj4ASA5wRPGGCRrPIAeNqVNkzY8M= +cloud.google.com/go/longrunning v0.1.1/go.mod h1:UUFxuDWkv22EuY93jjmDMFT5GPQKeFVJBIF6QlTqdsE= +cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI0YSmW3iIlLJc= +cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo= +cloud.google.com/go/longrunning v0.4.2/go.mod h1:OHrnaYyLUV6oqwh0xiS7e5sLQhP1m0QU9R+WhGDMgIQ= +cloud.google.com/go/longrunning v0.5.0/go.mod h1:0JNuqRShmscVAhIACGtskSAWtqtOoPkwP0YF1oVEchc= +cloud.google.com/go/longrunning v0.5.1 h1:Fr7TXftcqTudoyRJa113hyaqlGdiBQkp0Gq7tErFDWI= +cloud.google.com/go/longrunning v0.5.1/go.mod h1:spvimkwdz6SPWKEt/XBij79E9fiTkHSQl/fRUUQJYJc= +cloud.google.com/go/managedidentities v1.3.0/go.mod h1:UzlW3cBOiPrzucO5qWkNkh0w33KFtBJU281hacNvsdE= +cloud.google.com/go/managedidentities v1.4.0/go.mod h1:NWSBYbEMgqmbZsLIyKvxrYbtqOsxY1ZrGM+9RgDqInM= +cloud.google.com/go/managedidentities v1.5.0/go.mod h1:+dWcZ0JlUmpuxpIDfyP5pP5y0bLdRwOS4Lp7gMni/LA= +cloud.google.com/go/managedidentities v1.6.1/go.mod h1:h/irGhTN2SkZ64F43tfGPMbHnypMbu4RB3yl8YcuEak= +cloud.google.com/go/maps v0.1.0/go.mod h1:BQM97WGyfw9FWEmQMpZ5T6cpovXXSd1cGmFma94eubI= +cloud.google.com/go/maps v0.6.0/go.mod h1:o6DAMMfb+aINHz/p/jbcY+mYeXBoZoxTfdSQ8VAJaCw= +cloud.google.com/go/maps v0.7.0/go.mod h1:3GnvVl3cqeSvgMcpRlQidXsPYuDGQ8naBis7MVzpXsY= +cloud.google.com/go/mediatranslation v0.5.0/go.mod h1:jGPUhGTybqsPQn91pNXw0xVHfuJ3leR1wj37oU3y1f4= +cloud.google.com/go/mediatranslation v0.6.0/go.mod h1:hHdBCTYNigsBxshbznuIMFNe5QXEowAuNmmC7h8pu5w= +cloud.google.com/go/mediatranslation v0.7.0/go.mod h1:LCnB/gZr90ONOIQLgSXagp8XUW1ODs2UmUMvcgMfI2I= +cloud.google.com/go/mediatranslation v0.8.1/go.mod h1:L/7hBdEYbYHQJhX2sldtTO5SZZ1C1vkapubj0T2aGig= +cloud.google.com/go/memcache v1.4.0/go.mod h1:rTOfiGZtJX1AaFUrOgsMHX5kAzaTQ8azHiuDoTPzNsE= +cloud.google.com/go/memcache v1.5.0/go.mod h1:dk3fCK7dVo0cUU2c36jKb4VqKPS22BTkf81Xq617aWM= +cloud.google.com/go/memcache v1.6.0/go.mod h1:XS5xB0eQZdHtTuTF9Hf8eJkKtR3pVRCcvJwtm68T3rA= +cloud.google.com/go/memcache v1.7.0/go.mod h1:ywMKfjWhNtkQTxrWxCkCFkoPjLHPW6A7WOTVI8xy3LY= +cloud.google.com/go/memcache v1.9.0/go.mod h1:8oEyzXCu+zo9RzlEaEjHl4KkgjlNDaXbCQeQWlzNFJM= +cloud.google.com/go/memcache v1.10.1/go.mod h1:47YRQIarv4I3QS5+hoETgKO40InqzLP6kpNLvyXuyaA= +cloud.google.com/go/metastore v1.5.0/go.mod h1:2ZNrDcQwghfdtCwJ33nM0+GrBGlVuh8rakL3vdPY3XY= +cloud.google.com/go/metastore v1.6.0/go.mod h1:6cyQTls8CWXzk45G55x57DVQ9gWg7RiH65+YgPsNh9s= +cloud.google.com/go/metastore v1.7.0/go.mod h1:s45D0B4IlsINu87/AsWiEVYbLaIMeUSoxlKKDqBGFS8= +cloud.google.com/go/metastore v1.8.0/go.mod h1:zHiMc4ZUpBiM7twCIFQmJ9JMEkDSyZS9U12uf7wHqSI= +cloud.google.com/go/metastore v1.10.0/go.mod h1:fPEnH3g4JJAk+gMRnrAnoqyv2lpUCqJPWOodSaf45Eo= +cloud.google.com/go/metastore v1.11.1/go.mod h1:uZuSo80U3Wd4zi6C22ZZliOUJ3XeM/MlYi/z5OAOWRA= +cloud.google.com/go/monitoring v1.1.0/go.mod h1:L81pzz7HKn14QCMaCs6NTQkdBnE87TElyanS95vIcl4= +cloud.google.com/go/monitoring v1.7.0/go.mod h1:HpYse6kkGo//7p6sT0wsIC6IBDET0RhIsnmlA53dvEk= +cloud.google.com/go/monitoring v1.8.0/go.mod h1:E7PtoMJ1kQXWxPjB6mv2fhC5/15jInuulFdYYtlcvT4= +cloud.google.com/go/monitoring v1.12.0/go.mod h1:yx8Jj2fZNEkL/GYZyTLS4ZtZEZN8WtDEiEqG4kLK50w= +cloud.google.com/go/monitoring v1.13.0/go.mod h1:k2yMBAB1H9JT/QETjNkgdCGD9bPF712XiLTVr+cBrpw= +cloud.google.com/go/monitoring v1.15.1 h1:65JhLMd+JiYnXr6j5Z63dUYCuOg770p8a/VC+gil/58= +cloud.google.com/go/monitoring v1.15.1/go.mod h1:lADlSAlFdbqQuwwpaImhsJXu1QSdd3ojypXrFSMr2rM= +cloud.google.com/go/networkconnectivity v1.4.0/go.mod h1:nOl7YL8odKyAOtzNX73/M5/mGZgqqMeryi6UPZTk/rA= +cloud.google.com/go/networkconnectivity v1.5.0/go.mod h1:3GzqJx7uhtlM3kln0+x5wyFvuVH1pIBJjhCpjzSt75o= +cloud.google.com/go/networkconnectivity v1.6.0/go.mod h1:OJOoEXW+0LAxHh89nXd64uGG+FbQoeH8DtxCHVOMlaM= +cloud.google.com/go/networkconnectivity v1.7.0/go.mod h1:RMuSbkdbPwNMQjB5HBWD5MpTBnNm39iAVpC3TmsExt8= +cloud.google.com/go/networkconnectivity v1.10.0/go.mod h1:UP4O4sWXJG13AqrTdQCD9TnLGEbtNRqjuaaA7bNjF5E= +cloud.google.com/go/networkconnectivity v1.11.0/go.mod h1:iWmDD4QF16VCDLXUqvyspJjIEtBR/4zq5hwnY2X3scM= +cloud.google.com/go/networkconnectivity v1.12.1/go.mod h1:PelxSWYM7Sh9/guf8CFhi6vIqf19Ir/sbfZRUwXh92E= +cloud.google.com/go/networkmanagement v1.4.0/go.mod h1:Q9mdLLRn60AsOrPc8rs8iNV6OHXaGcDdsIQe1ohekq8= +cloud.google.com/go/networkmanagement v1.5.0/go.mod h1:ZnOeZ/evzUdUsnvRt792H0uYEnHQEMaz+REhhzJRcf4= +cloud.google.com/go/networkmanagement v1.6.0/go.mod h1:5pKPqyXjB/sgtvB5xqOemumoQNB7y95Q7S+4rjSOPYY= +cloud.google.com/go/networkmanagement v1.8.0/go.mod h1:Ho/BUGmtyEqrttTgWEe7m+8vDdK74ibQc+Be0q7Fof0= +cloud.google.com/go/networksecurity v0.5.0/go.mod h1:xS6fOCoqpVC5zx15Z/MqkfDwH4+m/61A3ODiDV1xmiQ= +cloud.google.com/go/networksecurity v0.6.0/go.mod h1:Q5fjhTr9WMI5mbpRYEbiexTzROf7ZbDzvzCrNl14nyU= +cloud.google.com/go/networksecurity v0.7.0/go.mod h1:mAnzoxx/8TBSyXEeESMy9OOYwo1v+gZ5eMRnsT5bC8k= +cloud.google.com/go/networksecurity v0.8.0/go.mod h1:B78DkqsxFG5zRSVuwYFRZ9Xz8IcQ5iECsNrPn74hKHU= +cloud.google.com/go/networksecurity v0.9.1/go.mod h1:MCMdxOKQ30wsBI1eI659f9kEp4wuuAueoC9AJKSPWZQ= +cloud.google.com/go/notebooks v1.2.0/go.mod h1:9+wtppMfVPUeJ8fIWPOq1UnATHISkGXGqTkxeieQ6UY= +cloud.google.com/go/notebooks v1.3.0/go.mod h1:bFR5lj07DtCPC7YAAJ//vHskFBxA5JzYlH68kXVdk34= +cloud.google.com/go/notebooks v1.4.0/go.mod h1:4QPMngcwmgb6uw7Po99B2xv5ufVoIQ7nOGDyL4P8AgA= +cloud.google.com/go/notebooks v1.5.0/go.mod h1:q8mwhnP9aR8Hpfnrc5iN5IBhrXUy8S2vuYs+kBJ/gu0= +cloud.google.com/go/notebooks v1.7.0/go.mod h1:PVlaDGfJgj1fl1S3dUwhFMXFgfYGhYQt2164xOMONmE= +cloud.google.com/go/notebooks v1.8.0/go.mod h1:Lq6dYKOYOWUCTvw5t2q1gp1lAp0zxAxRycayS0iJcqQ= +cloud.google.com/go/notebooks v1.9.1/go.mod h1:zqG9/gk05JrzgBt4ghLzEepPHNwE5jgPcHZRKhlC1A8= +cloud.google.com/go/optimization v1.1.0/go.mod h1:5po+wfvX5AQlPznyVEZjGJTMr4+CAkJf2XSTQOOl9l4= +cloud.google.com/go/optimization v1.2.0/go.mod h1:Lr7SOHdRDENsh+WXVmQhQTrzdu9ybg0NecjHidBq6xs= +cloud.google.com/go/optimization v1.3.1/go.mod h1:IvUSefKiwd1a5p0RgHDbWCIbDFgKuEdB+fPPuP0IDLI= +cloud.google.com/go/optimization v1.4.1/go.mod h1:j64vZQP7h9bO49m2rVaTVoNM0vEBEN5eKPUPbZyXOrk= +cloud.google.com/go/orchestration v1.3.0/go.mod h1:Sj5tq/JpWiB//X/q3Ngwdl5K7B7Y0KZ7bfv0wL6fqVA= +cloud.google.com/go/orchestration v1.4.0/go.mod h1:6W5NLFWs2TlniBphAViZEVhrXRSMgUGDfW7vrWKvsBk= +cloud.google.com/go/orchestration v1.6.0/go.mod h1:M62Bevp7pkxStDfFfTuCOaXgaaqRAga1yKyoMtEoWPQ= +cloud.google.com/go/orchestration v1.8.1/go.mod h1:4sluRF3wgbYVRqz7zJ1/EUNc90TTprliq9477fGobD8= +cloud.google.com/go/orgpolicy v1.4.0/go.mod h1:xrSLIV4RePWmP9P3tBl8S93lTmlAxjm06NSm2UTmKvE= +cloud.google.com/go/orgpolicy v1.5.0/go.mod h1:hZEc5q3wzwXJaKrsx5+Ewg0u1LxJ51nNFlext7Tanwc= +cloud.google.com/go/orgpolicy v1.10.0/go.mod h1:w1fo8b7rRqlXlIJbVhOMPrwVljyuW5mqssvBtU18ONc= +cloud.google.com/go/orgpolicy v1.11.0/go.mod h1:2RK748+FtVvnfuynxBzdnyu7sygtoZa1za/0ZfpOs1M= +cloud.google.com/go/orgpolicy v1.11.1/go.mod h1:8+E3jQcpZJQliP+zaFfayC2Pg5bmhuLK755wKhIIUCE= +cloud.google.com/go/osconfig v1.7.0/go.mod h1:oVHeCeZELfJP7XLxcBGTMBvRO+1nQ5tFG9VQTmYS2Fs= +cloud.google.com/go/osconfig v1.8.0/go.mod h1:EQqZLu5w5XA7eKizepumcvWx+m8mJUhEwiPqWiZeEdg= +cloud.google.com/go/osconfig v1.9.0/go.mod h1:Yx+IeIZJ3bdWmzbQU4fxNl8xsZ4amB+dygAwFPlvnNo= +cloud.google.com/go/osconfig v1.10.0/go.mod h1:uMhCzqC5I8zfD9zDEAfvgVhDS8oIjySWh+l4WK6GnWw= +cloud.google.com/go/osconfig v1.11.0/go.mod h1:aDICxrur2ogRd9zY5ytBLV89KEgT2MKB2L/n6x1ooPw= +cloud.google.com/go/osconfig v1.12.0/go.mod h1:8f/PaYzoS3JMVfdfTubkowZYGmAhUCjjwnjqWI7NVBc= +cloud.google.com/go/osconfig v1.12.1/go.mod h1:4CjBxND0gswz2gfYRCUoUzCm9zCABp91EeTtWXyz0tE= +cloud.google.com/go/oslogin v1.4.0/go.mod h1:YdgMXWRaElXz/lDk1Na6Fh5orF7gvmJ0FGLIs9LId4E= +cloud.google.com/go/oslogin v1.5.0/go.mod h1:D260Qj11W2qx/HVF29zBg+0fd6YCSjSqLUkY/qEenQU= +cloud.google.com/go/oslogin v1.6.0/go.mod h1:zOJ1O3+dTU8WPlGEkFSh7qeHPPSoxrcMbbK1Nm2iX70= +cloud.google.com/go/oslogin v1.7.0/go.mod h1:e04SN0xO1UNJ1M5GP0vzVBFicIe4O53FOfcixIqTyXo= +cloud.google.com/go/oslogin v1.9.0/go.mod h1:HNavntnH8nzrn8JCTT5fj18FuJLFJc4NaZJtBnQtKFs= +cloud.google.com/go/oslogin v1.10.1/go.mod h1:x692z7yAue5nE7CsSnoG0aaMbNoRJRXO4sn73R+ZqAs= +cloud.google.com/go/phishingprotection v0.5.0/go.mod h1:Y3HZknsK9bc9dMi+oE8Bim0lczMU6hrX0UpADuMefr0= +cloud.google.com/go/phishingprotection v0.6.0/go.mod h1:9Y3LBLgy0kDTcYET8ZH3bq/7qni15yVUoAxiFxnlSUA= +cloud.google.com/go/phishingprotection v0.7.0/go.mod h1:8qJI4QKHoda/sb/7/YmMQ2omRLSLYSu9bU0EKCNI+Lk= +cloud.google.com/go/phishingprotection v0.8.1/go.mod h1:AxonW7GovcA8qdEk13NfHq9hNx5KPtfxXNeUxTDxB6I= +cloud.google.com/go/policytroubleshooter v1.3.0/go.mod h1:qy0+VwANja+kKrjlQuOzmlvscn4RNsAc0e15GGqfMxg= +cloud.google.com/go/policytroubleshooter v1.4.0/go.mod h1:DZT4BcRw3QoO8ota9xw/LKtPa8lKeCByYeKTIf/vxdE= +cloud.google.com/go/policytroubleshooter v1.5.0/go.mod h1:Rz1WfV+1oIpPdN2VvvuboLVRsB1Hclg3CKQ53j9l8vw= +cloud.google.com/go/policytroubleshooter v1.6.0/go.mod h1:zYqaPTsmfvpjm5ULxAyD/lINQxJ0DDsnWOP/GZ7xzBc= +cloud.google.com/go/policytroubleshooter v1.7.1/go.mod h1:0NaT5v3Ag1M7U5r0GfDCpUFkWd9YqpubBWsQlhanRv0= +cloud.google.com/go/privatecatalog v0.5.0/go.mod h1:XgosMUvvPyxDjAVNDYxJ7wBW8//hLDDYmnsNcMGq1K0= +cloud.google.com/go/privatecatalog v0.6.0/go.mod h1:i/fbkZR0hLN29eEWiiwue8Pb+GforiEIBnV9yrRUOKI= +cloud.google.com/go/privatecatalog v0.7.0/go.mod h1:2s5ssIFO69F5csTXcwBP7NPFTZvps26xGzvQ2PQaBYg= +cloud.google.com/go/privatecatalog v0.8.0/go.mod h1:nQ6pfaegeDAq/Q5lrfCQzQLhubPiZhSaNhIgfJlnIXs= +cloud.google.com/go/privatecatalog v0.9.1/go.mod h1:0XlDXW2unJXdf9zFz968Hp35gl/bhF4twwpXZAW50JA= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/pubsub v1.26.0/go.mod h1:QgBH3U/jdJy/ftjPhTkyXNj543Tin1pRYcdcPRnFIRI= +cloud.google.com/go/pubsub v1.27.1/go.mod h1:hQN39ymbV9geqBnfQq6Xf63yNhUAhv9CZhzp5O6qsW0= +cloud.google.com/go/pubsub v1.28.0/go.mod h1:vuXFpwaVoIPQMGXqRyUQigu/AX1S3IWugR9xznmcXX8= +cloud.google.com/go/pubsub v1.30.0/go.mod h1:qWi1OPS0B+b5L+Sg6Gmc9zD1Y+HaM0MdUr7LsupY1P4= +cloud.google.com/go/pubsub v1.32.0/go.mod h1:f+w71I33OMyxf9VpMVcZbnG5KSUkCOUHYpFd5U1GdRc= +cloud.google.com/go/pubsublite v1.5.0/go.mod h1:xapqNQ1CuLfGi23Yda/9l4bBCKz/wC3KIJ5gKcxveZg= +cloud.google.com/go/pubsublite v1.6.0/go.mod h1:1eFCS0U11xlOuMFV/0iBqw3zP12kddMeCbj/F3FSj9k= +cloud.google.com/go/pubsublite v1.7.0/go.mod h1:8hVMwRXfDfvGm3fahVbtDbiLePT3gpoiJYJY+vxWxVM= +cloud.google.com/go/pubsublite v1.8.1/go.mod h1:fOLdU4f5xldK4RGJrBMm+J7zMWNj/k4PxwEZXy39QS0= +cloud.google.com/go/recaptchaenterprise v1.3.1/go.mod h1:OdD+q+y4XGeAlxRaMn1Y7/GveP6zmq76byL6tjPE7d4= +cloud.google.com/go/recaptchaenterprise/v2 v2.1.0/go.mod h1:w9yVqajwroDNTfGuhmOjPDN//rZGySaf6PtFVcSCa7o= +cloud.google.com/go/recaptchaenterprise/v2 v2.2.0/go.mod h1:/Zu5jisWGeERrd5HnlS3EUGb/D335f9k51B/FVil0jk= +cloud.google.com/go/recaptchaenterprise/v2 v2.3.0/go.mod h1:O9LwGCjrhGHBQET5CA7dd5NwwNQUErSgEDit1DLNTdo= +cloud.google.com/go/recaptchaenterprise/v2 v2.4.0/go.mod h1:Am3LHfOuBstrLrNCBrlI5sbwx9LBg3te2N6hGvHn2mE= +cloud.google.com/go/recaptchaenterprise/v2 v2.5.0/go.mod h1:O8LzcHXN3rz0j+LBC91jrwI3R+1ZSZEWrfL7XHgNo9U= +cloud.google.com/go/recaptchaenterprise/v2 v2.6.0/go.mod h1:RPauz9jeLtB3JVzg6nCbe12qNoaa8pXc4d/YukAmcnA= +cloud.google.com/go/recaptchaenterprise/v2 v2.7.0/go.mod h1:19wVj/fs5RtYtynAPJdDTb69oW0vNHYDBTbB4NvMD9c= +cloud.google.com/go/recaptchaenterprise/v2 v2.7.2/go.mod h1:kR0KjsJS7Jt1YSyWFkseQ756D45kaYNTlDPPaRAvDBU= +cloud.google.com/go/recommendationengine v0.5.0/go.mod h1:E5756pJcVFeVgaQv3WNpImkFP8a+RptV6dDLGPILjvg= +cloud.google.com/go/recommendationengine v0.6.0/go.mod h1:08mq2umu9oIqc7tDy8sx+MNJdLG0fUi3vaSVbztHgJ4= +cloud.google.com/go/recommendationengine v0.7.0/go.mod h1:1reUcE3GIu6MeBz/h5xZJqNLuuVjNg1lmWMPyjatzac= +cloud.google.com/go/recommendationengine v0.8.1/go.mod h1:MrZihWwtFYWDzE6Hz5nKcNz3gLizXVIDI/o3G1DLcrE= +cloud.google.com/go/recommender v1.5.0/go.mod h1:jdoeiBIVrJe9gQjwd759ecLJbxCDED4A6p+mqoqDvTg= +cloud.google.com/go/recommender v1.6.0/go.mod h1:+yETpm25mcoiECKh9DEScGzIRyDKpZ0cEhWGo+8bo+c= +cloud.google.com/go/recommender v1.7.0/go.mod h1:XLHs/W+T8olwlGOgfQenXBTbIseGclClff6lhFVe9Bs= +cloud.google.com/go/recommender v1.8.0/go.mod h1:PkjXrTT05BFKwxaUxQmtIlrtj0kph108r02ZZQ5FE70= +cloud.google.com/go/recommender v1.9.0/go.mod h1:PnSsnZY7q+VL1uax2JWkt/UegHssxjUVVCrX52CuEmQ= +cloud.google.com/go/recommender v1.10.1/go.mod h1:XFvrE4Suqn5Cq0Lf+mCP6oBHD/yRMA8XxP5sb7Q7gpA= +cloud.google.com/go/redis v1.7.0/go.mod h1:V3x5Jq1jzUcg+UNsRvdmsfuFnit1cfe3Z/PGyq/lm4Y= +cloud.google.com/go/redis v1.8.0/go.mod h1:Fm2szCDavWzBk2cDKxrkmWBqoCiL1+Ctwq7EyqBCA/A= +cloud.google.com/go/redis v1.9.0/go.mod h1:HMYQuajvb2D0LvMgZmLDZW8V5aOC/WxstZHiy4g8OiA= +cloud.google.com/go/redis v1.10.0/go.mod h1:ThJf3mMBQtW18JzGgh41/Wld6vnDDc/F/F35UolRZPM= +cloud.google.com/go/redis v1.11.0/go.mod h1:/X6eicana+BWcUda5PpwZC48o37SiFVTFSs0fWAJ7uQ= +cloud.google.com/go/redis v1.13.1/go.mod h1:VP7DGLpE91M6bcsDdMuyCm2hIpB6Vp2hI090Mfd1tcg= +cloud.google.com/go/resourcemanager v1.3.0/go.mod h1:bAtrTjZQFJkiWTPDb1WBjzvc6/kifjj4QBYuKCCoqKA= +cloud.google.com/go/resourcemanager v1.4.0/go.mod h1:MwxuzkumyTX7/a3n37gmsT3py7LIXwrShilPh3P1tR0= +cloud.google.com/go/resourcemanager v1.5.0/go.mod h1:eQoXNAiAvCf5PXxWxXjhKQoTMaUSNrEfg+6qdf/wots= +cloud.google.com/go/resourcemanager v1.6.0/go.mod h1:YcpXGRs8fDzcUl1Xw8uOVmI8JEadvhRIkoXXUNVYcVo= +cloud.google.com/go/resourcemanager v1.7.0/go.mod h1:HlD3m6+bwhzj9XCouqmeiGuni95NTrExfhoSrkC/3EI= +cloud.google.com/go/resourcemanager v1.9.1/go.mod h1:dVCuosgrh1tINZ/RwBufr8lULmWGOkPS8gL5gqyjdT8= +cloud.google.com/go/resourcesettings v1.3.0/go.mod h1:lzew8VfESA5DQ8gdlHwMrqZs1S9V87v3oCnKCWoOuQU= +cloud.google.com/go/resourcesettings v1.4.0/go.mod h1:ldiH9IJpcrlC3VSuCGvjR5of/ezRrOxFtpJoJo5SmXg= +cloud.google.com/go/resourcesettings v1.5.0/go.mod h1:+xJF7QSG6undsQDfsCJyqWXyBwUoJLhetkRMDRnIoXA= +cloud.google.com/go/resourcesettings v1.6.1/go.mod h1:M7mk9PIZrC5Fgsu1kZJci6mpgN8o0IUzVx3eJU3y4Jw= +cloud.google.com/go/retail v1.8.0/go.mod h1:QblKS8waDmNUhghY2TI9O3JLlFk8jybHeV4BF19FrE4= +cloud.google.com/go/retail v1.9.0/go.mod h1:g6jb6mKuCS1QKnH/dpu7isX253absFl6iE92nHwlBUY= +cloud.google.com/go/retail v1.10.0/go.mod h1:2gDk9HsL4HMS4oZwz6daui2/jmKvqShXKQuB2RZ+cCc= +cloud.google.com/go/retail v1.11.0/go.mod h1:MBLk1NaWPmh6iVFSz9MeKG/Psyd7TAgm6y/9L2B4x9Y= +cloud.google.com/go/retail v1.12.0/go.mod h1:UMkelN/0Z8XvKymXFbD4EhFJlYKRx1FGhQkVPU5kF14= +cloud.google.com/go/retail v1.14.1/go.mod h1:y3Wv3Vr2k54dLNIrCzenyKG8g8dhvhncT2NcNjb/6gE= +cloud.google.com/go/run v0.2.0/go.mod h1:CNtKsTA1sDcnqqIFR3Pb5Tq0usWxJJvsWOCPldRU3Do= +cloud.google.com/go/run v0.3.0/go.mod h1:TuyY1+taHxTjrD0ZFk2iAR+xyOXEA0ztb7U3UNA0zBo= +cloud.google.com/go/run v0.8.0/go.mod h1:VniEnuBwqjigv0A7ONfQUaEItaiCRVujlMqerPPiktM= +cloud.google.com/go/run v0.9.0/go.mod h1:Wwu+/vvg8Y+JUApMwEDfVfhetv30hCG4ZwDR/IXl2Qg= +cloud.google.com/go/scheduler v1.4.0/go.mod h1:drcJBmxF3aqZJRhmkHQ9b3uSSpQoltBPGPxGAWROx6s= +cloud.google.com/go/scheduler v1.5.0/go.mod h1:ri073ym49NW3AfT6DZi21vLZrG07GXr5p3H1KxN5QlI= +cloud.google.com/go/scheduler v1.6.0/go.mod h1:SgeKVM7MIwPn3BqtcBntpLyrIJftQISRrYB5ZtT+KOk= +cloud.google.com/go/scheduler v1.7.0/go.mod h1:jyCiBqWW956uBjjPMMuX09n3x37mtyPJegEWKxRsn44= +cloud.google.com/go/scheduler v1.8.0/go.mod h1:TCET+Y5Gp1YgHT8py4nlg2Sew8nUHMqcpousDgXJVQc= +cloud.google.com/go/scheduler v1.9.0/go.mod h1:yexg5t+KSmqu+njTIh3b7oYPheFtBWGcbVUYF1GGMIc= +cloud.google.com/go/scheduler v1.10.1/go.mod h1:R63Ldltd47Bs4gnhQkmNDse5w8gBRrhObZ54PxgR2Oo= +cloud.google.com/go/secretmanager v1.6.0/go.mod h1:awVa/OXF6IiyaU1wQ34inzQNc4ISIDIrId8qE5QGgKA= +cloud.google.com/go/secretmanager v1.8.0/go.mod h1:hnVgi/bN5MYHd3Gt0SPuTPPp5ENina1/LxM+2W9U9J4= +cloud.google.com/go/secretmanager v1.9.0/go.mod h1:b71qH2l1yHmWQHt9LC80akm86mX8AL6X1MA01dW8ht4= +cloud.google.com/go/secretmanager v1.10.0/go.mod h1:MfnrdvKMPNra9aZtQFvBcvRU54hbPD8/HayQdlUgJpU= +cloud.google.com/go/secretmanager v1.11.1/go.mod h1:znq9JlXgTNdBeQk9TBW/FnR/W4uChEKGeqQWAJ8SXFw= +cloud.google.com/go/security v1.5.0/go.mod h1:lgxGdyOKKjHL4YG3/YwIL2zLqMFCKs0UbQwgyZmfJl4= +cloud.google.com/go/security v1.7.0/go.mod h1:mZklORHl6Bg7CNnnjLH//0UlAlaXqiG7Lb9PsPXLfD0= +cloud.google.com/go/security v1.8.0/go.mod h1:hAQOwgmaHhztFhiQ41CjDODdWP0+AE1B3sX4OFlq+GU= +cloud.google.com/go/security v1.9.0/go.mod h1:6Ta1bO8LXI89nZnmnsZGp9lVoVWXqsVbIq/t9dzI+2Q= +cloud.google.com/go/security v1.10.0/go.mod h1:QtOMZByJVlibUT2h9afNDWRZ1G96gVywH8T5GUSb9IA= +cloud.google.com/go/security v1.12.0/go.mod h1:rV6EhrpbNHrrxqlvW0BWAIawFWq3X90SduMJdFwtLB8= +cloud.google.com/go/security v1.13.0/go.mod h1:Q1Nvxl1PAgmeW0y3HTt54JYIvUdtcpYKVfIB8AOMZ+0= +cloud.google.com/go/security v1.15.1/go.mod h1:MvTnnbsWnehoizHi09zoiZob0iCHVcL4AUBj76h9fXA= +cloud.google.com/go/securitycenter v1.13.0/go.mod h1:cv5qNAqjY84FCN6Y9z28WlkKXyWsgLO832YiWwkCWcU= +cloud.google.com/go/securitycenter v1.14.0/go.mod h1:gZLAhtyKv85n52XYWt6RmeBdydyxfPeTrpToDPw4Auc= +cloud.google.com/go/securitycenter v1.15.0/go.mod h1:PeKJ0t8MoFmmXLXWm41JidyzI3PJjd8sXWaVqg43WWk= +cloud.google.com/go/securitycenter v1.16.0/go.mod h1:Q9GMaLQFUD+5ZTabrbujNWLtSLZIZF7SAR0wWECrjdk= +cloud.google.com/go/securitycenter v1.18.1/go.mod h1:0/25gAzCM/9OL9vVx4ChPeM/+DlfGQJDwBy/UC8AKK0= +cloud.google.com/go/securitycenter v1.19.0/go.mod h1:LVLmSg8ZkkyaNy4u7HCIshAngSQ8EcIRREP3xBnyfag= +cloud.google.com/go/securitycenter v1.23.0/go.mod h1:8pwQ4n+Y9WCWM278R8W3nF65QtY172h4S8aXyI9/hsQ= +cloud.google.com/go/servicecontrol v1.4.0/go.mod h1:o0hUSJ1TXJAmi/7fLJAedOovnujSEvjKCAFNXPQ1RaU= +cloud.google.com/go/servicecontrol v1.5.0/go.mod h1:qM0CnXHhyqKVuiZnGKrIurvVImCs8gmqWsDoqe9sU1s= +cloud.google.com/go/servicecontrol v1.10.0/go.mod h1:pQvyvSRh7YzUF2efw7H87V92mxU8FnFDawMClGCNuAA= +cloud.google.com/go/servicecontrol v1.11.0/go.mod h1:kFmTzYzTUIuZs0ycVqRHNaNhgR+UMUpw9n02l/pY+mc= +cloud.google.com/go/servicecontrol v1.11.1/go.mod h1:aSnNNlwEFBY+PWGQ2DoM0JJ/QUXqV5/ZD9DOLB7SnUk= +cloud.google.com/go/servicedirectory v1.4.0/go.mod h1:gH1MUaZCgtP7qQiI+F+A+OpeKF/HQWgtAddhTbhL2bs= +cloud.google.com/go/servicedirectory v1.5.0/go.mod h1:QMKFL0NUySbpZJ1UZs3oFAmdvVxhhxB6eJ/Vlp73dfg= +cloud.google.com/go/servicedirectory v1.6.0/go.mod h1:pUlbnWsLH9c13yGkxCmfumWEPjsRs1RlmJ4pqiNjVL4= +cloud.google.com/go/servicedirectory v1.7.0/go.mod h1:5p/U5oyvgYGYejufvxhgwjL8UVXjkuw7q5XcG10wx1U= +cloud.google.com/go/servicedirectory v1.8.0/go.mod h1:srXodfhY1GFIPvltunswqXpVxFPpZjf8nkKQT7XcXaY= +cloud.google.com/go/servicedirectory v1.9.0/go.mod h1:29je5JjiygNYlmsGz8k6o+OZ8vd4f//bQLtvzkPPT/s= +cloud.google.com/go/servicedirectory v1.10.1/go.mod h1:Xv0YVH8s4pVOwfM/1eMTl0XJ6bzIOSLDt8f8eLaGOxQ= +cloud.google.com/go/servicemanagement v1.4.0/go.mod h1:d8t8MDbezI7Z2R1O/wu8oTggo3BI2GKYbdG4y/SJTco= +cloud.google.com/go/servicemanagement v1.5.0/go.mod h1:XGaCRe57kfqu4+lRxaFEAuqmjzF0r+gWHjWqKqBvKFo= +cloud.google.com/go/servicemanagement v1.6.0/go.mod h1:aWns7EeeCOtGEX4OvZUWCCJONRZeFKiptqKf1D0l/Jc= +cloud.google.com/go/servicemanagement v1.8.0/go.mod h1:MSS2TDlIEQD/fzsSGfCdJItQveu9NXnUniTrq/L8LK4= +cloud.google.com/go/serviceusage v1.3.0/go.mod h1:Hya1cozXM4SeSKTAgGXgj97GlqUvF5JaoXacR1JTP/E= +cloud.google.com/go/serviceusage v1.4.0/go.mod h1:SB4yxXSaYVuUBYUml6qklyONXNLt83U0Rb+CXyhjEeU= +cloud.google.com/go/serviceusage v1.5.0/go.mod h1:w8U1JvqUqwJNPEOTQjrMHkw3IaIFLoLsPLvsE3xueec= +cloud.google.com/go/serviceusage v1.6.0/go.mod h1:R5wwQcbOWsyuOfbP9tGdAnCAc6B9DRwPG1xtWMDeuPA= +cloud.google.com/go/shell v1.3.0/go.mod h1:VZ9HmRjZBsjLGXusm7K5Q5lzzByZmJHf1d0IWHEN5X4= +cloud.google.com/go/shell v1.4.0/go.mod h1:HDxPzZf3GkDdhExzD/gs8Grqk+dmYcEjGShZgYa9URw= +cloud.google.com/go/shell v1.6.0/go.mod h1:oHO8QACS90luWgxP3N9iZVuEiSF84zNyLytb+qE2f9A= +cloud.google.com/go/shell v1.7.1/go.mod h1:u1RaM+huXFaTojTbW4g9P5emOrrmLE69KrxqQahKn4g= +cloud.google.com/go/spanner v1.41.0/go.mod h1:MLYDBJR/dY4Wt7ZaMIQ7rXOTLjYrmxLE/5ve9vFfWos= +cloud.google.com/go/spanner v1.44.0/go.mod h1:G8XIgYdOK+Fbcpbs7p2fiprDw4CaZX63whnSMLVBxjk= +cloud.google.com/go/spanner v1.45.0/go.mod h1:FIws5LowYz8YAE1J8fOS7DJup8ff7xJeetWEo5REA2M= +cloud.google.com/go/spanner v1.47.0/go.mod h1:IXsJwVW2j4UKs0eYDqodab6HgGuA1bViSqW4uH9lfUI= +cloud.google.com/go/speech v1.6.0/go.mod h1:79tcr4FHCimOp56lwC01xnt/WPJZc4v3gzyT7FoBkCM= +cloud.google.com/go/speech v1.7.0/go.mod h1:KptqL+BAQIhMsj1kOP2la5DSEEerPDuOP/2mmkhHhZQ= +cloud.google.com/go/speech v1.8.0/go.mod h1:9bYIl1/tjsAnMgKGHKmBZzXKEkGgtU+MpdDPTE9f7y0= +cloud.google.com/go/speech v1.9.0/go.mod h1:xQ0jTcmnRFFM2RfX/U+rk6FQNUF6DQlydUSyoooSpco= +cloud.google.com/go/speech v1.14.1/go.mod h1:gEosVRPJ9waG7zqqnsHpYTOoAS4KouMRLDFMekpJ0J0= +cloud.google.com/go/speech v1.15.0/go.mod h1:y6oH7GhqCaZANH7+Oe0BhgIogsNInLlz542tg3VqeYI= +cloud.google.com/go/speech v1.17.1/go.mod h1:8rVNzU43tQvxDaGvqOhpDqgkJTFowBpDvCJ14kGlJYo= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= +cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= +cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc= +cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= +cloud.google.com/go/storage v1.28.1/go.mod h1:Qnisd4CqDdo6BGs2AD5LLnEsmSQ80wQ5ogcBBKhU86Y= +cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4= +cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7BiccwkR7+P7gN8E= +cloud.google.com/go/storagetransfer v1.5.0/go.mod h1:dxNzUopWy7RQevYFHewchb29POFv3/AaBgnhqzqiK0w= +cloud.google.com/go/storagetransfer v1.6.0/go.mod h1:y77xm4CQV/ZhFZH75PLEXY0ROiS7Gh6pSKrM8dJyg6I= +cloud.google.com/go/storagetransfer v1.7.0/go.mod h1:8Giuj1QNb1kfLAiWM1bN6dHzfdlDAVC9rv9abHot2W4= +cloud.google.com/go/storagetransfer v1.8.0/go.mod h1:JpegsHHU1eXg7lMHkvf+KE5XDJ7EQu0GwNJbbVGanEw= +cloud.google.com/go/storagetransfer v1.10.0/go.mod h1:DM4sTlSmGiNczmV6iZyceIh2dbs+7z2Ayg6YAiQlYfA= +cloud.google.com/go/talent v1.1.0/go.mod h1:Vl4pt9jiHKvOgF9KoZo6Kob9oV4lwd/ZD5Cto54zDRw= +cloud.google.com/go/talent v1.2.0/go.mod h1:MoNF9bhFQbiJ6eFD3uSsg0uBALw4n4gaCaEjBw9zo8g= +cloud.google.com/go/talent v1.3.0/go.mod h1:CmcxwJ/PKfRgd1pBjQgU6W3YBwiewmUzQYH5HHmSCmM= +cloud.google.com/go/talent v1.4.0/go.mod h1:ezFtAgVuRf8jRsvyE6EwmbTK5LKciD4KVnHuDEFmOOA= +cloud.google.com/go/talent v1.5.0/go.mod h1:G+ODMj9bsasAEJkQSzO2uHQWXHHXUomArjWQQYkqK6c= +cloud.google.com/go/talent v1.6.2/go.mod h1:CbGvmKCG61mkdjcqTcLOkb2ZN1SrQI8MDyma2l7VD24= +cloud.google.com/go/texttospeech v1.4.0/go.mod h1:FX8HQHA6sEpJ7rCMSfXuzBcysDAuWusNNNvN9FELDd8= +cloud.google.com/go/texttospeech v1.5.0/go.mod h1:oKPLhR4n4ZdQqWKURdwxMy0uiTS1xU161C8W57Wkea4= +cloud.google.com/go/texttospeech v1.6.0/go.mod h1:YmwmFT8pj1aBblQOI3TfKmwibnsfvhIBzPXcW4EBovc= +cloud.google.com/go/texttospeech v1.7.1/go.mod h1:m7QfG5IXxeneGqTapXNxv2ItxP/FS0hCZBwXYqucgSk= +cloud.google.com/go/tpu v1.3.0/go.mod h1:aJIManG0o20tfDQlRIej44FcwGGl/cD0oiRyMKG19IQ= +cloud.google.com/go/tpu v1.4.0/go.mod h1:mjZaX8p0VBgllCzF6wcU2ovUXN9TONFLd7iz227X2Xg= +cloud.google.com/go/tpu v1.5.0/go.mod h1:8zVo1rYDFuW2l4yZVY0R0fb/v44xLh3llq7RuV61fPM= +cloud.google.com/go/tpu v1.6.1/go.mod h1:sOdcHVIgDEEOKuqUoi6Fq53MKHJAtOwtz0GuKsWSH3E= +cloud.google.com/go/trace v1.0.0/go.mod h1:4iErSByzxkyHWzzlAj63/Gmjz0NH1ASqhJguHpGcr6A= +cloud.google.com/go/trace v1.3.0/go.mod h1:FFUE83d9Ca57C+K8rDl/Ih8LwOzWIV1krKgxg6N0G28= +cloud.google.com/go/trace v1.4.0/go.mod h1:UG0v8UBqzusp+z63o7FK74SdFE+AXpCLdFb1rshXG+Y= +cloud.google.com/go/trace v1.8.0/go.mod h1:zH7vcsbAhklH8hWFig58HvxcxyQbaIqMarMg9hn5ECA= +cloud.google.com/go/trace v1.9.0/go.mod h1:lOQqpE5IaWY0Ixg7/r2SjixMuc6lfTFeO4QGM4dQWOk= +cloud.google.com/go/trace v1.10.1 h1:EwGdOLCNfYOOPtgqo+D2sDLZmRCEO1AagRTJCU6ztdg= +cloud.google.com/go/trace v1.10.1/go.mod h1:gbtL94KE5AJLH3y+WVpfWILmqgc6dXcqgNXdOPAQTYk= +cloud.google.com/go/translate v1.3.0/go.mod h1:gzMUwRjvOqj5i69y/LYLd8RrNQk+hOmIXTi9+nb3Djs= +cloud.google.com/go/translate v1.4.0/go.mod h1:06Dn/ppvLD6WvA5Rhdp029IX2Mi3Mn7fpMRLPvXT5Wg= +cloud.google.com/go/translate v1.5.0/go.mod h1:29YDSYveqqpA1CQFD7NQuP49xymq17RXNaUDdc0mNu0= +cloud.google.com/go/translate v1.6.0/go.mod h1:lMGRudH1pu7I3n3PETiOB2507gf3HnfLV8qlkHZEyos= +cloud.google.com/go/translate v1.7.0/go.mod h1:lMGRudH1pu7I3n3PETiOB2507gf3HnfLV8qlkHZEyos= +cloud.google.com/go/translate v1.8.1/go.mod h1:d1ZH5aaOA0CNhWeXeC8ujd4tdCFw8XoNWRljklu5RHs= +cloud.google.com/go/video v1.8.0/go.mod h1:sTzKFc0bUSByE8Yoh8X0mn8bMymItVGPfTuUBUyRgxk= +cloud.google.com/go/video v1.9.0/go.mod h1:0RhNKFRF5v92f8dQt0yhaHrEuH95m068JYOvLZYnJSw= +cloud.google.com/go/video v1.12.0/go.mod h1:MLQew95eTuaNDEGriQdcYn0dTwf9oWiA4uYebxM5kdg= +cloud.google.com/go/video v1.13.0/go.mod h1:ulzkYlYgCp15N2AokzKjy7MQ9ejuynOJdf1tR5lGthk= +cloud.google.com/go/video v1.14.0/go.mod h1:SkgaXwT+lIIAKqWAJfktHT/RbgjSuY6DobxEp0C5yTQ= +cloud.google.com/go/video v1.15.0/go.mod h1:SkgaXwT+lIIAKqWAJfktHT/RbgjSuY6DobxEp0C5yTQ= +cloud.google.com/go/video v1.17.1/go.mod h1:9qmqPqw/Ib2tLqaeHgtakU+l5TcJxCJbhFXM7UJjVzU= +cloud.google.com/go/videointelligence v1.6.0/go.mod h1:w0DIDlVRKtwPCn/C4iwZIJdvC69yInhW0cfi+p546uU= +cloud.google.com/go/videointelligence v1.7.0/go.mod h1:k8pI/1wAhjznARtVT9U1llUaFNPh7muw8QyOUpavru4= +cloud.google.com/go/videointelligence v1.8.0/go.mod h1:dIcCn4gVDdS7yte/w+koiXn5dWVplOZkE+xwG9FgK+M= +cloud.google.com/go/videointelligence v1.9.0/go.mod h1:29lVRMPDYHikk3v8EdPSaL8Ku+eMzDljjuvRs105XoU= +cloud.google.com/go/videointelligence v1.10.0/go.mod h1:LHZngX1liVtUhZvi2uNS0VQuOzNi2TkY1OakiuoUOjU= +cloud.google.com/go/videointelligence v1.11.1/go.mod h1:76xn/8InyQHarjTWsBR058SmlPCwQjgcvoW0aZykOvo= +cloud.google.com/go/vision v1.2.0/go.mod h1:SmNwgObm5DpFBme2xpyOyasvBc1aPdjvMk2bBk0tKD0= +cloud.google.com/go/vision/v2 v2.2.0/go.mod h1:uCdV4PpN1S0jyCyq8sIM42v2Y6zOLkZs+4R9LrGYwFo= +cloud.google.com/go/vision/v2 v2.3.0/go.mod h1:UO61abBx9QRMFkNBbf1D8B1LXdS2cGiiCRx0vSpZoUo= +cloud.google.com/go/vision/v2 v2.4.0/go.mod h1:VtI579ll9RpVTrdKdkMzckdnwMyX2JILb+MhPqRbPsY= +cloud.google.com/go/vision/v2 v2.5.0/go.mod h1:MmaezXOOE+IWa+cS7OhRRLK2cNv1ZL98zhqFFZaaH2E= +cloud.google.com/go/vision/v2 v2.6.0/go.mod h1:158Hes0MvOS9Z/bDMSFpjwsUrZ5fPrdwuyyvKSGAGMY= +cloud.google.com/go/vision/v2 v2.7.0/go.mod h1:H89VysHy21avemp6xcf9b9JvZHVehWbET0uT/bcuY/0= +cloud.google.com/go/vision/v2 v2.7.2/go.mod h1:jKa8oSYBWhYiXarHPvP4USxYANYUEdEsQrloLjrSwJU= +cloud.google.com/go/vmmigration v1.2.0/go.mod h1:IRf0o7myyWFSmVR1ItrBSFLFD/rJkfDCUTO4vLlJvsE= +cloud.google.com/go/vmmigration v1.3.0/go.mod h1:oGJ6ZgGPQOFdjHuocGcLqX4lc98YQ7Ygq8YQwHh9A7g= +cloud.google.com/go/vmmigration v1.5.0/go.mod h1:E4YQ8q7/4W9gobHjQg4JJSgXXSgY21nA5r8swQV+Xxc= +cloud.google.com/go/vmmigration v1.6.0/go.mod h1:bopQ/g4z+8qXzichC7GW1w2MjbErL54rk3/C843CjfY= +cloud.google.com/go/vmmigration v1.7.1/go.mod h1:WD+5z7a/IpZ5bKK//YmT9E047AD+rjycCAvyMxGJbro= +cloud.google.com/go/vmwareengine v0.1.0/go.mod h1:RsdNEf/8UDvKllXhMz5J40XxDrNJNN4sagiox+OI208= +cloud.google.com/go/vmwareengine v0.2.2/go.mod h1:sKdctNJxb3KLZkE/6Oui94iw/xs9PRNC2wnNLXsHvH8= +cloud.google.com/go/vmwareengine v0.3.0/go.mod h1:wvoyMvNWdIzxMYSpH/R7y2h5h3WFkx6d+1TIsP39WGY= +cloud.google.com/go/vmwareengine v0.4.1/go.mod h1:Px64x+BvjPZwWuc4HdmVhoygcXqEkGHXoa7uyfTgSI0= +cloud.google.com/go/vpcaccess v1.4.0/go.mod h1:aQHVbTWDYUR1EbTApSVvMq1EnT57ppDmQzZ3imqIk4w= +cloud.google.com/go/vpcaccess v1.5.0/go.mod h1:drmg4HLk9NkZpGfCmZ3Tz0Bwnm2+DKqViEpeEpOq0m8= +cloud.google.com/go/vpcaccess v1.6.0/go.mod h1:wX2ILaNhe7TlVa4vC5xce1bCnqE3AeH27RV31lnmZes= +cloud.google.com/go/vpcaccess v1.7.1/go.mod h1:FogoD46/ZU+JUBX9D606X21EnxiszYi2tArQwLY4SXs= +cloud.google.com/go/webrisk v1.4.0/go.mod h1:Hn8X6Zr+ziE2aNd8SliSDWpEnSS1u4R9+xXZmFiHmGE= +cloud.google.com/go/webrisk v1.5.0/go.mod h1:iPG6fr52Tv7sGk0H6qUFzmL3HHZev1htXuWDEEsqMTg= +cloud.google.com/go/webrisk v1.6.0/go.mod h1:65sW9V9rOosnc9ZY7A7jsy1zoHS5W9IAXv6dGqhMQMc= +cloud.google.com/go/webrisk v1.7.0/go.mod h1:mVMHgEYH0r337nmt1JyLthzMr6YxwN1aAIEc2fTcq7A= +cloud.google.com/go/webrisk v1.8.0/go.mod h1:oJPDuamzHXgUc+b8SiHRcVInZQuybnvEW72PqTc7sSg= +cloud.google.com/go/webrisk v1.9.1/go.mod h1:4GCmXKcOa2BZcZPn6DCEvE7HypmEJcJkr4mtM+sqYPc= +cloud.google.com/go/websecurityscanner v1.3.0/go.mod h1:uImdKm2wyeXQevQJXeh8Uun/Ym1VqworNDlBXQevGMo= +cloud.google.com/go/websecurityscanner v1.4.0/go.mod h1:ebit/Fp0a+FWu5j4JOmJEV8S8CzdTkAS77oDsiSqYWQ= +cloud.google.com/go/websecurityscanner v1.5.0/go.mod h1:Y6xdCPy81yi0SQnDY1xdNTNpfY1oAgXUlcfN3B3eSng= +cloud.google.com/go/websecurityscanner v1.6.1/go.mod h1:Njgaw3rttgRHXzwCB8kgCYqv5/rGpFCsBOvPbYgszpg= +cloud.google.com/go/workflows v1.6.0/go.mod h1:6t9F5h/unJz41YqfBmqSASJSXccBLtD1Vwf+KmJENM0= +cloud.google.com/go/workflows v1.7.0/go.mod h1:JhSrZuVZWuiDfKEFxU0/F1PQjmpnpcoISEXH2bcHC3M= +cloud.google.com/go/workflows v1.8.0/go.mod h1:ysGhmEajwZxGn1OhGOGKsTXc5PyxOc0vfKf5Af+to4M= +cloud.google.com/go/workflows v1.9.0/go.mod h1:ZGkj1aFIOd9c8Gerkjjq7OW7I5+l6cSvT3ujaO/WwSA= +cloud.google.com/go/workflows v1.10.0/go.mod h1:fZ8LmRmZQWacon9UCX1r/g/DfAXx5VcPALq2CxzdePw= +cloud.google.com/go/workflows v1.11.1/go.mod h1:Z+t10G1wF7h8LgdY/EmRcQY8ptBD/nvofaL6FqlET6g= +contrib.go.opencensus.io/exporter/stackdriver v0.13.12 h1:bjBKzIf7/TAkxd7L2utGaLM78bmUWlCval5K9UeElbY= +contrib.go.opencensus.io/exporter/stackdriver v0.13.12/go.mod h1:mmxnWlrvrFdpiOHOhxBaVi1rkc0WOqhgfknj4Yg0SeQ= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zumjgTw83q2ge/PI+yyw8= +git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3pc= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= +github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm/4RlzPXRlREEwqTHAN3T56Bv2ITsFT3gY= +github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19/go.mod h1:T13YZdzov6OU0A1+RfKZiZN9ca6VeKdBdyDV+BY97Tk= +github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= +github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b/go.mod h1:1KcenG0jGWcpt8ov532z81sp/kMMUG485J2InIOyADM= +github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/apache/arrow/go/v10 v10.0.1/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0= +github.com/apache/arrow/go/v11 v11.0.0/go.mod h1:Eg5OsL5H+e299f7u5ssuXsuHQVEGC4xei5aX110hRiI= +github.com/apache/arrow/go/v12 v12.0.0/go.mod h1:d+tV/eHZZ7Dz7RPrFKtPK02tpr+c9/PEd/zm8mDS9Vg= +github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= +github.com/aws/aws-sdk-go v1.37.0/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= +github.com/aws/aws-sdk-go v1.44.162 h1:hKAd+X+/BLxVMzH+4zKxbQcQQGrk2UhFX0OTu1Mhon8= +github.com/aws/aws-sdk-go v1.44.162/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= +github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g= +github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20230310173818-32f1caf87195/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20230428030218-4003588d1b74/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= +github.com/envoyproxy/go-control-plane v0.10.3/go.mod h1:fJJn/j26vwOu972OllsvAgJJM//w9BV6Fxbg2LuVd34= +github.com/envoyproxy/go-control-plane v0.11.0/go.mod h1:VnHyVMpzcLvCFt9yUz1UnCwHLhwx1WguiVDV7pTG/tI= +github.com/envoyproxy/go-control-plane v0.11.1/go.mod h1:uhMcXKCQMEJHiAb0w+YGefQLaTEw+YhGluxZkrTmD0g= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v0.6.7/go.mod h1:dyJXwwfPK2VSqiB9Klm1J6romD608Ba7Hij42vrOBCo= +github.com/envoyproxy/protoc-gen-validate v0.9.1/go.mod h1:OKNgG7TCp5pF4d6XftA0++PMirau2/yoOwVac3AbF2w= +github.com/envoyproxy/protoc-gen-validate v0.10.0/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss= +github.com/envoyproxy/protoc-gen-validate v1.0.1/go.mod h1:0vj8bNkYbSTNS2PIyH87KZaeN4x9zpL9Qt8fQC7d+vs= +github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= +github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-fonts/dejavu v0.1.0/go.mod h1:4Wt4I4OU2Nq9asgDCteaAaWZOV24E+0/Pwo0gppep4g= +github.com/go-fonts/latin-modern v0.2.0/go.mod h1:rQVLdDMK+mK1xscDwsqM5J8U2jrRa3T0ecnM9pNujks= +github.com/go-fonts/liberation v0.1.1/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY= +github.com/go-fonts/liberation v0.2.0/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY= +github.com/go-fonts/stix v0.1.0/go.mod h1:w/c1f0ldAUlJmLBvlbkvVXLAD+tAMqobIIQpmnUIzUY= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2CSIqUrmQPqA0gdRIlnLEY0gK5JGjh37zN5U= +github.com/go-latex/latex v0.0.0-20210823091927-c0d11ff05a81/go.mod h1:SX0U8uGpxhq9o2S/CELCSUxEWWAuoCUcVCQWv7G2OCk= +github.com/go-pdf/fpdf v0.5.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= +github.com/go-pdf/fpdf v0.6.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= +github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/flatbuffers v2.0.8+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/s2a-go v0.1.0/go.mod h1:OJpEgntRZo8ugHpF9hkoLJbS5dSI20XZeXJ9JVywLlM= +github.com/google/s2a-go v0.1.3/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= +github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= +github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= +github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= +github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= +github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= +github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k= +github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= +github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= +github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= +github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= +github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= +github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo= +github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY= +github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= +github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= +github.com/googleapis/gax-go/v2 v2.8.0/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= +github.com/googleapis/gax-go/v2 v2.10.0/go.mod h1:4UOEnMCrxsSqQ940WnTiD6qJ63le2ev3xfyagutxiPw= +github.com/googleapis/gax-go/v2 v2.11.0 h1:9V9PWXEsWnPpQhu/PeQIkS4eGzMlTLGgt80cUUI8Ki4= +github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI= +github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= +github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU0GSB0f8Nhgmxx0VIRUvaC0w= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= +github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lyft/protoc-gen-star v0.6.0/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= +github.com/lyft/protoc-gen-star v0.6.1/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= +github.com/lyft/protoc-gen-star/v2 v2.0.1/go.mod h1:RcCdONR2ScXaYnQC5tUzxzlpA3WVYF7/opLeUgcQs/o= +github.com/lyft/protoc-gen-star/v2 v2.0.3/go.mod h1:amey7yeodaJhXSbf/TlLvWiqQfLOSpEk//mLlc+axEk= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= +github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2dXMnm1mY= +github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= +github.com/phpdave11/gofpdi v1.0.13/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= +github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= +github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= +github.com/prometheus/prometheus v2.5.0+incompatible h1:7QPitgO2kOFG8ecuRn9O/4L9+10He72rVRJvMXrE9Hg= +github.com/prometheus/prometheus v2.5.0+incompatible/go.mod h1:oAIUtOny2rjMX0OWN5vPR5/q/twIROJvdqnQKDdil/s= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w= +github.com/ruudk/golang-pdf417 v0.0.0-20201230142125-a7e3863a1245/go.mod h1:pQAZKsJ8yyVxGRWYNEm9oFB8ieLgKFnamEyDmSA0BRk= +github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= +github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= +github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= +go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= +golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= +golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= +golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= +golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3/go.mod h1:NOZ3BPKG0ec/BKJQgnvsSFpcKLM5xXVWnvZS97DWHgE= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20220827204233-334a2380cb91/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= +golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20190910094157-69e4b8554b2a/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200430140353-33d19683fad8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200618115811-c13761719519/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20201208152932-35266b937fa6/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20210216034530-4410531fe030/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20210607152325-775e3b0c77b9/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= +golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= +golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= +golang.org/x/image v0.0.0-20220302094943-723b81ca9867/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.0.0-20221012135044-0b7e1fb9d458/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= +golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= +golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= +golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= +golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.0.0-20221006150949-b44042a4b9c1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= +golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= +golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= +golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= +golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= +golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210304124612-50617c2ba197/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210917161153-d61c044b1678/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220829200755-d48e67d00261/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= +golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= +golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= +golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= +golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= +golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= +gonum.org/v1/gonum v0.9.3/go.mod h1:TZumC3NeyVQskjXqmyWt4S3bINhy7B4eYwW69EbyX+0= +gonum.org/v1/gonum v0.11.0/go.mod h1:fSG4YDCxxUZQJ7rKsQrj0gMOg00Il0Z96/qMA4bVQhA= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= +gonum.org/v1/plot v0.9.0/go.mod h1:3Pcqqmp6RHvJI72kgb8fThyUnav364FOsdDo2aGW5lY= +gonum.org/v1/plot v0.10.1/go.mod h1:VZW5OlhkL1mysU9vaqNHnsy86inf6Ot+jB3r+BczCEo= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= +google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= +google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= +google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= +google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= +google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= +google.golang.org/api v0.58.0/go.mod h1:cAbP2FsxoGVNwtgNAmmn3y5G1TWAiVYRmg4yku3lv+E= +google.golang.org/api v0.59.0/go.mod h1:sT2boj7M9YJxZzgeZqXogmhfmRWDtPzT31xkieUbuZU= +google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= +google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= +google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g= +google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA= +google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8= +google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= +google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= +google.golang.org/api v0.77.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= +google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw= +google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg= +google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o= +google.golang.org/api v0.85.0/go.mod h1:AqZf8Ep9uZ2pyTvgL+x0D3Zt0eoT9b5E8fmzfu6FO2g= +google.golang.org/api v0.90.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= +google.golang.org/api v0.93.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= +google.golang.org/api v0.95.0/go.mod h1:eADj+UBuxkh5zlrSntJghuNeg8HwQ1w5lTKkuqaETEI= +google.golang.org/api v0.96.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= +google.golang.org/api v0.97.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= +google.golang.org/api v0.98.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= +google.golang.org/api v0.99.0/go.mod h1:1YOf74vkVndF7pG6hIHuINsM7eWwpVTAfNMNiL91A08= +google.golang.org/api v0.100.0/go.mod h1:ZE3Z2+ZOr87Rx7dqFsdRQkRBk36kDtp/h+QpHbB7a70= +google.golang.org/api v0.102.0/go.mod h1:3VFl6/fzoA+qNuS1N1/VfXY4LjoXN/wzeIp7TweWwGo= +google.golang.org/api v0.103.0/go.mod h1:hGtW6nK1AC+d9si/UBhw8Xli+QMOf6xyNAyJw4qU9w0= +google.golang.org/api v0.106.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= +google.golang.org/api v0.107.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= +google.golang.org/api v0.108.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= +google.golang.org/api v0.110.0/go.mod h1:7FC4Vvx1Mooxh8C5HWjzZHcavuS2f6pmJpZx60ca7iI= +google.golang.org/api v0.111.0/go.mod h1:qtFHvU9mhgTJegR31csQ+rwxyUTHOKFqCKWp1J0fdw0= +google.golang.org/api v0.114.0/go.mod h1:ifYI2ZsFK6/uGddGfAD5BMxlnkBqCmqHSDUVi45N5Yg= +google.golang.org/api v0.118.0/go.mod h1:76TtD3vkgmZ66zZzp72bUUklpmQmKlhh6sYtIjYK+5E= +google.golang.org/api v0.122.0/go.mod h1:gcitW0lvnyWjSp9nKxAbdHKIZ6vF4aajGueeslZOyms= +google.golang.org/api v0.124.0/go.mod h1:xu2HQurE5gi/3t1aFCvhPD781p0a3p11sdunTJ2BlP4= +google.golang.org/api v0.125.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw= +google.golang.org/api v0.126.0 h1:q4GJq+cAdMAC7XP7njvQ4tvohGLiSlytuL4BQxbIZ+o= +google.golang.org/api v0.126.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210329143202-679c6ae281ee/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= +google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210917145530-b395a37504d4/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210921142501-181ce0d877f6/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211008145708-270636b82663/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211018162055-cf77aa76bad2/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= +google.golang.org/genproto v0.0.0-20220329172620-7be39ac1afc7/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220628213854-d9e0b6570c03/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220722212130-b98a9ff5e252/go.mod h1:GkXuJDJ6aQ7lnJcRF+SJVgFdQhypqgl3LB1C9vabdRE= +google.golang.org/genproto v0.0.0-20220801145646-83ce21fca29f/go.mod h1:iHe1svFLAZg9VWz891+QbRMwUv9O/1Ww+/mngYeThbc= +google.golang.org/genproto v0.0.0-20220815135757-37a418bb8959/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220817144833-d7fd3f11b9b1/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220829144015-23454907ede3/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220829175752-36a9c930ecbf/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220913154956-18f8339a66a5/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220914142337-ca0e39ece12f/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220915135415-7fd63a7952de/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220916172020-2692e8806bfa/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220919141832-68c03719ef51/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220920201722-2b89144ce006/go.mod h1:ht8XFiar2npT/g4vkk7O0WYS1sHOHbdujxbEp7CJWbw= +google.golang.org/genproto v0.0.0-20220926165614-551eb538f295/go.mod h1:woMGP53BroOrRY3xTxlbr8Y3eB/nzAvvFM83q7kG2OI= +google.golang.org/genproto v0.0.0-20220926220553-6981cbe3cfce/go.mod h1:woMGP53BroOrRY3xTxlbr8Y3eB/nzAvvFM83q7kG2OI= +google.golang.org/genproto v0.0.0-20221010155953-15ba04fc1c0e/go.mod h1:3526vdqwhZAwq4wsRUaVG555sVgsNmIjRtO7t/JH29U= +google.golang.org/genproto v0.0.0-20221014173430-6e2ab493f96b/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= +google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= +google.golang.org/genproto v0.0.0-20221024153911-1573dae28c9c/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= +google.golang.org/genproto v0.0.0-20221024183307-1bc688fe9f3e/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= +google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c/go.mod h1:CGI5F/G+E5bKwmfYo09AXuVN4dD894kIKUFmVbP2/Fo= +google.golang.org/genproto v0.0.0-20221109142239-94d6d90a7d66/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221114212237-e4508ebdbee1/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221117204609-8f9c96812029/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221201204527-e3fa12d562f3/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd/go.mod h1:cTsE614GARnxrLsqKREzmNYJACSWWpAWdNMwnD7c2BE= +google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230112194545-e10362b5ecf9/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230113154510-dbe35b8444a5/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230123190316-2c411cf9d197/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230124163310-31e0e69b6fc2/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230125152338-dcaf20b6aeaa/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230127162408-596548ed4efa/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230209215440-0dfe4f8abfcc/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230216225411-c8e22ba71e44/go.mod h1:8B0gmkoRebU8ukX6HP+4wrVQUY1+6PkQ44BSyIlflHA= +google.golang.org/genproto v0.0.0-20230222225845-10f96fb3dbec/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw= +google.golang.org/genproto v0.0.0-20230223222841-637eb2293923/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw= +google.golang.org/genproto v0.0.0-20230303212802-e74f57abe488/go.mod h1:TvhZT5f700eVlTNwND1xoEZQeWTB2RY/65kplwl/bFA= +google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= +google.golang.org/genproto v0.0.0-20230320184635-7606e756e683/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= +google.golang.org/genproto v0.0.0-20230323212658-478b75c54725/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/genproto v0.0.0-20230330154414-c0448cd141ea/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= +google.golang.org/genproto v0.0.0-20230525234025-438c736192d0/go.mod h1:9ExIQyXL5hZrHzQceCwuSYwZZ5QZBazOcprJ5rgs3lY= +google.golang.org/genproto v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:zqTuNwFlFRsw5zIts5VnzLQxSRqh+CGOTVMlYbY0Eyk= +google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= +google.golang.org/genproto v0.0.0-20230629202037-9506855d4529/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= +google.golang.org/genproto v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:O9kGHb51iE/nOGvQaDUuadVYqovW56s5emA88lQnj6Y= +google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 h1:Z0hjGZePRE0ZBWotvtrwxFNrNE9CUAGtplaDK5NNI/g= +google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98/go.mod h1:S7mY02OqCJTD0E1OiQy1F72PWFB4bZJ87cAtLPYgDR0= +google.golang.org/genproto/googleapis/api v0.0.0-20230525234020-1aefcd67740a/go.mod h1:ts19tUU+Z0ZShN1y3aPyq2+O3d5FUNNgT6FtOzmrNn8= +google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto/googleapis/api v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto/googleapis/api v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:mPBs5jNgx2GuQGvFwUvVKqtn6HsUw9nP64BedgvqEsQ= +google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 h1:FmF5cCW94Ij59cfpoLiwTgodWmm60eEV0CjlsVg2fuw= +google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= +google.golang.org/genproto/googleapis/bytestream v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:ylj+BE99M198VPbBh6A8d9n3w8fChvyLK3wwBOjXBFA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234015-3fc162c6f38a/go.mod h1:xURIpW9ES5+/GZhnV6beoEtxQrnkRGIfP5VQG2tCBLc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:8mL13HKkDa+IuJ8yruA3ci0q+0vsUz4m//+ottjwS5o= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 h1:bVf09lpb+OJbByTj913DRJioFFAjf/ZGxEz7MajTp2U= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= +google.golang.org/grpc/stats/opencensus v1.0.0 h1:evSYcRZaSToQp+borzWE52+03joezZeXcKJvZDfkUJA= +google.golang.org/grpc/stats/opencensus v1.0.0/go.mod h1:FhdkeYvN43wLYUnapVuRJJ9JXkNwe403iLUW2LKSnjs= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= +lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= +lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= +modernc.org/cc/v3 v3.36.0/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= +modernc.org/cc/v3 v3.36.2/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= +modernc.org/cc/v3 v3.36.3/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= +modernc.org/cc/v3 v3.37.0/go.mod h1:vtL+3mdHx/wcj3iEGz84rQa8vEqR6XM84v5Lcvfph20= +modernc.org/cc/v3 v3.40.0/go.mod h1:/bTg4dnWkSXowUO6ssQKnOV0yMVxDYNIsIrzqTFDGH0= +modernc.org/ccgo/v3 v3.0.0-20220428102840-41399a37e894/go.mod h1:eI31LL8EwEBKPpNpA4bU1/i+sKOwOrQy8D87zWUcRZc= +modernc.org/ccgo/v3 v3.0.0-20220430103911-bc99d88307be/go.mod h1:bwdAnOoaIt8Ax9YdWGjxWsdkPcZyRPHqrOvJxaKAKGw= +modernc.org/ccgo/v3 v3.0.0-20220904174949-82d86e1b6d56/go.mod h1:YSXjPL62P2AMSxBphRHPn7IkzhVHqkvOnRKAKh+W6ZI= +modernc.org/ccgo/v3 v3.16.4/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ= +modernc.org/ccgo/v3 v3.16.6/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ= +modernc.org/ccgo/v3 v3.16.8/go.mod h1:zNjwkizS+fIFDrDjIAgBSCLkWbJuHF+ar3QRn+Z9aws= +modernc.org/ccgo/v3 v3.16.9/go.mod h1:zNMzC9A9xeNUepy6KuZBbugn3c0Mc9TeiJO4lgvkJDo= +modernc.org/ccgo/v3 v3.16.13-0.20221017192402-261537637ce8/go.mod h1:fUB3Vn0nVPReA+7IG7yZDfjv1TMWjhQP8gCxrFAtL5g= +modernc.org/ccgo/v3 v3.16.13/go.mod h1:2Quk+5YgpImhPjv2Qsob1DnZ/4som1lJTodubIcoUkY= +modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ= +modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM= +modernc.org/libc v0.0.0-20220428101251-2d5f3daf273b/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA= +modernc.org/libc v1.16.0/go.mod h1:N4LD6DBE9cf+Dzf9buBlzVJndKr/iJHG97vGLHYnb5A= +modernc.org/libc v1.16.1/go.mod h1:JjJE0eu4yeK7tab2n4S1w8tlWd9MxXLRzheaRnAKymU= +modernc.org/libc v1.16.17/go.mod h1:hYIV5VZczAmGZAnG15Vdngn5HSF5cSkbvfz2B7GRuVU= +modernc.org/libc v1.16.19/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA= +modernc.org/libc v1.17.0/go.mod h1:XsgLldpP4aWlPlsjqKRdHPqCxCjISdHfM/yeWC5GyW0= +modernc.org/libc v1.17.1/go.mod h1:FZ23b+8LjxZs7XtFMbSzL/EhPxNbfZbErxEHc7cbD9s= +modernc.org/libc v1.17.4/go.mod h1:WNg2ZH56rDEwdropAJeZPQkXmDwh+JCA1s/htl6r2fA= +modernc.org/libc v1.18.0/go.mod h1:vj6zehR5bfc98ipowQOM2nIDUZnVew/wNC/2tOGS+q0= +modernc.org/libc v1.20.3/go.mod h1:ZRfIaEkgrYgZDl6pa4W39HgN5G/yDW+NRmNKZBDFrk0= +modernc.org/libc v1.21.4/go.mod h1:przBsL5RDOZajTVslkugzLBj1evTue36jEomFQOoYuI= +modernc.org/libc v1.22.2/go.mod h1:uvQavJ1pZ0hIoC/jfqNoMLURIMhKzINIWypNM17puug= +modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/mathutil v1.4.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/memory v1.1.1/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw= +modernc.org/memory v1.2.0/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw= +modernc.org/memory v1.2.1/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= +modernc.org/memory v1.3.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= +modernc.org/memory v1.4.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= +modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= +modernc.org/opt v0.1.1/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/sqlite v1.18.1/go.mod h1:6ho+Gow7oX5V+OiOQ6Tr4xeqbx13UZ6t+Fw9IRUG4d4= +modernc.org/sqlite v1.18.2/go.mod h1:kvrTLEWgxUcHa2GfHBQtanR1H9ht3hTJNtKpzH9k1u0= +modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw= +modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw= +modernc.org/tcl v1.13.1/go.mod h1:XOLfOwzhkljL4itZkK6T72ckMgvj0BDsnKNdZVUOecw= +modernc.org/tcl v1.13.2/go.mod h1:7CLiGIPo1M8Rv1Mitpv5akc2+8fxUd2y2UzC/MfMzy0= +modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +modernc.org/token v1.0.1/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +modernc.org/z v1.5.1/go.mod h1:eWFB510QWW5Th9YGZT81s+LwvaAs3Q2yr4sP0rmLkv8= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/gcp/observability/logging.go b/gcp/observability/logging.go new file mode 100644 index 000000000000..0ffbd93b3922 --- /dev/null +++ b/gcp/observability/logging.go @@ -0,0 +1,505 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package observability + +import ( + "bytes" + "context" + "encoding/base64" + "errors" + "fmt" + "strings" + "time" + + gcplogging "cloud.google.com/go/logging" + "github.com/google/uuid" + "go.opencensus.io/trace" + + "google.golang.org/grpc" + binlogpb "google.golang.org/grpc/binarylog/grpc_binarylog_v1" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal" + iblog "google.golang.org/grpc/internal/binarylog" + "google.golang.org/grpc/internal/grpcutil" + "google.golang.org/grpc/stats/opencensus" +) + +var lExporter loggingExporter + +var newLoggingExporter = newCloudLoggingExporter + +var canonicalString = internal.CanonicalString.(func(codes.Code) string) + +// translateMetadata translates the metadata from Binary Logging format to +// its GrpcLogEntry equivalent. +func translateMetadata(m *binlogpb.Metadata) map[string]string { + metadata := make(map[string]string) + for _, entry := range m.GetEntry() { + entryKey := entry.GetKey() + var newVal string + if strings.HasSuffix(entryKey, "-bin") { // bin header + newVal = base64.StdEncoding.EncodeToString(entry.GetValue()) + } else { // normal header + newVal = string(entry.GetValue()) + } + var oldVal string + var ok bool + if oldVal, ok = metadata[entryKey]; !ok { + metadata[entryKey] = newVal + continue + } + metadata[entryKey] = oldVal + "," + newVal + } + return metadata +} + +func setPeerIfPresent(binlogEntry *binlogpb.GrpcLogEntry, grpcLogEntry *grpcLogEntry) { + if binlogEntry.GetPeer() != nil { + grpcLogEntry.Peer.Type = addrType(binlogEntry.GetPeer().GetType()) + grpcLogEntry.Peer.Address = binlogEntry.GetPeer().GetAddress() + grpcLogEntry.Peer.IPPort = binlogEntry.GetPeer().GetIpPort() + } +} + +var loggerTypeToEventLogger = map[binlogpb.GrpcLogEntry_Logger]loggerType{ + binlogpb.GrpcLogEntry_LOGGER_UNKNOWN: loggerUnknown, + binlogpb.GrpcLogEntry_LOGGER_CLIENT: loggerClient, + binlogpb.GrpcLogEntry_LOGGER_SERVER: loggerServer, +} + +type eventType int + +const ( + // eventTypeUnknown is an unknown event type. + eventTypeUnknown eventType = iota + // eventTypeClientHeader is a header sent from client to server. + eventTypeClientHeader + // eventTypeServerHeader is a header sent from server to client. + eventTypeServerHeader + // eventTypeClientMessage is a message sent from client to server. + eventTypeClientMessage + // eventTypeServerMessage is a message sent from server to client. + eventTypeServerMessage + // eventTypeClientHalfClose is a signal that the loggerClient is done sending. + eventTypeClientHalfClose + // eventTypeServerTrailer indicated the end of a gRPC call. + eventTypeServerTrailer + // eventTypeCancel is a signal that the rpc is canceled. + eventTypeCancel +) + +func (t eventType) MarshalJSON() ([]byte, error) { + buffer := bytes.NewBufferString(`"`) + switch t { + case eventTypeUnknown: + buffer.WriteString("EVENT_TYPE_UNKNOWN") + case eventTypeClientHeader: + buffer.WriteString("CLIENT_HEADER") + case eventTypeServerHeader: + buffer.WriteString("SERVER_HEADER") + case eventTypeClientMessage: + buffer.WriteString("CLIENT_MESSAGE") + case eventTypeServerMessage: + buffer.WriteString("SERVER_MESSAGE") + case eventTypeClientHalfClose: + buffer.WriteString("CLIENT_HALF_CLOSE") + case eventTypeServerTrailer: + buffer.WriteString("SERVER_TRAILER") + case eventTypeCancel: + buffer.WriteString("CANCEL") + } + buffer.WriteString(`"`) + return buffer.Bytes(), nil +} + +type loggerType int + +const ( + loggerUnknown loggerType = iota + loggerClient + loggerServer +) + +func (t loggerType) MarshalJSON() ([]byte, error) { + buffer := bytes.NewBufferString(`"`) + switch t { + case loggerUnknown: + buffer.WriteString("LOGGER_UNKNOWN") + case loggerClient: + buffer.WriteString("CLIENT") + case loggerServer: + buffer.WriteString("SERVER") + } + buffer.WriteString(`"`) + return buffer.Bytes(), nil +} + +type payload struct { + Metadata map[string]string `json:"metadata,omitempty"` + // Timeout is the RPC timeout value. + Timeout time.Duration `json:"timeout,omitempty"` + // StatusCode is the gRPC status code. + StatusCode string `json:"statusCode,omitempty"` + // StatusMessage is the gRPC status message. + StatusMessage string `json:"statusMessage,omitempty"` + // StatusDetails is the value of the grpc-status-details-bin metadata key, + // if any. This is always an encoded google.rpc.Status message. + StatusDetails []byte `json:"statusDetails,omitempty"` + // MessageLength is the length of the message. + MessageLength uint32 `json:"messageLength,omitempty"` + // Message is the message of this entry. This is populated in the case of a + // message event. + Message []byte `json:"message,omitempty"` +} + +type addrType int + +const ( + typeUnknown addrType = iota // `json:"TYPE_UNKNOWN"` + ipv4 // `json:"IPV4"` + ipv6 // `json:"IPV6"` + unix // `json:"UNIX"` +) + +func (at addrType) MarshalJSON() ([]byte, error) { + buffer := bytes.NewBufferString(`"`) + switch at { + case typeUnknown: + buffer.WriteString("TYPE_UNKNOWN") + case ipv4: + buffer.WriteString("IPV4") + case ipv6: + buffer.WriteString("IPV6") + case unix: + buffer.WriteString("UNIX") + } + buffer.WriteString(`"`) + return buffer.Bytes(), nil +} + +type address struct { + // Type is the address type of the address of the peer of the RPC. + Type addrType `json:"type,omitempty"` + // Address is the address of the peer of the RPC. + Address string `json:"address,omitempty"` + // IPPort is the ip and port in string form. It is used only for addrType + // typeIPv4 and typeIPv6. + IPPort uint32 `json:"ipPort,omitempty"` +} + +type grpcLogEntry struct { + // CallID is a uuid which uniquely identifies a call. Each call may have + // several log entries. They will all have the same CallID. Nothing is + // guaranteed about their value other than they are unique across different + // RPCs in the same gRPC process. + CallID string `json:"callId,omitempty"` + // SequenceID is the entry sequence ID for this call. The first message has + // a value of 1, to disambiguate from an unset value. The purpose of this + // field is to detect missing entries in environments where durability or + // ordering is not guaranteed. + SequenceID uint64 `json:"sequenceId,omitempty"` + // Type is the type of binary logging event being logged. + Type eventType `json:"type,omitempty"` + // Logger is the entity that generates the log entry. + Logger loggerType `json:"logger,omitempty"` + // Payload is the payload of this log entry. + Payload payload `json:"payload,omitempty"` + // PayloadTruncated is whether the message or metadata field is either + // truncated or emitted due to options specified in the configuration. + PayloadTruncated bool `json:"payloadTruncated,omitempty"` + // Peer is information about the Peer of the RPC. + Peer address `json:"peer,omitempty"` + // A single process may be used to run multiple virtual servers with + // different identities. + // Authority is the name of such a server identify. It is typically a + // portion of the URI in the form of or :. + Authority string `json:"authority,omitempty"` + // ServiceName is the name of the service. + ServiceName string `json:"serviceName,omitempty"` + // MethodName is the name of the RPC method. + MethodName string `json:"methodName,omitempty"` +} + +type methodLoggerBuilder interface { + Build(iblog.LogEntryConfig) *binlogpb.GrpcLogEntry +} + +type binaryMethodLogger struct { + callID, serviceName, methodName, authority, projectID string + + mlb methodLoggerBuilder + exporter loggingExporter + clientSide bool +} + +// buildGCPLoggingEntry converts the binary log log entry into a gcp logging +// entry. +func (bml *binaryMethodLogger) buildGCPLoggingEntry(ctx context.Context, c iblog.LogEntryConfig) gcplogging.Entry { + binLogEntry := bml.mlb.Build(c) + + grpcLogEntry := &grpcLogEntry{ + CallID: bml.callID, + SequenceID: binLogEntry.GetSequenceIdWithinCall(), + Logger: loggerTypeToEventLogger[binLogEntry.Logger], + } + + switch binLogEntry.GetType() { + case binlogpb.GrpcLogEntry_EVENT_TYPE_UNKNOWN: + grpcLogEntry.Type = eventTypeUnknown + case binlogpb.GrpcLogEntry_EVENT_TYPE_CLIENT_HEADER: + grpcLogEntry.Type = eventTypeClientHeader + if binLogEntry.GetClientHeader() != nil { + methodName := binLogEntry.GetClientHeader().MethodName + // Example method name: /grpc.testing.TestService/UnaryCall + if strings.Contains(methodName, "/") { + tokens := strings.Split(methodName, "/") + if len(tokens) == 3 { + // Record service name and method name for all events. + bml.serviceName = tokens[1] + bml.methodName = tokens[2] + } else { + logger.Infof("Malformed method name: %v", methodName) + } + } + bml.authority = binLogEntry.GetClientHeader().GetAuthority() + grpcLogEntry.Payload.Timeout = binLogEntry.GetClientHeader().GetTimeout().AsDuration() + grpcLogEntry.Payload.Metadata = translateMetadata(binLogEntry.GetClientHeader().GetMetadata()) + } + grpcLogEntry.PayloadTruncated = binLogEntry.GetPayloadTruncated() + setPeerIfPresent(binLogEntry, grpcLogEntry) + case binlogpb.GrpcLogEntry_EVENT_TYPE_SERVER_HEADER: + grpcLogEntry.Type = eventTypeServerHeader + if binLogEntry.GetServerHeader() != nil { + grpcLogEntry.Payload.Metadata = translateMetadata(binLogEntry.GetServerHeader().GetMetadata()) + } + grpcLogEntry.PayloadTruncated = binLogEntry.GetPayloadTruncated() + setPeerIfPresent(binLogEntry, grpcLogEntry) + case binlogpb.GrpcLogEntry_EVENT_TYPE_CLIENT_MESSAGE: + grpcLogEntry.Type = eventTypeClientMessage + grpcLogEntry.Payload.Message = binLogEntry.GetMessage().GetData() + grpcLogEntry.Payload.MessageLength = binLogEntry.GetMessage().GetLength() + grpcLogEntry.PayloadTruncated = binLogEntry.GetPayloadTruncated() + case binlogpb.GrpcLogEntry_EVENT_TYPE_SERVER_MESSAGE: + grpcLogEntry.Type = eventTypeServerMessage + grpcLogEntry.Payload.Message = binLogEntry.GetMessage().GetData() + grpcLogEntry.Payload.MessageLength = binLogEntry.GetMessage().GetLength() + grpcLogEntry.PayloadTruncated = binLogEntry.GetPayloadTruncated() + case binlogpb.GrpcLogEntry_EVENT_TYPE_CLIENT_HALF_CLOSE: + grpcLogEntry.Type = eventTypeClientHalfClose + case binlogpb.GrpcLogEntry_EVENT_TYPE_SERVER_TRAILER: + grpcLogEntry.Type = eventTypeServerTrailer + grpcLogEntry.Payload.Metadata = translateMetadata(binLogEntry.GetTrailer().Metadata) + grpcLogEntry.Payload.StatusCode = canonicalString(codes.Code(binLogEntry.GetTrailer().GetStatusCode())) + grpcLogEntry.Payload.StatusMessage = binLogEntry.GetTrailer().GetStatusMessage() + grpcLogEntry.Payload.StatusDetails = binLogEntry.GetTrailer().GetStatusDetails() + grpcLogEntry.PayloadTruncated = binLogEntry.GetPayloadTruncated() + setPeerIfPresent(binLogEntry, grpcLogEntry) + case binlogpb.GrpcLogEntry_EVENT_TYPE_CANCEL: + grpcLogEntry.Type = eventTypeCancel + } + grpcLogEntry.ServiceName = bml.serviceName + grpcLogEntry.MethodName = bml.methodName + grpcLogEntry.Authority = bml.authority + + var sc trace.SpanContext + var ok bool + if bml.clientSide { + // client side span, populated through opencensus trace package. + if span := trace.FromContext(ctx); span != nil { + sc = span.SpanContext() + ok = true + } + } else { + // server side span, populated through stats/opencensus package. + sc, ok = opencensus.SpanContextFromContext(ctx) + } + gcploggingEntry := gcplogging.Entry{ + Timestamp: binLogEntry.GetTimestamp().AsTime(), + Severity: 100, + Payload: grpcLogEntry, + } + if ok { + gcploggingEntry.Trace = "projects/" + bml.projectID + "/traces/" + sc.TraceID.String() + gcploggingEntry.SpanID = sc.SpanID.String() + gcploggingEntry.TraceSampled = sc.IsSampled() + } + return gcploggingEntry +} + +func (bml *binaryMethodLogger) Log(ctx context.Context, c iblog.LogEntryConfig) { + bml.exporter.EmitGcpLoggingEntry(bml.buildGCPLoggingEntry(ctx, c)) +} + +type eventConfig struct { + // ServiceMethod has /s/m syntax for fast matching. + ServiceMethod map[string]bool + Services map[string]bool + MatchAll bool + + // If true, won't log anything. + Exclude bool + HeaderBytes uint64 + MessageBytes uint64 +} + +type binaryLogger struct { + EventConfigs []eventConfig + projectID string + exporter loggingExporter + clientSide bool +} + +func (bl *binaryLogger) GetMethodLogger(methodName string) iblog.MethodLogger { + s, _, err := grpcutil.ParseMethod(methodName) + if err != nil { + logger.Infof("binarylogging: failed to parse %q: %v", methodName, err) + return nil + } + for _, eventConfig := range bl.EventConfigs { + if eventConfig.MatchAll || eventConfig.ServiceMethod[methodName] || eventConfig.Services[s] { + if eventConfig.Exclude { + return nil + } + + return &binaryMethodLogger{ + exporter: bl.exporter, + mlb: iblog.NewTruncatingMethodLogger(eventConfig.HeaderBytes, eventConfig.MessageBytes), + callID: uuid.NewString(), + projectID: bl.projectID, + clientSide: bl.clientSide, + } + } + } + return nil +} + +// parseMethod splits service and method from the input. It expects format +// "service/method". +func parseMethod(method string) (string, string, error) { + pos := strings.Index(method, "/") + if pos < 0 { + // Shouldn't happen, config already validated. + return "", "", errors.New("invalid method name: no / found") + } + return method[:pos], method[pos+1:], nil +} + +func registerClientRPCEvents(config *config, exporter loggingExporter) { + clientRPCEvents := config.CloudLogging.ClientRPCEvents + if len(clientRPCEvents) == 0 { + return + } + var eventConfigs []eventConfig + for _, clientRPCEvent := range clientRPCEvents { + eventConfig := eventConfig{ + Exclude: clientRPCEvent.Exclude, + HeaderBytes: uint64(clientRPCEvent.MaxMetadataBytes), + MessageBytes: uint64(clientRPCEvent.MaxMessageBytes), + } + for _, method := range clientRPCEvent.Methods { + eventConfig.ServiceMethod = make(map[string]bool) + eventConfig.Services = make(map[string]bool) + if method == "*" { + eventConfig.MatchAll = true + continue + } + s, m, err := parseMethod(method) + if err != nil { + continue + } + if m == "*" { + eventConfig.Services[s] = true + continue + } + eventConfig.ServiceMethod["/"+method] = true + } + eventConfigs = append(eventConfigs, eventConfig) + } + clientSideLogger := &binaryLogger{ + EventConfigs: eventConfigs, + exporter: exporter, + projectID: config.ProjectID, + clientSide: true, + } + internal.AddGlobalDialOptions.(func(opt ...grpc.DialOption))(internal.WithBinaryLogger.(func(bl iblog.Logger) grpc.DialOption)(clientSideLogger)) +} + +func registerServerRPCEvents(config *config, exporter loggingExporter) { + serverRPCEvents := config.CloudLogging.ServerRPCEvents + if len(serverRPCEvents) == 0 { + return + } + var eventConfigs []eventConfig + for _, serverRPCEvent := range serverRPCEvents { + eventConfig := eventConfig{ + Exclude: serverRPCEvent.Exclude, + HeaderBytes: uint64(serverRPCEvent.MaxMetadataBytes), + MessageBytes: uint64(serverRPCEvent.MaxMessageBytes), + } + for _, method := range serverRPCEvent.Methods { + eventConfig.ServiceMethod = make(map[string]bool) + eventConfig.Services = make(map[string]bool) + if method == "*" { + eventConfig.MatchAll = true + continue + } + s, m, err := parseMethod(method) + if err != nil { + continue + } + if m == "*" { + eventConfig.Services[s] = true + continue + } + eventConfig.ServiceMethod["/"+method] = true + } + eventConfigs = append(eventConfigs, eventConfig) + } + serverSideLogger := &binaryLogger{ + EventConfigs: eventConfigs, + exporter: exporter, + projectID: config.ProjectID, + clientSide: false, + } + internal.AddGlobalServerOptions.(func(opt ...grpc.ServerOption))(internal.BinaryLogger.(func(bl iblog.Logger) grpc.ServerOption)(serverSideLogger)) +} + +func startLogging(ctx context.Context, config *config) error { + if config == nil || config.CloudLogging == nil { + return nil + } + var err error + lExporter, err = newLoggingExporter(ctx, config) + if err != nil { + return fmt.Errorf("unable to create CloudLogging exporter: %v", err) + } + + registerClientRPCEvents(config, lExporter) + registerServerRPCEvents(config, lExporter) + return nil +} + +func stopLogging() { + internal.ClearGlobalDialOptions() + internal.ClearGlobalServerOptions() + if lExporter != nil { + // This Close() call handles the flushing of the logging buffer. + lExporter.Close() + } +} diff --git a/gcp/observability/logging_test.go b/gcp/observability/logging_test.go new file mode 100644 index 000000000000..5e2bab44be48 --- /dev/null +++ b/gcp/observability/logging_test.go @@ -0,0 +1,1346 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package observability + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "strings" + "sync" + "testing" + + gcplogging "cloud.google.com/go/logging" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/metadata" + + binlogpb "google.golang.org/grpc/binarylog/grpc_binarylog_v1" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +func cmpLoggingEntryList(got []*grpcLogEntry, want []*grpcLogEntry) error { + if diff := cmp.Diff(got, want, + // For nondeterministic metadata iteration. + cmp.Comparer(func(a map[string]string, b map[string]string) bool { + if len(a) > len(b) { + a, b = b, a + } + if len(a) == 0 && len(a) != len(b) { // No metadata for one and the other comparator wants metadata. + return false + } + for k, v := range a { + if b[k] != v { + return false + } + } + return true + }), + cmpopts.IgnoreFields(grpcLogEntry{}, "CallID", "Peer"), + cmpopts.IgnoreFields(address{}, "IPPort", "Type"), + cmpopts.IgnoreFields(payload{}, "Timeout")); diff != "" { + return fmt.Errorf("got unexpected grpcLogEntry list, diff (-got, +want): %v", diff) + } + return nil +} + +type fakeLoggingExporter struct { + t *testing.T + + mu sync.Mutex + entries []*grpcLogEntry + + idsSeen []*traceAndSpanIDString +} + +func (fle *fakeLoggingExporter) EmitGcpLoggingEntry(entry gcplogging.Entry) { + fle.mu.Lock() + defer fle.mu.Unlock() + if entry.Severity != 100 { + fle.t.Errorf("entry.Severity is not 100, this should be hardcoded") + } + + ids := &traceAndSpanIDString{ + traceID: entry.Trace, + spanID: entry.SpanID, + isSampled: entry.TraceSampled, + } + fle.idsSeen = append(fle.idsSeen, ids) + + grpcLogEntry, ok := entry.Payload.(*grpcLogEntry) + if !ok { + fle.t.Errorf("payload passed in isn't grpcLogEntry") + } + fle.entries = append(fle.entries, grpcLogEntry) +} + +func (fle *fakeLoggingExporter) Close() error { + return nil +} + +// setupObservabilitySystemWithConfig sets up the observability system with the +// specified config, and returns a function which cleans up the observability +// system. +func setupObservabilitySystemWithConfig(cfg *config) (func(), error) { + validConfigJSON, err := json.Marshal(cfg) + if err != nil { + return nil, fmt.Errorf("failed to convert config to JSON: %v", err) + } + oldObservabilityConfig := envconfig.ObservabilityConfig + envconfig.ObservabilityConfig = string(validConfigJSON) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + err = Start(ctx) + cleanup := func() { + End() + envconfig.ObservabilityConfig = oldObservabilityConfig + } + if err != nil { + return cleanup, fmt.Errorf("error in Start: %v", err) + } + return cleanup, nil +} + +// TestClientRPCEventsLogAll tests the observability system configured with a +// client RPC event that logs every call. It performs a Unary and Bidirectional +// Streaming RPC, and expects certain grpcLogEntries to make it's way to the +// exporter. +func (s) TestClientRPCEventsLogAll(t *testing.T) { + fle := &fakeLoggingExporter{ + t: t, + } + defer func(ne func(ctx context.Context, config *config) (loggingExporter, error)) { + newLoggingExporter = ne + }(newLoggingExporter) + + newLoggingExporter = func(ctx context.Context, config *config) (loggingExporter, error) { + return fle, nil + } + + clientRPCEventLogAllConfig := &config{ + ProjectID: "fake", + CloudLogging: &cloudLogging{ + ClientRPCEvents: []clientRPCEvents{ + { + Methods: []string{"*"}, + MaxMetadataBytes: 30, + MaxMessageBytes: 30, + }, + }, + }, + } + cleanup, err := setupObservabilitySystemWithConfig(clientRPCEventLogAllConfig) + if err != nil { + t.Fatalf("error setting up observability: %v", err) + } + defer cleanup() + + ss := &stubserver.StubServer{ + UnaryCallF: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{}, nil + }, + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + if _, err := stream.Recv(); err != nil { + return err + } + if err := stream.Send(&testpb.StreamingOutputCallResponse{}); err != nil { + return err + } + if _, err := stream.Recv(); err != io.EOF { + return err + } + return nil + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil { + t.Fatalf("Unexpected error from UnaryCall: %v", err) + } + + grpcLogEntriesWant := []*grpcLogEntry{ + { + Type: eventTypeClientHeader, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "UnaryCall", + Authority: ss.Address, + SequenceID: 1, + Payload: payload{ + Metadata: map[string]string{}, + }, + }, + { + Type: eventTypeClientMessage, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "UnaryCall", + SequenceID: 2, + Authority: ss.Address, + Payload: payload{ + Message: []uint8{}, + }, + }, + { + Type: eventTypeServerHeader, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "UnaryCall", + SequenceID: 3, + Authority: ss.Address, + Payload: payload{ + Metadata: map[string]string{}, + }, + }, + { + Type: eventTypeServerMessage, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "UnaryCall", + Authority: ss.Address, + SequenceID: 4, + }, + { + Type: eventTypeServerTrailer, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "UnaryCall", + SequenceID: 5, + Authority: ss.Address, + Payload: payload{ + Metadata: map[string]string{}, + StatusCode: "OK", + }, + }, + } + fle.mu.Lock() + if err := cmpLoggingEntryList(fle.entries, grpcLogEntriesWant); err != nil { + fle.mu.Unlock() + t.Fatalf("error in logging entry list comparison %v", err) + } + + fle.entries = nil + fle.mu.Unlock() + + // Make a streaming RPC. This should cause Log calls on the MethodLogger. + stream, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("ss.Client.FullDuplexCall failed: %f", err) + } + if err := stream.Send(&testpb.StreamingOutputCallRequest{}); err != nil { + t.Fatalf("stream.Send() failed: %v", err) + } + if _, err := stream.Recv(); err != nil { + t.Fatalf("stream.Recv() failed: %v", err) + } + if err := stream.CloseSend(); err != nil { + t.Fatalf("stream.CloseSend()() failed: %v", err) + } + if _, err = stream.Recv(); err != io.EOF { + t.Fatalf("unexpected error: %v, expected an EOF error", err) + } + grpcLogEntriesWant = []*grpcLogEntry{ + { + Type: eventTypeClientHeader, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "FullDuplexCall", + Authority: ss.Address, + SequenceID: 1, + Payload: payload{ + Metadata: map[string]string{}, + }, + }, + { + Type: eventTypeClientMessage, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "FullDuplexCall", + SequenceID: 2, + Authority: ss.Address, + Payload: payload{ + Message: []uint8{}, + }, + }, + { + Type: eventTypeServerHeader, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "FullDuplexCall", + SequenceID: 3, + Authority: ss.Address, + Payload: payload{ + Metadata: map[string]string{}, + }, + }, + { + Type: eventTypeServerMessage, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "FullDuplexCall", + SequenceID: 4, + Authority: ss.Address, + }, + { + Type: eventTypeClientHalfClose, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "FullDuplexCall", + SequenceID: 5, + Authority: ss.Address, + }, + { + Type: eventTypeServerTrailer, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "FullDuplexCall", + Authority: ss.Address, + SequenceID: 6, + Payload: payload{ + Metadata: map[string]string{}, + StatusCode: "OK", + }, + }, + } + fle.mu.Lock() + if err := cmpLoggingEntryList(fle.entries, grpcLogEntriesWant); err != nil { + fle.mu.Unlock() + t.Fatalf("error in logging entry list comparison %v", err) + } + fle.mu.Unlock() +} + +func (s) TestServerRPCEventsLogAll(t *testing.T) { + fle := &fakeLoggingExporter{ + t: t, + } + defer func(ne func(ctx context.Context, config *config) (loggingExporter, error)) { + newLoggingExporter = ne + }(newLoggingExporter) + + newLoggingExporter = func(ctx context.Context, config *config) (loggingExporter, error) { + return fle, nil + } + + serverRPCEventLogAllConfig := &config{ + ProjectID: "fake", + CloudLogging: &cloudLogging{ + ServerRPCEvents: []serverRPCEvents{ + { + Methods: []string{"*"}, + MaxMetadataBytes: 30, + MaxMessageBytes: 30, + }, + }, + }, + } + cleanup, err := setupObservabilitySystemWithConfig(serverRPCEventLogAllConfig) + if err != nil { + t.Fatalf("error setting up observability %v", err) + } + defer cleanup() + + ss := &stubserver.StubServer{ + UnaryCallF: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{}, nil + }, + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + if _, err := stream.Recv(); err != nil { + return err + } + if err := stream.Send(&testpb.StreamingOutputCallResponse{}); err != nil { + return err + } + if _, err := stream.Recv(); err != io.EOF { + return err + } + return nil + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil { + t.Fatalf("Unexpected error from UnaryCall: %v", err) + } + grpcLogEntriesWant := []*grpcLogEntry{ + { + Type: eventTypeClientHeader, + Logger: loggerServer, + ServiceName: "grpc.testing.TestService", + MethodName: "UnaryCall", + Authority: ss.Address, + SequenceID: 1, + Payload: payload{ + Metadata: map[string]string{}, + }, + }, + { + Type: eventTypeClientMessage, + Logger: loggerServer, + ServiceName: "grpc.testing.TestService", + MethodName: "UnaryCall", + SequenceID: 2, + Authority: ss.Address, + }, + { + Type: eventTypeServerHeader, + Logger: loggerServer, + ServiceName: "grpc.testing.TestService", + MethodName: "UnaryCall", + SequenceID: 3, + Authority: ss.Address, + Payload: payload{ + Metadata: map[string]string{}, + }, + }, + { + Type: eventTypeServerMessage, + Logger: loggerServer, + ServiceName: "grpc.testing.TestService", + MethodName: "UnaryCall", + Authority: ss.Address, + SequenceID: 4, + Payload: payload{ + Message: []uint8{}, + }, + }, + { + Type: eventTypeServerTrailer, + Logger: loggerServer, + ServiceName: "grpc.testing.TestService", + MethodName: "UnaryCall", + SequenceID: 5, + Authority: ss.Address, + Payload: payload{ + Metadata: map[string]string{}, + StatusCode: "OK", + }, + }, + } + fle.mu.Lock() + if err := cmpLoggingEntryList(fle.entries, grpcLogEntriesWant); err != nil { + fle.mu.Unlock() + t.Fatalf("error in logging entry list comparison %v", err) + } + fle.entries = nil + fle.mu.Unlock() + + stream, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("ss.Client.FullDuplexCall failed: %f", err) + } + if err := stream.Send(&testpb.StreamingOutputCallRequest{}); err != nil { + t.Fatalf("stream.Send() failed: %v", err) + } + if _, err := stream.Recv(); err != nil { + t.Fatalf("stream.Recv() failed: %v", err) + } + if err := stream.CloseSend(); err != nil { + t.Fatalf("stream.CloseSend()() failed: %v", err) + } + if _, err = stream.Recv(); err != io.EOF { + t.Fatalf("unexpected error: %v, expected an EOF error", err) + } + + grpcLogEntriesWant = []*grpcLogEntry{ + { + Type: eventTypeClientHeader, + Logger: loggerServer, + ServiceName: "grpc.testing.TestService", + MethodName: "FullDuplexCall", + Authority: ss.Address, + SequenceID: 1, + Payload: payload{ + Metadata: map[string]string{}, + }, + }, + { + Type: eventTypeClientMessage, + Logger: loggerServer, + ServiceName: "grpc.testing.TestService", + MethodName: "FullDuplexCall", + SequenceID: 2, + Authority: ss.Address, + }, + { + Type: eventTypeServerHeader, + Logger: loggerServer, + ServiceName: "grpc.testing.TestService", + MethodName: "FullDuplexCall", + SequenceID: 3, + Authority: ss.Address, + Payload: payload{ + Metadata: map[string]string{}, + }, + }, + { + Type: eventTypeServerMessage, + Logger: loggerServer, + ServiceName: "grpc.testing.TestService", + MethodName: "FullDuplexCall", + SequenceID: 4, + Authority: ss.Address, + Payload: payload{ + Message: []uint8{}, + }, + }, + { + Type: eventTypeClientHalfClose, + Logger: loggerServer, + ServiceName: "grpc.testing.TestService", + MethodName: "FullDuplexCall", + SequenceID: 5, + Authority: ss.Address, + }, + { + Type: eventTypeServerTrailer, + Logger: loggerServer, + ServiceName: "grpc.testing.TestService", + MethodName: "FullDuplexCall", + Authority: ss.Address, + SequenceID: 6, + Payload: payload{ + Metadata: map[string]string{}, + StatusCode: "OK", + }, + }, + } + fle.mu.Lock() + if err := cmpLoggingEntryList(fle.entries, grpcLogEntriesWant); err != nil { + fle.mu.Unlock() + t.Fatalf("error in logging entry list comparison %v", err) + } + fle.mu.Unlock() +} + +// TestBothClientAndServerRPCEvents tests the scenario where you have both +// Client and Server RPC Events configured to log. Both sides should log and +// share the exporter, so the exporter should receive the collective amount of +// calls for both a client stream (corresponding to a Client RPC Event) and a +// server stream (corresponding ot a Server RPC Event). The specificity of the +// entries are tested in previous tests. +func (s) TestBothClientAndServerRPCEvents(t *testing.T) { + fle := &fakeLoggingExporter{ + t: t, + } + defer func(ne func(ctx context.Context, config *config) (loggingExporter, error)) { + newLoggingExporter = ne + }(newLoggingExporter) + + newLoggingExporter = func(ctx context.Context, config *config) (loggingExporter, error) { + return fle, nil + } + + serverRPCEventLogAllConfig := &config{ + ProjectID: "fake", + CloudLogging: &cloudLogging{ + ClientRPCEvents: []clientRPCEvents{ + { + Methods: []string{"*"}, + MaxMetadataBytes: 30, + MaxMessageBytes: 30, + }, + }, + ServerRPCEvents: []serverRPCEvents{ + { + Methods: []string{"*"}, + MaxMetadataBytes: 30, + MaxMessageBytes: 30, + }, + }, + }, + } + + cleanup, err := setupObservabilitySystemWithConfig(serverRPCEventLogAllConfig) + if err != nil { + t.Fatalf("error setting up observability %v", err) + } + defer cleanup() + + ss := &stubserver.StubServer{ + UnaryCallF: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{}, nil + }, + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + _, err := stream.Recv() + if err != io.EOF { + return err + } + return nil + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + // Make a Unary RPC. Both client side and server side streams should log + // entries, which share the same exporter. The exporter should thus receive + // entries from both the client and server streams (the specificity of + // entries is checked in previous tests). + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil { + t.Fatalf("Unexpected error from UnaryCall: %v", err) + } + fle.mu.Lock() + if len(fle.entries) != 10 { + fle.mu.Unlock() + t.Fatalf("Unexpected length of entries %v, want 10 (collective of client and server)", len(fle.entries)) + } + fle.mu.Unlock() + stream, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("ss.Client.FullDuplexCall failed: %f", err) + } + + stream.CloseSend() + if _, err = stream.Recv(); err != io.EOF { + t.Fatalf("unexpected error: %v, expected an EOF error", err) + } + fle.mu.Lock() + if len(fle.entries) != 16 { + fle.mu.Unlock() + t.Fatalf("Unexpected length of entries %v, want 16 (collective of client and server)", len(fle.entries)) + } + fle.mu.Unlock() +} + +// TestClientRPCEventsLogAll tests the observability system configured with a +// client RPC event that logs every call and that truncates headers and +// messages. It performs a Unary RPC, and expects events with truncated payloads +// and payloadTruncated set to true, signifying the system properly truncated +// headers and messages logged. +func (s) TestClientRPCEventsTruncateHeaderAndMetadata(t *testing.T) { + fle := &fakeLoggingExporter{ + t: t, + } + defer func(ne func(ctx context.Context, config *config) (loggingExporter, error)) { + newLoggingExporter = ne + }(newLoggingExporter) + + newLoggingExporter = func(ctx context.Context, config *config) (loggingExporter, error) { + return fle, nil + } + + clientRPCEventLogAllConfig := &config{ + ProjectID: "fake", + CloudLogging: &cloudLogging{ + ClientRPCEvents: []clientRPCEvents{ + { + Methods: []string{"*"}, + MaxMetadataBytes: 10, + MaxMessageBytes: 2, + }, + }, + }, + } + cleanup, err := setupObservabilitySystemWithConfig(clientRPCEventLogAllConfig) + if err != nil { + t.Fatalf("error setting up observability: %v", err) + } + defer cleanup() + + ss := &stubserver.StubServer{ + UnaryCallF: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{}, nil + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + md := metadata.MD{ + "key1": []string{"value1"}, + "key2": []string{"value2"}, + } + ctx = metadata.NewOutgoingContext(ctx, md) + if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{Payload: &testpb.Payload{Body: []byte("00000")}}); err != nil { + t.Fatalf("Unexpected error from UnaryCall: %v", err) + } + grpcLogEntriesWant := []*grpcLogEntry{ + { + Type: eventTypeClientHeader, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "UnaryCall", + Authority: ss.Address, + SequenceID: 1, + Payload: payload{ + Metadata: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + }, + PayloadTruncated: true, + }, + { + Type: eventTypeClientMessage, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "UnaryCall", + SequenceID: 2, + Authority: ss.Address, + Payload: payload{ + MessageLength: 9, + Message: []uint8{ + 0x1a, + 0x07, + }, + }, + PayloadTruncated: true, + }, + { + Type: eventTypeServerHeader, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "UnaryCall", + SequenceID: 3, + Authority: ss.Address, + Payload: payload{ + Metadata: map[string]string{}, + }, + }, + { + Type: eventTypeServerMessage, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "UnaryCall", + Authority: ss.Address, + SequenceID: 4, + }, + { + Type: eventTypeServerTrailer, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "UnaryCall", + SequenceID: 5, + Authority: ss.Address, + Payload: payload{ + Metadata: map[string]string{}, + StatusCode: "OK", + }, + }, + } + fle.mu.Lock() + if err := cmpLoggingEntryList(fle.entries, grpcLogEntriesWant); err != nil { + fle.mu.Unlock() + t.Fatalf("error in logging entry list comparison %v", err) + } + // Only one metadata entry should have been present in logging due to + // truncation. + if mdLen := len(fle.entries[0].Payload.Metadata); mdLen != 1 { + t.Fatalf("Metadata should have only 1 entry due to truncation, got %v", mdLen) + } + fle.mu.Unlock() +} + +// TestPrecedenceOrderingInConfiguration tests the scenario where the logging +// part of observability is configured with three client RPC events, the first +// two on specific methods in the service, the last one for any method within +// the service. This test sends three RPC's, one corresponding to each log +// entry. The logging logic dictated by that specific event should be what is +// used for emission. The second event will specify to exclude logging on RPC's, +// which should generate no log entries if an RPC gets to and matches that +// event. +func (s) TestPrecedenceOrderingInConfiguration(t *testing.T) { + fle := &fakeLoggingExporter{ + t: t, + } + + defer func(ne func(ctx context.Context, config *config) (loggingExporter, error)) { + newLoggingExporter = ne + }(newLoggingExporter) + + newLoggingExporter = func(ctx context.Context, config *config) (loggingExporter, error) { + return fle, nil + } + + threeEventsConfig := &config{ + ProjectID: "fake", + CloudLogging: &cloudLogging{ + ClientRPCEvents: []clientRPCEvents{ + { + Methods: []string{"grpc.testing.TestService/UnaryCall"}, + MaxMetadataBytes: 30, + MaxMessageBytes: 30, + }, + { + Methods: []string{"grpc.testing.TestService/EmptyCall"}, + Exclude: true, + MaxMetadataBytes: 30, + MaxMessageBytes: 30, + }, + { + Methods: []string{"grpc.testing.TestService/*"}, + MaxMetadataBytes: 30, + MaxMessageBytes: 30, + }, + }, + }, + } + + cleanup, err := setupObservabilitySystemWithConfig(threeEventsConfig) + if err != nil { + t.Fatalf("error setting up observability %v", err) + } + defer cleanup() + + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + UnaryCallF: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{}, nil + }, + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + _, err := stream.Recv() + if err != io.EOF { + return err + } + return nil + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + // A Unary RPC should match with first event and logs should correspond + // accordingly. The first event it matches to should be used for the + // configuration, even though it could potentially match to events in the + // future. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil { + t.Fatalf("Unexpected error from UnaryCall: %v", err) + } + grpcLogEntriesWant := []*grpcLogEntry{ + { + Type: eventTypeClientHeader, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "UnaryCall", + Authority: ss.Address, + SequenceID: 1, + Payload: payload{ + Metadata: map[string]string{}, + }, + }, + { + Type: eventTypeClientMessage, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "UnaryCall", + SequenceID: 2, + Authority: ss.Address, + Payload: payload{ + Message: []uint8{}, + }, + }, + { + Type: eventTypeServerHeader, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "UnaryCall", + SequenceID: 3, + Authority: ss.Address, + Payload: payload{ + Metadata: map[string]string{}, + }, + }, + { + Type: eventTypeServerMessage, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "UnaryCall", + Authority: ss.Address, + SequenceID: 4, + }, + { + Type: eventTypeServerTrailer, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "UnaryCall", + SequenceID: 5, + Authority: ss.Address, + Payload: payload{ + Metadata: map[string]string{}, + StatusCode: "OK", + }, + }, + } + + fle.mu.Lock() + if err := cmpLoggingEntryList(fle.entries, grpcLogEntriesWant); err != nil { + fle.mu.Unlock() + t.Fatalf("error in logging entry list comparison %v", err) + } + fle.entries = nil + fle.mu.Unlock() + + // A unary empty RPC should match with the second event, which has the exclude + // flag set. Thus, a unary empty RPC should cause no downstream logs. + if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("Unexpected error from EmptyCall: %v", err) + } + // The exporter should have received no new log entries due to this call. + fle.mu.Lock() + if len(fle.entries) != 0 { + fle.mu.Unlock() + t.Fatalf("Unexpected length of entries %v, want 0", len(fle.entries)) + } + fle.mu.Unlock() + + // A third RPC, a full duplex call, which doesn't match with first two and + // matches to last one, due to being a wildcard for every method in the + // service, should log accordingly to the last event's logic. + stream, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("ss.Client.FullDuplexCall failed: %f", err) + } + + stream.CloseSend() + if _, err = stream.Recv(); err != io.EOF { + t.Fatalf("unexpected error: %v, expected an EOF error", err) + } + + grpcLogEntriesWant = []*grpcLogEntry{ + { + Type: eventTypeClientHeader, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "FullDuplexCall", + Authority: ss.Address, + SequenceID: 1, + Payload: payload{ + Metadata: map[string]string{}, + }, + }, + { + Type: eventTypeClientHalfClose, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "FullDuplexCall", + SequenceID: 2, + Authority: ss.Address, + }, + { + Type: eventTypeServerTrailer, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "FullDuplexCall", + Authority: ss.Address, + SequenceID: 3, + Payload: payload{ + Metadata: map[string]string{}, + StatusCode: "OK", + }, + }, + } + fle.mu.Lock() + if err := cmpLoggingEntryList(fle.entries, grpcLogEntriesWant); err != nil { + fle.mu.Unlock() + t.Fatalf("error in logging entry list comparison %v", err) + } + fle.mu.Unlock() +} + +func (s) TestTranslateMetadata(t *testing.T) { + concatBinLogValue := base64.StdEncoding.EncodeToString([]byte("value1")) + "," + base64.StdEncoding.EncodeToString([]byte("value2")) + tests := []struct { + name string + binLogMD *binlogpb.Metadata + wantMD map[string]string + }{ + { + name: "two-entries-different-key", + binLogMD: &binlogpb.Metadata{ + Entry: []*binlogpb.MetadataEntry{ + { + Key: "header1", + Value: []byte("value1"), + }, + { + Key: "header2", + Value: []byte("value2"), + }, + }, + }, + wantMD: map[string]string{ + "header1": "value1", + "header2": "value2", + }, + }, + { + name: "two-entries-same-key", + binLogMD: &binlogpb.Metadata{ + Entry: []*binlogpb.MetadataEntry{ + { + Key: "header1", + Value: []byte("value1"), + }, + { + Key: "header1", + Value: []byte("value2"), + }, + }, + }, + wantMD: map[string]string{ + "header1": "value1,value2", + }, + }, + { + name: "two-entries-same-key-bin-header", + binLogMD: &binlogpb.Metadata{ + Entry: []*binlogpb.MetadataEntry{ + { + Key: "header1-bin", + Value: []byte("value1"), + }, + { + Key: "header1-bin", + Value: []byte("value2"), + }, + }, + }, + wantMD: map[string]string{ + "header1-bin": concatBinLogValue, + }, + }, + { + name: "four-entries-two-keys", + binLogMD: &binlogpb.Metadata{ + Entry: []*binlogpb.MetadataEntry{ + { + Key: "header1", + Value: []byte("value1"), + }, + { + Key: "header1", + Value: []byte("value2"), + }, + { + Key: "header1-bin", + Value: []byte("value1"), + }, + { + Key: "header1-bin", + Value: []byte("value2"), + }, + }, + }, + wantMD: map[string]string{ + "header1": "value1,value2", + "header1-bin": concatBinLogValue, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if gotMD := translateMetadata(test.binLogMD); !cmp.Equal(gotMD, test.wantMD) { + t.Fatalf("translateMetadata(%v) = %v, want %v", test.binLogMD, gotMD, test.wantMD) + } + }) + } +} + +func (s) TestMarshalJSON(t *testing.T) { + logEntry := &grpcLogEntry{ + CallID: "300-300-300", + SequenceID: 3, + Type: eventTypeUnknown, + Logger: loggerClient, + Payload: payload{ + Metadata: map[string]string{"header1": "value1"}, + Timeout: 20, + StatusCode: "UNKNOWN", + StatusMessage: "ok", + StatusDetails: []byte("ok"), + MessageLength: 3, + Message: []byte("wow"), + }, + Peer: address{ + Type: ipv4, + Address: "localhost", + IPPort: 16000, + }, + PayloadTruncated: false, + Authority: "server", + ServiceName: "grpc-testing", + MethodName: "UnaryRPC", + } + if _, err := json.Marshal(logEntry); err != nil { + t.Fatalf("json.Marshal(%v) failed with error: %v", logEntry, err) + } +} + +// TestMetadataTruncationAccountsKey tests that the metadata truncation takes +// into account both the key and value of metadata. It configures an +// observability system with a maximum byte length for metadata, which is +// greater than just the byte length of the metadata value but less than the +// byte length of the metadata key + metadata value. Thus, in the ClientHeader +// logging event, no metadata should be logged. +func (s) TestMetadataTruncationAccountsKey(t *testing.T) { + fle := &fakeLoggingExporter{ + t: t, + } + defer func(ne func(ctx context.Context, config *config) (loggingExporter, error)) { + newLoggingExporter = ne + }(newLoggingExporter) + + newLoggingExporter = func(ctx context.Context, config *config) (loggingExporter, error) { + return fle, nil + } + + const mdValue = "value" + configMetadataLimit := &config{ + ProjectID: "fake", + CloudLogging: &cloudLogging{ + ClientRPCEvents: []clientRPCEvents{ + { + Methods: []string{"*"}, + MaxMetadataBytes: len(mdValue) + 1, + }, + }, + }, + } + + cleanup, err := setupObservabilitySystemWithConfig(configMetadataLimit) + if err != nil { + t.Fatalf("error setting up observability %v", err) + } + defer cleanup() + + ss := &stubserver.StubServer{ + UnaryCallF: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{}, nil + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // the set config MaxMetdataBytes is in between len(mdValue) and len("key") + // + len(mdValue), and thus shouldn't log this metadata entry. + md := metadata.MD{ + "key": []string{mdValue}, + } + ctx = metadata.NewOutgoingContext(ctx, md) + if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{Payload: &testpb.Payload{Body: []byte("00000")}}); err != nil { + t.Fatalf("Unexpected error from UnaryCall: %v", err) + } + + grpcLogEntriesWant := []*grpcLogEntry{ + { + Type: eventTypeClientHeader, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "UnaryCall", + Authority: ss.Address, + SequenceID: 1, + Payload: payload{ + Metadata: map[string]string{}, + }, + PayloadTruncated: true, + }, + { + Type: eventTypeClientMessage, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "UnaryCall", + SequenceID: 2, + Authority: ss.Address, + Payload: payload{ + MessageLength: 9, + Message: []uint8{}, + }, + PayloadTruncated: true, + }, + { + Type: eventTypeServerHeader, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "UnaryCall", + SequenceID: 3, + Authority: ss.Address, + Payload: payload{ + Metadata: map[string]string{}, + }, + }, + { + Type: eventTypeServerMessage, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "UnaryCall", + Authority: ss.Address, + SequenceID: 4, + }, + { + Type: eventTypeServerTrailer, + Logger: loggerClient, + ServiceName: "grpc.testing.TestService", + MethodName: "UnaryCall", + SequenceID: 5, + Authority: ss.Address, + Payload: payload{ + Metadata: map[string]string{}, + StatusCode: "OK", + }, + }, + } + fle.mu.Lock() + if err := cmpLoggingEntryList(fle.entries, grpcLogEntriesWant); err != nil { + fle.mu.Unlock() + t.Fatalf("error in logging entry list comparison %v", err) + } + fle.mu.Unlock() +} + +// TestMethodInConfiguration tests different method names with an expectation on +// whether they should error or not. +func (s) TestMethodInConfiguration(t *testing.T) { + // To skip creating a stackdriver exporter. + fle := &fakeLoggingExporter{ + t: t, + } + + defer func(ne func(ctx context.Context, config *config) (loggingExporter, error)) { + newLoggingExporter = ne + }(newLoggingExporter) + + newLoggingExporter = func(ctx context.Context, config *config) (loggingExporter, error) { + return fle, nil + } + + tests := []struct { + name string + config *config + wantErr string + }{ + { + name: "leading-slash", + config: &config{ + ProjectID: "fake", + CloudLogging: &cloudLogging{ + ClientRPCEvents: []clientRPCEvents{ + { + Methods: []string{"/service/method"}, + }, + }, + }, + }, + wantErr: "cannot have a leading slash", + }, + { + name: "wildcard service/method", + config: &config{ + ProjectID: "fake", + CloudLogging: &cloudLogging{ + ClientRPCEvents: []clientRPCEvents{ + { + Methods: []string{"*/method"}, + }, + }, + }, + }, + wantErr: "cannot have service wildcard *", + }, + { + name: "/ in service name", + config: &config{ + ProjectID: "fake", + CloudLogging: &cloudLogging{ + ClientRPCEvents: []clientRPCEvents{ + { + Methods: []string{"ser/vice/method"}, + }, + }, + }, + }, + wantErr: "only one /", + }, + { + name: "empty method name", + config: &config{ + ProjectID: "fake", + CloudLogging: &cloudLogging{ + ClientRPCEvents: []clientRPCEvents{ + { + Methods: []string{"service/"}, + }, + }, + }, + }, + wantErr: "method name must be non empty", + }, + { + name: "normal", + config: &config{ + ProjectID: "fake", + CloudLogging: &cloudLogging{ + ClientRPCEvents: []clientRPCEvents{ + { + Methods: []string{"service/method"}, + }, + }, + }, + }, + wantErr: "", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cleanup, gotErr := setupObservabilitySystemWithConfig(test.config) + if cleanup != nil { + defer cleanup() + } + if gotErr != nil && !strings.Contains(gotErr.Error(), test.wantErr) { + t.Fatalf("Start(%v) = %v, wantErr %v", test.config, gotErr, test.wantErr) + } + if (gotErr != nil) != (test.wantErr != "") { + t.Fatalf("Start(%v) = %v, wantErr %v", test.config, gotErr, test.wantErr) + } + }) + } +} diff --git a/gcp/observability/observability.go b/gcp/observability/observability.go new file mode 100644 index 000000000000..6b7d4b1f762a --- /dev/null +++ b/gcp/observability/observability.go @@ -0,0 +1,92 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package observability implements the tracing, metrics, and logging data +// collection, and provides controlling knobs via a config file. +// +// # Experimental +// +// Notice: This package is EXPERIMENTAL and may be changed or removed in a +// later release. +package observability + +import ( + "context" + "fmt" + + "google.golang.org/grpc/grpclog" +) + +var logger = grpclog.Component("observability") + +// Start is the opt-in API for gRPC Observability plugin. This function should +// be invoked in the main function, and before creating any gRPC clients or +// servers, otherwise, they might not be instrumented. At high-level, this +// module does the following: +// +// - it loads observability config from environment; +// - it registers default exporters if not disabled by the config; +// - it sets up telemetry collectors (binary logging sink or StatsHandlers). +// +// Note: this method should only be invoked once. +// Note: handle the error +func Start(ctx context.Context) error { + config, err := parseObservabilityConfig() + if err != nil { + return err + } + if config == nil { + return fmt.Errorf("no ObservabilityConfig found") + } + + // Set the project ID if it isn't configured manually. + if err = ensureProjectIDInObservabilityConfig(ctx, config); err != nil { + return err + } + + // Cleanup any created resources this function created in case this function + // errors. + defer func() { + if err != nil { + End() + } + }() + + // Enabling tracing and metrics via OpenCensus + if err = startOpenCensus(config); err != nil { + return fmt.Errorf("failed to instrument OpenCensus: %v", err) + } + + if err = startLogging(ctx, config); err != nil { + return fmt.Errorf("failed to start logging: %v", err) + } + + // Logging is controlled by the config at methods level. + return nil +} + +// End is the clean-up API for gRPC Observability plugin. It is expected to be +// invoked in the main function of the application. The suggested usage is +// "defer observability.End()". This function also flushes data to upstream, and +// cleanup resources. +// +// Note: this method should only be invoked once. +func End() { + stopLogging() + stopOpenCensus() +} diff --git a/gcp/observability/observability_test.go b/gcp/observability/observability_test.go new file mode 100644 index 000000000000..b2030d86b2fa --- /dev/null +++ b/gcp/observability/observability_test.go @@ -0,0 +1,1079 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package observability + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "strings" + "sync" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "go.opencensus.io/stats/view" + "go.opencensus.io/trace" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/leakcheck" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/metadata" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func init() { + // OpenCensus, once included in binary, will spawn a global goroutine + // recorder that is not controllable by application. + // https://github.com/census-instrumentation/opencensus-go/issues/1191 + leakcheck.RegisterIgnoreGoroutine("go.opencensus.io/stats/view.(*worker).start") + // google-cloud-go leaks HTTP client. They are aware of this: + // https://github.com/googleapis/google-cloud-go/issues/1183 + leakcheck.RegisterIgnoreGoroutine("internal/poll.runtime_pollWait") +} + +var ( + defaultTestTimeout = 10 * time.Second + testHeaderMetadata = metadata.MD{"header": []string{"HeADer"}} + testTrailerMetadata = metadata.MD{"trailer": []string{"TrAileR"}} + testOkPayload = []byte{72, 101, 108, 108, 111, 32, 87, 111, 114, 108, 100} + testErrorPayload = []byte{77, 97, 114, 116, 104, 97} + testErrorMessage = "test case injected error" + infinitySizeBytes int32 = 1024 * 1024 * 1024 + defaultRequestCount = 24 +) + +const ( + TypeOpenCensusViewDistribution string = "distribution" + TypeOpenCensusViewCount = "count" + TypeOpenCensusViewSum = "sum" + TypeOpenCensusViewLastValue = "last_value" +) + +type fakeOpenCensusExporter struct { + // The map of the observed View name and type + SeenViews map[string]string + // Number of spans + SeenSpans int + + idCh *testutils.Channel + + t *testing.T + mu sync.RWMutex +} + +func (fe *fakeOpenCensusExporter) ExportView(vd *view.Data) { + fe.mu.Lock() + defer fe.mu.Unlock() + for _, row := range vd.Rows { + fe.t.Logf("Metrics[%s]", vd.View.Name) + switch row.Data.(type) { + case *view.DistributionData: + fe.SeenViews[vd.View.Name] = TypeOpenCensusViewDistribution + case *view.CountData: + fe.SeenViews[vd.View.Name] = TypeOpenCensusViewCount + case *view.SumData: + fe.SeenViews[vd.View.Name] = TypeOpenCensusViewSum + case *view.LastValueData: + fe.SeenViews[vd.View.Name] = TypeOpenCensusViewLastValue + } + } +} + +type traceAndSpanID struct { + spanName string + traceID trace.TraceID + spanID trace.SpanID + isSampled bool + spanKind int +} + +type traceAndSpanIDString struct { + traceID string + spanID string + isSampled bool + // SpanKind is the type of span. + SpanKind int +} + +// idsToString is a helper that converts from generated trace and span IDs to +// the string version stored in trace message events. +func (tasi *traceAndSpanID) idsToString(projectID string) traceAndSpanIDString { + return traceAndSpanIDString{ + traceID: "projects/" + projectID + "/traces/" + tasi.traceID.String(), + spanID: tasi.spanID.String(), + isSampled: tasi.isSampled, + SpanKind: tasi.spanKind, + } +} + +func (fe *fakeOpenCensusExporter) ExportSpan(vd *trace.SpanData) { + if fe.idCh != nil { + // This is what export span sees representing the trace/span ID which + // will populate different contexts throughout the system, convert in + // caller to string version as the logging code does. + fe.idCh.Send(traceAndSpanID{ + spanName: vd.Name, + traceID: vd.TraceID, + spanID: vd.SpanID, + isSampled: vd.IsSampled(), + spanKind: vd.SpanKind, + }) + } + + fe.mu.Lock() + defer fe.mu.Unlock() + fe.SeenSpans++ + fe.t.Logf("Span[%v]", vd.Name) +} + +func (fe *fakeOpenCensusExporter) Flush() {} + +func (fe *fakeOpenCensusExporter) Close() error { + return nil +} + +func (s) TestRefuseStartWithInvalidPatterns(t *testing.T) { + invalidConfig := &config{ + ProjectID: "fake", + CloudLogging: &cloudLogging{ + ClientRPCEvents: []clientRPCEvents{ + { + Methods: []string{":-)"}, + MaxMetadataBytes: 30, + MaxMessageBytes: 30, + }, + }, + }, + } + invalidConfigJSON, err := json.Marshal(invalidConfig) + if err != nil { + t.Fatalf("failed to convert config to JSON: %v", err) + } + oldObservabilityConfig := envconfig.ObservabilityConfig + oldObservabilityConfigFile := envconfig.ObservabilityConfigFile + envconfig.ObservabilityConfig = string(invalidConfigJSON) + envconfig.ObservabilityConfigFile = "" + defer func() { + envconfig.ObservabilityConfig = oldObservabilityConfig + envconfig.ObservabilityConfigFile = oldObservabilityConfigFile + }() + // If there is at least one invalid pattern, which should not be silently tolerated. + if err := Start(context.Background()); err == nil { + t.Fatalf("Invalid patterns not triggering error") + } +} + +// TestRefuseStartWithExcludeAndWildCardAll tests the sceanrio where an +// observability configuration is provided with client RPC event specifying to +// exclude, and which matches on the '*' wildcard (any). This should cause an +// error when trying to start the observability system. +func (s) TestRefuseStartWithExcludeAndWildCardAll(t *testing.T) { + invalidConfig := &config{ + ProjectID: "fake", + CloudLogging: &cloudLogging{ + ClientRPCEvents: []clientRPCEvents{ + { + Methods: []string{"*"}, + Exclude: true, + MaxMetadataBytes: 30, + MaxMessageBytes: 30, + }, + }, + }, + } + invalidConfigJSON, err := json.Marshal(invalidConfig) + if err != nil { + t.Fatalf("failed to convert config to JSON: %v", err) + } + oldObservabilityConfig := envconfig.ObservabilityConfig + oldObservabilityConfigFile := envconfig.ObservabilityConfigFile + envconfig.ObservabilityConfig = string(invalidConfigJSON) + envconfig.ObservabilityConfigFile = "" + defer func() { + envconfig.ObservabilityConfig = oldObservabilityConfig + envconfig.ObservabilityConfigFile = oldObservabilityConfigFile + }() + // If there is at least one invalid pattern, which should not be silently tolerated. + if err := Start(context.Background()); err == nil { + t.Fatalf("Invalid patterns not triggering error") + } +} + +// createTmpConfigInFileSystem creates a random observability config at a random +// place in the temporary portion of the file system dependent on system. It +// also sets the environment variable GRPC_CONFIG_OBSERVABILITY_JSON to point to +// this created config. +func createTmpConfigInFileSystem(rawJSON string) (func(), error) { + configJSONFile, err := os.CreateTemp(os.TempDir(), "configJSON-") + if err != nil { + return nil, fmt.Errorf("cannot create file %v: %v", configJSONFile.Name(), err) + } + _, err = configJSONFile.Write(json.RawMessage(rawJSON)) + if err != nil { + return nil, fmt.Errorf("cannot write marshalled JSON: %v", err) + } + oldObservabilityConfigFile := envconfig.ObservabilityConfigFile + envconfig.ObservabilityConfigFile = configJSONFile.Name() + return func() { + configJSONFile.Close() + envconfig.ObservabilityConfigFile = oldObservabilityConfigFile + }, nil +} + +// TestJSONEnvVarSet tests a valid observability configuration specified by the +// GRPC_CONFIG_OBSERVABILITY_JSON environment variable, whose value represents a +// file path pointing to a JSON encoded config. +func (s) TestJSONEnvVarSet(t *testing.T) { + configJSON := `{ + "project_id": "fake" + }` + cleanup, err := createTmpConfigInFileSystem(configJSON) + defer cleanup() + + if err != nil { + t.Fatalf("failed to create config in file system: %v", err) + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := Start(ctx); err != nil { + t.Fatalf("error starting observability with valid config through file system: %v", err) + } + defer End() +} + +// TestBothConfigEnvVarsSet tests the scenario where both configuration +// environment variables are set. The file system environment variable should +// take precedence, and an error should return in the case of the file system +// configuration being invalid, even if the direct configuration environment +// variable is set and valid. +func (s) TestBothConfigEnvVarsSet(t *testing.T) { + invalidConfig := &config{ + ProjectID: "fake", + CloudLogging: &cloudLogging{ + ClientRPCEvents: []clientRPCEvents{ + { + Methods: []string{":-)"}, + MaxMetadataBytes: 30, + MaxMessageBytes: 30, + }, + }, + }, + } + invalidConfigJSON, err := json.Marshal(invalidConfig) + if err != nil { + t.Fatalf("failed to convert config to JSON: %v", err) + } + cleanup, err := createTmpConfigInFileSystem(string(invalidConfigJSON)) + defer cleanup() + if err != nil { + t.Fatalf("failed to create config in file system: %v", err) + } + // This configuration should be ignored, as precedence 2. + validConfig := &config{ + ProjectID: "fake", + CloudLogging: &cloudLogging{ + ClientRPCEvents: []clientRPCEvents{ + { + Methods: []string{"*"}, + MaxMetadataBytes: 30, + MaxMessageBytes: 30, + }, + }, + }, + } + validConfigJSON, err := json.Marshal(validConfig) + if err != nil { + t.Fatalf("failed to convert config to JSON: %v", err) + } + oldObservabilityConfig := envconfig.ObservabilityConfig + envconfig.ObservabilityConfig = string(validConfigJSON) + defer func() { + envconfig.ObservabilityConfig = oldObservabilityConfig + }() + if err := Start(context.Background()); err == nil { + t.Fatalf("Invalid patterns not triggering error") + } +} + +// TestErrInFileSystemEnvVar tests the scenario where an observability +// configuration is specified with environment variable that specifies a +// location in the file system for configuration, and this location doesn't have +// a file (or valid configuration). +func (s) TestErrInFileSystemEnvVar(t *testing.T) { + oldObservabilityConfigFile := envconfig.ObservabilityConfigFile + envconfig.ObservabilityConfigFile = "/this-file/does-not-exist" + defer func() { + envconfig.ObservabilityConfigFile = oldObservabilityConfigFile + }() + if err := Start(context.Background()); err == nil { + t.Fatalf("Invalid file system path not triggering error") + } +} + +func (s) TestNoEnvSet(t *testing.T) { + oldObservabilityConfig := envconfig.ObservabilityConfig + oldObservabilityConfigFile := envconfig.ObservabilityConfigFile + envconfig.ObservabilityConfig = "" + envconfig.ObservabilityConfigFile = "" + defer func() { + envconfig.ObservabilityConfig = oldObservabilityConfig + envconfig.ObservabilityConfigFile = oldObservabilityConfigFile + }() + // If there is no observability config set at all, the Start should return an error. + if err := Start(context.Background()); err == nil { + t.Fatalf("Invalid patterns not triggering error") + } +} + +func (s) TestOpenCensusIntegration(t *testing.T) { + defaultMetricsReportingInterval = time.Millisecond * 100 + fe := &fakeOpenCensusExporter{SeenViews: make(map[string]string), t: t} + + defer func(ne func(config *config) (tracingMetricsExporter, error)) { + newExporter = ne + }(newExporter) + + newExporter = func(config *config) (tracingMetricsExporter, error) { + return fe, nil + } + + openCensusOnConfig := &config{ + ProjectID: "fake", + CloudMonitoring: &cloudMonitoring{}, + CloudTrace: &cloudTrace{ + SamplingRate: 1.0, + }, + } + cleanup, err := setupObservabilitySystemWithConfig(openCensusOnConfig) + if err != nil { + t.Fatalf("error setting up observability %v", err) + } + defer cleanup() + + ss := &stubserver.StubServer{ + UnaryCallF: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{}, nil + }, + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + for { + _, err := stream.Recv() + if err == io.EOF { + return nil + } + } + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + for i := 0; i < defaultRequestCount; i++ { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{Payload: &testpb.Payload{Body: testOkPayload}}); err != nil { + t.Fatalf("Unexpected error from UnaryCall: %v", err) + } + } + t.Logf("unary call passed count=%v", defaultRequestCount) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + stream, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("ss.Client.FullDuplexCall failed: %f", err) + } + + stream.CloseSend() + if _, err = stream.Recv(); err != io.EOF { + t.Fatalf("unexpected error: %v, expected an EOF error", err) + } + + var errs []error + for ctx.Err() == nil { + errs = nil + fe.mu.RLock() + if value := fe.SeenViews["grpc.io/client/api_latency"]; value != TypeOpenCensusViewDistribution { + errs = append(errs, fmt.Errorf("unexpected type for grpc.io/client/api_latency: %s != %s", value, TypeOpenCensusViewDistribution)) + } + if value := fe.SeenViews["grpc.io/client/started_rpcs"]; value != TypeOpenCensusViewCount { + errs = append(errs, fmt.Errorf("unexpected type for grpc.io/client/started_rpcs: %s != %s", value, TypeOpenCensusViewCount)) + } + if value := fe.SeenViews["grpc.io/server/started_rpcs"]; value != TypeOpenCensusViewCount { + errs = append(errs, fmt.Errorf("unexpected type for grpc.io/server/started_rpcs: %s != %s", value, TypeOpenCensusViewCount)) + } + + if value := fe.SeenViews["grpc.io/client/completed_rpcs"]; value != TypeOpenCensusViewCount { + errs = append(errs, fmt.Errorf("unexpected type for grpc.io/client/completed_rpcs: %s != %s", value, TypeOpenCensusViewCount)) + } + if value := fe.SeenViews["grpc.io/server/completed_rpcs"]; value != TypeOpenCensusViewCount { + errs = append(errs, fmt.Errorf("unexpected type for grpc.io/server/completed_rpcs: %s != %s", value, TypeOpenCensusViewCount)) + } + if value := fe.SeenViews["grpc.io/client/roundtrip_latency"]; value != TypeOpenCensusViewDistribution { + errs = append(errs, fmt.Errorf("unexpected type for grpc.io/client/completed_rpcs: %s != %s", value, TypeOpenCensusViewDistribution)) + } + if value := fe.SeenViews["grpc.io/server/server_latency"]; value != TypeOpenCensusViewDistribution { + errs = append(errs, fmt.Errorf("grpc.io/server/server_latency: %s != %s", value, TypeOpenCensusViewDistribution)) + } + if value := fe.SeenViews["grpc.io/client/sent_compressed_message_bytes_per_rpc"]; value != TypeOpenCensusViewDistribution { + errs = append(errs, fmt.Errorf("unexpected type for grpc.io/client/sent_compressed_message_bytes_per_rpc: %s != %s", value, TypeOpenCensusViewDistribution)) + } + if value := fe.SeenViews["grpc.io/client/received_compressed_message_bytes_per_rpc"]; value != TypeOpenCensusViewDistribution { + errs = append(errs, fmt.Errorf("unexpected type for grpc.io/client/received_compressed_message_bytes_per_rpc: %s != %s", value, TypeOpenCensusViewDistribution)) + } + if value := fe.SeenViews["grpc.io/server/sent_compressed_message_bytes_per_rpc"]; value != TypeOpenCensusViewDistribution { + errs = append(errs, fmt.Errorf("unexpected type for grpc.io/server/sent_compressed_message_bytes_per_rpc: %s != %s", value, TypeOpenCensusViewDistribution)) + } + if value := fe.SeenViews["grpc.io/server/received_compressed_message_bytes_per_rpc"]; value != TypeOpenCensusViewDistribution { + errs = append(errs, fmt.Errorf("unexpected type for grpc.io/server/received_compressed_message_bytes_per_rpc: %s != %s", value, TypeOpenCensusViewDistribution)) + } + if fe.SeenSpans <= 0 { + errs = append(errs, fmt.Errorf("unexpected number of seen spans: %v <= 0", fe.SeenSpans)) + } + fe.mu.RUnlock() + if len(errs) == 0 { + break + } + time.Sleep(100 * time.Millisecond) + } + if len(errs) != 0 { + t.Fatalf("Invalid OpenCensus export data: %v", errs) + } +} + +// TestCustomTagsTracingMetrics verifies that the custom tags defined in our +// observability configuration and set to two hardcoded values are passed to the +// function to create an exporter. +func (s) TestCustomTagsTracingMetrics(t *testing.T) { + defer func(ne func(config *config) (tracingMetricsExporter, error)) { + newExporter = ne + }(newExporter) + fe := &fakeOpenCensusExporter{SeenViews: make(map[string]string), t: t} + newExporter = func(config *config) (tracingMetricsExporter, error) { + ct := config.Labels + if len(ct) < 1 { + t.Fatalf("less than 2 custom tags sent in") + } + if val, ok := ct["customtag1"]; !ok || val != "wow" { + t.Fatalf("incorrect custom tag: got %v, want %v", val, "wow") + } + if val, ok := ct["customtag2"]; !ok || val != "nice" { + t.Fatalf("incorrect custom tag: got %v, want %v", val, "nice") + } + return fe, nil + } + + // This configuration present in file system and it's defined custom tags should make it + // to the created exporter. + configJSON := `{ + "project_id": "fake", + "cloud_trace": {}, + "cloud_monitoring": {"sampling_rate": 1.0}, + "labels":{"customtag1":"wow","customtag2":"nice"} + }` + + cleanup, err := createTmpConfigInFileSystem(configJSON) + defer cleanup() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + err = Start(ctx) + defer End() + if err != nil { + t.Fatalf("Start() failed with err: %v", err) + } +} + +// TestStartErrorsThenEnd tests that an End call after Start errors works +// without problems, as this is a possible codepath in the public observability +// API. +func (s) TestStartErrorsThenEnd(t *testing.T) { + invalidConfig := &config{ + ProjectID: "fake", + CloudLogging: &cloudLogging{ + ClientRPCEvents: []clientRPCEvents{ + { + Methods: []string{":-)"}, + MaxMetadataBytes: 30, + MaxMessageBytes: 30, + }, + }, + }, + } + invalidConfigJSON, err := json.Marshal(invalidConfig) + if err != nil { + t.Fatalf("failed to convert config to JSON: %v", err) + } + oldObservabilityConfig := envconfig.ObservabilityConfig + oldObservabilityConfigFile := envconfig.ObservabilityConfigFile + envconfig.ObservabilityConfig = string(invalidConfigJSON) + envconfig.ObservabilityConfigFile = "" + defer func() { + envconfig.ObservabilityConfig = oldObservabilityConfig + envconfig.ObservabilityConfigFile = oldObservabilityConfigFile + }() + if err := Start(context.Background()); err == nil { + t.Fatalf("Invalid patterns not triggering error") + } + End() +} + +// TestLoggingLinkedWithTraceClientSide tests that client side logs get the +// trace and span id corresponding to the created Call Level Span for the RPC. +func (s) TestLoggingLinkedWithTraceClientSide(t *testing.T) { + fle := &fakeLoggingExporter{ + t: t, + } + oldNewLoggingExporter := newLoggingExporter + defer func() { + newLoggingExporter = oldNewLoggingExporter + }() + + newLoggingExporter = func(ctx context.Context, config *config) (loggingExporter, error) { + return fle, nil + } + + idCh := testutils.NewChannel() + + fe := &fakeOpenCensusExporter{ + t: t, + idCh: idCh, + } + oldNewExporter := newExporter + defer func() { + newExporter = oldNewExporter + }() + + newExporter = func(config *config) (tracingMetricsExporter, error) { + return fe, nil + } + + const projectID = "project-id" + tracesAndLogsConfig := &config{ + ProjectID: projectID, + CloudLogging: &cloudLogging{ + ClientRPCEvents: []clientRPCEvents{ + { + Methods: []string{"*"}, + MaxMetadataBytes: 30, + MaxMessageBytes: 30, + }, + }, + }, + CloudTrace: &cloudTrace{ + SamplingRate: 1.0, + }, + } + cleanup, err := setupObservabilitySystemWithConfig(tracesAndLogsConfig) + if err != nil { + t.Fatalf("error setting up observability %v", err) + } + defer cleanup() + ss := &stubserver.StubServer{ + UnaryCallF: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{}, nil + }, + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + _, err := stream.Recv() + if err != io.EOF { + return err + } + return nil + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Spawn a goroutine to receive the trace and span ids received by the + // exporter corresponding to a Unary RPC. + readerErrCh := testutils.NewChannel() + unaryDone := grpcsync.NewEvent() + go func() { + var traceAndSpanIDs []traceAndSpanID + val, err := idCh.Receive(ctx) + if err != nil { + readerErrCh.Send(fmt.Errorf("error while waiting for IDs: %v", err)) + } + + tasi, ok := val.(traceAndSpanID) + if !ok { + readerErrCh.Send(fmt.Errorf("received wrong type from channel: %T", val)) + } + traceAndSpanIDs = append(traceAndSpanIDs, tasi) + + val, err = idCh.Receive(ctx) + if err != nil { + readerErrCh.Send(fmt.Errorf("error while waiting for IDs: %v", err)) + } + + tasi, ok = val.(traceAndSpanID) + if !ok { + readerErrCh.Send(fmt.Errorf("received wrong type from channel: %T", val)) + } + traceAndSpanIDs = append(traceAndSpanIDs, tasi) + + val, err = idCh.Receive(ctx) + if err != nil { + readerErrCh.Send(fmt.Errorf("error while waiting for IDs: %v", err)) + } + tasi, ok = val.(traceAndSpanID) + if !ok { + readerErrCh.Send(fmt.Errorf("received wrong type from channel: %T", val)) + } + traceAndSpanIDs = append(traceAndSpanIDs, tasi) + <-unaryDone.Done() + var tasiSent traceAndSpanIDString + for _, tasi := range traceAndSpanIDs { + if strings.HasPrefix(tasi.spanName, "grpc.") && tasi.spanKind == trace.SpanKindClient { + tasiSent = tasi.idsToString(projectID) + continue + } + } + + fle.mu.Lock() + for _, tasiSeen := range fle.idsSeen { + if diff := cmp.Diff(tasiSeen, &tasiSent, cmp.AllowUnexported(traceAndSpanIDString{}), cmpopts.IgnoreFields(traceAndSpanIDString{}, "SpanKind")); diff != "" { + readerErrCh.Send(fmt.Errorf("got unexpected id, should be a client span (-got, +want): %v", diff)) + } + } + + fle.entries = nil + fle.mu.Unlock() + readerErrCh.Send(nil) + }() + if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{Payload: &testpb.Payload{Body: testOkPayload}}); err != nil { + t.Fatalf("Unexpected error from UnaryCall: %v", err) + } + unaryDone.Fire() + if chErr, err := readerErrCh.Receive(ctx); chErr != nil || err != nil { + if err != nil { + t.Fatalf("Should have received something from error channel: %v", err) + } + if chErr != nil { + t.Fatalf("Should have received a nil error from channel, instead received: %v", chErr) + } + } +} + +// TestLoggingLinkedWithTraceServerSide tests that server side logs get the +// trace and span id corresponding to the created Server Span for the RPC. +func (s) TestLoggingLinkedWithTraceServerSide(t *testing.T) { + fle := &fakeLoggingExporter{ + t: t, + } + oldNewLoggingExporter := newLoggingExporter + defer func() { + newLoggingExporter = oldNewLoggingExporter + }() + + newLoggingExporter = func(ctx context.Context, config *config) (loggingExporter, error) { + return fle, nil + } + + idCh := testutils.NewChannel() + + fe := &fakeOpenCensusExporter{ + t: t, + idCh: idCh, + } + oldNewExporter := newExporter + defer func() { + newExporter = oldNewExporter + }() + + newExporter = func(config *config) (tracingMetricsExporter, error) { + return fe, nil + } + + const projectID = "project-id" + tracesAndLogsConfig := &config{ + ProjectID: projectID, + CloudLogging: &cloudLogging{ + ServerRPCEvents: []serverRPCEvents{ + { + Methods: []string{"*"}, + MaxMetadataBytes: 30, + MaxMessageBytes: 30, + }, + }, + }, + CloudTrace: &cloudTrace{ + SamplingRate: 1.0, + }, + } + cleanup, err := setupObservabilitySystemWithConfig(tracesAndLogsConfig) + if err != nil { + t.Fatalf("error setting up observability %v", err) + } + defer cleanup() + ss := &stubserver.StubServer{ + UnaryCallF: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{}, nil + }, + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + _, err := stream.Recv() + if err != io.EOF { + return err + } + return nil + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Spawn a goroutine to receive the trace and span ids received by the + // exporter corresponding to a Unary RPC. + readerErrCh := testutils.NewChannel() + unaryDone := grpcsync.NewEvent() + go func() { + var traceAndSpanIDs []traceAndSpanID + val, err := idCh.Receive(ctx) + if err != nil { + readerErrCh.Send(fmt.Errorf("error while waiting for IDs: %v", err)) + } + + tasi, ok := val.(traceAndSpanID) + if !ok { + readerErrCh.Send(fmt.Errorf("received wrong type from channel: %T", val)) + } + traceAndSpanIDs = append(traceAndSpanIDs, tasi) + + val, err = idCh.Receive(ctx) + if err != nil { + readerErrCh.Send(fmt.Errorf("error while waiting for IDs: %v", err)) + } + + tasi, ok = val.(traceAndSpanID) + if !ok { + readerErrCh.Send(fmt.Errorf("received wrong type from channel: %T", val)) + } + traceAndSpanIDs = append(traceAndSpanIDs, tasi) + + val, err = idCh.Receive(ctx) + if err != nil { + readerErrCh.Send(fmt.Errorf("error while waiting for IDs: %v", err)) + } + tasi, ok = val.(traceAndSpanID) + if !ok { + readerErrCh.Send(fmt.Errorf("received wrong type from channel: %T", val)) + } + traceAndSpanIDs = append(traceAndSpanIDs, tasi) + <-unaryDone.Done() + var tasiServer traceAndSpanIDString + for _, tasi := range traceAndSpanIDs { + if strings.HasPrefix(tasi.spanName, "grpc.") && tasi.spanKind == trace.SpanKindServer { + tasiServer = tasi.idsToString(projectID) + continue + } + } + + fle.mu.Lock() + for _, tasiSeen := range fle.idsSeen { + if diff := cmp.Diff(tasiSeen, &tasiServer, cmp.AllowUnexported(traceAndSpanIDString{}), cmpopts.IgnoreFields(traceAndSpanIDString{}, "SpanKind")); diff != "" { + readerErrCh.Send(fmt.Errorf("got unexpected id, should be a server span (-got, +want): %v", diff)) + } + } + + fle.entries = nil + fle.mu.Unlock() + readerErrCh.Send(nil) + }() + if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{Payload: &testpb.Payload{Body: testOkPayload}}); err != nil { + t.Fatalf("Unexpected error from UnaryCall: %v", err) + } + unaryDone.Fire() + if chErr, err := readerErrCh.Receive(ctx); chErr != nil || err != nil { + if err != nil { + t.Fatalf("Should have received something from error channel: %v", err) + } + if chErr != nil { + t.Fatalf("Should have received a nil error from channel, instead received: %v", chErr) + } + } +} + +// TestLoggingLinkedWithTrace tests that client and server side logs get the +// trace and span id corresponding to either the Call Level Span or Server Span +// (no determinism, so can only assert one or the other), for Unary and +// Streaming RPCs. +func (s) TestLoggingLinkedWithTrace(t *testing.T) { + fle := &fakeLoggingExporter{ + t: t, + } + oldNewLoggingExporter := newLoggingExporter + defer func() { + newLoggingExporter = oldNewLoggingExporter + }() + + newLoggingExporter = func(ctx context.Context, config *config) (loggingExporter, error) { + return fle, nil + } + + idCh := testutils.NewChannel() + + fe := &fakeOpenCensusExporter{ + t: t, + idCh: idCh, + } + oldNewExporter := newExporter + defer func() { + newExporter = oldNewExporter + }() + + newExporter = func(config *config) (tracingMetricsExporter, error) { + return fe, nil + } + + const projectID = "project-id" + tracesAndLogsConfig := &config{ + ProjectID: projectID, + CloudLogging: &cloudLogging{ + ClientRPCEvents: []clientRPCEvents{ + { + Methods: []string{"*"}, + MaxMetadataBytes: 30, + MaxMessageBytes: 30, + }, + }, + ServerRPCEvents: []serverRPCEvents{ + { + Methods: []string{"*"}, + MaxMetadataBytes: 30, + MaxMessageBytes: 30, + }, + }, + }, + CloudTrace: &cloudTrace{ + SamplingRate: 1.0, + }, + } + cleanup, err := setupObservabilitySystemWithConfig(tracesAndLogsConfig) + if err != nil { + t.Fatalf("error setting up observability %v", err) + } + defer cleanup() + ss := &stubserver.StubServer{ + UnaryCallF: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{}, nil + }, + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + _, err := stream.Recv() + if err != io.EOF { + return err + } + return nil + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Spawn a goroutine to receive the trace and span ids received by the + // exporter corresponding to a Unary RPC. + readerErrCh := testutils.NewChannel() + unaryDone := grpcsync.NewEvent() + go func() { + var traceAndSpanIDs []traceAndSpanID + val, err := idCh.Receive(ctx) + if err != nil { + readerErrCh.Send(fmt.Errorf("error while waiting for IDs: %v", err)) + } + + tasi, ok := val.(traceAndSpanID) + if !ok { + readerErrCh.Send(fmt.Errorf("received wrong type from channel: %T", val)) + } + traceAndSpanIDs = append(traceAndSpanIDs, tasi) + + val, err = idCh.Receive(ctx) + if err != nil { + readerErrCh.Send(fmt.Errorf("error while waiting for IDs: %v", err)) + } + + tasi, ok = val.(traceAndSpanID) + if !ok { + readerErrCh.Send(fmt.Errorf("received wrong type from channel: %T", val)) + } + traceAndSpanIDs = append(traceAndSpanIDs, tasi) + + val, err = idCh.Receive(ctx) + if err != nil { + readerErrCh.Send(fmt.Errorf("error while waiting for IDs: %v", err)) + } + tasi, ok = val.(traceAndSpanID) + if !ok { + readerErrCh.Send(fmt.Errorf("received wrong type from channel: %T", val)) + } + traceAndSpanIDs = append(traceAndSpanIDs, tasi) + <-unaryDone.Done() + var tasiSent traceAndSpanIDString + var tasiServer traceAndSpanIDString + for _, tasi := range traceAndSpanIDs { + if strings.HasPrefix(tasi.spanName, "grpc.") && tasi.spanKind == trace.SpanKindClient { + tasiSent = tasi.idsToString(projectID) + continue + } + if strings.HasPrefix(tasi.spanName, "grpc.") && tasi.spanKind == trace.SpanKindServer { + tasiServer = tasi.idsToString(projectID) + } + } + + fle.mu.Lock() + for _, tasiSeen := range fle.idsSeen { + if diff := cmp.Diff(tasiSeen, &tasiSent, cmp.AllowUnexported(traceAndSpanIDString{}), cmpopts.IgnoreFields(traceAndSpanIDString{}, "SpanKind")); diff != "" { + if diff2 := cmp.Diff(tasiSeen, &tasiServer, cmp.AllowUnexported(traceAndSpanIDString{}), cmpopts.IgnoreFields(traceAndSpanIDString{}, "SpanKind")); diff2 != "" { + readerErrCh.Send(fmt.Errorf("got unexpected id, should be a client or server span (-got, +want): %v, %v", diff, diff2)) + } + } + } + + fle.entries = nil + fle.mu.Unlock() + readerErrCh.Send(nil) + }() + if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{Payload: &testpb.Payload{Body: testOkPayload}}); err != nil { + t.Fatalf("Unexpected error from UnaryCall: %v", err) + } + unaryDone.Fire() + if chErr, err := readerErrCh.Receive(ctx); chErr != nil || err != nil { + if err != nil { + t.Fatalf("Should have received something from error channel: %v", err) + } + if chErr != nil { + t.Fatalf("Should have received a nil error from channel, instead received: %v", chErr) + } + } + + fle.mu.Lock() + fle.idsSeen = nil + fle.mu.Unlock() + + // Test streaming. Spawn a goroutine to receive the trace and span ids + // received by the exporter corresponding to a streaming RPC. + readerErrCh = testutils.NewChannel() + streamDone := grpcsync.NewEvent() + go func() { + var traceAndSpanIDs []traceAndSpanID + + val, err := idCh.Receive(ctx) + if err != nil { + readerErrCh.Send(fmt.Errorf("error while waiting for IDs: %v", err)) + } + + tasi, ok := val.(traceAndSpanID) + if !ok { + readerErrCh.Send(fmt.Errorf("received wrong type from channel: %T", val)) + } + traceAndSpanIDs = append(traceAndSpanIDs, tasi) + val, err = idCh.Receive(ctx) + if err != nil { + readerErrCh.Send(fmt.Errorf("error while waiting for IDs: %v", err)) + } + + tasi, ok = val.(traceAndSpanID) + if !ok { + readerErrCh.Send(fmt.Errorf("received wrong type from channel: %T", val)) + } + traceAndSpanIDs = append(traceAndSpanIDs, tasi) + + val, err = idCh.Receive(ctx) + if err != nil { + readerErrCh.Send(fmt.Errorf("error while waiting for IDs: %v", err)) + } + tasi, ok = val.(traceAndSpanID) + if !ok { + readerErrCh.Send(fmt.Errorf("received wrong type from channel: %T", val)) + } + traceAndSpanIDs = append(traceAndSpanIDs, tasi) + <-streamDone.Done() + var tasiSent traceAndSpanIDString + var tasiServer traceAndSpanIDString + for _, tasi := range traceAndSpanIDs { + if strings.HasPrefix(tasi.spanName, "grpc.") && tasi.spanKind == trace.SpanKindClient { + tasiSent = tasi.idsToString(projectID) + continue + } + if strings.HasPrefix(tasi.spanName, "grpc.") && tasi.spanKind == trace.SpanKindServer { + tasiServer = tasi.idsToString(projectID) + } + } + + fle.mu.Lock() + for _, tasiSeen := range fle.idsSeen { + if diff := cmp.Diff(tasiSeen, &tasiSent, cmp.AllowUnexported(traceAndSpanIDString{}), cmpopts.IgnoreFields(traceAndSpanIDString{}, "SpanKind")); diff != "" { + if diff2 := cmp.Diff(tasiSeen, &tasiServer, cmp.AllowUnexported(traceAndSpanIDString{}), cmpopts.IgnoreFields(traceAndSpanIDString{}, "SpanKind")); diff2 != "" { + readerErrCh.Send(fmt.Errorf("got unexpected id, should be a client or server span (-got, +want): %v, %v", diff, diff2)) + } + } + } + + fle.entries = nil + fle.mu.Unlock() + readerErrCh.Send(nil) + }() + + stream, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("ss.Client.FullDuplexCall failed: %f", err) + } + + stream.CloseSend() + if _, err = stream.Recv(); err != io.EOF { + t.Fatalf("unexpected error: %v, expected an EOF error", err) + } + streamDone.Fire() + + if chErr, err := readerErrCh.Receive(ctx); chErr != nil || err != nil { + if err != nil { + t.Fatalf("Should have received something from error channel: %v", err) + } + if chErr != nil { + t.Fatalf("Should have received a nil error from channel, instead received: %v", chErr) + } + } +} diff --git a/gcp/observability/opencensus.go b/gcp/observability/opencensus.go new file mode 100644 index 000000000000..9b262e4ad278 --- /dev/null +++ b/gcp/observability/opencensus.go @@ -0,0 +1,179 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package observability + +import ( + "fmt" + "os" + "strconv" + "time" + + "contrib.go.opencensus.io/exporter/stackdriver" + "contrib.go.opencensus.io/exporter/stackdriver/monitoredresource" + + "go.opencensus.io/stats/view" + "go.opencensus.io/trace" + "google.golang.org/grpc" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/stats/opencensus" +) + +var ( + // It's a variable instead of const to speed up testing + defaultMetricsReportingInterval = time.Second * 30 + defaultViews = []*view.View{ + opencensus.ClientStartedRPCsView, + opencensus.ClientCompletedRPCsView, + opencensus.ClientRoundtripLatencyView, + opencensus.ClientSentCompressedMessageBytesPerRPCView, + opencensus.ClientReceivedCompressedMessageBytesPerRPCView, + opencensus.ClientAPILatencyView, + opencensus.ServerStartedRPCsView, + opencensus.ServerCompletedRPCsView, + opencensus.ServerSentCompressedMessageBytesPerRPCView, + opencensus.ServerReceivedCompressedMessageBytesPerRPCView, + opencensus.ServerLatencyView, + } +) + +func labelsToMonitoringLabels(labels map[string]string) *stackdriver.Labels { + sdLabels := &stackdriver.Labels{} + for k, v := range labels { + sdLabels.Set(k, v, "") + } + return sdLabels +} + +func labelsToTraceAttributes(labels map[string]string) map[string]any { + ta := make(map[string]any, len(labels)) + for k, v := range labels { + ta[k] = v + } + return ta +} + +type tracingMetricsExporter interface { + trace.Exporter + view.Exporter + Flush() + Close() error +} + +var exporter tracingMetricsExporter + +var newExporter = newStackdriverExporter + +func newStackdriverExporter(config *config) (tracingMetricsExporter, error) { + // Create the Stackdriver exporter, which is shared between tracing and stats + mr := monitoredresource.Autodetect() + logger.Infof("Detected MonitoredResource:: %+v", mr) + var err error + // Custom labels completly overwrite any labels generated in the OpenCensus + // library, including their label that uniquely identifies the process. + // Thus, generate a unique process identifier here to uniquely identify + // process for metrics exporting to function correctly. + metricsLabels := make(map[string]string, len(config.Labels)+1) + for k, v := range config.Labels { + metricsLabels[k] = v + } + metricsLabels["opencensus_task"] = generateUniqueProcessIdentifier() + exporter, err := stackdriver.NewExporter(stackdriver.Options{ + ProjectID: config.ProjectID, + MonitoredResource: mr, + DefaultMonitoringLabels: labelsToMonitoringLabels(metricsLabels), + DefaultTraceAttributes: labelsToTraceAttributes(config.Labels), + MonitoringClientOptions: cOptsDisableLogTrace, + TraceClientOptions: cOptsDisableLogTrace, + }) + if err != nil { + return nil, fmt.Errorf("failed to create Stackdriver exporter: %v", err) + } + return exporter, nil +} + +// generateUniqueProcessIdentifier returns a unique process identifier for the +// process this code is running in. This is the same way the OpenCensus library +// generates the unique process identifier, in the format of +// "go-@". +func generateUniqueProcessIdentifier() string { + hostname, err := os.Hostname() + if err != nil { + hostname = "localhost" + } + return "go-" + strconv.Itoa(os.Getpid()) + "@" + hostname +} + +// This method accepts config and exporter; the exporter argument is exposed to +// assist unit testing of the OpenCensus behavior. +func startOpenCensus(config *config) error { + // If both tracing and metrics are disabled, there's no point inject default + // StatsHandler. + if config == nil || (config.CloudTrace == nil && config.CloudMonitoring == nil) { + return nil + } + + var err error + exporter, err = newExporter(config) + if err != nil { + return err + } + + var to opencensus.TraceOptions + if config.CloudTrace != nil { + to.TS = trace.ProbabilitySampler(config.CloudTrace.SamplingRate) + trace.RegisterExporter(exporter.(trace.Exporter)) + logger.Infof("Start collecting and exporting trace spans with global_trace_sampling_rate=%.2f", config.CloudTrace.SamplingRate) + } + + if config.CloudMonitoring != nil { + if err := view.Register(defaultViews...); err != nil { + return fmt.Errorf("failed to register observability views: %v", err) + } + view.SetReportingPeriod(defaultMetricsReportingInterval) + view.RegisterExporter(exporter.(view.Exporter)) + logger.Infof("Start collecting and exporting metrics") + } + + internal.AddGlobalServerOptions.(func(opt ...grpc.ServerOption))(opencensus.ServerOption(to)) + internal.AddGlobalDialOptions.(func(opt ...grpc.DialOption))(opencensus.DialOption(to)) + logger.Infof("Enabled OpenCensus StatsHandlers for clients and servers") + + return nil +} + +// stopOpenCensus flushes the exporter's and cleans up globals across all +// packages if exporter was created. +func stopOpenCensus() { + if exporter != nil { + internal.ClearGlobalDialOptions() + internal.ClearGlobalServerOptions() + // This Unregister call guarantees the data recorded gets sent to + // exporter, synchronising the view package and exporter. Doesn't matter + // if views not registered, will be a noop if not registered. + view.Unregister(defaultViews...) + // Call these unconditionally, doesn't matter if not registered, will be + // a noop if not registered. + trace.UnregisterExporter(exporter) + view.UnregisterExporter(exporter) + + // This Flush call makes sure recorded telemetry get sent to backend. + exporter.Flush() + exporter.Close() + } +} diff --git a/go.mod b/go.mod index 31f2b01f64e8..5e3bd51e4ac0 100644 --- a/go.mod +++ b/go.mod @@ -1,15 +1,31 @@ module google.golang.org/grpc -go 1.11 +go 1.19 require ( - github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f - github.com/envoyproxy/go-control-plane v0.9.4 - github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b - github.com/golang/protobuf v1.3.3 - github.com/google/go-cmp v0.4.0 - golang.org/x/net v0.0.0-20190311183353-d8887717615a - golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be - golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a - google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 + github.com/cespare/xxhash/v2 v2.2.0 + github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe + github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 + github.com/envoyproxy/go-control-plane v0.11.1 + github.com/golang/glog v1.1.0 + github.com/golang/protobuf v1.5.3 + github.com/google/go-cmp v0.5.9 + github.com/google/uuid v1.3.0 + golang.org/x/net v0.12.0 + golang.org/x/oauth2 v0.10.0 + golang.org/x/sync v0.3.0 + golang.org/x/sys v0.10.0 + google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 + google.golang.org/protobuf v1.31.0 +) + +require ( + cloud.google.com/go/compute v1.21.0 // indirect + cloud.google.com/go/compute/metadata v0.2.3 // indirect + github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect + github.com/envoyproxy/protoc-gen-validate v1.0.2 // indirect + golang.org/x/text v0.11.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 // indirect ) diff --git a/go.sum b/go.sum index be8078eace22..808537d01262 100644 --- a/go.sum +++ b/go.sum @@ -1,68 +1,99 @@ -cloud.google.com/go v0.26.0 h1:e0WKqKTd5BnrG8aKH3J3h+QvEIQtSUcf2n5UZ5ZgLtQ= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +cloud.google.com/go/compute v1.21.0 h1:JNBsyXVoOoNJtTQcnEY5uYpZIbeCTYIeDe0Xh1bySMk= +cloud.google.com/go/compute v1.21.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI= +github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g= +github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f h1:WBZRG4aNOuI15bLRrCgN8fCq8E5Xuty6jGbmSNEvSsU= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe h1:QQ3GSy+MqSHxm/d8nCtnAiZdYFd45cYZPs8vOOIYKfk= +github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= +github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4 h1:rEvIZUSZ3fx39WIi3JkQqQBitGwpELBIYWeBVh6wn+E= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/protoc-gen-validate v0.1.0 h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A= +github.com/envoyproxy/go-control-plane v0.11.1 h1:wSUXTlLfiAQRWs2F+p+EKOY9rUyis1MyGqJ2DIk5HpM= +github.com/envoyproxy/go-control-plane v0.11.1/go.mod h1:uhMcXKCQMEJHiAb0w+YGefQLaTEw+YhGluxZkrTmD0g= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= +github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA= +github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/mock v1.1.1 h1:G5FRp8JnTd7RQH5kemVNlMeyXQAztQ3mOWV95KxsXH8= +github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= +github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 h1:XQyxROzUlZH+WIQwySDgnISgOivlhjIEwaQaJEJrrN0= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= +golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= +golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135 h1:5Beo0mZN8dRzgrMMkDp0jc8YXQKx9DiJ2k1dkvGsn5A= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 h1:Z0hjGZePRE0ZBWotvtrwxFNrNE9CUAGtplaDK5NNI/g= +google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98/go.mod h1:S7mY02OqCJTD0E1OiQy1F72PWFB4bZJ87cAtLPYgDR0= +google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 h1:FmF5cCW94Ij59cfpoLiwTgodWmm60eEV0CjlsVg2fuw= +google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 h1:bVf09lpb+OJbByTj913DRJioFFAjf/ZGxEz7MajTp2U= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc h1:/hemPrYIhOhy8zYrNj+069zDB68us2sMGsfkFJO0iZs= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/grpclog/component.go b/grpclog/component.go index 8358dd6e2abb..ac73c9ced255 100644 --- a/grpclog/component.go +++ b/grpclog/component.go @@ -31,71 +31,71 @@ type componentData struct { var cache = map[string]*componentData{} -func (c *componentData) InfoDepth(depth int, args ...interface{}) { - args = append([]interface{}{"[" + string(c.name) + "]"}, args...) +func (c *componentData) InfoDepth(depth int, args ...any) { + args = append([]any{"[" + string(c.name) + "]"}, args...) grpclog.InfoDepth(depth+1, args...) } -func (c *componentData) WarningDepth(depth int, args ...interface{}) { - args = append([]interface{}{"[" + string(c.name) + "]"}, args...) +func (c *componentData) WarningDepth(depth int, args ...any) { + args = append([]any{"[" + string(c.name) + "]"}, args...) grpclog.WarningDepth(depth+1, args...) } -func (c *componentData) ErrorDepth(depth int, args ...interface{}) { - args = append([]interface{}{"[" + string(c.name) + "]"}, args...) +func (c *componentData) ErrorDepth(depth int, args ...any) { + args = append([]any{"[" + string(c.name) + "]"}, args...) grpclog.ErrorDepth(depth+1, args...) } -func (c *componentData) FatalDepth(depth int, args ...interface{}) { - args = append([]interface{}{"[" + string(c.name) + "]"}, args...) +func (c *componentData) FatalDepth(depth int, args ...any) { + args = append([]any{"[" + string(c.name) + "]"}, args...) grpclog.FatalDepth(depth+1, args...) } -func (c *componentData) Info(args ...interface{}) { +func (c *componentData) Info(args ...any) { c.InfoDepth(1, args...) } -func (c *componentData) Warning(args ...interface{}) { +func (c *componentData) Warning(args ...any) { c.WarningDepth(1, args...) } -func (c *componentData) Error(args ...interface{}) { +func (c *componentData) Error(args ...any) { c.ErrorDepth(1, args...) } -func (c *componentData) Fatal(args ...interface{}) { +func (c *componentData) Fatal(args ...any) { c.FatalDepth(1, args...) } -func (c *componentData) Infof(format string, args ...interface{}) { +func (c *componentData) Infof(format string, args ...any) { c.InfoDepth(1, fmt.Sprintf(format, args...)) } -func (c *componentData) Warningf(format string, args ...interface{}) { +func (c *componentData) Warningf(format string, args ...any) { c.WarningDepth(1, fmt.Sprintf(format, args...)) } -func (c *componentData) Errorf(format string, args ...interface{}) { +func (c *componentData) Errorf(format string, args ...any) { c.ErrorDepth(1, fmt.Sprintf(format, args...)) } -func (c *componentData) Fatalf(format string, args ...interface{}) { +func (c *componentData) Fatalf(format string, args ...any) { c.FatalDepth(1, fmt.Sprintf(format, args...)) } -func (c *componentData) Infoln(args ...interface{}) { +func (c *componentData) Infoln(args ...any) { c.InfoDepth(1, args...) } -func (c *componentData) Warningln(args ...interface{}) { +func (c *componentData) Warningln(args ...any) { c.WarningDepth(1, args...) } -func (c *componentData) Errorln(args ...interface{}) { +func (c *componentData) Errorln(args ...any) { c.ErrorDepth(1, args...) } -func (c *componentData) Fatalln(args ...interface{}) { +func (c *componentData) Fatalln(args ...any) { c.FatalDepth(1, args...) } diff --git a/grpclog/glogger/glogger.go b/grpclog/glogger/glogger.go index 4427dc078bb4..fdabb602f55e 100644 --- a/grpclog/glogger/glogger.go +++ b/grpclog/glogger/glogger.go @@ -35,67 +35,67 @@ func init() { type glogger struct{} -func (g *glogger) Info(args ...interface{}) { +func (g *glogger) Info(args ...any) { glog.InfoDepth(d, args...) } -func (g *glogger) Infoln(args ...interface{}) { +func (g *glogger) Infoln(args ...any) { glog.InfoDepth(d, fmt.Sprintln(args...)) } -func (g *glogger) Infof(format string, args ...interface{}) { +func (g *glogger) Infof(format string, args ...any) { glog.InfoDepth(d, fmt.Sprintf(format, args...)) } -func (g *glogger) InfoDepth(depth int, args ...interface{}) { +func (g *glogger) InfoDepth(depth int, args ...any) { glog.InfoDepth(depth+d, args...) } -func (g *glogger) Warning(args ...interface{}) { +func (g *glogger) Warning(args ...any) { glog.WarningDepth(d, args...) } -func (g *glogger) Warningln(args ...interface{}) { +func (g *glogger) Warningln(args ...any) { glog.WarningDepth(d, fmt.Sprintln(args...)) } -func (g *glogger) Warningf(format string, args ...interface{}) { +func (g *glogger) Warningf(format string, args ...any) { glog.WarningDepth(d, fmt.Sprintf(format, args...)) } -func (g *glogger) WarningDepth(depth int, args ...interface{}) { +func (g *glogger) WarningDepth(depth int, args ...any) { glog.WarningDepth(depth+d, args...) } -func (g *glogger) Error(args ...interface{}) { +func (g *glogger) Error(args ...any) { glog.ErrorDepth(d, args...) } -func (g *glogger) Errorln(args ...interface{}) { +func (g *glogger) Errorln(args ...any) { glog.ErrorDepth(d, fmt.Sprintln(args...)) } -func (g *glogger) Errorf(format string, args ...interface{}) { +func (g *glogger) Errorf(format string, args ...any) { glog.ErrorDepth(d, fmt.Sprintf(format, args...)) } -func (g *glogger) ErrorDepth(depth int, args ...interface{}) { +func (g *glogger) ErrorDepth(depth int, args ...any) { glog.ErrorDepth(depth+d, args...) } -func (g *glogger) Fatal(args ...interface{}) { +func (g *glogger) Fatal(args ...any) { glog.FatalDepth(d, args...) } -func (g *glogger) Fatalln(args ...interface{}) { +func (g *glogger) Fatalln(args ...any) { glog.FatalDepth(d, fmt.Sprintln(args...)) } -func (g *glogger) Fatalf(format string, args ...interface{}) { +func (g *glogger) Fatalf(format string, args ...any) { glog.FatalDepth(d, fmt.Sprintf(format, args...)) } -func (g *glogger) FatalDepth(depth int, args ...interface{}) { +func (g *glogger) FatalDepth(depth int, args ...any) { glog.FatalDepth(depth+d, args...) } diff --git a/grpclog/grpclog.go b/grpclog/grpclog.go index c8bb2be34bf5..16928c9cb993 100644 --- a/grpclog/grpclog.go +++ b/grpclog/grpclog.go @@ -42,53 +42,53 @@ func V(l int) bool { } // Info logs to the INFO log. -func Info(args ...interface{}) { +func Info(args ...any) { grpclog.Logger.Info(args...) } // Infof logs to the INFO log. Arguments are handled in the manner of fmt.Printf. -func Infof(format string, args ...interface{}) { +func Infof(format string, args ...any) { grpclog.Logger.Infof(format, args...) } // Infoln logs to the INFO log. Arguments are handled in the manner of fmt.Println. -func Infoln(args ...interface{}) { +func Infoln(args ...any) { grpclog.Logger.Infoln(args...) } // Warning logs to the WARNING log. -func Warning(args ...interface{}) { +func Warning(args ...any) { grpclog.Logger.Warning(args...) } // Warningf logs to the WARNING log. Arguments are handled in the manner of fmt.Printf. -func Warningf(format string, args ...interface{}) { +func Warningf(format string, args ...any) { grpclog.Logger.Warningf(format, args...) } // Warningln logs to the WARNING log. Arguments are handled in the manner of fmt.Println. -func Warningln(args ...interface{}) { +func Warningln(args ...any) { grpclog.Logger.Warningln(args...) } // Error logs to the ERROR log. -func Error(args ...interface{}) { +func Error(args ...any) { grpclog.Logger.Error(args...) } // Errorf logs to the ERROR log. Arguments are handled in the manner of fmt.Printf. -func Errorf(format string, args ...interface{}) { +func Errorf(format string, args ...any) { grpclog.Logger.Errorf(format, args...) } // Errorln logs to the ERROR log. Arguments are handled in the manner of fmt.Println. -func Errorln(args ...interface{}) { +func Errorln(args ...any) { grpclog.Logger.Errorln(args...) } // Fatal logs to the FATAL log. Arguments are handled in the manner of fmt.Print. // It calls os.Exit() with exit code 1. -func Fatal(args ...interface{}) { +func Fatal(args ...any) { grpclog.Logger.Fatal(args...) // Make sure fatal logs will exit. os.Exit(1) @@ -96,7 +96,7 @@ func Fatal(args ...interface{}) { // Fatalf logs to the FATAL log. Arguments are handled in the manner of fmt.Printf. // It calls os.Exit() with exit code 1. -func Fatalf(format string, args ...interface{}) { +func Fatalf(format string, args ...any) { grpclog.Logger.Fatalf(format, args...) // Make sure fatal logs will exit. os.Exit(1) @@ -104,7 +104,7 @@ func Fatalf(format string, args ...interface{}) { // Fatalln logs to the FATAL log. Arguments are handled in the manner of fmt.Println. // It calle os.Exit()) with exit code 1. -func Fatalln(args ...interface{}) { +func Fatalln(args ...any) { grpclog.Logger.Fatalln(args...) // Make sure fatal logs will exit. os.Exit(1) @@ -113,20 +113,20 @@ func Fatalln(args ...interface{}) { // Print prints to the logger. Arguments are handled in the manner of fmt.Print. // // Deprecated: use Info. -func Print(args ...interface{}) { +func Print(args ...any) { grpclog.Logger.Info(args...) } // Printf prints to the logger. Arguments are handled in the manner of fmt.Printf. // // Deprecated: use Infof. -func Printf(format string, args ...interface{}) { +func Printf(format string, args ...any) { grpclog.Logger.Infof(format, args...) } // Println prints to the logger. Arguments are handled in the manner of fmt.Println. // // Deprecated: use Infoln. -func Println(args ...interface{}) { +func Println(args ...any) { grpclog.Logger.Infoln(args...) } diff --git a/grpclog/logger.go b/grpclog/logger.go index ef06a4822b70..b1674d8267ca 100644 --- a/grpclog/logger.go +++ b/grpclog/logger.go @@ -24,12 +24,12 @@ import "google.golang.org/grpc/internal/grpclog" // // Deprecated: use LoggerV2. type Logger interface { - Fatal(args ...interface{}) - Fatalf(format string, args ...interface{}) - Fatalln(args ...interface{}) - Print(args ...interface{}) - Printf(format string, args ...interface{}) - Println(args ...interface{}) + Fatal(args ...any) + Fatalf(format string, args ...any) + Fatalln(args ...any) + Print(args ...any) + Printf(format string, args ...any) + Println(args ...any) } // SetLogger sets the logger that is used in grpc. Call only from @@ -45,39 +45,39 @@ type loggerWrapper struct { Logger } -func (g *loggerWrapper) Info(args ...interface{}) { +func (g *loggerWrapper) Info(args ...any) { g.Logger.Print(args...) } -func (g *loggerWrapper) Infoln(args ...interface{}) { +func (g *loggerWrapper) Infoln(args ...any) { g.Logger.Println(args...) } -func (g *loggerWrapper) Infof(format string, args ...interface{}) { +func (g *loggerWrapper) Infof(format string, args ...any) { g.Logger.Printf(format, args...) } -func (g *loggerWrapper) Warning(args ...interface{}) { +func (g *loggerWrapper) Warning(args ...any) { g.Logger.Print(args...) } -func (g *loggerWrapper) Warningln(args ...interface{}) { +func (g *loggerWrapper) Warningln(args ...any) { g.Logger.Println(args...) } -func (g *loggerWrapper) Warningf(format string, args ...interface{}) { +func (g *loggerWrapper) Warningf(format string, args ...any) { g.Logger.Printf(format, args...) } -func (g *loggerWrapper) Error(args ...interface{}) { +func (g *loggerWrapper) Error(args ...any) { g.Logger.Print(args...) } -func (g *loggerWrapper) Errorln(args ...interface{}) { +func (g *loggerWrapper) Errorln(args ...any) { g.Logger.Println(args...) } -func (g *loggerWrapper) Errorf(format string, args ...interface{}) { +func (g *loggerWrapper) Errorf(format string, args ...any) { g.Logger.Printf(format, args...) } diff --git a/grpclog/loggerv2.go b/grpclog/loggerv2.go index 8eba2d0e0eff..ecfd36d71303 100644 --- a/grpclog/loggerv2.go +++ b/grpclog/loggerv2.go @@ -19,11 +19,13 @@ package grpclog import ( + "encoding/json" + "fmt" "io" - "io/ioutil" "log" "os" "strconv" + "strings" "google.golang.org/grpc/internal/grpclog" ) @@ -31,35 +33,35 @@ import ( // LoggerV2 does underlying logging work for grpclog. type LoggerV2 interface { // Info logs to INFO log. Arguments are handled in the manner of fmt.Print. - Info(args ...interface{}) + Info(args ...any) // Infoln logs to INFO log. Arguments are handled in the manner of fmt.Println. - Infoln(args ...interface{}) + Infoln(args ...any) // Infof logs to INFO log. Arguments are handled in the manner of fmt.Printf. - Infof(format string, args ...interface{}) + Infof(format string, args ...any) // Warning logs to WARNING log. Arguments are handled in the manner of fmt.Print. - Warning(args ...interface{}) + Warning(args ...any) // Warningln logs to WARNING log. Arguments are handled in the manner of fmt.Println. - Warningln(args ...interface{}) + Warningln(args ...any) // Warningf logs to WARNING log. Arguments are handled in the manner of fmt.Printf. - Warningf(format string, args ...interface{}) + Warningf(format string, args ...any) // Error logs to ERROR log. Arguments are handled in the manner of fmt.Print. - Error(args ...interface{}) + Error(args ...any) // Errorln logs to ERROR log. Arguments are handled in the manner of fmt.Println. - Errorln(args ...interface{}) + Errorln(args ...any) // Errorf logs to ERROR log. Arguments are handled in the manner of fmt.Printf. - Errorf(format string, args ...interface{}) + Errorf(format string, args ...any) // Fatal logs to ERROR log. Arguments are handled in the manner of fmt.Print. // gRPC ensures that all Fatal logs will exit with os.Exit(1). // Implementations may also call os.Exit() with a non-zero exit code. - Fatal(args ...interface{}) + Fatal(args ...any) // Fatalln logs to ERROR log. Arguments are handled in the manner of fmt.Println. // gRPC ensures that all Fatal logs will exit with os.Exit(1). // Implementations may also call os.Exit() with a non-zero exit code. - Fatalln(args ...interface{}) + Fatalln(args ...any) // Fatalf logs to ERROR log. Arguments are handled in the manner of fmt.Printf. // gRPC ensures that all Fatal logs will exit with os.Exit(1). // Implementations may also call os.Exit() with a non-zero exit code. - Fatalf(format string, args ...interface{}) + Fatalf(format string, args ...any) // V reports whether verbosity level l is at least the requested verbose level. V(l int) bool } @@ -95,8 +97,9 @@ var severityName = []string{ // loggerT is the default logger used by grpclog. type loggerT struct { - m []*log.Logger - v int + m []*log.Logger + v int + jsonFormat bool } // NewLoggerV2 creates a loggerV2 with the provided writers. @@ -105,27 +108,40 @@ type loggerT struct { // Warning logs will be written to warningW and infoW. // Info logs will be written to infoW. func NewLoggerV2(infoW, warningW, errorW io.Writer) LoggerV2 { - return NewLoggerV2WithVerbosity(infoW, warningW, errorW, 0) + return newLoggerV2WithConfig(infoW, warningW, errorW, loggerV2Config{}) } // NewLoggerV2WithVerbosity creates a loggerV2 with the provided writers and // verbosity level. func NewLoggerV2WithVerbosity(infoW, warningW, errorW io.Writer, v int) LoggerV2 { + return newLoggerV2WithConfig(infoW, warningW, errorW, loggerV2Config{verbose: v}) +} + +type loggerV2Config struct { + verbose int + jsonFormat bool +} + +func newLoggerV2WithConfig(infoW, warningW, errorW io.Writer, c loggerV2Config) LoggerV2 { var m []*log.Logger - m = append(m, log.New(infoW, severityName[infoLog]+": ", log.LstdFlags)) - m = append(m, log.New(io.MultiWriter(infoW, warningW), severityName[warningLog]+": ", log.LstdFlags)) + flag := log.LstdFlags + if c.jsonFormat { + flag = 0 + } + m = append(m, log.New(infoW, "", flag)) + m = append(m, log.New(io.MultiWriter(infoW, warningW), "", flag)) ew := io.MultiWriter(infoW, warningW, errorW) // ew will be used for error and fatal. - m = append(m, log.New(ew, severityName[errorLog]+": ", log.LstdFlags)) - m = append(m, log.New(ew, severityName[fatalLog]+": ", log.LstdFlags)) - return &loggerT{m: m, v: v} + m = append(m, log.New(ew, "", flag)) + m = append(m, log.New(ew, "", flag)) + return &loggerT{m: m, v: c.verbose, jsonFormat: c.jsonFormat} } // newLoggerV2 creates a loggerV2 to be used as default logger. // All logs are written to stderr. func newLoggerV2() LoggerV2 { - errorW := ioutil.Discard - warningW := ioutil.Discard - infoW := ioutil.Discard + errorW := io.Discard + warningW := io.Discard + infoW := io.Discard logLevel := os.Getenv("GRPC_GO_LOG_SEVERITY_LEVEL") switch logLevel { @@ -142,58 +158,79 @@ func newLoggerV2() LoggerV2 { if vl, err := strconv.Atoi(vLevel); err == nil { v = vl } - return NewLoggerV2WithVerbosity(infoW, warningW, errorW, v) + + jsonFormat := strings.EqualFold(os.Getenv("GRPC_GO_LOG_FORMATTER"), "json") + + return newLoggerV2WithConfig(infoW, warningW, errorW, loggerV2Config{ + verbose: v, + jsonFormat: jsonFormat, + }) } -func (g *loggerT) Info(args ...interface{}) { - g.m[infoLog].Print(args...) +func (g *loggerT) output(severity int, s string) { + sevStr := severityName[severity] + if !g.jsonFormat { + g.m[severity].Output(2, fmt.Sprintf("%v: %v", sevStr, s)) + return + } + // TODO: we can also include the logging component, but that needs more + // (API) changes. + b, _ := json.Marshal(map[string]string{ + "severity": sevStr, + "message": s, + }) + g.m[severity].Output(2, string(b)) } -func (g *loggerT) Infoln(args ...interface{}) { - g.m[infoLog].Println(args...) +func (g *loggerT) Info(args ...any) { + g.output(infoLog, fmt.Sprint(args...)) } -func (g *loggerT) Infof(format string, args ...interface{}) { - g.m[infoLog].Printf(format, args...) +func (g *loggerT) Infoln(args ...any) { + g.output(infoLog, fmt.Sprintln(args...)) } -func (g *loggerT) Warning(args ...interface{}) { - g.m[warningLog].Print(args...) +func (g *loggerT) Infof(format string, args ...any) { + g.output(infoLog, fmt.Sprintf(format, args...)) } -func (g *loggerT) Warningln(args ...interface{}) { - g.m[warningLog].Println(args...) +func (g *loggerT) Warning(args ...any) { + g.output(warningLog, fmt.Sprint(args...)) } -func (g *loggerT) Warningf(format string, args ...interface{}) { - g.m[warningLog].Printf(format, args...) +func (g *loggerT) Warningln(args ...any) { + g.output(warningLog, fmt.Sprintln(args...)) } -func (g *loggerT) Error(args ...interface{}) { - g.m[errorLog].Print(args...) +func (g *loggerT) Warningf(format string, args ...any) { + g.output(warningLog, fmt.Sprintf(format, args...)) } -func (g *loggerT) Errorln(args ...interface{}) { - g.m[errorLog].Println(args...) +func (g *loggerT) Error(args ...any) { + g.output(errorLog, fmt.Sprint(args...)) } -func (g *loggerT) Errorf(format string, args ...interface{}) { - g.m[errorLog].Printf(format, args...) +func (g *loggerT) Errorln(args ...any) { + g.output(errorLog, fmt.Sprintln(args...)) } -func (g *loggerT) Fatal(args ...interface{}) { - g.m[fatalLog].Fatal(args...) - // No need to call os.Exit() again because log.Logger.Fatal() calls os.Exit(). +func (g *loggerT) Errorf(format string, args ...any) { + g.output(errorLog, fmt.Sprintf(format, args...)) } -func (g *loggerT) Fatalln(args ...interface{}) { - g.m[fatalLog].Fatalln(args...) - // No need to call os.Exit() again because log.Logger.Fatal() calls os.Exit(). +func (g *loggerT) Fatal(args ...any) { + g.output(fatalLog, fmt.Sprint(args...)) + os.Exit(1) } -func (g *loggerT) Fatalf(format string, args ...interface{}) { - g.m[fatalLog].Fatalf(format, args...) - // No need to call os.Exit() again because log.Logger.Fatal() calls os.Exit(). +func (g *loggerT) Fatalln(args ...any) { + g.output(fatalLog, fmt.Sprintln(args...)) + os.Exit(1) +} + +func (g *loggerT) Fatalf(format string, args ...any) { + g.output(fatalLog, fmt.Sprintf(format, args...)) + os.Exit(1) } func (g *loggerT) V(l int) bool { @@ -204,15 +241,18 @@ func (g *loggerT) V(l int) bool { // DepthLoggerV2, the below functions will be called with the appropriate stack // depth set for trivial functions the logger may ignore. // -// This API is EXPERIMENTAL. +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. type DepthLoggerV2 interface { LoggerV2 - // InfoDepth logs to INFO log at the specified depth. Arguments are handled in the manner of fmt.Print. - InfoDepth(depth int, args ...interface{}) - // WarningDepth logs to WARNING log at the specified depth. Arguments are handled in the manner of fmt.Print. - WarningDepth(depth int, args ...interface{}) - // ErrorDetph logs to ERROR log at the specified depth. Arguments are handled in the manner of fmt.Print. - ErrorDepth(depth int, args ...interface{}) - // FatalDepth logs to FATAL log at the specified depth. Arguments are handled in the manner of fmt.Print. - FatalDepth(depth int, args ...interface{}) + // InfoDepth logs to INFO log at the specified depth. Arguments are handled in the manner of fmt.Println. + InfoDepth(depth int, args ...any) + // WarningDepth logs to WARNING log at the specified depth. Arguments are handled in the manner of fmt.Println. + WarningDepth(depth int, args ...any) + // ErrorDepth logs to ERROR log at the specified depth. Arguments are handled in the manner of fmt.Println. + ErrorDepth(depth int, args ...any) + // FatalDepth logs to FATAL log at the specified depth. Arguments are handled in the manner of fmt.Println. + FatalDepth(depth int, args ...any) } diff --git a/grpclog/loggerv2_test.go b/grpclog/loggerv2_test.go index 756f215f9c86..119cea4c6ecd 100644 --- a/grpclog/loggerv2_test.go +++ b/grpclog/loggerv2_test.go @@ -52,9 +52,10 @@ func TestLoggerV2Severity(t *testing.T) { } // check if b is in the format of: -// WARNING: 2017/04/07 14:55:42 WARNING +// +// 2017/04/07 14:55:42 WARNING: WARNING func checkLogForSeverity(s int, b []byte) error { - expected := regexp.MustCompile(fmt.Sprintf(`^%s: [0-9]{4}/[0-9]{2}/[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} %s\n$`, severityName[s], severityName[s])) + expected := regexp.MustCompile(fmt.Sprintf(`^[0-9]{4}/[0-9]{2}/[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} %s: %s\n$`, severityName[s], severityName[s])) if m := expected.Match(b); !m { return fmt.Errorf("got: %v, want string in format of: %v", string(b), severityName[s]+": 2016/10/05 17:09:26 "+severityName[s]) } diff --git a/health/client.go b/health/client.go index b5bee4838024..740745c45f63 100644 --- a/health/client.go +++ b/health/client.go @@ -56,7 +56,7 @@ const healthCheckMethod = "/grpc.health.v1.Health/Watch" // This function implements the protocol defined at: // https://github.com/grpc/grpc/blob/master/doc/health-checking.md -func clientHealthCheck(ctx context.Context, newStream func(string) (interface{}, error), setConnectivityState func(connectivity.State, error), service string) error { +func clientHealthCheck(ctx context.Context, newStream func(string) (any, error), setConnectivityState func(connectivity.State, error), service string) error { tryCnt := 0 retryConnection: diff --git a/health/client_test.go b/health/client_test.go index fa218afada72..cc11bd212ba3 100644 --- a/health/client_test.go +++ b/health/client_test.go @@ -28,6 +28,8 @@ import ( "google.golang.org/grpc/connectivity" ) +const defaultTestTimeout = 10 * time.Second + func (s) TestClientHealthCheckBackoff(t *testing.T) { const maxRetries = 5 @@ -37,7 +39,7 @@ func (s) TestClientHealthCheckBackoff(t *testing.T) { } var got []time.Duration - newStream := func(string) (interface{}, error) { + newStream := func(string) (any, error) { if len(got) < maxRetries { return nil, errors.New("backoff") } @@ -51,7 +53,9 @@ func (s) TestClientHealthCheckBackoff(t *testing.T) { } defer func() { backoffFunc = oldBackoffFunc }() - clientHealthCheck(context.Background(), newStream, func(connectivity.State, error) {}, "test") + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + clientHealthCheck(ctx, newStream, func(connectivity.State, error) {}, "test") if !reflect.DeepEqual(got, want) { t.Fatalf("Backoff durations for %v retries are %v. (expected: %v)", maxRetries, got, want) diff --git a/health/grpc_health_v1/health.pb.go b/health/grpc_health_v1/health.pb.go index 4c2a527ec597..24299efd63f7 100644 --- a/health/grpc_health_v1/health.pb.go +++ b/health/grpc_health_v1/health.pb.go @@ -1,28 +1,41 @@ +// Copyright 2015 The gRPC Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The canonical version of this proto can be found at +// https://github.com/grpc/grpc-proto/blob/master/grpc/health/v1/health.proto + // Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.31.0 +// protoc v4.22.0 // source: grpc/health/v1/health.proto package grpc_health_v1 import ( - context "context" - fmt "fmt" - proto "github.com/golang/protobuf/proto" - grpc "google.golang.org/grpc" - codes "google.golang.org/grpc/codes" - status "google.golang.org/grpc/status" - math "math" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" ) -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) type HealthCheckResponse_ServingStatus int32 @@ -30,314 +43,266 @@ const ( HealthCheckResponse_UNKNOWN HealthCheckResponse_ServingStatus = 0 HealthCheckResponse_SERVING HealthCheckResponse_ServingStatus = 1 HealthCheckResponse_NOT_SERVING HealthCheckResponse_ServingStatus = 2 - HealthCheckResponse_SERVICE_UNKNOWN HealthCheckResponse_ServingStatus = 3 + HealthCheckResponse_SERVICE_UNKNOWN HealthCheckResponse_ServingStatus = 3 // Used only by the Watch method. ) -var HealthCheckResponse_ServingStatus_name = map[int32]string{ - 0: "UNKNOWN", - 1: "SERVING", - 2: "NOT_SERVING", - 3: "SERVICE_UNKNOWN", -} +// Enum value maps for HealthCheckResponse_ServingStatus. +var ( + HealthCheckResponse_ServingStatus_name = map[int32]string{ + 0: "UNKNOWN", + 1: "SERVING", + 2: "NOT_SERVING", + 3: "SERVICE_UNKNOWN", + } + HealthCheckResponse_ServingStatus_value = map[string]int32{ + "UNKNOWN": 0, + "SERVING": 1, + "NOT_SERVING": 2, + "SERVICE_UNKNOWN": 3, + } +) -var HealthCheckResponse_ServingStatus_value = map[string]int32{ - "UNKNOWN": 0, - "SERVING": 1, - "NOT_SERVING": 2, - "SERVICE_UNKNOWN": 3, +func (x HealthCheckResponse_ServingStatus) Enum() *HealthCheckResponse_ServingStatus { + p := new(HealthCheckResponse_ServingStatus) + *p = x + return p } func (x HealthCheckResponse_ServingStatus) String() string { - return proto.EnumName(HealthCheckResponse_ServingStatus_name, int32(x)) + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) } -func (HealthCheckResponse_ServingStatus) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_e265fd9d4e077217, []int{1, 0} +func (HealthCheckResponse_ServingStatus) Descriptor() protoreflect.EnumDescriptor { + return file_grpc_health_v1_health_proto_enumTypes[0].Descriptor() } -type HealthCheckRequest struct { - Service string `protobuf:"bytes,1,opt,name=service,proto3" json:"service,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` +func (HealthCheckResponse_ServingStatus) Type() protoreflect.EnumType { + return &file_grpc_health_v1_health_proto_enumTypes[0] } -func (m *HealthCheckRequest) Reset() { *m = HealthCheckRequest{} } -func (m *HealthCheckRequest) String() string { return proto.CompactTextString(m) } -func (*HealthCheckRequest) ProtoMessage() {} -func (*HealthCheckRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_e265fd9d4e077217, []int{0} +func (x HealthCheckResponse_ServingStatus) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) } -func (m *HealthCheckRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_HealthCheckRequest.Unmarshal(m, b) -} -func (m *HealthCheckRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_HealthCheckRequest.Marshal(b, m, deterministic) -} -func (m *HealthCheckRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_HealthCheckRequest.Merge(m, src) -} -func (m *HealthCheckRequest) XXX_Size() int { - return xxx_messageInfo_HealthCheckRequest.Size(m) -} -func (m *HealthCheckRequest) XXX_DiscardUnknown() { - xxx_messageInfo_HealthCheckRequest.DiscardUnknown(m) +// Deprecated: Use HealthCheckResponse_ServingStatus.Descriptor instead. +func (HealthCheckResponse_ServingStatus) EnumDescriptor() ([]byte, []int) { + return file_grpc_health_v1_health_proto_rawDescGZIP(), []int{1, 0} } -var xxx_messageInfo_HealthCheckRequest proto.InternalMessageInfo - -func (m *HealthCheckRequest) GetService() string { - if m != nil { - return m.Service - } - return "" -} +type HealthCheckRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields -type HealthCheckResponse struct { - Status HealthCheckResponse_ServingStatus `protobuf:"varint,1,opt,name=status,proto3,enum=grpc.health.v1.HealthCheckResponse_ServingStatus" json:"status,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Service string `protobuf:"bytes,1,opt,name=service,proto3" json:"service,omitempty"` } -func (m *HealthCheckResponse) Reset() { *m = HealthCheckResponse{} } -func (m *HealthCheckResponse) String() string { return proto.CompactTextString(m) } -func (*HealthCheckResponse) ProtoMessage() {} -func (*HealthCheckResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_e265fd9d4e077217, []int{1} +func (x *HealthCheckRequest) Reset() { + *x = HealthCheckRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_health_v1_health_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *HealthCheckResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_HealthCheckResponse.Unmarshal(m, b) -} -func (m *HealthCheckResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_HealthCheckResponse.Marshal(b, m, deterministic) -} -func (m *HealthCheckResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_HealthCheckResponse.Merge(m, src) -} -func (m *HealthCheckResponse) XXX_Size() int { - return xxx_messageInfo_HealthCheckResponse.Size(m) -} -func (m *HealthCheckResponse) XXX_DiscardUnknown() { - xxx_messageInfo_HealthCheckResponse.DiscardUnknown(m) +func (x *HealthCheckRequest) String() string { + return protoimpl.X.MessageStringOf(x) } -var xxx_messageInfo_HealthCheckResponse proto.InternalMessageInfo +func (*HealthCheckRequest) ProtoMessage() {} -func (m *HealthCheckResponse) GetStatus() HealthCheckResponse_ServingStatus { - if m != nil { - return m.Status +func (x *HealthCheckRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_health_v1_health_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms } - return HealthCheckResponse_UNKNOWN -} - -func init() { - proto.RegisterEnum("grpc.health.v1.HealthCheckResponse_ServingStatus", HealthCheckResponse_ServingStatus_name, HealthCheckResponse_ServingStatus_value) - proto.RegisterType((*HealthCheckRequest)(nil), "grpc.health.v1.HealthCheckRequest") - proto.RegisterType((*HealthCheckResponse)(nil), "grpc.health.v1.HealthCheckResponse") + return mi.MessageOf(x) } -func init() { proto.RegisterFile("grpc/health/v1/health.proto", fileDescriptor_e265fd9d4e077217) } - -var fileDescriptor_e265fd9d4e077217 = []byte{ - // 297 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x4e, 0x2f, 0x2a, 0x48, - 0xd6, 0xcf, 0x48, 0x4d, 0xcc, 0x29, 0xc9, 0xd0, 0x2f, 0x33, 0x84, 0xb2, 0xf4, 0x0a, 0x8a, 0xf2, - 0x4b, 0xf2, 0x85, 0xf8, 0x40, 0x92, 0x7a, 0x50, 0xa1, 0x32, 0x43, 0x25, 0x3d, 0x2e, 0x21, 0x0f, - 0x30, 0xc7, 0x39, 0x23, 0x35, 0x39, 0x3b, 0x28, 0xb5, 0xb0, 0x34, 0xb5, 0xb8, 0x44, 0x48, 0x82, - 0x8b, 0xbd, 0x38, 0xb5, 0xa8, 0x2c, 0x33, 0x39, 0x55, 0x82, 0x51, 0x81, 0x51, 0x83, 0x33, 0x08, - 0xc6, 0x55, 0xda, 0xc8, 0xc8, 0x25, 0x8c, 0xa2, 0xa1, 0xb8, 0x20, 0x3f, 0xaf, 0x38, 0x55, 0xc8, - 0x93, 0x8b, 0xad, 0xb8, 0x24, 0xb1, 0xa4, 0xb4, 0x18, 0xac, 0x81, 0xcf, 0xc8, 0x50, 0x0f, 0xd5, - 0x22, 0x3d, 0x2c, 0x9a, 0xf4, 0x82, 0x41, 0x86, 0xe6, 0xa5, 0x07, 0x83, 0x35, 0x06, 0x41, 0x0d, - 0x50, 0xf2, 0xe7, 0xe2, 0x45, 0x91, 0x10, 0xe2, 0xe6, 0x62, 0x0f, 0xf5, 0xf3, 0xf6, 0xf3, 0x0f, - 0xf7, 0x13, 0x60, 0x00, 0x71, 0x82, 0x5d, 0x83, 0xc2, 0x3c, 0xfd, 0xdc, 0x05, 0x18, 0x85, 0xf8, - 0xb9, 0xb8, 0xfd, 0xfc, 0x43, 0xe2, 0x61, 0x02, 0x4c, 0x42, 0xc2, 0x5c, 0xfc, 0x60, 0x8e, 0xb3, - 0x6b, 0x3c, 0x4c, 0x0b, 0xb3, 0xd1, 0x3a, 0x46, 0x2e, 0x36, 0x88, 0xf5, 0x42, 0x01, 0x5c, 0xac, - 0x60, 0x27, 0x08, 0x29, 0xe1, 0x75, 0x1f, 0x38, 0x14, 0xa4, 0x94, 0x89, 0xf0, 0x83, 0x50, 0x10, - 0x17, 0x6b, 0x78, 0x62, 0x49, 0x72, 0x06, 0xd5, 0x4c, 0x34, 0x60, 0x74, 0x4a, 0xe4, 0x12, 0xcc, - 0xcc, 0x47, 0x53, 0xea, 0xc4, 0x0d, 0x51, 0x1b, 0x00, 0x8a, 0xc6, 0x00, 0xc6, 0x28, 0x9d, 0xf4, - 0xfc, 0xfc, 0xf4, 0x9c, 0x54, 0xbd, 0xf4, 0xfc, 0x9c, 0xc4, 0xbc, 0x74, 0xbd, 0xfc, 0xa2, 0x74, - 0x7d, 0xe4, 0x78, 0x07, 0xb1, 0xe3, 0x21, 0xec, 0xf8, 0x32, 0xc3, 0x55, 0x4c, 0x7c, 0xee, 0x20, - 0xd3, 0x20, 0x46, 0xe8, 0x85, 0x19, 0x26, 0xb1, 0x81, 0x93, 0x83, 0x31, 0x20, 0x00, 0x00, 0xff, - 0xff, 0x12, 0x7d, 0x96, 0xcb, 0x2d, 0x02, 0x00, 0x00, -} - -// Reference imports to suppress errors if they are not otherwise used. -var _ context.Context -var _ grpc.ClientConnInterface - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the grpc package it is being compiled against. -const _ = grpc.SupportPackageIsVersion6 - -// HealthClient is the client API for Health service. -// -// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. -type HealthClient interface { - // If the requested service is unknown, the call will fail with status - // NOT_FOUND. - Check(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (*HealthCheckResponse, error) - // Performs a watch for the serving status of the requested service. - // The server will immediately send back a message indicating the current - // serving status. It will then subsequently send a new message whenever - // the service's serving status changes. - // - // If the requested service is unknown when the call is received, the - // server will send a message setting the serving status to - // SERVICE_UNKNOWN but will *not* terminate the call. If at some - // future point, the serving status of the service becomes known, the - // server will send a new message with the service's serving status. - // - // If the call terminates with status UNIMPLEMENTED, then clients - // should assume this method is not supported and should not retry the - // call. If the call terminates with any other status (including OK), - // clients should retry the call with appropriate exponential backoff. - Watch(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (Health_WatchClient, error) +// Deprecated: Use HealthCheckRequest.ProtoReflect.Descriptor instead. +func (*HealthCheckRequest) Descriptor() ([]byte, []int) { + return file_grpc_health_v1_health_proto_rawDescGZIP(), []int{0} } -type healthClient struct { - cc grpc.ClientConnInterface +func (x *HealthCheckRequest) GetService() string { + if x != nil { + return x.Service + } + return "" } -func NewHealthClient(cc grpc.ClientConnInterface) HealthClient { - return &healthClient{cc} -} +type HealthCheckResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields -func (c *healthClient) Check(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (*HealthCheckResponse, error) { - out := new(HealthCheckResponse) - err := c.cc.Invoke(ctx, "/grpc.health.v1.Health/Check", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil + Status HealthCheckResponse_ServingStatus `protobuf:"varint,1,opt,name=status,proto3,enum=grpc.health.v1.HealthCheckResponse_ServingStatus" json:"status,omitempty"` } -func (c *healthClient) Watch(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (Health_WatchClient, error) { - stream, err := c.cc.NewStream(ctx, &_Health_serviceDesc.Streams[0], "/grpc.health.v1.Health/Watch", opts...) - if err != nil { - return nil, err - } - x := &healthWatchClient{stream} - if err := x.ClientStream.SendMsg(in); err != nil { - return nil, err - } - if err := x.ClientStream.CloseSend(); err != nil { - return nil, err +func (x *HealthCheckResponse) Reset() { + *x = HealthCheckResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_health_v1_health_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } - return x, nil } -type Health_WatchClient interface { - Recv() (*HealthCheckResponse, error) - grpc.ClientStream +func (x *HealthCheckResponse) String() string { + return protoimpl.X.MessageStringOf(x) } -type healthWatchClient struct { - grpc.ClientStream -} +func (*HealthCheckResponse) ProtoMessage() {} -func (x *healthWatchClient) Recv() (*HealthCheckResponse, error) { - m := new(HealthCheckResponse) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err +func (x *HealthCheckResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_health_v1_health_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms } - return m, nil + return mi.MessageOf(x) } -// HealthServer is the server API for Health service. -type HealthServer interface { - // If the requested service is unknown, the call will fail with status - // NOT_FOUND. - Check(context.Context, *HealthCheckRequest) (*HealthCheckResponse, error) - // Performs a watch for the serving status of the requested service. - // The server will immediately send back a message indicating the current - // serving status. It will then subsequently send a new message whenever - // the service's serving status changes. - // - // If the requested service is unknown when the call is received, the - // server will send a message setting the serving status to - // SERVICE_UNKNOWN but will *not* terminate the call. If at some - // future point, the serving status of the service becomes known, the - // server will send a new message with the service's serving status. - // - // If the call terminates with status UNIMPLEMENTED, then clients - // should assume this method is not supported and should not retry the - // call. If the call terminates with any other status (including OK), - // clients should retry the call with appropriate exponential backoff. - Watch(*HealthCheckRequest, Health_WatchServer) error -} - -// UnimplementedHealthServer can be embedded to have forward compatible implementations. -type UnimplementedHealthServer struct { +// Deprecated: Use HealthCheckResponse.ProtoReflect.Descriptor instead. +func (*HealthCheckResponse) Descriptor() ([]byte, []int) { + return file_grpc_health_v1_health_proto_rawDescGZIP(), []int{1} } -func (*UnimplementedHealthServer) Check(ctx context.Context, req *HealthCheckRequest) (*HealthCheckResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method Check not implemented") -} -func (*UnimplementedHealthServer) Watch(req *HealthCheckRequest, srv Health_WatchServer) error { - return status.Errorf(codes.Unimplemented, "method Watch not implemented") +func (x *HealthCheckResponse) GetStatus() HealthCheckResponse_ServingStatus { + if x != nil { + return x.Status + } + return HealthCheckResponse_UNKNOWN } -func RegisterHealthServer(s *grpc.Server, srv HealthServer) { - s.RegisterService(&_Health_serviceDesc, srv) -} +var File_grpc_health_v1_health_proto protoreflect.FileDescriptor + +var file_grpc_health_v1_health_proto_rawDesc = []byte{ + 0x0a, 0x1b, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x2f, 0x76, 0x31, + 0x2f, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0e, 0x67, + 0x72, 0x70, 0x63, 0x2e, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x2e, 0x76, 0x31, 0x22, 0x2e, 0x0a, + 0x12, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x22, 0xb1, 0x01, + 0x0a, 0x13, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x49, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x31, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x68, 0x65, 0x61, + 0x6c, 0x74, 0x68, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, + 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, + 0x6e, 0x67, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x22, 0x4f, 0x0a, 0x0d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x0b, + 0x0a, 0x07, 0x53, 0x45, 0x52, 0x56, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x0f, 0x0a, 0x0b, 0x4e, + 0x4f, 0x54, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x49, 0x4e, 0x47, 0x10, 0x02, 0x12, 0x13, 0x0a, 0x0f, + 0x53, 0x45, 0x52, 0x56, 0x49, 0x43, 0x45, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, + 0x03, 0x32, 0xae, 0x01, 0x0a, 0x06, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x12, 0x50, 0x0a, 0x05, + 0x43, 0x68, 0x65, 0x63, 0x6b, 0x12, 0x22, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x68, 0x65, 0x61, + 0x6c, 0x74, 0x68, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, + 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, + 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x52, + 0x0a, 0x05, 0x57, 0x61, 0x74, 0x63, 0x68, 0x12, 0x22, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x68, + 0x65, 0x61, 0x6c, 0x74, 0x68, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x43, + 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x67, 0x72, + 0x70, 0x63, 0x2e, 0x68, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x65, 0x61, + 0x6c, 0x74, 0x68, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x30, 0x01, 0x42, 0x61, 0x0a, 0x11, 0x69, 0x6f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x68, 0x65, + 0x61, 0x6c, 0x74, 0x68, 0x2e, 0x76, 0x31, 0x42, 0x0b, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x50, + 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x2c, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67, + 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x68, + 0x65, 0x61, 0x6c, 0x74, 0x68, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x68, 0x65, 0x61, 0x6c, 0x74, + 0x68, 0x5f, 0x76, 0x31, 0xaa, 0x02, 0x0e, 0x47, 0x72, 0x70, 0x63, 0x2e, 0x48, 0x65, 0x61, 0x6c, + 0x74, 0x68, 0x2e, 0x56, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_grpc_health_v1_health_proto_rawDescOnce sync.Once + file_grpc_health_v1_health_proto_rawDescData = file_grpc_health_v1_health_proto_rawDesc +) -func _Health_Check_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(HealthCheckRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(HealthServer).Check(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/grpc.health.v1.Health/Check", +func file_grpc_health_v1_health_proto_rawDescGZIP() []byte { + file_grpc_health_v1_health_proto_rawDescOnce.Do(func() { + file_grpc_health_v1_health_proto_rawDescData = protoimpl.X.CompressGZIP(file_grpc_health_v1_health_proto_rawDescData) + }) + return file_grpc_health_v1_health_proto_rawDescData +} + +var file_grpc_health_v1_health_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_grpc_health_v1_health_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_grpc_health_v1_health_proto_goTypes = []interface{}{ + (HealthCheckResponse_ServingStatus)(0), // 0: grpc.health.v1.HealthCheckResponse.ServingStatus + (*HealthCheckRequest)(nil), // 1: grpc.health.v1.HealthCheckRequest + (*HealthCheckResponse)(nil), // 2: grpc.health.v1.HealthCheckResponse +} +var file_grpc_health_v1_health_proto_depIdxs = []int32{ + 0, // 0: grpc.health.v1.HealthCheckResponse.status:type_name -> grpc.health.v1.HealthCheckResponse.ServingStatus + 1, // 1: grpc.health.v1.Health.Check:input_type -> grpc.health.v1.HealthCheckRequest + 1, // 2: grpc.health.v1.Health.Watch:input_type -> grpc.health.v1.HealthCheckRequest + 2, // 3: grpc.health.v1.Health.Check:output_type -> grpc.health.v1.HealthCheckResponse + 2, // 4: grpc.health.v1.Health.Watch:output_type -> grpc.health.v1.HealthCheckResponse + 3, // [3:5] is the sub-list for method output_type + 1, // [1:3] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name +} + +func init() { file_grpc_health_v1_health_proto_init() } +func file_grpc_health_v1_health_proto_init() { + if File_grpc_health_v1_health_proto != nil { + return } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(HealthServer).Check(ctx, req.(*HealthCheckRequest)) + if !protoimpl.UnsafeEnabled { + file_grpc_health_v1_health_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*HealthCheckRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_health_v1_health_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*HealthCheckResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } - return interceptor(ctx, in, info, handler) -} - -func _Health_Watch_Handler(srv interface{}, stream grpc.ServerStream) error { - m := new(HealthCheckRequest) - if err := stream.RecvMsg(m); err != nil { - return err - } - return srv.(HealthServer).Watch(m, &healthWatchServer{stream}) -} - -type Health_WatchServer interface { - Send(*HealthCheckResponse) error - grpc.ServerStream -} - -type healthWatchServer struct { - grpc.ServerStream -} - -func (x *healthWatchServer) Send(m *HealthCheckResponse) error { - return x.ServerStream.SendMsg(m) -} - -var _Health_serviceDesc = grpc.ServiceDesc{ - ServiceName: "grpc.health.v1.Health", - HandlerType: (*HealthServer)(nil), - Methods: []grpc.MethodDesc{ - { - MethodName: "Check", - Handler: _Health_Check_Handler, - }, - }, - Streams: []grpc.StreamDesc{ - { - StreamName: "Watch", - Handler: _Health_Watch_Handler, - ServerStreams: true, + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_grpc_health_v1_health_proto_rawDesc, + NumEnums: 1, + NumMessages: 2, + NumExtensions: 0, + NumServices: 1, }, - }, - Metadata: "grpc/health/v1/health.proto", + GoTypes: file_grpc_health_v1_health_proto_goTypes, + DependencyIndexes: file_grpc_health_v1_health_proto_depIdxs, + EnumInfos: file_grpc_health_v1_health_proto_enumTypes, + MessageInfos: file_grpc_health_v1_health_proto_msgTypes, + }.Build() + File_grpc_health_v1_health_proto = out.File + file_grpc_health_v1_health_proto_rawDesc = nil + file_grpc_health_v1_health_proto_goTypes = nil + file_grpc_health_v1_health_proto_depIdxs = nil } diff --git a/health/grpc_health_v1/health_grpc.pb.go b/health/grpc_health_v1/health_grpc.pb.go index 716f8c0153a4..a01a1b4d54bd 100644 --- a/health/grpc_health_v1/health_grpc.pb.go +++ b/health/grpc_health_v1/health_grpc.pb.go @@ -1,4 +1,25 @@ +// Copyright 2015 The gRPC Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The canonical version of this proto can be found at +// https://github.com/grpc/grpc-proto/blob/master/grpc/health/v1/health.proto + // Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.3.0 +// - protoc v4.22.0 +// source: grpc/health/v1/health.proto package grpc_health_v1 @@ -11,16 +32,21 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. const _ = grpc.SupportPackageIsVersion7 -// HealthService is the service API for Health service. -// Fields should be assigned to their respective handler implementations only before -// RegisterHealthService is called. Any unassigned fields will result in the -// handler for that method returning an Unimplemented error. -type HealthService struct { +const ( + Health_Check_FullMethodName = "/grpc.health.v1.Health/Check" + Health_Watch_FullMethodName = "/grpc.health.v1.Health/Watch" +) + +// HealthClient is the client API for Health service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type HealthClient interface { // If the requested service is unknown, the call will fail with status // NOT_FOUND. - Check func(context.Context, *HealthCheckRequest) (*HealthCheckResponse, error) + Check(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (*HealthCheckResponse, error) // Performs a watch for the serving status of the requested service. // The server will immediately send back a message indicating the current // serving status. It will then subsequently send a new message whenever @@ -36,89 +62,62 @@ type HealthService struct { // should assume this method is not supported and should not retry the // call. If the call terminates with any other status (including OK), // clients should retry the call with appropriate exponential backoff. - Watch func(*HealthCheckRequest, Health_WatchServer) error + Watch(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (Health_WatchClient, error) } -func (s *HealthService) check(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - if s.Check == nil { - return nil, status.Errorf(codes.Unimplemented, "method Check not implemented") - } - in := new(HealthCheckRequest) - if err := dec(in); err != nil { +type healthClient struct { + cc grpc.ClientConnInterface +} + +func NewHealthClient(cc grpc.ClientConnInterface) HealthClient { + return &healthClient{cc} +} + +func (c *healthClient) Check(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (*HealthCheckResponse, error) { + out := new(HealthCheckResponse) + err := c.cc.Invoke(ctx, Health_Check_FullMethodName, in, out, opts...) + if err != nil { return nil, err } - if interceptor == nil { - return s.Check(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: s, - FullMethod: "/grpc.health.v1.Health/Check", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return s.Check(ctx, req.(*HealthCheckRequest)) - } - return interceptor(ctx, in, info, handler) + return out, nil } -func (s *HealthService) watch(_ interface{}, stream grpc.ServerStream) error { - if s.Watch == nil { - return status.Errorf(codes.Unimplemented, "method Watch not implemented") + +func (c *healthClient) Watch(ctx context.Context, in *HealthCheckRequest, opts ...grpc.CallOption) (Health_WatchClient, error) { + stream, err := c.cc.NewStream(ctx, &Health_ServiceDesc.Streams[0], Health_Watch_FullMethodName, opts...) + if err != nil { + return nil, err } - m := new(HealthCheckRequest) - if err := stream.RecvMsg(m); err != nil { - return err + x := &healthWatchClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err } - return s.Watch(m, &healthWatchServer{stream}) -} - -// RegisterHealthService registers a service implementation with a gRPC server. -func RegisterHealthService(s grpc.ServiceRegistrar, srv *HealthService) { - sd := grpc.ServiceDesc{ - ServiceName: "grpc.health.v1.Health", - Methods: []grpc.MethodDesc{ - { - MethodName: "Check", - Handler: srv.check, - }, - }, - Streams: []grpc.StreamDesc{ - { - StreamName: "Watch", - Handler: srv.watch, - ServerStreams: true, - }, - }, - Metadata: "grpc/health/v1/health.proto", + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err } + return x, nil +} - s.RegisterService(&sd, nil) -} - -// NewHealthService creates a new HealthService containing the -// implemented methods of the Health service in s. Any unimplemented -// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client. -// This includes situations where the method handler is misspelled or has the wrong -// signature. For this reason, this function should be used with great care and -// is not recommended to be used by most users. -func NewHealthService(s interface{}) *HealthService { - ns := &HealthService{} - if h, ok := s.(interface { - Check(context.Context, *HealthCheckRequest) (*HealthCheckResponse, error) - }); ok { - ns.Check = h.Check - } - if h, ok := s.(interface { - Watch(*HealthCheckRequest, Health_WatchServer) error - }); ok { - ns.Watch = h.Watch +type Health_WatchClient interface { + Recv() (*HealthCheckResponse, error) + grpc.ClientStream +} + +type healthWatchClient struct { + grpc.ClientStream +} + +func (x *healthWatchClient) Recv() (*HealthCheckResponse, error) { + m := new(HealthCheckResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err } - return ns + return m, nil } -// UnstableHealthService is the service API for Health service. -// New methods may be added to this interface if they are added to the service -// definition, which is not a backward-compatible change. For this reason, -// use of this type is not recommended. -type UnstableHealthService interface { +// HealthServer is the server API for Health service. +// All implementations should embed UnimplementedHealthServer +// for forward compatibility +type HealthServer interface { // If the requested service is unknown, the call will fail with status // NOT_FOUND. Check(context.Context, *HealthCheckRequest) (*HealthCheckResponse, error) @@ -139,3 +138,86 @@ type UnstableHealthService interface { // clients should retry the call with appropriate exponential backoff. Watch(*HealthCheckRequest, Health_WatchServer) error } + +// UnimplementedHealthServer should be embedded to have forward compatible implementations. +type UnimplementedHealthServer struct { +} + +func (UnimplementedHealthServer) Check(context.Context, *HealthCheckRequest) (*HealthCheckResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method Check not implemented") +} +func (UnimplementedHealthServer) Watch(*HealthCheckRequest, Health_WatchServer) error { + return status.Errorf(codes.Unimplemented, "method Watch not implemented") +} + +// UnsafeHealthServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to HealthServer will +// result in compilation errors. +type UnsafeHealthServer interface { + mustEmbedUnimplementedHealthServer() +} + +func RegisterHealthServer(s grpc.ServiceRegistrar, srv HealthServer) { + s.RegisterService(&Health_ServiceDesc, srv) +} + +func _Health_Check_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(HealthCheckRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(HealthServer).Check(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Health_Check_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(HealthServer).Check(ctx, req.(*HealthCheckRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Health_Watch_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(HealthCheckRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(HealthServer).Watch(m, &healthWatchServer{stream}) +} + +type Health_WatchServer interface { + Send(*HealthCheckResponse) error + grpc.ServerStream +} + +type healthWatchServer struct { + grpc.ServerStream +} + +func (x *healthWatchServer) Send(m *HealthCheckResponse) error { + return x.ServerStream.SendMsg(m) +} + +// Health_ServiceDesc is the grpc.ServiceDesc for Health service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var Health_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.health.v1.Health", + HandlerType: (*HealthServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Check", + Handler: _Health_Check_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "Watch", + Handler: _Health_Watch_Handler, + ServerStreams: true, + }, + }, + Metadata: "grpc/health/v1/health.proto", +} diff --git a/interceptor.go b/interceptor.go index 8b7350022ad7..877d78fc3d00 100644 --- a/interceptor.go +++ b/interceptor.go @@ -23,41 +23,68 @@ import ( ) // UnaryInvoker is called by UnaryClientInterceptor to complete RPCs. -type UnaryInvoker func(ctx context.Context, method string, req, reply interface{}, cc *ClientConn, opts ...CallOption) error +type UnaryInvoker func(ctx context.Context, method string, req, reply any, cc *ClientConn, opts ...CallOption) error -// UnaryClientInterceptor intercepts the execution of a unary RPC on the client. invoker is the handler to complete the RPC -// and it is the responsibility of the interceptor to call it. -// This is an EXPERIMENTAL API. -type UnaryClientInterceptor func(ctx context.Context, method string, req, reply interface{}, cc *ClientConn, invoker UnaryInvoker, opts ...CallOption) error +// UnaryClientInterceptor intercepts the execution of a unary RPC on the client. +// Unary interceptors can be specified as a DialOption, using +// WithUnaryInterceptor() or WithChainUnaryInterceptor(), when creating a +// ClientConn. When a unary interceptor(s) is set on a ClientConn, gRPC +// delegates all unary RPC invocations to the interceptor, and it is the +// responsibility of the interceptor to call invoker to complete the processing +// of the RPC. +// +// method is the RPC name. req and reply are the corresponding request and +// response messages. cc is the ClientConn on which the RPC was invoked. invoker +// is the handler to complete the RPC and it is the responsibility of the +// interceptor to call it. opts contain all applicable call options, including +// defaults from the ClientConn as well as per-call options. +// +// The returned error must be compatible with the status package. +type UnaryClientInterceptor func(ctx context.Context, method string, req, reply any, cc *ClientConn, invoker UnaryInvoker, opts ...CallOption) error // Streamer is called by StreamClientInterceptor to create a ClientStream. type Streamer func(ctx context.Context, desc *StreamDesc, cc *ClientConn, method string, opts ...CallOption) (ClientStream, error) -// StreamClientInterceptor intercepts the creation of ClientStream. It may return a custom ClientStream to intercept all I/O -// operations. streamer is the handler to create a ClientStream and it is the responsibility of the interceptor to call it. -// This is an EXPERIMENTAL API. +// StreamClientInterceptor intercepts the creation of a ClientStream. Stream +// interceptors can be specified as a DialOption, using WithStreamInterceptor() +// or WithChainStreamInterceptor(), when creating a ClientConn. When a stream +// interceptor(s) is set on the ClientConn, gRPC delegates all stream creations +// to the interceptor, and it is the responsibility of the interceptor to call +// streamer. +// +// desc contains a description of the stream. cc is the ClientConn on which the +// RPC was invoked. streamer is the handler to create a ClientStream and it is +// the responsibility of the interceptor to call it. opts contain all applicable +// call options, including defaults from the ClientConn as well as per-call +// options. +// +// StreamClientInterceptor may return a custom ClientStream to intercept all I/O +// operations. The returned error must be compatible with the status package. type StreamClientInterceptor func(ctx context.Context, desc *StreamDesc, cc *ClientConn, method string, streamer Streamer, opts ...CallOption) (ClientStream, error) // UnaryServerInfo consists of various information about a unary RPC on // server side. All per-rpc information may be mutated by the interceptor. type UnaryServerInfo struct { // Server is the service implementation the user provides. This is read-only. - Server interface{} + Server any // FullMethod is the full RPC method string, i.e., /package.service/method. FullMethod string } // UnaryHandler defines the handler invoked by UnaryServerInterceptor to complete the normal -// execution of a unary RPC. If a UnaryHandler returns an error, it should be produced by the -// status package, or else gRPC will use codes.Unknown as the status code and err.Error() as -// the status message of the RPC. -type UnaryHandler func(ctx context.Context, req interface{}) (interface{}, error) +// execution of a unary RPC. +// +// If a UnaryHandler returns an error, it should either be produced by the +// status package, or be one of the context errors. Otherwise, gRPC will use +// codes.Unknown as the status code and err.Error() as the status message of the +// RPC. +type UnaryHandler func(ctx context.Context, req any) (any, error) // UnaryServerInterceptor provides a hook to intercept the execution of a unary RPC on the server. info // contains all the information of this RPC the interceptor can operate on. And handler is the wrapper // of the service method implementation. It is the responsibility of the interceptor to invoke handler // to complete the RPC. -type UnaryServerInterceptor func(ctx context.Context, req interface{}, info *UnaryServerInfo, handler UnaryHandler) (resp interface{}, err error) +type UnaryServerInterceptor func(ctx context.Context, req any, info *UnaryServerInfo, handler UnaryHandler) (resp any, err error) // StreamServerInfo consists of various information about a streaming RPC on // server side. All per-rpc information may be mutated by the interceptor. @@ -74,4 +101,4 @@ type StreamServerInfo struct { // info contains all the information of this RPC the interceptor can operate on. And handler is the // service method implementation. It is the responsibility of the interceptor to invoke handler to // complete the RPC. -type StreamServerInterceptor func(srv interface{}, ss ServerStream, info *StreamServerInfo, handler StreamHandler) error +type StreamServerInterceptor func(srv any, ss ServerStream, info *StreamServerInfo, handler StreamHandler) error diff --git a/internal/admin/admin.go b/internal/admin/admin.go new file mode 100644 index 000000000000..a9285ee74842 --- /dev/null +++ b/internal/admin/admin.go @@ -0,0 +1,60 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package admin contains internal implementation for admin service. +package admin + +import "google.golang.org/grpc" + +// services is a map from name to service register functions. +var services []func(grpc.ServiceRegistrar) (func(), error) + +// AddService adds a service to the list of admin services. +// +// NOTE: this function must only be called during initialization time (i.e. in +// an init() function), and is not thread-safe. +// +// If multiple services with the same service name are added (e.g. two services +// for `grpc.channelz.v1.Channelz`), the server will panic on `Register()`. +func AddService(f func(grpc.ServiceRegistrar) (func(), error)) { + services = append(services, f) +} + +// Register registers the set of admin services to the given server. +func Register(s grpc.ServiceRegistrar) (cleanup func(), _ error) { + var cleanups []func() + for _, f := range services { + cleanup, err := f(s) + if err != nil { + callFuncs(cleanups) + return nil, err + } + if cleanup != nil { + cleanups = append(cleanups, cleanup) + } + } + return func() { + callFuncs(cleanups) + }, nil +} + +func callFuncs(fs []func()) { + for _, f := range fs { + f() + } +} diff --git a/internal/balancer/gracefulswitch/gracefulswitch.go b/internal/balancer/gracefulswitch/gracefulswitch.go new file mode 100644 index 000000000000..3c594e6e4e55 --- /dev/null +++ b/internal/balancer/gracefulswitch/gracefulswitch.go @@ -0,0 +1,385 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package gracefulswitch implements a graceful switch load balancer. +package gracefulswitch + +import ( + "errors" + "fmt" + "sync" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/base" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/resolver" +) + +var errBalancerClosed = errors.New("gracefulSwitchBalancer is closed") +var _ balancer.Balancer = (*Balancer)(nil) + +// NewBalancer returns a graceful switch Balancer. +func NewBalancer(cc balancer.ClientConn, opts balancer.BuildOptions) *Balancer { + return &Balancer{ + cc: cc, + bOpts: opts, + } +} + +// Balancer is a utility to gracefully switch from one balancer to +// a new balancer. It implements the balancer.Balancer interface. +type Balancer struct { + bOpts balancer.BuildOptions + cc balancer.ClientConn + + // mu protects the following fields and all fields within balancerCurrent + // and balancerPending. mu does not need to be held when calling into the + // child balancers, as all calls into these children happen only as a direct + // result of a call into the gracefulSwitchBalancer, which are also + // guaranteed to be synchronous. There is one exception: an UpdateState call + // from a child balancer when current and pending are populated can lead to + // calling Close() on the current. To prevent that racing with an + // UpdateSubConnState from the channel, we hold currentMu during Close and + // UpdateSubConnState calls. + mu sync.Mutex + balancerCurrent *balancerWrapper + balancerPending *balancerWrapper + closed bool // set to true when this balancer is closed + + // currentMu must be locked before mu. This mutex guards against this + // sequence of events: UpdateSubConnState() called, finds the + // balancerCurrent, gives up lock, updateState comes in, causes Close() on + // balancerCurrent before the UpdateSubConnState is called on the + // balancerCurrent. + currentMu sync.Mutex +} + +// swap swaps out the current lb with the pending lb and updates the ClientConn. +// The caller must hold gsb.mu. +func (gsb *Balancer) swap() { + gsb.cc.UpdateState(gsb.balancerPending.lastState) + cur := gsb.balancerCurrent + gsb.balancerCurrent = gsb.balancerPending + gsb.balancerPending = nil + go func() { + gsb.currentMu.Lock() + defer gsb.currentMu.Unlock() + cur.Close() + }() +} + +// Helper function that checks if the balancer passed in is current or pending. +// The caller must hold gsb.mu. +func (gsb *Balancer) balancerCurrentOrPending(bw *balancerWrapper) bool { + return bw == gsb.balancerCurrent || bw == gsb.balancerPending +} + +// SwitchTo initializes the graceful switch process, which completes based on +// connectivity state changes on the current/pending balancer. Thus, the switch +// process is not complete when this method returns. This method must be called +// synchronously alongside the rest of the balancer.Balancer methods this +// Graceful Switch Balancer implements. +func (gsb *Balancer) SwitchTo(builder balancer.Builder) error { + gsb.mu.Lock() + if gsb.closed { + gsb.mu.Unlock() + return errBalancerClosed + } + bw := &balancerWrapper{ + gsb: gsb, + lastState: balancer.State{ + ConnectivityState: connectivity.Connecting, + Picker: base.NewErrPicker(balancer.ErrNoSubConnAvailable), + }, + subconns: make(map[balancer.SubConn]bool), + } + balToClose := gsb.balancerPending // nil if there is no pending balancer + if gsb.balancerCurrent == nil { + gsb.balancerCurrent = bw + } else { + gsb.balancerPending = bw + } + gsb.mu.Unlock() + balToClose.Close() + // This function takes a builder instead of a balancer because builder.Build + // can call back inline, and this utility needs to handle the callbacks. + newBalancer := builder.Build(bw, gsb.bOpts) + if newBalancer == nil { + // This is illegal and should never happen; we clear the balancerWrapper + // we were constructing if it happens to avoid a potential panic. + gsb.mu.Lock() + if gsb.balancerPending != nil { + gsb.balancerPending = nil + } else { + gsb.balancerCurrent = nil + } + gsb.mu.Unlock() + return balancer.ErrBadResolverState + } + + // This write doesn't need to take gsb.mu because this field never gets read + // or written to on any calls from the current or pending. Calls from grpc + // to this balancer are guaranteed to be called synchronously, so this + // bw.Balancer field will never be forwarded to until this SwitchTo() + // function returns. + bw.Balancer = newBalancer + return nil +} + +// Returns nil if the graceful switch balancer is closed. +func (gsb *Balancer) latestBalancer() *balancerWrapper { + gsb.mu.Lock() + defer gsb.mu.Unlock() + if gsb.balancerPending != nil { + return gsb.balancerPending + } + return gsb.balancerCurrent +} + +// UpdateClientConnState forwards the update to the latest balancer created. +func (gsb *Balancer) UpdateClientConnState(state balancer.ClientConnState) error { + // The resolver data is only relevant to the most recent LB Policy. + balToUpdate := gsb.latestBalancer() + if balToUpdate == nil { + return errBalancerClosed + } + // Perform this call without gsb.mu to prevent deadlocks if the child calls + // back into the channel. The latest balancer can never be closed during a + // call from the channel, even without gsb.mu held. + return balToUpdate.UpdateClientConnState(state) +} + +// ResolverError forwards the error to the latest balancer created. +func (gsb *Balancer) ResolverError(err error) { + // The resolver data is only relevant to the most recent LB Policy. + balToUpdate := gsb.latestBalancer() + if balToUpdate == nil { + return + } + // Perform this call without gsb.mu to prevent deadlocks if the child calls + // back into the channel. The latest balancer can never be closed during a + // call from the channel, even without gsb.mu held. + balToUpdate.ResolverError(err) +} + +// ExitIdle forwards the call to the latest balancer created. +// +// If the latest balancer does not support ExitIdle, the subConns are +// re-connected to manually. +func (gsb *Balancer) ExitIdle() { + balToUpdate := gsb.latestBalancer() + if balToUpdate == nil { + return + } + // There is no need to protect this read with a mutex, as the write to the + // Balancer field happens in SwitchTo, which completes before this can be + // called. + if ei, ok := balToUpdate.Balancer.(balancer.ExitIdler); ok { + ei.ExitIdle() + return + } + gsb.mu.Lock() + defer gsb.mu.Unlock() + for sc := range balToUpdate.subconns { + sc.Connect() + } +} + +// updateSubConnState forwards the update to the appropriate child. +func (gsb *Balancer) updateSubConnState(sc balancer.SubConn, state balancer.SubConnState, cb func(balancer.SubConnState)) { + gsb.currentMu.Lock() + defer gsb.currentMu.Unlock() + gsb.mu.Lock() + // Forward update to the appropriate child. Even if there is a pending + // balancer, the current balancer should continue to get SubConn updates to + // maintain the proper state while the pending is still connecting. + var balToUpdate *balancerWrapper + if gsb.balancerCurrent != nil && gsb.balancerCurrent.subconns[sc] { + balToUpdate = gsb.balancerCurrent + } else if gsb.balancerPending != nil && gsb.balancerPending.subconns[sc] { + balToUpdate = gsb.balancerPending + } + if balToUpdate == nil { + // SubConn belonged to a stale lb policy that has not yet fully closed, + // or the balancer was already closed. + gsb.mu.Unlock() + return + } + if state.ConnectivityState == connectivity.Shutdown { + delete(balToUpdate.subconns, sc) + } + gsb.mu.Unlock() + if cb != nil { + cb(state) + } else { + balToUpdate.UpdateSubConnState(sc, state) + } +} + +// UpdateSubConnState forwards the update to the appropriate child. +func (gsb *Balancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { + gsb.updateSubConnState(sc, state, nil) +} + +// Close closes any active child balancers. +func (gsb *Balancer) Close() { + gsb.mu.Lock() + gsb.closed = true + currentBalancerToClose := gsb.balancerCurrent + gsb.balancerCurrent = nil + pendingBalancerToClose := gsb.balancerPending + gsb.balancerPending = nil + gsb.mu.Unlock() + + currentBalancerToClose.Close() + pendingBalancerToClose.Close() +} + +// balancerWrapper wraps a balancer.Balancer, and overrides some Balancer +// methods to help cleanup SubConns created by the wrapped balancer. +// +// It implements the balancer.ClientConn interface and is passed down in that +// capacity to the wrapped balancer. It maintains a set of subConns created by +// the wrapped balancer and calls from the latter to create/update/shutdown +// SubConns update this set before being forwarded to the parent ClientConn. +// State updates from the wrapped balancer can result in invocation of the +// graceful switch logic. +type balancerWrapper struct { + balancer.Balancer + gsb *Balancer + + lastState balancer.State + subconns map[balancer.SubConn]bool // subconns created by this balancer +} + +// Close closes the underlying LB policy and shuts down the subconns it +// created. bw must not be referenced via balancerCurrent or balancerPending in +// gsb when called. gsb.mu must not be held. Does not panic with a nil +// receiver. +func (bw *balancerWrapper) Close() { + // before Close is called. + if bw == nil { + return + } + // There is no need to protect this read with a mutex, as Close() is + // impossible to be called concurrently with the write in SwitchTo(). The + // callsites of Close() for this balancer in Graceful Switch Balancer will + // never be called until SwitchTo() returns. + bw.Balancer.Close() + bw.gsb.mu.Lock() + for sc := range bw.subconns { + sc.Shutdown() + } + bw.gsb.mu.Unlock() +} + +func (bw *balancerWrapper) UpdateState(state balancer.State) { + // Hold the mutex for this entire call to ensure it cannot occur + // concurrently with other updateState() calls. This causes updates to + // lastState and calls to cc.UpdateState to happen atomically. + bw.gsb.mu.Lock() + defer bw.gsb.mu.Unlock() + bw.lastState = state + + if !bw.gsb.balancerCurrentOrPending(bw) { + return + } + + if bw == bw.gsb.balancerCurrent { + // In the case that the current balancer exits READY, and there is a pending + // balancer, you can forward the pending balancer's cached State up to + // ClientConn and swap the pending into the current. This is because there + // is no reason to gracefully switch from and keep using the old policy as + // the ClientConn is not connected to any backends. + if state.ConnectivityState != connectivity.Ready && bw.gsb.balancerPending != nil { + bw.gsb.swap() + return + } + // Even if there is a pending balancer waiting to be gracefully switched to, + // continue to forward current balancer updates to the Client Conn. Ignoring + // state + picker from the current would cause undefined behavior/cause the + // system to behave incorrectly from the current LB policies perspective. + // Also, the current LB is still being used by grpc to choose SubConns per + // RPC, and thus should use the most updated form of the current balancer. + bw.gsb.cc.UpdateState(state) + return + } + // This method is now dealing with a state update from the pending balancer. + // If the current balancer is currently in a state other than READY, the new + // policy can be swapped into place immediately. This is because there is no + // reason to gracefully switch from and keep using the old policy as the + // ClientConn is not connected to any backends. + if state.ConnectivityState != connectivity.Connecting || bw.gsb.balancerCurrent.lastState.ConnectivityState != connectivity.Ready { + bw.gsb.swap() + } +} + +func (bw *balancerWrapper) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) { + bw.gsb.mu.Lock() + if !bw.gsb.balancerCurrentOrPending(bw) { + bw.gsb.mu.Unlock() + return nil, fmt.Errorf("%T at address %p that called NewSubConn is deleted", bw, bw) + } + bw.gsb.mu.Unlock() + + var sc balancer.SubConn + oldListener := opts.StateListener + opts.StateListener = func(state balancer.SubConnState) { bw.gsb.updateSubConnState(sc, state, oldListener) } + sc, err := bw.gsb.cc.NewSubConn(addrs, opts) + if err != nil { + return nil, err + } + bw.gsb.mu.Lock() + if !bw.gsb.balancerCurrentOrPending(bw) { // balancer was closed during this call + sc.Shutdown() + bw.gsb.mu.Unlock() + return nil, fmt.Errorf("%T at address %p that called NewSubConn is deleted", bw, bw) + } + bw.subconns[sc] = true + bw.gsb.mu.Unlock() + return sc, nil +} + +func (bw *balancerWrapper) ResolveNow(opts resolver.ResolveNowOptions) { + // Ignore ResolveNow requests from anything other than the most recent + // balancer, because older balancers were already removed from the config. + if bw != bw.gsb.latestBalancer() { + return + } + bw.gsb.cc.ResolveNow(opts) +} + +func (bw *balancerWrapper) RemoveSubConn(sc balancer.SubConn) { + // Note: existing third party balancers may call this, so it must remain + // until RemoveSubConn is fully removed. + sc.Shutdown() +} + +func (bw *balancerWrapper) UpdateAddresses(sc balancer.SubConn, addrs []resolver.Address) { + bw.gsb.mu.Lock() + if !bw.gsb.balancerCurrentOrPending(bw) { + bw.gsb.mu.Unlock() + return + } + bw.gsb.mu.Unlock() + bw.gsb.cc.UpdateAddresses(sc, addrs) +} + +func (bw *balancerWrapper) Target() string { + return bw.gsb.cc.Target() +} diff --git a/internal/balancer/gracefulswitch/gracefulswitch_test.go b/internal/balancer/gracefulswitch/gracefulswitch_test.go new file mode 100644 index 000000000000..e92b09dbe4cf --- /dev/null +++ b/internal/balancer/gracefulswitch/gracefulswitch_test.go @@ -0,0 +1,1103 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package gracefulswitch + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" +) + +const ( + defaultTestTimeout = 5 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func setup(t *testing.T) (*testutils.TestClientConn, *Balancer) { + tcc := testutils.NewTestClientConn(t) + return tcc, NewBalancer(tcc, balancer.BuildOptions{}) +} + +// TestSuccessfulFirstUpdate tests a basic scenario for the graceful switch load +// balancer, where it is setup with a balancer which should populate the current +// load balancer. Any ClientConn updates should then be forwarded to this +// current load balancer. +func (s) TestSuccessfulFirstUpdate(t *testing.T) { + _, gsb := setup(t) + if err := gsb.SwitchTo(mockBalancerBuilder1{}); err != nil { + t.Fatalf("Balancer.SwitchTo failed with error: %v", err) + } + if gsb.balancerCurrent == nil { + t.Fatal("current balancer not populated after a successful call to SwitchTo()") + } + // This will be used to update the graceful switch balancer. This update + // should simply be forwarded down to the current load balancing policy. + ccs := balancer.ClientConnState{ + BalancerConfig: mockBalancerConfig{}, + } + + // Updating ClientConnState should forward the update exactly as is to the + // current balancer. + if err := gsb.UpdateClientConnState(ccs); err != nil { + t.Fatalf("Balancer.UpdateClientConnState(%v) failed: %v", ccs, err) + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := gsb.balancerCurrent.Balancer.(*mockBalancer).waitForClientConnUpdate(ctx, ccs); err != nil { + t.Fatal(err) + } +} + +// TestTwoBalancersSameType tests the scenario where there is a graceful switch +// load balancer setup with a current and pending load balancer of the same +// type. Any ClientConn update should be forwarded to the current lb if there is +// a current lb and no pending lb, and the only the pending lb if the graceful +// switch balancer contains both a current lb and a pending lb. The pending load +// balancer should also swap into current whenever it updates with a +// connectivity state other than CONNECTING. +func (s) TestTwoBalancersSameType(t *testing.T) { + tcc, gsb := setup(t) + // This will be used to update the graceful switch balancer. This update + // should simply be forwarded down to either the current or pending load + // balancing policy. + ccs := balancer.ClientConnState{ + BalancerConfig: mockBalancerConfig{}, + } + + gsb.SwitchTo(mockBalancerBuilder1{}) + gsb.UpdateClientConnState(ccs) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := gsb.balancerCurrent.Balancer.(*mockBalancer).waitForClientConnUpdate(ctx, ccs); err != nil { + t.Fatal(err) + } + + // The current balancer reporting READY should cause this state + // to be forwarded to the ClientConn. + gsb.balancerCurrent.Balancer.(*mockBalancer).updateState(balancer.State{ + ConnectivityState: connectivity.Ready, + Picker: &neverErrPicker{}, + }) + + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for a UpdateState call on the ClientConn") + case state := <-tcc.NewStateCh: + if state != connectivity.Ready { + t.Fatalf("current balancer reports connectivity state %v, want %v", state, connectivity.Ready) + } + } + + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for a UpdateState call on the ClientConn") + case picker := <-tcc.NewPickerCh: + // Should receive a never err picker. + if _, err := picker.Pick(balancer.PickInfo{}); err != nil { + t.Fatalf("ClientConn should have received a never err picker from an UpdateState call") + } + } + + // An explicit call to switchTo, even if the same type, should cause the + // balancer to build a new balancer for pending. + gsb.SwitchTo(mockBalancerBuilder1{}) + if gsb.balancerPending == nil { + t.Fatal("pending balancer not populated after another call to SwitchTo()") + } + + // A ClientConn update received should be forwarded to the new pending LB + // policy, and not the current one. + gsb.UpdateClientConnState(ccs) + sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + if err := gsb.balancerCurrent.Balancer.(*mockBalancer).waitForClientConnUpdate(sCtx, ccs); err == nil { + t.Fatal("current balancer received a ClientConn update when there is a pending balancer") + } + if err := gsb.balancerPending.Balancer.(*mockBalancer).waitForClientConnUpdate(ctx, ccs); err != nil { + t.Fatal(err) + } + + // If the pending load balancer reports that is CONNECTING, no update should + // be sent to the ClientConn. + gsb.balancerPending.Balancer.(*mockBalancer).updateState(balancer.State{ + ConnectivityState: connectivity.Connecting, + }) + sCtx, sCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + select { + case <-tcc.NewStateCh: + t.Fatal("balancerPending reporting CONNECTING should not forward up to the ClientConn") + case <-sCtx.Done(): + } + + currBal := gsb.balancerCurrent.Balancer.(*mockBalancer) + // If the pending load balancer reports a state other than CONNECTING, the + // pending load balancer is logically warmed up, and the ClientConn should + // be updated with the State and Picker to start using the new policy. The + // pending load balancing policy should also be switched into the current + // load balancer. + gsb.balancerPending.Balancer.(*mockBalancer).updateState(balancer.State{ + ConnectivityState: connectivity.Ready, + Picker: &neverErrPicker{}, + }) + + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for a UpdateState call on the ClientConn") + case state := <-tcc.NewStateCh: + if state != connectivity.Ready { + t.Fatalf("pending balancer reports connectivity state %v, want %v", state, connectivity.Ready) + } + } + + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for a UpdateState call on the ClientConn") + case picker := <-tcc.NewPickerCh: + // This picker should be the recent one sent from UpdateState(), a never + // err picker, not the nil picker from two updateState() calls previous. + if picker == nil { + t.Fatalf("ClientConn should have received a never err picker, which is the most recent picker, from an UpdateState call") + } + if _, err := picker.Pick(balancer.PickInfo{}); err != nil { + t.Fatalf("ClientConn should have received a never err picker, which is the most recent picker, from an UpdateState call") + } + } + // The current balancer should be closed as a result of the swap. + if err := currBal.waitForClose(ctx); err != nil { + t.Fatal(err) + } +} + +// TestCurrentNotReadyPendingUpdate tests the scenario where there is a current +// and pending load balancer setup in the graceful switch load balancer, and the +// current LB is not in the connectivity state READY. Any update from the +// pending load balancer should cause the graceful switch load balancer to swap +// the pending into current, and update the ClientConn with the pending load +// balancers state. +func (s) TestCurrentNotReadyPendingUpdate(t *testing.T) { + tcc, gsb := setup(t) + gsb.SwitchTo(mockBalancerBuilder1{}) + gsb.SwitchTo(mockBalancerBuilder1{}) + if gsb.balancerPending == nil { + t.Fatal("pending balancer not populated after another call to SwitchTo()") + } + currBal := gsb.balancerCurrent.Balancer.(*mockBalancer) + // Due to the current load balancer not being in state READY, any update + // from the pending load balancer should cause that update to be forwarded + // to the ClientConn and also cause the pending load balancer to swap into + // the current one. + gsb.balancerPending.Balancer.(*mockBalancer).updateState(balancer.State{ + ConnectivityState: connectivity.Connecting, + Picker: &neverErrPicker{}, + }) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + select { + case <-ctx.Done(): + t.Fatalf("timeout waiting for an UpdateState call on the ClientConn") + case state := <-tcc.NewStateCh: + if state != connectivity.Connecting { + t.Fatalf("ClientConn received connectivity state %v, want %v (from pending)", state, connectivity.Connecting) + } + } + select { + case <-ctx.Done(): + t.Fatalf("timeout waiting for an UpdateState call on the ClientConn") + case picker := <-tcc.NewPickerCh: + // Should receive a never err picker. + if _, err := picker.Pick(balancer.PickInfo{}); err != nil { + t.Fatalf("ClientConn should have received a never err picker from an UpdateState call") + } + } + + // The current balancer should be closed as a result of the swap. + if err := currBal.waitForClose(ctx); err != nil { + t.Fatal(err) + } +} + +// TestCurrentLeavingReady tests the scenario where there is a current and +// pending load balancer setup in the graceful switch load balancer, with the +// current load balancer being in the state READY, and the current load balancer +// then transitions into a state other than READY. This should cause the pending +// load balancer to swap into the current load balancer, and the ClientConn to +// be updated with the cached pending load balancing state. Also, once the +// current is cleared from the graceful switch load balancer, any updates sent +// should be intercepted and not forwarded to the ClientConn, as the balancer +// has already been cleared. +func (s) TestCurrentLeavingReady(t *testing.T) { + tcc, gsb := setup(t) + gsb.SwitchTo(mockBalancerBuilder1{}) + currBal := gsb.balancerCurrent.Balancer.(*mockBalancer) + currBal.updateState(balancer.State{ + ConnectivityState: connectivity.Ready, + }) + + gsb.SwitchTo(mockBalancerBuilder2{}) + // Sends CONNECTING, shouldn't make it's way to ClientConn. + gsb.balancerPending.Balancer.(*mockBalancer).updateState(balancer.State{ + ConnectivityState: connectivity.Connecting, + Picker: &neverErrPicker{}, + }) + + // The current balancer leaving READY should cause the pending balancer to + // swap to the current balancer. This swap from current to pending should + // also update the ClientConn with the pending balancers cached state and + // picker. + currBal.updateState(balancer.State{ + ConnectivityState: connectivity.Idle, + }) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for a UpdateState call on the ClientConn") + case state := <-tcc.NewStateCh: + if state != connectivity.Connecting { + t.Fatalf("current balancer reports connectivity state %v, want %v", state, connectivity.Connecting) + } + } + + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for a UpdateState call on the ClientConn") + case picker := <-tcc.NewPickerCh: + // Should receive a never err picker cached from pending LB's updateState() call, which + // was cached. + if _, err := picker.Pick(balancer.PickInfo{}); err != nil { + t.Fatalf("ClientConn should have received a never err picker, the cached picker, from an UpdateState call") + } + } + + // The current balancer should be closed as a result of the swap. + if err := currBal.waitForClose(ctx); err != nil { + t.Fatal(err) + } + + // The current balancer is now cleared from the graceful switch load + // balancer. Thus, any update from the old current should be intercepted by + // the graceful switch load balancer and not forward up to the ClientConn. + currBal.updateState(balancer.State{ + ConnectivityState: connectivity.Ready, + Picker: &neverErrPicker{}, + }) + + // This update should not be forwarded to the ClientConn. + sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + select { + case <-sCtx.Done(): + case <-tcc.NewStateCh: + t.Fatal("UpdateState() from a cleared balancer should not make it's way to ClientConn") + } + + if _, err := currBal.newSubConn([]resolver.Address{}, balancer.NewSubConnOptions{}); err == nil { + t.Fatal("newSubConn() from a cleared balancer should have returned an error") + } + + // This newSubConn call should also not reach the ClientConn. + sCtx, sCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + select { + case <-sCtx.Done(): + case <-tcc.NewSubConnCh: + t.Fatal("newSubConn() from a cleared balancer should not make it's way to ClientConn") + } +} + +// TestBalancerSubconns tests the SubConn functionality of the graceful switch +// load balancer. This tests the SubConn update flow in both directions, and +// make sure updates end up at the correct component. +func (s) TestBalancerSubconns(t *testing.T) { + tcc, gsb := setup(t) + gsb.SwitchTo(mockBalancerBuilder1{}) + gsb.SwitchTo(mockBalancerBuilder2{}) + + // A child balancer creating a new SubConn should eventually be forwarded to + // the ClientConn held by the graceful switch load balancer. + sc1, err := gsb.balancerCurrent.Balancer.(*mockBalancer).newSubConn([]resolver.Address{}, balancer.NewSubConnOptions{}) + if err != nil { + t.Fatalf("error constructing newSubConn in gsb: %v", err) + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for an NewSubConn call on the ClientConn") + case sc := <-tcc.NewSubConnCh: + if sc != sc1 { + t.Fatalf("NewSubConn, want %v, got %v", sc1, sc) + } + } + + // The other child balancer creating a new SubConn should also be eventually + // be forwarded to the ClientConn held by the graceful switch load balancer. + sc2, err := gsb.balancerPending.Balancer.(*mockBalancer).newSubConn([]resolver.Address{}, balancer.NewSubConnOptions{}) + if err != nil { + t.Fatalf("error constructing newSubConn in gsb: %v", err) + } + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for an NewSubConn call on the ClientConn") + case sc := <-tcc.NewSubConnCh: + if sc != sc2 { + t.Fatalf("NewSubConn, want %v, got %v", sc2, sc) + } + } + scState := balancer.SubConnState{ConnectivityState: connectivity.Ready} + // Updating the SubConnState for sc1 should cause the graceful switch + // balancer to forward the Update to balancerCurrent for sc1, as that is the + // balancer that created this SubConn. + sc1.(*testutils.TestSubConn).UpdateState(scState) + + // Updating the SubConnState for sc2 should cause the graceful switch + // balancer to forward the Update to balancerPending for sc2, as that is the + // balancer that created this SubConn. + sc2.(*testutils.TestSubConn).UpdateState(scState) + + // Updating the addresses for both SubConns and removing both SubConns + // should get forwarded to the ClientConn. + + // Updating the addresses for sc1 should get forwarded to the ClientConn. + gsb.balancerCurrent.Balancer.(*mockBalancer).updateAddresses(sc1, []resolver.Address{}) + + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for an UpdateAddresses call on the ClientConn") + case <-tcc.UpdateAddressesAddrsCh: + } + + // Updating the addresses for sc2 should also get forwarded to the ClientConn. + gsb.balancerPending.Balancer.(*mockBalancer).updateAddresses(sc2, []resolver.Address{}) + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for an UpdateAddresses call on the ClientConn") + case <-tcc.UpdateAddressesAddrsCh: + } + + // balancerCurrent removing sc1 should get forwarded to the ClientConn. + sc1.Shutdown() + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for an UpdateAddresses call on the ClientConn") + case sc := <-tcc.ShutdownSubConnCh: + if sc != sc1 { + t.Fatalf("ShutdownSubConn, want %v, got %v", sc1, sc) + } + } + // balancerPending removing sc2 should get forwarded to the ClientConn. + sc2.Shutdown() + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for an UpdateAddresses call on the ClientConn") + case sc := <-tcc.ShutdownSubConnCh: + if sc != sc2 { + t.Fatalf("ShutdownSubConn, want %v, got %v", sc2, sc) + } + } +} + +// TestBalancerClose tests the graceful switch balancer's Close() +// functionality. From the Close() call, the graceful switch balancer should +// shut down any created Subconns and Close() the current and pending load +// balancers. This Close() call should also cause any other events (calls to +// entrance functions) to be no-ops. +func (s) TestBalancerClose(t *testing.T) { + // Setup gsb balancer with current, pending, and one created SubConn on both + // current and pending. + tcc, gsb := setup(t) + gsb.SwitchTo(mockBalancerBuilder1{}) + gsb.SwitchTo(mockBalancerBuilder2{}) + + sc1, err := gsb.balancerCurrent.Balancer.(*mockBalancer).newSubConn([]resolver.Address{}, balancer.NewSubConnOptions{}) + // Will eventually get back a SubConn with an identifying property id 1 + if err != nil { + t.Fatalf("error constructing newSubConn in gsb: %v", err) + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for an NewSubConn call on the ClientConn") + case <-tcc.NewSubConnCh: + } + + sc2, err := gsb.balancerPending.Balancer.(*mockBalancer).newSubConn([]resolver.Address{}, balancer.NewSubConnOptions{}) + // Will eventually get back a SubConn with an identifying property id 2 + if err != nil { + t.Fatalf("error constructing newSubConn in gsb: %v", err) + } + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for an NewSubConn call on the ClientConn") + case <-tcc.NewSubConnCh: + } + + currBal := gsb.balancerCurrent.Balancer.(*mockBalancer) + pendBal := gsb.balancerPending.Balancer.(*mockBalancer) + + // Closing the graceful switch load balancer should lead to removing any + // created SubConns, and closing both the current and pending load balancer. + gsb.Close() + + // The order of SubConns the graceful switch load balancer tells the Client + // Conn to shut down is non deterministic, as it is stored in a + // map. However, the first SubConn shut down should be either sc1 or sc2. + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for an UpdateAddresses call on the ClientConn") + case sc := <-tcc.ShutdownSubConnCh: + if sc != sc1 && sc != sc2 { + t.Fatalf("ShutdownSubConn, want either %v or %v, got %v", sc1, sc2, sc) + } + } + + // The graceful switch load balancer should then tell the ClientConn to + // shut down the other SubConn. + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for an UpdateAddresses call on the ClientConn") + case sc := <-tcc.ShutdownSubConnCh: + if sc != sc1 && sc != sc2 { + t.Fatalf("ShutdownSubConn, want either %v or %v, got %v", sc1, sc2, sc) + } + } + + // The current balancer should get closed as a result of the graceful switch balancer being closed. + if err := currBal.waitForClose(ctx); err != nil { + t.Fatal(err) + } + // The pending balancer should also get closed as a result of the graceful switch balancer being closed. + if err := pendBal.waitForClose(ctx); err != nil { + t.Fatal(err) + } + + // Once the graceful switch load balancer has been closed, any entrance + // function should be a no-op and return errBalancerClosed if the function + // returns an error. + + // SwitchTo() should return an error due to the graceful switch load + // balancer having been closed already. + if err := gsb.SwitchTo(mockBalancerBuilder1{}); err != errBalancerClosed { + t.Fatalf("gsb.SwitchTo(%v) returned error %v, want %v", mockBalancerBuilder1{}, err, errBalancerClosed) + } + + // UpdateClientConnState() should return an error due to the graceful switch + // load balancer having been closed already. + ccs := balancer.ClientConnState{ + BalancerConfig: mockBalancerConfig{}, + } + if err := gsb.UpdateClientConnState(ccs); err != errBalancerClosed { + t.Fatalf("gsb.UpdateCLientConnState(%v) returned error %v, want %v", ccs, err, errBalancerClosed) + } + + // After the graceful switch load balancer has been closed, any resolver error + // shouldn't forward to either balancer, as the resolver error is a no-op + // and also even if not, the balancers should have been cleared from the + // graceful switch load balancer. + gsb.ResolverError(balancer.ErrBadResolverState) + sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + if err := currBal.waitForResolverError(sCtx, balancer.ErrBadResolverState); !strings.Contains(err.Error(), sCtx.Err().Error()) { + t.Fatal("the current balancer should not have received the resolver error after close") + } + sCtx, sCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + if err := pendBal.waitForResolverError(sCtx, balancer.ErrBadResolverState); !strings.Contains(err.Error(), sCtx.Err().Error()) { + t.Fatal("the pending balancer should not have received the resolver error after close") + } +} + +// TestResolverError tests the functionality of a Resolver Error. If there is a +// current balancer, but no pending, the error should be forwarded to the +// current balancer. If there is both a current and pending balancer, the error +// should be forwarded to only the pending balancer. +func (s) TestResolverError(t *testing.T) { + _, gsb := setup(t) + gsb.SwitchTo(mockBalancerBuilder1{}) + currBal := gsb.balancerCurrent.Balancer.(*mockBalancer) + // If there is only a current balancer present, the resolver error should be + // forwarded to the current balancer. + gsb.ResolverError(balancer.ErrBadResolverState) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := currBal.waitForResolverError(ctx, balancer.ErrBadResolverState); err != nil { + t.Fatal(err) + } + + gsb.SwitchTo(mockBalancerBuilder1{}) + + // If there is a pending balancer present, then a resolver error should be + // forwarded to only the pending balancer, not the current. + pendBal := gsb.balancerPending.Balancer.(*mockBalancer) + gsb.ResolverError(balancer.ErrBadResolverState) + + // The Resolver Error should not be forwarded to the current load balancer. + sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + if err := currBal.waitForResolverError(sCtx, balancer.ErrBadResolverState); !strings.Contains(err.Error(), sCtx.Err().Error()) { + t.Fatal("the current balancer should not have received the resolver error after close") + } + + // The Resolver Error should be forwarded to the pending load balancer. + if err := pendBal.waitForResolverError(ctx, balancer.ErrBadResolverState); err != nil { + t.Fatal(err) + } +} + +// TestPendingReplacedByAnotherPending tests the scenario where a graceful +// switch balancer has a current and pending load balancer, and receives a +// SwitchTo() call, which then replaces the pending. This should cause the +// graceful switch balancer to clear pending state, close old pending SubConns, +// and Close() the pending balancer being replaced. +func (s) TestPendingReplacedByAnotherPending(t *testing.T) { + tcc, gsb := setup(t) + gsb.SwitchTo(mockBalancerBuilder1{}) + currBal := gsb.balancerCurrent.Balancer.(*mockBalancer) + currBal.updateState(balancer.State{ + ConnectivityState: connectivity.Ready, + }) + + // Populate pending with a SwitchTo() call. + gsb.SwitchTo(mockBalancerBuilder2{}) + + pendBal := gsb.balancerPending.Balancer.(*mockBalancer) + sc1, err := pendBal.newSubConn([]resolver.Address{}, balancer.NewSubConnOptions{}) + if err != nil { + t.Fatalf("error constructing newSubConn in gsb: %v", err) + } + // This picker never returns an error, which can help this this test verify + // whether this cached state will get cleared on a new pending balancer + // (will replace it with a picker that always errors). + pendBal.updateState(balancer.State{ + ConnectivityState: connectivity.Connecting, + Picker: &neverErrPicker{}, + }) + + // Replace pending with a SwitchTo() call. + gsb.SwitchTo(mockBalancerBuilder2{}) + // The pending balancer being replaced should cause the graceful switch + // balancer to Shutdown() any created SubConns for the old pending balancer + // and also Close() the old pending balancer. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for a SubConn.Shutdown") + case sc := <-tcc.ShutdownSubConnCh: + if sc != sc1 { + t.Fatalf("ShutdownSubConn, want %v, got %v", sc1, sc) + } + } + + if err := pendBal.waitForClose(ctx); err != nil { + t.Fatal(err) + } + + // Switching the current out of READY should cause the pending LB to swap + // into current, causing the graceful switch balancer to update the + // ClientConn with the cached pending state. Since the new pending hasn't + // sent an Update, the default state with connectivity state CONNECTING and + // an errPicker should be sent to the ClientConn. + currBal.updateState(balancer.State{ + ConnectivityState: connectivity.Idle, + }) + + // The update should contain a default connectivity state CONNECTING for the + // state of the new pending LB policy. + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for an UpdateState() call on the ClientConn") + case state := <-tcc.NewStateCh: + if state != connectivity.Connecting { + t.Fatalf("UpdateState(), want connectivity state %v, got %v", connectivity.Connecting, state) + } + } + // The update should contain a default picker ErrPicker in the picker sent + // for the state of the new pending LB policy. + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for an UpdateState() call on the ClientConn") + case picker := <-tcc.NewPickerCh: + if _, err := picker.Pick(balancer.PickInfo{}); err != balancer.ErrNoSubConnAvailable { + t.Fatalf("ClientConn should have received a never err picker from an UpdateState call") + } + } +} + +// Picker which never errors here for test purposes (can fill up tests further up with this) +type neverErrPicker struct{} + +func (p *neverErrPicker) Pick(info balancer.PickInfo) (balancer.PickResult, error) { + return balancer.PickResult{}, nil +} + +// TestUpdateSubConnStateRace tests the race condition when the graceful switch +// load balancer receives a SubConnUpdate concurrently with an UpdateState() +// call, which can cause the balancer to forward the update to to be closed and +// cleared. The balancer API guarantees to never call any method the balancer +// after a Close() call, and the test verifies that doesn't happen within the +// graceful switch load balancer. +func (s) TestUpdateSubConnStateRace(t *testing.T) { + tcc, gsb := setup(t) + gsb.SwitchTo(verifyBalancerBuilder{}) + gsb.SwitchTo(mockBalancerBuilder1{}) + currBal := gsb.balancerCurrent.Balancer.(*verifyBalancer) + currBal.t = t + pendBal := gsb.balancerPending.Balancer.(*mockBalancer) + sc, err := currBal.newSubConn([]resolver.Address{}, balancer.NewSubConnOptions{}) + if err != nil { + t.Fatalf("error constructing newSubConn in gsb: %v", err) + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for an NewSubConn call on the ClientConn") + case <-tcc.NewSubConnCh: + } + // Spawn a goroutine that constantly calls UpdateSubConn for the current + // balancer, which will get deleted in this testing goroutine. + finished := make(chan struct{}) + go func() { + for { + select { + case <-finished: + return + default: + } + sc.(*testutils.TestSubConn).UpdateState(balancer.SubConnState{ + ConnectivityState: connectivity.Ready, + }) + } + }() + time.Sleep(time.Millisecond) + // This UpdateState call causes current to be closed/cleared. + pendBal.updateState(balancer.State{ + ConnectivityState: connectivity.Ready, + }) + // From this, either one of two things happen. Either the graceful switch + // load balancer doesn't Close() the current balancer before it forwards the + // SubConn update to the child, and the call gets forwarded down to the + // current balancer, or it can Close() the current balancer in between + // reading the balancer pointer and writing to it, and in that case the old + // current balancer should not be updated, as the balancer has already been + // closed and the balancer API guarantees it. + close(finished) +} + +// TestInlineCallbackInBuild tests the scenario where a balancer calls back into +// the balancer.ClientConn API inline from it's build function. +func (s) TestInlineCallbackInBuild(t *testing.T) { + tcc, gsb := setup(t) + // This build call should cause all of the inline updates to forward to the + // ClientConn. + gsb.SwitchTo(buildCallbackBalancerBuilder{}) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for an UpdateState() call on the ClientConn") + case <-tcc.NewStateCh: + } + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for a NewSubConn() call on the ClientConn") + case <-tcc.NewSubConnCh: + } + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for an UpdateAddresses() call on the ClientConn") + case <-tcc.UpdateAddressesAddrsCh: + } + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for a Shutdown() call on the SubConn") + case <-tcc.ShutdownSubConnCh: + } + oldCurrent := gsb.balancerCurrent.Balancer.(*buildCallbackBal) + + // Since the callback reports a state READY, this new inline balancer should + // be swapped to the current. + gsb.SwitchTo(buildCallbackBalancerBuilder{}) + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for an UpdateState() call on the ClientConn") + case <-tcc.NewStateCh: + } + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for a NewSubConn() call on the ClientConn") + case <-tcc.NewSubConnCh: + } + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for an UpdateAddresses() call on the ClientConn") + case <-tcc.UpdateAddressesAddrsCh: + } + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for a Shutdown() call on the SubConn") + case <-tcc.ShutdownSubConnCh: + } + + // The current balancer should be closed as a result of the swap. + if err := oldCurrent.waitForClose(ctx); err != nil { + t.Fatalf("error waiting for balancer close: %v", err) + } + + // The old balancer should be deprecated and any calls from it should be a no-op. + oldCurrent.newSubConn([]resolver.Address{}, balancer.NewSubConnOptions{}) + sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + select { + case <-tcc.NewSubConnCh: + t.Fatal("Deprecated LB calling NewSubConn() should not forward up to the ClientConn") + case <-sCtx.Done(): + } +} + +// TestExitIdle tests the ExitIdle operation on the Graceful Switch Balancer for +// both possible codepaths, one where the child implements ExitIdler interface +// and one where the child doesn't implement ExitIdler interface. +func (s) TestExitIdle(t *testing.T) { + _, gsb := setup(t) + // switch to a balancer that implements ExitIdle{} (will populate current). + gsb.SwitchTo(mockBalancerBuilder1{}) + currBal := gsb.balancerCurrent.Balancer.(*mockBalancer) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + // exitIdle on the Graceful Switch Balancer should get forwarded to the + // current child as it implements exitIdle. + gsb.ExitIdle() + if err := currBal.waitForExitIdle(ctx); err != nil { + t.Fatal(err) + } + + // switch to a balancer that doesn't implement ExitIdle{} (will populate + // pending). + gsb.SwitchTo(verifyBalancerBuilder{}) + // call exitIdle concurrently with newSubConn to make sure there is not a + // data race. + done := make(chan struct{}) + go func() { + gsb.ExitIdle() + close(done) + }() + pendBal := gsb.balancerPending.Balancer.(*verifyBalancer) + for i := 0; i < 10; i++ { + pendBal.newSubConn([]resolver.Address{}, balancer.NewSubConnOptions{}) + } + <-done +} + +const balancerName1 = "mock_balancer_1" +const balancerName2 = "mock_balancer_2" +const verifyBalName = "verifyNoSubConnUpdateAfterCloseBalancer" +const buildCallbackBalName = "callbackInBuildBalancer" + +type mockBalancerBuilder1 struct{} + +func (mockBalancerBuilder1) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { + return &mockBalancer{ + ccsCh: testutils.NewChannel(), + scStateCh: testutils.NewChannel(), + resolverErrCh: testutils.NewChannel(), + closeCh: testutils.NewChannel(), + exitIdleCh: testutils.NewChannel(), + cc: cc, + } +} + +func (mockBalancerBuilder1) Name() string { + return balancerName1 +} + +type mockBalancerConfig struct { + serviceconfig.LoadBalancingConfig +} + +// mockBalancer is a fake balancer used to verify different actions from +// the gracefulswitch. It contains a bunch of channels to signal different events +// to the test. +type mockBalancer struct { + // ccsCh is a channel used to signal the receipt of a ClientConn update. + ccsCh *testutils.Channel + // scStateCh is a channel used to signal the receipt of a SubConn update. + scStateCh *testutils.Channel + // resolverErrCh is a channel used to signal a resolver error. + resolverErrCh *testutils.Channel + // closeCh is a channel used to signal the closing of this balancer. + closeCh *testutils.Channel + // exitIdleCh is a channel used to signal the receipt of an ExitIdle call. + exitIdleCh *testutils.Channel + // Hold onto ClientConn wrapper to communicate with it + cc balancer.ClientConn +} + +type subConnWithState struct { + sc balancer.SubConn + state balancer.SubConnState +} + +func (mb1 *mockBalancer) UpdateClientConnState(ccs balancer.ClientConnState) error { + // Need to verify this call...use a channel?...all of these will need verification + mb1.ccsCh.Send(ccs) + return nil +} + +func (mb1 *mockBalancer) ResolverError(err error) { + mb1.resolverErrCh.Send(err) +} + +func (mb1 *mockBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { + panic(fmt.Sprintf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, state)) +} + +func (mb1 *mockBalancer) Close() { + mb1.closeCh.Send(struct{}{}) +} + +func (mb1 *mockBalancer) ExitIdle() { + mb1.exitIdleCh.Send(struct{}{}) +} + +// waitForClientConnUpdate verifies if the mockBalancer receives the +// provided ClientConnState within a reasonable amount of time. +func (mb1 *mockBalancer) waitForClientConnUpdate(ctx context.Context, wantCCS balancer.ClientConnState) error { + ccs, err := mb1.ccsCh.Receive(ctx) + if err != nil { + return fmt.Errorf("error waiting for ClientConnUpdate: %v", err) + } + gotCCS := ccs.(balancer.ClientConnState) + if diff := cmp.Diff(gotCCS, wantCCS, cmpopts.IgnoreFields(resolver.State{}, "Attributes")); diff != "" { + return fmt.Errorf("error in ClientConnUpdate: received unexpected ClientConnState, diff (-got +want): %v", diff) + } + return nil +} + +// waitForResolverError verifies if the mockBalancer receives the provided +// resolver error before the context expires. +func (mb1 *mockBalancer) waitForResolverError(ctx context.Context, wantErr error) error { + gotErr, err := mb1.resolverErrCh.Receive(ctx) + if err != nil { + return fmt.Errorf("error waiting for resolver error: %v", err) + } + if gotErr != wantErr { + return fmt.Errorf("received resolver error: %v, want %v", gotErr, wantErr) + } + return nil +} + +// waitForClose verifies that the mockBalancer is closed before the context +// expires. +func (mb1 *mockBalancer) waitForClose(ctx context.Context) error { + if _, err := mb1.closeCh.Receive(ctx); err != nil { + return fmt.Errorf("error waiting for Close(): %v", err) + } + return nil +} + +// waitForExitIdle verifies that ExitIdle gets called on the mockBalancer before +// the context expires. +func (mb1 *mockBalancer) waitForExitIdle(ctx context.Context) error { + if _, err := mb1.exitIdleCh.Receive(ctx); err != nil { + return fmt.Errorf("error waiting for ExitIdle(): %v", err) + } + return nil +} + +func (mb1 *mockBalancer) updateState(state balancer.State) { + mb1.cc.UpdateState(state) +} + +func (mb1 *mockBalancer) newSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (sc balancer.SubConn, err error) { + if opts.StateListener == nil { + opts.StateListener = func(state balancer.SubConnState) { + mb1.scStateCh.Send(subConnWithState{sc: sc, state: state}) + } + } + defer func() { + if sc != nil { + sc.Connect() + } + }() + return mb1.cc.NewSubConn(addrs, opts) +} + +func (mb1 *mockBalancer) updateAddresses(sc balancer.SubConn, addrs []resolver.Address) { + mb1.cc.UpdateAddresses(sc, addrs) +} + +type mockBalancerBuilder2 struct{} + +func (mockBalancerBuilder2) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { + return &mockBalancer{ + ccsCh: testutils.NewChannel(), + scStateCh: testutils.NewChannel(), + resolverErrCh: testutils.NewChannel(), + closeCh: testutils.NewChannel(), + cc: cc, + } +} + +func (mockBalancerBuilder2) Name() string { + return balancerName2 +} + +type verifyBalancerBuilder struct{} + +func (verifyBalancerBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { + return &verifyBalancer{ + closed: grpcsync.NewEvent(), + cc: cc, + } +} + +func (verifyBalancerBuilder) Name() string { + return verifyBalName +} + +// verifyBalancer is a balancer that verifies that after a Close() call, a +// StateListener() call never happens. +type verifyBalancer struct { + closed *grpcsync.Event + // Hold onto the ClientConn wrapper to communicate with it. + cc balancer.ClientConn + // To fail the test if StateListener gets called after Close(). + t *testing.T +} + +func (vb *verifyBalancer) UpdateClientConnState(ccs balancer.ClientConnState) error { + return nil +} + +func (vb *verifyBalancer) ResolverError(err error) {} + +func (vb *verifyBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { + panic(fmt.Sprintf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, state)) +} + +func (vb *verifyBalancer) Close() { + vb.closed.Fire() +} + +func (vb *verifyBalancer) newSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (sc balancer.SubConn, err error) { + if opts.StateListener == nil { + opts.StateListener = func(state balancer.SubConnState) { + if vb.closed.HasFired() { + vb.t.Fatalf("StateListener(%+v) was called after Close(), which breaks the balancer API", state) + } + } + } + defer func() { sc.Connect() }() + return vb.cc.NewSubConn(addrs, opts) +} + +type buildCallbackBalancerBuilder struct{} + +func (buildCallbackBalancerBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { + b := &buildCallbackBal{ + cc: cc, + closeCh: testutils.NewChannel(), + } + b.updateState(balancer.State{ + ConnectivityState: connectivity.Connecting, + }) + sc, err := b.newSubConn([]resolver.Address{}, balancer.NewSubConnOptions{}) + if err != nil { + return nil + } + b.updateAddresses(sc, []resolver.Address{}) + sc.Shutdown() + return b +} + +func (buildCallbackBalancerBuilder) Name() string { + return buildCallbackBalName +} + +type buildCallbackBal struct { + // Hold onto the ClientConn wrapper to communicate with it. + cc balancer.ClientConn + // closeCh is a channel used to signal the closing of this balancer. + closeCh *testutils.Channel +} + +func (bcb *buildCallbackBal) UpdateClientConnState(ccs balancer.ClientConnState) error { + return nil +} + +func (bcb *buildCallbackBal) ResolverError(err error) {} + +func (bcb *buildCallbackBal) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { + panic(fmt.Sprintf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, state)) +} + +func (bcb *buildCallbackBal) Close() { + bcb.closeCh.Send(struct{}{}) +} + +func (bcb *buildCallbackBal) updateState(state balancer.State) { + bcb.cc.UpdateState(state) +} + +func (bcb *buildCallbackBal) newSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (sc balancer.SubConn, err error) { + defer func() { + if sc != nil { + sc.Connect() + } + }() + return bcb.cc.NewSubConn(addrs, opts) +} + +func (bcb *buildCallbackBal) updateAddresses(sc balancer.SubConn, addrs []resolver.Address) { + bcb.cc.UpdateAddresses(sc, addrs) +} + +// waitForClose verifies that the mockBalancer is closed before the context +// expires. +func (bcb *buildCallbackBal) waitForClose(ctx context.Context) error { + if _, err := bcb.closeCh.Receive(ctx); err != nil { + return err + } + return nil +} diff --git a/internal/balancer/nop/nop.go b/internal/balancer/nop/nop.go new file mode 100644 index 000000000000..0c96f1b81186 --- /dev/null +++ b/internal/balancer/nop/nop.go @@ -0,0 +1,62 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package nop implements a balancer with all of its balancer operations as +// no-ops, other than returning a Transient Failure Picker on a Client Conn +// update. +package nop + +import ( + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/base" + "google.golang.org/grpc/connectivity" +) + +// bal is a balancer with all of its balancer operations as no-ops, other than +// returning a Transient Failure Picker on a Client Conn update. +type bal struct { + cc balancer.ClientConn + err error +} + +// NewBalancer returns a no-op balancer. +func NewBalancer(cc balancer.ClientConn, err error) balancer.Balancer { + return &bal{ + cc: cc, + err: err, + } +} + +// UpdateClientConnState updates the bal's Client Conn with an Error Picker +// and a Connectivity State of TRANSIENT_FAILURE. +func (b *bal) UpdateClientConnState(_ balancer.ClientConnState) error { + b.cc.UpdateState(balancer.State{ + Picker: base.NewErrPicker(b.err), + ConnectivityState: connectivity.TransientFailure, + }) + return nil +} + +// ResolverError is a no-op. +func (b *bal) ResolverError(_ error) {} + +// UpdateSubConnState is a no-op. +func (b *bal) UpdateSubConnState(_ balancer.SubConn, _ balancer.SubConnState) {} + +// Close is a no-op. +func (b *bal) Close() {} diff --git a/internal/balancer/stub/stub.go b/internal/balancer/stub/stub.go index 2ddaf5f804d2..71e3cb619d9c 100644 --- a/internal/balancer/stub/stub.go +++ b/internal/balancer/stub/stub.go @@ -19,7 +19,13 @@ // Package stub implements a balancer for testing purposes. package stub -import "google.golang.org/grpc/balancer" +import ( + "encoding/json" + "fmt" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/serviceconfig" +) // BalancerFuncs contains all balancer.Balancer functions with a preceding // *BalancerData parameter for passing additional instance information. Any @@ -28,11 +34,13 @@ type BalancerFuncs struct { // Init is called after ClientConn and BuildOptions are set in // BalancerData. It may be used to initialize BalancerData.Data. Init func(*BalancerData) + // ParseConfig is used for parsing LB configs, if specified. + ParseConfig func(json.RawMessage) (serviceconfig.LoadBalancingConfig, error) UpdateClientConnState func(*BalancerData, balancer.ClientConnState) error ResolverError func(*BalancerData, error) - UpdateSubConnState func(*BalancerData, balancer.SubConn, balancer.SubConnState) Close func(*BalancerData) + ExitIdle func(*BalancerData) } // BalancerData contains data relevant to a stub balancer. @@ -42,14 +50,10 @@ type BalancerData struct { // BuildOptions is set by the builder. BuildOptions balancer.BuildOptions // Data may be used to store arbitrary user data. - Data interface{} + Data any } type bal struct { - // TODO: Remove this once the legacy balancer API is removed. See - // https://github.com/grpc/grpc-go/pull/3431. - balancer.Balancer - bf BalancerFuncs bd *BalancerData } @@ -68,9 +72,7 @@ func (b *bal) ResolverError(e error) { } func (b *bal) UpdateSubConnState(sc balancer.SubConn, scs balancer.SubConnState) { - if b.bf.UpdateSubConnState != nil { - b.bf.UpdateSubConnState(b.bd, sc, scs) - } + panic(fmt.Sprintf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, scs)) } func (b *bal) Close() { @@ -79,6 +81,12 @@ func (b *bal) Close() { } } +func (b *bal) ExitIdle() { + if b.bf.ExitIdle != nil { + b.bf.ExitIdle(b.bd) + } +} + type bb struct { name string bf BalancerFuncs @@ -94,6 +102,13 @@ func (bb bb) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer. func (bb bb) Name() string { return bb.name } +func (bb bb) ParseConfig(lbCfg json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + if bb.bf.ParseConfig != nil { + return bb.bf.ParseConfig(lbCfg) + } + return nil, nil +} + // Register registers a stub balancer builder which will call the provided // functions. The name used should be unique. func Register(name string, bf BalancerFuncs) { diff --git a/xds/internal/balancer/balancergroup/balancergroup.go b/internal/balancergroup/balancergroup.go similarity index 63% rename from xds/internal/balancer/balancergroup/balancergroup.go rename to internal/balancergroup/balancergroup.go index 84de4b63ecfe..8177fb58da9a 100644 --- a/xds/internal/balancer/balancergroup/balancergroup.go +++ b/internal/balancergroup/balancergroup.go @@ -23,14 +23,12 @@ import ( "sync" "time" - orcapb "github.com/cncf/udpa/go/udpa/data/orca/v1" "google.golang.org/grpc/balancer" "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/internal/balancer/gracefulswitch" "google.golang.org/grpc/internal/cache" "google.golang.org/grpc/internal/grpclog" "google.golang.org/grpc/resolver" - "google.golang.org/grpc/xds/internal" - "google.golang.org/grpc/xds/internal/balancer/lrs" ) // subBalancerWrapper is used to keep the configurations that will be used to start @@ -51,7 +49,7 @@ type subBalancerWrapper struct { // of the actions are forwarded to the parent ClientConn with no change. // Some are forward to balancer group with the sub-balancer ID. balancer.ClientConn - id internal.LocalityID + id string group *BalancerGroup mu sync.Mutex @@ -60,6 +58,8 @@ type subBalancerWrapper struct { // The static part of sub-balancer. Keeps balancerBuilders and addresses. // To be used when restarting sub-balancer. builder balancer.Builder + // Options to be passed to sub-balancer at the time of creation. + buildOpts balancer.BuildOptions // ccState is a cache of the addresses/balancer config, so when the balancer // is restarted after close, it will get the previous update. It's a pointer // and is set to nil at init, so when the balancer is built for the first @@ -68,7 +68,7 @@ type subBalancerWrapper struct { ccState *balancer.ClientConnState // The dynamic part of sub-balancer. Only used when balancer group is // started. Gets cleared when sub-balancer is closed. - balancer balancer.Balancer + balancer *gracefulswitch.Balancer } // UpdateState overrides balancer.ClientConn, to keep state and picker. @@ -94,24 +94,25 @@ func (sbc *subBalancerWrapper) updateBalancerStateWithCachedPicker() { } func (sbc *subBalancerWrapper) startBalancer() { - b := sbc.builder.Build(sbc, balancer.BuildOptions{}) - sbc.group.logger.Infof("Created child policy %p of type %v", b, sbc.builder.Name()) - sbc.balancer = b + if sbc.balancer == nil { + sbc.balancer = gracefulswitch.NewBalancer(sbc, sbc.buildOpts) + } + sbc.group.logger.Infof("Creating child policy of type %q for locality %q", sbc.builder.Name(), sbc.id) + sbc.balancer.SwitchTo(sbc.builder) if sbc.ccState != nil { - b.UpdateClientConnState(*sbc.ccState) + sbc.balancer.UpdateClientConnState(*sbc.ccState) } } -func (sbc *subBalancerWrapper) updateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { +// exitIdle invokes the sub-balancer's ExitIdle method. Returns a boolean +// indicating whether or not the operation was completed. +func (sbc *subBalancerWrapper) exitIdle() (complete bool) { b := sbc.balancer if b == nil { - // This sub-balancer was closed. This can happen when EDS removes a - // locality. The balancer for this locality was already closed, and the - // SubConns are being deleted. But SubConn state change can still - // happen. - return + return true } - b.UpdateSubConnState(sc, state) + b.ExitIdle() + return true } func (sbc *subBalancerWrapper) updateClientConnState(s balancer.ClientConnState) error { @@ -147,7 +148,24 @@ func (sbc *subBalancerWrapper) resolverError(err error) { b.ResolverError(err) } +func (sbc *subBalancerWrapper) gracefulSwitch(builder balancer.Builder) { + sbc.builder = builder + b := sbc.balancer + // Even if you get an add and it persists builder but doesn't start + // balancer, this would leave graceful switch being nil, in which we are + // correctly overwriting with the recent builder here as well to use later. + // The graceful switch balancer's presence is an invariant of whether the + // balancer group is closed or not (if closed, nil, if started, present). + if sbc.balancer != nil { + sbc.group.logger.Infof("Switching child policy %v to type %v", sbc.id, sbc.builder.Name()) + b.SwitchTo(sbc.builder) + } +} + func (sbc *subBalancerWrapper) stopBalancer() { + if sbc.balancer == nil { + return + } sbc.balancer.Close() sbc.balancer = nil } @@ -158,20 +176,19 @@ func (sbc *subBalancerWrapper) stopBalancer() { // intended to be used directly as a balancer. It's expected to be used as a // sub-balancer manager by a high level balancer. // -// Updates from ClientConn are forwarded to sub-balancers -// - service config update -// - Not implemented -// - address update -// - subConn state change -// - find the corresponding balancer and forward +// Updates from ClientConn are forwarded to sub-balancers +// - service config update +// - address update +// - subConn state change +// - find the corresponding balancer and forward // -// Actions from sub-balances are forwarded to parent ClientConn -// - new/remove SubConn -// - picker update and health states change -// - sub-pickers are sent to an aggregator provided by the parent, which -// will group them into a group-picker. The aggregated connectivity state is -// also handled by the aggregator. -// - resolveNow +// Actions from sub-balances are forwarded to parent ClientConn +// - new/remove SubConn +// - picker update and health states change +// - sub-pickers are sent to an aggregator provided by the parent, which +// will group them into a group-picker. The aggregated connectivity state is +// also handled by the aggregator. +// - resolveNow // // Sub-balancers are only built when the balancer group is started. If the // balancer group is closed, the sub-balancers are also closed. And it's @@ -179,8 +196,8 @@ func (sbc *subBalancerWrapper) stopBalancer() { // balancer group. type BalancerGroup struct { cc balancer.ClientConn + buildOpts balancer.BuildOptions logger *grpclog.PrefixLogger - loadStore lrs.Store // stateAggregator is where the state/picker updates will be sent to. It's // provided by the parent balancer, to build a picker with all the @@ -195,9 +212,10 @@ type BalancerGroup struct { // to sub-balancers after they are closed. outgoingMu sync.Mutex outgoingStarted bool - idToBalancerConfig map[internal.LocalityID]*subBalancerWrapper - // Cache for sub-balancers when they are removed. - balancerCache *cache.TimeoutCache + idToBalancerConfig map[string]*subBalancerWrapper + // Cache for sub-balancers when they are removed. This is `nil` if caching + // is disabled by passing `0` for Options.SubBalancerCloseTimeout`. + deletedBalancerCache *cache.TimeoutCache // incomingMu is to make sure this balancer group doesn't send updates to cc // after it's closed. @@ -215,7 +233,7 @@ type BalancerGroup struct { // incomingMu guards all operations in the direction: // Sub-balancer-->ClientConn. Including NewSubConn, RemoveSubConn. It also // guards the map from SubConn to balancer ID, so updateSubConnState needs - // to hold it shortly to find the sub-balancer to forward the update. + // to hold it shortly to potentially delete from the map. // // UpdateState is called by the balancer state aggretator, and it will // decide when and whether to call. @@ -227,25 +245,40 @@ type BalancerGroup struct { scToSubBalancer map[balancer.SubConn]*subBalancerWrapper } -// DefaultSubBalancerCloseTimeout is defined as a variable instead of const for -// testing. -// -// TODO: make it a parameter for New(). -var DefaultSubBalancerCloseTimeout = 15 * time.Minute +// Options wraps the arguments to be passed to the BalancerGroup ctor. +type Options struct { + // CC is a reference to the parent balancer.ClientConn. + CC balancer.ClientConn + // BuildOpts contains build options to be used when creating sub-balancers. + BuildOpts balancer.BuildOptions + // StateAggregator is an implementation of the BalancerStateAggregator + // interface to aggregate picker and connectivity states from sub-balancers. + StateAggregator BalancerStateAggregator + // Logger is a group specific prefix logger. + Logger *grpclog.PrefixLogger + // SubBalancerCloseTimeout is the amount of time deleted sub-balancers spend + // in the idle cache. A value of zero here disables caching of deleted + // sub-balancers. + SubBalancerCloseTimeout time.Duration +} // New creates a new BalancerGroup. Note that the BalancerGroup // needs to be started to work. -func New(cc balancer.ClientConn, stateAggregator BalancerStateAggregator, loadStore lrs.Store, logger *grpclog.PrefixLogger) *BalancerGroup { - return &BalancerGroup{ - cc: cc, - logger: logger, - loadStore: loadStore, - - stateAggregator: stateAggregator, +func New(opts Options) *BalancerGroup { + var bc *cache.TimeoutCache + if opts.SubBalancerCloseTimeout != time.Duration(0) { + bc = cache.NewTimeoutCache(opts.SubBalancerCloseTimeout) + } - idToBalancerConfig: make(map[internal.LocalityID]*subBalancerWrapper), - balancerCache: cache.NewTimeoutCache(DefaultSubBalancerCloseTimeout), - scToSubBalancer: make(map[balancer.SubConn]*subBalancerWrapper), + return &BalancerGroup{ + cc: opts.CC, + buildOpts: opts.BuildOpts, + stateAggregator: opts.StateAggregator, + logger: opts.Logger, + + deletedBalancerCache: bc, + idToBalancerConfig: make(map[string]*subBalancerWrapper), + scToSubBalancer: make(map[balancer.SubConn]*subBalancerWrapper), } } @@ -273,15 +306,28 @@ func (bg *BalancerGroup) Start() { bg.outgoingMu.Unlock() } -// Add adds a balancer built by builder to the group, with given id. -func (bg *BalancerGroup) Add(id internal.LocalityID, builder balancer.Builder) { +// AddWithClientConn adds a balancer with the given id to the group. The +// balancer is built with a balancer builder registered with balancerName. The +// given ClientConn is passed to the newly built balancer instead of the +// onepassed to balancergroup.New(). +// +// TODO: Get rid of the existing Add() API and replace it with this. +func (bg *BalancerGroup) AddWithClientConn(id, balancerName string, cc balancer.ClientConn) error { + bg.logger.Infof("Adding child policy of type %q for locality %q", balancerName, id) + builder := balancer.Get(balancerName) + if builder == nil { + return fmt.Errorf("unregistered balancer name %q", balancerName) + } + // Store data in static map, and then check to see if bg is started. bg.outgoingMu.Lock() + defer bg.outgoingMu.Unlock() var sbc *subBalancerWrapper // If outgoingStarted is true, search in the cache. Otherwise, cache is - // guaranteed to be empty, searching is unnecessary. - if bg.outgoingStarted { - if old, ok := bg.balancerCache.Remove(id); ok { + // guaranteed to be empty, searching is unnecessary. Also, skip the cache if + // caching is disabled. + if bg.outgoingStarted && bg.deletedBalancerCache != nil { + if old, ok := bg.deletedBalancerCache.Remove(id); ok { sbc, _ = old.(*subBalancerWrapper) if sbc != nil && sbc.builder != builder { // If the sub-balancer in cache was built with a different @@ -301,10 +347,11 @@ func (bg *BalancerGroup) Add(id internal.LocalityID, builder balancer.Builder) { } if sbc == nil { sbc = &subBalancerWrapper{ - ClientConn: bg.cc, + ClientConn: cc, id: id, group: bg, builder: builder, + buildOpts: bg.buildOpts, } if bg.outgoingStarted { // Only start the balancer if bg is started. Otherwise, we only keep the @@ -317,6 +364,30 @@ func (bg *BalancerGroup) Add(id internal.LocalityID, builder balancer.Builder) { sbc.updateBalancerStateWithCachedPicker() } bg.idToBalancerConfig[id] = sbc + return nil +} + +// Add adds a balancer built by builder to the group, with given id. +func (bg *BalancerGroup) Add(id string, builder balancer.Builder) { + bg.AddWithClientConn(id, builder.Name(), bg.cc) +} + +// UpdateBuilder updates the builder for a current child, starting the Graceful +// Switch process for that child. +// +// TODO: update this API to take the name of the new builder instead. +func (bg *BalancerGroup) UpdateBuilder(id string, builder balancer.Builder) { + bg.outgoingMu.Lock() + // This does not deal with the balancer cache because this call should come + // after an Add call for a given child balancer. If the child is removed, + // the caller will call Add if the child balancer comes back which would + // then deal with the balancer cache. + sbc := bg.idToBalancerConfig[id] + if sbc == nil { + // simply ignore it if not present, don't error + return + } + sbc.gracefulSwitch(builder) bg.outgoingMu.Unlock() } @@ -325,27 +396,49 @@ func (bg *BalancerGroup) Add(id internal.LocalityID, builder balancer.Builder) { // But doesn't close the balancer. The balancer is kept in a cache, and will be // closed after timeout. Cleanup work (closing sub-balancer and removing // subconns) will be done after timeout. -func (bg *BalancerGroup) Remove(id internal.LocalityID) { +func (bg *BalancerGroup) Remove(id string) { + bg.logger.Infof("Removing child policy for locality %q", id) + bg.outgoingMu.Lock() - if sbToRemove, ok := bg.idToBalancerConfig[id]; ok { - if bg.outgoingStarted { - bg.balancerCache.Add(id, sbToRemove, func() { - // After timeout, when sub-balancer is removed from cache, need - // to close the underlying sub-balancer, and remove all its - // subconns. - bg.outgoingMu.Lock() - if bg.outgoingStarted { - sbToRemove.stopBalancer() - } - bg.outgoingMu.Unlock() - bg.cleanupSubConns(sbToRemove) - }) - } - delete(bg.idToBalancerConfig, id) - } else { + + sbToRemove, ok := bg.idToBalancerConfig[id] + if !ok { bg.logger.Infof("balancer group: trying to remove a non-existing locality from balancer group: %v", id) + bg.outgoingMu.Unlock() + return + } + + // Unconditionally remove the sub-balancer config from the map. + delete(bg.idToBalancerConfig, id) + if !bg.outgoingStarted { + // Nothing needs to be done here, since we wouldn't have created the + // sub-balancer. + bg.outgoingMu.Unlock() + return + } + + if bg.deletedBalancerCache != nil { + bg.deletedBalancerCache.Add(id, sbToRemove, func() { + // A sub-balancer evicted from the timeout cache needs to closed + // and its subConns need to removed, unconditionally. There is a + // possibility that a sub-balancer might be removed (thereby + // moving it to the cache) around the same time that the + // balancergroup is closed, and by the time we get here the + // balancergroup might be closed. Check for `outgoingStarted == + // true` at that point can lead to a leaked sub-balancer. + bg.outgoingMu.Lock() + sbToRemove.stopBalancer() + bg.outgoingMu.Unlock() + bg.cleanupSubConns(sbToRemove) + }) + bg.outgoingMu.Unlock() + return } + + // Remove the sub-balancer with immediate effect if we are not caching. + sbToRemove.stopBalancer() bg.outgoingMu.Unlock() + bg.cleanupSubConns(sbToRemove) } // bg.remove(id) doesn't do cleanup for the sub-balancer. This function does @@ -362,21 +455,30 @@ func (bg *BalancerGroup) cleanupSubConns(config *subBalancerWrapper) { // sub-balancers. for sc, b := range bg.scToSubBalancer { if b == config { - bg.cc.RemoveSubConn(sc) delete(bg.scToSubBalancer, sc) } } bg.incomingMu.Unlock() } +// connect attempts to connect to all subConns belonging to sb. +func (bg *BalancerGroup) connect(sb *subBalancerWrapper) { + bg.incomingMu.Lock() + for sc, b := range bg.scToSubBalancer { + if b == sb { + sc.Connect() + } + } + bg.incomingMu.Unlock() +} + // Following are actions from the parent grpc.ClientConn, forward to sub-balancers. -// UpdateSubConnState handles the state for the subconn. It finds the -// corresponding balancer and forwards the update. -func (bg *BalancerGroup) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { +// updateSubConnState forwards the update to cb and updates scToSubBalancer if +// needed. +func (bg *BalancerGroup) updateSubConnState(sc balancer.SubConn, state balancer.SubConnState, cb func(balancer.SubConnState)) { bg.incomingMu.Lock() - config, ok := bg.scToSubBalancer[sc] - if !ok { + if _, ok := bg.scToSubBalancer[sc]; !ok { bg.incomingMu.Unlock() return } @@ -387,13 +489,21 @@ func (bg *BalancerGroup) UpdateSubConnState(sc balancer.SubConn, state balancer. bg.incomingMu.Unlock() bg.outgoingMu.Lock() - config.updateSubConnState(sc, state) + if cb != nil { + cb(state) + } bg.outgoingMu.Unlock() } +// UpdateSubConnState handles the state for the subconn. It finds the +// corresponding balancer and forwards the update. +func (bg *BalancerGroup) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { + bg.logger.Errorf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, state) +} + // UpdateClientConnState handles ClientState (including balancer config and // addresses) from resolver. It finds the balancer and forwards the update. -func (bg *BalancerGroup) UpdateClientConnState(id internal.LocalityID, s balancer.ClientConnState) error { +func (bg *BalancerGroup) UpdateClientConnState(id string, s balancer.ClientConnState) error { bg.outgoingMu.Lock() defer bg.outgoingMu.Unlock() if config, ok := bg.idToBalancerConfig[id]; ok { @@ -429,6 +539,9 @@ func (bg *BalancerGroup) newSubConn(config *subBalancerWrapper, addrs []resolver bg.incomingMu.Unlock() return nil, fmt.Errorf("NewSubConn is called after balancer group is closed") } + var sc balancer.SubConn + oldListener := opts.StateListener + opts.StateListener = func(state balancer.SubConnState) { bg.updateSubConnState(sc, state, oldListener) } sc, err := bg.cc.NewSubConn(addrs, opts) if err != nil { bg.incomingMu.Unlock() @@ -442,12 +555,8 @@ func (bg *BalancerGroup) newSubConn(config *subBalancerWrapper, addrs []resolver // updateBalancerState: forward the new state to balancer state aggregator. The // aggregator will create an aggregated picker and an aggregated connectivity // state, then forward to ClientConn. -func (bg *BalancerGroup) updateBalancerState(id internal.LocalityID, state balancer.State) { +func (bg *BalancerGroup) updateBalancerState(id string, state balancer.State) { bg.logger.Infof("Balancer state update from locality %v, new state: %+v", id, state) - if bg.loadStore != nil { - // Only wrap the picker to do load reporting if loadStore was set. - state.Picker = newLoadReportPicker(state.Picker, id, bg.loadStore) - } // Send new state to the aggregator, without holding the incomingMu. // incomingMu is to protect all calls to the parent ClientConn, this update @@ -466,12 +575,18 @@ func (bg *BalancerGroup) Close() { bg.incomingStarted = false // Also remove all SubConns. for sc := range bg.scToSubBalancer { - bg.cc.RemoveSubConn(sc) + sc.Shutdown() delete(bg.scToSubBalancer, sc) } } bg.incomingMu.Unlock() + // Clear(true) runs clear function to close sub-balancers in cache. It + // must be called out of outgoing mutex. + if bg.deletedBalancerCache != nil { + bg.deletedBalancerCache.Clear(true) + } + bg.outgoingMu.Lock() if bg.outgoingStarted { bg.outgoingStarted = false @@ -480,52 +595,29 @@ func (bg *BalancerGroup) Close() { } } bg.outgoingMu.Unlock() - // Clear(true) runs clear function to close sub-balancers in cache. It - // must be called out of outgoing mutex. - bg.balancerCache.Clear(true) } -const ( - serverLoadCPUName = "cpu_utilization" - serverLoadMemoryName = "mem_utilization" -) - -type loadReportPicker struct { - p balancer.Picker - - id internal.LocalityID - loadStore lrs.Store -} - -func newLoadReportPicker(p balancer.Picker, id internal.LocalityID, loadStore lrs.Store) *loadReportPicker { - return &loadReportPicker{ - p: p, - id: id, - loadStore: loadStore, +// ExitIdle should be invoked when the parent LB policy's ExitIdle is invoked. +// It will trigger this on all sub-balancers, or reconnect their subconns if +// not supported. +func (bg *BalancerGroup) ExitIdle() { + bg.outgoingMu.Lock() + for _, config := range bg.idToBalancerConfig { + if !config.exitIdle() { + bg.connect(config) + } } + bg.outgoingMu.Unlock() } -func (lrp *loadReportPicker) Pick(info balancer.PickInfo) (balancer.PickResult, error) { - res, err := lrp.p.Pick(info) - if lrp.loadStore != nil && err == nil { - lrp.loadStore.CallStarted(lrp.id) - td := res.Done - res.Done = func(info balancer.DoneInfo) { - lrp.loadStore.CallFinished(lrp.id, info.Err) - if load, ok := info.ServerLoad.(*orcapb.OrcaLoadReport); ok { - lrp.loadStore.CallServerLoad(lrp.id, serverLoadCPUName, load.CpuUtilization) - lrp.loadStore.CallServerLoad(lrp.id, serverLoadMemoryName, load.MemUtilization) - for n, d := range load.RequestCost { - lrp.loadStore.CallServerLoad(lrp.id, n, d) - } - for n, d := range load.Utilization { - lrp.loadStore.CallServerLoad(lrp.id, n, d) - } - } - if td != nil { - td(info) - } +// ExitIdleOne instructs the sub-balancer `id` to exit IDLE state, if +// appropriate and possible. +func (bg *BalancerGroup) ExitIdleOne(id string) { + bg.outgoingMu.Lock() + if config := bg.idToBalancerConfig[id]; config != nil { + if !config.exitIdle() { + bg.connect(config) } } - return res, err + bg.outgoingMu.Unlock() } diff --git a/internal/balancergroup/balancergroup_test.go b/internal/balancergroup/balancergroup_test.go new file mode 100644 index 000000000000..c57cf60ca84b --- /dev/null +++ b/internal/balancergroup/balancergroup_test.go @@ -0,0 +1,673 @@ +/* + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package balancergroup + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/roundrobin" + "google.golang.org/grpc/balancer/weightedtarget/weightedaggregator" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/balancer/stub" + "google.golang.org/grpc/internal/channelz" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/resolver" +) + +const ( + defaultTestTimeout = 5 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond +) + +var ( + rrBuilder = balancer.Get(roundrobin.Name) + testBalancerIDs = []string{"b1", "b2", "b3"} + testBackendAddrs []resolver.Address +) + +const testBackendAddrsCount = 12 + +func init() { + for i := 0; i < testBackendAddrsCount; i++ { + testBackendAddrs = append(testBackendAddrs, resolver.Address{Addr: fmt.Sprintf("%d.%d.%d.%d:%d", i, i, i, i, i)}) + } +} + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// Create a new balancer group, add balancer and backends, but not start. +// - b1, weight 2, backends [0,1] +// - b2, weight 1, backends [2,3] +// Start the balancer group and check behavior. +// +// Close the balancer group, call add/remove/change weight/change address. +// - b2, weight 3, backends [0,3] +// - b3, weight 1, backends [1,2] +// Start the balancer group again and check for behavior. +func (s) TestBalancerGroup_start_close(t *testing.T) { + cc := testutils.NewTestClientConn(t) + gator := weightedaggregator.New(cc, nil, testutils.NewTestWRR) + gator.Start() + bg := New(Options{ + CC: cc, + BuildOpts: balancer.BuildOptions{}, + StateAggregator: gator, + Logger: nil, + SubBalancerCloseTimeout: time.Duration(0), + }) + + // Add two balancers to group and send two resolved addresses to both + // balancers. + gator.Add(testBalancerIDs[0], 2) + bg.Add(testBalancerIDs[0], rrBuilder) + bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:2]}}) + gator.Add(testBalancerIDs[1], 1) + bg.Add(testBalancerIDs[1], rrBuilder) + bg.UpdateClientConnState(testBalancerIDs[1], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[2:4]}}) + + bg.Start() + + m1 := make(map[resolver.Address]balancer.SubConn) + for i := 0; i < 4; i++ { + addrs := <-cc.NewSubConnAddrsCh + sc := <-cc.NewSubConnCh + m1[addrs[0]] = sc + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + } + + // Test roundrobin on the last picker. + p1 := <-cc.NewPickerCh + want := []balancer.SubConn{ + m1[testBackendAddrs[0]], m1[testBackendAddrs[0]], + m1[testBackendAddrs[1]], m1[testBackendAddrs[1]], + m1[testBackendAddrs[2]], m1[testBackendAddrs[3]], + } + if err := testutils.IsRoundRobin(want, testutils.SubConnFromPicker(p1)); err != nil { + t.Fatalf("want %v, got %v", want, err) + } + + gator.Stop() + bg.Close() + for i := 0; i < 4; i++ { + (<-cc.ShutdownSubConnCh).UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Shutdown}) + } + + // Add b3, weight 1, backends [1,2]. + gator.Add(testBalancerIDs[2], 1) + bg.Add(testBalancerIDs[2], rrBuilder) + bg.UpdateClientConnState(testBalancerIDs[2], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[1:3]}}) + + // Remove b1. + gator.Remove(testBalancerIDs[0]) + bg.Remove(testBalancerIDs[0]) + + // Update b2 to weight 3, backends [0,3]. + gator.UpdateWeight(testBalancerIDs[1], 3) + bg.UpdateClientConnState(testBalancerIDs[1], balancer.ClientConnState{ResolverState: resolver.State{Addresses: append([]resolver.Address(nil), testBackendAddrs[0], testBackendAddrs[3])}}) + + gator.Start() + bg.Start() + + m2 := make(map[resolver.Address]balancer.SubConn) + for i := 0; i < 4; i++ { + addrs := <-cc.NewSubConnAddrsCh + sc := <-cc.NewSubConnCh + m2[addrs[0]] = sc + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + } + + // Test roundrobin on the last picker. + p2 := <-cc.NewPickerCh + want = []balancer.SubConn{ + m2[testBackendAddrs[0]], m2[testBackendAddrs[0]], m2[testBackendAddrs[0]], + m2[testBackendAddrs[3]], m2[testBackendAddrs[3]], m2[testBackendAddrs[3]], + m2[testBackendAddrs[1]], m2[testBackendAddrs[2]], + } + if err := testutils.IsRoundRobin(want, testutils.SubConnFromPicker(p2)); err != nil { + t.Fatalf("want %v, got %v", want, err) + } +} + +// Test that balancer group start() doesn't deadlock if the balancer calls back +// into balancer group inline when it gets an update. +// +// The potential deadlock can happen if we +// - hold a lock and send updates to balancer (e.g. update resolved addresses) +// - the balancer calls back (NewSubConn or update picker) in line +// +// The callback will try to hold hte same lock again, which will cause a +// deadlock. +// +// This test starts the balancer group with a test balancer, will updates picker +// whenever it gets an address update. It's expected that start() doesn't block +// because of deadlock. +func (s) TestBalancerGroup_start_close_deadlock(t *testing.T) { + const balancerName = "stub-TestBalancerGroup_start_close_deadlock" + stub.Register(balancerName, stub.BalancerFuncs{}) + builder := balancer.Get(balancerName) + + cc := testutils.NewTestClientConn(t) + gator := weightedaggregator.New(cc, nil, testutils.NewTestWRR) + gator.Start() + bg := New(Options{ + CC: cc, + BuildOpts: balancer.BuildOptions{}, + StateAggregator: gator, + Logger: nil, + SubBalancerCloseTimeout: time.Duration(0), + }) + + gator.Add(testBalancerIDs[0], 2) + bg.Add(testBalancerIDs[0], builder) + bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:2]}}) + gator.Add(testBalancerIDs[1], 1) + bg.Add(testBalancerIDs[1], builder) + bg.UpdateClientConnState(testBalancerIDs[1], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[2:4]}}) + + bg.Start() +} + +// initBalancerGroupForCachingTest creates a balancer group, and initialize it +// to be ready for caching tests. +// +// Two rr balancers are added to bg, each with 2 ready subConns. A sub-balancer +// is removed later, so the balancer group returned has one sub-balancer in its +// own map, and one sub-balancer in cache. +func initBalancerGroupForCachingTest(t *testing.T, idleCacheTimeout time.Duration) (*weightedaggregator.Aggregator, *BalancerGroup, *testutils.TestClientConn, map[resolver.Address]*testutils.TestSubConn) { + cc := testutils.NewTestClientConn(t) + gator := weightedaggregator.New(cc, nil, testutils.NewTestWRR) + gator.Start() + bg := New(Options{ + CC: cc, + BuildOpts: balancer.BuildOptions{}, + StateAggregator: gator, + Logger: nil, + SubBalancerCloseTimeout: idleCacheTimeout, + }) + + // Add two balancers to group and send two resolved addresses to both + // balancers. + gator.Add(testBalancerIDs[0], 2) + bg.Add(testBalancerIDs[0], rrBuilder) + bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:2]}}) + gator.Add(testBalancerIDs[1], 1) + bg.Add(testBalancerIDs[1], rrBuilder) + bg.UpdateClientConnState(testBalancerIDs[1], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[2:4]}}) + + bg.Start() + + m1 := make(map[resolver.Address]*testutils.TestSubConn) + for i := 0; i < 4; i++ { + addrs := <-cc.NewSubConnAddrsCh + sc := <-cc.NewSubConnCh + m1[addrs[0]] = sc + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + } + + // Test roundrobin on the last picker. + p1 := <-cc.NewPickerCh + want := []balancer.SubConn{ + m1[testBackendAddrs[0]], m1[testBackendAddrs[0]], + m1[testBackendAddrs[1]], m1[testBackendAddrs[1]], + m1[testBackendAddrs[2]], m1[testBackendAddrs[3]], + } + if err := testutils.IsRoundRobin(want, testutils.SubConnFromPicker(p1)); err != nil { + t.Fatalf("want %v, got %v", want, err) + } + + gator.Remove(testBalancerIDs[1]) + bg.Remove(testBalancerIDs[1]) + // Don't wait for SubConns to be removed after close, because they are only + // removed after close timeout. + for i := 0; i < 10; i++ { + select { + case sc := <-cc.ShutdownSubConnCh: + t.Fatalf("Got request to shut down subconn %v, want no shut down subconn (because subconns were still in cache)", sc) + default: + } + time.Sleep(time.Millisecond) + } + // Test roundrobin on the with only sub-balancer0. + p2 := <-cc.NewPickerCh + want = []balancer.SubConn{ + m1[testBackendAddrs[0]], m1[testBackendAddrs[1]], + } + if err := testutils.IsRoundRobin(want, testutils.SubConnFromPicker(p2)); err != nil { + t.Fatalf("want %v, got %v", want, err) + } + + return gator, bg, cc, m1 +} + +// Test that if a sub-balancer is removed, and re-added within close timeout, +// the subConns won't be re-created. +func (s) TestBalancerGroup_locality_caching(t *testing.T) { + gator, bg, cc, addrToSC := initBalancerGroupForCachingTest(t, defaultTestTimeout) + + // Turn down subconn for addr2, shouldn't get picker update because + // sub-balancer1 was removed. + addrToSC[testBackendAddrs[2]].UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) + for i := 0; i < 10; i++ { + select { + case <-cc.NewPickerCh: + t.Fatalf("Got new picker, want no new picker (because the sub-balancer was removed)") + default: + } + time.Sleep(defaultTestShortTimeout) + } + + // Re-add sub-balancer-1, because subconns were in cache, no new subconns + // should be created. But a new picker will still be generated, with subconn + // states update to date. + gator.Add(testBalancerIDs[1], 1) + bg.Add(testBalancerIDs[1], rrBuilder) + + p3 := <-cc.NewPickerCh + want := []balancer.SubConn{ + addrToSC[testBackendAddrs[0]], addrToSC[testBackendAddrs[0]], + addrToSC[testBackendAddrs[1]], addrToSC[testBackendAddrs[1]], + // addr2 is down, b2 only has addr3 in READY state. + addrToSC[testBackendAddrs[3]], addrToSC[testBackendAddrs[3]], + } + if err := testutils.IsRoundRobin(want, testutils.SubConnFromPicker(p3)); err != nil { + t.Fatalf("want %v, got %v", want, err) + } + + for i := 0; i < 10; i++ { + select { + case <-cc.NewSubConnAddrsCh: + t.Fatalf("Got new subconn, want no new subconn (because subconns were still in cache)") + default: + } + time.Sleep(defaultTestShortTimeout) + } +} + +// Sub-balancers are put in cache when they are shut down. If balancer group is +// closed within close timeout, all subconns should still be rmeoved +// immediately. +func (s) TestBalancerGroup_locality_caching_close_group(t *testing.T) { + _, bg, cc, addrToSC := initBalancerGroupForCachingTest(t, defaultTestTimeout) + + bg.Close() + // The balancer group is closed. The subconns should be shutdown immediately. + shutdownTimeout := time.After(time.Millisecond * 500) + scToShutdown := map[balancer.SubConn]int{ + addrToSC[testBackendAddrs[0]]: 1, + addrToSC[testBackendAddrs[1]]: 1, + addrToSC[testBackendAddrs[2]]: 1, + addrToSC[testBackendAddrs[3]]: 1, + } + for i := 0; i < len(scToShutdown); i++ { + select { + case sc := <-cc.ShutdownSubConnCh: + c := scToShutdown[sc] + if c == 0 { + t.Fatalf("Got Shutdown for %v when there's %d shutdown expected", sc, c) + } + scToShutdown[sc] = c - 1 + case <-shutdownTimeout: + t.Fatalf("timeout waiting for subConns (from balancer in cache) to be shut down") + } + } +} + +// Sub-balancers in cache will be closed if not re-added within timeout, and +// subConns will be shut down. +func (s) TestBalancerGroup_locality_caching_not_readd_within_timeout(t *testing.T) { + _, _, cc, addrToSC := initBalancerGroupForCachingTest(t, time.Second) + + // The sub-balancer is not re-added within timeout. The subconns should be + // shut down. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + scToShutdown := map[balancer.SubConn]int{ + addrToSC[testBackendAddrs[2]]: 1, + addrToSC[testBackendAddrs[3]]: 1, + } + for i := 0; i < len(scToShutdown); i++ { + select { + case sc := <-cc.ShutdownSubConnCh: + c := scToShutdown[sc] + if c == 0 { + t.Fatalf("Got Shutdown for %v when there's %d shutdown expected", sc, c) + } + scToShutdown[sc] = c - 1 + case <-ctx.Done(): + t.Fatalf("timeout waiting for subConns (from balancer in cache) to be shut down") + } + } +} + +// Wrap the rr builder, so it behaves the same, but has a different name. +type noopBalancerBuilderWrapper struct { + balancer.Builder +} + +func init() { + balancer.Register(&noopBalancerBuilderWrapper{Builder: rrBuilder}) +} + +func (*noopBalancerBuilderWrapper) Name() string { + return "noopBalancerBuilderWrapper" +} + +// After removing a sub-balancer, re-add with same ID, but different balancer +// builder. Old subconns should be shut down, and new subconns should be created. +func (s) TestBalancerGroup_locality_caching_readd_with_different_builder(t *testing.T) { + gator, bg, cc, addrToSC := initBalancerGroupForCachingTest(t, defaultTestTimeout) + + // Re-add sub-balancer-1, but with a different balancer builder. The + // sub-balancer was still in cache, but cann't be reused. This should cause + // old sub-balancer's subconns to be shut down immediately, and new + // subconns to be created. + gator.Add(testBalancerIDs[1], 1) + bg.Add(testBalancerIDs[1], &noopBalancerBuilderWrapper{rrBuilder}) + + // The cached sub-balancer should be closed, and the subconns should be + // shut down immediately. + shutdownTimeout := time.After(time.Millisecond * 500) + scToShutdown := map[balancer.SubConn]int{ + addrToSC[testBackendAddrs[2]]: 1, + addrToSC[testBackendAddrs[3]]: 1, + } + for i := 0; i < len(scToShutdown); i++ { + select { + case sc := <-cc.ShutdownSubConnCh: + c := scToShutdown[sc] + if c == 0 { + t.Fatalf("Got Shutdown for %v when there's %d shutdown expected", sc, c) + } + scToShutdown[sc] = c - 1 + case <-shutdownTimeout: + t.Fatalf("timeout waiting for subConns (from balancer in cache) to be shut down") + } + } + + bg.UpdateClientConnState(testBalancerIDs[1], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[4:6]}}) + + newSCTimeout := time.After(time.Millisecond * 500) + scToAdd := map[resolver.Address]int{ + testBackendAddrs[4]: 1, + testBackendAddrs[5]: 1, + } + for i := 0; i < len(scToAdd); i++ { + select { + case addr := <-cc.NewSubConnAddrsCh: + c := scToAdd[addr[0]] + if c == 0 { + t.Fatalf("Got newSubConn for %v when there's %d new expected", addr, c) + } + scToAdd[addr[0]] = c - 1 + sc := <-cc.NewSubConnCh + addrToSC[addr[0]] = sc + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + case <-newSCTimeout: + t.Fatalf("timeout waiting for subConns (from new sub-balancer) to be newed") + } + } + + // Test roundrobin on the new picker. + p3 := <-cc.NewPickerCh + want := []balancer.SubConn{ + addrToSC[testBackendAddrs[0]], addrToSC[testBackendAddrs[0]], + addrToSC[testBackendAddrs[1]], addrToSC[testBackendAddrs[1]], + addrToSC[testBackendAddrs[4]], addrToSC[testBackendAddrs[5]], + } + if err := testutils.IsRoundRobin(want, testutils.SubConnFromPicker(p3)); err != nil { + t.Fatalf("want %v, got %v", want, err) + } +} + +// After removing a sub-balancer, it will be kept in cache. Make sure that this +// sub-balancer's Close is called when the balancer group is closed. +func (s) TestBalancerGroup_CloseStopsBalancerInCache(t *testing.T) { + const balancerName = "stub-TestBalancerGroup_check_close" + closed := make(chan struct{}) + stub.Register(balancerName, stub.BalancerFuncs{Close: func(_ *stub.BalancerData) { + close(closed) + }}) + builder := balancer.Get(balancerName) + + gator, bg, _, _ := initBalancerGroupForCachingTest(t, time.Second) + + // Add balancer, and remove + gator.Add(testBalancerIDs[2], 1) + bg.Add(testBalancerIDs[2], builder) + gator.Remove(testBalancerIDs[2]) + bg.Remove(testBalancerIDs[2]) + + // Immediately close balancergroup, before the cache timeout. + bg.Close() + + // Make sure the removed child balancer is closed eventually. + select { + case <-closed: + case <-time.After(time.Second * 2): + t.Fatalf("timeout waiting for the child balancer in cache to be closed") + } +} + +// TestBalancerGroupBuildOptions verifies that the balancer.BuildOptions passed +// to the balancergroup at creation time is passed to child policies. +func (s) TestBalancerGroupBuildOptions(t *testing.T) { + const ( + balancerName = "stubBalancer-TestBalancerGroupBuildOptions" + userAgent = "ua" + ) + + // Setup the stub balancer such that we can read the build options passed to + // it in the UpdateClientConnState method. + bOpts := balancer.BuildOptions{ + DialCreds: insecure.NewCredentials(), + ChannelzParentID: channelz.NewIdentifierForTesting(channelz.RefChannel, 1234, nil), + CustomUserAgent: userAgent, + } + stub.Register(balancerName, stub.BalancerFuncs{ + UpdateClientConnState: func(bd *stub.BalancerData, _ balancer.ClientConnState) error { + if !cmp.Equal(bd.BuildOptions, bOpts) { + return fmt.Errorf("buildOptions in child balancer: %v, want %v", bd, bOpts) + } + return nil + }, + }) + cc := testutils.NewTestClientConn(t) + bg := New(Options{ + CC: cc, + BuildOpts: bOpts, + StateAggregator: nil, + Logger: nil, + }) + bg.Start() + + // Add the stub balancer build above as a child policy. + balancerBuilder := balancer.Get(balancerName) + bg.Add(testBalancerIDs[0], balancerBuilder) + + // Send an empty clientConn state change. This should trigger the + // verification of the buildOptions being passed to the child policy. + if err := bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{}); err != nil { + t.Fatal(err) + } +} + +func (s) TestBalancerExitIdleOne(t *testing.T) { + const balancerName = "stub-balancer-test-balancergroup-exit-idle-one" + exitIdleCh := make(chan struct{}, 1) + stub.Register(balancerName, stub.BalancerFuncs{ + ExitIdle: func(*stub.BalancerData) { + exitIdleCh <- struct{}{} + }, + }) + cc := testutils.NewTestClientConn(t) + bg := New(Options{ + CC: cc, + BuildOpts: balancer.BuildOptions{}, + StateAggregator: nil, + Logger: nil, + }) + bg.Start() + defer bg.Close() + + // Add the stub balancer build above as a child policy. + builder := balancer.Get(balancerName) + bg.Add(testBalancerIDs[0], builder) + + // Call ExitIdle on the child policy. + bg.ExitIdleOne(testBalancerIDs[0]) + select { + case <-time.After(time.Second): + t.Fatal("Timeout when waiting for ExitIdle to be invoked on child policy") + case <-exitIdleCh: + } +} + +// TestBalancerGracefulSwitch tests the graceful switch functionality for a +// child of the balancer group. At first, the child is configured as a round +// robin load balancer, and thus should behave accordingly. The test then +// gracefully switches this child to a custom type which only creates a SubConn +// for the second passed in address and also only picks that created SubConn. +// The new aggregated picker should reflect this change for the child. +func (s) TestBalancerGracefulSwitch(t *testing.T) { + cc := testutils.NewTestClientConn(t) + gator := weightedaggregator.New(cc, nil, testutils.NewTestWRR) + gator.Start() + bg := New(Options{ + CC: cc, + BuildOpts: balancer.BuildOptions{}, + StateAggregator: gator, + Logger: nil, + }) + gator.Add(testBalancerIDs[0], 1) + bg.Add(testBalancerIDs[0], rrBuilder) + bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:2]}}) + + bg.Start() + + m1 := make(map[resolver.Address]balancer.SubConn) + scs := make(map[balancer.SubConn]bool) + for i := 0; i < 2; i++ { + addrs := <-cc.NewSubConnAddrsCh + sc := <-cc.NewSubConnCh + m1[addrs[0]] = sc + scs[sc] = true + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + } + + p1 := <-cc.NewPickerCh + want := []balancer.SubConn{ + m1[testBackendAddrs[0]], m1[testBackendAddrs[1]], + } + if err := testutils.IsRoundRobin(want, testutils.SubConnFromPicker(p1)); err != nil { + t.Fatal(err) + } + + // The balancer type for testBalancersIDs[0] is currently Round Robin. Now, + // change it to a balancer that has separate behavior logically (creating + // SubConn for second address in address list and always picking that + // SubConn), and see if the downstream behavior reflects that change. + childPolicyName := t.Name() + stub.Register(childPolicyName, stub.BalancerFuncs{ + Init: func(bd *stub.BalancerData) { + bd.Data = balancer.Get(grpc.PickFirstBalancerName).Build(bd.ClientConn, bd.BuildOptions) + }, + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + ccs.ResolverState.Addresses = ccs.ResolverState.Addresses[1:] + bal := bd.Data.(balancer.Balancer) + return bal.UpdateClientConnState(ccs) + }, + }) + builder := balancer.Get(childPolicyName) + bg.UpdateBuilder(testBalancerIDs[0], builder) + if err := bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[2:4]}}); err != nil { + t.Fatalf("error updating ClientConn state: %v", err) + } + + addrs := <-cc.NewSubConnAddrsCh + if addrs[0].Addr != testBackendAddrs[3].Addr { + // Verifies forwarded to new created balancer, as the wrapped pick first + // balancer will delete first address. + t.Fatalf("newSubConn called with wrong address, want: %v, got : %v", testBackendAddrs[3].Addr, addrs[0].Addr) + } + sc := <-cc.NewSubConnCh + + // Update the pick first balancers SubConn as CONNECTING. This will cause + // the pick first balancer to UpdateState() with CONNECTING, which shouldn't send + // a Picker update back, as the Graceful Switch process is not complete. + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer cancel() + select { + case <-cc.NewPickerCh: + t.Fatalf("No new picker should have been sent due to the Graceful Switch process not completing") + case <-ctx.Done(): + } + + // Update the pick first balancers SubConn as READY. This will cause + // the pick first balancer to UpdateState() with READY, which should send a + // Picker update back, as the Graceful Switch process is complete. This + // Picker should always pick the pick first's created SubConn which + // corresponds to address 3. + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + p2 := <-cc.NewPickerCh + pr, err := p2.Pick(balancer.PickInfo{}) + if err != nil { + t.Fatalf("error picking: %v", err) + } + if pr.SubConn != sc { + t.Fatalf("picker.Pick(), want %v, got %v", sc, pr.SubConn) + } + + // The Graceful Switch process completing for the child should cause the + // SubConns for the balancer being gracefully switched from to get deleted. + ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + for i := 0; i < 2; i++ { + select { + case <-ctx.Done(): + t.Fatalf("error waiting for Shutdown()") + case sc := <-cc.ShutdownSubConnCh: + // The SubConn shut down should have been one of the two created + // SubConns, and both should be deleted. + if ok := scs[sc]; ok { + delete(scs, sc) + continue + } else { + t.Fatalf("Shutdown called for wrong SubConn %v, want in %v", sc, scs) + } + } + } +} diff --git a/xds/internal/balancer/balancergroup/balancerstateaggregator.go b/internal/balancergroup/balancerstateaggregator.go similarity index 85% rename from xds/internal/balancer/balancergroup/balancerstateaggregator.go rename to internal/balancergroup/balancerstateaggregator.go index 0a555f3d9ba2..816869555323 100644 --- a/xds/internal/balancer/balancergroup/balancerstateaggregator.go +++ b/internal/balancergroup/balancerstateaggregator.go @@ -20,19 +20,18 @@ package balancergroup import ( "google.golang.org/grpc/balancer" - "google.golang.org/grpc/xds/internal" ) // BalancerStateAggregator aggregates sub-picker and connectivity states into a // state. // // It takes care of merging sub-picker into one picker. The picking config is -// passed directly from the the parent to the aggregator implementation (instead +// passed directly from the parent to the aggregator implementation (instead // via balancer group). type BalancerStateAggregator interface { // UpdateState updates the state of the id. // // It's up to the implementation whether this will trigger an update to the // parent ClientConn. - UpdateState(id internal.LocalityID, state balancer.State) + UpdateState(id string, state balancer.State) } diff --git a/internal/balancerload/load.go b/internal/balancerload/load.go index 3a905d96657e..94a08d6875a9 100644 --- a/internal/balancerload/load.go +++ b/internal/balancerload/load.go @@ -25,7 +25,7 @@ import ( // Parser converts loads from metadata into a concrete type. type Parser interface { // Parse parses loads from metadata. - Parse(md metadata.MD) interface{} + Parse(md metadata.MD) any } var parser Parser @@ -38,7 +38,7 @@ func SetParser(lr Parser) { } // Parse calls parser.Read(). -func Parse(md metadata.MD) interface{} { +func Parse(md metadata.MD) any { if parser == nil { return nil } diff --git a/internal/binarylog/binarylog.go b/internal/binarylog/binarylog.go index 5cc3aeddb213..755fdebc1b15 100644 --- a/internal/binarylog/binarylog.go +++ b/internal/binarylog/binarylog.go @@ -28,38 +28,48 @@ import ( "google.golang.org/grpc/internal/grpcutil" ) -// Logger is the global binary logger. It can be used to get binary logger for -// each method. +var grpclogLogger = grpclog.Component("binarylog") + +// Logger specifies MethodLoggers for method names with a Log call that +// takes a context. +// +// This is used in the 1.0 release of gcp/observability, and thus must not be +// deleted or changed. type Logger interface { - getMethodLogger(methodName string) *MethodLogger + GetMethodLogger(methodName string) MethodLogger } // binLogger is the global binary logger for the binary. One of this should be // built at init time from the configuration (environment variable or flags). // -// It is used to get a methodLogger for each individual method. +// It is used to get a MethodLogger for each individual method. var binLogger Logger -var grpclogLogger = grpclog.Component("binarylog") - -// SetLogger sets the binarg logger. +// SetLogger sets the binary logger. // // Only call this at init time. func SetLogger(l Logger) { binLogger = l } -// GetMethodLogger returns the methodLogger for the given methodName. +// GetLogger gets the binary logger. +// +// Only call this at init time. +func GetLogger() Logger { + return binLogger +} + +// GetMethodLogger returns the MethodLogger for the given methodName. // // methodName should be in the format of "/service/method". // -// Each methodLogger returned by this method is a new instance. This is to +// Each MethodLogger returned by this method is a new instance. This is to // generate sequence id within the call. -func GetMethodLogger(methodName string) *MethodLogger { +func GetMethodLogger(methodName string) MethodLogger { if binLogger == nil { return nil } - return binLogger.getMethodLogger(methodName) + return binLogger.GetMethodLogger(methodName) } func init() { @@ -68,17 +78,29 @@ func init() { binLogger = NewLoggerFromConfigString(configStr) } -type methodLoggerConfig struct { +// MethodLoggerConfig contains the setting for logging behavior of a method +// logger. Currently, it contains the max length of header and message. +type MethodLoggerConfig struct { // Max length of header and message. - hdr, msg uint64 + Header, Message uint64 +} + +// LoggerConfig contains the config for loggers to create method loggers. +type LoggerConfig struct { + All *MethodLoggerConfig + Services map[string]*MethodLoggerConfig + Methods map[string]*MethodLoggerConfig + + Blacklist map[string]struct{} } type logger struct { - all *methodLoggerConfig - services map[string]*methodLoggerConfig - methods map[string]*methodLoggerConfig + config LoggerConfig +} - blacklist map[string]struct{} +// NewLoggerFromConfig builds a logger with the given LoggerConfig. +func NewLoggerFromConfig(config LoggerConfig) Logger { + return &logger{config: config} } // newEmptyLogger creates an empty logger. The map fields need to be filled in @@ -88,83 +110,83 @@ func newEmptyLogger() *logger { } // Set method logger for "*". -func (l *logger) setDefaultMethodLogger(ml *methodLoggerConfig) error { - if l.all != nil { +func (l *logger) setDefaultMethodLogger(ml *MethodLoggerConfig) error { + if l.config.All != nil { return fmt.Errorf("conflicting global rules found") } - l.all = ml + l.config.All = ml return nil } // Set method logger for "service/*". // -// New methodLogger with same service overrides the old one. -func (l *logger) setServiceMethodLogger(service string, ml *methodLoggerConfig) error { - if _, ok := l.services[service]; ok { +// New MethodLogger with same service overrides the old one. +func (l *logger) setServiceMethodLogger(service string, ml *MethodLoggerConfig) error { + if _, ok := l.config.Services[service]; ok { return fmt.Errorf("conflicting service rules for service %v found", service) } - if l.services == nil { - l.services = make(map[string]*methodLoggerConfig) + if l.config.Services == nil { + l.config.Services = make(map[string]*MethodLoggerConfig) } - l.services[service] = ml + l.config.Services[service] = ml return nil } // Set method logger for "service/method". // -// New methodLogger with same method overrides the old one. -func (l *logger) setMethodMethodLogger(method string, ml *methodLoggerConfig) error { - if _, ok := l.blacklist[method]; ok { +// New MethodLogger with same method overrides the old one. +func (l *logger) setMethodMethodLogger(method string, ml *MethodLoggerConfig) error { + if _, ok := l.config.Blacklist[method]; ok { return fmt.Errorf("conflicting blacklist rules for method %v found", method) } - if _, ok := l.methods[method]; ok { + if _, ok := l.config.Methods[method]; ok { return fmt.Errorf("conflicting method rules for method %v found", method) } - if l.methods == nil { - l.methods = make(map[string]*methodLoggerConfig) + if l.config.Methods == nil { + l.config.Methods = make(map[string]*MethodLoggerConfig) } - l.methods[method] = ml + l.config.Methods[method] = ml return nil } // Set blacklist method for "-service/method". func (l *logger) setBlacklist(method string) error { - if _, ok := l.blacklist[method]; ok { + if _, ok := l.config.Blacklist[method]; ok { return fmt.Errorf("conflicting blacklist rules for method %v found", method) } - if _, ok := l.methods[method]; ok { + if _, ok := l.config.Methods[method]; ok { return fmt.Errorf("conflicting method rules for method %v found", method) } - if l.blacklist == nil { - l.blacklist = make(map[string]struct{}) + if l.config.Blacklist == nil { + l.config.Blacklist = make(map[string]struct{}) } - l.blacklist[method] = struct{}{} + l.config.Blacklist[method] = struct{}{} return nil } -// getMethodLogger returns the methodLogger for the given methodName. +// getMethodLogger returns the MethodLogger for the given methodName. // // methodName should be in the format of "/service/method". // -// Each methodLogger returned by this method is a new instance. This is to +// Each MethodLogger returned by this method is a new instance. This is to // generate sequence id within the call. -func (l *logger) getMethodLogger(methodName string) *MethodLogger { +func (l *logger) GetMethodLogger(methodName string) MethodLogger { s, m, err := grpcutil.ParseMethod(methodName) if err != nil { grpclogLogger.Infof("binarylogging: failed to parse %q: %v", methodName, err) return nil } - if ml, ok := l.methods[s+"/"+m]; ok { - return newMethodLogger(ml.hdr, ml.msg) + if ml, ok := l.config.Methods[s+"/"+m]; ok { + return NewTruncatingMethodLogger(ml.Header, ml.Message) } - if _, ok := l.blacklist[s+"/"+m]; ok { + if _, ok := l.config.Blacklist[s+"/"+m]; ok { return nil } - if ml, ok := l.services[s]; ok { - return newMethodLogger(ml.hdr, ml.msg) + if ml, ok := l.config.Services[s]; ok { + return NewTruncatingMethodLogger(ml.Header, ml.Message) } - if l.all == nil { + if l.config.All == nil { return nil } - return newMethodLogger(l.all.hdr, l.all.msg) + return NewTruncatingMethodLogger(l.config.All.Header, l.config.All.Message) } diff --git a/internal/binarylog/binarylog_test.go b/internal/binarylog/binarylog_test.go index cbf2ba0d1bf8..47f6a541e767 100644 --- a/internal/binarylog/binarylog_test.go +++ b/internal/binarylog/binarylog_test.go @@ -93,12 +93,11 @@ func (s) TestGetMethodLogger(t *testing.T) { t.Errorf("in: %q, failed to create logger from config string", tc.in) continue } - ml := l.getMethodLogger(tc.method) + ml := l.GetMethodLogger(tc.method).(*TruncatingMethodLogger) if ml == nil { t.Errorf("in: %q, method logger is nil, want non-nil", tc.in) continue } - if ml.headerMaxLen != tc.hdr || ml.messageMaxLen != tc.msg { t.Errorf("in: %q, want header: %v, message: %v, got header: %v, message: %v", tc.in, tc.hdr, tc.msg, ml.headerMaxLen, ml.messageMaxLen) } @@ -149,7 +148,7 @@ func (s) TestGetMethodLoggerOff(t *testing.T) { t.Errorf("in: %q, failed to create logger from config string", tc.in) continue } - ml := l.getMethodLogger(tc.method) + ml := l.GetMethodLogger(tc.method) if ml != nil { t.Errorf("in: %q, method logger is non-nil, want nil", tc.in) } diff --git a/internal/binarylog/env_config.go b/internal/binarylog/env_config.go index d8f4e7602fde..f9e80e27ab68 100644 --- a/internal/binarylog/env_config.go +++ b/internal/binarylog/env_config.go @@ -30,15 +30,15 @@ import ( // to build a new logger and assign it to binarylog.Logger. // // Example filter config strings: -// - "" Nothing will be logged -// - "*" All headers and messages will be fully logged. -// - "*{h}" Only headers will be logged. -// - "*{m:256}" Only the first 256 bytes of each message will be logged. -// - "Foo/*" Logs every method in service Foo -// - "Foo/*,-Foo/Bar" Logs every method in service Foo except method /Foo/Bar -// - "Foo/*,Foo/Bar{m:256}" Logs the first 256 bytes of each message in method -// /Foo/Bar, logs all headers and messages in every other method in service -// Foo. +// - "" Nothing will be logged +// - "*" All headers and messages will be fully logged. +// - "*{h}" Only headers will be logged. +// - "*{m:256}" Only the first 256 bytes of each message will be logged. +// - "Foo/*" Logs every method in service Foo +// - "Foo/*,-Foo/Bar" Logs every method in service Foo except method /Foo/Bar +// - "Foo/*,Foo/Bar{m:256}" Logs the first 256 bytes of each message in method +// /Foo/Bar, logs all headers and messages in every other method in service +// Foo. // // If two configs exist for one certain method or service, the one specified // later overrides the previous config. @@ -57,7 +57,7 @@ func NewLoggerFromConfigString(s string) Logger { return l } -// fillMethodLoggerWithConfigString parses config, creates methodLogger and adds +// fillMethodLoggerWithConfigString parses config, creates TruncatingMethodLogger and adds // it to the right map in the logger. func (l *logger) fillMethodLoggerWithConfigString(config string) error { // "" is invalid. @@ -89,7 +89,7 @@ func (l *logger) fillMethodLoggerWithConfigString(config string) error { if err != nil { return fmt.Errorf("invalid config: %q, %v", config, err) } - if err := l.setDefaultMethodLogger(&methodLoggerConfig{hdr: hdr, msg: msg}); err != nil { + if err := l.setDefaultMethodLogger(&MethodLoggerConfig{Header: hdr, Message: msg}); err != nil { return fmt.Errorf("invalid config: %v", err) } return nil @@ -104,11 +104,11 @@ func (l *logger) fillMethodLoggerWithConfigString(config string) error { return fmt.Errorf("invalid header/message length config: %q, %v", suffix, err) } if m == "*" { - if err := l.setServiceMethodLogger(s, &methodLoggerConfig{hdr: hdr, msg: msg}); err != nil { + if err := l.setServiceMethodLogger(s, &MethodLoggerConfig{Header: hdr, Message: msg}); err != nil { return fmt.Errorf("invalid config: %v", err) } } else { - if err := l.setMethodMethodLogger(s+"/"+m, &methodLoggerConfig{hdr: hdr, msg: msg}); err != nil { + if err := l.setMethodMethodLogger(s+"/"+m, &MethodLoggerConfig{Header: hdr, Message: msg}); err != nil { return fmt.Errorf("invalid config: %v", err) } } diff --git a/internal/binarylog/env_config_test.go b/internal/binarylog/env_config_test.go index f67b4fd60326..9f888ad870ea 100644 --- a/internal/binarylog/env_config_test.go +++ b/internal/binarylog/env_config_test.go @@ -36,29 +36,29 @@ func (s) TestNewLoggerFromConfigString(t *testing.T) { c := fmt.Sprintf("*{h:1;m:2},%s{h},%s{m},%s{h;m}", s1+"/*", fullM1, fullM2) l := NewLoggerFromConfigString(c).(*logger) - if l.all.hdr != 1 || l.all.msg != 2 { - t.Errorf("l.all = %#v, want headerLen: 1, messageLen: 2", l.all) + if l.config.All.Header != 1 || l.config.All.Message != 2 { + t.Errorf("l.config.All = %#v, want headerLen: 1, messageLen: 2", l.config.All) } - if ml, ok := l.services[s1]; ok { - if ml.hdr != maxUInt || ml.msg != 0 { - t.Errorf("want maxUInt header, 0 message, got header: %v, message: %v", ml.hdr, ml.msg) + if ml, ok := l.config.Services[s1]; ok { + if ml.Header != maxUInt || ml.Message != 0 { + t.Errorf("want maxUInt header, 0 message, got header: %v, message: %v", ml.Header, ml.Message) } } else { t.Errorf("service/* is not set") } - if ml, ok := l.methods[fullM1]; ok { - if ml.hdr != 0 || ml.msg != maxUInt { - t.Errorf("want 0 header, maxUInt message, got header: %v, message: %v", ml.hdr, ml.msg) + if ml, ok := l.config.Methods[fullM1]; ok { + if ml.Header != 0 || ml.Message != maxUInt { + t.Errorf("want 0 header, maxUInt message, got header: %v, message: %v", ml.Header, ml.Message) } } else { t.Errorf("service/method{h} is not set") } - if ml, ok := l.methods[fullM2]; ok { - if ml.hdr != maxUInt || ml.msg != maxUInt { - t.Errorf("want maxUInt header, maxUInt message, got header: %v, message: %v", ml.hdr, ml.msg) + if ml, ok := l.config.Methods[fullM2]; ok { + if ml.Header != maxUInt || ml.Message != maxUInt { + t.Errorf("want maxUInt header, maxUInt message, got header: %v, message: %v", ml.Header, ml.Message) } } else { t.Errorf("service/method{h;m} is not set") @@ -249,7 +249,7 @@ func (s) TestFillMethodLoggerWithConfigStringBlacklist(t *testing.T) { t.Errorf("returned err %v, want nil", err) continue } - _, ok := l.blacklist[tc] + _, ok := l.config.Blacklist[tc] if !ok { t.Errorf("blacklist[%q] is not set", tc) } @@ -306,15 +306,15 @@ func (s) TestFillMethodLoggerWithConfigStringGlobal(t *testing.T) { t.Errorf("returned err %v, want nil", err) continue } - if l.all == nil { - t.Errorf("l.all is not set") + if l.config.All == nil { + t.Errorf("l.config.All is not set") continue } - if hdr := l.all.hdr; hdr != tc.hdr { + if hdr := l.config.All.Header; hdr != tc.hdr { t.Errorf("header length = %v, want %v", hdr, tc.hdr) } - if msg := l.all.msg; msg != tc.msg { + if msg := l.config.All.Message; msg != tc.msg { t.Errorf("message length = %v, want %v", msg, tc.msg) } } @@ -371,16 +371,16 @@ func (s) TestFillMethodLoggerWithConfigStringPerService(t *testing.T) { t.Errorf("returned err %v, want nil", err) continue } - ml, ok := l.services[serviceName] + ml, ok := l.config.Services[serviceName] if !ok { t.Errorf("l.service[%q] is not set", serviceName) continue } - if hdr := ml.hdr; hdr != tc.hdr { + if hdr := ml.Header; hdr != tc.hdr { t.Errorf("header length = %v, want %v", hdr, tc.hdr) } - if msg := ml.msg; msg != tc.msg { + if msg := ml.Message; msg != tc.msg { t.Errorf("message length = %v, want %v", msg, tc.msg) } } @@ -441,16 +441,16 @@ func (s) TestFillMethodLoggerWithConfigStringPerMethod(t *testing.T) { t.Errorf("returned err %v, want nil", err) continue } - ml, ok := l.methods[fullMethodName] + ml, ok := l.config.Methods[fullMethodName] if !ok { - t.Errorf("l.methods[%q] is not set", fullMethodName) + t.Errorf("l.config.Methods[%q] is not set", fullMethodName) continue } - if hdr := ml.hdr; hdr != tc.hdr { + if hdr := ml.Header; hdr != tc.hdr { t.Errorf("header length = %v, want %v", hdr, tc.hdr) } - if msg := ml.msg; msg != tc.msg { + if msg := ml.Message; msg != tc.msg { t.Errorf("message length = %v, want %v", msg, tc.msg) } } diff --git a/internal/binarylog/method_logger.go b/internal/binarylog/method_logger.go index 5e1083539b49..0f31274a3ccc 100644 --- a/internal/binarylog/method_logger.go +++ b/internal/binarylog/method_logger.go @@ -19,6 +19,7 @@ package binarylog import ( + "context" "net" "strings" "sync/atomic" @@ -26,7 +27,7 @@ import ( "github.com/golang/protobuf/proto" "github.com/golang/protobuf/ptypes" - pb "google.golang.org/grpc/binarylog/grpc_binarylog_v1" + binlogpb "google.golang.org/grpc/binarylog/grpc_binarylog_v1" "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" ) @@ -48,7 +49,16 @@ func (g *callIDGenerator) reset() { var idGen callIDGenerator // MethodLogger is the sub-logger for each method. -type MethodLogger struct { +// +// This is used in the 1.0 release of gcp/observability, and thus must not be +// deleted or changed. +type MethodLogger interface { + Log(context.Context, LogEntryConfig) +} + +// TruncatingMethodLogger is a method logger that truncates headers and messages +// based on configured fields. +type TruncatingMethodLogger struct { headerMaxLen, messageMaxLen uint64 callID uint64 @@ -57,20 +67,26 @@ type MethodLogger struct { sink Sink // TODO(blog): make this plugable. } -func newMethodLogger(h, m uint64) *MethodLogger { - return &MethodLogger{ +// NewTruncatingMethodLogger returns a new truncating method logger. +// +// This is used in the 1.0 release of gcp/observability, and thus must not be +// deleted or changed. +func NewTruncatingMethodLogger(h, m uint64) *TruncatingMethodLogger { + return &TruncatingMethodLogger{ headerMaxLen: h, messageMaxLen: m, callID: idGen.next(), idWithinCallGen: &callIDGenerator{}, - sink: defaultSink, // TODO(blog): make it plugable. + sink: DefaultSink, // TODO(blog): make it plugable. } } -// Log creates a proto binary log entry, and logs it to the sink. -func (ml *MethodLogger) Log(c LogEntryConfig) { +// Build is an internal only method for building the proto message out of the +// input event. It's made public to enable other library to reuse as much logic +// in TruncatingMethodLogger as possible. +func (ml *TruncatingMethodLogger) Build(c LogEntryConfig) *binlogpb.GrpcLogEntry { m := c.toProto() timestamp, _ := ptypes.TimestampProto(time.Now()) m.Timestamp = timestamp @@ -78,18 +94,22 @@ func (ml *MethodLogger) Log(c LogEntryConfig) { m.SequenceIdWithinCall = ml.idWithinCallGen.next() switch pay := m.Payload.(type) { - case *pb.GrpcLogEntry_ClientHeader: + case *binlogpb.GrpcLogEntry_ClientHeader: m.PayloadTruncated = ml.truncateMetadata(pay.ClientHeader.GetMetadata()) - case *pb.GrpcLogEntry_ServerHeader: + case *binlogpb.GrpcLogEntry_ServerHeader: m.PayloadTruncated = ml.truncateMetadata(pay.ServerHeader.GetMetadata()) - case *pb.GrpcLogEntry_Message: + case *binlogpb.GrpcLogEntry_Message: m.PayloadTruncated = ml.truncateMessage(pay.Message) } + return m +} - ml.sink.Write(m) +// Log creates a proto binary log entry, and logs it to the sink. +func (ml *TruncatingMethodLogger) Log(ctx context.Context, c LogEntryConfig) { + ml.sink.Write(ml.Build(c)) } -func (ml *MethodLogger) truncateMetadata(mdPb *pb.Metadata) (truncated bool) { +func (ml *TruncatingMethodLogger) truncateMetadata(mdPb *binlogpb.Metadata) (truncated bool) { if ml.headerMaxLen == maxUInt { return false } @@ -108,7 +128,7 @@ func (ml *MethodLogger) truncateMetadata(mdPb *pb.Metadata) (truncated bool) { // but not counted towards the size limit. continue } - currentEntryLen := uint64(len(entry.Value)) + currentEntryLen := uint64(len(entry.GetKey())) + uint64(len(entry.GetValue())) if currentEntryLen > bytesLimit { break } @@ -119,7 +139,7 @@ func (ml *MethodLogger) truncateMetadata(mdPb *pb.Metadata) (truncated bool) { return truncated } -func (ml *MethodLogger) truncateMessage(msgPb *pb.Message) (truncated bool) { +func (ml *TruncatingMethodLogger) truncateMessage(msgPb *binlogpb.Message) (truncated bool) { if ml.messageMaxLen == maxUInt { return false } @@ -131,8 +151,11 @@ func (ml *MethodLogger) truncateMessage(msgPb *pb.Message) (truncated bool) { } // LogEntryConfig represents the configuration for binary log entry. +// +// This is used in the 1.0 release of gcp/observability, and thus must not be +// deleted or changed. type LogEntryConfig interface { - toProto() *pb.GrpcLogEntry + toProto() *binlogpb.GrpcLogEntry } // ClientHeader configs the binary log entry to be a ClientHeader entry. @@ -146,10 +169,10 @@ type ClientHeader struct { PeerAddr net.Addr } -func (c *ClientHeader) toProto() *pb.GrpcLogEntry { +func (c *ClientHeader) toProto() *binlogpb.GrpcLogEntry { // This function doesn't need to set all the fields (e.g. seq ID). The Log // function will set the fields when necessary. - clientHeader := &pb.ClientHeader{ + clientHeader := &binlogpb.ClientHeader{ Metadata: mdToMetadataProto(c.Header), MethodName: c.MethodName, Authority: c.Authority, @@ -157,16 +180,16 @@ func (c *ClientHeader) toProto() *pb.GrpcLogEntry { if c.Timeout > 0 { clientHeader.Timeout = ptypes.DurationProto(c.Timeout) } - ret := &pb.GrpcLogEntry{ - Type: pb.GrpcLogEntry_EVENT_TYPE_CLIENT_HEADER, - Payload: &pb.GrpcLogEntry_ClientHeader{ + ret := &binlogpb.GrpcLogEntry{ + Type: binlogpb.GrpcLogEntry_EVENT_TYPE_CLIENT_HEADER, + Payload: &binlogpb.GrpcLogEntry_ClientHeader{ ClientHeader: clientHeader, }, } if c.OnClientSide { - ret.Logger = pb.GrpcLogEntry_LOGGER_CLIENT + ret.Logger = binlogpb.GrpcLogEntry_LOGGER_CLIENT } else { - ret.Logger = pb.GrpcLogEntry_LOGGER_SERVER + ret.Logger = binlogpb.GrpcLogEntry_LOGGER_SERVER } if c.PeerAddr != nil { ret.Peer = addrToProto(c.PeerAddr) @@ -182,19 +205,19 @@ type ServerHeader struct { PeerAddr net.Addr } -func (c *ServerHeader) toProto() *pb.GrpcLogEntry { - ret := &pb.GrpcLogEntry{ - Type: pb.GrpcLogEntry_EVENT_TYPE_SERVER_HEADER, - Payload: &pb.GrpcLogEntry_ServerHeader{ - ServerHeader: &pb.ServerHeader{ +func (c *ServerHeader) toProto() *binlogpb.GrpcLogEntry { + ret := &binlogpb.GrpcLogEntry{ + Type: binlogpb.GrpcLogEntry_EVENT_TYPE_SERVER_HEADER, + Payload: &binlogpb.GrpcLogEntry_ServerHeader{ + ServerHeader: &binlogpb.ServerHeader{ Metadata: mdToMetadataProto(c.Header), }, }, } if c.OnClientSide { - ret.Logger = pb.GrpcLogEntry_LOGGER_CLIENT + ret.Logger = binlogpb.GrpcLogEntry_LOGGER_CLIENT } else { - ret.Logger = pb.GrpcLogEntry_LOGGER_SERVER + ret.Logger = binlogpb.GrpcLogEntry_LOGGER_SERVER } if c.PeerAddr != nil { ret.Peer = addrToProto(c.PeerAddr) @@ -207,10 +230,10 @@ type ClientMessage struct { OnClientSide bool // Message can be a proto.Message or []byte. Other messages formats are not // supported. - Message interface{} + Message any } -func (c *ClientMessage) toProto() *pb.GrpcLogEntry { +func (c *ClientMessage) toProto() *binlogpb.GrpcLogEntry { var ( data []byte err error @@ -225,19 +248,19 @@ func (c *ClientMessage) toProto() *pb.GrpcLogEntry { } else { grpclogLogger.Infof("binarylogging: message to log is neither proto.message nor []byte") } - ret := &pb.GrpcLogEntry{ - Type: pb.GrpcLogEntry_EVENT_TYPE_CLIENT_MESSAGE, - Payload: &pb.GrpcLogEntry_Message{ - Message: &pb.Message{ + ret := &binlogpb.GrpcLogEntry{ + Type: binlogpb.GrpcLogEntry_EVENT_TYPE_CLIENT_MESSAGE, + Payload: &binlogpb.GrpcLogEntry_Message{ + Message: &binlogpb.Message{ Length: uint32(len(data)), Data: data, }, }, } if c.OnClientSide { - ret.Logger = pb.GrpcLogEntry_LOGGER_CLIENT + ret.Logger = binlogpb.GrpcLogEntry_LOGGER_CLIENT } else { - ret.Logger = pb.GrpcLogEntry_LOGGER_SERVER + ret.Logger = binlogpb.GrpcLogEntry_LOGGER_SERVER } return ret } @@ -247,10 +270,10 @@ type ServerMessage struct { OnClientSide bool // Message can be a proto.Message or []byte. Other messages formats are not // supported. - Message interface{} + Message any } -func (c *ServerMessage) toProto() *pb.GrpcLogEntry { +func (c *ServerMessage) toProto() *binlogpb.GrpcLogEntry { var ( data []byte err error @@ -265,19 +288,19 @@ func (c *ServerMessage) toProto() *pb.GrpcLogEntry { } else { grpclogLogger.Infof("binarylogging: message to log is neither proto.message nor []byte") } - ret := &pb.GrpcLogEntry{ - Type: pb.GrpcLogEntry_EVENT_TYPE_SERVER_MESSAGE, - Payload: &pb.GrpcLogEntry_Message{ - Message: &pb.Message{ + ret := &binlogpb.GrpcLogEntry{ + Type: binlogpb.GrpcLogEntry_EVENT_TYPE_SERVER_MESSAGE, + Payload: &binlogpb.GrpcLogEntry_Message{ + Message: &binlogpb.Message{ Length: uint32(len(data)), Data: data, }, }, } if c.OnClientSide { - ret.Logger = pb.GrpcLogEntry_LOGGER_CLIENT + ret.Logger = binlogpb.GrpcLogEntry_LOGGER_CLIENT } else { - ret.Logger = pb.GrpcLogEntry_LOGGER_SERVER + ret.Logger = binlogpb.GrpcLogEntry_LOGGER_SERVER } return ret } @@ -287,15 +310,15 @@ type ClientHalfClose struct { OnClientSide bool } -func (c *ClientHalfClose) toProto() *pb.GrpcLogEntry { - ret := &pb.GrpcLogEntry{ - Type: pb.GrpcLogEntry_EVENT_TYPE_CLIENT_HALF_CLOSE, +func (c *ClientHalfClose) toProto() *binlogpb.GrpcLogEntry { + ret := &binlogpb.GrpcLogEntry{ + Type: binlogpb.GrpcLogEntry_EVENT_TYPE_CLIENT_HALF_CLOSE, Payload: nil, // No payload here. } if c.OnClientSide { - ret.Logger = pb.GrpcLogEntry_LOGGER_CLIENT + ret.Logger = binlogpb.GrpcLogEntry_LOGGER_CLIENT } else { - ret.Logger = pb.GrpcLogEntry_LOGGER_SERVER + ret.Logger = binlogpb.GrpcLogEntry_LOGGER_SERVER } return ret } @@ -311,7 +334,7 @@ type ServerTrailer struct { PeerAddr net.Addr } -func (c *ServerTrailer) toProto() *pb.GrpcLogEntry { +func (c *ServerTrailer) toProto() *binlogpb.GrpcLogEntry { st, ok := status.FromError(c.Err) if !ok { grpclogLogger.Info("binarylogging: error in trailer is not a status error") @@ -327,10 +350,10 @@ func (c *ServerTrailer) toProto() *pb.GrpcLogEntry { grpclogLogger.Infof("binarylogging: failed to marshal status proto: %v", err) } } - ret := &pb.GrpcLogEntry{ - Type: pb.GrpcLogEntry_EVENT_TYPE_SERVER_TRAILER, - Payload: &pb.GrpcLogEntry_Trailer{ - Trailer: &pb.Trailer{ + ret := &binlogpb.GrpcLogEntry{ + Type: binlogpb.GrpcLogEntry_EVENT_TYPE_SERVER_TRAILER, + Payload: &binlogpb.GrpcLogEntry_Trailer{ + Trailer: &binlogpb.Trailer{ Metadata: mdToMetadataProto(c.Trailer), StatusCode: uint32(st.Code()), StatusMessage: st.Message(), @@ -339,9 +362,9 @@ func (c *ServerTrailer) toProto() *pb.GrpcLogEntry { }, } if c.OnClientSide { - ret.Logger = pb.GrpcLogEntry_LOGGER_CLIENT + ret.Logger = binlogpb.GrpcLogEntry_LOGGER_CLIENT } else { - ret.Logger = pb.GrpcLogEntry_LOGGER_SERVER + ret.Logger = binlogpb.GrpcLogEntry_LOGGER_SERVER } if c.PeerAddr != nil { ret.Peer = addrToProto(c.PeerAddr) @@ -354,15 +377,15 @@ type Cancel struct { OnClientSide bool } -func (c *Cancel) toProto() *pb.GrpcLogEntry { - ret := &pb.GrpcLogEntry{ - Type: pb.GrpcLogEntry_EVENT_TYPE_CANCEL, +func (c *Cancel) toProto() *binlogpb.GrpcLogEntry { + ret := &binlogpb.GrpcLogEntry{ + Type: binlogpb.GrpcLogEntry_EVENT_TYPE_CANCEL, Payload: nil, } if c.OnClientSide { - ret.Logger = pb.GrpcLogEntry_LOGGER_CLIENT + ret.Logger = binlogpb.GrpcLogEntry_LOGGER_CLIENT } else { - ret.Logger = pb.GrpcLogEntry_LOGGER_SERVER + ret.Logger = binlogpb.GrpcLogEntry_LOGGER_SERVER } return ret } @@ -379,15 +402,15 @@ func metadataKeyOmit(key string) bool { return strings.HasPrefix(key, "grpc-") } -func mdToMetadataProto(md metadata.MD) *pb.Metadata { - ret := &pb.Metadata{} +func mdToMetadataProto(md metadata.MD) *binlogpb.Metadata { + ret := &binlogpb.Metadata{} for k, vv := range md { if metadataKeyOmit(k) { continue } for _, v := range vv { ret.Entry = append(ret.Entry, - &pb.MetadataEntry{ + &binlogpb.MetadataEntry{ Key: k, Value: []byte(v), }, @@ -397,26 +420,26 @@ func mdToMetadataProto(md metadata.MD) *pb.Metadata { return ret } -func addrToProto(addr net.Addr) *pb.Address { - ret := &pb.Address{} +func addrToProto(addr net.Addr) *binlogpb.Address { + ret := &binlogpb.Address{} switch a := addr.(type) { case *net.TCPAddr: if a.IP.To4() != nil { - ret.Type = pb.Address_TYPE_IPV4 + ret.Type = binlogpb.Address_TYPE_IPV4 } else if a.IP.To16() != nil { - ret.Type = pb.Address_TYPE_IPV6 + ret.Type = binlogpb.Address_TYPE_IPV6 } else { - ret.Type = pb.Address_TYPE_UNKNOWN + ret.Type = binlogpb.Address_TYPE_UNKNOWN // Do not set address and port fields. break } ret.Address = a.IP.String() ret.IpPort = uint32(a.Port) case *net.UnixAddr: - ret.Type = pb.Address_TYPE_UNIX + ret.Type = binlogpb.Address_TYPE_UNIX ret.Address = a.String() default: - ret.Type = pb.Address_TYPE_UNKNOWN + ret.Type = binlogpb.Address_TYPE_UNKNOWN } return ret } diff --git a/internal/binarylog/method_logger_test.go b/internal/binarylog/method_logger_test.go index a99360bd92df..11255bb338b4 100644 --- a/internal/binarylog/method_logger_test.go +++ b/internal/binarylog/method_logger_test.go @@ -20,21 +20,22 @@ package binarylog import ( "bytes" + "context" "fmt" "net" "testing" "time" "github.com/golang/protobuf/proto" - dpb "github.com/golang/protobuf/ptypes/duration" - pb "google.golang.org/grpc/binarylog/grpc_binarylog_v1" + binlogpb "google.golang.org/grpc/binarylog/grpc_binarylog_v1" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/durationpb" ) func (s) TestLog(t *testing.T) { idGen.reset() - ml := newMethodLogger(10, 10) + ml := NewTruncatingMethodLogger(10, 10) // Set sink to testing buffer. buf := bytes.NewBuffer(nil) ml.sink = newWriterSink(buf) @@ -46,7 +47,7 @@ func (s) TestLog(t *testing.T) { port6 := 796 tcpAddr6, _ := net.ResolveTCPAddr("tcp", fmt.Sprintf("[%v]:%d", addr6, port6)) - testProtoMsg := &pb.Message{ + testProtoMsg := &binlogpb.Message{ Length: 1, Data: []byte{'a'}, } @@ -54,7 +55,7 @@ func (s) TestLog(t *testing.T) { testCases := []struct { config LogEntryConfig - want *pb.GrpcLogEntry + want *binlogpb.GrpcLogEntry }{ { config: &ClientHeader{ @@ -67,31 +68,31 @@ func (s) TestLog(t *testing.T) { Timeout: 2*time.Second + 3*time.Nanosecond, PeerAddr: tcpAddr, }, - want: &pb.GrpcLogEntry{ + want: &binlogpb.GrpcLogEntry{ Timestamp: nil, CallId: 1, SequenceIdWithinCall: 0, - Type: pb.GrpcLogEntry_EVENT_TYPE_CLIENT_HEADER, - Logger: pb.GrpcLogEntry_LOGGER_SERVER, - Payload: &pb.GrpcLogEntry_ClientHeader{ - ClientHeader: &pb.ClientHeader{ - Metadata: &pb.Metadata{ - Entry: []*pb.MetadataEntry{ + Type: binlogpb.GrpcLogEntry_EVENT_TYPE_CLIENT_HEADER, + Logger: binlogpb.GrpcLogEntry_LOGGER_SERVER, + Payload: &binlogpb.GrpcLogEntry_ClientHeader{ + ClientHeader: &binlogpb.ClientHeader{ + Metadata: &binlogpb.Metadata{ + Entry: []*binlogpb.MetadataEntry{ {Key: "a", Value: []byte{'b'}}, {Key: "a", Value: []byte{'b', 'b'}}, }, }, MethodName: "testservice/testmethod", Authority: "test.service.io", - Timeout: &dpb.Duration{ + Timeout: &durationpb.Duration{ Seconds: 2, Nanos: 3, }, }, }, PayloadTruncated: false, - Peer: &pb.Address{ - Type: pb.Address_TYPE_IPV4, + Peer: &binlogpb.Address{ + Type: binlogpb.Address_TYPE_IPV4, Address: addr, IpPort: uint32(port), }, @@ -103,15 +104,15 @@ func (s) TestLog(t *testing.T) { MethodName: "testservice/testmethod", Authority: "test.service.io", }, - want: &pb.GrpcLogEntry{ + want: &binlogpb.GrpcLogEntry{ Timestamp: nil, CallId: 1, SequenceIdWithinCall: 0, - Type: pb.GrpcLogEntry_EVENT_TYPE_CLIENT_HEADER, - Logger: pb.GrpcLogEntry_LOGGER_SERVER, - Payload: &pb.GrpcLogEntry_ClientHeader{ - ClientHeader: &pb.ClientHeader{ - Metadata: &pb.Metadata{}, + Type: binlogpb.GrpcLogEntry_EVENT_TYPE_CLIENT_HEADER, + Logger: binlogpb.GrpcLogEntry_LOGGER_SERVER, + Payload: &binlogpb.GrpcLogEntry_ClientHeader{ + ClientHeader: &binlogpb.ClientHeader{ + Metadata: &binlogpb.Metadata{}, MethodName: "testservice/testmethod", Authority: "test.service.io", }, @@ -127,16 +128,16 @@ func (s) TestLog(t *testing.T) { }, PeerAddr: tcpAddr6, }, - want: &pb.GrpcLogEntry{ + want: &binlogpb.GrpcLogEntry{ Timestamp: nil, CallId: 1, SequenceIdWithinCall: 0, - Type: pb.GrpcLogEntry_EVENT_TYPE_SERVER_HEADER, - Logger: pb.GrpcLogEntry_LOGGER_CLIENT, - Payload: &pb.GrpcLogEntry_ServerHeader{ - ServerHeader: &pb.ServerHeader{ - Metadata: &pb.Metadata{ - Entry: []*pb.MetadataEntry{ + Type: binlogpb.GrpcLogEntry_EVENT_TYPE_SERVER_HEADER, + Logger: binlogpb.GrpcLogEntry_LOGGER_CLIENT, + Payload: &binlogpb.GrpcLogEntry_ServerHeader{ + ServerHeader: &binlogpb.ServerHeader{ + Metadata: &binlogpb.Metadata{ + Entry: []*binlogpb.MetadataEntry{ {Key: "a", Value: []byte{'b'}}, {Key: "a", Value: []byte{'b', 'b'}}, }, @@ -144,8 +145,8 @@ func (s) TestLog(t *testing.T) { }, }, PayloadTruncated: false, - Peer: &pb.Address{ - Type: pb.Address_TYPE_IPV6, + Peer: &binlogpb.Address{ + Type: binlogpb.Address_TYPE_IPV6, Address: addr6, IpPort: uint32(port6), }, @@ -156,14 +157,14 @@ func (s) TestLog(t *testing.T) { OnClientSide: true, Message: testProtoMsg, }, - want: &pb.GrpcLogEntry{ + want: &binlogpb.GrpcLogEntry{ Timestamp: nil, CallId: 1, SequenceIdWithinCall: 0, - Type: pb.GrpcLogEntry_EVENT_TYPE_CLIENT_MESSAGE, - Logger: pb.GrpcLogEntry_LOGGER_CLIENT, - Payload: &pb.GrpcLogEntry_Message{ - Message: &pb.Message{ + Type: binlogpb.GrpcLogEntry_EVENT_TYPE_CLIENT_MESSAGE, + Logger: binlogpb.GrpcLogEntry_LOGGER_CLIENT, + Payload: &binlogpb.GrpcLogEntry_Message{ + Message: &binlogpb.Message{ Length: uint32(len(testProtoBytes)), Data: testProtoBytes, }, @@ -177,14 +178,14 @@ func (s) TestLog(t *testing.T) { OnClientSide: false, Message: testProtoMsg, }, - want: &pb.GrpcLogEntry{ + want: &binlogpb.GrpcLogEntry{ Timestamp: nil, CallId: 1, SequenceIdWithinCall: 0, - Type: pb.GrpcLogEntry_EVENT_TYPE_SERVER_MESSAGE, - Logger: pb.GrpcLogEntry_LOGGER_SERVER, - Payload: &pb.GrpcLogEntry_Message{ - Message: &pb.Message{ + Type: binlogpb.GrpcLogEntry_EVENT_TYPE_SERVER_MESSAGE, + Logger: binlogpb.GrpcLogEntry_LOGGER_SERVER, + Payload: &binlogpb.GrpcLogEntry_Message{ + Message: &binlogpb.Message{ Length: uint32(len(testProtoBytes)), Data: testProtoBytes, }, @@ -197,12 +198,12 @@ func (s) TestLog(t *testing.T) { config: &ClientHalfClose{ OnClientSide: false, }, - want: &pb.GrpcLogEntry{ + want: &binlogpb.GrpcLogEntry{ Timestamp: nil, CallId: 1, SequenceIdWithinCall: 0, - Type: pb.GrpcLogEntry_EVENT_TYPE_CLIENT_HALF_CLOSE, - Logger: pb.GrpcLogEntry_LOGGER_SERVER, + Type: binlogpb.GrpcLogEntry_EVENT_TYPE_CLIENT_HALF_CLOSE, + Logger: binlogpb.GrpcLogEntry_LOGGER_SERVER, Payload: nil, PayloadTruncated: false, Peer: nil, @@ -214,23 +215,23 @@ func (s) TestLog(t *testing.T) { Err: status.Errorf(codes.Unavailable, "test"), PeerAddr: tcpAddr, }, - want: &pb.GrpcLogEntry{ + want: &binlogpb.GrpcLogEntry{ Timestamp: nil, CallId: 1, SequenceIdWithinCall: 0, - Type: pb.GrpcLogEntry_EVENT_TYPE_SERVER_TRAILER, - Logger: pb.GrpcLogEntry_LOGGER_CLIENT, - Payload: &pb.GrpcLogEntry_Trailer{ - Trailer: &pb.Trailer{ - Metadata: &pb.Metadata{}, + Type: binlogpb.GrpcLogEntry_EVENT_TYPE_SERVER_TRAILER, + Logger: binlogpb.GrpcLogEntry_LOGGER_CLIENT, + Payload: &binlogpb.GrpcLogEntry_Trailer{ + Trailer: &binlogpb.Trailer{ + Metadata: &binlogpb.Metadata{}, StatusCode: uint32(codes.Unavailable), StatusMessage: "test", StatusDetails: nil, }, }, PayloadTruncated: false, - Peer: &pb.Address{ - Type: pb.Address_TYPE_IPV4, + Peer: &binlogpb.Address{ + Type: binlogpb.Address_TYPE_IPV4, Address: addr, IpPort: uint32(port), }, @@ -240,15 +241,15 @@ func (s) TestLog(t *testing.T) { config: &ServerTrailer{ OnClientSide: true, }, - want: &pb.GrpcLogEntry{ + want: &binlogpb.GrpcLogEntry{ Timestamp: nil, CallId: 1, SequenceIdWithinCall: 0, - Type: pb.GrpcLogEntry_EVENT_TYPE_SERVER_TRAILER, - Logger: pb.GrpcLogEntry_LOGGER_CLIENT, - Payload: &pb.GrpcLogEntry_Trailer{ - Trailer: &pb.Trailer{ - Metadata: &pb.Metadata{}, + Type: binlogpb.GrpcLogEntry_EVENT_TYPE_SERVER_TRAILER, + Logger: binlogpb.GrpcLogEntry_LOGGER_CLIENT, + Payload: &binlogpb.GrpcLogEntry_Trailer{ + Trailer: &binlogpb.Trailer{ + Metadata: &binlogpb.Metadata{}, StatusCode: uint32(codes.OK), StatusMessage: "", StatusDetails: nil, @@ -262,12 +263,12 @@ func (s) TestLog(t *testing.T) { config: &Cancel{ OnClientSide: true, }, - want: &pb.GrpcLogEntry{ + want: &binlogpb.GrpcLogEntry{ Timestamp: nil, CallId: 1, SequenceIdWithinCall: 0, - Type: pb.GrpcLogEntry_EVENT_TYPE_CANCEL, - Logger: pb.GrpcLogEntry_LOGGER_CLIENT, + Type: binlogpb.GrpcLogEntry_EVENT_TYPE_CANCEL, + Logger: binlogpb.GrpcLogEntry_LOGGER_CLIENT, Payload: nil, PayloadTruncated: false, Peer: nil, @@ -284,16 +285,16 @@ func (s) TestLog(t *testing.T) { "a": {"b", "bb"}, }, }, - want: &pb.GrpcLogEntry{ + want: &binlogpb.GrpcLogEntry{ Timestamp: nil, CallId: 1, SequenceIdWithinCall: 0, - Type: pb.GrpcLogEntry_EVENT_TYPE_CLIENT_HEADER, - Logger: pb.GrpcLogEntry_LOGGER_SERVER, - Payload: &pb.GrpcLogEntry_ClientHeader{ - ClientHeader: &pb.ClientHeader{ - Metadata: &pb.Metadata{ - Entry: []*pb.MetadataEntry{ + Type: binlogpb.GrpcLogEntry_EVENT_TYPE_CLIENT_HEADER, + Logger: binlogpb.GrpcLogEntry_LOGGER_SERVER, + Payload: &binlogpb.GrpcLogEntry_ClientHeader{ + ClientHeader: &binlogpb.ClientHeader{ + Metadata: &binlogpb.Metadata{ + Entry: []*binlogpb.MetadataEntry{ {Key: "a", Value: []byte{'b'}}, {Key: "a", Value: []byte{'b', 'b'}}, }, @@ -312,16 +313,16 @@ func (s) TestLog(t *testing.T) { "a": {"b", "bb"}, }, }, - want: &pb.GrpcLogEntry{ + want: &binlogpb.GrpcLogEntry{ Timestamp: nil, CallId: 1, SequenceIdWithinCall: 0, - Type: pb.GrpcLogEntry_EVENT_TYPE_SERVER_HEADER, - Logger: pb.GrpcLogEntry_LOGGER_CLIENT, - Payload: &pb.GrpcLogEntry_ServerHeader{ - ServerHeader: &pb.ServerHeader{ - Metadata: &pb.Metadata{ - Entry: []*pb.MetadataEntry{ + Type: binlogpb.GrpcLogEntry_EVENT_TYPE_SERVER_HEADER, + Logger: binlogpb.GrpcLogEntry_LOGGER_CLIENT, + Payload: &binlogpb.GrpcLogEntry_ServerHeader{ + ServerHeader: &binlogpb.ServerHeader{ + Metadata: &binlogpb.Metadata{ + Entry: []*binlogpb.MetadataEntry{ {Key: "a", Value: []byte{'b'}}, {Key: "a", Value: []byte{'b', 'b'}}, }, @@ -335,8 +336,8 @@ func (s) TestLog(t *testing.T) { for i, tc := range testCases { buf.Reset() tc.want.SequenceIdWithinCall = uint64(i + 1) - ml.Log(tc.config) - inSink := new(pb.GrpcLogEntry) + ml.Log(context.Background(), tc.config) + inSink := new(binlogpb.GrpcLogEntry) if err := proto.Unmarshal(buf.Bytes()[4:], inSink); err != nil { t.Errorf("failed to unmarshal bytes in sink to proto: %v", err) continue @@ -350,45 +351,45 @@ func (s) TestLog(t *testing.T) { func (s) TestTruncateMetadataNotTruncated(t *testing.T) { testCases := []struct { - ml *MethodLogger - mpPb *pb.Metadata + ml *TruncatingMethodLogger + mpPb *binlogpb.Metadata }{ { - ml: newMethodLogger(maxUInt, maxUInt), - mpPb: &pb.Metadata{ - Entry: []*pb.MetadataEntry{ + ml: NewTruncatingMethodLogger(maxUInt, maxUInt), + mpPb: &binlogpb.Metadata{ + Entry: []*binlogpb.MetadataEntry{ {Key: "", Value: []byte{1}}, }, }, }, { - ml: newMethodLogger(2, maxUInt), - mpPb: &pb.Metadata{ - Entry: []*pb.MetadataEntry{ + ml: NewTruncatingMethodLogger(2, maxUInt), + mpPb: &binlogpb.Metadata{ + Entry: []*binlogpb.MetadataEntry{ {Key: "", Value: []byte{1}}, }, }, }, { - ml: newMethodLogger(1, maxUInt), - mpPb: &pb.Metadata{ - Entry: []*pb.MetadataEntry{ + ml: NewTruncatingMethodLogger(1, maxUInt), + mpPb: &binlogpb.Metadata{ + Entry: []*binlogpb.MetadataEntry{ {Key: "", Value: nil}, }, }, }, { - ml: newMethodLogger(2, maxUInt), - mpPb: &pb.Metadata{ - Entry: []*pb.MetadataEntry{ + ml: NewTruncatingMethodLogger(2, maxUInt), + mpPb: &binlogpb.Metadata{ + Entry: []*binlogpb.MetadataEntry{ {Key: "", Value: []byte{1, 1}}, }, }, }, { - ml: newMethodLogger(2, maxUInt), - mpPb: &pb.Metadata{ - Entry: []*pb.MetadataEntry{ + ml: NewTruncatingMethodLogger(2, maxUInt), + mpPb: &binlogpb.Metadata{ + Entry: []*binlogpb.MetadataEntry{ {Key: "", Value: []byte{1}}, {Key: "", Value: []byte{1}}, }, @@ -397,9 +398,9 @@ func (s) TestTruncateMetadataNotTruncated(t *testing.T) { // "grpc-trace-bin" is kept in log but not counted towards the size // limit. { - ml: newMethodLogger(1, maxUInt), - mpPb: &pb.Metadata{ - Entry: []*pb.MetadataEntry{ + ml: NewTruncatingMethodLogger(1, maxUInt), + mpPb: &binlogpb.Metadata{ + Entry: []*binlogpb.MetadataEntry{ {Key: "", Value: []byte{1}}, {Key: "grpc-trace-bin", Value: []byte("some.trace.key")}, }, @@ -417,24 +418,24 @@ func (s) TestTruncateMetadataNotTruncated(t *testing.T) { func (s) TestTruncateMetadataTruncated(t *testing.T) { testCases := []struct { - ml *MethodLogger - mpPb *pb.Metadata + ml *TruncatingMethodLogger + mpPb *binlogpb.Metadata entryLen int }{ { - ml: newMethodLogger(2, maxUInt), - mpPb: &pb.Metadata{ - Entry: []*pb.MetadataEntry{ + ml: NewTruncatingMethodLogger(2, maxUInt), + mpPb: &binlogpb.Metadata{ + Entry: []*binlogpb.MetadataEntry{ {Key: "", Value: []byte{1, 1, 1}}, }, }, entryLen: 0, }, { - ml: newMethodLogger(2, maxUInt), - mpPb: &pb.Metadata{ - Entry: []*pb.MetadataEntry{ + ml: NewTruncatingMethodLogger(2, maxUInt), + mpPb: &binlogpb.Metadata{ + Entry: []*binlogpb.MetadataEntry{ {Key: "", Value: []byte{1}}, {Key: "", Value: []byte{1}}, {Key: "", Value: []byte{1}}, @@ -443,9 +444,9 @@ func (s) TestTruncateMetadataTruncated(t *testing.T) { entryLen: 2, }, { - ml: newMethodLogger(2, maxUInt), - mpPb: &pb.Metadata{ - Entry: []*pb.MetadataEntry{ + ml: NewTruncatingMethodLogger(2, maxUInt), + mpPb: &binlogpb.Metadata{ + Entry: []*binlogpb.MetadataEntry{ {Key: "", Value: []byte{1, 1}}, {Key: "", Value: []byte{1}}, }, @@ -453,9 +454,9 @@ func (s) TestTruncateMetadataTruncated(t *testing.T) { entryLen: 1, }, { - ml: newMethodLogger(2, maxUInt), - mpPb: &pb.Metadata{ - Entry: []*pb.MetadataEntry{ + ml: NewTruncatingMethodLogger(2, maxUInt), + mpPb: &binlogpb.Metadata{ + Entry: []*binlogpb.MetadataEntry{ {Key: "", Value: []byte{1}}, {Key: "", Value: []byte{1, 1}}, }, @@ -478,24 +479,24 @@ func (s) TestTruncateMetadataTruncated(t *testing.T) { func (s) TestTruncateMessageNotTruncated(t *testing.T) { testCases := []struct { - ml *MethodLogger - msgPb *pb.Message + ml *TruncatingMethodLogger + msgPb *binlogpb.Message }{ { - ml: newMethodLogger(maxUInt, maxUInt), - msgPb: &pb.Message{ + ml: NewTruncatingMethodLogger(maxUInt, maxUInt), + msgPb: &binlogpb.Message{ Data: []byte{1}, }, }, { - ml: newMethodLogger(maxUInt, 3), - msgPb: &pb.Message{ + ml: NewTruncatingMethodLogger(maxUInt, 3), + msgPb: &binlogpb.Message{ Data: []byte{1, 1}, }, }, { - ml: newMethodLogger(maxUInt, 2), - msgPb: &pb.Message{ + ml: NewTruncatingMethodLogger(maxUInt, 2), + msgPb: &binlogpb.Message{ Data: []byte{1, 1}, }, }, @@ -511,14 +512,14 @@ func (s) TestTruncateMessageNotTruncated(t *testing.T) { func (s) TestTruncateMessageTruncated(t *testing.T) { testCases := []struct { - ml *MethodLogger - msgPb *pb.Message + ml *TruncatingMethodLogger + msgPb *binlogpb.Message oldLength uint32 }{ { - ml: newMethodLogger(maxUInt, 2), - msgPb: &pb.Message{ + ml: NewTruncatingMethodLogger(maxUInt, 2), + msgPb: &binlogpb.Message{ Length: 3, Data: []byte{1, 1, 1}, }, diff --git a/internal/binarylog/sink.go b/internal/binarylog/sink.go index 835f51040cb0..264de387c2a5 100644 --- a/internal/binarylog/sink.go +++ b/internal/binarylog/sink.go @@ -21,44 +21,36 @@ package binarylog import ( "bufio" "encoding/binary" - "fmt" "io" - "io/ioutil" "sync" "time" "github.com/golang/protobuf/proto" - pb "google.golang.org/grpc/binarylog/grpc_binarylog_v1" + binlogpb "google.golang.org/grpc/binarylog/grpc_binarylog_v1" ) var ( - defaultSink Sink = &noopSink{} // TODO(blog): change this default (file in /tmp). + // DefaultSink is the sink where the logs will be written to. It's exported + // for the binarylog package to update. + DefaultSink Sink = &noopSink{} // TODO(blog): change this default (file in /tmp). ) -// SetDefaultSink sets the sink where binary logs will be written to. -// -// Not thread safe. Only set during initialization. -func SetDefaultSink(s Sink) { - if defaultSink != nil { - defaultSink.Close() - } - defaultSink = s -} - // Sink writes log entry into the binary log sink. +// +// sink is a copy of the exported binarylog.Sink, to avoid circular dependency. type Sink interface { // Write will be called to write the log entry into the sink. // // It should be thread-safe so it can be called in parallel. - Write(*pb.GrpcLogEntry) error + Write(*binlogpb.GrpcLogEntry) error // Close will be called when the Sink is replaced by a new Sink. Close() error } type noopSink struct{} -func (ns *noopSink) Write(*pb.GrpcLogEntry) error { return nil } -func (ns *noopSink) Close() error { return nil } +func (ns *noopSink) Write(*binlogpb.GrpcLogEntry) error { return nil } +func (ns *noopSink) Close() error { return nil } // newWriterSink creates a binary log sink with the given writer. // @@ -66,7 +58,7 @@ func (ns *noopSink) Close() error { return nil } // message is prefixed with a 4 byte big endian unsigned integer as the length. // // No buffer is done, Close() doesn't try to close the writer. -func newWriterSink(w io.Writer) *writerSink { +func newWriterSink(w io.Writer) Sink { return &writerSink{out: w} } @@ -74,10 +66,11 @@ type writerSink struct { out io.Writer } -func (ws *writerSink) Write(e *pb.GrpcLogEntry) error { +func (ws *writerSink) Write(e *binlogpb.GrpcLogEntry) error { b, err := proto.Marshal(e) if err != nil { - grpclogLogger.Infof("binary logging: failed to marshal proto message: %v", err) + grpclogLogger.Errorf("binary logging: failed to marshal proto message: %v", err) + return err } hdr := make([]byte, 4) binary.BigEndian.PutUint32(hdr, uint32(len(b))) @@ -92,25 +85,28 @@ func (ws *writerSink) Write(e *pb.GrpcLogEntry) error { func (ws *writerSink) Close() error { return nil } -type bufWriteCloserSink struct { - mu sync.Mutex - closer io.Closer - out *writerSink // out is built on buf. - buf *bufio.Writer // buf is kept for flush. +type bufferedSink struct { + mu sync.Mutex + closer io.Closer + out Sink // out is built on buf. + buf *bufio.Writer // buf is kept for flush. + flusherStarted bool - writeStartOnce sync.Once - writeTicker *time.Ticker + writeTicker *time.Ticker + done chan struct{} } -func (fs *bufWriteCloserSink) Write(e *pb.GrpcLogEntry) error { - // Start the write loop when Write is called. - fs.writeStartOnce.Do(fs.startFlushGoroutine) +func (fs *bufferedSink) Write(e *binlogpb.GrpcLogEntry) error { fs.mu.Lock() + defer fs.mu.Unlock() + if !fs.flusherStarted { + // Start the write loop when Write is called. + fs.startFlushGoroutine() + fs.flusherStarted = true + } if err := fs.out.Write(e); err != nil { - fs.mu.Unlock() return err } - fs.mu.Unlock() return nil } @@ -118,44 +114,57 @@ const ( bufFlushDuration = 60 * time.Second ) -func (fs *bufWriteCloserSink) startFlushGoroutine() { +func (fs *bufferedSink) startFlushGoroutine() { fs.writeTicker = time.NewTicker(bufFlushDuration) go func() { - for range fs.writeTicker.C { + for { + select { + case <-fs.done: + return + case <-fs.writeTicker.C: + } fs.mu.Lock() - fs.buf.Flush() + if err := fs.buf.Flush(); err != nil { + grpclogLogger.Warningf("failed to flush to Sink: %v", err) + } fs.mu.Unlock() } }() } -func (fs *bufWriteCloserSink) Close() error { +func (fs *bufferedSink) Close() error { + fs.mu.Lock() + defer fs.mu.Unlock() if fs.writeTicker != nil { fs.writeTicker.Stop() } - fs.mu.Lock() - fs.buf.Flush() - fs.closer.Close() - fs.out.Close() - fs.mu.Unlock() + close(fs.done) + if err := fs.buf.Flush(); err != nil { + grpclogLogger.Warningf("failed to flush to Sink: %v", err) + } + if err := fs.closer.Close(); err != nil { + grpclogLogger.Warningf("failed to close the underlying WriterCloser: %v", err) + } + if err := fs.out.Close(); err != nil { + grpclogLogger.Warningf("failed to close the Sink: %v", err) + } return nil } -func newBufWriteCloserSink(o io.WriteCloser) Sink { +// NewBufferedSink creates a binary log sink with the given WriteCloser. +// +// Write() marshals the proto message and writes it to the given writer. Each +// message is prefixed with a 4 byte big endian unsigned integer as the length. +// +// Content is kept in a buffer, and is flushed every 60 seconds. +// +// Close closes the WriteCloser. +func NewBufferedSink(o io.WriteCloser) Sink { bufW := bufio.NewWriter(o) - return &bufWriteCloserSink{ + return &bufferedSink{ closer: o, out: newWriterSink(bufW), buf: bufW, + done: make(chan struct{}), } } - -// NewTempFileSink creates a temp file and returns a Sink that writes to this -// file. -func NewTempFileSink() (Sink, error) { - tempFile, err := ioutil.TempFile("/tmp", "grpcgo_binarylog_*.txt") - if err != nil { - return nil, fmt.Errorf("failed to create temp file: %v", err) - } - return newBufWriteCloserSink(tempFile), nil -} diff --git a/internal/buffer/unbounded.go b/internal/buffer/unbounded.go index 9f6a0c1200db..4399c3df4959 100644 --- a/internal/buffer/unbounded.go +++ b/internal/buffer/unbounded.go @@ -28,35 +28,38 @@ import "sync" // the underlying mutex used for synchronization. // // Unbounded supports values of any type to be stored in it by using a channel -// of `interface{}`. This means that a call to Put() incurs an extra memory -// allocation, and also that users need a type assertion while reading. For -// performance critical code paths, using Unbounded is strongly discouraged and -// defining a new type specific implementation of this buffer is preferred. See +// of `any`. This means that a call to Put() incurs an extra memory allocation, +// and also that users need a type assertion while reading. For performance +// critical code paths, using Unbounded is strongly discouraged and defining a +// new type specific implementation of this buffer is preferred. See // internal/transport/transport.go for an example of this. type Unbounded struct { - c chan interface{} + c chan any + closed bool mu sync.Mutex - backlog []interface{} + backlog []any } // NewUnbounded returns a new instance of Unbounded. func NewUnbounded() *Unbounded { - return &Unbounded{c: make(chan interface{}, 1)} + return &Unbounded{c: make(chan any, 1)} } // Put adds t to the unbounded buffer. -func (b *Unbounded) Put(t interface{}) { +func (b *Unbounded) Put(t any) { b.mu.Lock() + defer b.mu.Unlock() + if b.closed { + return + } if len(b.backlog) == 0 { select { case b.c <- t: - b.mu.Unlock() return default: } } b.backlog = append(b.backlog, t) - b.mu.Unlock() } // Load sends the earliest buffered data, if any, onto the read channel @@ -64,6 +67,10 @@ func (b *Unbounded) Put(t interface{}) { // value from the read channel. func (b *Unbounded) Load() { b.mu.Lock() + defer b.mu.Unlock() + if b.closed { + return + } if len(b.backlog) > 0 { select { case b.c <- b.backlog[0]: @@ -72,7 +79,6 @@ func (b *Unbounded) Load() { default: } } - b.mu.Unlock() } // Get returns a read channel on which values added to the buffer, via Put(), @@ -80,6 +86,20 @@ func (b *Unbounded) Load() { // // Upon reading a value from this channel, users are expected to call Load() to // send the next buffered value onto the channel if there is any. -func (b *Unbounded) Get() <-chan interface{} { +// +// If the unbounded buffer is closed, the read channel returned by this method +// is closed. +func (b *Unbounded) Get() <-chan any { return b.c } + +// Close closes the unbounded buffer. +func (b *Unbounded) Close() { + b.mu.Lock() + defer b.mu.Unlock() + if b.closed { + return + } + b.closed = true + close(b.c) +} diff --git a/internal/buffer/unbounded_test.go b/internal/buffer/unbounded_test.go index 8cb800dd0f09..1708391e7f27 100644 --- a/internal/buffer/unbounded_test.go +++ b/internal/buffer/unbounded_test.go @@ -119,3 +119,19 @@ func (s) TestMultipleWriters(t *testing.T) { t.Errorf("reads: %#v, wantReads: %#v", reads, wantReads) } } + +// TestClose closes the buffer and makes sure that nothing is sent after the +// buffer is closed. +func (s) TestClose(t *testing.T) { + ub := NewUnbounded() + ub.Close() + if v, ok := <-ub.Get(); ok { + t.Errorf("Unbounded.Get() = %v, want closed channel", v) + } + ub.Put(1) + ub.Load() + if v, ok := <-ub.Get(); ok { + t.Errorf("Unbounded.Get() = %v, want closed channel", v) + } + ub.Close() +} diff --git a/internal/cache/timeoutCache.go b/internal/cache/timeoutCache.go index 200b499ec81e..3f2d47302c4e 100644 --- a/internal/cache/timeoutCache.go +++ b/internal/cache/timeoutCache.go @@ -23,7 +23,7 @@ import ( ) type cacheEntry struct { - item interface{} + item any // Note that to avoid deadlocks (potentially caused by lock ordering), // callback can only be called without holding cache's mutex. callback func() @@ -38,14 +38,14 @@ type cacheEntry struct { type TimeoutCache struct { mu sync.Mutex timeout time.Duration - cache map[interface{}]*cacheEntry + cache map[any]*cacheEntry } // NewTimeoutCache creates a TimeoutCache with the given timeout. func NewTimeoutCache(timeout time.Duration) *TimeoutCache { return &TimeoutCache{ timeout: timeout, - cache: make(map[interface{}]*cacheEntry), + cache: make(map[any]*cacheEntry), } } @@ -57,7 +57,7 @@ func NewTimeoutCache(timeout time.Duration) *TimeoutCache { // If the Add was successful, it returns (newly added item, true). If there is // an existing entry for the specified key, the cache entry is not be updated // with the specified item and it returns (existing item, false). -func (c *TimeoutCache) Add(key, item interface{}, callback func()) (interface{}, bool) { +func (c *TimeoutCache) Add(key, item any, callback func()) (any, bool) { c.mu.Lock() defer c.mu.Unlock() if e, ok := c.cache[key]; ok { @@ -88,7 +88,7 @@ func (c *TimeoutCache) Add(key, item interface{}, callback func()) (interface{}, // If the specified key exists in the cache, it returns (item associated with // key, true) and the callback associated with the item is guaranteed to be not // called. If the given key is not found in the cache, it returns (nil, false) -func (c *TimeoutCache) Remove(key interface{}) (item interface{}, ok bool) { +func (c *TimeoutCache) Remove(key any) (item any, ok bool) { c.mu.Lock() defer c.mu.Unlock() entry, ok := c.removeInternal(key) @@ -101,7 +101,7 @@ func (c *TimeoutCache) Remove(key interface{}) (item interface{}, ok bool) { // removeInternal removes and returns the item with key. // // caller must hold c.mu. -func (c *TimeoutCache) removeInternal(key interface{}) (*cacheEntry, bool) { +func (c *TimeoutCache) removeInternal(key any) (*cacheEntry, bool) { entry, ok := c.cache[key] if !ok { return nil, false diff --git a/internal/cache/timeoutCache_test.go b/internal/cache/timeoutCache_test.go index 9129f47e72f1..106ea7c911d7 100644 --- a/internal/cache/timeoutCache_test.go +++ b/internal/cache/timeoutCache_test.go @@ -38,7 +38,7 @@ func Test(t *testing.T) { grpctest.RunSubTests(t, s{}) } -func (c *TimeoutCache) getForTesting(key interface{}) (*cacheEntry, bool) { +func (c *TimeoutCache) getForTesting(key any) (*cacheEntry, bool) { c.mu.Lock() defer c.mu.Unlock() r, ok := c.cache[key] diff --git a/internal/channelz/funcs.go b/internal/channelz/funcs.go index 81d3dd33e62c..5395e77529cd 100644 --- a/internal/channelz/funcs.go +++ b/internal/channelz/funcs.go @@ -24,7 +24,7 @@ package channelz import ( - "fmt" + "errors" "sort" "sync" "sync/atomic" @@ -38,8 +38,11 @@ const ( ) var ( - db dbWrapper - idGen idGenerator + // IDGen is the global channelz entity ID generator. It should not be used + // outside this package except by tests. + IDGen IDGenerator + + db dbWrapper // EntryPerPage defines the number of channelz entries to be shown on a web page. EntryPerPage = int64(50) curState int32 @@ -49,14 +52,15 @@ var ( // TurnOn turns on channelz data collection. func TurnOn() { if !IsOn() { - NewChannelzStorage() + db.set(newChannelMap()) + IDGen.Reset() atomic.StoreInt32(&curState, 1) } } // IsOn returns whether channelz data collection is on. func IsOn() bool { - return atomic.CompareAndSwapInt32(&curState, 1, 1) + return atomic.LoadInt32(&curState) == 1 } // SetMaxTraceEntry sets maximum number of trace entry per entity (i.e. channel/subchannel). @@ -94,49 +98,6 @@ func (d *dbWrapper) get() *channelMap { return d.DB } -// NewChannelzStorage initializes channelz data storage and id generator. -// -// This function returns a cleanup function to wait for all channelz state to be reset by the -// grpc goroutines when those entities get closed. By using this cleanup function, we make sure tests -// don't mess up each other, i.e. lingering goroutine from previous test doing entity removal happen -// to remove some entity just register by the new test, since the id space is the same. -// -// Note: This function is exported for testing purpose only. User should not call -// it in most cases. -func NewChannelzStorage() (cleanup func() error) { - db.set(&channelMap{ - topLevelChannels: make(map[int64]struct{}), - channels: make(map[int64]*channel), - listenSockets: make(map[int64]*listenSocket), - normalSockets: make(map[int64]*normalSocket), - servers: make(map[int64]*server), - subChannels: make(map[int64]*subChannel), - }) - idGen.reset() - return func() error { - var err error - cm := db.get() - if cm == nil { - return nil - } - for i := 0; i < 1000; i++ { - cm.mu.Lock() - if len(cm.topLevelChannels) == 0 && len(cm.servers) == 0 && len(cm.channels) == 0 && len(cm.subChannels) == 0 && len(cm.listenSockets) == 0 && len(cm.normalSockets) == 0 { - cm.mu.Unlock() - // all things stored in the channelz map have been cleared. - return nil - } - cm.mu.Unlock() - time.Sleep(10 * time.Millisecond) - } - - cm.mu.Lock() - err = fmt.Errorf("after 10s the channelz map has not been cleaned up yet, topchannels: %d, servers: %d, channels: %d, subchannels: %d, listen sockets: %d, normal sockets: %d", len(cm.topLevelChannels), len(cm.servers), len(cm.channels), len(cm.subChannels), len(cm.listenSockets), len(cm.normalSockets)) - cm.mu.Unlock() - return err - } -} - // GetTopChannels returns a slice of top channel's ChannelMetric, along with a // boolean indicating whether there's more top channels to be queried for. // @@ -188,54 +149,77 @@ func GetServer(id int64) *ServerMetric { return db.get().GetServer(id) } -// RegisterChannel registers the given channel c in channelz database with ref -// as its reference name, and add it to the child list of its parent (identified -// by pid). pid = 0 means no parent. It returns the unique channelz tracking id -// assigned to this channel. -func RegisterChannel(c Channel, pid int64, ref string) int64 { - id := idGen.genID() +// RegisterChannel registers the given channel c in the channelz database with +// ref as its reference name, and adds it to the child list of its parent +// (identified by pid). pid == nil means no parent. +// +// Returns a unique channelz identifier assigned to this channel. +// +// If channelz is not turned ON, the channelz database is not mutated. +func RegisterChannel(c Channel, pid *Identifier, ref string) *Identifier { + id := IDGen.genID() + var parent int64 + isTopChannel := true + if pid != nil { + isTopChannel = false + parent = pid.Int() + } + + if !IsOn() { + return newIdentifer(RefChannel, id, pid) + } + cn := &channel{ refName: ref, c: c, subChans: make(map[int64]string), nestedChans: make(map[int64]string), id: id, - pid: pid, + pid: parent, trace: &channelTrace{createdTime: time.Now(), events: make([]*TraceEvent, 0, getMaxTraceEntry())}, } - if pid == 0 { - db.get().addChannel(id, cn, true, pid, ref) - } else { - db.get().addChannel(id, cn, false, pid, ref) - } - return id + db.get().addChannel(id, cn, isTopChannel, parent) + return newIdentifer(RefChannel, id, pid) } -// RegisterSubChannel registers the given channel c in channelz database with ref -// as its reference name, and add it to the child list of its parent (identified -// by pid). It returns the unique channelz tracking id assigned to this subchannel. -func RegisterSubChannel(c Channel, pid int64, ref string) int64 { - if pid == 0 { - logger.Error("a SubChannel's parent id cannot be 0") - return 0 +// RegisterSubChannel registers the given subChannel c in the channelz database +// with ref as its reference name, and adds it to the child list of its parent +// (identified by pid). +// +// Returns a unique channelz identifier assigned to this subChannel. +// +// If channelz is not turned ON, the channelz database is not mutated. +func RegisterSubChannel(c Channel, pid *Identifier, ref string) (*Identifier, error) { + if pid == nil { + return nil, errors.New("a SubChannel's parent id cannot be nil") } - id := idGen.genID() + id := IDGen.genID() + if !IsOn() { + return newIdentifer(RefSubChannel, id, pid), nil + } + sc := &subChannel{ refName: ref, c: c, sockets: make(map[int64]string), id: id, - pid: pid, + pid: pid.Int(), trace: &channelTrace{createdTime: time.Now(), events: make([]*TraceEvent, 0, getMaxTraceEntry())}, } - db.get().addSubChannel(id, sc, pid, ref) - return id + db.get().addSubChannel(id, sc, pid.Int()) + return newIdentifer(RefSubChannel, id, pid), nil } // RegisterServer registers the given server s in channelz database. It returns // the unique channelz tracking id assigned to this server. -func RegisterServer(s Server, ref string) int64 { - id := idGen.genID() +// +// If channelz is not turned ON, the channelz database is not mutated. +func RegisterServer(s Server, ref string) *Identifier { + id := IDGen.genID() + if !IsOn() { + return newIdentifer(RefServer, id, nil) + } + svr := &server{ refName: ref, s: s, @@ -244,73 +228,92 @@ func RegisterServer(s Server, ref string) int64 { id: id, } db.get().addServer(id, svr) - return id + return newIdentifer(RefServer, id, nil) } // RegisterListenSocket registers the given listen socket s in channelz database // with ref as its reference name, and add it to the child list of its parent // (identified by pid). It returns the unique channelz tracking id assigned to // this listen socket. -func RegisterListenSocket(s Socket, pid int64, ref string) int64 { - if pid == 0 { - logger.Error("a ListenSocket's parent id cannot be 0") - return 0 +// +// If channelz is not turned ON, the channelz database is not mutated. +func RegisterListenSocket(s Socket, pid *Identifier, ref string) (*Identifier, error) { + if pid == nil { + return nil, errors.New("a ListenSocket's parent id cannot be 0") } - id := idGen.genID() - ls := &listenSocket{refName: ref, s: s, id: id, pid: pid} - db.get().addListenSocket(id, ls, pid, ref) - return id + id := IDGen.genID() + if !IsOn() { + return newIdentifer(RefListenSocket, id, pid), nil + } + + ls := &listenSocket{refName: ref, s: s, id: id, pid: pid.Int()} + db.get().addListenSocket(id, ls, pid.Int()) + return newIdentifer(RefListenSocket, id, pid), nil } // RegisterNormalSocket registers the given normal socket s in channelz database -// with ref as its reference name, and add it to the child list of its parent +// with ref as its reference name, and adds it to the child list of its parent // (identified by pid). It returns the unique channelz tracking id assigned to // this normal socket. -func RegisterNormalSocket(s Socket, pid int64, ref string) int64 { - if pid == 0 { - logger.Error("a NormalSocket's parent id cannot be 0") - return 0 +// +// If channelz is not turned ON, the channelz database is not mutated. +func RegisterNormalSocket(s Socket, pid *Identifier, ref string) (*Identifier, error) { + if pid == nil { + return nil, errors.New("a NormalSocket's parent id cannot be 0") + } + id := IDGen.genID() + if !IsOn() { + return newIdentifer(RefNormalSocket, id, pid), nil } - id := idGen.genID() - ns := &normalSocket{refName: ref, s: s, id: id, pid: pid} - db.get().addNormalSocket(id, ns, pid, ref) - return id + + ns := &normalSocket{refName: ref, s: s, id: id, pid: pid.Int()} + db.get().addNormalSocket(id, ns, pid.Int()) + return newIdentifer(RefNormalSocket, id, pid), nil } -// RemoveEntry removes an entry with unique channelz trakcing id to be id from +// RemoveEntry removes an entry with unique channelz tracking id to be id from // channelz database. -func RemoveEntry(id int64) { - db.get().removeEntry(id) +// +// If channelz is not turned ON, this function is a no-op. +func RemoveEntry(id *Identifier) { + if !IsOn() { + return + } + db.get().removeEntry(id.Int()) } -// TraceEventDesc is what the caller of AddTraceEvent should provide to describe the event to be added -// to the channel trace. -// The Parent field is optional. It is used for event that will be recorded in the entity's parent -// trace also. +// TraceEventDesc is what the caller of AddTraceEvent should provide to describe +// the event to be added to the channel trace. +// +// The Parent field is optional. It is used for an event that will be recorded +// in the entity's parent trace. type TraceEventDesc struct { Desc string Severity Severity Parent *TraceEventDesc } -// AddTraceEvent adds trace related to the entity with specified id, using the provided TraceEventDesc. -func AddTraceEvent(l grpclog.DepthLoggerV2, id int64, depth int, desc *TraceEventDesc) { - for d := desc; d != nil; d = d.Parent { - switch d.Severity { - case CtUNKNOWN: - l.InfoDepth(depth+1, d.Desc) - case CtINFO: - l.InfoDepth(depth+1, d.Desc) - case CtWarning: - l.WarningDepth(depth+1, d.Desc) - case CtError: - l.ErrorDepth(depth+1, d.Desc) - } +// AddTraceEvent adds trace related to the entity with specified id, using the +// provided TraceEventDesc. +// +// If channelz is not turned ON, this will simply log the event descriptions. +func AddTraceEvent(l grpclog.DepthLoggerV2, id *Identifier, depth int, desc *TraceEventDesc) { + // Log only the trace description associated with the bottom most entity. + switch desc.Severity { + case CtUnknown, CtInfo: + l.InfoDepth(depth+1, withParens(id)+desc.Desc) + case CtWarning: + l.WarningDepth(depth+1, withParens(id)+desc.Desc) + case CtError: + l.ErrorDepth(depth+1, withParens(id)+desc.Desc) } + if getMaxTraceEntry() == 0 { return } - db.get().traceEvent(id, desc) + if IsOn() { + db.get().traceEvent(id.Int(), desc) + } } // channelMap is the storage data structure for channelz. @@ -328,6 +331,17 @@ type channelMap struct { normalSockets map[int64]*normalSocket } +func newChannelMap() *channelMap { + return &channelMap{ + topLevelChannels: make(map[int64]struct{}), + channels: make(map[int64]*channel), + listenSockets: make(map[int64]*listenSocket), + normalSockets: make(map[int64]*normalSocket), + servers: make(map[int64]*server), + subChannels: make(map[int64]*subChannel), + } +} + func (c *channelMap) addServer(id int64, s *server) { c.mu.Lock() s.cm = c @@ -335,7 +349,7 @@ func (c *channelMap) addServer(id int64, s *server) { c.mu.Unlock() } -func (c *channelMap) addChannel(id int64, cn *channel, isTopChannel bool, pid int64, ref string) { +func (c *channelMap) addChannel(id int64, cn *channel, isTopChannel bool, pid int64) { c.mu.Lock() cn.cm = c cn.trace.cm = c @@ -348,7 +362,7 @@ func (c *channelMap) addChannel(id int64, cn *channel, isTopChannel bool, pid in c.mu.Unlock() } -func (c *channelMap) addSubChannel(id int64, sc *subChannel, pid int64, ref string) { +func (c *channelMap) addSubChannel(id int64, sc *subChannel, pid int64) { c.mu.Lock() sc.cm = c sc.trace.cm = c @@ -357,7 +371,7 @@ func (c *channelMap) addSubChannel(id int64, sc *subChannel, pid int64, ref stri c.mu.Unlock() } -func (c *channelMap) addListenSocket(id int64, ls *listenSocket, pid int64, ref string) { +func (c *channelMap) addListenSocket(id int64, ls *listenSocket, pid int64) { c.mu.Lock() ls.cm = c c.listenSockets[id] = ls @@ -365,7 +379,7 @@ func (c *channelMap) addListenSocket(id int64, ls *listenSocket, pid int64, ref c.mu.Unlock() } -func (c *channelMap) addNormalSocket(id int64, ns *normalSocket, pid int64, ref string) { +func (c *channelMap) addNormalSocket(id int64, ns *normalSocket, pid int64) { c.mu.Lock() ns.cm = c c.normalSockets[id] = ns @@ -632,7 +646,7 @@ func (c *channelMap) GetServerSockets(id int64, startID int64, maxResults int64) if count == 0 { end = true } - var s []*SocketMetric + s := make([]*SocketMetric, 0, len(sks)) for _, ns := range sks { sm := &SocketMetric{} sm.SocketData = ns.s.ChannelzMetric() @@ -726,14 +740,17 @@ func (c *channelMap) GetServer(id int64) *ServerMetric { return sm } -type idGenerator struct { +// IDGenerator is an incrementing atomic that tracks IDs for channelz entities. +type IDGenerator struct { id int64 } -func (i *idGenerator) reset() { +// Reset resets the generated ID back to zero. Should only be used at +// initialization or by tests sensitive to the ID number. +func (i *IDGenerator) Reset() { atomic.StoreInt64(&i.id, 0) } -func (i *idGenerator) genID() int64 { +func (i *IDGenerator) genID() int64 { return atomic.AddInt64(&i.id, 1) } diff --git a/internal/channelz/id.go b/internal/channelz/id.go new file mode 100644 index 000000000000..c9a27acd3710 --- /dev/null +++ b/internal/channelz/id.go @@ -0,0 +1,75 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package channelz + +import "fmt" + +// Identifier is an opaque identifier which uniquely identifies an entity in the +// channelz database. +type Identifier struct { + typ RefChannelType + id int64 + str string + pid *Identifier +} + +// Type returns the entity type corresponding to id. +func (id *Identifier) Type() RefChannelType { + return id.typ +} + +// Int returns the integer identifier corresponding to id. +func (id *Identifier) Int() int64 { + return id.id +} + +// String returns a string representation of the entity corresponding to id. +// +// This includes some information about the parent as well. Examples: +// Top-level channel: [Channel #channel-number] +// Nested channel: [Channel #parent-channel-number Channel #channel-number] +// Sub channel: [Channel #parent-channel SubChannel #subchannel-number] +func (id *Identifier) String() string { + return id.str +} + +// Equal returns true if other is the same as id. +func (id *Identifier) Equal(other *Identifier) bool { + if (id != nil) != (other != nil) { + return false + } + if id == nil && other == nil { + return true + } + return id.typ == other.typ && id.id == other.id && id.pid == other.pid +} + +// NewIdentifierForTesting returns a new opaque identifier to be used only for +// testing purposes. +func NewIdentifierForTesting(typ RefChannelType, id int64, pid *Identifier) *Identifier { + return newIdentifer(typ, id, pid) +} + +func newIdentifer(typ RefChannelType, id int64, pid *Identifier) *Identifier { + str := fmt.Sprintf("%s #%d", typ, id) + if pid != nil { + str = fmt.Sprintf("%s %s", pid, str) + } + return &Identifier{typ: typ, id: id, str: str, pid: pid} +} diff --git a/internal/channelz/logging.go b/internal/channelz/logging.go index e94039ee20b5..f89e6f77bbd0 100644 --- a/internal/channelz/logging.go +++ b/internal/channelz/logging.go @@ -26,77 +26,54 @@ import ( var logger = grpclog.Component("channelz") +func withParens(id *Identifier) string { + return "[" + id.String() + "] " +} + // Info logs and adds a trace event if channelz is on. -func Info(l grpclog.DepthLoggerV2, id int64, args ...interface{}) { - if IsOn() { - AddTraceEvent(l, id, 1, &TraceEventDesc{ - Desc: fmt.Sprint(args...), - Severity: CtINFO, - }) - } else { - l.InfoDepth(1, args...) - } +func Info(l grpclog.DepthLoggerV2, id *Identifier, args ...any) { + AddTraceEvent(l, id, 1, &TraceEventDesc{ + Desc: fmt.Sprint(args...), + Severity: CtInfo, + }) } // Infof logs and adds a trace event if channelz is on. -func Infof(l grpclog.DepthLoggerV2, id int64, format string, args ...interface{}) { - msg := fmt.Sprintf(format, args...) - if IsOn() { - AddTraceEvent(l, id, 1, &TraceEventDesc{ - Desc: msg, - Severity: CtINFO, - }) - } else { - l.InfoDepth(1, msg) - } +func Infof(l grpclog.DepthLoggerV2, id *Identifier, format string, args ...any) { + AddTraceEvent(l, id, 1, &TraceEventDesc{ + Desc: fmt.Sprintf(format, args...), + Severity: CtInfo, + }) } // Warning logs and adds a trace event if channelz is on. -func Warning(l grpclog.DepthLoggerV2, id int64, args ...interface{}) { - if IsOn() { - AddTraceEvent(l, id, 1, &TraceEventDesc{ - Desc: fmt.Sprint(args...), - Severity: CtWarning, - }) - } else { - l.WarningDepth(1, args...) - } +func Warning(l grpclog.DepthLoggerV2, id *Identifier, args ...any) { + AddTraceEvent(l, id, 1, &TraceEventDesc{ + Desc: fmt.Sprint(args...), + Severity: CtWarning, + }) } // Warningf logs and adds a trace event if channelz is on. -func Warningf(l grpclog.DepthLoggerV2, id int64, format string, args ...interface{}) { - msg := fmt.Sprintf(format, args...) - if IsOn() { - AddTraceEvent(l, id, 1, &TraceEventDesc{ - Desc: msg, - Severity: CtWarning, - }) - } else { - l.WarningDepth(1, msg) - } +func Warningf(l grpclog.DepthLoggerV2, id *Identifier, format string, args ...any) { + AddTraceEvent(l, id, 1, &TraceEventDesc{ + Desc: fmt.Sprintf(format, args...), + Severity: CtWarning, + }) } // Error logs and adds a trace event if channelz is on. -func Error(l grpclog.DepthLoggerV2, id int64, args ...interface{}) { - if IsOn() { - AddTraceEvent(l, id, 1, &TraceEventDesc{ - Desc: fmt.Sprint(args...), - Severity: CtError, - }) - } else { - l.ErrorDepth(1, args...) - } +func Error(l grpclog.DepthLoggerV2, id *Identifier, args ...any) { + AddTraceEvent(l, id, 1, &TraceEventDesc{ + Desc: fmt.Sprint(args...), + Severity: CtError, + }) } // Errorf logs and adds a trace event if channelz is on. -func Errorf(l grpclog.DepthLoggerV2, id int64, format string, args ...interface{}) { - msg := fmt.Sprintf(format, args...) - if IsOn() { - AddTraceEvent(l, id, 1, &TraceEventDesc{ - Desc: msg, - Severity: CtError, - }) - } else { - l.ErrorDepth(1, msg) - } +func Errorf(l grpclog.DepthLoggerV2, id *Identifier, format string, args ...any) { + AddTraceEvent(l, id, 1, &TraceEventDesc{ + Desc: fmt.Sprintf(format, args...), + Severity: CtError, + }) } diff --git a/internal/channelz/types.go b/internal/channelz/types.go index 075dc7d16714..1d4020f53795 100644 --- a/internal/channelz/types.go +++ b/internal/channelz/types.go @@ -273,10 +273,10 @@ func (c *channel) deleteSelfFromMap() (delete bool) { // deleteSelfIfReady tries to delete the channel itself from the channelz database. // The delete process includes two steps: -// 1. delete the channel from the entry relation tree, i.e. delete the channel reference from its -// parent's child list. -// 2. delete the channel from the map, i.e. delete the channel entirely from channelz. Lookup by id -// will return entry not found error. +// 1. delete the channel from the entry relation tree, i.e. delete the channel reference from its +// parent's child list. +// 2. delete the channel from the map, i.e. delete the channel entirely from channelz. Lookup by id +// will return entry not found error. func (c *channel) deleteSelfIfReady() { if !c.deleteSelfFromTree() { return @@ -381,10 +381,10 @@ func (sc *subChannel) deleteSelfFromMap() (delete bool) { // deleteSelfIfReady tries to delete the subchannel itself from the channelz database. // The delete process includes two steps: -// 1. delete the subchannel from the entry relation tree, i.e. delete the subchannel reference from -// its parent's child list. -// 2. delete the subchannel from the map, i.e. delete the subchannel entirely from channelz. Lookup -// by id will return entry not found error. +// 1. delete the subchannel from the entry relation tree, i.e. delete the subchannel reference from +// its parent's child list. +// 2. delete the subchannel from the map, i.e. delete the subchannel entirely from channelz. Lookup +// by id will return entry not found error. func (sc *subChannel) deleteSelfIfReady() { if !sc.deleteSelfFromTree() { return @@ -628,6 +628,7 @@ type tracedChannel interface { type channelTrace struct { cm *channelMap + clearCalled bool createdTime time.Time eventCount int64 mu sync.Mutex @@ -656,6 +657,10 @@ func (c *channelTrace) append(e *TraceEvent) { } func (c *channelTrace) clear() { + if c.clearCalled { + return + } + c.clearCalled = true c.mu.Lock() for _, e := range c.events { if e.RefID != 0 { @@ -672,10 +677,10 @@ func (c *channelTrace) clear() { type Severity int const ( - // CtUNKNOWN indicates unknown severity of a trace event. - CtUNKNOWN Severity = iota - // CtINFO indicates info level severity of a trace event. - CtINFO + // CtUnknown indicates unknown severity of a trace event. + CtUnknown Severity = iota + // CtInfo indicates info level severity of a trace event. + CtInfo // CtWarning indicates warning level severity of a trace event. CtWarning // CtError indicates error level severity of a trace event. @@ -686,12 +691,33 @@ const ( type RefChannelType int const ( + // RefUnknown indicates an unknown entity type, the zero value for this type. + RefUnknown RefChannelType = iota // RefChannel indicates the referenced entity is a Channel. - RefChannel RefChannelType = iota + RefChannel // RefSubChannel indicates the referenced entity is a SubChannel. RefSubChannel + // RefServer indicates the referenced entity is a Server. + RefServer + // RefListenSocket indicates the referenced entity is a ListenSocket. + RefListenSocket + // RefNormalSocket indicates the referenced entity is a NormalSocket. + RefNormalSocket ) +var refChannelTypeToString = map[RefChannelType]string{ + RefUnknown: "Unknown", + RefChannel: "Channel", + RefSubChannel: "SubChannel", + RefServer: "Server", + RefListenSocket: "ListenSocket", + RefNormalSocket: "NormalSocket", +} + +func (r RefChannelType) String() string { + return refChannelTypeToString[r] +} + func (c *channelTrace) dumpData() *ChannelTrace { c.mu.Lock() ct := &ChannelTrace{EventNum: c.eventCount, CreationTime: c.createdTime} diff --git a/internal/channelz/types_nonlinux.go b/internal/channelz/types_nonlinux.go index da7351e9709a..8b06eed1ab8b 100644 --- a/internal/channelz/types_nonlinux.go +++ b/internal/channelz/types_nonlinux.go @@ -1,4 +1,5 @@ -// +build !linux appengine +//go:build !linux +// +build !linux /* * @@ -37,6 +38,6 @@ type SocketOptionData struct { // Windows OS doesn't support Socket Option func (s *SocketOptionData) Getsockopt(fd uintptr) { once.Do(func() { - logger.Warning("Channelz: socket options are not supported on non-linux os.") + logger.Warning("Channelz: socket options are not supported on non-linux environments") }) } diff --git a/internal/channelz/util_linux.go b/internal/channelz/util_linux.go index 49432bebf5f4..98288c3f866f 100644 --- a/internal/channelz/util_linux.go +++ b/internal/channelz/util_linux.go @@ -1,5 +1,3 @@ -// +build linux - /* * * Copyright 2018 gRPC authors. @@ -25,7 +23,7 @@ import ( ) // GetSocketOption gets the socket option info of the conn. -func GetSocketOption(socket interface{}) *SocketOptionData { +func GetSocketOption(socket any) *SocketOptionData { c, ok := socket.(syscall.Conn) if !ok { return nil diff --git a/internal/channelz/util_nonlinux.go b/internal/channelz/util_nonlinux.go index 8864a0811164..b5568b22e208 100644 --- a/internal/channelz/util_nonlinux.go +++ b/internal/channelz/util_nonlinux.go @@ -1,4 +1,5 @@ -// +build !linux appengine +//go:build !linux +// +build !linux /* * @@ -21,6 +22,6 @@ package channelz // GetSocketOption gets the socket option info of the conn. -func GetSocketOption(c interface{}) *SocketOptionData { +func GetSocketOption(c any) *SocketOptionData { return nil } diff --git a/internal/channelz/util_test.go b/internal/channelz/util_test.go index 2621e7410d61..da0fd30b1905 100644 --- a/internal/channelz/util_test.go +++ b/internal/channelz/util_test.go @@ -1,3 +1,4 @@ +//go:build linux // +build linux /* @@ -18,10 +19,6 @@ * */ -// The test in this file should be run in an environment that has go1.10 or later, -// as the function SyscallConn() (required to get socket option) was introduced -// to net.TCPListener in go1.10. - package channelz_test import ( diff --git a/internal/credentials/credentials.go b/internal/credentials/credentials.go new file mode 100644 index 000000000000..9deee7f6513e --- /dev/null +++ b/internal/credentials/credentials.go @@ -0,0 +1,49 @@ +/* + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package credentials + +import ( + "context" +) + +// requestInfoKey is a struct to be used as the key to store RequestInfo in a +// context. +type requestInfoKey struct{} + +// NewRequestInfoContext creates a context with ri. +func NewRequestInfoContext(ctx context.Context, ri any) context.Context { + return context.WithValue(ctx, requestInfoKey{}, ri) +} + +// RequestInfoFromContext extracts the RequestInfo from ctx. +func RequestInfoFromContext(ctx context.Context) any { + return ctx.Value(requestInfoKey{}) +} + +// clientHandshakeInfoKey is a struct used as the key to store +// ClientHandshakeInfo in a context. +type clientHandshakeInfoKey struct{} + +// ClientHandshakeInfoFromContext extracts the ClientHandshakeInfo from ctx. +func ClientHandshakeInfoFromContext(ctx context.Context) any { + return ctx.Value(clientHandshakeInfoKey{}) +} + +// NewClientHandshakeInfoContext creates a context with chi. +func NewClientHandshakeInfoContext(ctx context.Context, chi any) context.Context { + return context.WithValue(ctx, clientHandshakeInfoKey{}, chi) +} diff --git a/internal/credentials/spiffe.go b/internal/credentials/spiffe.go index 406023035288..25ade623058e 100644 --- a/internal/credentials/spiffe.go +++ b/internal/credentials/spiffe.go @@ -23,6 +23,7 @@ package credentials import ( "crypto/tls" + "crypto/x509" "net/url" "google.golang.org/grpc/grpclog" @@ -36,8 +37,17 @@ func SPIFFEIDFromState(state tls.ConnectionState) *url.URL { if len(state.PeerCertificates) == 0 || len(state.PeerCertificates[0].URIs) == 0 { return nil } + return SPIFFEIDFromCert(state.PeerCertificates[0]) +} + +// SPIFFEIDFromCert parses the SPIFFE ID from x509.Certificate. If the SPIFFE +// ID format is invalid, return nil with warning. +func SPIFFEIDFromCert(cert *x509.Certificate) *url.URL { + if cert == nil || cert.URIs == nil { + return nil + } var spiffeID *url.URL - for _, uri := range state.PeerCertificates[0].URIs { + for _, uri := range cert.URIs { if uri == nil || uri.Scheme != "spiffe" || uri.Opaque != "" || (uri.User != nil && uri.User.Username() != "") { continue } @@ -46,7 +56,7 @@ func SPIFFEIDFromState(state tls.ConnectionState) *url.URL { logger.Warning("invalid SPIFFE ID: total ID length larger than 2048 bytes") return nil } - if len(uri.Host) == 0 || len(uri.RawPath) == 0 || len(uri.Path) == 0 { + if len(uri.Host) == 0 || len(uri.Path) == 0 { logger.Warning("invalid SPIFFE ID: domain or workload ID is empty") return nil } @@ -55,7 +65,7 @@ func SPIFFEIDFromState(state tls.ConnectionState) *url.URL { return nil } // A valid SPIFFE certificate can only have exactly one URI SAN field. - if len(state.PeerCertificates[0].URIs) > 1 { + if len(cert.URIs) > 1 { logger.Warning("invalid SPIFFE ID: multiple URI SANs") return nil } diff --git a/internal/credentials/spiffe_test.go b/internal/credentials/spiffe_test.go index 324874e6e396..0011ed012bbd 100644 --- a/internal/credentials/spiffe_test.go +++ b/internal/credentials/spiffe_test.go @@ -21,12 +21,17 @@ package credentials import ( "crypto/tls" "crypto/x509" + "encoding/pem" "net/url" + "os" "testing" "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/testdata" ) +const wantURI = "spiffe://foo.bar.com/client/workload/1" + type s struct { grpctest.Tester } @@ -40,12 +45,12 @@ func (s) TestSPIFFEIDFromState(t *testing.T) { name string urls []*url.URL // If we expect a SPIFFE ID to be returned. - expectID bool + wantID bool }{ { - name: "empty URIs", - urls: []*url.URL{}, - expectID: false, + name: "empty URIs", + urls: []*url.URL{}, + wantID: false, }, { name: "good SPIFFE ID", @@ -57,7 +62,7 @@ func (s) TestSPIFFEIDFromState(t *testing.T) { RawPath: "workload/wl1", }, }, - expectID: true, + wantID: true, }, { name: "invalid host", @@ -69,7 +74,7 @@ func (s) TestSPIFFEIDFromState(t *testing.T) { RawPath: "workload/wl1", }, }, - expectID: false, + wantID: false, }, { name: "invalid path", @@ -81,7 +86,7 @@ func (s) TestSPIFFEIDFromState(t *testing.T) { RawPath: "", }, }, - expectID: false, + wantID: false, }, { name: "large path", @@ -93,7 +98,7 @@ func (s) TestSPIFFEIDFromState(t *testing.T) { RawPath: string(make([]byte, 2050)), }, }, - expectID: false, + wantID: false, }, { name: "large host", @@ -105,7 +110,7 @@ func (s) TestSPIFFEIDFromState(t *testing.T) { RawPath: "workload/wl1", }, }, - expectID: false, + wantID: false, }, { name: "multiple URI SANs", @@ -129,7 +134,7 @@ func (s) TestSPIFFEIDFromState(t *testing.T) { RawPath: "workload/wl1", }, }, - expectID: false, + wantID: false, }, { name: "multiple URI SANs without SPIFFE ID", @@ -147,7 +152,7 @@ func (s) TestSPIFFEIDFromState(t *testing.T) { RawPath: "workload/wl1", }, }, - expectID: false, + wantID: false, }, { name: "multiple URI SANs with one SPIFFE ID", @@ -165,15 +170,63 @@ func (s) TestSPIFFEIDFromState(t *testing.T) { RawPath: "workload/wl1", }, }, - expectID: false, + wantID: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { state := tls.ConnectionState{PeerCertificates: []*x509.Certificate{{URIs: tt.urls}}} id := SPIFFEIDFromState(state) - if got, want := id != nil, tt.expectID; got != want { - t.Errorf("want expectID = %v, but SPIFFE ID is %v", want, id) + if got, want := id != nil, tt.wantID; got != want { + t.Errorf("want wantID = %v, but SPIFFE ID is %v", want, id) + } + }) + } +} + +func (s) TestSPIFFEIDFromCert(t *testing.T) { + tests := []struct { + name string + dataPath string + // If we expect a SPIFFE ID to be returned. + wantID bool + }{ + { + name: "good certificate with SPIFFE ID", + dataPath: "x509/spiffe_cert.pem", + wantID: true, + }, + { + name: "bad certificate with SPIFFE ID and another URI", + dataPath: "x509/multiple_uri_cert.pem", + wantID: false, + }, + { + name: "certificate without SPIFFE ID", + dataPath: "x509/client1_cert.pem", + wantID: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + data, err := os.ReadFile(testdata.Path(tt.dataPath)) + if err != nil { + t.Fatalf("os.ReadFile(%s) failed: %v", testdata.Path(tt.dataPath), err) + } + block, _ := pem.Decode(data) + if block == nil { + t.Fatalf("Failed to parse the certificate: byte block is nil") + } + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + t.Fatalf("x509.ParseCertificate(%b) failed: %v", block.Bytes, err) + } + uri := SPIFFEIDFromCert(cert) + if (uri != nil) != tt.wantID { + t.Fatalf("wantID got and want mismatch, got %t, want %t", uri != nil, tt.wantID) + } + if uri != nil && uri.String() != wantURI { + t.Fatalf("SPIFFE ID not expected, got %s, want %s", uri.String(), wantURI) } }) } diff --git a/internal/credentials/util.go b/internal/credentials/util.go index 55664fa46b81..f792fd22cafc 100644 --- a/internal/credentials/util.go +++ b/internal/credentials/util.go @@ -18,7 +18,9 @@ package credentials -import "crypto/tls" +import ( + "crypto/tls" +) const alpnProtoStrH2 = "h2" diff --git a/internal/credentials/xds/handshake_info.go b/internal/credentials/xds/handshake_info.go new file mode 100644 index 000000000000..b6f1fa520fc4 --- /dev/null +++ b/internal/credentials/xds/handshake_info.go @@ -0,0 +1,333 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package xds contains non-user facing functionality of the xds credentials. +package xds + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "strings" + "sync" + + "google.golang.org/grpc/attributes" + "google.golang.org/grpc/credentials/tls/certprovider" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/xds/matcher" + "google.golang.org/grpc/resolver" +) + +func init() { + internal.GetXDSHandshakeInfoForTesting = GetHandshakeInfo +} + +// handshakeAttrKey is the type used as the key to store HandshakeInfo in +// the Attributes field of resolver.Address. +type handshakeAttrKey struct{} + +// Equal reports whether the handshake info structs are identical. +func (hi *HandshakeInfo) Equal(other *HandshakeInfo) bool { + if hi == nil && other == nil { + return true + } + if hi == nil || other == nil { + return false + } + if hi.rootProvider != other.rootProvider || + hi.identityProvider != other.identityProvider || + hi.requireClientCert != other.requireClientCert || + len(hi.sanMatchers) != len(other.sanMatchers) { + return false + } + for i := range hi.sanMatchers { + if !hi.sanMatchers[i].Equal(other.sanMatchers[i]) { + return false + } + } + return true +} + +// SetHandshakeInfo returns a copy of addr in which the Attributes field is +// updated with hInfo. +func SetHandshakeInfo(addr resolver.Address, hInfo *HandshakeInfo) resolver.Address { + addr.Attributes = addr.Attributes.WithValue(handshakeAttrKey{}, hInfo) + return addr +} + +// GetHandshakeInfo returns a pointer to the HandshakeInfo stored in attr. +func GetHandshakeInfo(attr *attributes.Attributes) *HandshakeInfo { + v := attr.Value(handshakeAttrKey{}) + hi, _ := v.(*HandshakeInfo) + return hi +} + +// HandshakeInfo wraps all the security configuration required by client and +// server handshake methods in xds credentials. The xDS implementation will be +// responsible for populating these fields. +// +// Safe for concurrent access. +type HandshakeInfo struct { + mu sync.Mutex + rootProvider certprovider.Provider + identityProvider certprovider.Provider + sanMatchers []matcher.StringMatcher // Only on the client side. + requireClientCert bool // Only on server side. +} + +// SetRootCertProvider updates the root certificate provider. +func (hi *HandshakeInfo) SetRootCertProvider(root certprovider.Provider) { + hi.mu.Lock() + hi.rootProvider = root + hi.mu.Unlock() +} + +// SetIdentityCertProvider updates the identity certificate provider. +func (hi *HandshakeInfo) SetIdentityCertProvider(identity certprovider.Provider) { + hi.mu.Lock() + hi.identityProvider = identity + hi.mu.Unlock() +} + +// SetSANMatchers updates the list of SAN matchers. +func (hi *HandshakeInfo) SetSANMatchers(sanMatchers []matcher.StringMatcher) { + hi.mu.Lock() + hi.sanMatchers = sanMatchers + hi.mu.Unlock() +} + +// SetRequireClientCert updates whether a client cert is required during the +// ServerHandshake(). A value of true indicates that we are performing mTLS. +func (hi *HandshakeInfo) SetRequireClientCert(require bool) { + hi.mu.Lock() + hi.requireClientCert = require + hi.mu.Unlock() +} + +// UseFallbackCreds returns true when fallback credentials are to be used based +// on the contents of the HandshakeInfo. +func (hi *HandshakeInfo) UseFallbackCreds() bool { + if hi == nil { + return true + } + + hi.mu.Lock() + defer hi.mu.Unlock() + return hi.identityProvider == nil && hi.rootProvider == nil +} + +// GetSANMatchersForTesting returns the SAN matchers stored in HandshakeInfo. +// To be used only for testing purposes. +func (hi *HandshakeInfo) GetSANMatchersForTesting() []matcher.StringMatcher { + hi.mu.Lock() + defer hi.mu.Unlock() + return append([]matcher.StringMatcher{}, hi.sanMatchers...) +} + +// ClientSideTLSConfig constructs a tls.Config to be used in a client-side +// handshake based on the contents of the HandshakeInfo. +func (hi *HandshakeInfo) ClientSideTLSConfig(ctx context.Context) (*tls.Config, error) { + hi.mu.Lock() + // On the client side, rootProvider is mandatory. IdentityProvider is + // optional based on whether the client is doing TLS or mTLS. + if hi.rootProvider == nil { + return nil, errors.New("xds: CertificateProvider to fetch trusted roots is missing, cannot perform TLS handshake. Please check configuration on the management server") + } + // Since the call to KeyMaterial() can block, we read the providers under + // the lock but call the actual function after releasing the lock. + rootProv, idProv := hi.rootProvider, hi.identityProvider + hi.mu.Unlock() + + // InsecureSkipVerify needs to be set to true because we need to perform + // custom verification to check the SAN on the received certificate. + // Currently the Go stdlib does complete verification of the cert (which + // includes hostname verification) or none. We are forced to go with the + // latter and perform the normal cert validation ourselves. + cfg := &tls.Config{ + InsecureSkipVerify: true, + NextProtos: []string{"h2"}, + } + + km, err := rootProv.KeyMaterial(ctx) + if err != nil { + return nil, fmt.Errorf("xds: fetching trusted roots from CertificateProvider failed: %v", err) + } + cfg.RootCAs = km.Roots + + if idProv != nil { + km, err := idProv.KeyMaterial(ctx) + if err != nil { + return nil, fmt.Errorf("xds: fetching identity certificates from CertificateProvider failed: %v", err) + } + cfg.Certificates = km.Certs + } + return cfg, nil +} + +// ServerSideTLSConfig constructs a tls.Config to be used in a server-side +// handshake based on the contents of the HandshakeInfo. +func (hi *HandshakeInfo) ServerSideTLSConfig(ctx context.Context) (*tls.Config, error) { + cfg := &tls.Config{ + ClientAuth: tls.NoClientCert, + NextProtos: []string{"h2"}, + } + hi.mu.Lock() + // On the server side, identityProvider is mandatory. RootProvider is + // optional based on whether the server is doing TLS or mTLS. + if hi.identityProvider == nil { + return nil, errors.New("xds: CertificateProvider to fetch identity certificate is missing, cannot perform TLS handshake. Please check configuration on the management server") + } + // Since the call to KeyMaterial() can block, we read the providers under + // the lock but call the actual function after releasing the lock. + rootProv, idProv := hi.rootProvider, hi.identityProvider + if hi.requireClientCert { + cfg.ClientAuth = tls.RequireAndVerifyClientCert + } + hi.mu.Unlock() + + // identityProvider is mandatory on the server side. + km, err := idProv.KeyMaterial(ctx) + if err != nil { + return nil, fmt.Errorf("xds: fetching identity certificates from CertificateProvider failed: %v", err) + } + cfg.Certificates = km.Certs + + if rootProv != nil { + km, err := rootProv.KeyMaterial(ctx) + if err != nil { + return nil, fmt.Errorf("xds: fetching trusted roots from CertificateProvider failed: %v", err) + } + cfg.ClientCAs = km.Roots + } + return cfg, nil +} + +// MatchingSANExists returns true if the SANs contained in cert match the +// criteria enforced by the list of SAN matchers in HandshakeInfo. +// +// If the list of SAN matchers in the HandshakeInfo is empty, this function +// returns true for all input certificates. +func (hi *HandshakeInfo) MatchingSANExists(cert *x509.Certificate) bool { + hi.mu.Lock() + defer hi.mu.Unlock() + if len(hi.sanMatchers) == 0 { + return true + } + + // SANs can be specified in any of these four fields on the parsed cert. + for _, san := range cert.DNSNames { + if hi.matchSAN(san, true) { + return true + } + } + for _, san := range cert.EmailAddresses { + if hi.matchSAN(san, false) { + return true + } + } + for _, san := range cert.IPAddresses { + if hi.matchSAN(san.String(), false) { + return true + } + } + for _, san := range cert.URIs { + if hi.matchSAN(san.String(), false) { + return true + } + } + return false +} + +// Caller must hold mu. +func (hi *HandshakeInfo) matchSAN(san string, isDNS bool) bool { + for _, matcher := range hi.sanMatchers { + if em := matcher.ExactMatch(); em != "" && isDNS { + // This is a special case which is documented in the xDS protos. + // If the DNS SAN is a wildcard entry, and the match criteria is + // `exact`, then we need to perform DNS wildcard matching + // instead of regular string comparison. + if dnsMatch(em, san) { + return true + } + continue + } + if matcher.Match(san) { + return true + } + } + return false +} + +// dnsMatch implements a DNS wildcard matching algorithm based on RFC2828 and +// grpc-java's implementation in `OkHostnameVerifier` class. +// +// NOTE: Here the `host` argument is the one from the set of string matchers in +// the xDS proto and the `san` argument is a DNS SAN from the certificate, and +// this is the one which can potentially contain a wildcard pattern. +func dnsMatch(host, san string) bool { + // Add trailing "." and turn them into absolute domain names. + if !strings.HasSuffix(host, ".") { + host += "." + } + if !strings.HasSuffix(san, ".") { + san += "." + } + // Domain names are case-insensitive. + host = strings.ToLower(host) + san = strings.ToLower(san) + + // If san does not contain a wildcard, do exact match. + if !strings.Contains(san, "*") { + return host == san + } + + // Wildcard dns matching rules + // - '*' is only permitted in the left-most label and must be the only + // character in that label. For example, *.example.com is permitted, while + // *a.example.com, a*.example.com, a*b.example.com, a.*.example.com are + // not permitted. + // - '*' matches a single domain name component. For example, *.example.com + // matches test.example.com but does not match sub.test.example.com. + // - Wildcard patterns for single-label domain names are not permitted. + if san == "*." || !strings.HasPrefix(san, "*.") || strings.Contains(san[1:], "*") { + return false + } + // Optimization: at this point, we know that the san contains a '*' and + // is the first domain component of san. So, the host name must be at + // least as long as the san to be able to match. + if len(host) < len(san) { + return false + } + // Hostname must end with the non-wildcard portion of san. + if !strings.HasSuffix(host, san[1:]) { + return false + } + // At this point we know that the hostName and san share the same suffix + // (the non-wildcard portion of san). Now, we just need to make sure + // that the '*' does not match across domain components. + hostPrefix := strings.TrimSuffix(host, san[1:]) + return !strings.Contains(hostPrefix, ".") +} + +// NewHandshakeInfo returns a new instance of HandshakeInfo with the given root +// and identity certificate providers. +func NewHandshakeInfo(root, identity certprovider.Provider) *HandshakeInfo { + return &HandshakeInfo{rootProvider: root, identityProvider: identity} +} diff --git a/internal/credentials/xds/handshake_info_test.go b/internal/credentials/xds/handshake_info_test.go new file mode 100644 index 000000000000..91257a1925da --- /dev/null +++ b/internal/credentials/xds/handshake_info_test.go @@ -0,0 +1,304 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds + +import ( + "crypto/x509" + "net" + "net/url" + "regexp" + "testing" + + "google.golang.org/grpc/internal/xds/matcher" +) + +func TestDNSMatch(t *testing.T) { + tests := []struct { + desc string + host string + pattern string + wantMatch bool + }{ + { + desc: "invalid wildcard 1", + host: "aa.example.com", + pattern: "*a.example.com", + wantMatch: false, + }, + { + desc: "invalid wildcard 2", + host: "aa.example.com", + pattern: "a*.example.com", + wantMatch: false, + }, + { + desc: "invalid wildcard 3", + host: "abc.example.com", + pattern: "a*c.example.com", + wantMatch: false, + }, + { + desc: "wildcard in one of the middle components", + host: "abc.test.example.com", + pattern: "abc.*.example.com", + wantMatch: false, + }, + { + desc: "single component wildcard", + host: "a.example.com", + pattern: "*", + wantMatch: false, + }, + { + desc: "short host name", + host: "a.com", + pattern: "*.example.com", + wantMatch: false, + }, + { + desc: "suffix mismatch", + host: "a.notexample.com", + pattern: "*.example.com", + wantMatch: false, + }, + { + desc: "wildcard match across components", + host: "sub.test.example.com", + pattern: "*.example.com.", + wantMatch: false, + }, + { + desc: "host doesn't end in period", + host: "test.example.com", + pattern: "test.example.com.", + wantMatch: true, + }, + { + desc: "pattern doesn't end in period", + host: "test.example.com.", + pattern: "test.example.com", + wantMatch: true, + }, + { + desc: "case insensitive", + host: "TEST.EXAMPLE.COM.", + pattern: "test.example.com.", + wantMatch: true, + }, + { + desc: "simple match", + host: "test.example.com", + pattern: "test.example.com", + wantMatch: true, + }, + { + desc: "good wildcard", + host: "a.example.com", + pattern: "*.example.com", + wantMatch: true, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + gotMatch := dnsMatch(test.host, test.pattern) + if gotMatch != test.wantMatch { + t.Fatalf("dnsMatch(%s, %s) = %v, want %v", test.host, test.pattern, gotMatch, test.wantMatch) + } + }) + } +} + +func TestMatchingSANExists_FailureCases(t *testing.T) { + url1, err := url.Parse("http://golang.org") + if err != nil { + t.Fatalf("url.Parse() failed: %v", err) + } + url2, err := url.Parse("https://github.com/grpc/grpc-go") + if err != nil { + t.Fatalf("url.Parse() failed: %v", err) + } + inputCert := &x509.Certificate{ + DNSNames: []string{"foo.bar.example.com", "bar.baz.test.com", "*.example.com"}, + EmailAddresses: []string{"foobar@example.com", "barbaz@test.com"}, + IPAddresses: []net.IP{net.ParseIP("192.0.0.1"), net.ParseIP("2001:db8::68")}, + URIs: []*url.URL{url1, url2}, + } + + tests := []struct { + desc string + sanMatchers []matcher.StringMatcher + }{ + { + desc: "exact match", + sanMatchers: []matcher.StringMatcher{ + matcher.StringMatcherForTesting(newStringP("abcd.test.com"), nil, nil, nil, nil, false), + matcher.StringMatcherForTesting(newStringP("http://golang"), nil, nil, nil, nil, false), + matcher.StringMatcherForTesting(newStringP("HTTP://GOLANG.ORG"), nil, nil, nil, nil, false), + }, + }, + { + desc: "prefix match", + sanMatchers: []matcher.StringMatcher{ + matcher.StringMatcherForTesting(nil, newStringP("i-aint-the-one"), nil, nil, nil, false), + matcher.StringMatcherForTesting(nil, newStringP("192.168.1.1"), nil, nil, nil, false), + matcher.StringMatcherForTesting(nil, newStringP("FOO.BAR"), nil, nil, nil, false), + }, + }, + { + desc: "suffix match", + sanMatchers: []matcher.StringMatcher{ + matcher.StringMatcherForTesting(nil, nil, newStringP("i-aint-the-one"), nil, nil, false), + matcher.StringMatcherForTesting(nil, nil, newStringP("1::68"), nil, nil, false), + matcher.StringMatcherForTesting(nil, nil, newStringP(".COM"), nil, nil, false), + }, + }, + { + desc: "regex match", + sanMatchers: []matcher.StringMatcher{ + matcher.StringMatcherForTesting(nil, nil, nil, nil, regexp.MustCompile(`.*\.examples\.com`), false), + matcher.StringMatcherForTesting(nil, nil, nil, nil, regexp.MustCompile(`192\.[0-9]{1,3}\.1\.1`), false), + }, + }, + { + desc: "contains match", + sanMatchers: []matcher.StringMatcher{ + matcher.StringMatcherForTesting(nil, nil, nil, newStringP("i-aint-the-one"), nil, false), + matcher.StringMatcherForTesting(nil, nil, nil, newStringP("2001:db8:1:1::68"), nil, false), + matcher.StringMatcherForTesting(nil, nil, nil, newStringP("GRPC"), nil, false), + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + hi := NewHandshakeInfo(nil, nil) + hi.SetSANMatchers(test.sanMatchers) + + if hi.MatchingSANExists(inputCert) { + t.Fatalf("hi.MatchingSANExists(%+v) with SAN matchers +%v succeeded when expected to fail", inputCert, test.sanMatchers) + } + }) + } +} + +func TestMatchingSANExists_Success(t *testing.T) { + url1, err := url.Parse("http://golang.org") + if err != nil { + t.Fatalf("url.Parse() failed: %v", err) + } + url2, err := url.Parse("https://github.com/grpc/grpc-go") + if err != nil { + t.Fatalf("url.Parse() failed: %v", err) + } + inputCert := &x509.Certificate{ + DNSNames: []string{"baz.test.com", "*.example.com"}, + EmailAddresses: []string{"foobar@example.com", "barbaz@test.com"}, + IPAddresses: []net.IP{net.ParseIP("192.0.0.1"), net.ParseIP("2001:db8::68")}, + URIs: []*url.URL{url1, url2}, + } + + tests := []struct { + desc string + sanMatchers []matcher.StringMatcher + }{ + { + desc: "no san matchers", + }, + { + desc: "exact match dns wildcard", + sanMatchers: []matcher.StringMatcher{ + matcher.StringMatcherForTesting(nil, newStringP("192.168.1.1"), nil, nil, nil, false), + matcher.StringMatcherForTesting(newStringP("https://github.com/grpc/grpc-java"), nil, nil, nil, nil, false), + matcher.StringMatcherForTesting(newStringP("abc.example.com"), nil, nil, nil, nil, false), + }, + }, + { + desc: "exact match ignore case", + sanMatchers: []matcher.StringMatcher{ + matcher.StringMatcherForTesting(newStringP("FOOBAR@EXAMPLE.COM"), nil, nil, nil, nil, true), + }, + }, + { + desc: "prefix match", + sanMatchers: []matcher.StringMatcher{ + matcher.StringMatcherForTesting(nil, nil, newStringP(".co.in"), nil, nil, false), + matcher.StringMatcherForTesting(nil, newStringP("192.168.1.1"), nil, nil, nil, false), + matcher.StringMatcherForTesting(nil, newStringP("baz.test"), nil, nil, nil, false), + }, + }, + { + desc: "prefix match ignore case", + sanMatchers: []matcher.StringMatcher{ + matcher.StringMatcherForTesting(nil, newStringP("BAZ.test"), nil, nil, nil, true), + }, + }, + { + desc: "suffix match", + sanMatchers: []matcher.StringMatcher{ + matcher.StringMatcherForTesting(nil, nil, nil, nil, regexp.MustCompile(`192\.[0-9]{1,3}\.1\.1`), false), + matcher.StringMatcherForTesting(nil, nil, newStringP("192.168.1.1"), nil, nil, false), + matcher.StringMatcherForTesting(nil, nil, newStringP("@test.com"), nil, nil, false), + }, + }, + { + desc: "suffix match ignore case", + sanMatchers: []matcher.StringMatcher{ + matcher.StringMatcherForTesting(nil, nil, newStringP("@test.COM"), nil, nil, true), + }, + }, + { + desc: "regex match", + sanMatchers: []matcher.StringMatcher{ + matcher.StringMatcherForTesting(nil, nil, nil, newStringP("https://github.com/grpc/grpc-java"), nil, false), + matcher.StringMatcherForTesting(nil, nil, nil, nil, regexp.MustCompile(`192\.[0-9]{1,3}\.1\.1`), false), + matcher.StringMatcherForTesting(nil, nil, nil, nil, regexp.MustCompile(`.*\.test\.com`), false), + }, + }, + { + desc: "contains match", + sanMatchers: []matcher.StringMatcher{ + matcher.StringMatcherForTesting(newStringP("https://github.com/grpc/grpc-java"), nil, nil, nil, nil, false), + matcher.StringMatcherForTesting(nil, nil, nil, newStringP("2001:68::db8"), nil, false), + matcher.StringMatcherForTesting(nil, nil, nil, newStringP("192.0.0"), nil, false), + }, + }, + { + desc: "contains match ignore case", + sanMatchers: []matcher.StringMatcher{ + matcher.StringMatcherForTesting(nil, nil, nil, newStringP("GRPC"), nil, true), + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + hi := NewHandshakeInfo(nil, nil) + hi.SetSANMatchers(test.sanMatchers) + + if !hi.MatchingSANExists(inputCert) { + t.Fatalf("hi.MatchingSANExists(%+v) with SAN matchers +%v failed when expected to succeed", inputCert, test.sanMatchers) + } + }) + } +} + +func newStringP(s string) *string { + return &s +} diff --git a/internal/envconfig/envconfig.go b/internal/envconfig/envconfig.go index 73931a94bcad..3cf10ddfbd4c 100644 --- a/internal/envconfig/envconfig.go +++ b/internal/envconfig/envconfig.go @@ -21,18 +21,52 @@ package envconfig import ( "os" + "strconv" "strings" ) -const ( - prefix = "GRPC_GO_" - retryStr = prefix + "RETRY" - txtErrIgnoreStr = prefix + "IGNORE_TXT_ERRORS" -) - var ( - // Retry is set if retry is explicitly enabled via "GRPC_GO_RETRY=on". - Retry = strings.EqualFold(os.Getenv(retryStr), "on") // TXTErrIgnore is set if TXT errors should be ignored ("GRPC_GO_IGNORE_TXT_ERRORS" is not "false"). - TXTErrIgnore = !strings.EqualFold(os.Getenv(txtErrIgnoreStr), "false") + TXTErrIgnore = boolFromEnv("GRPC_GO_IGNORE_TXT_ERRORS", true) + // AdvertiseCompressors is set if registered compressor should be advertised + // ("GRPC_GO_ADVERTISE_COMPRESSORS" is not "false"). + AdvertiseCompressors = boolFromEnv("GRPC_GO_ADVERTISE_COMPRESSORS", true) + // RingHashCap indicates the maximum ring size which defaults to 4096 + // entries but may be overridden by setting the environment variable + // "GRPC_RING_HASH_CAP". This does not override the default bounds + // checking which NACKs configs specifying ring sizes > 8*1024*1024 (~8M). + RingHashCap = uint64FromEnv("GRPC_RING_HASH_CAP", 4096, 1, 8*1024*1024) + // PickFirstLBConfig is set if we should support configuration of the + // pick_first LB policy. + PickFirstLBConfig = boolFromEnv("GRPC_EXPERIMENTAL_PICKFIRST_LB_CONFIG", true) + // LeastRequestLB is set if we should support the least_request_experimental + // LB policy, which can be enabled by setting the environment variable + // "GRPC_EXPERIMENTAL_ENABLE_LEAST_REQUEST" to "true". + LeastRequestLB = boolFromEnv("GRPC_EXPERIMENTAL_ENABLE_LEAST_REQUEST", false) + // ALTSMaxConcurrentHandshakes is the maximum number of concurrent ALTS + // handshakes that can be performed. + ALTSMaxConcurrentHandshakes = uint64FromEnv("GRPC_ALTS_MAX_CONCURRENT_HANDSHAKES", 100, 1, 100) ) + +func boolFromEnv(envVar string, def bool) bool { + if def { + // The default is true; return true unless the variable is "false". + return !strings.EqualFold(os.Getenv(envVar), "false") + } + // The default is false; return false unless the variable is "true". + return strings.EqualFold(os.Getenv(envVar), "true") +} + +func uint64FromEnv(envVar string, def, min, max uint64) uint64 { + v, err := strconv.ParseUint(os.Getenv(envVar), 10, 64) + if err != nil { + return def + } + if v < min { + return min + } + if v > max { + return max + } + return v +} diff --git a/internal/envconfig/envconfig_test.go b/internal/envconfig/envconfig_test.go new file mode 100644 index 000000000000..68fdf6c73a7f --- /dev/null +++ b/internal/envconfig/envconfig_test.go @@ -0,0 +1,103 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package envconfig + +import ( + "os" + "testing" + + "google.golang.org/grpc/internal/grpctest" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func (s) TestUint64FromEnv(t *testing.T) { + var testCases = []struct { + name string + val string + def, min, max uint64 + want uint64 + }{ + { + name: "error parsing", + val: "asdf", def: 5, want: 5, + }, { + name: "unset", + val: "", def: 5, want: 5, + }, { + name: "too low", + val: "5", min: 10, want: 10, + }, { + name: "too high", + val: "5", max: 2, want: 2, + }, { + name: "in range", + val: "17391", def: 13000, min: 12000, max: 18000, want: 17391, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + const testVar = "testvar" + if tc.val == "" { + os.Unsetenv(testVar) + } else { + os.Setenv(testVar, tc.val) + } + if got := uint64FromEnv(testVar, tc.def, tc.min, tc.max); got != tc.want { + t.Errorf("uint64FromEnv(%q(=%q), %v, %v, %v) = %v; want %v", testVar, tc.val, tc.def, tc.min, tc.max, got, tc.want) + } + }) + } +} + +func (s) TestBoolFromEnv(t *testing.T) { + var testCases = []struct { + val string + def bool + want bool + }{ + {val: "", def: true, want: true}, + {val: "", def: false, want: false}, + {val: "true", def: true, want: true}, + {val: "true", def: false, want: true}, + {val: "false", def: true, want: false}, + {val: "false", def: false, want: false}, + {val: "asdf", def: true, want: true}, + {val: "asdf", def: false, want: false}, + } + for _, tc := range testCases { + t.Run("", func(t *testing.T) { + const testVar = "testvar" + if tc.val == "" { + os.Unsetenv(testVar) + } else { + os.Setenv(testVar, tc.val) + } + if got := boolFromEnv(testVar, tc.def); got != tc.want { + t.Errorf("boolFromEnv(%q(=%q), %v) = %v; want %v", testVar, tc.val, tc.def, got, tc.want) + } + }) + } +} diff --git a/internal/envconfig/observability.go b/internal/envconfig/observability.go new file mode 100644 index 000000000000..dd314cfb18f4 --- /dev/null +++ b/internal/envconfig/observability.go @@ -0,0 +1,42 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package envconfig + +import "os" + +const ( + envObservabilityConfig = "GRPC_GCP_OBSERVABILITY_CONFIG" + envObservabilityConfigFile = "GRPC_GCP_OBSERVABILITY_CONFIG_FILE" +) + +var ( + // ObservabilityConfig is the json configuration for the gcp/observability + // package specified directly in the envObservabilityConfig env var. + // + // This is used in the 1.0 release of gcp/observability, and thus must not be + // deleted or changed. + ObservabilityConfig = os.Getenv(envObservabilityConfig) + // ObservabilityConfigFile is the json configuration for the + // gcp/observability specified in a file with the location specified in + // envObservabilityConfigFile env var. + // + // This is used in the 1.0 release of gcp/observability, and thus must not be + // deleted or changed. + ObservabilityConfigFile = os.Getenv(envObservabilityConfigFile) +) diff --git a/internal/envconfig/xds.go b/internal/envconfig/xds.go new file mode 100644 index 000000000000..02b4b6a1c109 --- /dev/null +++ b/internal/envconfig/xds.go @@ -0,0 +1,95 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package envconfig + +import ( + "os" +) + +const ( + // XDSBootstrapFileNameEnv is the env variable to set bootstrap file name. + // Do not use this and read from env directly. Its value is read and kept in + // variable XDSBootstrapFileName. + // + // When both bootstrap FileName and FileContent are set, FileName is used. + XDSBootstrapFileNameEnv = "GRPC_XDS_BOOTSTRAP" + // XDSBootstrapFileContentEnv is the env variable to set bootstrap file + // content. Do not use this and read from env directly. Its value is read + // and kept in variable XDSBootstrapFileContent. + // + // When both bootstrap FileName and FileContent are set, FileName is used. + XDSBootstrapFileContentEnv = "GRPC_XDS_BOOTSTRAP_CONFIG" +) + +var ( + // XDSBootstrapFileName holds the name of the file which contains xDS + // bootstrap configuration. Users can specify the location of the bootstrap + // file by setting the environment variable "GRPC_XDS_BOOTSTRAP". + // + // When both bootstrap FileName and FileContent are set, FileName is used. + XDSBootstrapFileName = os.Getenv(XDSBootstrapFileNameEnv) + // XDSBootstrapFileContent holds the content of the xDS bootstrap + // configuration. Users can specify the bootstrap config by setting the + // environment variable "GRPC_XDS_BOOTSTRAP_CONFIG". + // + // When both bootstrap FileName and FileContent are set, FileName is used. + XDSBootstrapFileContent = os.Getenv(XDSBootstrapFileContentEnv) + // XDSRingHash indicates whether ring hash support is enabled, which can be + // disabled by setting the environment variable + // "GRPC_XDS_EXPERIMENTAL_ENABLE_RING_HASH" to "false". + XDSRingHash = boolFromEnv("GRPC_XDS_EXPERIMENTAL_ENABLE_RING_HASH", true) + // XDSClientSideSecurity is used to control processing of security + // configuration on the client-side. + // + // Note that there is no env var protection for the server-side because we + // have a brand new API on the server-side and users explicitly need to use + // the new API to get security integration on the server. + XDSClientSideSecurity = boolFromEnv("GRPC_XDS_EXPERIMENTAL_SECURITY_SUPPORT", true) + // XDSAggregateAndDNS indicates whether processing of aggregated cluster and + // DNS cluster is enabled, which can be disabled by setting the environment + // variable "GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER" + // to "false". + XDSAggregateAndDNS = boolFromEnv("GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER", true) + + // XDSRBAC indicates whether xDS configured RBAC HTTP Filter is enabled, + // which can be disabled by setting the environment variable + // "GRPC_XDS_EXPERIMENTAL_RBAC" to "false". + XDSRBAC = boolFromEnv("GRPC_XDS_EXPERIMENTAL_RBAC", true) + // XDSOutlierDetection indicates whether outlier detection support is + // enabled, which can be disabled by setting the environment variable + // "GRPC_EXPERIMENTAL_ENABLE_OUTLIER_DETECTION" to "false". + XDSOutlierDetection = boolFromEnv("GRPC_EXPERIMENTAL_ENABLE_OUTLIER_DETECTION", true) + // XDSFederation indicates whether federation support is enabled, which can + // be enabled by setting the environment variable + // "GRPC_EXPERIMENTAL_XDS_FEDERATION" to "true". + XDSFederation = boolFromEnv("GRPC_EXPERIMENTAL_XDS_FEDERATION", true) + + // XDSRLS indicates whether processing of Cluster Specifier plugins and + // support for the RLS CLuster Specifier is enabled, which can be disabled by + // setting the environment variable "GRPC_EXPERIMENTAL_XDS_RLS_LB" to + // "false". + XDSRLS = boolFromEnv("GRPC_EXPERIMENTAL_XDS_RLS_LB", true) + + // C2PResolverTestOnlyTrafficDirectorURI is the TD URI for testing. + C2PResolverTestOnlyTrafficDirectorURI = os.Getenv("GRPC_TEST_ONLY_GOOGLE_C2P_RESOLVER_TRAFFIC_DIRECTOR_URI") + // XDSCustomLBPolicy indicates whether Custom LB Policies are enabled, which + // can be disabled by setting the environment variable + // "GRPC_EXPERIMENTAL_XDS_CUSTOM_LB_CONFIG" to "false". + XDSCustomLBPolicy = boolFromEnv("GRPC_EXPERIMENTAL_XDS_CUSTOM_LB_CONFIG", true) +) diff --git a/internal/googlecloud/googlecloud.go b/internal/googlecloud/googlecloud.go new file mode 100644 index 000000000000..6717b757f80d --- /dev/null +++ b/internal/googlecloud/googlecloud.go @@ -0,0 +1,72 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package googlecloud contains internal helpful functions for google cloud. +package googlecloud + +import ( + "runtime" + "strings" + "sync" + + "google.golang.org/grpc/grpclog" + internalgrpclog "google.golang.org/grpc/internal/grpclog" +) + +const logPrefix = "[googlecloud]" + +var ( + vmOnGCEOnce sync.Once + vmOnGCE bool + + logger = internalgrpclog.NewPrefixLogger(grpclog.Component("googlecloud"), logPrefix) +) + +// OnGCE returns whether the client is running on GCE. +// +// It provides similar functionality as metadata.OnGCE from the cloud library +// package. We keep this to avoid depending on the cloud library module. +func OnGCE() bool { + vmOnGCEOnce.Do(func() { + mf, err := manufacturer() + if err != nil { + logger.Infof("failed to read manufacturer, setting onGCE=false: %v") + return + } + vmOnGCE = isRunningOnGCE(mf, runtime.GOOS) + }) + return vmOnGCE +} + +// isRunningOnGCE checks whether the local system, without doing a network request, is +// running on GCP. +func isRunningOnGCE(manufacturer []byte, goos string) bool { + name := string(manufacturer) + switch goos { + case "linux": + name = strings.TrimSpace(name) + return name == "Google" || name == "Google Compute Engine" + case "windows": + name = strings.Replace(name, " ", "", -1) + name = strings.Replace(name, "\n", "", -1) + name = strings.Replace(name, "\r", "", -1) + return name == "Google" + default: + return false + } +} diff --git a/internal/googlecloud/googlecloud_test.go b/internal/googlecloud/googlecloud_test.go new file mode 100644 index 000000000000..69ab2fd4c5f2 --- /dev/null +++ b/internal/googlecloud/googlecloud_test.go @@ -0,0 +1,46 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package googlecloud + +import ( + "testing" +) + +func TestIsRunningOnGCE(t *testing.T) { + for _, tc := range []struct { + description string + testOS string + testManufacturer string + out bool + }{ + // Linux tests. + {"linux: not a GCP platform", "linux", "not GCP", false}, + {"Linux: GCP platform (Google)", "linux", "Google", true}, + {"Linux: GCP platform (Google Compute Engine)", "linux", "Google Compute Engine", true}, + {"Linux: GCP platform (Google Compute Engine) with extra spaces", "linux", " Google Compute Engine ", true}, + // Windows tests. + {"windows: not a GCP platform", "windows", "not GCP", false}, + {"windows: GCP platform (Google)", "windows", "Google", true}, + {"windows: GCP platform (Google) with extra spaces", "windows", " Google ", true}, + } { + if got, want := isRunningOnGCE([]byte(tc.testManufacturer), tc.testOS), tc.out; got != want { + t.Errorf("%v: isRunningOnGCE()=%v, want %v", tc.description, got, want) + } + } +} diff --git a/internal/googlecloud/manufacturer.go b/internal/googlecloud/manufacturer.go new file mode 100644 index 000000000000..ffa0f1ddee5d --- /dev/null +++ b/internal/googlecloud/manufacturer.go @@ -0,0 +1,26 @@ +//go:build !(linux || windows) +// +build !linux,!windows + +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package googlecloud + +func manufacturer() ([]byte, error) { + return nil, nil +} diff --git a/internal/googlecloud/manufacturer_linux.go b/internal/googlecloud/manufacturer_linux.go new file mode 100644 index 000000000000..6e455fb0a822 --- /dev/null +++ b/internal/googlecloud/manufacturer_linux.go @@ -0,0 +1,27 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package googlecloud + +import "os" + +const linuxProductNameFile = "/sys/class/dmi/id/product_name" + +func manufacturer() ([]byte, error) { + return os.ReadFile(linuxProductNameFile) +} diff --git a/internal/googlecloud/manufacturer_windows.go b/internal/googlecloud/manufacturer_windows.go new file mode 100644 index 000000000000..2d7aaaaa70fe --- /dev/null +++ b/internal/googlecloud/manufacturer_windows.go @@ -0,0 +1,50 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package googlecloud + +import ( + "errors" + "os/exec" + "regexp" + "strings" +) + +const ( + windowsCheckCommand = "powershell.exe" + windowsCheckCommandArgs = "Get-WmiObject -Class Win32_BIOS" + powershellOutputFilter = "Manufacturer" + windowsManufacturerRegex = ":(.*)" +) + +func manufacturer() ([]byte, error) { + cmd := exec.Command(windowsCheckCommand, windowsCheckCommandArgs) + out, err := cmd.Output() + if err != nil { + return nil, err + } + for _, line := range strings.Split(strings.TrimSuffix(string(out), "\n"), "\n") { + if strings.HasPrefix(line, powershellOutputFilter) { + re := regexp.MustCompile(windowsManufacturerRegex) + name := re.FindString(line) + name = strings.TrimLeft(name, ":") + return []byte(name), nil + } + } + return nil, errors.New("cannot determine the machine's manufacturer") +} diff --git a/internal/grpclog/grpclog.go b/internal/grpclog/grpclog.go index 745a166f02cf..bfc45102ab24 100644 --- a/internal/grpclog/grpclog.go +++ b/internal/grpclog/grpclog.go @@ -30,7 +30,7 @@ var Logger LoggerV2 var DepthLogger DepthLoggerV2 // InfoDepth logs to the INFO log at the specified depth. -func InfoDepth(depth int, args ...interface{}) { +func InfoDepth(depth int, args ...any) { if DepthLogger != nil { DepthLogger.InfoDepth(depth, args...) } else { @@ -39,7 +39,7 @@ func InfoDepth(depth int, args ...interface{}) { } // WarningDepth logs to the WARNING log at the specified depth. -func WarningDepth(depth int, args ...interface{}) { +func WarningDepth(depth int, args ...any) { if DepthLogger != nil { DepthLogger.WarningDepth(depth, args...) } else { @@ -48,7 +48,7 @@ func WarningDepth(depth int, args ...interface{}) { } // ErrorDepth logs to the ERROR log at the specified depth. -func ErrorDepth(depth int, args ...interface{}) { +func ErrorDepth(depth int, args ...any) { if DepthLogger != nil { DepthLogger.ErrorDepth(depth, args...) } else { @@ -57,7 +57,7 @@ func ErrorDepth(depth int, args ...interface{}) { } // FatalDepth logs to the FATAL log at the specified depth. -func FatalDepth(depth int, args ...interface{}) { +func FatalDepth(depth int, args ...any) { if DepthLogger != nil { DepthLogger.FatalDepth(depth, args...) } else { @@ -71,35 +71,35 @@ func FatalDepth(depth int, args ...interface{}) { // is defined here to avoid a circular dependency. type LoggerV2 interface { // Info logs to INFO log. Arguments are handled in the manner of fmt.Print. - Info(args ...interface{}) + Info(args ...any) // Infoln logs to INFO log. Arguments are handled in the manner of fmt.Println. - Infoln(args ...interface{}) + Infoln(args ...any) // Infof logs to INFO log. Arguments are handled in the manner of fmt.Printf. - Infof(format string, args ...interface{}) + Infof(format string, args ...any) // Warning logs to WARNING log. Arguments are handled in the manner of fmt.Print. - Warning(args ...interface{}) + Warning(args ...any) // Warningln logs to WARNING log. Arguments are handled in the manner of fmt.Println. - Warningln(args ...interface{}) + Warningln(args ...any) // Warningf logs to WARNING log. Arguments are handled in the manner of fmt.Printf. - Warningf(format string, args ...interface{}) + Warningf(format string, args ...any) // Error logs to ERROR log. Arguments are handled in the manner of fmt.Print. - Error(args ...interface{}) + Error(args ...any) // Errorln logs to ERROR log. Arguments are handled in the manner of fmt.Println. - Errorln(args ...interface{}) + Errorln(args ...any) // Errorf logs to ERROR log. Arguments are handled in the manner of fmt.Printf. - Errorf(format string, args ...interface{}) + Errorf(format string, args ...any) // Fatal logs to ERROR log. Arguments are handled in the manner of fmt.Print. // gRPC ensures that all Fatal logs will exit with os.Exit(1). // Implementations may also call os.Exit() with a non-zero exit code. - Fatal(args ...interface{}) + Fatal(args ...any) // Fatalln logs to ERROR log. Arguments are handled in the manner of fmt.Println. // gRPC ensures that all Fatal logs will exit with os.Exit(1). // Implementations may also call os.Exit() with a non-zero exit code. - Fatalln(args ...interface{}) + Fatalln(args ...any) // Fatalf logs to ERROR log. Arguments are handled in the manner of fmt.Printf. // gRPC ensures that all Fatal logs will exit with os.Exit(1). // Implementations may also call os.Exit() with a non-zero exit code. - Fatalf(format string, args ...interface{}) + Fatalf(format string, args ...any) // V reports whether verbosity level l is at least the requested verbose level. V(l int) bool } @@ -110,14 +110,17 @@ type LoggerV2 interface { // This is a copy of the DepthLoggerV2 defined in the external grpclog package. // It is defined here to avoid a circular dependency. // -// This API is EXPERIMENTAL. +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. type DepthLoggerV2 interface { - // InfoDepth logs to INFO log at the specified depth. Arguments are handled in the manner of fmt.Print. - InfoDepth(depth int, args ...interface{}) - // WarningDepth logs to WARNING log at the specified depth. Arguments are handled in the manner of fmt.Print. - WarningDepth(depth int, args ...interface{}) - // ErrorDetph logs to ERROR log at the specified depth. Arguments are handled in the manner of fmt.Print. - ErrorDepth(depth int, args ...interface{}) - // FatalDepth logs to FATAL log at the specified depth. Arguments are handled in the manner of fmt.Print. - FatalDepth(depth int, args ...interface{}) + // InfoDepth logs to INFO log at the specified depth. Arguments are handled in the manner of fmt.Println. + InfoDepth(depth int, args ...any) + // WarningDepth logs to WARNING log at the specified depth. Arguments are handled in the manner of fmt.Println. + WarningDepth(depth int, args ...any) + // ErrorDepth logs to ERROR log at the specified depth. Arguments are handled in the manner of fmt.Println. + ErrorDepth(depth int, args ...any) + // FatalDepth logs to FATAL log at the specified depth. Arguments are handled in the manner of fmt.Println. + FatalDepth(depth int, args ...any) } diff --git a/internal/grpclog/prefixLogger.go b/internal/grpclog/prefixLogger.go index 82af70e96f15..faa998de7632 100644 --- a/internal/grpclog/prefixLogger.go +++ b/internal/grpclog/prefixLogger.go @@ -31,7 +31,7 @@ type PrefixLogger struct { } // Infof does info logging. -func (pl *PrefixLogger) Infof(format string, args ...interface{}) { +func (pl *PrefixLogger) Infof(format string, args ...any) { if pl != nil { // Handle nil, so the tests can pass in a nil logger. format = pl.prefix + format @@ -42,7 +42,7 @@ func (pl *PrefixLogger) Infof(format string, args ...interface{}) { } // Warningf does warning logging. -func (pl *PrefixLogger) Warningf(format string, args ...interface{}) { +func (pl *PrefixLogger) Warningf(format string, args ...any) { if pl != nil { format = pl.prefix + format pl.logger.WarningDepth(1, fmt.Sprintf(format, args...)) @@ -52,7 +52,7 @@ func (pl *PrefixLogger) Warningf(format string, args ...interface{}) { } // Errorf does error logging. -func (pl *PrefixLogger) Errorf(format string, args ...interface{}) { +func (pl *PrefixLogger) Errorf(format string, args ...any) { if pl != nil { format = pl.prefix + format pl.logger.ErrorDepth(1, fmt.Sprintf(format, args...)) @@ -62,7 +62,10 @@ func (pl *PrefixLogger) Errorf(format string, args ...interface{}) { } // Debugf does info logging at verbose level 2. -func (pl *PrefixLogger) Debugf(format string, args ...interface{}) { +func (pl *PrefixLogger) Debugf(format string, args ...any) { + // TODO(6044): Refactor interfaces LoggerV2 and DepthLogger, and maybe + // rewrite PrefixLogger a little to ensure that we don't use the global + // `Logger` here, and instead use the `logger` field. if !Logger.V(2) { return } @@ -73,6 +76,15 @@ func (pl *PrefixLogger) Debugf(format string, args ...interface{}) { return } InfoDepth(1, fmt.Sprintf(format, args...)) + +} + +// V reports whether verbosity level l is at least the requested verbose level. +func (pl *PrefixLogger) V(l int) bool { + // TODO(6044): Refactor interfaces LoggerV2 and DepthLogger, and maybe + // rewrite PrefixLogger a little to ensure that we don't use the global + // `Logger` here, and instead use the `logger` field. + return Logger.V(l) } // NewPrefixLogger creates a prefix logger with the given prefix. diff --git a/internal/grpcrand/grpcrand.go b/internal/grpcrand/grpcrand.go index 200b115ca209..aa97273e7d13 100644 --- a/internal/grpcrand/grpcrand.go +++ b/internal/grpcrand/grpcrand.go @@ -31,26 +31,65 @@ var ( mu sync.Mutex ) +// Int implements rand.Int on the grpcrand global source. +func Int() int { + mu.Lock() + defer mu.Unlock() + return r.Int() +} + // Int63n implements rand.Int63n on the grpcrand global source. func Int63n(n int64) int64 { mu.Lock() - res := r.Int63n(n) - mu.Unlock() - return res + defer mu.Unlock() + return r.Int63n(n) } // Intn implements rand.Intn on the grpcrand global source. func Intn(n int) int { mu.Lock() - res := r.Intn(n) - mu.Unlock() - return res + defer mu.Unlock() + return r.Intn(n) +} + +// Int31n implements rand.Int31n on the grpcrand global source. +func Int31n(n int32) int32 { + mu.Lock() + defer mu.Unlock() + return r.Int31n(n) } // Float64 implements rand.Float64 on the grpcrand global source. func Float64() float64 { mu.Lock() - res := r.Float64() - mu.Unlock() - return res + defer mu.Unlock() + return r.Float64() +} + +// Uint64 implements rand.Uint64 on the grpcrand global source. +func Uint64() uint64 { + mu.Lock() + defer mu.Unlock() + return r.Uint64() +} + +// Uint32 implements rand.Uint32 on the grpcrand global source. +func Uint32() uint32 { + mu.Lock() + defer mu.Unlock() + return r.Uint32() +} + +// ExpFloat64 implements rand.ExpFloat64 on the grpcrand global source. +func ExpFloat64() float64 { + mu.Lock() + defer mu.Unlock() + return r.ExpFloat64() +} + +// Shuffle implements rand.Shuffle on the grpcrand global source. +var Shuffle = func(n int, f func(int, int)) { + mu.Lock() + defer mu.Unlock() + r.Shuffle(n, f) } diff --git a/internal/grpcsync/callback_serializer.go b/internal/grpcsync/callback_serializer.go new file mode 100644 index 000000000000..900917dbe6c1 --- /dev/null +++ b/internal/grpcsync/callback_serializer.go @@ -0,0 +1,125 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpcsync + +import ( + "context" + "sync" + + "google.golang.org/grpc/internal/buffer" +) + +// CallbackSerializer provides a mechanism to schedule callbacks in a +// synchronized manner. It provides a FIFO guarantee on the order of execution +// of scheduled callbacks. New callbacks can be scheduled by invoking the +// Schedule() method. +// +// This type is safe for concurrent access. +type CallbackSerializer struct { + // done is closed once the serializer is shut down completely, i.e all + // scheduled callbacks are executed and the serializer has deallocated all + // its resources. + done chan struct{} + + callbacks *buffer.Unbounded + closedMu sync.Mutex + closed bool +} + +// NewCallbackSerializer returns a new CallbackSerializer instance. The provided +// context will be passed to the scheduled callbacks. Users should cancel the +// provided context to shutdown the CallbackSerializer. It is guaranteed that no +// callbacks will be added once this context is canceled, and any pending un-run +// callbacks will be executed before the serializer is shut down. +func NewCallbackSerializer(ctx context.Context) *CallbackSerializer { + cs := &CallbackSerializer{ + done: make(chan struct{}), + callbacks: buffer.NewUnbounded(), + } + go cs.run(ctx) + return cs +} + +// Schedule adds a callback to be scheduled after existing callbacks are run. +// +// Callbacks are expected to honor the context when performing any blocking +// operations, and should return early when the context is canceled. +// +// Return value indicates if the callback was successfully added to the list of +// callbacks to be executed by the serializer. It is not possible to add +// callbacks once the context passed to NewCallbackSerializer is cancelled. +func (cs *CallbackSerializer) Schedule(f func(ctx context.Context)) bool { + cs.closedMu.Lock() + defer cs.closedMu.Unlock() + + if cs.closed { + return false + } + cs.callbacks.Put(f) + return true +} + +func (cs *CallbackSerializer) run(ctx context.Context) { + var backlog []func(context.Context) + + defer close(cs.done) + for ctx.Err() == nil { + select { + case <-ctx.Done(): + // Do nothing here. Next iteration of the for loop will not happen, + // since ctx.Err() would be non-nil. + case callback, ok := <-cs.callbacks.Get(): + if !ok { + return + } + cs.callbacks.Load() + callback.(func(ctx context.Context))(ctx) + } + } + + // Fetch pending callbacks if any, and execute them before returning from + // this method and closing cs.done. + cs.closedMu.Lock() + cs.closed = true + backlog = cs.fetchPendingCallbacks() + cs.callbacks.Close() + cs.closedMu.Unlock() + for _, b := range backlog { + b(ctx) + } +} + +func (cs *CallbackSerializer) fetchPendingCallbacks() []func(context.Context) { + var backlog []func(context.Context) + for { + select { + case b := <-cs.callbacks.Get(): + backlog = append(backlog, b.(func(context.Context))) + cs.callbacks.Load() + default: + return backlog + } + } +} + +// Done returns a channel that is closed after the context passed to +// NewCallbackSerializer is canceled and all callbacks have been executed. +func (cs *CallbackSerializer) Done() <-chan struct{} { + return cs.done +} diff --git a/internal/grpcsync/callback_serializer_test.go b/internal/grpcsync/callback_serializer_test.go new file mode 100644 index 000000000000..b7c3f5fcec4a --- /dev/null +++ b/internal/grpcsync/callback_serializer_test.go @@ -0,0 +1,206 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpcsync + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/google/go-cmp/cmp" +) + +const ( + defaultTestTimeout = 5 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond // For events expected to *not* happen. +) + +// TestCallbackSerializer_Schedule_FIFO verifies that callbacks are executed in +// the same order in which they were scheduled. +func (s) TestCallbackSerializer_Schedule_FIFO(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + cs := NewCallbackSerializer(ctx) + defer cancel() + + // We have two channels, one to record the order of scheduling, and the + // other to record the order of execution. We spawn a bunch of goroutines + // which record the order of scheduling and call the actual Schedule() + // method as well. The callbacks record the order of execution. + // + // We need to grab a lock to record order of scheduling to guarantee that + // the act of recording and the act of calling Schedule() happen atomically. + const numCallbacks = 100 + var mu sync.Mutex + scheduleOrderCh := make(chan int, numCallbacks) + executionOrderCh := make(chan int, numCallbacks) + for i := 0; i < numCallbacks; i++ { + go func(id int) { + mu.Lock() + defer mu.Unlock() + scheduleOrderCh <- id + cs.Schedule(func(ctx context.Context) { + select { + case <-ctx.Done(): + return + case executionOrderCh <- id: + } + }) + }(i) + } + + // Spawn a couple of goroutines to capture the order or scheduling and the + // order of execution. + scheduleOrder := make([]int, numCallbacks) + executionOrder := make([]int, numCallbacks) + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + for i := 0; i < numCallbacks; i++ { + select { + case <-ctx.Done(): + return + case id := <-scheduleOrderCh: + scheduleOrder[i] = id + } + } + }() + go func() { + defer wg.Done() + for i := 0; i < numCallbacks; i++ { + select { + case <-ctx.Done(): + return + case id := <-executionOrderCh: + executionOrder[i] = id + } + } + }() + wg.Wait() + + if diff := cmp.Diff(executionOrder, scheduleOrder); diff != "" { + t.Fatalf("Callbacks are not executed in scheduled order. diff(-want, +got):\n%s", diff) + } +} + +// TestCallbackSerializer_Schedule_Concurrent verifies that all concurrently +// scheduled callbacks get executed. +func (s) TestCallbackSerializer_Schedule_Concurrent(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + cs := NewCallbackSerializer(ctx) + defer cancel() + + // Schedule callbacks concurrently by calling Schedule() from goroutines. + // The execution of the callbacks call Done() on the waitgroup, which + // eventually unblocks the test and allows it to complete. + const numCallbacks = 100 + var wg sync.WaitGroup + wg.Add(numCallbacks) + for i := 0; i < numCallbacks; i++ { + go func() { + cs.Schedule(func(context.Context) { + wg.Done() + }) + }() + } + + // We call Wait() on the waitgroup from a goroutine so that we can select on + // the Wait() being unblocked and the overall test deadline expiring. + done := make(chan struct{}) + go func() { + wg.Wait() + close(done) + }() + + select { + case <-ctx.Done(): + t.Fatal("Timeout waiting for all scheduled callbacks to be executed") + case <-done: + } +} + +// TestCallbackSerializer_Schedule_Close verifies that callbacks in the queue +// are not executed once Close() returns. +func (s) TestCallbackSerializer_Schedule_Close(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + serializerCtx, serializerCancel := context.WithTimeout(context.Background(), defaultTestTimeout) + cs := NewCallbackSerializer(serializerCtx) + + // Schedule a callback which blocks until the context passed to it is + // canceled. It also closes a channel to signal that it has started. + firstCallbackStartedCh := make(chan struct{}) + cs.Schedule(func(ctx context.Context) { + close(firstCallbackStartedCh) + <-ctx.Done() + }) + + // Schedule a bunch of callbacks. These should be exeuted since the are + // scheduled before the serializer is closed. + const numCallbacks = 10 + callbackCh := make(chan int, numCallbacks) + for i := 0; i < numCallbacks; i++ { + num := i + if !cs.Schedule(func(context.Context) { callbackCh <- num }) { + t.Fatal("Schedule failed to accept a callback when the serializer is yet to be closed") + } + } + + // Ensure that none of the newer callbacks are executed at this point. + select { + case <-time.After(defaultTestShortTimeout): + case <-callbackCh: + t.Fatal("Newer callback executed when older one is still executing") + } + + // Wait for the first callback to start before closing the scheduler. + <-firstCallbackStartedCh + + // Cancel the context which will unblock the first callback. All of the + // other callbacks (which have not started executing at this point) should + // be executed after this. + serializerCancel() + + // Ensure that the newer callbacks are executed. + for i := 0; i < numCallbacks; i++ { + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for callback scheduled before close to be executed") + case num := <-callbackCh: + if num != i { + t.Fatalf("Executing callback %d, want %d", num, i) + } + } + } + <-cs.Done() + + done := make(chan struct{}) + if cs.Schedule(func(context.Context) { close(done) }) { + t.Fatal("Scheduled a callback after closing the serializer") + } + + // Ensure that the lates callback is executed at this point. + select { + case <-time.After(defaultTestShortTimeout): + case <-done: + t.Fatal("Newer callback executed when scheduled after closing serializer") + } +} diff --git a/internal/resolver/dns/go113.go b/internal/grpcsync/oncefunc.go similarity index 66% rename from internal/resolver/dns/go113.go rename to internal/grpcsync/oncefunc.go index 8783a8cf8214..6635f7bca96d 100644 --- a/internal/resolver/dns/go113.go +++ b/internal/grpcsync/oncefunc.go @@ -1,8 +1,6 @@ -// +build go1.13 - /* * - * Copyright 2019 gRPC authors. + * Copyright 2022 gRPC authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,16 +16,17 @@ * */ -package dns +package grpcsync -import "net" +import ( + "sync" +) -func init() { - filterError = func(err error) error { - if dnsErr, ok := err.(*net.DNSError); ok && dnsErr.IsNotFound { - // The name does not exist; not an error. - return nil - } - return err +// OnceFunc returns a function wrapping f which ensures f is only executed +// once even if the returned function is executed multiple times. +func OnceFunc(f func()) func() { + var once sync.Once + return func() { + once.Do(f) } } diff --git a/internal/grpcsync/oncefunc_test.go b/internal/grpcsync/oncefunc_test.go new file mode 100644 index 000000000000..2b0db8d3eaa3 --- /dev/null +++ b/internal/grpcsync/oncefunc_test.go @@ -0,0 +1,53 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpcsync + +import ( + "sync" + "sync/atomic" + "testing" + "time" +) + +// TestOnceFunc tests that a OnceFunc is executed only once even with multiple +// simultaneous callers of it. +func (s) TestOnceFunc(t *testing.T) { + var v int32 + inc := OnceFunc(func() { atomic.AddInt32(&v, 1) }) + + const numWorkers = 100 + var wg sync.WaitGroup // Blocks until all workers have called inc. + wg.Add(numWorkers) + + block := NewEvent() // Signal to worker goroutines to call inc + + for i := 0; i < numWorkers; i++ { + go func() { + <-block.Done() // Wait for a signal. + inc() // Call the OnceFunc. + wg.Done() + }() + } + time.Sleep(time.Millisecond) // Allow goroutines to get to the block. + block.Fire() // Unblock them. + wg.Wait() // Wait for them to complete. + if v != 1 { + t.Fatalf("OnceFunc() called %v times; want 1", v) + } +} diff --git a/internal/grpcsync/pubsub.go b/internal/grpcsync/pubsub.go new file mode 100644 index 000000000000..aef8cec1ab0c --- /dev/null +++ b/internal/grpcsync/pubsub.go @@ -0,0 +1,121 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpcsync + +import ( + "context" + "sync" +) + +// Subscriber represents an entity that is subscribed to messages published on +// a PubSub. It wraps the callback to be invoked by the PubSub when a new +// message is published. +type Subscriber interface { + // OnMessage is invoked when a new message is published. Implementations + // must not block in this method. + OnMessage(msg any) +} + +// PubSub is a simple one-to-many publish-subscribe system that supports +// messages of arbitrary type. It guarantees that messages are delivered in +// the same order in which they were published. +// +// Publisher invokes the Publish() method to publish new messages, while +// subscribers interested in receiving these messages register a callback +// via the Subscribe() method. +// +// Once a PubSub is stopped, no more messages can be published, but any pending +// published messages will be delivered to the subscribers. Done may be used +// to determine when all published messages have been delivered. +type PubSub struct { + cs *CallbackSerializer + + // Access to the below fields are guarded by this mutex. + mu sync.Mutex + msg any + subscribers map[Subscriber]bool +} + +// NewPubSub returns a new PubSub instance. Users should cancel the +// provided context to shutdown the PubSub. +func NewPubSub(ctx context.Context) *PubSub { + return &PubSub{ + cs: NewCallbackSerializer(ctx), + subscribers: map[Subscriber]bool{}, + } +} + +// Subscribe registers the provided Subscriber to the PubSub. +// +// If the PubSub contains a previously published message, the Subscriber's +// OnMessage() callback will be invoked asynchronously with the existing +// message to begin with, and subsequently for every newly published message. +// +// The caller is responsible for invoking the returned cancel function to +// unsubscribe itself from the PubSub. +func (ps *PubSub) Subscribe(sub Subscriber) (cancel func()) { + ps.mu.Lock() + defer ps.mu.Unlock() + + ps.subscribers[sub] = true + + if ps.msg != nil { + msg := ps.msg + ps.cs.Schedule(func(context.Context) { + ps.mu.Lock() + defer ps.mu.Unlock() + if !ps.subscribers[sub] { + return + } + sub.OnMessage(msg) + }) + } + + return func() { + ps.mu.Lock() + defer ps.mu.Unlock() + delete(ps.subscribers, sub) + } +} + +// Publish publishes the provided message to the PubSub, and invokes +// callbacks registered by subscribers asynchronously. +func (ps *PubSub) Publish(msg any) { + ps.mu.Lock() + defer ps.mu.Unlock() + + ps.msg = msg + for sub := range ps.subscribers { + s := sub + ps.cs.Schedule(func(context.Context) { + ps.mu.Lock() + defer ps.mu.Unlock() + if !ps.subscribers[s] { + return + } + s.OnMessage(msg) + }) + } +} + +// Done returns a channel that is closed after the context passed to NewPubSub +// is canceled and all updates have been sent to subscribers. +func (ps *PubSub) Done() <-chan struct{} { + return ps.cs.Done() +} diff --git a/internal/grpcsync/pubsub_test.go b/internal/grpcsync/pubsub_test.go new file mode 100644 index 000000000000..c9f9889c6933 --- /dev/null +++ b/internal/grpcsync/pubsub_test.go @@ -0,0 +1,196 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpcsync + +import ( + "context" + "sync" + "testing" + "time" +) + +type testSubscriber struct { + onMsgCh chan int +} + +func newTestSubscriber(chSize int) *testSubscriber { + return &testSubscriber{onMsgCh: make(chan int, chSize)} +} + +func (ts *testSubscriber) OnMessage(msg any) { + select { + case ts.onMsgCh <- msg.(int): + default: + } +} + +func (s) TestPubSub_PublishNoMsg(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + pubsub := NewPubSub(ctx) + + ts := newTestSubscriber(1) + pubsub.Subscribe(ts) + + select { + case <-ts.onMsgCh: + t.Fatal("Subscriber callback invoked when no message was published") + case <-time.After(defaultTestShortTimeout): + } +} + +func (s) TestPubSub_PublishMsgs_RegisterSubs_And_Stop(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + pubsub := NewPubSub(ctx) + + const numPublished = 10 + + ts1 := newTestSubscriber(numPublished) + pubsub.Subscribe(ts1) + + var wg sync.WaitGroup + wg.Add(2) + // Publish ten messages on the pubsub and ensure that they are received in order by the subscriber. + go func() { + for i := 0; i < numPublished; i++ { + pubsub.Publish(i) + } + wg.Done() + }() + + go func() { + defer wg.Done() + for i := 0; i < numPublished; i++ { + select { + case m := <-ts1.onMsgCh: + if m != i { + t.Errorf("Received unexpected message: %q; want: %q", m, i) + return + } + case <-time.After(defaultTestTimeout): + t.Error("Timeout when expecting the onMessage() callback to be invoked") + return + } + } + }() + wg.Wait() + if t.Failed() { + t.FailNow() + } + + // Register another subscriber and ensure that it receives the last published message. + ts2 := newTestSubscriber(numPublished) + pubsub.Subscribe(ts2) + + select { + case m := <-ts2.onMsgCh: + if m != numPublished-1 { + t.Fatalf("Received unexpected message: %q; want: %q", m, numPublished-1) + } + case <-time.After(defaultTestShortTimeout): + t.Fatal("Timeout when expecting the onMessage() callback to be invoked") + } + + wg.Add(3) + // Publish ten messages on the pubsub and ensure that they are received in order by the subscribers. + go func() { + for i := 0; i < numPublished; i++ { + pubsub.Publish(i) + } + wg.Done() + }() + go func() { + defer wg.Done() + for i := 0; i < numPublished; i++ { + select { + case m := <-ts1.onMsgCh: + if m != i { + t.Errorf("Received unexpected message: %q; want: %q", m, i) + return + } + case <-time.After(defaultTestTimeout): + t.Error("Timeout when expecting the onMessage() callback to be invoked") + return + } + } + + }() + go func() { + defer wg.Done() + for i := 0; i < numPublished; i++ { + select { + case m := <-ts2.onMsgCh: + if m != i { + t.Errorf("Received unexpected message: %q; want: %q", m, i) + return + } + case <-time.After(defaultTestTimeout): + t.Error("Timeout when expecting the onMessage() callback to be invoked") + return + } + } + }() + wg.Wait() + if t.Failed() { + t.FailNow() + } + + cancel() + <-pubsub.Done() + + go func() { + pubsub.Publish(99) + }() + // Ensure that the subscriber callback is not invoked as instantiated + // pubsub has already closed. + select { + case <-ts1.onMsgCh: + t.Fatal("The callback was invoked after pubsub being stopped") + case <-ts2.onMsgCh: + t.Fatal("The callback was invoked after pubsub being stopped") + case <-time.After(defaultTestShortTimeout): + } +} + +func (s) TestPubSub_PublishMsgs_BeforeRegisterSub(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + pubsub := NewPubSub(ctx) + + const numPublished = 3 + for i := 0; i < numPublished; i++ { + pubsub.Publish(i) + } + + ts := newTestSubscriber(numPublished) + pubsub.Subscribe(ts) + + // Ensure that the subscriber callback is invoked with a previously + // published message. + select { + case d := <-ts.onMsgCh: + if d != numPublished-1 { + t.Fatalf("Unexpected message received: %q; %q", d, numPublished-1) + } + + case <-time.After(defaultTestShortTimeout): + t.Fatal("Timeout when expecting the onMessage() callback to be invoked") + } +} diff --git a/internal/grpctest/grpctest.go b/internal/grpctest/grpctest.go index 53d1c24f4da5..53a39d56c0da 100644 --- a/internal/grpctest/grpctest.go +++ b/internal/grpctest/grpctest.go @@ -34,7 +34,7 @@ type errorer struct { t *testing.T } -func (e errorer) Errorf(format string, args ...interface{}) { +func (e errorer) Errorf(format string, args ...any) { atomic.StoreUint32(&lcFailed, 1) e.t.Errorf(format, args...) } @@ -62,6 +62,12 @@ func (Tester) Teardown(t *testing.T) { TLogger.EndTest(t) } +// Interface defines Tester's methods for use in this package. +type Interface interface { + Setup(*testing.T) + Teardown(*testing.T) +} + func getTestFunc(t *testing.T, xv reflect.Value, name string) func(*testing.T) { if m := xv.MethodByName(name); m.IsValid() { if f, ok := m.Interface().(func(*testing.T)); ok { @@ -74,22 +80,20 @@ func getTestFunc(t *testing.T, xv reflect.Value, name string) func(*testing.T) { } // RunSubTests runs all "Test___" functions that are methods of x as subtests -// of the current test. If x contains methods "Setup(*testing.T)" or -// "Teardown(*testing.T)", those are run before or after each of the test -// functions, respectively. +// of the current test. Setup is run before the test function and Teardown is +// run after. // // For example usage, see example_test.go. Run it using: -// $ go test -v -run TestExample . +// +// $ go test -v -run TestExample . // // To run a specific test/subtest: -// $ go test -v -run 'TestExample/^Something$' . -func RunSubTests(t *testing.T, x interface{}) { +// +// $ go test -v -run 'TestExample/^Something$' . +func RunSubTests(t *testing.T, x Interface) { xt := reflect.TypeOf(x) xv := reflect.ValueOf(x) - setup := getTestFunc(t, xv, "Setup") - teardown := getTestFunc(t, xv, "Teardown") - for i := 0; i < xt.NumMethod(); i++ { methodName := xt.Method(i).Name if !strings.HasPrefix(methodName, "Test") { @@ -97,9 +101,13 @@ func RunSubTests(t *testing.T, x interface{}) { } tfunc := getTestFunc(t, xv, methodName) t.Run(strings.TrimPrefix(methodName, "Test"), func(t *testing.T) { - setup(t) - // defer teardown to guarantee it is run even if tfunc uses t.Fatal() - defer teardown(t) + // Run leakcheck in t.Cleanup() to guarantee it is run even if tfunc + // or setup uses t.Fatal(). + // + // Note that a defer would run before t.Cleanup, so if a goroutine + // is closed by a test's t.Cleanup, a deferred leakcheck would fail. + t.Cleanup(func() { x.Teardown(t) }) + x.Setup(t) tfunc(t) }) } diff --git a/internal/grpctest/grpctest_test.go b/internal/grpctest/grpctest_test.go index 5fd646aac2ee..7bc704cc04b9 100644 --- a/internal/grpctest/grpctest_test.go +++ b/internal/grpctest/grpctest_test.go @@ -44,20 +44,3 @@ func TestRunSubTests(t *testing.T) { t.Fatalf("x = %v; want all fields true", x) } } - -type tNoST struct { - test bool -} - -func (t *tNoST) TestSubTest(*testing.T) { - t.test = true -} - -func TestNoSetupOrTeardown(t *testing.T) { - // Ensures nothing panics or fails if Setup/Teardown are omitted. - x := &tNoST{} - RunSubTests(t, x) - if want := (&tNoST{test: true}); !reflect.DeepEqual(x, want) { - t.Fatalf("x = %v; want %v", x, want) - } -} diff --git a/internal/grpctest/tlogger.go b/internal/grpctest/tlogger.go index 4558fc10176b..f7f6da15284f 100644 --- a/internal/grpctest/tlogger.go +++ b/internal/grpctest/tlogger.go @@ -22,12 +22,13 @@ import ( "errors" "fmt" "os" + "path" "regexp" - "runtime/debug" + "runtime" "strconv" - "strings" "sync" "testing" + "time" "google.golang.org/grpc/grpclog" ) @@ -40,18 +41,34 @@ const callingFrame = 4 type logType int +func (l logType) String() string { + switch l { + case infoLog: + return "INFO" + case warningLog: + return "WARNING" + case errorLog: + return "ERROR" + case fatalLog: + return "FATAL" + } + return "UNKNOWN" +} + const ( - logLog logType = iota + infoLog logType = iota + warningLog errorLog fatalLog ) type tLogger struct { v int - t *testing.T initialized bool - m sync.Mutex // protects errors + mu sync.Mutex // guards t, start, and errors + t *testing.T + start time.Time errors map[*regexp.Regexp]int } @@ -63,25 +80,27 @@ func init() { } } -// getStackFrame gets, from the stack byte string, the appropriate stack frame. -func getStackFrame(stack []byte, frame int) (string, error) { - s := strings.Split(string(stack), "\n") - if frame >= (len(s)-1)/2 { +// getCallingPrefix returns the at the given depth from the stack. +func getCallingPrefix(depth int) (string, error) { + _, file, line, ok := runtime.Caller(depth) + if !ok { return "", errors.New("frame request out-of-bounds") } - split := strings.Split(strings.Fields(s[(frame*2)+2][1:])[0], "/") - return fmt.Sprintf("%v:", split[len(split)-1]), nil + return fmt.Sprintf("%s:%d", path.Base(file), line), nil } // log logs the message with the specified parameters to the tLogger. -func (g *tLogger) log(ltype logType, depth int, format string, args ...interface{}) { - s := debug.Stack() - prefix, err := getStackFrame(s, callingFrame+depth) - args = append([]interface{}{prefix}, args...) +func (g *tLogger) log(ltype logType, depth int, format string, args ...any) { + g.mu.Lock() + defer g.mu.Unlock() + prefix, err := getCallingPrefix(callingFrame + depth) if err != nil { g.t.Error(err) return } + args = append([]any{ltype.String() + " " + prefix}, args...) + args = append(args, fmt.Sprintf(" (t=+%s)", time.Since(g.start))) + if format == "" { switch ltype { case errorLog: @@ -97,7 +116,8 @@ func (g *tLogger) log(ltype logType, depth int, format string, args ...interface g.t.Log(args...) } } else { - format = "%v " + format + // Add formatting directives for the callingPrefix and timeSuffix. + format = "%v " + format + "%s" switch ltype { case errorLog: if g.expected(fmt.Sprintf(format, args...)) { @@ -116,13 +136,14 @@ func (g *tLogger) log(ltype logType, depth int, format string, args ...interface // Update updates the testing.T that the testing logger logs to. Should be done // before every test. It also initializes the tLogger if it has not already. func (g *tLogger) Update(t *testing.T) { + g.mu.Lock() + defer g.mu.Unlock() if !g.initialized { grpclog.SetLoggerV2(TLogger) g.initialized = true } g.t = t - g.m.Lock() - defer g.m.Unlock() + g.start = time.Now() g.errors = map[*regexp.Regexp]int{} } @@ -137,20 +158,20 @@ func (g *tLogger) ExpectError(expr string) { // ExpectErrorN declares an error to be expected n times. func (g *tLogger) ExpectErrorN(expr string, n int) { + g.mu.Lock() + defer g.mu.Unlock() re, err := regexp.Compile(expr) if err != nil { g.t.Error(err) return } - g.m.Lock() - defer g.m.Unlock() g.errors[re] += n } // EndTest checks if expected errors were not encountered. func (g *tLogger) EndTest(t *testing.T) { - g.m.Lock() - defer g.m.Unlock() + g.mu.Lock() + defer g.mu.Unlock() for re, count := range g.errors { if count > 0 { t.Errorf("Expected error '%v' not encountered", re.String()) @@ -161,8 +182,6 @@ func (g *tLogger) EndTest(t *testing.T) { // expected determines if the error string is protected or not. func (g *tLogger) expected(s string) bool { - g.m.Lock() - defer g.m.Unlock() for re, count := range g.errors { if re.FindStringIndex(s) != nil { g.errors[re]-- @@ -175,67 +194,67 @@ func (g *tLogger) expected(s string) bool { return false } -func (g *tLogger) Info(args ...interface{}) { - g.log(logLog, 0, "", args...) +func (g *tLogger) Info(args ...any) { + g.log(infoLog, 0, "", args...) } -func (g *tLogger) Infoln(args ...interface{}) { - g.log(logLog, 0, "", args...) +func (g *tLogger) Infoln(args ...any) { + g.log(infoLog, 0, "", args...) } -func (g *tLogger) Infof(format string, args ...interface{}) { - g.log(logLog, 0, format, args...) +func (g *tLogger) Infof(format string, args ...any) { + g.log(infoLog, 0, format, args...) } -func (g *tLogger) InfoDepth(depth int, args ...interface{}) { - g.log(logLog, depth, "", args...) +func (g *tLogger) InfoDepth(depth int, args ...any) { + g.log(infoLog, depth, "", args...) } -func (g *tLogger) Warning(args ...interface{}) { - g.log(logLog, 0, "", args...) +func (g *tLogger) Warning(args ...any) { + g.log(warningLog, 0, "", args...) } -func (g *tLogger) Warningln(args ...interface{}) { - g.log(logLog, 0, "", args...) +func (g *tLogger) Warningln(args ...any) { + g.log(warningLog, 0, "", args...) } -func (g *tLogger) Warningf(format string, args ...interface{}) { - g.log(logLog, 0, format, args...) +func (g *tLogger) Warningf(format string, args ...any) { + g.log(warningLog, 0, format, args...) } -func (g *tLogger) WarningDepth(depth int, args ...interface{}) { - g.log(logLog, depth, "", args...) +func (g *tLogger) WarningDepth(depth int, args ...any) { + g.log(warningLog, depth, "", args...) } -func (g *tLogger) Error(args ...interface{}) { +func (g *tLogger) Error(args ...any) { g.log(errorLog, 0, "", args...) } -func (g *tLogger) Errorln(args ...interface{}) { +func (g *tLogger) Errorln(args ...any) { g.log(errorLog, 0, "", args...) } -func (g *tLogger) Errorf(format string, args ...interface{}) { +func (g *tLogger) Errorf(format string, args ...any) { g.log(errorLog, 0, format, args...) } -func (g *tLogger) ErrorDepth(depth int, args ...interface{}) { +func (g *tLogger) ErrorDepth(depth int, args ...any) { g.log(errorLog, depth, "", args...) } -func (g *tLogger) Fatal(args ...interface{}) { +func (g *tLogger) Fatal(args ...any) { g.log(fatalLog, 0, "", args...) } -func (g *tLogger) Fatalln(args ...interface{}) { +func (g *tLogger) Fatalln(args ...any) { g.log(fatalLog, 0, "", args...) } -func (g *tLogger) Fatalf(format string, args ...interface{}) { +func (g *tLogger) Fatalf(format string, args ...any) { g.log(fatalLog, 0, format, args...) } -func (g *tLogger) FatalDepth(depth int, args ...interface{}) { +func (g *tLogger) FatalDepth(depth int, args ...any) { g.log(fatalLog, depth, "", args...) } diff --git a/internal/grpcutil/compressor.go b/internal/grpcutil/compressor.go new file mode 100644 index 000000000000..9f4090967980 --- /dev/null +++ b/internal/grpcutil/compressor.go @@ -0,0 +1,47 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpcutil + +import ( + "strings" + + "google.golang.org/grpc/internal/envconfig" +) + +// RegisteredCompressorNames holds names of the registered compressors. +var RegisteredCompressorNames []string + +// IsCompressorNameRegistered returns true when name is available in registry. +func IsCompressorNameRegistered(name string) bool { + for _, compressor := range RegisteredCompressorNames { + if compressor == name { + return true + } + } + return false +} + +// RegisteredCompressors returns a string of registered compressor names +// separated by comma. +func RegisteredCompressors() string { + if !envconfig.AdvertiseCompressors { + return "" + } + return strings.Join(RegisteredCompressorNames, ",") +} diff --git a/internal/grpcutil/compressor_test.go b/internal/grpcutil/compressor_test.go new file mode 100644 index 000000000000..0d639422a9a0 --- /dev/null +++ b/internal/grpcutil/compressor_test.go @@ -0,0 +1,46 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpcutil + +import ( + "testing" + + "google.golang.org/grpc/internal/envconfig" +) + +func TestRegisteredCompressors(t *testing.T) { + defer func(c []string) { RegisteredCompressorNames = c }(RegisteredCompressorNames) + defer func(v bool) { envconfig.AdvertiseCompressors = v }(envconfig.AdvertiseCompressors) + RegisteredCompressorNames = []string{"gzip", "snappy"} + tests := []struct { + desc string + enabled bool + want string + }{ + {desc: "compressor_ad_disabled", enabled: false, want: ""}, + {desc: "compressor_ad_enabled", enabled: true, want: "gzip,snappy"}, + } + for _, tt := range tests { + envconfig.AdvertiseCompressors = tt.enabled + compressors := RegisteredCompressors() + if compressors != tt.want { + t.Fatalf("Unexpected compressors got:%s, want:%s", compressors, tt.want) + } + } +} diff --git a/internal/grpcutil/grpcutil.go b/internal/grpcutil/grpcutil.go new file mode 100644 index 000000000000..e2f948e8f4f4 --- /dev/null +++ b/internal/grpcutil/grpcutil.go @@ -0,0 +1,20 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package grpcutil provides utility functions used across the gRPC codebase. +package grpcutil diff --git a/internal/grpcutil/method.go b/internal/grpcutil/method.go index 4e7475060c1c..ec62b4775e5b 100644 --- a/internal/grpcutil/method.go +++ b/internal/grpcutil/method.go @@ -25,7 +25,6 @@ import ( // ParseMethod splits service and method from the input. It expects format // "/service/method". -// func ParseMethod(methodName string) (service, method string, _ error) { if !strings.HasPrefix(methodName, "/") { return "", "", errors.New("invalid method name: should start with /") @@ -39,6 +38,11 @@ func ParseMethod(methodName string) (service, method string, _ error) { return methodName[:pos], methodName[pos+1:], nil } +// baseContentType is the base content-type for gRPC. This is a valid +// content-type on it's own, but can also include a content-subtype such as +// "proto" as a suffix after "+" or ";". See +// https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests +// for more details. const baseContentType = "application/grpc" // ContentSubtype returns the content-subtype for the given content-type. The diff --git a/internal/grpcutil/regex.go b/internal/grpcutil/regex.go new file mode 100644 index 000000000000..7a092b2b8041 --- /dev/null +++ b/internal/grpcutil/regex.go @@ -0,0 +1,31 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpcutil + +import "regexp" + +// FullMatchWithRegex returns whether the full text matches the regex provided. +func FullMatchWithRegex(re *regexp.Regexp, text string) bool { + if len(text) == 0 { + return re.MatchString(text) + } + re.Longest() + rem := re.FindString(text) + return len(rem) == len(text) +} diff --git a/internal/grpcutil/regex_test.go b/internal/grpcutil/regex_test.go new file mode 100644 index 000000000000..4c12804fed5f --- /dev/null +++ b/internal/grpcutil/regex_test.go @@ -0,0 +1,72 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpcutil + +import ( + "regexp" + "testing" +) + +func TestFullMatchWithRegex(t *testing.T) { + tests := []struct { + name string + regexStr string + string string + want bool + }{ + { + name: "not match because only partial", + regexStr: "^a+$", + string: "ab", + want: false, + }, + { + name: "match because fully match", + regexStr: "^a+$", + string: "aa", + want: true, + }, + { + name: "longest", + regexStr: "a(|b)", + string: "ab", + want: true, + }, + { + name: "match all", + regexStr: ".*", + string: "", + want: true, + }, + { + name: "matches non-empty strings", + regexStr: ".+", + string: "", + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + hrm := regexp.MustCompile(tt.regexStr) + if got := FullMatchWithRegex(hrm, tt.string); got != tt.want { + t.Errorf("match() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/grpcutil/target.go b/internal/grpcutil/target.go deleted file mode 100644 index 80b33cdaf905..000000000000 --- a/internal/grpcutil/target.go +++ /dev/null @@ -1,55 +0,0 @@ -/* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -// Package grpcutil provides a bunch of utility functions to be used across the -// gRPC codebase. -package grpcutil - -import ( - "strings" - - "google.golang.org/grpc/resolver" -) - -// split2 returns the values from strings.SplitN(s, sep, 2). -// If sep is not found, it returns ("", "", false) instead. -func split2(s, sep string) (string, string, bool) { - spl := strings.SplitN(s, sep, 2) - if len(spl) < 2 { - return "", "", false - } - return spl[0], spl[1], true -} - -// ParseTarget splits target into a resolver.Target struct containing scheme, -// authority and endpoint. -// -// If target is not a valid scheme://authority/endpoint, it returns {Endpoint: -// target}. -func ParseTarget(target string) (ret resolver.Target) { - var ok bool - ret.Scheme, ret.Endpoint, ok = split2(target, "://") - if !ok { - return resolver.Target{Endpoint: target} - } - ret.Authority, ret.Endpoint, ok = split2(ret.Endpoint, "/") - if !ok { - return resolver.Target{Endpoint: target} - } - return ret -} diff --git a/internal/grpcutil/target_test.go b/internal/grpcutil/target_test.go deleted file mode 100644 index 92bdb63d213d..000000000000 --- a/internal/grpcutil/target_test.go +++ /dev/null @@ -1,79 +0,0 @@ -/* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package grpcutil - -import ( - "testing" - - "google.golang.org/grpc/resolver" -) - -func TestParseTarget(t *testing.T) { - for _, test := range []resolver.Target{ - {Scheme: "dns", Authority: "", Endpoint: "google.com"}, - {Scheme: "dns", Authority: "a.server.com", Endpoint: "google.com"}, - {Scheme: "dns", Authority: "a.server.com", Endpoint: "google.com/?a=b"}, - {Scheme: "passthrough", Authority: "", Endpoint: "/unix/socket/address"}, - } { - str := test.Scheme + "://" + test.Authority + "/" + test.Endpoint - got := ParseTarget(str) - if got != test { - t.Errorf("ParseTarget(%q) = %+v, want %+v", str, got, test) - } - } -} - -func TestParseTargetString(t *testing.T) { - for _, test := range []struct { - targetStr string - want resolver.Target - }{ - {targetStr: "", want: resolver.Target{Scheme: "", Authority: "", Endpoint: ""}}, - {targetStr: ":///", want: resolver.Target{Scheme: "", Authority: "", Endpoint: ""}}, - {targetStr: "a:///", want: resolver.Target{Scheme: "a", Authority: "", Endpoint: ""}}, - {targetStr: "://a/", want: resolver.Target{Scheme: "", Authority: "a", Endpoint: ""}}, - {targetStr: ":///a", want: resolver.Target{Scheme: "", Authority: "", Endpoint: "a"}}, - {targetStr: "a://b/", want: resolver.Target{Scheme: "a", Authority: "b", Endpoint: ""}}, - {targetStr: "a:///b", want: resolver.Target{Scheme: "a", Authority: "", Endpoint: "b"}}, - {targetStr: "://a/b", want: resolver.Target{Scheme: "", Authority: "a", Endpoint: "b"}}, - {targetStr: "a://b/c", want: resolver.Target{Scheme: "a", Authority: "b", Endpoint: "c"}}, - {targetStr: "dns:///google.com", want: resolver.Target{Scheme: "dns", Authority: "", Endpoint: "google.com"}}, - {targetStr: "dns://a.server.com/google.com", want: resolver.Target{Scheme: "dns", Authority: "a.server.com", Endpoint: "google.com"}}, - {targetStr: "dns://a.server.com/google.com/?a=b", want: resolver.Target{Scheme: "dns", Authority: "a.server.com", Endpoint: "google.com/?a=b"}}, - - {targetStr: "/", want: resolver.Target{Scheme: "", Authority: "", Endpoint: "/"}}, - {targetStr: "google.com", want: resolver.Target{Scheme: "", Authority: "", Endpoint: "google.com"}}, - {targetStr: "google.com/?a=b", want: resolver.Target{Scheme: "", Authority: "", Endpoint: "google.com/?a=b"}}, - {targetStr: "/unix/socket/address", want: resolver.Target{Scheme: "", Authority: "", Endpoint: "/unix/socket/address"}}, - - // If we can only parse part of the target. - {targetStr: "://", want: resolver.Target{Scheme: "", Authority: "", Endpoint: "://"}}, - {targetStr: "unix://domain", want: resolver.Target{Scheme: "", Authority: "", Endpoint: "unix://domain"}}, - {targetStr: "a:b", want: resolver.Target{Scheme: "", Authority: "", Endpoint: "a:b"}}, - {targetStr: "a/b", want: resolver.Target{Scheme: "", Authority: "", Endpoint: "a/b"}}, - {targetStr: "a:/b", want: resolver.Target{Scheme: "", Authority: "", Endpoint: "a:/b"}}, - {targetStr: "a//b", want: resolver.Target{Scheme: "", Authority: "", Endpoint: "a//b"}}, - {targetStr: "a://b", want: resolver.Target{Scheme: "", Authority: "", Endpoint: "a://b"}}, - } { - got := ParseTarget(test.targetStr) - if got != test.want { - t.Errorf("ParseTarget(%q) = %+v, want %+v", test.targetStr, got, test.want) - } - } -} diff --git a/internal/hierarchy/hierarchy.go b/internal/hierarchy/hierarchy.go index 17185d95d38e..c3baac3643ce 100644 --- a/internal/hierarchy/hierarchy.go +++ b/internal/hierarchy/hierarchy.go @@ -23,7 +23,6 @@ package hierarchy import ( - "google.golang.org/grpc/attributes" "google.golang.org/grpc/resolver" ) @@ -31,26 +30,37 @@ type pathKeyType string const pathKey = pathKeyType("grpc.internal.address.hierarchical_path") +type pathValue []string + +func (p pathValue) Equal(o any) bool { + op, ok := o.(pathValue) + if !ok { + return false + } + if len(op) != len(p) { + return false + } + for i, v := range p { + if v != op[i] { + return false + } + } + return true +} + // Get returns the hierarchical path of addr. func Get(addr resolver.Address) []string { - attrs := addr.Attributes + attrs := addr.BalancerAttributes if attrs == nil { return nil } - path, ok := attrs.Value(pathKey).([]string) - if !ok { - return nil - } - return path + path, _ := attrs.Value(pathKey).(pathValue) + return ([]string)(path) } // Set overrides the hierarchical path in addr with path. func Set(addr resolver.Address, path []string) resolver.Address { - if addr.Attributes == nil { - addr.Attributes = attributes.New(pathKey, path) - return addr - } - addr.Attributes = addr.Attributes.WithValues(pathKey, path) + addr.BalancerAttributes = addr.BalancerAttributes.WithValue(pathKey, pathValue(path)) return addr } @@ -60,26 +70,29 @@ func Set(addr resolver.Address, path []string) resolver.Address { // // Input: // [ -// {addr0, path: [p0, wt0]} -// {addr1, path: [p0, wt1]} -// {addr2, path: [p1, wt2]} -// {addr3, path: [p1, wt3]} +// +// {addr0, path: [p0, wt0]} +// {addr1, path: [p0, wt1]} +// {addr2, path: [p1, wt2]} +// {addr3, path: [p1, wt3]} +// // ] // // Addresses will be split into p0/p1, and the p0/p1 will be removed from the // path. // // Output: -// { -// p0: [ -// {addr0, path: [wt0]}, -// {addr1, path: [wt1]}, -// ], -// p1: [ -// {addr2, path: [wt2]}, -// {addr3, path: [wt3]}, -// ], -// } +// +// { +// p0: [ +// {addr0, path: [wt0]}, +// {addr1, path: [wt1]}, +// ], +// p1: [ +// {addr2, path: [wt2]}, +// {addr3, path: [wt3]}, +// ], +// } // // If hierarchical path is not set, or has no path in it, the address is // dropped. diff --git a/internal/hierarchy/hierarchy_test.go b/internal/hierarchy/hierarchy_test.go index fc62f82b0850..1043d5f81dfa 100644 --- a/internal/hierarchy/hierarchy_test.go +++ b/internal/hierarchy/hierarchy_test.go @@ -40,7 +40,7 @@ func TestGet(t *testing.T) { { name: "set", addr: resolver.Address{ - Attributes: attributes.New(pathKey, []string{"a", "b"}), + BalancerAttributes: attributes.New(pathKey, pathValue{"a", "b"}), }, want: []string{"a", "b"}, }, @@ -68,7 +68,7 @@ func TestSet(t *testing.T) { { name: "before is set", addr: resolver.Address{ - Attributes: attributes.New(pathKey, []string{"before", "a", "b"}), + BalancerAttributes: attributes.New(pathKey, pathValue{"before", "a", "b"}), }, path: []string{"a", "b"}, }, @@ -93,19 +93,19 @@ func TestGroup(t *testing.T) { { name: "all with hierarchy", addrs: []resolver.Address{ - {Addr: "a0", Attributes: attributes.New(pathKey, []string{"a"})}, - {Addr: "a1", Attributes: attributes.New(pathKey, []string{"a"})}, - {Addr: "b0", Attributes: attributes.New(pathKey, []string{"b"})}, - {Addr: "b1", Attributes: attributes.New(pathKey, []string{"b"})}, + {Addr: "a0", BalancerAttributes: attributes.New(pathKey, pathValue{"a"})}, + {Addr: "a1", BalancerAttributes: attributes.New(pathKey, pathValue{"a"})}, + {Addr: "b0", BalancerAttributes: attributes.New(pathKey, pathValue{"b"})}, + {Addr: "b1", BalancerAttributes: attributes.New(pathKey, pathValue{"b"})}, }, want: map[string][]resolver.Address{ "a": { - {Addr: "a0", Attributes: attributes.New(pathKey, []string{})}, - {Addr: "a1", Attributes: attributes.New(pathKey, []string{})}, + {Addr: "a0", BalancerAttributes: attributes.New(pathKey, pathValue{})}, + {Addr: "a1", BalancerAttributes: attributes.New(pathKey, pathValue{})}, }, "b": { - {Addr: "b0", Attributes: attributes.New(pathKey, []string{})}, - {Addr: "b1", Attributes: attributes.New(pathKey, []string{})}, + {Addr: "b0", BalancerAttributes: attributes.New(pathKey, pathValue{})}, + {Addr: "b1", BalancerAttributes: attributes.New(pathKey, pathValue{})}, }, }, }, @@ -113,15 +113,15 @@ func TestGroup(t *testing.T) { // Addresses without hierarchy are ignored. name: "without hierarchy", addrs: []resolver.Address{ - {Addr: "a0", Attributes: attributes.New(pathKey, []string{"a"})}, - {Addr: "a1", Attributes: attributes.New(pathKey, []string{"a"})}, - {Addr: "b0", Attributes: nil}, - {Addr: "b1", Attributes: nil}, + {Addr: "a0", BalancerAttributes: attributes.New(pathKey, pathValue{"a"})}, + {Addr: "a1", BalancerAttributes: attributes.New(pathKey, pathValue{"a"})}, + {Addr: "b0", BalancerAttributes: nil}, + {Addr: "b1", BalancerAttributes: nil}, }, want: map[string][]resolver.Address{ "a": { - {Addr: "a0", Attributes: attributes.New(pathKey, []string{})}, - {Addr: "a1", Attributes: attributes.New(pathKey, []string{})}, + {Addr: "a0", BalancerAttributes: attributes.New(pathKey, pathValue{})}, + {Addr: "a1", BalancerAttributes: attributes.New(pathKey, pathValue{})}, }, }, }, @@ -130,15 +130,15 @@ func TestGroup(t *testing.T) { // the address is ignored. name: "wrong type", addrs: []resolver.Address{ - {Addr: "a0", Attributes: attributes.New(pathKey, []string{"a"})}, - {Addr: "a1", Attributes: attributes.New(pathKey, []string{"a"})}, - {Addr: "b0", Attributes: attributes.New(pathKey, "b")}, - {Addr: "b1", Attributes: attributes.New(pathKey, 314)}, + {Addr: "a0", BalancerAttributes: attributes.New(pathKey, pathValue{"a"})}, + {Addr: "a1", BalancerAttributes: attributes.New(pathKey, pathValue{"a"})}, + {Addr: "b0", BalancerAttributes: attributes.New(pathKey, "b")}, + {Addr: "b1", BalancerAttributes: attributes.New(pathKey, 314)}, }, want: map[string][]resolver.Address{ "a": { - {Addr: "a0", Attributes: attributes.New(pathKey, []string{})}, - {Addr: "a1", Attributes: attributes.New(pathKey, []string{})}, + {Addr: "a0", BalancerAttributes: attributes.New(pathKey, pathValue{})}, + {Addr: "a1", BalancerAttributes: attributes.New(pathKey, pathValue{})}, }, }, }, @@ -167,14 +167,14 @@ func TestGroupE2E(t *testing.T) { var addrsWithHierarchy []resolver.Address for p, wts := range hierarchy { - path1 := []string{p} + path1 := pathValue{p} for wt, addrs := range wts { - path2 := append([]string(nil), path1...) + path2 := append(pathValue(nil), path1...) path2 = append(path2, wt) for _, addr := range addrs { a := resolver.Address{ - Addr: addr, - Attributes: attributes.New(pathKey, path2), + Addr: addr, + BalancerAttributes: attributes.New(pathKey, path2), } addrsWithHierarchy = append(addrsWithHierarchy, a) } diff --git a/internal/idle/idle.go b/internal/idle/idle.go new file mode 100644 index 000000000000..6c272476e5ef --- /dev/null +++ b/internal/idle/idle.go @@ -0,0 +1,301 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package idle contains a component for managing idleness (entering and exiting) +// based on RPC activity. +package idle + +import ( + "fmt" + "math" + "sync" + "sync/atomic" + "time" + + "google.golang.org/grpc/grpclog" +) + +// For overriding in unit tests. +var timeAfterFunc = func(d time.Duration, f func()) *time.Timer { + return time.AfterFunc(d, f) +} + +// Enforcer is the functionality provided by grpc.ClientConn to enter +// and exit from idle mode. +type Enforcer interface { + ExitIdleMode() error + EnterIdleMode() error +} + +// Manager defines the functionality required to track RPC activity on a +// channel. +type Manager interface { + OnCallBegin() error + OnCallEnd() + Close() +} + +type noopManager struct{} + +func (noopManager) OnCallBegin() error { return nil } +func (noopManager) OnCallEnd() {} +func (noopManager) Close() {} + +// manager implements the Manager interface. It uses atomic operations to +// synchronize access to shared state and a mutex to guarantee mutual exclusion +// in a critical section. +type manager struct { + // State accessed atomically. + lastCallEndTime int64 // Unix timestamp in nanos; time when the most recent RPC completed. + activeCallsCount int32 // Count of active RPCs; -math.MaxInt32 means channel is idle or is trying to get there. + activeSinceLastTimerCheck int32 // Boolean; True if there was an RPC since the last timer callback. + closed int32 // Boolean; True when the manager is closed. + + // Can be accessed without atomics or mutex since these are set at creation + // time and read-only after that. + enforcer Enforcer // Functionality provided by grpc.ClientConn. + timeout int64 // Idle timeout duration nanos stored as an int64. + logger grpclog.LoggerV2 + + // idleMu is used to guarantee mutual exclusion in two scenarios: + // - Opposing intentions: + // - a: Idle timeout has fired and handleIdleTimeout() is trying to put + // the channel in idle mode because the channel has been inactive. + // - b: At the same time an RPC is made on the channel, and OnCallBegin() + // is trying to prevent the channel from going idle. + // - Competing intentions: + // - The channel is in idle mode and there are multiple RPCs starting at + // the same time, all trying to move the channel out of idle. Only one + // of them should succeed in doing so, while the other RPCs should + // piggyback on the first one and be successfully handled. + idleMu sync.RWMutex + actuallyIdle bool + timer *time.Timer +} + +// ManagerOptions is a collection of options used by +// NewManager. +type ManagerOptions struct { + Enforcer Enforcer + Timeout time.Duration + Logger grpclog.LoggerV2 +} + +// NewManager creates a new idleness manager implementation for the +// given idle timeout. +func NewManager(opts ManagerOptions) Manager { + if opts.Timeout == 0 { + return noopManager{} + } + + m := &manager{ + enforcer: opts.Enforcer, + timeout: int64(opts.Timeout), + logger: opts.Logger, + } + m.timer = timeAfterFunc(opts.Timeout, m.handleIdleTimeout) + return m +} + +// resetIdleTimer resets the idle timer to the given duration. This method +// should only be called from the timer callback. +func (m *manager) resetIdleTimer(d time.Duration) { + m.idleMu.Lock() + defer m.idleMu.Unlock() + + if m.timer == nil { + // Only close sets timer to nil. We are done. + return + } + + // It is safe to ignore the return value from Reset() because this method is + // only ever called from the timer callback, which means the timer has + // already fired. + m.timer.Reset(d) +} + +// handleIdleTimeout is the timer callback that is invoked upon expiry of the +// configured idle timeout. The channel is considered inactive if there are no +// ongoing calls and no RPC activity since the last time the timer fired. +func (m *manager) handleIdleTimeout() { + if m.isClosed() { + return + } + + if atomic.LoadInt32(&m.activeCallsCount) > 0 { + m.resetIdleTimer(time.Duration(m.timeout)) + return + } + + // There has been activity on the channel since we last got here. Reset the + // timer and return. + if atomic.LoadInt32(&m.activeSinceLastTimerCheck) == 1 { + // Set the timer to fire after a duration of idle timeout, calculated + // from the time the most recent RPC completed. + atomic.StoreInt32(&m.activeSinceLastTimerCheck, 0) + m.resetIdleTimer(time.Duration(atomic.LoadInt64(&m.lastCallEndTime) + m.timeout - time.Now().UnixNano())) + return + } + + // This CAS operation is extremely likely to succeed given that there has + // been no activity since the last time we were here. Setting the + // activeCallsCount to -math.MaxInt32 indicates to OnCallBegin() that the + // channel is either in idle mode or is trying to get there. + if !atomic.CompareAndSwapInt32(&m.activeCallsCount, 0, -math.MaxInt32) { + // This CAS operation can fail if an RPC started after we checked for + // activity at the top of this method, or one was ongoing from before + // the last time we were here. In both case, reset the timer and return. + m.resetIdleTimer(time.Duration(m.timeout)) + return + } + + // Now that we've set the active calls count to -math.MaxInt32, it's time to + // actually move to idle mode. + if m.tryEnterIdleMode() { + // Successfully entered idle mode. No timer needed until we exit idle. + return + } + + // Failed to enter idle mode due to a concurrent RPC that kept the channel + // active, or because of an error from the channel. Undo the attempt to + // enter idle, and reset the timer to try again later. + atomic.AddInt32(&m.activeCallsCount, math.MaxInt32) + m.resetIdleTimer(time.Duration(m.timeout)) +} + +// tryEnterIdleMode instructs the channel to enter idle mode. But before +// that, it performs a last minute check to ensure that no new RPC has come in, +// making the channel active. +// +// Return value indicates whether or not the channel moved to idle mode. +// +// Holds idleMu which ensures mutual exclusion with exitIdleMode. +func (m *manager) tryEnterIdleMode() bool { + m.idleMu.Lock() + defer m.idleMu.Unlock() + + if atomic.LoadInt32(&m.activeCallsCount) != -math.MaxInt32 { + // We raced and lost to a new RPC. Very rare, but stop entering idle. + return false + } + if atomic.LoadInt32(&m.activeSinceLastTimerCheck) == 1 { + // An very short RPC could have come in (and also finished) after we + // checked for calls count and activity in handleIdleTimeout(), but + // before the CAS operation. So, we need to check for activity again. + return false + } + + // No new RPCs have come in since we last set the active calls count value + // -math.MaxInt32 in the timer callback. And since we have the lock, it is + // safe to enter idle mode now. + if err := m.enforcer.EnterIdleMode(); err != nil { + m.logger.Errorf("Failed to enter idle mode: %v", err) + return false + } + + // Successfully entered idle mode. + m.actuallyIdle = true + return true +} + +// OnCallBegin is invoked at the start of every RPC. +func (m *manager) OnCallBegin() error { + if m.isClosed() { + return nil + } + + if atomic.AddInt32(&m.activeCallsCount, 1) > 0 { + // Channel is not idle now. Set the activity bit and allow the call. + atomic.StoreInt32(&m.activeSinceLastTimerCheck, 1) + return nil + } + + // Channel is either in idle mode or is in the process of moving to idle + // mode. Attempt to exit idle mode to allow this RPC. + if err := m.exitIdleMode(); err != nil { + // Undo the increment to calls count, and return an error causing the + // RPC to fail. + atomic.AddInt32(&m.activeCallsCount, -1) + return err + } + + atomic.StoreInt32(&m.activeSinceLastTimerCheck, 1) + return nil +} + +// exitIdleMode instructs the channel to exit idle mode. +// +// Holds idleMu which ensures mutual exclusion with tryEnterIdleMode. +func (m *manager) exitIdleMode() error { + m.idleMu.Lock() + defer m.idleMu.Unlock() + + if !m.actuallyIdle { + // This can happen in two scenarios: + // - handleIdleTimeout() set the calls count to -math.MaxInt32 and called + // tryEnterIdleMode(). But before the latter could grab the lock, an RPC + // came in and OnCallBegin() noticed that the calls count is negative. + // - Channel is in idle mode, and multiple new RPCs come in at the same + // time, all of them notice a negative calls count in OnCallBegin and get + // here. The first one to get the lock would got the channel to exit idle. + // + // Either way, nothing to do here. + return nil + } + + if err := m.enforcer.ExitIdleMode(); err != nil { + return fmt.Errorf("channel failed to exit idle mode: %v", err) + } + + // Undo the idle entry process. This also respects any new RPC attempts. + atomic.AddInt32(&m.activeCallsCount, math.MaxInt32) + m.actuallyIdle = false + + // Start a new timer to fire after the configured idle timeout. + m.timer = timeAfterFunc(time.Duration(m.timeout), m.handleIdleTimeout) + return nil +} + +// OnCallEnd is invoked at the end of every RPC. +func (m *manager) OnCallEnd() { + if m.isClosed() { + return + } + + // Record the time at which the most recent call finished. + atomic.StoreInt64(&m.lastCallEndTime, time.Now().UnixNano()) + + // Decrement the active calls count. This count can temporarily go negative + // when the timer callback is in the process of moving the channel to idle + // mode, but one or more RPCs come in and complete before the timer callback + // can get done with the process of moving to idle mode. + atomic.AddInt32(&m.activeCallsCount, -1) +} + +func (m *manager) isClosed() bool { + return atomic.LoadInt32(&m.closed) == 1 +} + +func (m *manager) Close() { + atomic.StoreInt32(&m.closed, 1) + + m.idleMu.Lock() + m.timer.Stop() + m.timer = nil + m.idleMu.Unlock() +} diff --git a/internal/idle/idle_e2e_test.go b/internal/idle/idle_e2e_test.go new file mode 100644 index 000000000000..de88046bc362 --- /dev/null +++ b/internal/idle/idle_e2e_test.go @@ -0,0 +1,456 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package idle_test + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/channelz" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/status" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +func init() { + channelz.TurnOn() +} + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +const ( + defaultTestTimeout = 10 * time.Second + defaultTestShortTimeout = 100 * time.Millisecond + defaultTestShortIdleTimeout = 500 * time.Millisecond +) + +// channelzTraceEventFound looks up the top-channels in channelz (expects a +// single one), and checks if there is a trace event on the channel matching the +// provided description string. +func channelzTraceEventFound(ctx context.Context, wantDesc string) error { + for ctx.Err() == nil { + tcs, _ := channelz.GetTopChannels(0, 0) + if l := len(tcs); l != 1 { + return fmt.Errorf("when looking for channelz trace event with description %q, found %d top-level channels, want 1", wantDesc, l) + } + if tcs[0].Trace == nil { + return fmt.Errorf("when looking for channelz trace event with description %q, no trace events found for top-level channel", wantDesc) + } + + for _, e := range tcs[0].Trace.Events { + if strings.Contains(e.Desc, wantDesc) { + return nil + } + } + } + return fmt.Errorf("when looking for channelz trace event with description %q, %w", wantDesc, ctx.Err()) +} + +// channelzTraceEventNotFound looks up the top-channels in channelz (expects a +// single one), and verifies that there is no trace event on the channel +// matching the provided description string. +func channelzTraceEventNotFound(ctx context.Context, wantDesc string) error { + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + + err := channelzTraceEventFound(sCtx, wantDesc) + if err == nil { + return fmt.Errorf("found channelz trace event with description %q, when expected not to", wantDesc) + } + if !errors.Is(err, context.DeadlineExceeded) { + return err + } + return nil +} + +// Tests the case where channel idleness is disabled by passing an idle_timeout +// of 0. Verifies that a READY channel with no RPCs does not move to IDLE. +func (s) TestChannelIdleness_Disabled_NoActivity(t *testing.T) { + // Create a ClientConn with idle_timeout set to 0. + r := manual.NewBuilderWithScheme("whatever") + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(r), + grpc.WithIdleTimeout(0), // Disable idleness. + grpc.WithDefaultServiceConfig(`{"loadBalancingConfig": [{"round_robin":{}}]}`), + } + cc, err := grpc.Dial(r.Scheme()+":///test.server", dopts...) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) + } + t.Cleanup(func() { cc.Close() }) + + // Start a test backend and push an address update via the resolver. + backend := stubserver.StartTestService(t, nil) + t.Cleanup(backend.Stop) + r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: backend.Address}}}) + + // Verify that the ClientConn moves to READY. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + + // Verify that the ClientConn stay in READY. + sCtx, sCancel := context.WithTimeout(ctx, 3*defaultTestShortIdleTimeout) + defer sCancel() + testutils.AwaitNoStateChange(sCtx, t, cc, connectivity.Ready) + + // Verify that there are no idleness related channelz events. + if err := channelzTraceEventNotFound(ctx, "entering idle mode"); err != nil { + t.Fatal(err) + } + if err := channelzTraceEventNotFound(ctx, "exiting idle mode"); err != nil { + t.Fatal(err) + } +} + +// Tests the case where channel idleness is enabled by passing a small value for +// idle_timeout. Verifies that a READY channel with no RPCs moves to IDLE. +func (s) TestChannelIdleness_Enabled_NoActivity(t *testing.T) { + // Create a ClientConn with a short idle_timeout. + r := manual.NewBuilderWithScheme("whatever") + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(r), + grpc.WithIdleTimeout(defaultTestShortIdleTimeout), + grpc.WithDefaultServiceConfig(`{"loadBalancingConfig": [{"round_robin":{}}]}`), + } + cc, err := grpc.Dial(r.Scheme()+":///test.server", dopts...) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) + } + t.Cleanup(func() { cc.Close() }) + + // Start a test backend and push an address update via the resolver. + backend := stubserver.StartTestService(t, nil) + t.Cleanup(backend.Stop) + r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: backend.Address}}}) + + // Verify that the ClientConn moves to READY. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + + // Verify that the ClientConn moves to IDLE as there is no activity. + testutils.AwaitState(ctx, t, cc, connectivity.Idle) + + // Verify idleness related channelz events. + if err := channelzTraceEventFound(ctx, "entering idle mode"); err != nil { + t.Fatal(err) + } +} + +// Tests the case where channel idleness is enabled by passing a small value for +// idle_timeout. Verifies that a READY channel with an ongoing RPC stays READY. +func (s) TestChannelIdleness_Enabled_OngoingCall(t *testing.T) { + // Create a ClientConn with a short idle_timeout. + r := manual.NewBuilderWithScheme("whatever") + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(r), + grpc.WithIdleTimeout(defaultTestShortIdleTimeout), + grpc.WithDefaultServiceConfig(`{"loadBalancingConfig": [{"round_robin":{}}]}`), + } + cc, err := grpc.Dial(r.Scheme()+":///test.server", dopts...) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) + } + t.Cleanup(func() { cc.Close() }) + + // Start a test backend which keeps a unary RPC call active by blocking on a + // channel that is closed by the test later on. Also push an address update + // via the resolver. + blockCh := make(chan struct{}) + backend := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { + <-blockCh + return &testpb.Empty{}, nil + }, + } + if err := backend.StartServer(); err != nil { + t.Fatalf("Failed to start backend: %v", err) + } + t.Cleanup(backend.Stop) + r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: backend.Address}}}) + + // Verify that the ClientConn moves to READY. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + + // Spawn a goroutine which checks expected state transitions and idleness + // channelz trace events. It eventually closes `blockCh`, thereby unblocking + // the server RPC handler and the unary call below. + errCh := make(chan error, 1) + go func() { + defer close(blockCh) + // Verify that the ClientConn stays in READY. + sCtx, sCancel := context.WithTimeout(ctx, 3*defaultTestShortIdleTimeout) + defer sCancel() + testutils.AwaitNoStateChange(sCtx, t, cc, connectivity.Ready) + + // Verify that there are no idleness related channelz events. + if err := channelzTraceEventNotFound(ctx, "entering idle mode"); err != nil { + errCh <- err + return + } + if err := channelzTraceEventNotFound(ctx, "exiting idle mode"); err != nil { + errCh <- err + return + } + + // Unblock the unary RPC on the server. + errCh <- nil + }() + + // Make a unary RPC that blocks on the server, thereby ensuring that the + // count of active RPCs on the client is non-zero. + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Errorf("EmptyCall RPC failed: %v", err) + } + + select { + case err := <-errCh: + if err != nil { + t.Fatal(err) + } + case <-ctx.Done(): + t.Fatalf("Timeout when trying to verify that an active RPC keeps channel from moving to IDLE") + } +} + +// Tests the case where channel idleness is enabled by passing a small value for +// idle_timeout. Verifies that activity on a READY channel (frequent and short +// RPCs) keeps it from moving to IDLE. +func (s) TestChannelIdleness_Enabled_ActiveSinceLastCheck(t *testing.T) { + // Create a ClientConn with a short idle_timeout. + r := manual.NewBuilderWithScheme("whatever") + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(r), + grpc.WithIdleTimeout(defaultTestShortIdleTimeout), + grpc.WithDefaultServiceConfig(`{"loadBalancingConfig": [{"round_robin":{}}]}`), + } + cc, err := grpc.Dial(r.Scheme()+":///test.server", dopts...) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) + } + t.Cleanup(func() { cc.Close() }) + + // Start a test backend and push an address update via the resolver. + backend := stubserver.StartTestService(t, nil) + t.Cleanup(backend.Stop) + r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: backend.Address}}}) + + // Verify that the ClientConn moves to READY. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + + // For a duration of three times the configured idle timeout, making RPCs + // every now and then and ensure that the channel does not move out of + // READY. + sCtx, sCancel := context.WithTimeout(ctx, 3*defaultTestShortIdleTimeout) + defer sCancel() + go func() { + for ; sCtx.Err() == nil; <-time.After(defaultTestShortIdleTimeout / 4) { + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(sCtx, &testpb.Empty{}); err != nil { + // While iterating through this for loop, at some point in time, + // the context deadline will expire. It is safe to ignore that + // error code. + if status.Code(err) != codes.DeadlineExceeded { + t.Errorf("EmptyCall RPC failed: %v", err) + return + } + } + } + }() + + // Verify that the ClientConn stay in READY. + testutils.AwaitNoStateChange(sCtx, t, cc, connectivity.Ready) + + // Verify that there are no idleness related channelz events. + if err := channelzTraceEventNotFound(ctx, "entering idle mode"); err != nil { + t.Fatal(err) + } + if err := channelzTraceEventNotFound(ctx, "exiting idle mode"); err != nil { + t.Fatal(err) + } +} + +// Tests the case where channel idleness is enabled by passing a small value for +// idle_timeout. Verifies that a READY channel with no RPCs moves to IDLE. Also +// verifies that a subsequent RPC on the IDLE channel kicks it out of IDLE. +func (s) TestChannelIdleness_Enabled_ExitIdleOnRPC(t *testing.T) { + // Start a test backend and set the bootstrap state of the resolver to + // include this address. This will ensure that when the resolver is + // restarted when exiting idle, it will push the same address to grpc again. + r := manual.NewBuilderWithScheme("whatever") + backend := stubserver.StartTestService(t, nil) + t.Cleanup(backend.Stop) + r.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: backend.Address}}}) + + // Create a ClientConn with a short idle_timeout. + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(r), + grpc.WithIdleTimeout(defaultTestShortIdleTimeout), + grpc.WithDefaultServiceConfig(`{"loadBalancingConfig": [{"round_robin":{}}]}`), + } + cc, err := grpc.Dial(r.Scheme()+":///test.server", dopts...) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) + } + t.Cleanup(func() { cc.Close() }) + + // Verify that the ClientConn moves to READY. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + + // Verify that the ClientConn moves to IDLE as there is no activity. + testutils.AwaitState(ctx, t, cc, connectivity.Idle) + + // Verify idleness related channelz events. + if err := channelzTraceEventFound(ctx, "entering idle mode"); err != nil { + t.Fatal(err) + } + + // Make an RPC and ensure that it succeeds and moves the channel back to + // READY. + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall RPC failed: %v", err) + } + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + if err := channelzTraceEventFound(ctx, "exiting idle mode"); err != nil { + t.Fatal(err) + } +} + +// Tests the case where channel idleness is enabled by passing a small value for +// idle_timeout. Simulates a race between the idle timer firing and RPCs being +// initiated, after a period of inactivity on the channel. +// +// After a period of inactivity (for the configured idle timeout duration), when +// RPCs are started, there are two possibilities: +// - the idle timer wins the race and puts the channel in idle. The RPCs then +// kick it out of idle. +// - the RPCs win the race, and therefore the channel never moves to idle. +// +// In either of these cases, all RPCs must succeed. +func (s) TestChannelIdleness_Enabled_IdleTimeoutRacesWithRPCs(t *testing.T) { + // Start a test backend and set the bootstrap state of the resolver to + // include this address. This will ensure that when the resolver is + // restarted when exiting idle, it will push the same address to grpc again. + r := manual.NewBuilderWithScheme("whatever") + backend := stubserver.StartTestService(t, nil) + t.Cleanup(backend.Stop) + r.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: backend.Address}}}) + + // Create a ClientConn with a short idle_timeout. + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(r), + grpc.WithIdleTimeout(defaultTestShortTimeout), + grpc.WithDefaultServiceConfig(`{"loadBalancingConfig": [{"round_robin":{}}]}`), + } + cc, err := grpc.Dial(r.Scheme()+":///test.server", dopts...) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) + } + t.Cleanup(func() { cc.Close() }) + + // Verify that the ClientConn moves to READY. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Errorf("EmptyCall RPC failed: %v", err) + } + + // Make an RPC every defaultTestShortTimeout duration so as to race with the + // idle timeout. Whether the idle timeout wins the race or the RPC wins the + // race, RPCs must succeed. + for i := 0; i < 20; i++ { + <-time.After(defaultTestShortTimeout) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall RPC failed: %v", err) + } + t.Logf("Iteration %d succeeded", i) + } +} + +// Tests the case where the channel is IDLE and we call cc.Connect. +func (s) TestChannelIdleness_Connect(t *testing.T) { + // Start a test backend and set the bootstrap state of the resolver to + // include this address. This will ensure that when the resolver is + // restarted when exiting idle, it will push the same address to grpc again. + r := manual.NewBuilderWithScheme("whatever") + backend := stubserver.StartTestService(t, nil) + t.Cleanup(backend.Stop) + r.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: backend.Address}}}) + + // Create a ClientConn with a short idle_timeout. + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(r), + grpc.WithIdleTimeout(defaultTestShortIdleTimeout), + grpc.WithDefaultServiceConfig(`{"loadBalancingConfig": [{"round_robin":{}}]}`), + } + cc, err := grpc.Dial(r.Scheme()+":///test.server", dopts...) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) + } + t.Cleanup(func() { cc.Close() }) + + // Verify that the ClientConn moves to IDLE. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + testutils.AwaitState(ctx, t, cc, connectivity.Idle) + + // Connect should exit channel idleness. + cc.Connect() + + // Verify that the ClientConn moves back to READY. + testutils.AwaitState(ctx, t, cc, connectivity.Ready) +} diff --git a/internal/idle/idle_test.go b/internal/idle/idle_test.go new file mode 100644 index 000000000000..22bde3ba1422 --- /dev/null +++ b/internal/idle/idle_test.go @@ -0,0 +1,372 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package idle + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "testing" + "time" + + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal/grpctest" +) + +const ( + defaultTestTimeout = 10 * time.Second + defaultTestIdleTimeout = 500 * time.Millisecond // A short idle_timeout for tests. + defaultTestShortTimeout = 10 * time.Millisecond // A small deadline to wait for events expected to not happen. +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +type testEnforcer struct { + exitIdleCh chan struct{} + enterIdleCh chan struct{} +} + +func (ti *testEnforcer) ExitIdleMode() error { + ti.exitIdleCh <- struct{}{} + return nil + +} + +func (ti *testEnforcer) EnterIdleMode() error { + ti.enterIdleCh <- struct{}{} + return nil + +} + +func newTestEnforcer() *testEnforcer { + return &testEnforcer{ + exitIdleCh: make(chan struct{}, 1), + enterIdleCh: make(chan struct{}, 1), + } +} + +// overrideNewTimer overrides the new timer creation function by ensuring that a +// message is pushed on the returned channel everytime the timer fires. +func overrideNewTimer(t *testing.T) <-chan struct{} { + t.Helper() + + ch := make(chan struct{}, 1) + origTimeAfterFunc := timeAfterFunc + timeAfterFunc = func(d time.Duration, callback func()) *time.Timer { + return time.AfterFunc(d, func() { + select { + case ch <- struct{}{}: + default: + } + callback() + }) + } + t.Cleanup(func() { timeAfterFunc = origTimeAfterFunc }) + return ch +} + +// TestManager_Disabled tests the case where the idleness manager is +// disabled by passing an idle_timeout of 0. Verifies the following things: +// - timer callback does not fire +// - an RPC does not trigger a call to ExitIdleMode on the ClientConn +// - more calls to RPC termination (as compared to RPC initiation) does not +// result in an error log +func (s) TestManager_Disabled(t *testing.T) { + callbackCh := overrideNewTimer(t) + + // Create an idleness manager that is disabled because of idleTimeout being + // set to `0`. + enforcer := newTestEnforcer() + mgr := NewManager(ManagerOptions{Enforcer: enforcer, Timeout: time.Duration(0), Logger: grpclog.Component("test")}) + + // Ensure that the timer callback does not fire within a short deadline. + select { + case <-callbackCh: + t.Fatal("Idle timer callback fired when manager is disabled") + case <-time.After(defaultTestShortTimeout): + } + + // The first invocation of OnCallBegin() would lead to a call to + // ExitIdleMode() on the enforcer, unless the idleness manager is disabled. + mgr.OnCallBegin() + select { + case <-enforcer.exitIdleCh: + t.Fatalf("ExitIdleMode() called on enforcer when manager is disabled") + case <-time.After(defaultTestShortTimeout): + } + + // If the number of calls to OnCallEnd() exceeds the number of calls to + // OnCallBegin(), the idleness manager is expected to throw an error log + // (which will cause our TestLogger to fail the test). But since the manager + // is disabled, this should not happen. + mgr.OnCallEnd() + mgr.OnCallEnd() + + // The idleness manager is explicitly not closed here. But since the manager + // is disabled, it will not start the run goroutine, and hence we expect the + // leakchecker to not find any leaked goroutines. +} + +// TestManager_Enabled_TimerFires tests the case where the idle manager +// is enabled. Ensures that when there are no RPCs, the timer callback is +// invoked and the EnterIdleMode() method is invoked on the enforcer. +func (s) TestManager_Enabled_TimerFires(t *testing.T) { + callbackCh := overrideNewTimer(t) + + enforcer := newTestEnforcer() + mgr := NewManager(ManagerOptions{Enforcer: enforcer, Timeout: time.Duration(defaultTestIdleTimeout), Logger: grpclog.Component("test")}) + defer mgr.Close() + + // Ensure that the timer callback fires within a appropriate amount of time. + select { + case <-callbackCh: + case <-time.After(2 * defaultTestIdleTimeout): + t.Fatal("Timeout waiting for idle timer callback to fire") + } + + // Ensure that the channel moves to idle mode eventually. + select { + case <-enforcer.enterIdleCh: + case <-time.After(defaultTestTimeout): + t.Fatal("Timeout waiting for channel to move to idle") + } +} + +// TestManager_Enabled_OngoingCall tests the case where the idle manager +// is enabled. Ensures that when there is an ongoing RPC, the channel does not +// enter idle mode. +func (s) TestManager_Enabled_OngoingCall(t *testing.T) { + callbackCh := overrideNewTimer(t) + + enforcer := newTestEnforcer() + mgr := NewManager(ManagerOptions{Enforcer: enforcer, Timeout: time.Duration(defaultTestIdleTimeout), Logger: grpclog.Component("test")}) + defer mgr.Close() + + // Fire up a goroutine that simulates an ongoing RPC that is terminated + // after the timer callback fires for the first time. + timerFired := make(chan struct{}) + go func() { + mgr.OnCallBegin() + <-timerFired + mgr.OnCallEnd() + }() + + // Ensure that the timer callback fires and unblock the above goroutine. + select { + case <-callbackCh: + close(timerFired) + case <-time.After(2 * defaultTestIdleTimeout): + t.Fatal("Timeout waiting for idle timer callback to fire") + } + + // The invocation of the timer callback should not put the channel in idle + // mode since we had an ongoing RPC. + select { + case <-enforcer.enterIdleCh: + t.Fatalf("EnterIdleMode() called on enforcer when active RPC exists") + case <-time.After(defaultTestShortTimeout): + } + + // Since we terminated the ongoing RPC and we have no other active RPCs, the + // channel must move to idle eventually. + select { + case <-enforcer.enterIdleCh: + case <-time.After(defaultTestTimeout): + t.Fatal("Timeout waiting for channel to move to idle") + } +} + +// TestManager_Enabled_ActiveSinceLastCheck tests the case where the +// idle manager is enabled. Ensures that when there are active RPCs in the last +// period (even though there is no active call when the timer fires), the +// channel does not enter idle mode. +func (s) TestManager_Enabled_ActiveSinceLastCheck(t *testing.T) { + callbackCh := overrideNewTimer(t) + + enforcer := newTestEnforcer() + mgr := NewManager(ManagerOptions{Enforcer: enforcer, Timeout: time.Duration(defaultTestIdleTimeout), Logger: grpclog.Component("test")}) + defer mgr.Close() + + // Fire up a goroutine that simulates unary RPCs until the timer callback + // fires. + timerFired := make(chan struct{}) + go func() { + for ; ; <-time.After(defaultTestShortTimeout) { + mgr.OnCallBegin() + mgr.OnCallEnd() + + select { + case <-timerFired: + return + default: + } + } + }() + + // Ensure that the timer callback fires, and that we don't enter idle as + // part of this invocation of the timer callback, since we had some RPCs in + // this period. + select { + case <-callbackCh: + close(timerFired) + case <-time.After(2 * defaultTestIdleTimeout): + t.Fatal("Timeout waiting for idle timer callback to fire") + } + select { + case <-enforcer.enterIdleCh: + t.Fatalf("EnterIdleMode() called on enforcer when one RPC completed in the last period") + case <-time.After(defaultTestShortTimeout): + } + + // Since the unrary RPC terminated and we have no other active RPCs, the + // channel must move to idle eventually. + select { + case <-enforcer.enterIdleCh: + case <-time.After(defaultTestTimeout): + t.Fatal("Timeout waiting for channel to move to idle") + } +} + +// TestManager_Enabled_ExitIdleOnRPC tests the case where the idle +// manager is enabled. Ensures that the channel moves out of idle when an RPC is +// initiated. +func (s) TestManager_Enabled_ExitIdleOnRPC(t *testing.T) { + overrideNewTimer(t) + + enforcer := newTestEnforcer() + mgr := NewManager(ManagerOptions{Enforcer: enforcer, Timeout: time.Duration(defaultTestIdleTimeout), Logger: grpclog.Component("test")}) + defer mgr.Close() + + // Ensure that the channel moves to idle since there are no RPCs. + select { + case <-enforcer.enterIdleCh: + case <-time.After(2 * defaultTestIdleTimeout): + t.Fatal("Timeout waiting for channel to move to idle mode") + } + + for i := 0; i < 100; i++ { + // A call to OnCallBegin and OnCallEnd simulates an RPC. + go func() { + if err := mgr.OnCallBegin(); err != nil { + t.Errorf("OnCallBegin() failed: %v", err) + } + mgr.OnCallEnd() + }() + } + + // Ensure that the channel moves out of idle as a result of the above RPC. + select { + case <-enforcer.exitIdleCh: + case <-time.After(2 * defaultTestIdleTimeout): + t.Fatal("Timeout waiting for channel to move out of idle mode") + } + + // Ensure that only one call to exit idle mode is made to the CC. + sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + select { + case <-enforcer.exitIdleCh: + t.Fatal("More than one call to exit idle mode on the ClientConn; only one expected") + case <-sCtx.Done(): + } +} + +type racyState int32 + +const ( + stateInital racyState = iota + stateEnteredIdle + stateExitedIdle + stateActiveRPCs +) + +// racyIdlnessEnforcer is a test idleness enforcer used specifically to test the +// race between idle timeout and incoming RPCs. +type racyEnforcer struct { + state *racyState // Accessed atomically. +} + +// ExitIdleMode sets the internal state to stateExitedIdle. We should only ever +// exit idle when we are currently in idle. +func (ri *racyEnforcer) ExitIdleMode() error { + if !atomic.CompareAndSwapInt32((*int32)(ri.state), int32(stateEnteredIdle), int32(stateExitedIdle)) { + return fmt.Errorf("idleness enforcer asked to exit idle when it did not enter idle earlier") + } + return nil +} + +// EnterIdleMode attempts to set the internal state to stateEnteredIdle. We should only ever enter idle before RPCs start. +func (ri *racyEnforcer) EnterIdleMode() error { + if !atomic.CompareAndSwapInt32((*int32)(ri.state), int32(stateInital), int32(stateEnteredIdle)) { + return fmt.Errorf("idleness enforcer asked to enter idle after rpcs started") + } + return nil +} + +// TestManager_IdleTimeoutRacesWithOnCallBegin tests the case where +// firing of the idle timeout races with an incoming RPC. The test verifies that +// if the timer callback win the race and puts the channel in idle, the RPCs can +// kick it out of idle. And if the RPCs win the race and keep the channel +// active, then the timer callback should not attempt to put the channel in idle +// mode. +func (s) TestManager_IdleTimeoutRacesWithOnCallBegin(t *testing.T) { + // Run multiple iterations to simulate different possibilities. + for i := 0; i < 20; i++ { + t.Run(fmt.Sprintf("iteration=%d", i), func(t *testing.T) { + var idlenessState racyState + enforcer := &racyEnforcer{state: &idlenessState} + + // Configure a large idle timeout so that we can control the + // race between the timer callback and RPCs. + mgr := NewManager(ManagerOptions{Enforcer: enforcer, Timeout: time.Duration(10 * time.Minute), Logger: grpclog.Component("test")}) + defer mgr.Close() + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + m := mgr.(interface{ handleIdleTimeout() }) + <-time.After(defaultTestIdleTimeout / 10) + m.handleIdleTimeout() + }() + for j := 0; j < 100; j++ { + wg.Add(1) + go func() { + defer wg.Done() + // Wait for the configured idle timeout and simulate an RPC to + // race with the idle timeout timer callback. + <-time.After(defaultTestIdleTimeout / 10) + if err := mgr.OnCallBegin(); err != nil { + t.Errorf("OnCallBegin() failed: %v", err) + } + atomic.StoreInt32((*int32)(&idlenessState), int32(stateActiveRPCs)) + mgr.OnCallEnd() + }() + } + wg.Wait() + }) + } +} diff --git a/internal/internal.go b/internal/internal.go index 818ca857998b..c8a8c76d628c 100644 --- a/internal/internal.go +++ b/internal/internal.go @@ -30,7 +30,7 @@ import ( var ( // WithHealthCheckFunc is set by dialoptions.go - WithHealthCheckFunc interface{} // func (HealthChecker) DialOption + WithHealthCheckFunc any // func (HealthChecker) DialOption // HealthCheckFunc is used to provide client-side LB channel health checking HealthCheckFunc HealthChecker // BalancerUnregister is exported by package balancer to unregister a balancer. @@ -38,20 +38,143 @@ var ( // KeepaliveMinPingTime is the minimum ping interval. This must be 10s by // default, but tests may wish to set it lower for convenience. KeepaliveMinPingTime = 10 * time.Second - // NewRequestInfoContext creates a new context based on the argument context attaching - // the passed in RequestInfo to the new context. - NewRequestInfoContext interface{} // func(context.Context, credentials.RequestInfo) context.Context - // NewClientHandshakeInfoContext returns a copy of the input context with - // the passed in ClientHandshakeInfo struct added to it. - NewClientHandshakeInfoContext interface{} // func(context.Context, credentials.ClientHandshakeInfo) context.Context - // ParseServiceConfigForTesting is for creating a fake - // ClientConn for resolver testing only - ParseServiceConfigForTesting interface{} // func(string) *serviceconfig.ParseResult + // KeepaliveMinServerPingTime is the minimum ping interval for servers. + // This must be 1s by default, but tests may wish to set it lower for + // convenience. + KeepaliveMinServerPingTime = time.Second + // ParseServiceConfig parses a JSON representation of the service config. + ParseServiceConfig any // func(string) *serviceconfig.ParseResult // EqualServiceConfigForTesting is for testing service config generation and - // parsing. Both a and b should be returned by ParseServiceConfigForTesting. + // parsing. Both a and b should be returned by ParseServiceConfig. // This function compares the config without rawJSON stripped, in case the // there's difference in white space. EqualServiceConfigForTesting func(a, b serviceconfig.Config) bool + // GetCertificateProviderBuilder returns the registered builder for the + // given name. This is set by package certprovider for use from xDS + // bootstrap code while parsing certificate provider configs in the + // bootstrap file. + GetCertificateProviderBuilder any // func(string) certprovider.Builder + // GetXDSHandshakeInfoForTesting returns a pointer to the xds.HandshakeInfo + // stored in the passed in attributes. This is set by + // credentials/xds/xds.go. + GetXDSHandshakeInfoForTesting any // func (*attributes.Attributes) *xds.HandshakeInfo + // GetServerCredentials returns the transport credentials configured on a + // gRPC server. An xDS-enabled server needs to know what type of credentials + // is configured on the underlying gRPC server. This is set by server.go. + GetServerCredentials any // func (*grpc.Server) credentials.TransportCredentials + // CanonicalString returns the canonical string of the code defined here: + // https://github.com/grpc/grpc/blob/master/doc/statuscodes.md. + // + // This is used in the 1.0 release of gcp/observability, and thus must not be + // deleted or changed. + CanonicalString any // func (codes.Code) string + // DrainServerTransports initiates a graceful close of existing connections + // on a gRPC server accepted on the provided listener address. An + // xDS-enabled server invokes this method on a grpc.Server when a particular + // listener moves to "not-serving" mode. + DrainServerTransports any // func(*grpc.Server, string) + // AddGlobalServerOptions adds an array of ServerOption that will be + // effective globally for newly created servers. The priority will be: 1. + // user-provided; 2. this method; 3. default values. + // + // This is used in the 1.0 release of gcp/observability, and thus must not be + // deleted or changed. + AddGlobalServerOptions any // func(opt ...ServerOption) + // ClearGlobalServerOptions clears the array of extra ServerOption. This + // method is useful in testing and benchmarking. + // + // This is used in the 1.0 release of gcp/observability, and thus must not be + // deleted or changed. + ClearGlobalServerOptions func() + // AddGlobalDialOptions adds an array of DialOption that will be effective + // globally for newly created client channels. The priority will be: 1. + // user-provided; 2. this method; 3. default values. + // + // This is used in the 1.0 release of gcp/observability, and thus must not be + // deleted or changed. + AddGlobalDialOptions any // func(opt ...DialOption) + // DisableGlobalDialOptions returns a DialOption that prevents the + // ClientConn from applying the global DialOptions (set via + // AddGlobalDialOptions). + // + // This is used in the 1.0 release of gcp/observability, and thus must not be + // deleted or changed. + DisableGlobalDialOptions any // func() grpc.DialOption + // ClearGlobalDialOptions clears the array of extra DialOption. This + // method is useful in testing and benchmarking. + // + // This is used in the 1.0 release of gcp/observability, and thus must not be + // deleted or changed. + ClearGlobalDialOptions func() + // JoinDialOptions combines the dial options passed as arguments into a + // single dial option. + JoinDialOptions any // func(...grpc.DialOption) grpc.DialOption + // JoinServerOptions combines the server options passed as arguments into a + // single server option. + JoinServerOptions any // func(...grpc.ServerOption) grpc.ServerOption + + // WithBinaryLogger returns a DialOption that specifies the binary logger + // for a ClientConn. + // + // This is used in the 1.0 release of gcp/observability, and thus must not be + // deleted or changed. + WithBinaryLogger any // func(binarylog.Logger) grpc.DialOption + // BinaryLogger returns a ServerOption that can set the binary logger for a + // server. + // + // This is used in the 1.0 release of gcp/observability, and thus must not be + // deleted or changed. + BinaryLogger any // func(binarylog.Logger) grpc.ServerOption + + // SubscribeToConnectivityStateChanges adds a grpcsync.Subscriber to a provided grpc.ClientConn + SubscribeToConnectivityStateChanges any // func(*grpc.ClientConn, grpcsync.Subscriber) + + // NewXDSResolverWithConfigForTesting creates a new xds resolver builder using + // the provided xds bootstrap config instead of the global configuration from + // the supported environment variables. The resolver.Builder is meant to be + // used in conjunction with the grpc.WithResolvers DialOption. + // + // Testing Only + // + // This function should ONLY be used for testing and may not work with some + // other features, including the CSDS service. + NewXDSResolverWithConfigForTesting any // func([]byte) (resolver.Builder, error) + + // RegisterRLSClusterSpecifierPluginForTesting registers the RLS Cluster + // Specifier Plugin for testing purposes, regardless of the XDSRLS environment + // variable. + // + // TODO: Remove this function once the RLS env var is removed. + RegisterRLSClusterSpecifierPluginForTesting func() + + // UnregisterRLSClusterSpecifierPluginForTesting unregisters the RLS Cluster + // Specifier Plugin for testing purposes. This is needed because there is no way + // to unregister the RLS Cluster Specifier Plugin after registering it solely + // for testing purposes using RegisterRLSClusterSpecifierPluginForTesting(). + // + // TODO: Remove this function once the RLS env var is removed. + UnregisterRLSClusterSpecifierPluginForTesting func() + + // RegisterRBACHTTPFilterForTesting registers the RBAC HTTP Filter for testing + // purposes, regardless of the RBAC environment variable. + // + // TODO: Remove this function once the RBAC env var is removed. + RegisterRBACHTTPFilterForTesting func() + + // UnregisterRBACHTTPFilterForTesting unregisters the RBAC HTTP Filter for + // testing purposes. This is needed because there is no way to unregister the + // HTTP Filter after registering it solely for testing purposes using + // RegisterRBACHTTPFilterForTesting(). + // + // TODO: Remove this function once the RBAC env var is removed. + UnregisterRBACHTTPFilterForTesting func() + + // ORCAAllowAnyMinReportingInterval is for examples/orca use ONLY. + ORCAAllowAnyMinReportingInterval any // func(so *orca.ServiceOptions) + + // GRPCResolverSchemeExtraMetadata determines when gRPC will add extra + // metadata to RPCs. + GRPCResolverSchemeExtraMetadata string = "xds" ) // HealthChecker defines the signature of the client-side LB channel health checking function. @@ -62,7 +185,7 @@ var ( // // The health checking protocol is defined at: // https://github.com/grpc/grpc/blob/master/doc/health-checking.md -type HealthChecker func(ctx context.Context, newStream func(string) (interface{}, error), setConnectivityState func(connectivity.State, error), serviceName string) error +type HealthChecker func(ctx context.Context, newStream func(string) (any, error), setConnectivityState func(connectivity.State, error), serviceName string) error const ( // CredsBundleModeFallback switches GoogleDefaultCreds to fallback mode. @@ -74,3 +197,9 @@ const ( // that supports backend returned by grpclb balancer. CredsBundleModeBackendFromBalancer = "backend-from-balancer" ) + +// RLSLoadBalancingPolicyName is the name of the RLS LB policy. +// +// It currently has an experimental suffix which would be removed once +// end-to-end testing of the policy is completed. +const RLSLoadBalancingPolicyName = "rls_experimental" diff --git a/internal/leakcheck/leakcheck.go b/internal/leakcheck/leakcheck.go index 946c575f140f..68c37fe4184d 100644 --- a/internal/leakcheck/leakcheck.go +++ b/internal/leakcheck/leakcheck.go @@ -42,6 +42,12 @@ var goroutinesToIgnore = []string{ "runtime_mcall", "(*loggingT).flushDaemon", "goroutine in C code", + // Ignore the http read/write goroutines. gce metadata.OnGCE() was leaking + // these, root cause unknown. + // + // https://github.com/grpc/grpc-go/issues/5171 + // https://github.com/grpc/grpc-go/issues/5173 + "created by net/http.(*Transport).dialConn", } // RegisterIgnoreGoroutine appends s into the ignore goroutine list. The @@ -91,7 +97,7 @@ func interestingGoroutines() (gs []string) { // Errorfer is the interface that wraps the Errorf method. It's a subset of // testing.TB to make it easy to use Check. type Errorfer interface { - Errorf(format string, args ...interface{}) + Errorf(format string, args ...any) } func check(efer Errorfer, timeout time.Duration) { diff --git a/internal/leakcheck/leakcheck_test.go b/internal/leakcheck/leakcheck_test.go index 50927e9db4cd..606632cd2a21 100644 --- a/internal/leakcheck/leakcheck_test.go +++ b/internal/leakcheck/leakcheck_test.go @@ -30,7 +30,7 @@ type testErrorfer struct { errors []string } -func (e *testErrorfer) Errorf(format string, args ...interface{}) { +func (e *testErrorfer) Errorf(format string, args ...any) { e.errors = append(e.errors, fmt.Sprintf(format, args...)) e.errorCount++ } diff --git a/internal/metadata/metadata.go b/internal/metadata/metadata.go new file mode 100644 index 000000000000..900bfb716080 --- /dev/null +++ b/internal/metadata/metadata.go @@ -0,0 +1,132 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package metadata contains functions to set and get metadata from addresses. +// +// This package is experimental. +package metadata + +import ( + "fmt" + "strings" + + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/resolver" +) + +type mdKeyType string + +const mdKey = mdKeyType("grpc.internal.address.metadata") + +type mdValue metadata.MD + +func (m mdValue) Equal(o any) bool { + om, ok := o.(mdValue) + if !ok { + return false + } + if len(m) != len(om) { + return false + } + for k, v := range m { + ov := om[k] + if len(ov) != len(v) { + return false + } + for i, ve := range v { + if ov[i] != ve { + return false + } + } + } + return true +} + +// Get returns the metadata of addr. +func Get(addr resolver.Address) metadata.MD { + attrs := addr.Attributes + if attrs == nil { + return nil + } + md, _ := attrs.Value(mdKey).(mdValue) + return metadata.MD(md) +} + +// Set sets (overrides) the metadata in addr. +// +// When a SubConn is created with this address, the RPCs sent on it will all +// have this metadata. +func Set(addr resolver.Address, md metadata.MD) resolver.Address { + addr.Attributes = addr.Attributes.WithValue(mdKey, mdValue(md)) + return addr +} + +// Validate validates every pair in md with ValidatePair. +func Validate(md metadata.MD) error { + for k, vals := range md { + if err := ValidatePair(k, vals...); err != nil { + return err + } + } + return nil +} + +// hasNotPrintable return true if msg contains any characters which are not in %x20-%x7E +func hasNotPrintable(msg string) bool { + // for i that saving a conversion if not using for range + for i := 0; i < len(msg); i++ { + if msg[i] < 0x20 || msg[i] > 0x7E { + return true + } + } + return false +} + +// ValidatePair validate a key-value pair with the following rules (the pseudo-header will be skipped) : +// +// - key must contain one or more characters. +// - the characters in the key must be contained in [0-9 a-z _ - .]. +// - if the key ends with a "-bin" suffix, no validation of the corresponding value is performed. +// - the characters in the every value must be printable (in [%x20-%x7E]). +func ValidatePair(key string, vals ...string) error { + // key should not be empty + if key == "" { + return fmt.Errorf("there is an empty key in the header") + } + // pseudo-header will be ignored + if key[0] == ':' { + return nil + } + // check key, for i that saving a conversion if not using for range + for i := 0; i < len(key); i++ { + r := key[i] + if !(r >= 'a' && r <= 'z') && !(r >= '0' && r <= '9') && r != '.' && r != '-' && r != '_' { + return fmt.Errorf("header key %q contains illegal characters not in [0-9a-z-_.]", key) + } + } + if strings.HasSuffix(key, "-bin") { + return nil + } + // check value + for _, val := range vals { + if hasNotPrintable(val) { + return fmt.Errorf("header key %q contains value with non-printable ASCII characters", key) + } + } + return nil +} diff --git a/internal/metadata/metadata_test.go b/internal/metadata/metadata_test.go new file mode 100644 index 000000000000..8f0e430e5ed4 --- /dev/null +++ b/internal/metadata/metadata_test.go @@ -0,0 +1,117 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package metadata + +import ( + "errors" + "reflect" + "testing" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/attributes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/resolver" +) + +func TestGet(t *testing.T) { + tests := []struct { + name string + addr resolver.Address + want metadata.MD + }{ + { + name: "not set", + addr: resolver.Address{}, + want: nil, + }, + { + name: "not set", + addr: resolver.Address{ + Attributes: attributes.New(mdKey, mdValue(metadata.Pairs("k", "v"))), + }, + want: metadata.Pairs("k", "v"), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := Get(tt.addr); !cmp.Equal(got, tt.want) { + t.Errorf("Get() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestSet(t *testing.T) { + tests := []struct { + name string + addr resolver.Address + md metadata.MD + }{ + { + name: "unset before", + addr: resolver.Address{}, + md: metadata.Pairs("k", "v"), + }, + { + name: "set before", + addr: resolver.Address{ + Attributes: attributes.New(mdKey, mdValue(metadata.Pairs("bef", "ore"))), + }, + md: metadata.Pairs("k", "v"), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + newAddr := Set(tt.addr, tt.md) + newMD := Get(newAddr) + if !cmp.Equal(newMD, tt.md) { + t.Errorf("md after Set() = %v, want %v", newMD, tt.md) + } + }) + } +} + +func TestValidate(t *testing.T) { + for _, test := range []struct { + md metadata.MD + want error + }{ + { + md: map[string][]string{string(rune(0x19)): {"testVal"}}, + want: errors.New("header key \"\\x19\" contains illegal characters not in [0-9a-z-_.]"), + }, + { + md: map[string][]string{"test": {string(rune(0x19))}}, + want: errors.New("header key \"test\" contains value with non-printable ASCII characters"), + }, + { + md: map[string][]string{"": {"valid"}}, + want: errors.New("there is an empty key in the header"), + }, + { + md: map[string][]string{"test-bin": {string(rune(0x19))}}, + want: nil, + }, + } { + err := Validate(test.md) + if !reflect.DeepEqual(err, test.want) { + t.Errorf("validating metadata which is %v got err :%v, want err :%v", test.md, err, test.want) + } + } +} diff --git a/internal/pretty/pretty.go b/internal/pretty/pretty.go new file mode 100644 index 000000000000..7033191375de --- /dev/null +++ b/internal/pretty/pretty.go @@ -0,0 +1,82 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package pretty defines helper functions to pretty-print structs for logging. +package pretty + +import ( + "bytes" + "encoding/json" + "fmt" + + "github.com/golang/protobuf/jsonpb" + protov1 "github.com/golang/protobuf/proto" + "google.golang.org/protobuf/encoding/protojson" + protov2 "google.golang.org/protobuf/proto" +) + +const jsonIndent = " " + +// ToJSON marshals the input into a json string. +// +// If marshal fails, it falls back to fmt.Sprintf("%+v"). +func ToJSON(e any) string { + switch ee := e.(type) { + case protov1.Message: + mm := jsonpb.Marshaler{Indent: jsonIndent} + ret, err := mm.MarshalToString(ee) + if err != nil { + // This may fail for proto.Anys, e.g. for xDS v2, LDS, the v2 + // messages are not imported, and this will fail because the message + // is not found. + return fmt.Sprintf("%+v", ee) + } + return ret + case protov2.Message: + mm := protojson.MarshalOptions{ + Multiline: true, + Indent: jsonIndent, + } + ret, err := mm.Marshal(ee) + if err != nil { + // This may fail for proto.Anys, e.g. for xDS v2, LDS, the v2 + // messages are not imported, and this will fail because the message + // is not found. + return fmt.Sprintf("%+v", ee) + } + return string(ret) + default: + ret, err := json.MarshalIndent(ee, "", jsonIndent) + if err != nil { + return fmt.Sprintf("%+v", ee) + } + return string(ret) + } +} + +// FormatJSON formats the input json bytes with indentation. +// +// If Indent fails, it returns the unchanged input as string. +func FormatJSON(b []byte) string { + var out bytes.Buffer + err := json.Indent(&out, b, "", jsonIndent) + if err != nil { + return string(b) + } + return out.String() +} diff --git a/internal/profiling/buffer/buffer.go b/internal/profiling/buffer/buffer.go index f4cd4201de11..27c8a2003cb1 100644 --- a/internal/profiling/buffer/buffer.go +++ b/internal/profiling/buffer/buffer.go @@ -170,7 +170,7 @@ func NewCircularBuffer(size uint32) (*CircularBuffer, error) { // a finite number of steps (also lock-free). Does not guarantee that push // order will be retained. Does not guarantee that the operation will succeed // if a Drain operation concurrently begins execution. -func (cb *CircularBuffer) Push(x interface{}) { +func (cb *CircularBuffer) Push(x any) { n := atomic.AddUint32(&cb.qpn, 1) & cb.qpMask qptr := atomic.LoadPointer(&cb.qp[n].q) q := (*queue)(qptr) @@ -221,10 +221,10 @@ func (cb *CircularBuffer) Push(x interface{}) { // arr that are copied is [from, to). Assumes that the result slice is already // allocated and is large enough to hold all the elements that might be copied. // Also assumes mutual exclusion on the array of pointers. -func dereferenceAppend(result []interface{}, arr []unsafe.Pointer, from, to uint32) []interface{} { +func dereferenceAppend(result []any, arr []unsafe.Pointer, from, to uint32) []any { for i := from; i < to; i++ { // We have mutual exclusion on arr, there's no need for atomics. - x := (*interface{})(arr[i]) + x := (*any)(arr[i]) if x != nil { result = append(result, *x) } @@ -235,7 +235,7 @@ func dereferenceAppend(result []interface{}, arr []unsafe.Pointer, from, to uint // Drain allocates and returns an array of things Pushed in to the circular // buffer. Push order is not maintained; that is, if B was Pushed after A, // drain may return B at a lower index than A in the returned array. -func (cb *CircularBuffer) Drain() []interface{} { +func (cb *CircularBuffer) Drain() []any { cb.drainMutex.Lock() qs := make([]*queue, len(cb.qp)) @@ -244,7 +244,7 @@ func (cb *CircularBuffer) Drain() []interface{} { } var wg sync.WaitGroup - wg.Add(int(len(qs))) + wg.Add(len(qs)) for i := 0; i < len(qs); i++ { go func(qi int) { qs[qi].drainWait() @@ -253,7 +253,7 @@ func (cb *CircularBuffer) Drain() []interface{} { } wg.Wait() - result := make([]interface{}, 0) + result := make([]any, 0) for i := 0; i < len(qs); i++ { if acquired := atomic.LoadUint32(&qs[i].acquired); acquired < qs[i].size { result = dereferenceAppend(result, qs[i].arr, 0, acquired) diff --git a/internal/profiling/buffer/buffer_test.go b/internal/profiling/buffer/buffer_test.go index a7f3b61e4afa..c816a87dac27 100644 --- a/internal/profiling/buffer/buffer_test.go +++ b/internal/profiling/buffer/buffer_test.go @@ -37,7 +37,7 @@ func Test(t *testing.T) { func (s) TestCircularBufferSerial(t *testing.T) { var size, i uint32 - var result []interface{} + var result []any size = 1 << 15 cb, err := NewCircularBuffer(size) @@ -78,7 +78,7 @@ func (s) TestCircularBufferSerial(t *testing.T) { func (s) TestCircularBufferOverflow(t *testing.T) { var size, i uint32 - var result []interface{} + var result []any size = 1 << 10 cb, err := NewCircularBuffer(size) @@ -106,7 +106,7 @@ func (s) TestCircularBufferOverflow(t *testing.T) { func (s) TestCircularBufferConcurrent(t *testing.T) { for tn := 0; tn < 2; tn++ { var size uint32 - var result []interface{} + var result []any size = 1 << 6 cb, err := NewCircularBuffer(size) diff --git a/internal/profiling/goid_modified.go b/internal/profiling/goid_modified.go index b186499cd0d1..f2bae99b2a95 100644 --- a/internal/profiling/goid_modified.go +++ b/internal/profiling/goid_modified.go @@ -1,3 +1,4 @@ +//go:build grpcgoid // +build grpcgoid /* @@ -32,48 +33,48 @@ import ( // // Several other approaches were considered before arriving at this: // -// 1. Using a CGO module: CGO usually has access to some things that regular -// Go does not. Till go1.4, CGO used to have access to the goroutine struct -// because the Go runtime was written in C. However, 1.5+ uses a native Go -// runtime; as a result, CGO does not have access to the goroutine structure -// anymore in modern Go. Besides, CGO interop wasn't fast enough (estimated -// to be ~170ns/op). This would also make building grpc require a C -// compiler, which isn't a requirement currently, breaking a lot of stuff. +// 1. Using a CGO module: CGO usually has access to some things that regular +// Go does not. Till go1.4, CGO used to have access to the goroutine struct +// because the Go runtime was written in C. However, 1.5+ uses a native Go +// runtime; as a result, CGO does not have access to the goroutine structure +// anymore in modern Go. Besides, CGO interop wasn't fast enough (estimated +// to be ~170ns/op). This would also make building grpc require a C +// compiler, which isn't a requirement currently, breaking a lot of stuff. // -// 2. Using runtime.Stack stacktrace: While this would remove the need for a -// modified Go runtime, this is ridiculously slow, thanks to the all the -// string processing shenanigans required to extract the goroutine ID (about -// ~2000ns/op). +// 2. Using runtime.Stack stacktrace: While this would remove the need for a +// modified Go runtime, this is ridiculously slow, thanks to the all the +// string processing shenanigans required to extract the goroutine ID (about +// ~2000ns/op). // -// 3. Using Go version-specific build tags: For any given Go version, the -// goroutine struct has a fixed structure. As a result, the goroutine ID -// could be extracted if we know the offset using some assembly. This would -// be faster then #1 and #2, but is harder to maintain. This would require -// special Go code that's both architecture-specific and go version-specific -// (a quadratic number of variants to maintain). +// 3. Using Go version-specific build tags: For any given Go version, the +// goroutine struct has a fixed structure. As a result, the goroutine ID +// could be extracted if we know the offset using some assembly. This would +// be faster then #1 and #2, but is harder to maintain. This would require +// special Go code that's both architecture-specific and go version-specific +// (a quadratic number of variants to maintain). // -// 4. This approach, which requires a simple modification [1] to the Go runtime -// to expose the current goroutine's ID. This is the chosen approach and it -// takes about ~2 ns/op, which is negligible in the face of the tens of -// microseconds that grpc takes to complete a RPC request. +// 4. This approach, which requires a simple modification [1] to the Go runtime +// to expose the current goroutine's ID. This is the chosen approach and it +// takes about ~2 ns/op, which is negligible in the face of the tens of +// microseconds that grpc takes to complete a RPC request. // // [1] To make the goroutine ID visible to Go programs apply the following // change to the runtime2.go file in your Go runtime installation: // -// diff --git a/src/runtime/runtime2.go b/src/runtime/runtime2.go -// --- a/src/runtime/runtime2.go -// +++ b/src/runtime/runtime2.go -// @@ -392,6 +392,10 @@ type stack struct { -// hi uintptr -// } +// diff --git a/src/runtime/runtime2.go b/src/runtime/runtime2.go +// --- a/src/runtime/runtime2.go +// +++ b/src/runtime/runtime2.go +// @@ -392,6 +392,10 @@ type stack struct { +// hi uintptr +// } // -// +func Goid() int64 { -// + return getg().goid -// +} -// + -// type g struct { -// // Stack parameters. -// // stack describes the actual stack memory: [stack.lo, stack.hi). +// +func Goid() int64 { +// + return getg().goid +// +} +// + +// type g struct { +// // Stack parameters. +// // stack describes the actual stack memory: [stack.lo, stack.hi). // // The exposed runtime.Goid() function will return a int64 goroutine ID. func goid() int64 { diff --git a/internal/profiling/goid_regular.go b/internal/profiling/goid_regular.go index 891c2e98f9db..042933227d81 100644 --- a/internal/profiling/goid_regular.go +++ b/internal/profiling/goid_regular.go @@ -1,3 +1,4 @@ +//go:build !grpcgoid // +build !grpcgoid /* diff --git a/internal/profiling/profiling.go b/internal/profiling/profiling.go index 4d8e68aab433..58f71423459e 100644 --- a/internal/profiling/profiling.go +++ b/internal/profiling/profiling.go @@ -26,10 +26,10 @@ // example, if one wants to profile the load balancing layer, which is // independent of RPC queries, a separate CircularBuffer can be used. // -// Note that the circular buffer simply takes any interface{}. In the future, -// more types of measurements (such as the number of memory allocations) could -// be measured, which might require a different type of object being pushed -// into the circular buffer. +// Note that the circular buffer simply takes any type. In the future, more +// types of measurements (such as the number of memory allocations) could be +// measured, which might require a different type of object being pushed into +// the circular buffer. package profiling import ( diff --git a/internal/proto/grpc_lookup_v1/rls.pb.go b/internal/proto/grpc_lookup_v1/rls.pb.go new file mode 100644 index 000000000000..2f0417bd8db6 --- /dev/null +++ b/internal/proto/grpc_lookup_v1/rls.pb.go @@ -0,0 +1,357 @@ +// Copyright 2020 The gRPC Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.31.0 +// protoc v4.22.0 +// source: grpc/lookup/v1/rls.proto + +package grpc_lookup_v1 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Possible reasons for making a request. +type RouteLookupRequest_Reason int32 + +const ( + RouteLookupRequest_REASON_UNKNOWN RouteLookupRequest_Reason = 0 // Unused + RouteLookupRequest_REASON_MISS RouteLookupRequest_Reason = 1 // No data available in local cache + RouteLookupRequest_REASON_STALE RouteLookupRequest_Reason = 2 // Data in local cache is stale +) + +// Enum value maps for RouteLookupRequest_Reason. +var ( + RouteLookupRequest_Reason_name = map[int32]string{ + 0: "REASON_UNKNOWN", + 1: "REASON_MISS", + 2: "REASON_STALE", + } + RouteLookupRequest_Reason_value = map[string]int32{ + "REASON_UNKNOWN": 0, + "REASON_MISS": 1, + "REASON_STALE": 2, + } +) + +func (x RouteLookupRequest_Reason) Enum() *RouteLookupRequest_Reason { + p := new(RouteLookupRequest_Reason) + *p = x + return p +} + +func (x RouteLookupRequest_Reason) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (RouteLookupRequest_Reason) Descriptor() protoreflect.EnumDescriptor { + return file_grpc_lookup_v1_rls_proto_enumTypes[0].Descriptor() +} + +func (RouteLookupRequest_Reason) Type() protoreflect.EnumType { + return &file_grpc_lookup_v1_rls_proto_enumTypes[0] +} + +func (x RouteLookupRequest_Reason) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use RouteLookupRequest_Reason.Descriptor instead. +func (RouteLookupRequest_Reason) EnumDescriptor() ([]byte, []int) { + return file_grpc_lookup_v1_rls_proto_rawDescGZIP(), []int{0, 0} +} + +type RouteLookupRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Target type allows the client to specify what kind of target format it + // would like from RLS to allow it to find the regional server, e.g. "grpc". + TargetType string `protobuf:"bytes,3,opt,name=target_type,json=targetType,proto3" json:"target_type,omitempty"` + // Reason for making this request. + Reason RouteLookupRequest_Reason `protobuf:"varint,5,opt,name=reason,proto3,enum=grpc.lookup.v1.RouteLookupRequest_Reason" json:"reason,omitempty"` + // For REASON_STALE, the header_data from the stale response, if any. + StaleHeaderData string `protobuf:"bytes,6,opt,name=stale_header_data,json=staleHeaderData,proto3" json:"stale_header_data,omitempty"` + // Map of key values extracted via key builders for the gRPC or HTTP request. + KeyMap map[string]string `protobuf:"bytes,4,rep,name=key_map,json=keyMap,proto3" json:"key_map,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` +} + +func (x *RouteLookupRequest) Reset() { + *x = RouteLookupRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_lookup_v1_rls_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RouteLookupRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RouteLookupRequest) ProtoMessage() {} + +func (x *RouteLookupRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_lookup_v1_rls_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RouteLookupRequest.ProtoReflect.Descriptor instead. +func (*RouteLookupRequest) Descriptor() ([]byte, []int) { + return file_grpc_lookup_v1_rls_proto_rawDescGZIP(), []int{0} +} + +func (x *RouteLookupRequest) GetTargetType() string { + if x != nil { + return x.TargetType + } + return "" +} + +func (x *RouteLookupRequest) GetReason() RouteLookupRequest_Reason { + if x != nil { + return x.Reason + } + return RouteLookupRequest_REASON_UNKNOWN +} + +func (x *RouteLookupRequest) GetStaleHeaderData() string { + if x != nil { + return x.StaleHeaderData + } + return "" +} + +func (x *RouteLookupRequest) GetKeyMap() map[string]string { + if x != nil { + return x.KeyMap + } + return nil +} + +type RouteLookupResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Prioritized list (best one first) of addressable entities to use + // for routing, using syntax requested by the request target_type. + // The targets will be tried in order until a healthy one is found. + Targets []string `protobuf:"bytes,3,rep,name=targets,proto3" json:"targets,omitempty"` + // Optional header value to pass along to AFE in the X-Google-RLS-Data header. + // Cached with "target" and sent with all requests that match the request key. + // Allows the RLS to pass its work product to the eventual target. + HeaderData string `protobuf:"bytes,2,opt,name=header_data,json=headerData,proto3" json:"header_data,omitempty"` +} + +func (x *RouteLookupResponse) Reset() { + *x = RouteLookupResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_lookup_v1_rls_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RouteLookupResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RouteLookupResponse) ProtoMessage() {} + +func (x *RouteLookupResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_lookup_v1_rls_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RouteLookupResponse.ProtoReflect.Descriptor instead. +func (*RouteLookupResponse) Descriptor() ([]byte, []int) { + return file_grpc_lookup_v1_rls_proto_rawDescGZIP(), []int{1} +} + +func (x *RouteLookupResponse) GetTargets() []string { + if x != nil { + return x.Targets + } + return nil +} + +func (x *RouteLookupResponse) GetHeaderData() string { + if x != nil { + return x.HeaderData + } + return "" +} + +var File_grpc_lookup_v1_rls_proto protoreflect.FileDescriptor + +var file_grpc_lookup_v1_rls_proto_rawDesc = []byte{ + 0x0a, 0x18, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2f, 0x76, 0x31, + 0x2f, 0x72, 0x6c, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0e, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2e, 0x76, 0x31, 0x22, 0x83, 0x03, 0x0a, 0x12, 0x52, + 0x6f, 0x75, 0x74, 0x65, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x54, 0x79, + 0x70, 0x65, 0x12, 0x41, 0x0a, 0x06, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x01, + 0x28, 0x0e, 0x32, 0x29, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, + 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x52, 0x06, 0x72, + 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x12, 0x2a, 0x0a, 0x11, 0x73, 0x74, 0x61, 0x6c, 0x65, 0x5f, 0x68, + 0x65, 0x61, 0x64, 0x65, 0x72, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0f, 0x73, 0x74, 0x61, 0x6c, 0x65, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x44, 0x61, 0x74, + 0x61, 0x12, 0x47, 0x0a, 0x07, 0x6b, 0x65, 0x79, 0x5f, 0x6d, 0x61, 0x70, 0x18, 0x04, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, + 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x4b, 0x65, 0x79, 0x4d, 0x61, 0x70, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x52, 0x06, 0x6b, 0x65, 0x79, 0x4d, 0x61, 0x70, 0x1a, 0x39, 0x0a, 0x0b, 0x4b, 0x65, + 0x79, 0x4d, 0x61, 0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x3f, 0x0a, 0x06, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x12, + 0x12, 0x0a, 0x0e, 0x52, 0x45, 0x41, 0x53, 0x4f, 0x4e, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, + 0x4e, 0x10, 0x00, 0x12, 0x0f, 0x0a, 0x0b, 0x52, 0x45, 0x41, 0x53, 0x4f, 0x4e, 0x5f, 0x4d, 0x49, + 0x53, 0x53, 0x10, 0x01, 0x12, 0x10, 0x0a, 0x0c, 0x52, 0x45, 0x41, 0x53, 0x4f, 0x4e, 0x5f, 0x53, + 0x54, 0x41, 0x4c, 0x45, 0x10, 0x02, 0x4a, 0x04, 0x08, 0x01, 0x10, 0x02, 0x4a, 0x04, 0x08, 0x02, + 0x10, 0x03, 0x52, 0x06, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, + 0x22, 0x5e, 0x0a, 0x13, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, + 0x74, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, + 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x5f, 0x64, 0x61, 0x74, 0x61, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x44, 0x61, + 0x74, 0x61, 0x4a, 0x04, 0x08, 0x01, 0x10, 0x02, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, + 0x32, 0x6e, 0x0a, 0x12, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x53, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x58, 0x0a, 0x0b, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x4c, + 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x12, 0x22, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x6f, 0x6f, + 0x6b, 0x75, 0x70, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x4c, 0x6f, 0x6f, 0x6b, + 0x75, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x6f, 0x75, 0x74, 0x65, + 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, + 0x42, 0x4d, 0x0a, 0x11, 0x69, 0x6f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x6f, 0x6f, 0x6b, + 0x75, 0x70, 0x2e, 0x76, 0x31, 0x42, 0x08, 0x52, 0x6c, 0x73, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, + 0x01, 0x5a, 0x2c, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, + 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, + 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x5f, 0x76, 0x31, 0x62, + 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_grpc_lookup_v1_rls_proto_rawDescOnce sync.Once + file_grpc_lookup_v1_rls_proto_rawDescData = file_grpc_lookup_v1_rls_proto_rawDesc +) + +func file_grpc_lookup_v1_rls_proto_rawDescGZIP() []byte { + file_grpc_lookup_v1_rls_proto_rawDescOnce.Do(func() { + file_grpc_lookup_v1_rls_proto_rawDescData = protoimpl.X.CompressGZIP(file_grpc_lookup_v1_rls_proto_rawDescData) + }) + return file_grpc_lookup_v1_rls_proto_rawDescData +} + +var file_grpc_lookup_v1_rls_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_grpc_lookup_v1_rls_proto_msgTypes = make([]protoimpl.MessageInfo, 3) +var file_grpc_lookup_v1_rls_proto_goTypes = []interface{}{ + (RouteLookupRequest_Reason)(0), // 0: grpc.lookup.v1.RouteLookupRequest.Reason + (*RouteLookupRequest)(nil), // 1: grpc.lookup.v1.RouteLookupRequest + (*RouteLookupResponse)(nil), // 2: grpc.lookup.v1.RouteLookupResponse + nil, // 3: grpc.lookup.v1.RouteLookupRequest.KeyMapEntry +} +var file_grpc_lookup_v1_rls_proto_depIdxs = []int32{ + 0, // 0: grpc.lookup.v1.RouteLookupRequest.reason:type_name -> grpc.lookup.v1.RouteLookupRequest.Reason + 3, // 1: grpc.lookup.v1.RouteLookupRequest.key_map:type_name -> grpc.lookup.v1.RouteLookupRequest.KeyMapEntry + 1, // 2: grpc.lookup.v1.RouteLookupService.RouteLookup:input_type -> grpc.lookup.v1.RouteLookupRequest + 2, // 3: grpc.lookup.v1.RouteLookupService.RouteLookup:output_type -> grpc.lookup.v1.RouteLookupResponse + 3, // [3:4] is the sub-list for method output_type + 2, // [2:3] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name +} + +func init() { file_grpc_lookup_v1_rls_proto_init() } +func file_grpc_lookup_v1_rls_proto_init() { + if File_grpc_lookup_v1_rls_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_grpc_lookup_v1_rls_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RouteLookupRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_lookup_v1_rls_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RouteLookupResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_grpc_lookup_v1_rls_proto_rawDesc, + NumEnums: 1, + NumMessages: 3, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_grpc_lookup_v1_rls_proto_goTypes, + DependencyIndexes: file_grpc_lookup_v1_rls_proto_depIdxs, + EnumInfos: file_grpc_lookup_v1_rls_proto_enumTypes, + MessageInfos: file_grpc_lookup_v1_rls_proto_msgTypes, + }.Build() + File_grpc_lookup_v1_rls_proto = out.File + file_grpc_lookup_v1_rls_proto_rawDesc = nil + file_grpc_lookup_v1_rls_proto_goTypes = nil + file_grpc_lookup_v1_rls_proto_depIdxs = nil +} diff --git a/internal/proto/grpc_lookup_v1/rls_config.pb.go b/internal/proto/grpc_lookup_v1/rls_config.pb.go new file mode 100644 index 000000000000..0c687504b30b --- /dev/null +++ b/internal/proto/grpc_lookup_v1/rls_config.pb.go @@ -0,0 +1,938 @@ +// Copyright 2020 The gRPC Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.31.0 +// protoc v4.22.0 +// source: grpc/lookup/v1/rls_config.proto + +package grpc_lookup_v1 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + durationpb "google.golang.org/protobuf/types/known/durationpb" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Extract a key based on a given name (e.g. header name or query parameter +// name). The name must match one of the names listed in the "name" field. If +// the "required_match" field is true, one of the specified names must be +// present for the keybuilder to match. +type NameMatcher struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // The name that will be used in the RLS key_map to refer to this value. + // If required_match is true, you may omit this field or set it to an empty + // string, in which case the matcher will require a match, but won't update + // the key_map. + Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + // Ordered list of names (headers or query parameter names) that can supply + // this value; the first one with a non-empty value is used. + Names []string `protobuf:"bytes,2,rep,name=names,proto3" json:"names,omitempty"` + // If true, make this extraction required; the key builder will not match + // if no value is found. + RequiredMatch bool `protobuf:"varint,3,opt,name=required_match,json=requiredMatch,proto3" json:"required_match,omitempty"` +} + +func (x *NameMatcher) Reset() { + *x = NameMatcher{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_lookup_v1_rls_config_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *NameMatcher) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*NameMatcher) ProtoMessage() {} + +func (x *NameMatcher) ProtoReflect() protoreflect.Message { + mi := &file_grpc_lookup_v1_rls_config_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use NameMatcher.ProtoReflect.Descriptor instead. +func (*NameMatcher) Descriptor() ([]byte, []int) { + return file_grpc_lookup_v1_rls_config_proto_rawDescGZIP(), []int{0} +} + +func (x *NameMatcher) GetKey() string { + if x != nil { + return x.Key + } + return "" +} + +func (x *NameMatcher) GetNames() []string { + if x != nil { + return x.Names + } + return nil +} + +func (x *NameMatcher) GetRequiredMatch() bool { + if x != nil { + return x.RequiredMatch + } + return false +} + +// A GrpcKeyBuilder applies to a given gRPC service, name, and headers. +type GrpcKeyBuilder struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Names []*GrpcKeyBuilder_Name `protobuf:"bytes,1,rep,name=names,proto3" json:"names,omitempty"` + ExtraKeys *GrpcKeyBuilder_ExtraKeys `protobuf:"bytes,3,opt,name=extra_keys,json=extraKeys,proto3" json:"extra_keys,omitempty"` + // Extract keys from all listed headers. + // For gRPC, it is an error to specify "required_match" on the NameMatcher + // protos. + Headers []*NameMatcher `protobuf:"bytes,2,rep,name=headers,proto3" json:"headers,omitempty"` + // You can optionally set one or more specific key/value pairs to be added to + // the key_map. This can be useful to identify which builder built the key, + // for example if you are suppressing the actual method, but need to + // separately cache and request all the matched methods. + ConstantKeys map[string]string `protobuf:"bytes,4,rep,name=constant_keys,json=constantKeys,proto3" json:"constant_keys,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` +} + +func (x *GrpcKeyBuilder) Reset() { + *x = GrpcKeyBuilder{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_lookup_v1_rls_config_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GrpcKeyBuilder) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GrpcKeyBuilder) ProtoMessage() {} + +func (x *GrpcKeyBuilder) ProtoReflect() protoreflect.Message { + mi := &file_grpc_lookup_v1_rls_config_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GrpcKeyBuilder.ProtoReflect.Descriptor instead. +func (*GrpcKeyBuilder) Descriptor() ([]byte, []int) { + return file_grpc_lookup_v1_rls_config_proto_rawDescGZIP(), []int{1} +} + +func (x *GrpcKeyBuilder) GetNames() []*GrpcKeyBuilder_Name { + if x != nil { + return x.Names + } + return nil +} + +func (x *GrpcKeyBuilder) GetExtraKeys() *GrpcKeyBuilder_ExtraKeys { + if x != nil { + return x.ExtraKeys + } + return nil +} + +func (x *GrpcKeyBuilder) GetHeaders() []*NameMatcher { + if x != nil { + return x.Headers + } + return nil +} + +func (x *GrpcKeyBuilder) GetConstantKeys() map[string]string { + if x != nil { + return x.ConstantKeys + } + return nil +} + +// An HttpKeyBuilder applies to a given HTTP URL and headers. +// +// Path and host patterns use the matching syntax from gRPC transcoding to +// extract named key/value pairs from the path and host components of the URL: +// https://github.com/googleapis/googleapis/blob/master/google/api/http.proto +// +// It is invalid to specify the same key name in multiple places in a pattern. +// +// For a service where the project id can be expressed either as a subdomain or +// in the path, separate HttpKeyBuilders must be used: +// +// host_pattern: 'example.com' path_pattern: '/{id}/{object}/**' +// host_pattern: '{id}.example.com' path_pattern: '/{object}/**' +// +// If the host is exactly 'example.com', the first path segment will be used as +// the id and the second segment as the object. If the host has a subdomain, the +// subdomain will be used as the id and the first segment as the object. If +// neither pattern matches, no keys will be extracted. +type HttpKeyBuilder struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // host_pattern is an ordered list of host template patterns for the desired + // value. If any host_pattern values are specified, then at least one must + // match, and the last one wins and sets any specified variables. A host + // consists of labels separated by dots. Each label is matched against the + // label in the pattern as follows: + // - "*": Matches any single label. + // - "**": Matches zero or more labels (first or last part of host only). + // - "{=...}": One or more label capture, where "..." can be any + // template that does not include a capture. + // - "{}": A single label capture. Identical to {=*}. + // + // Examples: + // - "example.com": Only applies to the exact host example.com. + // - "*.example.com": Matches subdomains of example.com. + // - "**.example.com": matches example.com, and all levels of subdomains. + // - "{project}.example.com": Extracts the third level subdomain. + // - "{project=**}.example.com": Extracts the third level+ subdomains. + // - "{project=**}": Extracts the entire host. + HostPatterns []string `protobuf:"bytes,1,rep,name=host_patterns,json=hostPatterns,proto3" json:"host_patterns,omitempty"` + // path_pattern is an ordered list of path template patterns for the desired + // value. If any path_pattern values are specified, then at least one must + // match, and the last one wins and sets any specified variables. A path + // consists of segments separated by slashes. Each segment is matched against + // the segment in the pattern as follows: + // - "*": Matches any single segment. + // - "**": Matches zero or more segments (first or last part of path only). + // - "{=...}": One or more segment capture, where "..." can be any + // template that does not include a capture. + // - "{}": A single segment capture. Identical to {=*}. + // + // A custom method may also be specified by appending ":" and the custom + // method name or "*" to indicate any custom method (including no custom + // method). For example, "/*/projects/{project_id}/**:*" extracts + // `{project_id}` for any version, resource and custom method that includes + // it. By default, any custom method will be matched. + // + // Examples: + // - "/v1/{name=messages/*}": extracts a name like "messages/12345". + // - "/v1/messages/{message_id}": extracts a message_id like "12345". + // - "/v1/users/{user_id}/messages/{message_id}": extracts two key values. + PathPatterns []string `protobuf:"bytes,2,rep,name=path_patterns,json=pathPatterns,proto3" json:"path_patterns,omitempty"` + // List of query parameter names to try to match. + // For example: ["parent", "name", "resource.name"] + // We extract all the specified query_parameters (case-sensitively). If any + // are marked as "required_match" and are not present, this keybuilder fails + // to match. If a given parameter appears multiple times (?foo=a&foo=b) we + // will report it as a comma-separated string (foo=a,b). + QueryParameters []*NameMatcher `protobuf:"bytes,3,rep,name=query_parameters,json=queryParameters,proto3" json:"query_parameters,omitempty"` + // List of headers to try to match. + // We extract all the specified header values (case-insensitively). If any + // are marked as "required_match" and are not present, this keybuilder fails + // to match. If a given header appears multiple times in the request we will + // report it as a comma-separated string, in standard HTTP fashion. + Headers []*NameMatcher `protobuf:"bytes,4,rep,name=headers,proto3" json:"headers,omitempty"` + // You can optionally set one or more specific key/value pairs to be added to + // the key_map. This can be useful to identify which builder built the key, + // for example if you are suppressing a lot of information from the URL, but + // need to separately cache and request URLs with that content. + ConstantKeys map[string]string `protobuf:"bytes,5,rep,name=constant_keys,json=constantKeys,proto3" json:"constant_keys,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` +} + +func (x *HttpKeyBuilder) Reset() { + *x = HttpKeyBuilder{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_lookup_v1_rls_config_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *HttpKeyBuilder) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HttpKeyBuilder) ProtoMessage() {} + +func (x *HttpKeyBuilder) ProtoReflect() protoreflect.Message { + mi := &file_grpc_lookup_v1_rls_config_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HttpKeyBuilder.ProtoReflect.Descriptor instead. +func (*HttpKeyBuilder) Descriptor() ([]byte, []int) { + return file_grpc_lookup_v1_rls_config_proto_rawDescGZIP(), []int{2} +} + +func (x *HttpKeyBuilder) GetHostPatterns() []string { + if x != nil { + return x.HostPatterns + } + return nil +} + +func (x *HttpKeyBuilder) GetPathPatterns() []string { + if x != nil { + return x.PathPatterns + } + return nil +} + +func (x *HttpKeyBuilder) GetQueryParameters() []*NameMatcher { + if x != nil { + return x.QueryParameters + } + return nil +} + +func (x *HttpKeyBuilder) GetHeaders() []*NameMatcher { + if x != nil { + return x.Headers + } + return nil +} + +func (x *HttpKeyBuilder) GetConstantKeys() map[string]string { + if x != nil { + return x.ConstantKeys + } + return nil +} + +type RouteLookupConfig struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Ordered specifications for constructing keys for HTTP requests. Last + // match wins. If no HttpKeyBuilder matches, an empty key_map will be sent to + // the lookup service; it should likely reply with a global default route + // and raise an alert. + HttpKeybuilders []*HttpKeyBuilder `protobuf:"bytes,1,rep,name=http_keybuilders,json=httpKeybuilders,proto3" json:"http_keybuilders,omitempty"` + // Unordered specifications for constructing keys for gRPC requests. All + // GrpcKeyBuilders on this list must have unique "name" fields so that the + // client is free to prebuild a hash map keyed by name. If no GrpcKeyBuilder + // matches, an empty key_map will be sent to the lookup service; it should + // likely reply with a global default route and raise an alert. + GrpcKeybuilders []*GrpcKeyBuilder `protobuf:"bytes,2,rep,name=grpc_keybuilders,json=grpcKeybuilders,proto3" json:"grpc_keybuilders,omitempty"` + // The name of the lookup service as a gRPC URI. Typically, this will be + // a subdomain of the target, such as "lookup.datastore.googleapis.com". + LookupService string `protobuf:"bytes,3,opt,name=lookup_service,json=lookupService,proto3" json:"lookup_service,omitempty"` + // Configure a timeout value for lookup service requests. + // Defaults to 10 seconds if not specified. + LookupServiceTimeout *durationpb.Duration `protobuf:"bytes,4,opt,name=lookup_service_timeout,json=lookupServiceTimeout,proto3" json:"lookup_service_timeout,omitempty"` + // How long are responses valid for (like HTTP Cache-Control). + // If omitted or zero, the longest valid cache time is used. + // This value is clamped to 5 minutes to avoid unflushable bad responses. + MaxAge *durationpb.Duration `protobuf:"bytes,5,opt,name=max_age,json=maxAge,proto3" json:"max_age,omitempty"` + // After a response has been in the client cache for this amount of time + // and is re-requested, start an asynchronous RPC to re-validate it. + // This value should be less than max_age by at least the length of a + // typical RTT to the Route Lookup Service to fully mask the RTT latency. + // If omitted, keys are only re-requested after they have expired. + StaleAge *durationpb.Duration `protobuf:"bytes,6,opt,name=stale_age,json=staleAge,proto3" json:"stale_age,omitempty"` + // Rough indicator of amount of memory to use for the client cache. Some of + // the data structure overhead is not accounted for, so actual memory consumed + // will be somewhat greater than this value. If this field is omitted or set + // to zero, a client default will be used. The value may be capped to a lower + // amount based on client configuration. + CacheSizeBytes int64 `protobuf:"varint,7,opt,name=cache_size_bytes,json=cacheSizeBytes,proto3" json:"cache_size_bytes,omitempty"` + // This is a list of all the possible targets that can be returned by the + // lookup service. If a target not on this list is returned, it will be + // treated the same as an unhealthy target. + ValidTargets []string `protobuf:"bytes,8,rep,name=valid_targets,json=validTargets,proto3" json:"valid_targets,omitempty"` + // This value provides a default target to use if needed. If set, it will be + // used if RLS returns an error, times out, or returns an invalid response. + // Note that requests can be routed only to a subdomain of the original + // target, e.g. "us_east_1.cloudbigtable.googleapis.com". + DefaultTarget string `protobuf:"bytes,9,opt,name=default_target,json=defaultTarget,proto3" json:"default_target,omitempty"` +} + +func (x *RouteLookupConfig) Reset() { + *x = RouteLookupConfig{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_lookup_v1_rls_config_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RouteLookupConfig) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RouteLookupConfig) ProtoMessage() {} + +func (x *RouteLookupConfig) ProtoReflect() protoreflect.Message { + mi := &file_grpc_lookup_v1_rls_config_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RouteLookupConfig.ProtoReflect.Descriptor instead. +func (*RouteLookupConfig) Descriptor() ([]byte, []int) { + return file_grpc_lookup_v1_rls_config_proto_rawDescGZIP(), []int{3} +} + +func (x *RouteLookupConfig) GetHttpKeybuilders() []*HttpKeyBuilder { + if x != nil { + return x.HttpKeybuilders + } + return nil +} + +func (x *RouteLookupConfig) GetGrpcKeybuilders() []*GrpcKeyBuilder { + if x != nil { + return x.GrpcKeybuilders + } + return nil +} + +func (x *RouteLookupConfig) GetLookupService() string { + if x != nil { + return x.LookupService + } + return "" +} + +func (x *RouteLookupConfig) GetLookupServiceTimeout() *durationpb.Duration { + if x != nil { + return x.LookupServiceTimeout + } + return nil +} + +func (x *RouteLookupConfig) GetMaxAge() *durationpb.Duration { + if x != nil { + return x.MaxAge + } + return nil +} + +func (x *RouteLookupConfig) GetStaleAge() *durationpb.Duration { + if x != nil { + return x.StaleAge + } + return nil +} + +func (x *RouteLookupConfig) GetCacheSizeBytes() int64 { + if x != nil { + return x.CacheSizeBytes + } + return 0 +} + +func (x *RouteLookupConfig) GetValidTargets() []string { + if x != nil { + return x.ValidTargets + } + return nil +} + +func (x *RouteLookupConfig) GetDefaultTarget() string { + if x != nil { + return x.DefaultTarget + } + return "" +} + +// RouteLookupClusterSpecifier is used in xDS to represent a cluster specifier +// plugin for RLS. +type RouteLookupClusterSpecifier struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // The RLS config for this cluster specifier plugin instance. + RouteLookupConfig *RouteLookupConfig `protobuf:"bytes,1,opt,name=route_lookup_config,json=routeLookupConfig,proto3" json:"route_lookup_config,omitempty"` +} + +func (x *RouteLookupClusterSpecifier) Reset() { + *x = RouteLookupClusterSpecifier{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_lookup_v1_rls_config_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RouteLookupClusterSpecifier) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RouteLookupClusterSpecifier) ProtoMessage() {} + +func (x *RouteLookupClusterSpecifier) ProtoReflect() protoreflect.Message { + mi := &file_grpc_lookup_v1_rls_config_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RouteLookupClusterSpecifier.ProtoReflect.Descriptor instead. +func (*RouteLookupClusterSpecifier) Descriptor() ([]byte, []int) { + return file_grpc_lookup_v1_rls_config_proto_rawDescGZIP(), []int{4} +} + +func (x *RouteLookupClusterSpecifier) GetRouteLookupConfig() *RouteLookupConfig { + if x != nil { + return x.RouteLookupConfig + } + return nil +} + +// To match, one of the given Name fields must match; the service and method +// fields are specified as fixed strings. The service name is required and +// includes the proto package name. The method name may be omitted, in +// which case any method on the given service is matched. +type GrpcKeyBuilder_Name struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Service string `protobuf:"bytes,1,opt,name=service,proto3" json:"service,omitempty"` + Method string `protobuf:"bytes,2,opt,name=method,proto3" json:"method,omitempty"` +} + +func (x *GrpcKeyBuilder_Name) Reset() { + *x = GrpcKeyBuilder_Name{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_lookup_v1_rls_config_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GrpcKeyBuilder_Name) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GrpcKeyBuilder_Name) ProtoMessage() {} + +func (x *GrpcKeyBuilder_Name) ProtoReflect() protoreflect.Message { + mi := &file_grpc_lookup_v1_rls_config_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GrpcKeyBuilder_Name.ProtoReflect.Descriptor instead. +func (*GrpcKeyBuilder_Name) Descriptor() ([]byte, []int) { + return file_grpc_lookup_v1_rls_config_proto_rawDescGZIP(), []int{1, 0} +} + +func (x *GrpcKeyBuilder_Name) GetService() string { + if x != nil { + return x.Service + } + return "" +} + +func (x *GrpcKeyBuilder_Name) GetMethod() string { + if x != nil { + return x.Method + } + return "" +} + +// If you wish to include the host, service, or method names as keys in the +// generated RouteLookupRequest, specify key names to use in the extra_keys +// submessage. If a key name is empty, no key will be set for that value. +// If this submessage is specified, the normal host/path fields will be left +// unset in the RouteLookupRequest. We are deprecating host/path in the +// RouteLookupRequest, so services should migrate to the ExtraKeys approach. +type GrpcKeyBuilder_ExtraKeys struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Host string `protobuf:"bytes,1,opt,name=host,proto3" json:"host,omitempty"` + Service string `protobuf:"bytes,2,opt,name=service,proto3" json:"service,omitempty"` + Method string `protobuf:"bytes,3,opt,name=method,proto3" json:"method,omitempty"` +} + +func (x *GrpcKeyBuilder_ExtraKeys) Reset() { + *x = GrpcKeyBuilder_ExtraKeys{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_lookup_v1_rls_config_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GrpcKeyBuilder_ExtraKeys) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GrpcKeyBuilder_ExtraKeys) ProtoMessage() {} + +func (x *GrpcKeyBuilder_ExtraKeys) ProtoReflect() protoreflect.Message { + mi := &file_grpc_lookup_v1_rls_config_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GrpcKeyBuilder_ExtraKeys.ProtoReflect.Descriptor instead. +func (*GrpcKeyBuilder_ExtraKeys) Descriptor() ([]byte, []int) { + return file_grpc_lookup_v1_rls_config_proto_rawDescGZIP(), []int{1, 1} +} + +func (x *GrpcKeyBuilder_ExtraKeys) GetHost() string { + if x != nil { + return x.Host + } + return "" +} + +func (x *GrpcKeyBuilder_ExtraKeys) GetService() string { + if x != nil { + return x.Service + } + return "" +} + +func (x *GrpcKeyBuilder_ExtraKeys) GetMethod() string { + if x != nil { + return x.Method + } + return "" +} + +var File_grpc_lookup_v1_rls_config_proto protoreflect.FileDescriptor + +var file_grpc_lookup_v1_rls_config_proto_rawDesc = []byte{ + 0x0a, 0x1f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2f, 0x76, 0x31, + 0x2f, 0x72, 0x6c, 0x73, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x12, 0x0e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2e, 0x76, + 0x31, 0x1a, 0x1e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2f, 0x64, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x22, 0x5c, 0x0a, 0x0b, 0x4e, 0x61, 0x6d, 0x65, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x72, + 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, + 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, + 0x09, 0x52, 0x05, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x72, 0x65, 0x71, 0x75, + 0x69, 0x72, 0x65, 0x64, 0x5f, 0x6d, 0x61, 0x74, 0x63, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x0d, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x22, + 0xf0, 0x03, 0x0a, 0x0e, 0x47, 0x72, 0x70, 0x63, 0x4b, 0x65, 0x79, 0x42, 0x75, 0x69, 0x6c, 0x64, + 0x65, 0x72, 0x12, 0x39, 0x0a, 0x05, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x23, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2e, + 0x76, 0x31, 0x2e, 0x47, 0x72, 0x70, 0x63, 0x4b, 0x65, 0x79, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x65, + 0x72, 0x2e, 0x4e, 0x61, 0x6d, 0x65, 0x52, 0x05, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x12, 0x47, 0x0a, + 0x0a, 0x65, 0x78, 0x74, 0x72, 0x61, 0x5f, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x28, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2e, + 0x76, 0x31, 0x2e, 0x47, 0x72, 0x70, 0x63, 0x4b, 0x65, 0x79, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x65, + 0x72, 0x2e, 0x45, 0x78, 0x74, 0x72, 0x61, 0x4b, 0x65, 0x79, 0x73, 0x52, 0x09, 0x65, 0x78, 0x74, + 0x72, 0x61, 0x4b, 0x65, 0x79, 0x73, 0x12, 0x35, 0x0a, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, + 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, + 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x61, 0x6d, 0x65, 0x4d, 0x61, 0x74, + 0x63, 0x68, 0x65, 0x72, 0x52, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x12, 0x55, 0x0a, + 0x0d, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x04, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x6f, 0x6f, 0x6b, + 0x75, 0x70, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x72, 0x70, 0x63, 0x4b, 0x65, 0x79, 0x42, 0x75, 0x69, + 0x6c, 0x64, 0x65, 0x72, 0x2e, 0x43, 0x6f, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x74, 0x4b, 0x65, 0x79, + 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0c, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x74, + 0x4b, 0x65, 0x79, 0x73, 0x1a, 0x38, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, + 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x73, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x1a, 0x51, + 0x0a, 0x09, 0x45, 0x78, 0x74, 0x72, 0x61, 0x4b, 0x65, 0x79, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x68, + 0x6f, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x12, + 0x18, 0x0a, 0x07, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x07, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x6d, 0x65, 0x74, + 0x68, 0x6f, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6d, 0x65, 0x74, 0x68, 0x6f, + 0x64, 0x1a, 0x3f, 0x0a, 0x11, 0x43, 0x6f, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x74, 0x4b, 0x65, 0x79, + 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, + 0x38, 0x01, 0x22, 0xf1, 0x02, 0x0a, 0x0e, 0x48, 0x74, 0x74, 0x70, 0x4b, 0x65, 0x79, 0x42, 0x75, + 0x69, 0x6c, 0x64, 0x65, 0x72, 0x12, 0x23, 0x0a, 0x0d, 0x68, 0x6f, 0x73, 0x74, 0x5f, 0x70, 0x61, + 0x74, 0x74, 0x65, 0x72, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x68, 0x6f, + 0x73, 0x74, 0x50, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x70, 0x61, + 0x74, 0x68, 0x5f, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, + 0x09, 0x52, 0x0c, 0x70, 0x61, 0x74, 0x68, 0x50, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x73, 0x12, + 0x46, 0x0a, 0x10, 0x71, 0x75, 0x65, 0x72, 0x79, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, + 0x65, 0x72, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x61, 0x6d, 0x65, 0x4d, + 0x61, 0x74, 0x63, 0x68, 0x65, 0x72, 0x52, 0x0f, 0x71, 0x75, 0x65, 0x72, 0x79, 0x50, 0x61, 0x72, + 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x12, 0x35, 0x0a, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65, + 0x72, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, + 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x61, 0x6d, 0x65, 0x4d, 0x61, + 0x74, 0x63, 0x68, 0x65, 0x72, 0x52, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x12, 0x55, + 0x0a, 0x0d, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x73, 0x18, + 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x6f, 0x6f, + 0x6b, 0x75, 0x70, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x74, 0x74, 0x70, 0x4b, 0x65, 0x79, 0x42, 0x75, + 0x69, 0x6c, 0x64, 0x65, 0x72, 0x2e, 0x43, 0x6f, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x74, 0x4b, 0x65, + 0x79, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0c, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x61, 0x6e, + 0x74, 0x4b, 0x65, 0x79, 0x73, 0x1a, 0x3f, 0x0a, 0x11, 0x43, 0x6f, 0x6e, 0x73, 0x74, 0x61, 0x6e, + 0x74, 0x4b, 0x65, 0x79, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xa6, 0x04, 0x0a, 0x11, 0x52, 0x6f, 0x75, 0x74, 0x65, + 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x49, 0x0a, 0x10, + 0x68, 0x74, 0x74, 0x70, 0x5f, 0x6b, 0x65, 0x79, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x65, 0x72, 0x73, + 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x6f, + 0x6f, 0x6b, 0x75, 0x70, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x74, 0x74, 0x70, 0x4b, 0x65, 0x79, 0x42, + 0x75, 0x69, 0x6c, 0x64, 0x65, 0x72, 0x52, 0x0f, 0x68, 0x74, 0x74, 0x70, 0x4b, 0x65, 0x79, 0x62, + 0x75, 0x69, 0x6c, 0x64, 0x65, 0x72, 0x73, 0x12, 0x49, 0x0a, 0x10, 0x67, 0x72, 0x70, 0x63, 0x5f, + 0x6b, 0x65, 0x79, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x65, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x1e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2e, + 0x76, 0x31, 0x2e, 0x47, 0x72, 0x70, 0x63, 0x4b, 0x65, 0x79, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x65, + 0x72, 0x52, 0x0f, 0x67, 0x72, 0x70, 0x63, 0x4b, 0x65, 0x79, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x65, + 0x72, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x5f, 0x73, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x6c, 0x6f, 0x6f, 0x6b, + 0x75, 0x70, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x4f, 0x0a, 0x16, 0x6c, 0x6f, 0x6f, + 0x6b, 0x75, 0x70, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x74, 0x69, 0x6d, 0x65, + 0x6f, 0x75, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x14, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x53, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x12, 0x32, 0x0a, 0x07, 0x6d, 0x61, + 0x78, 0x5f, 0x61, 0x67, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x06, 0x6d, 0x61, 0x78, 0x41, 0x67, 0x65, 0x12, 0x36, + 0x0a, 0x09, 0x73, 0x74, 0x61, 0x6c, 0x65, 0x5f, 0x61, 0x67, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x08, 0x73, 0x74, + 0x61, 0x6c, 0x65, 0x41, 0x67, 0x65, 0x12, 0x28, 0x0a, 0x10, 0x63, 0x61, 0x63, 0x68, 0x65, 0x5f, + 0x73, 0x69, 0x7a, 0x65, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, + 0x52, 0x0e, 0x63, 0x61, 0x63, 0x68, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x42, 0x79, 0x74, 0x65, 0x73, + 0x12, 0x23, 0x0a, 0x0d, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x5f, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, + 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x54, 0x61, + 0x72, 0x67, 0x65, 0x74, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, + 0x5f, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x64, + 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x4a, 0x04, 0x08, 0x0a, + 0x10, 0x0b, 0x52, 0x1b, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x70, 0x72, 0x6f, 0x63, + 0x65, 0x73, 0x73, 0x69, 0x6e, 0x67, 0x5f, 0x73, 0x74, 0x72, 0x61, 0x74, 0x65, 0x67, 0x79, 0x22, + 0x70, 0x0a, 0x1b, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x43, 0x6c, + 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x70, 0x65, 0x63, 0x69, 0x66, 0x69, 0x65, 0x72, 0x12, 0x51, + 0x0a, 0x13, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x5f, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x5f, 0x63, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x67, 0x72, + 0x70, 0x63, 0x2e, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x6f, 0x75, + 0x74, 0x65, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x11, + 0x72, 0x6f, 0x75, 0x74, 0x65, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x42, 0x53, 0x0a, 0x11, 0x69, 0x6f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x6c, 0x6f, 0x6f, + 0x6b, 0x75, 0x70, 0x2e, 0x76, 0x31, 0x42, 0x0e, 0x52, 0x6c, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x2c, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, + 0x2e, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, + 0x2f, 0x6c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x6c, 0x6f, 0x6f, + 0x6b, 0x75, 0x70, 0x5f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_grpc_lookup_v1_rls_config_proto_rawDescOnce sync.Once + file_grpc_lookup_v1_rls_config_proto_rawDescData = file_grpc_lookup_v1_rls_config_proto_rawDesc +) + +func file_grpc_lookup_v1_rls_config_proto_rawDescGZIP() []byte { + file_grpc_lookup_v1_rls_config_proto_rawDescOnce.Do(func() { + file_grpc_lookup_v1_rls_config_proto_rawDescData = protoimpl.X.CompressGZIP(file_grpc_lookup_v1_rls_config_proto_rawDescData) + }) + return file_grpc_lookup_v1_rls_config_proto_rawDescData +} + +var file_grpc_lookup_v1_rls_config_proto_msgTypes = make([]protoimpl.MessageInfo, 9) +var file_grpc_lookup_v1_rls_config_proto_goTypes = []interface{}{ + (*NameMatcher)(nil), // 0: grpc.lookup.v1.NameMatcher + (*GrpcKeyBuilder)(nil), // 1: grpc.lookup.v1.GrpcKeyBuilder + (*HttpKeyBuilder)(nil), // 2: grpc.lookup.v1.HttpKeyBuilder + (*RouteLookupConfig)(nil), // 3: grpc.lookup.v1.RouteLookupConfig + (*RouteLookupClusterSpecifier)(nil), // 4: grpc.lookup.v1.RouteLookupClusterSpecifier + (*GrpcKeyBuilder_Name)(nil), // 5: grpc.lookup.v1.GrpcKeyBuilder.Name + (*GrpcKeyBuilder_ExtraKeys)(nil), // 6: grpc.lookup.v1.GrpcKeyBuilder.ExtraKeys + nil, // 7: grpc.lookup.v1.GrpcKeyBuilder.ConstantKeysEntry + nil, // 8: grpc.lookup.v1.HttpKeyBuilder.ConstantKeysEntry + (*durationpb.Duration)(nil), // 9: google.protobuf.Duration +} +var file_grpc_lookup_v1_rls_config_proto_depIdxs = []int32{ + 5, // 0: grpc.lookup.v1.GrpcKeyBuilder.names:type_name -> grpc.lookup.v1.GrpcKeyBuilder.Name + 6, // 1: grpc.lookup.v1.GrpcKeyBuilder.extra_keys:type_name -> grpc.lookup.v1.GrpcKeyBuilder.ExtraKeys + 0, // 2: grpc.lookup.v1.GrpcKeyBuilder.headers:type_name -> grpc.lookup.v1.NameMatcher + 7, // 3: grpc.lookup.v1.GrpcKeyBuilder.constant_keys:type_name -> grpc.lookup.v1.GrpcKeyBuilder.ConstantKeysEntry + 0, // 4: grpc.lookup.v1.HttpKeyBuilder.query_parameters:type_name -> grpc.lookup.v1.NameMatcher + 0, // 5: grpc.lookup.v1.HttpKeyBuilder.headers:type_name -> grpc.lookup.v1.NameMatcher + 8, // 6: grpc.lookup.v1.HttpKeyBuilder.constant_keys:type_name -> grpc.lookup.v1.HttpKeyBuilder.ConstantKeysEntry + 2, // 7: grpc.lookup.v1.RouteLookupConfig.http_keybuilders:type_name -> grpc.lookup.v1.HttpKeyBuilder + 1, // 8: grpc.lookup.v1.RouteLookupConfig.grpc_keybuilders:type_name -> grpc.lookup.v1.GrpcKeyBuilder + 9, // 9: grpc.lookup.v1.RouteLookupConfig.lookup_service_timeout:type_name -> google.protobuf.Duration + 9, // 10: grpc.lookup.v1.RouteLookupConfig.max_age:type_name -> google.protobuf.Duration + 9, // 11: grpc.lookup.v1.RouteLookupConfig.stale_age:type_name -> google.protobuf.Duration + 3, // 12: grpc.lookup.v1.RouteLookupClusterSpecifier.route_lookup_config:type_name -> grpc.lookup.v1.RouteLookupConfig + 13, // [13:13] is the sub-list for method output_type + 13, // [13:13] is the sub-list for method input_type + 13, // [13:13] is the sub-list for extension type_name + 13, // [13:13] is the sub-list for extension extendee + 0, // [0:13] is the sub-list for field type_name +} + +func init() { file_grpc_lookup_v1_rls_config_proto_init() } +func file_grpc_lookup_v1_rls_config_proto_init() { + if File_grpc_lookup_v1_rls_config_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_grpc_lookup_v1_rls_config_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*NameMatcher); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_lookup_v1_rls_config_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GrpcKeyBuilder); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_lookup_v1_rls_config_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*HttpKeyBuilder); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_lookup_v1_rls_config_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RouteLookupConfig); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_lookup_v1_rls_config_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RouteLookupClusterSpecifier); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_lookup_v1_rls_config_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GrpcKeyBuilder_Name); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_lookup_v1_rls_config_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GrpcKeyBuilder_ExtraKeys); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_grpc_lookup_v1_rls_config_proto_rawDesc, + NumEnums: 0, + NumMessages: 9, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_grpc_lookup_v1_rls_config_proto_goTypes, + DependencyIndexes: file_grpc_lookup_v1_rls_config_proto_depIdxs, + MessageInfos: file_grpc_lookup_v1_rls_config_proto_msgTypes, + }.Build() + File_grpc_lookup_v1_rls_config_proto = out.File + file_grpc_lookup_v1_rls_config_proto_rawDesc = nil + file_grpc_lookup_v1_rls_config_proto_goTypes = nil + file_grpc_lookup_v1_rls_config_proto_depIdxs = nil +} diff --git a/internal/proto/grpc_lookup_v1/rls_grpc.pb.go b/internal/proto/grpc_lookup_v1/rls_grpc.pb.go new file mode 100644 index 000000000000..2435fbc9a9b9 --- /dev/null +++ b/internal/proto/grpc_lookup_v1/rls_grpc.pb.go @@ -0,0 +1,125 @@ +// Copyright 2020 The gRPC Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.3.0 +// - protoc v4.22.0 +// source: grpc/lookup/v1/rls.proto + +package grpc_lookup_v1 + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 + +const ( + RouteLookupService_RouteLookup_FullMethodName = "/grpc.lookup.v1.RouteLookupService/RouteLookup" +) + +// RouteLookupServiceClient is the client API for RouteLookupService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type RouteLookupServiceClient interface { + // Lookup returns a target for a single key. + RouteLookup(ctx context.Context, in *RouteLookupRequest, opts ...grpc.CallOption) (*RouteLookupResponse, error) +} + +type routeLookupServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewRouteLookupServiceClient(cc grpc.ClientConnInterface) RouteLookupServiceClient { + return &routeLookupServiceClient{cc} +} + +func (c *routeLookupServiceClient) RouteLookup(ctx context.Context, in *RouteLookupRequest, opts ...grpc.CallOption) (*RouteLookupResponse, error) { + out := new(RouteLookupResponse) + err := c.cc.Invoke(ctx, RouteLookupService_RouteLookup_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// RouteLookupServiceServer is the server API for RouteLookupService service. +// All implementations must embed UnimplementedRouteLookupServiceServer +// for forward compatibility +type RouteLookupServiceServer interface { + // Lookup returns a target for a single key. + RouteLookup(context.Context, *RouteLookupRequest) (*RouteLookupResponse, error) + mustEmbedUnimplementedRouteLookupServiceServer() +} + +// UnimplementedRouteLookupServiceServer must be embedded to have forward compatible implementations. +type UnimplementedRouteLookupServiceServer struct { +} + +func (UnimplementedRouteLookupServiceServer) RouteLookup(context.Context, *RouteLookupRequest) (*RouteLookupResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method RouteLookup not implemented") +} +func (UnimplementedRouteLookupServiceServer) mustEmbedUnimplementedRouteLookupServiceServer() {} + +// UnsafeRouteLookupServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to RouteLookupServiceServer will +// result in compilation errors. +type UnsafeRouteLookupServiceServer interface { + mustEmbedUnimplementedRouteLookupServiceServer() +} + +func RegisterRouteLookupServiceServer(s grpc.ServiceRegistrar, srv RouteLookupServiceServer) { + s.RegisterService(&RouteLookupService_ServiceDesc, srv) +} + +func _RouteLookupService_RouteLookup_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RouteLookupRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(RouteLookupServiceServer).RouteLookup(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: RouteLookupService_RouteLookup_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(RouteLookupServiceServer).RouteLookup(ctx, req.(*RouteLookupRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// RouteLookupService_ServiceDesc is the grpc.ServiceDesc for RouteLookupService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var RouteLookupService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.lookup.v1.RouteLookupService", + HandlerType: (*RouteLookupServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "RouteLookup", + Handler: _RouteLookupService_RouteLookup_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "grpc/lookup/v1/rls.proto", +} diff --git a/internal/proto/grpc_service_config/example_test.go b/internal/proto/grpc_service_config/example_test.go deleted file mode 100644 index b707d8b05e39..000000000000 --- a/internal/proto/grpc_service_config/example_test.go +++ /dev/null @@ -1,66 +0,0 @@ -/* - * - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package grpc_service_config_test - -import ( - "testing" - - "github.com/golang/protobuf/jsonpb" - wrapperspb "github.com/golang/protobuf/ptypes/wrappers" - "google.golang.org/grpc/internal/grpctest" - scpb "google.golang.org/grpc/internal/proto/grpc_service_config" -) - -type s struct { - grpctest.Tester -} - -func Test(t *testing.T) { - grpctest.RunSubTests(t, s{}) -} - -// TestXdsConfigMarshalToJSON is an example to print json format of xds_config. -func (s) TestXdsConfigMarshalToJSON(t *testing.T) { - c := &scpb.XdsConfig{ - ChildPolicy: []*scpb.LoadBalancingConfig{ - {Policy: &scpb.LoadBalancingConfig_Grpclb{ - Grpclb: &scpb.GrpcLbConfig{}, - }}, - {Policy: &scpb.LoadBalancingConfig_RoundRobin{ - RoundRobin: &scpb.RoundRobinConfig{}, - }}, - }, - FallbackPolicy: []*scpb.LoadBalancingConfig{ - {Policy: &scpb.LoadBalancingConfig_Grpclb{ - Grpclb: &scpb.GrpcLbConfig{}, - }}, - {Policy: &scpb.LoadBalancingConfig_PickFirst{ - PickFirst: &scpb.PickFirstConfig{}, - }}, - }, - EdsServiceName: "eds.service.name", - LrsLoadReportingServerName: &wrapperspb.StringValue{ - Value: "lrs.server.name", - }, - } - j, err := (&jsonpb.Marshaler{}).MarshalToString(c) - if err != nil { - t.Fatalf("failed to marshal proto to json: %v", err) - } - t.Logf(j) -} diff --git a/internal/proto/grpc_service_config/service_config.pb.go b/internal/proto/grpc_service_config/service_config.pb.go deleted file mode 100644 index c0436b62a64b..000000000000 --- a/internal/proto/grpc_service_config/service_config.pb.go +++ /dev/null @@ -1,1668 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. -// source: grpc/service_config/service_config.proto - -package grpc_service_config - -import ( - fmt "fmt" - proto "github.com/golang/protobuf/proto" - duration "github.com/golang/protobuf/ptypes/duration" - wrappers "github.com/golang/protobuf/ptypes/wrappers" - code "google.golang.org/genproto/googleapis/rpc/code" - math "math" -) - -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package - -// Load balancing policy. -// -// Note that load_balancing_policy is deprecated in favor of -// load_balancing_config; the former will be used only if the latter -// is unset. -// -// If no LB policy is configured here, then the default is pick_first. -// If the policy name is set via the client API, that value overrides -// the value specified here. -// -// If the deprecated load_balancing_policy field is used, note that if the -// resolver returns at least one balancer address (as opposed to backend -// addresses), gRPC will use grpclb (see -// https://github.com/grpc/grpc/blob/master/doc/load-balancing.md), -// regardless of what policy is configured here. However, if the resolver -// returns at least one backend address in addition to the balancer -// address(es), the client may fall back to the requested policy if it -// is unable to reach any of the grpclb load balancers. -type ServiceConfig_LoadBalancingPolicy int32 - -const ( - ServiceConfig_UNSPECIFIED ServiceConfig_LoadBalancingPolicy = 0 - ServiceConfig_ROUND_ROBIN ServiceConfig_LoadBalancingPolicy = 1 -) - -var ServiceConfig_LoadBalancingPolicy_name = map[int32]string{ - 0: "UNSPECIFIED", - 1: "ROUND_ROBIN", -} - -var ServiceConfig_LoadBalancingPolicy_value = map[string]int32{ - "UNSPECIFIED": 0, - "ROUND_ROBIN": 1, -} - -func (x ServiceConfig_LoadBalancingPolicy) String() string { - return proto.EnumName(ServiceConfig_LoadBalancingPolicy_name, int32(x)) -} - -func (ServiceConfig_LoadBalancingPolicy) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_e32d3cb2c41c77ce, []int{11, 0} -} - -// Configuration for a method. -type MethodConfig struct { - Name []*MethodConfig_Name `protobuf:"bytes,1,rep,name=name,proto3" json:"name,omitempty"` - // Whether RPCs sent to this method should wait until the connection is - // ready by default. If false, the RPC will abort immediately if there is - // a transient failure connecting to the server. Otherwise, gRPC will - // attempt to connect until the deadline is exceeded. - // - // The value specified via the gRPC client API will override the value - // set here. However, note that setting the value in the client API will - // also affect transient errors encountered during name resolution, which - // cannot be caught by the value here, since the service config is - // obtained by the gRPC client via name resolution. - WaitForReady *wrappers.BoolValue `protobuf:"bytes,2,opt,name=wait_for_ready,json=waitForReady,proto3" json:"wait_for_ready,omitempty"` - // The default timeout in seconds for RPCs sent to this method. This can be - // overridden in code. If no reply is received in the specified amount of - // time, the request is aborted and a DEADLINE_EXCEEDED error status - // is returned to the caller. - // - // The actual deadline used will be the minimum of the value specified here - // and the value set by the application via the gRPC client API. If either - // one is not set, then the other will be used. If neither is set, then the - // request has no deadline. - Timeout *duration.Duration `protobuf:"bytes,3,opt,name=timeout,proto3" json:"timeout,omitempty"` - // The maximum allowed payload size for an individual request or object in a - // stream (client->server) in bytes. The size which is measured is the - // serialized payload after per-message compression (but before stream - // compression) in bytes. This applies both to streaming and non-streaming - // requests. - // - // The actual value used is the minimum of the value specified here and the - // value set by the application via the gRPC client API. If either one is - // not set, then the other will be used. If neither is set, then the - // built-in default is used. - // - // If a client attempts to send an object larger than this value, it will not - // be sent and the client will see a ClientError. - // Note that 0 is a valid value, meaning that the request message - // must be empty. - MaxRequestMessageBytes *wrappers.UInt32Value `protobuf:"bytes,4,opt,name=max_request_message_bytes,json=maxRequestMessageBytes,proto3" json:"max_request_message_bytes,omitempty"` - // The maximum allowed payload size for an individual response or object in a - // stream (server->client) in bytes. The size which is measured is the - // serialized payload after per-message compression (but before stream - // compression) in bytes. This applies both to streaming and non-streaming - // requests. - // - // The actual value used is the minimum of the value specified here and the - // value set by the application via the gRPC client API. If either one is - // not set, then the other will be used. If neither is set, then the - // built-in default is used. - // - // If a server attempts to send an object larger than this value, it will not - // be sent, and a ServerError will be sent to the client instead. - // Note that 0 is a valid value, meaning that the response message - // must be empty. - MaxResponseMessageBytes *wrappers.UInt32Value `protobuf:"bytes,5,opt,name=max_response_message_bytes,json=maxResponseMessageBytes,proto3" json:"max_response_message_bytes,omitempty"` - // Only one of retry_policy or hedging_policy may be set. If neither is set, - // RPCs will not be retried or hedged. - // - // Types that are valid to be assigned to RetryOrHedgingPolicy: - // *MethodConfig_RetryPolicy_ - // *MethodConfig_HedgingPolicy_ - RetryOrHedgingPolicy isMethodConfig_RetryOrHedgingPolicy `protobuf_oneof:"retry_or_hedging_policy"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *MethodConfig) Reset() { *m = MethodConfig{} } -func (m *MethodConfig) String() string { return proto.CompactTextString(m) } -func (*MethodConfig) ProtoMessage() {} -func (*MethodConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_e32d3cb2c41c77ce, []int{0} -} - -func (m *MethodConfig) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_MethodConfig.Unmarshal(m, b) -} -func (m *MethodConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_MethodConfig.Marshal(b, m, deterministic) -} -func (m *MethodConfig) XXX_Merge(src proto.Message) { - xxx_messageInfo_MethodConfig.Merge(m, src) -} -func (m *MethodConfig) XXX_Size() int { - return xxx_messageInfo_MethodConfig.Size(m) -} -func (m *MethodConfig) XXX_DiscardUnknown() { - xxx_messageInfo_MethodConfig.DiscardUnknown(m) -} - -var xxx_messageInfo_MethodConfig proto.InternalMessageInfo - -func (m *MethodConfig) GetName() []*MethodConfig_Name { - if m != nil { - return m.Name - } - return nil -} - -func (m *MethodConfig) GetWaitForReady() *wrappers.BoolValue { - if m != nil { - return m.WaitForReady - } - return nil -} - -func (m *MethodConfig) GetTimeout() *duration.Duration { - if m != nil { - return m.Timeout - } - return nil -} - -func (m *MethodConfig) GetMaxRequestMessageBytes() *wrappers.UInt32Value { - if m != nil { - return m.MaxRequestMessageBytes - } - return nil -} - -func (m *MethodConfig) GetMaxResponseMessageBytes() *wrappers.UInt32Value { - if m != nil { - return m.MaxResponseMessageBytes - } - return nil -} - -type isMethodConfig_RetryOrHedgingPolicy interface { - isMethodConfig_RetryOrHedgingPolicy() -} - -type MethodConfig_RetryPolicy_ struct { - RetryPolicy *MethodConfig_RetryPolicy `protobuf:"bytes,6,opt,name=retry_policy,json=retryPolicy,proto3,oneof"` -} - -type MethodConfig_HedgingPolicy_ struct { - HedgingPolicy *MethodConfig_HedgingPolicy `protobuf:"bytes,7,opt,name=hedging_policy,json=hedgingPolicy,proto3,oneof"` -} - -func (*MethodConfig_RetryPolicy_) isMethodConfig_RetryOrHedgingPolicy() {} - -func (*MethodConfig_HedgingPolicy_) isMethodConfig_RetryOrHedgingPolicy() {} - -func (m *MethodConfig) GetRetryOrHedgingPolicy() isMethodConfig_RetryOrHedgingPolicy { - if m != nil { - return m.RetryOrHedgingPolicy - } - return nil -} - -func (m *MethodConfig) GetRetryPolicy() *MethodConfig_RetryPolicy { - if x, ok := m.GetRetryOrHedgingPolicy().(*MethodConfig_RetryPolicy_); ok { - return x.RetryPolicy - } - return nil -} - -func (m *MethodConfig) GetHedgingPolicy() *MethodConfig_HedgingPolicy { - if x, ok := m.GetRetryOrHedgingPolicy().(*MethodConfig_HedgingPolicy_); ok { - return x.HedgingPolicy - } - return nil -} - -// XXX_OneofWrappers is for the internal use of the proto package. -func (*MethodConfig) XXX_OneofWrappers() []interface{} { - return []interface{}{ - (*MethodConfig_RetryPolicy_)(nil), - (*MethodConfig_HedgingPolicy_)(nil), - } -} - -// The names of the methods to which this configuration applies. -// - MethodConfig without names (empty list) will be skipped. -// - Each name entry must be unique across the entire ServiceConfig. -// - If the 'method' field is empty, this MethodConfig specifies the defaults -// for all methods for the specified service. -// - If the 'service' field is empty, the 'method' field must be empty, and -// this MethodConfig specifies the default for all methods (it's the default -// config). -// -// When determining which MethodConfig to use for a given RPC, the most -// specific match wins. For example, let's say that the service config -// contains the following MethodConfig entries: -// -// method_config { name { } ... } -// method_config { name { service: "MyService" } ... } -// method_config { name { service: "MyService" method: "Foo" } ... } -// -// MyService/Foo will use the third entry, because it exactly matches the -// service and method name. MyService/Bar will use the second entry, because -// it provides the default for all methods of MyService. AnotherService/Baz -// will use the first entry, because it doesn't match the other two. -// -// In JSON representation, value "", value `null`, and not present are the -// same. The following are the same Name: -// - { "service": "s" } -// - { "service": "s", "method": null } -// - { "service": "s", "method": "" } -type MethodConfig_Name struct { - Service string `protobuf:"bytes,1,opt,name=service,proto3" json:"service,omitempty"` - Method string `protobuf:"bytes,2,opt,name=method,proto3" json:"method,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *MethodConfig_Name) Reset() { *m = MethodConfig_Name{} } -func (m *MethodConfig_Name) String() string { return proto.CompactTextString(m) } -func (*MethodConfig_Name) ProtoMessage() {} -func (*MethodConfig_Name) Descriptor() ([]byte, []int) { - return fileDescriptor_e32d3cb2c41c77ce, []int{0, 0} -} - -func (m *MethodConfig_Name) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_MethodConfig_Name.Unmarshal(m, b) -} -func (m *MethodConfig_Name) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_MethodConfig_Name.Marshal(b, m, deterministic) -} -func (m *MethodConfig_Name) XXX_Merge(src proto.Message) { - xxx_messageInfo_MethodConfig_Name.Merge(m, src) -} -func (m *MethodConfig_Name) XXX_Size() int { - return xxx_messageInfo_MethodConfig_Name.Size(m) -} -func (m *MethodConfig_Name) XXX_DiscardUnknown() { - xxx_messageInfo_MethodConfig_Name.DiscardUnknown(m) -} - -var xxx_messageInfo_MethodConfig_Name proto.InternalMessageInfo - -func (m *MethodConfig_Name) GetService() string { - if m != nil { - return m.Service - } - return "" -} - -func (m *MethodConfig_Name) GetMethod() string { - if m != nil { - return m.Method - } - return "" -} - -// The retry policy for outgoing RPCs. -type MethodConfig_RetryPolicy struct { - // The maximum number of RPC attempts, including the original attempt. - // - // This field is required and must be greater than 1. - // Any value greater than 5 will be treated as if it were 5. - MaxAttempts uint32 `protobuf:"varint,1,opt,name=max_attempts,json=maxAttempts,proto3" json:"max_attempts,omitempty"` - // Exponential backoff parameters. The initial retry attempt will occur at - // random(0, initial_backoff). In general, the nth attempt will occur at - // random(0, - // min(initial_backoff*backoff_multiplier**(n-1), max_backoff)). - // Required. Must be greater than zero. - InitialBackoff *duration.Duration `protobuf:"bytes,2,opt,name=initial_backoff,json=initialBackoff,proto3" json:"initial_backoff,omitempty"` - // Required. Must be greater than zero. - MaxBackoff *duration.Duration `protobuf:"bytes,3,opt,name=max_backoff,json=maxBackoff,proto3" json:"max_backoff,omitempty"` - BackoffMultiplier float32 `protobuf:"fixed32,4,opt,name=backoff_multiplier,json=backoffMultiplier,proto3" json:"backoff_multiplier,omitempty"` - // The set of status codes which may be retried. - // - // This field is required and must be non-empty. - RetryableStatusCodes []code.Code `protobuf:"varint,5,rep,packed,name=retryable_status_codes,json=retryableStatusCodes,proto3,enum=google.rpc.Code" json:"retryable_status_codes,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *MethodConfig_RetryPolicy) Reset() { *m = MethodConfig_RetryPolicy{} } -func (m *MethodConfig_RetryPolicy) String() string { return proto.CompactTextString(m) } -func (*MethodConfig_RetryPolicy) ProtoMessage() {} -func (*MethodConfig_RetryPolicy) Descriptor() ([]byte, []int) { - return fileDescriptor_e32d3cb2c41c77ce, []int{0, 1} -} - -func (m *MethodConfig_RetryPolicy) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_MethodConfig_RetryPolicy.Unmarshal(m, b) -} -func (m *MethodConfig_RetryPolicy) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_MethodConfig_RetryPolicy.Marshal(b, m, deterministic) -} -func (m *MethodConfig_RetryPolicy) XXX_Merge(src proto.Message) { - xxx_messageInfo_MethodConfig_RetryPolicy.Merge(m, src) -} -func (m *MethodConfig_RetryPolicy) XXX_Size() int { - return xxx_messageInfo_MethodConfig_RetryPolicy.Size(m) -} -func (m *MethodConfig_RetryPolicy) XXX_DiscardUnknown() { - xxx_messageInfo_MethodConfig_RetryPolicy.DiscardUnknown(m) -} - -var xxx_messageInfo_MethodConfig_RetryPolicy proto.InternalMessageInfo - -func (m *MethodConfig_RetryPolicy) GetMaxAttempts() uint32 { - if m != nil { - return m.MaxAttempts - } - return 0 -} - -func (m *MethodConfig_RetryPolicy) GetInitialBackoff() *duration.Duration { - if m != nil { - return m.InitialBackoff - } - return nil -} - -func (m *MethodConfig_RetryPolicy) GetMaxBackoff() *duration.Duration { - if m != nil { - return m.MaxBackoff - } - return nil -} - -func (m *MethodConfig_RetryPolicy) GetBackoffMultiplier() float32 { - if m != nil { - return m.BackoffMultiplier - } - return 0 -} - -func (m *MethodConfig_RetryPolicy) GetRetryableStatusCodes() []code.Code { - if m != nil { - return m.RetryableStatusCodes - } - return nil -} - -// The hedging policy for outgoing RPCs. Hedged RPCs may execute more than -// once on the server, so only idempotent methods should specify a hedging -// policy. -type MethodConfig_HedgingPolicy struct { - // The hedging policy will send up to max_requests RPCs. - // This number represents the total number of all attempts, including - // the original attempt. - // - // This field is required and must be greater than 1. - // Any value greater than 5 will be treated as if it were 5. - MaxAttempts uint32 `protobuf:"varint,1,opt,name=max_attempts,json=maxAttempts,proto3" json:"max_attempts,omitempty"` - // The first RPC will be sent immediately, but the max_requests-1 subsequent - // hedged RPCs will be sent at intervals of every hedging_delay. Set this - // to 0 to immediately send all max_requests RPCs. - HedgingDelay *duration.Duration `protobuf:"bytes,2,opt,name=hedging_delay,json=hedgingDelay,proto3" json:"hedging_delay,omitempty"` - // The set of status codes which indicate other hedged RPCs may still - // succeed. If a non-fatal status code is returned by the server, hedged - // RPCs will continue. Otherwise, outstanding requests will be canceled and - // the error returned to the client application layer. - // - // This field is optional. - NonFatalStatusCodes []code.Code `protobuf:"varint,3,rep,packed,name=non_fatal_status_codes,json=nonFatalStatusCodes,proto3,enum=google.rpc.Code" json:"non_fatal_status_codes,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *MethodConfig_HedgingPolicy) Reset() { *m = MethodConfig_HedgingPolicy{} } -func (m *MethodConfig_HedgingPolicy) String() string { return proto.CompactTextString(m) } -func (*MethodConfig_HedgingPolicy) ProtoMessage() {} -func (*MethodConfig_HedgingPolicy) Descriptor() ([]byte, []int) { - return fileDescriptor_e32d3cb2c41c77ce, []int{0, 2} -} - -func (m *MethodConfig_HedgingPolicy) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_MethodConfig_HedgingPolicy.Unmarshal(m, b) -} -func (m *MethodConfig_HedgingPolicy) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_MethodConfig_HedgingPolicy.Marshal(b, m, deterministic) -} -func (m *MethodConfig_HedgingPolicy) XXX_Merge(src proto.Message) { - xxx_messageInfo_MethodConfig_HedgingPolicy.Merge(m, src) -} -func (m *MethodConfig_HedgingPolicy) XXX_Size() int { - return xxx_messageInfo_MethodConfig_HedgingPolicy.Size(m) -} -func (m *MethodConfig_HedgingPolicy) XXX_DiscardUnknown() { - xxx_messageInfo_MethodConfig_HedgingPolicy.DiscardUnknown(m) -} - -var xxx_messageInfo_MethodConfig_HedgingPolicy proto.InternalMessageInfo - -func (m *MethodConfig_HedgingPolicy) GetMaxAttempts() uint32 { - if m != nil { - return m.MaxAttempts - } - return 0 -} - -func (m *MethodConfig_HedgingPolicy) GetHedgingDelay() *duration.Duration { - if m != nil { - return m.HedgingDelay - } - return nil -} - -func (m *MethodConfig_HedgingPolicy) GetNonFatalStatusCodes() []code.Code { - if m != nil { - return m.NonFatalStatusCodes - } - return nil -} - -// Configuration for pick_first LB policy. -type PickFirstConfig struct { - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *PickFirstConfig) Reset() { *m = PickFirstConfig{} } -func (m *PickFirstConfig) String() string { return proto.CompactTextString(m) } -func (*PickFirstConfig) ProtoMessage() {} -func (*PickFirstConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_e32d3cb2c41c77ce, []int{1} -} - -func (m *PickFirstConfig) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_PickFirstConfig.Unmarshal(m, b) -} -func (m *PickFirstConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_PickFirstConfig.Marshal(b, m, deterministic) -} -func (m *PickFirstConfig) XXX_Merge(src proto.Message) { - xxx_messageInfo_PickFirstConfig.Merge(m, src) -} -func (m *PickFirstConfig) XXX_Size() int { - return xxx_messageInfo_PickFirstConfig.Size(m) -} -func (m *PickFirstConfig) XXX_DiscardUnknown() { - xxx_messageInfo_PickFirstConfig.DiscardUnknown(m) -} - -var xxx_messageInfo_PickFirstConfig proto.InternalMessageInfo - -// Configuration for round_robin LB policy. -type RoundRobinConfig struct { - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *RoundRobinConfig) Reset() { *m = RoundRobinConfig{} } -func (m *RoundRobinConfig) String() string { return proto.CompactTextString(m) } -func (*RoundRobinConfig) ProtoMessage() {} -func (*RoundRobinConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_e32d3cb2c41c77ce, []int{2} -} - -func (m *RoundRobinConfig) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_RoundRobinConfig.Unmarshal(m, b) -} -func (m *RoundRobinConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_RoundRobinConfig.Marshal(b, m, deterministic) -} -func (m *RoundRobinConfig) XXX_Merge(src proto.Message) { - xxx_messageInfo_RoundRobinConfig.Merge(m, src) -} -func (m *RoundRobinConfig) XXX_Size() int { - return xxx_messageInfo_RoundRobinConfig.Size(m) -} -func (m *RoundRobinConfig) XXX_DiscardUnknown() { - xxx_messageInfo_RoundRobinConfig.DiscardUnknown(m) -} - -var xxx_messageInfo_RoundRobinConfig proto.InternalMessageInfo - -// Configuration for priority LB policy. -type PriorityLoadBalancingPolicyConfig struct { - Children map[string]*PriorityLoadBalancingPolicyConfig_Child `protobuf:"bytes,1,rep,name=children,proto3" json:"children,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` - // A list of child names in decreasing priority order - // (i.e., first element is the highest priority). - Priorities []string `protobuf:"bytes,2,rep,name=priorities,proto3" json:"priorities,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *PriorityLoadBalancingPolicyConfig) Reset() { *m = PriorityLoadBalancingPolicyConfig{} } -func (m *PriorityLoadBalancingPolicyConfig) String() string { return proto.CompactTextString(m) } -func (*PriorityLoadBalancingPolicyConfig) ProtoMessage() {} -func (*PriorityLoadBalancingPolicyConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_e32d3cb2c41c77ce, []int{3} -} - -func (m *PriorityLoadBalancingPolicyConfig) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_PriorityLoadBalancingPolicyConfig.Unmarshal(m, b) -} -func (m *PriorityLoadBalancingPolicyConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_PriorityLoadBalancingPolicyConfig.Marshal(b, m, deterministic) -} -func (m *PriorityLoadBalancingPolicyConfig) XXX_Merge(src proto.Message) { - xxx_messageInfo_PriorityLoadBalancingPolicyConfig.Merge(m, src) -} -func (m *PriorityLoadBalancingPolicyConfig) XXX_Size() int { - return xxx_messageInfo_PriorityLoadBalancingPolicyConfig.Size(m) -} -func (m *PriorityLoadBalancingPolicyConfig) XXX_DiscardUnknown() { - xxx_messageInfo_PriorityLoadBalancingPolicyConfig.DiscardUnknown(m) -} - -var xxx_messageInfo_PriorityLoadBalancingPolicyConfig proto.InternalMessageInfo - -func (m *PriorityLoadBalancingPolicyConfig) GetChildren() map[string]*PriorityLoadBalancingPolicyConfig_Child { - if m != nil { - return m.Children - } - return nil -} - -func (m *PriorityLoadBalancingPolicyConfig) GetPriorities() []string { - if m != nil { - return m.Priorities - } - return nil -} - -// A map of name to child policy configuration. -// The names are used to allow the priority policy to update -// existing child policies instead of creating new ones every -// time it receives a config update. -type PriorityLoadBalancingPolicyConfig_Child struct { - Config []*LoadBalancingConfig `protobuf:"bytes,1,rep,name=config,proto3" json:"config,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *PriorityLoadBalancingPolicyConfig_Child) Reset() { - *m = PriorityLoadBalancingPolicyConfig_Child{} -} -func (m *PriorityLoadBalancingPolicyConfig_Child) String() string { return proto.CompactTextString(m) } -func (*PriorityLoadBalancingPolicyConfig_Child) ProtoMessage() {} -func (*PriorityLoadBalancingPolicyConfig_Child) Descriptor() ([]byte, []int) { - return fileDescriptor_e32d3cb2c41c77ce, []int{3, 0} -} - -func (m *PriorityLoadBalancingPolicyConfig_Child) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_PriorityLoadBalancingPolicyConfig_Child.Unmarshal(m, b) -} -func (m *PriorityLoadBalancingPolicyConfig_Child) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_PriorityLoadBalancingPolicyConfig_Child.Marshal(b, m, deterministic) -} -func (m *PriorityLoadBalancingPolicyConfig_Child) XXX_Merge(src proto.Message) { - xxx_messageInfo_PriorityLoadBalancingPolicyConfig_Child.Merge(m, src) -} -func (m *PriorityLoadBalancingPolicyConfig_Child) XXX_Size() int { - return xxx_messageInfo_PriorityLoadBalancingPolicyConfig_Child.Size(m) -} -func (m *PriorityLoadBalancingPolicyConfig_Child) XXX_DiscardUnknown() { - xxx_messageInfo_PriorityLoadBalancingPolicyConfig_Child.DiscardUnknown(m) -} - -var xxx_messageInfo_PriorityLoadBalancingPolicyConfig_Child proto.InternalMessageInfo - -func (m *PriorityLoadBalancingPolicyConfig_Child) GetConfig() []*LoadBalancingConfig { - if m != nil { - return m.Config - } - return nil -} - -// Configuration for weighted_target LB policy. -type WeightedTargetLoadBalancingPolicyConfig struct { - Targets map[string]*WeightedTargetLoadBalancingPolicyConfig_Target `protobuf:"bytes,1,rep,name=targets,proto3" json:"targets,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *WeightedTargetLoadBalancingPolicyConfig) Reset() { - *m = WeightedTargetLoadBalancingPolicyConfig{} -} -func (m *WeightedTargetLoadBalancingPolicyConfig) String() string { return proto.CompactTextString(m) } -func (*WeightedTargetLoadBalancingPolicyConfig) ProtoMessage() {} -func (*WeightedTargetLoadBalancingPolicyConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_e32d3cb2c41c77ce, []int{4} -} - -func (m *WeightedTargetLoadBalancingPolicyConfig) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_WeightedTargetLoadBalancingPolicyConfig.Unmarshal(m, b) -} -func (m *WeightedTargetLoadBalancingPolicyConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_WeightedTargetLoadBalancingPolicyConfig.Marshal(b, m, deterministic) -} -func (m *WeightedTargetLoadBalancingPolicyConfig) XXX_Merge(src proto.Message) { - xxx_messageInfo_WeightedTargetLoadBalancingPolicyConfig.Merge(m, src) -} -func (m *WeightedTargetLoadBalancingPolicyConfig) XXX_Size() int { - return xxx_messageInfo_WeightedTargetLoadBalancingPolicyConfig.Size(m) -} -func (m *WeightedTargetLoadBalancingPolicyConfig) XXX_DiscardUnknown() { - xxx_messageInfo_WeightedTargetLoadBalancingPolicyConfig.DiscardUnknown(m) -} - -var xxx_messageInfo_WeightedTargetLoadBalancingPolicyConfig proto.InternalMessageInfo - -func (m *WeightedTargetLoadBalancingPolicyConfig) GetTargets() map[string]*WeightedTargetLoadBalancingPolicyConfig_Target { - if m != nil { - return m.Targets - } - return nil -} - -type WeightedTargetLoadBalancingPolicyConfig_Target struct { - Weight uint32 `protobuf:"varint,1,opt,name=weight,proto3" json:"weight,omitempty"` - ChildPolicy []*LoadBalancingConfig `protobuf:"bytes,2,rep,name=child_policy,json=childPolicy,proto3" json:"child_policy,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *WeightedTargetLoadBalancingPolicyConfig_Target) Reset() { - *m = WeightedTargetLoadBalancingPolicyConfig_Target{} -} -func (m *WeightedTargetLoadBalancingPolicyConfig_Target) String() string { - return proto.CompactTextString(m) -} -func (*WeightedTargetLoadBalancingPolicyConfig_Target) ProtoMessage() {} -func (*WeightedTargetLoadBalancingPolicyConfig_Target) Descriptor() ([]byte, []int) { - return fileDescriptor_e32d3cb2c41c77ce, []int{4, 0} -} - -func (m *WeightedTargetLoadBalancingPolicyConfig_Target) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_WeightedTargetLoadBalancingPolicyConfig_Target.Unmarshal(m, b) -} -func (m *WeightedTargetLoadBalancingPolicyConfig_Target) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_WeightedTargetLoadBalancingPolicyConfig_Target.Marshal(b, m, deterministic) -} -func (m *WeightedTargetLoadBalancingPolicyConfig_Target) XXX_Merge(src proto.Message) { - xxx_messageInfo_WeightedTargetLoadBalancingPolicyConfig_Target.Merge(m, src) -} -func (m *WeightedTargetLoadBalancingPolicyConfig_Target) XXX_Size() int { - return xxx_messageInfo_WeightedTargetLoadBalancingPolicyConfig_Target.Size(m) -} -func (m *WeightedTargetLoadBalancingPolicyConfig_Target) XXX_DiscardUnknown() { - xxx_messageInfo_WeightedTargetLoadBalancingPolicyConfig_Target.DiscardUnknown(m) -} - -var xxx_messageInfo_WeightedTargetLoadBalancingPolicyConfig_Target proto.InternalMessageInfo - -func (m *WeightedTargetLoadBalancingPolicyConfig_Target) GetWeight() uint32 { - if m != nil { - return m.Weight - } - return 0 -} - -func (m *WeightedTargetLoadBalancingPolicyConfig_Target) GetChildPolicy() []*LoadBalancingConfig { - if m != nil { - return m.ChildPolicy - } - return nil -} - -// Configuration for grpclb LB policy. -type GrpcLbConfig struct { - // Optional. What LB policy to use for routing between the backend - // addresses. If unset, defaults to round_robin. - // Currently, the only supported values are round_robin and pick_first. - // Note that this will be used both in balancer mode and in fallback mode. - // Multiple LB policies can be specified; clients will iterate through - // the list in order and stop at the first policy that they support. - ChildPolicy []*LoadBalancingConfig `protobuf:"bytes,1,rep,name=child_policy,json=childPolicy,proto3" json:"child_policy,omitempty"` - // Optional. If specified, overrides the name of the service to be sent to - // the balancer. - ServiceName string `protobuf:"bytes,2,opt,name=service_name,json=serviceName,proto3" json:"service_name,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *GrpcLbConfig) Reset() { *m = GrpcLbConfig{} } -func (m *GrpcLbConfig) String() string { return proto.CompactTextString(m) } -func (*GrpcLbConfig) ProtoMessage() {} -func (*GrpcLbConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_e32d3cb2c41c77ce, []int{5} -} - -func (m *GrpcLbConfig) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_GrpcLbConfig.Unmarshal(m, b) -} -func (m *GrpcLbConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_GrpcLbConfig.Marshal(b, m, deterministic) -} -func (m *GrpcLbConfig) XXX_Merge(src proto.Message) { - xxx_messageInfo_GrpcLbConfig.Merge(m, src) -} -func (m *GrpcLbConfig) XXX_Size() int { - return xxx_messageInfo_GrpcLbConfig.Size(m) -} -func (m *GrpcLbConfig) XXX_DiscardUnknown() { - xxx_messageInfo_GrpcLbConfig.DiscardUnknown(m) -} - -var xxx_messageInfo_GrpcLbConfig proto.InternalMessageInfo - -func (m *GrpcLbConfig) GetChildPolicy() []*LoadBalancingConfig { - if m != nil { - return m.ChildPolicy - } - return nil -} - -func (m *GrpcLbConfig) GetServiceName() string { - if m != nil { - return m.ServiceName - } - return "" -} - -// Configuration for the cds LB policy. -type CdsConfig struct { - Cluster string `protobuf:"bytes,1,opt,name=cluster,proto3" json:"cluster,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *CdsConfig) Reset() { *m = CdsConfig{} } -func (m *CdsConfig) String() string { return proto.CompactTextString(m) } -func (*CdsConfig) ProtoMessage() {} -func (*CdsConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_e32d3cb2c41c77ce, []int{6} -} - -func (m *CdsConfig) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_CdsConfig.Unmarshal(m, b) -} -func (m *CdsConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_CdsConfig.Marshal(b, m, deterministic) -} -func (m *CdsConfig) XXX_Merge(src proto.Message) { - xxx_messageInfo_CdsConfig.Merge(m, src) -} -func (m *CdsConfig) XXX_Size() int { - return xxx_messageInfo_CdsConfig.Size(m) -} -func (m *CdsConfig) XXX_DiscardUnknown() { - xxx_messageInfo_CdsConfig.DiscardUnknown(m) -} - -var xxx_messageInfo_CdsConfig proto.InternalMessageInfo - -func (m *CdsConfig) GetCluster() string { - if m != nil { - return m.Cluster - } - return "" -} - -// Configuration for xds LB policy. -type XdsConfig struct { - // Name of balancer to connect to. - BalancerName string `protobuf:"bytes,1,opt,name=balancer_name,json=balancerName,proto3" json:"balancer_name,omitempty"` // Deprecated: Do not use. - // Optional. What LB policy to use for intra-locality routing. - // If unset, will use whatever algorithm is specified by the balancer. - // Multiple LB policies can be specified; clients will iterate through - // the list in order and stop at the first policy that they support. - ChildPolicy []*LoadBalancingConfig `protobuf:"bytes,2,rep,name=child_policy,json=childPolicy,proto3" json:"child_policy,omitempty"` - // Optional. What LB policy to use in fallback mode. If not - // specified, defaults to round_robin. - // Multiple LB policies can be specified; clients will iterate through - // the list in order and stop at the first policy that they support. - FallbackPolicy []*LoadBalancingConfig `protobuf:"bytes,3,rep,name=fallback_policy,json=fallbackPolicy,proto3" json:"fallback_policy,omitempty"` - // Optional. Name to use in EDS query. If not present, defaults to - // the server name from the target URI. - EdsServiceName string `protobuf:"bytes,4,opt,name=eds_service_name,json=edsServiceName,proto3" json:"eds_service_name,omitempty"` - // LRS server to send load reports to. - // If not present, load reporting will be disabled. - // If set to the empty string, load reporting will be sent to the same - // server that we obtained CDS data from. - LrsLoadReportingServerName *wrappers.StringValue `protobuf:"bytes,5,opt,name=lrs_load_reporting_server_name,json=lrsLoadReportingServerName,proto3" json:"lrs_load_reporting_server_name,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *XdsConfig) Reset() { *m = XdsConfig{} } -func (m *XdsConfig) String() string { return proto.CompactTextString(m) } -func (*XdsConfig) ProtoMessage() {} -func (*XdsConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_e32d3cb2c41c77ce, []int{7} -} - -func (m *XdsConfig) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_XdsConfig.Unmarshal(m, b) -} -func (m *XdsConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_XdsConfig.Marshal(b, m, deterministic) -} -func (m *XdsConfig) XXX_Merge(src proto.Message) { - xxx_messageInfo_XdsConfig.Merge(m, src) -} -func (m *XdsConfig) XXX_Size() int { - return xxx_messageInfo_XdsConfig.Size(m) -} -func (m *XdsConfig) XXX_DiscardUnknown() { - xxx_messageInfo_XdsConfig.DiscardUnknown(m) -} - -var xxx_messageInfo_XdsConfig proto.InternalMessageInfo - -// Deprecated: Do not use. -func (m *XdsConfig) GetBalancerName() string { - if m != nil { - return m.BalancerName - } - return "" -} - -func (m *XdsConfig) GetChildPolicy() []*LoadBalancingConfig { - if m != nil { - return m.ChildPolicy - } - return nil -} - -func (m *XdsConfig) GetFallbackPolicy() []*LoadBalancingConfig { - if m != nil { - return m.FallbackPolicy - } - return nil -} - -func (m *XdsConfig) GetEdsServiceName() string { - if m != nil { - return m.EdsServiceName - } - return "" -} - -func (m *XdsConfig) GetLrsLoadReportingServerName() *wrappers.StringValue { - if m != nil { - return m.LrsLoadReportingServerName - } - return nil -} - -// Configuration for eds LB policy. -type EdsLoadBalancingPolicyConfig struct { - // Cluster name. Required. - Cluster string `protobuf:"bytes,1,opt,name=cluster,proto3" json:"cluster,omitempty"` - // EDS service name, as returned in CDS. - // May be unset if not specified in CDS. - EdsServiceName string `protobuf:"bytes,2,opt,name=eds_service_name,json=edsServiceName,proto3" json:"eds_service_name,omitempty"` - // Server to send load reports to. - // If unset, no load reporting is done. - // If set to empty string, load reporting will be sent to the same - // server as we are getting xds data from. - LrsLoadReportingServerName *wrappers.StringValue `protobuf:"bytes,3,opt,name=lrs_load_reporting_server_name,json=lrsLoadReportingServerName,proto3" json:"lrs_load_reporting_server_name,omitempty"` - // Locality-picking policy. - // This policy's config is expected to be in the format used - // by the weighted_target policy. Note that the config should include - // an empty value for the "targets" field; that empty value will be - // replaced by one that is dynamically generated based on the EDS data. - // Optional; defaults to "weighted_target". - LocalityPickingPolicy []*LoadBalancingConfig `protobuf:"bytes,4,rep,name=locality_picking_policy,json=localityPickingPolicy,proto3" json:"locality_picking_policy,omitempty"` - // Endpoint-picking policy. - // This will be configured as the policy for each child in the - // locality-policy's config. - // Optional; defaults to "round_robin". - EndpointPickingPolicy []*LoadBalancingConfig `protobuf:"bytes,5,rep,name=endpoint_picking_policy,json=endpointPickingPolicy,proto3" json:"endpoint_picking_policy,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *EdsLoadBalancingPolicyConfig) Reset() { *m = EdsLoadBalancingPolicyConfig{} } -func (m *EdsLoadBalancingPolicyConfig) String() string { return proto.CompactTextString(m) } -func (*EdsLoadBalancingPolicyConfig) ProtoMessage() {} -func (*EdsLoadBalancingPolicyConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_e32d3cb2c41c77ce, []int{8} -} - -func (m *EdsLoadBalancingPolicyConfig) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_EdsLoadBalancingPolicyConfig.Unmarshal(m, b) -} -func (m *EdsLoadBalancingPolicyConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_EdsLoadBalancingPolicyConfig.Marshal(b, m, deterministic) -} -func (m *EdsLoadBalancingPolicyConfig) XXX_Merge(src proto.Message) { - xxx_messageInfo_EdsLoadBalancingPolicyConfig.Merge(m, src) -} -func (m *EdsLoadBalancingPolicyConfig) XXX_Size() int { - return xxx_messageInfo_EdsLoadBalancingPolicyConfig.Size(m) -} -func (m *EdsLoadBalancingPolicyConfig) XXX_DiscardUnknown() { - xxx_messageInfo_EdsLoadBalancingPolicyConfig.DiscardUnknown(m) -} - -var xxx_messageInfo_EdsLoadBalancingPolicyConfig proto.InternalMessageInfo - -func (m *EdsLoadBalancingPolicyConfig) GetCluster() string { - if m != nil { - return m.Cluster - } - return "" -} - -func (m *EdsLoadBalancingPolicyConfig) GetEdsServiceName() string { - if m != nil { - return m.EdsServiceName - } - return "" -} - -func (m *EdsLoadBalancingPolicyConfig) GetLrsLoadReportingServerName() *wrappers.StringValue { - if m != nil { - return m.LrsLoadReportingServerName - } - return nil -} - -func (m *EdsLoadBalancingPolicyConfig) GetLocalityPickingPolicy() []*LoadBalancingConfig { - if m != nil { - return m.LocalityPickingPolicy - } - return nil -} - -func (m *EdsLoadBalancingPolicyConfig) GetEndpointPickingPolicy() []*LoadBalancingConfig { - if m != nil { - return m.EndpointPickingPolicy - } - return nil -} - -// Configuration for lrs LB policy. -type LrsLoadBalancingPolicyConfig struct { - // Cluster name. Required. - ClusterName string `protobuf:"bytes,1,opt,name=cluster_name,json=clusterName,proto3" json:"cluster_name,omitempty"` - // EDS service name, as returned in CDS. - // May be unset if not specified in CDS. - EdsServiceName string `protobuf:"bytes,2,opt,name=eds_service_name,json=edsServiceName,proto3" json:"eds_service_name,omitempty"` - // Server to send load reports to. Required. - // If set to empty string, load reporting will be sent to the same - // server as we are getting xds data from. - LrsLoadReportingServerName string `protobuf:"bytes,3,opt,name=lrs_load_reporting_server_name,json=lrsLoadReportingServerName,proto3" json:"lrs_load_reporting_server_name,omitempty"` - Locality *LrsLoadBalancingPolicyConfig_Locality `protobuf:"bytes,4,opt,name=locality,proto3" json:"locality,omitempty"` - // Endpoint-picking policy. - ChildPolicy []*LoadBalancingConfig `protobuf:"bytes,5,rep,name=child_policy,json=childPolicy,proto3" json:"child_policy,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *LrsLoadBalancingPolicyConfig) Reset() { *m = LrsLoadBalancingPolicyConfig{} } -func (m *LrsLoadBalancingPolicyConfig) String() string { return proto.CompactTextString(m) } -func (*LrsLoadBalancingPolicyConfig) ProtoMessage() {} -func (*LrsLoadBalancingPolicyConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_e32d3cb2c41c77ce, []int{9} -} - -func (m *LrsLoadBalancingPolicyConfig) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_LrsLoadBalancingPolicyConfig.Unmarshal(m, b) -} -func (m *LrsLoadBalancingPolicyConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_LrsLoadBalancingPolicyConfig.Marshal(b, m, deterministic) -} -func (m *LrsLoadBalancingPolicyConfig) XXX_Merge(src proto.Message) { - xxx_messageInfo_LrsLoadBalancingPolicyConfig.Merge(m, src) -} -func (m *LrsLoadBalancingPolicyConfig) XXX_Size() int { - return xxx_messageInfo_LrsLoadBalancingPolicyConfig.Size(m) -} -func (m *LrsLoadBalancingPolicyConfig) XXX_DiscardUnknown() { - xxx_messageInfo_LrsLoadBalancingPolicyConfig.DiscardUnknown(m) -} - -var xxx_messageInfo_LrsLoadBalancingPolicyConfig proto.InternalMessageInfo - -func (m *LrsLoadBalancingPolicyConfig) GetClusterName() string { - if m != nil { - return m.ClusterName - } - return "" -} - -func (m *LrsLoadBalancingPolicyConfig) GetEdsServiceName() string { - if m != nil { - return m.EdsServiceName - } - return "" -} - -func (m *LrsLoadBalancingPolicyConfig) GetLrsLoadReportingServerName() string { - if m != nil { - return m.LrsLoadReportingServerName - } - return "" -} - -func (m *LrsLoadBalancingPolicyConfig) GetLocality() *LrsLoadBalancingPolicyConfig_Locality { - if m != nil { - return m.Locality - } - return nil -} - -func (m *LrsLoadBalancingPolicyConfig) GetChildPolicy() []*LoadBalancingConfig { - if m != nil { - return m.ChildPolicy - } - return nil -} - -// The locality for which this policy will report load. Required. -type LrsLoadBalancingPolicyConfig_Locality struct { - Region string `protobuf:"bytes,1,opt,name=region,proto3" json:"region,omitempty"` - Zone string `protobuf:"bytes,2,opt,name=zone,proto3" json:"zone,omitempty"` - Subzone string `protobuf:"bytes,3,opt,name=subzone,proto3" json:"subzone,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *LrsLoadBalancingPolicyConfig_Locality) Reset() { *m = LrsLoadBalancingPolicyConfig_Locality{} } -func (m *LrsLoadBalancingPolicyConfig_Locality) String() string { return proto.CompactTextString(m) } -func (*LrsLoadBalancingPolicyConfig_Locality) ProtoMessage() {} -func (*LrsLoadBalancingPolicyConfig_Locality) Descriptor() ([]byte, []int) { - return fileDescriptor_e32d3cb2c41c77ce, []int{9, 0} -} - -func (m *LrsLoadBalancingPolicyConfig_Locality) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_LrsLoadBalancingPolicyConfig_Locality.Unmarshal(m, b) -} -func (m *LrsLoadBalancingPolicyConfig_Locality) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_LrsLoadBalancingPolicyConfig_Locality.Marshal(b, m, deterministic) -} -func (m *LrsLoadBalancingPolicyConfig_Locality) XXX_Merge(src proto.Message) { - xxx_messageInfo_LrsLoadBalancingPolicyConfig_Locality.Merge(m, src) -} -func (m *LrsLoadBalancingPolicyConfig_Locality) XXX_Size() int { - return xxx_messageInfo_LrsLoadBalancingPolicyConfig_Locality.Size(m) -} -func (m *LrsLoadBalancingPolicyConfig_Locality) XXX_DiscardUnknown() { - xxx_messageInfo_LrsLoadBalancingPolicyConfig_Locality.DiscardUnknown(m) -} - -var xxx_messageInfo_LrsLoadBalancingPolicyConfig_Locality proto.InternalMessageInfo - -func (m *LrsLoadBalancingPolicyConfig_Locality) GetRegion() string { - if m != nil { - return m.Region - } - return "" -} - -func (m *LrsLoadBalancingPolicyConfig_Locality) GetZone() string { - if m != nil { - return m.Zone - } - return "" -} - -func (m *LrsLoadBalancingPolicyConfig_Locality) GetSubzone() string { - if m != nil { - return m.Subzone - } - return "" -} - -// Selects LB policy and provides corresponding configuration. -// -// In general, all instances of this field should be repeated. Clients will -// iterate through the list in order and stop at the first policy that they -// support. This allows the service config to specify custom policies that may -// not be known to all clients. -// -// - If the config for the first supported policy is invalid, the whole service -// config is invalid. -// - If the list doesn't contain any supported policy, the whole service config -// is invalid. -type LoadBalancingConfig struct { - // Exactly one LB policy may be configured. - // - // Types that are valid to be assigned to Policy: - // *LoadBalancingConfig_PickFirst - // *LoadBalancingConfig_RoundRobin - // *LoadBalancingConfig_Grpclb - // *LoadBalancingConfig_Priority - // *LoadBalancingConfig_WeightedTarget - // *LoadBalancingConfig_Cds - // *LoadBalancingConfig_Eds - // *LoadBalancingConfig_Lrs - // *LoadBalancingConfig_Xds - // *LoadBalancingConfig_XdsExperimental - Policy isLoadBalancingConfig_Policy `protobuf_oneof:"policy"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *LoadBalancingConfig) Reset() { *m = LoadBalancingConfig{} } -func (m *LoadBalancingConfig) String() string { return proto.CompactTextString(m) } -func (*LoadBalancingConfig) ProtoMessage() {} -func (*LoadBalancingConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_e32d3cb2c41c77ce, []int{10} -} - -func (m *LoadBalancingConfig) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_LoadBalancingConfig.Unmarshal(m, b) -} -func (m *LoadBalancingConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_LoadBalancingConfig.Marshal(b, m, deterministic) -} -func (m *LoadBalancingConfig) XXX_Merge(src proto.Message) { - xxx_messageInfo_LoadBalancingConfig.Merge(m, src) -} -func (m *LoadBalancingConfig) XXX_Size() int { - return xxx_messageInfo_LoadBalancingConfig.Size(m) -} -func (m *LoadBalancingConfig) XXX_DiscardUnknown() { - xxx_messageInfo_LoadBalancingConfig.DiscardUnknown(m) -} - -var xxx_messageInfo_LoadBalancingConfig proto.InternalMessageInfo - -type isLoadBalancingConfig_Policy interface { - isLoadBalancingConfig_Policy() -} - -type LoadBalancingConfig_PickFirst struct { - PickFirst *PickFirstConfig `protobuf:"bytes,4,opt,name=pick_first,proto3,oneof"` -} - -type LoadBalancingConfig_RoundRobin struct { - RoundRobin *RoundRobinConfig `protobuf:"bytes,1,opt,name=round_robin,proto3,oneof"` -} - -type LoadBalancingConfig_Grpclb struct { - Grpclb *GrpcLbConfig `protobuf:"bytes,3,opt,name=grpclb,proto3,oneof"` -} - -type LoadBalancingConfig_Priority struct { - Priority *PriorityLoadBalancingPolicyConfig `protobuf:"bytes,9,opt,name=priority,proto3,oneof"` -} - -type LoadBalancingConfig_WeightedTarget struct { - WeightedTarget *WeightedTargetLoadBalancingPolicyConfig `protobuf:"bytes,10,opt,name=weighted_target,json=weightedTarget,proto3,oneof"` -} - -type LoadBalancingConfig_Cds struct { - Cds *CdsConfig `protobuf:"bytes,6,opt,name=cds,proto3,oneof"` -} - -type LoadBalancingConfig_Eds struct { - Eds *EdsLoadBalancingPolicyConfig `protobuf:"bytes,7,opt,name=eds,proto3,oneof"` -} - -type LoadBalancingConfig_Lrs struct { - Lrs *LrsLoadBalancingPolicyConfig `protobuf:"bytes,8,opt,name=lrs,proto3,oneof"` -} - -type LoadBalancingConfig_Xds struct { - Xds *XdsConfig `protobuf:"bytes,2,opt,name=xds,proto3,oneof"` -} - -type LoadBalancingConfig_XdsExperimental struct { - XdsExperimental *XdsConfig `protobuf:"bytes,5,opt,name=xds_experimental,proto3,oneof"` -} - -func (*LoadBalancingConfig_PickFirst) isLoadBalancingConfig_Policy() {} - -func (*LoadBalancingConfig_RoundRobin) isLoadBalancingConfig_Policy() {} - -func (*LoadBalancingConfig_Grpclb) isLoadBalancingConfig_Policy() {} - -func (*LoadBalancingConfig_Priority) isLoadBalancingConfig_Policy() {} - -func (*LoadBalancingConfig_WeightedTarget) isLoadBalancingConfig_Policy() {} - -func (*LoadBalancingConfig_Cds) isLoadBalancingConfig_Policy() {} - -func (*LoadBalancingConfig_Eds) isLoadBalancingConfig_Policy() {} - -func (*LoadBalancingConfig_Lrs) isLoadBalancingConfig_Policy() {} - -func (*LoadBalancingConfig_Xds) isLoadBalancingConfig_Policy() {} - -func (*LoadBalancingConfig_XdsExperimental) isLoadBalancingConfig_Policy() {} - -func (m *LoadBalancingConfig) GetPolicy() isLoadBalancingConfig_Policy { - if m != nil { - return m.Policy - } - return nil -} - -func (m *LoadBalancingConfig) GetPickFirst() *PickFirstConfig { - if x, ok := m.GetPolicy().(*LoadBalancingConfig_PickFirst); ok { - return x.PickFirst - } - return nil -} - -func (m *LoadBalancingConfig) GetRoundRobin() *RoundRobinConfig { - if x, ok := m.GetPolicy().(*LoadBalancingConfig_RoundRobin); ok { - return x.RoundRobin - } - return nil -} - -func (m *LoadBalancingConfig) GetGrpclb() *GrpcLbConfig { - if x, ok := m.GetPolicy().(*LoadBalancingConfig_Grpclb); ok { - return x.Grpclb - } - return nil -} - -func (m *LoadBalancingConfig) GetPriority() *PriorityLoadBalancingPolicyConfig { - if x, ok := m.GetPolicy().(*LoadBalancingConfig_Priority); ok { - return x.Priority - } - return nil -} - -func (m *LoadBalancingConfig) GetWeightedTarget() *WeightedTargetLoadBalancingPolicyConfig { - if x, ok := m.GetPolicy().(*LoadBalancingConfig_WeightedTarget); ok { - return x.WeightedTarget - } - return nil -} - -func (m *LoadBalancingConfig) GetCds() *CdsConfig { - if x, ok := m.GetPolicy().(*LoadBalancingConfig_Cds); ok { - return x.Cds - } - return nil -} - -func (m *LoadBalancingConfig) GetEds() *EdsLoadBalancingPolicyConfig { - if x, ok := m.GetPolicy().(*LoadBalancingConfig_Eds); ok { - return x.Eds - } - return nil -} - -func (m *LoadBalancingConfig) GetLrs() *LrsLoadBalancingPolicyConfig { - if x, ok := m.GetPolicy().(*LoadBalancingConfig_Lrs); ok { - return x.Lrs - } - return nil -} - -// Deprecated: Do not use. -func (m *LoadBalancingConfig) GetXds() *XdsConfig { - if x, ok := m.GetPolicy().(*LoadBalancingConfig_Xds); ok { - return x.Xds - } - return nil -} - -// Deprecated: Do not use. -func (m *LoadBalancingConfig) GetXdsExperimental() *XdsConfig { - if x, ok := m.GetPolicy().(*LoadBalancingConfig_XdsExperimental); ok { - return x.XdsExperimental - } - return nil -} - -// XXX_OneofWrappers is for the internal use of the proto package. -func (*LoadBalancingConfig) XXX_OneofWrappers() []interface{} { - return []interface{}{ - (*LoadBalancingConfig_PickFirst)(nil), - (*LoadBalancingConfig_RoundRobin)(nil), - (*LoadBalancingConfig_Grpclb)(nil), - (*LoadBalancingConfig_Priority)(nil), - (*LoadBalancingConfig_WeightedTarget)(nil), - (*LoadBalancingConfig_Cds)(nil), - (*LoadBalancingConfig_Eds)(nil), - (*LoadBalancingConfig_Lrs)(nil), - (*LoadBalancingConfig_Xds)(nil), - (*LoadBalancingConfig_XdsExperimental)(nil), - } -} - -// A ServiceConfig represents information about a service but is not specific to -// any name resolver. -type ServiceConfig struct { - LoadBalancingPolicy ServiceConfig_LoadBalancingPolicy `protobuf:"varint,1,opt,name=load_balancing_policy,json=loadBalancingPolicy,proto3,enum=grpc.service_config.ServiceConfig_LoadBalancingPolicy" json:"load_balancing_policy,omitempty"` // Deprecated: Do not use. - // Multiple LB policies can be specified; clients will iterate through - // the list in order and stop at the first policy that they support. If none - // are supported, the service config is considered invalid. - LoadBalancingConfig []*LoadBalancingConfig `protobuf:"bytes,4,rep,name=load_balancing_config,json=loadBalancingConfig,proto3" json:"load_balancing_config,omitempty"` - // Per-method configuration. - MethodConfig []*MethodConfig `protobuf:"bytes,2,rep,name=method_config,json=methodConfig,proto3" json:"method_config,omitempty"` - RetryThrottling *ServiceConfig_RetryThrottlingPolicy `protobuf:"bytes,3,opt,name=retry_throttling,json=retryThrottling,proto3" json:"retry_throttling,omitempty"` - HealthCheckConfig *ServiceConfig_HealthCheckConfig `protobuf:"bytes,5,opt,name=health_check_config,json=healthCheckConfig,proto3" json:"health_check_config,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *ServiceConfig) Reset() { *m = ServiceConfig{} } -func (m *ServiceConfig) String() string { return proto.CompactTextString(m) } -func (*ServiceConfig) ProtoMessage() {} -func (*ServiceConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_e32d3cb2c41c77ce, []int{11} -} - -func (m *ServiceConfig) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ServiceConfig.Unmarshal(m, b) -} -func (m *ServiceConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ServiceConfig.Marshal(b, m, deterministic) -} -func (m *ServiceConfig) XXX_Merge(src proto.Message) { - xxx_messageInfo_ServiceConfig.Merge(m, src) -} -func (m *ServiceConfig) XXX_Size() int { - return xxx_messageInfo_ServiceConfig.Size(m) -} -func (m *ServiceConfig) XXX_DiscardUnknown() { - xxx_messageInfo_ServiceConfig.DiscardUnknown(m) -} - -var xxx_messageInfo_ServiceConfig proto.InternalMessageInfo - -// Deprecated: Do not use. -func (m *ServiceConfig) GetLoadBalancingPolicy() ServiceConfig_LoadBalancingPolicy { - if m != nil { - return m.LoadBalancingPolicy - } - return ServiceConfig_UNSPECIFIED -} - -func (m *ServiceConfig) GetLoadBalancingConfig() []*LoadBalancingConfig { - if m != nil { - return m.LoadBalancingConfig - } - return nil -} - -func (m *ServiceConfig) GetMethodConfig() []*MethodConfig { - if m != nil { - return m.MethodConfig - } - return nil -} - -func (m *ServiceConfig) GetRetryThrottling() *ServiceConfig_RetryThrottlingPolicy { - if m != nil { - return m.RetryThrottling - } - return nil -} - -func (m *ServiceConfig) GetHealthCheckConfig() *ServiceConfig_HealthCheckConfig { - if m != nil { - return m.HealthCheckConfig - } - return nil -} - -// If a RetryThrottlingPolicy is provided, gRPC will automatically throttle -// retry attempts and hedged RPCs when the client's ratio of failures to -// successes exceeds a threshold. -// -// For each server name, the gRPC client will maintain a token_count which is -// initially set to max_tokens. Every outgoing RPC (regardless of service or -// method invoked) will change token_count as follows: -// -// - Every failed RPC will decrement the token_count by 1. -// - Every successful RPC will increment the token_count by token_ratio. -// -// If token_count is less than or equal to max_tokens / 2, then RPCs will not -// be retried and hedged RPCs will not be sent. -type ServiceConfig_RetryThrottlingPolicy struct { - // The number of tokens starts at max_tokens. The token_count will always be - // between 0 and max_tokens. - // - // This field is required and must be greater than zero. - MaxTokens uint32 `protobuf:"varint,1,opt,name=max_tokens,json=maxTokens,proto3" json:"max_tokens,omitempty"` - // The amount of tokens to add on each successful RPC. Typically this will - // be some number between 0 and 1, e.g., 0.1. - // - // This field is required and must be greater than zero. Up to 3 decimal - // places are supported. - TokenRatio float32 `protobuf:"fixed32,2,opt,name=token_ratio,json=tokenRatio,proto3" json:"token_ratio,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *ServiceConfig_RetryThrottlingPolicy) Reset() { *m = ServiceConfig_RetryThrottlingPolicy{} } -func (m *ServiceConfig_RetryThrottlingPolicy) String() string { return proto.CompactTextString(m) } -func (*ServiceConfig_RetryThrottlingPolicy) ProtoMessage() {} -func (*ServiceConfig_RetryThrottlingPolicy) Descriptor() ([]byte, []int) { - return fileDescriptor_e32d3cb2c41c77ce, []int{11, 0} -} - -func (m *ServiceConfig_RetryThrottlingPolicy) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ServiceConfig_RetryThrottlingPolicy.Unmarshal(m, b) -} -func (m *ServiceConfig_RetryThrottlingPolicy) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ServiceConfig_RetryThrottlingPolicy.Marshal(b, m, deterministic) -} -func (m *ServiceConfig_RetryThrottlingPolicy) XXX_Merge(src proto.Message) { - xxx_messageInfo_ServiceConfig_RetryThrottlingPolicy.Merge(m, src) -} -func (m *ServiceConfig_RetryThrottlingPolicy) XXX_Size() int { - return xxx_messageInfo_ServiceConfig_RetryThrottlingPolicy.Size(m) -} -func (m *ServiceConfig_RetryThrottlingPolicy) XXX_DiscardUnknown() { - xxx_messageInfo_ServiceConfig_RetryThrottlingPolicy.DiscardUnknown(m) -} - -var xxx_messageInfo_ServiceConfig_RetryThrottlingPolicy proto.InternalMessageInfo - -func (m *ServiceConfig_RetryThrottlingPolicy) GetMaxTokens() uint32 { - if m != nil { - return m.MaxTokens - } - return 0 -} - -func (m *ServiceConfig_RetryThrottlingPolicy) GetTokenRatio() float32 { - if m != nil { - return m.TokenRatio - } - return 0 -} - -type ServiceConfig_HealthCheckConfig struct { - // Service name to use in the health-checking request. - ServiceName *wrappers.StringValue `protobuf:"bytes,1,opt,name=service_name,json=serviceName,proto3" json:"service_name,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *ServiceConfig_HealthCheckConfig) Reset() { *m = ServiceConfig_HealthCheckConfig{} } -func (m *ServiceConfig_HealthCheckConfig) String() string { return proto.CompactTextString(m) } -func (*ServiceConfig_HealthCheckConfig) ProtoMessage() {} -func (*ServiceConfig_HealthCheckConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_e32d3cb2c41c77ce, []int{11, 1} -} - -func (m *ServiceConfig_HealthCheckConfig) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ServiceConfig_HealthCheckConfig.Unmarshal(m, b) -} -func (m *ServiceConfig_HealthCheckConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ServiceConfig_HealthCheckConfig.Marshal(b, m, deterministic) -} -func (m *ServiceConfig_HealthCheckConfig) XXX_Merge(src proto.Message) { - xxx_messageInfo_ServiceConfig_HealthCheckConfig.Merge(m, src) -} -func (m *ServiceConfig_HealthCheckConfig) XXX_Size() int { - return xxx_messageInfo_ServiceConfig_HealthCheckConfig.Size(m) -} -func (m *ServiceConfig_HealthCheckConfig) XXX_DiscardUnknown() { - xxx_messageInfo_ServiceConfig_HealthCheckConfig.DiscardUnknown(m) -} - -var xxx_messageInfo_ServiceConfig_HealthCheckConfig proto.InternalMessageInfo - -func (m *ServiceConfig_HealthCheckConfig) GetServiceName() *wrappers.StringValue { - if m != nil { - return m.ServiceName - } - return nil -} - -func init() { - proto.RegisterEnum("grpc.service_config.ServiceConfig_LoadBalancingPolicy", ServiceConfig_LoadBalancingPolicy_name, ServiceConfig_LoadBalancingPolicy_value) - proto.RegisterType((*MethodConfig)(nil), "grpc.service_config.MethodConfig") - proto.RegisterType((*MethodConfig_Name)(nil), "grpc.service_config.MethodConfig.Name") - proto.RegisterType((*MethodConfig_RetryPolicy)(nil), "grpc.service_config.MethodConfig.RetryPolicy") - proto.RegisterType((*MethodConfig_HedgingPolicy)(nil), "grpc.service_config.MethodConfig.HedgingPolicy") - proto.RegisterType((*PickFirstConfig)(nil), "grpc.service_config.PickFirstConfig") - proto.RegisterType((*RoundRobinConfig)(nil), "grpc.service_config.RoundRobinConfig") - proto.RegisterType((*PriorityLoadBalancingPolicyConfig)(nil), "grpc.service_config.PriorityLoadBalancingPolicyConfig") - proto.RegisterMapType((map[string]*PriorityLoadBalancingPolicyConfig_Child)(nil), "grpc.service_config.PriorityLoadBalancingPolicyConfig.ChildrenEntry") - proto.RegisterType((*PriorityLoadBalancingPolicyConfig_Child)(nil), "grpc.service_config.PriorityLoadBalancingPolicyConfig.Child") - proto.RegisterType((*WeightedTargetLoadBalancingPolicyConfig)(nil), "grpc.service_config.WeightedTargetLoadBalancingPolicyConfig") - proto.RegisterMapType((map[string]*WeightedTargetLoadBalancingPolicyConfig_Target)(nil), "grpc.service_config.WeightedTargetLoadBalancingPolicyConfig.TargetsEntry") - proto.RegisterType((*WeightedTargetLoadBalancingPolicyConfig_Target)(nil), "grpc.service_config.WeightedTargetLoadBalancingPolicyConfig.Target") - proto.RegisterType((*GrpcLbConfig)(nil), "grpc.service_config.GrpcLbConfig") - proto.RegisterType((*CdsConfig)(nil), "grpc.service_config.CdsConfig") - proto.RegisterType((*XdsConfig)(nil), "grpc.service_config.XdsConfig") - proto.RegisterType((*EdsLoadBalancingPolicyConfig)(nil), "grpc.service_config.EdsLoadBalancingPolicyConfig") - proto.RegisterType((*LrsLoadBalancingPolicyConfig)(nil), "grpc.service_config.LrsLoadBalancingPolicyConfig") - proto.RegisterType((*LrsLoadBalancingPolicyConfig_Locality)(nil), "grpc.service_config.LrsLoadBalancingPolicyConfig.Locality") - proto.RegisterType((*LoadBalancingConfig)(nil), "grpc.service_config.LoadBalancingConfig") - proto.RegisterType((*ServiceConfig)(nil), "grpc.service_config.ServiceConfig") - proto.RegisterType((*ServiceConfig_RetryThrottlingPolicy)(nil), "grpc.service_config.ServiceConfig.RetryThrottlingPolicy") - proto.RegisterType((*ServiceConfig_HealthCheckConfig)(nil), "grpc.service_config.ServiceConfig.HealthCheckConfig") -} - -func init() { - proto.RegisterFile("grpc/service_config/service_config.proto", fileDescriptor_e32d3cb2c41c77ce) -} - -var fileDescriptor_e32d3cb2c41c77ce = []byte{ - // 1589 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0x5b, 0x73, 0x1a, 0x47, - 0x16, 0x16, 0xa0, 0x1b, 0x07, 0x84, 0x50, 0x6b, 0x25, 0x61, 0xca, 0xab, 0x95, 0xa8, 0xf5, 0x9a, - 0x17, 0xa3, 0x5a, 0x79, 0xcb, 0xeb, 0xd2, 0x6e, 0xed, 0x3a, 0xe8, 0x12, 0x54, 0x91, 0x65, 0xd2, - 0x92, 0x6f, 0x55, 0xa9, 0x1a, 0x0d, 0x33, 0x0d, 0x4c, 0x69, 0x66, 0x9a, 0xf4, 0x34, 0x16, 0xe4, - 0x21, 0x79, 0xcd, 0x2f, 0x49, 0x7e, 0x43, 0xfe, 0x49, 0x9e, 0x52, 0x79, 0xcf, 0x3f, 0xc8, 0x43, - 0x2a, 0xd5, 0x97, 0x41, 0x0c, 0x8c, 0x10, 0xb2, 0xfc, 0xc6, 0x9c, 0xee, 0xef, 0x3b, 0xd7, 0x3e, - 0x7d, 0x1a, 0x28, 0xb7, 0x58, 0xc7, 0xda, 0x09, 0x08, 0xfb, 0xe0, 0x58, 0xc4, 0xb0, 0xa8, 0xdf, - 0x74, 0x5a, 0x23, 0x9f, 0x95, 0x0e, 0xa3, 0x9c, 0xa2, 0x55, 0xb1, 0xb3, 0x12, 0x5d, 0x2a, 0x6e, - 0xb6, 0x28, 0x6d, 0xb9, 0x64, 0x47, 0x6e, 0x69, 0x74, 0x9b, 0x3b, 0x76, 0x97, 0x99, 0xdc, 0xa1, - 0xbe, 0x02, 0x8d, 0xaf, 0x5f, 0x31, 0xb3, 0xd3, 0x21, 0x2c, 0xd0, 0xeb, 0x6b, 0x7a, 0x5d, 0x18, - 0x61, 0x51, 0x9b, 0x28, 0x71, 0xe9, 0x97, 0x45, 0xc8, 0xbe, 0x24, 0xbc, 0x4d, 0xed, 0x7d, 0xa9, - 0x07, 0xed, 0xc1, 0xac, 0x6f, 0x7a, 0xa4, 0x90, 0xd8, 0x4a, 0x95, 0x33, 0xbb, 0xff, 0xa8, 0xc4, - 0xd8, 0x52, 0x19, 0x06, 0x54, 0x4e, 0x4d, 0x8f, 0x60, 0x89, 0x41, 0x2f, 0x20, 0x77, 0x65, 0x3a, - 0xdc, 0x68, 0x52, 0x66, 0x30, 0x62, 0xda, 0xfd, 0x42, 0x72, 0x2b, 0x51, 0xce, 0xec, 0x16, 0x2b, - 0x4a, 0x79, 0x25, 0x34, 0xae, 0x52, 0xa5, 0xd4, 0x7d, 0x63, 0xba, 0x5d, 0x82, 0xb3, 0x02, 0x71, - 0x44, 0x19, 0x16, 0xfb, 0xd1, 0x53, 0x58, 0xe0, 0x8e, 0x47, 0x68, 0x97, 0x17, 0x52, 0x12, 0xfa, - 0x60, 0x0c, 0x7a, 0xa0, 0xfd, 0xc6, 0xe1, 0x4e, 0xf4, 0x16, 0x1e, 0x78, 0x66, 0xcf, 0x60, 0xe4, - 0xeb, 0x2e, 0x09, 0xb8, 0xe1, 0x91, 0x20, 0x30, 0x5b, 0xc4, 0x68, 0xf4, 0x39, 0x09, 0x0a, 0xb3, - 0x92, 0xe6, 0xe1, 0x18, 0xcd, 0xeb, 0x63, 0x9f, 0x3f, 0xdd, 0x55, 0x36, 0xac, 0x7b, 0x66, 0x0f, - 0x2b, 0xf4, 0x4b, 0x05, 0xae, 0x0a, 0x2c, 0x7a, 0x0f, 0x45, 0x45, 0x1c, 0x74, 0xa8, 0x1f, 0x90, - 0x11, 0xe6, 0xb9, 0x29, 0x98, 0x37, 0x24, 0xb3, 0x82, 0x47, 0xa8, 0x31, 0x64, 0x19, 0xe1, 0xac, - 0x6f, 0x74, 0xa8, 0xeb, 0x58, 0xfd, 0xc2, 0xbc, 0x24, 0x7b, 0x72, 0x7b, 0xb8, 0xb1, 0x40, 0xd5, - 0x25, 0xa8, 0x36, 0x83, 0x33, 0xec, 0xfa, 0x13, 0xbd, 0x83, 0x5c, 0x9b, 0xd8, 0x2d, 0xc7, 0x6f, - 0x85, 0xac, 0x0b, 0x92, 0x75, 0xe7, 0x76, 0xd6, 0x9a, 0xc2, 0x0d, 0x78, 0x97, 0xda, 0xc3, 0x82, - 0xe2, 0x73, 0x98, 0x15, 0x69, 0x46, 0x05, 0x58, 0xd0, 0x2c, 0x85, 0xc4, 0x56, 0xa2, 0x9c, 0xc6, - 0xe1, 0x27, 0x5a, 0x87, 0x79, 0x4f, 0x12, 0xca, 0x94, 0xa7, 0xb1, 0xfe, 0x2a, 0xfe, 0x98, 0x84, - 0xcc, 0x90, 0xc9, 0x68, 0x1b, 0xb2, 0x22, 0xa4, 0x26, 0xe7, 0xc4, 0xeb, 0xf0, 0x40, 0xd2, 0x2c, - 0xe1, 0x8c, 0x67, 0xf6, 0x3e, 0xd3, 0x22, 0x54, 0x85, 0x65, 0xc7, 0x77, 0xb8, 0x63, 0xba, 0x46, - 0xc3, 0xb4, 0x2e, 0x69, 0xb3, 0xa9, 0xcb, 0x68, 0x42, 0x2d, 0xe4, 0x34, 0xa2, 0xaa, 0x00, 0x68, - 0x0f, 0x04, 0xe5, 0x00, 0x7f, 0x6b, 0x2d, 0x81, 0x67, 0xf6, 0x42, 0xec, 0x13, 0x40, 0x1a, 0x67, - 0x78, 0x5d, 0x97, 0x3b, 0x1d, 0xd7, 0x21, 0x4c, 0xd6, 0x51, 0x12, 0xaf, 0xe8, 0x95, 0x97, 0x83, - 0x05, 0x74, 0x04, 0xeb, 0x32, 0x09, 0x66, 0xc3, 0x25, 0x46, 0xc0, 0x4d, 0xde, 0x0d, 0x0c, 0x71, - 0xc0, 0x44, 0x81, 0xa4, 0xca, 0xb9, 0xdd, 0x7c, 0xa8, 0x55, 0xe4, 0x60, 0x9f, 0xda, 0x04, 0xff, - 0x65, 0xb0, 0xff, 0x4c, 0x6e, 0x17, 0xc2, 0xa0, 0xf8, 0x53, 0x02, 0x96, 0x22, 0x69, 0x98, 0x26, - 0x56, 0xff, 0x83, 0x30, 0x53, 0x86, 0x4d, 0x5c, 0xb3, 0x7f, 0x7b, 0xa4, 0xb2, 0x7a, 0xff, 0x81, - 0xd8, 0x8e, 0x0e, 0x61, 0xdd, 0xa7, 0xbe, 0xd1, 0x34, 0xb9, 0xe9, 0x46, 0x8d, 0x4f, 0xdd, 0x60, - 0xfc, 0xaa, 0x4f, 0xfd, 0x23, 0xb1, 0x7d, 0xc8, 0xf6, 0xea, 0x03, 0xd8, 0x50, 0xd5, 0x4c, 0x99, - 0x11, 0x2d, 0xc1, 0xd2, 0x0a, 0x2c, 0xd7, 0x1d, 0xeb, 0xf2, 0xc8, 0x61, 0x01, 0x57, 0xc5, 0x56, - 0x42, 0x90, 0xc7, 0xb4, 0xeb, 0xdb, 0x98, 0x36, 0x1c, 0x5f, 0xcb, 0x7e, 0x4b, 0xc2, 0x76, 0x9d, - 0x39, 0x94, 0x39, 0xbc, 0x7f, 0x42, 0x4d, 0xbb, 0x6a, 0xba, 0xa6, 0x6f, 0x0d, 0x62, 0xa1, 0x9b, - 0xd3, 0x05, 0x2c, 0x5a, 0x6d, 0xc7, 0xb5, 0x19, 0xf1, 0x75, 0x83, 0x3a, 0x88, 0xad, 0xed, 0x5b, - 0x99, 0x2a, 0xfb, 0x9a, 0xe6, 0xd0, 0xe7, 0xac, 0x8f, 0x07, 0xac, 0x68, 0x13, 0xa0, 0xa3, 0xc0, - 0x0e, 0x09, 0x0a, 0xc9, 0xad, 0x54, 0x39, 0x8d, 0x87, 0x24, 0xc5, 0x63, 0x98, 0x93, 0x50, 0xf4, - 0x02, 0xe6, 0x95, 0x32, 0x6d, 0x48, 0x39, 0xd6, 0x90, 0x88, 0x01, 0x4a, 0x35, 0xd6, 0xb8, 0x62, - 0x1f, 0x96, 0x22, 0x56, 0xa0, 0x3c, 0xa4, 0x2e, 0x49, 0x5f, 0x9f, 0x2c, 0xf1, 0x13, 0x61, 0x98, - 0xfb, 0x20, 0xfa, 0x88, 0x4e, 0xeb, 0x7f, 0xef, 0xe3, 0x2c, 0x56, 0x54, 0x7b, 0xc9, 0xe7, 0x89, - 0xd2, 0x1f, 0x49, 0x78, 0xfc, 0x96, 0x38, 0xad, 0x36, 0x27, 0xf6, 0xb9, 0xc9, 0x5a, 0x84, 0xdf, - 0x1c, 0x73, 0x0b, 0x16, 0xb8, 0xdc, 0x12, 0x68, 0x4f, 0x8f, 0x63, 0xad, 0x98, 0x92, 0xae, 0xa2, - 0xd6, 0x03, 0x15, 0xf7, 0x90, 0xb9, 0xe8, 0xc1, 0xbc, 0x5a, 0x10, 0x8d, 0xe4, 0x4a, 0x52, 0xe9, - 0x72, 0xd7, 0x5f, 0xe8, 0x0b, 0xc8, 0xca, 0x24, 0x85, 0xad, 0x2d, 0x79, 0xc7, 0xa8, 0x67, 0x24, - 0x5a, 0xf7, 0xb3, 0xef, 0x20, 0x3b, 0x6c, 0x47, 0x4c, 0xe4, 0xdf, 0x47, 0x23, 0xbf, 0xff, 0x09, - 0x7c, 0x1e, 0x4e, 0xc0, 0xb7, 0x90, 0xfd, 0x9c, 0x75, 0xac, 0x93, 0x86, 0x0e, 0xf2, 0xa8, 0x77, - 0x89, 0x7b, 0x78, 0x27, 0xfa, 0x46, 0x08, 0x91, 0x57, 0xb9, 0xea, 0xc8, 0x19, 0x2d, 0x13, 0x8d, - 0xbc, 0xf4, 0x08, 0xd2, 0xfb, 0x76, 0xa0, 0x95, 0x17, 0x60, 0xc1, 0x72, 0xbb, 0x01, 0x27, 0x2c, - 0xec, 0xea, 0xfa, 0xb3, 0xf4, 0x6b, 0x12, 0xd2, 0xef, 0x06, 0xfb, 0x1e, 0xc3, 0x52, 0x43, 0xea, - 0x25, 0xcc, 0xd0, 0x33, 0x42, 0xa2, 0x9c, 0xae, 0x26, 0x0b, 0x09, 0x9c, 0x0d, 0x17, 0xe4, 0x35, - 0xf1, 0x29, 0x73, 0x85, 0xbe, 0x84, 0xe5, 0xa6, 0xe9, 0xba, 0xa2, 0xf1, 0x86, 0x7c, 0xa9, 0x3b, - 0xf2, 0xe5, 0x42, 0x02, 0x4d, 0x59, 0x86, 0x3c, 0xb1, 0x03, 0x23, 0x12, 0xa4, 0x59, 0xe9, 0x79, - 0x8e, 0xd8, 0xc1, 0xd9, 0x75, 0x9c, 0xd0, 0x05, 0x6c, 0xba, 0x2c, 0x30, 0x5c, 0x6a, 0xda, 0x06, - 0x23, 0x1d, 0xca, 0xb8, 0x68, 0x6d, 0x02, 0x18, 0xc6, 0xe0, 0xa6, 0x29, 0xe0, 0x8c, 0x33, 0xc7, - 0x6f, 0xa9, 0x29, 0xa0, 0xe8, 0xb2, 0x40, 0xd8, 0x85, 0x43, 0x86, 0x33, 0x49, 0x20, 0x33, 0xf1, - 0x7d, 0x0a, 0x1e, 0x1e, 0xda, 0xc1, 0xcd, 0xe7, 0xef, 0xc6, 0xec, 0xc4, 0xba, 0x91, 0xfc, 0x48, - 0x37, 0x52, 0xf7, 0x73, 0x03, 0x5d, 0xc0, 0x86, 0x4b, 0x2d, 0xd3, 0x75, 0x78, 0xdf, 0xe8, 0x38, - 0xd6, 0xe5, 0xd0, 0x10, 0x32, 0x7b, 0xc7, 0x6c, 0xad, 0x85, 0x44, 0x75, 0xc5, 0xa3, 0x93, 0x76, - 0x01, 0x1b, 0xc4, 0xb7, 0x3b, 0xd4, 0xf1, 0xf9, 0xa8, 0x86, 0xb9, 0xbb, 0x6a, 0x08, 0x89, 0x22, - 0x1a, 0x4a, 0x3f, 0xa4, 0xe0, 0xe1, 0x09, 0x9b, 0x90, 0x8a, 0x6d, 0xc8, 0xea, 0xd8, 0x0f, 0xd5, - 0x3f, 0xce, 0x68, 0x99, 0x8c, 0xc3, 0xf4, 0x39, 0xa9, 0x4e, 0x95, 0x93, 0xf4, 0xc4, 0xa8, 0xbf, - 0x81, 0xc5, 0x30, 0x58, 0x7a, 0xd0, 0xdd, 0x8b, 0x0f, 0xc2, 0x04, 0xaf, 0x2a, 0x27, 0x9a, 0x01, - 0x0f, 0xb8, 0xc6, 0x0e, 0xf0, 0xdc, 0x7d, 0x9a, 0x6d, 0x1d, 0x16, 0x43, 0x15, 0xa2, 0xbb, 0x33, - 0xd2, 0x72, 0xa8, 0xaf, 0x63, 0xa7, 0xbf, 0x10, 0x82, 0xd9, 0x6f, 0xa8, 0x1f, 0x86, 0x4a, 0xfe, - 0x96, 0xc3, 0x66, 0xb7, 0x21, 0xc5, 0x29, 0x3d, 0x6c, 0xaa, 0xcf, 0xd2, 0xef, 0x73, 0xb0, 0x1a, - 0xa3, 0x16, 0x1d, 0x01, 0x88, 0xca, 0x30, 0x9a, 0x62, 0xd8, 0xd0, 0x01, 0xf9, 0x7b, 0xfc, 0x9d, - 0x19, 0x1d, 0x49, 0x6a, 0x33, 0x78, 0x08, 0x89, 0x8e, 0x21, 0xc3, 0xc4, 0x80, 0x62, 0x30, 0x31, - 0xa1, 0x48, 0x53, 0x33, 0xbb, 0x8f, 0x62, 0x89, 0x46, 0x07, 0x19, 0x39, 0x93, 0x5f, 0x63, 0xd1, - 0x7f, 0x60, 0x5e, 0xc0, 0xdc, 0x86, 0x3e, 0x61, 0xdb, 0xb1, 0x2c, 0xc3, 0x77, 0x41, 0x6d, 0x06, - 0x6b, 0x08, 0x3a, 0x87, 0x45, 0x3d, 0x7a, 0xf4, 0x0b, 0x69, 0x09, 0x7f, 0xf6, 0x71, 0x13, 0x40, - 0x6d, 0x06, 0x0f, 0x98, 0x50, 0x0b, 0x96, 0xaf, 0xf4, 0xc5, 0x65, 0xa8, 0xfb, 0xb7, 0x00, 0x13, - 0xc6, 0x8b, 0x29, 0x2f, 0xb9, 0xda, 0x0c, 0xce, 0x5d, 0x45, 0xb6, 0xa2, 0x5d, 0x48, 0x59, 0x76, - 0xa0, 0x9f, 0x36, 0x9b, 0xb1, 0xe4, 0x83, 0x4b, 0xa8, 0x36, 0x83, 0xc5, 0x66, 0x74, 0x08, 0x29, - 0x62, 0x07, 0xfa, 0xe1, 0xf2, 0xcf, 0x58, 0xcc, 0xa4, 0x6e, 0x29, 0x68, 0x88, 0xa2, 0x71, 0x59, - 0x50, 0x58, 0x9c, 0x40, 0x33, 0xe9, 0x4c, 0x08, 0x1a, 0x97, 0x05, 0xe8, 0x19, 0xa4, 0x7a, 0x76, - 0xa0, 0x67, 0x80, 0x78, 0x0f, 0x06, 0xd7, 0xa3, 0xb8, 0x07, 0x05, 0xae, 0x67, 0x07, 0xa8, 0x0e, - 0xf9, 0x9e, 0x1d, 0x18, 0xa4, 0xd7, 0x21, 0xcc, 0xf1, 0x88, 0xcf, 0x4d, 0x57, 0x5f, 0x14, 0xd3, - 0x91, 0x8c, 0xa1, 0xab, 0x8b, 0x30, 0xaf, 0x07, 0xea, 0x9f, 0xe7, 0x60, 0x49, 0xf7, 0x11, 0x5d, - 0xf6, 0x3e, 0xac, 0xc9, 0x2e, 0xd2, 0x08, 0x5d, 0xb9, 0x9e, 0x22, 0x12, 0xe5, 0xdc, 0x0d, 0x35, - 0x13, 0xa1, 0xa8, 0xc4, 0x44, 0x42, 0xde, 0xeb, 0xab, 0xee, 0xf8, 0x02, 0xfa, 0x6a, 0x4c, 0x9f, - 0x9e, 0x84, 0xef, 0xda, 0xe9, 0xa3, 0xec, 0x83, 0x43, 0xbc, 0xa4, 0xde, 0x8e, 0x21, 0xab, 0x9a, - 0x1e, 0xb6, 0x6f, 0x7d, 0xc4, 0xe2, 0xac, 0x37, 0xfc, 0x47, 0x86, 0x05, 0x79, 0xf5, 0x26, 0xe1, - 0x6d, 0x46, 0x39, 0x77, 0x1d, 0xbf, 0xa5, 0xcf, 0xe0, 0xf3, 0x29, 0x02, 0x22, 0xdf, 0xac, 0xe7, - 0x03, 0xa4, 0xf2, 0x1c, 0x2f, 0xb3, 0xa8, 0x18, 0xd9, 0xb0, 0xda, 0x26, 0xa6, 0xcb, 0xdb, 0x86, - 0xd5, 0x26, 0xd6, 0x65, 0x68, 0xb2, 0xca, 0xf5, 0xbf, 0xa6, 0xd0, 0x53, 0x93, 0xe8, 0x7d, 0x01, - 0xd6, 0x5e, 0xac, 0xb4, 0x47, 0x45, 0xc5, 0xb7, 0xb0, 0x16, 0x6b, 0x0f, 0xfa, 0x2b, 0x88, 0x87, - 0xab, 0xc1, 0xe9, 0x25, 0xf1, 0xc3, 0xf7, 0x61, 0xda, 0x33, 0x7b, 0xe7, 0x52, 0x80, 0xfe, 0x06, - 0x19, 0xb9, 0x64, 0xc8, 0xb7, 0x9f, 0x2c, 0xe3, 0x24, 0x06, 0x29, 0xc2, 0x42, 0x52, 0x3c, 0x87, - 0x95, 0x31, 0x03, 0xd0, 0xff, 0x47, 0xc6, 0xc7, 0xc4, 0x14, 0xa3, 0x41, 0x64, 0xb8, 0xfc, 0xf7, - 0x48, 0x77, 0xd6, 0xc6, 0x2e, 0x43, 0xe6, 0xf5, 0xe9, 0x59, 0xfd, 0x70, 0xff, 0xf8, 0xe8, 0xf8, - 0xf0, 0x20, 0x3f, 0x23, 0x04, 0xf8, 0xd5, 0xeb, 0xd3, 0x03, 0x03, 0xbf, 0xaa, 0x1e, 0x9f, 0xe6, - 0x13, 0xd5, 0x27, 0xb0, 0xe6, 0xd0, 0x48, 0xd0, 0x54, 0xcc, 0xaa, 0x28, 0x12, 0xb4, 0xba, 0xb0, - 0xa0, 0x9e, 0x68, 0xcc, 0x4b, 0x53, 0x9e, 0xfe, 0x19, 0x00, 0x00, 0xff, 0xff, 0xae, 0xd3, 0x19, - 0x42, 0x5a, 0x13, 0x00, 0x00, -} diff --git a/internal/resolver/config_selector.go b/internal/resolver/config_selector.go new file mode 100644 index 000000000000..f0603871c93a --- /dev/null +++ b/internal/resolver/config_selector.go @@ -0,0 +1,167 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package resolver provides internal resolver-related functionality. +package resolver + +import ( + "context" + "sync" + + "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/resolver" +) + +// ConfigSelector controls what configuration to use for every RPC. +type ConfigSelector interface { + // Selects the configuration for the RPC, or terminates it using the error. + // This error will be converted by the gRPC library to a status error with + // code UNKNOWN if it is not returned as a status error. + SelectConfig(RPCInfo) (*RPCConfig, error) +} + +// RPCInfo contains RPC information needed by a ConfigSelector. +type RPCInfo struct { + // Context is the user's context for the RPC and contains headers and + // application timeout. It is passed for interception purposes and for + // efficiency reasons. SelectConfig should not be blocking. + Context context.Context + Method string // i.e. "/Service/Method" +} + +// RPCConfig describes the configuration to use for each RPC. +type RPCConfig struct { + // The context to use for the remainder of the RPC; can pass info to LB + // policy or affect timeout or metadata. + Context context.Context + MethodConfig serviceconfig.MethodConfig // configuration to use for this RPC + OnCommitted func() // Called when the RPC has been committed (retries no longer possible) + Interceptor ClientInterceptor +} + +// ClientStream is the same as grpc.ClientStream, but defined here for circular +// dependency reasons. +type ClientStream interface { + // Header returns the header metadata received from the server if there + // is any. It blocks if the metadata is not ready to read. + Header() (metadata.MD, error) + // Trailer returns the trailer metadata from the server, if there is any. + // It must only be called after stream.CloseAndRecv has returned, or + // stream.Recv has returned a non-nil error (including io.EOF). + Trailer() metadata.MD + // CloseSend closes the send direction of the stream. It closes the stream + // when non-nil error is met. It is also not safe to call CloseSend + // concurrently with SendMsg. + CloseSend() error + // Context returns the context for this stream. + // + // It should not be called until after Header or RecvMsg has returned. Once + // called, subsequent client-side retries are disabled. + Context() context.Context + // SendMsg is generally called by generated code. On error, SendMsg aborts + // the stream. If the error was generated by the client, the status is + // returned directly; otherwise, io.EOF is returned and the status of + // the stream may be discovered using RecvMsg. + // + // SendMsg blocks until: + // - There is sufficient flow control to schedule m with the transport, or + // - The stream is done, or + // - The stream breaks. + // + // SendMsg does not wait until the message is received by the server. An + // untimely stream closure may result in lost messages. To ensure delivery, + // users should ensure the RPC completed successfully using RecvMsg. + // + // It is safe to have a goroutine calling SendMsg and another goroutine + // calling RecvMsg on the same stream at the same time, but it is not safe + // to call SendMsg on the same stream in different goroutines. It is also + // not safe to call CloseSend concurrently with SendMsg. + SendMsg(m any) error + // RecvMsg blocks until it receives a message into m or the stream is + // done. It returns io.EOF when the stream completes successfully. On + // any other error, the stream is aborted and the error contains the RPC + // status. + // + // It is safe to have a goroutine calling SendMsg and another goroutine + // calling RecvMsg on the same stream at the same time, but it is not + // safe to call RecvMsg on the same stream in different goroutines. + RecvMsg(m any) error +} + +// ClientInterceptor is an interceptor for gRPC client streams. +type ClientInterceptor interface { + // NewStream produces a ClientStream for an RPC which may optionally use + // the provided function to produce a stream for delegation. Note: + // RPCInfo.Context should not be used (will be nil). + // + // done is invoked when the RPC is finished using its connection, or could + // not be assigned a connection. RPC operations may still occur on + // ClientStream after done is called, since the interceptor is invoked by + // application-layer operations. done must never be nil when called. + NewStream(ctx context.Context, ri RPCInfo, done func(), newStream func(ctx context.Context, done func()) (ClientStream, error)) (ClientStream, error) +} + +// ServerInterceptor is an interceptor for incoming RPC's on gRPC server side. +type ServerInterceptor interface { + // AllowRPC checks if an incoming RPC is allowed to proceed based on + // information about connection RPC was received on, and HTTP Headers. This + // information will be piped into context. + AllowRPC(ctx context.Context) error // TODO: Make this a real interceptor for filters such as rate limiting. +} + +type csKeyType string + +const csKey = csKeyType("grpc.internal.resolver.configSelector") + +// SetConfigSelector sets the config selector in state and returns the new +// state. +func SetConfigSelector(state resolver.State, cs ConfigSelector) resolver.State { + state.Attributes = state.Attributes.WithValue(csKey, cs) + return state +} + +// GetConfigSelector retrieves the config selector from state, if present, and +// returns it or nil if absent. +func GetConfigSelector(state resolver.State) ConfigSelector { + cs, _ := state.Attributes.Value(csKey).(ConfigSelector) + return cs +} + +// SafeConfigSelector allows for safe switching of ConfigSelector +// implementations such that previous values are guaranteed to not be in use +// when UpdateConfigSelector returns. +type SafeConfigSelector struct { + mu sync.RWMutex + cs ConfigSelector +} + +// UpdateConfigSelector swaps to the provided ConfigSelector and blocks until +// all uses of the previous ConfigSelector have completed. +func (scs *SafeConfigSelector) UpdateConfigSelector(cs ConfigSelector) { + scs.mu.Lock() + defer scs.mu.Unlock() + scs.cs = cs +} + +// SelectConfig defers to the current ConfigSelector in scs. +func (scs *SafeConfigSelector) SelectConfig(r RPCInfo) (*RPCConfig, error) { + scs.mu.RLock() + defer scs.mu.RUnlock() + return scs.cs.SelectConfig(r) +} diff --git a/internal/resolver/config_selector_test.go b/internal/resolver/config_selector_test.go new file mode 100644 index 000000000000..7a8a5dbd693f --- /dev/null +++ b/internal/resolver/config_selector_test.go @@ -0,0 +1,156 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package resolver + +import ( + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/serviceconfig" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +type fakeConfigSelector struct { + selectConfig func(RPCInfo) (*RPCConfig, error) +} + +func (f *fakeConfigSelector) SelectConfig(r RPCInfo) (*RPCConfig, error) { + return f.selectConfig(r) +} + +func (s) TestSafeConfigSelector(t *testing.T) { + testRPCInfo := RPCInfo{Method: "test method"} + + retChan1 := make(chan *RPCConfig) + retChan2 := make(chan *RPCConfig) + defer close(retChan1) + defer close(retChan2) + + one := 1 + two := 2 + + resp1 := &RPCConfig{MethodConfig: serviceconfig.MethodConfig{MaxReqSize: &one}} + resp2 := &RPCConfig{MethodConfig: serviceconfig.MethodConfig{MaxReqSize: &two}} + + cs1Called := make(chan struct{}, 1) + cs2Called := make(chan struct{}, 1) + + cs1 := &fakeConfigSelector{ + selectConfig: func(r RPCInfo) (*RPCConfig, error) { + cs1Called <- struct{}{} + if diff := cmp.Diff(r, testRPCInfo); diff != "" { + t.Errorf("SelectConfig(%v) called; want %v\n Diffs:\n%s", r, testRPCInfo, diff) + } + return <-retChan1, nil + }, + } + cs2 := &fakeConfigSelector{ + selectConfig: func(r RPCInfo) (*RPCConfig, error) { + cs2Called <- struct{}{} + if diff := cmp.Diff(r, testRPCInfo); diff != "" { + t.Errorf("SelectConfig(%v) called; want %v\n Diffs:\n%s", r, testRPCInfo, diff) + } + return <-retChan2, nil + }, + } + + scs := &SafeConfigSelector{} + scs.UpdateConfigSelector(cs1) + + cs1Returned := make(chan struct{}) + go func() { + got, err := scs.SelectConfig(testRPCInfo) // blocks until send to retChan1 + if err != nil || got != resp1 { + t.Errorf("SelectConfig(%v) = %v, %v; want %v, nil", testRPCInfo, got, err, resp1) + } + close(cs1Returned) + }() + + // cs1 is blocked but should be called + select { + case <-time.After(500 * time.Millisecond): + t.Fatalf("timed out waiting for cs1 to be called") + case <-cs1Called: + } + + // swap in cs2 now that cs1 is called + csSwapped := make(chan struct{}) + go func() { + // wait awhile first to ensure cs1 could be called below. + time.Sleep(50 * time.Millisecond) + scs.UpdateConfigSelector(cs2) // Blocks until cs1 done + close(csSwapped) + }() + + // Allow cs1 to return and cs2 to eventually be swapped in. + retChan1 <- resp1 + + cs1Done := false // set when cs2 is first called + for dl := time.Now().Add(150 * time.Millisecond); !time.Now().After(dl); { + gotConfigChan := make(chan *RPCConfig, 1) + go func() { + cfg, _ := scs.SelectConfig(testRPCInfo) + gotConfigChan <- cfg + }() + select { + case <-time.After(500 * time.Millisecond): + t.Fatalf("timed out waiting for cs1 or cs2 to be called") + case <-cs1Called: + // Initially, before swapping to cs2, cs1 should be called + retChan1 <- resp1 + go func() { <-gotConfigChan }() + if cs1Done { + t.Fatalf("cs1 called after cs2") + } + case <-cs2Called: + // Success! the new config selector is being called + if !cs1Done { + select { + case <-csSwapped: + case <-time.After(50 * time.Millisecond): + t.Fatalf("timed out waiting for UpdateConfigSelector to return") + } + select { + case <-cs1Returned: + case <-time.After(50 * time.Millisecond): + t.Fatalf("timed out waiting for cs1 to return") + } + cs1Done = true + } + retChan2 <- resp2 + got := <-gotConfigChan + if diff := cmp.Diff(got, resp2); diff != "" { + t.Fatalf("SelectConfig(%v) = %v; want %v\n Diffs:\n%s", testRPCInfo, got, resp2, diff) + } + } + time.Sleep(10 * time.Millisecond) + } + if !cs1Done { + t.Fatalf("timed out waiting for cs2 to be called") + } +} diff --git a/internal/resolver/dns/dns_resolver.go b/internal/resolver/dns/dns_resolver.go index 304235566589..99e1e5b36c89 100644 --- a/internal/resolver/dns/dns_resolver.go +++ b/internal/resolver/dns/dns_resolver.go @@ -34,6 +34,7 @@ import ( grpclbstate "google.golang.org/grpc/balancer/grpclb/state" "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal/backoff" "google.golang.org/grpc/internal/envconfig" "google.golang.org/grpc/internal/grpcrand" "google.golang.org/grpc/resolver" @@ -46,6 +47,13 @@ var EnableSRVLookups = false var logger = grpclog.Component("dns") +// Globals to stub out in tests. TODO: Perhaps these two can be combined into a +// single variable for testing the resolver? +var ( + newTimer = time.NewTimer + newTimerDNSResRate = time.NewTimer +) + func init() { resolver.Register(NewBuilder()) } @@ -54,7 +62,8 @@ const ( defaultPort = "443" defaultDNSSvrPort = "53" golang = "GO" - // txtPrefix is the prefix string to be prepended to the host name for txt record lookup. + // txtPrefix is the prefix string to be prepended to the host name for txt + // record lookup. txtPrefix = "_grpc_config." // In DNS, service config is encoded in a TXT record via the mechanism // described in RFC-1464 using the attribute name grpc_config. @@ -78,14 +87,14 @@ var ( minDNSResRate = 30 * time.Second ) -var customAuthorityDialler = func(authority string) func(ctx context.Context, network, address string) (net.Conn, error) { - return func(ctx context.Context, network, address string) (net.Conn, error) { +var addressDialer = func(address string) func(context.Context, string, string) (net.Conn, error) { + return func(ctx context.Context, network, _ string) (net.Conn, error) { var dialer net.Dialer - return dialer.DialContext(ctx, network, authority) + return dialer.DialContext(ctx, network, address) } } -var customAuthorityResolver = func(authority string) (netResolver, error) { +var newNetResolver = func(authority string) (netResolver, error) { host, port, err := parseTarget(authority, defaultDNSSvrPort) if err != nil { return nil, err @@ -95,7 +104,7 @@ var customAuthorityResolver = func(authority string) (netResolver, error) { return &net.Resolver{ PreferGo: true, - Dial: customAuthorityDialler(authorityWithPort), + Dial: addressDialer(authorityWithPort), }, nil } @@ -106,9 +115,10 @@ func NewBuilder() resolver.Builder { type dnsBuilder struct{} -// Build creates and starts a DNS resolver that watches the name resolution of the target. +// Build creates and starts a DNS resolver that watches the name resolution of +// the target. func (b *dnsBuilder) Build(target resolver.Target, cc resolver.ClientConn, opts resolver.BuildOptions) (resolver.Resolver, error) { - host, port, err := parseTarget(target.Endpoint, defaultPort) + host, port, err := parseTarget(target.Endpoint(), defaultPort) if err != nil { return nil, err } @@ -132,10 +142,10 @@ func (b *dnsBuilder) Build(target resolver.Target, cc resolver.ClientConn, opts disableServiceConfig: opts.DisableServiceConfig, } - if target.Authority == "" { + if target.URL.Host == "" { d.resolver = defaultResolver } else { - d.resolver, err = customAuthorityResolver(target.Authority) + d.resolver, err = newNetResolver(target.URL.Host) if err != nil { return nil, err } @@ -143,7 +153,6 @@ func (b *dnsBuilder) Build(target resolver.Target, cc resolver.ClientConn, opts d.wg.Add(1) go d.watcher() - d.ResolveNow(resolver.ResolveNowOptions{}) return d, nil } @@ -173,19 +182,22 @@ type dnsResolver struct { ctx context.Context cancel context.CancelFunc cc resolver.ClientConn - // rn channel is used by ResolveNow() to force an immediate resolution of the target. + // rn channel is used by ResolveNow() to force an immediate resolution of the + // target. rn chan struct{} - // wg is used to enforce Close() to return after the watcher() goroutine has finished. - // Otherwise, data race will be possible. [Race Example] in dns_resolver_test we - // replace the real lookup functions with mocked ones to facilitate testing. - // If Close() doesn't wait for watcher() goroutine finishes, race detector sometimes - // will warns lookup (READ the lookup function pointers) inside watcher() goroutine - // has data race with replaceNetFunc (WRITE the lookup function pointers). + // wg is used to enforce Close() to return after the watcher() goroutine has + // finished. Otherwise, data race will be possible. [Race Example] in + // dns_resolver_test we replace the real lookup functions with mocked ones to + // facilitate testing. If Close() doesn't wait for watcher() goroutine + // finishes, race detector sometimes will warns lookup (READ the lookup + // function pointers) inside watcher() goroutine has data race with + // replaceNetFunc (WRITE the lookup function pointers). wg sync.WaitGroup disableServiceConfig bool } -// ResolveNow invoke an immediate resolution of the target that this dnsResolver watches. +// ResolveNow invoke an immediate resolution of the target that this +// dnsResolver watches. func (d *dnsResolver) ResolveNow(resolver.ResolveNowOptions) { select { case d.rn <- struct{}{}: @@ -201,28 +213,39 @@ func (d *dnsResolver) Close() { func (d *dnsResolver) watcher() { defer d.wg.Done() + backoffIndex := 1 for { - select { - case <-d.ctx.Done(): - return - case <-d.rn: - } - state, err := d.lookup() if err != nil { + // Report error to the underlying grpc.ClientConn. d.cc.ReportError(err) } else { - d.cc.UpdateState(*state) + err = d.cc.UpdateState(*state) } - // Sleep to prevent excessive re-resolutions. Incoming resolution requests - // will be queued in d.rn. - t := time.NewTimer(minDNSResRate) + var timer *time.Timer + if err == nil { + // Success resolving, wait for the next ResolveNow. However, also wait 30 + // seconds at the very least to prevent constantly re-resolving. + backoffIndex = 1 + timer = newTimerDNSResRate(minDNSResRate) + select { + case <-d.ctx.Done(): + timer.Stop() + return + case <-d.rn: + } + } else { + // Poll on an error found in DNS Resolver or an error received from + // ClientConn. + timer = newTimer(backoff.DefaultExponential.Backoff(backoffIndex)) + backoffIndex++ + } select { - case <-t.C: case <-d.ctx.Done(): - t.Stop() + timer.Stop() return + case <-timer.C: } } } @@ -260,18 +283,14 @@ func (d *dnsResolver) lookupSRV() ([]resolver.Address, error) { return newAddrs, nil } -var filterError = func(err error) error { - if dnsErr, ok := err.(*net.DNSError); ok && !dnsErr.IsTimeout && !dnsErr.IsTemporary { +func handleDNSError(err error, lookupType string) error { + dnsErr, ok := err.(*net.DNSError) + if ok && !dnsErr.IsTimeout && !dnsErr.IsTemporary { // Timeouts and temporary errors should be communicated to gRPC to // attempt another DNS query (with backoff). Other errors should be // suppressed (they may represent the absence of a TXT record). return nil } - return err -} - -func handleDNSError(err error, lookupType string) error { - err = filterError(err) if err != nil { err = fmt.Errorf("dns: %v record lookup error: %v", lookupType, err) logger.Info(err) @@ -295,10 +314,12 @@ func (d *dnsResolver) lookupTXT() *serviceconfig.ParseResult { res += s } - // TXT record must have "grpc_config=" attribute in order to be used as service config. + // TXT record must have "grpc_config=" attribute in order to be used as + // service config. if !strings.HasPrefix(res, txtAttribute) { logger.Warningf("dns: TXT record %v missing %v attribute", res, txtAttribute) - // This is not an error; it is the equivalent of not having a service config. + // This is not an error; it is the equivalent of not having a service + // config. return nil } sc := canaryingSC(strings.TrimPrefix(res, txtAttribute)) @@ -306,12 +327,12 @@ func (d *dnsResolver) lookupTXT() *serviceconfig.ParseResult { } func (d *dnsResolver) lookupHost() ([]resolver.Address, error) { - var newAddrs []resolver.Address addrs, err := d.resolver.LookupHost(d.ctx, d.host) if err != nil { err = handleDNSError(err, "A") return nil, err } + newAddrs := make([]resolver.Address, 0, len(addrs)) for _, a := range addrs { ip, ok := formatIP(a) if !ok { @@ -340,9 +361,10 @@ func (d *dnsResolver) lookup() (*resolver.State, error) { return &state, nil } -// formatIP returns ok = false if addr is not a valid textual representation of an IP address. -// If addr is an IPv4 address, return the addr and ok = true. -// If addr is an IPv6 address, return the addr enclosed in square brackets and ok = true. +// formatIP returns ok = false if addr is not a valid textual representation of +// an IP address. If addr is an IPv4 address, return the addr and ok = true. +// If addr is an IPv6 address, return the addr enclosed in square brackets and +// ok = true. func formatIP(addr string) (addrIP string, ok bool) { ip := net.ParseIP(addr) if ip == nil { @@ -354,10 +376,10 @@ func formatIP(addr string) (addrIP string, ok bool) { return "[" + addr + "]", true } -// parseTarget takes the user input target string and default port, returns formatted host and port info. -// If target doesn't specify a port, set the port to be the defaultPort. -// If target is in IPv6 format and host-name is enclosed in square brackets, brackets -// are stripped when setting the host. +// parseTarget takes the user input target string and default port, returns +// formatted host and port info. If target doesn't specify a port, set the port +// to be the defaultPort. If target is in IPv6 format and host-name is enclosed +// in square brackets, brackets are stripped when setting the host. // examples: // target: "www.google.com" defaultPort: "443" returns host: "www.google.com", port: "443" // target: "ipv4-host:80" defaultPort: "443" returns host: "ipv4-host", port: "80" @@ -373,12 +395,14 @@ func parseTarget(target, defaultPort string) (host, port string, err error) { } if host, port, err = net.SplitHostPort(target); err == nil { if port == "" { - // If the port field is empty (target ends with colon), e.g. "[::1]:", this is an error. + // If the port field is empty (target ends with colon), e.g. "[::1]:", + // this is an error. return "", "", errEndsWithColon } // target has port, i.e ipv4-host:port, [ipv6-host]:port, host-name:port if host == "" { - // Keep consistent with net.Dial(): If the host is empty, as in ":80", the local system is assumed. + // Keep consistent with net.Dial(): If the host is empty, as in ":80", + // the local system is assumed. host = "localhost" } return host, port, nil diff --git a/internal/resolver/dns/dns_resolver_test.go b/internal/resolver/dns/dns_resolver_test.go index 1c8469a275a7..a66ffffd3ce1 100644 --- a/internal/resolver/dns/dns_resolver_test.go +++ b/internal/resolver/dns/dns_resolver_test.go @@ -30,9 +30,13 @@ import ( "testing" "time" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "google.golang.org/grpc/balancer" grpclbstate "google.golang.org/grpc/balancer/grpclb/state" "google.golang.org/grpc/internal/envconfig" "google.golang.org/grpc/internal/leakcheck" + "google.golang.org/grpc/internal/testutils" "google.golang.org/grpc/resolver" "google.golang.org/grpc/serviceconfig" ) @@ -41,13 +45,15 @@ func TestMain(m *testing.M) { // Set a non-zero duration only for tests which are actually testing that // feature. replaceDNSResRate(time.Duration(0)) // No nead to clean up since we os.Exit - replaceNetFunc(nil) // No nead to clean up since we os.Exit + overrideDefaultResolver(false) // No nead to clean up since we os.Exit code := m.Run() os.Exit(code) } const ( - txtBytesLimit = 255 + txtBytesLimit = 255 + defaultTestTimeout = 10 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond ) type testClientConn struct { @@ -57,13 +63,17 @@ type testClientConn struct { state resolver.State updateStateCalls int errChan chan error + updateStateErr error } -func (t *testClientConn) UpdateState(s resolver.State) { +func (t *testClientConn) UpdateState(s resolver.State) error { t.m1.Lock() defer t.m1.Unlock() t.state = s t.updateStateCalls++ + // This error determines whether DNS Resolver actually decides to exponentially backoff or not. + // This can be any error. + return t.updateStateErr } func (t *testClientConn) getState() (resolver.State, int) { @@ -99,12 +109,12 @@ type testResolver struct { // A write to this channel is made when this resolver receives a resolution // request. Tests can rely on reading from this channel to be notified about // resolution requests instead of sleeping for a predefined period of time. - ch chan struct{} + lookupHostCh *testutils.Channel } func (tr *testResolver) LookupHost(ctx context.Context, host string) ([]string, error) { - if tr.ch != nil { - tr.ch <- struct{}{} + if tr.lookupHostCh != nil { + tr.lookupHostCh.Send(nil) } return hostLookup(host) } @@ -117,9 +127,17 @@ func (*testResolver) LookupTXT(ctx context.Context, host string) ([]string, erro return txtLookup(host) } -func replaceNetFunc(ch chan struct{}) func() { +// overrideDefaultResolver overrides the defaultResolver used by the code with +// an instance of the testResolver. pushOnLookup controls whether the +// testResolver created here pushes lookupHost events on its channel. +func overrideDefaultResolver(pushOnLookup bool) func() { oldResolver := defaultResolver - defaultResolver = &testResolver{ch: ch} + + var lookupHostCh *testutils.Channel + if pushOnLookup { + lookupHostCh = testutils.NewChannel() + } + defaultResolver = &testResolver{lookupHostCh: lookupHostCh} return func() { defaultResolver = oldResolver @@ -669,6 +687,13 @@ func TestResolve(t *testing.T) { func testDNSResolver(t *testing.T) { defer leakcheck.Check(t) + defer func(nt func(d time.Duration) *time.Timer) { + newTimer = nt + }(newTimer) + newTimer = func(_ time.Duration) *time.Timer { + // Will never fire on its own, will protect from triggering exponential backoff. + return time.NewTimer(time.Hour) + } tests := []struct { target string addrWant []resolver.Address @@ -709,7 +734,7 @@ func testDNSResolver(t *testing.T) { for _, a := range tests { b := NewBuilder() cc := &testClientConn{target: a.target} - r, err := b.Build(resolver.Target{Endpoint: a.target}, cc, resolver.BuildOptions{}) + r, err := b.Build(resolver.Target{URL: *testutils.MustParseURL(fmt.Sprintf("scheme:///%s", a.target))}, cc, resolver.BuildOptions{}) if err != nil { t.Fatalf("%v\n", err) } @@ -725,7 +750,7 @@ func testDNSResolver(t *testing.T) { if cnt == 0 { t.Fatalf("UpdateState not called after 2s; aborting") } - if !reflect.DeepEqual(a.addrWant, state.Addresses) { + if !cmp.Equal(a.addrWant, state.Addresses, cmpopts.EquateEmpty()) { t.Errorf("Resolved addresses of target: %q = %+v, want %+v", a.target, state.Addresses, a.addrWant) } sc := scFromState(state) @@ -736,12 +761,151 @@ func testDNSResolver(t *testing.T) { } } +// DNS Resolver immediately starts polling on an error from grpc. This should continue until the ClientConn doesn't +// send back an error from updating the DNS Resolver's state. +func TestDNSResolverExponentialBackoff(t *testing.T) { + defer leakcheck.Check(t) + defer func(nt func(d time.Duration) *time.Timer) { + newTimer = nt + }(newTimer) + timerChan := testutils.NewChannel() + newTimer = func(d time.Duration) *time.Timer { + // Will never fire on its own, allows this test to call timer immediately. + t := time.NewTimer(time.Hour) + timerChan.Send(t) + return t + } + tests := []struct { + name string + target string + addrWant []resolver.Address + scWant string + }{ + { + "happy case default port", + "foo.bar.com", + []resolver.Address{{Addr: "1.2.3.4" + colonDefaultPort}, {Addr: "5.6.7.8" + colonDefaultPort}}, + generateSC("foo.bar.com"), + }, + { + "happy case specified port", + "foo.bar.com:1234", + []resolver.Address{{Addr: "1.2.3.4:1234"}, {Addr: "5.6.7.8:1234"}}, + generateSC("foo.bar.com"), + }, + { + "happy case another default port", + "srv.ipv4.single.fake", + []resolver.Address{{Addr: "2.4.6.8" + colonDefaultPort}}, + generateSC("srv.ipv4.single.fake"), + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + b := NewBuilder() + cc := &testClientConn{target: test.target} + // Cause ClientConn to return an error. + cc.updateStateErr = balancer.ErrBadResolverState + r, err := b.Build(resolver.Target{URL: *testutils.MustParseURL(fmt.Sprintf("scheme:///%s", test.target))}, cc, resolver.BuildOptions{}) + if err != nil { + t.Fatalf("Error building resolver for target %v: %v", test.target, err) + } + var state resolver.State + var cnt int + for i := 0; i < 2000; i++ { + state, cnt = cc.getState() + if cnt > 0 { + break + } + time.Sleep(time.Millisecond) + } + if cnt == 0 { + t.Fatalf("UpdateState not called after 2s; aborting") + } + if !reflect.DeepEqual(test.addrWant, state.Addresses) { + t.Errorf("Resolved addresses of target: %q = %+v, want %+v", test.target, state.Addresses, test.addrWant) + } + sc := scFromState(state) + if test.scWant != sc { + t.Errorf("Resolved service config of target: %q = %+v, want %+v", test.target, sc, test.scWant) + } + ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer ctxCancel() + // Cause timer to go off 10 times, and see if it calls updateState() correctly. + for i := 0; i < 10; i++ { + timer, err := timerChan.Receive(ctx) + if err != nil { + t.Fatalf("Error receiving timer from mock NewTimer call: %v", err) + } + timerPointer := timer.(*time.Timer) + timerPointer.Reset(0) + } + // Poll to see if DNS Resolver updated state the correct number of times, which allows time for the DNS Resolver to call + // ClientConn update state. + deadline := time.Now().Add(defaultTestTimeout) + for { + cc.m1.Lock() + got := cc.updateStateCalls + cc.m1.Unlock() + if got == 11 { + break + } + + if time.Now().After(deadline) { + t.Fatalf("Exponential backoff is not working as expected - should update state 11 times instead of %d", got) + } + + time.Sleep(time.Millisecond) + } + + // Update resolver.ClientConn to not return an error anymore - this should stop it from backing off. + cc.updateStateErr = nil + timer, err := timerChan.Receive(ctx) + if err != nil { + t.Fatalf("Error receiving timer from mock NewTimer call: %v", err) + } + timerPointer := timer.(*time.Timer) + timerPointer.Reset(0) + // Poll to see if DNS Resolver updated state the correct number of times, which allows time for the DNS Resolver to call + // ClientConn update state the final time. The DNS Resolver should then stop polling. + deadline = time.Now().Add(defaultTestTimeout) + for { + cc.m1.Lock() + got := cc.updateStateCalls + cc.m1.Unlock() + if got == 12 { + break + } + + if time.Now().After(deadline) { + t.Fatalf("Exponential backoff is not working as expected - should stop backing off at 12 total UpdateState calls instead of %d", got) + } + + _, err := timerChan.ReceiveOrFail() + if err { + t.Fatalf("Should not poll again after Client Conn stops returning error.") + } + + time.Sleep(time.Millisecond) + } + r.Close() + }) + } +} + func testDNSResolverWithSRV(t *testing.T) { EnableSRVLookups = true defer func() { EnableSRVLookups = false }() defer leakcheck.Check(t) + defer func(nt func(d time.Duration) *time.Timer) { + newTimer = nt + }(newTimer) + newTimer = func(_ time.Duration) *time.Timer { + // Will never fire on its own, will protect from triggering exponential backoff. + return time.NewTimer(time.Hour) + } tests := []struct { target string addrWant []resolver.Address @@ -797,7 +961,7 @@ func testDNSResolverWithSRV(t *testing.T) { for _, a := range tests { b := NewBuilder() cc := &testClientConn{target: a.target} - r, err := b.Build(resolver.Target{Endpoint: a.target}, cc, resolver.BuildOptions{}) + r, err := b.Build(resolver.Target{URL: *testutils.MustParseURL(fmt.Sprintf("scheme:///%s", a.target))}, cc, resolver.BuildOptions{}) if err != nil { t.Fatalf("%v\n", err) } @@ -814,7 +978,7 @@ func testDNSResolverWithSRV(t *testing.T) { if cnt == 0 { t.Fatalf("UpdateState not called after 2s; aborting") } - if !reflect.DeepEqual(a.addrWant, state.Addresses) { + if !cmp.Equal(a.addrWant, state.Addresses, cmpopts.EquateEmpty()) { t.Errorf("Resolved addresses of target: %q = %+v, want %+v", a.target, state.Addresses, a.addrWant) } gs := grpclbstate.Get(state) @@ -855,6 +1019,13 @@ func mutateTbl(target string) func() { func testDNSResolveNow(t *testing.T) { defer leakcheck.Check(t) + defer func(nt func(d time.Duration) *time.Timer) { + newTimer = nt + }(newTimer) + newTimer = func(_ time.Duration) *time.Timer { + // Will never fire on its own, will protect from triggering exponential backoff. + return time.NewTimer(time.Hour) + } tests := []struct { target string addrWant []resolver.Address @@ -874,7 +1045,7 @@ func testDNSResolveNow(t *testing.T) { for _, a := range tests { b := NewBuilder() cc := &testClientConn{target: a.target} - r, err := b.Build(resolver.Target{Endpoint: a.target}, cc, resolver.BuildOptions{}) + r, err := b.Build(resolver.Target{URL: *testutils.MustParseURL(fmt.Sprintf("scheme:///%s", a.target))}, cc, resolver.BuildOptions{}) if err != nil { t.Fatalf("%v\n", err) } @@ -926,6 +1097,13 @@ const colonDefaultPort = ":" + defaultPort func testIPResolver(t *testing.T) { defer leakcheck.Check(t) + defer func(nt func(d time.Duration) *time.Timer) { + newTimer = nt + }(newTimer) + newTimer = func(_ time.Duration) *time.Timer { + // Will never fire on its own, will protect from triggering exponential backoff. + return time.NewTimer(time.Hour) + } tests := []struct { target string want []resolver.Address @@ -945,7 +1123,7 @@ func testIPResolver(t *testing.T) { for _, v := range tests { b := NewBuilder() cc := &testClientConn{target: v.target} - r, err := b.Build(resolver.Target{Endpoint: v.target}, cc, resolver.BuildOptions{}) + r, err := b.Build(resolver.Target{URL: *testutils.MustParseURL(fmt.Sprintf("scheme:///%s", v.target))}, cc, resolver.BuildOptions{}) if err != nil { t.Fatalf("%v\n", err) } @@ -975,6 +1153,13 @@ func testIPResolver(t *testing.T) { func TestResolveFunc(t *testing.T) { defer leakcheck.Check(t) + defer func(nt func(d time.Duration) *time.Timer) { + newTimer = nt + }(newTimer) + newTimer = func(d time.Duration) *time.Timer { + // Will never fire on its own, will protect from triggering exponential backoff. + return time.NewTimer(time.Hour) + } tests := []struct { addr string want error @@ -989,7 +1174,7 @@ func TestResolveFunc(t *testing.T) { {"[2001:db8:a0b:12f0::1]:21", nil}, {":80", nil}, {"127.0.0...1:12345", nil}, - {"[fe80::1%lo0]:80", nil}, + {"[fe80::1%25lo0]:80", nil}, {"golang.org:http", nil}, {"[2001:db8::1]:http", nil}, {"[2001:db8::1]:", errEndsWithColon}, @@ -1001,7 +1186,7 @@ func TestResolveFunc(t *testing.T) { b := NewBuilder() for _, v := range tests { cc := &testClientConn{target: v.addr, errChan: make(chan error, 1)} - r, err := b.Build(resolver.Target{Endpoint: v.addr}, cc, resolver.BuildOptions{}) + r, err := b.Build(resolver.Target{URL: *testutils.MustParseURL(fmt.Sprintf("scheme:///%s", v.addr))}, cc, resolver.BuildOptions{}) if err == nil { r.Close() } @@ -1013,6 +1198,13 @@ func TestResolveFunc(t *testing.T) { func TestDisableServiceConfig(t *testing.T) { defer leakcheck.Check(t) + defer func(nt func(d time.Duration) *time.Timer) { + newTimer = nt + }(newTimer) + newTimer = func(d time.Duration) *time.Timer { + // Will never fire on its own, will protect from triggering exponential backoff. + return time.NewTimer(time.Hour) + } tests := []struct { target string scWant string @@ -1033,7 +1225,7 @@ func TestDisableServiceConfig(t *testing.T) { for _, a := range tests { b := NewBuilder() cc := &testClientConn{target: a.target} - r, err := b.Build(resolver.Target{Endpoint: a.target}, cc, resolver.BuildOptions{DisableServiceConfig: a.disableServiceConfig}) + r, err := b.Build(resolver.Target{URL: *testutils.MustParseURL(fmt.Sprintf("scheme:///%s", a.target))}, cc, resolver.BuildOptions{DisableServiceConfig: a.disableServiceConfig}) if err != nil { t.Fatalf("%v\n", err) } @@ -1059,12 +1251,19 @@ func TestDisableServiceConfig(t *testing.T) { func TestTXTError(t *testing.T) { defer leakcheck.Check(t) + defer func(nt func(d time.Duration) *time.Timer) { + newTimer = nt + }(newTimer) + newTimer = func(d time.Duration) *time.Timer { + // Will never fire on its own, will protect from triggering exponential backoff. + return time.NewTimer(time.Hour) + } defer func(v bool) { envconfig.TXTErrIgnore = v }(envconfig.TXTErrIgnore) for _, ignore := range []bool{false, true} { envconfig.TXTErrIgnore = ignore b := NewBuilder() cc := &testClientConn{target: "ipv4.single.fake"} // has A records but not TXT records. - r, err := b.Build(resolver.Target{Endpoint: "ipv4.single.fake"}, cc, resolver.BuildOptions{}) + r, err := b.Build(resolver.Target{URL: *testutils.MustParseURL(fmt.Sprintf("scheme:///%s", "ipv4.single.fake"))}, cc, resolver.BuildOptions{}) if err != nil { t.Fatalf("%v\n", err) } @@ -1090,10 +1289,17 @@ func TestTXTError(t *testing.T) { } func TestDNSResolverRetry(t *testing.T) { + defer func(nt func(d time.Duration) *time.Timer) { + newTimer = nt + }(newTimer) + newTimer = func(d time.Duration) *time.Timer { + // Will never fire on its own, will protect from triggering exponential backoff. + return time.NewTimer(time.Hour) + } b := NewBuilder() target := "ipv4.single.fake" cc := &testClientConn{target: target} - r, err := b.Build(resolver.Target{Endpoint: target}, cc, resolver.BuildOptions{}) + r, err := b.Build(resolver.Target{URL: *testutils.MustParseURL(fmt.Sprintf("scheme:///%s", target))}, cc, resolver.BuildOptions{}) if err != nil { t.Fatalf("%v\n", err) } @@ -1144,6 +1350,13 @@ func TestDNSResolverRetry(t *testing.T) { func TestCustomAuthority(t *testing.T) { defer leakcheck.Check(t) + defer func(nt func(d time.Duration) *time.Timer) { + newTimer = nt + }(newTimer) + newTimer = func(d time.Duration) *time.Timer { + // Will never fire on its own, will protect from triggering exponential backoff. + return time.NewTimer(time.Hour) + } tests := []struct { authority string @@ -1206,14 +1419,14 @@ func TestCustomAuthority(t *testing.T) { true, }, } - oldCustomAuthorityDialler := customAuthorityDialler + oldAddressDialer := addressDialer defer func() { - customAuthorityDialler = oldCustomAuthorityDialler + addressDialer = oldAddressDialer }() for _, a := range tests { errChan := make(chan error, 1) - customAuthorityDialler = func(authority string) func(ctx context.Context, network, address string) (net.Conn, error) { + addressDialer = func(authority string) func(ctx context.Context, network, address string) (net.Conn, error) { if authority != a.authorityWant { errChan <- fmt.Errorf("wrong custom authority passed to resolver. input: %s expected: %s actual: %s", a.authority, a.authorityWant, authority) } else { @@ -1224,9 +1437,13 @@ func TestCustomAuthority(t *testing.T) { } } + mockEndpointTarget := "foo.bar.com" b := NewBuilder() - cc := &testClientConn{target: "foo.bar.com", errChan: make(chan error, 1)} - r, err := b.Build(resolver.Target{Endpoint: "foo.bar.com", Authority: a.authority}, cc, resolver.BuildOptions{}) + cc := &testClientConn{target: mockEndpointTarget, errChan: make(chan error, 1)} + target := resolver.Target{ + URL: *testutils.MustParseURL(fmt.Sprintf("scheme://%s/%s", a.authority, mockEndpointTarget)), + } + r, err := b.Build(target, cc, resolver.BuildOptions{}) if err == nil { r.Close() @@ -1249,23 +1466,40 @@ func TestCustomAuthority(t *testing.T) { // requests. It sets the re-resolution rate to a small value and repeatedly // calls ResolveNow() and ensures only the expected number of resolution // requests are made. + func TestRateLimitedResolve(t *testing.T) { defer leakcheck.Check(t) - - const dnsResRate = 10 * time.Millisecond - dc := replaceDNSResRate(dnsResRate) - defer dc() + defer func(nt func(d time.Duration) *time.Timer) { + newTimer = nt + }(newTimer) + newTimer = func(d time.Duration) *time.Timer { + // Will never fire on its own, will protect from triggering exponential + // backoff. + return time.NewTimer(time.Hour) + } + defer func(nt func(d time.Duration) *time.Timer) { + newTimerDNSResRate = nt + }(newTimerDNSResRate) + + timerChan := testutils.NewChannel() + newTimerDNSResRate = func(d time.Duration) *time.Timer { + // Will never fire on its own, allows this test to call timer + // immediately. + t := time.NewTimer(time.Hour) + timerChan.Send(t) + return t + } // Create a new testResolver{} for this test because we want the exact count // of the number of times the resolver was invoked. - nc := replaceNetFunc(make(chan struct{})) + nc := overrideDefaultResolver(true) defer nc() target := "foo.bar.com" b := NewBuilder() cc := &testClientConn{target: target} - r, err := b.Build(resolver.Target{Endpoint: target}, cc, resolver.BuildOptions{}) + r, err := b.Build(resolver.Target{URL: *testutils.MustParseURL(fmt.Sprintf("scheme:///%s", target))}, cc, resolver.BuildOptions{}) if err != nil { t.Fatalf("resolver.Build() returned error: %v\n", err) } @@ -1281,55 +1515,65 @@ func TestRateLimitedResolve(t *testing.T) { t.Fatalf("delegate resolver returned unexpected type: %T\n", tr) } - // Observe the time before unblocking the lookupHost call. The 100ms rate - // limiting timer will begin immediately after that. This means the next - // resolution could happen less than 100ms if we read the time *after* - // receiving from tr.ch - start := time.Now() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() // Wait for the first resolution request to be done. This happens as part - // of the first iteration of the for loop in watcher() because we call - // ResolveNow in Build. - <-tr.ch - - // Here we start a couple of goroutines. One repeatedly calls ResolveNow() - // until asked to stop, and the other waits for two resolution requests to be - // made to our testResolver and stops the former. We measure the start and - // end times, and expect the duration elapsed to be in the interval - // {wantCalls*dnsResRate, wantCalls*dnsResRate} - done := make(chan struct{}) - go func() { - for { - select { - case <-done: - return - default: - r.ResolveNow(resolver.ResolveNowOptions{}) - time.Sleep(1 * time.Millisecond) - } - } - }() + // of the first iteration of the for loop in watcher(). + if _, err := tr.lookupHostCh.Receive(ctx); err != nil { + t.Fatalf("Timed out waiting for lookup() call.") + } - gotCalls := 0 - const wantCalls = 3 - min, max := wantCalls*dnsResRate, (wantCalls+1)*dnsResRate - tMax := time.NewTimer(max) - for gotCalls != wantCalls { - select { - case <-tr.ch: - gotCalls++ - case <-tMax.C: - t.Fatalf("Timed out waiting for %v calls after %v; got %v", wantCalls, max, gotCalls) - } + // Call Resolve Now 100 times, shouldn't continue onto next iteration of + // watcher, thus shouldn't lookup again. + for i := 0; i <= 100; i++ { + r.ResolveNow(resolver.ResolveNowOptions{}) } - close(done) - elapsed := time.Since(start) - if gotCalls != wantCalls { - t.Fatalf("resolve count mismatch for target: %q = %+v, want %+v\n", target, gotCalls, wantCalls) + continueCtx, continueCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer continueCancel() + + if _, err := tr.lookupHostCh.Receive(continueCtx); err == nil { + t.Fatalf("Should not have looked up again as DNS Min Res Rate timer has not gone off.") } - if elapsed < min { - t.Fatalf("elapsed time: %v, wanted it to be between {%v and %v}", elapsed, min, max) + + // Make the DNSMinResRate timer fire immediately (by receiving it, then + // resetting to 0), this will unblock the resolver which is currently + // blocked on the DNS Min Res Rate timer going off, which will allow it to + // continue to the next iteration of the watcher loop. + timer, err := timerChan.Receive(ctx) + if err != nil { + t.Fatalf("Error receiving timer from mock NewTimer call: %v", err) + } + timerPointer := timer.(*time.Timer) + timerPointer.Reset(0) + + // Now that DNS Min Res Rate timer has gone off, it should lookup again. + if _, err := tr.lookupHostCh.Receive(ctx); err != nil { + t.Fatalf("Timed out waiting for lookup() call.") + } + + // Resolve Now 1000 more times, shouldn't lookup again as DNS Min Res Rate + // timer has not gone off. + for i := 0; i < 1000; i++ { + r.ResolveNow(resolver.ResolveNowOptions{}) + } + + if _, err = tr.lookupHostCh.Receive(continueCtx); err == nil { + t.Fatalf("Should not have looked up again as DNS Min Res Rate timer has not gone off.") + } + + // Make the DNSMinResRate timer fire immediately again. + timer, err = timerChan.Receive(ctx) + if err != nil { + t.Fatalf("Error receiving timer from mock NewTimer call: %v", err) + } + timerPointer = timer.(*time.Timer) + timerPointer.Reset(0) + + // Now that DNS Min Res Rate timer has gone off, it should lookup again. + if _, err = tr.lookupHostCh.Receive(ctx); err != nil { + t.Fatalf("Timed out waiting for lookup() call.") } wantAddrs := []resolver.Address{{Addr: "1.2.3.4" + colonDefaultPort}, {Addr: "5.6.7.8" + colonDefaultPort}} @@ -1347,21 +1591,66 @@ func TestRateLimitedResolve(t *testing.T) { } } +// DNS Resolver immediately starts polling on an error. This will cause the re-resolution to return another error. +// Thus, test that it constantly sends errors to the grpc.ClientConn. func TestReportError(t *testing.T) { const target = "notfoundaddress" + defer func(nt func(d time.Duration) *time.Timer) { + newTimer = nt + }(newTimer) + timerChan := testutils.NewChannel() + newTimer = func(d time.Duration) *time.Timer { + // Will never fire on its own, allows this test to call timer immediately. + t := time.NewTimer(time.Hour) + timerChan.Send(t) + return t + } cc := &testClientConn{target: target, errChan: make(chan error)} + totalTimesCalledError := 0 b := NewBuilder() - r, err := b.Build(resolver.Target{Endpoint: target}, cc, resolver.BuildOptions{}) + r, err := b.Build(resolver.Target{URL: *testutils.MustParseURL(fmt.Sprintf("scheme:///%s", target))}, cc, resolver.BuildOptions{}) if err != nil { - t.Fatalf("%v\n", err) + t.Fatalf("Error building resolver for target %v: %v", target, err) + } + // Should receive first error. + err = <-cc.errChan + if !strings.Contains(err.Error(), "hostLookup error") { + t.Fatalf(`ReportError(err=%v) called; want err contains "hostLookupError"`, err) } + totalTimesCalledError++ + ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer ctxCancel() + timer, err := timerChan.Receive(ctx) + if err != nil { + t.Fatalf("Error receiving timer from mock NewTimer call: %v", err) + } + timerPointer := timer.(*time.Timer) + timerPointer.Reset(0) defer r.Close() - select { - case err := <-cc.errChan: + + // Cause timer to go off 10 times, and see if it matches DNS Resolver updating Error. + for i := 0; i < 10; i++ { + // Should call ReportError(). + err = <-cc.errChan if !strings.Contains(err.Error(), "hostLookup error") { t.Fatalf(`ReportError(err=%v) called; want err contains "hostLookupError"`, err) } - case <-time.After(time.Second): - t.Fatalf("did not receive error after 1s") + totalTimesCalledError++ + timer, err := timerChan.Receive(ctx) + if err != nil { + t.Fatalf("Error receiving timer from mock NewTimer call: %v", err) + } + timerPointer := timer.(*time.Timer) + timerPointer.Reset(0) + } + + if totalTimesCalledError != 11 { + t.Errorf("ReportError() not called 11 times, instead called %d times.", totalTimesCalledError) + } + // Clean up final watcher iteration. + <-cc.errChan + _, err = timerChan.Receive(ctx) + if err != nil { + t.Fatalf("Error receiving timer from mock NewTimer call: %v", err) } } diff --git a/internal/resolver/passthrough/passthrough.go b/internal/resolver/passthrough/passthrough.go index 520d9229e1ed..afac56572ad5 100644 --- a/internal/resolver/passthrough/passthrough.go +++ b/internal/resolver/passthrough/passthrough.go @@ -20,13 +20,20 @@ // name without scheme back to gRPC as resolved address. package passthrough -import "google.golang.org/grpc/resolver" +import ( + "errors" + + "google.golang.org/grpc/resolver" +) const scheme = "passthrough" type passthroughBuilder struct{} func (*passthroughBuilder) Build(target resolver.Target, cc resolver.ClientConn, opts resolver.BuildOptions) (resolver.Resolver, error) { + if target.Endpoint() == "" && opts.Dialer == nil { + return nil, errors.New("passthrough: received empty target in Build()") + } r := &passthroughResolver{ target: target, cc: cc, @@ -45,7 +52,7 @@ type passthroughResolver struct { } func (r *passthroughResolver) start() { - r.cc.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: r.target.Endpoint}}}) + r.cc.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: r.target.Endpoint()}}}) } func (*passthroughResolver) ResolveNow(o resolver.ResolveNowOptions) {} diff --git a/internal/resolver/unix/unix.go b/internal/resolver/unix/unix.go new file mode 100644 index 000000000000..160911687738 --- /dev/null +++ b/internal/resolver/unix/unix.go @@ -0,0 +1,74 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package unix implements a resolver for unix targets. +package unix + +import ( + "fmt" + + "google.golang.org/grpc/internal/transport/networktype" + "google.golang.org/grpc/resolver" +) + +const unixScheme = "unix" +const unixAbstractScheme = "unix-abstract" + +type builder struct { + scheme string +} + +func (b *builder) Build(target resolver.Target, cc resolver.ClientConn, _ resolver.BuildOptions) (resolver.Resolver, error) { + if target.URL.Host != "" { + return nil, fmt.Errorf("invalid (non-empty) authority: %v", target.URL.Host) + } + + // gRPC was parsing the dial target manually before PR #4817, and we + // switched to using url.Parse() in that PR. To avoid breaking existing + // resolver implementations we ended up stripping the leading "/" from the + // endpoint. This obviously does not work for the "unix" scheme. Hence we + // end up using the parsed URL instead. + endpoint := target.URL.Path + if endpoint == "" { + endpoint = target.URL.Opaque + } + addr := resolver.Address{Addr: endpoint} + if b.scheme == unixAbstractScheme { + // We can not prepend \0 as c++ gRPC does, as in Golang '@' is used to signify we do + // not want trailing \0 in address. + addr.Addr = "@" + addr.Addr + } + cc.UpdateState(resolver.State{Addresses: []resolver.Address{networktype.Set(addr, "unix")}}) + return &nopResolver{}, nil +} + +func (b *builder) Scheme() string { + return b.scheme +} + +type nopResolver struct { +} + +func (*nopResolver) ResolveNow(resolver.ResolveNowOptions) {} + +func (*nopResolver) Close() {} + +func init() { + resolver.Register(&builder{scheme: unixScheme}) + resolver.Register(&builder{scheme: unixAbstractScheme}) +} diff --git a/internal/serviceconfig/duration.go b/internal/serviceconfig/duration.go new file mode 100644 index 000000000000..11d82afcc7ec --- /dev/null +++ b/internal/serviceconfig/duration.go @@ -0,0 +1,130 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package serviceconfig + +import ( + "encoding/json" + "fmt" + "math" + "strconv" + "strings" + "time" +) + +// Duration defines JSON marshal and unmarshal methods to conform to the +// protobuf JSON spec defined [here]. +// +// [here]: https://protobuf.dev/reference/protobuf/google.protobuf/#duration +type Duration time.Duration + +func (d Duration) String() string { + return fmt.Sprint(time.Duration(d)) +} + +// MarshalJSON converts from d to a JSON string output. +func (d Duration) MarshalJSON() ([]byte, error) { + ns := time.Duration(d).Nanoseconds() + sec := ns / int64(time.Second) + ns = ns % int64(time.Second) + + var sign string + if sec < 0 || ns < 0 { + sign, sec, ns = "-", -1*sec, -1*ns + } + + // Generated output always contains 0, 3, 6, or 9 fractional digits, + // depending on required precision. + str := fmt.Sprintf("%s%d.%09d", sign, sec, ns) + str = strings.TrimSuffix(str, "000") + str = strings.TrimSuffix(str, "000") + str = strings.TrimSuffix(str, ".000") + return []byte(fmt.Sprintf("\"%ss\"", str)), nil +} + +// UnmarshalJSON unmarshals b as a duration JSON string into d. +func (d *Duration) UnmarshalJSON(b []byte) error { + var s string + if err := json.Unmarshal(b, &s); err != nil { + return err + } + if !strings.HasSuffix(s, "s") { + return fmt.Errorf("malformed duration %q: missing seconds unit", s) + } + neg := false + if s[0] == '-' { + neg = true + s = s[1:] + } + ss := strings.SplitN(s[:len(s)-1], ".", 3) + if len(ss) > 2 { + return fmt.Errorf("malformed duration %q: too many decimals", s) + } + // hasDigits is set if either the whole or fractional part of the number is + // present, since both are optional but one is required. + hasDigits := false + var sec, ns int64 + if len(ss[0]) > 0 { + var err error + if sec, err = strconv.ParseInt(ss[0], 10, 64); err != nil { + return fmt.Errorf("malformed duration %q: %v", s, err) + } + // Maximum seconds value per the durationpb spec. + const maxProtoSeconds = 315_576_000_000 + if sec > maxProtoSeconds { + return fmt.Errorf("out of range: %q", s) + } + hasDigits = true + } + if len(ss) == 2 && len(ss[1]) > 0 { + if len(ss[1]) > 9 { + return fmt.Errorf("malformed duration %q: too many digits after decimal", s) + } + var err error + if ns, err = strconv.ParseInt(ss[1], 10, 64); err != nil { + return fmt.Errorf("malformed duration %q: %v", s, err) + } + for i := 9; i > len(ss[1]); i-- { + ns *= 10 + } + hasDigits = true + } + if !hasDigits { + return fmt.Errorf("malformed duration %q: contains no numbers", s) + } + + if neg { + sec *= -1 + ns *= -1 + } + + // Maximum/minimum seconds/nanoseconds representable by Go's time.Duration. + const maxSeconds = math.MaxInt64 / int64(time.Second) + const maxNanosAtMaxSeconds = math.MaxInt64 % int64(time.Second) + const minSeconds = math.MinInt64 / int64(time.Second) + const minNanosAtMinSeconds = math.MinInt64 % int64(time.Second) + + if sec > maxSeconds || (sec == maxSeconds && ns >= maxNanosAtMaxSeconds) { + *d = Duration(math.MaxInt64) + } else if sec < minSeconds || (sec == minSeconds && ns <= minNanosAtMinSeconds) { + *d = Duration(math.MinInt64) + } else { + *d = Duration(sec*int64(time.Second) + ns) + } + return nil +} diff --git a/internal/serviceconfig/duration_test.go b/internal/serviceconfig/duration_test.go new file mode 100644 index 000000000000..5696541aa870 --- /dev/null +++ b/internal/serviceconfig/duration_test.go @@ -0,0 +1,87 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package serviceconfig + +import ( + "fmt" + "math" + "strings" + "testing" + "time" + + "google.golang.org/grpc/internal/grpcrand" +) + +// Tests both marshalling and unmarshalling of Durations. +func TestDuration_MarshalUnmarshal(t *testing.T) { + testCases := []struct { + json string + td time.Duration + unmarshalErr error + noMarshal bool + }{ + // Basic values. + {json: `"1s"`, td: time.Second}, + {json: `"-100.700s"`, td: -100*time.Second - 700*time.Millisecond}, + {json: `".050s"`, td: 50 * time.Millisecond, noMarshal: true}, + {json: `"-.001s"`, td: -1 * time.Millisecond, noMarshal: true}, + {json: `"-0.200s"`, td: -200 * time.Millisecond}, + // Positive near / out of bounds. + {json: `"9223372036s"`, td: 9223372036 * time.Second}, + {json: `"9223372037s"`, td: math.MaxInt64, noMarshal: true}, + {json: `"9223372036.854775807s"`, td: math.MaxInt64}, + {json: `"9223372036.854775808s"`, td: math.MaxInt64, noMarshal: true}, + {json: `"315576000000s"`, td: math.MaxInt64, noMarshal: true}, + {json: `"315576000001s"`, unmarshalErr: fmt.Errorf("out of range")}, + // Negative near / out of bounds. + {json: `"-9223372036s"`, td: -9223372036 * time.Second}, + {json: `"-9223372037s"`, td: math.MinInt64, noMarshal: true}, + {json: `"-9223372036.854775808s"`, td: math.MinInt64}, + {json: `"-9223372036.854775809s"`, td: math.MinInt64, noMarshal: true}, + {json: `"-315576000000s"`, td: math.MinInt64, noMarshal: true}, + {json: `"-315576000001s"`, unmarshalErr: fmt.Errorf("out of range")}, + // Parse errors. + {json: `123s`, unmarshalErr: fmt.Errorf("invalid character")}, + {json: `"5m"`, unmarshalErr: fmt.Errorf("malformed duration")}, + {json: `"5.3.2s"`, unmarshalErr: fmt.Errorf("malformed duration")}, + {json: `"x.3s"`, unmarshalErr: fmt.Errorf("malformed duration")}, + {json: `"3.xs"`, unmarshalErr: fmt.Errorf("malformed duration")}, + {json: `"3.1234567890s"`, unmarshalErr: fmt.Errorf("malformed duration")}, + {json: `".s"`, unmarshalErr: fmt.Errorf("malformed duration")}, + {json: `"s"`, unmarshalErr: fmt.Errorf("malformed duration")}, + } + for _, tc := range testCases { + // Seed `got` with a random value to ensure we properly reset it in all + // non-error cases. + got := Duration(grpcrand.Uint64()) + err := got.UnmarshalJSON([]byte(tc.json)) + if (err == nil && time.Duration(got) != tc.td) || + (err != nil) != (tc.unmarshalErr != nil) || !strings.Contains(fmt.Sprint(err), fmt.Sprint(tc.unmarshalErr)) { + t.Errorf("UnmarshalJSON of %v = %v, %v; want %v, %v", tc.json, time.Duration(got), err, tc.td, tc.unmarshalErr) + } + + if tc.unmarshalErr == nil && !tc.noMarshal { + d := Duration(tc.td) + got, err := d.MarshalJSON() + if string(got) != tc.json || err != nil { + t.Errorf("MarshalJSON of %v = %v, %v; want %v, nil", d, string(got), err, tc.json) + } + } + } +} diff --git a/internal/serviceconfig/serviceconfig.go b/internal/serviceconfig/serviceconfig.go index af3e2b5f7e8b..51e733e495a3 100644 --- a/internal/serviceconfig/serviceconfig.go +++ b/internal/serviceconfig/serviceconfig.go @@ -22,8 +22,10 @@ package serviceconfig import ( "encoding/json" "fmt" + "time" "google.golang.org/grpc/balancer" + "google.golang.org/grpc/codes" "google.golang.org/grpc/grpclog" externalserviceconfig "google.golang.org/grpc/serviceconfig" ) @@ -44,15 +46,31 @@ type BalancerConfig struct { type intermediateBalancerConfig []map[string]json.RawMessage +// MarshalJSON implements the json.Marshaler interface. +// +// It marshals the balancer and config into a length-1 slice +// ([]map[string]config). +func (bc *BalancerConfig) MarshalJSON() ([]byte, error) { + if bc.Config == nil { + // If config is nil, return empty config `{}`. + return []byte(fmt.Sprintf(`[{%q: %v}]`, bc.Name, "{}")), nil + } + c, err := json.Marshal(bc.Config) + if err != nil { + return nil, err + } + return []byte(fmt.Sprintf(`[{%q: %s}]`, bc.Name, c)), nil +} + // UnmarshalJSON implements the json.Unmarshaler interface. // // ServiceConfig contains a list of loadBalancingConfigs, each with a name and // config. This method iterates through that list in order, and stops at the // first policy that is supported. -// - If the config for the first supported policy is invalid, the whole service -// config is invalid. -// - If the list doesn't contain any supported policy, the whole service config -// is invalid. +// - If the config for the first supported policy is invalid, the whole service +// config is invalid. +// - If the list doesn't contain any supported policy, the whole service config +// is invalid. func (bc *BalancerConfig) UnmarshalJSON(b []byte) error { var ir intermediateBalancerConfig err := json.Unmarshal(b, &ir) @@ -60,6 +78,7 @@ func (bc *BalancerConfig) UnmarshalJSON(b []byte) error { return err } + var names []string for i, lbcfg := range ir { if len(lbcfg) != 1 { return fmt.Errorf("invalid loadBalancingConfig: entry %v does not contain exactly 1 policy/config pair: %q", i, lbcfg) @@ -74,6 +93,7 @@ func (bc *BalancerConfig) UnmarshalJSON(b []byte) error { for name, jsonCfg = range lbcfg { } + names = append(names, name) builder := balancer.Get(name) if builder == nil { // If the balancer is not registered, move on to the next config. @@ -102,5 +122,59 @@ func (bc *BalancerConfig) UnmarshalJSON(b []byte) error { // return. This means we had a loadBalancingConfig slice but did not // encounter a registered policy. The config is considered invalid in this // case. - return fmt.Errorf("invalid loadBalancingConfig: no supported policies found") + return fmt.Errorf("invalid loadBalancingConfig: no supported policies found in %v", names) +} + +// MethodConfig defines the configuration recommended by the service providers for a +// particular method. +type MethodConfig struct { + // WaitForReady indicates whether RPCs sent to this method should wait until + // the connection is ready by default (!failfast). The value specified via the + // gRPC client API will override the value set here. + WaitForReady *bool + // Timeout is the default timeout for RPCs sent to this method. The actual + // deadline used will be the minimum of the value specified here and the value + // set by the application via the gRPC client API. If either one is not set, + // then the other will be used. If neither is set, then the RPC has no deadline. + Timeout *time.Duration + // MaxReqSize is the maximum allowed payload size for an individual request in a + // stream (client->server) in bytes. The size which is measured is the serialized + // payload after per-message compression (but before stream compression) in bytes. + // The actual value used is the minimum of the value specified here and the value set + // by the application via the gRPC client API. If either one is not set, then the other + // will be used. If neither is set, then the built-in default is used. + MaxReqSize *int + // MaxRespSize is the maximum allowed payload size for an individual response in a + // stream (server->client) in bytes. + MaxRespSize *int + // RetryPolicy configures retry options for the method. + RetryPolicy *RetryPolicy +} + +// RetryPolicy defines the go-native version of the retry policy defined by the +// service config here: +// https://github.com/grpc/proposal/blob/master/A6-client-retries.md#integration-with-service-config +type RetryPolicy struct { + // MaxAttempts is the maximum number of attempts, including the original RPC. + // + // This field is required and must be two or greater. + MaxAttempts int + + // Exponential backoff parameters. The initial retry attempt will occur at + // random(0, initialBackoff). In general, the nth attempt will occur at + // random(0, + // min(initialBackoff*backoffMultiplier**(n-1), maxBackoff)). + // + // These fields are required and must be greater than zero. + InitialBackoff time.Duration + MaxBackoff time.Duration + BackoffMultiplier float64 + + // The set of status codes which may be retried. + // + // Status codes are specified as strings, e.g., "UNAVAILABLE". + // + // This field is required and must be non-empty. + // Note: a set is used to store this for easy lookup. + RetryableStatusCodes map[codes.Code]bool } diff --git a/internal/serviceconfig/serviceconfig_test.go b/internal/serviceconfig/serviceconfig_test.go index b8abaae027ef..3a725685db01 100644 --- a/internal/serviceconfig/serviceconfig_test.go +++ b/internal/serviceconfig/serviceconfig_test.go @@ -29,16 +29,18 @@ import ( ) type testBalancerConfigType struct { - externalserviceconfig.LoadBalancingConfig + externalserviceconfig.LoadBalancingConfig `json:"-"` + + Check bool `json:"check"` } -var testBalancerConfig = testBalancerConfigType{} +var testBalancerConfig = testBalancerConfigType{Check: true} const ( testBalancerBuilderName = "test-bb" testBalancerBuilderNotParserName = "test-bb-not-parser" - testBalancerConfigJSON = `{"test-balancer-config":"true"}` + testBalancerConfigJSON = `{"check":true}` ) type testBalancerBuilder struct { @@ -133,3 +135,48 @@ func TestBalancerConfigUnmarshalJSON(t *testing.T) { }) } } + +func TestBalancerConfigMarshalJSON(t *testing.T) { + tests := []struct { + name string + bc BalancerConfig + wantJSON string + }{ + { + name: "OK", + bc: BalancerConfig{ + Name: testBalancerBuilderName, + Config: testBalancerConfig, + }, + wantJSON: `[{"test-bb": {"check":true}}]`, + }, + { + name: "OK config is nil", + bc: BalancerConfig{ + Name: testBalancerBuilderNotParserName, + Config: nil, // nil should be marshalled to an empty config "{}". + }, + wantJSON: `[{"test-bb-not-parser": {}}]`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b, err := tt.bc.MarshalJSON() + if err != nil { + t.Fatalf("failed to marshal: %v", err) + } + + if str := string(b); str != tt.wantJSON { + t.Fatalf("got str %q, want %q", str, tt.wantJSON) + } + + var bc BalancerConfig + if err := bc.UnmarshalJSON(b); err != nil { + t.Errorf("failed to unmarshal: %v", err) + } + if !cmp.Equal(bc, tt.bc) { + t.Errorf("diff: %v", cmp.Diff(bc, tt.bc)) + } + }) + } +} diff --git a/internal/status/status.go b/internal/status/status.go index 710223b8ded0..4cf85cad9f81 100644 --- a/internal/status/status.go +++ b/internal/status/status.go @@ -49,7 +49,7 @@ func New(c codes.Code, msg string) *Status { } // Newf returns New(c, fmt.Sprintf(format, a...)). -func Newf(c codes.Code, format string, a ...interface{}) *Status { +func Newf(c codes.Code, format string, a ...any) *Status { return New(c, fmt.Sprintf(format, a...)) } @@ -64,7 +64,7 @@ func Err(c codes.Code, msg string) error { } // Errorf returns Error(c, fmt.Sprintf(format, a...)). -func Errorf(c codes.Code, format string, a ...interface{}) error { +func Errorf(c codes.Code, format string, a ...any) error { return Err(c, fmt.Sprintf(format, a...)) } @@ -97,7 +97,7 @@ func (s *Status) Err() error { if s.Code() == codes.OK { return nil } - return &Error{e: s.Proto()} + return &Error{s: s} } // WithDetails returns a new status with the provided details messages appended to the status. @@ -120,11 +120,11 @@ func (s *Status) WithDetails(details ...proto.Message) (*Status, error) { // Details returns a slice of details messages attached to the status. // If a detail cannot be decoded, the error is returned in place of the detail. -func (s *Status) Details() []interface{} { +func (s *Status) Details() []any { if s == nil || s.s == nil { return nil } - details := make([]interface{}, 0, len(s.s.Details)) + details := make([]any, 0, len(s.s.Details)) for _, any := range s.s.Details { detail := &ptypes.DynamicAny{} if err := ptypes.UnmarshalAny(any, detail); err != nil { @@ -136,19 +136,23 @@ func (s *Status) Details() []interface{} { return details } +func (s *Status) String() string { + return fmt.Sprintf("rpc error: code = %s desc = %s", s.Code(), s.Message()) +} + // Error wraps a pointer of a status proto. It implements error and Status, // and a nil *Error should never be returned by this package. type Error struct { - e *spb.Status + s *Status } func (e *Error) Error() string { - return fmt.Sprintf("rpc error: code = %s desc = %s", codes.Code(e.e.GetCode()), e.e.GetMessage()) + return e.s.String() } // GRPCStatus returns the Status represented by se. func (e *Error) GRPCStatus() *Status { - return FromProto(e.e) + return e.s } // Is implements future error.Is functionality. @@ -158,5 +162,15 @@ func (e *Error) Is(target error) bool { if !ok { return false } - return proto.Equal(e.e, tse.e) + return proto.Equal(e.s.s, tse.s.s) +} + +// IsRestrictedControlPlaneCode returns whether the status includes a code +// restricted for control plane usage as defined by gRFC A54. +func IsRestrictedControlPlaneCode(s *Status) bool { + switch s.Code() { + case codes.InvalidArgument, codes.NotFound, codes.AlreadyExists, codes.FailedPrecondition, codes.Aborted, codes.OutOfRange, codes.DataLoss: + return true + } + return false } diff --git a/internal/stubserver/stubserver.go b/internal/stubserver/stubserver.go new file mode 100644 index 000000000000..39c291500547 --- /dev/null +++ b/internal/stubserver/stubserver.go @@ -0,0 +1,232 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package stubserver is a stubbable implementation of +// google.golang.org/grpc/interop/grpc_testing for testing purposes. +package stubserver + +import ( + "context" + "fmt" + "net" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/serviceconfig" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +// StubServer is a server that is easy to customize within individual test +// cases. +type StubServer struct { + // Guarantees we satisfy this interface; panics if unimplemented methods are called. + testgrpc.TestServiceServer + + // Customizable implementations of server handlers. + EmptyCallF func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) + UnaryCallF func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) + FullDuplexCallF func(stream testgrpc.TestService_FullDuplexCallServer) error + + // A client connected to this service the test may use. Created in Start(). + Client testgrpc.TestServiceClient + CC *grpc.ClientConn + S *grpc.Server + + // Parameters for Listen and Dial. Defaults will be used if these are empty + // before Start. + Network string + Address string + Target string + + // Custom listener to use for serving. If unspecified, a new listener is + // created on a local port. + Listener net.Listener + + cleanups []func() // Lambdas executed in Stop(); populated by Start(). + + // Set automatically if Target == "" + R *manual.Resolver +} + +// EmptyCall is the handler for testpb.EmptyCall +func (ss *StubServer) EmptyCall(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { + return ss.EmptyCallF(ctx, in) +} + +// UnaryCall is the handler for testpb.UnaryCall +func (ss *StubServer) UnaryCall(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return ss.UnaryCallF(ctx, in) +} + +// FullDuplexCall is the handler for testpb.FullDuplexCall +func (ss *StubServer) FullDuplexCall(stream testgrpc.TestService_FullDuplexCallServer) error { + return ss.FullDuplexCallF(stream) +} + +// Start starts the server and creates a client connected to it. +func (ss *StubServer) Start(sopts []grpc.ServerOption, dopts ...grpc.DialOption) error { + if err := ss.StartServer(sopts...); err != nil { + return err + } + if err := ss.StartClient(dopts...); err != nil { + ss.Stop() + return err + } + return nil +} + +type registerServiceServerOption struct { + grpc.EmptyServerOption + f func(*grpc.Server) +} + +// RegisterServiceServerOption returns a ServerOption that will run f() in +// Start or StartServer with the grpc.Server created before serving. This +// allows other services to be registered on the test server (e.g. ORCA, +// health, or reflection). +func RegisterServiceServerOption(f func(*grpc.Server)) grpc.ServerOption { + return ®isterServiceServerOption{f: f} +} + +// StartServer only starts the server. It does not create a client to it. +func (ss *StubServer) StartServer(sopts ...grpc.ServerOption) error { + if ss.Network == "" { + ss.Network = "tcp" + } + if ss.Address == "" { + ss.Address = "localhost:0" + } + if ss.Target == "" { + ss.R = manual.NewBuilderWithScheme("whatever") + } + + lis := ss.Listener + if lis == nil { + var err error + lis, err = net.Listen(ss.Network, ss.Address) + if err != nil { + return fmt.Errorf("net.Listen(%q, %q) = %v", ss.Network, ss.Address, err) + } + } + ss.Address = lis.Addr().String() + ss.cleanups = append(ss.cleanups, func() { lis.Close() }) + + s := grpc.NewServer(sopts...) + for _, so := range sopts { + switch x := so.(type) { + case *registerServiceServerOption: + x.f(s) + } + } + + testgrpc.RegisterTestServiceServer(s, ss) + go s.Serve(lis) + ss.cleanups = append(ss.cleanups, s.Stop) + ss.S = s + return nil +} + +// StartClient creates a client connected to this service that the test may use. +// The newly created client will be available in the Client field of StubServer. +func (ss *StubServer) StartClient(dopts ...grpc.DialOption) error { + opts := append([]grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())}, dopts...) + if ss.R != nil { + ss.Target = ss.R.Scheme() + ":///" + ss.Address + opts = append(opts, grpc.WithResolvers(ss.R)) + } + + cc, err := grpc.Dial(ss.Target, opts...) + if err != nil { + return fmt.Errorf("grpc.Dial(%q) = %v", ss.Target, err) + } + ss.CC = cc + if ss.R != nil { + ss.R.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: ss.Address}}}) + } + if err := waitForReady(cc); err != nil { + cc.Close() + return err + } + + ss.cleanups = append(ss.cleanups, func() { cc.Close() }) + + ss.Client = testgrpc.NewTestServiceClient(cc) + return nil +} + +// NewServiceConfig applies sc to ss.Client using the resolver (if present). +func (ss *StubServer) NewServiceConfig(sc string) { + if ss.R != nil { + ss.R.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: ss.Address}}, ServiceConfig: parseCfg(ss.R, sc)}) + } +} + +func waitForReady(cc *grpc.ClientConn) error { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + for { + s := cc.GetState() + if s == connectivity.Ready { + return nil + } + if !cc.WaitForStateChange(ctx, s) { + // ctx got timeout or canceled. + return ctx.Err() + } + } +} + +// Stop stops ss and cleans up all resources it consumed. +func (ss *StubServer) Stop() { + for i := len(ss.cleanups) - 1; i >= 0; i-- { + ss.cleanups[i]() + } +} + +func parseCfg(r *manual.Resolver, s string) *serviceconfig.ParseResult { + g := r.CC.ParseServiceConfig(s) + if g.Err != nil { + panic(fmt.Sprintf("Error parsing config %q: %v", s, g.Err)) + } + return g +} + +// StartTestService spins up a stub server exposing the TestService on a local +// port. If the passed in server is nil, a stub server that implements only the +// EmptyCall and UnaryCall RPCs is started. +func StartTestService(t *testing.T, server *StubServer, sopts ...grpc.ServerOption) *StubServer { + if server == nil { + server = &StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil }, + UnaryCallF: func(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{}, nil + }, + } + } + server.StartServer(sopts...) + + t.Logf("Started test service backend at %q", server.Address) + return server +} diff --git a/internal/syscall/syscall_linux.go b/internal/syscall/syscall_linux.go index 3743bd45afbc..b3a72276dee4 100644 --- a/internal/syscall/syscall_linux.go +++ b/internal/syscall/syscall_linux.go @@ -42,25 +42,23 @@ func GetCPUTime() int64 { } // Rusage is an alias for syscall.Rusage under linux environment. -type Rusage syscall.Rusage +type Rusage = syscall.Rusage // GetRusage returns the resource usage of current process. -func GetRusage() (rusage *Rusage) { - rusage = new(Rusage) - syscall.Getrusage(syscall.RUSAGE_SELF, (*syscall.Rusage)(rusage)) - return +func GetRusage() *Rusage { + rusage := new(Rusage) + syscall.Getrusage(syscall.RUSAGE_SELF, rusage) + return rusage } // CPUTimeDiff returns the differences of user CPU time and system CPU time used // between two Rusage structs. func CPUTimeDiff(first *Rusage, latest *Rusage) (float64, float64) { - f := (*syscall.Rusage)(first) - l := (*syscall.Rusage)(latest) var ( - utimeDiffs = l.Utime.Sec - f.Utime.Sec - utimeDiffus = l.Utime.Usec - f.Utime.Usec - stimeDiffs = l.Stime.Sec - f.Stime.Sec - stimeDiffus = l.Stime.Usec - f.Stime.Usec + utimeDiffs = latest.Utime.Sec - first.Utime.Sec + utimeDiffus = latest.Utime.Usec - first.Utime.Usec + stimeDiffs = latest.Stime.Sec - first.Stime.Sec + stimeDiffus = latest.Stime.Usec - first.Stime.Usec ) uTimeElapsed := float64(utimeDiffs) + float64(utimeDiffus)*1.0e-6 diff --git a/internal/syscall/syscall_nonlinux.go b/internal/syscall/syscall_nonlinux.go index 51159e638d77..999f52cd75bd 100644 --- a/internal/syscall/syscall_nonlinux.go +++ b/internal/syscall/syscall_nonlinux.go @@ -1,4 +1,5 @@ -// +build !linux appengine +//go:build !linux +// +build !linux /* * @@ -35,41 +36,41 @@ var logger = grpclog.Component("core") func log() { once.Do(func() { - logger.Info("CPU time info is unavailable on non-linux environment.") + logger.Info("CPU time info is unavailable on non-linux environments.") }) } -// GetCPUTime returns the how much CPU time has passed since the start of this process. -// It always returns 0 under non-linux environment. +// GetCPUTime returns the how much CPU time has passed since the start of this +// process. It always returns 0 under non-linux environments. func GetCPUTime() int64 { log() return 0 } -// Rusage is an empty struct under non-linux environment. +// Rusage is an empty struct under non-linux environments. type Rusage struct{} -// GetRusage is a no-op function under non-linux environment. -func GetRusage() (rusage *Rusage) { +// GetRusage is a no-op function under non-linux environments. +func GetRusage() *Rusage { log() return nil } // CPUTimeDiff returns the differences of user CPU time and system CPU time used -// between two Rusage structs. It a no-op function for non-linux environment. +// between two Rusage structs. It a no-op function for non-linux environments. func CPUTimeDiff(first *Rusage, latest *Rusage) (float64, float64) { log() return 0, 0 } -// SetTCPUserTimeout is a no-op function under non-linux environments +// SetTCPUserTimeout is a no-op function under non-linux environments. func SetTCPUserTimeout(conn net.Conn, timeout time.Duration) error { log() return nil } -// GetTCPUserTimeout is a no-op function under non-linux environments -// a negative return value indicates the operation is not supported +// GetTCPUserTimeout is a no-op function under non-linux environments. +// A negative return value indicates the operation is not supported func GetTCPUserTimeout(conn net.Conn) (int, error) { log() return -1, nil diff --git a/internal/testutils/balancer.go b/internal/testutils/balancer.go new file mode 100644 index 000000000000..43bbbf9ae560 --- /dev/null +++ b/internal/testutils/balancer.go @@ -0,0 +1,394 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package testutils + +import ( + "context" + "errors" + "fmt" + "testing" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/resolver" +) + +// testingLogger wraps the logging methods from testing.T. +type testingLogger interface { + Log(args ...any) + Logf(format string, args ...any) + Errorf(format string, args ...any) +} + +// TestSubConn implements the SubConn interface, to be used in tests. +type TestSubConn struct { + tcc *TestClientConn // the CC that owns this SubConn + id string + ConnectCh chan struct{} + stateListener func(balancer.SubConnState) + connectCalled *grpcsync.Event +} + +// NewTestSubConn returns a newly initialized SubConn. Typically, subconns +// should be created via TestClientConn.NewSubConn instead, but can be useful +// for some tests. +func NewTestSubConn(id string) *TestSubConn { + return &TestSubConn{ + ConnectCh: make(chan struct{}, 1), + connectCalled: grpcsync.NewEvent(), + id: id, + } +} + +// UpdateAddresses is a no-op. +func (tsc *TestSubConn) UpdateAddresses([]resolver.Address) {} + +// Connect is a no-op. +func (tsc *TestSubConn) Connect() { + tsc.connectCalled.Fire() + select { + case tsc.ConnectCh <- struct{}{}: + default: + } +} + +// GetOrBuildProducer is a no-op. +func (tsc *TestSubConn) GetOrBuildProducer(balancer.ProducerBuilder) (balancer.Producer, func()) { + return nil, nil +} + +// UpdateState pushes the state to the listener, if one is registered. +func (tsc *TestSubConn) UpdateState(state balancer.SubConnState) { + <-tsc.connectCalled.Done() + if tsc.stateListener != nil { + tsc.stateListener(state) + return + } +} + +// Shutdown pushes the SubConn to the ShutdownSubConn channel in the parent +// TestClientConn. +func (tsc *TestSubConn) Shutdown() { + tsc.tcc.logger.Logf("SubConn %s: Shutdown", tsc) + select { + case tsc.tcc.ShutdownSubConnCh <- tsc: + default: + } +} + +// String implements stringer to print human friendly error message. +func (tsc *TestSubConn) String() string { + return tsc.id +} + +// TestClientConn is a mock balancer.ClientConn used in tests. +type TestClientConn struct { + logger testingLogger + + NewSubConnAddrsCh chan []resolver.Address // the last 10 []Address to create subconn. + NewSubConnCh chan *TestSubConn // the last 10 subconn created. + ShutdownSubConnCh chan *TestSubConn // the last 10 subconn removed. + UpdateAddressesAddrsCh chan []resolver.Address // last updated address via UpdateAddresses(). + + NewPickerCh chan balancer.Picker // the last picker updated. + NewStateCh chan connectivity.State // the last state. + ResolveNowCh chan resolver.ResolveNowOptions // the last ResolveNow(). + + subConnIdx int +} + +// NewTestClientConn creates a TestClientConn. +func NewTestClientConn(t *testing.T) *TestClientConn { + return &TestClientConn{ + logger: t, + + NewSubConnAddrsCh: make(chan []resolver.Address, 10), + NewSubConnCh: make(chan *TestSubConn, 10), + ShutdownSubConnCh: make(chan *TestSubConn, 10), + UpdateAddressesAddrsCh: make(chan []resolver.Address, 1), + + NewPickerCh: make(chan balancer.Picker, 1), + NewStateCh: make(chan connectivity.State, 1), + ResolveNowCh: make(chan resolver.ResolveNowOptions, 1), + } +} + +// NewSubConn creates a new SubConn. +func (tcc *TestClientConn) NewSubConn(a []resolver.Address, o balancer.NewSubConnOptions) (balancer.SubConn, error) { + sc := &TestSubConn{ + tcc: tcc, + id: fmt.Sprintf("sc%d", tcc.subConnIdx), + ConnectCh: make(chan struct{}, 1), + stateListener: o.StateListener, + connectCalled: grpcsync.NewEvent(), + } + tcc.subConnIdx++ + tcc.logger.Logf("testClientConn: NewSubConn(%v, %+v) => %s", a, o, sc) + select { + case tcc.NewSubConnAddrsCh <- a: + default: + } + + select { + case tcc.NewSubConnCh <- sc: + default: + } + + return sc, nil +} + +// RemoveSubConn is a nop; tests should all be updated to use sc.Shutdown() +// instead. +func (tcc *TestClientConn) RemoveSubConn(sc balancer.SubConn) { + tcc.logger.Errorf("RemoveSubConn(%v) called unexpectedly", sc) +} + +// UpdateAddresses updates the addresses on the SubConn. +func (tcc *TestClientConn) UpdateAddresses(sc balancer.SubConn, addrs []resolver.Address) { + tcc.logger.Logf("testClientConn: UpdateAddresses(%v, %+v)", sc, addrs) + select { + case tcc.UpdateAddressesAddrsCh <- addrs: + default: + } +} + +// UpdateState updates connectivity state and picker. +func (tcc *TestClientConn) UpdateState(bs balancer.State) { + tcc.logger.Logf("testClientConn: UpdateState(%v)", bs) + select { + case <-tcc.NewStateCh: + default: + } + tcc.NewStateCh <- bs.ConnectivityState + + select { + case <-tcc.NewPickerCh: + default: + } + tcc.NewPickerCh <- bs.Picker +} + +// ResolveNow panics. +func (tcc *TestClientConn) ResolveNow(o resolver.ResolveNowOptions) { + select { + case <-tcc.ResolveNowCh: + default: + } + tcc.ResolveNowCh <- o +} + +// Target panics. +func (tcc *TestClientConn) Target() string { + panic("not implemented") +} + +// WaitForErrPicker waits until an error picker is pushed to this ClientConn. +// Returns error if the provided context expires or a non-error picker is pushed +// to the ClientConn. +func (tcc *TestClientConn) WaitForErrPicker(ctx context.Context) error { + select { + case <-ctx.Done(): + return errors.New("timeout when waiting for an error picker") + case picker := <-tcc.NewPickerCh: + if _, perr := picker.Pick(balancer.PickInfo{}); perr == nil { + return fmt.Errorf("balancer returned a picker which is not an error picker") + } + } + return nil +} + +// WaitForPickerWithErr waits until an error picker is pushed to this +// ClientConn with the error matching the wanted error. Returns an error if +// the provided context expires, including the last received picker error (if +// any). +func (tcc *TestClientConn) WaitForPickerWithErr(ctx context.Context, want error) error { + lastErr := errors.New("received no picker") + for { + select { + case <-ctx.Done(): + return fmt.Errorf("timeout when waiting for an error picker with %v; last picker error: %v", want, lastErr) + case picker := <-tcc.NewPickerCh: + if _, lastErr = picker.Pick(balancer.PickInfo{}); lastErr != nil && lastErr.Error() == want.Error() { + return nil + } + } + } +} + +// WaitForConnectivityState waits until the state pushed to this ClientConn +// matches the wanted state. Returns an error if the provided context expires, +// including the last received state (if any). +func (tcc *TestClientConn) WaitForConnectivityState(ctx context.Context, want connectivity.State) error { + var lastState connectivity.State = -1 + for { + select { + case <-ctx.Done(): + return fmt.Errorf("timeout when waiting for state to be %s; last state: %s", want, lastState) + case s := <-tcc.NewStateCh: + if s == want { + return nil + } + lastState = s + } + } +} + +// WaitForRoundRobinPicker waits for a picker that passes IsRoundRobin. Also +// drains the matching state channel and requires it to be READY (if an entry +// is pending) to be considered. Returns an error if the provided context +// expires, including the last received error from IsRoundRobin or the picker +// (if any). +func (tcc *TestClientConn) WaitForRoundRobinPicker(ctx context.Context, want ...balancer.SubConn) error { + lastErr := errors.New("received no picker") + for { + select { + case <-ctx.Done(): + return fmt.Errorf("timeout when waiting for round robin picker with %v; last error: %v", want, lastErr) + case p := <-tcc.NewPickerCh: + s := connectivity.Ready + select { + case s = <-tcc.NewStateCh: + default: + } + if s != connectivity.Ready { + lastErr = fmt.Errorf("received state %v instead of ready", s) + break + } + var pickerErr error + if err := IsRoundRobin(want, func() balancer.SubConn { + sc, err := p.Pick(balancer.PickInfo{}) + if err != nil { + pickerErr = err + } else if sc.Done != nil { + sc.Done(balancer.DoneInfo{}) + } + return sc.SubConn + }); pickerErr != nil { + lastErr = pickerErr + continue + } else if err != nil { + lastErr = err + continue + } + return nil + } + } +} + +// WaitForPicker waits for a picker that results in f returning nil. If the +// context expires, returns the last error returned by f (if any). +func (tcc *TestClientConn) WaitForPicker(ctx context.Context, f func(balancer.Picker) error) error { + lastErr := errors.New("received no picker") + for { + select { + case <-ctx.Done(): + return fmt.Errorf("timeout when waiting for picker; last error: %v", lastErr) + case p := <-tcc.NewPickerCh: + if err := f(p); err != nil { + lastErr = err + continue + } + return nil + } + } +} + +// IsRoundRobin checks whether f's return value is roundrobin of elements from +// want. But it doesn't check for the order. Note that want can contain +// duplicate items, which makes it weight-round-robin. +// +// Step 1. the return values of f should form a permutation of all elements in +// want, but not necessary in the same order. E.g. if want is {a,a,b}, the check +// fails if f returns: +// - {a,a,a}: third a is returned before b +// - {a,b,b}: second b is returned before the second a +// +// If error is found in this step, the returned error contains only the first +// iteration until where it goes wrong. +// +// Step 2. the return values of f should be repetitions of the same permutation. +// E.g. if want is {a,a,b}, the check failes if f returns: +// - {a,b,a,b,a,a}: though it satisfies step 1, the second iteration is not +// repeating the first iteration. +// +// If error is found in this step, the returned error contains the first +// iteration + the second iteration until where it goes wrong. +func IsRoundRobin(want []balancer.SubConn, f func() balancer.SubConn) error { + wantSet := make(map[balancer.SubConn]int) // SubConn -> count, for weighted RR. + for _, sc := range want { + wantSet[sc]++ + } + + // The first iteration: makes sure f's return values form a permutation of + // elements in want. + // + // Also keep the returns values in a slice, so we can compare the order in + // the second iteration. + gotSliceFirstIteration := make([]balancer.SubConn, 0, len(want)) + for range want { + got := f() + gotSliceFirstIteration = append(gotSliceFirstIteration, got) + wantSet[got]-- + if wantSet[got] < 0 { + return fmt.Errorf("non-roundrobin want: %v, result: %v", want, gotSliceFirstIteration) + } + } + + // The second iteration should repeat the first iteration. + var gotSliceSecondIteration []balancer.SubConn + for i := 0; i < 2; i++ { + for _, w := range gotSliceFirstIteration { + g := f() + gotSliceSecondIteration = append(gotSliceSecondIteration, g) + if w != g { + return fmt.Errorf("non-roundrobin, first iter: %v, second iter: %v", gotSliceFirstIteration, gotSliceSecondIteration) + } + } + } + + return nil +} + +// SubConnFromPicker returns a function which returns a SubConn by calling the +// Pick() method of the provided picker. There is no caching of SubConns here. +// Every invocation of the returned function results in a new pick. +func SubConnFromPicker(p balancer.Picker) func() balancer.SubConn { + return func() balancer.SubConn { + scst, _ := p.Pick(balancer.PickInfo{}) + return scst.SubConn + } +} + +// ErrTestConstPicker is error returned by test const picker. +var ErrTestConstPicker = fmt.Errorf("const picker error") + +// TestConstPicker is a const picker for tests. +type TestConstPicker struct { + Err error + SC balancer.SubConn +} + +// Pick returns the const SubConn or the error. +func (tcp *TestConstPicker) Pick(info balancer.PickInfo) (balancer.PickResult, error) { + if tcp.Err != nil { + return balancer.PickResult{}, tcp.Err + } + return balancer.PickResult{SubConn: tcp.SC}, nil +} diff --git a/internal/testutils/channel.go b/internal/testutils/channel.go index 35f67ea285c1..991d05cdde74 100644 --- a/internal/testutils/channel.go +++ b/internal/testutils/channel.go @@ -18,45 +18,81 @@ package testutils import ( - "errors" - "time" + "context" ) -// ErrRecvTimeout is an error to indicate that a receive operation on the -// channel timed out. -var ErrRecvTimeout = errors.New("timed out when waiting for value on channel") - -const ( - // DefaultChanRecvTimeout is the default timeout for receive operations on the - // underlying channel. - DefaultChanRecvTimeout = 1 * time.Second - // DefaultChanBufferSize is the default buffer size of the underlying channel. - DefaultChanBufferSize = 1 -) +// DefaultChanBufferSize is the default buffer size of the underlying channel. +const DefaultChanBufferSize = 1 // Channel wraps a generic channel and provides a timed receive operation. type Channel struct { - ch chan interface{} + ch chan any } // Send sends value on the underlying channel. -func (cwt *Channel) Send(value interface{}) { - cwt.ch <- value +func (c *Channel) Send(value any) { + c.ch <- value } -// Receive returns the value received on the underlying channel, or -// ErrRecvTimeout if DefaultChanRecvTimeout amount of time elapses. -func (cwt *Channel) Receive() (interface{}, error) { - timer := time.NewTimer(DefaultChanRecvTimeout) +// SendContext sends value on the underlying channel, or returns an error if +// the context expires. +func (c *Channel) SendContext(ctx context.Context, value any) error { select { - case <-timer.C: - return nil, ErrRecvTimeout - case got := <-cwt.ch: - timer.Stop() + case c.ch <- value: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// SendOrFail attempts to send value on the underlying channel. Returns true +// if successful or false if the channel was full. +func (c *Channel) SendOrFail(value any) bool { + select { + case c.ch <- value: + return true + default: + return false + } +} + +// ReceiveOrFail returns the value on the underlying channel and true, or nil +// and false if the channel was empty. +func (c *Channel) ReceiveOrFail() (any, bool) { + select { + case got := <-c.ch: + return got, true + default: + return nil, false + } +} + +// Receive returns the value received on the underlying channel, or the error +// returned by ctx if it is closed or cancelled. +func (c *Channel) Receive(ctx context.Context) (any, error) { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case got := <-c.ch: return got, nil } } +// Replace clears the value on the underlying channel, and sends the new value. +// +// It's expected to be used with a size-1 channel, to only keep the most +// up-to-date item. This method is inherently racy when invoked concurrently +// from multiple goroutines. +func (c *Channel) Replace(value any) { + for { + select { + case c.ch <- value: + return + case <-c.ch: + } + } +} + // NewChannel returns a new Channel. func NewChannel() *Channel { return NewChannelWithSize(DefaultChanBufferSize) @@ -64,5 +100,5 @@ func NewChannel() *Channel { // NewChannelWithSize returns a new Channel with a buffer of bufSize. func NewChannelWithSize(bufSize int) *Channel { - return &Channel{ch: make(chan interface{}, bufSize)} + return &Channel{ch: make(chan any, bufSize)} } diff --git a/internal/testutils/fakegrpclb/server.go b/internal/testutils/fakegrpclb/server.go new file mode 100644 index 000000000000..82be2c1af1a4 --- /dev/null +++ b/internal/testutils/fakegrpclb/server.go @@ -0,0 +1,249 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package fakegrpclb provides a fake implementation of the grpclb server. +package fakegrpclb + +import ( + "errors" + "fmt" + "io" + "net" + "strconv" + "sync" + "time" + + "google.golang.org/grpc" + lbgrpc "google.golang.org/grpc/balancer/grpclb/grpc_lb_v1" + lbpb "google.golang.org/grpc/balancer/grpclb/grpc_lb_v1" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/status" +) + +var logger = grpclog.Component("fake_grpclb") + +// ServerParams wraps options passed while creating a Server. +type ServerParams struct { + ListenPort int // Listening port for the balancer server. + ServerOptions []grpc.ServerOption // gRPC options for the balancer server. + + LoadBalancedServiceName string // Service name being load balanced for. + LoadBalancedServicePort int // Service port being load balanced for. + BackendAddresses []string // Service backends to balance load across. + ShortStream bool // End balancer stream after sending server list. +} + +// Server is a fake implementation of the grpclb LoadBalancer service. It does +// not support stats reporting from clients, and always sends back a static list +// of backends to the client to balance load across. +// +// It is safe for concurrent access. +type Server struct { + lbgrpc.UnimplementedLoadBalancerServer + + // Options copied over from ServerParams passed to NewServer. + sOpts []grpc.ServerOption // gRPC server options. + serviceName string // Service name being load balanced for. + servicePort int // Service port being load balanced for. + shortStream bool // End balancer stream after sending server list. + + // Values initialized using ServerParams passed to NewServer. + backends []*lbpb.Server // Service backends to balance load across. + lis net.Listener // Listener for grpc connections to the LoadBalancer service. + + // mu guards access to below fields. + mu sync.Mutex + grpcServer *grpc.Server // Underlying grpc server. + address string // Actual listening address. + + stopped chan struct{} // Closed when Stop() is called. +} + +// NewServer creates a new Server with passed in params. Returns a non-nil error +// if the params are invalid. +func NewServer(params ServerParams) (*Server, error) { + var servers []*lbpb.Server + for _, addr := range params.BackendAddresses { + ipStr, portStr, err := net.SplitHostPort(addr) + if err != nil { + return nil, fmt.Errorf("failed to parse list of backend address %q: %v", addr, err) + } + ip := net.ParseIP(ipStr) + if ip == nil { + return nil, fmt.Errorf("failed to parse ip: %q", ipStr) + } + port, err := strconv.Atoi(portStr) + if err != nil { + return nil, fmt.Errorf("failed to convert port %q to int", portStr) + } + logger.Infof("Adding backend ip: %q, port: %d to server list", ip.String(), port) + servers = append(servers, &lbpb.Server{ + IpAddress: ip, + Port: int32(port), + }) + } + + lis, err := net.Listen("tcp", "localhost:"+strconv.Itoa(params.ListenPort)) + if err != nil { + return nil, fmt.Errorf("failed to listen on port %q: %v", params.ListenPort, err) + } + + return &Server{ + sOpts: params.ServerOptions, + serviceName: params.LoadBalancedServiceName, + servicePort: params.LoadBalancedServicePort, + shortStream: params.ShortStream, + backends: servers, + lis: lis, + address: lis.Addr().String(), + stopped: make(chan struct{}), + }, nil +} + +// Serve starts serving the LoadBalancer service on a gRPC server. +// +// It returns early with a non-nil error if it is unable to start serving. +// Otherwise, it blocks until Stop() is called, at which point it returns the +// error returned by the underlying grpc.Server's Serve() method. +func (s *Server) Serve() error { + s.mu.Lock() + if s.grpcServer != nil { + s.mu.Unlock() + return errors.New("Serve() called multiple times") + } + + server := grpc.NewServer(s.sOpts...) + s.grpcServer = server + s.mu.Unlock() + + logger.Infof("Begin listening on %s", s.lis.Addr().String()) + lbgrpc.RegisterLoadBalancerServer(server, s) + return server.Serve(s.lis) // This call will block. +} + +// Stop stops serving the LoadBalancer service and unblocks the preceding call +// to Serve(). +func (s *Server) Stop() { + defer close(s.stopped) + s.mu.Lock() + if s.grpcServer != nil { + s.grpcServer.Stop() + s.grpcServer = nil + } + s.mu.Unlock() +} + +// Address returns the host:port on which the LoadBalancer service is serving. +func (s *Server) Address() string { + s.mu.Lock() + defer s.mu.Unlock() + return s.address +} + +// BalanceLoad provides a fake implementation of the LoadBalancer service. +func (s *Server) BalanceLoad(stream lbgrpc.LoadBalancer_BalanceLoadServer) error { + logger.Info("New BalancerLoad stream started") + + req, err := stream.Recv() + if err == io.EOF { + logger.Warning("Received EOF when reading from the stream") + return nil + } + if err != nil { + logger.Warning("Failed to read LoadBalanceRequest from stream: %v", err) + return err + } + logger.Infof("Received LoadBalancerRequest:\n%s", pretty.ToJSON(req)) + + // Initial request contains the service being load balanced for. + initialReq := req.GetInitialRequest() + if initialReq == nil { + logger.Info("First message on the stream does not contain an InitialLoadBalanceRequest") + return status.Error(codes.Unknown, "First request not an InitialLoadBalanceRequest") + } + + // Basic validation of the service name and port from the incoming request. + // + // Clients targeting service:port can sometimes include the ":port" suffix in + // their requested names; handle this case. + serviceName, port, err := net.SplitHostPort(initialReq.Name) + if err != nil { + // Requested name did not contain a port. So, use the name as is. + serviceName = initialReq.Name + } else { + p, err := strconv.Atoi(port) + if err != nil { + logger.Info("Failed to parse requested service port %q to integer", port) + return status.Error(codes.Unknown, "Bad requested service port number") + } + if p != s.servicePort { + logger.Info("Requested service port number %q does not match expected", port, s.servicePort) + return status.Error(codes.Unknown, "Bad requested service port number") + } + } + if serviceName != s.serviceName { + logger.Info("Requested service name %q does not match expected %q", serviceName, s.serviceName) + return status.Error(codes.NotFound, "Bad requested service name") + } + + // Empty initial response disables stats reporting from the client. Stats + // reporting from the client is used to determine backend load and is not + // required for the purposes of this fake. + initResp := &lbpb.LoadBalanceResponse{ + LoadBalanceResponseType: &lbpb.LoadBalanceResponse_InitialResponse{ + InitialResponse: &lbpb.InitialLoadBalanceResponse{}, + }, + } + if err := stream.Send(initResp); err != nil { + logger.Warningf("Failed to send InitialLoadBalanceResponse on the stream: %v", err) + return err + } + + resp := &lbpb.LoadBalanceResponse{ + LoadBalanceResponseType: &lbpb.LoadBalanceResponse_ServerList{ + ServerList: &lbpb.ServerList{Servers: s.backends}, + }, + } + logger.Infof("Sending response with server list: %s", pretty.ToJSON(resp)) + if err := stream.Send(resp); err != nil { + logger.Warningf("Failed to send InitialLoadBalanceResponse on the stream: %v", err) + return err + } + + if s.shortStream { + logger.Info("Ending stream early as the short stream option was set") + return nil + } + + for { + select { + case <-stream.Context().Done(): + return nil + case <-s.stopped: + return nil + case <-time.After(10 * time.Second): + logger.Infof("Sending response with server list: %s", pretty.ToJSON(resp)) + if err := stream.Send(resp); err != nil { + logger.Warningf("Failed to send InitialLoadBalanceResponse on the stream: %v", err) + return err + } + } + } +} diff --git a/internal/testutils/http_client.go b/internal/testutils/http_client.go new file mode 100644 index 000000000000..9832bf305756 --- /dev/null +++ b/internal/testutils/http_client.go @@ -0,0 +1,63 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package testutils + +import ( + "context" + "net/http" + "time" +) + +// DefaultHTTPRequestTimeout is the default timeout value for the amount of time +// this client waits for a response to be pushed on RespChan before it fails the +// Do() call. +const DefaultHTTPRequestTimeout = 1 * time.Second + +// FakeHTTPClient helps mock out HTTP calls made by the code under test. It +// makes HTTP requests made by the code under test available through a channel, +// and makes it possible to inject various responses. +type FakeHTTPClient struct { + // ReqChan exposes the HTTP.Request made by the code under test. + ReqChan *Channel + // RespChan is a channel on which this fake client accepts responses to be + // sent to the code under test. + RespChan *Channel + // Err, if set, is returned by Do(). + Err error + // RecvTimeout is the amount of the time this client waits for a response to + // be pushed on RespChan before it fails the Do() call. If this field is + // left unspecified, DefaultHTTPRequestTimeout is used. + RecvTimeout time.Duration +} + +// Do pushes req on ReqChan and returns the response available on RespChan. +func (fc *FakeHTTPClient) Do(req *http.Request) (*http.Response, error) { + fc.ReqChan.Send(req) + + timeout := fc.RecvTimeout + if timeout == 0 { + timeout = DefaultHTTPRequestTimeout + } + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + val, err := fc.RespChan.Receive(ctx) + if err != nil { + return nil, err + } + return val.(*http.Response), fc.Err +} diff --git a/internal/credentials/spiffe_appengine.go b/internal/testutils/local_listener.go similarity index 74% rename from internal/credentials/spiffe_appengine.go rename to internal/testutils/local_listener.go index af6f57719768..f831b95f4133 100644 --- a/internal/credentials/spiffe_appengine.go +++ b/internal/testutils/local_listener.go @@ -1,5 +1,3 @@ -// +build appengine - /* * * Copyright 2020 gRPC authors. @@ -18,14 +16,11 @@ * */ -package credentials +package testutils -import ( - "crypto/tls" - "net/url" -) +import "net" -// SPIFFEIDFromState is a no-op for appengine builds. -func SPIFFEIDFromState(state tls.ConnectionState) *url.URL { - return nil +// LocalTCPListener returns a net.Listener listening on local address and port. +func LocalTCPListener() (net.Listener, error) { + return net.Listen("tcp", "localhost:0") } diff --git a/internal/testutils/marshal_any.go b/internal/testutils/marshal_any.go new file mode 100644 index 000000000000..9ddef6de15d6 --- /dev/null +++ b/internal/testutils/marshal_any.go @@ -0,0 +1,36 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package testutils + +import ( + "fmt" + + "github.com/golang/protobuf/proto" + "github.com/golang/protobuf/ptypes" + "google.golang.org/protobuf/types/known/anypb" +) + +// MarshalAny is a convenience function to marshal protobuf messages into any +// protos. It will panic if the marshaling fails. +func MarshalAny(m proto.Message) *anypb.Any { + a, err := ptypes.MarshalAny(m) + if err != nil { + panic(fmt.Sprintf("ptypes.MarshalAny(%+v) failed: %v", m, err)) + } + return a +} diff --git a/internal/testutils/parse_port.go b/internal/testutils/parse_port.go new file mode 100644 index 000000000000..c633af06a7db --- /dev/null +++ b/internal/testutils/parse_port.go @@ -0,0 +1,39 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package testutils + +import ( + "net" + "strconv" + "testing" +) + +// ParsePort returns the port from the given address string, as a unit32. +func ParsePort(t *testing.T, addr string) uint32 { + t.Helper() + + _, p, err := net.SplitHostPort(addr) + if err != nil { + t.Fatalf("Invalid serving address: %v", err) + } + port, err := strconv.ParseUint(p, 10, 32) + if err != nil { + t.Fatalf("Invalid serving port: %v", err) + } + return uint32(port) +} diff --git a/internal/testutils/parse_url.go b/internal/testutils/parse_url.go new file mode 100644 index 000000000000..ff276e4d0c38 --- /dev/null +++ b/internal/testutils/parse_url.go @@ -0,0 +1,34 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package testutils + +import ( + "fmt" + "net/url" +) + +// MustParseURL attempts to parse the provided target using url.Parse() +// and panics if parsing fails. +func MustParseURL(target string) *url.URL { + u, err := url.Parse(target) + if err != nil { + panic(fmt.Sprintf("Error parsing target(%s): %v", target, err)) + } + return u +} diff --git a/internal/testutils/pickfirst/pickfirst.go b/internal/testutils/pickfirst/pickfirst.go new file mode 100644 index 000000000000..aa90ffc531f4 --- /dev/null +++ b/internal/testutils/pickfirst/pickfirst.go @@ -0,0 +1,82 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package pickfirst contains helper functions to check for pickfirst load +// balancing of RPCs in tests. +package pickfirst + +import ( + "context" + "fmt" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/resolver" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +// CheckRPCsToBackend makes a bunch of RPCs on the given ClientConn and verifies +// if the RPCs are routed to a peer matching wantAddr. +// +// Returns a non-nil error if context deadline expires before all RPCs begin to +// be routed to the peer matching wantAddr, or if the backend returns RPC errors. +func CheckRPCsToBackend(ctx context.Context, cc *grpc.ClientConn, wantAddr resolver.Address) error { + client := testgrpc.NewTestServiceClient(cc) + peer := &peer.Peer{} + // Make sure that 20 RPCs in a row reach the expected backend. Some + // tests switch from round_robin back to pick_first and call this + // function. None of our tests spin up more than 10 backends. So, + // waiting for 20 RPCs to reach a single backend would a decent + // indicator of having switched to pick_first. + count := 0 + for { + time.Sleep(time.Millisecond) + if ctx.Err() != nil { + return fmt.Errorf("timeout waiting for RPC to be routed to %s", wantAddr.Addr) + } + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer)); err != nil { + // Some tests remove backends and check if pick_first is happening across + // the remaining backends. In such cases, RPCs can initially fail on the + // connection using the removed backend. Just keep retrying and eventually + // the connection using the removed backend will shutdown and will be + // removed. + continue + } + if peer.Addr.String() != wantAddr.Addr { + count = 0 + continue + } + count++ + if count > 20 { + break + } + } + // Make sure subsequent RPCs are all routed to the same backend. + for i := 0; i < 10; i++ { + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer)); err != nil { + return fmt.Errorf("EmptyCall() = %v, want ", err) + } + if gotAddr := peer.Addr.String(); gotAddr != wantAddr.Addr { + return fmt.Errorf("rpc sent to peer %q, want peer %q", gotAddr, wantAddr) + } + } + return nil +} diff --git a/internal/testutils/restartable_listener.go b/internal/testutils/restartable_listener.go new file mode 100644 index 000000000000..efe4019a08c2 --- /dev/null +++ b/internal/testutils/restartable_listener.go @@ -0,0 +1,98 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package testutils + +import ( + "net" + "sync" +) + +type tempError struct{} + +func (*tempError) Error() string { + return "restartable listener temporary error" +} +func (*tempError) Temporary() bool { + return true +} + +// RestartableListener wraps a net.Listener and supports stopping and restarting +// the latter. +type RestartableListener struct { + lis net.Listener + + mu sync.Mutex + stopped bool + conns []net.Conn +} + +// NewRestartableListener returns a new RestartableListener wrapping l. +func NewRestartableListener(l net.Listener) *RestartableListener { + return &RestartableListener{lis: l} +} + +// Accept waits for and returns the next connection to the listener. +// +// If the listener is currently not accepting new connections, because `Stop` +// was called on it, the connection is immediately closed after accepting +// without any bytes being sent on it. +func (l *RestartableListener) Accept() (net.Conn, error) { + conn, err := l.lis.Accept() + if err != nil { + return nil, err + } + + l.mu.Lock() + defer l.mu.Unlock() + if l.stopped { + conn.Close() + return nil, &tempError{} + } + l.conns = append(l.conns, conn) + return conn, nil +} + +// Close closes the listener. +func (l *RestartableListener) Close() error { + return l.lis.Close() +} + +// Addr returns the listener's network address. +func (l *RestartableListener) Addr() net.Addr { + return l.lis.Addr() +} + +// Stop closes existing connections on the listener and prevents new connections +// from being accepted. +func (l *RestartableListener) Stop() { + l.mu.Lock() + l.stopped = true + for _, conn := range l.conns { + conn.Close() + } + l.conns = nil + l.mu.Unlock() +} + +// Restart gets a previously stopped listener to start accepting connections. +func (l *RestartableListener) Restart() { + l.mu.Lock() + l.stopped = false + l.mu.Unlock() +} diff --git a/internal/testutils/rls/fake_rls_server.go b/internal/testutils/rls/fake_rls_server.go new file mode 100644 index 000000000000..e64c9de3ae7f --- /dev/null +++ b/internal/testutils/rls/fake_rls_server.go @@ -0,0 +1,134 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package rls contains utilities for RouteLookupService e2e tests. +package rls + +import ( + "context" + "net" + "sync" + "testing" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + rlsgrpc "google.golang.org/grpc/internal/proto/grpc_lookup_v1" + rlspb "google.golang.org/grpc/internal/proto/grpc_lookup_v1" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/status" +) + +// RouteLookupResponse wraps an RLS response and the associated error to be sent +// to a client when the RouteLookup RPC is invoked. +type RouteLookupResponse struct { + Resp *rlspb.RouteLookupResponse + Err error +} + +// SetupFakeRLSServer starts and returns a fake RouteLookupService server +// listening on the given listener or on a random local port. Also returns a +// channel for tests to get notified whenever the RouteLookup RPC is invoked on +// the fake server. +// +// This function sets up the fake server to respond with an empty response for +// the RouteLookup RPCs. Tests can override this by calling the +// SetResponseCallback() method on the returned fake server. +func SetupFakeRLSServer(t *testing.T, lis net.Listener, opts ...grpc.ServerOption) (*FakeRouteLookupServer, chan struct{}) { + s, cancel := StartFakeRouteLookupServer(t, lis, opts...) + t.Logf("Started fake RLS server at %q", s.Address) + + ch := make(chan struct{}, 1) + s.SetRequestCallback(func(request *rlspb.RouteLookupRequest) { + select { + case ch <- struct{}{}: + default: + } + }) + t.Cleanup(cancel) + return s, ch +} + +// FakeRouteLookupServer is a fake implementation of the RouteLookupService. +// +// It is safe for concurrent use. +type FakeRouteLookupServer struct { + rlsgrpc.UnimplementedRouteLookupServiceServer + Address string + + mu sync.Mutex + respCb func(context.Context, *rlspb.RouteLookupRequest) *RouteLookupResponse + reqCb func(*rlspb.RouteLookupRequest) +} + +// StartFakeRouteLookupServer starts a fake RLS server listening for requests on +// lis. If lis is nil, it creates a new listener on a random local port. The +// returned cancel function should be invoked by the caller upon completion of +// the test. +func StartFakeRouteLookupServer(t *testing.T, lis net.Listener, opts ...grpc.ServerOption) (*FakeRouteLookupServer, func()) { + t.Helper() + + if lis == nil { + var err error + lis, err = testutils.LocalTCPListener() + if err != nil { + t.Fatalf("net.Listen() failed: %v", err) + } + } + + s := &FakeRouteLookupServer{Address: lis.Addr().String()} + server := grpc.NewServer(opts...) + rlsgrpc.RegisterRouteLookupServiceServer(server, s) + go server.Serve(lis) + return s, func() { server.Stop() } +} + +// RouteLookup implements the RouteLookupService. +func (s *FakeRouteLookupServer) RouteLookup(ctx context.Context, req *rlspb.RouteLookupRequest) (*rlspb.RouteLookupResponse, error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.reqCb != nil { + s.reqCb(req) + } + if err := ctx.Err(); err != nil { + return nil, status.Error(codes.DeadlineExceeded, err.Error()) + } + if s.respCb == nil { + return &rlspb.RouteLookupResponse{}, nil + } + resp := s.respCb(ctx, req) + return resp.Resp, resp.Err +} + +// SetResponseCallback sets a callback to be invoked on every RLS request. If +// this callback is set, the response returned by the fake server depends on the +// value returned by the callback. If this callback is not set, the fake server +// responds with an empty response. +func (s *FakeRouteLookupServer) SetResponseCallback(f func(context.Context, *rlspb.RouteLookupRequest) *RouteLookupResponse) { + s.mu.Lock() + s.respCb = f + s.mu.Unlock() +} + +// SetRequestCallback sets a callback to be invoked on every RLS request. The +// callback is given the incoming request, and tests can use this to verify that +// the request matches its expectations. +func (s *FakeRouteLookupServer) SetRequestCallback(f func(*rlspb.RouteLookupRequest)) { + s.mu.Lock() + s.reqCb = f + s.mu.Unlock() +} diff --git a/internal/testutils/roundrobin/roundrobin.go b/internal/testutils/roundrobin/roundrobin.go new file mode 100644 index 000000000000..ba595735364d --- /dev/null +++ b/internal/testutils/roundrobin/roundrobin.go @@ -0,0 +1,223 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package roundrobin contains helper functions to check for roundrobin and +// weighted-roundrobin load balancing of RPCs in tests. +package roundrobin + +import ( + "context" + "fmt" + "math" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/resolver" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +var logger = grpclog.Component("testutils-roundrobin") + +// waitForTrafficToReachBackends repeatedly makes RPCs using the provided +// TestServiceClient until RPCs reach all backends specified in addrs, or the +// context expires, in which case a non-nil error is returned. +func waitForTrafficToReachBackends(ctx context.Context, client testgrpc.TestServiceClient, addrs []resolver.Address) error { + // Make sure connections to all backends are up. We need to do this two + // times (to be sure that round_robin has kicked in) because the channel + // could have been configured with a different LB policy before the switch + // to round_robin. And the previous LB policy could be sharing backends with + // round_robin, and therefore in the first iteration of this loop, RPCs + // could land on backends owned by the previous LB policy. + for j := 0; j < 2; j++ { + for i := 0; i < len(addrs); i++ { + for { + time.Sleep(time.Millisecond) + if ctx.Err() != nil { + return fmt.Errorf("timeout waiting for connection to %q to be up", addrs[i].Addr) + } + var peer peer.Peer + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&peer)); err != nil { + // Some tests remove backends and check if round robin is + // happening across the remaining backends. In such cases, + // RPCs can initially fail on the connection using the + // removed backend. Just keep retrying and eventually the + // connection using the removed backend will shutdown and + // will be removed. + continue + } + if peer.Addr.String() == addrs[i].Addr { + break + } + } + } + } + return nil +} + +// CheckRoundRobinRPCs verifies that EmptyCall RPCs on the given ClientConn, +// connected to a server exposing the test.grpc_testing.TestService, are +// roundrobined across the given backend addresses. +// +// Returns a non-nil error if context deadline expires before RPCs start to get +// roundrobined across the given backends. +func CheckRoundRobinRPCs(ctx context.Context, client testgrpc.TestServiceClient, addrs []resolver.Address) error { + if err := waitForTrafficToReachBackends(ctx, client, addrs); err != nil { + return err + } + + // At this point, RPCs are getting successfully executed at the backends + // that we care about. To support duplicate addresses (in addrs) and + // backends being removed from the list of addresses passed to the + // roundrobin LB, we do the following: + // 1. Determine the count of RPCs that we expect each of our backends to + // receive per iteration. + // 2. Wait until the same pattern repeats a few times, or the context + // deadline expires. + wantAddrCount := make(map[string]int) + for _, addr := range addrs { + wantAddrCount[addr.Addr]++ + } + for ; ctx.Err() == nil; <-time.After(time.Millisecond) { + // Perform 3 more iterations. + var iterations [][]string + for i := 0; i < 3; i++ { + iteration := make([]string, len(addrs)) + for c := 0; c < len(addrs); c++ { + var peer peer.Peer + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&peer)); err != nil { + return fmt.Errorf("EmptyCall() = %v, want ", err) + } + iteration[c] = peer.Addr.String() + } + iterations = append(iterations, iteration) + } + // Ensure the first iteration contains all addresses in addrs. + gotAddrCount := make(map[string]int) + for _, addr := range iterations[0] { + gotAddrCount[addr]++ + } + if diff := cmp.Diff(gotAddrCount, wantAddrCount); diff != "" { + logger.Infof("non-roundrobin, got address count in one iteration: %v, want: %v, Diff: %s", gotAddrCount, wantAddrCount, diff) + continue + } + // Ensure all three iterations contain the same addresses. + if !cmp.Equal(iterations[0], iterations[1]) || !cmp.Equal(iterations[0], iterations[2]) { + logger.Infof("non-roundrobin, first iter: %v, second iter: %v, third iter: %v", iterations[0], iterations[1], iterations[2]) + continue + } + return nil + } + return fmt.Errorf("timeout when waiting for roundrobin distribution of RPCs across addresses: %v", addrs) +} + +// CheckWeightedRoundRobinRPCs verifies that EmptyCall RPCs on the given +// ClientConn, connected to a server exposing the test.grpc_testing.TestService, +// are weighted roundrobined (with randomness) across the given backend +// addresses. +// +// Returns a non-nil error if context deadline expires before RPCs start to get +// roundrobined across the given backends. +func CheckWeightedRoundRobinRPCs(ctx context.Context, client testgrpc.TestServiceClient, addrs []resolver.Address) error { + if err := waitForTrafficToReachBackends(ctx, client, addrs); err != nil { + return err + } + + // At this point, RPCs are getting successfully executed at the backends + // that we care about. To take the randomness of the WRR into account, we + // look for approximate distribution instead of exact. + wantAddrCount := make(map[string]int) + for _, addr := range addrs { + wantAddrCount[addr.Addr]++ + } + wantRatio := make(map[string]float64) + for addr, count := range wantAddrCount { + wantRatio[addr] = float64(count) / float64(len(addrs)) + } + + // There is a small possibility that RPCs are reaching backends that we + // don't expect them to reach here. The can happen because: + // - at time T0, the list of backends [A, B, C, D]. + // - at time T1, the test updates the list of backends to [A, B, C], and + // immediately starts attempting to check the distribution of RPCs to the + // new backends. + // - there is no way for the test to wait for a new picker to be pushed on + // to the channel (which contains the updated list of backends) before + // starting to attempt the RPC distribution checks. + // - This is usually a transitory state and will eventually fix itself when + // the new picker is pushed on the channel, and RPCs will start getting + // routed to only backends that we care about. + // + // We work around this situation by using two loops. The inner loop contains + // the meat of the calculations, and includes the logic which factors out + // the randomness in weighted roundrobin. If we ever see an RPCs getting + // routed to a backend that we dont expect it to get routed to, we break + // from the inner loop thereby resetting all state and start afresh. + for { + results := make(map[string]float64) + totalCount := float64(0) + InnerLoop: + for { + if ctx.Err() != nil { + return fmt.Errorf("timeout when waiting for roundrobin distribution of RPCs across addresses: %v", addrs) + } + for i := 0; i < len(addrs); i++ { + var peer peer.Peer + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&peer)); err != nil { + return fmt.Errorf("EmptyCall() = %v, want ", err) + } + if addr := peer.Addr.String(); wantAddrCount[addr] == 0 { + break InnerLoop + } + results[peer.Addr.String()]++ + } + totalCount += float64(len(addrs)) + + gotRatio := make(map[string]float64) + for addr, count := range results { + gotRatio[addr] = count / totalCount + } + if equalApproximate(gotRatio, wantRatio) { + return nil + } + logger.Infof("non-weighted-roundrobin, gotRatio: %v, wantRatio: %v", gotRatio, wantRatio) + } + <-time.After(time.Millisecond) + } +} + +func equalApproximate(got, want map[string]float64) bool { + if len(got) != len(want) { + return false + } + opt := cmp.Comparer(func(x, y float64) bool { + delta := math.Abs(x - y) + mean := math.Abs(x+y) / 2.0 + return delta/mean < 0.05 + }) + for addr := range want { + if !cmp.Equal(got[addr], want[addr], opt) { + return false + } + } + return true +} diff --git a/internal/testutils/state.go b/internal/testutils/state.go new file mode 100644 index 000000000000..246b07a7ea19 --- /dev/null +++ b/internal/testutils/state.go @@ -0,0 +1,85 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package testutils + +import ( + "context" + "testing" + + "google.golang.org/grpc/connectivity" +) + +// A StateChanger reports state changes, e.g. a grpc.ClientConn. +type StateChanger interface { + // Connect begins connecting the StateChanger. + Connect() + // GetState returns the current state of the StateChanger. + GetState() connectivity.State + // WaitForStateChange returns true when the state becomes s, or returns + // false if ctx is canceled first. + WaitForStateChange(ctx context.Context, s connectivity.State) bool +} + +// StayConnected makes sc stay connected by repeatedly calling sc.Connect() +// until the state becomes Shutdown or until ithe context expires. +func StayConnected(ctx context.Context, sc StateChanger) { + for { + state := sc.GetState() + switch state { + case connectivity.Idle: + sc.Connect() + case connectivity.Shutdown: + return + } + if !sc.WaitForStateChange(ctx, state) { + return + } + } +} + +// AwaitState waits for sc to enter stateWant or fatal errors if it doesn't +// happen before ctx expires. +func AwaitState(ctx context.Context, t *testing.T, sc StateChanger, stateWant connectivity.State) { + t.Helper() + for state := sc.GetState(); state != stateWant; state = sc.GetState() { + if !sc.WaitForStateChange(ctx, state) { + t.Fatalf("Timed out waiting for state change. got %v; want %v", state, stateWant) + } + } +} + +// AwaitNotState waits for sc to leave stateDoNotWant or fatal errors if it +// doesn't happen before ctx expires. +func AwaitNotState(ctx context.Context, t *testing.T, sc StateChanger, stateDoNotWant connectivity.State) { + t.Helper() + for state := sc.GetState(); state == stateDoNotWant; state = sc.GetState() { + if !sc.WaitForStateChange(ctx, state) { + t.Fatalf("Timed out waiting for state change. got %v; want NOT %v", state, stateDoNotWant) + } + } +} + +// AwaitNoStateChange expects ctx to be canceled before sc's state leaves +// currState, and fatal errors otherwise. +func AwaitNoStateChange(ctx context.Context, t *testing.T, sc StateChanger, currState connectivity.State) { + t.Helper() + if sc.WaitForStateChange(ctx, currState) { + t.Fatalf("State changed from %q to %q when no state change was expected", currState, sc.GetState()) + } +} diff --git a/internal/testutils/wrappers.go b/internal/testutils/wrappers.go new file mode 100644 index 000000000000..c9b596d8851c --- /dev/null +++ b/internal/testutils/wrappers.go @@ -0,0 +1,74 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package testutils + +import ( + "net" + "testing" +) + +// ConnWrapper wraps a net.Conn and pushes on a channel when closed. +type ConnWrapper struct { + net.Conn + CloseCh *Channel +} + +// Close closes the connection and sends a value on the close channel. +func (cw *ConnWrapper) Close() error { + err := cw.Conn.Close() + cw.CloseCh.Replace(nil) + return err +} + +// ListenerWrapper wraps a net.Listener and the returned net.Conn. +// +// It pushes on a channel whenever it accepts a new connection. +type ListenerWrapper struct { + net.Listener + NewConnCh *Channel +} + +// Accept wraps the Listener Accept and sends the accepted connection on a +// channel. +func (l *ListenerWrapper) Accept() (net.Conn, error) { + c, err := l.Listener.Accept() + if err != nil { + return nil, err + } + closeCh := NewChannel() + conn := &ConnWrapper{Conn: c, CloseCh: closeCh} + l.NewConnCh.Send(conn) + return conn, nil +} + +// NewListenerWrapper returns a ListenerWrapper. +func NewListenerWrapper(t *testing.T, lis net.Listener) *ListenerWrapper { + if lis == nil { + var err error + lis, err = LocalTCPListener() + if err != nil { + t.Fatal(err) + } + } + + return &ListenerWrapper{ + Listener: lis, + NewConnCh: NewChannel(), + } +} diff --git a/internal/testutils/wrr.go b/internal/testutils/wrr.go new file mode 100644 index 000000000000..5dcc7a8d37a4 --- /dev/null +++ b/internal/testutils/wrr.go @@ -0,0 +1,73 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package testutils + +import ( + "fmt" + "sync" + + "google.golang.org/grpc/internal/wrr" +) + +// testWRR is a deterministic WRR implementation. +// +// The real implementation does random WRR. testWRR makes the balancer behavior +// deterministic and easier to test. +// +// With {a: 2, b: 3}, the Next() results will be {a, a, b, b, b}. +type testWRR struct { + itemsWithWeight []struct { + item any + weight int64 + } + length int + + mu sync.Mutex + idx int // The index of the item that will be picked + count int64 // The number of times the current item has been picked. +} + +// NewTestWRR return a WRR for testing. It's deterministic instead of random. +func NewTestWRR() wrr.WRR { + return &testWRR{} +} + +func (twrr *testWRR) Add(item any, weight int64) { + twrr.itemsWithWeight = append(twrr.itemsWithWeight, struct { + item any + weight int64 + }{item: item, weight: weight}) + twrr.length++ +} + +func (twrr *testWRR) Next() any { + twrr.mu.Lock() + iww := twrr.itemsWithWeight[twrr.idx] + twrr.count++ + if twrr.count >= iww.weight { + twrr.idx = (twrr.idx + 1) % twrr.length + twrr.count = 0 + } + twrr.mu.Unlock() + return iww.item +} + +func (twrr *testWRR) String() string { + return fmt.Sprint(twrr.itemsWithWeight) +} diff --git a/internal/testutils/xds/bootstrap/bootstrap.go b/internal/testutils/xds/bootstrap/bootstrap.go new file mode 100644 index 000000000000..190cf028f0b5 --- /dev/null +++ b/internal/testutils/xds/bootstrap/bootstrap.go @@ -0,0 +1,167 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package bootstrap provides functionality to generate bootstrap configuration. +package bootstrap + +import ( + "encoding/json" + "fmt" + "os" + + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal/envconfig" +) + +var logger = grpclog.Component("internal/xds") + +// Options wraps the parameters used to generate bootstrap configuration. +type Options struct { + // NodeID is the node identifier of the gRPC client/server node in the + // proxyless service mesh. + NodeID string + // ServerURI is the address of the management server. + ServerURI string + // IgnoreResourceDeletion, if true, results in a bootstrap config with the + // `server_features` list containing `ignore_resource_deletion`. This results + // in gRPC ignoring resource deletions from the management server, as per A53. + IgnoreResourceDeletion bool + // ClientDefaultListenerResourceNameTemplate is the default listener + // resource name template to be used on the gRPC client. + ClientDefaultListenerResourceNameTemplate string + // ServerListenerResourceNameTemplate is the listener resource name template + // to be used on the gRPC server. + ServerListenerResourceNameTemplate string + // CertificateProviders is the certificate providers configuration. + CertificateProviders map[string]json.RawMessage + // Authorities is a list of non-default authorities. + // + // In the config, an authority contains {ServerURI, xds-version, creds, + // features, etc}. Note that this fields only has ServerURI (it's a + // map[authority-name]ServerURI). The other fields (version, creds, + // features) are assumed to be the same as the default authority (they can + // be added later if needed). + // + // If the env var corresponding to federation (envconfig.XDSFederation) is + // set, an entry with empty string as the key and empty server config as + // value will be added. This will be used by new style resource names with + // an empty authority. + Authorities map[string]string +} + +// CreateFile creates a temporary file with bootstrap contents, based on the +// passed in options, and updates the bootstrap environment variable to point to +// this file. +// +// Returns a cleanup function which will be non-nil if the setup process was +// completed successfully. It is the responsibility of the caller to invoke the +// cleanup function at the end of the test. +func CreateFile(opts Options) (func(), error) { + bootstrapContents, err := Contents(opts) + if err != nil { + return nil, err + } + f, err := os.CreateTemp("", "test_xds_bootstrap_*") + if err != nil { + return nil, fmt.Errorf("failed to created bootstrap file: %v", err) + } + + if err := os.WriteFile(f.Name(), bootstrapContents, 0644); err != nil { + return nil, fmt.Errorf("failed to created bootstrap file: %v", err) + } + logger.Infof("Created bootstrap file at %q with contents: %s\n", f.Name(), bootstrapContents) + + origBootstrapFileName := envconfig.XDSBootstrapFileName + envconfig.XDSBootstrapFileName = f.Name() + return func() { + os.Remove(f.Name()) + envconfig.XDSBootstrapFileName = origBootstrapFileName + }, nil +} + +// Contents returns the contents to go into a bootstrap file, environment, or +// configuration passed to xds.NewXDSResolverWithConfigForTesting. +func Contents(opts Options) ([]byte, error) { + cfg := &bootstrapConfig{ + XdsServers: []server{ + { + ServerURI: opts.ServerURI, + ChannelCreds: []creds{{Type: "insecure"}}, + }, + }, + Node: node{ + ID: opts.NodeID, + }, + CertificateProviders: opts.CertificateProviders, + ClientDefaultListenerResourceNameTemplate: opts.ClientDefaultListenerResourceNameTemplate, + ServerListenerResourceNameTemplate: opts.ServerListenerResourceNameTemplate, + } + cfg.XdsServers[0].ServerFeatures = append(cfg.XdsServers[0].ServerFeatures, "xds_v3") + if opts.IgnoreResourceDeletion { + cfg.XdsServers[0].ServerFeatures = append(cfg.XdsServers[0].ServerFeatures, "ignore_resource_deletion") + } + + auths := make(map[string]authority) + if envconfig.XDSFederation { + // This will end up using the top-level server list for new style + // resources with empty authority. + auths[""] = authority{} + } + for n, auURI := range opts.Authorities { + auths[n] = authority{XdsServers: []server{{ + ServerURI: auURI, + ChannelCreds: []creds{{Type: "insecure"}}, + ServerFeatures: cfg.XdsServers[0].ServerFeatures, + }}} + } + cfg.Authorities = auths + + bootstrapContents, err := json.MarshalIndent(cfg, "", " ") + if err != nil { + return nil, fmt.Errorf("failed to created bootstrap file: %v", err) + } + return bootstrapContents, nil +} + +type bootstrapConfig struct { + XdsServers []server `json:"xds_servers,omitempty"` + Node node `json:"node,omitempty"` + CertificateProviders map[string]json.RawMessage `json:"certificate_providers,omitempty"` + ClientDefaultListenerResourceNameTemplate string `json:"client_default_listener_resource_name_template,omitempty"` + ServerListenerResourceNameTemplate string `json:"server_listener_resource_name_template,omitempty"` + Authorities map[string]authority `json:"authorities,omitempty"` +} + +type authority struct { + XdsServers []server `json:"xds_servers,omitempty"` +} + +type server struct { + ServerURI string `json:"server_uri,omitempty"` + ChannelCreds []creds `json:"channel_creds,omitempty"` + ServerFeatures []string `json:"server_features,omitempty"` +} + +type creds struct { + Type string `json:"type,omitempty"` + Config any `json:"config,omitempty"` +} + +type node struct { + ID string `json:"id,omitempty"` +} diff --git a/internal/testutils/xds/e2e/bootstrap.go b/internal/testutils/xds/e2e/bootstrap.go new file mode 100644 index 000000000000..99702032f817 --- /dev/null +++ b/internal/testutils/xds/e2e/bootstrap.go @@ -0,0 +1,39 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package e2e + +import ( + "encoding/json" + "fmt" +) + +// DefaultFileWatcherConfig is a helper function to create a default certificate +// provider plugin configuration. The test is expected to have setup the files +// appropriately before this configuration is used to instantiate providers. +func DefaultFileWatcherConfig(certPath, keyPath, caPath string) json.RawMessage { + return json.RawMessage(fmt.Sprintf(`{ + "plugin_name": "file_watcher", + "config": { + "certificate_file": %q, + "private_key_file": %q, + "ca_certificate_file": %q, + "refresh_interval": "600s" + } + }`, certPath, keyPath, caPath)) +} diff --git a/internal/testutils/xds/e2e/clientresources.go b/internal/testutils/xds/e2e/clientresources.go new file mode 100644 index 000000000000..7f219b5d569e --- /dev/null +++ b/internal/testutils/xds/e2e/clientresources.go @@ -0,0 +1,647 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package e2e + +import ( + "fmt" + "net" + "strconv" + + "github.com/envoyproxy/go-control-plane/pkg/wellknown" + "github.com/golang/protobuf/proto" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/protobuf/types/known/anypb" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3routerpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3" + v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" + v3typepb "github.com/envoyproxy/go-control-plane/envoy/type/v3" + wrapperspb "github.com/golang/protobuf/ptypes/wrappers" +) + +const ( + // ServerListenerResourceNameTemplate is the Listener resource name template + // used on the server side. + ServerListenerResourceNameTemplate = "grpc/server?xds.resource.listening_address=%s" + // ClientSideCertProviderInstance is the certificate provider instance name + // used in the Cluster resource on the client side. + ClientSideCertProviderInstance = "client-side-certificate-provider-instance" + // ServerSideCertProviderInstance is the certificate provider instance name + // used in the Listener resource on the server side. + ServerSideCertProviderInstance = "server-side-certificate-provider-instance" +) + +// SecurityLevel allows the test to control the security level to be used in the +// resource returned by this package. +type SecurityLevel int + +const ( + // SecurityLevelNone is used when no security configuration is required. + SecurityLevelNone SecurityLevel = iota + // SecurityLevelTLS is used when security configuration corresponding to TLS + // is required. Only the server presents an identity certificate in this + // configuration. + SecurityLevelTLS + // SecurityLevelMTLS is used when security ocnfiguration corresponding to + // mTLS is required. Both client and server present identity certificates in + // this configuration. + SecurityLevelMTLS +) + +// ResourceParams wraps the arguments to be passed to DefaultClientResources. +type ResourceParams struct { + // DialTarget is the client's dial target. This is used as the name of the + // Listener resource. + DialTarget string + // NodeID is the id of the xdsClient to which this update is to be pushed. + NodeID string + // Host is the host of the default Endpoint resource. + Host string + // port is the port of the default Endpoint resource. + Port uint32 + // SecLevel controls the security configuration in the Cluster resource. + SecLevel SecurityLevel +} + +// DefaultClientResources returns a set of resources (LDS, RDS, CDS, EDS) for a +// client to generically connect to one server. +func DefaultClientResources(params ResourceParams) UpdateOptions { + routeConfigName := "route-" + params.DialTarget + clusterName := "cluster-" + params.DialTarget + endpointsName := "endpoints-" + params.DialTarget + return UpdateOptions{ + NodeID: params.NodeID, + Listeners: []*v3listenerpb.Listener{DefaultClientListener(params.DialTarget, routeConfigName)}, + Routes: []*v3routepb.RouteConfiguration{DefaultRouteConfig(routeConfigName, params.DialTarget, clusterName)}, + Clusters: []*v3clusterpb.Cluster{DefaultCluster(clusterName, endpointsName, params.SecLevel)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{DefaultEndpoint(endpointsName, params.Host, []uint32{params.Port})}, + } +} + +// RouterHTTPFilter is the HTTP Filter configuration for the Router filter. +var RouterHTTPFilter = HTTPFilter("router", &v3routerpb.Router{}) + +// DefaultClientListener returns a basic xds Listener resource to be used on +// the client side. +func DefaultClientListener(target, routeName string) *v3listenerpb.Listener { + hcm := testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{Rds: &v3httppb.Rds{ + ConfigSource: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}, + }, + RouteConfigName: routeName, + }}, + HttpFilters: []*v3httppb.HttpFilter{HTTPFilter("router", &v3routerpb.Router{})}, // router fields are unused by grpc + }) + return &v3listenerpb.Listener{ + Name: target, + ApiListener: &v3listenerpb.ApiListener{ApiListener: hcm}, + FilterChains: []*v3listenerpb.FilterChain{{ + Name: "filter-chain-name", + Filters: []*v3listenerpb.Filter{{ + Name: wellknown.HTTPConnectionManager, + ConfigType: &v3listenerpb.Filter_TypedConfig{TypedConfig: hcm}, + }}, + }}, + } +} + +// DefaultServerListener returns a basic xds Listener resource to be used on +// the server side. +func DefaultServerListener(host string, port uint32, secLevel SecurityLevel) *v3listenerpb.Listener { + var tlsContext *v3tlspb.DownstreamTlsContext + switch secLevel { + case SecurityLevelNone: + case SecurityLevelTLS: + tlsContext = &v3tlspb.DownstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: ServerSideCertProviderInstance, + }, + }, + } + case SecurityLevelMTLS: + tlsContext = &v3tlspb.DownstreamTlsContext{ + RequireClientCertificate: &wrapperspb.BoolValue{Value: true}, + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: ServerSideCertProviderInstance, + }, + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{ + ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: ServerSideCertProviderInstance, + }, + }, + }, + } + } + + var ts *v3corepb.TransportSocket + if tlsContext != nil { + ts = &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(tlsContext), + }, + } + } + return &v3listenerpb.Listener{ + Name: fmt.Sprintf(ServerListenerResourceNameTemplate, net.JoinHostPort(host, strconv.Itoa(int(port)))), + Address: &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: host, + PortSpecifier: &v3corepb.SocketAddress_PortValue{ + PortValue: port, + }, + }, + }, + }, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "v4-wildcard", + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{ + { + AddressPrefix: "0.0.0.0", + PrefixLen: &wrapperspb.UInt32Value{ + Value: uint32(0), + }, + }, + }, + SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK, + SourcePrefixRanges: []*v3corepb.CidrRange{ + { + AddressPrefix: "0.0.0.0", + PrefixLen: &wrapperspb.UInt32Value{ + Value: uint32(0), + }, + }, + }, + }, + Filters: []*v3listenerpb.Filter{ + { + Name: "filter-1", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: &v3routepb.RouteConfiguration{ + Name: "routeName", + VirtualHosts: []*v3routepb.VirtualHost{{ + // This "*" string matches on any incoming authority. This is to ensure any + // incoming RPC matches to Route_NonForwardingAction and will proceed as + // normal. + Domains: []string{"*"}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}, + }, + Action: &v3routepb.Route_NonForwardingAction{}, + }}}}}, + }, + HttpFilters: []*v3httppb.HttpFilter{RouterHTTPFilter}, + }), + }, + }, + }, + TransportSocket: ts, + }, + { + Name: "v6-wildcard", + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{ + { + AddressPrefix: "::", + PrefixLen: &wrapperspb.UInt32Value{ + Value: uint32(0), + }, + }, + }, + SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK, + SourcePrefixRanges: []*v3corepb.CidrRange{ + { + AddressPrefix: "::", + PrefixLen: &wrapperspb.UInt32Value{ + Value: uint32(0), + }, + }, + }, + }, + Filters: []*v3listenerpb.Filter{ + { + Name: "filter-1", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: &v3routepb.RouteConfiguration{ + Name: "routeName", + VirtualHosts: []*v3routepb.VirtualHost{{ + // This "*" string matches on any incoming authority. This is to ensure any + // incoming RPC matches to Route_NonForwardingAction and will proceed as + // normal. + Domains: []string{"*"}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}, + }, + Action: &v3routepb.Route_NonForwardingAction{}, + }}}}}, + }, + HttpFilters: []*v3httppb.HttpFilter{RouterHTTPFilter}, + }), + }, + }, + }, + TransportSocket: ts, + }, + }, + } +} + +// HTTPFilter constructs an xds HttpFilter with the provided name and config. +func HTTPFilter(name string, config proto.Message) *v3httppb.HttpFilter { + return &v3httppb.HttpFilter{ + Name: name, + ConfigType: &v3httppb.HttpFilter_TypedConfig{ + TypedConfig: testutils.MarshalAny(config), + }, + } +} + +// DefaultRouteConfig returns a basic xds RouteConfig resource. +func DefaultRouteConfig(routeName, ldsTarget, clusterName string) *v3routepb.RouteConfiguration { + return &v3routepb.RouteConfiguration{ + Name: routeName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{ldsTarget}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}}, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{WeightedClusters: &v3routepb.WeightedCluster{ + Clusters: []*v3routepb.WeightedCluster_ClusterWeight{ + { + Name: clusterName, + Weight: &wrapperspb.UInt32Value{Value: 100}, + }, + }, + }}, + }}, + }}, + }}, + } +} + +// RouteConfigClusterSpecifierType determines the cluster specifier type for the +// route actions configured in the returned RouteConfiguration resource. +type RouteConfigClusterSpecifierType int + +const ( + // RouteConfigClusterSpecifierTypeCluster results in the cluster specifier + // being set to a RouteAction_Cluster. + RouteConfigClusterSpecifierTypeCluster RouteConfigClusterSpecifierType = iota + // RouteConfigClusterSpecifierTypeWeightedCluster results in the cluster + // specifier being set to RouteAction_WeightedClusters. + RouteConfigClusterSpecifierTypeWeightedCluster + // RouteConfigClusterSpecifierTypeClusterSpecifierPlugin results in the + // cluster specifier being set to a RouteAction_ClusterSpecifierPlugin. + RouteConfigClusterSpecifierTypeClusterSpecifierPlugin +) + +// RouteConfigOptions contains options to configure a RouteConfiguration +// resource. +type RouteConfigOptions struct { + // RouteConfigName is the name of the RouteConfiguration resource. + RouteConfigName string + // ListenerName is the name of the Listener resource which uses this + // RouteConfiguration. + ListenerName string + // ClusterSpecifierType determines the cluster specifier type. + ClusterSpecifierType RouteConfigClusterSpecifierType + + // ClusterName is name of the cluster resource used when the cluster + // specifier type is set to RouteConfigClusterSpecifierTypeCluster. + // + // Default value of "A" is used if left unspecified. + ClusterName string + // WeightedClusters is a map from cluster name to weights, and is used when + // the cluster specifier type is set to + // RouteConfigClusterSpecifierTypeWeightedCluster. + // + // Default value of {"A": 75, "B": 25} is used if left unspecified. + WeightedClusters map[string]int + // The below two fields specify the name of the cluster specifier plugin and + // its configuration, and are used when the cluster specifier type is set to + // RouteConfigClusterSpecifierTypeClusterSpecifierPlugin. Tests are expected + // to provide valid values for these fields when appropriate. + ClusterSpecifierPluginName string + ClusterSpecifierPluginConfig *anypb.Any +} + +// RouteConfigResourceWithOptions returns a RouteConfiguration resource +// configured with the provided options. +func RouteConfigResourceWithOptions(opts RouteConfigOptions) *v3routepb.RouteConfiguration { + switch opts.ClusterSpecifierType { + case RouteConfigClusterSpecifierTypeCluster: + clusterName := opts.ClusterName + if clusterName == "" { + clusterName = "A" + } + return &v3routepb.RouteConfiguration{ + Name: opts.RouteConfigName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{opts.ListenerName}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}}, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName}, + }}, + }}, + }}, + } + case RouteConfigClusterSpecifierTypeWeightedCluster: + weightedClusters := opts.WeightedClusters + if weightedClusters == nil { + weightedClusters = map[string]int{"A": 75, "B": 25} + } + clusters := []*v3routepb.WeightedCluster_ClusterWeight{} + for name, weight := range weightedClusters { + clusters = append(clusters, &v3routepb.WeightedCluster_ClusterWeight{ + Name: name, + Weight: &wrapperspb.UInt32Value{Value: uint32(weight)}, + }) + } + return &v3routepb.RouteConfiguration{ + Name: opts.RouteConfigName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{opts.ListenerName}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}}, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{WeightedClusters: &v3routepb.WeightedCluster{Clusters: clusters}}, + }}, + }}, + }}, + } + case RouteConfigClusterSpecifierTypeClusterSpecifierPlugin: + return &v3routepb.RouteConfiguration{ + Name: opts.RouteConfigName, + ClusterSpecifierPlugins: []*v3routepb.ClusterSpecifierPlugin{{ + Extension: &v3corepb.TypedExtensionConfig{ + Name: opts.ClusterSpecifierPluginName, + TypedConfig: opts.ClusterSpecifierPluginConfig, + }}, + }, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{opts.ListenerName}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}}, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_ClusterSpecifierPlugin{ClusterSpecifierPlugin: opts.ClusterSpecifierPluginName}, + }}, + }}, + }}, + } + default: + panic(fmt.Sprintf("unsupported cluster specifier plugin type: %v", opts.ClusterSpecifierType)) + } +} + +// DefaultCluster returns a basic xds Cluster resource. +func DefaultCluster(clusterName, edsServiceName string, secLevel SecurityLevel) *v3clusterpb.Cluster { + return ClusterResourceWithOptions(ClusterOptions{ + ClusterName: clusterName, + ServiceName: edsServiceName, + Policy: LoadBalancingPolicyRoundRobin, + SecurityLevel: secLevel, + }) +} + +// LoadBalancingPolicy determines the policy used for balancing load across +// endpoints in the Cluster. +type LoadBalancingPolicy int + +const ( + // LoadBalancingPolicyRoundRobin results in the use of the weighted_target + // LB policy to balance load across localities and endpoints in the cluster. + LoadBalancingPolicyRoundRobin LoadBalancingPolicy = iota + // LoadBalancingPolicyRingHash results in the use of the ring_hash LB policy + // as the leaf policy. + LoadBalancingPolicyRingHash +) + +// ClusterOptions contains options to configure a Cluster resource. +type ClusterOptions struct { + // ClusterName is the name of the Cluster resource. + ClusterName string + // ServiceName is the EDS service name of the Cluster. + ServiceName string + // Policy is the LB policy to be used. + Policy LoadBalancingPolicy + // SecurityLevel determines the security configuration for the Cluster. + SecurityLevel SecurityLevel + // EnableLRS adds a load reporting configuration with a config source + // pointing to self. + EnableLRS bool +} + +// ClusterResourceWithOptions returns an xDS Cluster resource configured with +// the provided options. +func ClusterResourceWithOptions(opts ClusterOptions) *v3clusterpb.Cluster { + var tlsContext *v3tlspb.UpstreamTlsContext + switch opts.SecurityLevel { + case SecurityLevelNone: + case SecurityLevelTLS: + tlsContext = &v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{ + ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: ClientSideCertProviderInstance, + }, + }, + }, + } + case SecurityLevelMTLS: + tlsContext = &v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{ + ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: ClientSideCertProviderInstance, + }, + }, + TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: ClientSideCertProviderInstance, + }, + }, + } + } + + var lbPolicy v3clusterpb.Cluster_LbPolicy + switch opts.Policy { + case LoadBalancingPolicyRoundRobin: + lbPolicy = v3clusterpb.Cluster_ROUND_ROBIN + case LoadBalancingPolicyRingHash: + lbPolicy = v3clusterpb.Cluster_RING_HASH + } + cluster := &v3clusterpb.Cluster{ + Name: opts.ClusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: opts.ServiceName, + }, + LbPolicy: lbPolicy, + } + if tlsContext != nil { + cluster.TransportSocket = &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(tlsContext), + }, + } + } + if opts.EnableLRS { + cluster.LrsServer = &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Self{ + Self: &v3corepb.SelfConfigSource{}, + }, + } + } + return cluster +} + +// LocalityOptions contains options to configure a Locality. +type LocalityOptions struct { + // Name is the unique locality name. + Name string + // Weight is the weight of the locality, used for load balancing. + Weight uint32 + // Backends is a set of backends belonging to this locality. + Backends []BackendOptions +} + +// BackendOptions contains options to configure individual backends in a +// locality. +type BackendOptions struct { + // Port number on which the backend is accepting connections. All backends + // are expected to run on localhost, hence host name is not stored here. + Port uint32 + // Health status of the backend. Default is UNKNOWN which is treated the + // same as HEALTHY. + HealthStatus v3corepb.HealthStatus +} + +// EndpointOptions contains options to configure an Endpoint (or +// ClusterLoadAssignment) resource. +type EndpointOptions struct { + // ClusterName is the name of the Cluster resource (or EDS service name) + // containing the endpoints specified below. + ClusterName string + // Host is the hostname of the endpoints. In our e2e tests, hostname must + // always be "localhost". + Host string + // Localities is a set of localities belonging to this resource. + Localities []LocalityOptions + // DropPercents is a map from drop category to a drop percentage. If unset, + // no drops are configured. + DropPercents map[string]int +} + +// DefaultEndpoint returns a basic xds Endpoint resource. +func DefaultEndpoint(clusterName string, host string, ports []uint32) *v3endpointpb.ClusterLoadAssignment { + var bOpts []BackendOptions + for _, p := range ports { + bOpts = append(bOpts, BackendOptions{Port: p}) + } + return EndpointResourceWithOptions(EndpointOptions{ + ClusterName: clusterName, + Host: host, + Localities: []LocalityOptions{ + { + Backends: bOpts, + Weight: 1, + }, + }, + }) +} + +// EndpointResourceWithOptions returns an xds Endpoint resource configured with +// the provided options. +func EndpointResourceWithOptions(opts EndpointOptions) *v3endpointpb.ClusterLoadAssignment { + var endpoints []*v3endpointpb.LocalityLbEndpoints + for i, locality := range opts.Localities { + var lbEndpoints []*v3endpointpb.LbEndpoint + for _, b := range locality.Backends { + lbEndpoints = append(lbEndpoints, &v3endpointpb.LbEndpoint{ + HostIdentifier: &v3endpointpb.LbEndpoint_Endpoint{Endpoint: &v3endpointpb.Endpoint{ + Address: &v3corepb.Address{Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Protocol: v3corepb.SocketAddress_TCP, + Address: opts.Host, + PortSpecifier: &v3corepb.SocketAddress_PortValue{PortValue: b.Port}, + }, + }}, + }}, + HealthStatus: b.HealthStatus, + LoadBalancingWeight: &wrapperspb.UInt32Value{Value: 1}, + }) + } + + endpoints = append(endpoints, &v3endpointpb.LocalityLbEndpoints{ + Locality: &v3corepb.Locality{ + Region: fmt.Sprintf("region-%d", i+1), + Zone: fmt.Sprintf("zone-%d", i+1), + SubZone: fmt.Sprintf("subzone-%d", i+1), + }, + LbEndpoints: lbEndpoints, + LoadBalancingWeight: &wrapperspb.UInt32Value{Value: locality.Weight}, + Priority: 0, + }) + } + + cla := &v3endpointpb.ClusterLoadAssignment{ + ClusterName: opts.ClusterName, + Endpoints: endpoints, + } + + var drops []*v3endpointpb.ClusterLoadAssignment_Policy_DropOverload + for category, val := range opts.DropPercents { + drops = append(drops, &v3endpointpb.ClusterLoadAssignment_Policy_DropOverload{ + Category: category, + DropPercentage: &v3typepb.FractionalPercent{ + Numerator: uint32(val), + Denominator: v3typepb.FractionalPercent_HUNDRED, + }, + }) + } + if len(drops) != 0 { + cla.Policy = &v3endpointpb.ClusterLoadAssignment_Policy{ + DropOverloads: drops, + } + } + return cla +} diff --git a/internal/testutils/xds/e2e/logging.go b/internal/testutils/xds/e2e/logging.go new file mode 100644 index 000000000000..2a0925a13060 --- /dev/null +++ b/internal/testutils/xds/e2e/logging.go @@ -0,0 +1,48 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package e2e + +import ( + "fmt" + + "google.golang.org/grpc/grpclog" +) + +var logger = grpclog.Component("xds-e2e") + +// serverLogger implements the Logger interface defined at +// envoyproxy/go-control-plane/pkg/log. This is passed to the Snapshot cache. +type serverLogger struct{} + +func (l serverLogger) Debugf(format string, args ...any) { + msg := fmt.Sprintf(format, args...) + logger.InfoDepth(1, msg) +} +func (l serverLogger) Infof(format string, args ...any) { + msg := fmt.Sprintf(format, args...) + logger.InfoDepth(1, msg) +} +func (l serverLogger) Warnf(format string, args ...any) { + msg := fmt.Sprintf(format, args...) + logger.WarningDepth(1, msg) +} +func (l serverLogger) Errorf(format string, args ...any) { + msg := fmt.Sprintf(format, args...) + logger.ErrorDepth(1, msg) +} diff --git a/internal/testutils/xds/e2e/server.go b/internal/testutils/xds/e2e/server.go new file mode 100644 index 000000000000..e3bf0de5572c --- /dev/null +++ b/internal/testutils/xds/e2e/server.go @@ -0,0 +1,247 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package e2e provides utilities for end2end testing of xDS functionality. +package e2e + +import ( + "context" + "fmt" + "net" + "reflect" + "strconv" + + "github.com/envoyproxy/go-control-plane/pkg/cache/types" + "google.golang.org/grpc" + "google.golang.org/grpc/internal/testutils/xds/fakeserver" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3discoverygrpc "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + v3lrsgrpc "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v3" + v3cache "github.com/envoyproxy/go-control-plane/pkg/cache/v3" + v3resource "github.com/envoyproxy/go-control-plane/pkg/resource/v3" + v3server "github.com/envoyproxy/go-control-plane/pkg/server/v3" +) + +// ManagementServer is a thin wrapper around the xDS control plane +// implementation provided by envoyproxy/go-control-plane. +type ManagementServer struct { + // Address is the host:port on which the management server is listening for + // new connections. + Address string + + // LRSServer points to the fake LRS server implementation. Set only if the + // SupportLoadReportingService option was set to true when creating this + // management server. + LRSServer *fakeserver.Server + + cancel context.CancelFunc // To stop the v3 ADS service. + xs v3server.Server // v3 implementation of ADS. + gs *grpc.Server // gRPC server which exports the ADS service. + cache v3cache.SnapshotCache // Resource snapshot. + version int // Version of resource snapshot. +} + +// ManagementServerOptions contains options to be passed to the management +// server during creation. +type ManagementServerOptions struct { + // Listener to accept connections on. If nil, a TPC listener on a local port + // will be created and used. + Listener net.Listener + + // SupportLoadReportingService, if set, results in the load reporting + // service being registered on the same port as that of ADS. + SupportLoadReportingService bool + + // AllowResourceSubSet allows the management server to respond to requests + // before all configured resources are explicitly named in the request. The + // default behavior that we want is for the management server to wait for + // all configured resources to be requested before responding to any of + // them, since this is how we have run our tests historically, and should be + // set to true only for tests which explicitly require the other behavior. + AllowResourceSubset bool + + // ServerFeaturesIgnoreResourceDeletion, if set, results in a bootstrap config + // where the server features list contains `ignore_resource_deletion`. This + // results in gRPC ignoring resource deletions from the management server, as + // per A53. + ServerFeaturesIgnoreResourceDeletion bool + + // The callbacks defined below correspond to the state of the world (sotw) + // version of the xDS API on the management server. + + // OnStreamOpen is called when an xDS stream is opened. The callback is + // invoked with the assigned stream ID and the type URL from the incoming + // request (or "" for ADS). + // + // Returning an error from this callback will end processing and close the + // stream. OnStreamClosed will still be called. + OnStreamOpen func(context.Context, int64, string) error + + // OnStreamClosed is called immediately prior to closing an xDS stream. The + // callback is invoked with the stream ID of the stream being closed. + OnStreamClosed func(int64, *v3corepb.Node) + + // OnStreamRequest is called when a request is received on the stream. The + // callback is invoked with the stream ID of the stream on which the request + // was received and the received request. + // + // Returning an error from this callback will end processing and close the + // stream. OnStreamClosed will still be called. + OnStreamRequest func(int64, *v3discoverypb.DiscoveryRequest) error + + // OnStreamResponse is called immediately prior to sending a response on the + // stream. The callback is invoked with the stream ID of the stream on which + // the response is being sent along with the incoming request and the outgoing + // response. + OnStreamResponse func(context.Context, int64, *v3discoverypb.DiscoveryRequest, *v3discoverypb.DiscoveryResponse) +} + +// StartManagementServer initializes a management server which implements the +// AggregatedDiscoveryService endpoint. The management server is initialized +// with no resources. Tests should call the Update() method to change the +// resource snapshot held by the management server, as required by the test +// logic. When the test is done, it should call the Stop() method to cleanup +// resources allocated by the management server. +func StartManagementServer(opts ManagementServerOptions) (*ManagementServer, error) { + // Create a snapshot cache. The first parameter to NewSnapshotCache() + // controls whether the server should wait for all resources to be + // explicitly named in the request before responding to any of them. + wait := !opts.AllowResourceSubset + cache := v3cache.NewSnapshotCache(wait, v3cache.IDHash{}, serverLogger{}) + logger.Infof("Created new snapshot cache...") + + lis := opts.Listener + if lis == nil { + var err error + lis, err = net.Listen("tcp", "localhost:0") + if err != nil { + return nil, fmt.Errorf("listening on local host and port: %v", err) + } + } + + // Cancelling the context passed to the server is the only way of stopping it + // at the end of the test. + ctx, cancel := context.WithCancel(context.Background()) + callbacks := v3server.CallbackFuncs{ + StreamOpenFunc: opts.OnStreamOpen, + StreamClosedFunc: opts.OnStreamClosed, + StreamRequestFunc: opts.OnStreamRequest, + StreamResponseFunc: opts.OnStreamResponse, + } + + // Create an xDS management server and register the ADS implementation + // provided by it on a gRPC server. + xs := v3server.NewServer(ctx, cache, callbacks) + gs := grpc.NewServer() + v3discoverygrpc.RegisterAggregatedDiscoveryServiceServer(gs, xs) + logger.Infof("Registered Aggregated Discovery Service (ADS)...") + + mgmtServer := &ManagementServer{ + Address: lis.Addr().String(), + cancel: cancel, + version: 0, + gs: gs, + xs: xs, + cache: cache, + } + if opts.SupportLoadReportingService { + lrs := fakeserver.NewServer(lis.Addr().String()) + v3lrsgrpc.RegisterLoadReportingServiceServer(gs, lrs) + mgmtServer.LRSServer = lrs + logger.Infof("Registered Load Reporting Service (LRS)...") + } + + // Start serving. + go gs.Serve(lis) + logger.Infof("xDS management server serving at: %v...", lis.Addr().String()) + + return mgmtServer, nil +} + +// UpdateOptions wraps parameters to be passed to the Update() method. +type UpdateOptions struct { + // NodeID is the id of the client to which this update is to be pushed. + NodeID string + // Endpoints, Clusters, Routes, and Listeners are the updated list of xds + // resources for the server. All must be provided with each Update. + Endpoints []*v3endpointpb.ClusterLoadAssignment + Clusters []*v3clusterpb.Cluster + Routes []*v3routepb.RouteConfiguration + Listeners []*v3listenerpb.Listener + // SkipValidation indicates whether we want to skip validation (by not + // calling snapshot.Consistent()). It can be useful for negative tests, + // where we send updates that the client will NACK. + SkipValidation bool +} + +// Update changes the resource snapshot held by the management server, which +// updates connected clients as required. +func (s *ManagementServer) Update(ctx context.Context, opts UpdateOptions) error { + s.version++ + + // Create a snapshot with the passed in resources. + resources := map[v3resource.Type][]types.Resource{ + v3resource.ListenerType: resourceSlice(opts.Listeners), + v3resource.RouteType: resourceSlice(opts.Routes), + v3resource.ClusterType: resourceSlice(opts.Clusters), + v3resource.EndpointType: resourceSlice(opts.Endpoints), + } + snapshot, err := v3cache.NewSnapshot(strconv.Itoa(s.version), resources) + if err != nil { + return fmt.Errorf("failed to create new snapshot cache: %v", err) + + } + if !opts.SkipValidation { + if err := snapshot.Consistent(); err != nil { + return fmt.Errorf("failed to create new resource snapshot: %v", err) + } + } + logger.Infof("Created new resource snapshot...") + + // Update the cache with the new resource snapshot. + if err := s.cache.SetSnapshot(ctx, opts.NodeID, snapshot); err != nil { + return fmt.Errorf("failed to update resource snapshot in management server: %v", err) + } + logger.Infof("Updated snapshot cache with resource snapshot...") + return nil +} + +// Stop stops the management server. +func (s *ManagementServer) Stop() { + if s.cancel != nil { + s.cancel() + } + s.gs.Stop() +} + +// resourceSlice accepts a slice of any type of proto messages and returns a +// slice of types.Resource. Will panic if there is an input type mismatch. +func resourceSlice(i any) []types.Resource { + v := reflect.ValueOf(i) + rs := make([]types.Resource, v.Len()) + for i := 0; i < v.Len(); i++ { + rs[i] = v.Index(i).Interface().(types.Resource) + } + return rs +} diff --git a/internal/testutils/xds/e2e/setup_certs.go b/internal/testutils/xds/e2e/setup_certs.go new file mode 100644 index 000000000000..799e18564879 --- /dev/null +++ b/internal/testutils/xds/e2e/setup_certs.go @@ -0,0 +1,97 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package e2e + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "os" + "path" + "testing" + + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/testdata" +) + +const ( + // Names of files inside tempdir, for certprovider plugin to watch. + certFile = "cert.pem" + keyFile = "key.pem" + rootFile = "ca.pem" +) + +func createTmpFile(src, dst string) error { + data, err := os.ReadFile(src) + if err != nil { + return fmt.Errorf("os.ReadFile(%q) failed: %v", src, err) + } + if err := os.WriteFile(dst, data, os.ModePerm); err != nil { + return fmt.Errorf("os.WriteFile(%q) failed: %v", dst, err) + } + return nil +} + +// createTempDirWithFiles creates a temporary directory under the system default +// tempDir with the given dirSuffix. It also reads from certSrc, keySrc and +// rootSrc files are creates appropriate files under the newly create tempDir. +// Returns the name of the created tempDir. +func createTmpDirWithFiles(dirSuffix, certSrc, keySrc, rootSrc string) (string, error) { + // Create a temp directory. Passing an empty string for the first argument + // uses the system temp directory. + dir, err := os.MkdirTemp("", dirSuffix) + if err != nil { + return "", fmt.Errorf("os.MkdirTemp() failed: %v", err) + } + + if err := createTmpFile(testdata.Path(certSrc), path.Join(dir, certFile)); err != nil { + return "", err + } + if err := createTmpFile(testdata.Path(keySrc), path.Join(dir, keyFile)); err != nil { + return "", err + } + if err := createTmpFile(testdata.Path(rootSrc), path.Join(dir, rootFile)); err != nil { + return "", err + } + return dir, nil +} + +// CreateClientTLSCredentials creates client-side TLS transport credentials +// using certificate and key files from testdata/x509 directory. +func CreateClientTLSCredentials(t *testing.T) credentials.TransportCredentials { + t.Helper() + + cert, err := tls.LoadX509KeyPair(testdata.Path("x509/client1_cert.pem"), testdata.Path("x509/client1_key.pem")) + if err != nil { + t.Fatalf("tls.LoadX509KeyPair(x509/client1_cert.pem, x509/client1_key.pem) failed: %v", err) + } + b, err := os.ReadFile(testdata.Path("x509/server_ca_cert.pem")) + if err != nil { + t.Fatalf("os.ReadFile(x509/server_ca_cert.pem) failed: %v", err) + } + roots := x509.NewCertPool() + if !roots.AppendCertsFromPEM(b) { + t.Fatal("failed to append certificates") + } + return credentials.NewTLS(&tls.Config{ + Certificates: []tls.Certificate{cert}, + RootCAs: roots, + ServerName: "x.test.example.com", + }) +} diff --git a/internal/testutils/xds/e2e/setup_management_server.go b/internal/testutils/xds/e2e/setup_management_server.go new file mode 100644 index 000000000000..0218d4f2ce2a --- /dev/null +++ b/internal/testutils/xds/e2e/setup_management_server.go @@ -0,0 +1,112 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package e2e + +import ( + "encoding/json" + "fmt" + "path" + "testing" + + "github.com/google/uuid" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/testutils/xds/bootstrap" + "google.golang.org/grpc/resolver" +) + +// SetupManagementServer performs the following: +// - spin up an xDS management server on a local port +// - set up certificates for consumption by the file_watcher plugin +// - creates a bootstrap file in a temporary location +// - creates an xDS resolver using the above bootstrap contents +// +// Returns the following: +// - management server +// - nodeID to be used by the client when connecting to the management server +// - bootstrap contents to be used by the client +// - xDS resolver builder to be used by the client +// - a cleanup function to be invoked at the end of the test +func SetupManagementServer(t *testing.T, opts ManagementServerOptions) (*ManagementServer, string, []byte, resolver.Builder, func()) { + t.Helper() + + // Spin up an xDS management server on a local port. + server, err := StartManagementServer(opts) + if err != nil { + t.Fatalf("Failed to spin up the xDS management server: %v", err) + } + defer func() { + if err != nil { + server.Stop() + } + }() + + nodeID := uuid.New().String() + bootstrapContents, err := DefaultBootstrapContents(nodeID, server.Address) + if err != nil { + server.Stop() + t.Fatal(err) + } + var rb resolver.Builder + if newResolver := internal.NewXDSResolverWithConfigForTesting; newResolver != nil { + rb, err = newResolver.(func([]byte) (resolver.Builder, error))(bootstrapContents) + if err != nil { + server.Stop() + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + } + + return server, nodeID, bootstrapContents, rb, func() { server.Stop() } +} + +// DefaultBootstrapContents creates a default bootstrap configuration with the +// given node ID and server URI. It also creates certificate provider +// configuration and sets the listener resource name template to be used on the +// server side. +func DefaultBootstrapContents(nodeID, serverURI string) ([]byte, error) { + // Create a directory to hold certs and key files used on the server side. + serverDir, err := createTmpDirWithFiles("testServerSideXDS*", "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem") + if err != nil { + return nil, fmt.Errorf("failed to create bootstrap configuration: %v", err) + } + + // Create a directory to hold certs and key files used on the client side. + clientDir, err := createTmpDirWithFiles("testClientSideXDS*", "x509/client1_cert.pem", "x509/client1_key.pem", "x509/server_ca_cert.pem") + if err != nil { + return nil, fmt.Errorf("failed to create bootstrap configuration: %v", err) + } + + // Create certificate providers section of the bootstrap config with entries + // for both the client and server sides. + cpc := map[string]json.RawMessage{ + ServerSideCertProviderInstance: DefaultFileWatcherConfig(path.Join(serverDir, certFile), path.Join(serverDir, keyFile), path.Join(serverDir, rootFile)), + ClientSideCertProviderInstance: DefaultFileWatcherConfig(path.Join(clientDir, certFile), path.Join(clientDir, keyFile), path.Join(clientDir, rootFile)), + } + + // Create the bootstrap configuration. + bs, err := bootstrap.Contents(bootstrap.Options{ + NodeID: nodeID, + ServerURI: serverURI, + CertificateProviders: cpc, + ServerListenerResourceNameTemplate: ServerListenerResourceNameTemplate, + }) + if err != nil { + return nil, fmt.Errorf("failed to create bootstrap configuration: %v", err) + } + return bs, nil +} diff --git a/xds/internal/testutils/fakeserver/server.go b/internal/testutils/xds/fakeserver/server.go similarity index 52% rename from xds/internal/testutils/fakeserver/server.go rename to internal/testutils/xds/fakeserver/server.go index e1663c60444c..e2f2fb39e0dd 100644 --- a/xds/internal/testutils/fakeserver/server.go +++ b/internal/testutils/xds/fakeserver/server.go @@ -16,11 +16,14 @@ * */ -// Package fakeserver provides a fake implementation of an xDS server. +// Package fakeserver provides a fake implementation of the management server. +// +// This package is recommended only for scenarios which cannot be tested using +// the xDS management server (which uses envoy-go-control-plane) provided by the +// `internal/testutils/xds/e2e` package. package fakeserver import ( - "context" "fmt" "io" "net" @@ -29,13 +32,13 @@ import ( "github.com/golang/protobuf/proto" "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal/testutils" "google.golang.org/grpc/status" - "google.golang.org/grpc/xds/internal/testutils" - discoverypb "github.com/envoyproxy/go-control-plane/envoy/api/v2" - adsgrpc "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v2" - lrsgrpc "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v2" - lrspb "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v2" + v3discoverygrpc "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + v3lrsgrpc "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v3" + v3lrspb "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v3" ) const ( @@ -62,6 +65,10 @@ type Response struct { // Server is a fake implementation of xDS and LRS protocols. It listens on the // same port for both services and exposes a bunch of channels to send/receive // messages. +// +// This server is recommended only for scenarios which cannot be tested using +// the xDS management server (which uses envoy-go-control-plane) provided by the +// `internal/testutils/xds/e2e` package. type Server struct { // XDSRequestChan is a channel on which received xDS requests are made // available to the users of this Server. @@ -75,6 +82,12 @@ type Server struct { // LRSResponseChan is a channel on which the Server accepts the LRS // response to be sent to the client. LRSResponseChan chan *Response + // LRSStreamOpenChan is a channel on which the Server sends notifications + // when a new LRS stream is created. + LRSStreamOpenChan *testutils.Channel + // LRSStreamCloseChan is a channel on which the Server sends notifications + // when an existing LRS stream is closed. + LRSStreamCloseChan *testutils.Channel // NewConnChan is a channel on which the fake server notifies receipt of new // connection attempts. Tests can gate on this event before proceeding to // other actions which depend on a connection to the fake server being up. @@ -83,8 +96,8 @@ type Server struct { Address string // The underlying fake implementation of xDS and LRS. - xdsS *xdsServer - lrsS *lrsServer + *xdsServerV3 + *lrsServerV3 } type wrappedListener struct { @@ -101,56 +114,59 @@ func (wl *wrappedListener) Accept() (net.Conn, error) { return c, err } -// StartServer makes a new Server and gets it to start listening on a local -// port for gRPC requests. The returned cancel function should be invoked by -// the caller upon completion of the test. -func StartServer() (*Server, func(), error) { - lis, err := net.Listen("tcp", "localhost:0") - if err != nil { - return nil, func() {}, fmt.Errorf("net.Listen() failed: %v", err) +// StartServer makes a new Server and gets it to start listening on the given +// net.Listener. If the given net.Listener is nil, a new one is created on a +// local port for gRPC requests. The returned cancel function should be invoked +// by the caller upon completion of the test. +func StartServer(lis net.Listener) (*Server, func(), error) { + if lis == nil { + var err error + lis, err = net.Listen("tcp", "localhost:0") + if err != nil { + return nil, func() {}, fmt.Errorf("net.Listen() failed: %v", err) + } } - s := &Server{ - XDSRequestChan: testutils.NewChannelWithSize(defaultChannelBufferSize), - LRSRequestChan: testutils.NewChannelWithSize(defaultChannelBufferSize), - NewConnChan: testutils.NewChannelWithSize(defaultChannelBufferSize), - XDSResponseChan: make(chan *Response, defaultChannelBufferSize), - LRSResponseChan: make(chan *Response, 1), // The server only ever sends one response. - Address: lis.Addr().String(), - } - s.xdsS = &xdsServer{reqChan: s.XDSRequestChan, respChan: s.XDSResponseChan} - s.lrsS = &lrsServer{reqChan: s.LRSRequestChan, respChan: s.LRSResponseChan} + s := NewServer(lis.Addr().String()) wp := &wrappedListener{ Listener: lis, server: s, } server := grpc.NewServer() - lrsgrpc.RegisterLoadReportingServiceServer(server, s.lrsS) - adsgrpc.RegisterAggregatedDiscoveryServiceServer(server, s.xdsS) + v3lrsgrpc.RegisterLoadReportingServiceServer(server, s) + v3discoverygrpc.RegisterAggregatedDiscoveryServiceServer(server, s) go server.Serve(wp) return s, func() { server.Stop() }, nil } -// XDSClientConn returns a grpc.ClientConn connected to the fakeServer. -func (xdsS *Server) XDSClientConn() (*grpc.ClientConn, func(), error) { - ctx, cancel := context.WithTimeout(context.Background(), defaultDialTimeout) - defer cancel() - - cc, err := grpc.DialContext(ctx, xdsS.Address, grpc.WithInsecure(), grpc.WithBlock()) - if err != nil { - return nil, nil, fmt.Errorf("grpc.DialContext(%s) failed: %v", xdsS.Address, err) +// NewServer returns a new instance of Server, set to accept requests on addr. +// It is the responsibility of the caller to register the exported ADS and LRS +// services on an appropriate gRPC server. Most usages should prefer +// StartServer() instead of this. +func NewServer(addr string) *Server { + s := &Server{ + XDSRequestChan: testutils.NewChannelWithSize(defaultChannelBufferSize), + LRSRequestChan: testutils.NewChannelWithSize(defaultChannelBufferSize), + NewConnChan: testutils.NewChannelWithSize(defaultChannelBufferSize), + XDSResponseChan: make(chan *Response, defaultChannelBufferSize), + LRSResponseChan: make(chan *Response, 1), // The server only ever sends one response. + LRSStreamOpenChan: testutils.NewChannelWithSize(defaultChannelBufferSize), + LRSStreamCloseChan: testutils.NewChannelWithSize(defaultChannelBufferSize), + Address: addr, } - return cc, func() { cc.Close() }, nil + s.xdsServerV3 = &xdsServerV3{reqChan: s.XDSRequestChan, respChan: s.XDSResponseChan} + s.lrsServerV3 = &lrsServerV3{reqChan: s.LRSRequestChan, respChan: s.LRSResponseChan, streamOpenChan: s.LRSStreamOpenChan, streamCloseChan: s.LRSStreamCloseChan} + return s } -type xdsServer struct { +type xdsServerV3 struct { reqChan *testutils.Channel respChan chan *Response } -func (xdsS *xdsServer) StreamAggregatedResources(s adsgrpc.AggregatedDiscoveryService_StreamAggregatedResourcesServer) error { +func (xdsS *xdsServerV3) StreamAggregatedResources(s v3discoverygrpc.AggregatedDiscoveryService_StreamAggregatedResourcesServer) error { errCh := make(chan error, 2) go func() { for { @@ -175,7 +191,7 @@ func (xdsS *xdsServer) StreamAggregatedResources(s adsgrpc.AggregatedDiscoverySe retErr = r.Err return } - if err := s.Send(r.Resp.(*discoverypb.DiscoveryResponse)); err != nil { + if err := s.Send(r.Resp.(*v3discoverypb.DiscoveryResponse)); err != nil { retErr = err return } @@ -192,28 +208,33 @@ func (xdsS *xdsServer) StreamAggregatedResources(s adsgrpc.AggregatedDiscoverySe return nil } -func (xdsS *xdsServer) DeltaAggregatedResources(adsgrpc.AggregatedDiscoveryService_DeltaAggregatedResourcesServer) error { +func (xdsS *xdsServerV3) DeltaAggregatedResources(v3discoverygrpc.AggregatedDiscoveryService_DeltaAggregatedResourcesServer) error { return status.Error(codes.Unimplemented, "") } -type lrsServer struct { - reqChan *testutils.Channel - respChan chan *Response +type lrsServerV3 struct { + reqChan *testutils.Channel + respChan chan *Response + streamOpenChan *testutils.Channel + streamCloseChan *testutils.Channel } -func (lrsS *lrsServer) StreamLoadStats(s lrsgrpc.LoadReportingService_StreamLoadStatsServer) error { +func (lrsS *lrsServerV3) StreamLoadStats(s v3lrsgrpc.LoadReportingService_StreamLoadStatsServer) error { + lrsS.streamOpenChan.Send(nil) + defer lrsS.streamCloseChan.Send(nil) + req, err := s.Recv() + lrsS.reqChan.Send(&Request{req, err}) if err != nil { return err } - lrsS.reqChan.Send(&Request{req, err}) select { case r := <-lrsS.respChan: if r.Err != nil { return r.Err } - if err := s.Send(r.Resp.(*lrspb.LoadStatsResponse)); err != nil { + if err := s.Send(r.Resp.(*v3lrspb.LoadStatsResponse)); err != nil { return err } case <-s.Context().Done(): @@ -222,12 +243,12 @@ func (lrsS *lrsServer) StreamLoadStats(s lrsgrpc.LoadReportingService_StreamLoad for { req, err := s.Recv() + lrsS.reqChan.Send(&Request{req, err}) if err != nil { if err == io.EOF { return nil } return err } - lrsS.reqChan.Send(&Request{req, err}) } } diff --git a/internal/transport/controlbuf.go b/internal/transport/controlbuf.go index 40ef23923fda..b330ccedc8ab 100644 --- a/internal/transport/controlbuf.go +++ b/internal/transport/controlbuf.go @@ -20,13 +20,19 @@ package transport import ( "bytes" + "errors" "fmt" + "net" "runtime" + "strconv" "sync" "sync/atomic" "golang.org/x/net/http2" "golang.org/x/net/http2/hpack" + "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/grpcutil" + "google.golang.org/grpc/status" ) var updateHeaderTblSize = func(e *hpack.Encoder, v uint32) { @@ -34,7 +40,7 @@ var updateHeaderTblSize = func(e *hpack.Encoder, v uint32) { } type itemNode struct { - it interface{} + it any next *itemNode } @@ -43,7 +49,7 @@ type itemList struct { tail *itemNode } -func (il *itemList) enqueue(i interface{}) { +func (il *itemList) enqueue(i any) { n := &itemNode{it: i} if il.tail == nil { il.head, il.tail = n, n @@ -55,11 +61,11 @@ func (il *itemList) enqueue(i interface{}) { // peek returns the first item in the list without removing it from the // list. -func (il *itemList) peek() interface{} { +func (il *itemList) peek() any { return il.head.it } -func (il *itemList) dequeue() interface{} { +func (il *itemList) dequeue() any { if il.head == nil { return nil } @@ -128,6 +134,16 @@ type cleanupStream struct { func (c *cleanupStream) isTransportResponseFrame() bool { return c.rst } // Results in a RST_STREAM +type earlyAbortStream struct { + httpStatus uint32 + streamID uint32 + contentSubtype string + status *status.Status + rst bool +} + +func (*earlyAbortStream) isTransportResponseFrame() bool { return false } + type dataFrame struct { streamID uint32 endStream bool @@ -177,7 +193,7 @@ type goAway struct { code http2.ErrCode debugData []byte headsUp bool - closeConn bool + closeConn error // if set, loopyWriter will exit, resulting in conn closure } func (*goAway) isTransportResponseFrame() bool { return false } @@ -195,6 +211,14 @@ type outFlowControlSizeRequest struct { func (*outFlowControlSizeRequest) isTransportResponseFrame() bool { return false } +// closeConnection is an instruction to tell the loopy writer to flush the +// framer and exit, which will cause the transport's connection to be closed +// (by the client or server). The transport itself will close after the reader +// encounters the EOF caused by the connection closure. +type closeConnection struct{} + +func (closeConnection) isTransportResponseFrame() bool { return false } + type outStreamState int const ( @@ -284,7 +308,7 @@ type controlBuffer struct { // closed and nilled when transportResponseFrames drops below the // threshold. Both fields are protected by mu. transportResponseFrames int - trfChan atomic.Value // *chan struct{} + trfChan atomic.Value // chan struct{} } func newControlBuffer(done <-chan struct{}) *controlBuffer { @@ -298,10 +322,10 @@ func newControlBuffer(done <-chan struct{}) *controlBuffer { // throttle blocks if there are too many incomingSettings/cleanupStreams in the // controlbuf. func (c *controlBuffer) throttle() { - ch, _ := c.trfChan.Load().(*chan struct{}) + ch, _ := c.trfChan.Load().(chan struct{}) if ch != nil { select { - case <-*ch: + case <-ch: case <-c.done: } } @@ -312,7 +336,7 @@ func (c *controlBuffer) put(it cbItem) error { return err } -func (c *controlBuffer) executeAndPut(f func(it interface{}) bool, it cbItem) (bool, error) { +func (c *controlBuffer) executeAndPut(f func(it any) bool, it cbItem) (bool, error) { var wakeUp bool c.mu.Lock() if c.err != nil { @@ -335,8 +359,7 @@ func (c *controlBuffer) executeAndPut(f func(it interface{}) bool, it cbItem) (b if c.transportResponseFrames == maxQueuedTransportResponseFrames { // We are adding the frame that puts us over the threshold; create // a throttling channel. - ch := make(chan struct{}) - c.trfChan.Store(&ch) + c.trfChan.Store(make(chan struct{})) } } c.mu.Unlock() @@ -350,7 +373,7 @@ func (c *controlBuffer) executeAndPut(f func(it interface{}) bool, it cbItem) (b } // Note argument f should never be nil. -func (c *controlBuffer) execute(f func(it interface{}) bool, it interface{}) (bool, error) { +func (c *controlBuffer) execute(f func(it any) bool, it any) (bool, error) { c.mu.Lock() if c.err != nil { c.mu.Unlock() @@ -364,7 +387,7 @@ func (c *controlBuffer) execute(f func(it interface{}) bool, it interface{}) (bo return true, nil } -func (c *controlBuffer) get(block bool) (interface{}, error) { +func (c *controlBuffer) get(block bool) (any, error) { for { c.mu.Lock() if c.err != nil { @@ -377,9 +400,9 @@ func (c *controlBuffer) get(block bool) (interface{}, error) { if c.transportResponseFrames == maxQueuedTransportResponseFrames { // We are removing the frame that put us over the // threshold; close and clear the throttling channel. - ch := c.trfChan.Load().(*chan struct{}) - close(*ch) - c.trfChan.Store((*chan struct{})(nil)) + ch := c.trfChan.Load().(chan struct{}) + close(ch) + c.trfChan.Store((chan struct{})(nil)) } c.transportResponseFrames-- } @@ -395,8 +418,7 @@ func (c *controlBuffer) get(block bool) (interface{}, error) { select { case <-c.ch: case <-c.done: - c.finish() - return nil, ErrConnClosing + return nil, errors.New("transport closed by client") } } } @@ -420,6 +442,14 @@ func (c *controlBuffer) finish() { hdr.onOrphaned(ErrConnClosing) } } + // In case throttle() is currently in flight, it needs to be unblocked. + // Otherwise, the transport may not close, since the transport is closed by + // the reader encountering the connection error. + ch, _ := c.trfChan.Load().(chan struct{}) + if ch != nil { + close(ch) + } + c.trfChan.Store((chan struct{})(nil)) c.mu.Unlock() } @@ -458,12 +488,14 @@ type loopyWriter struct { hEnc *hpack.Encoder // HPACK encoder. bdpEst *bdpEstimator draining bool + conn net.Conn + logger *grpclog.PrefixLogger // Side-specific handlers ssGoAwayHandler func(*goAway) (bool, error) } -func newLoopyWriter(s side, fr *framer, cbuf *controlBuffer, bdpEst *bdpEstimator) *loopyWriter { +func newLoopyWriter(s side, fr *framer, cbuf *controlBuffer, bdpEst *bdpEstimator, conn net.Conn, logger *grpclog.PrefixLogger) *loopyWriter { var buf bytes.Buffer l := &loopyWriter{ side: s, @@ -476,6 +508,8 @@ func newLoopyWriter(s side, fr *framer, cbuf *controlBuffer, bdpEst *bdpEstimato hBuf: &buf, hEnc: hpack.NewEncoder(&buf), bdpEst: bdpEst, + conn: conn, + logger: logger, } return l } @@ -493,23 +527,26 @@ const minBatchSize = 1000 // 2. Stream level flow control quota available. // // In each iteration of run loop, other than processing the incoming control -// frame, loopy calls processData, which processes one node from the activeStreams linked-list. -// This results in writing of HTTP2 frames into an underlying write buffer. -// When there's no more control frames to read from controlBuf, loopy flushes the write buffer. -// As an optimization, to increase the batch size for each flush, loopy yields the processor, once -// if the batch size is too low to give stream goroutines a chance to fill it up. +// frame, loopy calls processData, which processes one node from the +// activeStreams linked-list. This results in writing of HTTP2 frames into an +// underlying write buffer. When there's no more control frames to read from +// controlBuf, loopy flushes the write buffer. As an optimization, to increase +// the batch size for each flush, loopy yields the processor, once if the batch +// size is too low to give stream goroutines a chance to fill it up. +// +// Upon exiting, if the error causing the exit is not an I/O error, run() +// flushes and closes the underlying connection. Otherwise, the connection is +// left open to allow the I/O error to be encountered by the reader instead. func (l *loopyWriter) run() (err error) { defer func() { - if err == ErrConnClosing { - // Don't log ErrConnClosing as error since it happens - // 1. When the connection is closed by some other known issue. - // 2. User closed the connection. - // 3. A graceful close of connection. - if logger.V(logLevel) { - logger.Infof("transport: loopyWriter.run returning. %v", err) - } - err = nil + if l.logger.V(logLevel) { + l.logger.Infof("loopyWriter exiting with error: %v", err) } + if !isIOError(err) { + l.framer.writer.Flush() + l.conn.Close() + } + l.cbuf.finish() }() for { it, err := l.cbuf.get(true) @@ -554,7 +591,6 @@ func (l *loopyWriter) run() (err error) { } l.framer.writer.Flush() break hasdata - } } } @@ -563,11 +599,11 @@ func (l *loopyWriter) outgoingWindowUpdateHandler(w *outgoingWindowUpdate) error return l.framer.fr.WriteWindowUpdate(w.streamID, w.increment) } -func (l *loopyWriter) incomingWindowUpdateHandler(w *incomingWindowUpdate) error { +func (l *loopyWriter) incomingWindowUpdateHandler(w *incomingWindowUpdate) { // Otherwise update the quota. if w.streamID == 0 { l.sendQuota += w.increment - return nil + return } // Find the stream and update it. if str, ok := l.estdStreams[w.streamID]; ok { @@ -575,10 +611,9 @@ func (l *loopyWriter) incomingWindowUpdateHandler(w *incomingWindowUpdate) error if strQuota := int(l.oiws) - str.bytesOutStanding; strQuota > 0 && str.state == waitingOnStreamQuota { str.state = active l.activeStreams.enqueue(str) - return nil + return } } - return nil } func (l *loopyWriter) outgoingSettingsHandler(s *outgoingSettings) error { @@ -586,13 +621,11 @@ func (l *loopyWriter) outgoingSettingsHandler(s *outgoingSettings) error { } func (l *loopyWriter) incomingSettingsHandler(s *incomingSettings) error { - if err := l.applySettings(s.ss); err != nil { - return err - } + l.applySettings(s.ss) return l.framer.fr.WriteSettingsAck() } -func (l *loopyWriter) registerStreamHandler(h *registerStream) error { +func (l *loopyWriter) registerStreamHandler(h *registerStream) { str := &outStream{ id: h.streamID, state: empty, @@ -600,15 +633,14 @@ func (l *loopyWriter) registerStreamHandler(h *registerStream) error { wq: h.wq, } l.estdStreams[h.streamID] = str - return nil } func (l *loopyWriter) headerHandler(h *headerFrame) error { if l.side == serverSide { str, ok := l.estdStreams[h.streamID] if !ok { - if logger.V(logLevel) { - logger.Warningf("transport: loopy doesn't recognize the stream: %d", h.streamID) + if l.logger.V(logLevel) { + l.logger.Infof("Unrecognized streamID %d in loopyWriter", h.streamID) } return nil } @@ -635,19 +667,20 @@ func (l *loopyWriter) headerHandler(h *headerFrame) error { itl: &itemList{}, wq: h.wq, } - str.itl.enqueue(h) - return l.originateStream(str) + return l.originateStream(str, h) } -func (l *loopyWriter) originateStream(str *outStream) error { - hdr := str.itl.dequeue().(*headerFrame) - if err := hdr.initStream(str.id); err != nil { - if err == ErrConnClosing { - return err - } - // Other errors(errStreamDrain) need not close transport. +func (l *loopyWriter) originateStream(str *outStream, hdr *headerFrame) error { + // l.draining is set when handling GoAway. In which case, we want to avoid + // creating new streams. + if l.draining { + // TODO: provide a better error with the reason we are in draining. + hdr.onOrphaned(errStreamDrain) return nil } + if err := hdr.initStream(str.id); err != nil { + return err + } if err := l.writeHeader(str.id, hdr.endStream, hdr.hf, hdr.onWrite); err != nil { return err } @@ -662,8 +695,8 @@ func (l *loopyWriter) writeHeader(streamID uint32, endStream bool, hf []hpack.He l.hBuf.Reset() for _, f := range hf { if err := l.hEnc.WriteField(f); err != nil { - if logger.V(logLevel) { - logger.Warningf("transport: loopyWriter.writeHeader encountered error while encoding headers: %v", err) + if l.logger.V(logLevel) { + l.logger.Warningf("Encountered error while encoding headers: %v", err) } } } @@ -701,10 +734,10 @@ func (l *loopyWriter) writeHeader(streamID uint32, endStream bool, hf []hpack.He return nil } -func (l *loopyWriter) preprocessData(df *dataFrame) error { +func (l *loopyWriter) preprocessData(df *dataFrame) { str, ok := l.estdStreams[df.streamID] if !ok { - return nil + return } // If we got data for a stream it means that // stream was originated and the headers were sent out. @@ -713,7 +746,6 @@ func (l *loopyWriter) preprocessData(df *dataFrame) error { str.state = active l.activeStreams.enqueue(str) } - return nil } func (l *loopyWriter) pingHandler(p *ping) error { @@ -724,9 +756,8 @@ func (l *loopyWriter) pingHandler(p *ping) error { } -func (l *loopyWriter) outFlowControlSizeRequestHandler(o *outFlowControlSizeRequest) error { +func (l *loopyWriter) outFlowControlSizeRequestHandler(o *outFlowControlSizeRequest) { o.resp <- l.sendQuota - return nil } func (l *loopyWriter) cleanupStreamHandler(c *cleanupStream) error { @@ -743,8 +774,35 @@ func (l *loopyWriter) cleanupStreamHandler(c *cleanupStream) error { return err } } - if l.side == clientSide && l.draining && len(l.estdStreams) == 0 { - return ErrConnClosing + if l.draining && len(l.estdStreams) == 0 { + // Flush and close the connection; we are done with it. + return errors.New("finished processing active streams while in draining mode") + } + return nil +} + +func (l *loopyWriter) earlyAbortStreamHandler(eas *earlyAbortStream) error { + if l.side == clientSide { + return errors.New("earlyAbortStream not handled on client") + } + // In case the caller forgets to set the http status, default to 200. + if eas.httpStatus == 0 { + eas.httpStatus = 200 + } + headerFields := []hpack.HeaderField{ + {Name: ":status", Value: strconv.Itoa(int(eas.httpStatus))}, + {Name: "content-type", Value: grpcutil.ContentType(eas.contentSubtype)}, + {Name: "grpc-status", Value: strconv.Itoa(int(eas.status.Code()))}, + {Name: "grpc-message", Value: encodeGrpcMessage(eas.status.Message())}, + } + + if err := l.writeHeader(eas.streamID, true, headerFields, nil); err != nil { + return err + } + if eas.rst { + if err := l.framer.fr.WriteRSTStream(eas.streamID, http2.ErrCodeNo); err != nil { + return err + } } return nil } @@ -753,7 +811,8 @@ func (l *loopyWriter) incomingGoAwayHandler(*incomingGoAway) error { if l.side == clientSide { l.draining = true if len(l.estdStreams) == 0 { - return ErrConnClosing + // Flush and close the connection; we are done with it. + return errors.New("received GOAWAY with no active streams") } } return nil @@ -771,10 +830,10 @@ func (l *loopyWriter) goAwayHandler(g *goAway) error { return nil } -func (l *loopyWriter) handle(i interface{}) error { +func (l *loopyWriter) handle(i any) error { switch i := i.(type) { case *incomingWindowUpdate: - return l.incomingWindowUpdateHandler(i) + l.incomingWindowUpdateHandler(i) case *outgoingWindowUpdate: return l.outgoingWindowUpdateHandler(i) case *incomingSettings: @@ -784,25 +843,32 @@ func (l *loopyWriter) handle(i interface{}) error { case *headerFrame: return l.headerHandler(i) case *registerStream: - return l.registerStreamHandler(i) + l.registerStreamHandler(i) case *cleanupStream: return l.cleanupStreamHandler(i) + case *earlyAbortStream: + return l.earlyAbortStreamHandler(i) case *incomingGoAway: return l.incomingGoAwayHandler(i) case *dataFrame: - return l.preprocessData(i) + l.preprocessData(i) case *ping: return l.pingHandler(i) case *goAway: return l.goAwayHandler(i) case *outFlowControlSizeRequest: - return l.outFlowControlSizeRequestHandler(i) + l.outFlowControlSizeRequestHandler(i) + case closeConnection: + // Just return a non-I/O error and run() will flush and close the + // connection. + return ErrConnClosing default: return fmt.Errorf("transport: unknown control message type %T", i) } + return nil } -func (l *loopyWriter) applySettings(ss []http2.Setting) error { +func (l *loopyWriter) applySettings(ss []http2.Setting) { for _, s := range ss { switch s.ID { case http2.SettingInitialWindowSize: @@ -821,7 +887,6 @@ func (l *loopyWriter) applySettings(ss []http2.Setting) error { updateHeaderTblSize(l.hEnc, s.Val) } } - return nil } // processData removes the first stream from active streams, writes out at most 16KB @@ -838,9 +903,9 @@ func (l *loopyWriter) processData() (bool, error) { dataItem := str.itl.peek().(*dataFrame) // Peek at the first data item this stream. // A data item is represented by a dataFrame, since it later translates into // multiple HTTP2 data frames. - // Every dataFrame has two buffers; h that keeps grpc-message header and d that is acutal data. + // Every dataFrame has two buffers; h that keeps grpc-message header and d that is actual data. // As an optimization to keep wire traffic low, data from d is copied to h to make as big as the - // maximum possilbe HTTP2 frame size. + // maximum possible HTTP2 frame size. if len(dataItem.h) == 0 && len(dataItem.d) == 0 { // Empty data frame // Client sends out empty data frame with endStream = true @@ -855,7 +920,7 @@ func (l *loopyWriter) processData() (bool, error) { return false, err } if err := l.cleanupStreamHandler(trailer.cleanup); err != nil { - return false, nil + return false, err } } else { l.activeStreams.enqueue(str) diff --git a/internal/transport/defaults.go b/internal/transport/defaults.go index 9fa306b2e07a..bc8ee0747496 100644 --- a/internal/transport/defaults.go +++ b/internal/transport/defaults.go @@ -47,3 +47,9 @@ const ( defaultClientMaxHeaderListSize = uint32(16 << 20) defaultServerMaxHeaderListSize = uint32(16 << 20) ) + +// MaxStreamID is the upper bound for the stream ID before the current +// transport gracefully closes and new transport is created for subsequent RPCs. +// This is set to 75% of 2^31-1. Streams are identified with an unsigned 31-bit +// integer. It's exported so that tests can override it. +var MaxStreamID = uint32(math.MaxInt32 * 3 / 4) diff --git a/internal/transport/flowcontrol.go b/internal/transport/flowcontrol.go index f262edd8ecda..97198c515889 100644 --- a/internal/transport/flowcontrol.go +++ b/internal/transport/flowcontrol.go @@ -136,12 +136,10 @@ type inFlow struct { // newLimit updates the inflow window to a new value n. // It assumes that n is always greater than the old limit. -func (f *inFlow) newLimit(n uint32) uint32 { +func (f *inFlow) newLimit(n uint32) { f.mu.Lock() - d := n - f.limit f.limit = n f.mu.Unlock() - return d } func (f *inFlow) maybeAdjust(n uint32) uint32 { diff --git a/internal/transport/handler_server.go b/internal/transport/handler_server.go index 05d3871e628d..98f80e3fa00a 100644 --- a/internal/transport/handler_server.go +++ b/internal/transport/handler_server.go @@ -39,6 +39,7 @@ import ( "golang.org/x/net/http2" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/internal/grpclog" "google.golang.org/grpc/internal/grpcutil" "google.golang.org/grpc/metadata" "google.golang.org/grpc/peer" @@ -46,24 +47,32 @@ import ( "google.golang.org/grpc/status" ) -// NewServerHandlerTransport returns a ServerTransport handling gRPC -// from inside an http.Handler. It requires that the http Server -// supports HTTP/2. -func NewServerHandlerTransport(w http.ResponseWriter, r *http.Request, stats stats.Handler) (ServerTransport, error) { +// NewServerHandlerTransport returns a ServerTransport handling gRPC from +// inside an http.Handler, or writes an HTTP error to w and returns an error. +// It requires that the http Server supports HTTP/2. +func NewServerHandlerTransport(w http.ResponseWriter, r *http.Request, stats []stats.Handler) (ServerTransport, error) { if r.ProtoMajor != 2 { - return nil, errors.New("gRPC requires HTTP/2") + msg := "gRPC requires HTTP/2" + http.Error(w, msg, http.StatusBadRequest) + return nil, errors.New(msg) } if r.Method != "POST" { - return nil, errors.New("invalid gRPC request method") + msg := fmt.Sprintf("invalid gRPC request method %q", r.Method) + http.Error(w, msg, http.StatusBadRequest) + return nil, errors.New(msg) } contentType := r.Header.Get("Content-Type") // TODO: do we assume contentType is lowercase? we did before contentSubtype, validContentType := grpcutil.ContentSubtype(contentType) if !validContentType { - return nil, errors.New("invalid gRPC request content-type") + msg := fmt.Sprintf("invalid gRPC request content-type %q", contentType) + http.Error(w, msg, http.StatusUnsupportedMediaType) + return nil, errors.New(msg) } if _, ok := w.(http.Flusher); !ok { - return nil, errors.New("gRPC requires a ResponseWriter supporting http.Flusher") + msg := "gRPC requires a ResponseWriter supporting http.Flusher" + http.Error(w, msg, http.StatusInternalServerError) + return nil, errors.New(msg) } st := &serverHandlerTransport{ @@ -75,11 +84,14 @@ func NewServerHandlerTransport(w http.ResponseWriter, r *http.Request, stats sta contentSubtype: contentSubtype, stats: stats, } + st.logger = prefixLoggerForServerHandlerTransport(st) if v := r.Header.Get("grpc-timeout"); v != "" { to, err := decodeTimeout(v) if err != nil { - return nil, status.Errorf(codes.Internal, "malformed time-out: %v", err) + msg := fmt.Sprintf("malformed grpc-timeout: %v", err) + http.Error(w, msg, http.StatusBadRequest) + return nil, status.Error(codes.Internal, msg) } st.timeoutSet = true st.timeout = to @@ -97,7 +109,9 @@ func NewServerHandlerTransport(w http.ResponseWriter, r *http.Request, stats sta for _, v := range vv { v, err := decodeMetadataHeader(k, v) if err != nil { - return nil, status.Errorf(codes.Internal, "malformed binary metadata: %v", err) + msg := fmt.Sprintf("malformed binary metadata %q in header %q: %v", v, k, err) + http.Error(w, msg, http.StatusBadRequest) + return nil, status.Error(codes.Internal, msg) } metakv = append(metakv, k, v) } @@ -138,16 +152,19 @@ type serverHandlerTransport struct { // TODO make sure this is consistent across handler_server and http2_server contentSubtype string - stats stats.Handler + stats []stats.Handler + logger *grpclog.PrefixLogger } -func (ht *serverHandlerTransport) Close() error { - ht.closeOnce.Do(ht.closeCloseChanOnce) - return nil +func (ht *serverHandlerTransport) Close(err error) { + ht.closeOnce.Do(func() { + if ht.logger.V(logLevel) { + ht.logger.Infof("Closing: %v", err) + } + close(ht.closedCh) + }) } -func (ht *serverHandlerTransport) closeCloseChanOnce() { close(ht.closedCh) } - func (ht *serverHandlerTransport) RemoteAddr() net.Addr { return strAddr(ht.req.RemoteAddr) } // strAddr is a net.Addr backed by either a TCP "ip:port" string, or @@ -229,15 +246,15 @@ func (ht *serverHandlerTransport) WriteStatus(s *Stream, st *status.Status) erro }) if err == nil { // transport has not been closed - if ht.stats != nil { - // Note: The trailer fields are compressed with hpack after this call returns. - // No WireLength field is set here. - ht.stats.HandleRPC(s.Context(), &stats.OutTrailer{ + // Note: The trailer fields are compressed with hpack after this call returns. + // No WireLength field is set here. + for _, sh := range ht.stats { + sh.HandleRPC(s.Context(), &stats.OutTrailer{ Trailer: s.trailer.Copy(), }) } } - ht.Close() + ht.Close(errors.New("finished writing status")) return err } @@ -315,10 +332,10 @@ func (ht *serverHandlerTransport) WriteHeader(s *Stream, md metadata.MD) error { }) if err == nil { - if ht.stats != nil { + for _, sh := range ht.stats { // Note: The header fields are compressed with hpack after this call returns. // No WireLength field is set here. - ht.stats.HandleRPC(s.Context(), &stats.OutHeader{ + sh.HandleRPC(s.Context(), &stats.OutHeader{ Header: md.Copy(), Compression: s.sendCompress, }) @@ -347,7 +364,7 @@ func (ht *serverHandlerTransport) HandleStreams(startStream func(*Stream), trace case <-ht.req.Context().Done(): } cancel() - ht.Close() + ht.Close(errors.New("request is done processing")) }() req := ht.req @@ -370,14 +387,14 @@ func (ht *serverHandlerTransport) HandleStreams(startStream func(*Stream), trace } ctx = metadata.NewIncomingContext(ctx, ht.headerMD) s.ctx = peer.NewContext(ctx, pr) - if ht.stats != nil { - s.ctx = ht.stats.TagRPC(s.ctx, &stats.RPCTagInfo{FullMethodName: s.method}) + for _, sh := range ht.stats { + s.ctx = sh.TagRPC(s.ctx, &stats.RPCTagInfo{FullMethodName: s.method}) inHeader := &stats.InHeader{ FullMethod: s.method, RemoteAddr: ht.RemoteAddr(), Compression: s.recvCompress, } - ht.stats.HandleRPC(s.ctx, inHeader) + sh.HandleRPC(s.ctx, inHeader) } s.trReader = &transportReader{ reader: &recvBufferReader{ctx: s.ctx, ctxDone: s.ctx.Done(), recv: s.buf, freeBuffer: func(*bytes.Buffer) {}}, @@ -436,17 +453,17 @@ func (ht *serverHandlerTransport) IncrMsgSent() {} func (ht *serverHandlerTransport) IncrMsgRecv() {} -func (ht *serverHandlerTransport) Drain() { +func (ht *serverHandlerTransport) Drain(debugData string) { panic("Drain() is not implemented") } // mapRecvMsgError returns the non-nil err into the appropriate // error value as expected by callers of *grpc.parser.recvMsg. // In particular, in can only be: -// * io.EOF -// * io.ErrUnexpectedEOF -// * of type transport.ConnectionError -// * an error from the status package +// - io.EOF +// - io.ErrUnexpectedEOF +// - of type transport.ConnectionError +// - an error from the status package func mapRecvMsgError(err error) error { if err == io.EOF || err == io.ErrUnexpectedEOF { return err diff --git a/internal/transport/handler_server_test.go b/internal/transport/handler_server_test.go index f9efdfb0716d..36b0864177e7 100644 --- a/internal/transport/handler_server_test.go +++ b/internal/transport/handler_server_test.go @@ -41,11 +41,12 @@ import ( func (s) TestHandlerTransport_NewServerHandlerTransport(t *testing.T) { type testCase struct { - name string - req *http.Request - wantErr string - modrw func(http.ResponseWriter) http.ResponseWriter - check func(*serverHandlerTransport, *testCase) error + name string + req *http.Request + wantErr string + wantErrCode int + modrw func(http.ResponseWriter) http.ResponseWriter + check func(*serverHandlerTransport, *testCase) error } tests := []testCase{ { @@ -54,7 +55,8 @@ func (s) TestHandlerTransport_NewServerHandlerTransport(t *testing.T) { ProtoMajor: 1, ProtoMinor: 1, }, - wantErr: "gRPC requires HTTP/2", + wantErr: "gRPC requires HTTP/2", + wantErrCode: http.StatusBadRequest, }, { name: "bad method", @@ -62,9 +64,9 @@ func (s) TestHandlerTransport_NewServerHandlerTransport(t *testing.T) { ProtoMajor: 2, Method: "GET", Header: http.Header{}, - RequestURI: "/", }, - wantErr: "invalid gRPC request method", + wantErr: `invalid gRPC request method "GET"`, + wantErrCode: http.StatusBadRequest, }, { name: "bad content type", @@ -74,9 +76,9 @@ func (s) TestHandlerTransport_NewServerHandlerTransport(t *testing.T) { Header: http.Header{ "Content-Type": {"application/foo"}, }, - RequestURI: "/service/foo.bar", }, - wantErr: "invalid gRPC request content-type", + wantErr: `invalid gRPC request content-type "application/foo"`, + wantErrCode: http.StatusUnsupportedMediaType, }, { name: "not flusher", @@ -86,7 +88,6 @@ func (s) TestHandlerTransport_NewServerHandlerTransport(t *testing.T) { Header: http.Header{ "Content-Type": {"application/grpc"}, }, - RequestURI: "/service/foo.bar", }, modrw: func(w http.ResponseWriter) http.ResponseWriter { // Return w without its Flush method @@ -96,7 +97,8 @@ func (s) TestHandlerTransport_NewServerHandlerTransport(t *testing.T) { } return struct{ onlyCloseNotifier }{w.(onlyCloseNotifier)} }, - wantErr: "gRPC requires a ResponseWriter supporting http.Flusher", + wantErr: "gRPC requires a ResponseWriter supporting http.Flusher", + wantErrCode: http.StatusInternalServerError, }, { name: "valid", @@ -109,7 +111,6 @@ func (s) TestHandlerTransport_NewServerHandlerTransport(t *testing.T) { URL: &url.URL{ Path: "/service/foo.bar", }, - RequestURI: "/service/foo.bar", }, check: func(t *serverHandlerTransport, tt *testCase) error { if t.req != tt.req { @@ -133,7 +134,6 @@ func (s) TestHandlerTransport_NewServerHandlerTransport(t *testing.T) { URL: &url.URL{ Path: "/service/foo.bar", }, - RequestURI: "/service/foo.bar", }, check: func(t *serverHandlerTransport, tt *testCase) error { if !t.timeoutSet { @@ -157,9 +157,9 @@ func (s) TestHandlerTransport_NewServerHandlerTransport(t *testing.T) { URL: &url.URL{ Path: "/service/foo.bar", }, - RequestURI: "/service/foo.bar", }, - wantErr: `rpc error: code = Internal desc = malformed time-out: transport: timeout unit is not recognized: "tomorrow"`, + wantErr: `rpc error: code = Internal desc = malformed grpc-timeout: transport: timeout unit is not recognized: "tomorrow"`, + wantErrCode: http.StatusBadRequest, }, { name: "with metadata", @@ -175,7 +175,6 @@ func (s) TestHandlerTransport_NewServerHandlerTransport(t *testing.T) { URL: &url.URL{ Path: "/service/foo.bar", }, - RequestURI: "/service/foo.bar", }, check: func(ht *serverHandlerTransport, tt *testCase) error { want := metadata.MD{ @@ -194,7 +193,12 @@ func (s) TestHandlerTransport_NewServerHandlerTransport(t *testing.T) { } for _, tt := range tests { - rw := newTestHandlerResponseWriter() + rrec := httptest.NewRecorder() + rw := http.ResponseWriter(testHandlerResponseWriter{ + ResponseRecorder: rrec, + closeNotify: make(chan bool, 1), + }) + if tt.modrw != nil { rw = tt.modrw(rw) } @@ -203,6 +207,13 @@ func (s) TestHandlerTransport_NewServerHandlerTransport(t *testing.T) { t.Errorf("%s: error = %q; want %q", tt.name, gotErr.Error(), tt.wantErr) continue } + if tt.wantErrCode == 0 { + tt.wantErrCode = http.StatusOK + } + if rrec.Code != tt.wantErrCode { + t.Errorf("%s: code = %d; want %d", tt.name, rrec.Code, tt.wantErrCode) + continue + } if gotErr != nil { continue } @@ -247,8 +258,7 @@ func newHandleStreamTest(t *testing.T) *handleStreamTest { URL: &url.URL{ Path: "/service/foo.bar", }, - RequestURI: "/service/foo.bar", - Body: bodyr, + Body: bodyr, } rw := newTestHandlerResponseWriter().(testHandlerResponseWriter) ht, err := NewServerHandlerTransport(rw, req, nil) @@ -270,31 +280,36 @@ func (s) TestHandlerTransport_HandleStreams(t *testing.T) { t.Errorf("stream method = %q; want %q", s.method, want) } - err := s.SetHeader(metadata.Pairs("custom-header", "Custom header value")) - if err != nil { + if err := s.SetHeader(metadata.Pairs("custom-header", "Custom header value")); err != nil { + t.Error(err) + } + + if err := s.SetTrailer(metadata.Pairs("custom-trailer", "Custom trailer value")); err != nil { t.Error(err) } - err = s.SetTrailer(metadata.Pairs("custom-trailer", "Custom trailer value")) - if err != nil { + + if err := s.SetSendCompress("gzip"); err != nil { t.Error(err) } md := metadata.Pairs("custom-header", "Another custom header value") - err = s.SendHeader(md) - delete(md, "custom-header") - if err != nil { + if err := s.SendHeader(md); err != nil { t.Error(err) } + delete(md, "custom-header") - err = s.SetHeader(metadata.Pairs("too-late", "Header value that should be ignored")) - if err == nil { + if err := s.SetHeader(metadata.Pairs("too-late", "Header value that should be ignored")); err == nil { t.Error("expected SetHeader call after SendHeader to fail") } - err = s.SendHeader(metadata.Pairs("too-late", "This header value should be ignored as well")) - if err == nil { + + if err := s.SendHeader(metadata.Pairs("too-late", "This header value should be ignored as well")); err == nil { t.Error("expected second SendHeader call to fail") } + if err := s.SetSendCompress("snappy"); err == nil { + t.Error("expected second SetSendCompress call to fail") + } + st.bodyw.Close() // no body st.ht.WriteStatus(s, status.New(codes.OK, "")) } @@ -303,10 +318,11 @@ func (s) TestHandlerTransport_HandleStreams(t *testing.T) { func(ctx context.Context, method string) context.Context { return ctx }, ) wantHeader := http.Header{ - "Date": {}, + "Date": nil, "Content-Type": {"application/grpc"}, "Trailer": {"Grpc-Status", "Grpc-Message", "Grpc-Status-Details-Bin"}, "Custom-Header": {"Custom header value", "Another custom header value"}, + "Grpc-Encoding": {"gzip"}, } wantTrailer := http.Header{ "Grpc-Status": {"0"}, @@ -336,7 +352,7 @@ func handleStreamCloseBodyTest(t *testing.T, statusCode codes.Code, msg string) func(ctx context.Context, method string) context.Context { return ctx }, ) wantHeader := http.Header{ - "Date": {}, + "Date": nil, "Content-Type": {"application/grpc"}, "Trailer": {"Grpc-Status", "Grpc-Message", "Grpc-Status-Details-Bin"}, } @@ -359,8 +375,7 @@ func (s) TestHandlerTransport_HandleStreams_Timeout(t *testing.T) { URL: &url.URL{ Path: "/service/foo.bar", }, - RequestURI: "/service/foo.bar", - Body: bodyr, + Body: bodyr, } rw := newTestHandlerResponseWriter().(testHandlerResponseWriter) ht, err := NewServerHandlerTransport(rw, req, nil) @@ -387,7 +402,7 @@ func (s) TestHandlerTransport_HandleStreams_Timeout(t *testing.T) { func(ctx context.Context, method string) context.Context { return ctx }, ) wantHeader := http.Header{ - "Date": {}, + "Date": nil, "Content-Type": {"application/grpc"}, "Trailer": {"Grpc-Status", "Grpc-Message", "Grpc-Status-Details-Bin"}, } @@ -474,7 +489,7 @@ func (s) TestHandlerTransport_HandleStreams_ErrDetails(t *testing.T) { func(ctx context.Context, method string) context.Context { return ctx }, ) wantHeader := http.Header{ - "Date": {}, + "Date": nil, "Content-Type": {"application/grpc"}, "Trailer": {"Grpc-Status", "Grpc-Message", "Grpc-Status-Details-Bin"}, } @@ -487,11 +502,20 @@ func (s) TestHandlerTransport_HandleStreams_ErrDetails(t *testing.T) { checkHeaderAndTrailer(t, hst.rw, wantHeader, wantTrailer) } +// TestHandlerTransport_Drain verifies that Drain() is not implemented +// by `serverHandlerTransport`. +func (s) TestHandlerTransport_Drain(t *testing.T) { + defer func() { recover() }() + st := newHandleStreamTest(t) + st.ht.Drain("whatever") + t.Errorf("serverHandlerTransport.Drain() should have panicked") +} + // checkHeaderAndTrailer checks that the resulting header and trailer matches the expectation. func checkHeaderAndTrailer(t *testing.T, rw testHandlerResponseWriter, wantHeader, wantTrailer http.Header) { // For trailer-only responses, the trailer values might be reported as part of the Header. They will however // be present in Trailer in either case. Hence, normalize the header by removing all trailer values. - actualHeader := cloneHeader(rw.Result().Header) + actualHeader := rw.Result().Header.Clone() for _, trailerKey := range actualHeader["Trailer"] { actualHeader.Del(trailerKey) } @@ -503,21 +527,3 @@ func checkHeaderAndTrailer(t *testing.T, rw testHandlerResponseWriter, wantHeade t.Errorf("Trailer mismatch.\n got: %#v\n want: %#v", actualTrailer, wantTrailer) } } - -// cloneHeader performs a deep clone of an http.Header, since the (http.Header).Clone() method was only added in -// Go 1.13. -func cloneHeader(hdr http.Header) http.Header { - if hdr == nil { - return nil - } - - hdrClone := make(http.Header, len(hdr)) - - for k, vv := range hdr { - vvClone := make([]string, len(vv)) - copy(vvClone, vv) - hdrClone[k] = vvClone - } - - return hdrClone -} diff --git a/internal/transport/http2_client.go b/internal/transport/http2_client.go index e7f2321131e4..badab8acf3b1 100644 --- a/internal/transport/http2_client.go +++ b/internal/transport/http2_client.go @@ -24,6 +24,8 @@ import ( "io" "math" "net" + "net/http" + "path/filepath" "strconv" "strings" "sync" @@ -32,13 +34,17 @@ import ( "golang.org/x/net/http2" "golang.org/x/net/http2/hpack" - "google.golang.org/grpc/internal/grpcutil" - "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" - "google.golang.org/grpc/internal" "google.golang.org/grpc/internal/channelz" + icredentials "google.golang.org/grpc/internal/credentials" + "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/grpcutil" + imetadata "google.golang.org/grpc/internal/metadata" + istatus "google.golang.org/grpc/internal/status" "google.golang.org/grpc/internal/syscall" + "google.golang.org/grpc/internal/transport/networktype" "google.golang.org/grpc/keepalive" "google.golang.org/grpc/metadata" "google.golang.org/grpc/peer" @@ -54,12 +60,16 @@ var clientConnectionCounter uint64 // http2Client implements the ClientTransport interface with HTTP2. type http2Client struct { - lastRead int64 // Keep this field 64-bit aligned. Accessed atomically. - ctx context.Context - cancel context.CancelFunc - ctxDone <-chan struct{} // Cache the ctx.Done() chan. - userAgent string - md interface{} + lastRead int64 // Keep this field 64-bit aligned. Accessed atomically. + ctx context.Context + cancel context.CancelFunc + ctxDone <-chan struct{} // Cache the ctx.Done() chan. + userAgent string + // address contains the resolver returned address for this transport. + // If the `ServerName` field is set, it takes precedence over `CallHdr.Host` + // passed to `NewStream`, when determining the :authority header. + address resolver.Address + md metadata.MD conn net.Conn // underlying communication channel loopy *loopyWriter remoteAddr net.Addr @@ -75,6 +85,7 @@ type http2Client struct { framer *framer // controlBuf delivers all the control related tasks (e.g., window // updates, reset streams, and various settings) to the controller. + // Do not access controlBuf with mu held. controlBuf *controlBuffer fc *trInFlow // The scheme used: https if TLS is on, http otherwise. @@ -87,7 +98,7 @@ type http2Client struct { kp keepalive.ClientParameters keepaliveEnabled bool - statsHandler stats.Handler + statsHandlers []stats.Handler initialWindowSize int32 @@ -95,17 +106,15 @@ type http2Client struct { maxSendHeaderListSize *uint32 bdpEst *bdpEstimator - // onPrefaceReceipt is a callback that client transport calls upon - // receiving server preface to signal that a succefull HTTP2 - // connection was established. - onPrefaceReceipt func() maxConcurrentStreams uint32 streamQuota int64 streamsQuotaAvailable chan struct{} waitingStreams uint32 nextID uint32 + registeredCompressors string + // Do not access controlBuf with mu held. mu sync.Mutex // guard the following variables state transportState activeStreams map[uint32]*Stream @@ -114,6 +123,9 @@ type http2Client struct { // goAwayReason records the http2.ErrCode and debug data received with the // GoAway frame. goAwayReason GoAwayReason + // goAwayDebugMessage contains a detailed human readable string about a + // GoAway frame, useful for error messages. + goAwayDebugMessage string // A condition variable used to signal when the keepalive goroutine should // go dormant. The condition for dormancy is based on the number of active // streams and the `PermitWithoutStream` keepalive client parameter. And @@ -126,22 +138,45 @@ type http2Client struct { kpDormant bool // Fields below are for channelz metric collection. - channelzID int64 // channelz unique identification number + channelzID *channelz.Identifier czData *channelzData - onGoAway func(GoAwayReason) - onClose func() + onClose func(GoAwayReason) bufferPool *bufferPool connectionID uint64 + logger *grpclog.PrefixLogger } -func dial(ctx context.Context, fn func(context.Context, string) (net.Conn, error), addr string) (net.Conn, error) { +func dial(ctx context.Context, fn func(context.Context, string) (net.Conn, error), addr resolver.Address, useProxy bool, grpcUA string) (net.Conn, error) { + address := addr.Addr + networkType, ok := networktype.Get(addr) if fn != nil { - return fn(ctx, addr) + // Special handling for unix scheme with custom dialer. Back in the day, + // we did not have a unix resolver and therefore targets with a unix + // scheme would end up using the passthrough resolver. So, user's used a + // custom dialer in this case and expected the original dial target to + // be passed to the custom dialer. Now, we have a unix resolver. But if + // a custom dialer is specified, we want to retain the old behavior in + // terms of the address being passed to the custom dialer. + if networkType == "unix" && !strings.HasPrefix(address, "\x00") { + // Supported unix targets are either "unix://absolute-path" or + // "unix:relative-path". + if filepath.IsAbs(address) { + return fn(ctx, "unix://"+address) + } + return fn(ctx, "unix:"+address) + } + return fn(ctx, address) } - return (&net.Dialer{}).DialContext(ctx, "tcp", addr) + if !ok { + networkType, address = parseDialTarget(address) + } + if networkType == "tcp" && useProxy { + return proxyDial(ctx, address, grpcUA) + } + return (&net.Dialer{}).DialContext(ctx, networkType, address) } func isTemporary(err error) bool { @@ -163,7 +198,7 @@ func isTemporary(err error) bool { // newHTTP2Client constructs a connected ClientTransport to addr based on HTTP2 // and starts to receive messages on it. Non-nil error returns if construction // fails. -func newHTTP2Client(connectCtx, ctx context.Context, addr resolver.Address, opts ConnectOptions, onPrefaceReceipt func(), onGoAway func(GoAwayReason), onClose func()) (_ *http2Client, err error) { +func newHTTP2Client(connectCtx, ctx context.Context, addr resolver.Address, opts ConnectOptions, onClose func(GoAwayReason)) (_ *http2Client, err error) { scheme := "http" ctx, cancel := context.WithCancel(ctx) defer func() { @@ -172,19 +207,51 @@ func newHTTP2Client(connectCtx, ctx context.Context, addr resolver.Address, opts } }() - conn, err := dial(connectCtx, opts.Dialer, addr.Addr) + // gRPC, resolver, balancer etc. can specify arbitrary data in the + // Attributes field of resolver.Address, which is shoved into connectCtx + // and passed to the dialer and credential handshaker. This makes it possible for + // address specific arbitrary data to reach custom dialers and credential handshakers. + connectCtx = icredentials.NewClientHandshakeInfoContext(connectCtx, credentials.ClientHandshakeInfo{Attributes: addr.Attributes}) + + conn, err := dial(connectCtx, opts.Dialer, addr, opts.UseProxy, opts.UserAgent) if err != nil { if opts.FailOnNonTempDialError { return nil, connectionErrorf(isTemporary(err), err, "transport: error while dialing: %v", err) } - return nil, connectionErrorf(true, err, "transport: Error while dialing %v", err) + return nil, connectionErrorf(true, err, "transport: Error while dialing: %v", err) } + // Any further errors will close the underlying connection defer func(conn net.Conn) { if err != nil { conn.Close() } }(conn) + + // The following defer and goroutine monitor the connectCtx for cancelation + // and deadline. On context expiration, the connection is hard closed and + // this function will naturally fail as a result. Otherwise, the defer + // waits for the goroutine to exit to prevent the context from being + // monitored (and to prevent the connection from ever being closed) after + // returning from this function. + ctxMonitorDone := grpcsync.NewEvent() + newClientCtx, newClientDone := context.WithCancel(connectCtx) + defer func() { + newClientDone() // Awaken the goroutine below if connectCtx hasn't expired. + <-ctxMonitorDone.Done() // Wait for the goroutine below to exit. + }() + go func(conn net.Conn) { + defer ctxMonitorDone.Fire() // Signal this goroutine has exited. + <-newClientCtx.Done() // Block until connectCtx expires or the defer above executes. + if err := connectCtx.Err(); err != nil { + // connectCtx expired before exiting the function. Hard close the connection. + if logger.V(logLevel) { + logger.Infof("Aborting due to connect deadline expiring: %v", err) + } + conn.Close() + } + }(conn) + kp := opts.KeepaliveParams // Validate keepalive parameters. if kp.Time == 0 { @@ -216,16 +283,22 @@ func newHTTP2Client(connectCtx, ctx context.Context, addr resolver.Address, opts } } if transportCreds != nil { - // gRPC, resolver, balancer etc. can specify arbitrary data in the - // Attributes field of resolver.Address, which is shoved into connectCtx - // and passed to the credential handshaker. This makes it possible for - // address specific arbitrary data to reach the credential handshaker. - contextWithHandshakeInfo := internal.NewClientHandshakeInfoContext.(func(context.Context, credentials.ClientHandshakeInfo) context.Context) - connectCtx = contextWithHandshakeInfo(connectCtx, credentials.ClientHandshakeInfo{Attributes: addr.Attributes}) conn, authInfo, err = transportCreds.ClientHandshake(connectCtx, addr.ServerName, conn) if err != nil { return nil, connectionErrorf(isTemporary(err), err, "transport: authentication handshake failed: %v", err) } + for _, cd := range perRPCCreds { + if cd.RequireTransportSecurity() { + if ci, ok := authInfo.(interface { + GetCommonAuthInfo() credentials.CommonAuthInfo + }); ok { + secLevel := ci.GetCommonAuthInfo().SecurityLevel + if secLevel != credentials.InvalidSecurityLevel && secLevel < credentials.PrivacyAndIntegrity { + return nil, connectionErrorf(true, nil, "transport: cannot send secure credentials on an insecure connection") + } + } + } + } isSecure = true if transportCreds.Info().SecurityProtocol == "tls" { scheme = "https" @@ -248,7 +321,8 @@ func newHTTP2Client(connectCtx, ctx context.Context, addr resolver.Address, opts ctxDone: ctx.Done(), // Cache Done chan. cancel: cancel, userAgent: opts.UserAgent, - md: addr.Metadata, + registeredCompressors: grpcutil.RegisteredCompressors(), + address: addr, conn: conn, remoteAddr: conn.RemoteAddr(), localAddr: conn.LocalAddr(), @@ -256,25 +330,32 @@ func newHTTP2Client(connectCtx, ctx context.Context, addr resolver.Address, opts readerDone: make(chan struct{}), writerDone: make(chan struct{}), goAway: make(chan struct{}), - framer: newFramer(conn, writeBufSize, readBufSize, maxHeaderListSize), + framer: newFramer(conn, writeBufSize, readBufSize, opts.SharedWriteBuffer, maxHeaderListSize), fc: &trInFlow{limit: uint32(icwz)}, scheme: scheme, activeStreams: make(map[uint32]*Stream), isSecure: isSecure, perRPCCreds: perRPCCreds, kp: kp, - statsHandler: opts.StatsHandler, + statsHandlers: opts.StatsHandlers, initialWindowSize: initialWindowSize, - onPrefaceReceipt: onPrefaceReceipt, nextID: 1, maxConcurrentStreams: defaultMaxStreamsClient, streamQuota: defaultMaxStreamsClient, streamsQuotaAvailable: make(chan struct{}, 1), czData: new(channelzData), - onGoAway: onGoAway, - onClose: onClose, keepaliveEnabled: keepaliveEnabled, bufferPool: newBufferPool(), + onClose: onClose, + } + t.logger = prefixLoggerForClientTransport(t) + // Add peer information to the http2client context. + t.ctx = peer.NewContext(t.ctx, t.getPeer()) + + if md, ok := addr.Metadata.(*metadata.MD); ok { + t.md = *md + } else if md := imetadata.Get(addr); md != nil { + t.md = md } t.controlBuf = newControlBuffer(t.ctxDone) if opts.InitialWindowSize >= defaultWindowSize { @@ -287,37 +368,51 @@ func newHTTP2Client(connectCtx, ctx context.Context, addr resolver.Address, opts updateFlowControl: t.updateFlowControl, } } - if t.statsHandler != nil { - t.ctx = t.statsHandler.TagConn(t.ctx, &stats.ConnTagInfo{ + for _, sh := range t.statsHandlers { + t.ctx = sh.TagConn(t.ctx, &stats.ConnTagInfo{ RemoteAddr: t.remoteAddr, LocalAddr: t.localAddr, }) connBegin := &stats.ConnBegin{ Client: true, } - t.statsHandler.HandleConn(t.ctx, connBegin) + sh.HandleConn(t.ctx, connBegin) } - if channelz.IsOn() { - t.channelzID = channelz.RegisterNormalSocket(t, opts.ChannelzParentID, fmt.Sprintf("%s -> %s", t.localAddr, t.remoteAddr)) + t.channelzID, err = channelz.RegisterNormalSocket(t, opts.ChannelzParentID, fmt.Sprintf("%s -> %s", t.localAddr, t.remoteAddr)) + if err != nil { + return nil, err } if t.keepaliveEnabled { t.kpDormancyCond = sync.NewCond(&t.mu) go t.keepalive() } - // Start the reader goroutine for incoming message. Each transport has - // a dedicated goroutine which reads HTTP2 frame from network. Then it - // dispatches the frame to the corresponding stream entity. - go t.reader() + + // Start the reader goroutine for incoming messages. Each transport has a + // dedicated goroutine which reads HTTP2 frames from the network. Then it + // dispatches the frame to the corresponding stream entity. When the + // server preface is received, readerErrCh is closed. If an error occurs + // first, an error is pushed to the channel. This must be checked before + // returning from this function. + readerErrCh := make(chan error, 1) + go t.reader(readerErrCh) + defer func() { + if err == nil { + err = <-readerErrCh + } + if err != nil { + t.Close(err) + } + }() // Send connection preface to server. n, err := t.conn.Write(clientPreface) if err != nil { - t.Close() - return nil, connectionErrorf(true, err, "transport: failed to write client preface: %v", err) + err = connectionErrorf(true, err, "transport: failed to write client preface: %v", err) + return nil, err } if n != len(clientPreface) { - t.Close() - return nil, connectionErrorf(true, err, "transport: preface mismatch, wrote %d bytes; want %d", n, len(clientPreface)) + err = connectionErrorf(true, nil, "transport: preface mismatch, wrote %d bytes; want %d", n, len(clientPreface)) + return nil, err } var ss []http2.Setting @@ -335,14 +430,14 @@ func newHTTP2Client(connectCtx, ctx context.Context, addr resolver.Address, opts } err = t.framer.fr.WriteSettings(ss...) if err != nil { - t.Close() - return nil, connectionErrorf(true, err, "transport: failed to write initial settings frame: %v", err) + err = connectionErrorf(true, err, "transport: failed to write initial settings frame: %v", err) + return nil, err } // Adjust the connection flow control window if needed. if delta := uint32(icwz - defaultWindowSize); delta > 0 { if err := t.framer.fr.WriteWindowUpdate(0, delta); err != nil { - t.Close() - return nil, connectionErrorf(true, err, "transport: failed to write window update: %v", err) + err = connectionErrorf(true, err, "transport: failed to write window update: %v", err) + return nil, err } } @@ -352,18 +447,8 @@ func newHTTP2Client(connectCtx, ctx context.Context, addr resolver.Address, opts return nil, err } go func() { - t.loopy = newLoopyWriter(clientSide, t.framer, t.controlBuf, t.bdpEst) - err := t.loopy.run() - if err != nil { - if logger.V(logLevel) { - logger.Errorf("transport: loopyWriter.run returning. Err: %v", err) - } - } - // If it's a connection error, let reader goroutine handle it - // since there might be data in the buffers. - if _, ok := err.(net.Error); !ok { - t.conn.Close() - } + t.loopy = newLoopyWriter(clientSide, t.framer, t.controlBuf, t.bdpEst, t.conn, t.logger) + t.loopy.run() close(t.writerDone) }() return t, nil @@ -379,6 +464,7 @@ func (t *http2Client) newStream(ctx context.Context, callHdr *CallHdr) *Stream { buf: newRecvBuffer(), headerChan: make(chan struct{}), contentSubtype: callHdr.ContentSubtype, + doneFunc: callHdr.DoneFunc, } s.wq = newWriteQuota(defaultWriteQuota, s.done) s.requestRead = func(n int) { @@ -408,7 +494,7 @@ func (t *http2Client) newStream(ctx context.Context, callHdr *CallHdr) *Stream { func (t *http2Client) getPeer() *peer.Peer { return &peer.Peer{ Addr: t.remoteAddr, - AuthInfo: t.authInfo, + AuthInfo: t.authInfo, // Can be nil } } @@ -418,7 +504,7 @@ func (t *http2Client) createHeaderFields(ctx context.Context, callHdr *CallHdr) Method: callHdr.Method, AuthInfo: t.authInfo, } - ctxWithRequestInfo := internal.NewRequestInfoContext.(func(context.Context, credentials.RequestInfo) context.Context)(ctx, ri) + ctxWithRequestInfo := icredentials.NewRequestInfoContext(ctx, ri) authData, err := t.getTrAuthData(ctxWithRequestInfo, aud) if err != nil { return nil, err @@ -444,9 +530,22 @@ func (t *http2Client) createHeaderFields(ctx context.Context, callHdr *CallHdr) headerFields = append(headerFields, hpack.HeaderField{Name: "grpc-previous-rpc-attempts", Value: strconv.Itoa(callHdr.PreviousAttempts)}) } + registeredCompressors := t.registeredCompressors if callHdr.SendCompress != "" { headerFields = append(headerFields, hpack.HeaderField{Name: "grpc-encoding", Value: callHdr.SendCompress}) - headerFields = append(headerFields, hpack.HeaderField{Name: "grpc-accept-encoding", Value: callHdr.SendCompress}) + // Include the outgoing compressor name when compressor is not registered + // via encoding.RegisterCompressor. This is possible when client uses + // WithCompressor dial option. + if !grpcutil.IsCompressorNameRegistered(callHdr.SendCompress) { + if registeredCompressors != "" { + registeredCompressors += "," + } + registeredCompressors += callHdr.SendCompress + } + } + + if registeredCompressors != "" { + headerFields = append(headerFields, hpack.HeaderField{Name: "grpc-accept-encoding", Value: registeredCompressors}) } if dl, ok := ctx.Deadline(); ok { // Send out timeout regardless its value. The server can detect timeout context by itself. @@ -481,25 +580,23 @@ func (t *http2Client) createHeaderFields(ctx context.Context, callHdr *CallHdr) for _, vv := range added { for i, v := range vv { if i%2 == 0 { - k = v + k = strings.ToLower(v) continue } // HTTP doesn't allow you to set pseudoheaders after non pseudoheaders were set. if isReservedHeader(k) { continue } - headerFields = append(headerFields, hpack.HeaderField{Name: strings.ToLower(k), Value: encodeMetadataHeader(k, v)}) + headerFields = append(headerFields, hpack.HeaderField{Name: k, Value: encodeMetadataHeader(k, v)}) } } } - if md, ok := t.md.(*metadata.MD); ok { - for k, vv := range *md { - if isReservedHeader(k) { - continue - } - for _, v := range vv { - headerFields = append(headerFields, hpack.HeaderField{Name: k, Value: encodeMetadataHeader(k, v)}) - } + for k, vv := range t.md { + if isReservedHeader(k) { + continue + } + for _, v := range vv { + headerFields = append(headerFields, hpack.HeaderField{Name: k, Value: encodeMetadataHeader(k, v)}) } } return headerFields, nil @@ -528,11 +625,15 @@ func (t *http2Client) getTrAuthData(ctx context.Context, audience string) (map[s for _, c := range t.perRPCCreds { data, err := c.GetRequestMetadata(ctx, audience) if err != nil { - if _, ok := status.FromError(err); ok { + if st, ok := status.FromError(err); ok { + // Restrict the code to the list allowed by gRFC A54. + if istatus.IsRestrictedControlPlaneCode(st) { + err = status.Errorf(codes.Internal, "transport: received per-RPC creds error with illegal status: %v", err) + } return nil, err } - return nil, status.Errorf(codes.Unauthenticated, "transport: %v", err) + return nil, status.Errorf(codes.Unauthenticated, "transport: per-RPC creds failed due to error: %v", err) } for k, v := range data { // Capital header names are illegal in HTTP/2. @@ -549,12 +650,22 @@ func (t *http2Client) getCallAuthData(ctx context.Context, audience string, call // Note: if these credentials are provided both via dial options and call // options, then both sets of credentials will be applied. if callCreds := callHdr.Creds; callCreds != nil { - if !t.isSecure && callCreds.RequireTransportSecurity() { - return nil, status.Error(codes.Unauthenticated, "transport: cannot send secure credentials on an insecure connection") + if callCreds.RequireTransportSecurity() { + ri, _ := credentials.RequestInfoFromContext(ctx) + if !t.isSecure || credentials.CheckSecurityLevel(ri.AuthInfo, credentials.PrivacyAndIntegrity) != nil { + return nil, status.Error(codes.Unauthenticated, "transport: cannot send secure credentials on an insecure connection") + } } data, err := callCreds.GetRequestMetadata(ctx, audience) if err != nil { - return nil, status.Errorf(codes.Internal, "transport: %v", err) + if st, ok := status.FromError(err); ok { + // Restrict the code to the list allowed by gRFC A54. + if istatus.IsRestrictedControlPlaneCode(st) { + err = status.Errorf(codes.Internal, "transport: received per-RPC creds error with illegal status: %v", err) + } + return nil, err + } + return nil, status.Errorf(codes.Internal, "transport: per-RPC creds failed due to error: %v", err) } callAuthData = make(map[string]string, len(data)) for k, v := range data { @@ -566,26 +677,46 @@ func (t *http2Client) getCallAuthData(ctx context.Context, audience string, call return callAuthData, nil } -// PerformedIOError wraps an error to indicate IO may have been performed -// before the error occurred. -type PerformedIOError struct { +// NewStreamError wraps an error and reports additional information. Typically +// NewStream errors result in transparent retry, as they mean nothing went onto +// the wire. However, there are two notable exceptions: +// +// 1. If the stream headers violate the max header list size allowed by the +// server. It's possible this could succeed on another transport, even if +// it's unlikely, but do not transparently retry. +// 2. If the credentials errored when requesting their headers. In this case, +// it's possible a retry can fix the problem, but indefinitely transparently +// retrying is not appropriate as it is likely the credentials, if they can +// eventually succeed, would need I/O to do so. +type NewStreamError struct { Err error + + AllowTransparentRetry bool } -// Error implements error. -func (p PerformedIOError) Error() string { - return p.Err.Error() +func (e NewStreamError) Error() string { + return e.Err.Error() } // NewStream creates a stream and registers it into the transport as "active" -// streams. -func (t *http2Client) NewStream(ctx context.Context, callHdr *CallHdr) (_ *Stream, err error) { +// streams. All non-nil errors returned will be *NewStreamError. +func (t *http2Client) NewStream(ctx context.Context, callHdr *CallHdr) (*Stream, error) { ctx = peer.NewContext(ctx, t.getPeer()) + + // ServerName field of the resolver returned address takes precedence over + // Host field of CallHdr to determine the :authority header. This is because, + // the ServerName field takes precedence for server authentication during + // TLS handshake, and the :authority header should match the value used + // for server authentication. + if t.address.ServerName != "" { + newCallHdr := *callHdr + newCallHdr.Host = t.address.ServerName + callHdr = &newCallHdr + } + headerFields, err := t.createHeaderFields(ctx, callHdr) if err != nil { - // We may have performed I/O in the per-RPC creds callback, so do not - // allow transparent retry. - return nil, PerformedIOError{err} + return nil, &NewStreamError{Err: err, AllowTransparentRetry: false} } s := t.newStream(ctx, callHdr) cleanup := func(err error) { @@ -607,17 +738,13 @@ func (t *http2Client) NewStream(ctx context.Context, callHdr *CallHdr) (_ *Strea endStream: false, initStream: func(id uint32) error { t.mu.Lock() - if state := t.state; state != reachable { + // TODO: handle transport closure in loopy instead and remove this + // initStream is never called when transport is draining. + if t.state == closing { t.mu.Unlock() - // Do a quick cleanup. - err := error(errStreamDrain) - if state == closing { - err = ErrConnClosing - } - cleanup(err) - return err + cleanup(ErrConnClosing) + return ErrConnClosing } - t.activeStreams[id] = s if channelz.IsOn() { atomic.AddInt64(&t.czData.streamsStarted, 1) atomic.StoreInt64(&t.czData.lastStreamCreatedTime, time.Now().UnixNano()) @@ -634,7 +761,8 @@ func (t *http2Client) NewStream(ctx context.Context, callHdr *CallHdr) (_ *Strea } firstTry := true var ch chan struct{} - checkForStreamQuota := func(it interface{}) bool { + transportDrainRequired := false + checkForStreamQuota := func(it any) bool { if t.streamQuota <= 0 { // Can go negative if server decreases it. if firstTry { t.waitingStreams++ @@ -649,8 +777,20 @@ func (t *http2Client) NewStream(ctx context.Context, callHdr *CallHdr) (_ *Strea h := it.(*headerFrame) h.streamID = t.nextID t.nextID += 2 + + // Drain client transport if nextID > MaxStreamID which signals gRPC that + // the connection is closed and a new one must be created for subsequent RPCs. + transportDrainRequired = t.nextID > MaxStreamID + s.id = h.streamID s.fc = &inFlow{limit: uint32(t.initialWindowSize)} + t.mu.Lock() + if t.state == draining || t.activeStreams == nil { // Can be niled from Close(). + t.mu.Unlock() + return false // Don't create a stream if the transport is already closed. + } + t.activeStreams[s.id] = s + t.mu.Unlock() if t.streamQuota > 0 && t.waitingStreams > 0 { select { case t.streamsQuotaAvailable <- struct{}{}: @@ -660,7 +800,7 @@ func (t *http2Client) NewStream(ctx context.Context, callHdr *CallHdr) (_ *Strea return true } var hdrListSizeErr error - checkForHeaderListSize := func(it interface{}) bool { + checkForHeaderListSize := func(it any) bool { if t.maxSendHeaderListSize == nil { return true } @@ -675,53 +815,57 @@ func (t *http2Client) NewStream(ctx context.Context, callHdr *CallHdr) (_ *Strea return true } for { - success, err := t.controlBuf.executeAndPut(func(it interface{}) bool { - if !checkForStreamQuota(it) { - return false - } - if !checkForHeaderListSize(it) { - return false - } - return true + success, err := t.controlBuf.executeAndPut(func(it any) bool { + return checkForHeaderListSize(it) && checkForStreamQuota(it) }, hdr) if err != nil { - return nil, err + // Connection closed. + return nil, &NewStreamError{Err: err, AllowTransparentRetry: true} } if success { break } if hdrListSizeErr != nil { - return nil, hdrListSizeErr + return nil, &NewStreamError{Err: hdrListSizeErr} } firstTry = false select { case <-ch: - case <-s.ctx.Done(): - return nil, ContextErr(s.ctx.Err()) + case <-ctx.Done(): + return nil, &NewStreamError{Err: ContextErr(ctx.Err())} case <-t.goAway: - return nil, errStreamDrain + return nil, &NewStreamError{Err: errStreamDrain, AllowTransparentRetry: true} case <-t.ctx.Done(): - return nil, ErrConnClosing + return nil, &NewStreamError{Err: ErrConnClosing, AllowTransparentRetry: true} } } - if t.statsHandler != nil { + if len(t.statsHandlers) != 0 { header, ok := metadata.FromOutgoingContext(ctx) if ok { header.Set("user-agent", t.userAgent) } else { header = metadata.Pairs("user-agent", t.userAgent) } - // Note: The header fields are compressed with hpack after this call returns. - // No WireLength field is set here. - outHeader := &stats.OutHeader{ - Client: true, - FullMethod: callHdr.Method, - RemoteAddr: t.remoteAddr, - LocalAddr: t.localAddr, - Compression: callHdr.SendCompress, - Header: header, + for _, sh := range t.statsHandlers { + // Note: The header fields are compressed with hpack after this call returns. + // No WireLength field is set here. + // Note: Creating a new stats object to prevent pollution. + outHeader := &stats.OutHeader{ + Client: true, + FullMethod: callHdr.Method, + RemoteAddr: t.remoteAddr, + LocalAddr: t.localAddr, + Compression: callHdr.SendCompress, + Header: header, + } + sh.HandleRPC(s.ctx, outHeader) + } + } + if transportDrainRequired { + if t.logger.V(logLevel) { + t.logger.Infof("Draining transport: t.nextID > MaxStreamID") } - t.statsHandler.HandleRPC(s.ctx, outHeader) + t.GracefulClose() } return s, nil } @@ -783,7 +927,7 @@ func (t *http2Client) closeStream(s *Stream, err error, rst bool, rstCode http2. rst: rst, rstCode: rstCode, } - addBackStreamQuota := func(interface{}) bool { + addBackStreamQuota := func(any) bool { t.streamQuota++ if t.streamQuota > 0 && t.waitingStreams > 0 { select { @@ -796,25 +940,29 @@ func (t *http2Client) closeStream(s *Stream, err error, rst bool, rstCode http2. t.controlBuf.executeAndPut(addBackStreamQuota, cleanup) // This will unblock write. close(s.done) + if s.doneFunc != nil { + s.doneFunc() + } } // Close kicks off the shutdown process of the transport. This should be called // only once on a transport. Once it is called, the transport should not be // accessed any more. -// -// This method blocks until the addrConn that initiated this transport is -// re-connected. This happens because t.onClose() begins reconnect logic at the -// addrConn level and blocks until the addrConn is successfully connected. -func (t *http2Client) Close() error { +func (t *http2Client) Close(err error) { t.mu.Lock() - // Make sure we only Close once. + // Make sure we only close once. if t.state == closing { t.mu.Unlock() - return nil + return + } + if t.logger.V(logLevel) { + t.logger.Infof("Closing: %v", err) + } + // Call t.onClose ASAP to prevent the client from attempting to create new + // streams. + if t.state != draining { + t.onClose(GoAwayInvalid) } - // Call t.onClose before setting the state to closing to prevent the client - // from attempting to create new streams ASAP. - t.onClose() t.state = closing streams := t.activeStreams t.activeStreams = nil @@ -826,21 +974,30 @@ func (t *http2Client) Close() error { t.mu.Unlock() t.controlBuf.finish() t.cancel() - err := t.conn.Close() - if channelz.IsOn() { - channelz.RemoveEntry(t.channelzID) + t.conn.Close() + channelz.RemoveEntry(t.channelzID) + // Append info about previous goaways if there were any, since this may be important + // for understanding the root cause for this connection to be closed. + _, goAwayDebugMessage := t.GetGoAwayReason() + + var st *status.Status + if len(goAwayDebugMessage) > 0 { + st = status.Newf(codes.Unavailable, "closing transport due to: %v, received prior goaway: %v", err, goAwayDebugMessage) + err = st.Err() + } else { + st = status.New(codes.Unavailable, err.Error()) } + // Notify all active streams. for _, s := range streams { - t.closeStream(s, ErrConnClosing, false, http2.ErrCodeNo, status.New(codes.Unavailable, ErrConnClosing.Desc), nil, false) + t.closeStream(s, err, false, http2.ErrCodeNo, st, nil, false) } - if t.statsHandler != nil { + for _, sh := range t.statsHandlers { connEnd := &stats.ConnEnd{ Client: true, } - t.statsHandler.HandleConn(t.ctx, connEnd) + sh.HandleConn(t.ctx, connEnd) } - return err } // GracefulClose sets the state to draining, which prevents new streams from @@ -855,11 +1012,15 @@ func (t *http2Client) GracefulClose() { t.mu.Unlock() return } + if t.logger.V(logLevel) { + t.logger.Infof("GracefulClose called") + } + t.onClose(GoAwayInvalid) t.state = draining active := len(t.activeStreams) t.mu.Unlock() if active == 0 { - t.Close() + t.Close(connectionErrorf(true, nil, "no active streams left to process while draining")) return } t.controlBuf.put(&incomingGoAway{}) @@ -919,13 +1080,13 @@ func (t *http2Client) updateWindow(s *Stream, n uint32) { // for the transport and the stream based on the current bdp // estimation. func (t *http2Client) updateFlowControl(n uint32) { - t.mu.Lock() - for _, s := range t.activeStreams { - s.fc.newLimit(n) - } - t.mu.Unlock() - updateIWS := func(interface{}) bool { + updateIWS := func(any) bool { t.initialWindowSize = int32(n) + t.mu.Lock() + for _, s := range t.activeStreams { + s.fc.newLimit(n) + } + t.mu.Unlock() return true } t.controlBuf.executeAndPut(updateIWS, &outgoingWindowUpdate{streamID: 0, increment: t.fc.newLimit(n)}) @@ -1000,7 +1161,7 @@ func (t *http2Client) handleData(f *http2.DataFrame) { } // The server has closed the stream without sending trailers. Record that // the read direction is closed, and set the status appropriately. - if f.FrameHeader.Flags.Has(http2.FlagDataEndStream) { + if f.StreamEnded() { t.closeStream(s, io.EOF, false, http2.ErrCodeNo, status.New(codes.Internal, "server closed the stream without sending trailers"), nil, true) } } @@ -1016,8 +1177,8 @@ func (t *http2Client) handleRSTStream(f *http2.RSTStreamFrame) { } statusCode, ok := http2ErrConvTab[f.ErrCode] if !ok { - if logger.V(logLevel) { - logger.Warningf("transport: http2Client.handleRSTStream found no mapped gRPC status for the received http2 error %v", f.ErrCode) + if t.logger.V(logLevel) { + t.logger.Infof("Received a RST_STREAM frame with code %q, but found no mapped gRPC status", f.ErrCode) } statusCode = codes.Unknown } @@ -1072,7 +1233,7 @@ func (t *http2Client) handleSettings(f *http2.SettingsFrame, isFirst bool) { } updateFuncs = append(updateFuncs, updateStreamQuota) } - t.controlBuf.executeAndPut(func(interface{}) bool { + t.controlBuf.executeAndPut(func(any) bool { for _, f := range updateFuncs { f() } @@ -1099,15 +1260,17 @@ func (t *http2Client) handleGoAway(f *http2.GoAwayFrame) { t.mu.Unlock() return } - if f.ErrCode == http2.ErrCodeEnhanceYourCalm { - if logger.V(logLevel) { - logger.Infof("Client received GoAway with http2.ErrCodeEnhanceYourCalm.") - } + if f.ErrCode == http2.ErrCodeEnhanceYourCalm && string(f.DebugData()) == "too_many_pings" { + // When a client receives a GOAWAY with error code ENHANCE_YOUR_CALM and debug + // data equal to ASCII "too_many_pings", it should log the occurrence at a log level that is + // enabled by default and double the configure KEEPALIVE_TIME used for new connections + // on that channel. + logger.Errorf("Client received GoAway with error code ENHANCE_YOUR_CALM and debug data equal to ASCII \"too_many_pings\".") } id := f.LastStreamID - if id > 0 && id%2 != 1 { + if id > 0 && id%2 == 0 { t.mu.Unlock() - t.Close() + t.Close(connectionErrorf(true, nil, "received goaway with non-zero even-numbered numbered stream id: %v", id)) return } // A client can receive multiple GoAways from the server (see @@ -1125,18 +1288,20 @@ func (t *http2Client) handleGoAway(f *http2.GoAwayFrame) { // If there are multiple GoAways the first one should always have an ID greater than the following ones. if id > t.prevGoAwayID { t.mu.Unlock() - t.Close() + t.Close(connectionErrorf(true, nil, "received goaway with stream id: %v, which exceeds stream id of previous goaway: %v", id, t.prevGoAwayID)) return } default: t.setGoAwayReason(f) close(t.goAway) - t.controlBuf.put(&incomingGoAway{}) + defer t.controlBuf.put(&incomingGoAway{}) // Defer as t.mu is currently held. // Notify the clientconn about the GOAWAY before we set the state to // draining, to allow the client to stop attempting to create streams // before disallowing new streams on this connection. - t.onGoAway(t.goAwayReason) - t.state = draining + if t.state != draining { + t.onClose(t.goAwayReason) + t.state = draining + } } // All streams with IDs greater than the GoAwayId // and smaller than the previous GoAway ID should be killed. @@ -1144,24 +1309,35 @@ func (t *http2Client) handleGoAway(f *http2.GoAwayFrame) { if upperLimit == 0 { // This is the first GoAway Frame. upperLimit = math.MaxUint32 // Kill all streams after the GoAway ID. } + + t.prevGoAwayID = id + if len(t.activeStreams) == 0 { + t.mu.Unlock() + t.Close(connectionErrorf(true, nil, "received goaway and there are no active streams")) + return + } + + streamsToClose := make([]*Stream, 0) for streamID, stream := range t.activeStreams { if streamID > id && streamID <= upperLimit { // The stream was unprocessed by the server. - atomic.StoreUint32(&stream.unprocessed, 1) - t.closeStream(stream, errStreamDrain, false, http2.ErrCodeNo, statusGoAway, nil, false) + if streamID > id && streamID <= upperLimit { + atomic.StoreUint32(&stream.unprocessed, 1) + streamsToClose = append(streamsToClose, stream) + } } } - t.prevGoAwayID = id - active := len(t.activeStreams) t.mu.Unlock() - if active == 0 { - t.Close() + // Called outside t.mu because closeStream can take controlBuf's mu, which + // could induce deadlock and is not allowed. + for _, stream := range streamsToClose { + t.closeStream(stream, errStreamDrain, false, http2.ErrCodeNo, statusGoAway, nil, false) } } // setGoAwayReason sets the value of t.goAwayReason based // on the GoAway frame received. -// It expects a lock on transport's mutext to be held by +// It expects a lock on transport's mutex to be held by // the caller. func (t *http2Client) setGoAwayReason(f *http2.GoAwayFrame) { t.goAwayReason = GoAwayNoReason @@ -1171,12 +1347,17 @@ func (t *http2Client) setGoAwayReason(f *http2.GoAwayFrame) { t.goAwayReason = GoAwayTooManyPings } } + if len(f.DebugData()) == 0 { + t.goAwayDebugMessage = fmt.Sprintf("code: %s", f.ErrCode) + } else { + t.goAwayDebugMessage = fmt.Sprintf("code: %s, debug data: %q", f.ErrCode, string(f.DebugData())) + } } -func (t *http2Client) GetGoAwayReason() GoAwayReason { +func (t *http2Client) GetGoAwayReason() (GoAwayReason, string) { t.mu.Lock() defer t.mu.Unlock() - return t.goAwayReason + return t.goAwayReason, t.goAwayDebugMessage } func (t *http2Client) handleWindowUpdate(f *http2.WindowUpdateFrame) { @@ -1203,90 +1384,209 @@ func (t *http2Client) operateHeaders(frame *http2.MetaHeadersFrame) { return } - state := &decodeState{} - // Initialize isGRPC value to be !initialHeader, since if a gRPC Response-Headers has already been received, then it means that the peer is speaking gRPC and we are in gRPC mode. - state.data.isGRPC = !initialHeader - if err := state.decodeHeader(frame); err != nil { - t.closeStream(s, err, true, http2.ErrCodeProtocol, status.Convert(err), nil, endStream) + // frame.Truncated is set to true when framer detects that the current header + // list size hits MaxHeaderListSize limit. + if frame.Truncated { + se := status.New(codes.Internal, "peer header list size exceeded limit") + t.closeStream(s, se.Err(), true, http2.ErrCodeFrameSize, se, nil, endStream) return } - isHeader := false - defer func() { - if t.statsHandler != nil { - if isHeader { - inHeader := &stats.InHeader{ - Client: true, - WireLength: int(frame.Header().Length), - Header: s.header.Copy(), - Compression: s.recvCompress, - } - t.statsHandler.HandleRPC(s.ctx, inHeader) - } else { - inTrailer := &stats.InTrailer{ - Client: true, - WireLength: int(frame.Header().Length), - Trailer: s.trailer.Copy(), - } - t.statsHandler.HandleRPC(s.ctx, inTrailer) + var ( + // If a gRPC Response-Headers has already been received, then it means + // that the peer is speaking gRPC and we are in gRPC mode. + isGRPC = !initialHeader + mdata = make(map[string][]string) + contentTypeErr = "malformed header: missing HTTP content-type" + grpcMessage string + statusGen *status.Status + recvCompress string + httpStatusCode *int + httpStatusErr string + rawStatusCode = codes.Unknown + // headerError is set if an error is encountered while parsing the headers + headerError string + ) + + if initialHeader { + httpStatusErr = "malformed header: missing HTTP status" + } + + for _, hf := range frame.Fields { + switch hf.Name { + case "content-type": + if _, validContentType := grpcutil.ContentSubtype(hf.Value); !validContentType { + contentTypeErr = fmt.Sprintf("transport: received unexpected content-type %q", hf.Value) + break + } + contentTypeErr = "" + mdata[hf.Name] = append(mdata[hf.Name], hf.Value) + isGRPC = true + case "grpc-encoding": + recvCompress = hf.Value + case "grpc-status": + code, err := strconv.ParseInt(hf.Value, 10, 32) + if err != nil { + se := status.New(codes.Internal, fmt.Sprintf("transport: malformed grpc-status: %v", err)) + t.closeStream(s, se.Err(), true, http2.ErrCodeProtocol, se, nil, endStream) + return + } + rawStatusCode = codes.Code(uint32(code)) + case "grpc-message": + grpcMessage = decodeGrpcMessage(hf.Value) + case "grpc-status-details-bin": + var err error + statusGen, err = decodeGRPCStatusDetails(hf.Value) + if err != nil { + headerError = fmt.Sprintf("transport: malformed grpc-status-details-bin: %v", err) } + case ":status": + if hf.Value == "200" { + httpStatusErr = "" + statusCode := 200 + httpStatusCode = &statusCode + break + } + + c, err := strconv.ParseInt(hf.Value, 10, 32) + if err != nil { + se := status.New(codes.Internal, fmt.Sprintf("transport: malformed http-status: %v", err)) + t.closeStream(s, se.Err(), true, http2.ErrCodeProtocol, se, nil, endStream) + return + } + statusCode := int(c) + httpStatusCode = &statusCode + + httpStatusErr = fmt.Sprintf( + "unexpected HTTP status code received from server: %d (%s)", + statusCode, + http.StatusText(statusCode), + ) + default: + if isReservedHeader(hf.Name) && !isWhitelistedHeader(hf.Name) { + break + } + v, err := decodeMetadataHeader(hf.Name, hf.Value) + if err != nil { + headerError = fmt.Sprintf("transport: malformed %s: %v", hf.Name, err) + logger.Warningf("Failed to decode metadata header (%q, %q): %v", hf.Name, hf.Value, err) + break + } + mdata[hf.Name] = append(mdata[hf.Name], v) } - }() + } - // If headerChan hasn't been closed yet - if atomic.CompareAndSwapUint32(&s.headerChanClosed, 0, 1) { - s.headerValid = true - if !endStream { - // HEADERS frame block carries a Response-Headers. - isHeader = true + if !isGRPC || httpStatusErr != "" { + var code = codes.Internal // when header does not include HTTP status, return INTERNAL + + if httpStatusCode != nil { + var ok bool + code, ok = HTTPStatusConvTab[*httpStatusCode] + if !ok { + code = codes.Unknown + } + } + var errs []string + if httpStatusErr != "" { + errs = append(errs, httpStatusErr) + } + if contentTypeErr != "" { + errs = append(errs, contentTypeErr) + } + // Verify the HTTP response is a 200. + se := status.New(code, strings.Join(errs, "; ")) + t.closeStream(s, se.Err(), true, http2.ErrCodeProtocol, se, nil, endStream) + return + } + + if headerError != "" { + se := status.New(codes.Internal, headerError) + t.closeStream(s, se.Err(), true, http2.ErrCodeProtocol, se, nil, endStream) + return + } + + // For headers, set them in s.header and close headerChan. For trailers or + // trailers-only, closeStream will set the trailers and close headerChan as + // needed. + if !endStream { + // If headerChan hasn't been closed yet (expected, given we checked it + // above, but something else could have potentially closed the whole + // stream). + if atomic.CompareAndSwapUint32(&s.headerChanClosed, 0, 1) { + s.headerValid = true // These values can be set without any synchronization because // stream goroutine will read it only after seeing a closed // headerChan which we'll close after setting this. - s.recvCompress = state.data.encoding - if len(state.data.mdata) > 0 { - s.header = state.data.mdata + s.recvCompress = recvCompress + if len(mdata) > 0 { + s.header = mdata } + close(s.headerChan) + } + } + + for _, sh := range t.statsHandlers { + if !endStream { + inHeader := &stats.InHeader{ + Client: true, + WireLength: int(frame.Header().Length), + Header: metadata.MD(mdata).Copy(), + Compression: s.recvCompress, + } + sh.HandleRPC(s.ctx, inHeader) } else { - // HEADERS frame block carries a Trailers-Only. - s.noHeaders = true + inTrailer := &stats.InTrailer{ + Client: true, + WireLength: int(frame.Header().Length), + Trailer: metadata.MD(mdata).Copy(), + } + sh.HandleRPC(s.ctx, inTrailer) } - close(s.headerChan) } if !endStream { return } - // if client received END_STREAM from server while stream was still active, send RST_STREAM - rst := s.getState() == streamActive - t.closeStream(s, io.EOF, rst, http2.ErrCodeNo, state.status(), state.data.mdata, true) + if statusGen == nil { + statusGen = status.New(rawStatusCode, grpcMessage) + } + + // If client received END_STREAM from server while stream was still active, + // send RST_STREAM. + rstStream := s.getState() == streamActive + t.closeStream(s, io.EOF, rstStream, http2.ErrCodeNo, statusGen, mdata, true) } -// reader runs as a separate goroutine in charge of reading data from network -// connection. -// -// TODO(zhaoq): currently one reader per transport. Investigate whether this is -// optimal. -// TODO(zhaoq): Check the validity of the incoming frame sequence. -func (t *http2Client) reader() { - defer close(t.readerDone) - // Check the validity of server preface. +// readServerPreface reads and handles the initial settings frame from the +// server. +func (t *http2Client) readServerPreface() error { frame, err := t.framer.fr.ReadFrame() if err != nil { - t.Close() // this kicks off resetTransport, so must be last before return - return - } - t.conn.SetReadDeadline(time.Time{}) // reset deadline once we get the settings frame (we didn't time out, yay!) - if t.keepaliveEnabled { - atomic.StoreInt64(&t.lastRead, time.Now().UnixNano()) + return connectionErrorf(true, err, "error reading server preface: %v", err) } sf, ok := frame.(*http2.SettingsFrame) if !ok { - t.Close() // this kicks off resetTransport, so must be last before return - return + return connectionErrorf(true, nil, "initial http2 frame from server is not a settings frame: %T", frame) } - t.onPrefaceReceipt() t.handleSettings(sf, true) + return nil +} + +// reader verifies the server preface and reads all subsequent data from +// network connection. If the server preface is not read successfully, an +// error is pushed to errCh; otherwise errCh is closed with no error. +func (t *http2Client) reader(errCh chan<- error) { + defer close(t.readerDone) + + if err := t.readServerPreface(); err != nil { + errCh <- err + return + } + close(errCh) + if t.keepaliveEnabled { + atomic.StoreInt64(&t.lastRead, time.Now().UnixNano()) + } // loop to keep reading incoming messages on this transport. for { @@ -1306,13 +1606,19 @@ func (t *http2Client) reader() { if s != nil { // use error detail to provide better err message code := http2ErrConvTab[se.Code] - msg := t.framer.fr.ErrorDetail().Error() + errorDetail := t.framer.fr.ErrorDetail() + var msg string + if errorDetail != nil { + msg = errorDetail.Error() + } else { + msg = "received invalid frame" + } t.closeStream(s, status.Error(code, msg), true, http2.ErrCodeProtocol, status.New(code, msg), nil, false) } continue } else { // Transport error. - t.Close() + t.Close(connectionErrorf(true, err, "error reading from server: %v", err)) return } } @@ -1346,7 +1652,7 @@ func minTime(a, b time.Duration) time.Duration { return b } -// keepalive running in a separate goroutune makes sure the connection is alive by sending pings. +// keepalive running in a separate goroutine makes sure the connection is alive by sending pings. func (t *http2Client) keepalive() { p := &ping{data: [8]byte{}} // True iff a ping has been sent, and no data has been received since then. @@ -1371,7 +1677,7 @@ func (t *http2Client) keepalive() { continue } if outstandingPing && timeoutLeft <= 0 { - t.Close() + t.Close(connectionErrorf(true, nil, "keepalive ping failed to receive ACK within timeout")) return } t.mu.Lock() @@ -1483,3 +1789,9 @@ func (t *http2Client) getOutFlowWindow() int64 { return -2 } } + +func (t *http2Client) stateForTesting() transportState { + t.mu.Lock() + defer t.mu.Unlock() + return t.state +} diff --git a/internal/transport/http2_server.go b/internal/transport/http2_server.go index 3be22fee426c..8d3a353c1d58 100644 --- a/internal/transport/http2_server.go +++ b/internal/transport/http2_server.go @@ -26,6 +26,7 @@ import ( "io" "math" "net" + "net/http" "strconv" "sync" "sync/atomic" @@ -34,12 +35,16 @@ import ( "github.com/golang/protobuf/proto" "golang.org/x/net/http2" "golang.org/x/net/http2/hpack" + "google.golang.org/grpc/internal/grpclog" "google.golang.org/grpc/internal/grpcutil" + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/internal/syscall" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" "google.golang.org/grpc/internal/channelz" "google.golang.org/grpc/internal/grpcrand" + "google.golang.org/grpc/internal/grpcsync" "google.golang.org/grpc/keepalive" "google.golang.org/grpc/metadata" "google.golang.org/grpc/peer" @@ -51,10 +56,10 @@ import ( var ( // ErrIllegalHeaderWrite indicates that setting header is illegal because of // the stream's state. - ErrIllegalHeaderWrite = errors.New("transport: the stream is done or WriteHeader was already called") + ErrIllegalHeaderWrite = status.Error(codes.Internal, "transport: SendHeader called multiple times") // ErrHeaderListSizeLimitViolation indicates that the header list size is larger // than the limit set by peer. - ErrHeaderListSizeLimitViolation = errors.New("transport: trying to send header list size larger than the limit set by peer") + ErrHeaderListSizeLimitViolation = status.Error(codes.Internal, "transport: trying to send header list size larger than the limit set by peer") ) // serverConnectionCounter counts the number of connections a server has seen @@ -72,7 +77,6 @@ type http2Server struct { writerDone chan struct{} // sync point to enable testing. remoteAddr net.Addr localAddr net.Addr - maxStreamID uint32 // max stream ID ever seen authInfo credentials.AuthInfo // auth info about the connection inTapHandle tap.ServerInHandle framer *framer @@ -82,7 +86,7 @@ type http2Server struct { // updates, reset streams, and various settings) to the controller. controlBuf *controlBuffer fc *trInFlow - stats stats.Handler + stats []stats.Handler // Keepalive and max-age parameters for the server. kp keepalive.ServerParameters // Keepalive enforcement policy. @@ -101,13 +105,13 @@ type http2Server struct { mu sync.Mutex // guard the following - // drainChan is initialized when drain(...) is called the first time. - // After which the server writes out the first GoAway(with ID 2^31-1) frame. - // Then an independent goroutine will be launched to later send the second GoAway. - // During this time we don't want to write another first GoAway(with ID 2^31 -1) frame. - // Thus call to drain(...) will be a no-op if drainChan is already initialized since draining is - // already underway. - drainChan chan struct{} + // drainEvent is initialized when Drain() is called the first time. After + // which the server writes out the first GoAway(with ID 2^31-1) frame. Then + // an independent goroutine will be launched to later send the second + // GoAway. During this time we don't want to write another first GoAway(with + // ID 2^31 -1) frame. Thus call to Drain() will be a no-op if drainEvent is + // already initialized since draining is already underway. + drainEvent *grpcsync.Event state transportState activeStreams map[uint32]*Stream // idle is the time instant when the connection went idle. @@ -117,23 +121,51 @@ type http2Server struct { idle time.Time // Fields below are for channelz metric collection. - channelzID int64 // channelz unique identification number + channelzID *channelz.Identifier czData *channelzData bufferPool *bufferPool connectionID uint64 + + // maxStreamMu guards the maximum stream ID + // This lock may not be taken if mu is already held. + maxStreamMu sync.Mutex + maxStreamID uint32 // max stream ID ever seen + + logger *grpclog.PrefixLogger } -// newHTTP2Server constructs a ServerTransport based on HTTP2. ConnectionError is -// returned if something goes wrong. -func newHTTP2Server(conn net.Conn, config *ServerConfig) (_ ServerTransport, err error) { +// NewServerTransport creates a http2 transport with conn and configuration +// options from config. +// +// It returns a non-nil transport and a nil error on success. On failure, it +// returns a nil transport and a non-nil error. For a special case where the +// underlying conn gets closed before the client preface could be read, it +// returns a nil transport and a nil error. +func NewServerTransport(conn net.Conn, config *ServerConfig) (_ ServerTransport, err error) { + var authInfo credentials.AuthInfo + rawConn := conn + if config.Credentials != nil { + var err error + conn, authInfo, err = config.Credentials.ServerHandshake(rawConn) + if err != nil { + // ErrConnDispatched means that the connection was dispatched away + // from gRPC; those connections should be left open. io.EOF means + // the connection was closed before handshaking completed, which can + // happen naturally from probers. Return these errors directly. + if err == credentials.ErrConnDispatched || err == io.EOF { + return nil, err + } + return nil, connectionErrorf(false, err, "ServerHandshake(%q) failed: %v", rawConn.RemoteAddr(), err) + } + } writeBufSize := config.WriteBufferSize readBufSize := config.ReadBufferSize maxHeaderListSize := defaultServerMaxHeaderListSize if config.MaxHeaderListSize != nil { maxHeaderListSize = *config.MaxHeaderListSize } - framer := newFramer(conn, writeBufSize, readBufSize, maxHeaderListSize) + framer := newFramer(conn, writeBufSize, readBufSize, config.SharedWriteBuffer, maxHeaderListSize) // Send initial settings as connection preface to client. isettings := []http2.Setting{{ ID: http2.SettingMaxFrameSize, @@ -205,18 +237,24 @@ func newHTTP2Server(conn net.Conn, config *ServerConfig) (_ ServerTransport, err if kp.Timeout == 0 { kp.Timeout = defaultServerKeepaliveTimeout } + if kp.Time != infinity { + if err = syscall.SetTCPUserTimeout(rawConn, kp.Timeout); err != nil { + return nil, connectionErrorf(false, err, "transport: failed to set TCP_USER_TIMEOUT: %v", err) + } + } kep := config.KeepalivePolicy if kep.MinTime == 0 { kep.MinTime = defaultKeepalivePolicyMinTime } + done := make(chan struct{}) t := &http2Server{ - ctx: context.Background(), + ctx: setConnection(context.Background(), rawConn), done: done, conn: conn, remoteAddr: conn.RemoteAddr(), localAddr: conn.LocalAddr(), - authInfo: config.AuthInfo, + authInfo: authInfo, framer: framer, readerDone: make(chan struct{}), writerDone: make(chan struct{}), @@ -225,7 +263,7 @@ func newHTTP2Server(conn net.Conn, config *ServerConfig) (_ ServerTransport, err fc: &trInFlow{limit: uint32(icwz)}, state: reachable, activeStreams: make(map[uint32]*Stream), - stats: config.StatsHandler, + stats: config.StatsHandlers, kp: kp, idle: time.Now(), kep: kep, @@ -233,6 +271,10 @@ func newHTTP2Server(conn net.Conn, config *ServerConfig) (_ ServerTransport, err czData: new(channelzData), bufferPool: newBufferPool(), } + t.logger = prefixLoggerForServerTransport(t) + // Add peer information to the http2server context. + t.ctx = peer.NewContext(t.ctx, t.getPeer()) + t.controlBuf = newControlBuffer(t.done) if dynamicWindow { t.bdpEst = &bdpEstimator{ @@ -240,31 +282,39 @@ func newHTTP2Server(conn net.Conn, config *ServerConfig) (_ ServerTransport, err updateFlowControl: t.updateFlowControl, } } - if t.stats != nil { - t.ctx = t.stats.TagConn(t.ctx, &stats.ConnTagInfo{ + for _, sh := range t.stats { + t.ctx = sh.TagConn(t.ctx, &stats.ConnTagInfo{ RemoteAddr: t.remoteAddr, LocalAddr: t.localAddr, }) connBegin := &stats.ConnBegin{} - t.stats.HandleConn(t.ctx, connBegin) + sh.HandleConn(t.ctx, connBegin) } - if channelz.IsOn() { - t.channelzID = channelz.RegisterNormalSocket(t, config.ChannelzParentID, fmt.Sprintf("%s -> %s", t.remoteAddr, t.localAddr)) + t.channelzID, err = channelz.RegisterNormalSocket(t, config.ChannelzParentID, fmt.Sprintf("%s -> %s", t.remoteAddr, t.localAddr)) + if err != nil { + return nil, err } t.connectionID = atomic.AddUint64(&serverConnectionCounter, 1) - t.framer.writer.Flush() defer func() { if err != nil { - t.Close() + t.Close(err) } }() // Check the validity of client preface. preface := make([]byte, len(clientPreface)) if _, err := io.ReadFull(t.conn, preface); err != nil { + // In deployments where a gRPC server runs behind a cloud load balancer + // which performs regular TCP level health checks, the connection is + // closed immediately by the latter. Returning io.EOF here allows the + // grpc server implementation to recognize this scenario and suppress + // logging to reduce spam. + if err == io.EOF { + return nil, io.EOF + } return nil, connectionErrorf(false, err, "transport: http2Server.HandleStreams failed to receive the preface from client: %v", err) } if !bytes.Equal(preface, clientPreface) { @@ -286,100 +336,206 @@ func newHTTP2Server(conn net.Conn, config *ServerConfig) (_ ServerTransport, err t.handleSettings(sf) go func() { - t.loopy = newLoopyWriter(serverSide, t.framer, t.controlBuf, t.bdpEst) + t.loopy = newLoopyWriter(serverSide, t.framer, t.controlBuf, t.bdpEst, t.conn, t.logger) t.loopy.ssGoAwayHandler = t.outgoingGoAwayHandler - if err := t.loopy.run(); err != nil { - if logger.V(logLevel) { - logger.Errorf("transport: loopyWriter.run returning. Err: %v", err) - } - } - t.conn.Close() + t.loopy.run() close(t.writerDone) }() go t.keepalive() return t, nil } -// operateHeader takes action on the decoded headers. -func (t *http2Server) operateHeaders(frame *http2.MetaHeadersFrame, handle func(*Stream), traceCtx func(context.Context, string) context.Context) (fatal bool) { +// operateHeaders takes action on the decoded headers. Returns an error if fatal +// error encountered and transport needs to close, otherwise returns nil. +func (t *http2Server) operateHeaders(frame *http2.MetaHeadersFrame, handle func(*Stream), traceCtx func(context.Context, string) context.Context) error { + // Acquire max stream ID lock for entire duration + t.maxStreamMu.Lock() + defer t.maxStreamMu.Unlock() + streamID := frame.Header().StreamID - state := &decodeState{ - serverSide: true, - } - if err := state.decodeHeader(frame); err != nil { - if se, ok := status.FromError(err); ok { - t.controlBuf.put(&cleanupStream{ - streamID: streamID, - rst: true, - rstCode: statusCodeConvTab[se.Code()], - onWrite: func() {}, - }) - } - return false + + // frame.Truncated is set to true when framer detects that the current header + // list size hits MaxHeaderListSize limit. + if frame.Truncated { + t.controlBuf.put(&cleanupStream{ + streamID: streamID, + rst: true, + rstCode: http2.ErrCodeFrameSize, + onWrite: func() {}, + }) + return nil } + if streamID%2 != 1 || streamID <= t.maxStreamID { + // illegal gRPC stream id. + return fmt.Errorf("received an illegal stream id: %v. headers frame: %+v", streamID, frame) + } + t.maxStreamID = streamID + buf := newRecvBuffer() s := &Stream{ - id: streamID, - st: t, - buf: buf, - fc: &inFlow{limit: uint32(t.initialWindowSize)}, - recvCompress: state.data.encoding, - method: state.data.method, - contentSubtype: state.data.contentSubtype, + id: streamID, + st: t, + buf: buf, + fc: &inFlow{limit: uint32(t.initialWindowSize)}, + } + var ( + // if false, content-type was missing or invalid + isGRPC = false + contentType = "" + mdata = make(metadata.MD, len(frame.Fields)) + httpMethod string + // these are set if an error is encountered while parsing the headers + protocolError bool + headerError *status.Status + + timeoutSet bool + timeout time.Duration + ) + + for _, hf := range frame.Fields { + switch hf.Name { + case "content-type": + contentSubtype, validContentType := grpcutil.ContentSubtype(hf.Value) + if !validContentType { + contentType = hf.Value + break + } + mdata[hf.Name] = append(mdata[hf.Name], hf.Value) + s.contentSubtype = contentSubtype + isGRPC = true + + case "grpc-accept-encoding": + mdata[hf.Name] = append(mdata[hf.Name], hf.Value) + if hf.Value == "" { + continue + } + compressors := hf.Value + if s.clientAdvertisedCompressors != "" { + compressors = s.clientAdvertisedCompressors + "," + compressors + } + s.clientAdvertisedCompressors = compressors + case "grpc-encoding": + s.recvCompress = hf.Value + case ":method": + httpMethod = hf.Value + case ":path": + s.method = hf.Value + case "grpc-timeout": + timeoutSet = true + var err error + if timeout, err = decodeTimeout(hf.Value); err != nil { + headerError = status.Newf(codes.Internal, "malformed grpc-timeout: %v", err) + } + // "Transports must consider requests containing the Connection header + // as malformed." - A41 + case "connection": + if t.logger.V(logLevel) { + t.logger.Infof("Received a HEADERS frame with a :connection header which makes the request malformed, as per the HTTP/2 spec") + } + protocolError = true + default: + if isReservedHeader(hf.Name) && !isWhitelistedHeader(hf.Name) { + break + } + v, err := decodeMetadataHeader(hf.Name, hf.Value) + if err != nil { + headerError = status.Newf(codes.Internal, "malformed binary metadata %q in header %q: %v", hf.Value, hf.Name, err) + t.logger.Warningf("Failed to decode metadata header (%q, %q): %v", hf.Name, hf.Value, err) + break + } + mdata[hf.Name] = append(mdata[hf.Name], v) + } + } + + // "If multiple Host headers or multiple :authority headers are present, the + // request must be rejected with an HTTP status code 400 as required by Host + // validation in RFC 7230 §5.4, gRPC status code INTERNAL, or RST_STREAM + // with HTTP/2 error code PROTOCOL_ERROR." - A41. Since this is a HTTP/2 + // error, this takes precedence over a client not speaking gRPC. + if len(mdata[":authority"]) > 1 || len(mdata["host"]) > 1 { + errMsg := fmt.Sprintf("num values of :authority: %v, num values of host: %v, both must only have 1 value as per HTTP/2 spec", len(mdata[":authority"]), len(mdata["host"])) + if t.logger.V(logLevel) { + t.logger.Infof("Aborting the stream early: %v", errMsg) + } + t.controlBuf.put(&earlyAbortStream{ + httpStatus: http.StatusBadRequest, + streamID: streamID, + contentSubtype: s.contentSubtype, + status: status.New(codes.Internal, errMsg), + rst: !frame.StreamEnded(), + }) + return nil } + + if protocolError { + t.controlBuf.put(&cleanupStream{ + streamID: streamID, + rst: true, + rstCode: http2.ErrCodeProtocol, + onWrite: func() {}, + }) + return nil + } + if !isGRPC { + t.controlBuf.put(&earlyAbortStream{ + httpStatus: http.StatusUnsupportedMediaType, + streamID: streamID, + contentSubtype: s.contentSubtype, + status: status.Newf(codes.InvalidArgument, "invalid gRPC request content-type %q", contentType), + rst: !frame.StreamEnded(), + }) + return nil + } + if headerError != nil { + t.controlBuf.put(&earlyAbortStream{ + httpStatus: http.StatusBadRequest, + streamID: streamID, + contentSubtype: s.contentSubtype, + status: headerError, + rst: !frame.StreamEnded(), + }) + return nil + } + + // "If :authority is missing, Host must be renamed to :authority." - A41 + if len(mdata[":authority"]) == 0 { + // No-op if host isn't present, no eventual :authority header is a valid + // RPC. + if host, ok := mdata["host"]; ok { + mdata[":authority"] = host + delete(mdata, "host") + } + } else { + // "If :authority is present, Host must be discarded" - A41 + delete(mdata, "host") + } + if frame.StreamEnded() { // s is just created by the caller. No lock needed. s.state = streamReadDone } - if state.data.timeoutSet { - s.ctx, s.cancel = context.WithTimeout(t.ctx, state.data.timeout) + if timeoutSet { + s.ctx, s.cancel = context.WithTimeout(t.ctx, timeout) } else { s.ctx, s.cancel = context.WithCancel(t.ctx) } - pr := &peer.Peer{ - Addr: t.remoteAddr, - } - // Attach Auth info if there is any. - if t.authInfo != nil { - pr.AuthInfo = t.authInfo - } - s.ctx = peer.NewContext(s.ctx, pr) + // Attach the received metadata to the context. - if len(state.data.mdata) > 0 { - s.ctx = metadata.NewIncomingContext(s.ctx, state.data.mdata) - } - if state.data.statsTags != nil { - s.ctx = stats.SetIncomingTags(s.ctx, state.data.statsTags) - } - if state.data.statsTrace != nil { - s.ctx = stats.SetIncomingTrace(s.ctx, state.data.statsTrace) - } - if t.inTapHandle != nil { - var err error - info := &tap.Info{ - FullMethodName: state.data.method, + if len(mdata) > 0 { + s.ctx = metadata.NewIncomingContext(s.ctx, mdata) + if statsTags := mdata["grpc-tags-bin"]; len(statsTags) > 0 { + s.ctx = stats.SetIncomingTags(s.ctx, []byte(statsTags[len(statsTags)-1])) } - s.ctx, err = t.inTapHandle(s.ctx, info) - if err != nil { - if logger.V(logLevel) { - logger.Warningf("transport: http2Server.operateHeaders got an error from InTapHandle: %v", err) - } - t.controlBuf.put(&cleanupStream{ - streamID: s.id, - rst: true, - rstCode: http2.ErrCodeRefusedStream, - onWrite: func() {}, - }) - s.cancel() - return false + if statsTrace := mdata["grpc-trace-bin"]; len(statsTrace) > 0 { + s.ctx = stats.SetIncomingTrace(s.ctx, []byte(statsTrace[len(statsTrace)-1])) } } t.mu.Lock() if t.state != reachable { t.mu.Unlock() s.cancel() - return false + return nil } if uint32(len(t.activeStreams)) >= t.maxStreams { t.mu.Unlock() @@ -390,18 +546,45 @@ func (t *http2Server) operateHeaders(frame *http2.MetaHeadersFrame, handle func( onWrite: func() {}, }) s.cancel() - return false + return nil } - if streamID%2 != 1 || streamID <= t.maxStreamID { + if httpMethod != http.MethodPost { t.mu.Unlock() - // illegal gRPC stream id. - if logger.V(logLevel) { - logger.Errorf("transport: http2Server.HandleStreams received an illegal stream id: %v", streamID) + errMsg := fmt.Sprintf("Received a HEADERS frame with :method %q which should be POST", httpMethod) + if t.logger.V(logLevel) { + t.logger.Infof("Aborting the stream early: %v", errMsg) } + t.controlBuf.put(&earlyAbortStream{ + httpStatus: 405, + streamID: streamID, + contentSubtype: s.contentSubtype, + status: status.New(codes.Internal, errMsg), + rst: !frame.StreamEnded(), + }) s.cancel() - return true + return nil + } + if t.inTapHandle != nil { + var err error + if s.ctx, err = t.inTapHandle(s.ctx, &tap.Info{FullMethodName: s.method}); err != nil { + t.mu.Unlock() + if t.logger.V(logLevel) { + t.logger.Infof("Aborting the stream early due to InTapHandle failure: %v", err) + } + stat, ok := status.FromError(err) + if !ok { + stat = status.New(codes.PermissionDenied, err.Error()) + } + t.controlBuf.put(&earlyAbortStream{ + httpStatus: 200, + streamID: s.id, + contentSubtype: s.contentSubtype, + status: stat, + rst: !frame.StreamEnded(), + }) + return nil + } } - t.maxStreamID = streamID t.activeStreams[streamID] = s if len(t.activeStreams) == 1 { t.idle = time.Time{} @@ -415,17 +598,17 @@ func (t *http2Server) operateHeaders(frame *http2.MetaHeadersFrame, handle func( t.adjustWindow(s, uint32(n)) } s.ctx = traceCtx(s.ctx, s.method) - if t.stats != nil { - s.ctx = t.stats.TagRPC(s.ctx, &stats.RPCTagInfo{FullMethodName: s.method}) + for _, sh := range t.stats { + s.ctx = sh.TagRPC(s.ctx, &stats.RPCTagInfo{FullMethodName: s.method}) inHeader := &stats.InHeader{ FullMethod: s.method, RemoteAddr: t.remoteAddr, LocalAddr: t.localAddr, Compression: s.recvCompress, WireLength: int(frame.Header().Length), - Header: metadata.MD(state.data.mdata).Copy(), + Header: mdata.Copy(), } - t.stats.HandleRPC(s.ctx, inHeader) + sh.HandleRPC(s.ctx, inHeader) } s.ctxDone = s.ctx.Done() s.wq = newWriteQuota(defaultWriteQuota, s.ctxDone) @@ -446,7 +629,7 @@ func (t *http2Server) operateHeaders(frame *http2.MetaHeadersFrame, handle func( wq: s.wq, }) handle(s) - return false + return nil } // HandleStreams receives incoming streams using the given handler. This is @@ -460,8 +643,8 @@ func (t *http2Server) HandleStreams(handle func(*Stream), traceCtx func(context. atomic.StoreInt64(&t.lastRead, time.Now().UnixNano()) if err != nil { if se, ok := err.(http2.StreamError); ok { - if logger.V(logLevel) { - logger.Warningf("transport: http2Server.HandleStreams encountered http2.StreamError: %v", se) + if t.logger.V(logLevel) { + t.logger.Warningf("Encountered http2.StreamError: %v", se) } t.mu.Lock() s := t.activeStreams[se.StreamID] @@ -479,19 +662,16 @@ func (t *http2Server) HandleStreams(handle func(*Stream), traceCtx func(context. continue } if err == io.EOF || err == io.ErrUnexpectedEOF { - t.Close() + t.Close(err) return } - if logger.V(logLevel) { - logger.Warningf("transport: http2Server.HandleStreams failed to read frame: %v", err) - } - t.Close() + t.Close(err) return } switch frame := frame.(type) { case *http2.MetaHeadersFrame: - if t.operateHeaders(frame, handle, traceCtx) { - t.Close() + if err := t.operateHeaders(frame, handle, traceCtx); err != nil { + t.Close(err) break } case *http2.DataFrame: @@ -507,8 +687,8 @@ func (t *http2Server) HandleStreams(handle func(*Stream), traceCtx func(context. case *http2.GoAwayFrame: // TODO: Handle GoAway from the client appropriately. default: - if logger.V(logLevel) { - logger.Errorf("transport: http2Server.HandleStreams found unhandled frame type %v.", frame) + if t.logger.V(logLevel) { + t.logger.Infof("Received unsupported frame type %T", frame) } } } @@ -635,7 +815,7 @@ func (t *http2Server) handleData(f *http2.DataFrame) { s.write(recvMsg{buffer: buffer}) } } - if f.Header().Flags.Has(http2.FlagDataEndStream) { + if f.StreamEnded() { // Received the end of stream from the client. s.compareAndSwapState(streamActive, streamReadDone) s.write(recvMsg{err: io.EOF}) @@ -675,7 +855,7 @@ func (t *http2Server) handleSettings(f *http2.SettingsFrame) { } return nil }) - t.controlBuf.executeAndPut(func(interface{}) bool { + t.controlBuf.executeAndPut(func(any) bool { for _, f := range updateFuncs { f() } @@ -692,8 +872,8 @@ const ( func (t *http2Server) handlePing(f *http2.PingFrame) { if f.IsAck() { - if f.Data == goAwayPing.data && t.drainChan != nil { - close(t.drainChan) + if f.Data == goAwayPing.data && t.drainEvent != nil { + t.drainEvent.Fire() return } // Maybe it's a BDP ping. @@ -735,10 +915,7 @@ func (t *http2Server) handlePing(f *http2.PingFrame) { if t.pingStrikes > maxPingStrikes { // Send goaway and close the connection. - if logger.V(logLevel) { - logger.Errorf("transport: Got too many pings from the client, closing the connection.") - } - t.controlBuf.put(&goAway{code: http2.ErrCodeEnhanceYourCalm, debugData: []byte("too_many_pings"), closeConn: true}) + t.controlBuf.put(&goAway{code: http2.ErrCodeEnhanceYourCalm, debugData: []byte("too_many_pings"), closeConn: errors.New("got too many pings from the client")}) } } @@ -762,7 +939,7 @@ func appendHeaderFieldsFromMD(headerFields []hpack.HeaderField, md metadata.MD) return headerFields } -func (t *http2Server) checkForHeaderListSize(it interface{}) bool { +func (t *http2Server) checkForHeaderListSize(it any) bool { if t.maxSendHeaderListSize == nil { return true } @@ -770,8 +947,8 @@ func (t *http2Server) checkForHeaderListSize(it interface{}) bool { var sz int64 for _, f := range hdrFrame.hf { if sz += int64(f.Size()); sz > int64(*t.maxSendHeaderListSize) { - if logger.V(logLevel) { - logger.Errorf("header list size to send violates the maximum size (%d bytes) set by client", *t.maxSendHeaderListSize) + if t.logger.V(logLevel) { + t.logger.Infof("Header list size to send violates the maximum size (%d bytes) set by client", *t.maxSendHeaderListSize) } return false } @@ -779,12 +956,27 @@ func (t *http2Server) checkForHeaderListSize(it interface{}) bool { return true } +func (t *http2Server) streamContextErr(s *Stream) error { + select { + case <-t.done: + return ErrConnClosing + default: + } + return ContextErr(s.ctx.Err()) +} + // WriteHeader sends the header metadata md back to the client. func (t *http2Server) WriteHeader(s *Stream, md metadata.MD) error { - if s.updateHeaderSent() || s.getState() == streamDone { + s.hdrMu.Lock() + defer s.hdrMu.Unlock() + if s.getState() == streamDone { + return t.streamContextErr(s) + } + + if s.updateHeaderSent() { return ErrIllegalHeaderWrite } - s.hdrMu.Lock() + if md.Len() > 0 { if s.header.Len() > 0 { s.header = metadata.Join(s.header, md) @@ -793,10 +985,8 @@ func (t *http2Server) WriteHeader(s *Stream, md metadata.MD) error { } } if err := t.writeHeaderLocked(s); err != nil { - s.hdrMu.Unlock() - return err + return status.Convert(err).Err() } - s.hdrMu.Unlock() return nil } @@ -827,14 +1017,14 @@ func (t *http2Server) writeHeaderLocked(s *Stream) error { t.closeStream(s, true, http2.ErrCodeInternal, false) return ErrHeaderListSizeLimitViolation } - if t.stats != nil { + for _, sh := range t.stats { // Note: Headers are compressed with hpack after this call returns. // No WireLength field is set here. outHeader := &stats.OutHeader{ Header: s.header.Copy(), Compression: s.sendCompress, } - t.stats.HandleRPC(s.Context(), outHeader) + sh.HandleRPC(s.Context(), outHeader) } return nil } @@ -844,17 +1034,19 @@ func (t *http2Server) writeHeaderLocked(s *Stream) error { // TODO(zhaoq): Now it indicates the end of entire stream. Revisit if early // OK is adopted. func (t *http2Server) WriteStatus(s *Stream, st *status.Status) error { + s.hdrMu.Lock() + defer s.hdrMu.Unlock() + if s.getState() == streamDone { return nil } - s.hdrMu.Lock() + // TODO(mmukhi): Benchmark if the performance gets better if count the metadata and other header fields // first and create a slice of that exact size. headerFields := make([]hpack.HeaderField, 0, 2) // grpc-status and grpc-message will be there if none else. if !s.updateHeaderSent() { // No headers have been sent. if len(s.header) > 0 { // Send a separate header frame. if err := t.writeHeaderLocked(s); err != nil { - s.hdrMu.Unlock() return err } } else { // Send a trailer only response. @@ -869,7 +1061,7 @@ func (t *http2Server) WriteStatus(s *Stream, st *status.Status) error { stBytes, err := proto.Marshal(p) if err != nil { // TODO: return error instead, when callers are able to handle it. - logger.Errorf("transport: failed to marshal rpc status: %v, error: %v", p, err) + t.logger.Errorf("Failed to marshal rpc status: %s, error: %v", pretty.ToJSON(p), err) } else { headerFields = append(headerFields, hpack.HeaderField{Name: "grpc-status-details-bin", Value: encodeBinHeader(stBytes)}) } @@ -883,7 +1075,7 @@ func (t *http2Server) WriteStatus(s *Stream, st *status.Status) error { endStream: true, onWrite: t.setResetPingStrikes, } - s.hdrMu.Unlock() + success, err := t.controlBuf.execute(t.checkForHeaderListSize, trailingHeader) if !success { if err != nil { @@ -895,10 +1087,10 @@ func (t *http2Server) WriteStatus(s *Stream, st *status.Status) error { // Send a RST_STREAM after the trailers if the client has not already half-closed. rst := s.getState() == streamActive t.finishStream(s, rst, http2.ErrCodeNo, trailingHeader, true) - if t.stats != nil { + for _, sh := range t.stats { // Note: The trailer fields are compressed with hpack after this call returns. // No WireLength field is set here. - t.stats.HandleRPC(s.Context(), &stats.OutTrailer{ + sh.HandleRPC(s.Context(), &stats.OutTrailer{ Trailer: s.trailer.Copy(), }) } @@ -910,23 +1102,12 @@ func (t *http2Server) WriteStatus(s *Stream, st *status.Status) error { func (t *http2Server) Write(s *Stream, hdr []byte, data []byte, opts *Options) error { if !s.isHeaderSent() { // Headers haven't been written yet. if err := t.WriteHeader(s, nil); err != nil { - if _, ok := err.(ConnectionError); ok { - return err - } - // TODO(mmukhi, dfawley): Make sure this is the right code to return. - return status.Errorf(codes.Internal, "transport: %v", err) + return err } } else { // Writing headers checks for this condition. if s.getState() == streamDone { - // TODO(mmukhi, dfawley): Should the server write also return io.EOF? - s.cancel() - select { - case <-t.done: - return ErrConnClosing - default: - } - return ContextErr(s.ctx.Err()) + return t.streamContextErr(s) } } df := &dataFrame{ @@ -936,12 +1117,7 @@ func (t *http2Server) Write(s *Stream, hdr []byte, data []byte, opts *Options) e onEachWrite: t.setResetPingStrikes, } if err := s.wq.get(int32(len(hdr) + len(data))); err != nil { - select { - case <-t.done: - return ErrConnClosing - default: - } - return ContextErr(s.ctx.Err()) + return t.streamContextErr(s) } return t.controlBuf.put(df) } @@ -990,20 +1166,20 @@ func (t *http2Server) keepalive() { if val <= 0 { // The connection has been idle for a duration of keepalive.MaxConnectionIdle or more. // Gracefully close the connection. - t.drain(http2.ErrCodeNo, []byte{}) + t.Drain("max_idle") return } idleTimer.Reset(val) case <-ageTimer.C: - t.drain(http2.ErrCodeNo, []byte{}) + t.Drain("max_age") ageTimer.Reset(t.kp.MaxConnectionAgeGrace) select { case <-ageTimer.C: // Close the connection after grace period. - if logger.V(logLevel) { - logger.Infof("transport: closing server transport due to maximum connection age.") + if t.logger.V(logLevel) { + t.logger.Infof("Closing server transport due to maximum connection age") } - t.Close() + t.controlBuf.put(closeConnection{}) case <-t.done: } return @@ -1019,10 +1195,7 @@ func (t *http2Server) keepalive() { continue } if outstandingPing && kpTimeoutLeft <= 0 { - if logger.V(logLevel) { - logger.Infof("transport: closing server transport due to idleness.") - } - t.Close() + t.Close(fmt.Errorf("keepalive ping not acked within timeout %s", t.kp.Time)) return } if !outstandingPing { @@ -1049,11 +1222,14 @@ func (t *http2Server) keepalive() { // Close starts shutting down the http2Server transport. // TODO(zhaoq): Now the destruction is not blocked on any pending streams. This // could cause some resource issue. Revisit this later. -func (t *http2Server) Close() error { +func (t *http2Server) Close(err error) { t.mu.Lock() if t.state == closing { t.mu.Unlock() - return errors.New("transport: Close() was already called") + return + } + if t.logger.V(logLevel) { + t.logger.Infof("Closing: %v", err) } t.state = closing streams := t.activeStreams @@ -1061,27 +1237,22 @@ func (t *http2Server) Close() error { t.mu.Unlock() t.controlBuf.finish() close(t.done) - err := t.conn.Close() - if channelz.IsOn() { - channelz.RemoveEntry(t.channelzID) + if err := t.conn.Close(); err != nil && t.logger.V(logLevel) { + t.logger.Infof("Error closing underlying net.Conn during Close: %v", err) } + channelz.RemoveEntry(t.channelzID) // Cancel all active streams. for _, s := range streams { s.cancel() } - if t.stats != nil { + for _, sh := range t.stats { connEnd := &stats.ConnEnd{} - t.stats.HandleConn(t.ctx, connEnd) + sh.HandleConn(t.ctx, connEnd) } - return err } // deleteStream deletes the stream s from transport's active streams. func (t *http2Server) deleteStream(s *Stream, eosReceived bool) { - // In case stream sending and receiving are invoked in separate - // goroutines (e.g., bi-directional streaming), cancel needs to be - // called to interrupt the potential blocking on other goroutines. - s.cancel() t.mu.Lock() if _, ok := t.activeStreams[s.id]; ok { @@ -1103,6 +1274,11 @@ func (t *http2Server) deleteStream(s *Stream, eosReceived bool) { // finishStream closes the stream and puts the trailing headerFrame into controlbuf. func (t *http2Server) finishStream(s *Stream, rst bool, rstCode http2.ErrCode, hdr *headerFrame, eosReceived bool) { + // In case stream sending and receiving are invoked in separate + // goroutines (e.g., bi-directional streaming), cancel needs to be + // called to interrupt the potential blocking on other goroutines. + s.cancel() + oldState := s.swapState(streamDone) if oldState == streamDone { // If the stream was already done, return. @@ -1122,6 +1298,11 @@ func (t *http2Server) finishStream(s *Stream, rst bool, rstCode http2.ErrCode, h // closeStream clears the footprint of a stream when the stream is not needed any more. func (t *http2Server) closeStream(s *Stream, rst bool, rstCode http2.ErrCode, eosReceived bool) { + // In case stream sending and receiving are invoked in separate + // goroutines (e.g., bi-directional streaming), cancel needs to be + // called to interrupt the potential blocking on other goroutines. + s.cancel() + s.swapState(streamDone) t.deleteStream(s, eosReceived) @@ -1137,18 +1318,14 @@ func (t *http2Server) RemoteAddr() net.Addr { return t.remoteAddr } -func (t *http2Server) Drain() { - t.drain(http2.ErrCodeNo, []byte{}) -} - -func (t *http2Server) drain(code http2.ErrCode, debugData []byte) { +func (t *http2Server) Drain(debugData string) { t.mu.Lock() defer t.mu.Unlock() - if t.drainChan != nil { + if t.drainEvent != nil { return } - t.drainChan = make(chan struct{}) - t.controlBuf.put(&goAway{code: code, debugData: debugData, headsUp: true}) + t.drainEvent = grpcsync.NewEvent() + t.controlBuf.put(&goAway{code: http2.ErrCodeNo, debugData: []byte(debugData), headsUp: true}) } var goAwayPing = &ping{data: [8]byte{1, 6, 1, 8, 0, 3, 3, 9}} @@ -1156,39 +1333,41 @@ var goAwayPing = &ping{data: [8]byte{1, 6, 1, 8, 0, 3, 3, 9}} // Handles outgoing GoAway and returns true if loopy needs to put itself // in draining mode. func (t *http2Server) outgoingGoAwayHandler(g *goAway) (bool, error) { + t.maxStreamMu.Lock() t.mu.Lock() if t.state == closing { // TODO(mmukhi): This seems unnecessary. t.mu.Unlock() + t.maxStreamMu.Unlock() // The transport is closing. return false, ErrConnClosing } - sid := t.maxStreamID if !g.headsUp { // Stop accepting more streams now. t.state = draining + sid := t.maxStreamID + retErr := g.closeConn if len(t.activeStreams) == 0 { - g.closeConn = true + retErr = errors.New("second GOAWAY written and no active streams left to process") } t.mu.Unlock() + t.maxStreamMu.Unlock() if err := t.framer.fr.WriteGoAway(sid, g.code, g.debugData); err != nil { return false, err } - if g.closeConn { - // Abruptly close the connection following the GoAway (via - // loopywriter). But flush out what's inside the buffer first. - t.framer.writer.Flush() - return false, fmt.Errorf("transport: Connection closing") + if retErr != nil { + return false, retErr } return true, nil } t.mu.Unlock() + t.maxStreamMu.Unlock() // For a graceful close, send out a GoAway with stream ID of MaxUInt32, // Follow that with a ping and wait for the ack to come back or a timer // to expire. During this time accept new streams since they might have // originated before the GoAway reaches the client. // After getting the ack or timer expiration send out another GoAway this // time with an ID of the max stream server intends to process. - if err := t.framer.fr.WriteGoAway(math.MaxUint32, http2.ErrCodeNo, []byte{}); err != nil { + if err := t.framer.fr.WriteGoAway(math.MaxUint32, http2.ErrCodeNo, g.debugData); err != nil { return false, err } if err := t.framer.fr.WritePing(false, goAwayPing.data); err != nil { @@ -1198,7 +1377,7 @@ func (t *http2Server) outgoingGoAwayHandler(g *goAway) (bool, error) { timer := time.NewTimer(time.Minute) defer timer.Stop() select { - case <-t.drainChan: + case <-t.drainEvent.Done(): case <-timer.C: case <-t.done: return @@ -1257,6 +1436,13 @@ func (t *http2Server) getOutFlowWindow() int64 { } } +func (t *http2Server) getPeer() *peer.Peer { + return &peer.Peer{ + Addr: t.remoteAddr, + AuthInfo: t.authInfo, // Can be nil + } +} + func getJitter(v time.Duration) time.Duration { if v == infinity { return 0 @@ -1266,3 +1452,18 @@ func getJitter(v time.Duration) time.Duration { j := grpcrand.Int63n(2*r) - r return time.Duration(j) } + +type connectionKey struct{} + +// GetConnection gets the connection from the context. +func GetConnection(ctx context.Context) net.Conn { + conn, _ := ctx.Value(connectionKey{}).(net.Conn) + return conn +} + +// SetConnection adds the connection to the context to be able to get +// information about the destination ip and port for an incoming RPC. This also +// allows any unary or streaming interceptors to see the connection. +func setConnection(ctx context.Context, conn net.Conn) context.Context { + return context.WithValue(ctx, connectionKey{}, conn) +} diff --git a/internal/transport/http_util.go b/internal/transport/http_util.go index 5e1e7a65da2b..1958140082b3 100644 --- a/internal/transport/http_util.go +++ b/internal/transport/http_util.go @@ -20,15 +20,17 @@ package transport import ( "bufio" - "bytes" "encoding/base64" + "errors" "fmt" "io" "math" "net" "net/http" + "net/url" "strconv" "strings" + "sync" "time" "unicode/utf8" @@ -37,22 +39,14 @@ import ( "golang.org/x/net/http2/hpack" spb "google.golang.org/genproto/googleapis/rpc/status" "google.golang.org/grpc/codes" - "google.golang.org/grpc/grpclog" - "google.golang.org/grpc/internal/grpcutil" "google.golang.org/grpc/status" ) const ( // http2MaxFrameLen specifies the max length of a HTTP2 frame. http2MaxFrameLen = 16384 // 16KB frame - // http://http2.github.io/http2-spec/#SettingValues + // https://httpwg.org/specs/rfc7540.html#SettingValues http2InitHeaderTableSize = 4096 - // baseContentType is the base content-type for gRPC. This is a valid - // content-type on it's own, but can also include a content-subtype such as - // "proto" as a suffix after "+" or ";". See - // https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests - // for more details. - ) var ( @@ -73,13 +67,6 @@ var ( http2.ErrCodeInadequateSecurity: codes.PermissionDenied, http2.ErrCodeHTTP11Required: codes.Internal, } - statusCodeConvTab = map[codes.Code]http2.ErrCode{ - codes.Internal: http2.ErrCodeInternal, - codes.Canceled: http2.ErrCodeCancel, - codes.Unavailable: http2.ErrCodeRefusedStream, - codes.ResourceExhausted: http2.ErrCodeEnhanceYourCalm, - codes.PermissionDenied: http2.ErrCodeInadequateSecurity, - } // HTTPStatusConvTab is the HTTP status code to gRPC error code conversion table. HTTPStatusConvTab = map[int]codes.Code{ // 400 Bad Request - INTERNAL. @@ -99,55 +86,8 @@ var ( // 504 Gateway timeout - UNAVAILABLE. http.StatusGatewayTimeout: codes.Unavailable, } - logger = grpclog.Component("transport") ) -type parsedHeaderData struct { - encoding string - // statusGen caches the stream status received from the trailer the server - // sent. Client side only. Do not access directly. After all trailers are - // parsed, use the status method to retrieve the status. - statusGen *status.Status - // rawStatusCode and rawStatusMsg are set from the raw trailer fields and are not - // intended for direct access outside of parsing. - rawStatusCode *int - rawStatusMsg string - httpStatus *int - // Server side only fields. - timeoutSet bool - timeout time.Duration - method string - // key-value metadata map from the peer. - mdata map[string][]string - statsTags []byte - statsTrace []byte - contentSubtype string - - // isGRPC field indicates whether the peer is speaking gRPC (otherwise HTTP). - // - // We are in gRPC mode (peer speaking gRPC) if: - // * We are client side and have already received a HEADER frame that indicates gRPC peer. - // * The header contains valid a content-type, i.e. a string starts with "application/grpc" - // And we should handle error specific to gRPC. - // - // Otherwise (i.e. a content-type string starts without "application/grpc", or does not exist), we - // are in HTTP fallback mode, and should handle error specific to HTTP. - isGRPC bool - grpcErr error - httpErr error - contentTypeErr string -} - -// decodeState configures decoding criteria and records the decoded data. -type decodeState struct { - // whether decoding on server side or not - serverSide bool - - // Records the states during HPACK decoding. It will be filled with info parsed from HTTP HEADERS - // frame once decodeHeader function has been invoked and returned. - data parsedHeaderData -} - // isReservedHeader checks whether hdr belongs to HTTP2 headers // reserved by gRPC protocol. Any other headers are classified as the // user-specified metadata. @@ -185,14 +125,6 @@ func isWhitelistedHeader(hdr string) bool { } } -func (d *decodeState) status() *status.Status { - if d.data.statusGen == nil { - // No status-details were provided; generate status using code/msg. - d.data.statusGen = status.New(codes.Code(int32(*(d.data.rawStatusCode))), d.data.rawStatusMsg) - } - return d.data.statusGen -} - const binHdrSuffix = "-bin" func encodeBinHeader(v []byte) string { @@ -222,166 +154,16 @@ func decodeMetadataHeader(k, v string) (string, error) { return v, nil } -func (d *decodeState) decodeHeader(frame *http2.MetaHeadersFrame) error { - // frame.Truncated is set to true when framer detects that the current header - // list size hits MaxHeaderListSize limit. - if frame.Truncated { - return status.Error(codes.Internal, "peer header list size exceeded limit") - } - - for _, hf := range frame.Fields { - d.processHeaderField(hf) - } - - if d.data.isGRPC { - if d.data.grpcErr != nil { - return d.data.grpcErr - } - if d.serverSide { - return nil - } - if d.data.rawStatusCode == nil && d.data.statusGen == nil { - // gRPC status doesn't exist. - // Set rawStatusCode to be unknown and return nil error. - // So that, if the stream has ended this Unknown status - // will be propagated to the user. - // Otherwise, it will be ignored. In which case, status from - // a later trailer, that has StreamEnded flag set, is propagated. - code := int(codes.Unknown) - d.data.rawStatusCode = &code - } - return nil - } - - // HTTP fallback mode - if d.data.httpErr != nil { - return d.data.httpErr - } - - var ( - code = codes.Internal // when header does not include HTTP status, return INTERNAL - ok bool - ) - - if d.data.httpStatus != nil { - code, ok = HTTPStatusConvTab[*(d.data.httpStatus)] - if !ok { - code = codes.Unknown - } - } - - return status.Error(code, d.constructHTTPErrMsg()) -} - -// constructErrMsg constructs error message to be returned in HTTP fallback mode. -// Format: HTTP status code and its corresponding message + content-type error message. -func (d *decodeState) constructHTTPErrMsg() string { - var errMsgs []string - - if d.data.httpStatus == nil { - errMsgs = append(errMsgs, "malformed header: missing HTTP status") - } else { - errMsgs = append(errMsgs, fmt.Sprintf("%s: HTTP status code %d", http.StatusText(*(d.data.httpStatus)), *d.data.httpStatus)) +func decodeGRPCStatusDetails(rawDetails string) (*status.Status, error) { + v, err := decodeBinHeader(rawDetails) + if err != nil { + return nil, err } - - if d.data.contentTypeErr == "" { - errMsgs = append(errMsgs, "transport: missing content-type field") - } else { - errMsgs = append(errMsgs, d.data.contentTypeErr) - } - - return strings.Join(errMsgs, "; ") -} - -func (d *decodeState) addMetadata(k, v string) { - if d.data.mdata == nil { - d.data.mdata = make(map[string][]string) - } - d.data.mdata[k] = append(d.data.mdata[k], v) -} - -func (d *decodeState) processHeaderField(f hpack.HeaderField) { - switch f.Name { - case "content-type": - contentSubtype, validContentType := grpcutil.ContentSubtype(f.Value) - if !validContentType { - d.data.contentTypeErr = fmt.Sprintf("transport: received the unexpected content-type %q", f.Value) - return - } - d.data.contentSubtype = contentSubtype - // TODO: do we want to propagate the whole content-type in the metadata, - // or come up with a way to just propagate the content-subtype if it was set? - // ie {"content-type": "application/grpc+proto"} or {"content-subtype": "proto"} - // in the metadata? - d.addMetadata(f.Name, f.Value) - d.data.isGRPC = true - case "grpc-encoding": - d.data.encoding = f.Value - case "grpc-status": - code, err := strconv.Atoi(f.Value) - if err != nil { - d.data.grpcErr = status.Errorf(codes.Internal, "transport: malformed grpc-status: %v", err) - return - } - d.data.rawStatusCode = &code - case "grpc-message": - d.data.rawStatusMsg = decodeGrpcMessage(f.Value) - case "grpc-status-details-bin": - v, err := decodeBinHeader(f.Value) - if err != nil { - d.data.grpcErr = status.Errorf(codes.Internal, "transport: malformed grpc-status-details-bin: %v", err) - return - } - s := &spb.Status{} - if err := proto.Unmarshal(v, s); err != nil { - d.data.grpcErr = status.Errorf(codes.Internal, "transport: malformed grpc-status-details-bin: %v", err) - return - } - d.data.statusGen = status.FromProto(s) - case "grpc-timeout": - d.data.timeoutSet = true - var err error - if d.data.timeout, err = decodeTimeout(f.Value); err != nil { - d.data.grpcErr = status.Errorf(codes.Internal, "transport: malformed time-out: %v", err) - } - case ":path": - d.data.method = f.Value - case ":status": - code, err := strconv.Atoi(f.Value) - if err != nil { - d.data.httpErr = status.Errorf(codes.Internal, "transport: malformed http-status: %v", err) - return - } - d.data.httpStatus = &code - case "grpc-tags-bin": - v, err := decodeBinHeader(f.Value) - if err != nil { - d.data.grpcErr = status.Errorf(codes.Internal, "transport: malformed grpc-tags-bin: %v", err) - return - } - d.data.statsTags = v - d.addMetadata(f.Name, string(v)) - case "grpc-trace-bin": - v, err := decodeBinHeader(f.Value) - if err != nil { - d.data.grpcErr = status.Errorf(codes.Internal, "transport: malformed grpc-trace-bin: %v", err) - return - } - d.data.statsTrace = v - d.addMetadata(f.Name, string(v)) - default: - if isReservedHeader(f.Name) && !isWhitelistedHeader(f.Name) { - break - } - v, err := decodeMetadataHeader(f.Name, f.Value) - if err != nil { - if logger.V(logLevel) { - logger.Errorf("Failed to decode metadata header (%q, %q): %v", f.Name, f.Value, err) - } - return - } - d.addMetadata(f.Name, v) + st := &spb.Status{} + if err = proto.Unmarshal(v, st); err != nil { + return nil, err } + return status.FromProto(st), nil } type timeoutUnit uint8 @@ -468,13 +250,13 @@ func encodeGrpcMessage(msg string) string { } func encodeGrpcMessageUnchecked(msg string) string { - var buf bytes.Buffer + var sb strings.Builder for len(msg) > 0 { r, size := utf8.DecodeRuneInString(msg) for _, b := range []byte(string(r)) { if size > 1 { // If size > 1, r is not ascii. Always do percent encoding. - buf.WriteString(fmt.Sprintf("%%%02X", b)) + fmt.Fprintf(&sb, "%%%02X", b) continue } @@ -483,14 +265,14 @@ func encodeGrpcMessageUnchecked(msg string) string { // // fmt.Sprintf("%%%02X", utf8.RuneError) gives "%FFFD". if b >= spaceByte && b <= tildeByte && b != percentByte { - buf.WriteByte(b) + sb.WriteByte(b) } else { - buf.WriteString(fmt.Sprintf("%%%02X", b)) + fmt.Fprintf(&sb, "%%%02X", b) } } msg = msg[size:] } - return buf.String() + return sb.String() } // decodeGrpcMessage decodes the msg encoded by encodeGrpcMessage. @@ -508,41 +290,45 @@ func decodeGrpcMessage(msg string) string { } func decodeGrpcMessageUnchecked(msg string) string { - var buf bytes.Buffer + var sb strings.Builder lenMsg := len(msg) for i := 0; i < lenMsg; i++ { c := msg[i] if c == percentByte && i+2 < lenMsg { parsed, err := strconv.ParseUint(msg[i+1:i+3], 16, 8) if err != nil { - buf.WriteByte(c) + sb.WriteByte(c) } else { - buf.WriteByte(byte(parsed)) + sb.WriteByte(byte(parsed)) i += 2 } } else { - buf.WriteByte(c) + sb.WriteByte(c) } } - return buf.String() + return sb.String() } type bufWriter struct { + pool *sync.Pool buf []byte offset int batchSize int conn net.Conn err error - - onFlush func() } -func newBufWriter(conn net.Conn, batchSize int) *bufWriter { - return &bufWriter{ - buf: make([]byte, batchSize*2), +func newBufWriter(conn net.Conn, batchSize int, pool *sync.Pool) *bufWriter { + w := &bufWriter{ batchSize: batchSize, conn: conn, + pool: pool, } + // this indicates that we should use non shared buf + if pool == nil { + w.buf = make([]byte, batchSize) + } + return w } func (w *bufWriter) Write(b []byte) (n int, err error) { @@ -550,7 +336,12 @@ func (w *bufWriter) Write(b []byte) (n int, err error) { return 0, w.err } if w.batchSize == 0 { // Buffer has been disabled. - return w.conn.Write(b) + n, err = w.conn.Write(b) + return n, toIOError(err) + } + if w.buf == nil { + b := w.pool.Get().(*[]byte) + w.buf = *b } for len(b) > 0 { nn := copy(w.buf[w.offset:], b) @@ -558,33 +349,64 @@ func (w *bufWriter) Write(b []byte) (n int, err error) { w.offset += nn n += nn if w.offset >= w.batchSize { - err = w.Flush() + err = w.flushKeepBuffer() } } return n, err } func (w *bufWriter) Flush() error { + err := w.flushKeepBuffer() + // Only release the buffer if we are in a "shared" mode + if w.buf != nil && w.pool != nil { + b := w.buf + w.pool.Put(&b) + w.buf = nil + } + return err +} + +func (w *bufWriter) flushKeepBuffer() error { if w.err != nil { return w.err } if w.offset == 0 { return nil } - if w.onFlush != nil { - w.onFlush() - } _, w.err = w.conn.Write(w.buf[:w.offset]) + w.err = toIOError(w.err) w.offset = 0 return w.err } +type ioError struct { + error +} + +func (i ioError) Unwrap() error { + return i.error +} + +func isIOError(err error) bool { + return errors.As(err, &ioError{}) +} + +func toIOError(err error) error { + if err == nil { + return nil + } + return ioError{error: err} +} + type framer struct { writer *bufWriter fr *http2.Framer } -func newFramer(conn net.Conn, writeBufferSize, readBufferSize int, maxHeaderListSize uint32) *framer { +var writeBufferPoolMap map[int]*sync.Pool = make(map[int]*sync.Pool) +var writeBufferMutex sync.Mutex + +func newFramer(conn net.Conn, writeBufferSize, readBufferSize int, sharedWriteBuffer bool, maxHeaderListSize uint32) *framer { if writeBufferSize < 0 { writeBufferSize = 0 } @@ -592,7 +414,11 @@ func newFramer(conn net.Conn, writeBufferSize, readBufferSize int, maxHeaderList if readBufferSize > 0 { r = bufio.NewReaderSize(r, readBufferSize) } - w := newBufWriter(conn, writeBufferSize) + var pool *sync.Pool + if sharedWriteBuffer { + pool = getWriteBufferPool(writeBufferSize) + } + w := newBufWriter(conn, writeBufferSize, pool) f := &framer{ writer: w, fr: http2.NewFramer(w, r), @@ -605,3 +431,49 @@ func newFramer(conn net.Conn, writeBufferSize, readBufferSize int, maxHeaderList f.fr.ReadMetaHeaders = hpack.NewDecoder(http2InitHeaderTableSize, nil) return f } + +func getWriteBufferPool(writeBufferSize int) *sync.Pool { + writeBufferMutex.Lock() + defer writeBufferMutex.Unlock() + size := writeBufferSize * 2 + pool, ok := writeBufferPoolMap[size] + if ok { + return pool + } + pool = &sync.Pool{ + New: func() any { + b := make([]byte, size) + return &b + }, + } + writeBufferPoolMap[size] = pool + return pool +} + +// parseDialTarget returns the network and address to pass to dialer. +func parseDialTarget(target string) (string, string) { + net := "tcp" + m1 := strings.Index(target, ":") + m2 := strings.Index(target, ":/") + // handle unix:addr which will fail with url.Parse + if m1 >= 0 && m2 < 0 { + if n := target[0:m1]; n == "unix" { + return n, target[m1+1:] + } + } + if m2 >= 0 { + t, err := url.Parse(target) + if err != nil { + return net, target + } + scheme := t.Scheme + addr := t.Path + if scheme == "unix" { + if addr == "" { + addr = t.Host + } + return scheme, addr + } + } + return net, target +} diff --git a/internal/transport/http_util_test.go b/internal/transport/http_util_test.go index a3616f7389f5..cc7807670b62 100644 --- a/internal/transport/http_util_test.go +++ b/internal/transport/http_util_test.go @@ -185,3 +185,56 @@ func (s) TestDecodeMetadataHeader(t *testing.T) { } } } + +func (s) TestParseDialTarget(t *testing.T) { + for _, test := range []struct { + target, wantNet, wantAddr string + }{ + {"unix:a", "unix", "a"}, + {"unix:a/b/c", "unix", "a/b/c"}, + {"unix:/a", "unix", "/a"}, + {"unix:/a/b/c", "unix", "/a/b/c"}, + {"unix://a", "unix", "a"}, + {"unix://a/b/c", "unix", "/b/c"}, + {"unix:///a", "unix", "/a"}, + {"unix:///a/b/c", "unix", "/a/b/c"}, + {"unix:etcd:0", "unix", "etcd:0"}, + {"unix:///tmp/unix-3", "unix", "/tmp/unix-3"}, + {"unix://domain", "unix", "domain"}, + {"unix://etcd:0", "unix", "etcd:0"}, + {"unix:///etcd:0", "unix", "/etcd:0"}, + {"passthrough://unix://domain", "tcp", "passthrough://unix://domain"}, + {"https://google.com:443", "tcp", "https://google.com:443"}, + {"dns:///google.com", "tcp", "dns:///google.com"}, + {"/unix/socket/address", "tcp", "/unix/socket/address"}, + } { + gotNet, gotAddr := parseDialTarget(test.target) + if gotNet != test.wantNet || gotAddr != test.wantAddr { + t.Errorf("parseDialTarget(%q) = %s, %s want %s, %s", test.target, gotNet, gotAddr, test.wantNet, test.wantAddr) + } + } +} + +func BenchmarkDecodeGrpcMessage(b *testing.B) { + input := "Hello, %E4%B8%96%E7%95%8C" + want := "Hello, 世界" + b.ReportAllocs() + for i := 0; i < b.N; i++ { + got := decodeGrpcMessage(input) + if got != want { + b.Fatalf("decodeGrpcMessage(%q) = %s, want %s", input, got, want) + } + } +} + +func BenchmarkEncodeGrpcMessage(b *testing.B) { + input := "Hello, 世界" + want := "Hello, %E4%B8%96%E7%95%8C" + b.ReportAllocs() + for i := 0; i < b.N; i++ { + got := encodeGrpcMessage(input) + if got != want { + b.Fatalf("encodeGrpcMessage(%q) = %s, want %s", input, got, want) + } + } +} diff --git a/internal/transport/keepalive_test.go b/internal/transport/keepalive_test.go index 37b77bb539c4..8144277fb6c1 100644 --- a/internal/transport/keepalive_test.go +++ b/internal/transport/keepalive_test.go @@ -24,121 +24,136 @@ package transport import ( "context" + "crypto/tls" + "crypto/x509" + "fmt" "io" "net" + "os" + "strings" "testing" "time" "golang.org/x/net/http2" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/internal/channelz" + "google.golang.org/grpc/internal/grpctest" "google.golang.org/grpc/internal/syscall" "google.golang.org/grpc/keepalive" + "google.golang.org/grpc/testdata" ) +const defaultTestTimeout = 10 * time.Second + // TestMaxConnectionIdle tests that a server will send GoAway to an idle // client. An idle client is one who doesn't make any RPC calls for a duration // of MaxConnectionIdle time. func (s) TestMaxConnectionIdle(t *testing.T) { serverConfig := &ServerConfig{ KeepaliveParams: keepalive.ServerParameters{ - MaxConnectionIdle: 2 * time.Second, + MaxConnectionIdle: 30 * time.Millisecond, }, } server, client, cancel := setUpWithOptions(t, 0, serverConfig, suspended, ConnectOptions{}) defer func() { - client.Close() + client.Close(fmt.Errorf("closed manually by test")) server.stop() cancel() }() - stream, err := client.NewStream(context.Background(), &CallHdr{}) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + stream, err := client.NewStream(ctx, &CallHdr{}) if err != nil { t.Fatalf("client.NewStream() failed: %v", err) } client.CloseStream(stream, io.EOF) - // Wait for the server's MaxConnectionIdle timeout to kick in, and for it - // to send a GoAway. - timeout := time.NewTimer(time.Second * 4) + // Verify the server sends a GoAway to client after MaxConnectionIdle timeout + // kicks in. select { - case <-client.Error(): - if !timeout.Stop() { - <-timeout.C - } - if reason := client.GetGoAwayReason(); reason != GoAwayNoReason { + case <-ctx.Done(): + t.Fatalf("context expired before receiving GoAway from the server.") + case <-client.GoAway(): + reason, debugMsg := client.GetGoAwayReason() + if reason != GoAwayNoReason { t.Fatalf("GoAwayReason is %v, want %v", reason, GoAwayNoReason) } - case <-timeout.C: - t.Fatalf("MaxConnectionIdle timeout expired, expected a GoAway from the server.") + if !strings.Contains(debugMsg, "max_idle") { + t.Fatalf("GoAwayDebugMessage is %v, want %v", debugMsg, "max_idle") + } } } -// TestMaxConenctionIdleBusyClient tests that a server will not send GoAway to +// TestMaxConnectionIdleBusyClient tests that a server will not send GoAway to // a busy client. func (s) TestMaxConnectionIdleBusyClient(t *testing.T) { serverConfig := &ServerConfig{ KeepaliveParams: keepalive.ServerParameters{ - MaxConnectionIdle: 2 * time.Second, + MaxConnectionIdle: 100 * time.Millisecond, }, } server, client, cancel := setUpWithOptions(t, 0, serverConfig, suspended, ConnectOptions{}) defer func() { - client.Close() + client.Close(fmt.Errorf("closed manually by test")) server.stop() cancel() }() - _, err := client.NewStream(context.Background(), &CallHdr{}) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + _, err := client.NewStream(ctx, &CallHdr{}) if err != nil { t.Fatalf("client.NewStream() failed: %v", err) } - // Wait for double the MaxConnectionIdle time to make sure the server does - // not send a GoAway, as the client has an open stream. - timeout := time.NewTimer(time.Second * 4) + // Verify the server does not send a GoAway to client even after MaxConnectionIdle + // timeout kicks in. + ctx, cancel = context.WithTimeout(context.Background(), time.Second) + defer cancel() select { case <-client.GoAway(): - if !timeout.Stop() { - <-timeout.C - } - t.Fatalf("A non-idle client received a GoAway.") - case <-timeout.C: + t.Fatalf("A busy client received a GoAway.") + case <-ctx.Done(): } } // TestMaxConnectionAge tests that a server will send GoAway after a duration // of MaxConnectionAge. func (s) TestMaxConnectionAge(t *testing.T) { + maxConnAge := 100 * time.Millisecond serverConfig := &ServerConfig{ KeepaliveParams: keepalive.ServerParameters{ - MaxConnectionAge: 1 * time.Second, - MaxConnectionAgeGrace: 1 * time.Second, + MaxConnectionAge: maxConnAge, + MaxConnectionAgeGrace: 10 * time.Millisecond, }, } server, client, cancel := setUpWithOptions(t, 0, serverConfig, suspended, ConnectOptions{}) defer func() { - client.Close() + client.Close(fmt.Errorf("closed manually by test")) server.stop() cancel() }() - _, err := client.NewStream(context.Background(), &CallHdr{}) - if err != nil { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := client.NewStream(ctx, &CallHdr{}); err != nil { t.Fatalf("client.NewStream() failed: %v", err) } - // Wait for the server's MaxConnectionAge timeout to kick in, and for it - // to send a GoAway. - timeout := time.NewTimer(4 * time.Second) + // Verify the server sends a GoAway to client even after client remains idle + // for more than MaxConnectionIdle time. select { - case <-client.Error(): - if !timeout.Stop() { - <-timeout.C - } - if reason := client.GetGoAwayReason(); reason != GoAwayNoReason { + case <-client.GoAway(): + reason, debugMsg := client.GetGoAwayReason() + if reason != GoAwayNoReason { t.Fatalf("GoAwayReason is %v, want %v", reason, GoAwayNoReason) } - case <-timeout.C: - t.Fatalf("MaxConnectionAge timeout expired, expected a GoAway from the server.") + if !strings.Contains(debugMsg, "max_age") { + t.Fatalf("GoAwayDebugMessage is %v, want %v", debugMsg, "max_age") + } + case <-ctx.Done(): + t.Fatalf("timed out before getting a GoAway from the server.") } } @@ -155,13 +170,13 @@ const ( func (s) TestKeepaliveServerClosesUnresponsiveClient(t *testing.T) { serverConfig := &ServerConfig{ KeepaliveParams: keepalive.ServerParameters{ - Time: 1 * time.Second, - Timeout: 1 * time.Second, + Time: 100 * time.Millisecond, + Timeout: 10 * time.Millisecond, }, } server, client, cancel := setUpWithOptions(t, 0, serverConfig, suspended, ConnectOptions{}) defer func() { - client.Close() + client.Close(fmt.Errorf("closed manually by test")) server.stop() cancel() }() @@ -176,7 +191,7 @@ func (s) TestKeepaliveServerClosesUnresponsiveClient(t *testing.T) { if n, err := conn.Write(clientPreface); err != nil || n != len(clientPreface) { t.Fatalf("conn.Write(clientPreface) failed: n=%v, err=%v", n, err) } - framer := newFramer(conn, defaultWriteBufSize, defaultReadBufSize, 0) + framer := newFramer(conn, defaultWriteBufSize, defaultReadBufSize, false, 0) if err := framer.fr.WriteSettings(http2.Setting{}); err != nil { t.Fatal("framer.WriteSettings(http2.Setting{}) failed:", err) } @@ -184,7 +199,7 @@ func (s) TestKeepaliveServerClosesUnresponsiveClient(t *testing.T) { // We read from the net.Conn till we get an error, which is expected when // the server closes the connection as part of the keepalive logic. - errCh := make(chan error) + errCh := make(chan error, 1) go func() { b := make([]byte, 24) for { @@ -197,15 +212,16 @@ func (s) TestKeepaliveServerClosesUnresponsiveClient(t *testing.T) { // Server waits for KeepaliveParams.Time seconds before sending out a ping, // and then waits for KeepaliveParams.Timeout for a ping ack. - timeout := time.NewTimer(4 * time.Second) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() select { case err := <-errCh: if err != io.EOF { t.Fatalf("client.Read(_) = _,%v, want io.EOF", err) } - case <-timeout.C: - t.Fatalf("keepalive timeout expired, server should have closed the connection.") + case <-ctx.Done(): + t.Fatalf("Test timed out before server closed the connection.") } } @@ -214,23 +230,22 @@ func (s) TestKeepaliveServerClosesUnresponsiveClient(t *testing.T) { func (s) TestKeepaliveServerWithResponsiveClient(t *testing.T) { serverConfig := &ServerConfig{ KeepaliveParams: keepalive.ServerParameters{ - Time: 1 * time.Second, - Timeout: 1 * time.Second, + Time: 100 * time.Millisecond, + Timeout: 100 * time.Millisecond, }, } server, client, cancel := setUpWithOptions(t, 0, serverConfig, suspended, ConnectOptions{}) defer func() { - client.Close() + client.Close(fmt.Errorf("closed manually by test")) server.stop() cancel() }() // Give keepalive logic some time by sleeping. - time.Sleep(4 * time.Second) + time.Sleep(500 * time.Millisecond) - // Make sure the client transport is healthy. - if _, err := client.NewStream(context.Background(), &CallHdr{}); err != nil { - t.Fatalf("client.NewStream() failed: %v", err) + if err := checkForHealthyStream(client); err != nil { + t.Fatalf("Stream creation failed: %v", err) } } @@ -241,13 +256,17 @@ func (s) TestKeepaliveServerWithResponsiveClient(t *testing.T) { // logic is running even without any active streams. func (s) TestKeepaliveClientClosesUnresponsiveServer(t *testing.T) { connCh := make(chan net.Conn, 1) - client, cancel := setUpWithNoPingServer(t, ConnectOptions{KeepaliveParams: keepalive.ClientParameters{ - Time: 1 * time.Second, - Timeout: 1 * time.Second, - PermitWithoutStream: true, - }}, connCh) + copts := ConnectOptions{ + ChannelzParentID: channelz.NewIdentifierForTesting(channelz.RefSubChannel, time.Now().Unix(), nil), + KeepaliveParams: keepalive.ClientParameters{ + Time: 10 * time.Millisecond, + Timeout: 10 * time.Millisecond, + PermitWithoutStream: true, + }, + } + client, cancel := setUpWithNoPingServer(t, copts, connCh) defer cancel() - defer client.Close() + defer client.Close(fmt.Errorf("closed manually by test")) conn, ok := <-connCh if !ok { @@ -255,12 +274,8 @@ func (s) TestKeepaliveClientClosesUnresponsiveServer(t *testing.T) { } defer conn.Close() - // Sleep for keepalive to close the connection. - time.Sleep(4 * time.Second) - - // Make sure the client transport is not healthy. - if _, err := client.NewStream(context.Background(), &CallHdr{}); err == nil { - t.Fatal("client.NewStream() should have failed, but succeeded") + if err := pollForStreamCreationError(client); err != nil { + t.Fatal(err) } } @@ -271,12 +286,16 @@ func (s) TestKeepaliveClientClosesUnresponsiveServer(t *testing.T) { // active streams, and therefore the transport stays open. func (s) TestKeepaliveClientOpenWithUnresponsiveServer(t *testing.T) { connCh := make(chan net.Conn, 1) - client, cancel := setUpWithNoPingServer(t, ConnectOptions{KeepaliveParams: keepalive.ClientParameters{ - Time: 1 * time.Second, - Timeout: 1 * time.Second, - }}, connCh) + copts := ConnectOptions{ + ChannelzParentID: channelz.NewIdentifierForTesting(channelz.RefSubChannel, time.Now().Unix(), nil), + KeepaliveParams: keepalive.ClientParameters{ + Time: 10 * time.Millisecond, + Timeout: 10 * time.Millisecond, + }, + } + client, cancel := setUpWithNoPingServer(t, copts, connCh) defer cancel() - defer client.Close() + defer client.Close(fmt.Errorf("closed manually by test")) conn, ok := <-connCh if !ok { @@ -285,11 +304,10 @@ func (s) TestKeepaliveClientOpenWithUnresponsiveServer(t *testing.T) { defer conn.Close() // Give keepalive some time. - time.Sleep(4 * time.Second) + time.Sleep(500 * time.Millisecond) - // Make sure the client transport is healthy. - if _, err := client.NewStream(context.Background(), &CallHdr{}); err != nil { - t.Fatalf("client.NewStream() failed: %v", err) + if err := checkForHealthyStream(client); err != nil { + t.Fatalf("Stream creation failed: %v", err) } } @@ -298,12 +316,18 @@ func (s) TestKeepaliveClientOpenWithUnresponsiveServer(t *testing.T) { // transport even when there is an active stream. func (s) TestKeepaliveClientClosesWithActiveStreams(t *testing.T) { connCh := make(chan net.Conn, 1) - client, cancel := setUpWithNoPingServer(t, ConnectOptions{KeepaliveParams: keepalive.ClientParameters{ - Time: 1 * time.Second, - Timeout: 1 * time.Second, - }}, connCh) + copts := ConnectOptions{ + ChannelzParentID: channelz.NewIdentifierForTesting(channelz.RefSubChannel, time.Now().Unix(), nil), + KeepaliveParams: keepalive.ClientParameters{ + Time: 500 * time.Millisecond, + Timeout: 500 * time.Millisecond, + }, + } + // TODO(i/6099): Setup a server which can ping and no-ping based on a flag to + // reduce the flakiness in this test. + client, cancel := setUpWithNoPingServer(t, copts, connCh) defer cancel() - defer client.Close() + defer client.Close(fmt.Errorf("closed manually by test")) conn, ok := <-connCh if !ok { @@ -311,17 +335,15 @@ func (s) TestKeepaliveClientClosesWithActiveStreams(t *testing.T) { } defer conn.Close() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() // Create a stream, but send no data on it. - if _, err := client.NewStream(context.Background(), &CallHdr{}); err != nil { - t.Fatalf("client.NewStream() failed: %v", err) + if _, err := client.NewStream(ctx, &CallHdr{}); err != nil { + t.Fatalf("Stream creation failed: %v", err) } - // Give keepalive some time. - time.Sleep(4 * time.Second) - - // Make sure the client transport is not healthy. - if _, err := client.NewStream(context.Background(), &CallHdr{}); err == nil { - t.Fatal("client.NewStream() should have failed, but succeeded") + if err := pollForStreamCreationError(client); err != nil { + t.Fatal(err) } } @@ -329,71 +351,66 @@ func (s) TestKeepaliveClientClosesWithActiveStreams(t *testing.T) { // responds to keepalive pings, and makes sure than a client transport stays // healthy without any active streams. func (s) TestKeepaliveClientStaysHealthyWithResponsiveServer(t *testing.T) { - server, client, cancel := setUpWithOptions(t, 0, &ServerConfig{}, normal, ConnectOptions{ - KeepaliveParams: keepalive.ClientParameters{ - Time: 1 * time.Second, - Timeout: 1 * time.Second, - PermitWithoutStream: true, - }}) + server, client, cancel := setUpWithOptions(t, 0, + &ServerConfig{ + KeepalivePolicy: keepalive.EnforcementPolicy{ + MinTime: 50 * time.Millisecond, + PermitWithoutStream: true, + }, + }, + normal, + ConnectOptions{ + KeepaliveParams: keepalive.ClientParameters{ + Time: 55 * time.Millisecond, + Timeout: time.Second, + PermitWithoutStream: true, + }}) defer func() { - client.Close() + client.Close(fmt.Errorf("closed manually by test")) server.stop() cancel() }() // Give keepalive some time. - time.Sleep(4 * time.Second) + time.Sleep(500 * time.Millisecond) - // Make sure the client transport is healthy. - if _, err := client.NewStream(context.Background(), &CallHdr{}); err != nil { - t.Fatalf("client.NewStream() failed: %v", err) + if err := checkForHealthyStream(client); err != nil { + t.Fatalf("Stream creation failed: %v", err) } } // TestKeepaliveClientFrequency creates a server which expects at most 1 client -// ping for every 1.2 seconds, while the client is configured to send a ping -// every 1 second. So, this configuration should end up with the client +// ping for every 100 ms, while the client is configured to send a ping +// every 50 ms. So, this configuration should end up with the client // transport being closed. But we had a bug wherein the client was sending one // ping every [Time+Timeout] instead of every [Time] period, and this test // explicitly makes sure the fix works and the client sends a ping every [Time] // period. func (s) TestKeepaliveClientFrequency(t *testing.T) { + grpctest.TLogger.ExpectError("Client received GoAway with error code ENHANCE_YOUR_CALM and debug data equal to ASCII \"too_many_pings\"") + serverConfig := &ServerConfig{ KeepalivePolicy: keepalive.EnforcementPolicy{ - MinTime: 1200 * time.Millisecond, // 1.2 seconds + MinTime: 100 * time.Millisecond, PermitWithoutStream: true, }, } clientOptions := ConnectOptions{ KeepaliveParams: keepalive.ClientParameters{ - Time: 1 * time.Second, - Timeout: 2 * time.Second, + Time: 50 * time.Millisecond, + Timeout: time.Second, PermitWithoutStream: true, }, } server, client, cancel := setUpWithOptions(t, 0, serverConfig, normal, clientOptions) defer func() { - client.Close() + client.Close(fmt.Errorf("closed manually by test")) server.stop() cancel() }() - timeout := time.NewTimer(6 * time.Second) - select { - case <-client.Error(): - if !timeout.Stop() { - <-timeout.C - } - if reason := client.GetGoAwayReason(); reason != GoAwayTooManyPings { - t.Fatalf("GoAwayReason is %v, want %v", reason, GoAwayTooManyPings) - } - case <-timeout.C: - t.Fatalf("client transport still healthy; expected GoAway from the server.") - } - - // Make sure the client transport is not healthy. - if _, err := client.NewStream(context.Background(), &CallHdr{}); err == nil { - t.Fatal("client.NewStream() should have failed, but succeeded") + if err := waitForGoAwayTooManyPings(client); err != nil { + t.Fatal(err) } } @@ -402,41 +419,29 @@ func (s) TestKeepaliveClientFrequency(t *testing.T) { // (when there are no active streams), based on the configured // EnforcementPolicy. func (s) TestKeepaliveServerEnforcementWithAbusiveClientNoRPC(t *testing.T) { + grpctest.TLogger.ExpectError("Client received GoAway with error code ENHANCE_YOUR_CALM and debug data equal to ASCII \"too_many_pings\"") + serverConfig := &ServerConfig{ KeepalivePolicy: keepalive.EnforcementPolicy{ - MinTime: 2 * time.Second, + MinTime: time.Second, }, } clientOptions := ConnectOptions{ KeepaliveParams: keepalive.ClientParameters{ - Time: 50 * time.Millisecond, - Timeout: 1 * time.Second, + Time: 20 * time.Millisecond, + Timeout: 100 * time.Millisecond, PermitWithoutStream: true, }, } server, client, cancel := setUpWithOptions(t, 0, serverConfig, normal, clientOptions) defer func() { - client.Close() + client.Close(fmt.Errorf("closed manually by test")) server.stop() cancel() }() - timeout := time.NewTimer(4 * time.Second) - select { - case <-client.Error(): - if !timeout.Stop() { - <-timeout.C - } - if reason := client.GetGoAwayReason(); reason != GoAwayTooManyPings { - t.Fatalf("GoAwayReason is %v, want %v", reason, GoAwayTooManyPings) - } - case <-timeout.C: - t.Fatalf("client transport still healthy; expected GoAway from the server.") - } - - // Make sure the client transport is not healthy. - if _, err := client.NewStream(context.Background(), &CallHdr{}); err == nil { - t.Fatal("client.NewStream() should have failed, but succeeded") + if err := waitForGoAwayTooManyPings(client); err != nil { + t.Fatal(err) } } @@ -445,44 +450,34 @@ func (s) TestKeepaliveServerEnforcementWithAbusiveClientNoRPC(t *testing.T) { // (even when there is an active stream), based on the configured // EnforcementPolicy. func (s) TestKeepaliveServerEnforcementWithAbusiveClientWithRPC(t *testing.T) { + grpctest.TLogger.ExpectError("Client received GoAway with error code ENHANCE_YOUR_CALM and debug data equal to ASCII \"too_many_pings\"") + serverConfig := &ServerConfig{ KeepalivePolicy: keepalive.EnforcementPolicy{ - MinTime: 2 * time.Second, + MinTime: time.Second, }, } clientOptions := ConnectOptions{ KeepaliveParams: keepalive.ClientParameters{ Time: 50 * time.Millisecond, - Timeout: 1 * time.Second, + Timeout: 100 * time.Millisecond, }, } server, client, cancel := setUpWithOptions(t, 0, serverConfig, suspended, clientOptions) defer func() { - client.Close() + client.Close(fmt.Errorf("closed manually by test")) server.stop() cancel() }() - if _, err := client.NewStream(context.Background(), &CallHdr{}); err != nil { - t.Fatalf("client.NewStream() failed: %v", err) - } - - timeout := time.NewTimer(4 * time.Second) - select { - case <-client.Error(): - if !timeout.Stop() { - <-timeout.C - } - if reason := client.GetGoAwayReason(); reason != GoAwayTooManyPings { - t.Fatalf("GoAwayReason is %v, want %v", reason, GoAwayTooManyPings) - } - case <-timeout.C: - t.Fatalf("client transport still healthy; expected GoAway from the server.") + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := client.NewStream(ctx, &CallHdr{}); err != nil { + t.Fatalf("Stream creation failed: %v", err) } - // Make sure the client transport is not healthy. - if _, err := client.NewStream(context.Background(), &CallHdr{}); err == nil { - t.Fatal("client.NewStream() should have failed, but succeeded") + if err := waitForGoAwayTooManyPings(client); err != nil { + t.Fatal(err) } } @@ -493,30 +488,30 @@ func (s) TestKeepaliveServerEnforcementWithAbusiveClientWithRPC(t *testing.T) { func (s) TestKeepaliveServerEnforcementWithObeyingClientNoRPC(t *testing.T) { serverConfig := &ServerConfig{ KeepalivePolicy: keepalive.EnforcementPolicy{ - MinTime: 100 * time.Millisecond, + MinTime: 40 * time.Millisecond, PermitWithoutStream: true, }, } clientOptions := ConnectOptions{ KeepaliveParams: keepalive.ClientParameters{ - Time: 101 * time.Millisecond, - Timeout: 1 * time.Second, + Time: 50 * time.Millisecond, + Timeout: time.Second, PermitWithoutStream: true, }, } server, client, cancel := setUpWithOptions(t, 0, serverConfig, normal, clientOptions) defer func() { - client.Close() + client.Close(fmt.Errorf("closed manually by test")) server.stop() cancel() }() - // Give keepalive enough time. - time.Sleep(3 * time.Second) + // Sleep for client to send ~10 keepalive pings. + time.Sleep(500 * time.Millisecond) - // Make sure the client transport is healthy. - if _, err := client.NewStream(context.Background(), &CallHdr{}); err != nil { - t.Fatalf("client.NewStream() failed: %v", err) + // Verify that the server does not close the client transport. + if err := checkForHealthyStream(client); err != nil { + t.Fatalf("Stream creation failed: %v", err) } } @@ -527,32 +522,30 @@ func (s) TestKeepaliveServerEnforcementWithObeyingClientNoRPC(t *testing.T) { func (s) TestKeepaliveServerEnforcementWithObeyingClientWithRPC(t *testing.T) { serverConfig := &ServerConfig{ KeepalivePolicy: keepalive.EnforcementPolicy{ - MinTime: 100 * time.Millisecond, + MinTime: 40 * time.Millisecond, }, } clientOptions := ConnectOptions{ KeepaliveParams: keepalive.ClientParameters{ - Time: 101 * time.Millisecond, - Timeout: 1 * time.Second, + Time: 50 * time.Millisecond, }, } server, client, cancel := setUpWithOptions(t, 0, serverConfig, suspended, clientOptions) defer func() { - client.Close() + client.Close(fmt.Errorf("closed manually by test")) server.stop() cancel() }() - if _, err := client.NewStream(context.Background(), &CallHdr{}); err != nil { - t.Fatalf("client.NewStream() failed: %v", err) + if err := checkForHealthyStream(client); err != nil { + t.Fatalf("Stream creation failed: %v", err) } // Give keepalive enough time. - time.Sleep(3 * time.Second) + time.Sleep(500 * time.Millisecond) - // Make sure the client transport is healthy. - if _, err := client.NewStream(context.Background(), &CallHdr{}); err != nil { - t.Fatalf("client.NewStream() failed: %v", err) + if err := checkForHealthyStream(client); err != nil { + t.Fatalf("Stream creation failed: %v", err) } } @@ -565,28 +558,27 @@ func (s) TestKeepaliveServerEnforcementWithObeyingClientWithRPC(t *testing.T) { func (s) TestKeepaliveServerEnforcementWithDormantKeepaliveOnClient(t *testing.T) { serverConfig := &ServerConfig{ KeepalivePolicy: keepalive.EnforcementPolicy{ - MinTime: 2 * time.Second, + MinTime: 100 * time.Millisecond, }, } clientOptions := ConnectOptions{ KeepaliveParams: keepalive.ClientParameters{ - Time: 50 * time.Millisecond, - Timeout: 1 * time.Second, + Time: 10 * time.Millisecond, + Timeout: 10 * time.Millisecond, }, } server, client, cancel := setUpWithOptions(t, 0, serverConfig, normal, clientOptions) defer func() { - client.Close() + client.Close(fmt.Errorf("closed manually by test")) server.stop() cancel() }() // No active streams on the client. Give keepalive enough time. - time.Sleep(5 * time.Second) + time.Sleep(500 * time.Millisecond) - // Make sure the client transport is healthy. - if _, err := client.NewStream(context.Background(), &CallHdr{}); err != nil { - t.Fatalf("client.NewStream() failed: %v", err) + if err := checkForHealthyStream(client); err != nil { + t.Fatalf("Stream creation failed: %v", err) } } @@ -594,60 +586,217 @@ func (s) TestKeepaliveServerEnforcementWithDormantKeepaliveOnClient(t *testing.T // the keepalive timeout, as detailed in proposal A18. func (s) TestTCPUserTimeout(t *testing.T) { tests := []struct { - time time.Duration - timeout time.Duration - wantTimeout time.Duration + tls bool + time time.Duration + timeout time.Duration + clientWantTimeout time.Duration + serverWantTimeout time.Duration }{ { + false, 10 * time.Second, 10 * time.Second, 10 * 1000 * time.Millisecond, + 10 * 1000 * time.Millisecond, }, { + false, + 0, + 0, + 0, + 20 * 1000 * time.Millisecond, + }, + { + false, + infinity, + infinity, + 0, + 0, + }, + { + true, + 10 * time.Second, + 10 * time.Second, + 10 * 1000 * time.Millisecond, + 10 * 1000 * time.Millisecond, + }, + { + true, + 0, 0, 0, + 20 * 1000 * time.Millisecond, + }, + { + true, + infinity, + infinity, + 0, 0, }, } for _, tt := range tests { + sopts := &ServerConfig{ + KeepaliveParams: keepalive.ServerParameters{ + Time: tt.time, + Timeout: tt.timeout, + }, + } + + copts := ConnectOptions{ + KeepaliveParams: keepalive.ClientParameters{ + Time: tt.time, + Timeout: tt.timeout, + }, + } + + if tt.tls { + copts.TransportCredentials = makeTLSCreds(t, "x509/client1_cert.pem", "x509/client1_key.pem", "x509/server_ca_cert.pem") + sopts.Credentials = makeTLSCreds(t, "x509/server1_cert.pem", "x509/server1_key.pem", "x509/client_ca_cert.pem") + + } + server, client, cancel := setUpWithOptions( t, 0, - &ServerConfig{ - KeepaliveParams: keepalive.ServerParameters{ - Time: tt.timeout, - Timeout: tt.timeout, - }, - }, + sopts, normal, - ConnectOptions{ - KeepaliveParams: keepalive.ClientParameters{ - Time: tt.time, - Timeout: tt.timeout, - }, - }, + copts, ) defer func() { - client.Close() + client.Close(fmt.Errorf("closed manually by test")) server.stop() cancel() }() - stream, err := client.NewStream(context.Background(), &CallHdr{}) + var sc *http2Server + var srawConn net.Conn + // Wait until the server transport is setup. + for { + server.mu.Lock() + if len(server.conns) == 0 { + server.mu.Unlock() + time.Sleep(time.Millisecond) + continue + } + for k := range server.conns { + var ok bool + sc, ok = k.(*http2Server) + if !ok { + t.Fatalf("Failed to convert %v to *http2Server", k) + } + srawConn = server.conns[k] + } + server.mu.Unlock() + break + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + stream, err := client.NewStream(ctx, &CallHdr{}) if err != nil { t.Fatalf("client.NewStream() failed: %v", err) } client.CloseStream(stream, io.EOF) - opt, err := syscall.GetTCPUserTimeout(client.conn) + // check client TCP user timeout only when non TLS + // TODO : find a way to get the underlying conn for client when TLS + if !tt.tls { + cltOpt, err := syscall.GetTCPUserTimeout(client.conn) + if err != nil { + t.Fatalf("syscall.GetTCPUserTimeout() failed: %v", err) + } + if cltOpt < 0 { + t.Skipf("skipping test on unsupported environment") + } + if gotTimeout := time.Duration(cltOpt) * time.Millisecond; gotTimeout != tt.clientWantTimeout { + t.Fatalf("syscall.GetTCPUserTimeout() = %d, want %d", gotTimeout, tt.clientWantTimeout) + } + } + scConn := sc.conn + if tt.tls { + if _, ok := sc.conn.(*net.TCPConn); ok { + t.Fatalf("sc.conn is should have wrapped conn with TLS") + } + scConn = srawConn + } + // verify the type of scConn (on which TCP user timeout will be got) + if _, ok := scConn.(*net.TCPConn); !ok { + t.Fatalf("server underlying conn is of type %T, want net.TCPConn", scConn) + } + srvOpt, err := syscall.GetTCPUserTimeout(scConn) if err != nil { t.Fatalf("syscall.GetTCPUserTimeout() failed: %v", err) } - if opt < 0 { - t.Skipf("skipping test on unsupported environment") + if gotTimeout := time.Duration(srvOpt) * time.Millisecond; gotTimeout != tt.serverWantTimeout { + t.Fatalf("syscall.GetTCPUserTimeout() = %d, want %d", gotTimeout, tt.serverWantTimeout) + } + + } +} + +func makeTLSCreds(t *testing.T, certPath, keyPath, rootsPath string) credentials.TransportCredentials { + cert, err := tls.LoadX509KeyPair(testdata.Path(certPath), testdata.Path(keyPath)) + if err != nil { + t.Fatalf("tls.LoadX509KeyPair(%q, %q) failed: %v", certPath, keyPath, err) + } + b, err := os.ReadFile(testdata.Path(rootsPath)) + if err != nil { + t.Fatalf("os.ReadFile(%q) failed: %v", rootsPath, err) + } + roots := x509.NewCertPool() + if !roots.AppendCertsFromPEM(b) { + t.Fatal("failed to append certificates") + } + return credentials.NewTLS(&tls.Config{ + Certificates: []tls.Certificate{cert}, + RootCAs: roots, + InsecureSkipVerify: true, + }) +} + +// checkForHealthyStream attempts to create a stream and return error if any. +// The stream created is closed right after to avoid any leakages. +func checkForHealthyStream(client *http2Client) error { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + stream, err := client.NewStream(ctx, &CallHdr{}) + client.CloseStream(stream, err) + return err +} + +func pollForStreamCreationError(client *http2Client) error { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + for { + if _, err := client.NewStream(ctx, &CallHdr{}); err != nil { + break } - if gotTimeout := time.Duration(opt) * time.Millisecond; gotTimeout != tt.wantTimeout { - t.Fatalf("syscall.GetTCPUserTimeout() = %d, want %d", gotTimeout, tt.wantTimeout) + time.Sleep(50 * time.Millisecond) + } + if ctx.Err() != nil { + return fmt.Errorf("test timed out before stream creation returned an error") + } + return nil +} + +// waitForGoAwayTooManyPings waits for client to receive a GoAwayTooManyPings +// from server. It also asserts that stream creation fails after receiving a +// GoAway. +func waitForGoAwayTooManyPings(client *http2Client) error { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + select { + case <-client.GoAway(): + if reason, _ := client.GetGoAwayReason(); reason != GoAwayTooManyPings { + return fmt.Errorf("goAwayReason is %v, want %v", reason, GoAwayTooManyPings) } + case <-ctx.Done(): + return fmt.Errorf("test timed out before getting GoAway with reason:GoAwayTooManyPings from server") + } + + if _, err := client.NewStream(ctx, &CallHdr{}); err == nil { + return fmt.Errorf("stream creation succeeded after receiving a GoAway from the server") } + return nil } diff --git a/internal/transport/logging.go b/internal/transport/logging.go new file mode 100644 index 000000000000..42ed2b07af66 --- /dev/null +++ b/internal/transport/logging.go @@ -0,0 +1,40 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package transport + +import ( + "fmt" + + "google.golang.org/grpc/grpclog" + internalgrpclog "google.golang.org/grpc/internal/grpclog" +) + +var logger = grpclog.Component("transport") + +func prefixLoggerForServerTransport(p *http2Server) *internalgrpclog.PrefixLogger { + return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf("[server-transport %p] ", p)) +} + +func prefixLoggerForServerHandlerTransport(p *serverHandlerTransport) *internalgrpclog.PrefixLogger { + return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf("[server-handler-transport %p] ", p)) +} + +func prefixLoggerForClientTransport(p *http2Client) *internalgrpclog.PrefixLogger { + return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf("[client-transport %p] ", p)) +} diff --git a/internal/transport/networktype/networktype.go b/internal/transport/networktype/networktype.go new file mode 100644 index 000000000000..c11b5278274f --- /dev/null +++ b/internal/transport/networktype/networktype.go @@ -0,0 +1,46 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package networktype declares the network type to be used in the default +// dialer. Attribute of a resolver.Address. +package networktype + +import ( + "google.golang.org/grpc/resolver" +) + +// keyType is the key to use for storing State in Attributes. +type keyType string + +const key = keyType("grpc.internal.transport.networktype") + +// Set returns a copy of the provided address with attributes containing networkType. +func Set(address resolver.Address, networkType string) resolver.Address { + address.Attributes = address.Attributes.WithValue(key, networkType) + return address +} + +// Get returns the network type in the resolver.Address and true, or "", false +// if not present. +func Get(address resolver.Address) (string, bool) { + v := address.Attributes.Value(key) + if v == nil { + return "", false + } + return v.(string), true +} diff --git a/proxy.go b/internal/transport/proxy.go similarity index 71% rename from proxy.go rename to internal/transport/proxy.go index f8f69bfb70fd..415961987870 100644 --- a/proxy.go +++ b/internal/transport/proxy.go @@ -16,13 +16,12 @@ * */ -package grpc +package transport import ( "bufio" "context" "encoding/base64" - "errors" "fmt" "io" "net" @@ -34,13 +33,11 @@ import ( const proxyAuthHeaderKey = "Proxy-Authorization" var ( - // errDisabled indicates that proxy is disabled for the address. - errDisabled = errors.New("proxy is disabled for the address") // The following variable will be overwritten in the tests. httpProxyFromEnvironment = http.ProxyFromEnvironment ) -func mapAddress(ctx context.Context, address string) (*url.URL, error) { +func mapAddress(address string) (*url.URL, error) { req := &http.Request{ URL: &url.URL{ Scheme: "https", @@ -51,9 +48,6 @@ func mapAddress(ctx context.Context, address string) (*url.URL, error) { if err != nil { return nil, err } - if url == nil { - return nil, errDisabled - } return url, nil } @@ -76,7 +70,7 @@ func basicAuth(username, password string) string { return base64.StdEncoding.EncodeToString([]byte(auth)) } -func doHTTPConnectHandshake(ctx context.Context, conn net.Conn, backendAddr string, proxyURL *url.URL) (_ net.Conn, err error) { +func doHTTPConnectHandshake(ctx context.Context, conn net.Conn, backendAddr string, proxyURL *url.URL, grpcUA string) (_ net.Conn, err error) { defer func() { if err != nil { conn.Close() @@ -115,32 +109,28 @@ func doHTTPConnectHandshake(ctx context.Context, conn net.Conn, backendAddr stri return &bufConn{Conn: conn, r: r}, nil } -// newProxyDialer returns a dialer that connects to proxy first if necessary. -// The returned dialer checks if a proxy is necessary, dial to the proxy with the -// provided dialer, does HTTP CONNECT handshake and returns the connection. -func newProxyDialer(dialer func(context.Context, string) (net.Conn, error)) func(context.Context, string) (net.Conn, error) { - return func(ctx context.Context, addr string) (conn net.Conn, err error) { - var newAddr string - proxyURL, err := mapAddress(ctx, addr) - if err != nil { - if err != errDisabled { - return nil, err - } - newAddr = addr - } else { - newAddr = proxyURL.Host - } +// proxyDial dials, connecting to a proxy first if necessary. Checks if a proxy +// is necessary, dials, does the HTTP CONNECT handshake, and returns the +// connection. +func proxyDial(ctx context.Context, addr string, grpcUA string) (conn net.Conn, err error) { + newAddr := addr + proxyURL, err := mapAddress(addr) + if err != nil { + return nil, err + } + if proxyURL != nil { + newAddr = proxyURL.Host + } - conn, err = dialer(ctx, newAddr) - if err != nil { - return - } - if proxyURL != nil { - // proxy is disabled if proxyURL is nil. - conn, err = doHTTPConnectHandshake(ctx, conn, addr, proxyURL) - } + conn, err = (&net.Dialer{}).DialContext(ctx, "tcp", newAddr) + if err != nil { return } + if proxyURL != nil { + // proxy is disabled if proxyURL is nil. + conn, err = doHTTPConnectHandshake(ctx, conn, addr, proxyURL, grpcUA) + } + return } func sendHTTPRequest(ctx context.Context, req *http.Request, conn net.Conn) error { diff --git a/proxy_test.go b/internal/transport/proxy_test.go similarity index 89% rename from proxy_test.go rename to internal/transport/proxy_test.go index c9604be62351..8abee1e7b383 100644 --- a/proxy_test.go +++ b/internal/transport/proxy_test.go @@ -1,3 +1,4 @@ +//go:build !race // +build !race /* @@ -18,7 +19,7 @@ * */ -package grpc +package transport import ( "bufio" @@ -119,7 +120,7 @@ func testHTTPConnect(t *testing.T, proxyURLModify func(*url.URL) *url.URL, proxy msg := []byte{4, 3, 5, 2} recvBuf := make([]byte, len(msg)) - done := make(chan error) + done := make(chan error, 1) go func() { in, err := blis.Accept() if err != nil { @@ -138,15 +139,9 @@ func testHTTPConnect(t *testing.T, proxyURLModify func(*url.URL) *url.URL, proxy defer overwrite(hpfe)() // Dial to proxy server. - dialer := newProxyDialer(func(ctx context.Context, addr string) (net.Conn, error) { - if deadline, ok := ctx.Deadline(); ok { - return net.DialTimeout("tcp", addr, time.Until(deadline)) - } - return net.Dial("tcp", addr) - }) ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() - c, err := dialer(ctx, blis.Addr().String()) + c, err := proxyDial(ctx, blis.Addr().String(), "test") if err != nil { t.Fatalf("http connect Dial failed: %v", err) } @@ -173,9 +168,6 @@ func (s) TestHTTPConnect(t *testing.T) { if req.Method != http.MethodConnect { return fmt.Errorf("unexpected Method %q, want %q", req.Method, http.MethodConnect) } - if req.UserAgent() != grpcUA { - return fmt.Errorf("unexpect user agent %q, want %q", req.UserAgent(), grpcUA) - } return nil }, ) @@ -195,9 +187,6 @@ func (s) TestHTTPConnectBasicAuth(t *testing.T) { if req.Method != http.MethodConnect { return fmt.Errorf("unexpected Method %q, want %q", req.Method, http.MethodConnect) } - if req.UserAgent() != grpcUA { - return fmt.Errorf("unexpect user agent %q, want %q", req.UserAgent(), grpcUA) - } wantProxyAuthStr := "Basic " + base64.StdEncoding.EncodeToString([]byte(user+":"+password)) if got := req.Header.Get(proxyAuthHeaderKey); got != wantProxyAuthStr { gotDecoded, _ := base64.StdEncoding.DecodeString(got) @@ -223,7 +212,7 @@ func (s) TestMapAddressEnv(t *testing.T) { defer overwrite(hpfe)() // envTestAddr should be handled by ProxyFromEnvironment. - got, err := mapAddress(context.Background(), envTestAddr) + got, err := mapAddress(envTestAddr) if err != nil { t.Error(err) } diff --git a/internal/transport/transport.go b/internal/transport/transport.go index b74030a96878..74a811fc0590 100644 --- a/internal/transport/transport.go +++ b/internal/transport/transport.go @@ -30,9 +30,11 @@ import ( "net" "sync" "sync/atomic" + "time" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/internal/channelz" "google.golang.org/grpc/keepalive" "google.golang.org/grpc/metadata" "google.golang.org/grpc/resolver" @@ -50,7 +52,7 @@ type bufferPool struct { func newBufferPool() *bufferPool { return &bufferPool{ pool: sync.Pool{ - New: func() interface{} { + New: func() any { return new(bytes.Buffer) }, }, @@ -241,6 +243,7 @@ type Stream struct { ctx context.Context // the associated context of the stream cancel context.CancelFunc // always nil for client side Stream done chan struct{} // closed at the end of stream to unblock writers. On the client side. + doneFunc func() // invoked at the end of stream on client side. ctxDone <-chan struct{} // same as done chan but for server side. Cache of ctx.Done() (for performance) method string // the associated RPC method of the stream recvCompress string @@ -250,6 +253,9 @@ type Stream struct { fc *inFlow wq *writeQuota + // Holds compressor names passed in grpc-accept-encoding metadata from the + // client. This is empty for the client side stream. + clientAdvertisedCompressors string // Callback to state application's intentions to read data. This // is used to adjust flow control, if needed. requestRead func(int) @@ -338,8 +344,24 @@ func (s *Stream) RecvCompress() string { } // SetSendCompress sets the compression algorithm to the stream. -func (s *Stream) SetSendCompress(str string) { - s.sendCompress = str +func (s *Stream) SetSendCompress(name string) error { + if s.isHeaderSent() || s.getState() == streamDone { + return errors.New("transport: set send compressor called after headers sent or stream done") + } + + s.sendCompress = name + return nil +} + +// SendCompress returns the send compressor name. +func (s *Stream) SendCompress() string { + return s.sendCompress +} + +// ClientAdvertisedCompressors returns the compressor names advertised by the +// client via grpc-accept-encoding header. +func (s *Stream) ClientAdvertisedCompressors() string { + return s.clientAdvertisedCompressors } // Done returns a channel which is closed when it receives the final status @@ -363,9 +385,11 @@ func (s *Stream) Header() (metadata.MD, error) { return s.header.Copy(), nil } s.waitOnHeader() - if !s.headerValid { + + if !s.headerValid || s.noHeaders { return nil, s.status.Err() } + return s.header.Copy(), nil } @@ -517,26 +541,22 @@ const ( // ServerConfig consists of all the configurations to establish a server transport. type ServerConfig struct { MaxStreams uint32 - AuthInfo credentials.AuthInfo + ConnectionTimeout time.Duration + Credentials credentials.TransportCredentials InTapHandle tap.ServerInHandle - StatsHandler stats.Handler + StatsHandlers []stats.Handler KeepaliveParams keepalive.ServerParameters KeepalivePolicy keepalive.EnforcementPolicy InitialWindowSize int32 InitialConnWindowSize int32 WriteBufferSize int ReadBufferSize int - ChannelzParentID int64 + SharedWriteBuffer bool + ChannelzParentID *channelz.Identifier MaxHeaderListSize *uint32 HeaderTableSize *uint32 } -// NewServerTransport creates a ServerTransport with conn or non-nil error -// if it fails. -func NewServerTransport(protocol string, conn net.Conn, config *ServerConfig) (ServerTransport, error) { - return newHTTP2Server(conn, config) -} - // ConnectOptions covers all relevant options for communicating with the server. type ConnectOptions struct { // UserAgent is the application user agent. @@ -555,8 +575,8 @@ type ConnectOptions struct { CredsBundle credentials.Bundle // KeepaliveParams stores the keepalive parameters. KeepaliveParams keepalive.ClientParameters - // StatsHandler stores the handler for stats. - StatsHandler stats.Handler + // StatsHandlers stores the handler for stats. + StatsHandlers []stats.Handler // InitialWindowSize sets the initial window size for a stream. InitialWindowSize int32 // InitialConnWindowSize sets the initial window size for a connection. @@ -565,16 +585,20 @@ type ConnectOptions struct { WriteBufferSize int // ReadBufferSize sets the size of read buffer, which in turn determines how much data can be read at most for one read syscall. ReadBufferSize int + // SharedWriteBuffer indicates whether connections should reuse write buffer + SharedWriteBuffer bool // ChannelzParentID sets the addrConn id which initiate the creation of this client transport. - ChannelzParentID int64 + ChannelzParentID *channelz.Identifier // MaxHeaderListSize sets the max (uncompressed) size of header list that is prepared to be received. MaxHeaderListSize *uint32 + // UseProxy specifies if a proxy should be used. + UseProxy bool } // NewClientTransport establishes the transport with the required ConnectOptions // and returns it to the caller. -func NewClientTransport(connectCtx, ctx context.Context, addr resolver.Address, opts ConnectOptions, onPrefaceReceipt func(), onGoAway func(GoAwayReason), onClose func()) (ClientTransport, error) { - return newHTTP2Client(connectCtx, ctx, addr, opts, onPrefaceReceipt, onGoAway, onClose) +func NewClientTransport(connectCtx, ctx context.Context, addr resolver.Address, opts ConnectOptions, onClose func(GoAwayReason)) (ClientTransport, error) { + return newHTTP2Client(connectCtx, ctx, addr, opts, onClose) } // Options provides additional hints and information for message @@ -609,6 +633,8 @@ type CallHdr struct { ContentSubtype string PreviousAttempts int // value of grpc-previous-rpc-attempts header to set + + DoneFunc func() // called when the stream is finished } // ClientTransport is the common interface for all gRPC client-side transport @@ -617,7 +643,7 @@ type ClientTransport interface { // Close tears down this transport. Once it returns, the transport // should not be accessed any more. The caller must make sure this // is called only once. - Close() error + Close(err error) // GracefulClose starts to tear down the transport: the transport will stop // accepting new RPCs and NewStream will return error. Once all streams are @@ -651,8 +677,9 @@ type ClientTransport interface { // HTTP/2). GoAway() <-chan struct{} - // GetGoAwayReason returns the reason why GoAway frame was received. - GetGoAwayReason() GoAwayReason + // GetGoAwayReason returns the reason why GoAway frame was received, along + // with a human readable string with debug info. + GetGoAwayReason() (GoAwayReason, string) // RemoteAddr returns the remote network address. RemoteAddr() net.Addr @@ -688,13 +715,13 @@ type ServerTransport interface { // Close tears down the transport. Once it is called, the transport // should not be accessed any more. All the pending streams and their // handlers will be terminated asynchronously. - Close() error + Close(err error) // RemoteAddr returns the remote network address. RemoteAddr() net.Addr // Drain notifies the client this ServerTransport stops accepting new RPCs. - Drain() + Drain(debugData string) // IncrMsgSent increments the number of message sent through this transport. IncrMsgSent() @@ -704,7 +731,7 @@ type ServerTransport interface { } // connectionErrorf creates an ConnectionError with the specified error description. -func connectionErrorf(temp bool, e error, format string, a ...interface{}) ConnectionError { +func connectionErrorf(temp bool, e error, format string, a ...any) ConnectionError { return ConnectionError{ Desc: fmt.Sprintf(format, a...), temp: temp, @@ -739,6 +766,12 @@ func (e ConnectionError) Origin() error { return e.err } +// Unwrap returns the original error of this connection error or nil when the +// origin is nil. +func (e ConnectionError) Unwrap() error { + return e.err +} + var ( // ErrConnClosing indicates that the transport is closing. ErrConnClosing = connectionErrorf(true, nil, "transport is closing") diff --git a/internal/transport/transport_test.go b/internal/transport/transport_test.go index 391ad9925c1a..258ef7411cf0 100644 --- a/internal/transport/transport_test.go +++ b/internal/transport/transport_test.go @@ -27,6 +27,7 @@ import ( "io" "math" "net" + "os" "runtime" "strconv" "strings" @@ -34,12 +35,15 @@ import ( "testing" "time" + "google.golang.org/grpc/peer" + "github.com/google/go-cmp/cmp" "golang.org/x/net/http2" "golang.org/x/net/http2/hpack" "google.golang.org/grpc/attributes" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/internal/channelz" "google.golang.org/grpc/internal/grpctest" "google.golang.org/grpc/internal/leakcheck" "google.golang.org/grpc/internal/testutils" @@ -55,16 +59,6 @@ func Test(t *testing.T) { grpctest.RunSubTests(t, s{}) } -type server struct { - lis net.Listener - port string - startedErr chan error // error (or nil) with server start value - mu sync.Mutex - conns map[ServerTransport]bool - h *testStreamHandler - ready chan struct{} -} - var ( expectedRequest = []byte("ping") expectedResponse = []byte("pong") @@ -194,12 +188,12 @@ func (h *testStreamHandler) handleStreamMisbehave(t *testing.T, s *Stream) { } } -func (h *testStreamHandler) handleStreamEncodingRequiredStatus(t *testing.T, s *Stream) { +func (h *testStreamHandler) handleStreamEncodingRequiredStatus(s *Stream) { // raw newline is not accepted by http2 framer so it must be encoded. h.t.WriteStatus(s, encodingTestStatus) } -func (h *testStreamHandler) handleStreamInvalidHeaderField(t *testing.T, s *Stream) { +func (h *testStreamHandler) handleStreamInvalidHeaderField(s *Stream) { headerFields := []hpack.HeaderField{} headerFields = append(headerFields, hpack.HeaderField{Name: "content-type", Value: expectedInvalidHeaderField}) h.t.controlBuf.put(&headerFrame{ @@ -298,6 +292,25 @@ func (h *testStreamHandler) handleStreamDelayRead(t *testing.T, s *Stream) { } } +type server struct { + lis net.Listener + port string + startedErr chan error // error (or nil) with server start value + mu sync.Mutex + conns map[ServerTransport]net.Conn + h *testStreamHandler + ready chan struct{} + channelzID *channelz.Identifier +} + +func newTestServer() *server { + return &server{ + startedErr: make(chan error, 1), + ready: make(chan struct{}), + channelzID: channelz.NewIdentifierForTesting(channelz.RefServer, time.Now().Unix(), nil), + } +} + // start starts server. Other goroutines should block on s.readyChan for further operations. func (s *server) start(t *testing.T, port int, serverConfig *ServerConfig, ht hType) { var err error @@ -316,24 +329,25 @@ func (s *server) start(t *testing.T, port int, serverConfig *ServerConfig, ht hT return } s.port = p - s.conns = make(map[ServerTransport]bool) + s.conns = make(map[ServerTransport]net.Conn) s.startedErr <- nil for { conn, err := s.lis.Accept() if err != nil { return } - transport, err := NewServerTransport("http2", conn, serverConfig) + rawConn := conn + transport, err := NewServerTransport(conn, serverConfig) if err != nil { return } s.mu.Lock() if s.conns == nil { s.mu.Unlock() - transport.Close() + transport.Close(errors.New("s.conns is nil")) return } - s.conns[transport] = true + s.conns[transport] = rawConn h := &testStreamHandler{t: transport.(*http2Server)} s.h = h s.mu.Unlock() @@ -356,13 +370,13 @@ func (s *server) start(t *testing.T, port int, serverConfig *ServerConfig, ht hT }) case encodingRequiredStatus: go transport.HandleStreams(func(s *Stream) { - go h.handleStreamEncodingRequiredStatus(t, s) + go h.handleStreamEncodingRequiredStatus(s) }, func(ctx context.Context, method string) context.Context { return ctx }) case invalidHeaderField: go transport.HandleStreams(func(s *Stream) { - go h.handleStreamInvalidHeaderField(t, s) + go h.handleStreamInvalidHeaderField(s) }, func(ctx context.Context, method string) context.Context { return ctx }) @@ -408,7 +422,7 @@ func (s *server) stop() { s.lis.Close() s.mu.Lock() for c := range s.conns { - c.Close() + c.Close(errors.New("server Stop called")) } s.conns = nil s.mu.Unlock() @@ -421,9 +435,10 @@ func (s *server) addr() string { return s.lis.Addr().String() } -func setUpServerOnly(t *testing.T, port int, serverConfig *ServerConfig, ht hType) *server { - server := &server{startedErr: make(chan error, 1), ready: make(chan struct{})} - go server.start(t, port, serverConfig, ht) +func setUpServerOnly(t *testing.T, port int, sc *ServerConfig, ht hType) *server { + server := newTestServer() + sc.ChannelzParentID = server.channelzID + go server.start(t, port, sc, ht) server.wait(t, 2*time.Second) return server } @@ -432,11 +447,13 @@ func setUp(t *testing.T, port int, maxStreams uint32, ht hType) (*server, *http2 return setUpWithOptions(t, port, &ServerConfig{MaxStreams: maxStreams}, ht, ConnectOptions{}) } -func setUpWithOptions(t *testing.T, port int, serverConfig *ServerConfig, ht hType, copts ConnectOptions) (*server, *http2Client, func()) { - server := setUpServerOnly(t, port, serverConfig, ht) +func setUpWithOptions(t *testing.T, port int, sc *ServerConfig, ht hType, copts ConnectOptions) (*server, *http2Client, func()) { + server := setUpServerOnly(t, port, sc, ht) addr := resolver.Address{Addr: "localhost:" + server.port} + copts.ChannelzParentID = channelz.NewIdentifierForTesting(channelz.RefSubChannel, time.Now().Unix(), nil) + connectCtx, cancel := context.WithDeadline(context.Background(), time.Now().Add(2*time.Second)) - ct, connErr := NewClientTransport(connectCtx, context.Background(), addr, copts, func() {}, func(GoAwayReason) {}, func() {}) + ct, connErr := NewClientTransport(connectCtx, context.Background(), addr, copts, func(GoAwayReason) {}) if connErr != nil { cancel() // Do not cancel in success path. t.Fatalf("failed to create transport: %v", connErr) @@ -458,10 +475,16 @@ func setUpWithNoPingServer(t *testing.T, copts ConnectOptions, connCh chan net.C close(connCh) return } + framer := http2.NewFramer(conn, conn) + if err := framer.WriteSettings(); err != nil { + t.Errorf("Error at server-side while writing settings: %v", err) + close(connCh) + return + } connCh <- conn }() connectCtx, cancel := context.WithDeadline(context.Background(), time.Now().Add(2*time.Second)) - tr, err := NewClientTransport(connectCtx, context.Background(), resolver.Address{Addr: lis.Addr().String()}, copts, func() {}, func(GoAwayReason) {}, func() {}) + tr, err := NewClientTransport(connectCtx, context.Background(), resolver.Address{Addr: lis.Addr().String()}, copts, func(GoAwayReason) {}) if err != nil { cancel() // Do not cancel in success path. // Server clean-up. @@ -481,9 +504,11 @@ func (s) TestInflightStreamClosing(t *testing.T) { server, client, cancel := setUpWithOptions(t, 0, serverConfig, suspended, ConnectOptions{}) defer cancel() defer server.stop() - defer client.Close() + defer client.Close(fmt.Errorf("closed manually by test")) - stream, err := client.NewStream(context.Background(), &CallHdr{}) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + stream, err := client.NewStream(ctx, &CallHdr{}) if err != nil { t.Fatalf("Client failed to create RPC request: %v", err) } @@ -512,6 +537,52 @@ func (s) TestInflightStreamClosing(t *testing.T) { } } +// Tests that when streamID > MaxStreamId, the current client transport drains. +func (s) TestClientTransportDrainsAfterStreamIDExhausted(t *testing.T) { + server, ct, cancel := setUp(t, 0, math.MaxUint32, normal) + defer cancel() + defer server.stop() + callHdr := &CallHdr{ + Host: "localhost", + Method: "foo.Small", + } + + originalMaxStreamID := MaxStreamID + MaxStreamID = 3 + defer func() { + MaxStreamID = originalMaxStreamID + }() + + ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer ctxCancel() + + s, err := ct.NewStream(ctx, callHdr) + if err != nil { + t.Fatalf("ct.NewStream() = %v", err) + } + if s.id != 1 { + t.Fatalf("Stream id: %d, want: 1", s.id) + } + + if got, want := ct.stateForTesting(), reachable; got != want { + t.Fatalf("Client transport state %v, want %v", got, want) + } + + // The expected stream ID here is 3 since stream IDs are incremented by 2. + s, err = ct.NewStream(ctx, callHdr) + if err != nil { + t.Fatalf("ct.NewStream() = %v", err) + } + if s.id != 3 { + t.Fatalf("Stream id: %d, want: 3", s.id) + } + + // Verifying that ct.state is draining when next stream ID > MaxStreamId. + if got, want := ct.stateForTesting(), draining; got != want { + t.Fatalf("Client transport state %v, want %v", got, want) + } +} + func (s) TestClientSendAndReceive(t *testing.T) { server, ct, cancel := setUp(t, 0, math.MaxUint32, normal) defer cancel() @@ -519,14 +590,16 @@ func (s) TestClientSendAndReceive(t *testing.T) { Host: "localhost", Method: "foo.Small", } - s1, err1 := ct.NewStream(context.Background(), callHdr) + ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer ctxCancel() + s1, err1 := ct.NewStream(ctx, callHdr) if err1 != nil { t.Fatalf("failed to open stream: %v", err1) } if s1.id != 1 { t.Fatalf("wrong stream id: %d", s1.id) } - s2, err2 := ct.NewStream(context.Background(), callHdr) + s2, err2 := ct.NewStream(ctx, callHdr) if err2 != nil { t.Fatalf("failed to open stream: %v", err2) } @@ -546,7 +619,7 @@ func (s) TestClientSendAndReceive(t *testing.T) { if recvErr != io.EOF { t.Fatalf("Error: %v; want ", recvErr) } - ct.Close() + ct.Close(fmt.Errorf("closed manually by test")) server.stop() } @@ -556,7 +629,7 @@ func (s) TestClientErrorNotify(t *testing.T) { go server.stop() // ct.reader should detect the error and activate ct.Error(). <-ct.Error() - ct.Close() + ct.Close(fmt.Errorf("closed manually by test")) } func performOneRPC(ct ClientTransport) { @@ -564,7 +637,9 @@ func performOneRPC(ct ClientTransport) { Host: "localhost", Method: "foo.Small", } - s, err := ct.NewStream(context.Background(), callHdr) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + s, err := ct.NewStream(ctx, callHdr) if err != nil { return } @@ -585,16 +660,13 @@ func performOneRPC(ct ClientTransport) { func (s) TestClientMix(t *testing.T) { s, ct, cancel := setUp(t, 0, math.MaxUint32, normal) defer cancel() - go func(s *server) { - time.Sleep(5 * time.Second) - s.stop() - }(s) + time.AfterFunc(time.Second, s.stop) go func(ct ClientTransport) { <-ct.Error() - ct.Close() + ct.Close(fmt.Errorf("closed manually by test")) }(ct) - for i := 0; i < 1000; i++ { - time.Sleep(10 * time.Millisecond) + for i := 0; i < 750; i++ { + time.Sleep(2 * time.Millisecond) go performOneRPC(ct) } } @@ -606,12 +678,14 @@ func (s) TestLargeMessage(t *testing.T) { Host: "localhost", Method: "foo.Large", } + ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer ctxCancel() var wg sync.WaitGroup for i := 0; i < 2; i++ { wg.Add(1) go func() { defer wg.Done() - s, err := ct.NewStream(context.Background(), callHdr) + s, err := ct.NewStream(ctx, callHdr) if err != nil { t.Errorf("%v.NewStream(_, _) = _, %v, want _, ", ct, err) } @@ -628,7 +702,7 @@ func (s) TestLargeMessage(t *testing.T) { }() } wg.Wait() - ct.Close() + ct.Close(fmt.Errorf("closed manually by test")) server.stop() } @@ -645,7 +719,7 @@ func (s) TestLargeMessageWithDelayRead(t *testing.T) { server, ct, cancel := setUpWithOptions(t, 0, sc, delayRead, co) defer cancel() defer server.stop() - defer ct.Close() + defer ct.Close(fmt.Errorf("closed manually by test")) server.mu.Lock() ready := server.ready server.mu.Unlock() @@ -729,6 +803,9 @@ func (s) TestLargeMessageWithDelayRead(t *testing.T) { } } +// TestGracefulClose ensures that GracefulClose allows in-flight streams to +// proceed until they complete naturally, while not allowing creation of new +// streams during this window. func (s) TestGracefulClose(t *testing.T) { server, ct, cancel := setUp(t, 0, math.MaxUint32, pingpong) defer cancel() @@ -744,6 +821,9 @@ func (s) TestGracefulClose(t *testing.T) { }() ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*10)) defer cancel() + + // Create a stream that will exist for this whole test and confirm basic + // functionality. s, err := ct.NewStream(ctx, &CallHdr{}) if err != nil { t.Fatalf("NewStream(_, _) = _, %v, want _, ", err) @@ -764,31 +844,31 @@ func (s) TestGracefulClose(t *testing.T) { if _, err := s.Read(recvMsg); err != nil { t.Fatalf("Error while reading: %v", err) } + + // Gracefully close the transport, which should not affect the existing + // stream. ct.GracefulClose() + var wg sync.WaitGroup - // Expect the failure for all the follow-up streams because ct has been closed gracefully. + // Expect errors creating new streams because the client transport has been + // gracefully closed. for i := 0; i < 200; i++ { wg.Add(1) go func() { defer wg.Done() - str, err := ct.NewStream(context.Background(), &CallHdr{}) - if err == ErrConnClosing { + _, err := ct.NewStream(ctx, &CallHdr{}) + if err != nil && err.(*NewStreamError).Err == ErrConnClosing && err.(*NewStreamError).AllowTransparentRetry { return - } else if err != nil { - t.Errorf("_.NewStream(_, _) = _, %v, want _, %v", err, ErrConnClosing) - return - } - ct.Write(str, nil, nil, &Options{Last: true}) - if _, err := str.Read(make([]byte, 8)); err != errStreamDrain && err != ErrConnClosing { - t.Errorf("_.Read(_) = _, %v, want _, %v or %v", err, errStreamDrain, ErrConnClosing) } + t.Errorf("_.NewStream(_, _) = _, %v, want _, %v", err, ErrConnClosing) }() } + + // Confirm the existing stream still functions as expected. ct.Write(s, nil, nil, &Options{Last: true}) if _, err := s.Read(incomingHeader); err != io.EOF { t.Fatalf("Client expected EOF from the server. Got: %v", err) } - // The stream which was created before graceful close can still proceed. wg.Wait() } @@ -823,7 +903,7 @@ func (s) TestLargeMessageSuspension(t *testing.T) { if _, err := s.Read(make([]byte, 8)); err.Error() != expectedErr.Error() { t.Fatalf("Read got %v of type %T, want %v", err, err, expectedErr) } - ct.Close() + ct.Close(fmt.Errorf("closed manually by test")) server.stop() } @@ -833,13 +913,15 @@ func (s) TestMaxStreams(t *testing.T) { } server, ct, cancel := setUpWithOptions(t, 0, serverConfig, suspended, ConnectOptions{}) defer cancel() - defer ct.Close() + defer ct.Close(fmt.Errorf("closed manually by test")) defer server.stop() callHdr := &CallHdr{ Host: "localhost", Method: "foo.Large", } - s, err := ct.NewStream(context.Background(), callHdr) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + s, err := ct.NewStream(ctx, callHdr) if err != nil { t.Fatalf("Failed to open stream: %v", err) } @@ -891,7 +973,7 @@ func (s) TestMaxStreams(t *testing.T) { // Close the first stream created so that the new stream can finally be created. ct.CloseStream(s, nil) <-done - ct.Close() + ct.Close(fmt.Errorf("closed manually by test")) <-ct.writerDone if ct.maxConcurrentStreams != 1 { t.Fatalf("ct.maxConcurrentStreams: %d, want 1", ct.maxConcurrentStreams) @@ -924,7 +1006,9 @@ func (s) TestServerContextCanceledOnClosedConnection(t *testing.T) { server.mu.Unlock() break } - s, err := ct.NewStream(context.Background(), callHdr) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + s, err := ct.NewStream(ctx, callHdr) if err != nil { t.Fatalf("Failed to open stream: %v", err) } @@ -948,7 +1032,7 @@ func (s) TestServerContextCanceledOnClosedConnection(t *testing.T) { sc.mu.Unlock() break } - ct.Close() + ct.Close(fmt.Errorf("closed manually by test")) select { case <-ss.Context().Done(): if ss.Context().Err() != context.Canceled { @@ -968,7 +1052,7 @@ func (s) TestClientConnDecoupledFromApplicationRead(t *testing.T) { server, client, cancel := setUpWithOptions(t, 0, &ServerConfig{}, notifyCall, connectOptions) defer cancel() defer server.stop() - defer client.Close() + defer client.Close(fmt.Errorf("closed manually by test")) waitWhileTrue(t, func() (bool, error) { server.mu.Lock() @@ -988,7 +1072,9 @@ func (s) TestClientConnDecoupledFromApplicationRead(t *testing.T) { notifyChan := make(chan struct{}) server.h.notify = notifyChan server.mu.Unlock() - cstream1, err := client.NewStream(context.Background(), &CallHdr{}) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + cstream1, err := client.NewStream(ctx, &CallHdr{}) if err != nil { t.Fatalf("Client failed to create first stream. Err: %v", err) } @@ -1015,7 +1101,7 @@ func (s) TestClientConnDecoupledFromApplicationRead(t *testing.T) { server.h.notify = notifyChan server.mu.Unlock() // Create another stream on client. - cstream2, err := client.NewStream(context.Background(), &CallHdr{}) + cstream2, err := client.NewStream(ctx, &CallHdr{}) if err != nil { t.Fatalf("Client failed to create second stream. Err: %v", err) } @@ -1055,7 +1141,7 @@ func (s) TestServerConnDecoupledFromApplicationRead(t *testing.T) { server, client, cancel := setUpWithOptions(t, 0, serverConfig, suspended, ConnectOptions{}) defer cancel() defer server.stop() - defer client.Close() + defer client.Close(fmt.Errorf("closed manually by test")) waitWhileTrue(t, func() (bool, error) { server.mu.Lock() defer server.mu.Unlock() @@ -1070,8 +1156,10 @@ func (s) TestServerConnDecoupledFromApplicationRead(t *testing.T) { for k := range server.conns { st = k.(*http2Server) } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() server.mu.Unlock() - cstream1, err := client.NewStream(context.Background(), &CallHdr{}) + cstream1, err := client.NewStream(ctx, &CallHdr{}) if err != nil { t.Fatalf("Failed to create 1st stream. Err: %v", err) } @@ -1080,7 +1168,7 @@ func (s) TestServerConnDecoupledFromApplicationRead(t *testing.T) { t.Fatalf("Client failed to write data. Err: %v", err) } //Client should be able to create another stream and send data on it. - cstream2, err := client.NewStream(context.Background(), &CallHdr{}) + cstream2, err := client.NewStream(ctx, &CallHdr{}) if err != nil { t.Fatalf("Failed to create 2nd stream. Err: %v", err) } @@ -1216,6 +1304,58 @@ func (s) TestServerWithMisbehavedClient(t *testing.T) { } } +func (s) TestClientHonorsConnectContext(t *testing.T) { + // Create a server that will not send a preface. + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Error while listening: %v", err) + } + defer lis.Close() + go func() { // Launch the misbehaving server. + sconn, err := lis.Accept() + if err != nil { + t.Errorf("Error while accepting: %v", err) + return + } + defer sconn.Close() + if _, err := io.ReadFull(sconn, make([]byte, len(clientPreface))); err != nil { + t.Errorf("Error while reading client preface: %v", err) + return + } + sfr := http2.NewFramer(sconn, sconn) + // Do not write a settings frame, but read from the conn forever. + for { + if _, err := sfr.ReadFrame(); err != nil { + return + } + } + }() + + // Test context cancelation. + timeBefore := time.Now() + connectCtx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + time.AfterFunc(100*time.Millisecond, cancel) + + copts := ConnectOptions{ChannelzParentID: channelz.NewIdentifierForTesting(channelz.RefSubChannel, time.Now().Unix(), nil)} + _, err = NewClientTransport(connectCtx, context.Background(), resolver.Address{Addr: lis.Addr().String()}, copts, func(GoAwayReason) {}) + if err == nil { + t.Fatalf("NewClientTransport() returned successfully; wanted error") + } + t.Logf("NewClientTransport() = _, %v", err) + if time.Since(timeBefore) > 3*time.Second { + t.Fatalf("NewClientTransport returned > 2.9s after context cancelation") + } + + // Test context deadline. + connectCtx, cancel = context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + _, err = NewClientTransport(connectCtx, context.Background(), resolver.Address{Addr: lis.Addr().String()}, copts, func(GoAwayReason) {}) + if err == nil { + t.Fatalf("NewClientTransport() returned successfully; wanted error") + } + t.Logf("NewClientTransport() = _, %v", err) +} + func (s) TestClientWithMisbehavedServer(t *testing.T) { // Create a misbehaving server. lis, err := net.Listen("tcp", "localhost:0") @@ -1234,10 +1374,14 @@ func (s) TestClientWithMisbehavedServer(t *testing.T) { } defer sconn.Close() if _, err := io.ReadFull(sconn, make([]byte, len(clientPreface))); err != nil { - t.Errorf("Error while reading clieng preface: %v", err) + t.Errorf("Error while reading client preface: %v", err) return } sfr := http2.NewFramer(sconn, sconn) + if err := sfr.WriteSettings(); err != nil { + t.Errorf("Error while writing settings: %v", err) + return + } if err := sfr.WriteSettingsAck(); err != nil { t.Errorf("Error while writing settings: %v", err) return @@ -1282,12 +1426,15 @@ func (s) TestClientWithMisbehavedServer(t *testing.T) { }() connectCtx, cancel := context.WithDeadline(context.Background(), time.Now().Add(2*time.Second)) defer cancel() - ct, err := NewClientTransport(connectCtx, context.Background(), resolver.Address{Addr: lis.Addr().String()}, ConnectOptions{}, func() {}, func(GoAwayReason) {}, func() {}) + + copts := ConnectOptions{ChannelzParentID: channelz.NewIdentifierForTesting(channelz.RefSubChannel, time.Now().Unix(), nil)} + ct, err := NewClientTransport(connectCtx, context.Background(), resolver.Address{Addr: lis.Addr().String()}, copts, func(GoAwayReason) {}) if err != nil { t.Fatalf("Error while creating client transport: %v", err) } - defer ct.Close() - str, err := ct.NewStream(context.Background(), &CallHdr{}) + defer ct.Close(fmt.Errorf("closed manually by test")) + + str, err := ct.NewStream(connectCtx, &CallHdr{}) if err != nil { t.Fatalf("Error while creating stream: %v", err) } @@ -1312,7 +1459,9 @@ func (s) TestEncodingRequiredStatus(t *testing.T) { Host: "localhost", Method: "foo", } - s, err := ct.NewStream(context.Background(), callHdr) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + s, err := ct.NewStream(ctx, callHdr) if err != nil { return } @@ -1327,7 +1476,7 @@ func (s) TestEncodingRequiredStatus(t *testing.T) { if !testutils.StatusErrEqual(s.Status().Err(), encodingTestStatus.Err()) { t.Fatalf("stream with status %v, want %v", s.Status(), encodingTestStatus) } - ct.Close() + ct.Close(fmt.Errorf("closed manually by test")) server.stop() } @@ -1338,7 +1487,9 @@ func (s) TestInvalidHeaderField(t *testing.T) { Host: "localhost", Method: "foo", } - s, err := ct.NewStream(context.Background(), callHdr) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + s, err := ct.NewStream(ctx, callHdr) if err != nil { return } @@ -1347,7 +1498,7 @@ func (s) TestInvalidHeaderField(t *testing.T) { if se, ok := status.FromError(err); !ok || se.Code() != codes.Internal || !strings.Contains(err.Error(), expectedInvalidHeaderField) { t.Fatalf("Read got error %v, want error with code %s and contains %q", err, codes.Internal, expectedInvalidHeaderField) } - ct.Close() + ct.Close(fmt.Errorf("closed manually by test")) server.stop() } @@ -1355,8 +1506,10 @@ func (s) TestHeaderChanClosedAfterReceivingAnInvalidHeader(t *testing.T) { server, ct, cancel := setUp(t, 0, math.MaxUint32, invalidHeaderField) defer cancel() defer server.stop() - defer ct.Close() - s, err := ct.NewStream(context.Background(), &CallHdr{Host: "localhost", Method: "foo"}) + defer ct.Close(fmt.Errorf("closed manually by test")) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + s, err := ct.NewStream(ctx, &CallHdr{Host: "localhost", Method: "foo"}) if err != nil { t.Fatalf("failed to create the stream") } @@ -1429,9 +1582,10 @@ func (s) TestAccountCheckWindowSizeWithLargeWindow(t *testing.T) { } func (s) TestAccountCheckWindowSizeWithSmallWindow(t *testing.T) { + // These settings disable dynamic window sizes based on BDP estimation; + // must be at least defaultWindowSize or the setting is ignored. wc := windowSizeConfig{ serverStream: defaultWindowSize, - // Note this is smaller than initialConnWindowSize which is the current default. serverConn: defaultWindowSize, clientStream: defaultWindowSize, clientConn: defaultWindowSize, @@ -1459,7 +1613,7 @@ func testFlowControlAccountCheck(t *testing.T, msgSize int, wc windowSizeConfig) server, client, cancel := setUpWithOptions(t, 0, sc, pingpong, co) defer cancel() defer server.stop() - defer client.Close() + defer client.Close(fmt.Errorf("closed manually by test")) waitWhileTrue(t, func() (bool, error) { server.mu.Lock() defer server.mu.Unlock() @@ -1474,11 +1628,14 @@ func testFlowControlAccountCheck(t *testing.T, msgSize int, wc windowSizeConfig) st = k.(*http2Server) } server.mu.Unlock() - const numStreams = 10 + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + const numStreams = 5 clientStreams := make([]*Stream, numStreams) for i := 0; i < numStreams; i++ { var err error - clientStreams[i], err = client.NewStream(context.Background(), &CallHdr{}) + clientStreams[i], err = client.NewStream(ctx, &CallHdr{}) if err != nil { t.Fatalf("Failed to create stream. Err: %v", err) } @@ -1494,26 +1651,27 @@ func testFlowControlAccountCheck(t *testing.T, msgSize int, wc windowSizeConfig) binary.BigEndian.PutUint32(buf[1:], uint32(msgSize)) opts := Options{} header := make([]byte, 5) - for i := 1; i <= 10; i++ { + for i := 1; i <= 5; i++ { if err := client.Write(stream, nil, buf, &opts); err != nil { - t.Errorf("Error on client while writing message: %v", err) + t.Errorf("Error on client while writing message %v on stream %v: %v", i, stream.id, err) return } if _, err := stream.Read(header); err != nil { - t.Errorf("Error on client while reading data frame header: %v", err) + t.Errorf("Error on client while reading data frame header %v on stream %v: %v", i, stream.id, err) return } sz := binary.BigEndian.Uint32(header[1:]) recvMsg := make([]byte, int(sz)) if _, err := stream.Read(recvMsg); err != nil { - t.Errorf("Error on client while reading data: %v", err) + t.Errorf("Error on client while reading data %v on stream %v: %v", i, stream.id, err) return } if len(recvMsg) != msgSize { - t.Errorf("Length of message received by client: %v, want: %v", len(recvMsg), msgSize) + t.Errorf("Length of message %v received by client on stream %v: %v, want: %v", i, stream.id, len(recvMsg), msgSize) return } } + t.Logf("stream %v done with pingpongs", stream.id) }(stream) } wg.Wait() @@ -1522,6 +1680,7 @@ func testFlowControlAccountCheck(t *testing.T, msgSize int, wc windowSizeConfig) loopyServerStreams := map[uint32]*outStream{} // Get all the streams from server reader and writer and client writer. st.mu.Lock() + client.mu.Lock() for _, stream := range clientStreams { id := stream.id serverStreams[id] = st.activeStreams[id] @@ -1529,6 +1688,7 @@ func testFlowControlAccountCheck(t *testing.T, msgSize int, wc windowSizeConfig) loopyClientStreams[id] = client.loopy.estdStreams[id] } + client.mu.Unlock() st.mu.Unlock() // Close all streams for _, stream := range clientStreams { @@ -1539,8 +1699,8 @@ func testFlowControlAccountCheck(t *testing.T, msgSize int, wc windowSizeConfig) } // Close down both server and client so that their internals can be read without data // races. - client.Close() - st.Close() + client.Close(errors.New("closed manually by test")) + st.Close(errors.New("closed manually by test")) <-st.readerDone <-st.writerDone <-client.readerDone @@ -1550,6 +1710,9 @@ func testFlowControlAccountCheck(t *testing.T, msgSize int, wc windowSizeConfig) sstream := serverStreams[id] loopyServerStream := loopyServerStreams[id] loopyClientStream := loopyClientStreams[id] + if loopyServerStream == nil { + t.Fatalf("Unexpected nil loopyServerStream") + } // Check stream flow control. if int(cstream.fc.limit+cstream.fc.delta-cstream.fc.pendingData-cstream.fc.pendingUpdate) != int(st.loopy.oiws)-loopyServerStream.bytesOutStanding { t.Fatalf("Account mismatch: client stream inflow limit(%d) + delta(%d) - pendingData(%d) - pendingUpdate(%d) != server outgoing InitialWindowSize(%d) - outgoingStream.bytesOutStanding(%d)", cstream.fc.limit, cstream.fc.delta, cstream.fc.pendingData, cstream.fc.pendingUpdate, st.loopy.oiws, loopyServerStream.bytesOutStanding) @@ -1639,6 +1802,357 @@ func (s) TestReadGivesSameErrorAfterAnyErrorOccurs(t *testing.T) { } } +// TestHeadersCausingStreamError tests headers that should cause a stream protocol +// error, which would end up with a RST_STREAM being sent to the client and also +// the server closing the stream. +func (s) TestHeadersCausingStreamError(t *testing.T) { + tests := []struct { + name string + headers []struct { + name string + values []string + } + }{ + // "Transports must consider requests containing the Connection header + // as malformed" - A41 Malformed requests map to a stream error of type + // PROTOCOL_ERROR. + { + name: "Connection header present", + headers: []struct { + name string + values []string + }{ + {name: ":method", values: []string{"POST"}}, + {name: ":path", values: []string{"foo"}}, + {name: ":authority", values: []string{"localhost"}}, + {name: "content-type", values: []string{"application/grpc"}}, + {name: "connection", values: []string{"not-supported"}}, + }, + }, + // multiple :authority or multiple Host headers would make the eventual + // :authority ambiguous as per A41. Since these headers won't have a + // content-type that corresponds to a grpc-client, the server should + // simply write a RST_STREAM to the wire. + { + // Note: multiple authority headers are handled by the framer + // itself, which will cause a stream error. Thus, it will never get + // to operateHeaders with the check in operateHeaders for stream + // error, but the server transport will still send a stream error. + name: "Multiple authority headers", + headers: []struct { + name string + values []string + }{ + {name: ":method", values: []string{"POST"}}, + {name: ":path", values: []string{"foo"}}, + {name: ":authority", values: []string{"localhost", "localhost2"}}, + {name: "host", values: []string{"localhost"}}, + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + server := setUpServerOnly(t, 0, &ServerConfig{}, suspended) + defer server.stop() + // Create a client directly to not tie what you can send to API of + // http2_client.go (i.e. control headers being sent). + mconn, err := net.Dial("tcp", server.lis.Addr().String()) + if err != nil { + t.Fatalf("Client failed to dial: %v", err) + } + defer mconn.Close() + + if n, err := mconn.Write(clientPreface); err != nil || n != len(clientPreface) { + t.Fatalf("mconn.Write(clientPreface) = %d, %v, want %d, ", n, err, len(clientPreface)) + } + + framer := http2.NewFramer(mconn, mconn) + if err := framer.WriteSettings(); err != nil { + t.Fatalf("Error while writing settings: %v", err) + } + + // result chan indicates that reader received a RSTStream from server. + // An error will be passed on it if any other frame is received. + result := testutils.NewChannel() + + // Launch a reader goroutine. + go func() { + for { + frame, err := framer.ReadFrame() + if err != nil { + return + } + switch frame := frame.(type) { + case *http2.SettingsFrame: + // Do nothing. A settings frame is expected from server preface. + case *http2.RSTStreamFrame: + if frame.Header().StreamID != 1 || http2.ErrCode(frame.ErrCode) != http2.ErrCodeProtocol { + // Client only created a single stream, so RST Stream should be for that single stream. + result.Send(fmt.Errorf("RST stream received with streamID: %d and code %v, want streamID: 1 and code: http.ErrCodeFlowControl", frame.Header().StreamID, http2.ErrCode(frame.ErrCode))) + } + // Records that client successfully received RST Stream frame. + result.Send(nil) + return + default: + // The server should send nothing but a single RST Stream frame. + result.Send(errors.New("the client received a frame other than RST Stream")) + } + } + }() + + var buf bytes.Buffer + henc := hpack.NewEncoder(&buf) + + // Needs to build headers deterministically to conform to gRPC over + // HTTP/2 spec. + for _, header := range test.headers { + for _, value := range header.values { + if err := henc.WriteField(hpack.HeaderField{Name: header.name, Value: value}); err != nil { + t.Fatalf("Error while encoding header: %v", err) + } + } + } + + if err := framer.WriteHeaders(http2.HeadersFrameParam{StreamID: 1, BlockFragment: buf.Bytes(), EndHeaders: true}); err != nil { + t.Fatalf("Error while writing headers: %v", err) + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + r, err := result.Receive(ctx) + if err != nil { + t.Fatalf("Error receiving from channel: %v", err) + } + if r != nil { + t.Fatalf("want nil, got %v", r) + } + }) + } +} + +// TestHeadersHTTPStatusGRPCStatus tests requests with certain headers get a +// certain HTTP and gRPC status back. +func (s) TestHeadersHTTPStatusGRPCStatus(t *testing.T) { + tests := []struct { + name string + headers []struct { + name string + values []string + } + httpStatusWant string + grpcStatusWant string + grpcMessageWant string + }{ + // Note: multiple authority headers are handled by the framer itself, + // which will cause a stream error. Thus, it will never get to + // operateHeaders with the check in operateHeaders for possible + // grpc-status sent back. + + // multiple :authority or multiple Host headers would make the eventual + // :authority ambiguous as per A41. This takes precedence even over the + // fact a request is non grpc. All of these requests should be rejected + // with grpc-status Internal. Thus, requests with multiple hosts should + // get rejected with HTTP Status 400 and gRPC status Internal, + // regardless of whether the client is speaking gRPC or not. + { + name: "Multiple host headers non grpc", + headers: []struct { + name string + values []string + }{ + {name: ":method", values: []string{"POST"}}, + {name: ":path", values: []string{"foo"}}, + {name: ":authority", values: []string{"localhost"}}, + {name: "host", values: []string{"localhost", "localhost2"}}, + }, + httpStatusWant: "400", + grpcStatusWant: "13", + grpcMessageWant: "both must only have 1 value as per HTTP/2 spec", + }, + { + name: "Multiple host headers grpc", + headers: []struct { + name string + values []string + }{ + {name: ":method", values: []string{"POST"}}, + {name: ":path", values: []string{"foo"}}, + {name: ":authority", values: []string{"localhost"}}, + {name: "content-type", values: []string{"application/grpc"}}, + {name: "host", values: []string{"localhost", "localhost2"}}, + }, + httpStatusWant: "400", + grpcStatusWant: "13", + grpcMessageWant: "both must only have 1 value as per HTTP/2 spec", + }, + // If the client sends an HTTP/2 request with a :method header with a + // value other than POST, as specified in the gRPC over HTTP/2 + // specification, the server should fail the RPC. + { + name: "Client Sending Wrong Method", + headers: []struct { + name string + values []string + }{ + {name: ":method", values: []string{"PUT"}}, + {name: ":path", values: []string{"foo"}}, + {name: ":authority", values: []string{"localhost"}}, + {name: "content-type", values: []string{"application/grpc"}}, + }, + httpStatusWant: "405", + grpcStatusWant: "13", + grpcMessageWant: "which should be POST", + }, + { + name: "Client Sending Wrong Content-Type", + headers: []struct { + name string + values []string + }{ + {name: ":method", values: []string{"POST"}}, + {name: ":path", values: []string{"foo"}}, + {name: ":authority", values: []string{"localhost"}}, + {name: "content-type", values: []string{"application/json"}}, + }, + httpStatusWant: "415", + grpcStatusWant: "3", + grpcMessageWant: `invalid gRPC request content-type "application/json"`, + }, + { + name: "Client Sending Bad Timeout", + headers: []struct { + name string + values []string + }{ + {name: ":method", values: []string{"POST"}}, + {name: ":path", values: []string{"foo"}}, + {name: ":authority", values: []string{"localhost"}}, + {name: "content-type", values: []string{"application/grpc"}}, + {name: "grpc-timeout", values: []string{"18f6n"}}, + }, + httpStatusWant: "400", + grpcStatusWant: "13", + grpcMessageWant: "malformed grpc-timeout", + }, + { + name: "Client Sending Bad Binary Header", + headers: []struct { + name string + values []string + }{ + {name: ":method", values: []string{"POST"}}, + {name: ":path", values: []string{"foo"}}, + {name: ":authority", values: []string{"localhost"}}, + {name: "content-type", values: []string{"application/grpc"}}, + {name: "foobar-bin", values: []string{"X()3e@#$-"}}, + }, + httpStatusWant: "400", + grpcStatusWant: "13", + grpcMessageWant: `header "foobar-bin": illegal base64 data`, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + server := setUpServerOnly(t, 0, &ServerConfig{}, suspended) + defer server.stop() + // Create a client directly to not tie what you can send to API of + // http2_client.go (i.e. control headers being sent). + mconn, err := net.Dial("tcp", server.lis.Addr().String()) + if err != nil { + t.Fatalf("Client failed to dial: %v", err) + } + defer mconn.Close() + + if n, err := mconn.Write(clientPreface); err != nil || n != len(clientPreface) { + t.Fatalf("mconn.Write(clientPreface) = %d, %v, want %d, ", n, err, len(clientPreface)) + } + + framer := http2.NewFramer(mconn, mconn) + framer.ReadMetaHeaders = hpack.NewDecoder(4096, nil) + if err := framer.WriteSettings(); err != nil { + t.Fatalf("Error while writing settings: %v", err) + } + + // result chan indicates that reader received a Headers Frame with + // desired grpc status and message from server. An error will be passed + // on it if any other frame is received. + result := testutils.NewChannel() + + // Launch a reader goroutine. + go func() { + for { + frame, err := framer.ReadFrame() + if err != nil { + return + } + switch frame := frame.(type) { + case *http2.SettingsFrame: + // Do nothing. A settings frame is expected from server preface. + case *http2.MetaHeadersFrame: + var httpStatus, grpcStatus, grpcMessage string + for _, header := range frame.Fields { + if header.Name == ":status" { + httpStatus = header.Value + } + if header.Name == "grpc-status" { + grpcStatus = header.Value + } + if header.Name == "grpc-message" { + grpcMessage = header.Value + } + } + if httpStatus != test.httpStatusWant { + result.Send(fmt.Errorf("incorrect HTTP Status got %v, want %v", httpStatus, test.httpStatusWant)) + return + } + if grpcStatus != test.grpcStatusWant { // grpc status code internal + result.Send(fmt.Errorf("incorrect gRPC Status got %v, want %v", grpcStatus, test.grpcStatusWant)) + return + } + if !strings.Contains(grpcMessage, test.grpcMessageWant) { + result.Send(fmt.Errorf("incorrect gRPC message, want %q got %q", test.grpcMessageWant, grpcMessage)) + return + } + + // Records that client successfully received a HeadersFrame + // with expected Trailers-Only response. + result.Send(nil) + return + default: + // The server should send nothing but a single Settings and Headers frame. + result.Send(errors.New("the client received a frame other than Settings or Headers")) + } + } + }() + + var buf bytes.Buffer + henc := hpack.NewEncoder(&buf) + + // Needs to build headers deterministically to conform to gRPC over + // HTTP/2 spec. + for _, header := range test.headers { + for _, value := range header.values { + if err := henc.WriteField(hpack.HeaderField{Name: header.name, Value: value}); err != nil { + t.Fatalf("Error while encoding header: %v", err) + } + } + } + + if err := framer.WriteHeaders(http2.HeadersFrameParam{StreamID: 1, BlockFragment: buf.Bytes(), EndHeaders: true}); err != nil { + t.Fatalf("Error while writing headers: %v", err) + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + r, err := result.Receive(ctx) + if err != nil { + t.Fatalf("Error receiving from channel: %v", err) + } + if r != nil { + t.Fatalf("want nil, got %v", r) + } + }) + } +} + func (s) TestPingPong1B(t *testing.T) { runPingPongTest(t, 1) } @@ -1655,12 +2169,12 @@ func (s) TestPingPong1MB(t *testing.T) { runPingPongTest(t, 1048576) } -//This is a stress-test of flow control logic. +// This is a stress-test of flow control logic. func runPingPongTest(t *testing.T, msgSize int) { server, client, cancel := setUp(t, 0, 0, pingpong) defer cancel() defer server.stop() - defer client.Close() + defer client.Close(fmt.Errorf("closed manually by test")) waitWhileTrue(t, func() (bool, error) { server.mu.Lock() defer server.mu.Unlock() @@ -1669,7 +2183,9 @@ func runPingPongTest(t *testing.T, msgSize int) { } return false, nil }) - stream, err := client.NewStream(context.Background(), &CallHdr{}) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + stream, err := client.NewStream(ctx, &CallHdr{}) if err != nil { t.Fatalf("Failed to create stream. Err: %v", err) } @@ -1679,33 +2195,26 @@ func runPingPongTest(t *testing.T, msgSize int) { binary.BigEndian.PutUint32(outgoingHeader[1:], uint32(msgSize)) opts := &Options{} incomingHeader := make([]byte, 5) - done := make(chan struct{}) - go func() { - timer := time.NewTimer(time.Second * 5) - <-timer.C - close(done) - }() - for { - select { - case <-done: - client.Write(stream, nil, nil, &Options{Last: true}) - if _, err := stream.Read(incomingHeader); err != io.EOF { - t.Fatalf("Client expected EOF from the server. Got: %v", err) - } - return - default: - if err := client.Write(stream, outgoingHeader, msg, opts); err != nil { - t.Fatalf("Error on client while writing message. Err: %v", err) - } - if _, err := stream.Read(incomingHeader); err != nil { - t.Fatalf("Error on client while reading data header. Err: %v", err) - } - sz := binary.BigEndian.Uint32(incomingHeader[1:]) - recvMsg := make([]byte, int(sz)) - if _, err := stream.Read(recvMsg); err != nil { - t.Fatalf("Error on client while reading data. Err: %v", err) - } + + ctx, cancel = context.WithTimeout(ctx, time.Second) + defer cancel() + for ctx.Err() == nil { + if err := client.Write(stream, outgoingHeader, msg, opts); err != nil { + t.Fatalf("Error on client while writing message. Err: %v", err) } + if _, err := stream.Read(incomingHeader); err != nil { + t.Fatalf("Error on client while reading data header. Err: %v", err) + } + sz := binary.BigEndian.Uint32(incomingHeader[1:]) + recvMsg := make([]byte, int(sz)) + if _, err := stream.Read(recvMsg); err != nil { + t.Fatalf("Error on client while reading data. Err: %v", err) + } + } + + client.Write(stream, nil, nil, &Options{Last: true}) + if _, err := stream.Read(incomingHeader); err != io.EOF { + t.Fatalf("Client expected EOF from the server. Got: %v", err) } } @@ -1746,9 +2255,11 @@ func (s) TestHeaderTblSize(t *testing.T) { server, ct, cancel := setUp(t, 0, math.MaxUint32, normal) defer cancel() - defer ct.Close() + defer ct.Close(fmt.Errorf("closed manually by test")) defer server.stop() - _, err := ct.NewStream(context.Background(), &CallHdr{}) + ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer ctxCancel() + _, err := ct.NewStream(ctx, &CallHdr{}) if err != nil { t.Fatalf("failed to open stream: %v", err) } @@ -1859,14 +2370,291 @@ func (s) TestClientHandshakeInfo(t *testing.T) { defer cancel() creds := &attrTransportCreds{} - tr, err := NewClientTransport(ctx, context.Background(), addr, ConnectOptions{TransportCredentials: creds}, func() {}, func(GoAwayReason) {}, func() {}) + copts := ConnectOptions{ + TransportCredentials: creds, + ChannelzParentID: channelz.NewIdentifierForTesting(channelz.RefSubChannel, time.Now().Unix(), nil), + } + tr, err := NewClientTransport(ctx, context.Background(), addr, copts, func(GoAwayReason) {}) if err != nil { t.Fatalf("NewClientTransport(): %v", err) } - defer tr.Close() + defer tr.Close(fmt.Errorf("closed manually by test")) wantAttr := attributes.New(testAttrKey, testAttrVal) if gotAttr := creds.attr; !cmp.Equal(gotAttr, wantAttr, cmp.AllowUnexported(attributes.Attributes{})) { t.Fatalf("received attributes %v in creds, want %v", gotAttr, wantAttr) } } + +// TestClientHandshakeInfoDialer adds attributes to the resolver.Address passes to +// NewClientTransport and verifies that these attributes are received by a custom +// dialer. +func (s) TestClientHandshakeInfoDialer(t *testing.T) { + server := setUpServerOnly(t, 0, &ServerConfig{}, pingpong) + defer server.stop() + + const ( + testAttrKey = "foo" + testAttrVal = "bar" + ) + addr := resolver.Address{ + Addr: "localhost:" + server.port, + Attributes: attributes.New(testAttrKey, testAttrVal), + } + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(2*time.Second)) + defer cancel() + + var attr *attributes.Attributes + dialer := func(ctx context.Context, addr string) (net.Conn, error) { + ai := credentials.ClientHandshakeInfoFromContext(ctx) + attr = ai.Attributes + return (&net.Dialer{}).DialContext(ctx, "tcp", addr) + } + + copts := ConnectOptions{ + Dialer: dialer, + ChannelzParentID: channelz.NewIdentifierForTesting(channelz.RefSubChannel, time.Now().Unix(), nil), + } + tr, err := NewClientTransport(ctx, context.Background(), addr, copts, func(GoAwayReason) {}) + if err != nil { + t.Fatalf("NewClientTransport(): %v", err) + } + defer tr.Close(fmt.Errorf("closed manually by test")) + + wantAttr := attributes.New(testAttrKey, testAttrVal) + if gotAttr := attr; !cmp.Equal(gotAttr, wantAttr, cmp.AllowUnexported(attributes.Attributes{})) { + t.Errorf("Received attributes %v in custom dialer, want %v", gotAttr, wantAttr) + } +} + +func (s) TestClientDecodeHeaderStatusErr(t *testing.T) { + testStream := func() *Stream { + return &Stream{ + done: make(chan struct{}), + headerChan: make(chan struct{}), + buf: &recvBuffer{ + c: make(chan recvMsg), + mu: sync.Mutex{}, + }, + } + } + + testClient := func(ts *Stream) *http2Client { + return &http2Client{ + mu: sync.Mutex{}, + activeStreams: map[uint32]*Stream{ + 0: ts, + }, + controlBuf: &controlBuffer{ + ch: make(chan struct{}), + done: make(chan struct{}), + list: &itemList{}, + }, + } + } + + for _, test := range []struct { + name string + // input + metaHeaderFrame *http2.MetaHeadersFrame + // output + wantStatus *status.Status + }{ + { + name: "valid header", + metaHeaderFrame: &http2.MetaHeadersFrame{ + Fields: []hpack.HeaderField{ + {Name: "content-type", Value: "application/grpc"}, + {Name: "grpc-status", Value: "0"}, + {Name: ":status", Value: "200"}, + }, + }, + // no error + wantStatus: status.New(codes.OK, ""), + }, + { + name: "missing content-type header", + metaHeaderFrame: &http2.MetaHeadersFrame{ + Fields: []hpack.HeaderField{ + {Name: "grpc-status", Value: "0"}, + {Name: ":status", Value: "200"}, + }, + }, + wantStatus: status.New( + codes.Unknown, + "malformed header: missing HTTP content-type", + ), + }, + { + name: "invalid grpc status header field", + metaHeaderFrame: &http2.MetaHeadersFrame{ + Fields: []hpack.HeaderField{ + {Name: "content-type", Value: "application/grpc"}, + {Name: "grpc-status", Value: "xxxx"}, + {Name: ":status", Value: "200"}, + }, + }, + wantStatus: status.New( + codes.Internal, + "transport: malformed grpc-status: strconv.ParseInt: parsing \"xxxx\": invalid syntax", + ), + }, + { + name: "invalid http content type", + metaHeaderFrame: &http2.MetaHeadersFrame{ + Fields: []hpack.HeaderField{ + {Name: "content-type", Value: "application/json"}, + }, + }, + wantStatus: status.New( + codes.Internal, + "malformed header: missing HTTP status; transport: received unexpected content-type \"application/json\"", + ), + }, + { + name: "http fallback and invalid http status", + metaHeaderFrame: &http2.MetaHeadersFrame{ + Fields: []hpack.HeaderField{ + // No content type provided then fallback into handling http error. + {Name: ":status", Value: "xxxx"}, + }, + }, + wantStatus: status.New( + codes.Internal, + "transport: malformed http-status: strconv.ParseInt: parsing \"xxxx\": invalid syntax", + ), + }, + { + name: "http2 frame size exceeds", + metaHeaderFrame: &http2.MetaHeadersFrame{ + Fields: nil, + Truncated: true, + }, + wantStatus: status.New( + codes.Internal, + "peer header list size exceeded limit", + ), + }, + { + name: "bad status in grpc mode", + metaHeaderFrame: &http2.MetaHeadersFrame{ + Fields: []hpack.HeaderField{ + {Name: "content-type", Value: "application/grpc"}, + {Name: "grpc-status", Value: "0"}, + {Name: ":status", Value: "504"}, + }, + }, + wantStatus: status.New( + codes.Unavailable, + "unexpected HTTP status code received from server: 504 (Gateway Timeout)", + ), + }, + { + name: "missing http status", + metaHeaderFrame: &http2.MetaHeadersFrame{ + Fields: []hpack.HeaderField{ + {Name: "content-type", Value: "application/grpc"}, + }, + }, + wantStatus: status.New( + codes.Internal, + "malformed header: missing HTTP status", + ), + }, + } { + + t.Run(test.name, func(t *testing.T) { + ts := testStream() + s := testClient(ts) + + test.metaHeaderFrame.HeadersFrame = &http2.HeadersFrame{ + FrameHeader: http2.FrameHeader{ + StreamID: 0, + }, + } + + s.operateHeaders(test.metaHeaderFrame) + + got := ts.status + want := test.wantStatus + if got.Code() != want.Code() || got.Message() != want.Message() { + t.Fatalf("operateHeaders(%v); status = \ngot: %s\nwant: %s", test.metaHeaderFrame, got, want) + } + }) + t.Run(fmt.Sprintf("%s-end_stream", test.name), func(t *testing.T) { + ts := testStream() + s := testClient(ts) + + test.metaHeaderFrame.HeadersFrame = &http2.HeadersFrame{ + FrameHeader: http2.FrameHeader{ + StreamID: 0, + Flags: http2.FlagHeadersEndStream, + }, + } + + s.operateHeaders(test.metaHeaderFrame) + + got := ts.status + want := test.wantStatus + if got.Code() != want.Code() || got.Message() != want.Message() { + t.Fatalf("operateHeaders(%v); status = \ngot: %s\nwant: %s", test.metaHeaderFrame, got, want) + } + }) + } +} + +func TestConnectionError_Unwrap(t *testing.T) { + err := connectionErrorf(false, os.ErrNotExist, "unwrap me") + if !errors.Is(err, os.ErrNotExist) { + t.Error("ConnectionError does not unwrap") + } +} + +func (s) TestPeerSetInServerContext(t *testing.T) { + // create client and server transports. + server, client, cancel := setUp(t, 0, math.MaxUint32, normal) + defer cancel() + defer server.stop() + defer client.Close(fmt.Errorf("closed manually by test")) + + // create a stream with client transport. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + stream, err := client.NewStream(ctx, &CallHdr{}) + if err != nil { + t.Fatalf("failed to create a stream: %v", err) + } + + waitWhileTrue(t, func() (bool, error) { + server.mu.Lock() + defer server.mu.Unlock() + + if len(server.conns) == 0 { + return true, fmt.Errorf("timed-out while waiting for connection to be created on the server") + } + return false, nil + }) + + // verify peer is set in client transport context. + if _, ok := peer.FromContext(client.ctx); !ok { + t.Fatalf("Peer expected in client transport's context, but actually not found.") + } + + // verify peer is set in stream context. + if _, ok := peer.FromContext(stream.ctx); !ok { + t.Fatalf("Peer expected in stream context, but actually not found.") + } + + // verify peer is set in server transport context. + server.mu.Lock() + for k := range server.conns { + sc, ok := k.(*http2Server) + if !ok { + t.Fatalf("ServerTransport is of type %T, want %T", k, &http2Server{}) + } + if _, ok = peer.FromContext(sc.ctx); !ok { + t.Fatalf("Peer expected in server transport's context, but actually not found.") + } + } + server.mu.Unlock() +} diff --git a/internal/wrr/edf.go b/internal/wrr/edf.go index b4fb3f9d3bea..a06656f46400 100644 --- a/internal/wrr/edf.go +++ b/internal/wrr/edf.go @@ -43,7 +43,7 @@ type edfEntry struct { deadline float64 weight int64 orderOffset uint64 - item interface{} + item any } // edfPriorityQueue is a heap.Interface implementation for edfEntry elements. @@ -55,17 +55,17 @@ func (pq edfPriorityQueue) Less(i, j int) bool { } func (pq edfPriorityQueue) Swap(i, j int) { pq[i], pq[j] = pq[j], pq[i] } -func (pq *edfPriorityQueue) Push(x interface{}) { +func (pq *edfPriorityQueue) Push(x any) { *pq = append(*pq, x.(*edfEntry)) } -func (pq *edfPriorityQueue) Pop() interface{} { +func (pq *edfPriorityQueue) Pop() any { old := *pq *pq = old[0 : len(old)-1] return old[len(old)-1] } -func (edf *edfWrr) Add(item interface{}, weight int64) { +func (edf *edfWrr) Add(item any, weight int64) { edf.lock.Lock() defer edf.lock.Unlock() entry := edfEntry{ @@ -78,7 +78,7 @@ func (edf *edfWrr) Add(item interface{}, weight int64) { heap.Push(&edf.items, &entry) } -func (edf *edfWrr) Next() interface{} { +func (edf *edfWrr) Next() any { edf.lock.Lock() defer edf.lock.Unlock() if len(edf.items) == 0 { diff --git a/internal/wrr/random.go b/internal/wrr/random.go index a43652dcb735..25bbd82594d6 100644 --- a/internal/wrr/random.go +++ b/internal/wrr/random.go @@ -18,6 +18,8 @@ package wrr import ( + "fmt" + "sort" "sync" "google.golang.org/grpc/internal/grpcrand" @@ -25,15 +27,21 @@ import ( // weightedItem is a wrapped weighted item that is used to implement weighted random algorithm. type weightedItem struct { - Item interface{} - Weight int64 + item any + weight int64 + accumulatedWeight int64 +} + +func (w *weightedItem) String() string { + return fmt.Sprint(*w) } // randomWRR is a struct that contains weighted items implement weighted random algorithm. type randomWRR struct { - mu sync.RWMutex - items []*weightedItem - sumOfWeights int64 + mu sync.RWMutex + items []*weightedItem + // Are all item's weights equal + equalWeights bool } // NewRandom creates a new WRR with random. @@ -43,28 +51,41 @@ func NewRandom() WRR { var grpcrandInt63n = grpcrand.Int63n -func (rw *randomWRR) Next() (item interface{}) { +func (rw *randomWRR) Next() (item any) { rw.mu.RLock() defer rw.mu.RUnlock() - if rw.sumOfWeights == 0 { + if len(rw.items) == 0 { return nil } - // Random number in [0, sum). - randomWeight := grpcrandInt63n(rw.sumOfWeights) - for _, item := range rw.items { - randomWeight = randomWeight - item.Weight - if randomWeight < 0 { - return item.Item - } + if rw.equalWeights { + return rw.items[grpcrandInt63n(int64(len(rw.items)))].item } - return rw.items[len(rw.items)-1].Item + sumOfWeights := rw.items[len(rw.items)-1].accumulatedWeight + // Random number in [0, sumOfWeights). + randomWeight := grpcrandInt63n(sumOfWeights) + // Item's accumulated weights are in ascending order, because item's weight >= 0. + // Binary search rw.items to find first item whose accumulatedWeight > randomWeight + // The return i is guaranteed to be in range [0, len(rw.items)) because randomWeight < last item's accumulatedWeight + i := sort.Search(len(rw.items), func(i int) bool { return rw.items[i].accumulatedWeight > randomWeight }) + return rw.items[i].item } -func (rw *randomWRR) Add(item interface{}, weight int64) { +func (rw *randomWRR) Add(item any, weight int64) { rw.mu.Lock() defer rw.mu.Unlock() - rItem := &weightedItem{Item: item, Weight: weight} + accumulatedWeight := weight + equalWeights := true + if len(rw.items) > 0 { + lastItem := rw.items[len(rw.items)-1] + accumulatedWeight = lastItem.accumulatedWeight + weight + equalWeights = rw.equalWeights && weight == lastItem.weight + } + rw.equalWeights = equalWeights + rItem := &weightedItem{item: item, weight: weight, accumulatedWeight: accumulatedWeight} rw.items = append(rw.items, rItem) - rw.sumOfWeights += weight +} + +func (rw *randomWRR) String() string { + return fmt.Sprint(rw.items) } diff --git a/internal/wrr/wrr.go b/internal/wrr/wrr.go index d46bfad86ef4..d0d82cf4f015 100644 --- a/internal/wrr/wrr.go +++ b/internal/wrr/wrr.go @@ -24,9 +24,9 @@ type WRR interface { // Add adds an item with weight to the WRR set. // // Add and Next need to be thread safe. - Add(item interface{}, weight int64) + Add(item any, weight int64) // Next returns the next picked item. // // Add and Next need to be thread safe. - Next() interface{} + Next() any } diff --git a/internal/wrr/wrr_test.go b/internal/wrr/wrr_test.go index 4565e34ffb9c..ce4f5e507a2c 100644 --- a/internal/wrr/wrr_test.go +++ b/internal/wrr/wrr_test.go @@ -21,6 +21,7 @@ import ( "errors" "math" "math/rand" + "strconv" "testing" "github.com/google/go-cmp/cmp" @@ -70,12 +71,22 @@ func testWRRNext(t *testing.T, newWRR func() WRR) { name: "17-23-37", weights: []int64{17, 23, 37}, }, + { + name: "no items", + weights: []int64{}, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - var sumOfWeights int64 - w := newWRR() + if len(tt.weights) == 0 { + if next := w.Next(); next != nil { + t.Fatalf("w.Next returns non nil value:%v when there is no item", next) + } + return + } + + var sumOfWeights int64 for i, weight := range tt.weights { w.Add(i, weight) sumOfWeights += weight @@ -112,6 +123,70 @@ func (s) TestEdfWrrNext(t *testing.T) { testWRRNext(t, NewEDF) } +func BenchmarkRandomWRRNext(b *testing.B) { + for _, n := range []int{100, 500, 1000} { + b.Run("equal-weights-"+strconv.Itoa(n)+"-items", func(b *testing.B) { + w := NewRandom() + sumOfWeights := n + for i := 0; i < n; i++ { + w.Add(i, 1) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + for i := 0; i < sumOfWeights; i++ { + w.Next() + } + } + }) + } + + var maxWeight int64 = 1024 + for _, n := range []int{100, 500, 1000} { + b.Run("random-weights-"+strconv.Itoa(n)+"-items", func(b *testing.B) { + w := NewRandom() + var sumOfWeights int64 + for i := 0; i < n; i++ { + weight := rand.Int63n(maxWeight + 1) + w.Add(i, weight) + sumOfWeights += weight + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + for i := 0; i < int(sumOfWeights); i++ { + w.Next() + } + } + }) + } + + itemsNum := 200 + heavyWeight := int64(itemsNum) + lightWeight := int64(1) + heavyIndices := []int{0, itemsNum / 2, itemsNum - 1} + for _, heavyIndex := range heavyIndices { + b.Run("skew-weights-heavy-index-"+strconv.Itoa(heavyIndex), func(b *testing.B) { + w := NewRandom() + var sumOfWeights int64 + for i := 0; i < itemsNum; i++ { + var weight int64 + if i == heavyIndex { + weight = heavyWeight + } else { + weight = lightWeight + } + sumOfWeights += weight + w.Add(i, weight) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + for i := 0; i < int(sumOfWeights); i++ { + w.Next() + } + } + }) + } +} + func init() { r := rand.New(rand.NewSource(0)) grpcrandInt63n = r.Int63n diff --git a/internal/xds/matcher/matcher_header.go b/internal/xds/matcher/matcher_header.go new file mode 100644 index 000000000000..01433f4122a2 --- /dev/null +++ b/internal/xds/matcher/matcher_header.go @@ -0,0 +1,274 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package matcher + +import ( + "fmt" + "regexp" + "strconv" + "strings" + + "google.golang.org/grpc/internal/grpcutil" + "google.golang.org/grpc/metadata" +) + +// HeaderMatcher is an interface for header matchers. These are +// documented in (EnvoyProxy link here?). These matchers will match on different +// aspects of HTTP header name/value pairs. +type HeaderMatcher interface { + Match(metadata.MD) bool + String() string +} + +// mdValuesFromOutgoingCtx retrieves metadata from context. If there are +// multiple values, the values are concatenated with "," (comma and no space). +// +// All header matchers only match against the comma-concatenated string. +func mdValuesFromOutgoingCtx(md metadata.MD, key string) (string, bool) { + vs, ok := md[key] + if !ok { + return "", false + } + return strings.Join(vs, ","), true +} + +// HeaderExactMatcher matches on an exact match of the value of the header. +type HeaderExactMatcher struct { + key string + exact string + invert bool +} + +// NewHeaderExactMatcher returns a new HeaderExactMatcher. +func NewHeaderExactMatcher(key, exact string, invert bool) *HeaderExactMatcher { + return &HeaderExactMatcher{key: key, exact: exact, invert: invert} +} + +// Match returns whether the passed in HTTP Headers match according to the +// HeaderExactMatcher. +func (hem *HeaderExactMatcher) Match(md metadata.MD) bool { + v, ok := mdValuesFromOutgoingCtx(md, hem.key) + if !ok { + return false + } + return (v == hem.exact) != hem.invert +} + +func (hem *HeaderExactMatcher) String() string { + return fmt.Sprintf("headerExact:%v:%v", hem.key, hem.exact) +} + +// HeaderRegexMatcher matches on whether the entire request header value matches +// the regex. +type HeaderRegexMatcher struct { + key string + re *regexp.Regexp + invert bool +} + +// NewHeaderRegexMatcher returns a new HeaderRegexMatcher. +func NewHeaderRegexMatcher(key string, re *regexp.Regexp, invert bool) *HeaderRegexMatcher { + return &HeaderRegexMatcher{key: key, re: re, invert: invert} +} + +// Match returns whether the passed in HTTP Headers match according to the +// HeaderRegexMatcher. +func (hrm *HeaderRegexMatcher) Match(md metadata.MD) bool { + v, ok := mdValuesFromOutgoingCtx(md, hrm.key) + if !ok { + return false + } + return grpcutil.FullMatchWithRegex(hrm.re, v) != hrm.invert +} + +func (hrm *HeaderRegexMatcher) String() string { + return fmt.Sprintf("headerRegex:%v:%v", hrm.key, hrm.re.String()) +} + +// HeaderRangeMatcher matches on whether the request header value is within the +// range. The header value must be an integer in base 10 notation. +type HeaderRangeMatcher struct { + key string + start, end int64 // represents [start, end). + invert bool +} + +// NewHeaderRangeMatcher returns a new HeaderRangeMatcher. +func NewHeaderRangeMatcher(key string, start, end int64, invert bool) *HeaderRangeMatcher { + return &HeaderRangeMatcher{key: key, start: start, end: end, invert: invert} +} + +// Match returns whether the passed in HTTP Headers match according to the +// HeaderRangeMatcher. +func (hrm *HeaderRangeMatcher) Match(md metadata.MD) bool { + v, ok := mdValuesFromOutgoingCtx(md, hrm.key) + if !ok { + return false + } + if i, err := strconv.ParseInt(v, 10, 64); err == nil && i >= hrm.start && i < hrm.end { + return !hrm.invert + } + return hrm.invert +} + +func (hrm *HeaderRangeMatcher) String() string { + return fmt.Sprintf("headerRange:%v:[%d,%d)", hrm.key, hrm.start, hrm.end) +} + +// HeaderPresentMatcher will match based on whether the header is present in the +// whole request. +type HeaderPresentMatcher struct { + key string + present bool +} + +// NewHeaderPresentMatcher returns a new HeaderPresentMatcher. +func NewHeaderPresentMatcher(key string, present bool, invert bool) *HeaderPresentMatcher { + if invert { + present = !present + } + return &HeaderPresentMatcher{key: key, present: present} +} + +// Match returns whether the passed in HTTP Headers match according to the +// HeaderPresentMatcher. +func (hpm *HeaderPresentMatcher) Match(md metadata.MD) bool { + vs, ok := mdValuesFromOutgoingCtx(md, hpm.key) + present := ok && len(vs) > 0 // TODO: Are we sure we need this len(vs) > 0? + return present == hpm.present +} + +func (hpm *HeaderPresentMatcher) String() string { + return fmt.Sprintf("headerPresent:%v:%v", hpm.key, hpm.present) +} + +// HeaderPrefixMatcher matches on whether the prefix of the header value matches +// the prefix passed into this struct. +type HeaderPrefixMatcher struct { + key string + prefix string + invert bool +} + +// NewHeaderPrefixMatcher returns a new HeaderPrefixMatcher. +func NewHeaderPrefixMatcher(key string, prefix string, invert bool) *HeaderPrefixMatcher { + return &HeaderPrefixMatcher{key: key, prefix: prefix, invert: invert} +} + +// Match returns whether the passed in HTTP Headers match according to the +// HeaderPrefixMatcher. +func (hpm *HeaderPrefixMatcher) Match(md metadata.MD) bool { + v, ok := mdValuesFromOutgoingCtx(md, hpm.key) + if !ok { + return false + } + return strings.HasPrefix(v, hpm.prefix) != hpm.invert +} + +func (hpm *HeaderPrefixMatcher) String() string { + return fmt.Sprintf("headerPrefix:%v:%v", hpm.key, hpm.prefix) +} + +// HeaderSuffixMatcher matches on whether the suffix of the header value matches +// the suffix passed into this struct. +type HeaderSuffixMatcher struct { + key string + suffix string + invert bool +} + +// NewHeaderSuffixMatcher returns a new HeaderSuffixMatcher. +func NewHeaderSuffixMatcher(key string, suffix string, invert bool) *HeaderSuffixMatcher { + return &HeaderSuffixMatcher{key: key, suffix: suffix, invert: invert} +} + +// Match returns whether the passed in HTTP Headers match according to the +// HeaderSuffixMatcher. +func (hsm *HeaderSuffixMatcher) Match(md metadata.MD) bool { + v, ok := mdValuesFromOutgoingCtx(md, hsm.key) + if !ok { + return false + } + return strings.HasSuffix(v, hsm.suffix) != hsm.invert +} + +func (hsm *HeaderSuffixMatcher) String() string { + return fmt.Sprintf("headerSuffix:%v:%v", hsm.key, hsm.suffix) +} + +// HeaderContainsMatcher matches on whether the header value contains the +// value passed into this struct. +type HeaderContainsMatcher struct { + key string + contains string + invert bool +} + +// NewHeaderContainsMatcher returns a new HeaderContainsMatcher. key is the HTTP +// Header key to match on, and contains is the value that the header should +// should contain for a successful match. An empty contains string does not +// work, use HeaderPresentMatcher in that case. +func NewHeaderContainsMatcher(key string, contains string, invert bool) *HeaderContainsMatcher { + return &HeaderContainsMatcher{key: key, contains: contains, invert: invert} +} + +// Match returns whether the passed in HTTP Headers match according to the +// HeaderContainsMatcher. +func (hcm *HeaderContainsMatcher) Match(md metadata.MD) bool { + v, ok := mdValuesFromOutgoingCtx(md, hcm.key) + if !ok { + return false + } + return strings.Contains(v, hcm.contains) != hcm.invert +} + +func (hcm *HeaderContainsMatcher) String() string { + return fmt.Sprintf("headerContains:%v%v", hcm.key, hcm.contains) +} + +// HeaderStringMatcher matches on whether the header value matches against the +// StringMatcher specified. +type HeaderStringMatcher struct { + key string + stringMatcher StringMatcher + invert bool +} + +// NewHeaderStringMatcher returns a new HeaderStringMatcher. +func NewHeaderStringMatcher(key string, sm StringMatcher, invert bool) *HeaderStringMatcher { + return &HeaderStringMatcher{ + key: key, + stringMatcher: sm, + invert: invert, + } +} + +// Match returns whether the passed in HTTP Headers match according to the +// specified StringMatcher. +func (hsm *HeaderStringMatcher) Match(md metadata.MD) bool { + v, ok := mdValuesFromOutgoingCtx(md, hsm.key) + if !ok { + return false + } + return hsm.stringMatcher.Match(v) != hsm.invert +} + +func (hsm *HeaderStringMatcher) String() string { + return fmt.Sprintf("headerString:%v:%v", hsm.key, hsm.stringMatcher) +} diff --git a/xds/internal/balancer/xdsrouting/matcher_header_test.go b/internal/xds/matcher/matcher_header_test.go similarity index 51% rename from xds/internal/balancer/xdsrouting/matcher_header_test.go rename to internal/xds/matcher/matcher_header_test.go index 3ec73ee90f78..9a20cf12b0f9 100644 --- a/xds/internal/balancer/xdsrouting/matcher_header_test.go +++ b/internal/xds/matcher/matcher_header_test.go @@ -16,7 +16,7 @@ * */ -package xdsrouting +package matcher import ( "regexp" @@ -31,6 +31,7 @@ func TestHeaderExactMatcherMatch(t *testing.T) { key, exact string md metadata.MD want bool + invert bool }{ { name: "one value one match", @@ -61,11 +62,35 @@ func TestHeaderExactMatcherMatch(t *testing.T) { md: metadata.Pairs("th", "abc"), want: false, }, + { + name: "invert header not present", + key: "th", + exact: "tv", + md: metadata.Pairs(":method", "GET"), + want: false, + invert: true, + }, + { + name: "invert header match", + key: "th", + exact: "tv", + md: metadata.Pairs("th", "tv"), + want: false, + invert: true, + }, + { + name: "invert header not match", + key: "th", + exact: "tv", + md: metadata.Pairs("th", "tvv"), + want: true, + invert: true, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - hem := newHeaderExactMatcher(tt.key, tt.exact) - if got := hem.match(tt.md); got != tt.want { + hem := NewHeaderExactMatcher(tt.key, tt.exact, tt.invert) + if got := hem.Match(tt.md); got != tt.want { t.Errorf("match() = %v, want %v", got, tt.want) } }) @@ -78,6 +103,7 @@ func TestHeaderRegexMatcherMatch(t *testing.T) { key, regexStr string md metadata.MD want bool + invert bool }{ { name: "one value one match", @@ -107,11 +133,49 @@ func TestHeaderRegexMatcherMatch(t *testing.T) { md: metadata.Pairs("th", "abc"), want: false, }, + { + name: "no match because only part of value matches with regex", + key: "header", + regexStr: "^a+$", + md: metadata.Pairs("header", "ab"), + want: false, + }, + { + name: "match because full value matches with regex", + key: "header", + regexStr: "^a+$", + md: metadata.Pairs("header", "aa"), + want: true, + }, + { + name: "invert header not present", + key: "th", + regexStr: "^t+v*$", + md: metadata.Pairs(":method", "GET"), + want: false, + invert: true, + }, + { + name: "invert header match", + key: "th", + regexStr: "^t+v*$", + md: metadata.Pairs("th", "tttvv"), + want: false, + invert: true, + }, + { + name: "invert header not match", + key: "th", + regexStr: "^t+v*$", + md: metadata.Pairs("th", "abc"), + want: true, + invert: true, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - hrm := newHeaderRegexMatcher(tt.key, regexp.MustCompile(tt.regexStr)) - if got := hrm.match(tt.md); got != tt.want { + hrm := NewHeaderRegexMatcher(tt.key, regexp.MustCompile(tt.regexStr), tt.invert) + if got := hrm.Match(tt.md); got != tt.want { t.Errorf("match() = %v, want %v", got, tt.want) } }) @@ -125,6 +189,7 @@ func TestHeaderRangeMatcherMatch(t *testing.T) { start, end int64 md metadata.MD want bool + invert bool }{ { name: "match", @@ -154,11 +219,35 @@ func TestHeaderRangeMatcherMatch(t *testing.T) { md: metadata.Pairs("th", "-5"), want: true, }, + { + name: "invert header not present", + key: "th", + start: 1, end: 10, + md: metadata.Pairs(":method", "GET"), + want: false, + invert: true, + }, + { + name: "invert header match", + key: "th", + start: 1, end: 10, + md: metadata.Pairs("th", "5"), + want: false, + invert: true, + }, + { + name: "invert header not match", + key: "th", + start: 1, end: 9, + md: metadata.Pairs("th", "10"), + want: true, + invert: true, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - hrm := newHeaderRangeMatcher(tt.key, tt.start, tt.end) - if got := hrm.match(tt.md); got != tt.want { + hrm := NewHeaderRangeMatcher(tt.key, tt.start, tt.end, tt.invert) + if got := hrm.Match(tt.md); got != tt.want { t.Errorf("match() = %v, want %v", got, tt.want) } }) @@ -172,6 +261,7 @@ func TestHeaderPresentMatcherMatch(t *testing.T) { present bool md metadata.MD want bool + invert bool }{ { name: "want present is present", @@ -201,11 +291,35 @@ func TestHeaderPresentMatcherMatch(t *testing.T) { md: metadata.Pairs("abc", "tv"), want: true, }, + { + name: "invert header not present", + key: "th", + present: true, + md: metadata.Pairs(":method", "GET"), + want: true, + invert: true, + }, + { + name: "invert header match", + key: "th", + present: true, + md: metadata.Pairs("th", "tv"), + want: false, + invert: true, + }, + { + name: "invert header not match", + key: "th", + present: true, + md: metadata.Pairs(":method", "GET"), + want: true, + invert: true, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - hpm := newHeaderPresentMatcher(tt.key, tt.present) - if got := hpm.match(tt.md); got != tt.want { + hpm := NewHeaderPresentMatcher(tt.key, tt.present, tt.invert) + if got := hpm.Match(tt.md); got != tt.want { t.Errorf("match() = %v, want %v", got, tt.want) } }) @@ -218,6 +332,7 @@ func TestHeaderPrefixMatcherMatch(t *testing.T) { key, prefix string md metadata.MD want bool + invert bool }{ { name: "one value one match", @@ -247,11 +362,35 @@ func TestHeaderPrefixMatcherMatch(t *testing.T) { md: metadata.Pairs("th", "abc"), want: false, }, + { + name: "invert header not present", + key: "th", + prefix: "tv", + md: metadata.Pairs(":method", "GET"), + want: false, + invert: true, + }, + { + name: "invert header match", + key: "th", + prefix: "tv", + md: metadata.Pairs("th", "tv123"), + want: false, + invert: true, + }, + { + name: "invert header not match", + key: "th", + prefix: "tv", + md: metadata.Pairs("th", "abc"), + want: true, + invert: true, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - hpm := newHeaderPrefixMatcher(tt.key, tt.prefix) - if got := hpm.match(tt.md); got != tt.want { + hpm := NewHeaderPrefixMatcher(tt.key, tt.prefix, tt.invert) + if got := hpm.Match(tt.md); got != tt.want { t.Errorf("match() = %v, want %v", got, tt.want) } }) @@ -264,6 +403,7 @@ func TestHeaderSuffixMatcherMatch(t *testing.T) { key, suffix string md metadata.MD want bool + invert bool }{ { name: "one value one match", @@ -293,40 +433,116 @@ func TestHeaderSuffixMatcherMatch(t *testing.T) { md: metadata.Pairs("th", "abc"), want: false, }, + { + name: "invert header not present", + key: "th", + suffix: "tv", + md: metadata.Pairs(":method", "GET"), + want: false, + invert: true, + }, + { + name: "invert header match", + key: "th", + suffix: "tv", + md: metadata.Pairs("th", "123tv"), + want: false, + invert: true, + }, + { + name: "invert header not match", + key: "th", + suffix: "tv", + md: metadata.Pairs("th", "abc"), + want: true, + invert: true, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - hsm := newHeaderSuffixMatcher(tt.key, tt.suffix) - if got := hsm.match(tt.md); got != tt.want { + hsm := NewHeaderSuffixMatcher(tt.key, tt.suffix, tt.invert) + if got := hsm.Match(tt.md); got != tt.want { t.Errorf("match() = %v, want %v", got, tt.want) } }) } } -func TestInvertMatcherMatch(t *testing.T) { +func TestHeaderStringMatch(t *testing.T) { tests := []struct { - name string - m headerMatcherInterface - md metadata.MD + name string + key string + sm StringMatcher + invert bool + md metadata.MD + want bool }{ { - name: "true->false", - m: newHeaderExactMatcher("th", "tv"), - md: metadata.Pairs("th", "tv"), + name: "should-match", + key: "th", + sm: StringMatcher{ + exactMatch: newStringP("tv"), + }, + invert: false, + md: metadata.Pairs("th", "tv"), + want: true, + }, + { + name: "not match", + key: "th", + sm: StringMatcher{ + containsMatch: newStringP("tv"), + }, + invert: false, + md: metadata.Pairs("th", "not-match"), + want: false, + }, + { + name: "invert string match", + key: "th", + sm: StringMatcher{ + containsMatch: newStringP("tv"), + }, + invert: true, + md: metadata.Pairs("th", "not-match"), + want: true, + }, + { + name: "header missing", + key: "th", + sm: StringMatcher{ + containsMatch: newStringP("tv"), + }, + invert: false, + md: metadata.Pairs("not-specified-key", "not-match"), + want: false, }, { - name: "false->true", - m: newHeaderExactMatcher("th", "abc"), - md: metadata.Pairs("th", "tv"), + name: "header missing invert true", + key: "th", + sm: StringMatcher{ + containsMatch: newStringP("tv"), + }, + invert: true, + md: metadata.Pairs("not-specified-key", "not-match"), + want: false, + }, + { + name: "header empty string invert", + key: "th", + sm: StringMatcher{ + containsMatch: newStringP("tv"), + }, + invert: true, + md: metadata.Pairs("th", ""), + want: true, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := newInvertMatcher(tt.m).match(tt.md) - want := !tt.m.match(tt.md) - if got != want { - t.Errorf("match() = %v, want %v", got, want) + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + hsm := NewHeaderStringMatcher(test.key, test.sm, test.invert) + if got := hsm.Match(test.md); got != test.want { + t.Errorf("match() = %v, want %v", got, test.want) } }) } diff --git a/internal/xds/matcher/string_matcher.go b/internal/xds/matcher/string_matcher.go new file mode 100644 index 000000000000..c138f78735bc --- /dev/null +++ b/internal/xds/matcher/string_matcher.go @@ -0,0 +1,184 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package matcher contains types that need to be shared between code under +// google.golang.org/grpc/xds/... and the rest of gRPC. +package matcher + +import ( + "errors" + "fmt" + "regexp" + "strings" + + v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3" + "google.golang.org/grpc/internal/grpcutil" +) + +// StringMatcher contains match criteria for matching a string, and is an +// internal representation of the `StringMatcher` proto defined at +// https://github.com/envoyproxy/envoy/blob/main/api/envoy/type/matcher/v3/string.proto. +type StringMatcher struct { + // Since these match fields are part of a `oneof` in the corresponding xDS + // proto, only one of them is expected to be set. + exactMatch *string + prefixMatch *string + suffixMatch *string + regexMatch *regexp.Regexp + containsMatch *string + // If true, indicates the exact/prefix/suffix/contains matching should be + // case insensitive. This has no effect on the regex match. + ignoreCase bool +} + +// Match returns true if input matches the criteria in the given StringMatcher. +func (sm StringMatcher) Match(input string) bool { + if sm.ignoreCase { + input = strings.ToLower(input) + } + switch { + case sm.exactMatch != nil: + return input == *sm.exactMatch + case sm.prefixMatch != nil: + return strings.HasPrefix(input, *sm.prefixMatch) + case sm.suffixMatch != nil: + return strings.HasSuffix(input, *sm.suffixMatch) + case sm.regexMatch != nil: + return grpcutil.FullMatchWithRegex(sm.regexMatch, input) + case sm.containsMatch != nil: + return strings.Contains(input, *sm.containsMatch) + } + return false +} + +// StringMatcherFromProto is a helper function to create a StringMatcher from +// the corresponding StringMatcher proto. +// +// Returns a non-nil error if matcherProto is invalid. +func StringMatcherFromProto(matcherProto *v3matcherpb.StringMatcher) (StringMatcher, error) { + if matcherProto == nil { + return StringMatcher{}, errors.New("input StringMatcher proto is nil") + } + + matcher := StringMatcher{ignoreCase: matcherProto.GetIgnoreCase()} + switch mt := matcherProto.GetMatchPattern().(type) { + case *v3matcherpb.StringMatcher_Exact: + matcher.exactMatch = &mt.Exact + if matcher.ignoreCase { + *matcher.exactMatch = strings.ToLower(*matcher.exactMatch) + } + case *v3matcherpb.StringMatcher_Prefix: + if matcherProto.GetPrefix() == "" { + return StringMatcher{}, errors.New("empty prefix is not allowed in StringMatcher") + } + matcher.prefixMatch = &mt.Prefix + if matcher.ignoreCase { + *matcher.prefixMatch = strings.ToLower(*matcher.prefixMatch) + } + case *v3matcherpb.StringMatcher_Suffix: + if matcherProto.GetSuffix() == "" { + return StringMatcher{}, errors.New("empty suffix is not allowed in StringMatcher") + } + matcher.suffixMatch = &mt.Suffix + if matcher.ignoreCase { + *matcher.suffixMatch = strings.ToLower(*matcher.suffixMatch) + } + case *v3matcherpb.StringMatcher_SafeRegex: + regex := matcherProto.GetSafeRegex().GetRegex() + re, err := regexp.Compile(regex) + if err != nil { + return StringMatcher{}, fmt.Errorf("safe_regex matcher %q is invalid", regex) + } + matcher.regexMatch = re + case *v3matcherpb.StringMatcher_Contains: + if matcherProto.GetContains() == "" { + return StringMatcher{}, errors.New("empty contains is not allowed in StringMatcher") + } + matcher.containsMatch = &mt.Contains + if matcher.ignoreCase { + *matcher.containsMatch = strings.ToLower(*matcher.containsMatch) + } + default: + return StringMatcher{}, fmt.Errorf("unrecognized string matcher: %+v", matcherProto) + } + return matcher, nil +} + +// StringMatcherForTesting is a helper function to create a StringMatcher based +// on the given arguments. Intended only for testing purposes. +func StringMatcherForTesting(exact, prefix, suffix, contains *string, regex *regexp.Regexp, ignoreCase bool) StringMatcher { + sm := StringMatcher{ + exactMatch: exact, + prefixMatch: prefix, + suffixMatch: suffix, + regexMatch: regex, + containsMatch: contains, + ignoreCase: ignoreCase, + } + if ignoreCase { + switch { + case sm.exactMatch != nil: + *sm.exactMatch = strings.ToLower(*exact) + case sm.prefixMatch != nil: + *sm.prefixMatch = strings.ToLower(*prefix) + case sm.suffixMatch != nil: + *sm.suffixMatch = strings.ToLower(*suffix) + case sm.containsMatch != nil: + *sm.containsMatch = strings.ToLower(*contains) + } + } + return sm +} + +// ExactMatch returns the value of the configured exact match or an empty string +// if exact match criteria was not specified. +func (sm StringMatcher) ExactMatch() string { + if sm.exactMatch != nil { + return *sm.exactMatch + } + return "" +} + +// Equal returns true if other and sm are equivalent to each other. +func (sm StringMatcher) Equal(other StringMatcher) bool { + if sm.ignoreCase != other.ignoreCase { + return false + } + + if (sm.exactMatch != nil) != (other.exactMatch != nil) || + (sm.prefixMatch != nil) != (other.prefixMatch != nil) || + (sm.suffixMatch != nil) != (other.suffixMatch != nil) || + (sm.regexMatch != nil) != (other.regexMatch != nil) || + (sm.containsMatch != nil) != (other.containsMatch != nil) { + return false + } + + switch { + case sm.exactMatch != nil: + return *sm.exactMatch == *other.exactMatch + case sm.prefixMatch != nil: + return *sm.prefixMatch == *other.prefixMatch + case sm.suffixMatch != nil: + return *sm.suffixMatch == *other.suffixMatch + case sm.regexMatch != nil: + return sm.regexMatch.String() == other.regexMatch.String() + case sm.containsMatch != nil: + return *sm.containsMatch == *other.containsMatch + } + return true +} diff --git a/internal/xds/matcher/string_matcher_test.go b/internal/xds/matcher/string_matcher_test.go new file mode 100644 index 000000000000..9528b57e44a5 --- /dev/null +++ b/internal/xds/matcher/string_matcher_test.go @@ -0,0 +1,315 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package matcher + +import ( + "regexp" + "testing" + + v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3" + "github.com/google/go-cmp/cmp" +) + +func TestStringMatcherFromProto(t *testing.T) { + tests := []struct { + desc string + inputProto *v3matcherpb.StringMatcher + wantMatcher StringMatcher + wantErr bool + }{ + { + desc: "nil proto", + wantErr: true, + }, + { + desc: "empty prefix", + inputProto: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: ""}, + }, + wantErr: true, + }, + { + desc: "empty suffix", + inputProto: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Suffix{Suffix: ""}, + }, + wantErr: true, + }, + { + desc: "empty contains", + inputProto: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Contains{Contains: ""}, + }, + wantErr: true, + }, + { + desc: "invalid regex", + inputProto: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_SafeRegex{ + SafeRegex: &v3matcherpb.RegexMatcher{Regex: "??"}, + }, + }, + wantErr: true, + }, + { + desc: "happy case exact", + inputProto: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "exact"}, + }, + wantMatcher: StringMatcher{exactMatch: newStringP("exact")}, + }, + { + desc: "happy case exact ignore case", + inputProto: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "EXACT"}, + IgnoreCase: true, + }, + wantMatcher: StringMatcher{ + exactMatch: newStringP("exact"), + ignoreCase: true, + }, + }, + { + desc: "happy case prefix", + inputProto: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: "prefix"}, + }, + wantMatcher: StringMatcher{prefixMatch: newStringP("prefix")}, + }, + { + desc: "happy case prefix ignore case", + inputProto: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: "PREFIX"}, + IgnoreCase: true, + }, + wantMatcher: StringMatcher{ + prefixMatch: newStringP("prefix"), + ignoreCase: true, + }, + }, + { + desc: "happy case suffix", + inputProto: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Suffix{Suffix: "suffix"}, + }, + wantMatcher: StringMatcher{suffixMatch: newStringP("suffix")}, + }, + { + desc: "happy case suffix ignore case", + inputProto: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Suffix{Suffix: "SUFFIX"}, + IgnoreCase: true, + }, + wantMatcher: StringMatcher{ + suffixMatch: newStringP("suffix"), + ignoreCase: true, + }, + }, + { + desc: "happy case regex", + inputProto: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_SafeRegex{ + SafeRegex: &v3matcherpb.RegexMatcher{Regex: "good?regex?"}, + }, + }, + wantMatcher: StringMatcher{regexMatch: regexp.MustCompile("good?regex?")}, + }, + { + desc: "happy case contains", + inputProto: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Contains{Contains: "contains"}, + }, + wantMatcher: StringMatcher{containsMatch: newStringP("contains")}, + }, + { + desc: "happy case contains ignore case", + inputProto: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Contains{Contains: "CONTAINS"}, + IgnoreCase: true, + }, + wantMatcher: StringMatcher{ + containsMatch: newStringP("contains"), + ignoreCase: true, + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + gotMatcher, err := StringMatcherFromProto(test.inputProto) + if (err != nil) != test.wantErr { + t.Fatalf("StringMatcherFromProto(%+v) returned err: %v, wantErr: %v", test.inputProto, err, test.wantErr) + } + if diff := cmp.Diff(gotMatcher, test.wantMatcher, cmp.AllowUnexported(regexp.Regexp{})); diff != "" { + t.Fatalf("StringMatcherFromProto(%+v) returned unexpected diff (-got, +want):\n%s", test.inputProto, diff) + } + }) + } +} + +func TestMatch(t *testing.T) { + var ( + exactMatcher, _ = StringMatcherFromProto(&v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "exact"}}) + prefixMatcher, _ = StringMatcherFromProto(&v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: "prefix"}}) + suffixMatcher, _ = StringMatcherFromProto(&v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Suffix{Suffix: "suffix"}}) + regexMatcher, _ = StringMatcherFromProto(&v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_SafeRegex{SafeRegex: &v3matcherpb.RegexMatcher{Regex: "good?regex?"}}}) + containsMatcher, _ = StringMatcherFromProto(&v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Contains{Contains: "contains"}}) + exactMatcherIgnoreCase, _ = StringMatcherFromProto(&v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "exact"}, + IgnoreCase: true, + }) + prefixMatcherIgnoreCase, _ = StringMatcherFromProto(&v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: "prefix"}, + IgnoreCase: true, + }) + suffixMatcherIgnoreCase, _ = StringMatcherFromProto(&v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Suffix{Suffix: "suffix"}, + IgnoreCase: true, + }) + containsMatcherIgnoreCase, _ = StringMatcherFromProto(&v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Contains{Contains: "contains"}, + IgnoreCase: true, + }) + ) + + tests := []struct { + desc string + matcher StringMatcher + input string + wantMatch bool + }{ + { + desc: "exact match success", + matcher: exactMatcher, + input: "exact", + wantMatch: true, + }, + { + desc: "exact match failure", + matcher: exactMatcher, + input: "not-exact", + }, + { + desc: "exact match success with ignore case", + matcher: exactMatcherIgnoreCase, + input: "EXACT", + wantMatch: true, + }, + { + desc: "exact match failure with ignore case", + matcher: exactMatcherIgnoreCase, + input: "not-exact", + }, + { + desc: "prefix match success", + matcher: prefixMatcher, + input: "prefixIsHere", + wantMatch: true, + }, + { + desc: "prefix match failure", + matcher: prefixMatcher, + input: "not-prefix", + }, + { + desc: "prefix match success with ignore case", + matcher: prefixMatcherIgnoreCase, + input: "PREFIXisHere", + wantMatch: true, + }, + { + desc: "prefix match failure with ignore case", + matcher: prefixMatcherIgnoreCase, + input: "not-PREFIX", + }, + { + desc: "suffix match success", + matcher: suffixMatcher, + input: "hereIsThesuffix", + wantMatch: true, + }, + { + desc: "suffix match failure", + matcher: suffixMatcher, + input: "suffix-is-not-here", + }, + { + desc: "suffix match success with ignore case", + matcher: suffixMatcherIgnoreCase, + input: "hereIsTheSuFFix", + wantMatch: true, + }, + { + desc: "suffix match failure with ignore case", + matcher: suffixMatcherIgnoreCase, + input: "SUFFIX-is-not-here", + }, + { + desc: "regex match success", + matcher: regexMatcher, + input: "goodregex", + wantMatch: true, + }, + { + desc: "regex match failure because only part match", + matcher: regexMatcher, + input: "goodregexa", + wantMatch: false, + }, + { + desc: "regex match failure", + matcher: regexMatcher, + input: "regex-is-not-here", + }, + { + desc: "contains match success", + matcher: containsMatcher, + input: "IScontainsHERE", + wantMatch: true, + }, + { + desc: "contains match failure", + matcher: containsMatcher, + input: "con-tains-is-not-here", + }, + { + desc: "contains match success with ignore case", + matcher: containsMatcherIgnoreCase, + input: "isCONTAINShere", + wantMatch: true, + }, + { + desc: "contains match failure with ignore case", + matcher: containsMatcherIgnoreCase, + input: "CON-TAINS-is-not-here", + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + if gotMatch := test.matcher.Match(test.input); gotMatch != test.wantMatch { + t.Errorf("StringMatcher.Match(%s) returned %v, want %v", test.input, gotMatch, test.wantMatch) + } + }) + } +} + +func newStringP(s string) *string { + return &s +} diff --git a/internal/xds/rbac/converter.go b/internal/xds/rbac/converter.go new file mode 100644 index 000000000000..713e39cf31cb --- /dev/null +++ b/internal/xds/rbac/converter.go @@ -0,0 +1,101 @@ +/* + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package rbac + +import ( + "encoding/json" + "fmt" + "strings" + + v1xdsudpatypepb "github.com/cncf/xds/go/udpa/type/v1" + v3xdsxdstypepb "github.com/cncf/xds/go/xds/type/v3" + v3rbacpb "github.com/envoyproxy/go-control-plane/envoy/config/rbac/v3" + v3auditloggersstreampb "github.com/envoyproxy/go-control-plane/envoy/extensions/rbac/audit_loggers/stream/v3" + "google.golang.org/grpc/authz/audit" + "google.golang.org/grpc/authz/audit/stdout" + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/structpb" +) + +func buildLogger(loggerConfig *v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig) (audit.Logger, error) { + if loggerConfig.GetAuditLogger().GetTypedConfig() == nil { + return nil, fmt.Errorf("missing required field: TypedConfig") + } + customConfig, loggerName, err := getCustomConfig(loggerConfig.AuditLogger.TypedConfig) + if err != nil { + return nil, err + } + if loggerName == "" { + return nil, fmt.Errorf("field TypedConfig.TypeURL cannot be an empty string") + } + factory := audit.GetLoggerBuilder(loggerName) + if factory == nil { + if loggerConfig.IsOptional { + return nil, nil + } + return nil, fmt.Errorf("no builder registered for %v", loggerName) + } + auditLoggerConfig, err := factory.ParseLoggerConfig(customConfig) + if err != nil { + return nil, fmt.Errorf("custom config could not be parsed by registered factory. error: %v", err) + } + auditLogger := factory.Build(auditLoggerConfig) + return auditLogger, nil +} + +func getCustomConfig(config *anypb.Any) (json.RawMessage, string, error) { + any, err := config.UnmarshalNew() + if err != nil { + return nil, "", err + } + switch m := any.(type) { + case *v1xdsudpatypepb.TypedStruct: + return convertCustomConfig(m.TypeUrl, m.Value) + case *v3xdsxdstypepb.TypedStruct: + return convertCustomConfig(m.TypeUrl, m.Value) + case *v3auditloggersstreampb.StdoutAuditLog: + return convertStdoutConfig(m) + } + return nil, "", fmt.Errorf("custom config not implemented for type [%v]", config.GetTypeUrl()) +} + +func convertStdoutConfig(config *v3auditloggersstreampb.StdoutAuditLog) (json.RawMessage, string, error) { + json, err := protojson.Marshal(config) + return json, stdout.Name, err +} + +func convertCustomConfig(typeURL string, s *structpb.Struct) (json.RawMessage, string, error) { + // The gRPC policy name will be the "type name" part of the value of the + // type_url field in the TypedStruct. We get this by using the part after + // the last / character. Can assume a valid type_url from the control plane. + urls := strings.Split(typeURL, "/") + if len(urls) == 0 { + return nil, "", fmt.Errorf("error converting custom audit logger %v for %v: typeURL must have a url-like format with the typeName being the value after the last /", typeURL, s) + } + name := urls[len(urls)-1] + + rawJSON := []byte("{}") + var err error + if s != nil { + rawJSON, err = json.Marshal(s) + if err != nil { + return nil, "", fmt.Errorf("error converting custom audit logger %v for %v: %v", typeURL, s, err) + } + } + return rawJSON, name, nil +} diff --git a/internal/xds/rbac/converter_test.go b/internal/xds/rbac/converter_test.go new file mode 100644 index 000000000000..222e504d82c3 --- /dev/null +++ b/internal/xds/rbac/converter_test.go @@ -0,0 +1,170 @@ +/* + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package rbac + +import ( + "reflect" + "strings" + "testing" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3rbacpb "github.com/envoyproxy/go-control-plane/envoy/config/rbac/v3" + v3auditloggersstreampb "github.com/envoyproxy/go-control-plane/envoy/extensions/rbac/audit_loggers/stream/v3" + "google.golang.org/grpc/authz/audit" + "google.golang.org/grpc/authz/audit/stdout" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/protobuf/types/known/anypb" +) + +func (s) TestBuildLoggerErrors(t *testing.T) { + tests := []struct { + name string + loggerConfig *v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig + expectedLogger audit.Logger + expectedError string + }{ + { + name: "nil typed config", + loggerConfig: &v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + AuditLogger: &v3corepb.TypedExtensionConfig{ + TypedConfig: nil, + }, + }, + expectedError: "missing required field: TypedConfig", + }, + { + name: "Unsupported Type", + loggerConfig: &v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + AuditLogger: &v3corepb.TypedExtensionConfig{ + Name: "TestAuditLoggerBuffer", + TypedConfig: testutils.MarshalAny(&v3rbacpb.RBAC_AuditLoggingOptions{}), + }, + }, + expectedError: "custom config not implemented for type ", + }, + { + name: "Empty name", + loggerConfig: &v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + AuditLogger: &v3corepb.TypedExtensionConfig{ + Name: "TestAuditLoggerBuffer", + TypedConfig: createUDPATypedStruct(t, map[string]any{}, ""), + }, + }, + expectedError: "field TypedConfig.TypeURL cannot be an empty string", + }, + { + name: "No registered logger", + loggerConfig: &v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + AuditLogger: &v3corepb.TypedExtensionConfig{ + Name: "UnregisteredLogger", + TypedConfig: createUDPATypedStruct(t, map[string]any{}, "UnregisteredLogger"), + }, + IsOptional: false, + }, + expectedError: "no builder registered for UnregisteredLogger", + }, + { + name: "fail to parse custom config", + loggerConfig: &v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + AuditLogger: &v3corepb.TypedExtensionConfig{ + Name: "TestAuditLoggerCustomConfig", + TypedConfig: createUDPATypedStruct(t, map[string]any{"abc": "BADVALUE", "xyz": "123"}, "fail to parse custom config_TestAuditLoggerCustomConfig")}, + IsOptional: false, + }, + expectedError: "custom config could not be parsed", + }, + { + name: "no registered logger but optional passes", + loggerConfig: &v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + AuditLogger: &v3corepb.TypedExtensionConfig{ + Name: "UnregisteredLogger", + TypedConfig: createUDPATypedStruct(t, map[string]any{}, "no registered logger but optional passes_UnregisteredLogger"), + }, + IsOptional: true, + }, + expectedLogger: nil, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + b := TestAuditLoggerCustomConfigBuilder{testName: test.name} + audit.RegisterLoggerBuilder(&b) + logger, err := buildLogger(test.loggerConfig) + if err != nil && !strings.HasPrefix(err.Error(), test.expectedError) { + t.Fatalf("expected error: %v. got error: %v", test.expectedError, err) + } + if logger != test.expectedLogger { + t.Fatalf("expected logger: %v. got logger: %v", test.expectedLogger, logger) + } + + }) + } +} + +func (s) TestBuildLoggerKnownTypes(t *testing.T) { + tests := []struct { + name string + loggerConfig *v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig + expectedType reflect.Type + }{ + { + name: "stdout logger", + loggerConfig: &v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + AuditLogger: &v3corepb.TypedExtensionConfig{ + Name: stdout.Name, + TypedConfig: createStdoutPb(t), + }, + IsOptional: false, + }, + expectedType: reflect.TypeOf(audit.GetLoggerBuilder(stdout.Name).Build(nil)), + }, + { + name: "stdout logger with generic TypedConfig", + loggerConfig: &v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + AuditLogger: &v3corepb.TypedExtensionConfig{ + Name: stdout.Name, + TypedConfig: createXDSTypedStruct(t, map[string]any{}, stdout.Name), + }, + IsOptional: false, + }, + expectedType: reflect.TypeOf(audit.GetLoggerBuilder(stdout.Name).Build(nil)), + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + logger, err := buildLogger(test.loggerConfig) + if err != nil { + t.Fatalf("expected success. got error: %v", err) + } + loggerType := reflect.TypeOf(logger) + if test.expectedType != loggerType { + t.Fatalf("logger not of expected type. want: %v got: %v", test.expectedType, loggerType) + } + }) + } +} + +// Builds stdout config for audit logger proto. +func createStdoutPb(t *testing.T) *anypb.Any { + t.Helper() + pb := &v3auditloggersstreampb.StdoutAuditLog{} + customConfig, err := anypb.New(pb) + if err != nil { + t.Fatalf("createStdoutPb failed during anypb.New: %v", err) + } + return customConfig +} diff --git a/internal/xds/rbac/matchers.go b/internal/xds/rbac/matchers.go new file mode 100644 index 000000000000..c9f71d32cbb2 --- /dev/null +++ b/internal/xds/rbac/matchers.go @@ -0,0 +1,432 @@ +/* + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package rbac + +import ( + "errors" + "fmt" + "net" + "regexp" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3rbacpb "github.com/envoyproxy/go-control-plane/envoy/config/rbac/v3" + v3route_componentspb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3" + internalmatcher "google.golang.org/grpc/internal/xds/matcher" +) + +// matcher is an interface that takes data about incoming RPC's and returns +// whether it matches with whatever matcher implements this interface. +type matcher interface { + match(data *rpcData) bool +} + +// policyMatcher helps determine whether an incoming RPC call matches a policy. +// A policy is a logical role (e.g. Service Admin), which is comprised of +// permissions and principals. A principal is an identity (or identities) for a +// downstream subject which are assigned the policy (role), and a permission is +// an action(s) that a principal(s) can take. A policy matches if both a +// permission and a principal match, which will be determined by the child or +// permissions and principal matchers. policyMatcher implements the matcher +// interface. +type policyMatcher struct { + permissions *orMatcher + principals *orMatcher +} + +func newPolicyMatcher(policy *v3rbacpb.Policy) (*policyMatcher, error) { + permissions, err := matchersFromPermissions(policy.Permissions) + if err != nil { + return nil, err + } + principals, err := matchersFromPrincipals(policy.Principals) + if err != nil { + return nil, err + } + return &policyMatcher{ + permissions: &orMatcher{matchers: permissions}, + principals: &orMatcher{matchers: principals}, + }, nil +} + +func (pm *policyMatcher) match(data *rpcData) bool { + // A policy matches if and only if at least one of its permissions match the + // action taking place AND at least one if its principals match the + // downstream peer. + return pm.permissions.match(data) && pm.principals.match(data) +} + +// matchersFromPermissions takes a list of permissions (can also be +// a single permission, e.g. from a not matcher which is logically !permission) +// and returns a list of matchers which correspond to that permission. This will +// be called in many instances throughout the initial construction of the RBAC +// engine from the AND and OR matchers and also from the NOT matcher. +func matchersFromPermissions(permissions []*v3rbacpb.Permission) ([]matcher, error) { + var matchers []matcher + for _, permission := range permissions { + switch permission.GetRule().(type) { + case *v3rbacpb.Permission_AndRules: + mList, err := matchersFromPermissions(permission.GetAndRules().Rules) + if err != nil { + return nil, err + } + matchers = append(matchers, &andMatcher{matchers: mList}) + case *v3rbacpb.Permission_OrRules: + mList, err := matchersFromPermissions(permission.GetOrRules().Rules) + if err != nil { + return nil, err + } + matchers = append(matchers, &orMatcher{matchers: mList}) + case *v3rbacpb.Permission_Any: + matchers = append(matchers, &alwaysMatcher{}) + case *v3rbacpb.Permission_Header: + m, err := newHeaderMatcher(permission.GetHeader()) + if err != nil { + return nil, err + } + matchers = append(matchers, m) + case *v3rbacpb.Permission_UrlPath: + m, err := newURLPathMatcher(permission.GetUrlPath()) + if err != nil { + return nil, err + } + matchers = append(matchers, m) + case *v3rbacpb.Permission_DestinationIp: + // Due to this being on server side, the destination IP is the local + // IP. + m, err := newLocalIPMatcher(permission.GetDestinationIp()) + if err != nil { + return nil, err + } + matchers = append(matchers, m) + case *v3rbacpb.Permission_DestinationPort: + matchers = append(matchers, newPortMatcher(permission.GetDestinationPort())) + case *v3rbacpb.Permission_NotRule: + mList, err := matchersFromPermissions([]*v3rbacpb.Permission{{Rule: permission.GetNotRule().Rule}}) + if err != nil { + return nil, err + } + matchers = append(matchers, ¬Matcher{matcherToNot: mList[0]}) + case *v3rbacpb.Permission_Metadata: + // Never matches - so no-op if not inverted, always match if + // inverted. + if permission.GetMetadata().GetInvert() { // Test metadata being no-op and also metadata with invert always matching + matchers = append(matchers, &alwaysMatcher{}) + } + case *v3rbacpb.Permission_RequestedServerName: + // Not supported in gRPC RBAC currently - a permission typed as + // requested server name in the initial config will be a no-op. + } + } + return matchers, nil +} + +func matchersFromPrincipals(principals []*v3rbacpb.Principal) ([]matcher, error) { + var matchers []matcher + for _, principal := range principals { + switch principal.GetIdentifier().(type) { + case *v3rbacpb.Principal_AndIds: + mList, err := matchersFromPrincipals(principal.GetAndIds().Ids) + if err != nil { + return nil, err + } + matchers = append(matchers, &andMatcher{matchers: mList}) + case *v3rbacpb.Principal_OrIds: + mList, err := matchersFromPrincipals(principal.GetOrIds().Ids) + if err != nil { + return nil, err + } + matchers = append(matchers, &orMatcher{matchers: mList}) + case *v3rbacpb.Principal_Any: + matchers = append(matchers, &alwaysMatcher{}) + case *v3rbacpb.Principal_Authenticated_: + authenticatedMatcher, err := newAuthenticatedMatcher(principal.GetAuthenticated()) + if err != nil { + return nil, err + } + matchers = append(matchers, authenticatedMatcher) + case *v3rbacpb.Principal_DirectRemoteIp: + m, err := newRemoteIPMatcher(principal.GetDirectRemoteIp()) + if err != nil { + return nil, err + } + matchers = append(matchers, m) + case *v3rbacpb.Principal_Header: + // Do we need an error here? + m, err := newHeaderMatcher(principal.GetHeader()) + if err != nil { + return nil, err + } + matchers = append(matchers, m) + case *v3rbacpb.Principal_UrlPath: + m, err := newURLPathMatcher(principal.GetUrlPath()) + if err != nil { + return nil, err + } + matchers = append(matchers, m) + case *v3rbacpb.Principal_NotId: + mList, err := matchersFromPrincipals([]*v3rbacpb.Principal{{Identifier: principal.GetNotId().Identifier}}) + if err != nil { + return nil, err + } + matchers = append(matchers, ¬Matcher{matcherToNot: mList[0]}) + case *v3rbacpb.Principal_SourceIp: + // The source ip principal identifier is deprecated. Thus, a + // principal typed as a source ip in the identifier will be a no-op. + // The config should use DirectRemoteIp instead. + case *v3rbacpb.Principal_RemoteIp: + // RBAC in gRPC treats direct_remote_ip and remote_ip as logically + // equivalent, as per A41. + m, err := newRemoteIPMatcher(principal.GetRemoteIp()) + if err != nil { + return nil, err + } + matchers = append(matchers, m) + case *v3rbacpb.Principal_Metadata: + // Not supported in gRPC RBAC currently - a principal typed as + // Metadata in the initial config will be a no-op. + } + } + return matchers, nil +} + +// orMatcher is a matcher where it successfully matches if one of it's +// children successfully match. It also logically represents a principal or +// permission, but can also be it's own entity further down the tree of +// matchers. orMatcher implements the matcher interface. +type orMatcher struct { + matchers []matcher +} + +func (om *orMatcher) match(data *rpcData) bool { + // Range through child matchers and pass in data about incoming RPC, and + // only one child matcher has to match to be logically successful. + for _, m := range om.matchers { + if m.match(data) { + return true + } + } + return false +} + +// andMatcher is a matcher that is successful if every child matcher +// matches. andMatcher implements the matcher interface. +type andMatcher struct { + matchers []matcher +} + +func (am *andMatcher) match(data *rpcData) bool { + for _, m := range am.matchers { + if !m.match(data) { + return false + } + } + return true +} + +// alwaysMatcher is a matcher that will always match. This logically +// represents an any rule for a permission or a principal. alwaysMatcher +// implements the matcher interface. +type alwaysMatcher struct { +} + +func (am *alwaysMatcher) match(data *rpcData) bool { + return true +} + +// notMatcher is a matcher that nots an underlying matcher. notMatcher +// implements the matcher interface. +type notMatcher struct { + matcherToNot matcher +} + +func (nm *notMatcher) match(data *rpcData) bool { + return !nm.matcherToNot.match(data) +} + +// headerMatcher is a matcher that matches on incoming HTTP Headers present +// in the incoming RPC. headerMatcher implements the matcher interface. +type headerMatcher struct { + matcher internalmatcher.HeaderMatcher +} + +func newHeaderMatcher(headerMatcherConfig *v3route_componentspb.HeaderMatcher) (*headerMatcher, error) { + var m internalmatcher.HeaderMatcher + switch headerMatcherConfig.HeaderMatchSpecifier.(type) { + case *v3route_componentspb.HeaderMatcher_ExactMatch: + m = internalmatcher.NewHeaderExactMatcher(headerMatcherConfig.Name, headerMatcherConfig.GetExactMatch(), headerMatcherConfig.InvertMatch) + case *v3route_componentspb.HeaderMatcher_SafeRegexMatch: + regex, err := regexp.Compile(headerMatcherConfig.GetSafeRegexMatch().Regex) + if err != nil { + return nil, err + } + m = internalmatcher.NewHeaderRegexMatcher(headerMatcherConfig.Name, regex, headerMatcherConfig.InvertMatch) + case *v3route_componentspb.HeaderMatcher_RangeMatch: + m = internalmatcher.NewHeaderRangeMatcher(headerMatcherConfig.Name, headerMatcherConfig.GetRangeMatch().Start, headerMatcherConfig.GetRangeMatch().End, headerMatcherConfig.InvertMatch) + case *v3route_componentspb.HeaderMatcher_PresentMatch: + m = internalmatcher.NewHeaderPresentMatcher(headerMatcherConfig.Name, headerMatcherConfig.GetPresentMatch(), headerMatcherConfig.InvertMatch) + case *v3route_componentspb.HeaderMatcher_PrefixMatch: + m = internalmatcher.NewHeaderPrefixMatcher(headerMatcherConfig.Name, headerMatcherConfig.GetPrefixMatch(), headerMatcherConfig.InvertMatch) + case *v3route_componentspb.HeaderMatcher_SuffixMatch: + m = internalmatcher.NewHeaderSuffixMatcher(headerMatcherConfig.Name, headerMatcherConfig.GetSuffixMatch(), headerMatcherConfig.InvertMatch) + case *v3route_componentspb.HeaderMatcher_ContainsMatch: + m = internalmatcher.NewHeaderContainsMatcher(headerMatcherConfig.Name, headerMatcherConfig.GetContainsMatch(), headerMatcherConfig.InvertMatch) + case *v3route_componentspb.HeaderMatcher_StringMatch: + sm, err := internalmatcher.StringMatcherFromProto(headerMatcherConfig.GetStringMatch()) + if err != nil { + return nil, fmt.Errorf("invalid string matcher %+v: %v", headerMatcherConfig.GetStringMatch(), err) + } + m = internalmatcher.NewHeaderStringMatcher(headerMatcherConfig.Name, sm, headerMatcherConfig.InvertMatch) + default: + return nil, errors.New("unknown header matcher type") + } + return &headerMatcher{matcher: m}, nil +} + +func (hm *headerMatcher) match(data *rpcData) bool { + return hm.matcher.Match(data.md) +} + +// urlPathMatcher matches on the URL Path of the incoming RPC. In gRPC, this +// logically maps to the full method name the RPC is calling on the server side. +// urlPathMatcher implements the matcher interface. +type urlPathMatcher struct { + stringMatcher internalmatcher.StringMatcher +} + +func newURLPathMatcher(pathMatcher *v3matcherpb.PathMatcher) (*urlPathMatcher, error) { + stringMatcher, err := internalmatcher.StringMatcherFromProto(pathMatcher.GetPath()) + if err != nil { + return nil, err + } + return &urlPathMatcher{stringMatcher: stringMatcher}, nil +} + +func (upm *urlPathMatcher) match(data *rpcData) bool { + return upm.stringMatcher.Match(data.fullMethod) +} + +// remoteIPMatcher and localIPMatcher both are matchers that match against +// a CIDR Range. Two different matchers are needed as the remote and destination +// ip addresses come from different parts of the data about incoming RPC's +// passed in. Matching a CIDR Range means to determine whether the IP Address +// falls within the CIDR Range or not. They both implement the matcher +// interface. +type remoteIPMatcher struct { + // ipNet represents the CidrRange that this matcher was configured with. + // This is what will remote and destination IP's will be matched against. + ipNet *net.IPNet +} + +func newRemoteIPMatcher(cidrRange *v3corepb.CidrRange) (*remoteIPMatcher, error) { + // Convert configuration to a cidrRangeString, as Go standard library has + // methods that parse cidr string. + cidrRangeString := fmt.Sprintf("%s/%d", cidrRange.AddressPrefix, cidrRange.PrefixLen.Value) + _, ipNet, err := net.ParseCIDR(cidrRangeString) + if err != nil { + return nil, err + } + return &remoteIPMatcher{ipNet: ipNet}, nil +} + +func (sim *remoteIPMatcher) match(data *rpcData) bool { + return sim.ipNet.Contains(net.IP(net.ParseIP(data.peerInfo.Addr.String()))) +} + +type localIPMatcher struct { + ipNet *net.IPNet +} + +func newLocalIPMatcher(cidrRange *v3corepb.CidrRange) (*localIPMatcher, error) { + cidrRangeString := fmt.Sprintf("%s/%d", cidrRange.AddressPrefix, cidrRange.PrefixLen.Value) + _, ipNet, err := net.ParseCIDR(cidrRangeString) + if err != nil { + return nil, err + } + return &localIPMatcher{ipNet: ipNet}, nil +} + +func (dim *localIPMatcher) match(data *rpcData) bool { + return dim.ipNet.Contains(net.IP(net.ParseIP(data.localAddr.String()))) +} + +// portMatcher matches on whether the destination port of the RPC matches the +// destination port this matcher was instantiated with. portMatcher +// implements the matcher interface. +type portMatcher struct { + destinationPort uint32 +} + +func newPortMatcher(destinationPort uint32) *portMatcher { + return &portMatcher{destinationPort: destinationPort} +} + +func (pm *portMatcher) match(data *rpcData) bool { + return data.destinationPort == pm.destinationPort +} + +// authenticatedMatcher matches on the name of the Principal. If set, the URI +// SAN or DNS SAN in that order is used from the certificate, otherwise the +// subject field is used. If unset, it applies to any user that is +// authenticated. authenticatedMatcher implements the matcher interface. +type authenticatedMatcher struct { + stringMatcher *internalmatcher.StringMatcher +} + +func newAuthenticatedMatcher(authenticatedMatcherConfig *v3rbacpb.Principal_Authenticated) (*authenticatedMatcher, error) { + // Represents this line in the RBAC documentation = "If unset, it applies to + // any user that is authenticated" (see package-level comments). + if authenticatedMatcherConfig.PrincipalName == nil { + return &authenticatedMatcher{}, nil + } + stringMatcher, err := internalmatcher.StringMatcherFromProto(authenticatedMatcherConfig.PrincipalName) + if err != nil { + return nil, err + } + return &authenticatedMatcher{stringMatcher: &stringMatcher}, nil +} + +func (am *authenticatedMatcher) match(data *rpcData) bool { + if data.authType != "tls" { + // Connection is not authenticated. + return false + } + if am.stringMatcher == nil { + // Allows any authenticated user. + return true + } + // "If there is no client certificate (thus no SAN nor Subject), check if "" + // (empty string) matches. If it matches, the principal_name is said to + // match" - A41 + if len(data.certs) == 0 { + return am.stringMatcher.Match("") + } + cert := data.certs[0] + // The order of matching as per the RBAC documentation (see package-level comments) + // is as follows: URI SANs, DNS SANs, and then subject name. + for _, uriSAN := range cert.URIs { + if am.stringMatcher.Match(uriSAN.String()) { + return true + } + } + for _, dnsSAN := range cert.DNSNames { + if am.stringMatcher.Match(dnsSAN) { + return true + } + } + return am.stringMatcher.Match(cert.Subject.String()) +} diff --git a/internal/xds/rbac/rbac_engine.go b/internal/xds/rbac/rbac_engine.go new file mode 100644 index 000000000000..63237affe23f --- /dev/null +++ b/internal/xds/rbac/rbac_engine.go @@ -0,0 +1,317 @@ +/* + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package rbac provides service-level and method-level access control for a +// service. See +// https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/rbac/v3/rbac.proto#role-based-access-control-rbac +// for documentation. +package rbac + +import ( + "context" + "crypto/x509" + "errors" + "fmt" + "net" + "strconv" + + v3rbacpb "github.com/envoyproxy/go-control-plane/envoy/config/rbac/v3" + "google.golang.org/grpc" + "google.golang.org/grpc/authz/audit" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal/transport" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" +) + +var logger = grpclog.Component("rbac") + +var getConnection = transport.GetConnection + +// ChainEngine represents a chain of RBAC Engines, used to make authorization +// decisions on incoming RPCs. +type ChainEngine struct { + chainedEngines []*engine +} + +// NewChainEngine returns a chain of RBAC engines, used to make authorization +// decisions on incoming RPCs. Returns a non-nil error for invalid policies. +func NewChainEngine(policies []*v3rbacpb.RBAC, policyName string) (*ChainEngine, error) { + engines := make([]*engine, 0, len(policies)) + for _, policy := range policies { + engine, err := newEngine(policy, policyName) + if err != nil { + return nil, err + } + engines = append(engines, engine) + } + return &ChainEngine{chainedEngines: engines}, nil +} + +func (cre *ChainEngine) logRequestDetails(rpcData *rpcData) { + if logger.V(2) { + logger.Infof("checking request: url path=%s", rpcData.fullMethod) + if len(rpcData.certs) > 0 { + cert := rpcData.certs[0] + logger.Infof("uri sans=%q, dns sans=%q, subject=%v", cert.URIs, cert.DNSNames, cert.Subject) + } + } +} + +// IsAuthorized determines if an incoming RPC is authorized based on the chain of RBAC +// engines and their associated actions. +// +// Errors returned by this function are compatible with the status package. +func (cre *ChainEngine) IsAuthorized(ctx context.Context) error { + // This conversion step (i.e. pulling things out of ctx) can be done once, + // and then be used for the whole chain of RBAC Engines. + rpcData, err := newRPCData(ctx) + if err != nil { + logger.Errorf("newRPCData: %v", err) + return status.Errorf(codes.Internal, "gRPC RBAC: %v", err) + } + for _, engine := range cre.chainedEngines { + matchingPolicyName, ok := engine.findMatchingPolicy(rpcData) + if logger.V(2) && ok { + logger.Infof("incoming RPC matched to policy %v in engine with action %v", matchingPolicyName, engine.action) + } + + switch { + case engine.action == v3rbacpb.RBAC_ALLOW && !ok: + cre.logRequestDetails(rpcData) + engine.doAuditLogging(rpcData, matchingPolicyName, false) + return status.Errorf(codes.PermissionDenied, "incoming RPC did not match an allow policy") + case engine.action == v3rbacpb.RBAC_DENY && ok: + cre.logRequestDetails(rpcData) + engine.doAuditLogging(rpcData, matchingPolicyName, false) + return status.Errorf(codes.PermissionDenied, "incoming RPC matched a deny policy %q", matchingPolicyName) + } + // Every policy in the engine list must be queried. Thus, iterate to the + // next policy. + engine.doAuditLogging(rpcData, matchingPolicyName, true) + } + // If the incoming RPC gets through all of the engines successfully (i.e. + // doesn't not match an allow or match a deny engine), the RPC is authorized + // to proceed. + return nil +} + +// engine is used for matching incoming RPCs to policies. +type engine struct { + // TODO(gtcooke94) - differentiate between `policyName`, `policies`, and `rules` + policyName string + policies map[string]*policyMatcher + // action must be ALLOW or DENY. + action v3rbacpb.RBAC_Action + auditLoggers []audit.Logger + auditCondition v3rbacpb.RBAC_AuditLoggingOptions_AuditCondition +} + +// newEngine creates an RBAC Engine based on the contents of a policy. Returns a +// non-nil error if the policy is invalid. +func newEngine(config *v3rbacpb.RBAC, policyName string) (*engine, error) { + a := config.GetAction() + if a != v3rbacpb.RBAC_ALLOW && a != v3rbacpb.RBAC_DENY { + return nil, fmt.Errorf("unsupported action %s", config.Action) + } + + policies := make(map[string]*policyMatcher, len(config.GetPolicies())) + for name, policy := range config.GetPolicies() { + matcher, err := newPolicyMatcher(policy) + if err != nil { + return nil, err + } + policies[name] = matcher + } + + auditLoggers, auditCondition, err := parseAuditOptions(config.GetAuditLoggingOptions()) + if err != nil { + return nil, err + } + return &engine{ + policyName: policyName, + policies: policies, + action: a, + auditLoggers: auditLoggers, + auditCondition: auditCondition, + }, nil +} + +func parseAuditOptions(opts *v3rbacpb.RBAC_AuditLoggingOptions) ([]audit.Logger, v3rbacpb.RBAC_AuditLoggingOptions_AuditCondition, error) { + if opts == nil { + return nil, v3rbacpb.RBAC_AuditLoggingOptions_NONE, nil + } + var auditLoggers []audit.Logger + for _, logger := range opts.LoggerConfigs { + auditLogger, err := buildLogger(logger) + if err != nil { + return nil, v3rbacpb.RBAC_AuditLoggingOptions_NONE, err + } + if auditLogger == nil { + // This occurs when the audit logger is not registered but also + // marked optional. + continue + } + auditLoggers = append(auditLoggers, auditLogger) + } + return auditLoggers, opts.GetAuditCondition(), nil + +} + +// findMatchingPolicy determines if an incoming RPC matches a policy. On a +// successful match, it returns the name of the matching policy and a true bool +// to specify that there was a matching policy found. It returns false in +// the case of not finding a matching policy. +func (e *engine) findMatchingPolicy(rpcData *rpcData) (string, bool) { + for policy, matcher := range e.policies { + if matcher.match(rpcData) { + return policy, true + } + } + return "", false +} + +// newRPCData takes an incoming context (should be a context representing state +// needed for server RPC Call with metadata, peer info (used for source ip/port +// and TLS information) and connection (used for destination ip/port) piped into +// it) and the method name of the Service being called server side and populates +// an rpcData struct ready to be passed to the RBAC Engine to find a matching +// policy. +func newRPCData(ctx context.Context) (*rpcData, error) { + // The caller should populate all of these fields (i.e. for empty headers, + // pipe an empty md into context). + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return nil, errors.New("missing metadata in incoming context") + } + // ":method can be hard-coded to POST if unavailable" - A41 + md[":method"] = []string{"POST"} + // "If the transport exposes TE in Metadata, then RBAC must special-case the + // header to treat it as not present." - A41 + delete(md, "TE") + + pi, ok := peer.FromContext(ctx) + if !ok { + return nil, errors.New("missing peer info in incoming context") + } + + // The methodName will be available in the passed in ctx from a unary or streaming + // interceptor, as grpc.Server pipes in a transport stream which contains the methodName + // into contexts available in both unary or streaming interceptors. + mn, ok := grpc.Method(ctx) + if !ok { + return nil, errors.New("missing method in incoming context") + } + + // The connection is needed in order to find the destination address and + // port of the incoming RPC Call. + conn := getConnection(ctx) + if conn == nil { + return nil, errors.New("missing connection in incoming context") + } + _, dPort, err := net.SplitHostPort(conn.LocalAddr().String()) + if err != nil { + return nil, fmt.Errorf("error parsing local address: %v", err) + } + dp, err := strconv.ParseUint(dPort, 10, 32) + if err != nil { + return nil, fmt.Errorf("error parsing local address: %v", err) + } + + var authType string + var peerCertificates []*x509.Certificate + if pi.AuthInfo != nil { + tlsInfo, ok := pi.AuthInfo.(credentials.TLSInfo) + if ok { + authType = pi.AuthInfo.AuthType() + peerCertificates = tlsInfo.State.PeerCertificates + } + } + + return &rpcData{ + md: md, + peerInfo: pi, + fullMethod: mn, + destinationPort: uint32(dp), + localAddr: conn.LocalAddr(), + authType: authType, + certs: peerCertificates, + }, nil +} + +// rpcData wraps data pulled from an incoming RPC that the RBAC engine needs to +// find a matching policy. +type rpcData struct { + // md is the HTTP Headers that are present in the incoming RPC. + md metadata.MD + // peerInfo is information about the downstream peer. + peerInfo *peer.Peer + // fullMethod is the method name being called on the upstream service. + fullMethod string + // destinationPort is the port that the RPC is being sent to on the + // server. + destinationPort uint32 + // localAddr is the address that the RPC is being sent to. + localAddr net.Addr + // authType is the type of authentication e.g. "tls". + authType string + // certs are the certificates presented by the peer during a TLS + // handshake. + certs []*x509.Certificate +} + +func (e *engine) doAuditLogging(rpcData *rpcData, rule string, authorized bool) { + // In the RBAC world, we need to have a SPIFFE ID as the principal for this + // to be meaningful + principal := "" + if rpcData.peerInfo != nil && rpcData.peerInfo.AuthInfo != nil && rpcData.peerInfo.AuthInfo.AuthType() == "tls" { + // If AuthType = tls, then we can cast AuthInfo to TLSInfo. + tlsInfo := rpcData.peerInfo.AuthInfo.(credentials.TLSInfo) + if tlsInfo.SPIFFEID != nil { + principal = tlsInfo.SPIFFEID.String() + } + } + + //TODO(gtcooke94) check if we need to log before creating the event + event := &audit.Event{ + FullMethodName: rpcData.fullMethod, + Principal: principal, + PolicyName: e.policyName, + MatchedRule: rule, + Authorized: authorized, + } + for _, logger := range e.auditLoggers { + switch e.auditCondition { + case v3rbacpb.RBAC_AuditLoggingOptions_ON_DENY: + if !authorized { + logger.Log(event) + } + case v3rbacpb.RBAC_AuditLoggingOptions_ON_ALLOW: + if authorized { + logger.Log(event) + } + case v3rbacpb.RBAC_AuditLoggingOptions_ON_DENY_AND_ALLOW: + logger.Log(event) + } + } +} + +// This is used when converting a custom config from raw JSON to a TypedStruct. +// The TypeURL of the TypeStruct will be "grpc.authz.audit_logging/". +const typeURLPrefix = "grpc.authz.audit_logging/" diff --git a/internal/xds/rbac/rbac_engine_test.go b/internal/xds/rbac/rbac_engine_test.go new file mode 100644 index 000000000000..d0c3ae7af3e8 --- /dev/null +++ b/internal/xds/rbac/rbac_engine_test.go @@ -0,0 +1,1930 @@ +/* + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package rbac + +import ( + "context" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/json" + "fmt" + "net" + "net/url" + "reflect" + "testing" + + v1xdsudpatypepb "github.com/cncf/xds/go/udpa/type/v1" + v3xdsxdstypepb "github.com/cncf/xds/go/xds/type/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3rbacpb "github.com/envoyproxy/go-control-plane/envoy/config/rbac/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3" + v3typepb "github.com/envoyproxy/go-control-plane/envoy/type/v3" + wrapperspb "github.com/golang/protobuf/ptypes/wrappers" + "google.golang.org/grpc" + "google.golang.org/grpc/authz/audit" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/structpb" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +type addr struct { + ipAddress string +} + +func (addr) Network() string { return "" } +func (a *addr) String() string { return a.ipAddress } + +// TestNewChainEngine tests the construction of the ChainEngine. Due to some +// types of RBAC configuration being logically wrong and returning an error +// rather than successfully constructing the RBAC Engine, this test tests both +// RBAC Configurations deemed successful and also RBAC Configurations that will +// raise errors. +func (s) TestNewChainEngine(t *testing.T) { + tests := []struct { + name string + policies []*v3rbacpb.RBAC + wantErr bool + policyName string + }{ + { + name: "SuccessCaseAnyMatchSingular", + policies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "anyone": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + }, + { + name: "SuccessCaseAnyMatchMultiple", + policies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "anyone": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + { + Action: v3rbacpb.RBAC_DENY, + Policies: map[string]*v3rbacpb.Policy{ + "anyone": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + }, + { + name: "SuccessCaseSimplePolicySingular", + policies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "localhost-fan": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_DestinationPort{DestinationPort: 8080}}, + {Rule: &v3rbacpb.Permission_UrlPath{UrlPath: &v3matcherpb.PathMatcher{Rule: &v3matcherpb.PathMatcher_Path{Path: &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "localhost-fan-page"}}}}}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + }, + // SuccessCaseSimplePolicyTwoPolicies tests the construction of the + // chained engines in the case where there are two policies in a list, + // one with an allow policy and one with a deny policy. A situation + // where two policies (allow and deny) is a very common use case for + // this API, and should successfully build. + { + name: "SuccessCaseSimplePolicyTwoPolicies", + policies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "localhost-fan": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_DestinationPort{DestinationPort: 8080}}, + {Rule: &v3rbacpb.Permission_UrlPath{UrlPath: &v3matcherpb.PathMatcher{Rule: &v3matcherpb.PathMatcher_Path{Path: &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "localhost-fan-page"}}}}}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + { + Action: v3rbacpb.RBAC_DENY, + Policies: map[string]*v3rbacpb.Policy{ + "localhost-fan": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_DestinationPort{DestinationPort: 8080}}, + {Rule: &v3rbacpb.Permission_UrlPath{UrlPath: &v3matcherpb.PathMatcher{Rule: &v3matcherpb.PathMatcher_Path{Path: &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "localhost-fan-page"}}}}}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + }, + { + name: "SuccessCaseEnvoyExampleSingular", + policies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "service-admin": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Authenticated_{Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "cluster.local/ns/default/sa/admin"}}}}}, + {Identifier: &v3rbacpb.Principal_Authenticated_{Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "cluster.local/ns/default/sa/superuser"}}}}}, + }, + }, + "product-viewer": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_AndRules{AndRules: &v3rbacpb.Permission_Set{ + Rules: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Header{Header: &v3routepb.HeaderMatcher{Name: ":method", HeaderMatchSpecifier: &v3routepb.HeaderMatcher_ExactMatch{ExactMatch: "GET"}}}}, + {Rule: &v3rbacpb.Permission_UrlPath{UrlPath: &v3matcherpb.PathMatcher{Rule: &v3matcherpb.PathMatcher_Path{Path: &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: "/products"}}}}}}, + {Rule: &v3rbacpb.Permission_OrRules{OrRules: &v3rbacpb.Permission_Set{ + Rules: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_DestinationPort{DestinationPort: 80}}, + {Rule: &v3rbacpb.Permission_DestinationPort{DestinationPort: 443}}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + }, + { + name: "SourceIpMatcherSuccessSingular", + policies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "certain-source-ip": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_DirectRemoteIp{DirectRemoteIp: &v3corepb.CidrRange{AddressPrefix: "0.0.0.0", PrefixLen: &wrapperspb.UInt32Value{Value: uint32(10)}}}}, + }, + }, + }, + }, + }, + }, + { + name: "SourceIpMatcherFailureSingular", + policies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "certain-source-ip": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_DirectRemoteIp{DirectRemoteIp: &v3corepb.CidrRange{AddressPrefix: "not a correct address", PrefixLen: &wrapperspb.UInt32Value{Value: uint32(10)}}}}, + }, + }, + }, + }, + }, + wantErr: true, + }, + { + name: "DestinationIpMatcherSuccess", + policies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "certain-destination-ip": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_DestinationIp{DestinationIp: &v3corepb.CidrRange{AddressPrefix: "0.0.0.0", PrefixLen: &wrapperspb.UInt32Value{Value: uint32(10)}}}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + }, + { + name: "DestinationIpMatcherFailure", + policies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "certain-destination-ip": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_DestinationIp{DestinationIp: &v3corepb.CidrRange{AddressPrefix: "not a correct address", PrefixLen: &wrapperspb.UInt32Value{Value: uint32(10)}}}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + wantErr: true, + }, + { + name: "MatcherToNotPolicy", + policies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "not-secret-content": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_NotRule{NotRule: &v3rbacpb.Permission{Rule: &v3rbacpb.Permission_UrlPath{UrlPath: &v3matcherpb.PathMatcher{Rule: &v3matcherpb.PathMatcher_Path{Path: &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: "/secret-content"}}}}}}}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + }, + { + name: "MatcherToNotPrinicipal", + policies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "not-from-certain-ip": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_NotId{NotId: &v3rbacpb.Principal{Identifier: &v3rbacpb.Principal_DirectRemoteIp{DirectRemoteIp: &v3corepb.CidrRange{AddressPrefix: "0.0.0.0", PrefixLen: &wrapperspb.UInt32Value{Value: uint32(10)}}}}}}, + }, + }, + }, + }, + }, + }, + // PrinicpalProductViewer tests the construction of a chained engine + // with a policy that allows any downstream to send a GET request on a + // certain path. + { + name: "PrincipalProductViewer", + policies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "product-viewer": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + { + Identifier: &v3rbacpb.Principal_AndIds{AndIds: &v3rbacpb.Principal_Set{Ids: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Header{Header: &v3routepb.HeaderMatcher{Name: ":method", HeaderMatchSpecifier: &v3routepb.HeaderMatcher_ExactMatch{ExactMatch: "GET"}}}}, + {Identifier: &v3rbacpb.Principal_OrIds{OrIds: &v3rbacpb.Principal_Set{ + Ids: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_UrlPath{UrlPath: &v3matcherpb.PathMatcher{Rule: &v3matcherpb.PathMatcher_Path{Path: &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: "/books"}}}}}}, + {Identifier: &v3rbacpb.Principal_UrlPath{UrlPath: &v3matcherpb.PathMatcher{Rule: &v3matcherpb.PathMatcher_Path{Path: &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: "/cars"}}}}}}, + }, + }}}, + }}}, + }, + }, + }, + }, + }, + }, + }, + // Certain Headers tests the construction of a chained engine with a + // policy that allows any downstream to send an HTTP request with + // certain headers. + { + name: "CertainHeaders", + policies: []*v3rbacpb.RBAC{ + { + Policies: map[string]*v3rbacpb.Policy{ + "certain-headers": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + { + Identifier: &v3rbacpb.Principal_OrIds{OrIds: &v3rbacpb.Principal_Set{Ids: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Header{Header: &v3routepb.HeaderMatcher{Name: ":method", HeaderMatchSpecifier: &v3routepb.HeaderMatcher_ExactMatch{ExactMatch: "GET"}}}}, + {Identifier: &v3rbacpb.Principal_Header{Header: &v3routepb.HeaderMatcher{Name: ":method", HeaderMatchSpecifier: &v3routepb.HeaderMatcher_SafeRegexMatch{SafeRegexMatch: &v3matcherpb.RegexMatcher{Regex: "GET"}}}}}, + {Identifier: &v3rbacpb.Principal_Header{Header: &v3routepb.HeaderMatcher{Name: ":method", HeaderMatchSpecifier: &v3routepb.HeaderMatcher_RangeMatch{RangeMatch: &v3typepb.Int64Range{ + Start: 0, + End: 64, + }}}}}, + {Identifier: &v3rbacpb.Principal_Header{Header: &v3routepb.HeaderMatcher{Name: ":method", HeaderMatchSpecifier: &v3routepb.HeaderMatcher_PresentMatch{PresentMatch: true}}}}, + {Identifier: &v3rbacpb.Principal_Header{Header: &v3routepb.HeaderMatcher{Name: ":method", HeaderMatchSpecifier: &v3routepb.HeaderMatcher_PrefixMatch{PrefixMatch: "GET"}}}}, + {Identifier: &v3rbacpb.Principal_Header{Header: &v3routepb.HeaderMatcher{Name: ":method", HeaderMatchSpecifier: &v3routepb.HeaderMatcher_SuffixMatch{SuffixMatch: "GET"}}}}, + {Identifier: &v3rbacpb.Principal_Header{Header: &v3routepb.HeaderMatcher{Name: ":method", HeaderMatchSpecifier: &v3routepb.HeaderMatcher_ContainsMatch{ContainsMatch: "GET"}}}}, + {Identifier: &v3rbacpb.Principal_Header{Header: &v3routepb.HeaderMatcher{Name: ":method", HeaderMatchSpecifier: &v3routepb.HeaderMatcher_ContainsMatch{ContainsMatch: "GET"}}}}, + }}}, + }, + }, + }, + }, + }, + }, + }, + { + name: "LogAction", + policies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_LOG, + Policies: map[string]*v3rbacpb.Policy{ + "anyone": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + wantErr: true, + }, + { + name: "ActionNotSpecified", + policies: []*v3rbacpb.RBAC{ + { + Policies: map[string]*v3rbacpb.Policy{ + "anyone": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + }, + { + name: "SimpleAuditLogger", + policies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "anyone": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_ON_ALLOW, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{ + Name: "TestAuditLoggerBuffer", + TypedConfig: createUDPATypedStruct(t, map[string]any{}, "SimpleAuditLogger_TestAuditLoggerBuffer")}, + IsOptional: false, + }, + }, + }, + }, + }, + }, + { + name: "AuditLoggerCustomConfig", + policies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "anyone": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_ON_ALLOW, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{ + Name: "TestAuditLoggerCustomConfig", + TypedConfig: createUDPATypedStruct(t, map[string]any{"abc": 123, "xyz": "123"}, "AuditLoggerCustomConfig_TestAuditLoggerCustomConfig")}, + IsOptional: false, + }, + }, + }, + }, + }, + policyName: "test_policy", + }, + { + name: "AuditLoggerCustomConfigXdsTypedStruct", + policies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "anyone": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_ON_ALLOW, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{ + Name: "TestAuditLoggerCustomConfig", + TypedConfig: createXDSTypedStruct(t, map[string]any{"abc": 123, "xyz": "123"}, "AuditLoggerCustomConfigXdsTypedStruct_TestAuditLoggerCustomConfig")}, + IsOptional: false, + }, + }, + }, + }, + }, + policyName: "test_policy", + }, + { + name: "Missing Optional AuditLogger doesn't fail", + policies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "anyone": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_ON_ALLOW, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{ + Name: "UnsupportedLogger", + TypedConfig: createUDPATypedStruct(t, map[string]any{}, "Missing Optional AuditLogger doesn't fail_UnsupportedLogger")}, + IsOptional: true, + }, + }, + }, + }, + }, + }, + { + name: "Missing Non-Optional AuditLogger fails", + policies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "anyone": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_ON_ALLOW, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{ + Name: "UnsupportedLogger", + TypedConfig: createUDPATypedStruct(t, map[string]any{}, "Missing Non-Optional AuditLogger fails_UnsupportedLogger")}, + IsOptional: false, + }, + }, + }, + }, + }, + wantErr: true, + }, + { + name: "Cannot_parse_missing_CustomConfig", + policies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "anyone": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_ON_ALLOW, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{ + Name: "TestAuditLoggerCustomConfig", + }, + IsOptional: false, + }, + }, + }, + }, + }, + wantErr: true, + }, + { + name: "Cannot_parse_bad_CustomConfig", + policies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "anyone": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_ON_ALLOW, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{ + Name: "TestAuditLoggerCustomConfig", + TypedConfig: createUDPATypedStruct(t, map[string]any{"abc": "BADVALUE", "xyz": "123"}, "Cannot_parse_bad_CustomConfig_TestAuditLoggerCustomConfig")}, + IsOptional: false, + }, + }, + }, + }, + }, + wantErr: true, + }, + { + name: "Cannot_parse_missing_typedConfig_name", + policies: []*v3rbacpb.RBAC{ + { + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "anyone": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_ON_ALLOW, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{ + Name: "TestAuditLoggerCustomConfig", + TypedConfig: createUDPATypedStruct(t, map[string]any{"abc": 123, "xyz": "123"}, "")}, + IsOptional: false, + }, + }, + }, + }, + }, + wantErr: true, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + b := TestAuditLoggerBufferBuilder{testName: test.name} + audit.RegisterLoggerBuilder(&b) + b2 := TestAuditLoggerCustomConfigBuilder{testName: test.name} + audit.RegisterLoggerBuilder(&b2) + if _, err := NewChainEngine(test.policies, test.policyName); (err != nil) != test.wantErr { + t.Fatalf("NewChainEngine(%+v) returned err: %v, wantErr: %v", test.policies, err, test.wantErr) + } + }) + } +} + +type rbacQuery struct { + rpcData *rpcData + wantStatusCode codes.Code + wantAuditEvents []*audit.Event +} + +// TestChainEngine tests the chain of RBAC Engines by configuring the chain of +// engines in a certain way in different scenarios. After configuring the chain +// of engines in a certain way, this test pings the chain of engines with +// different types of data representing incoming RPC's (piped into a context), +// and verifies that it works as expected. +func (s) TestChainEngine(t *testing.T) { + defer func(gc func(ctx context.Context) net.Conn) { + getConnection = gc + }(getConnection) + tests := []struct { + name string + rbacConfigs []*v3rbacpb.RBAC + rbacQueries []rbacQuery + policyName string + }{ + // SuccessCaseAnyMatch tests a single RBAC Engine instantiated with + // a config with a policy with any rules for both permissions and + // principals, meaning that any data about incoming RPC's that the RBAC + // Engine is queried with should match that policy. + { + name: "SuccessCaseAnyMatch", + rbacConfigs: []*v3rbacpb.RBAC{ + { + Policies: map[string]*v3rbacpb.Policy{ + "anyone": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + rbacQueries: []rbacQuery{ + { + rpcData: &rpcData{ + fullMethod: "some method", + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0"}, + }, + }, + wantStatusCode: codes.OK, + }, + }, + }, + // SuccessCaseSimplePolicy is a test that tests a single policy + // that only allows an rpc to proceed if the rpc is calling with a certain + // path. + { + name: "SuccessCaseSimplePolicy", + rbacConfigs: []*v3rbacpb.RBAC{ + { + Policies: map[string]*v3rbacpb.Policy{ + "localhost-fan": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_UrlPath{UrlPath: &v3matcherpb.PathMatcher{Rule: &v3matcherpb.PathMatcher_Path{Path: &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "localhost-fan-page"}}}}}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + rbacQueries: []rbacQuery{ + // This RPC should match with the local host fan policy. Thus, + // this RPC should be allowed to proceed. + { + rpcData: &rpcData{ + fullMethod: "localhost-fan-page", + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0"}, + }, + }, + wantStatusCode: codes.OK, + }, + + // This RPC shouldn't match with the local host fan policy. Thus, + // this rpc shouldn't be allowed to proceed. + { + rpcData: &rpcData{ + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0"}, + }, + }, + wantStatusCode: codes.PermissionDenied, + }, + }, + }, + // SuccessCaseEnvoyExample is a test based on the example provided + // in the EnvoyProxy docs. The RBAC Config contains two policies, + // service admin and product viewer, that provides an example of a real + // RBAC Config that might be configured for a given for a given backend + // service. + { + name: "SuccessCaseEnvoyExample", + rbacConfigs: []*v3rbacpb.RBAC{ + { + Policies: map[string]*v3rbacpb.Policy{ + "service-admin": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Authenticated_{Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "//cluster.local/ns/default/sa/admin"}}}}}, + {Identifier: &v3rbacpb.Principal_Authenticated_{Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "//cluster.local/ns/default/sa/superuser"}}}}}, + }, + }, + "product-viewer": { + Permissions: []*v3rbacpb.Permission{ + { + Rule: &v3rbacpb.Permission_AndRules{AndRules: &v3rbacpb.Permission_Set{ + Rules: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Header{Header: &v3routepb.HeaderMatcher{Name: ":method", HeaderMatchSpecifier: &v3routepb.HeaderMatcher_ExactMatch{ExactMatch: "GET"}}}}, + {Rule: &v3rbacpb.Permission_UrlPath{UrlPath: &v3matcherpb.PathMatcher{Rule: &v3matcherpb.PathMatcher_Path{Path: &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: "/products"}}}}}}, + }, + }, + }, + }, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + rbacQueries: []rbacQuery{ + // This incoming RPC Call should match with the service admin + // policy. + { + rpcData: &rpcData{ + fullMethod: "some method", + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0"}, + AuthInfo: credentials.TLSInfo{ + State: tls.ConnectionState{ + PeerCertificates: []*x509.Certificate{ + { + URIs: []*url.URL{ + { + Host: "cluster.local", + Path: "/ns/default/sa/admin", + }, + }, + }, + }, + }, + }, + }, + }, + wantStatusCode: codes.OK, + }, + // These incoming RPC calls should not match any policy. + { + rpcData: &rpcData{ + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0"}, + }, + }, + wantStatusCode: codes.PermissionDenied, + }, + { + rpcData: &rpcData{ + fullMethod: "get-product-list", + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0"}, + }, + }, + wantStatusCode: codes.PermissionDenied, + }, + { + rpcData: &rpcData{ + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0"}, + AuthInfo: credentials.TLSInfo{ + State: tls.ConnectionState{ + PeerCertificates: []*x509.Certificate{ + { + Subject: pkix.Name{ + CommonName: "localhost", + }, + }, + }, + }, + }, + }, + }, + wantStatusCode: codes.PermissionDenied, + }, + }, + }, + { + name: "NotMatcher", + rbacConfigs: []*v3rbacpb.RBAC{ + { + Policies: map[string]*v3rbacpb.Policy{ + "not-secret-content": { + Permissions: []*v3rbacpb.Permission{ + { + Rule: &v3rbacpb.Permission_NotRule{ + NotRule: &v3rbacpb.Permission{Rule: &v3rbacpb.Permission_UrlPath{UrlPath: &v3matcherpb.PathMatcher{Rule: &v3matcherpb.PathMatcher_Path{Path: &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: "/secret-content"}}}}}}, + }, + }, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + rbacQueries: []rbacQuery{ + // This incoming RPC Call should match with the not-secret-content policy. + { + rpcData: &rpcData{ + fullMethod: "/regular-content", + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0"}, + }, + }, + wantStatusCode: codes.OK, + }, + // This incoming RPC Call shouldn't match with the not-secret-content-policy. + { + rpcData: &rpcData{ + fullMethod: "/secret-content", + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0"}, + }, + }, + wantStatusCode: codes.PermissionDenied, + }, + }, + }, + { + name: "DirectRemoteIpMatcher", + rbacConfigs: []*v3rbacpb.RBAC{ + { + Policies: map[string]*v3rbacpb.Policy{ + "certain-direct-remote-ip": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_DirectRemoteIp{DirectRemoteIp: &v3corepb.CidrRange{AddressPrefix: "0.0.0.0", PrefixLen: &wrapperspb.UInt32Value{Value: uint32(10)}}}}, + }, + }, + }, + }, + }, + rbacQueries: []rbacQuery{ + // This incoming RPC Call should match with the certain-direct-remote-ip policy. + { + rpcData: &rpcData{ + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0"}, + }, + }, + wantStatusCode: codes.OK, + }, + // This incoming RPC Call shouldn't match with the certain-direct-remote-ip policy. + { + rpcData: &rpcData{ + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "10.0.0.0"}, + }, + }, + wantStatusCode: codes.PermissionDenied, + }, + }, + }, + // This test tests a RBAC policy configured with a remote-ip policy. + // This should be logically equivalent to configuring a Engine with a + // direct-remote-ip policy, as per A41 - "allow equating RBAC's + // direct_remote_ip and remote_ip." + { + name: "RemoteIpMatcher", + rbacConfigs: []*v3rbacpb.RBAC{ + { + Policies: map[string]*v3rbacpb.Policy{ + "certain-remote-ip": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_RemoteIp{RemoteIp: &v3corepb.CidrRange{AddressPrefix: "0.0.0.0", PrefixLen: &wrapperspb.UInt32Value{Value: uint32(10)}}}}, + }, + }, + }, + }, + }, + rbacQueries: []rbacQuery{ + // This incoming RPC Call should match with the certain-remote-ip policy. + { + rpcData: &rpcData{ + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0"}, + }, + }, + wantStatusCode: codes.OK, + }, + // This incoming RPC Call shouldn't match with the certain-remote-ip policy. + { + rpcData: &rpcData{ + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "10.0.0.0"}, + }, + }, + wantStatusCode: codes.PermissionDenied, + }, + }, + }, + { + name: "DestinationIpMatcher", + rbacConfigs: []*v3rbacpb.RBAC{ + { + Policies: map[string]*v3rbacpb.Policy{ + "certain-destination-ip": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_DestinationIp{DestinationIp: &v3corepb.CidrRange{AddressPrefix: "0.0.0.0", PrefixLen: &wrapperspb.UInt32Value{Value: uint32(10)}}}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + rbacQueries: []rbacQuery{ + // This incoming RPC Call shouldn't match with the + // certain-destination-ip policy, as the test listens on local + // host. + { + rpcData: &rpcData{ + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "10.0.0.0"}, + }, + }, + wantStatusCode: codes.PermissionDenied, + }, + }, + }, + // AllowAndDenyPolicy tests a policy with an allow (on path) and + // deny (on port) policy chained together. This represents how a user + // configured interceptor would use this, and also is a potential + // configuration for a dynamic xds interceptor. + { + name: "AllowAndDenyPolicy", + rbacConfigs: []*v3rbacpb.RBAC{ + { + Policies: map[string]*v3rbacpb.Policy{ + "certain-source-ip": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_DirectRemoteIp{DirectRemoteIp: &v3corepb.CidrRange{AddressPrefix: "0.0.0.0", PrefixLen: &wrapperspb.UInt32Value{Value: uint32(10)}}}}, + }, + }, + }, + Action: v3rbacpb.RBAC_ALLOW, + }, + { + Policies: map[string]*v3rbacpb.Policy{ + "localhost-fan": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_UrlPath{UrlPath: &v3matcherpb.PathMatcher{Rule: &v3matcherpb.PathMatcher_Path{Path: &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "localhost-fan-page"}}}}}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + Action: v3rbacpb.RBAC_DENY, + }, + }, + rbacQueries: []rbacQuery{ + // This RPC should match with the allow policy, and shouldn't + // match with the deny and thus should be allowed to proceed. + { + rpcData: &rpcData{ + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0"}, + }, + }, + wantStatusCode: codes.OK, + }, + // This RPC should match with both the allow policy and deny policy + // and thus shouldn't be allowed to proceed as matched with deny. + { + rpcData: &rpcData{ + fullMethod: "localhost-fan-page", + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0"}, + }, + }, + wantStatusCode: codes.PermissionDenied, + }, + // This RPC shouldn't match with either policy, and thus + // shouldn't be allowed to proceed as didn't match with allow. + { + rpcData: &rpcData{ + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "10.0.0.0"}, + }, + }, + wantStatusCode: codes.PermissionDenied, + }, + // This RPC shouldn't match with allow, match with deny, and + // thus shouldn't be allowed to proceed. + { + rpcData: &rpcData{ + fullMethod: "localhost-fan-page", + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "10.0.0.0"}, + }, + }, + wantStatusCode: codes.PermissionDenied, + }, + }, + }, + // This test tests that when there are no SANs or Subject's + // distinguished name in incoming RPC's, that authenticated matchers + // match against the empty string. + { + name: "default-matching-no-credentials", + rbacConfigs: []*v3rbacpb.RBAC{ + { + Policies: map[string]*v3rbacpb.Policy{ + "service-admin": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Authenticated_{Authenticated: &v3rbacpb.Principal_Authenticated{PrincipalName: &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: ""}}}}}, + }, + }, + }, + }, + }, + rbacQueries: []rbacQuery{ + // This incoming RPC Call should match with the service admin + // policy. No authentication info is provided, so the + // authenticated matcher should match to the string matcher on + // the empty string, matching to the service-admin policy. + { + rpcData: &rpcData{ + fullMethod: "some method", + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0"}, + AuthInfo: credentials.TLSInfo{ + State: tls.ConnectionState{ + PeerCertificates: []*x509.Certificate{ + { + URIs: []*url.URL{ + { + Host: "cluster.local", + Path: "/ns/default/sa/admin", + }, + }, + }, + }, + }, + }, + }, + }, + wantStatusCode: codes.OK, + }, + }, + }, + // This test tests that an RBAC policy configured with a metadata + // matcher as a permission doesn't match with any incoming RPC. + { + name: "metadata-never-matches", + rbacConfigs: []*v3rbacpb.RBAC{ + { + Policies: map[string]*v3rbacpb.Policy{ + "metadata-never-matches": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Metadata{ + Metadata: &v3matcherpb.MetadataMatcher{}, + }}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + rbacQueries: []rbacQuery{ + { + rpcData: &rpcData{ + fullMethod: "some method", + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0"}, + }, + }, + wantStatusCode: codes.PermissionDenied, + }, + }, + }, + // This test tests that an RBAC policy configured with a metadata + // matcher with invert set to true as a permission always matches with + // any incoming RPC. + { + name: "metadata-invert-always-matches", + rbacConfigs: []*v3rbacpb.RBAC{ + { + Policies: map[string]*v3rbacpb.Policy{ + "metadata-invert-always-matches": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Metadata{ + Metadata: &v3matcherpb.MetadataMatcher{Invert: true}, + }}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + rbacQueries: []rbacQuery{ + { + rpcData: &rpcData{ + fullMethod: "some method", + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0"}, + }, + }, + wantStatusCode: codes.OK, + }, + }, + }, + // AllowAndDenyPolicy tests a policy with an allow (on path) and + // deny (on port) policy chained together. This represents how a user + // configured interceptor would use this, and also is a potential + // configuration for a dynamic xds interceptor. Further, it tests that + // the audit logger works properly in each scenario. + { + name: "AuditLoggingAllowAndDenyPolicy_ON_ALLOW", + policyName: "test_policy", + rbacConfigs: []*v3rbacpb.RBAC{ + { + Policies: map[string]*v3rbacpb.Policy{ + "localhost-fan": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_UrlPath{UrlPath: &v3matcherpb.PathMatcher{Rule: &v3matcherpb.PathMatcher_Path{Path: &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "localhost-fan-page"}}}}}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + Action: v3rbacpb.RBAC_DENY, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_NONE, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{ + Name: "TestAuditLoggerBuffer", + TypedConfig: createUDPATypedStruct(t, map[string]any{}, "AuditLoggingAllowAndDenyPolicy_ON_ALLOW_TestAuditLoggerBuffer")}, + IsOptional: false, + }, + }, + }, + }, + { + Policies: map[string]*v3rbacpb.Policy{ + "certain-source-ip": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_DirectRemoteIp{DirectRemoteIp: &v3corepb.CidrRange{AddressPrefix: "0.0.0.0", PrefixLen: &wrapperspb.UInt32Value{Value: uint32(10)}}}}, + }, + }, + }, + Action: v3rbacpb.RBAC_ALLOW, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_ON_ALLOW, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{ + Name: "TestAuditLoggerBuffer", + TypedConfig: createUDPATypedStruct(t, map[string]any{}, "AuditLoggingAllowAndDenyPolicy_ON_ALLOW_TestAuditLoggerBuffer")}, + IsOptional: false, + }, + }, + }, + }, + }, + rbacQueries: []rbacQuery{ + // This RPC should match with the allow policy, and shouldn't + // match with the deny and thus should be allowed to proceed. + { + rpcData: &rpcData{ + fullMethod: "", + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0"}, + AuthInfo: credentials.TLSInfo{ + State: tls.ConnectionState{ + PeerCertificates: []*x509.Certificate{ + { + URIs: []*url.URL{ + { + Scheme: "spiffe", + Host: "cluster.local", + Path: "/ns/default/sa/admin", + }, + }, + }, + }, + }, + SPIFFEID: &url.URL{ + Scheme: "spiffe", + Host: "cluster.local", + Path: "/ns/default/sa/admin", + }, + }, + }, + }, + wantStatusCode: codes.OK, + wantAuditEvents: []*audit.Event{ + { + FullMethodName: "", + Principal: "spiffe://cluster.local/ns/default/sa/admin", + PolicyName: "test_policy", + MatchedRule: "certain-source-ip", + Authorized: true, + }, + }, + }, + // This RPC should match with both the allow policy and deny policy + // and thus shouldn't be allowed to proceed as matched with deny. + { + rpcData: &rpcData{ + fullMethod: "localhost-fan-page", + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0"}, + }, + }, + wantStatusCode: codes.PermissionDenied, + }, + // This RPC shouldn't match with either policy, and thus + // shouldn't be allowed to proceed as didn't match with allow. + { + rpcData: &rpcData{ + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "10.0.0.0"}, + }, + }, + wantStatusCode: codes.PermissionDenied, + }, + // This RPC shouldn't match with allow, match with deny, and + // thus shouldn't be allowed to proceed. + { + rpcData: &rpcData{ + fullMethod: "localhost-fan-page", + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "10.0.0.0"}, + }, + }, + wantStatusCode: codes.PermissionDenied, + }, + }, + }, + { + name: "AuditLoggingAllowAndDenyPolicy_ON_DENY", + policyName: "test_policy", + rbacConfigs: []*v3rbacpb.RBAC{ + { + Policies: map[string]*v3rbacpb.Policy{ + "localhost-fan": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_UrlPath{UrlPath: &v3matcherpb.PathMatcher{Rule: &v3matcherpb.PathMatcher_Path{Path: &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "localhost-fan-page"}}}}}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + Action: v3rbacpb.RBAC_DENY, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_ON_DENY, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{ + Name: "TestAuditLoggerBuffer", + TypedConfig: createUDPATypedStruct(t, map[string]any{}, "AuditLoggingAllowAndDenyPolicy_ON_DENY_TestAuditLoggerBuffer")}, + IsOptional: false, + }, + }, + }, + }, + { + Policies: map[string]*v3rbacpb.Policy{ + "certain-source-ip": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_DirectRemoteIp{DirectRemoteIp: &v3corepb.CidrRange{AddressPrefix: "0.0.0.0", PrefixLen: &wrapperspb.UInt32Value{Value: uint32(10)}}}}, + }, + }, + }, + Action: v3rbacpb.RBAC_ALLOW, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_ON_DENY, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{ + Name: "TestAuditLoggerBuffer", + TypedConfig: createUDPATypedStruct(t, map[string]any{}, "AuditLoggingAllowAndDenyPolicy_ON_DENY_TestAuditLoggerBuffer")}, + IsOptional: false, + }, + }, + }, + }, + }, + rbacQueries: []rbacQuery{ + // This RPC should match with the allow policy, and shouldn't + // match with the deny and thus should be allowed to proceed. + // Audit logging matches with nothing. + { + rpcData: &rpcData{ + fullMethod: "", + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0"}, + }, + }, + wantStatusCode: codes.OK, + }, + // This RPC should match with both the allow policy and deny policy + // and thus shouldn't be allowed to proceed as matched with deny. + // Audit logging matches with deny and short circuits. + { + rpcData: &rpcData{ + fullMethod: "localhost-fan-page", + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0"}, + AuthInfo: credentials.TLSInfo{ + State: tls.ConnectionState{ + PeerCertificates: []*x509.Certificate{ + { + URIs: []*url.URL{ + { + Host: "cluster.local", + Path: "/ns/default/sa/admin", + }, + }, + }, + }, + }, + }, + }, + }, + wantStatusCode: codes.PermissionDenied, + wantAuditEvents: []*audit.Event{ + { + FullMethodName: "localhost-fan-page", + PolicyName: "test_policy", + MatchedRule: "localhost-fan", + Authorized: false, + }, + }, + }, + // This RPC shouldn't match with either policy, and thus + // shouldn't be allowed to proceed as didn't match with allow. + // Audit logging matches with the allow policy. + { + rpcData: &rpcData{ + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "10.0.0.0"}, + }, + }, + wantStatusCode: codes.PermissionDenied, + wantAuditEvents: []*audit.Event{ + { + FullMethodName: "", + PolicyName: "test_policy", + MatchedRule: "", + Authorized: false, + }, + }, + }, + // This RPC shouldn't match with allow, match with deny, and + // thus shouldn't be allowed to proceed. + // Audit logging will have the deny logged. + { + rpcData: &rpcData{ + fullMethod: "localhost-fan-page", + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "10.0.0.0"}, + }, + }, + wantStatusCode: codes.PermissionDenied, + wantAuditEvents: []*audit.Event{ + { + FullMethodName: "localhost-fan-page", + PolicyName: "test_policy", + MatchedRule: "localhost-fan", + Authorized: false, + }, + }, + }, + }, + }, + { + name: "AuditLoggingAllowAndDenyPolicy_NONE", + policyName: "test_policy", + rbacConfigs: []*v3rbacpb.RBAC{ + { + Policies: map[string]*v3rbacpb.Policy{ + "localhost-fan": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_UrlPath{UrlPath: &v3matcherpb.PathMatcher{Rule: &v3matcherpb.PathMatcher_Path{Path: &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "localhost-fan-page"}}}}}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + Action: v3rbacpb.RBAC_DENY, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_NONE, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{ + Name: "TestAuditLoggerBuffer", + TypedConfig: createUDPATypedStruct(t, map[string]any{}, "AuditLoggingAllowAndDenyPolicy_NONE_TestAuditLoggerBuffer")}, + IsOptional: false, + }, + }, + }, + }, + { + Policies: map[string]*v3rbacpb.Policy{ + "certain-source-ip": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_DirectRemoteIp{DirectRemoteIp: &v3corepb.CidrRange{AddressPrefix: "0.0.0.0", PrefixLen: &wrapperspb.UInt32Value{Value: uint32(10)}}}}, + }, + }, + }, + Action: v3rbacpb.RBAC_ALLOW, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_NONE, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{ + Name: "TestAuditLoggerBuffer", + TypedConfig: createUDPATypedStruct(t, map[string]any{}, "AuditLoggingAllowAndDenyPolicy_NONE_TestAuditLoggerBuffer")}, + IsOptional: false, + }, + }, + }, + }, + }, + rbacQueries: []rbacQuery{ + // This RPC should match with the allow policy, and shouldn't + // match with the deny and thus should be allowed to proceed. + // Audit logging is NONE. + { + rpcData: &rpcData{ + fullMethod: "", + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0"}, + }, + }, + wantStatusCode: codes.OK, + }, + // This RPC should match with both the allow policy and deny policy + // and thus shouldn't be allowed to proceed as matched with deny. + // Audit logging is NONE. + { + rpcData: &rpcData{ + fullMethod: "localhost-fan-page", + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0"}, + }, + }, + wantStatusCode: codes.PermissionDenied, + }, + // This RPC shouldn't match with either policy, and thus + // shouldn't be allowed to proceed as didn't match with allow. + // Audit logging is NONE. + { + rpcData: &rpcData{ + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "10.0.0.0"}, + }, + }, + wantStatusCode: codes.PermissionDenied, + }, + // This RPC shouldn't match with allow, match with deny, and + // thus shouldn't be allowed to proceed. + // Audit logging is NONE. + { + rpcData: &rpcData{ + fullMethod: "localhost-fan-page", + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "10.0.0.0"}, + }, + }, + wantStatusCode: codes.PermissionDenied, + }, + }, + }, + { + name: "AuditLoggingAllowAndDenyPolicy_ON_DENY_AND_ALLOW", + policyName: "test_policy", + rbacConfigs: []*v3rbacpb.RBAC{ + { + Policies: map[string]*v3rbacpb.Policy{ + "localhost-fan": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_UrlPath{UrlPath: &v3matcherpb.PathMatcher{Rule: &v3matcherpb.PathMatcher_Path{Path: &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "localhost-fan-page"}}}}}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + Action: v3rbacpb.RBAC_DENY, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_ON_DENY, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{ + Name: "TestAuditLoggerBuffer", + TypedConfig: createUDPATypedStruct(t, map[string]any{}, "AuditLoggingAllowAndDenyPolicy_ON_DENY_AND_ALLOW_TestAuditLoggerBuffer")}, + IsOptional: false, + }, + }, + }, + }, + { + Policies: map[string]*v3rbacpb.Policy{ + "certain-source-ip": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_DirectRemoteIp{DirectRemoteIp: &v3corepb.CidrRange{AddressPrefix: "0.0.0.0", PrefixLen: &wrapperspb.UInt32Value{Value: uint32(10)}}}}, + }, + }, + }, + Action: v3rbacpb.RBAC_ALLOW, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_ON_DENY_AND_ALLOW, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + {AuditLogger: &v3corepb.TypedExtensionConfig{ + Name: "TestAuditLoggerBuffer", + TypedConfig: createUDPATypedStruct(t, map[string]any{}, "AuditLoggingAllowAndDenyPolicy_ON_DENY_AND_ALLOW_TestAuditLoggerBuffer")}, + IsOptional: false, + }, + }, + }, + }, + }, + rbacQueries: []rbacQuery{ + // This RPC should match with the allow policy, and shouldn't + // match with the deny and thus should be allowed to proceed. + // Audit logging matches with nothing. + { + rpcData: &rpcData{ + fullMethod: "", + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0"}, + }, + }, + wantStatusCode: codes.OK, + wantAuditEvents: []*audit.Event{ + { + FullMethodName: "", + PolicyName: "test_policy", + MatchedRule: "certain-source-ip", + Authorized: true, + }, + }, + }, + // This RPC should match with both the allow policy and deny policy + // and thus shouldn't be allowed to proceed as matched with deny. + // Audit logging matches with deny and short circuits. + { + rpcData: &rpcData{ + fullMethod: "localhost-fan-page", + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "0.0.0.0"}, + }, + }, + wantStatusCode: codes.PermissionDenied, + wantAuditEvents: []*audit.Event{ + { + FullMethodName: "localhost-fan-page", + PolicyName: "test_policy", + MatchedRule: "localhost-fan", + Authorized: false, + }, + }, + }, + // This RPC shouldn't match with either policy, and thus + // shouldn't be allowed to proceed as didn't match with allow. + // Audit logging matches with the allow policy. + { + rpcData: &rpcData{ + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "10.0.0.0"}, + }, + }, + wantStatusCode: codes.PermissionDenied, + wantAuditEvents: []*audit.Event{ + { + FullMethodName: "", + PolicyName: "test_policy", + MatchedRule: "", + Authorized: false, + }, + }, + }, + // This RPC shouldn't match with allow, match with deny, and + // thus shouldn't be allowed to proceed. + // Audit logging will have the deny logged. + { + rpcData: &rpcData{ + fullMethod: "localhost-fan-page", + peerInfo: &peer.Peer{ + Addr: &addr{ipAddress: "10.0.0.0"}, + }, + }, + wantStatusCode: codes.PermissionDenied, + wantAuditEvents: []*audit.Event{ + { + FullMethodName: "localhost-fan-page", + PolicyName: "test_policy", + MatchedRule: "localhost-fan", + Authorized: false, + }, + }, + }, + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + b := TestAuditLoggerBufferBuilder{testName: test.name} + audit.RegisterLoggerBuilder(&b) + b2 := TestAuditLoggerCustomConfigBuilder{testName: test.name} + audit.RegisterLoggerBuilder(&b2) + + // Instantiate the chainedRBACEngine with different configurations that are + // interesting to test and to query. + cre, err := NewChainEngine(test.rbacConfigs, test.policyName) + if err != nil { + t.Fatalf("Error constructing RBAC Engine: %v", err) + } + // Query the created chain of RBAC Engines with different args to see + // if the chain of RBAC Engines configured as such works as intended. + for _, data := range test.rbacQueries { + func() { + // Construct the context with three data points that have enough + // information to represent incoming RPC's. This will be how a + // user uses this API. A user will have to put MD, PeerInfo, and + // the connection the RPC is sent on in the context. + ctx := metadata.NewIncomingContext(context.Background(), data.rpcData.md) + + // Make a TCP connection with a certain destination port. The + // address/port of this connection will be used to populate the + // destination ip/port in RPCData struct. This represents what + // the user of ChainEngine will have to place into context, + // as this is only way to get destination ip and port. + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Error listening: %v", err) + } + defer lis.Close() + connCh := make(chan net.Conn, 1) + go func() { + conn, err := lis.Accept() + if err != nil { + t.Errorf("Error accepting connection: %v", err) + return + } + connCh <- conn + }() + _, err = net.Dial("tcp", lis.Addr().String()) + if err != nil { + t.Fatalf("Error dialing: %v", err) + } + conn := <-connCh + defer conn.Close() + getConnection = func(context.Context) net.Conn { + return conn + } + ctx = peer.NewContext(ctx, data.rpcData.peerInfo) + stream := &ServerTransportStreamWithMethod{ + method: data.rpcData.fullMethod, + } + + ctx = grpc.NewContextWithServerTransportStream(ctx, stream) + err = cre.IsAuthorized(ctx) + if gotCode := status.Code(err); gotCode != data.wantStatusCode { + t.Fatalf("IsAuthorized(%+v, %+v) returned (%+v), want(%+v)", ctx, data.rpcData.fullMethod, gotCode, data.wantStatusCode) + } + if !reflect.DeepEqual(b.auditEvents, data.wantAuditEvents) { + t.Fatalf("Unexpected audit event for query:%v", data) + } + + // This builder's auditEvents can be shared for several queries, make sure it's empty. + b.auditEvents = nil + }() + } + }) + } +} + +type ServerTransportStreamWithMethod struct { + method string +} + +func (sts *ServerTransportStreamWithMethod) Method() string { + return sts.method +} + +func (sts *ServerTransportStreamWithMethod) SetHeader(md metadata.MD) error { + return nil +} + +func (sts *ServerTransportStreamWithMethod) SendHeader(md metadata.MD) error { + return nil +} + +func (sts *ServerTransportStreamWithMethod) SetTrailer(md metadata.MD) error { + return nil +} + +// An audit logger that will log to the auditEvents slice. +type TestAuditLoggerBuffer struct { + auditEvents *[]*audit.Event +} + +func (logger *TestAuditLoggerBuffer) Log(e *audit.Event) { + *(logger.auditEvents) = append(*(logger.auditEvents), e) +} + +// Builds TestAuditLoggerBuffer. +type TestAuditLoggerBufferBuilder struct { + auditEvents []*audit.Event + testName string +} + +// The required config for TestAuditLoggerBuffer. +type TestAuditLoggerBufferConfig struct { + audit.LoggerConfig +} + +func (b *TestAuditLoggerBufferBuilder) ParseLoggerConfig(configJSON json.RawMessage) (config audit.LoggerConfig, err error) { + return TestAuditLoggerBufferConfig{}, nil +} + +func (b *TestAuditLoggerBufferBuilder) Build(config audit.LoggerConfig) audit.Logger { + return &TestAuditLoggerBuffer{auditEvents: &b.auditEvents} +} + +func (b *TestAuditLoggerBufferBuilder) Name() string { + return b.testName + "_TestAuditLoggerBuffer" +} + +// An audit logger to test using a custom config. +type TestAuditLoggerCustomConfig struct{} + +func (logger *TestAuditLoggerCustomConfig) Log(*audit.Event) {} + +// Build TestAuditLoggerCustomConfig. This builds a TestAuditLoggerCustomConfig +// logger that uses a custom config. +type TestAuditLoggerCustomConfigBuilder struct { + testName string +} + +// The custom config for the TestAuditLoggerCustomConfig logger. +type TestAuditLoggerCustomConfigConfig struct { + audit.LoggerConfig + Abc int + Xyz string +} + +// Parses TestAuditLoggerCustomConfigConfig. Hard-coded to match with it's test +// case above. +func (b TestAuditLoggerCustomConfigBuilder) ParseLoggerConfig(configJSON json.RawMessage) (audit.LoggerConfig, error) { + c := TestAuditLoggerCustomConfigConfig{} + err := json.Unmarshal(configJSON, &c) + if err != nil { + return nil, fmt.Errorf("could not parse custom config: %v", err) + } + return c, nil +} + +func (b *TestAuditLoggerCustomConfigBuilder) Build(config audit.LoggerConfig) audit.Logger { + return &TestAuditLoggerCustomConfig{} +} + +func (b *TestAuditLoggerCustomConfigBuilder) Name() string { + return b.testName + "_TestAuditLoggerCustomConfig" +} + +// Builds custom configs for audit logger RBAC protos. +func createUDPATypedStruct(t *testing.T, in map[string]any, name string) *anypb.Any { + t.Helper() + pb, err := structpb.NewStruct(in) + if err != nil { + t.Fatalf("createUDPATypedStructFailed during structpb.NewStruct: %v", err) + } + typedURL := "" + if name != "" { + typedURL = typeURLPrefix + name + } + typedStruct := &v1xdsudpatypepb.TypedStruct{ + TypeUrl: typedURL, + Value: pb, + } + customConfig, err := anypb.New(typedStruct) + if err != nil { + t.Fatalf("createUDPATypedStructFailed during anypb.New: %v", err) + } + return customConfig +} + +// Builds custom configs for audit logger RBAC protos. +func createXDSTypedStruct(t *testing.T, in map[string]any, name string) *anypb.Any { + t.Helper() + pb, err := structpb.NewStruct(in) + if err != nil { + t.Fatalf("createXDSTypedStructFailed during structpb.NewStruct: %v", err) + } + typedStruct := &v3xdsxdstypepb.TypedStruct{ + TypeUrl: typeURLPrefix + name, + Value: pb, + } + customConfig, err := anypb.New(typedStruct) + if err != nil { + t.Fatalf("createXDSTypedStructFailed during anypb.New: %v", err) + } + return customConfig +} diff --git a/internal/xds_handshake_cluster.go b/internal/xds_handshake_cluster.go new file mode 100644 index 000000000000..e8b492774d1a --- /dev/null +++ b/internal/xds_handshake_cluster.go @@ -0,0 +1,40 @@ +/* + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package internal + +import ( + "google.golang.org/grpc/attributes" + "google.golang.org/grpc/resolver" +) + +// handshakeClusterNameKey is the type used as the key to store cluster name in +// the Attributes field of resolver.Address. +type handshakeClusterNameKey struct{} + +// SetXDSHandshakeClusterName returns a copy of addr in which the Attributes field +// is updated with the cluster name. +func SetXDSHandshakeClusterName(addr resolver.Address, clusterName string) resolver.Address { + addr.Attributes = addr.Attributes.WithValue(handshakeClusterNameKey{}, clusterName) + return addr +} + +// GetXDSHandshakeClusterName returns cluster name stored in attr. +func GetXDSHandshakeClusterName(attr *attributes.Attributes) (string, bool) { + v := attr.Value(handshakeClusterNameKey{}) + name, ok := v.(string) + return name, ok +} diff --git a/interop/alts/client/client.go b/interop/alts/client/client.go index 9fc0e3c15650..aef601ff885b 100644 --- a/interop/alts/client/client.go +++ b/interop/alts/client/client.go @@ -27,6 +27,8 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/credentials/alts" "google.golang.org/grpc/grpclog" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" testpb "google.golang.org/grpc/interop/grpc_testing" ) @@ -51,7 +53,7 @@ func main() { logger.Fatalf("gRPC Client: failed to dial the server at %v: %v", *serverAddr, err) } defer conn.Close() - grpcClient := testpb.NewTestServiceClient(conn) + grpcClient := testgrpc.NewTestServiceClient(conn) // Call the EmptyCall API. ctx := context.Background() diff --git a/interop/alts/server/server.go b/interop/alts/server/server.go index 0ac7678a51b5..9db6750252e0 100644 --- a/interop/alts/server/server.go +++ b/interop/alts/server/server.go @@ -29,8 +29,9 @@ import ( "google.golang.org/grpc/credentials/alts" "google.golang.org/grpc/grpclog" "google.golang.org/grpc/interop" - testpb "google.golang.org/grpc/interop/grpc_testing" "google.golang.org/grpc/tap" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" ) const ( @@ -64,7 +65,7 @@ func main() { } altsTC := alts.NewServerCreds(opts) grpcServer := grpc.NewServer(grpc.Creds(altsTC), grpc.InTapHandle(authz)) - testpb.RegisterTestServiceService(grpcServer, interop.NewTestServer()) + testgrpc.RegisterTestServiceServer(grpcServer, interop.NewTestServer()) grpcServer.Serve(lis) } diff --git a/interop/client/client.go b/interop/client/client.go index 2d7823062309..7f04be0c152e 100644 --- a/interop/client/client.go +++ b/interop/client/client.go @@ -17,24 +17,41 @@ */ // Binary client is an interop client. +// +// See interop test case descriptions [here]. +// +// [here]: https://github.com/grpc/grpc/blob/master/doc/interop-test-descriptions.md package main import ( + "context" + "crypto/tls" + "crypto/x509" "flag" "net" + "os" "strconv" + "strings" + "time" + "golang.org/x/oauth2" "google.golang.org/grpc" - _ "google.golang.org/grpc/balancer/grpclb" "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/alts" "google.golang.org/grpc/credentials/google" + "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/credentials/oauth" "google.golang.org/grpc/grpclog" "google.golang.org/grpc/interop" - testpb "google.golang.org/grpc/interop/grpc_testing" + "google.golang.org/grpc/metadata" "google.golang.org/grpc/resolver" "google.golang.org/grpc/testdata" + + _ "google.golang.org/grpc/balancer/grpclb" // Register the grpclb load balancing policy. + _ "google.golang.org/grpc/balancer/rls" // Register the RLS load balancing policy. + _ "google.golang.org/grpc/xds/googledirectpath" // Register xDS resolver required for c2p directpath. + + testgrpc "google.golang.org/grpc/interop/grpc_testing" ) const ( @@ -43,19 +60,28 @@ const ( ) var ( - caFile = flag.String("ca_file", "", "The file containning the CA root cert file") - useTLS = flag.Bool("use_tls", false, "Connection uses TLS if true") - useALTS = flag.Bool("use_alts", false, "Connection uses ALTS if true (this option can only be used on GCP)") - customCredentialsType = flag.String("custom_credentials_type", "", "Custom creds to use, excluding TLS or ALTS") - altsHSAddr = flag.String("alts_handshaker_service_address", "", "ALTS handshaker gRPC service address") - testCA = flag.Bool("use_test_ca", false, "Whether to replace platform root CAs with test CA as the CA root") - serviceAccountKeyFile = flag.String("service_account_key_file", "", "Path to service account json key file") - oauthScope = flag.String("oauth_scope", "", "The scope for OAuth2 tokens") - defaultServiceAccount = flag.String("default_service_account", "", "Email of GCE default service account") - serverHost = flag.String("server_host", "localhost", "The server host name") - serverPort = flag.Int("server_port", 10000, "The server port number") - tlsServerName = flag.String("server_host_override", "", "The server name use to verify the hostname returned by TLS handshake if it is not empty. Otherwise, --server_host is used.") - testCase = flag.String("test_case", "large_unary", + caFile = flag.String("ca_file", "", "The file containning the CA root cert file") + useTLS = flag.Bool("use_tls", false, "Connection uses TLS if true") + useALTS = flag.Bool("use_alts", false, "Connection uses ALTS if true (this option can only be used on GCP)") + customCredentialsType = flag.String("custom_credentials_type", "", "Custom creds to use, excluding TLS or ALTS") + altsHSAddr = flag.String("alts_handshaker_service_address", "", "ALTS handshaker gRPC service address") + testCA = flag.Bool("use_test_ca", false, "Whether to replace platform root CAs with test CA as the CA root") + serviceAccountKeyFile = flag.String("service_account_key_file", "", "Path to service account json key file") + oauthScope = flag.String("oauth_scope", "", "The scope for OAuth2 tokens") + defaultServiceAccount = flag.String("default_service_account", "", "Email of GCE default service account") + serverHost = flag.String("server_host", "localhost", "The server host name") + serverPort = flag.Int("server_port", 10000, "The server port number") + serviceConfigJSON = flag.String("service_config_json", "", "Disables service config lookups and sets the provided string as the default service config.") + soakIterations = flag.Int("soak_iterations", 10, "The number of iterations to use for the two soak tests: rpc_soak and channel_soak") + soakMaxFailures = flag.Int("soak_max_failures", 0, "The number of iterations in soak tests that are allowed to fail (either due to non-OK status code or exceeding the per-iteration max acceptable latency).") + soakPerIterationMaxAcceptableLatencyMs = flag.Int("soak_per_iteration_max_acceptable_latency_ms", 1000, "The number of milliseconds a single iteration in the two soak tests (rpc_soak and channel_soak) should take.") + soakOverallTimeoutSeconds = flag.Int("soak_overall_timeout_seconds", 10, "The overall number of seconds after which a soak test should stop and fail, if the desired number of iterations have not yet completed.") + soakMinTimeMsBetweenRPCs = flag.Int("soak_min_time_ms_between_rpcs", 0, "The minimum time in milliseconds between consecutive RPCs in a soak test (rpc_soak or channel_soak), useful for limiting QPS") + soakRequestSize = flag.Int("soak_request_size", 271828, "The request size in a soak RPC. The default value is set based on the interop large unary test case.") + soakResponseSize = flag.Int("soak_response_size", 314159, "The response size in a soak RPC. The default value is set based on the interop large unary test case.") + tlsServerName = flag.String("server_host_override", "", "The server name used to verify the hostname returned by TLS handshake if it is not empty. Otherwise, --server_host is used.") + additionalMetadata = flag.String("additional_metadata", "", "Additional metadata to send in each request, as a semicolon-separated list of key:value pairs.") + testCase = flag.String("test_case", "large_unary", `Configure different test cases. Valid options are: empty_unary : empty (zero bytes) request and response; large_unary : single request and (large) response; @@ -78,7 +104,9 @@ var ( custom_metadata: server will echo custom metadata; unimplemented_method: client attempts to call unimplemented method; unimplemented_service: client attempts to call unimplemented service; - pick_first_unary: all requests are sent to one server despite multiple servers are resolved.`) + pick_first_unary: all requests are sent to one server despite multiple servers are resolved; + orca_per_rpc: the client verifies ORCA per-RPC metrics are provided; + orca_oob: the client verifies ORCA out-of-band metrics are provided.`) logger = grpclog.Component("interop") ) @@ -93,8 +121,37 @@ const ( credsComputeEngineCreds ) +// Parses the --additional_metadata flag and returns metadata to send on each RPC, +// formatted as per https://pkg.go.dev/google.golang.org/grpc/metadata#Pairs. +// Allow any character but semicolons in values. If the flag is empty, return a nil map. +func parseAdditionalMetadataFlag() []string { + if len(*additionalMetadata) == 0 { + return nil + } + r := *additionalMetadata + addMd := make([]string, 0) + for len(r) > 0 { + i := strings.Index(r, ":") + if i < 0 { + logger.Fatalf("Error parsing --additional_metadata flag: missing colon separator") + } + addMd = append(addMd, r[:i]) // append key + r = r[i+1:] + i = strings.Index(r, ";") + // append value + if i < 0 { + addMd = append(addMd, r) + break + } + addMd = append(addMd, r[:i]) + r = r[i+1:] + } + return addMd +} + func main() { flag.Parse() + logger.Infof("Client running with test case %q", *testCase) var useGDC bool // use google default creds var useCEC bool // use compute engine creds if *customCredentialsType != "" { @@ -125,26 +182,32 @@ func main() { } resolver.SetDefaultScheme("dns") - serverAddr := net.JoinHostPort(*serverHost, strconv.Itoa(*serverPort)) + serverAddr := *serverHost + if *serverPort != 0 { + serverAddr = net.JoinHostPort(*serverHost, strconv.Itoa(*serverPort)) + } var opts []grpc.DialOption switch credsChosen { case credsTLS: - var sn string - if *tlsServerName != "" { - sn = *tlsServerName - } - var creds credentials.TransportCredentials + var roots *x509.CertPool if *testCA { - var err error if *caFile == "" { *caFile = testdata.Path("ca.pem") } - creds, err = credentials.NewClientTLSFromFile(*caFile, sn) + b, err := os.ReadFile(*caFile) if err != nil { - logger.Fatalf("Failed to create TLS credentials %v", err) + logger.Fatalf("Failed to read root certificate file %q: %v", *caFile, err) + } + roots = x509.NewCertPool() + if !roots.AppendCertsFromPEM(b) { + logger.Fatalf("Failed to append certificates: %s", string(b)) } + } + var creds credentials.TransportCredentials + if *tlsServerName != "" { + creds = credentials.NewClientTLSFromCert(roots, *tlsServerName) } else { - creds = credentials.NewClientTLSFromCert(nil, sn) + creds = credentials.NewTLS(&tls.Config{RootCAs: roots}) } opts = append(opts, grpc.WithTransportCredentials(creds)) case credsALTS: @@ -159,7 +222,7 @@ func main() { case credsComputeEngineCreds: opts = append(opts, grpc.WithCredentialsBundle(google.NewComputeEngineCredentials())) case credsNone: - opts = append(opts, grpc.WithInsecure()) + opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) default: logger.Fatal("Invalid creds") } @@ -179,16 +242,29 @@ func main() { } opts = append(opts, grpc.WithPerRPCCredentials(jwtCreds)) } else if *testCase == "oauth2_auth_token" { - opts = append(opts, grpc.WithPerRPCCredentials(oauth.NewOauthAccess(interop.GetToken(*serviceAccountKeyFile, *oauthScope)))) + opts = append(opts, grpc.WithPerRPCCredentials(oauth.TokenSource{TokenSource: oauth2.StaticTokenSource(interop.GetToken(*serviceAccountKeyFile, *oauthScope))})) + } + } + if len(*serviceConfigJSON) > 0 { + opts = append(opts, grpc.WithDisableServiceConfig(), grpc.WithDefaultServiceConfig(*serviceConfigJSON)) + } + if addMd := parseAdditionalMetadataFlag(); addMd != nil { + unaryAddMd := func(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + ctx = metadata.AppendToOutgoingContext(ctx, addMd...) + return invoker(ctx, method, req, reply, cc, opts...) + } + streamingAddMd := func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) { + ctx = metadata.AppendToOutgoingContext(ctx, addMd...) + return streamer(ctx, desc, cc, method, opts...) } + opts = append(opts, grpc.WithUnaryInterceptor(unaryAddMd), grpc.WithStreamInterceptor(streamingAddMd)) } - opts = append(opts, grpc.WithBlock()) conn, err := grpc.Dial(serverAddr, opts...) if err != nil { logger.Fatalf("Fail to dial: %v", err) } defer conn.Close() - tc := testpb.NewTestServiceClient(conn) + tc := testgrpc.NewTestServiceClient(conn) switch *testCase { case "empty_unary": interop.DoEmptyUnaryCall(tc) @@ -272,11 +348,23 @@ func main() { interop.DoUnimplementedMethod(conn) logger.Infoln("UnimplementedMethod done") case "unimplemented_service": - interop.DoUnimplementedService(testpb.NewUnimplementedServiceClient(conn)) + interop.DoUnimplementedService(testgrpc.NewUnimplementedServiceClient(conn)) logger.Infoln("UnimplementedService done") case "pick_first_unary": interop.DoPickFirstUnary(tc) logger.Infoln("PickFirstUnary done") + case "rpc_soak": + interop.DoSoakTest(tc, serverAddr, opts, false /* resetChannel */, *soakIterations, *soakMaxFailures, *soakRequestSize, *soakResponseSize, time.Duration(*soakPerIterationMaxAcceptableLatencyMs)*time.Millisecond, time.Duration(*soakMinTimeMsBetweenRPCs)*time.Millisecond, time.Now().Add(time.Duration(*soakOverallTimeoutSeconds)*time.Second)) + logger.Infoln("RpcSoak done") + case "channel_soak": + interop.DoSoakTest(tc, serverAddr, opts, true /* resetChannel */, *soakIterations, *soakMaxFailures, *soakRequestSize, *soakResponseSize, time.Duration(*soakPerIterationMaxAcceptableLatencyMs)*time.Millisecond, time.Duration(*soakMinTimeMsBetweenRPCs)*time.Millisecond, time.Now().Add(time.Duration(*soakOverallTimeoutSeconds)*time.Second)) + logger.Infoln("ChannelSoak done") + case "orca_per_rpc": + interop.DoORCAPerRPCTest(tc) + logger.Infoln("ORCAPerRPC done") + case "orca_oob": + interop.DoORCAOOBTest(tc) + logger.Infoln("ORCAOOB done") default: logger.Fatal("Unsupported test case: ", *testCase) } diff --git a/interop/fake_grpclb/fake_grpclb.go b/interop/fake_grpclb/fake_grpclb.go index 6804235486ba..00ae00a7f683 100644 --- a/interop/fake_grpclb/fake_grpclb.go +++ b/interop/fake_grpclb/fake_grpclb.go @@ -23,18 +23,13 @@ package main import ( "flag" - "net" - "strconv" "strings" - "time" "google.golang.org/grpc" - lbpb "google.golang.org/grpc/balancer/grpclb/grpc_lb_v1" - "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/alts" "google.golang.org/grpc/grpclog" - "google.golang.org/grpc/status" + "google.golang.org/grpc/internal/testutils/fakegrpclb" "google.golang.org/grpc/testdata" ) @@ -49,76 +44,16 @@ var ( logger = grpclog.Component("interop") ) -type loadBalancerServer struct { - lbpb.UnimplementedLoadBalancerServer - serverListResponse *lbpb.LoadBalanceResponse -} - -func (l *loadBalancerServer) BalanceLoad(stream lbpb.LoadBalancer_BalanceLoadServer) error { - logger.Info("Begin handling new BalancerLoad request.") - var lbReq *lbpb.LoadBalanceRequest - var err error - if lbReq, err = stream.Recv(); err != nil { - logger.Errorf("Error receiving LoadBalanceRequest: %v", err) - return err - } - logger.Info("LoadBalancerRequest received.") - initialReq := lbReq.GetInitialRequest() - if initialReq == nil { - logger.Info("Expected first request to be an InitialRequest. Got: %v", lbReq) - return status.Error(codes.Unknown, "First request not an InitialRequest") - } - // gRPC clients targeting foo.bar.com:443 can sometimes include the ":443" suffix in - // their requested names; handle this case. TODO: make 443 configurable? - var cleanedName string - var requestedNamePortNumber string - if cleanedName, requestedNamePortNumber, err = net.SplitHostPort(initialReq.Name); err != nil { - cleanedName = initialReq.Name - } else { - if requestedNamePortNumber != "443" { - logger.Info("Bad requested service name port number: %v.", requestedNamePortNumber) - return status.Error(codes.Unknown, "Bad requested service name port number") - } - } - if cleanedName != *serviceName { - logger.Info("Expected requested service name: %v. Got: %v", *serviceName, initialReq.Name) - return status.Error(codes.NotFound, "Bad requested service name") - } - if err := stream.Send(&lbpb.LoadBalanceResponse{ - LoadBalanceResponseType: &lbpb.LoadBalanceResponse_InitialResponse{ - InitialResponse: &lbpb.InitialLoadBalanceResponse{}, - }, - }); err != nil { - logger.Errorf("Error sending initial LB response: %v", err) - return status.Error(codes.Unknown, "Error sending initial response") - } - logger.Info("Send LoadBalanceResponse: %v", l.serverListResponse) - if err := stream.Send(l.serverListResponse); err != nil { - logger.Errorf("Error sending LB response: %v", err) - return status.Error(codes.Unknown, "Error sending response") - } - if *shortStream { - return nil - } - for { - logger.Info("Send LoadBalanceResponse: %v", l.serverListResponse) - if err := stream.Send(l.serverListResponse); err != nil { - logger.Errorf("Error sending LB response: %v", err) - return status.Error(codes.Unknown, "Error sending response") - } - time.Sleep(10 * time.Second) - } -} - func main() { flag.Parse() + var opts []grpc.ServerOption if *useTLS { certFile := testdata.Path("server1.pem") keyFile := testdata.Path("server1.key") creds, err := credentials.NewServerTLSFromFile(certFile, keyFile) if err != nil { - logger.Fatalf("Failed to generate credentials %v", err) + logger.Fatalf("Failed to generate credentials: %v", err) } opts = append(opts, grpc.Creds(creds)) } else if *useALTS { @@ -126,47 +61,23 @@ func main() { altsTC := alts.NewServerCreds(altsOpts) opts = append(opts, grpc.Creds(altsTC)) } - var serverList []*lbpb.Server - if len(*backendAddrs) == 0 { - serverList = make([]*lbpb.Server, 0) - } else { - rawBackendAddrs := strings.Split(*backendAddrs, ",") - serverList = make([]*lbpb.Server, len(rawBackendAddrs)) - for i := range rawBackendAddrs { - rawIP, rawPort, err := net.SplitHostPort(rawBackendAddrs[i]) - if err != nil { - logger.Fatalf("Failed to parse --backend_addrs[%d]=%v, error: %v", i, rawBackendAddrs[i], err) - } - ip := net.ParseIP(rawIP) - if ip == nil { - logger.Fatalf("Failed to parse ip: %v", rawIP) - } - numericPort, err := strconv.Atoi(rawPort) - if err != nil { - logger.Fatalf("Failed to convert port %v to int", rawPort) - } - logger.Infof("Adding backend ip: %v, port: %d", ip.String(), numericPort) - serverList[i] = &lbpb.Server{ - IpAddress: ip, - Port: int32(numericPort), - } - } - } - serverListResponse := &lbpb.LoadBalanceResponse{ - LoadBalanceResponseType: &lbpb.LoadBalanceResponse_ServerList{ - ServerList: &lbpb.ServerList{ - Servers: serverList, - }, - }, - } - server := grpc.NewServer(opts...) - logger.Infof("Begin listening on %d.", *port) - lis, err := net.Listen("tcp", ":"+strconv.Itoa(*port)) + + rawBackendAddrs := strings.Split(*backendAddrs, ",") + server, err := fakegrpclb.NewServer(fakegrpclb.ServerParams{ + ListenPort: *port, + ServerOptions: opts, + LoadBalancedServiceName: *serviceName, + LoadBalancedServicePort: 443, // TODO: make this configurable? + BackendAddresses: rawBackendAddrs, + ShortStream: *shortStream, + }) if err != nil { - logger.Fatalf("Failed to listen on port %v: %v", *port, err) + logger.Fatalf("Failed to create balancer server: %v", err) + } + + // Serve() starts serving and blocks until Stop() is called. We don't need to + // call Stop() here since we want the server to run until we are killed. + if err := server.Serve(); err != nil { + logger.Fatalf("Failed to start balancer server: %v", err) } - lbpb.RegisterLoadBalancerServer(server, &loadBalancerServer{ - serverListResponse: serverListResponse, - }) - server.Serve(lis) } diff --git a/interop/grpc_testing/benchmark_service.pb.go b/interop/grpc_testing/benchmark_service.pb.go new file mode 100644 index 000000000000..031e12ee2e06 --- /dev/null +++ b/interop/grpc_testing/benchmark_service.pb.go @@ -0,0 +1,124 @@ +// Copyright 2015 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// An integration test service that covers all the method signature permutations +// of unary/streaming requests/responses. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.31.0 +// protoc v4.22.0 +// source: grpc/testing/benchmark_service.proto + +package grpc_testing + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +var File_grpc_testing_benchmark_service_proto protoreflect.FileDescriptor + +var file_grpc_testing_benchmark_service_proto_rawDesc = []byte{ + 0x0a, 0x24, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x62, + 0x65, 0x6e, 0x63, 0x68, 0x6d, 0x61, 0x72, 0x6b, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, + 0x74, 0x69, 0x6e, 0x67, 0x1a, 0x1b, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x69, + 0x6e, 0x67, 0x2f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x32, 0xa6, 0x03, 0x0a, 0x10, 0x42, 0x65, 0x6e, 0x63, 0x68, 0x6d, 0x61, 0x72, 0x6b, 0x53, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x46, 0x0a, 0x09, 0x55, 0x6e, 0x61, 0x72, 0x79, 0x43, + 0x61, 0x6c, 0x6c, 0x12, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, + 0x6e, 0x67, 0x2e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, + 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4e, + 0x0a, 0x0d, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x43, 0x61, 0x6c, 0x6c, 0x12, + 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, + 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x67, + 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x69, 0x6d, 0x70, + 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x12, 0x52, + 0x0a, 0x13, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x46, 0x72, 0x6f, 0x6d, 0x43, + 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x12, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, + 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, + 0x67, 0x2e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x28, 0x01, 0x12, 0x52, 0x0a, 0x13, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x46, + 0x72, 0x6f, 0x6d, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, + 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x12, 0x52, 0x0a, 0x11, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, + 0x69, 0x6e, 0x67, 0x42, 0x6f, 0x74, 0x68, 0x57, 0x61, 0x79, 0x73, 0x12, 0x1b, 0x2e, 0x67, 0x72, + 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x69, 0x6d, 0x70, 0x6c, + 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, + 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x42, 0x2a, 0x0a, 0x0f, 0x69, 0x6f, + 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x42, 0x15, 0x42, + 0x65, 0x6e, 0x63, 0x68, 0x6d, 0x61, 0x72, 0x6b, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x50, + 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var file_grpc_testing_benchmark_service_proto_goTypes = []interface{}{ + (*SimpleRequest)(nil), // 0: grpc.testing.SimpleRequest + (*SimpleResponse)(nil), // 1: grpc.testing.SimpleResponse +} +var file_grpc_testing_benchmark_service_proto_depIdxs = []int32{ + 0, // 0: grpc.testing.BenchmarkService.UnaryCall:input_type -> grpc.testing.SimpleRequest + 0, // 1: grpc.testing.BenchmarkService.StreamingCall:input_type -> grpc.testing.SimpleRequest + 0, // 2: grpc.testing.BenchmarkService.StreamingFromClient:input_type -> grpc.testing.SimpleRequest + 0, // 3: grpc.testing.BenchmarkService.StreamingFromServer:input_type -> grpc.testing.SimpleRequest + 0, // 4: grpc.testing.BenchmarkService.StreamingBothWays:input_type -> grpc.testing.SimpleRequest + 1, // 5: grpc.testing.BenchmarkService.UnaryCall:output_type -> grpc.testing.SimpleResponse + 1, // 6: grpc.testing.BenchmarkService.StreamingCall:output_type -> grpc.testing.SimpleResponse + 1, // 7: grpc.testing.BenchmarkService.StreamingFromClient:output_type -> grpc.testing.SimpleResponse + 1, // 8: grpc.testing.BenchmarkService.StreamingFromServer:output_type -> grpc.testing.SimpleResponse + 1, // 9: grpc.testing.BenchmarkService.StreamingBothWays:output_type -> grpc.testing.SimpleResponse + 5, // [5:10] is the sub-list for method output_type + 0, // [0:5] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_grpc_testing_benchmark_service_proto_init() } +func file_grpc_testing_benchmark_service_proto_init() { + if File_grpc_testing_benchmark_service_proto != nil { + return + } + file_grpc_testing_messages_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_grpc_testing_benchmark_service_proto_rawDesc, + NumEnums: 0, + NumMessages: 0, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_grpc_testing_benchmark_service_proto_goTypes, + DependencyIndexes: file_grpc_testing_benchmark_service_proto_depIdxs, + }.Build() + File_grpc_testing_benchmark_service_proto = out.File + file_grpc_testing_benchmark_service_proto_rawDesc = nil + file_grpc_testing_benchmark_service_proto_goTypes = nil + file_grpc_testing_benchmark_service_proto_depIdxs = nil +} diff --git a/interop/grpc_testing/benchmark_service_grpc.pb.go b/interop/grpc_testing/benchmark_service_grpc.pb.go new file mode 100644 index 000000000000..84cd44e4d45d --- /dev/null +++ b/interop/grpc_testing/benchmark_service_grpc.pb.go @@ -0,0 +1,422 @@ +// Copyright 2015 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// An integration test service that covers all the method signature permutations +// of unary/streaming requests/responses. + +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.3.0 +// - protoc v4.22.0 +// source: grpc/testing/benchmark_service.proto + +package grpc_testing + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 + +const ( + BenchmarkService_UnaryCall_FullMethodName = "/grpc.testing.BenchmarkService/UnaryCall" + BenchmarkService_StreamingCall_FullMethodName = "/grpc.testing.BenchmarkService/StreamingCall" + BenchmarkService_StreamingFromClient_FullMethodName = "/grpc.testing.BenchmarkService/StreamingFromClient" + BenchmarkService_StreamingFromServer_FullMethodName = "/grpc.testing.BenchmarkService/StreamingFromServer" + BenchmarkService_StreamingBothWays_FullMethodName = "/grpc.testing.BenchmarkService/StreamingBothWays" +) + +// BenchmarkServiceClient is the client API for BenchmarkService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type BenchmarkServiceClient interface { + // One request followed by one response. + // The server returns the client payload as-is. + UnaryCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (*SimpleResponse, error) + // Repeated sequence of one request followed by one response. + // Should be called streaming ping-pong + // The server returns the client payload as-is on each response + StreamingCall(ctx context.Context, opts ...grpc.CallOption) (BenchmarkService_StreamingCallClient, error) + // Single-sided unbounded streaming from client to server + // The server returns the client payload as-is once the client does WritesDone + StreamingFromClient(ctx context.Context, opts ...grpc.CallOption) (BenchmarkService_StreamingFromClientClient, error) + // Single-sided unbounded streaming from server to client + // The server repeatedly returns the client payload as-is + StreamingFromServer(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (BenchmarkService_StreamingFromServerClient, error) + // Two-sided unbounded streaming between server to client + // Both sides send the content of their own choice to the other + StreamingBothWays(ctx context.Context, opts ...grpc.CallOption) (BenchmarkService_StreamingBothWaysClient, error) +} + +type benchmarkServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewBenchmarkServiceClient(cc grpc.ClientConnInterface) BenchmarkServiceClient { + return &benchmarkServiceClient{cc} +} + +func (c *benchmarkServiceClient) UnaryCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (*SimpleResponse, error) { + out := new(SimpleResponse) + err := c.cc.Invoke(ctx, BenchmarkService_UnaryCall_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *benchmarkServiceClient) StreamingCall(ctx context.Context, opts ...grpc.CallOption) (BenchmarkService_StreamingCallClient, error) { + stream, err := c.cc.NewStream(ctx, &BenchmarkService_ServiceDesc.Streams[0], BenchmarkService_StreamingCall_FullMethodName, opts...) + if err != nil { + return nil, err + } + x := &benchmarkServiceStreamingCallClient{stream} + return x, nil +} + +type BenchmarkService_StreamingCallClient interface { + Send(*SimpleRequest) error + Recv() (*SimpleResponse, error) + grpc.ClientStream +} + +type benchmarkServiceStreamingCallClient struct { + grpc.ClientStream +} + +func (x *benchmarkServiceStreamingCallClient) Send(m *SimpleRequest) error { + return x.ClientStream.SendMsg(m) +} + +func (x *benchmarkServiceStreamingCallClient) Recv() (*SimpleResponse, error) { + m := new(SimpleResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *benchmarkServiceClient) StreamingFromClient(ctx context.Context, opts ...grpc.CallOption) (BenchmarkService_StreamingFromClientClient, error) { + stream, err := c.cc.NewStream(ctx, &BenchmarkService_ServiceDesc.Streams[1], BenchmarkService_StreamingFromClient_FullMethodName, opts...) + if err != nil { + return nil, err + } + x := &benchmarkServiceStreamingFromClientClient{stream} + return x, nil +} + +type BenchmarkService_StreamingFromClientClient interface { + Send(*SimpleRequest) error + CloseAndRecv() (*SimpleResponse, error) + grpc.ClientStream +} + +type benchmarkServiceStreamingFromClientClient struct { + grpc.ClientStream +} + +func (x *benchmarkServiceStreamingFromClientClient) Send(m *SimpleRequest) error { + return x.ClientStream.SendMsg(m) +} + +func (x *benchmarkServiceStreamingFromClientClient) CloseAndRecv() (*SimpleResponse, error) { + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + m := new(SimpleResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *benchmarkServiceClient) StreamingFromServer(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (BenchmarkService_StreamingFromServerClient, error) { + stream, err := c.cc.NewStream(ctx, &BenchmarkService_ServiceDesc.Streams[2], BenchmarkService_StreamingFromServer_FullMethodName, opts...) + if err != nil { + return nil, err + } + x := &benchmarkServiceStreamingFromServerClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type BenchmarkService_StreamingFromServerClient interface { + Recv() (*SimpleResponse, error) + grpc.ClientStream +} + +type benchmarkServiceStreamingFromServerClient struct { + grpc.ClientStream +} + +func (x *benchmarkServiceStreamingFromServerClient) Recv() (*SimpleResponse, error) { + m := new(SimpleResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *benchmarkServiceClient) StreamingBothWays(ctx context.Context, opts ...grpc.CallOption) (BenchmarkService_StreamingBothWaysClient, error) { + stream, err := c.cc.NewStream(ctx, &BenchmarkService_ServiceDesc.Streams[3], BenchmarkService_StreamingBothWays_FullMethodName, opts...) + if err != nil { + return nil, err + } + x := &benchmarkServiceStreamingBothWaysClient{stream} + return x, nil +} + +type BenchmarkService_StreamingBothWaysClient interface { + Send(*SimpleRequest) error + Recv() (*SimpleResponse, error) + grpc.ClientStream +} + +type benchmarkServiceStreamingBothWaysClient struct { + grpc.ClientStream +} + +func (x *benchmarkServiceStreamingBothWaysClient) Send(m *SimpleRequest) error { + return x.ClientStream.SendMsg(m) +} + +func (x *benchmarkServiceStreamingBothWaysClient) Recv() (*SimpleResponse, error) { + m := new(SimpleResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +// BenchmarkServiceServer is the server API for BenchmarkService service. +// All implementations must embed UnimplementedBenchmarkServiceServer +// for forward compatibility +type BenchmarkServiceServer interface { + // One request followed by one response. + // The server returns the client payload as-is. + UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error) + // Repeated sequence of one request followed by one response. + // Should be called streaming ping-pong + // The server returns the client payload as-is on each response + StreamingCall(BenchmarkService_StreamingCallServer) error + // Single-sided unbounded streaming from client to server + // The server returns the client payload as-is once the client does WritesDone + StreamingFromClient(BenchmarkService_StreamingFromClientServer) error + // Single-sided unbounded streaming from server to client + // The server repeatedly returns the client payload as-is + StreamingFromServer(*SimpleRequest, BenchmarkService_StreamingFromServerServer) error + // Two-sided unbounded streaming between server to client + // Both sides send the content of their own choice to the other + StreamingBothWays(BenchmarkService_StreamingBothWaysServer) error + mustEmbedUnimplementedBenchmarkServiceServer() +} + +// UnimplementedBenchmarkServiceServer must be embedded to have forward compatible implementations. +type UnimplementedBenchmarkServiceServer struct { +} + +func (UnimplementedBenchmarkServiceServer) UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method UnaryCall not implemented") +} +func (UnimplementedBenchmarkServiceServer) StreamingCall(BenchmarkService_StreamingCallServer) error { + return status.Errorf(codes.Unimplemented, "method StreamingCall not implemented") +} +func (UnimplementedBenchmarkServiceServer) StreamingFromClient(BenchmarkService_StreamingFromClientServer) error { + return status.Errorf(codes.Unimplemented, "method StreamingFromClient not implemented") +} +func (UnimplementedBenchmarkServiceServer) StreamingFromServer(*SimpleRequest, BenchmarkService_StreamingFromServerServer) error { + return status.Errorf(codes.Unimplemented, "method StreamingFromServer not implemented") +} +func (UnimplementedBenchmarkServiceServer) StreamingBothWays(BenchmarkService_StreamingBothWaysServer) error { + return status.Errorf(codes.Unimplemented, "method StreamingBothWays not implemented") +} +func (UnimplementedBenchmarkServiceServer) mustEmbedUnimplementedBenchmarkServiceServer() {} + +// UnsafeBenchmarkServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to BenchmarkServiceServer will +// result in compilation errors. +type UnsafeBenchmarkServiceServer interface { + mustEmbedUnimplementedBenchmarkServiceServer() +} + +func RegisterBenchmarkServiceServer(s grpc.ServiceRegistrar, srv BenchmarkServiceServer) { + s.RegisterService(&BenchmarkService_ServiceDesc, srv) +} + +func _BenchmarkService_UnaryCall_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(SimpleRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(BenchmarkServiceServer).UnaryCall(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: BenchmarkService_UnaryCall_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(BenchmarkServiceServer).UnaryCall(ctx, req.(*SimpleRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _BenchmarkService_StreamingCall_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(BenchmarkServiceServer).StreamingCall(&benchmarkServiceStreamingCallServer{stream}) +} + +type BenchmarkService_StreamingCallServer interface { + Send(*SimpleResponse) error + Recv() (*SimpleRequest, error) + grpc.ServerStream +} + +type benchmarkServiceStreamingCallServer struct { + grpc.ServerStream +} + +func (x *benchmarkServiceStreamingCallServer) Send(m *SimpleResponse) error { + return x.ServerStream.SendMsg(m) +} + +func (x *benchmarkServiceStreamingCallServer) Recv() (*SimpleRequest, error) { + m := new(SimpleRequest) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func _BenchmarkService_StreamingFromClient_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(BenchmarkServiceServer).StreamingFromClient(&benchmarkServiceStreamingFromClientServer{stream}) +} + +type BenchmarkService_StreamingFromClientServer interface { + SendAndClose(*SimpleResponse) error + Recv() (*SimpleRequest, error) + grpc.ServerStream +} + +type benchmarkServiceStreamingFromClientServer struct { + grpc.ServerStream +} + +func (x *benchmarkServiceStreamingFromClientServer) SendAndClose(m *SimpleResponse) error { + return x.ServerStream.SendMsg(m) +} + +func (x *benchmarkServiceStreamingFromClientServer) Recv() (*SimpleRequest, error) { + m := new(SimpleRequest) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func _BenchmarkService_StreamingFromServer_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(SimpleRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(BenchmarkServiceServer).StreamingFromServer(m, &benchmarkServiceStreamingFromServerServer{stream}) +} + +type BenchmarkService_StreamingFromServerServer interface { + Send(*SimpleResponse) error + grpc.ServerStream +} + +type benchmarkServiceStreamingFromServerServer struct { + grpc.ServerStream +} + +func (x *benchmarkServiceStreamingFromServerServer) Send(m *SimpleResponse) error { + return x.ServerStream.SendMsg(m) +} + +func _BenchmarkService_StreamingBothWays_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(BenchmarkServiceServer).StreamingBothWays(&benchmarkServiceStreamingBothWaysServer{stream}) +} + +type BenchmarkService_StreamingBothWaysServer interface { + Send(*SimpleResponse) error + Recv() (*SimpleRequest, error) + grpc.ServerStream +} + +type benchmarkServiceStreamingBothWaysServer struct { + grpc.ServerStream +} + +func (x *benchmarkServiceStreamingBothWaysServer) Send(m *SimpleResponse) error { + return x.ServerStream.SendMsg(m) +} + +func (x *benchmarkServiceStreamingBothWaysServer) Recv() (*SimpleRequest, error) { + m := new(SimpleRequest) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +// BenchmarkService_ServiceDesc is the grpc.ServiceDesc for BenchmarkService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var BenchmarkService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.testing.BenchmarkService", + HandlerType: (*BenchmarkServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "UnaryCall", + Handler: _BenchmarkService_UnaryCall_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "StreamingCall", + Handler: _BenchmarkService_StreamingCall_Handler, + ServerStreams: true, + ClientStreams: true, + }, + { + StreamName: "StreamingFromClient", + Handler: _BenchmarkService_StreamingFromClient_Handler, + ClientStreams: true, + }, + { + StreamName: "StreamingFromServer", + Handler: _BenchmarkService_StreamingFromServer_Handler, + ServerStreams: true, + }, + { + StreamName: "StreamingBothWays", + Handler: _BenchmarkService_StreamingBothWays_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "grpc/testing/benchmark_service.proto", +} diff --git a/interop/grpc_testing/control.pb.go b/interop/grpc_testing/control.pb.go new file mode 100644 index 000000000000..e09b9bacc043 --- /dev/null +++ b/interop/grpc_testing/control.pb.go @@ -0,0 +1,2492 @@ +// Copyright 2015 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.31.0 +// protoc v4.22.0 +// source: grpc/testing/control.proto + +package grpc_testing + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type ClientType int32 + +const ( + // Many languages support a basic distinction between using + // sync or async client, and this allows the specification + ClientType_SYNC_CLIENT ClientType = 0 + ClientType_ASYNC_CLIENT ClientType = 1 + ClientType_OTHER_CLIENT ClientType = 2 // used for some language-specific variants + ClientType_CALLBACK_CLIENT ClientType = 3 +) + +// Enum value maps for ClientType. +var ( + ClientType_name = map[int32]string{ + 0: "SYNC_CLIENT", + 1: "ASYNC_CLIENT", + 2: "OTHER_CLIENT", + 3: "CALLBACK_CLIENT", + } + ClientType_value = map[string]int32{ + "SYNC_CLIENT": 0, + "ASYNC_CLIENT": 1, + "OTHER_CLIENT": 2, + "CALLBACK_CLIENT": 3, + } +) + +func (x ClientType) Enum() *ClientType { + p := new(ClientType) + *p = x + return p +} + +func (x ClientType) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (ClientType) Descriptor() protoreflect.EnumDescriptor { + return file_grpc_testing_control_proto_enumTypes[0].Descriptor() +} + +func (ClientType) Type() protoreflect.EnumType { + return &file_grpc_testing_control_proto_enumTypes[0] +} + +func (x ClientType) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use ClientType.Descriptor instead. +func (ClientType) EnumDescriptor() ([]byte, []int) { + return file_grpc_testing_control_proto_rawDescGZIP(), []int{0} +} + +type ServerType int32 + +const ( + ServerType_SYNC_SERVER ServerType = 0 + ServerType_ASYNC_SERVER ServerType = 1 + ServerType_ASYNC_GENERIC_SERVER ServerType = 2 + ServerType_OTHER_SERVER ServerType = 3 // used for some language-specific variants + ServerType_CALLBACK_SERVER ServerType = 4 +) + +// Enum value maps for ServerType. +var ( + ServerType_name = map[int32]string{ + 0: "SYNC_SERVER", + 1: "ASYNC_SERVER", + 2: "ASYNC_GENERIC_SERVER", + 3: "OTHER_SERVER", + 4: "CALLBACK_SERVER", + } + ServerType_value = map[string]int32{ + "SYNC_SERVER": 0, + "ASYNC_SERVER": 1, + "ASYNC_GENERIC_SERVER": 2, + "OTHER_SERVER": 3, + "CALLBACK_SERVER": 4, + } +) + +func (x ServerType) Enum() *ServerType { + p := new(ServerType) + *p = x + return p +} + +func (x ServerType) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (ServerType) Descriptor() protoreflect.EnumDescriptor { + return file_grpc_testing_control_proto_enumTypes[1].Descriptor() +} + +func (ServerType) Type() protoreflect.EnumType { + return &file_grpc_testing_control_proto_enumTypes[1] +} + +func (x ServerType) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use ServerType.Descriptor instead. +func (ServerType) EnumDescriptor() ([]byte, []int) { + return file_grpc_testing_control_proto_rawDescGZIP(), []int{1} +} + +type RpcType int32 + +const ( + RpcType_UNARY RpcType = 0 + RpcType_STREAMING RpcType = 1 + RpcType_STREAMING_FROM_CLIENT RpcType = 2 + RpcType_STREAMING_FROM_SERVER RpcType = 3 + RpcType_STREAMING_BOTH_WAYS RpcType = 4 +) + +// Enum value maps for RpcType. +var ( + RpcType_name = map[int32]string{ + 0: "UNARY", + 1: "STREAMING", + 2: "STREAMING_FROM_CLIENT", + 3: "STREAMING_FROM_SERVER", + 4: "STREAMING_BOTH_WAYS", + } + RpcType_value = map[string]int32{ + "UNARY": 0, + "STREAMING": 1, + "STREAMING_FROM_CLIENT": 2, + "STREAMING_FROM_SERVER": 3, + "STREAMING_BOTH_WAYS": 4, + } +) + +func (x RpcType) Enum() *RpcType { + p := new(RpcType) + *p = x + return p +} + +func (x RpcType) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (RpcType) Descriptor() protoreflect.EnumDescriptor { + return file_grpc_testing_control_proto_enumTypes[2].Descriptor() +} + +func (RpcType) Type() protoreflect.EnumType { + return &file_grpc_testing_control_proto_enumTypes[2] +} + +func (x RpcType) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use RpcType.Descriptor instead. +func (RpcType) EnumDescriptor() ([]byte, []int) { + return file_grpc_testing_control_proto_rawDescGZIP(), []int{2} +} + +// Parameters of poisson process distribution, which is a good representation +// of activity coming in from independent identical stationary sources. +type PoissonParams struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // The rate of arrivals (a.k.a. lambda parameter of the exp distribution). + OfferedLoad float64 `protobuf:"fixed64,1,opt,name=offered_load,json=offeredLoad,proto3" json:"offered_load,omitempty"` +} + +func (x *PoissonParams) Reset() { + *x = PoissonParams{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_control_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PoissonParams) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PoissonParams) ProtoMessage() {} + +func (x *PoissonParams) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_control_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PoissonParams.ProtoReflect.Descriptor instead. +func (*PoissonParams) Descriptor() ([]byte, []int) { + return file_grpc_testing_control_proto_rawDescGZIP(), []int{0} +} + +func (x *PoissonParams) GetOfferedLoad() float64 { + if x != nil { + return x.OfferedLoad + } + return 0 +} + +// Once an RPC finishes, immediately start a new one. +// No configuration parameters needed. +type ClosedLoopParams struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *ClosedLoopParams) Reset() { + *x = ClosedLoopParams{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_control_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ClosedLoopParams) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ClosedLoopParams) ProtoMessage() {} + +func (x *ClosedLoopParams) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_control_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ClosedLoopParams.ProtoReflect.Descriptor instead. +func (*ClosedLoopParams) Descriptor() ([]byte, []int) { + return file_grpc_testing_control_proto_rawDescGZIP(), []int{1} +} + +type LoadParams struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Types that are assignable to Load: + // + // *LoadParams_ClosedLoop + // *LoadParams_Poisson + Load isLoadParams_Load `protobuf_oneof:"load"` +} + +func (x *LoadParams) Reset() { + *x = LoadParams{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_control_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *LoadParams) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LoadParams) ProtoMessage() {} + +func (x *LoadParams) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_control_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LoadParams.ProtoReflect.Descriptor instead. +func (*LoadParams) Descriptor() ([]byte, []int) { + return file_grpc_testing_control_proto_rawDescGZIP(), []int{2} +} + +func (m *LoadParams) GetLoad() isLoadParams_Load { + if m != nil { + return m.Load + } + return nil +} + +func (x *LoadParams) GetClosedLoop() *ClosedLoopParams { + if x, ok := x.GetLoad().(*LoadParams_ClosedLoop); ok { + return x.ClosedLoop + } + return nil +} + +func (x *LoadParams) GetPoisson() *PoissonParams { + if x, ok := x.GetLoad().(*LoadParams_Poisson); ok { + return x.Poisson + } + return nil +} + +type isLoadParams_Load interface { + isLoadParams_Load() +} + +type LoadParams_ClosedLoop struct { + ClosedLoop *ClosedLoopParams `protobuf:"bytes,1,opt,name=closed_loop,json=closedLoop,proto3,oneof"` +} + +type LoadParams_Poisson struct { + Poisson *PoissonParams `protobuf:"bytes,2,opt,name=poisson,proto3,oneof"` +} + +func (*LoadParams_ClosedLoop) isLoadParams_Load() {} + +func (*LoadParams_Poisson) isLoadParams_Load() {} + +// presence of SecurityParams implies use of TLS +type SecurityParams struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + UseTestCa bool `protobuf:"varint,1,opt,name=use_test_ca,json=useTestCa,proto3" json:"use_test_ca,omitempty"` + ServerHostOverride string `protobuf:"bytes,2,opt,name=server_host_override,json=serverHostOverride,proto3" json:"server_host_override,omitempty"` + CredType string `protobuf:"bytes,3,opt,name=cred_type,json=credType,proto3" json:"cred_type,omitempty"` +} + +func (x *SecurityParams) Reset() { + *x = SecurityParams{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_control_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SecurityParams) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SecurityParams) ProtoMessage() {} + +func (x *SecurityParams) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_control_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SecurityParams.ProtoReflect.Descriptor instead. +func (*SecurityParams) Descriptor() ([]byte, []int) { + return file_grpc_testing_control_proto_rawDescGZIP(), []int{3} +} + +func (x *SecurityParams) GetUseTestCa() bool { + if x != nil { + return x.UseTestCa + } + return false +} + +func (x *SecurityParams) GetServerHostOverride() string { + if x != nil { + return x.ServerHostOverride + } + return "" +} + +func (x *SecurityParams) GetCredType() string { + if x != nil { + return x.CredType + } + return "" +} + +type ChannelArg struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + // Types that are assignable to Value: + // + // *ChannelArg_StrValue + // *ChannelArg_IntValue + Value isChannelArg_Value `protobuf_oneof:"value"` +} + +func (x *ChannelArg) Reset() { + *x = ChannelArg{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_control_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ChannelArg) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ChannelArg) ProtoMessage() {} + +func (x *ChannelArg) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_control_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ChannelArg.ProtoReflect.Descriptor instead. +func (*ChannelArg) Descriptor() ([]byte, []int) { + return file_grpc_testing_control_proto_rawDescGZIP(), []int{4} +} + +func (x *ChannelArg) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (m *ChannelArg) GetValue() isChannelArg_Value { + if m != nil { + return m.Value + } + return nil +} + +func (x *ChannelArg) GetStrValue() string { + if x, ok := x.GetValue().(*ChannelArg_StrValue); ok { + return x.StrValue + } + return "" +} + +func (x *ChannelArg) GetIntValue() int32 { + if x, ok := x.GetValue().(*ChannelArg_IntValue); ok { + return x.IntValue + } + return 0 +} + +type isChannelArg_Value interface { + isChannelArg_Value() +} + +type ChannelArg_StrValue struct { + StrValue string `protobuf:"bytes,2,opt,name=str_value,json=strValue,proto3,oneof"` +} + +type ChannelArg_IntValue struct { + IntValue int32 `protobuf:"varint,3,opt,name=int_value,json=intValue,proto3,oneof"` +} + +func (*ChannelArg_StrValue) isChannelArg_Value() {} + +func (*ChannelArg_IntValue) isChannelArg_Value() {} + +type ClientConfig struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // List of targets to connect to. At least one target needs to be specified. + ServerTargets []string `protobuf:"bytes,1,rep,name=server_targets,json=serverTargets,proto3" json:"server_targets,omitempty"` + ClientType ClientType `protobuf:"varint,2,opt,name=client_type,json=clientType,proto3,enum=grpc.testing.ClientType" json:"client_type,omitempty"` + SecurityParams *SecurityParams `protobuf:"bytes,3,opt,name=security_params,json=securityParams,proto3" json:"security_params,omitempty"` + // How many concurrent RPCs to start for each channel. + // For synchronous client, use a separate thread for each outstanding RPC. + OutstandingRpcsPerChannel int32 `protobuf:"varint,4,opt,name=outstanding_rpcs_per_channel,json=outstandingRpcsPerChannel,proto3" json:"outstanding_rpcs_per_channel,omitempty"` + // Number of independent client channels to create. + // i-th channel will connect to server_target[i % server_targets.size()] + ClientChannels int32 `protobuf:"varint,5,opt,name=client_channels,json=clientChannels,proto3" json:"client_channels,omitempty"` + // Only for async client. Number of threads to use to start/manage RPCs. + AsyncClientThreads int32 `protobuf:"varint,7,opt,name=async_client_threads,json=asyncClientThreads,proto3" json:"async_client_threads,omitempty"` + RpcType RpcType `protobuf:"varint,8,opt,name=rpc_type,json=rpcType,proto3,enum=grpc.testing.RpcType" json:"rpc_type,omitempty"` + // The requested load for the entire client (aggregated over all the threads). + LoadParams *LoadParams `protobuf:"bytes,10,opt,name=load_params,json=loadParams,proto3" json:"load_params,omitempty"` + PayloadConfig *PayloadConfig `protobuf:"bytes,11,opt,name=payload_config,json=payloadConfig,proto3" json:"payload_config,omitempty"` + HistogramParams *HistogramParams `protobuf:"bytes,12,opt,name=histogram_params,json=histogramParams,proto3" json:"histogram_params,omitempty"` + // Specify the cores we should run the client on, if desired + CoreList []int32 `protobuf:"varint,13,rep,packed,name=core_list,json=coreList,proto3" json:"core_list,omitempty"` + CoreLimit int32 `protobuf:"varint,14,opt,name=core_limit,json=coreLimit,proto3" json:"core_limit,omitempty"` + // If we use an OTHER_CLIENT client_type, this string gives more detail + OtherClientApi string `protobuf:"bytes,15,opt,name=other_client_api,json=otherClientApi,proto3" json:"other_client_api,omitempty"` + ChannelArgs []*ChannelArg `protobuf:"bytes,16,rep,name=channel_args,json=channelArgs,proto3" json:"channel_args,omitempty"` + // Number of threads that share each completion queue + ThreadsPerCq int32 `protobuf:"varint,17,opt,name=threads_per_cq,json=threadsPerCq,proto3" json:"threads_per_cq,omitempty"` + // Number of messages on a stream before it gets finished/restarted + MessagesPerStream int32 `protobuf:"varint,18,opt,name=messages_per_stream,json=messagesPerStream,proto3" json:"messages_per_stream,omitempty"` + // Use coalescing API when possible. + UseCoalesceApi bool `protobuf:"varint,19,opt,name=use_coalesce_api,json=useCoalesceApi,proto3" json:"use_coalesce_api,omitempty"` + // If 0, disabled. Else, specifies the period between gathering latency + // medians in milliseconds. + MedianLatencyCollectionIntervalMillis int32 `protobuf:"varint,20,opt,name=median_latency_collection_interval_millis,json=medianLatencyCollectionIntervalMillis,proto3" json:"median_latency_collection_interval_millis,omitempty"` + // Number of client processes. 0 indicates no restriction. + ClientProcesses int32 `protobuf:"varint,21,opt,name=client_processes,json=clientProcesses,proto3" json:"client_processes,omitempty"` +} + +func (x *ClientConfig) Reset() { + *x = ClientConfig{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_control_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ClientConfig) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ClientConfig) ProtoMessage() {} + +func (x *ClientConfig) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_control_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ClientConfig.ProtoReflect.Descriptor instead. +func (*ClientConfig) Descriptor() ([]byte, []int) { + return file_grpc_testing_control_proto_rawDescGZIP(), []int{5} +} + +func (x *ClientConfig) GetServerTargets() []string { + if x != nil { + return x.ServerTargets + } + return nil +} + +func (x *ClientConfig) GetClientType() ClientType { + if x != nil { + return x.ClientType + } + return ClientType_SYNC_CLIENT +} + +func (x *ClientConfig) GetSecurityParams() *SecurityParams { + if x != nil { + return x.SecurityParams + } + return nil +} + +func (x *ClientConfig) GetOutstandingRpcsPerChannel() int32 { + if x != nil { + return x.OutstandingRpcsPerChannel + } + return 0 +} + +func (x *ClientConfig) GetClientChannels() int32 { + if x != nil { + return x.ClientChannels + } + return 0 +} + +func (x *ClientConfig) GetAsyncClientThreads() int32 { + if x != nil { + return x.AsyncClientThreads + } + return 0 +} + +func (x *ClientConfig) GetRpcType() RpcType { + if x != nil { + return x.RpcType + } + return RpcType_UNARY +} + +func (x *ClientConfig) GetLoadParams() *LoadParams { + if x != nil { + return x.LoadParams + } + return nil +} + +func (x *ClientConfig) GetPayloadConfig() *PayloadConfig { + if x != nil { + return x.PayloadConfig + } + return nil +} + +func (x *ClientConfig) GetHistogramParams() *HistogramParams { + if x != nil { + return x.HistogramParams + } + return nil +} + +func (x *ClientConfig) GetCoreList() []int32 { + if x != nil { + return x.CoreList + } + return nil +} + +func (x *ClientConfig) GetCoreLimit() int32 { + if x != nil { + return x.CoreLimit + } + return 0 +} + +func (x *ClientConfig) GetOtherClientApi() string { + if x != nil { + return x.OtherClientApi + } + return "" +} + +func (x *ClientConfig) GetChannelArgs() []*ChannelArg { + if x != nil { + return x.ChannelArgs + } + return nil +} + +func (x *ClientConfig) GetThreadsPerCq() int32 { + if x != nil { + return x.ThreadsPerCq + } + return 0 +} + +func (x *ClientConfig) GetMessagesPerStream() int32 { + if x != nil { + return x.MessagesPerStream + } + return 0 +} + +func (x *ClientConfig) GetUseCoalesceApi() bool { + if x != nil { + return x.UseCoalesceApi + } + return false +} + +func (x *ClientConfig) GetMedianLatencyCollectionIntervalMillis() int32 { + if x != nil { + return x.MedianLatencyCollectionIntervalMillis + } + return 0 +} + +func (x *ClientConfig) GetClientProcesses() int32 { + if x != nil { + return x.ClientProcesses + } + return 0 +} + +type ClientStatus struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Stats *ClientStats `protobuf:"bytes,1,opt,name=stats,proto3" json:"stats,omitempty"` +} + +func (x *ClientStatus) Reset() { + *x = ClientStatus{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_control_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ClientStatus) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ClientStatus) ProtoMessage() {} + +func (x *ClientStatus) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_control_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ClientStatus.ProtoReflect.Descriptor instead. +func (*ClientStatus) Descriptor() ([]byte, []int) { + return file_grpc_testing_control_proto_rawDescGZIP(), []int{6} +} + +func (x *ClientStatus) GetStats() *ClientStats { + if x != nil { + return x.Stats + } + return nil +} + +// Request current stats +type Mark struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // if true, the stats will be reset after taking their snapshot. + Reset_ bool `protobuf:"varint,1,opt,name=reset,proto3" json:"reset,omitempty"` +} + +func (x *Mark) Reset() { + *x = Mark{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_control_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Mark) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Mark) ProtoMessage() {} + +func (x *Mark) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_control_proto_msgTypes[7] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Mark.ProtoReflect.Descriptor instead. +func (*Mark) Descriptor() ([]byte, []int) { + return file_grpc_testing_control_proto_rawDescGZIP(), []int{7} +} + +func (x *Mark) GetReset_() bool { + if x != nil { + return x.Reset_ + } + return false +} + +type ClientArgs struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Types that are assignable to Argtype: + // + // *ClientArgs_Setup + // *ClientArgs_Mark + Argtype isClientArgs_Argtype `protobuf_oneof:"argtype"` +} + +func (x *ClientArgs) Reset() { + *x = ClientArgs{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_control_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ClientArgs) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ClientArgs) ProtoMessage() {} + +func (x *ClientArgs) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_control_proto_msgTypes[8] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ClientArgs.ProtoReflect.Descriptor instead. +func (*ClientArgs) Descriptor() ([]byte, []int) { + return file_grpc_testing_control_proto_rawDescGZIP(), []int{8} +} + +func (m *ClientArgs) GetArgtype() isClientArgs_Argtype { + if m != nil { + return m.Argtype + } + return nil +} + +func (x *ClientArgs) GetSetup() *ClientConfig { + if x, ok := x.GetArgtype().(*ClientArgs_Setup); ok { + return x.Setup + } + return nil +} + +func (x *ClientArgs) GetMark() *Mark { + if x, ok := x.GetArgtype().(*ClientArgs_Mark); ok { + return x.Mark + } + return nil +} + +type isClientArgs_Argtype interface { + isClientArgs_Argtype() +} + +type ClientArgs_Setup struct { + Setup *ClientConfig `protobuf:"bytes,1,opt,name=setup,proto3,oneof"` +} + +type ClientArgs_Mark struct { + Mark *Mark `protobuf:"bytes,2,opt,name=mark,proto3,oneof"` +} + +func (*ClientArgs_Setup) isClientArgs_Argtype() {} + +func (*ClientArgs_Mark) isClientArgs_Argtype() {} + +type ServerConfig struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + ServerType ServerType `protobuf:"varint,1,opt,name=server_type,json=serverType,proto3,enum=grpc.testing.ServerType" json:"server_type,omitempty"` + SecurityParams *SecurityParams `protobuf:"bytes,2,opt,name=security_params,json=securityParams,proto3" json:"security_params,omitempty"` + // Port on which to listen. Zero means pick unused port. + Port int32 `protobuf:"varint,4,opt,name=port,proto3" json:"port,omitempty"` + // Only for async server. Number of threads used to serve the requests. + AsyncServerThreads int32 `protobuf:"varint,7,opt,name=async_server_threads,json=asyncServerThreads,proto3" json:"async_server_threads,omitempty"` + // Specify the number of cores to limit server to, if desired + CoreLimit int32 `protobuf:"varint,8,opt,name=core_limit,json=coreLimit,proto3" json:"core_limit,omitempty"` + // payload config, used in generic server. + // Note this must NOT be used in proto (non-generic) servers. For proto servers, + // 'response sizes' must be configured from the 'response_size' field of the + // 'SimpleRequest' objects in RPC requests. + PayloadConfig *PayloadConfig `protobuf:"bytes,9,opt,name=payload_config,json=payloadConfig,proto3" json:"payload_config,omitempty"` + // Specify the cores we should run the server on, if desired + CoreList []int32 `protobuf:"varint,10,rep,packed,name=core_list,json=coreList,proto3" json:"core_list,omitempty"` + // If we use an OTHER_SERVER client_type, this string gives more detail + OtherServerApi string `protobuf:"bytes,11,opt,name=other_server_api,json=otherServerApi,proto3" json:"other_server_api,omitempty"` + // Number of threads that share each completion queue + ThreadsPerCq int32 `protobuf:"varint,12,opt,name=threads_per_cq,json=threadsPerCq,proto3" json:"threads_per_cq,omitempty"` + // Buffer pool size (no buffer pool specified if unset) + ResourceQuotaSize int32 `protobuf:"varint,1001,opt,name=resource_quota_size,json=resourceQuotaSize,proto3" json:"resource_quota_size,omitempty"` + ChannelArgs []*ChannelArg `protobuf:"bytes,1002,rep,name=channel_args,json=channelArgs,proto3" json:"channel_args,omitempty"` + // Number of server processes. 0 indicates no restriction. + ServerProcesses int32 `protobuf:"varint,21,opt,name=server_processes,json=serverProcesses,proto3" json:"server_processes,omitempty"` +} + +func (x *ServerConfig) Reset() { + *x = ServerConfig{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_control_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ServerConfig) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ServerConfig) ProtoMessage() {} + +func (x *ServerConfig) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_control_proto_msgTypes[9] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ServerConfig.ProtoReflect.Descriptor instead. +func (*ServerConfig) Descriptor() ([]byte, []int) { + return file_grpc_testing_control_proto_rawDescGZIP(), []int{9} +} + +func (x *ServerConfig) GetServerType() ServerType { + if x != nil { + return x.ServerType + } + return ServerType_SYNC_SERVER +} + +func (x *ServerConfig) GetSecurityParams() *SecurityParams { + if x != nil { + return x.SecurityParams + } + return nil +} + +func (x *ServerConfig) GetPort() int32 { + if x != nil { + return x.Port + } + return 0 +} + +func (x *ServerConfig) GetAsyncServerThreads() int32 { + if x != nil { + return x.AsyncServerThreads + } + return 0 +} + +func (x *ServerConfig) GetCoreLimit() int32 { + if x != nil { + return x.CoreLimit + } + return 0 +} + +func (x *ServerConfig) GetPayloadConfig() *PayloadConfig { + if x != nil { + return x.PayloadConfig + } + return nil +} + +func (x *ServerConfig) GetCoreList() []int32 { + if x != nil { + return x.CoreList + } + return nil +} + +func (x *ServerConfig) GetOtherServerApi() string { + if x != nil { + return x.OtherServerApi + } + return "" +} + +func (x *ServerConfig) GetThreadsPerCq() int32 { + if x != nil { + return x.ThreadsPerCq + } + return 0 +} + +func (x *ServerConfig) GetResourceQuotaSize() int32 { + if x != nil { + return x.ResourceQuotaSize + } + return 0 +} + +func (x *ServerConfig) GetChannelArgs() []*ChannelArg { + if x != nil { + return x.ChannelArgs + } + return nil +} + +func (x *ServerConfig) GetServerProcesses() int32 { + if x != nil { + return x.ServerProcesses + } + return 0 +} + +type ServerArgs struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Types that are assignable to Argtype: + // + // *ServerArgs_Setup + // *ServerArgs_Mark + Argtype isServerArgs_Argtype `protobuf_oneof:"argtype"` +} + +func (x *ServerArgs) Reset() { + *x = ServerArgs{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_control_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ServerArgs) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ServerArgs) ProtoMessage() {} + +func (x *ServerArgs) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_control_proto_msgTypes[10] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ServerArgs.ProtoReflect.Descriptor instead. +func (*ServerArgs) Descriptor() ([]byte, []int) { + return file_grpc_testing_control_proto_rawDescGZIP(), []int{10} +} + +func (m *ServerArgs) GetArgtype() isServerArgs_Argtype { + if m != nil { + return m.Argtype + } + return nil +} + +func (x *ServerArgs) GetSetup() *ServerConfig { + if x, ok := x.GetArgtype().(*ServerArgs_Setup); ok { + return x.Setup + } + return nil +} + +func (x *ServerArgs) GetMark() *Mark { + if x, ok := x.GetArgtype().(*ServerArgs_Mark); ok { + return x.Mark + } + return nil +} + +type isServerArgs_Argtype interface { + isServerArgs_Argtype() +} + +type ServerArgs_Setup struct { + Setup *ServerConfig `protobuf:"bytes,1,opt,name=setup,proto3,oneof"` +} + +type ServerArgs_Mark struct { + Mark *Mark `protobuf:"bytes,2,opt,name=mark,proto3,oneof"` +} + +func (*ServerArgs_Setup) isServerArgs_Argtype() {} + +func (*ServerArgs_Mark) isServerArgs_Argtype() {} + +type ServerStatus struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Stats *ServerStats `protobuf:"bytes,1,opt,name=stats,proto3" json:"stats,omitempty"` + // the port bound by the server + Port int32 `protobuf:"varint,2,opt,name=port,proto3" json:"port,omitempty"` + // Number of cores available to the server + Cores int32 `protobuf:"varint,3,opt,name=cores,proto3" json:"cores,omitempty"` +} + +func (x *ServerStatus) Reset() { + *x = ServerStatus{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_control_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ServerStatus) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ServerStatus) ProtoMessage() {} + +func (x *ServerStatus) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_control_proto_msgTypes[11] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ServerStatus.ProtoReflect.Descriptor instead. +func (*ServerStatus) Descriptor() ([]byte, []int) { + return file_grpc_testing_control_proto_rawDescGZIP(), []int{11} +} + +func (x *ServerStatus) GetStats() *ServerStats { + if x != nil { + return x.Stats + } + return nil +} + +func (x *ServerStatus) GetPort() int32 { + if x != nil { + return x.Port + } + return 0 +} + +func (x *ServerStatus) GetCores() int32 { + if x != nil { + return x.Cores + } + return 0 +} + +type CoreRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *CoreRequest) Reset() { + *x = CoreRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_control_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *CoreRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CoreRequest) ProtoMessage() {} + +func (x *CoreRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_control_proto_msgTypes[12] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CoreRequest.ProtoReflect.Descriptor instead. +func (*CoreRequest) Descriptor() ([]byte, []int) { + return file_grpc_testing_control_proto_rawDescGZIP(), []int{12} +} + +type CoreResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Number of cores available on the server + Cores int32 `protobuf:"varint,1,opt,name=cores,proto3" json:"cores,omitempty"` +} + +func (x *CoreResponse) Reset() { + *x = CoreResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_control_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *CoreResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CoreResponse) ProtoMessage() {} + +func (x *CoreResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_control_proto_msgTypes[13] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CoreResponse.ProtoReflect.Descriptor instead. +func (*CoreResponse) Descriptor() ([]byte, []int) { + return file_grpc_testing_control_proto_rawDescGZIP(), []int{13} +} + +func (x *CoreResponse) GetCores() int32 { + if x != nil { + return x.Cores + } + return 0 +} + +type Void struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *Void) Reset() { + *x = Void{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_control_proto_msgTypes[14] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Void) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Void) ProtoMessage() {} + +func (x *Void) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_control_proto_msgTypes[14] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Void.ProtoReflect.Descriptor instead. +func (*Void) Descriptor() ([]byte, []int) { + return file_grpc_testing_control_proto_rawDescGZIP(), []int{14} +} + +// A single performance scenario: input to qps_json_driver +type Scenario struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Human readable name for this scenario + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + // Client configuration + ClientConfig *ClientConfig `protobuf:"bytes,2,opt,name=client_config,json=clientConfig,proto3" json:"client_config,omitempty"` + // Number of clients to start for the test + NumClients int32 `protobuf:"varint,3,opt,name=num_clients,json=numClients,proto3" json:"num_clients,omitempty"` + // Server configuration + ServerConfig *ServerConfig `protobuf:"bytes,4,opt,name=server_config,json=serverConfig,proto3" json:"server_config,omitempty"` + // Number of servers to start for the test + NumServers int32 `protobuf:"varint,5,opt,name=num_servers,json=numServers,proto3" json:"num_servers,omitempty"` + // Warmup period, in seconds + WarmupSeconds int32 `protobuf:"varint,6,opt,name=warmup_seconds,json=warmupSeconds,proto3" json:"warmup_seconds,omitempty"` + // Benchmark time, in seconds + BenchmarkSeconds int32 `protobuf:"varint,7,opt,name=benchmark_seconds,json=benchmarkSeconds,proto3" json:"benchmark_seconds,omitempty"` + // Number of workers to spawn locally (usually zero) + SpawnLocalWorkerCount int32 `protobuf:"varint,8,opt,name=spawn_local_worker_count,json=spawnLocalWorkerCount,proto3" json:"spawn_local_worker_count,omitempty"` +} + +func (x *Scenario) Reset() { + *x = Scenario{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_control_proto_msgTypes[15] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Scenario) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Scenario) ProtoMessage() {} + +func (x *Scenario) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_control_proto_msgTypes[15] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Scenario.ProtoReflect.Descriptor instead. +func (*Scenario) Descriptor() ([]byte, []int) { + return file_grpc_testing_control_proto_rawDescGZIP(), []int{15} +} + +func (x *Scenario) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *Scenario) GetClientConfig() *ClientConfig { + if x != nil { + return x.ClientConfig + } + return nil +} + +func (x *Scenario) GetNumClients() int32 { + if x != nil { + return x.NumClients + } + return 0 +} + +func (x *Scenario) GetServerConfig() *ServerConfig { + if x != nil { + return x.ServerConfig + } + return nil +} + +func (x *Scenario) GetNumServers() int32 { + if x != nil { + return x.NumServers + } + return 0 +} + +func (x *Scenario) GetWarmupSeconds() int32 { + if x != nil { + return x.WarmupSeconds + } + return 0 +} + +func (x *Scenario) GetBenchmarkSeconds() int32 { + if x != nil { + return x.BenchmarkSeconds + } + return 0 +} + +func (x *Scenario) GetSpawnLocalWorkerCount() int32 { + if x != nil { + return x.SpawnLocalWorkerCount + } + return 0 +} + +// A set of scenarios to be run with qps_json_driver +type Scenarios struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Scenarios []*Scenario `protobuf:"bytes,1,rep,name=scenarios,proto3" json:"scenarios,omitempty"` +} + +func (x *Scenarios) Reset() { + *x = Scenarios{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_control_proto_msgTypes[16] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Scenarios) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Scenarios) ProtoMessage() {} + +func (x *Scenarios) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_control_proto_msgTypes[16] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Scenarios.ProtoReflect.Descriptor instead. +func (*Scenarios) Descriptor() ([]byte, []int) { + return file_grpc_testing_control_proto_rawDescGZIP(), []int{16} +} + +func (x *Scenarios) GetScenarios() []*Scenario { + if x != nil { + return x.Scenarios + } + return nil +} + +// Basic summary that can be computed from ClientStats and ServerStats +// once the scenario has finished. +type ScenarioResultSummary struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Total number of operations per second over all clients. What is counted as 1 'operation' depends on the benchmark scenarios: + // For unary benchmarks, an operation is processing of a single unary RPC. + // For streaming benchmarks, an operation is processing of a single ping pong of request and response. + Qps float64 `protobuf:"fixed64,1,opt,name=qps,proto3" json:"qps,omitempty"` + // QPS per server core. + QpsPerServerCore float64 `protobuf:"fixed64,2,opt,name=qps_per_server_core,json=qpsPerServerCore,proto3" json:"qps_per_server_core,omitempty"` + // The total server cpu load based on system time across all server processes, expressed as percentage of a single cpu core. + // For example, 85 implies 85% of a cpu core, 125 implies 125% of a cpu core. Since we are accumulating the cpu load across all the server + // processes, the value could > 100 when there are multiple servers or a single server using multiple threads and cores. + // Same explanation for the total client cpu load below. + ServerSystemTime float64 `protobuf:"fixed64,3,opt,name=server_system_time,json=serverSystemTime,proto3" json:"server_system_time,omitempty"` + // The total server cpu load based on user time across all server processes, expressed as percentage of a single cpu core. (85 => 85%, 125 => 125%) + ServerUserTime float64 `protobuf:"fixed64,4,opt,name=server_user_time,json=serverUserTime,proto3" json:"server_user_time,omitempty"` + // The total client cpu load based on system time across all client processes, expressed as percentage of a single cpu core. (85 => 85%, 125 => 125%) + ClientSystemTime float64 `protobuf:"fixed64,5,opt,name=client_system_time,json=clientSystemTime,proto3" json:"client_system_time,omitempty"` + // The total client cpu load based on user time across all client processes, expressed as percentage of a single cpu core. (85 => 85%, 125 => 125%) + ClientUserTime float64 `protobuf:"fixed64,6,opt,name=client_user_time,json=clientUserTime,proto3" json:"client_user_time,omitempty"` + // X% latency percentiles (in nanoseconds) + Latency_50 float64 `protobuf:"fixed64,7,opt,name=latency_50,json=latency50,proto3" json:"latency_50,omitempty"` + Latency_90 float64 `protobuf:"fixed64,8,opt,name=latency_90,json=latency90,proto3" json:"latency_90,omitempty"` + Latency_95 float64 `protobuf:"fixed64,9,opt,name=latency_95,json=latency95,proto3" json:"latency_95,omitempty"` + Latency_99 float64 `protobuf:"fixed64,10,opt,name=latency_99,json=latency99,proto3" json:"latency_99,omitempty"` + Latency_999 float64 `protobuf:"fixed64,11,opt,name=latency_999,json=latency999,proto3" json:"latency_999,omitempty"` + // server cpu usage percentage + ServerCpuUsage float64 `protobuf:"fixed64,12,opt,name=server_cpu_usage,json=serverCpuUsage,proto3" json:"server_cpu_usage,omitempty"` + // Number of requests that succeeded/failed + SuccessfulRequestsPerSecond float64 `protobuf:"fixed64,13,opt,name=successful_requests_per_second,json=successfulRequestsPerSecond,proto3" json:"successful_requests_per_second,omitempty"` + FailedRequestsPerSecond float64 `protobuf:"fixed64,14,opt,name=failed_requests_per_second,json=failedRequestsPerSecond,proto3" json:"failed_requests_per_second,omitempty"` + // Number of polls called inside completion queue per request + ClientPollsPerRequest float64 `protobuf:"fixed64,15,opt,name=client_polls_per_request,json=clientPollsPerRequest,proto3" json:"client_polls_per_request,omitempty"` + ServerPollsPerRequest float64 `protobuf:"fixed64,16,opt,name=server_polls_per_request,json=serverPollsPerRequest,proto3" json:"server_polls_per_request,omitempty"` + // Queries per CPU-sec over all servers or clients + ServerQueriesPerCpuSec float64 `protobuf:"fixed64,17,opt,name=server_queries_per_cpu_sec,json=serverQueriesPerCpuSec,proto3" json:"server_queries_per_cpu_sec,omitempty"` + ClientQueriesPerCpuSec float64 `protobuf:"fixed64,18,opt,name=client_queries_per_cpu_sec,json=clientQueriesPerCpuSec,proto3" json:"client_queries_per_cpu_sec,omitempty"` + // Start and end time for the test scenario + StartTime *timestamppb.Timestamp `protobuf:"bytes,19,opt,name=start_time,json=startTime,proto3" json:"start_time,omitempty"` + EndTime *timestamppb.Timestamp `protobuf:"bytes,20,opt,name=end_time,json=endTime,proto3" json:"end_time,omitempty"` +} + +func (x *ScenarioResultSummary) Reset() { + *x = ScenarioResultSummary{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_control_proto_msgTypes[17] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ScenarioResultSummary) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ScenarioResultSummary) ProtoMessage() {} + +func (x *ScenarioResultSummary) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_control_proto_msgTypes[17] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ScenarioResultSummary.ProtoReflect.Descriptor instead. +func (*ScenarioResultSummary) Descriptor() ([]byte, []int) { + return file_grpc_testing_control_proto_rawDescGZIP(), []int{17} +} + +func (x *ScenarioResultSummary) GetQps() float64 { + if x != nil { + return x.Qps + } + return 0 +} + +func (x *ScenarioResultSummary) GetQpsPerServerCore() float64 { + if x != nil { + return x.QpsPerServerCore + } + return 0 +} + +func (x *ScenarioResultSummary) GetServerSystemTime() float64 { + if x != nil { + return x.ServerSystemTime + } + return 0 +} + +func (x *ScenarioResultSummary) GetServerUserTime() float64 { + if x != nil { + return x.ServerUserTime + } + return 0 +} + +func (x *ScenarioResultSummary) GetClientSystemTime() float64 { + if x != nil { + return x.ClientSystemTime + } + return 0 +} + +func (x *ScenarioResultSummary) GetClientUserTime() float64 { + if x != nil { + return x.ClientUserTime + } + return 0 +} + +func (x *ScenarioResultSummary) GetLatency_50() float64 { + if x != nil { + return x.Latency_50 + } + return 0 +} + +func (x *ScenarioResultSummary) GetLatency_90() float64 { + if x != nil { + return x.Latency_90 + } + return 0 +} + +func (x *ScenarioResultSummary) GetLatency_95() float64 { + if x != nil { + return x.Latency_95 + } + return 0 +} + +func (x *ScenarioResultSummary) GetLatency_99() float64 { + if x != nil { + return x.Latency_99 + } + return 0 +} + +func (x *ScenarioResultSummary) GetLatency_999() float64 { + if x != nil { + return x.Latency_999 + } + return 0 +} + +func (x *ScenarioResultSummary) GetServerCpuUsage() float64 { + if x != nil { + return x.ServerCpuUsage + } + return 0 +} + +func (x *ScenarioResultSummary) GetSuccessfulRequestsPerSecond() float64 { + if x != nil { + return x.SuccessfulRequestsPerSecond + } + return 0 +} + +func (x *ScenarioResultSummary) GetFailedRequestsPerSecond() float64 { + if x != nil { + return x.FailedRequestsPerSecond + } + return 0 +} + +func (x *ScenarioResultSummary) GetClientPollsPerRequest() float64 { + if x != nil { + return x.ClientPollsPerRequest + } + return 0 +} + +func (x *ScenarioResultSummary) GetServerPollsPerRequest() float64 { + if x != nil { + return x.ServerPollsPerRequest + } + return 0 +} + +func (x *ScenarioResultSummary) GetServerQueriesPerCpuSec() float64 { + if x != nil { + return x.ServerQueriesPerCpuSec + } + return 0 +} + +func (x *ScenarioResultSummary) GetClientQueriesPerCpuSec() float64 { + if x != nil { + return x.ClientQueriesPerCpuSec + } + return 0 +} + +func (x *ScenarioResultSummary) GetStartTime() *timestamppb.Timestamp { + if x != nil { + return x.StartTime + } + return nil +} + +func (x *ScenarioResultSummary) GetEndTime() *timestamppb.Timestamp { + if x != nil { + return x.EndTime + } + return nil +} + +// Results of a single benchmark scenario. +type ScenarioResult struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Inputs used to run the scenario. + Scenario *Scenario `protobuf:"bytes,1,opt,name=scenario,proto3" json:"scenario,omitempty"` + // Histograms from all clients merged into one histogram. + Latencies *HistogramData `protobuf:"bytes,2,opt,name=latencies,proto3" json:"latencies,omitempty"` + // Client stats for each client + ClientStats []*ClientStats `protobuf:"bytes,3,rep,name=client_stats,json=clientStats,proto3" json:"client_stats,omitempty"` + // Server stats for each server + ServerStats []*ServerStats `protobuf:"bytes,4,rep,name=server_stats,json=serverStats,proto3" json:"server_stats,omitempty"` + // Number of cores available to each server + ServerCores []int32 `protobuf:"varint,5,rep,packed,name=server_cores,json=serverCores,proto3" json:"server_cores,omitempty"` + // An after-the-fact computed summary + Summary *ScenarioResultSummary `protobuf:"bytes,6,opt,name=summary,proto3" json:"summary,omitempty"` + // Information on success or failure of each worker + ClientSuccess []bool `protobuf:"varint,7,rep,packed,name=client_success,json=clientSuccess,proto3" json:"client_success,omitempty"` + ServerSuccess []bool `protobuf:"varint,8,rep,packed,name=server_success,json=serverSuccess,proto3" json:"server_success,omitempty"` + // Number of failed requests (one row per status code seen) + RequestResults []*RequestResultCount `protobuf:"bytes,9,rep,name=request_results,json=requestResults,proto3" json:"request_results,omitempty"` +} + +func (x *ScenarioResult) Reset() { + *x = ScenarioResult{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_control_proto_msgTypes[18] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ScenarioResult) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ScenarioResult) ProtoMessage() {} + +func (x *ScenarioResult) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_control_proto_msgTypes[18] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ScenarioResult.ProtoReflect.Descriptor instead. +func (*ScenarioResult) Descriptor() ([]byte, []int) { + return file_grpc_testing_control_proto_rawDescGZIP(), []int{18} +} + +func (x *ScenarioResult) GetScenario() *Scenario { + if x != nil { + return x.Scenario + } + return nil +} + +func (x *ScenarioResult) GetLatencies() *HistogramData { + if x != nil { + return x.Latencies + } + return nil +} + +func (x *ScenarioResult) GetClientStats() []*ClientStats { + if x != nil { + return x.ClientStats + } + return nil +} + +func (x *ScenarioResult) GetServerStats() []*ServerStats { + if x != nil { + return x.ServerStats + } + return nil +} + +func (x *ScenarioResult) GetServerCores() []int32 { + if x != nil { + return x.ServerCores + } + return nil +} + +func (x *ScenarioResult) GetSummary() *ScenarioResultSummary { + if x != nil { + return x.Summary + } + return nil +} + +func (x *ScenarioResult) GetClientSuccess() []bool { + if x != nil { + return x.ClientSuccess + } + return nil +} + +func (x *ScenarioResult) GetServerSuccess() []bool { + if x != nil { + return x.ServerSuccess + } + return nil +} + +func (x *ScenarioResult) GetRequestResults() []*RequestResultCount { + if x != nil { + return x.RequestResults + } + return nil +} + +var File_grpc_testing_control_proto protoreflect.FileDescriptor + +var file_grpc_testing_control_proto_rawDesc = []byte{ + 0x0a, 0x1a, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x63, + 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x67, 0x72, + 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x1a, 0x1b, 0x67, 0x72, 0x70, 0x63, + 0x2f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, + 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x18, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x74, 0x65, + 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x73, 0x74, 0x61, 0x74, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x22, 0x32, 0x0a, 0x0d, 0x50, 0x6f, 0x69, 0x73, 0x73, 0x6f, 0x6e, 0x50, 0x61, 0x72, + 0x61, 0x6d, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x6f, 0x66, 0x66, 0x65, 0x72, 0x65, 0x64, 0x5f, 0x6c, + 0x6f, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0b, 0x6f, 0x66, 0x66, 0x65, 0x72, + 0x65, 0x64, 0x4c, 0x6f, 0x61, 0x64, 0x22, 0x12, 0x0a, 0x10, 0x43, 0x6c, 0x6f, 0x73, 0x65, 0x64, + 0x4c, 0x6f, 0x6f, 0x70, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x22, 0x90, 0x01, 0x0a, 0x0a, 0x4c, + 0x6f, 0x61, 0x64, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x41, 0x0a, 0x0b, 0x63, 0x6c, 0x6f, + 0x73, 0x65, 0x64, 0x5f, 0x6c, 0x6f, 0x6f, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, + 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43, 0x6c, + 0x6f, 0x73, 0x65, 0x64, 0x4c, 0x6f, 0x6f, 0x70, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x48, 0x00, + 0x52, 0x0a, 0x63, 0x6c, 0x6f, 0x73, 0x65, 0x64, 0x4c, 0x6f, 0x6f, 0x70, 0x12, 0x37, 0x0a, 0x07, + 0x70, 0x6f, 0x69, 0x73, 0x73, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, + 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x6f, 0x69, + 0x73, 0x73, 0x6f, 0x6e, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x48, 0x00, 0x52, 0x07, 0x70, 0x6f, + 0x69, 0x73, 0x73, 0x6f, 0x6e, 0x42, 0x06, 0x0a, 0x04, 0x6c, 0x6f, 0x61, 0x64, 0x22, 0x7f, 0x0a, + 0x0e, 0x53, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, + 0x1e, 0x0a, 0x0b, 0x75, 0x73, 0x65, 0x5f, 0x74, 0x65, 0x73, 0x74, 0x5f, 0x63, 0x61, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x75, 0x73, 0x65, 0x54, 0x65, 0x73, 0x74, 0x43, 0x61, 0x12, + 0x30, 0x0a, 0x14, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x68, 0x6f, 0x73, 0x74, 0x5f, 0x6f, + 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x73, + 0x65, 0x72, 0x76, 0x65, 0x72, 0x48, 0x6f, 0x73, 0x74, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, + 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x63, 0x72, 0x65, 0x64, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x63, 0x72, 0x65, 0x64, 0x54, 0x79, 0x70, 0x65, 0x22, 0x67, + 0x0a, 0x0a, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x41, 0x72, 0x67, 0x12, 0x12, 0x0a, 0x04, + 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, + 0x12, 0x1d, 0x0a, 0x09, 0x73, 0x74, 0x72, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x08, 0x73, 0x74, 0x72, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, + 0x1d, 0x0a, 0x09, 0x69, 0x6e, 0x74, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x05, 0x48, 0x00, 0x52, 0x08, 0x69, 0x6e, 0x74, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x42, 0x07, + 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0xf6, 0x07, 0x0a, 0x0c, 0x43, 0x6c, 0x69, 0x65, + 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x25, 0x0a, 0x0e, 0x73, 0x65, 0x72, 0x76, + 0x65, 0x72, 0x5f, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, + 0x52, 0x0d, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x12, + 0x39, 0x0a, 0x0b, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0e, 0x32, 0x18, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, + 0x69, 0x6e, 0x67, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0a, + 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x45, 0x0a, 0x0f, 0x73, 0x65, + 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, + 0x6e, 0x67, 0x2e, 0x53, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x50, 0x61, 0x72, 0x61, 0x6d, + 0x73, 0x52, 0x0e, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x50, 0x61, 0x72, 0x61, 0x6d, + 0x73, 0x12, 0x3f, 0x0a, 0x1c, 0x6f, 0x75, 0x74, 0x73, 0x74, 0x61, 0x6e, 0x64, 0x69, 0x6e, 0x67, + 0x5f, 0x72, 0x70, 0x63, 0x73, 0x5f, 0x70, 0x65, 0x72, 0x5f, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, + 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x19, 0x6f, 0x75, 0x74, 0x73, 0x74, 0x61, 0x6e, + 0x64, 0x69, 0x6e, 0x67, 0x52, 0x70, 0x63, 0x73, 0x50, 0x65, 0x72, 0x43, 0x68, 0x61, 0x6e, 0x6e, + 0x65, 0x6c, 0x12, 0x27, 0x0a, 0x0f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x63, 0x68, 0x61, + 0x6e, 0x6e, 0x65, 0x6c, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0e, 0x63, 0x6c, 0x69, + 0x65, 0x6e, 0x74, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x73, 0x12, 0x30, 0x0a, 0x14, 0x61, + 0x73, 0x79, 0x6e, 0x63, 0x5f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x68, 0x72, 0x65, + 0x61, 0x64, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x05, 0x52, 0x12, 0x61, 0x73, 0x79, 0x6e, 0x63, + 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x54, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x12, 0x30, 0x0a, + 0x08, 0x72, 0x70, 0x63, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0e, 0x32, + 0x15, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x52, + 0x70, 0x63, 0x54, 0x79, 0x70, 0x65, 0x52, 0x07, 0x72, 0x70, 0x63, 0x54, 0x79, 0x70, 0x65, 0x12, + 0x39, 0x0a, 0x0b, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x18, 0x0a, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, + 0x69, 0x6e, 0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x52, 0x0a, + 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x42, 0x0a, 0x0e, 0x70, 0x61, + 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x0b, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, + 0x67, 0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, + 0x0d, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x48, + 0x0a, 0x10, 0x68, 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x5f, 0x70, 0x61, 0x72, 0x61, + 0x6d, 0x73, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, + 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72, 0x61, + 0x6d, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x52, 0x0f, 0x68, 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72, + 0x61, 0x6d, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x63, 0x6f, 0x72, 0x65, + 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x18, 0x0d, 0x20, 0x03, 0x28, 0x05, 0x52, 0x08, 0x63, 0x6f, 0x72, + 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6f, 0x72, 0x65, 0x5f, 0x6c, 0x69, + 0x6d, 0x69, 0x74, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, 0x63, 0x6f, 0x72, 0x65, 0x4c, + 0x69, 0x6d, 0x69, 0x74, 0x12, 0x28, 0x0a, 0x10, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x5f, 0x63, 0x6c, + 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x61, 0x70, 0x69, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, + 0x6f, 0x74, 0x68, 0x65, 0x72, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x41, 0x70, 0x69, 0x12, 0x3b, + 0x0a, 0x0c, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x5f, 0x61, 0x72, 0x67, 0x73, 0x18, 0x10, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, + 0x69, 0x6e, 0x67, 0x2e, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x41, 0x72, 0x67, 0x52, 0x0b, + 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x41, 0x72, 0x67, 0x73, 0x12, 0x24, 0x0a, 0x0e, 0x74, + 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x5f, 0x70, 0x65, 0x72, 0x5f, 0x63, 0x71, 0x18, 0x11, 0x20, + 0x01, 0x28, 0x05, 0x52, 0x0c, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x50, 0x65, 0x72, 0x43, + 0x71, 0x12, 0x2e, 0x0a, 0x13, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x5f, 0x70, 0x65, + 0x72, 0x5f, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x18, 0x12, 0x20, 0x01, 0x28, 0x05, 0x52, 0x11, + 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x50, 0x65, 0x72, 0x53, 0x74, 0x72, 0x65, 0x61, + 0x6d, 0x12, 0x28, 0x0a, 0x10, 0x75, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x61, 0x6c, 0x65, 0x73, 0x63, + 0x65, 0x5f, 0x61, 0x70, 0x69, 0x18, 0x13, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x75, 0x73, 0x65, + 0x43, 0x6f, 0x61, 0x6c, 0x65, 0x73, 0x63, 0x65, 0x41, 0x70, 0x69, 0x12, 0x58, 0x0a, 0x29, 0x6d, + 0x65, 0x64, 0x69, 0x61, 0x6e, 0x5f, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x5f, 0x63, 0x6f, + 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, + 0x6c, 0x5f, 0x6d, 0x69, 0x6c, 0x6c, 0x69, 0x73, 0x18, 0x14, 0x20, 0x01, 0x28, 0x05, 0x52, 0x25, + 0x6d, 0x65, 0x64, 0x69, 0x61, 0x6e, 0x4c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x43, 0x6f, 0x6c, + 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x4d, + 0x69, 0x6c, 0x6c, 0x69, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, + 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x65, 0x73, 0x18, 0x15, 0x20, 0x01, 0x28, 0x05, 0x52, + 0x0f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x65, 0x73, + 0x22, 0x3f, 0x0a, 0x0c, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x12, 0x2f, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x19, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43, + 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, + 0x73, 0x22, 0x1c, 0x0a, 0x04, 0x4d, 0x61, 0x72, 0x6b, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x73, + 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x72, 0x65, 0x73, 0x65, 0x74, 0x22, + 0x75, 0x0a, 0x0a, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x41, 0x72, 0x67, 0x73, 0x12, 0x32, 0x0a, + 0x05, 0x73, 0x65, 0x74, 0x75, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, + 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43, 0x6c, 0x69, 0x65, + 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x48, 0x00, 0x52, 0x05, 0x73, 0x65, 0x74, 0x75, + 0x70, 0x12, 0x28, 0x0a, 0x04, 0x6d, 0x61, 0x72, 0x6b, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x12, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4d, + 0x61, 0x72, 0x6b, 0x48, 0x00, 0x52, 0x04, 0x6d, 0x61, 0x72, 0x6b, 0x42, 0x09, 0x0a, 0x07, 0x61, + 0x72, 0x67, 0x74, 0x79, 0x70, 0x65, 0x22, 0xc0, 0x04, 0x0a, 0x0c, 0x53, 0x65, 0x72, 0x76, 0x65, + 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x39, 0x0a, 0x0b, 0x73, 0x65, 0x72, 0x76, 0x65, + 0x72, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x18, 0x2e, 0x67, + 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x65, 0x72, 0x76, + 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0a, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x54, 0x79, + 0x70, 0x65, 0x12, 0x45, 0x0a, 0x0f, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x5f, 0x70, + 0x61, 0x72, 0x61, 0x6d, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x72, + 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x65, 0x63, 0x75, 0x72, + 0x69, 0x74, 0x79, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x52, 0x0e, 0x73, 0x65, 0x63, 0x75, 0x72, + 0x69, 0x74, 0x79, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, + 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x30, 0x0a, + 0x14, 0x61, 0x73, 0x79, 0x6e, 0x63, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x74, 0x68, + 0x72, 0x65, 0x61, 0x64, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x05, 0x52, 0x12, 0x61, 0x73, 0x79, + 0x6e, 0x63, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x54, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x12, + 0x1d, 0x0a, 0x0a, 0x63, 0x6f, 0x72, 0x65, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x08, 0x20, + 0x01, 0x28, 0x05, 0x52, 0x09, 0x63, 0x6f, 0x72, 0x65, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x12, 0x42, + 0x0a, 0x0e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, + 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x43, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x52, 0x0d, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x12, 0x1b, 0x0a, 0x09, 0x63, 0x6f, 0x72, 0x65, 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x18, + 0x0a, 0x20, 0x03, 0x28, 0x05, 0x52, 0x08, 0x63, 0x6f, 0x72, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, + 0x28, 0x0a, 0x10, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, + 0x61, 0x70, 0x69, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x6f, 0x74, 0x68, 0x65, 0x72, + 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x41, 0x70, 0x69, 0x12, 0x24, 0x0a, 0x0e, 0x74, 0x68, 0x72, + 0x65, 0x61, 0x64, 0x73, 0x5f, 0x70, 0x65, 0x72, 0x5f, 0x63, 0x71, 0x18, 0x0c, 0x20, 0x01, 0x28, + 0x05, 0x52, 0x0c, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x50, 0x65, 0x72, 0x43, 0x71, 0x12, + 0x2f, 0x0a, 0x13, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x71, 0x75, 0x6f, 0x74, + 0x61, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0xe9, 0x07, 0x20, 0x01, 0x28, 0x05, 0x52, 0x11, 0x72, + 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x51, 0x75, 0x6f, 0x74, 0x61, 0x53, 0x69, 0x7a, 0x65, + 0x12, 0x3c, 0x0a, 0x0c, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x5f, 0x61, 0x72, 0x67, 0x73, + 0x18, 0xea, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, + 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x41, 0x72, + 0x67, 0x52, 0x0b, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x41, 0x72, 0x67, 0x73, 0x12, 0x29, + 0x0a, 0x10, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, + 0x65, 0x73, 0x18, 0x15, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, + 0x50, 0x72, 0x6f, 0x63, 0x65, 0x73, 0x73, 0x65, 0x73, 0x22, 0x75, 0x0a, 0x0a, 0x53, 0x65, 0x72, + 0x76, 0x65, 0x72, 0x41, 0x72, 0x67, 0x73, 0x12, 0x32, 0x0a, 0x05, 0x73, 0x65, 0x74, 0x75, 0x70, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, + 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x48, 0x00, 0x52, 0x05, 0x73, 0x65, 0x74, 0x75, 0x70, 0x12, 0x28, 0x0a, 0x04, 0x6d, + 0x61, 0x72, 0x6b, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4d, 0x61, 0x72, 0x6b, 0x48, 0x00, 0x52, + 0x04, 0x6d, 0x61, 0x72, 0x6b, 0x42, 0x09, 0x0a, 0x07, 0x61, 0x72, 0x67, 0x74, 0x79, 0x70, 0x65, + 0x22, 0x69, 0x0a, 0x0c, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x12, 0x2f, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x19, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, + 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, + 0x73, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, + 0x04, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x72, 0x65, 0x73, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x63, 0x6f, 0x72, 0x65, 0x73, 0x22, 0x0d, 0x0a, 0x0b, 0x43, + 0x6f, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x24, 0x0a, 0x0c, 0x43, 0x6f, + 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, + 0x72, 0x65, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x63, 0x6f, 0x72, 0x65, 0x73, + 0x22, 0x06, 0x0a, 0x04, 0x56, 0x6f, 0x69, 0x64, 0x22, 0xef, 0x02, 0x0a, 0x08, 0x53, 0x63, 0x65, + 0x6e, 0x61, 0x72, 0x69, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x3f, 0x0a, 0x0d, 0x63, 0x6c, 0x69, + 0x65, 0x6e, 0x74, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, + 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0c, 0x63, 0x6c, + 0x69, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1f, 0x0a, 0x0b, 0x6e, 0x75, + 0x6d, 0x5f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, + 0x0a, 0x6e, 0x75, 0x6d, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x3f, 0x0a, 0x0d, 0x73, + 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, + 0x67, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x0c, + 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1f, 0x0a, 0x0b, + 0x6e, 0x75, 0x6d, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x05, 0x52, 0x0a, 0x6e, 0x75, 0x6d, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x12, 0x25, 0x0a, + 0x0e, 0x77, 0x61, 0x72, 0x6d, 0x75, 0x70, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, + 0x06, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x77, 0x61, 0x72, 0x6d, 0x75, 0x70, 0x53, 0x65, 0x63, + 0x6f, 0x6e, 0x64, 0x73, 0x12, 0x2b, 0x0a, 0x11, 0x62, 0x65, 0x6e, 0x63, 0x68, 0x6d, 0x61, 0x72, + 0x6b, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x05, 0x52, + 0x10, 0x62, 0x65, 0x6e, 0x63, 0x68, 0x6d, 0x61, 0x72, 0x6b, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, + 0x73, 0x12, 0x37, 0x0a, 0x18, 0x73, 0x70, 0x61, 0x77, 0x6e, 0x5f, 0x6c, 0x6f, 0x63, 0x61, 0x6c, + 0x5f, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x08, 0x20, + 0x01, 0x28, 0x05, 0x52, 0x15, 0x73, 0x70, 0x61, 0x77, 0x6e, 0x4c, 0x6f, 0x63, 0x61, 0x6c, 0x57, + 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x41, 0x0a, 0x09, 0x53, 0x63, + 0x65, 0x6e, 0x61, 0x72, 0x69, 0x6f, 0x73, 0x12, 0x34, 0x0a, 0x09, 0x73, 0x63, 0x65, 0x6e, 0x61, + 0x72, 0x69, 0x6f, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x67, 0x72, 0x70, + 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x63, 0x65, 0x6e, 0x61, 0x72, + 0x69, 0x6f, 0x52, 0x09, 0x73, 0x63, 0x65, 0x6e, 0x61, 0x72, 0x69, 0x6f, 0x73, 0x22, 0xad, 0x07, + 0x0a, 0x15, 0x53, 0x63, 0x65, 0x6e, 0x61, 0x72, 0x69, 0x6f, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, + 0x53, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x71, 0x70, 0x73, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x01, 0x52, 0x03, 0x71, 0x70, 0x73, 0x12, 0x2d, 0x0a, 0x13, 0x71, 0x70, 0x73, + 0x5f, 0x70, 0x65, 0x72, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x63, 0x6f, 0x72, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x10, 0x71, 0x70, 0x73, 0x50, 0x65, 0x72, 0x53, 0x65, + 0x72, 0x76, 0x65, 0x72, 0x43, 0x6f, 0x72, 0x65, 0x12, 0x2c, 0x0a, 0x12, 0x73, 0x65, 0x72, 0x76, + 0x65, 0x72, 0x5f, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x01, 0x52, 0x10, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x79, 0x73, 0x74, + 0x65, 0x6d, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x28, 0x0a, 0x10, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, + 0x5f, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x01, + 0x52, 0x0e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x55, 0x73, 0x65, 0x72, 0x54, 0x69, 0x6d, 0x65, + 0x12, 0x2c, 0x0a, 0x12, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x73, 0x79, 0x73, 0x74, 0x65, + 0x6d, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x01, 0x52, 0x10, 0x63, 0x6c, + 0x69, 0x65, 0x6e, 0x74, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x28, + 0x0a, 0x10, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x74, 0x69, + 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, + 0x55, 0x73, 0x65, 0x72, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x6c, 0x61, 0x74, 0x65, + 0x6e, 0x63, 0x79, 0x5f, 0x35, 0x30, 0x18, 0x07, 0x20, 0x01, 0x28, 0x01, 0x52, 0x09, 0x6c, 0x61, + 0x74, 0x65, 0x6e, 0x63, 0x79, 0x35, 0x30, 0x12, 0x1d, 0x0a, 0x0a, 0x6c, 0x61, 0x74, 0x65, 0x6e, + 0x63, 0x79, 0x5f, 0x39, 0x30, 0x18, 0x08, 0x20, 0x01, 0x28, 0x01, 0x52, 0x09, 0x6c, 0x61, 0x74, + 0x65, 0x6e, 0x63, 0x79, 0x39, 0x30, 0x12, 0x1d, 0x0a, 0x0a, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63, + 0x79, 0x5f, 0x39, 0x35, 0x18, 0x09, 0x20, 0x01, 0x28, 0x01, 0x52, 0x09, 0x6c, 0x61, 0x74, 0x65, + 0x6e, 0x63, 0x79, 0x39, 0x35, 0x12, 0x1d, 0x0a, 0x0a, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79, + 0x5f, 0x39, 0x39, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x01, 0x52, 0x09, 0x6c, 0x61, 0x74, 0x65, 0x6e, + 0x63, 0x79, 0x39, 0x39, 0x12, 0x1f, 0x0a, 0x0b, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x5f, + 0x39, 0x39, 0x39, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0a, 0x6c, 0x61, 0x74, 0x65, 0x6e, + 0x63, 0x79, 0x39, 0x39, 0x39, 0x12, 0x28, 0x0a, 0x10, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, + 0x63, 0x70, 0x75, 0x5f, 0x75, 0x73, 0x61, 0x67, 0x65, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x01, 0x52, + 0x0e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x43, 0x70, 0x75, 0x55, 0x73, 0x61, 0x67, 0x65, 0x12, + 0x43, 0x0a, 0x1e, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x66, 0x75, 0x6c, 0x5f, 0x72, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x5f, 0x70, 0x65, 0x72, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, + 0x64, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x01, 0x52, 0x1b, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, + 0x66, 0x75, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x50, 0x65, 0x72, 0x53, 0x65, + 0x63, 0x6f, 0x6e, 0x64, 0x12, 0x3b, 0x0a, 0x1a, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x5f, 0x72, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x5f, 0x70, 0x65, 0x72, 0x5f, 0x73, 0x65, 0x63, 0x6f, + 0x6e, 0x64, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x01, 0x52, 0x17, 0x66, 0x61, 0x69, 0x6c, 0x65, 0x64, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x50, 0x65, 0x72, 0x53, 0x65, 0x63, 0x6f, 0x6e, + 0x64, 0x12, 0x37, 0x0a, 0x18, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x70, 0x6f, 0x6c, 0x6c, + 0x73, 0x5f, 0x70, 0x65, 0x72, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x18, 0x0f, 0x20, + 0x01, 0x28, 0x01, 0x52, 0x15, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x50, 0x6f, 0x6c, 0x6c, 0x73, + 0x50, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x37, 0x0a, 0x18, 0x73, 0x65, + 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x6f, 0x6c, 0x6c, 0x73, 0x5f, 0x70, 0x65, 0x72, 0x5f, 0x72, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x18, 0x10, 0x20, 0x01, 0x28, 0x01, 0x52, 0x15, 0x73, 0x65, + 0x72, 0x76, 0x65, 0x72, 0x50, 0x6f, 0x6c, 0x6c, 0x73, 0x50, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x12, 0x3a, 0x0a, 0x1a, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x71, 0x75, + 0x65, 0x72, 0x69, 0x65, 0x73, 0x5f, 0x70, 0x65, 0x72, 0x5f, 0x63, 0x70, 0x75, 0x5f, 0x73, 0x65, + 0x63, 0x18, 0x11, 0x20, 0x01, 0x28, 0x01, 0x52, 0x16, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x51, + 0x75, 0x65, 0x72, 0x69, 0x65, 0x73, 0x50, 0x65, 0x72, 0x43, 0x70, 0x75, 0x53, 0x65, 0x63, 0x12, + 0x3a, 0x0a, 0x1a, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x71, 0x75, 0x65, 0x72, 0x69, 0x65, + 0x73, 0x5f, 0x70, 0x65, 0x72, 0x5f, 0x63, 0x70, 0x75, 0x5f, 0x73, 0x65, 0x63, 0x18, 0x12, 0x20, + 0x01, 0x28, 0x01, 0x52, 0x16, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x51, 0x75, 0x65, 0x72, 0x69, + 0x65, 0x73, 0x50, 0x65, 0x72, 0x43, 0x70, 0x75, 0x53, 0x65, 0x63, 0x12, 0x39, 0x0a, 0x0a, 0x73, + 0x74, 0x61, 0x72, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x13, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x73, 0x74, 0x61, + 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x35, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x5f, 0x74, 0x69, + 0x6d, 0x65, 0x18, 0x14, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, + 0x74, 0x61, 0x6d, 0x70, 0x52, 0x07, 0x65, 0x6e, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x22, 0xf6, 0x03, + 0x0a, 0x0e, 0x53, 0x63, 0x65, 0x6e, 0x61, 0x72, 0x69, 0x6f, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, + 0x12, 0x32, 0x0a, 0x08, 0x73, 0x63, 0x65, 0x6e, 0x61, 0x72, 0x69, 0x6f, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, + 0x67, 0x2e, 0x53, 0x63, 0x65, 0x6e, 0x61, 0x72, 0x69, 0x6f, 0x52, 0x08, 0x73, 0x63, 0x65, 0x6e, + 0x61, 0x72, 0x69, 0x6f, 0x12, 0x39, 0x0a, 0x09, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x69, 0x65, + 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, + 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72, 0x61, 0x6d, + 0x44, 0x61, 0x74, 0x61, 0x52, 0x09, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x12, + 0x3c, 0x0a, 0x0c, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x73, 0x18, + 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, + 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x73, + 0x52, 0x0b, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, 0x3c, 0x0a, + 0x0c, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x73, 0x18, 0x04, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, + 0x6e, 0x67, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x0b, + 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x73, + 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x63, 0x6f, 0x72, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, + 0x05, 0x52, 0x0b, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x43, 0x6f, 0x72, 0x65, 0x73, 0x12, 0x3d, + 0x0a, 0x07, 0x73, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x23, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, + 0x63, 0x65, 0x6e, 0x61, 0x72, 0x69, 0x6f, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x53, 0x75, 0x6d, + 0x6d, 0x61, 0x72, 0x79, 0x52, 0x07, 0x73, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x12, 0x25, 0x0a, + 0x0e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, + 0x07, 0x20, 0x03, 0x28, 0x08, 0x52, 0x0d, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x53, 0x75, 0x63, + 0x63, 0x65, 0x73, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x73, + 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x08, 0x52, 0x0d, 0x73, 0x65, + 0x72, 0x76, 0x65, 0x72, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x49, 0x0a, 0x0f, 0x72, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x09, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, + 0x69, 0x6e, 0x67, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, 0x65, 0x73, 0x75, 0x6c, + 0x74, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x0e, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, + 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x2a, 0x56, 0x0a, 0x0a, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, + 0x54, 0x79, 0x70, 0x65, 0x12, 0x0f, 0x0a, 0x0b, 0x53, 0x59, 0x4e, 0x43, 0x5f, 0x43, 0x4c, 0x49, + 0x45, 0x4e, 0x54, 0x10, 0x00, 0x12, 0x10, 0x0a, 0x0c, 0x41, 0x53, 0x59, 0x4e, 0x43, 0x5f, 0x43, + 0x4c, 0x49, 0x45, 0x4e, 0x54, 0x10, 0x01, 0x12, 0x10, 0x0a, 0x0c, 0x4f, 0x54, 0x48, 0x45, 0x52, + 0x5f, 0x43, 0x4c, 0x49, 0x45, 0x4e, 0x54, 0x10, 0x02, 0x12, 0x13, 0x0a, 0x0f, 0x43, 0x41, 0x4c, + 0x4c, 0x42, 0x41, 0x43, 0x4b, 0x5f, 0x43, 0x4c, 0x49, 0x45, 0x4e, 0x54, 0x10, 0x03, 0x2a, 0x70, + 0x0a, 0x0a, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0f, 0x0a, 0x0b, + 0x53, 0x59, 0x4e, 0x43, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x45, 0x52, 0x10, 0x00, 0x12, 0x10, 0x0a, + 0x0c, 0x41, 0x53, 0x59, 0x4e, 0x43, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x45, 0x52, 0x10, 0x01, 0x12, + 0x18, 0x0a, 0x14, 0x41, 0x53, 0x59, 0x4e, 0x43, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x49, 0x43, + 0x5f, 0x53, 0x45, 0x52, 0x56, 0x45, 0x52, 0x10, 0x02, 0x12, 0x10, 0x0a, 0x0c, 0x4f, 0x54, 0x48, + 0x45, 0x52, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x45, 0x52, 0x10, 0x03, 0x12, 0x13, 0x0a, 0x0f, 0x43, + 0x41, 0x4c, 0x4c, 0x42, 0x41, 0x43, 0x4b, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x45, 0x52, 0x10, 0x04, + 0x2a, 0x72, 0x0a, 0x07, 0x52, 0x70, 0x63, 0x54, 0x79, 0x70, 0x65, 0x12, 0x09, 0x0a, 0x05, 0x55, + 0x4e, 0x41, 0x52, 0x59, 0x10, 0x00, 0x12, 0x0d, 0x0a, 0x09, 0x53, 0x54, 0x52, 0x45, 0x41, 0x4d, + 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x19, 0x0a, 0x15, 0x53, 0x54, 0x52, 0x45, 0x41, 0x4d, 0x49, + 0x4e, 0x47, 0x5f, 0x46, 0x52, 0x4f, 0x4d, 0x5f, 0x43, 0x4c, 0x49, 0x45, 0x4e, 0x54, 0x10, 0x02, + 0x12, 0x19, 0x0a, 0x15, 0x53, 0x54, 0x52, 0x45, 0x41, 0x4d, 0x49, 0x4e, 0x47, 0x5f, 0x46, 0x52, + 0x4f, 0x4d, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x45, 0x52, 0x10, 0x03, 0x12, 0x17, 0x0a, 0x13, 0x53, + 0x54, 0x52, 0x45, 0x41, 0x4d, 0x49, 0x4e, 0x47, 0x5f, 0x42, 0x4f, 0x54, 0x48, 0x5f, 0x57, 0x41, + 0x59, 0x53, 0x10, 0x04, 0x42, 0x21, 0x0a, 0x0f, 0x69, 0x6f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, + 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x42, 0x0c, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, + 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_grpc_testing_control_proto_rawDescOnce sync.Once + file_grpc_testing_control_proto_rawDescData = file_grpc_testing_control_proto_rawDesc +) + +func file_grpc_testing_control_proto_rawDescGZIP() []byte { + file_grpc_testing_control_proto_rawDescOnce.Do(func() { + file_grpc_testing_control_proto_rawDescData = protoimpl.X.CompressGZIP(file_grpc_testing_control_proto_rawDescData) + }) + return file_grpc_testing_control_proto_rawDescData +} + +var file_grpc_testing_control_proto_enumTypes = make([]protoimpl.EnumInfo, 3) +var file_grpc_testing_control_proto_msgTypes = make([]protoimpl.MessageInfo, 19) +var file_grpc_testing_control_proto_goTypes = []interface{}{ + (ClientType)(0), // 0: grpc.testing.ClientType + (ServerType)(0), // 1: grpc.testing.ServerType + (RpcType)(0), // 2: grpc.testing.RpcType + (*PoissonParams)(nil), // 3: grpc.testing.PoissonParams + (*ClosedLoopParams)(nil), // 4: grpc.testing.ClosedLoopParams + (*LoadParams)(nil), // 5: grpc.testing.LoadParams + (*SecurityParams)(nil), // 6: grpc.testing.SecurityParams + (*ChannelArg)(nil), // 7: grpc.testing.ChannelArg + (*ClientConfig)(nil), // 8: grpc.testing.ClientConfig + (*ClientStatus)(nil), // 9: grpc.testing.ClientStatus + (*Mark)(nil), // 10: grpc.testing.Mark + (*ClientArgs)(nil), // 11: grpc.testing.ClientArgs + (*ServerConfig)(nil), // 12: grpc.testing.ServerConfig + (*ServerArgs)(nil), // 13: grpc.testing.ServerArgs + (*ServerStatus)(nil), // 14: grpc.testing.ServerStatus + (*CoreRequest)(nil), // 15: grpc.testing.CoreRequest + (*CoreResponse)(nil), // 16: grpc.testing.CoreResponse + (*Void)(nil), // 17: grpc.testing.Void + (*Scenario)(nil), // 18: grpc.testing.Scenario + (*Scenarios)(nil), // 19: grpc.testing.Scenarios + (*ScenarioResultSummary)(nil), // 20: grpc.testing.ScenarioResultSummary + (*ScenarioResult)(nil), // 21: grpc.testing.ScenarioResult + (*PayloadConfig)(nil), // 22: grpc.testing.PayloadConfig + (*HistogramParams)(nil), // 23: grpc.testing.HistogramParams + (*ClientStats)(nil), // 24: grpc.testing.ClientStats + (*ServerStats)(nil), // 25: grpc.testing.ServerStats + (*timestamppb.Timestamp)(nil), // 26: google.protobuf.Timestamp + (*HistogramData)(nil), // 27: grpc.testing.HistogramData + (*RequestResultCount)(nil), // 28: grpc.testing.RequestResultCount +} +var file_grpc_testing_control_proto_depIdxs = []int32{ + 4, // 0: grpc.testing.LoadParams.closed_loop:type_name -> grpc.testing.ClosedLoopParams + 3, // 1: grpc.testing.LoadParams.poisson:type_name -> grpc.testing.PoissonParams + 0, // 2: grpc.testing.ClientConfig.client_type:type_name -> grpc.testing.ClientType + 6, // 3: grpc.testing.ClientConfig.security_params:type_name -> grpc.testing.SecurityParams + 2, // 4: grpc.testing.ClientConfig.rpc_type:type_name -> grpc.testing.RpcType + 5, // 5: grpc.testing.ClientConfig.load_params:type_name -> grpc.testing.LoadParams + 22, // 6: grpc.testing.ClientConfig.payload_config:type_name -> grpc.testing.PayloadConfig + 23, // 7: grpc.testing.ClientConfig.histogram_params:type_name -> grpc.testing.HistogramParams + 7, // 8: grpc.testing.ClientConfig.channel_args:type_name -> grpc.testing.ChannelArg + 24, // 9: grpc.testing.ClientStatus.stats:type_name -> grpc.testing.ClientStats + 8, // 10: grpc.testing.ClientArgs.setup:type_name -> grpc.testing.ClientConfig + 10, // 11: grpc.testing.ClientArgs.mark:type_name -> grpc.testing.Mark + 1, // 12: grpc.testing.ServerConfig.server_type:type_name -> grpc.testing.ServerType + 6, // 13: grpc.testing.ServerConfig.security_params:type_name -> grpc.testing.SecurityParams + 22, // 14: grpc.testing.ServerConfig.payload_config:type_name -> grpc.testing.PayloadConfig + 7, // 15: grpc.testing.ServerConfig.channel_args:type_name -> grpc.testing.ChannelArg + 12, // 16: grpc.testing.ServerArgs.setup:type_name -> grpc.testing.ServerConfig + 10, // 17: grpc.testing.ServerArgs.mark:type_name -> grpc.testing.Mark + 25, // 18: grpc.testing.ServerStatus.stats:type_name -> grpc.testing.ServerStats + 8, // 19: grpc.testing.Scenario.client_config:type_name -> grpc.testing.ClientConfig + 12, // 20: grpc.testing.Scenario.server_config:type_name -> grpc.testing.ServerConfig + 18, // 21: grpc.testing.Scenarios.scenarios:type_name -> grpc.testing.Scenario + 26, // 22: grpc.testing.ScenarioResultSummary.start_time:type_name -> google.protobuf.Timestamp + 26, // 23: grpc.testing.ScenarioResultSummary.end_time:type_name -> google.protobuf.Timestamp + 18, // 24: grpc.testing.ScenarioResult.scenario:type_name -> grpc.testing.Scenario + 27, // 25: grpc.testing.ScenarioResult.latencies:type_name -> grpc.testing.HistogramData + 24, // 26: grpc.testing.ScenarioResult.client_stats:type_name -> grpc.testing.ClientStats + 25, // 27: grpc.testing.ScenarioResult.server_stats:type_name -> grpc.testing.ServerStats + 20, // 28: grpc.testing.ScenarioResult.summary:type_name -> grpc.testing.ScenarioResultSummary + 28, // 29: grpc.testing.ScenarioResult.request_results:type_name -> grpc.testing.RequestResultCount + 30, // [30:30] is the sub-list for method output_type + 30, // [30:30] is the sub-list for method input_type + 30, // [30:30] is the sub-list for extension type_name + 30, // [30:30] is the sub-list for extension extendee + 0, // [0:30] is the sub-list for field type_name +} + +func init() { file_grpc_testing_control_proto_init() } +func file_grpc_testing_control_proto_init() { + if File_grpc_testing_control_proto != nil { + return + } + file_grpc_testing_payloads_proto_init() + file_grpc_testing_stats_proto_init() + if !protoimpl.UnsafeEnabled { + file_grpc_testing_control_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PoissonParams); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_control_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ClosedLoopParams); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_control_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*LoadParams); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_control_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SecurityParams); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_control_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ChannelArg); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_control_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ClientConfig); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_control_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ClientStatus); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_control_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Mark); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_control_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ClientArgs); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_control_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ServerConfig); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_control_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ServerArgs); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_control_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ServerStatus); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_control_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*CoreRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_control_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*CoreResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_control_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Void); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_control_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Scenario); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_control_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Scenarios); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_control_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ScenarioResultSummary); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_control_proto_msgTypes[18].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ScenarioResult); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + file_grpc_testing_control_proto_msgTypes[2].OneofWrappers = []interface{}{ + (*LoadParams_ClosedLoop)(nil), + (*LoadParams_Poisson)(nil), + } + file_grpc_testing_control_proto_msgTypes[4].OneofWrappers = []interface{}{ + (*ChannelArg_StrValue)(nil), + (*ChannelArg_IntValue)(nil), + } + file_grpc_testing_control_proto_msgTypes[8].OneofWrappers = []interface{}{ + (*ClientArgs_Setup)(nil), + (*ClientArgs_Mark)(nil), + } + file_grpc_testing_control_proto_msgTypes[10].OneofWrappers = []interface{}{ + (*ServerArgs_Setup)(nil), + (*ServerArgs_Mark)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_grpc_testing_control_proto_rawDesc, + NumEnums: 3, + NumMessages: 19, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_grpc_testing_control_proto_goTypes, + DependencyIndexes: file_grpc_testing_control_proto_depIdxs, + EnumInfos: file_grpc_testing_control_proto_enumTypes, + MessageInfos: file_grpc_testing_control_proto_msgTypes, + }.Build() + File_grpc_testing_control_proto = out.File + file_grpc_testing_control_proto_rawDesc = nil + file_grpc_testing_control_proto_goTypes = nil + file_grpc_testing_control_proto_depIdxs = nil +} diff --git a/interop/grpc_testing/core/stats.pb.go b/interop/grpc_testing/core/stats.pb.go new file mode 100644 index 000000000000..e61bf8f5ce12 --- /dev/null +++ b/interop/grpc_testing/core/stats.pb.go @@ -0,0 +1,408 @@ +// Copyright 2017 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.31.0 +// protoc v4.22.0 +// source: grpc/core/stats.proto + +package core + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type Bucket struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Start float64 `protobuf:"fixed64,1,opt,name=start,proto3" json:"start,omitempty"` + Count uint64 `protobuf:"varint,2,opt,name=count,proto3" json:"count,omitempty"` +} + +func (x *Bucket) Reset() { + *x = Bucket{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_core_stats_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Bucket) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Bucket) ProtoMessage() {} + +func (x *Bucket) ProtoReflect() protoreflect.Message { + mi := &file_grpc_core_stats_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Bucket.ProtoReflect.Descriptor instead. +func (*Bucket) Descriptor() ([]byte, []int) { + return file_grpc_core_stats_proto_rawDescGZIP(), []int{0} +} + +func (x *Bucket) GetStart() float64 { + if x != nil { + return x.Start + } + return 0 +} + +func (x *Bucket) GetCount() uint64 { + if x != nil { + return x.Count + } + return 0 +} + +type Histogram struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Buckets []*Bucket `protobuf:"bytes,1,rep,name=buckets,proto3" json:"buckets,omitempty"` +} + +func (x *Histogram) Reset() { + *x = Histogram{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_core_stats_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Histogram) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Histogram) ProtoMessage() {} + +func (x *Histogram) ProtoReflect() protoreflect.Message { + mi := &file_grpc_core_stats_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Histogram.ProtoReflect.Descriptor instead. +func (*Histogram) Descriptor() ([]byte, []int) { + return file_grpc_core_stats_proto_rawDescGZIP(), []int{1} +} + +func (x *Histogram) GetBuckets() []*Bucket { + if x != nil { + return x.Buckets + } + return nil +} + +type Metric struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + // Types that are assignable to Value: + // + // *Metric_Count + // *Metric_Histogram + Value isMetric_Value `protobuf_oneof:"value"` +} + +func (x *Metric) Reset() { + *x = Metric{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_core_stats_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Metric) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Metric) ProtoMessage() {} + +func (x *Metric) ProtoReflect() protoreflect.Message { + mi := &file_grpc_core_stats_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Metric.ProtoReflect.Descriptor instead. +func (*Metric) Descriptor() ([]byte, []int) { + return file_grpc_core_stats_proto_rawDescGZIP(), []int{2} +} + +func (x *Metric) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (m *Metric) GetValue() isMetric_Value { + if m != nil { + return m.Value + } + return nil +} + +func (x *Metric) GetCount() uint64 { + if x, ok := x.GetValue().(*Metric_Count); ok { + return x.Count + } + return 0 +} + +func (x *Metric) GetHistogram() *Histogram { + if x, ok := x.GetValue().(*Metric_Histogram); ok { + return x.Histogram + } + return nil +} + +type isMetric_Value interface { + isMetric_Value() +} + +type Metric_Count struct { + Count uint64 `protobuf:"varint,10,opt,name=count,proto3,oneof"` +} + +type Metric_Histogram struct { + Histogram *Histogram `protobuf:"bytes,11,opt,name=histogram,proto3,oneof"` +} + +func (*Metric_Count) isMetric_Value() {} + +func (*Metric_Histogram) isMetric_Value() {} + +type Stats struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Metrics []*Metric `protobuf:"bytes,1,rep,name=metrics,proto3" json:"metrics,omitempty"` +} + +func (x *Stats) Reset() { + *x = Stats{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_core_stats_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Stats) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Stats) ProtoMessage() {} + +func (x *Stats) ProtoReflect() protoreflect.Message { + mi := &file_grpc_core_stats_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Stats.ProtoReflect.Descriptor instead. +func (*Stats) Descriptor() ([]byte, []int) { + return file_grpc_core_stats_proto_rawDescGZIP(), []int{3} +} + +func (x *Stats) GetMetrics() []*Metric { + if x != nil { + return x.Metrics + } + return nil +} + +var File_grpc_core_stats_proto protoreflect.FileDescriptor + +var file_grpc_core_stats_proto_rawDesc = []byte{ + 0x0a, 0x15, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x73, 0x74, 0x61, 0x74, + 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x09, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x6f, + 0x72, 0x65, 0x22, 0x34, 0x0a, 0x06, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x14, 0x0a, 0x05, + 0x73, 0x74, 0x61, 0x72, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, 0x05, 0x73, 0x74, 0x61, + 0x72, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x04, 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x38, 0x0a, 0x09, 0x48, 0x69, 0x73, 0x74, + 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x12, 0x2b, 0x0a, 0x07, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, + 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x6f, + 0x72, 0x65, 0x2e, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x07, 0x62, 0x75, 0x63, 0x6b, 0x65, + 0x74, 0x73, 0x22, 0x73, 0x0a, 0x06, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x12, 0x12, 0x0a, 0x04, + 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, + 0x12, 0x16, 0x0a, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x04, 0x48, + 0x00, 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x34, 0x0a, 0x09, 0x68, 0x69, 0x73, 0x74, + 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x67, 0x72, + 0x70, 0x63, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72, 0x61, + 0x6d, 0x48, 0x00, 0x52, 0x09, 0x68, 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x42, 0x07, + 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x34, 0x0a, 0x05, 0x53, 0x74, 0x61, 0x74, 0x73, + 0x12, 0x2b, 0x0a, 0x07, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x11, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4d, 0x65, + 0x74, 0x72, 0x69, 0x63, 0x52, 0x07, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x62, 0x06, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_grpc_core_stats_proto_rawDescOnce sync.Once + file_grpc_core_stats_proto_rawDescData = file_grpc_core_stats_proto_rawDesc +) + +func file_grpc_core_stats_proto_rawDescGZIP() []byte { + file_grpc_core_stats_proto_rawDescOnce.Do(func() { + file_grpc_core_stats_proto_rawDescData = protoimpl.X.CompressGZIP(file_grpc_core_stats_proto_rawDescData) + }) + return file_grpc_core_stats_proto_rawDescData +} + +var file_grpc_core_stats_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_grpc_core_stats_proto_goTypes = []interface{}{ + (*Bucket)(nil), // 0: grpc.core.Bucket + (*Histogram)(nil), // 1: grpc.core.Histogram + (*Metric)(nil), // 2: grpc.core.Metric + (*Stats)(nil), // 3: grpc.core.Stats +} +var file_grpc_core_stats_proto_depIdxs = []int32{ + 0, // 0: grpc.core.Histogram.buckets:type_name -> grpc.core.Bucket + 1, // 1: grpc.core.Metric.histogram:type_name -> grpc.core.Histogram + 2, // 2: grpc.core.Stats.metrics:type_name -> grpc.core.Metric + 3, // [3:3] is the sub-list for method output_type + 3, // [3:3] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name +} + +func init() { file_grpc_core_stats_proto_init() } +func file_grpc_core_stats_proto_init() { + if File_grpc_core_stats_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_grpc_core_stats_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Bucket); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_core_stats_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Histogram); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_core_stats_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Metric); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_core_stats_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Stats); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + file_grpc_core_stats_proto_msgTypes[2].OneofWrappers = []interface{}{ + (*Metric_Count)(nil), + (*Metric_Histogram)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_grpc_core_stats_proto_rawDesc, + NumEnums: 0, + NumMessages: 4, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_grpc_core_stats_proto_goTypes, + DependencyIndexes: file_grpc_core_stats_proto_depIdxs, + MessageInfos: file_grpc_core_stats_proto_msgTypes, + }.Build() + File_grpc_core_stats_proto = out.File + file_grpc_core_stats_proto_rawDesc = nil + file_grpc_core_stats_proto_goTypes = nil + file_grpc_core_stats_proto_depIdxs = nil +} diff --git a/interop/grpc_testing/empty.pb.go b/interop/grpc_testing/empty.pb.go new file mode 100644 index 000000000000..8030ccd055dc --- /dev/null +++ b/interop/grpc_testing/empty.pb.go @@ -0,0 +1,155 @@ +// Copyright 2015 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.31.0 +// protoc v4.22.0 +// source: grpc/testing/empty.proto + +package grpc_testing + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// An empty message that you can re-use to avoid defining duplicated empty +// messages in your project. A typical example is to use it as argument or the +// return value of a service API. For instance: +// +// service Foo { +// rpc Bar (grpc.testing.Empty) returns (grpc.testing.Empty) { }; +// }; +type Empty struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *Empty) Reset() { + *x = Empty{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_empty_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Empty) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Empty) ProtoMessage() {} + +func (x *Empty) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_empty_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Empty.ProtoReflect.Descriptor instead. +func (*Empty) Descriptor() ([]byte, []int) { + return file_grpc_testing_empty_proto_rawDescGZIP(), []int{0} +} + +var File_grpc_testing_empty_proto protoreflect.FileDescriptor + +var file_grpc_testing_empty_proto_rawDesc = []byte{ + 0x0a, 0x18, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x65, + 0x6d, 0x70, 0x74, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x22, 0x07, 0x0a, 0x05, 0x45, 0x6d, 0x70, 0x74, + 0x79, 0x42, 0x2a, 0x0a, 0x1b, 0x69, 0x6f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, + 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x42, 0x0b, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x62, 0x06, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_grpc_testing_empty_proto_rawDescOnce sync.Once + file_grpc_testing_empty_proto_rawDescData = file_grpc_testing_empty_proto_rawDesc +) + +func file_grpc_testing_empty_proto_rawDescGZIP() []byte { + file_grpc_testing_empty_proto_rawDescOnce.Do(func() { + file_grpc_testing_empty_proto_rawDescData = protoimpl.X.CompressGZIP(file_grpc_testing_empty_proto_rawDescData) + }) + return file_grpc_testing_empty_proto_rawDescData +} + +var file_grpc_testing_empty_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_grpc_testing_empty_proto_goTypes = []interface{}{ + (*Empty)(nil), // 0: grpc.testing.Empty +} +var file_grpc_testing_empty_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_grpc_testing_empty_proto_init() } +func file_grpc_testing_empty_proto_init() { + if File_grpc_testing_empty_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_grpc_testing_empty_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Empty); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_grpc_testing_empty_proto_rawDesc, + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_grpc_testing_empty_proto_goTypes, + DependencyIndexes: file_grpc_testing_empty_proto_depIdxs, + MessageInfos: file_grpc_testing_empty_proto_msgTypes, + }.Build() + File_grpc_testing_empty_proto = out.File + file_grpc_testing_empty_proto_rawDesc = nil + file_grpc_testing_empty_proto_goTypes = nil + file_grpc_testing_empty_proto_depIdxs = nil +} diff --git a/interop/grpc_testing/messages.pb.go b/interop/grpc_testing/messages.pb.go new file mode 100644 index 000000000000..de3cb7affcbe --- /dev/null +++ b/interop/grpc_testing/messages.pb.go @@ -0,0 +1,2902 @@ +// Copyright 2015-2016 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Message definitions to be used by integration test service definitions. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.31.0 +// protoc v4.22.0 +// source: grpc/testing/messages.proto + +package grpc_testing + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// The type of payload that should be returned. +type PayloadType int32 + +const ( + // Compressable text format. + PayloadType_COMPRESSABLE PayloadType = 0 +) + +// Enum value maps for PayloadType. +var ( + PayloadType_name = map[int32]string{ + 0: "COMPRESSABLE", + } + PayloadType_value = map[string]int32{ + "COMPRESSABLE": 0, + } +) + +func (x PayloadType) Enum() *PayloadType { + p := new(PayloadType) + *p = x + return p +} + +func (x PayloadType) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (PayloadType) Descriptor() protoreflect.EnumDescriptor { + return file_grpc_testing_messages_proto_enumTypes[0].Descriptor() +} + +func (PayloadType) Type() protoreflect.EnumType { + return &file_grpc_testing_messages_proto_enumTypes[0] +} + +func (x PayloadType) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use PayloadType.Descriptor instead. +func (PayloadType) EnumDescriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{0} +} + +// The type of route that a client took to reach a server w.r.t. gRPCLB. +// The server must fill in "fallback" if it detects that the RPC reached +// the server via the "gRPCLB fallback" path, and "backend" if it detects +// that the RPC reached the server via "gRPCLB backend" path (i.e. if it got +// the address of this server from the gRPCLB server BalanceLoad RPC). Exactly +// how this detection is done is context and server dependent. +type GrpclbRouteType int32 + +const ( + // Server didn't detect the route that a client took to reach it. + GrpclbRouteType_GRPCLB_ROUTE_TYPE_UNKNOWN GrpclbRouteType = 0 + // Indicates that a client reached a server via gRPCLB fallback. + GrpclbRouteType_GRPCLB_ROUTE_TYPE_FALLBACK GrpclbRouteType = 1 + // Indicates that a client reached a server as a gRPCLB-given backend. + GrpclbRouteType_GRPCLB_ROUTE_TYPE_BACKEND GrpclbRouteType = 2 +) + +// Enum value maps for GrpclbRouteType. +var ( + GrpclbRouteType_name = map[int32]string{ + 0: "GRPCLB_ROUTE_TYPE_UNKNOWN", + 1: "GRPCLB_ROUTE_TYPE_FALLBACK", + 2: "GRPCLB_ROUTE_TYPE_BACKEND", + } + GrpclbRouteType_value = map[string]int32{ + "GRPCLB_ROUTE_TYPE_UNKNOWN": 0, + "GRPCLB_ROUTE_TYPE_FALLBACK": 1, + "GRPCLB_ROUTE_TYPE_BACKEND": 2, + } +) + +func (x GrpclbRouteType) Enum() *GrpclbRouteType { + p := new(GrpclbRouteType) + *p = x + return p +} + +func (x GrpclbRouteType) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (GrpclbRouteType) Descriptor() protoreflect.EnumDescriptor { + return file_grpc_testing_messages_proto_enumTypes[1].Descriptor() +} + +func (GrpclbRouteType) Type() protoreflect.EnumType { + return &file_grpc_testing_messages_proto_enumTypes[1] +} + +func (x GrpclbRouteType) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use GrpclbRouteType.Descriptor instead. +func (GrpclbRouteType) EnumDescriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{1} +} + +type HookRequestCommand int32 + +const ( + // Start the HTTP endpoint + HookRequestCommand_START HookRequestCommand = 0 + // Stop + HookRequestCommand_STOP HookRequestCommand = 1 + // Return from HTTP GET/POST + HookRequestCommand_RETURN HookRequestCommand = 2 +) + +// Enum value maps for HookRequestCommand. +var ( + HookRequestCommand_name = map[int32]string{ + 0: "START", + 1: "STOP", + 2: "RETURN", + } + HookRequestCommand_value = map[string]int32{ + "START": 0, + "STOP": 1, + "RETURN": 2, + } +) + +func (x HookRequestCommand) Enum() *HookRequestCommand { + p := new(HookRequestCommand) + *p = x + return p +} + +func (x HookRequestCommand) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (HookRequestCommand) Descriptor() protoreflect.EnumDescriptor { + return file_grpc_testing_messages_proto_enumTypes[2].Descriptor() +} + +func (HookRequestCommand) Type() protoreflect.EnumType { + return &file_grpc_testing_messages_proto_enumTypes[2] +} + +func (x HookRequestCommand) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use HookRequestCommand.Descriptor instead. +func (HookRequestCommand) EnumDescriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{2} +} + +type LoadBalancerStatsResponse_MetadataType int32 + +const ( + LoadBalancerStatsResponse_Initial LoadBalancerStatsResponse_MetadataType = 0 + LoadBalancerStatsResponse_Trailing LoadBalancerStatsResponse_MetadataType = 1 +) + +// Enum value maps for LoadBalancerStatsResponse_MetadataType. +var ( + LoadBalancerStatsResponse_MetadataType_name = map[int32]string{ + 0: "Initial", + 1: "Trailing", + } + LoadBalancerStatsResponse_MetadataType_value = map[string]int32{ + "Initial": 0, + "Trailing": 1, + } +) + +func (x LoadBalancerStatsResponse_MetadataType) Enum() *LoadBalancerStatsResponse_MetadataType { + p := new(LoadBalancerStatsResponse_MetadataType) + *p = x + return p +} + +func (x LoadBalancerStatsResponse_MetadataType) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (LoadBalancerStatsResponse_MetadataType) Descriptor() protoreflect.EnumDescriptor { + return file_grpc_testing_messages_proto_enumTypes[3].Descriptor() +} + +func (LoadBalancerStatsResponse_MetadataType) Type() protoreflect.EnumType { + return &file_grpc_testing_messages_proto_enumTypes[3] +} + +func (x LoadBalancerStatsResponse_MetadataType) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use LoadBalancerStatsResponse_MetadataType.Descriptor instead. +func (LoadBalancerStatsResponse_MetadataType) EnumDescriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{13, 0} +} + +// Type of RPCs to send. +type ClientConfigureRequest_RpcType int32 + +const ( + ClientConfigureRequest_EMPTY_CALL ClientConfigureRequest_RpcType = 0 + ClientConfigureRequest_UNARY_CALL ClientConfigureRequest_RpcType = 1 +) + +// Enum value maps for ClientConfigureRequest_RpcType. +var ( + ClientConfigureRequest_RpcType_name = map[int32]string{ + 0: "EMPTY_CALL", + 1: "UNARY_CALL", + } + ClientConfigureRequest_RpcType_value = map[string]int32{ + "EMPTY_CALL": 0, + "UNARY_CALL": 1, + } +) + +func (x ClientConfigureRequest_RpcType) Enum() *ClientConfigureRequest_RpcType { + p := new(ClientConfigureRequest_RpcType) + *p = x + return p +} + +func (x ClientConfigureRequest_RpcType) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (ClientConfigureRequest_RpcType) Descriptor() protoreflect.EnumDescriptor { + return file_grpc_testing_messages_proto_enumTypes[4].Descriptor() +} + +func (ClientConfigureRequest_RpcType) Type() protoreflect.EnumType { + return &file_grpc_testing_messages_proto_enumTypes[4] +} + +func (x ClientConfigureRequest_RpcType) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use ClientConfigureRequest_RpcType.Descriptor instead. +func (ClientConfigureRequest_RpcType) EnumDescriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{16, 0} +} + +// TODO(dgq): Go back to using well-known types once +// https://github.com/grpc/grpc/issues/6980 has been fixed. +// import "google/protobuf/wrappers.proto"; +type BoolValue struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // The bool value. + Value bool `protobuf:"varint,1,opt,name=value,proto3" json:"value,omitempty"` +} + +func (x *BoolValue) Reset() { + *x = BoolValue{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_messages_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *BoolValue) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BoolValue) ProtoMessage() {} + +func (x *BoolValue) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BoolValue.ProtoReflect.Descriptor instead. +func (*BoolValue) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{0} +} + +func (x *BoolValue) GetValue() bool { + if x != nil { + return x.Value + } + return false +} + +// A block of data, to simply increase gRPC message size. +type Payload struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // The type of data in body. + Type PayloadType `protobuf:"varint,1,opt,name=type,proto3,enum=grpc.testing.PayloadType" json:"type,omitempty"` + // Primary contents of payload. + Body []byte `protobuf:"bytes,2,opt,name=body,proto3" json:"body,omitempty"` +} + +func (x *Payload) Reset() { + *x = Payload{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_messages_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Payload) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Payload) ProtoMessage() {} + +func (x *Payload) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Payload.ProtoReflect.Descriptor instead. +func (*Payload) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{1} +} + +func (x *Payload) GetType() PayloadType { + if x != nil { + return x.Type + } + return PayloadType_COMPRESSABLE +} + +func (x *Payload) GetBody() []byte { + if x != nil { + return x.Body + } + return nil +} + +// A protobuf representation for grpc status. This is used by test +// clients to specify a status that the server should attempt to return. +type EchoStatus struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Code int32 `protobuf:"varint,1,opt,name=code,proto3" json:"code,omitempty"` + Message string `protobuf:"bytes,2,opt,name=message,proto3" json:"message,omitempty"` +} + +func (x *EchoStatus) Reset() { + *x = EchoStatus{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_messages_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *EchoStatus) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EchoStatus) ProtoMessage() {} + +func (x *EchoStatus) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EchoStatus.ProtoReflect.Descriptor instead. +func (*EchoStatus) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{2} +} + +func (x *EchoStatus) GetCode() int32 { + if x != nil { + return x.Code + } + return 0 +} + +func (x *EchoStatus) GetMessage() string { + if x != nil { + return x.Message + } + return "" +} + +// Unary request. +type SimpleRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Desired payload type in the response from the server. + // If response_type is RANDOM, server randomly chooses one from other formats. + ResponseType PayloadType `protobuf:"varint,1,opt,name=response_type,json=responseType,proto3,enum=grpc.testing.PayloadType" json:"response_type,omitempty"` + // Desired payload size in the response from the server. + ResponseSize int32 `protobuf:"varint,2,opt,name=response_size,json=responseSize,proto3" json:"response_size,omitempty"` + // Optional input payload sent along with the request. + Payload *Payload `protobuf:"bytes,3,opt,name=payload,proto3" json:"payload,omitempty"` + // Whether SimpleResponse should include username. + FillUsername bool `protobuf:"varint,4,opt,name=fill_username,json=fillUsername,proto3" json:"fill_username,omitempty"` + // Whether SimpleResponse should include OAuth scope. + FillOauthScope bool `protobuf:"varint,5,opt,name=fill_oauth_scope,json=fillOauthScope,proto3" json:"fill_oauth_scope,omitempty"` + // Whether to request the server to compress the response. This field is + // "nullable" in order to interoperate seamlessly with clients not able to + // implement the full compression tests by introspecting the call to verify + // the response's compression status. + ResponseCompressed *BoolValue `protobuf:"bytes,6,opt,name=response_compressed,json=responseCompressed,proto3" json:"response_compressed,omitempty"` + // Whether server should return a given status + ResponseStatus *EchoStatus `protobuf:"bytes,7,opt,name=response_status,json=responseStatus,proto3" json:"response_status,omitempty"` + // Whether the server should expect this request to be compressed. + ExpectCompressed *BoolValue `protobuf:"bytes,8,opt,name=expect_compressed,json=expectCompressed,proto3" json:"expect_compressed,omitempty"` + // Whether SimpleResponse should include server_id. + FillServerId bool `protobuf:"varint,9,opt,name=fill_server_id,json=fillServerId,proto3" json:"fill_server_id,omitempty"` + // Whether SimpleResponse should include grpclb_route_type. + FillGrpclbRouteType bool `protobuf:"varint,10,opt,name=fill_grpclb_route_type,json=fillGrpclbRouteType,proto3" json:"fill_grpclb_route_type,omitempty"` + // If set the server should record this metrics report data for the current RPC. + OrcaPerQueryReport *TestOrcaReport `protobuf:"bytes,11,opt,name=orca_per_query_report,json=orcaPerQueryReport,proto3" json:"orca_per_query_report,omitempty"` +} + +func (x *SimpleRequest) Reset() { + *x = SimpleRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_messages_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SimpleRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SimpleRequest) ProtoMessage() {} + +func (x *SimpleRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SimpleRequest.ProtoReflect.Descriptor instead. +func (*SimpleRequest) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{3} +} + +func (x *SimpleRequest) GetResponseType() PayloadType { + if x != nil { + return x.ResponseType + } + return PayloadType_COMPRESSABLE +} + +func (x *SimpleRequest) GetResponseSize() int32 { + if x != nil { + return x.ResponseSize + } + return 0 +} + +func (x *SimpleRequest) GetPayload() *Payload { + if x != nil { + return x.Payload + } + return nil +} + +func (x *SimpleRequest) GetFillUsername() bool { + if x != nil { + return x.FillUsername + } + return false +} + +func (x *SimpleRequest) GetFillOauthScope() bool { + if x != nil { + return x.FillOauthScope + } + return false +} + +func (x *SimpleRequest) GetResponseCompressed() *BoolValue { + if x != nil { + return x.ResponseCompressed + } + return nil +} + +func (x *SimpleRequest) GetResponseStatus() *EchoStatus { + if x != nil { + return x.ResponseStatus + } + return nil +} + +func (x *SimpleRequest) GetExpectCompressed() *BoolValue { + if x != nil { + return x.ExpectCompressed + } + return nil +} + +func (x *SimpleRequest) GetFillServerId() bool { + if x != nil { + return x.FillServerId + } + return false +} + +func (x *SimpleRequest) GetFillGrpclbRouteType() bool { + if x != nil { + return x.FillGrpclbRouteType + } + return false +} + +func (x *SimpleRequest) GetOrcaPerQueryReport() *TestOrcaReport { + if x != nil { + return x.OrcaPerQueryReport + } + return nil +} + +// Unary response, as configured by the request. +type SimpleResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Payload to increase message size. + Payload *Payload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"` + // The user the request came from, for verifying authentication was + // successful when the client expected it. + Username string `protobuf:"bytes,2,opt,name=username,proto3" json:"username,omitempty"` + // OAuth scope. + OauthScope string `protobuf:"bytes,3,opt,name=oauth_scope,json=oauthScope,proto3" json:"oauth_scope,omitempty"` + // Server ID. This must be unique among different server instances, + // but the same across all RPC's made to a particular server instance. + ServerId string `protobuf:"bytes,4,opt,name=server_id,json=serverId,proto3" json:"server_id,omitempty"` + // gRPCLB Path. + GrpclbRouteType GrpclbRouteType `protobuf:"varint,5,opt,name=grpclb_route_type,json=grpclbRouteType,proto3,enum=grpc.testing.GrpclbRouteType" json:"grpclb_route_type,omitempty"` + // Server hostname. + Hostname string `protobuf:"bytes,6,opt,name=hostname,proto3" json:"hostname,omitempty"` +} + +func (x *SimpleResponse) Reset() { + *x = SimpleResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_messages_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SimpleResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SimpleResponse) ProtoMessage() {} + +func (x *SimpleResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SimpleResponse.ProtoReflect.Descriptor instead. +func (*SimpleResponse) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{4} +} + +func (x *SimpleResponse) GetPayload() *Payload { + if x != nil { + return x.Payload + } + return nil +} + +func (x *SimpleResponse) GetUsername() string { + if x != nil { + return x.Username + } + return "" +} + +func (x *SimpleResponse) GetOauthScope() string { + if x != nil { + return x.OauthScope + } + return "" +} + +func (x *SimpleResponse) GetServerId() string { + if x != nil { + return x.ServerId + } + return "" +} + +func (x *SimpleResponse) GetGrpclbRouteType() GrpclbRouteType { + if x != nil { + return x.GrpclbRouteType + } + return GrpclbRouteType_GRPCLB_ROUTE_TYPE_UNKNOWN +} + +func (x *SimpleResponse) GetHostname() string { + if x != nil { + return x.Hostname + } + return "" +} + +// Client-streaming request. +type StreamingInputCallRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Optional input payload sent along with the request. + Payload *Payload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"` + // Whether the server should expect this request to be compressed. This field + // is "nullable" in order to interoperate seamlessly with servers not able to + // implement the full compression tests by introspecting the call to verify + // the request's compression status. + ExpectCompressed *BoolValue `protobuf:"bytes,2,opt,name=expect_compressed,json=expectCompressed,proto3" json:"expect_compressed,omitempty"` +} + +func (x *StreamingInputCallRequest) Reset() { + *x = StreamingInputCallRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_messages_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *StreamingInputCallRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StreamingInputCallRequest) ProtoMessage() {} + +func (x *StreamingInputCallRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StreamingInputCallRequest.ProtoReflect.Descriptor instead. +func (*StreamingInputCallRequest) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{5} +} + +func (x *StreamingInputCallRequest) GetPayload() *Payload { + if x != nil { + return x.Payload + } + return nil +} + +func (x *StreamingInputCallRequest) GetExpectCompressed() *BoolValue { + if x != nil { + return x.ExpectCompressed + } + return nil +} + +// Client-streaming response. +type StreamingInputCallResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Aggregated size of payloads received from the client. + AggregatedPayloadSize int32 `protobuf:"varint,1,opt,name=aggregated_payload_size,json=aggregatedPayloadSize,proto3" json:"aggregated_payload_size,omitempty"` +} + +func (x *StreamingInputCallResponse) Reset() { + *x = StreamingInputCallResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_messages_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *StreamingInputCallResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StreamingInputCallResponse) ProtoMessage() {} + +func (x *StreamingInputCallResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StreamingInputCallResponse.ProtoReflect.Descriptor instead. +func (*StreamingInputCallResponse) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{6} +} + +func (x *StreamingInputCallResponse) GetAggregatedPayloadSize() int32 { + if x != nil { + return x.AggregatedPayloadSize + } + return 0 +} + +// Configuration for a particular response. +type ResponseParameters struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Desired payload sizes in responses from the server. + Size int32 `protobuf:"varint,1,opt,name=size,proto3" json:"size,omitempty"` + // Desired interval between consecutive responses in the response stream in + // microseconds. + IntervalUs int32 `protobuf:"varint,2,opt,name=interval_us,json=intervalUs,proto3" json:"interval_us,omitempty"` + // Whether to request the server to compress the response. This field is + // "nullable" in order to interoperate seamlessly with clients not able to + // implement the full compression tests by introspecting the call to verify + // the response's compression status. + Compressed *BoolValue `protobuf:"bytes,3,opt,name=compressed,proto3" json:"compressed,omitempty"` +} + +func (x *ResponseParameters) Reset() { + *x = ResponseParameters{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_messages_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ResponseParameters) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ResponseParameters) ProtoMessage() {} + +func (x *ResponseParameters) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[7] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ResponseParameters.ProtoReflect.Descriptor instead. +func (*ResponseParameters) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{7} +} + +func (x *ResponseParameters) GetSize() int32 { + if x != nil { + return x.Size + } + return 0 +} + +func (x *ResponseParameters) GetIntervalUs() int32 { + if x != nil { + return x.IntervalUs + } + return 0 +} + +func (x *ResponseParameters) GetCompressed() *BoolValue { + if x != nil { + return x.Compressed + } + return nil +} + +// Server-streaming request. +type StreamingOutputCallRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Desired payload type in the response from the server. + // If response_type is RANDOM, the payload from each response in the stream + // might be of different types. This is to simulate a mixed type of payload + // stream. + ResponseType PayloadType `protobuf:"varint,1,opt,name=response_type,json=responseType,proto3,enum=grpc.testing.PayloadType" json:"response_type,omitempty"` + // Configuration for each expected response message. + ResponseParameters []*ResponseParameters `protobuf:"bytes,2,rep,name=response_parameters,json=responseParameters,proto3" json:"response_parameters,omitempty"` + // Optional input payload sent along with the request. + Payload *Payload `protobuf:"bytes,3,opt,name=payload,proto3" json:"payload,omitempty"` + // Whether server should return a given status + ResponseStatus *EchoStatus `protobuf:"bytes,7,opt,name=response_status,json=responseStatus,proto3" json:"response_status,omitempty"` + // If set the server should update this metrics report data at the OOB server. + OrcaOobReport *TestOrcaReport `protobuf:"bytes,8,opt,name=orca_oob_report,json=orcaOobReport,proto3" json:"orca_oob_report,omitempty"` +} + +func (x *StreamingOutputCallRequest) Reset() { + *x = StreamingOutputCallRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_messages_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *StreamingOutputCallRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StreamingOutputCallRequest) ProtoMessage() {} + +func (x *StreamingOutputCallRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[8] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StreamingOutputCallRequest.ProtoReflect.Descriptor instead. +func (*StreamingOutputCallRequest) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{8} +} + +func (x *StreamingOutputCallRequest) GetResponseType() PayloadType { + if x != nil { + return x.ResponseType + } + return PayloadType_COMPRESSABLE +} + +func (x *StreamingOutputCallRequest) GetResponseParameters() []*ResponseParameters { + if x != nil { + return x.ResponseParameters + } + return nil +} + +func (x *StreamingOutputCallRequest) GetPayload() *Payload { + if x != nil { + return x.Payload + } + return nil +} + +func (x *StreamingOutputCallRequest) GetResponseStatus() *EchoStatus { + if x != nil { + return x.ResponseStatus + } + return nil +} + +func (x *StreamingOutputCallRequest) GetOrcaOobReport() *TestOrcaReport { + if x != nil { + return x.OrcaOobReport + } + return nil +} + +// Server-streaming response, as configured by the request and parameters. +type StreamingOutputCallResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Payload to increase response size. + Payload *Payload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"` +} + +func (x *StreamingOutputCallResponse) Reset() { + *x = StreamingOutputCallResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_messages_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *StreamingOutputCallResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StreamingOutputCallResponse) ProtoMessage() {} + +func (x *StreamingOutputCallResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[9] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StreamingOutputCallResponse.ProtoReflect.Descriptor instead. +func (*StreamingOutputCallResponse) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{9} +} + +func (x *StreamingOutputCallResponse) GetPayload() *Payload { + if x != nil { + return x.Payload + } + return nil +} + +// For reconnect interop test only. +// Client tells server what reconnection parameters it used. +type ReconnectParams struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + MaxReconnectBackoffMs int32 `protobuf:"varint,1,opt,name=max_reconnect_backoff_ms,json=maxReconnectBackoffMs,proto3" json:"max_reconnect_backoff_ms,omitempty"` +} + +func (x *ReconnectParams) Reset() { + *x = ReconnectParams{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_messages_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ReconnectParams) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReconnectParams) ProtoMessage() {} + +func (x *ReconnectParams) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[10] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReconnectParams.ProtoReflect.Descriptor instead. +func (*ReconnectParams) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{10} +} + +func (x *ReconnectParams) GetMaxReconnectBackoffMs() int32 { + if x != nil { + return x.MaxReconnectBackoffMs + } + return 0 +} + +// For reconnect interop test only. +// Server tells client whether its reconnects are following the spec and the +// reconnect backoffs it saw. +type ReconnectInfo struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Passed bool `protobuf:"varint,1,opt,name=passed,proto3" json:"passed,omitempty"` + BackoffMs []int32 `protobuf:"varint,2,rep,packed,name=backoff_ms,json=backoffMs,proto3" json:"backoff_ms,omitempty"` +} + +func (x *ReconnectInfo) Reset() { + *x = ReconnectInfo{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_messages_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ReconnectInfo) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReconnectInfo) ProtoMessage() {} + +func (x *ReconnectInfo) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[11] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReconnectInfo.ProtoReflect.Descriptor instead. +func (*ReconnectInfo) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{11} +} + +func (x *ReconnectInfo) GetPassed() bool { + if x != nil { + return x.Passed + } + return false +} + +func (x *ReconnectInfo) GetBackoffMs() []int32 { + if x != nil { + return x.BackoffMs + } + return nil +} + +type LoadBalancerStatsRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Request stats for the next num_rpcs sent by client. + NumRpcs int32 `protobuf:"varint,1,opt,name=num_rpcs,json=numRpcs,proto3" json:"num_rpcs,omitempty"` + // If num_rpcs have not completed within timeout_sec, return partial results. + TimeoutSec int32 `protobuf:"varint,2,opt,name=timeout_sec,json=timeoutSec,proto3" json:"timeout_sec,omitempty"` + // Response header + trailer metadata entries we want the values of. + // Matching of the keys is case-insensitive as per rfc7540#section-8.1.2 + // * (asterisk) is a special value that will return all metadata entries + MetadataKeys []string `protobuf:"bytes,3,rep,name=metadata_keys,json=metadataKeys,proto3" json:"metadata_keys,omitempty"` +} + +func (x *LoadBalancerStatsRequest) Reset() { + *x = LoadBalancerStatsRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_messages_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *LoadBalancerStatsRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LoadBalancerStatsRequest) ProtoMessage() {} + +func (x *LoadBalancerStatsRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[12] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LoadBalancerStatsRequest.ProtoReflect.Descriptor instead. +func (*LoadBalancerStatsRequest) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{12} +} + +func (x *LoadBalancerStatsRequest) GetNumRpcs() int32 { + if x != nil { + return x.NumRpcs + } + return 0 +} + +func (x *LoadBalancerStatsRequest) GetTimeoutSec() int32 { + if x != nil { + return x.TimeoutSec + } + return 0 +} + +func (x *LoadBalancerStatsRequest) GetMetadataKeys() []string { + if x != nil { + return x.MetadataKeys + } + return nil +} + +type LoadBalancerStatsResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // The number of completed RPCs for each peer. + RpcsByPeer map[string]int32 `protobuf:"bytes,1,rep,name=rpcs_by_peer,json=rpcsByPeer,proto3" json:"rpcs_by_peer,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` + // The number of RPCs that failed to record a remote peer. + NumFailures int32 `protobuf:"varint,2,opt,name=num_failures,json=numFailures,proto3" json:"num_failures,omitempty"` + RpcsByMethod map[string]*LoadBalancerStatsResponse_RpcsByPeer `protobuf:"bytes,3,rep,name=rpcs_by_method,json=rpcsByMethod,proto3" json:"rpcs_by_method,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // All the metadata of all RPCs for each peer. + MetadatasByPeer map[string]*LoadBalancerStatsResponse_MetadataByPeer `protobuf:"bytes,4,rep,name=metadatas_by_peer,json=metadatasByPeer,proto3" json:"metadatas_by_peer,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` +} + +func (x *LoadBalancerStatsResponse) Reset() { + *x = LoadBalancerStatsResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_messages_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *LoadBalancerStatsResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LoadBalancerStatsResponse) ProtoMessage() {} + +func (x *LoadBalancerStatsResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[13] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LoadBalancerStatsResponse.ProtoReflect.Descriptor instead. +func (*LoadBalancerStatsResponse) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{13} +} + +func (x *LoadBalancerStatsResponse) GetRpcsByPeer() map[string]int32 { + if x != nil { + return x.RpcsByPeer + } + return nil +} + +func (x *LoadBalancerStatsResponse) GetNumFailures() int32 { + if x != nil { + return x.NumFailures + } + return 0 +} + +func (x *LoadBalancerStatsResponse) GetRpcsByMethod() map[string]*LoadBalancerStatsResponse_RpcsByPeer { + if x != nil { + return x.RpcsByMethod + } + return nil +} + +func (x *LoadBalancerStatsResponse) GetMetadatasByPeer() map[string]*LoadBalancerStatsResponse_MetadataByPeer { + if x != nil { + return x.MetadatasByPeer + } + return nil +} + +// Request for retrieving a test client's accumulated stats. +type LoadBalancerAccumulatedStatsRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *LoadBalancerAccumulatedStatsRequest) Reset() { + *x = LoadBalancerAccumulatedStatsRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_messages_proto_msgTypes[14] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *LoadBalancerAccumulatedStatsRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LoadBalancerAccumulatedStatsRequest) ProtoMessage() {} + +func (x *LoadBalancerAccumulatedStatsRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[14] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LoadBalancerAccumulatedStatsRequest.ProtoReflect.Descriptor instead. +func (*LoadBalancerAccumulatedStatsRequest) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{14} +} + +// Accumulated stats for RPCs sent by a test client. +type LoadBalancerAccumulatedStatsResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // The total number of RPCs have ever issued for each type. + // Deprecated: use stats_per_method.rpcs_started instead. + // + // Deprecated: Marked as deprecated in grpc/testing/messages.proto. + NumRpcsStartedByMethod map[string]int32 `protobuf:"bytes,1,rep,name=num_rpcs_started_by_method,json=numRpcsStartedByMethod,proto3" json:"num_rpcs_started_by_method,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` + // The total number of RPCs have ever completed successfully for each type. + // Deprecated: use stats_per_method.result instead. + // + // Deprecated: Marked as deprecated in grpc/testing/messages.proto. + NumRpcsSucceededByMethod map[string]int32 `protobuf:"bytes,2,rep,name=num_rpcs_succeeded_by_method,json=numRpcsSucceededByMethod,proto3" json:"num_rpcs_succeeded_by_method,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` + // The total number of RPCs have ever failed for each type. + // Deprecated: use stats_per_method.result instead. + // + // Deprecated: Marked as deprecated in grpc/testing/messages.proto. + NumRpcsFailedByMethod map[string]int32 `protobuf:"bytes,3,rep,name=num_rpcs_failed_by_method,json=numRpcsFailedByMethod,proto3" json:"num_rpcs_failed_by_method,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` + // Per-method RPC statistics. The key is the RpcType in string form; e.g. + // 'EMPTY_CALL' or 'UNARY_CALL' + StatsPerMethod map[string]*LoadBalancerAccumulatedStatsResponse_MethodStats `protobuf:"bytes,4,rep,name=stats_per_method,json=statsPerMethod,proto3" json:"stats_per_method,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` +} + +func (x *LoadBalancerAccumulatedStatsResponse) Reset() { + *x = LoadBalancerAccumulatedStatsResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_messages_proto_msgTypes[15] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *LoadBalancerAccumulatedStatsResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LoadBalancerAccumulatedStatsResponse) ProtoMessage() {} + +func (x *LoadBalancerAccumulatedStatsResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[15] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LoadBalancerAccumulatedStatsResponse.ProtoReflect.Descriptor instead. +func (*LoadBalancerAccumulatedStatsResponse) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{15} +} + +// Deprecated: Marked as deprecated in grpc/testing/messages.proto. +func (x *LoadBalancerAccumulatedStatsResponse) GetNumRpcsStartedByMethod() map[string]int32 { + if x != nil { + return x.NumRpcsStartedByMethod + } + return nil +} + +// Deprecated: Marked as deprecated in grpc/testing/messages.proto. +func (x *LoadBalancerAccumulatedStatsResponse) GetNumRpcsSucceededByMethod() map[string]int32 { + if x != nil { + return x.NumRpcsSucceededByMethod + } + return nil +} + +// Deprecated: Marked as deprecated in grpc/testing/messages.proto. +func (x *LoadBalancerAccumulatedStatsResponse) GetNumRpcsFailedByMethod() map[string]int32 { + if x != nil { + return x.NumRpcsFailedByMethod + } + return nil +} + +func (x *LoadBalancerAccumulatedStatsResponse) GetStatsPerMethod() map[string]*LoadBalancerAccumulatedStatsResponse_MethodStats { + if x != nil { + return x.StatsPerMethod + } + return nil +} + +// Configurations for a test client. +type ClientConfigureRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // The types of RPCs the client sends. + Types []ClientConfigureRequest_RpcType `protobuf:"varint,1,rep,packed,name=types,proto3,enum=grpc.testing.ClientConfigureRequest_RpcType" json:"types,omitempty"` + // The collection of custom metadata to be attached to RPCs sent by the client. + Metadata []*ClientConfigureRequest_Metadata `protobuf:"bytes,2,rep,name=metadata,proto3" json:"metadata,omitempty"` + // The deadline to use, in seconds, for all RPCs. If unset or zero, the + // client will use the default from the command-line. + TimeoutSec int32 `protobuf:"varint,3,opt,name=timeout_sec,json=timeoutSec,proto3" json:"timeout_sec,omitempty"` +} + +func (x *ClientConfigureRequest) Reset() { + *x = ClientConfigureRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_messages_proto_msgTypes[16] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ClientConfigureRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ClientConfigureRequest) ProtoMessage() {} + +func (x *ClientConfigureRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[16] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ClientConfigureRequest.ProtoReflect.Descriptor instead. +func (*ClientConfigureRequest) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{16} +} + +func (x *ClientConfigureRequest) GetTypes() []ClientConfigureRequest_RpcType { + if x != nil { + return x.Types + } + return nil +} + +func (x *ClientConfigureRequest) GetMetadata() []*ClientConfigureRequest_Metadata { + if x != nil { + return x.Metadata + } + return nil +} + +func (x *ClientConfigureRequest) GetTimeoutSec() int32 { + if x != nil { + return x.TimeoutSec + } + return 0 +} + +// Response for updating a test client's configuration. +type ClientConfigureResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *ClientConfigureResponse) Reset() { + *x = ClientConfigureResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_messages_proto_msgTypes[17] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ClientConfigureResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ClientConfigureResponse) ProtoMessage() {} + +func (x *ClientConfigureResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[17] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ClientConfigureResponse.ProtoReflect.Descriptor instead. +func (*ClientConfigureResponse) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{17} +} + +type MemorySize struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Rss int64 `protobuf:"varint,1,opt,name=rss,proto3" json:"rss,omitempty"` +} + +func (x *MemorySize) Reset() { + *x = MemorySize{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_messages_proto_msgTypes[18] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *MemorySize) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MemorySize) ProtoMessage() {} + +func (x *MemorySize) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[18] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MemorySize.ProtoReflect.Descriptor instead. +func (*MemorySize) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{18} +} + +func (x *MemorySize) GetRss() int64 { + if x != nil { + return x.Rss + } + return 0 +} + +// Metrics data the server will update and send to the client. It mirrors orca load report +// https://github.com/cncf/xds/blob/eded343319d09f30032952beda9840bbd3dcf7ac/xds/data/orca/v3/orca_load_report.proto#L15, +// but avoids orca dependency. Used by both per-query and out-of-band reporting tests. +type TestOrcaReport struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + CpuUtilization float64 `protobuf:"fixed64,1,opt,name=cpu_utilization,json=cpuUtilization,proto3" json:"cpu_utilization,omitempty"` + MemoryUtilization float64 `protobuf:"fixed64,2,opt,name=memory_utilization,json=memoryUtilization,proto3" json:"memory_utilization,omitempty"` + RequestCost map[string]float64 `protobuf:"bytes,3,rep,name=request_cost,json=requestCost,proto3" json:"request_cost,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"fixed64,2,opt,name=value,proto3"` + Utilization map[string]float64 `protobuf:"bytes,4,rep,name=utilization,proto3" json:"utilization,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"fixed64,2,opt,name=value,proto3"` +} + +func (x *TestOrcaReport) Reset() { + *x = TestOrcaReport{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_messages_proto_msgTypes[19] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *TestOrcaReport) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TestOrcaReport) ProtoMessage() {} + +func (x *TestOrcaReport) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[19] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TestOrcaReport.ProtoReflect.Descriptor instead. +func (*TestOrcaReport) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{19} +} + +func (x *TestOrcaReport) GetCpuUtilization() float64 { + if x != nil { + return x.CpuUtilization + } + return 0 +} + +func (x *TestOrcaReport) GetMemoryUtilization() float64 { + if x != nil { + return x.MemoryUtilization + } + return 0 +} + +func (x *TestOrcaReport) GetRequestCost() map[string]float64 { + if x != nil { + return x.RequestCost + } + return nil +} + +func (x *TestOrcaReport) GetUtilization() map[string]float64 { + if x != nil { + return x.Utilization + } + return nil +} + +type HookRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Command HookRequestCommand `protobuf:"varint,1,opt,name=command,proto3,enum=grpc.testing.HookRequestCommand" json:"command,omitempty"` + GrpcCodeToReturn int32 `protobuf:"varint,2,opt,name=grpc_code_to_return,json=grpcCodeToReturn,proto3" json:"grpc_code_to_return,omitempty"` + GrpcStatusDescription string `protobuf:"bytes,3,opt,name=grpc_status_description,json=grpcStatusDescription,proto3" json:"grpc_status_description,omitempty"` + // Server port to listen to + ServerPort int32 `protobuf:"varint,4,opt,name=server_port,json=serverPort,proto3" json:"server_port,omitempty"` +} + +func (x *HookRequest) Reset() { + *x = HookRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_messages_proto_msgTypes[20] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *HookRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HookRequest) ProtoMessage() {} + +func (x *HookRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[20] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HookRequest.ProtoReflect.Descriptor instead. +func (*HookRequest) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{20} +} + +func (x *HookRequest) GetCommand() HookRequestCommand { + if x != nil { + return x.Command + } + return HookRequestCommand_START +} + +func (x *HookRequest) GetGrpcCodeToReturn() int32 { + if x != nil { + return x.GrpcCodeToReturn + } + return 0 +} + +func (x *HookRequest) GetGrpcStatusDescription() string { + if x != nil { + return x.GrpcStatusDescription + } + return "" +} + +func (x *HookRequest) GetServerPort() int32 { + if x != nil { + return x.ServerPort + } + return 0 +} + +type HookResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *HookResponse) Reset() { + *x = HookResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_messages_proto_msgTypes[21] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *HookResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HookResponse) ProtoMessage() {} + +func (x *HookResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[21] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HookResponse.ProtoReflect.Descriptor instead. +func (*HookResponse) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{21} +} + +type LoadBalancerStatsResponse_MetadataEntry struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Key, exactly as received from the server. Case may be different from what + // was requested in the LoadBalancerStatsRequest) + Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + // Value, exactly as received from the server. + Value string `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` + // Metadata type is either Initial or Trailing + Type LoadBalancerStatsResponse_MetadataType `protobuf:"varint,3,opt,name=type,proto3,enum=grpc.testing.LoadBalancerStatsResponse_MetadataType" json:"type,omitempty"` +} + +func (x *LoadBalancerStatsResponse_MetadataEntry) Reset() { + *x = LoadBalancerStatsResponse_MetadataEntry{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_messages_proto_msgTypes[22] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *LoadBalancerStatsResponse_MetadataEntry) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LoadBalancerStatsResponse_MetadataEntry) ProtoMessage() {} + +func (x *LoadBalancerStatsResponse_MetadataEntry) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[22] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LoadBalancerStatsResponse_MetadataEntry.ProtoReflect.Descriptor instead. +func (*LoadBalancerStatsResponse_MetadataEntry) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{13, 0} +} + +func (x *LoadBalancerStatsResponse_MetadataEntry) GetKey() string { + if x != nil { + return x.Key + } + return "" +} + +func (x *LoadBalancerStatsResponse_MetadataEntry) GetValue() string { + if x != nil { + return x.Value + } + return "" +} + +func (x *LoadBalancerStatsResponse_MetadataEntry) GetType() LoadBalancerStatsResponse_MetadataType { + if x != nil { + return x.Type + } + return LoadBalancerStatsResponse_Initial +} + +type LoadBalancerStatsResponse_RpcMetadata struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // metadata values for each rpc for the keys specified in + // LoadBalancerStatsRequest.metadata_keys. + Metadata []*LoadBalancerStatsResponse_MetadataEntry `protobuf:"bytes,1,rep,name=metadata,proto3" json:"metadata,omitempty"` +} + +func (x *LoadBalancerStatsResponse_RpcMetadata) Reset() { + *x = LoadBalancerStatsResponse_RpcMetadata{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_messages_proto_msgTypes[23] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *LoadBalancerStatsResponse_RpcMetadata) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LoadBalancerStatsResponse_RpcMetadata) ProtoMessage() {} + +func (x *LoadBalancerStatsResponse_RpcMetadata) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[23] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LoadBalancerStatsResponse_RpcMetadata.ProtoReflect.Descriptor instead. +func (*LoadBalancerStatsResponse_RpcMetadata) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{13, 1} +} + +func (x *LoadBalancerStatsResponse_RpcMetadata) GetMetadata() []*LoadBalancerStatsResponse_MetadataEntry { + if x != nil { + return x.Metadata + } + return nil +} + +type LoadBalancerStatsResponse_MetadataByPeer struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // List of RpcMetadata in for each RPC with a given peer + RpcMetadata []*LoadBalancerStatsResponse_RpcMetadata `protobuf:"bytes,1,rep,name=rpc_metadata,json=rpcMetadata,proto3" json:"rpc_metadata,omitempty"` +} + +func (x *LoadBalancerStatsResponse_MetadataByPeer) Reset() { + *x = LoadBalancerStatsResponse_MetadataByPeer{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_messages_proto_msgTypes[24] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *LoadBalancerStatsResponse_MetadataByPeer) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LoadBalancerStatsResponse_MetadataByPeer) ProtoMessage() {} + +func (x *LoadBalancerStatsResponse_MetadataByPeer) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[24] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LoadBalancerStatsResponse_MetadataByPeer.ProtoReflect.Descriptor instead. +func (*LoadBalancerStatsResponse_MetadataByPeer) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{13, 2} +} + +func (x *LoadBalancerStatsResponse_MetadataByPeer) GetRpcMetadata() []*LoadBalancerStatsResponse_RpcMetadata { + if x != nil { + return x.RpcMetadata + } + return nil +} + +type LoadBalancerStatsResponse_RpcsByPeer struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // The number of completed RPCs for each peer. + RpcsByPeer map[string]int32 `protobuf:"bytes,1,rep,name=rpcs_by_peer,json=rpcsByPeer,proto3" json:"rpcs_by_peer,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` +} + +func (x *LoadBalancerStatsResponse_RpcsByPeer) Reset() { + *x = LoadBalancerStatsResponse_RpcsByPeer{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_messages_proto_msgTypes[25] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *LoadBalancerStatsResponse_RpcsByPeer) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LoadBalancerStatsResponse_RpcsByPeer) ProtoMessage() {} + +func (x *LoadBalancerStatsResponse_RpcsByPeer) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[25] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LoadBalancerStatsResponse_RpcsByPeer.ProtoReflect.Descriptor instead. +func (*LoadBalancerStatsResponse_RpcsByPeer) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{13, 3} +} + +func (x *LoadBalancerStatsResponse_RpcsByPeer) GetRpcsByPeer() map[string]int32 { + if x != nil { + return x.RpcsByPeer + } + return nil +} + +type LoadBalancerAccumulatedStatsResponse_MethodStats struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // The number of RPCs that were started for this method. + RpcsStarted int32 `protobuf:"varint,1,opt,name=rpcs_started,json=rpcsStarted,proto3" json:"rpcs_started,omitempty"` + // The number of RPCs that completed with each status for this method. The + // key is the integral value of a google.rpc.Code; the value is the count. + Result map[int32]int32 `protobuf:"bytes,2,rep,name=result,proto3" json:"result,omitempty" protobuf_key:"varint,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` +} + +func (x *LoadBalancerAccumulatedStatsResponse_MethodStats) Reset() { + *x = LoadBalancerAccumulatedStatsResponse_MethodStats{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_messages_proto_msgTypes[33] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *LoadBalancerAccumulatedStatsResponse_MethodStats) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LoadBalancerAccumulatedStatsResponse_MethodStats) ProtoMessage() {} + +func (x *LoadBalancerAccumulatedStatsResponse_MethodStats) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[33] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LoadBalancerAccumulatedStatsResponse_MethodStats.ProtoReflect.Descriptor instead. +func (*LoadBalancerAccumulatedStatsResponse_MethodStats) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{15, 3} +} + +func (x *LoadBalancerAccumulatedStatsResponse_MethodStats) GetRpcsStarted() int32 { + if x != nil { + return x.RpcsStarted + } + return 0 +} + +func (x *LoadBalancerAccumulatedStatsResponse_MethodStats) GetResult() map[int32]int32 { + if x != nil { + return x.Result + } + return nil +} + +// Metadata to be attached for the given type of RPCs. +type ClientConfigureRequest_Metadata struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Type ClientConfigureRequest_RpcType `protobuf:"varint,1,opt,name=type,proto3,enum=grpc.testing.ClientConfigureRequest_RpcType" json:"type,omitempty"` + Key string `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"` + Value string `protobuf:"bytes,3,opt,name=value,proto3" json:"value,omitempty"` +} + +func (x *ClientConfigureRequest_Metadata) Reset() { + *x = ClientConfigureRequest_Metadata{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_messages_proto_msgTypes[36] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ClientConfigureRequest_Metadata) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ClientConfigureRequest_Metadata) ProtoMessage() {} + +func (x *ClientConfigureRequest_Metadata) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_messages_proto_msgTypes[36] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ClientConfigureRequest_Metadata.ProtoReflect.Descriptor instead. +func (*ClientConfigureRequest_Metadata) Descriptor() ([]byte, []int) { + return file_grpc_testing_messages_proto_rawDescGZIP(), []int{16, 0} +} + +func (x *ClientConfigureRequest_Metadata) GetType() ClientConfigureRequest_RpcType { + if x != nil { + return x.Type + } + return ClientConfigureRequest_EMPTY_CALL +} + +func (x *ClientConfigureRequest_Metadata) GetKey() string { + if x != nil { + return x.Key + } + return "" +} + +func (x *ClientConfigureRequest_Metadata) GetValue() string { + if x != nil { + return x.Value + } + return "" +} + +var File_grpc_testing_messages_proto protoreflect.FileDescriptor + +var file_grpc_testing_messages_proto_rawDesc = []byte{ + 0x0a, 0x1b, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x6d, + 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x67, + 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x22, 0x21, 0x0a, 0x09, 0x42, + 0x6f, 0x6f, 0x6c, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x4c, + 0x0a, 0x07, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x2d, 0x0a, 0x04, 0x74, 0x79, 0x70, + 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x19, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, + 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x54, 0x79, + 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x22, 0x3a, 0x0a, 0x0a, + 0x45, 0x63, 0x68, 0x6f, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x6f, + 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x12, 0x18, + 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0xf3, 0x04, 0x0a, 0x0d, 0x53, 0x69, 0x6d, + 0x70, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3e, 0x0a, 0x0d, 0x72, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0e, 0x32, 0x19, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, + 0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0c, 0x72, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x72, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x05, 0x52, 0x0c, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x12, + 0x2f, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x15, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, + 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, + 0x12, 0x23, 0x0a, 0x0d, 0x66, 0x69, 0x6c, 0x6c, 0x5f, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, + 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x66, 0x69, 0x6c, 0x6c, 0x55, 0x73, 0x65, + 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x28, 0x0a, 0x10, 0x66, 0x69, 0x6c, 0x6c, 0x5f, 0x6f, 0x61, + 0x75, 0x74, 0x68, 0x5f, 0x73, 0x63, 0x6f, 0x70, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x0e, 0x66, 0x69, 0x6c, 0x6c, 0x4f, 0x61, 0x75, 0x74, 0x68, 0x53, 0x63, 0x6f, 0x70, 0x65, 0x12, + 0x48, 0x0a, 0x13, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x63, 0x6f, 0x6d, 0x70, + 0x72, 0x65, 0x73, 0x73, 0x65, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x67, + 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x42, 0x6f, 0x6f, 0x6c, + 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x12, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x43, + 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x65, 0x64, 0x12, 0x41, 0x0a, 0x0f, 0x72, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x07, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, + 0x67, 0x2e, 0x45, 0x63, 0x68, 0x6f, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x0e, 0x72, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x44, 0x0a, 0x11, + 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x65, + 0x64, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, + 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x42, 0x6f, 0x6f, 0x6c, 0x56, 0x61, 0x6c, 0x75, 0x65, + 0x52, 0x10, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x43, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, + 0x65, 0x64, 0x12, 0x24, 0x0a, 0x0e, 0x66, 0x69, 0x6c, 0x6c, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, + 0x72, 0x5f, 0x69, 0x64, 0x18, 0x09, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x66, 0x69, 0x6c, 0x6c, + 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x49, 0x64, 0x12, 0x33, 0x0a, 0x16, 0x66, 0x69, 0x6c, 0x6c, + 0x5f, 0x67, 0x72, 0x70, 0x63, 0x6c, 0x62, 0x5f, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x5f, 0x74, 0x79, + 0x70, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x66, 0x69, 0x6c, 0x6c, 0x47, 0x72, + 0x70, 0x63, 0x6c, 0x62, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x4f, 0x0a, + 0x15, 0x6f, 0x72, 0x63, 0x61, 0x5f, 0x70, 0x65, 0x72, 0x5f, 0x71, 0x75, 0x65, 0x72, 0x79, 0x5f, + 0x72, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, + 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x54, 0x65, 0x73, 0x74, + 0x4f, 0x72, 0x63, 0x61, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x52, 0x12, 0x6f, 0x72, 0x63, 0x61, + 0x50, 0x65, 0x72, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x22, 0x82, + 0x02, 0x0a, 0x0e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x2f, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, + 0x67, 0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, + 0x61, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1f, + 0x0a, 0x0b, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x5f, 0x73, 0x63, 0x6f, 0x70, 0x65, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0a, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x53, 0x63, 0x6f, 0x70, 0x65, 0x12, + 0x1b, 0x0a, 0x09, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x08, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x49, 0x64, 0x12, 0x49, 0x0a, 0x11, + 0x67, 0x72, 0x70, 0x63, 0x6c, 0x62, 0x5f, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x5f, 0x74, 0x79, 0x70, + 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, + 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x72, 0x70, 0x63, 0x6c, 0x62, 0x52, 0x6f, 0x75, + 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0f, 0x67, 0x72, 0x70, 0x63, 0x6c, 0x62, 0x52, 0x6f, + 0x75, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x68, 0x6f, 0x73, 0x74, 0x6e, + 0x61, 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x68, 0x6f, 0x73, 0x74, 0x6e, + 0x61, 0x6d, 0x65, 0x22, 0x92, 0x01, 0x0a, 0x19, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, + 0x67, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x2f, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, + 0x67, 0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, + 0x61, 0x64, 0x12, 0x44, 0x0a, 0x11, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x5f, 0x63, 0x6f, 0x6d, + 0x70, 0x72, 0x65, 0x73, 0x73, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, + 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x42, 0x6f, 0x6f, + 0x6c, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x10, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x43, 0x6f, + 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x65, 0x64, 0x22, 0x54, 0x0a, 0x1a, 0x53, 0x74, 0x72, 0x65, + 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x17, 0x61, 0x67, 0x67, 0x72, 0x65, 0x67, + 0x61, 0x74, 0x65, 0x64, 0x5f, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x73, 0x69, 0x7a, + 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x15, 0x61, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, + 0x74, 0x65, 0x64, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x53, 0x69, 0x7a, 0x65, 0x22, 0x82, + 0x01, 0x0a, 0x12, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x50, 0x61, 0x72, 0x61, 0x6d, + 0x65, 0x74, 0x65, 0x72, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x05, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x69, 0x6e, 0x74, + 0x65, 0x72, 0x76, 0x61, 0x6c, 0x5f, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, + 0x69, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x55, 0x73, 0x12, 0x37, 0x0a, 0x0a, 0x63, 0x6f, + 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, + 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x42, 0x6f, + 0x6f, 0x6c, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x0a, 0x63, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, + 0x73, 0x65, 0x64, 0x22, 0xe9, 0x02, 0x0a, 0x1a, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, + 0x67, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x3e, 0x0a, 0x0d, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x74, + 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x19, 0x2e, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, + 0x54, 0x79, 0x70, 0x65, 0x52, 0x0c, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x54, 0x79, + 0x70, 0x65, 0x12, 0x51, 0x0a, 0x13, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x70, + 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x20, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, + 0x73, 0x52, 0x12, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x50, 0x61, 0x72, 0x61, 0x6d, + 0x65, 0x74, 0x65, 0x72, 0x73, 0x12, 0x2f, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, + 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x07, 0x70, + 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x41, 0x0a, 0x0f, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x18, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x45, + 0x63, 0x68, 0x6f, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x0e, 0x72, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x44, 0x0a, 0x0f, 0x6f, 0x72, 0x63, + 0x61, 0x5f, 0x6f, 0x6f, 0x62, 0x5f, 0x72, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x08, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, + 0x67, 0x2e, 0x54, 0x65, 0x73, 0x74, 0x4f, 0x72, 0x63, 0x61, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, + 0x52, 0x0d, 0x6f, 0x72, 0x63, 0x61, 0x4f, 0x6f, 0x62, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x22, + 0x4e, 0x0a, 0x1b, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f, 0x75, 0x74, 0x70, + 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2f, + 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x15, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x50, + 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x22, + 0x4a, 0x0a, 0x0f, 0x52, 0x65, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x50, 0x61, 0x72, 0x61, + 0x6d, 0x73, 0x12, 0x37, 0x0a, 0x18, 0x6d, 0x61, 0x78, 0x5f, 0x72, 0x65, 0x63, 0x6f, 0x6e, 0x6e, + 0x65, 0x63, 0x74, 0x5f, 0x62, 0x61, 0x63, 0x6b, 0x6f, 0x66, 0x66, 0x5f, 0x6d, 0x73, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x05, 0x52, 0x15, 0x6d, 0x61, 0x78, 0x52, 0x65, 0x63, 0x6f, 0x6e, 0x6e, 0x65, + 0x63, 0x74, 0x42, 0x61, 0x63, 0x6b, 0x6f, 0x66, 0x66, 0x4d, 0x73, 0x22, 0x46, 0x0a, 0x0d, 0x52, + 0x65, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x16, 0x0a, 0x06, + 0x70, 0x61, 0x73, 0x73, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x70, 0x61, + 0x73, 0x73, 0x65, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x62, 0x61, 0x63, 0x6b, 0x6f, 0x66, 0x66, 0x5f, + 0x6d, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x05, 0x52, 0x09, 0x62, 0x61, 0x63, 0x6b, 0x6f, 0x66, + 0x66, 0x4d, 0x73, 0x22, 0x7b, 0x0a, 0x18, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, + 0x63, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x19, 0x0a, 0x08, 0x6e, 0x75, 0x6d, 0x5f, 0x72, 0x70, 0x63, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x05, 0x52, 0x07, 0x6e, 0x75, 0x6d, 0x52, 0x70, 0x63, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x74, 0x69, + 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x5f, 0x73, 0x65, 0x63, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, + 0x0a, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x53, 0x65, 0x63, 0x12, 0x23, 0x0a, 0x0d, 0x6d, + 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x03, 0x20, 0x03, + 0x28, 0x09, 0x52, 0x0c, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x4b, 0x65, 0x79, 0x73, + 0x22, 0xc3, 0x09, 0x0a, 0x19, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, + 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x59, + 0x0a, 0x0c, 0x72, 0x70, 0x63, 0x73, 0x5f, 0x62, 0x79, 0x5f, 0x70, 0x65, 0x65, 0x72, 0x18, 0x01, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, + 0x69, 0x6e, 0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, + 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x70, + 0x63, 0x73, 0x42, 0x79, 0x50, 0x65, 0x65, 0x72, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x72, + 0x70, 0x63, 0x73, 0x42, 0x79, 0x50, 0x65, 0x65, 0x72, 0x12, 0x21, 0x0a, 0x0c, 0x6e, 0x75, 0x6d, + 0x5f, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, + 0x0b, 0x6e, 0x75, 0x6d, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x73, 0x12, 0x5f, 0x0a, 0x0e, + 0x72, 0x70, 0x63, 0x73, 0x5f, 0x62, 0x79, 0x5f, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x18, 0x03, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, + 0x69, 0x6e, 0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, + 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x70, + 0x63, 0x73, 0x42, 0x79, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, + 0x0c, 0x72, 0x70, 0x63, 0x73, 0x42, 0x79, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x12, 0x68, 0x0a, + 0x11, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x73, 0x5f, 0x62, 0x79, 0x5f, 0x70, 0x65, + 0x65, 0x72, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, + 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, + 0x6e, 0x63, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x73, 0x42, 0x79, 0x50, 0x65, 0x65, + 0x72, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x73, 0x42, 0x79, 0x50, 0x65, 0x65, 0x72, 0x1a, 0x81, 0x01, 0x0a, 0x0d, 0x4d, 0x65, 0x74, 0x61, + 0x64, 0x61, 0x74, 0x61, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x12, 0x48, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, + 0x34, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4c, + 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, + 0x61, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x1a, 0x60, 0x0a, 0x0b, 0x52, + 0x70, 0x63, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x51, 0x0a, 0x08, 0x6d, 0x65, + 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x35, 0x2e, 0x67, + 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, + 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x1a, 0x68, 0x0a, + 0x0e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x42, 0x79, 0x50, 0x65, 0x65, 0x72, 0x12, + 0x56, 0x0a, 0x0c, 0x72, 0x70, 0x63, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, + 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, + 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, + 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, + 0x70, 0x63, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x0b, 0x72, 0x70, 0x63, 0x4d, + 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x1a, 0xb1, 0x01, 0x0a, 0x0a, 0x52, 0x70, 0x63, 0x73, + 0x42, 0x79, 0x50, 0x65, 0x65, 0x72, 0x12, 0x64, 0x0a, 0x0c, 0x72, 0x70, 0x63, 0x73, 0x5f, 0x62, + 0x79, 0x5f, 0x70, 0x65, 0x65, 0x72, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x42, 0x2e, 0x67, + 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, + 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x70, 0x63, 0x73, 0x42, 0x79, 0x50, 0x65, 0x65, 0x72, + 0x2e, 0x52, 0x70, 0x63, 0x73, 0x42, 0x79, 0x50, 0x65, 0x65, 0x72, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x52, 0x0a, 0x72, 0x70, 0x63, 0x73, 0x42, 0x79, 0x50, 0x65, 0x65, 0x72, 0x1a, 0x3d, 0x0a, 0x0f, + 0x52, 0x70, 0x63, 0x73, 0x42, 0x79, 0x50, 0x65, 0x65, 0x72, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, + 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, + 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, + 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x3d, 0x0a, 0x0f, 0x52, + 0x70, 0x63, 0x73, 0x42, 0x79, 0x50, 0x65, 0x65, 0x72, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, + 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, + 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x73, 0x0a, 0x11, 0x52, 0x70, + 0x63, 0x73, 0x42, 0x79, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, + 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, + 0x79, 0x12, 0x48, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x32, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, + 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x70, 0x63, 0x73, 0x42, 0x79, + 0x50, 0x65, 0x65, 0x72, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, + 0x7a, 0x0a, 0x14, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x73, 0x42, 0x79, 0x50, 0x65, + 0x65, 0x72, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x4c, 0x0a, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x36, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, + 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, + 0x6e, 0x63, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x42, 0x79, 0x50, 0x65, 0x65, 0x72, + 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x29, 0x0a, 0x0c, 0x4d, + 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x49, + 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x54, 0x72, 0x61, 0x69, + 0x6c, 0x69, 0x6e, 0x67, 0x10, 0x01, 0x22, 0x25, 0x0a, 0x23, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, + 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x41, 0x63, 0x63, 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74, 0x65, + 0x64, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x86, 0x09, + 0x0a, 0x24, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x41, 0x63, + 0x63, 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x8e, 0x01, 0x0a, 0x1a, 0x6e, 0x75, 0x6d, 0x5f, 0x72, + 0x70, 0x63, 0x73, 0x5f, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x5f, 0x62, 0x79, 0x5f, 0x6d, + 0x65, 0x74, 0x68, 0x6f, 0x64, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x4e, 0x2e, 0x67, 0x72, + 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, + 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x41, 0x63, 0x63, 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74, + 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, + 0x4e, 0x75, 0x6d, 0x52, 0x70, 0x63, 0x73, 0x53, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x42, 0x79, + 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x42, 0x02, 0x18, 0x01, 0x52, + 0x16, 0x6e, 0x75, 0x6d, 0x52, 0x70, 0x63, 0x73, 0x53, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x42, + 0x79, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x12, 0x94, 0x01, 0x0a, 0x1c, 0x6e, 0x75, 0x6d, 0x5f, + 0x72, 0x70, 0x63, 0x73, 0x5f, 0x73, 0x75, 0x63, 0x63, 0x65, 0x65, 0x64, 0x65, 0x64, 0x5f, 0x62, + 0x79, 0x5f, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x50, + 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4c, 0x6f, + 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x41, 0x63, 0x63, 0x75, 0x6d, 0x75, + 0x6c, 0x61, 0x74, 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x2e, 0x4e, 0x75, 0x6d, 0x52, 0x70, 0x63, 0x73, 0x53, 0x75, 0x63, 0x63, 0x65, 0x65, + 0x64, 0x65, 0x64, 0x42, 0x79, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x42, 0x02, 0x18, 0x01, 0x52, 0x18, 0x6e, 0x75, 0x6d, 0x52, 0x70, 0x63, 0x73, 0x53, 0x75, 0x63, + 0x63, 0x65, 0x65, 0x64, 0x65, 0x64, 0x42, 0x79, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x12, 0x8b, + 0x01, 0x0a, 0x19, 0x6e, 0x75, 0x6d, 0x5f, 0x72, 0x70, 0x63, 0x73, 0x5f, 0x66, 0x61, 0x69, 0x6c, + 0x65, 0x64, 0x5f, 0x62, 0x79, 0x5f, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x18, 0x03, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x4d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, + 0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x41, 0x63, + 0x63, 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x4e, 0x75, 0x6d, 0x52, 0x70, 0x63, 0x73, 0x46, 0x61, + 0x69, 0x6c, 0x65, 0x64, 0x42, 0x79, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x42, 0x02, 0x18, 0x01, 0x52, 0x15, 0x6e, 0x75, 0x6d, 0x52, 0x70, 0x63, 0x73, 0x46, 0x61, + 0x69, 0x6c, 0x65, 0x64, 0x42, 0x79, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x12, 0x70, 0x0a, 0x10, + 0x73, 0x74, 0x61, 0x74, 0x73, 0x5f, 0x70, 0x65, 0x72, 0x5f, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, + 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x46, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, + 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, + 0x65, 0x72, 0x41, 0x63, 0x63, 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x53, 0x74, 0x61, + 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x73, + 0x50, 0x65, 0x72, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0e, + 0x73, 0x74, 0x61, 0x74, 0x73, 0x50, 0x65, 0x72, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x1a, 0x49, + 0x0a, 0x1b, 0x4e, 0x75, 0x6d, 0x52, 0x70, 0x63, 0x73, 0x53, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, + 0x42, 0x79, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, + 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, + 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x4b, 0x0a, 0x1d, 0x4e, 0x75, 0x6d, + 0x52, 0x70, 0x63, 0x73, 0x53, 0x75, 0x63, 0x63, 0x65, 0x65, 0x64, 0x65, 0x64, 0x42, 0x79, 0x4d, + 0x65, 0x74, 0x68, 0x6f, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x48, 0x0a, 0x1a, 0x4e, 0x75, 0x6d, 0x52, 0x70, 0x63, + 0x73, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x42, 0x79, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, + 0x1a, 0xcf, 0x01, 0x0a, 0x0b, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x53, 0x74, 0x61, 0x74, 0x73, + 0x12, 0x21, 0x0a, 0x0c, 0x72, 0x70, 0x63, 0x73, 0x5f, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0b, 0x72, 0x70, 0x63, 0x73, 0x53, 0x74, 0x61, 0x72, + 0x74, 0x65, 0x64, 0x12, 0x62, 0x0a, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x18, 0x02, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x4a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, + 0x6e, 0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x41, + 0x63, 0x63, 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x53, 0x74, + 0x61, 0x74, 0x73, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, + 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x1a, 0x39, 0x0a, 0x0b, 0x52, 0x65, 0x73, 0x75, 0x6c, + 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x05, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, + 0x38, 0x01, 0x1a, 0x81, 0x01, 0x0a, 0x13, 0x53, 0x74, 0x61, 0x74, 0x73, 0x50, 0x65, 0x72, 0x4d, + 0x65, 0x74, 0x68, 0x6f, 0x64, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x54, 0x0a, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3e, 0x2e, 0x67, 0x72, + 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, + 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x41, 0x63, 0x63, 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74, + 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, + 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xe9, 0x02, 0x0a, 0x16, 0x43, 0x6c, 0x69, 0x65, 0x6e, + 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x42, 0x0a, 0x05, 0x74, 0x79, 0x70, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0e, + 0x32, 0x2c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, + 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x52, 0x70, 0x63, 0x54, 0x79, 0x70, 0x65, 0x52, 0x05, + 0x74, 0x79, 0x70, 0x65, 0x73, 0x12, 0x49, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, + 0x61, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, + 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x4d, 0x65, + 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x12, 0x1f, 0x0a, 0x0b, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x5f, 0x73, 0x65, 0x63, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x53, 0x65, + 0x63, 0x1a, 0x74, 0x0a, 0x08, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x40, 0x0a, + 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2c, 0x2e, 0x67, 0x72, + 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, + 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x2e, 0x52, 0x70, 0x63, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, + 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, + 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x29, 0x0a, 0x07, 0x52, 0x70, 0x63, 0x54, 0x79, + 0x70, 0x65, 0x12, 0x0e, 0x0a, 0x0a, 0x45, 0x4d, 0x50, 0x54, 0x59, 0x5f, 0x43, 0x41, 0x4c, 0x4c, + 0x10, 0x00, 0x12, 0x0e, 0x0a, 0x0a, 0x55, 0x4e, 0x41, 0x52, 0x59, 0x5f, 0x43, 0x41, 0x4c, 0x4c, + 0x10, 0x01, 0x22, 0x19, 0x0a, 0x17, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x1e, 0x0a, + 0x0a, 0x4d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x72, + 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x03, 0x72, 0x73, 0x73, 0x22, 0x8b, 0x03, + 0x0a, 0x0e, 0x54, 0x65, 0x73, 0x74, 0x4f, 0x72, 0x63, 0x61, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, + 0x12, 0x27, 0x0a, 0x0f, 0x63, 0x70, 0x75, 0x5f, 0x75, 0x74, 0x69, 0x6c, 0x69, 0x7a, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0e, 0x63, 0x70, 0x75, 0x55, 0x74, + 0x69, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x2d, 0x0a, 0x12, 0x6d, 0x65, 0x6d, + 0x6f, 0x72, 0x79, 0x5f, 0x75, 0x74, 0x69, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x11, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x55, 0x74, 0x69, + 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x50, 0x0a, 0x0c, 0x72, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x5f, 0x63, 0x6f, 0x73, 0x74, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2d, + 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x54, 0x65, + 0x73, 0x74, 0x4f, 0x72, 0x63, 0x61, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, 0x73, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0b, 0x72, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, 0x73, 0x74, 0x12, 0x4f, 0x0a, 0x0b, 0x75, 0x74, + 0x69, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x2d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x54, + 0x65, 0x73, 0x74, 0x4f, 0x72, 0x63, 0x61, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x55, 0x74, + 0x69, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0b, + 0x75, 0x74, 0x69, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x1a, 0x3e, 0x0a, 0x10, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, 0x73, 0x74, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, + 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, + 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, + 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x3e, 0x0a, 0x10, 0x55, + 0x74, 0x69, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, + 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, + 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, + 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xd1, 0x01, 0x0a, 0x0b, + 0x48, 0x6f, 0x6f, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3a, 0x0a, 0x07, 0x63, + 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x20, 0x2e, 0x67, + 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x48, 0x6f, 0x6f, 0x6b, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x52, 0x07, + 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x12, 0x2d, 0x0a, 0x13, 0x67, 0x72, 0x70, 0x63, 0x5f, + 0x63, 0x6f, 0x64, 0x65, 0x5f, 0x74, 0x6f, 0x5f, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x05, 0x52, 0x10, 0x67, 0x72, 0x70, 0x63, 0x43, 0x6f, 0x64, 0x65, 0x54, 0x6f, + 0x52, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x12, 0x36, 0x0a, 0x17, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x73, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x5f, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x15, 0x67, 0x72, 0x70, 0x63, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1f, + 0x0a, 0x0b, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x05, 0x52, 0x0a, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x50, 0x6f, 0x72, 0x74, 0x22, + 0x0e, 0x0a, 0x0c, 0x48, 0x6f, 0x6f, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2a, + 0x1f, 0x0a, 0x0b, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x54, 0x79, 0x70, 0x65, 0x12, 0x10, + 0x0a, 0x0c, 0x43, 0x4f, 0x4d, 0x50, 0x52, 0x45, 0x53, 0x53, 0x41, 0x42, 0x4c, 0x45, 0x10, 0x00, + 0x2a, 0x6f, 0x0a, 0x0f, 0x47, 0x72, 0x70, 0x63, 0x6c, 0x62, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x54, + 0x79, 0x70, 0x65, 0x12, 0x1d, 0x0a, 0x19, 0x47, 0x52, 0x50, 0x43, 0x4c, 0x42, 0x5f, 0x52, 0x4f, + 0x55, 0x54, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, + 0x10, 0x00, 0x12, 0x1e, 0x0a, 0x1a, 0x47, 0x52, 0x50, 0x43, 0x4c, 0x42, 0x5f, 0x52, 0x4f, 0x55, + 0x54, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x46, 0x41, 0x4c, 0x4c, 0x42, 0x41, 0x43, 0x4b, + 0x10, 0x01, 0x12, 0x1d, 0x0a, 0x19, 0x47, 0x52, 0x50, 0x43, 0x4c, 0x42, 0x5f, 0x52, 0x4f, 0x55, + 0x54, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x41, 0x43, 0x4b, 0x45, 0x4e, 0x44, 0x10, + 0x02, 0x2a, 0x35, 0x0a, 0x12, 0x48, 0x6f, 0x6f, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x12, 0x09, 0x0a, 0x05, 0x53, 0x54, 0x41, 0x52, 0x54, + 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x53, 0x54, 0x4f, 0x50, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, + 0x52, 0x45, 0x54, 0x55, 0x52, 0x4e, 0x10, 0x02, 0x42, 0x1d, 0x0a, 0x1b, 0x69, 0x6f, 0x2e, 0x67, + 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x69, 0x6e, 0x74, 0x65, + 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_grpc_testing_messages_proto_rawDescOnce sync.Once + file_grpc_testing_messages_proto_rawDescData = file_grpc_testing_messages_proto_rawDesc +) + +func file_grpc_testing_messages_proto_rawDescGZIP() []byte { + file_grpc_testing_messages_proto_rawDescOnce.Do(func() { + file_grpc_testing_messages_proto_rawDescData = protoimpl.X.CompressGZIP(file_grpc_testing_messages_proto_rawDescData) + }) + return file_grpc_testing_messages_proto_rawDescData +} + +var file_grpc_testing_messages_proto_enumTypes = make([]protoimpl.EnumInfo, 5) +var file_grpc_testing_messages_proto_msgTypes = make([]protoimpl.MessageInfo, 39) +var file_grpc_testing_messages_proto_goTypes = []interface{}{ + (PayloadType)(0), // 0: grpc.testing.PayloadType + (GrpclbRouteType)(0), // 1: grpc.testing.GrpclbRouteType + (HookRequestCommand)(0), // 2: grpc.testing.HookRequestCommand + (LoadBalancerStatsResponse_MetadataType)(0), // 3: grpc.testing.LoadBalancerStatsResponse.MetadataType + (ClientConfigureRequest_RpcType)(0), // 4: grpc.testing.ClientConfigureRequest.RpcType + (*BoolValue)(nil), // 5: grpc.testing.BoolValue + (*Payload)(nil), // 6: grpc.testing.Payload + (*EchoStatus)(nil), // 7: grpc.testing.EchoStatus + (*SimpleRequest)(nil), // 8: grpc.testing.SimpleRequest + (*SimpleResponse)(nil), // 9: grpc.testing.SimpleResponse + (*StreamingInputCallRequest)(nil), // 10: grpc.testing.StreamingInputCallRequest + (*StreamingInputCallResponse)(nil), // 11: grpc.testing.StreamingInputCallResponse + (*ResponseParameters)(nil), // 12: grpc.testing.ResponseParameters + (*StreamingOutputCallRequest)(nil), // 13: grpc.testing.StreamingOutputCallRequest + (*StreamingOutputCallResponse)(nil), // 14: grpc.testing.StreamingOutputCallResponse + (*ReconnectParams)(nil), // 15: grpc.testing.ReconnectParams + (*ReconnectInfo)(nil), // 16: grpc.testing.ReconnectInfo + (*LoadBalancerStatsRequest)(nil), // 17: grpc.testing.LoadBalancerStatsRequest + (*LoadBalancerStatsResponse)(nil), // 18: grpc.testing.LoadBalancerStatsResponse + (*LoadBalancerAccumulatedStatsRequest)(nil), // 19: grpc.testing.LoadBalancerAccumulatedStatsRequest + (*LoadBalancerAccumulatedStatsResponse)(nil), // 20: grpc.testing.LoadBalancerAccumulatedStatsResponse + (*ClientConfigureRequest)(nil), // 21: grpc.testing.ClientConfigureRequest + (*ClientConfigureResponse)(nil), // 22: grpc.testing.ClientConfigureResponse + (*MemorySize)(nil), // 23: grpc.testing.MemorySize + (*TestOrcaReport)(nil), // 24: grpc.testing.TestOrcaReport + (*HookRequest)(nil), // 25: grpc.testing.HookRequest + (*HookResponse)(nil), // 26: grpc.testing.HookResponse + (*LoadBalancerStatsResponse_MetadataEntry)(nil), // 27: grpc.testing.LoadBalancerStatsResponse.MetadataEntry + (*LoadBalancerStatsResponse_RpcMetadata)(nil), // 28: grpc.testing.LoadBalancerStatsResponse.RpcMetadata + (*LoadBalancerStatsResponse_MetadataByPeer)(nil), // 29: grpc.testing.LoadBalancerStatsResponse.MetadataByPeer + (*LoadBalancerStatsResponse_RpcsByPeer)(nil), // 30: grpc.testing.LoadBalancerStatsResponse.RpcsByPeer + nil, // 31: grpc.testing.LoadBalancerStatsResponse.RpcsByPeerEntry + nil, // 32: grpc.testing.LoadBalancerStatsResponse.RpcsByMethodEntry + nil, // 33: grpc.testing.LoadBalancerStatsResponse.MetadatasByPeerEntry + nil, // 34: grpc.testing.LoadBalancerStatsResponse.RpcsByPeer.RpcsByPeerEntry + nil, // 35: grpc.testing.LoadBalancerAccumulatedStatsResponse.NumRpcsStartedByMethodEntry + nil, // 36: grpc.testing.LoadBalancerAccumulatedStatsResponse.NumRpcsSucceededByMethodEntry + nil, // 37: grpc.testing.LoadBalancerAccumulatedStatsResponse.NumRpcsFailedByMethodEntry + (*LoadBalancerAccumulatedStatsResponse_MethodStats)(nil), // 38: grpc.testing.LoadBalancerAccumulatedStatsResponse.MethodStats + nil, // 39: grpc.testing.LoadBalancerAccumulatedStatsResponse.StatsPerMethodEntry + nil, // 40: grpc.testing.LoadBalancerAccumulatedStatsResponse.MethodStats.ResultEntry + (*ClientConfigureRequest_Metadata)(nil), // 41: grpc.testing.ClientConfigureRequest.Metadata + nil, // 42: grpc.testing.TestOrcaReport.RequestCostEntry + nil, // 43: grpc.testing.TestOrcaReport.UtilizationEntry +} +var file_grpc_testing_messages_proto_depIdxs = []int32{ + 0, // 0: grpc.testing.Payload.type:type_name -> grpc.testing.PayloadType + 0, // 1: grpc.testing.SimpleRequest.response_type:type_name -> grpc.testing.PayloadType + 6, // 2: grpc.testing.SimpleRequest.payload:type_name -> grpc.testing.Payload + 5, // 3: grpc.testing.SimpleRequest.response_compressed:type_name -> grpc.testing.BoolValue + 7, // 4: grpc.testing.SimpleRequest.response_status:type_name -> grpc.testing.EchoStatus + 5, // 5: grpc.testing.SimpleRequest.expect_compressed:type_name -> grpc.testing.BoolValue + 24, // 6: grpc.testing.SimpleRequest.orca_per_query_report:type_name -> grpc.testing.TestOrcaReport + 6, // 7: grpc.testing.SimpleResponse.payload:type_name -> grpc.testing.Payload + 1, // 8: grpc.testing.SimpleResponse.grpclb_route_type:type_name -> grpc.testing.GrpclbRouteType + 6, // 9: grpc.testing.StreamingInputCallRequest.payload:type_name -> grpc.testing.Payload + 5, // 10: grpc.testing.StreamingInputCallRequest.expect_compressed:type_name -> grpc.testing.BoolValue + 5, // 11: grpc.testing.ResponseParameters.compressed:type_name -> grpc.testing.BoolValue + 0, // 12: grpc.testing.StreamingOutputCallRequest.response_type:type_name -> grpc.testing.PayloadType + 12, // 13: grpc.testing.StreamingOutputCallRequest.response_parameters:type_name -> grpc.testing.ResponseParameters + 6, // 14: grpc.testing.StreamingOutputCallRequest.payload:type_name -> grpc.testing.Payload + 7, // 15: grpc.testing.StreamingOutputCallRequest.response_status:type_name -> grpc.testing.EchoStatus + 24, // 16: grpc.testing.StreamingOutputCallRequest.orca_oob_report:type_name -> grpc.testing.TestOrcaReport + 6, // 17: grpc.testing.StreamingOutputCallResponse.payload:type_name -> grpc.testing.Payload + 31, // 18: grpc.testing.LoadBalancerStatsResponse.rpcs_by_peer:type_name -> grpc.testing.LoadBalancerStatsResponse.RpcsByPeerEntry + 32, // 19: grpc.testing.LoadBalancerStatsResponse.rpcs_by_method:type_name -> grpc.testing.LoadBalancerStatsResponse.RpcsByMethodEntry + 33, // 20: grpc.testing.LoadBalancerStatsResponse.metadatas_by_peer:type_name -> grpc.testing.LoadBalancerStatsResponse.MetadatasByPeerEntry + 35, // 21: grpc.testing.LoadBalancerAccumulatedStatsResponse.num_rpcs_started_by_method:type_name -> grpc.testing.LoadBalancerAccumulatedStatsResponse.NumRpcsStartedByMethodEntry + 36, // 22: grpc.testing.LoadBalancerAccumulatedStatsResponse.num_rpcs_succeeded_by_method:type_name -> grpc.testing.LoadBalancerAccumulatedStatsResponse.NumRpcsSucceededByMethodEntry + 37, // 23: grpc.testing.LoadBalancerAccumulatedStatsResponse.num_rpcs_failed_by_method:type_name -> grpc.testing.LoadBalancerAccumulatedStatsResponse.NumRpcsFailedByMethodEntry + 39, // 24: grpc.testing.LoadBalancerAccumulatedStatsResponse.stats_per_method:type_name -> grpc.testing.LoadBalancerAccumulatedStatsResponse.StatsPerMethodEntry + 4, // 25: grpc.testing.ClientConfigureRequest.types:type_name -> grpc.testing.ClientConfigureRequest.RpcType + 41, // 26: grpc.testing.ClientConfigureRequest.metadata:type_name -> grpc.testing.ClientConfigureRequest.Metadata + 42, // 27: grpc.testing.TestOrcaReport.request_cost:type_name -> grpc.testing.TestOrcaReport.RequestCostEntry + 43, // 28: grpc.testing.TestOrcaReport.utilization:type_name -> grpc.testing.TestOrcaReport.UtilizationEntry + 2, // 29: grpc.testing.HookRequest.command:type_name -> grpc.testing.HookRequestCommand + 3, // 30: grpc.testing.LoadBalancerStatsResponse.MetadataEntry.type:type_name -> grpc.testing.LoadBalancerStatsResponse.MetadataType + 27, // 31: grpc.testing.LoadBalancerStatsResponse.RpcMetadata.metadata:type_name -> grpc.testing.LoadBalancerStatsResponse.MetadataEntry + 28, // 32: grpc.testing.LoadBalancerStatsResponse.MetadataByPeer.rpc_metadata:type_name -> grpc.testing.LoadBalancerStatsResponse.RpcMetadata + 34, // 33: grpc.testing.LoadBalancerStatsResponse.RpcsByPeer.rpcs_by_peer:type_name -> grpc.testing.LoadBalancerStatsResponse.RpcsByPeer.RpcsByPeerEntry + 30, // 34: grpc.testing.LoadBalancerStatsResponse.RpcsByMethodEntry.value:type_name -> grpc.testing.LoadBalancerStatsResponse.RpcsByPeer + 29, // 35: grpc.testing.LoadBalancerStatsResponse.MetadatasByPeerEntry.value:type_name -> grpc.testing.LoadBalancerStatsResponse.MetadataByPeer + 40, // 36: grpc.testing.LoadBalancerAccumulatedStatsResponse.MethodStats.result:type_name -> grpc.testing.LoadBalancerAccumulatedStatsResponse.MethodStats.ResultEntry + 38, // 37: grpc.testing.LoadBalancerAccumulatedStatsResponse.StatsPerMethodEntry.value:type_name -> grpc.testing.LoadBalancerAccumulatedStatsResponse.MethodStats + 4, // 38: grpc.testing.ClientConfigureRequest.Metadata.type:type_name -> grpc.testing.ClientConfigureRequest.RpcType + 39, // [39:39] is the sub-list for method output_type + 39, // [39:39] is the sub-list for method input_type + 39, // [39:39] is the sub-list for extension type_name + 39, // [39:39] is the sub-list for extension extendee + 0, // [0:39] is the sub-list for field type_name +} + +func init() { file_grpc_testing_messages_proto_init() } +func file_grpc_testing_messages_proto_init() { + if File_grpc_testing_messages_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_grpc_testing_messages_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*BoolValue); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_messages_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Payload); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_messages_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*EchoStatus); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_messages_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SimpleRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_messages_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SimpleResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_messages_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*StreamingInputCallRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_messages_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*StreamingInputCallResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_messages_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ResponseParameters); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_messages_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*StreamingOutputCallRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_messages_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*StreamingOutputCallResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_messages_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ReconnectParams); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_messages_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ReconnectInfo); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_messages_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*LoadBalancerStatsRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_messages_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*LoadBalancerStatsResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_messages_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*LoadBalancerAccumulatedStatsRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_messages_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*LoadBalancerAccumulatedStatsResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_messages_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ClientConfigureRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_messages_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ClientConfigureResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_messages_proto_msgTypes[18].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*MemorySize); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_messages_proto_msgTypes[19].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*TestOrcaReport); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_messages_proto_msgTypes[20].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*HookRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_messages_proto_msgTypes[21].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*HookResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_messages_proto_msgTypes[22].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*LoadBalancerStatsResponse_MetadataEntry); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_messages_proto_msgTypes[23].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*LoadBalancerStatsResponse_RpcMetadata); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_messages_proto_msgTypes[24].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*LoadBalancerStatsResponse_MetadataByPeer); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_messages_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*LoadBalancerStatsResponse_RpcsByPeer); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_messages_proto_msgTypes[33].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*LoadBalancerAccumulatedStatsResponse_MethodStats); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_messages_proto_msgTypes[36].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ClientConfigureRequest_Metadata); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_grpc_testing_messages_proto_rawDesc, + NumEnums: 5, + NumMessages: 39, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_grpc_testing_messages_proto_goTypes, + DependencyIndexes: file_grpc_testing_messages_proto_depIdxs, + EnumInfos: file_grpc_testing_messages_proto_enumTypes, + MessageInfos: file_grpc_testing_messages_proto_msgTypes, + }.Build() + File_grpc_testing_messages_proto = out.File + file_grpc_testing_messages_proto_rawDesc = nil + file_grpc_testing_messages_proto_goTypes = nil + file_grpc_testing_messages_proto_depIdxs = nil +} diff --git a/interop/grpc_testing/payloads.pb.go b/interop/grpc_testing/payloads.pb.go new file mode 100644 index 000000000000..684fd10273f1 --- /dev/null +++ b/interop/grpc_testing/payloads.pb.go @@ -0,0 +1,425 @@ +// Copyright 2015 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.31.0 +// protoc v4.22.0 +// source: grpc/testing/payloads.proto + +package grpc_testing + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type ByteBufferParams struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + ReqSize int32 `protobuf:"varint,1,opt,name=req_size,json=reqSize,proto3" json:"req_size,omitempty"` + RespSize int32 `protobuf:"varint,2,opt,name=resp_size,json=respSize,proto3" json:"resp_size,omitempty"` +} + +func (x *ByteBufferParams) Reset() { + *x = ByteBufferParams{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_payloads_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ByteBufferParams) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ByteBufferParams) ProtoMessage() {} + +func (x *ByteBufferParams) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_payloads_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ByteBufferParams.ProtoReflect.Descriptor instead. +func (*ByteBufferParams) Descriptor() ([]byte, []int) { + return file_grpc_testing_payloads_proto_rawDescGZIP(), []int{0} +} + +func (x *ByteBufferParams) GetReqSize() int32 { + if x != nil { + return x.ReqSize + } + return 0 +} + +func (x *ByteBufferParams) GetRespSize() int32 { + if x != nil { + return x.RespSize + } + return 0 +} + +type SimpleProtoParams struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + ReqSize int32 `protobuf:"varint,1,opt,name=req_size,json=reqSize,proto3" json:"req_size,omitempty"` + RespSize int32 `protobuf:"varint,2,opt,name=resp_size,json=respSize,proto3" json:"resp_size,omitempty"` +} + +func (x *SimpleProtoParams) Reset() { + *x = SimpleProtoParams{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_payloads_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SimpleProtoParams) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SimpleProtoParams) ProtoMessage() {} + +func (x *SimpleProtoParams) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_payloads_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SimpleProtoParams.ProtoReflect.Descriptor instead. +func (*SimpleProtoParams) Descriptor() ([]byte, []int) { + return file_grpc_testing_payloads_proto_rawDescGZIP(), []int{1} +} + +func (x *SimpleProtoParams) GetReqSize() int32 { + if x != nil { + return x.ReqSize + } + return 0 +} + +func (x *SimpleProtoParams) GetRespSize() int32 { + if x != nil { + return x.RespSize + } + return 0 +} + +type ComplexProtoParams struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *ComplexProtoParams) Reset() { + *x = ComplexProtoParams{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_payloads_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ComplexProtoParams) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ComplexProtoParams) ProtoMessage() {} + +func (x *ComplexProtoParams) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_payloads_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ComplexProtoParams.ProtoReflect.Descriptor instead. +func (*ComplexProtoParams) Descriptor() ([]byte, []int) { + return file_grpc_testing_payloads_proto_rawDescGZIP(), []int{2} +} + +type PayloadConfig struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Types that are assignable to Payload: + // + // *PayloadConfig_BytebufParams + // *PayloadConfig_SimpleParams + // *PayloadConfig_ComplexParams + Payload isPayloadConfig_Payload `protobuf_oneof:"payload"` +} + +func (x *PayloadConfig) Reset() { + *x = PayloadConfig{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_payloads_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PayloadConfig) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PayloadConfig) ProtoMessage() {} + +func (x *PayloadConfig) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_payloads_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PayloadConfig.ProtoReflect.Descriptor instead. +func (*PayloadConfig) Descriptor() ([]byte, []int) { + return file_grpc_testing_payloads_proto_rawDescGZIP(), []int{3} +} + +func (m *PayloadConfig) GetPayload() isPayloadConfig_Payload { + if m != nil { + return m.Payload + } + return nil +} + +func (x *PayloadConfig) GetBytebufParams() *ByteBufferParams { + if x, ok := x.GetPayload().(*PayloadConfig_BytebufParams); ok { + return x.BytebufParams + } + return nil +} + +func (x *PayloadConfig) GetSimpleParams() *SimpleProtoParams { + if x, ok := x.GetPayload().(*PayloadConfig_SimpleParams); ok { + return x.SimpleParams + } + return nil +} + +func (x *PayloadConfig) GetComplexParams() *ComplexProtoParams { + if x, ok := x.GetPayload().(*PayloadConfig_ComplexParams); ok { + return x.ComplexParams + } + return nil +} + +type isPayloadConfig_Payload interface { + isPayloadConfig_Payload() +} + +type PayloadConfig_BytebufParams struct { + BytebufParams *ByteBufferParams `protobuf:"bytes,1,opt,name=bytebuf_params,json=bytebufParams,proto3,oneof"` +} + +type PayloadConfig_SimpleParams struct { + SimpleParams *SimpleProtoParams `protobuf:"bytes,2,opt,name=simple_params,json=simpleParams,proto3,oneof"` +} + +type PayloadConfig_ComplexParams struct { + ComplexParams *ComplexProtoParams `protobuf:"bytes,3,opt,name=complex_params,json=complexParams,proto3,oneof"` +} + +func (*PayloadConfig_BytebufParams) isPayloadConfig_Payload() {} + +func (*PayloadConfig_SimpleParams) isPayloadConfig_Payload() {} + +func (*PayloadConfig_ComplexParams) isPayloadConfig_Payload() {} + +var File_grpc_testing_payloads_proto protoreflect.FileDescriptor + +var file_grpc_testing_payloads_proto_rawDesc = []byte{ + 0x0a, 0x1b, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x70, + 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x67, + 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x22, 0x4a, 0x0a, 0x10, 0x42, + 0x79, 0x74, 0x65, 0x42, 0x75, 0x66, 0x66, 0x65, 0x72, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, + 0x19, 0x0a, 0x08, 0x72, 0x65, 0x71, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x05, 0x52, 0x07, 0x72, 0x65, 0x71, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x65, + 0x73, 0x70, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x72, + 0x65, 0x73, 0x70, 0x53, 0x69, 0x7a, 0x65, 0x22, 0x4b, 0x0a, 0x11, 0x53, 0x69, 0x6d, 0x70, 0x6c, + 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x19, 0x0a, 0x08, + 0x72, 0x65, 0x71, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, + 0x72, 0x65, 0x71, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x65, 0x73, 0x70, 0x5f, + 0x73, 0x69, 0x7a, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x72, 0x65, 0x73, 0x70, + 0x53, 0x69, 0x7a, 0x65, 0x22, 0x14, 0x0a, 0x12, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x78, 0x50, + 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x22, 0xf6, 0x01, 0x0a, 0x0d, 0x50, + 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x47, 0x0a, 0x0e, + 0x62, 0x79, 0x74, 0x65, 0x62, 0x75, 0x66, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, + 0x69, 0x6e, 0x67, 0x2e, 0x42, 0x79, 0x74, 0x65, 0x42, 0x75, 0x66, 0x66, 0x65, 0x72, 0x50, 0x61, + 0x72, 0x61, 0x6d, 0x73, 0x48, 0x00, 0x52, 0x0d, 0x62, 0x79, 0x74, 0x65, 0x62, 0x75, 0x66, 0x50, + 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x46, 0x0a, 0x0d, 0x73, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x5f, + 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x67, + 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x69, 0x6d, 0x70, + 0x6c, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x48, 0x00, 0x52, + 0x0c, 0x73, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x12, 0x49, 0x0a, + 0x0e, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x78, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, + 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x78, 0x50, 0x72, 0x6f, 0x74, + 0x6f, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x48, 0x00, 0x52, 0x0d, 0x63, 0x6f, 0x6d, 0x70, 0x6c, + 0x65, 0x78, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, 0x42, 0x09, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, + 0x6f, 0x61, 0x64, 0x42, 0x22, 0x0a, 0x0f, 0x69, 0x6f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, + 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x42, 0x0d, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x73, + 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_grpc_testing_payloads_proto_rawDescOnce sync.Once + file_grpc_testing_payloads_proto_rawDescData = file_grpc_testing_payloads_proto_rawDesc +) + +func file_grpc_testing_payloads_proto_rawDescGZIP() []byte { + file_grpc_testing_payloads_proto_rawDescOnce.Do(func() { + file_grpc_testing_payloads_proto_rawDescData = protoimpl.X.CompressGZIP(file_grpc_testing_payloads_proto_rawDescData) + }) + return file_grpc_testing_payloads_proto_rawDescData +} + +var file_grpc_testing_payloads_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_grpc_testing_payloads_proto_goTypes = []interface{}{ + (*ByteBufferParams)(nil), // 0: grpc.testing.ByteBufferParams + (*SimpleProtoParams)(nil), // 1: grpc.testing.SimpleProtoParams + (*ComplexProtoParams)(nil), // 2: grpc.testing.ComplexProtoParams + (*PayloadConfig)(nil), // 3: grpc.testing.PayloadConfig +} +var file_grpc_testing_payloads_proto_depIdxs = []int32{ + 0, // 0: grpc.testing.PayloadConfig.bytebuf_params:type_name -> grpc.testing.ByteBufferParams + 1, // 1: grpc.testing.PayloadConfig.simple_params:type_name -> grpc.testing.SimpleProtoParams + 2, // 2: grpc.testing.PayloadConfig.complex_params:type_name -> grpc.testing.ComplexProtoParams + 3, // [3:3] is the sub-list for method output_type + 3, // [3:3] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name +} + +func init() { file_grpc_testing_payloads_proto_init() } +func file_grpc_testing_payloads_proto_init() { + if File_grpc_testing_payloads_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_grpc_testing_payloads_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ByteBufferParams); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_payloads_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SimpleProtoParams); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_payloads_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ComplexProtoParams); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_payloads_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PayloadConfig); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + file_grpc_testing_payloads_proto_msgTypes[3].OneofWrappers = []interface{}{ + (*PayloadConfig_BytebufParams)(nil), + (*PayloadConfig_SimpleParams)(nil), + (*PayloadConfig_ComplexParams)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_grpc_testing_payloads_proto_rawDesc, + NumEnums: 0, + NumMessages: 4, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_grpc_testing_payloads_proto_goTypes, + DependencyIndexes: file_grpc_testing_payloads_proto_depIdxs, + MessageInfos: file_grpc_testing_payloads_proto_msgTypes, + }.Build() + File_grpc_testing_payloads_proto = out.File + file_grpc_testing_payloads_proto_rawDesc = nil + file_grpc_testing_payloads_proto_goTypes = nil + file_grpc_testing_payloads_proto_depIdxs = nil +} diff --git a/interop/grpc_testing/report_qps_scenario_service.pb.go b/interop/grpc_testing/report_qps_scenario_service.pb.go new file mode 100644 index 000000000000..366067714814 --- /dev/null +++ b/interop/grpc_testing/report_qps_scenario_service.pb.go @@ -0,0 +1,97 @@ +// Copyright 2015 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// An integration test service that covers all the method signature permutations +// of unary/streaming requests/responses. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.31.0 +// protoc v4.22.0 +// source: grpc/testing/report_qps_scenario_service.proto + +package grpc_testing + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +var File_grpc_testing_report_qps_scenario_service_proto protoreflect.FileDescriptor + +var file_grpc_testing_report_qps_scenario_service_proto_rawDesc = []byte{ + 0x0a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x72, + 0x65, 0x70, 0x6f, 0x72, 0x74, 0x5f, 0x71, 0x70, 0x73, 0x5f, 0x73, 0x63, 0x65, 0x6e, 0x61, 0x72, + 0x69, 0x6f, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x12, 0x0c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x1a, 0x1a, + 0x67, 0x72, 0x70, 0x63, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x32, 0x5e, 0x0a, 0x18, 0x52, 0x65, + 0x70, 0x6f, 0x72, 0x74, 0x51, 0x70, 0x73, 0x53, 0x63, 0x65, 0x6e, 0x61, 0x72, 0x69, 0x6f, 0x53, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x42, 0x0a, 0x0e, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, + 0x53, 0x63, 0x65, 0x6e, 0x61, 0x72, 0x69, 0x6f, 0x12, 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, + 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x63, 0x65, 0x6e, 0x61, 0x72, 0x69, 0x6f, + 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x1a, 0x12, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, + 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x56, 0x6f, 0x69, 0x64, 0x42, 0x32, 0x0a, 0x0f, 0x69, 0x6f, + 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x42, 0x1d, 0x52, + 0x65, 0x70, 0x6f, 0x72, 0x74, 0x51, 0x70, 0x73, 0x53, 0x63, 0x65, 0x6e, 0x61, 0x72, 0x69, 0x6f, + 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x62, 0x06, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var file_grpc_testing_report_qps_scenario_service_proto_goTypes = []interface{}{ + (*ScenarioResult)(nil), // 0: grpc.testing.ScenarioResult + (*Void)(nil), // 1: grpc.testing.Void +} +var file_grpc_testing_report_qps_scenario_service_proto_depIdxs = []int32{ + 0, // 0: grpc.testing.ReportQpsScenarioService.ReportScenario:input_type -> grpc.testing.ScenarioResult + 1, // 1: grpc.testing.ReportQpsScenarioService.ReportScenario:output_type -> grpc.testing.Void + 1, // [1:2] is the sub-list for method output_type + 0, // [0:1] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_grpc_testing_report_qps_scenario_service_proto_init() } +func file_grpc_testing_report_qps_scenario_service_proto_init() { + if File_grpc_testing_report_qps_scenario_service_proto != nil { + return + } + file_grpc_testing_control_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_grpc_testing_report_qps_scenario_service_proto_rawDesc, + NumEnums: 0, + NumMessages: 0, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_grpc_testing_report_qps_scenario_service_proto_goTypes, + DependencyIndexes: file_grpc_testing_report_qps_scenario_service_proto_depIdxs, + }.Build() + File_grpc_testing_report_qps_scenario_service_proto = out.File + file_grpc_testing_report_qps_scenario_service_proto_rawDesc = nil + file_grpc_testing_report_qps_scenario_service_proto_goTypes = nil + file_grpc_testing_report_qps_scenario_service_proto_depIdxs = nil +} diff --git a/interop/grpc_testing/report_qps_scenario_service_grpc.pb.go b/interop/grpc_testing/report_qps_scenario_service_grpc.pb.go new file mode 100644 index 000000000000..33392bc6ae33 --- /dev/null +++ b/interop/grpc_testing/report_qps_scenario_service_grpc.pb.go @@ -0,0 +1,129 @@ +// Copyright 2015 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// An integration test service that covers all the method signature permutations +// of unary/streaming requests/responses. + +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.3.0 +// - protoc v4.22.0 +// source: grpc/testing/report_qps_scenario_service.proto + +package grpc_testing + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 + +const ( + ReportQpsScenarioService_ReportScenario_FullMethodName = "/grpc.testing.ReportQpsScenarioService/ReportScenario" +) + +// ReportQpsScenarioServiceClient is the client API for ReportQpsScenarioService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type ReportQpsScenarioServiceClient interface { + // Report results of a QPS test benchmark scenario. + ReportScenario(ctx context.Context, in *ScenarioResult, opts ...grpc.CallOption) (*Void, error) +} + +type reportQpsScenarioServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewReportQpsScenarioServiceClient(cc grpc.ClientConnInterface) ReportQpsScenarioServiceClient { + return &reportQpsScenarioServiceClient{cc} +} + +func (c *reportQpsScenarioServiceClient) ReportScenario(ctx context.Context, in *ScenarioResult, opts ...grpc.CallOption) (*Void, error) { + out := new(Void) + err := c.cc.Invoke(ctx, ReportQpsScenarioService_ReportScenario_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// ReportQpsScenarioServiceServer is the server API for ReportQpsScenarioService service. +// All implementations must embed UnimplementedReportQpsScenarioServiceServer +// for forward compatibility +type ReportQpsScenarioServiceServer interface { + // Report results of a QPS test benchmark scenario. + ReportScenario(context.Context, *ScenarioResult) (*Void, error) + mustEmbedUnimplementedReportQpsScenarioServiceServer() +} + +// UnimplementedReportQpsScenarioServiceServer must be embedded to have forward compatible implementations. +type UnimplementedReportQpsScenarioServiceServer struct { +} + +func (UnimplementedReportQpsScenarioServiceServer) ReportScenario(context.Context, *ScenarioResult) (*Void, error) { + return nil, status.Errorf(codes.Unimplemented, "method ReportScenario not implemented") +} +func (UnimplementedReportQpsScenarioServiceServer) mustEmbedUnimplementedReportQpsScenarioServiceServer() { +} + +// UnsafeReportQpsScenarioServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to ReportQpsScenarioServiceServer will +// result in compilation errors. +type UnsafeReportQpsScenarioServiceServer interface { + mustEmbedUnimplementedReportQpsScenarioServiceServer() +} + +func RegisterReportQpsScenarioServiceServer(s grpc.ServiceRegistrar, srv ReportQpsScenarioServiceServer) { + s.RegisterService(&ReportQpsScenarioService_ServiceDesc, srv) +} + +func _ReportQpsScenarioService_ReportScenario_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ScenarioResult) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ReportQpsScenarioServiceServer).ReportScenario(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ReportQpsScenarioService_ReportScenario_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ReportQpsScenarioServiceServer).ReportScenario(ctx, req.(*ScenarioResult)) + } + return interceptor(ctx, in, info, handler) +} + +// ReportQpsScenarioService_ServiceDesc is the grpc.ServiceDesc for ReportQpsScenarioService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var ReportQpsScenarioService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.testing.ReportQpsScenarioService", + HandlerType: (*ReportQpsScenarioServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "ReportScenario", + Handler: _ReportQpsScenarioService_ReportScenario_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "grpc/testing/report_qps_scenario_service.proto", +} diff --git a/interop/grpc_testing/stats.pb.go b/interop/grpc_testing/stats.pb.go new file mode 100644 index 000000000000..aa6e9b3bc430 --- /dev/null +++ b/interop/grpc_testing/stats.pb.go @@ -0,0 +1,629 @@ +// Copyright 2015 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.31.0 +// protoc v4.22.0 +// source: grpc/testing/stats.proto + +package grpc_testing + +import ( + core "google.golang.org/grpc/interop/grpc_testing/core" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type ServerStats struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // wall clock time change in seconds since last reset + TimeElapsed float64 `protobuf:"fixed64,1,opt,name=time_elapsed,json=timeElapsed,proto3" json:"time_elapsed,omitempty"` + // change in user time (in seconds) used by the server since last reset + TimeUser float64 `protobuf:"fixed64,2,opt,name=time_user,json=timeUser,proto3" json:"time_user,omitempty"` + // change in server time (in seconds) used by the server process and all + // threads since last reset + TimeSystem float64 `protobuf:"fixed64,3,opt,name=time_system,json=timeSystem,proto3" json:"time_system,omitempty"` + // change in total cpu time of the server (data from proc/stat) + TotalCpuTime uint64 `protobuf:"varint,4,opt,name=total_cpu_time,json=totalCpuTime,proto3" json:"total_cpu_time,omitempty"` + // change in idle time of the server (data from proc/stat) + IdleCpuTime uint64 `protobuf:"varint,5,opt,name=idle_cpu_time,json=idleCpuTime,proto3" json:"idle_cpu_time,omitempty"` + // Number of polls called inside completion queue + CqPollCount uint64 `protobuf:"varint,6,opt,name=cq_poll_count,json=cqPollCount,proto3" json:"cq_poll_count,omitempty"` + // Core library stats + CoreStats *core.Stats `protobuf:"bytes,7,opt,name=core_stats,json=coreStats,proto3" json:"core_stats,omitempty"` +} + +func (x *ServerStats) Reset() { + *x = ServerStats{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_stats_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ServerStats) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ServerStats) ProtoMessage() {} + +func (x *ServerStats) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_stats_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ServerStats.ProtoReflect.Descriptor instead. +func (*ServerStats) Descriptor() ([]byte, []int) { + return file_grpc_testing_stats_proto_rawDescGZIP(), []int{0} +} + +func (x *ServerStats) GetTimeElapsed() float64 { + if x != nil { + return x.TimeElapsed + } + return 0 +} + +func (x *ServerStats) GetTimeUser() float64 { + if x != nil { + return x.TimeUser + } + return 0 +} + +func (x *ServerStats) GetTimeSystem() float64 { + if x != nil { + return x.TimeSystem + } + return 0 +} + +func (x *ServerStats) GetTotalCpuTime() uint64 { + if x != nil { + return x.TotalCpuTime + } + return 0 +} + +func (x *ServerStats) GetIdleCpuTime() uint64 { + if x != nil { + return x.IdleCpuTime + } + return 0 +} + +func (x *ServerStats) GetCqPollCount() uint64 { + if x != nil { + return x.CqPollCount + } + return 0 +} + +func (x *ServerStats) GetCoreStats() *core.Stats { + if x != nil { + return x.CoreStats + } + return nil +} + +// Histogram params based on grpc/support/histogram.c +type HistogramParams struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Resolution float64 `protobuf:"fixed64,1,opt,name=resolution,proto3" json:"resolution,omitempty"` // first bucket is [0, 1 + resolution) + MaxPossible float64 `protobuf:"fixed64,2,opt,name=max_possible,json=maxPossible,proto3" json:"max_possible,omitempty"` // use enough buckets to allow this value +} + +func (x *HistogramParams) Reset() { + *x = HistogramParams{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_stats_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *HistogramParams) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HistogramParams) ProtoMessage() {} + +func (x *HistogramParams) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_stats_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HistogramParams.ProtoReflect.Descriptor instead. +func (*HistogramParams) Descriptor() ([]byte, []int) { + return file_grpc_testing_stats_proto_rawDescGZIP(), []int{1} +} + +func (x *HistogramParams) GetResolution() float64 { + if x != nil { + return x.Resolution + } + return 0 +} + +func (x *HistogramParams) GetMaxPossible() float64 { + if x != nil { + return x.MaxPossible + } + return 0 +} + +// Histogram data based on grpc/support/histogram.c +type HistogramData struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Bucket []uint32 `protobuf:"varint,1,rep,packed,name=bucket,proto3" json:"bucket,omitempty"` + MinSeen float64 `protobuf:"fixed64,2,opt,name=min_seen,json=minSeen,proto3" json:"min_seen,omitempty"` + MaxSeen float64 `protobuf:"fixed64,3,opt,name=max_seen,json=maxSeen,proto3" json:"max_seen,omitempty"` + Sum float64 `protobuf:"fixed64,4,opt,name=sum,proto3" json:"sum,omitempty"` + SumOfSquares float64 `protobuf:"fixed64,5,opt,name=sum_of_squares,json=sumOfSquares,proto3" json:"sum_of_squares,omitempty"` + Count float64 `protobuf:"fixed64,6,opt,name=count,proto3" json:"count,omitempty"` +} + +func (x *HistogramData) Reset() { + *x = HistogramData{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_stats_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *HistogramData) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HistogramData) ProtoMessage() {} + +func (x *HistogramData) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_stats_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HistogramData.ProtoReflect.Descriptor instead. +func (*HistogramData) Descriptor() ([]byte, []int) { + return file_grpc_testing_stats_proto_rawDescGZIP(), []int{2} +} + +func (x *HistogramData) GetBucket() []uint32 { + if x != nil { + return x.Bucket + } + return nil +} + +func (x *HistogramData) GetMinSeen() float64 { + if x != nil { + return x.MinSeen + } + return 0 +} + +func (x *HistogramData) GetMaxSeen() float64 { + if x != nil { + return x.MaxSeen + } + return 0 +} + +func (x *HistogramData) GetSum() float64 { + if x != nil { + return x.Sum + } + return 0 +} + +func (x *HistogramData) GetSumOfSquares() float64 { + if x != nil { + return x.SumOfSquares + } + return 0 +} + +func (x *HistogramData) GetCount() float64 { + if x != nil { + return x.Count + } + return 0 +} + +type RequestResultCount struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + StatusCode int32 `protobuf:"varint,1,opt,name=status_code,json=statusCode,proto3" json:"status_code,omitempty"` + Count int64 `protobuf:"varint,2,opt,name=count,proto3" json:"count,omitempty"` +} + +func (x *RequestResultCount) Reset() { + *x = RequestResultCount{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_stats_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RequestResultCount) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RequestResultCount) ProtoMessage() {} + +func (x *RequestResultCount) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_stats_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RequestResultCount.ProtoReflect.Descriptor instead. +func (*RequestResultCount) Descriptor() ([]byte, []int) { + return file_grpc_testing_stats_proto_rawDescGZIP(), []int{3} +} + +func (x *RequestResultCount) GetStatusCode() int32 { + if x != nil { + return x.StatusCode + } + return 0 +} + +func (x *RequestResultCount) GetCount() int64 { + if x != nil { + return x.Count + } + return 0 +} + +type ClientStats struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Latency histogram. Data points are in nanoseconds. + Latencies *HistogramData `protobuf:"bytes,1,opt,name=latencies,proto3" json:"latencies,omitempty"` + // See ServerStats for details. + TimeElapsed float64 `protobuf:"fixed64,2,opt,name=time_elapsed,json=timeElapsed,proto3" json:"time_elapsed,omitempty"` + TimeUser float64 `protobuf:"fixed64,3,opt,name=time_user,json=timeUser,proto3" json:"time_user,omitempty"` + TimeSystem float64 `protobuf:"fixed64,4,opt,name=time_system,json=timeSystem,proto3" json:"time_system,omitempty"` + // Number of failed requests (one row per status code seen) + RequestResults []*RequestResultCount `protobuf:"bytes,5,rep,name=request_results,json=requestResults,proto3" json:"request_results,omitempty"` + // Number of polls called inside completion queue + CqPollCount uint64 `protobuf:"varint,6,opt,name=cq_poll_count,json=cqPollCount,proto3" json:"cq_poll_count,omitempty"` + // Core library stats + CoreStats *core.Stats `protobuf:"bytes,7,opt,name=core_stats,json=coreStats,proto3" json:"core_stats,omitempty"` +} + +func (x *ClientStats) Reset() { + *x = ClientStats{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_testing_stats_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ClientStats) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ClientStats) ProtoMessage() {} + +func (x *ClientStats) ProtoReflect() protoreflect.Message { + mi := &file_grpc_testing_stats_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ClientStats.ProtoReflect.Descriptor instead. +func (*ClientStats) Descriptor() ([]byte, []int) { + return file_grpc_testing_stats_proto_rawDescGZIP(), []int{4} +} + +func (x *ClientStats) GetLatencies() *HistogramData { + if x != nil { + return x.Latencies + } + return nil +} + +func (x *ClientStats) GetTimeElapsed() float64 { + if x != nil { + return x.TimeElapsed + } + return 0 +} + +func (x *ClientStats) GetTimeUser() float64 { + if x != nil { + return x.TimeUser + } + return 0 +} + +func (x *ClientStats) GetTimeSystem() float64 { + if x != nil { + return x.TimeSystem + } + return 0 +} + +func (x *ClientStats) GetRequestResults() []*RequestResultCount { + if x != nil { + return x.RequestResults + } + return nil +} + +func (x *ClientStats) GetCqPollCount() uint64 { + if x != nil { + return x.CqPollCount + } + return 0 +} + +func (x *ClientStats) GetCoreStats() *core.Stats { + if x != nil { + return x.CoreStats + } + return nil +} + +var File_grpc_testing_stats_proto protoreflect.FileDescriptor + +var file_grpc_testing_stats_proto_rawDesc = []byte{ + 0x0a, 0x18, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x73, + 0x74, 0x61, 0x74, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x1a, 0x15, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x63, + 0x6f, 0x72, 0x65, 0x2f, 0x73, 0x74, 0x61, 0x74, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, + 0x8d, 0x02, 0x0a, 0x0b, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, + 0x21, 0x0a, 0x0c, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x65, 0x6c, 0x61, 0x70, 0x73, 0x65, 0x64, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0b, 0x74, 0x69, 0x6d, 0x65, 0x45, 0x6c, 0x61, 0x70, 0x73, + 0x65, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x75, 0x73, 0x65, 0x72, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x08, 0x74, 0x69, 0x6d, 0x65, 0x55, 0x73, 0x65, 0x72, 0x12, + 0x1f, 0x0a, 0x0b, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x01, 0x52, 0x0a, 0x74, 0x69, 0x6d, 0x65, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, + 0x12, 0x24, 0x0a, 0x0e, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x63, 0x70, 0x75, 0x5f, 0x74, 0x69, + 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0c, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x43, + 0x70, 0x75, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x22, 0x0a, 0x0d, 0x69, 0x64, 0x6c, 0x65, 0x5f, 0x63, + 0x70, 0x75, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b, 0x69, + 0x64, 0x6c, 0x65, 0x43, 0x70, 0x75, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x22, 0x0a, 0x0d, 0x63, 0x71, + 0x5f, 0x70, 0x6f, 0x6c, 0x6c, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, + 0x04, 0x52, 0x0b, 0x63, 0x71, 0x50, 0x6f, 0x6c, 0x6c, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x2f, + 0x0a, 0x0a, 0x63, 0x6f, 0x72, 0x65, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x73, 0x18, 0x07, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, + 0x74, 0x61, 0x74, 0x73, 0x52, 0x09, 0x63, 0x6f, 0x72, 0x65, 0x53, 0x74, 0x61, 0x74, 0x73, 0x22, + 0x54, 0x0a, 0x0f, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x50, 0x61, 0x72, 0x61, + 0x6d, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x75, 0x74, 0x69, 0x6f, 0x6e, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0a, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x75, 0x74, 0x69, + 0x6f, 0x6e, 0x12, 0x21, 0x0a, 0x0c, 0x6d, 0x61, 0x78, 0x5f, 0x70, 0x6f, 0x73, 0x73, 0x69, 0x62, + 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0b, 0x6d, 0x61, 0x78, 0x50, 0x6f, 0x73, + 0x73, 0x69, 0x62, 0x6c, 0x65, 0x22, 0xab, 0x01, 0x0a, 0x0d, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x67, + 0x72, 0x61, 0x6d, 0x44, 0x61, 0x74, 0x61, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, + 0x74, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0d, 0x52, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, + 0x19, 0x0a, 0x08, 0x6d, 0x69, 0x6e, 0x5f, 0x73, 0x65, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x01, 0x52, 0x07, 0x6d, 0x69, 0x6e, 0x53, 0x65, 0x65, 0x6e, 0x12, 0x19, 0x0a, 0x08, 0x6d, 0x61, + 0x78, 0x5f, 0x73, 0x65, 0x65, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x01, 0x52, 0x07, 0x6d, 0x61, + 0x78, 0x53, 0x65, 0x65, 0x6e, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x75, 0x6d, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x01, 0x52, 0x03, 0x73, 0x75, 0x6d, 0x12, 0x24, 0x0a, 0x0e, 0x73, 0x75, 0x6d, 0x5f, 0x6f, + 0x66, 0x5f, 0x73, 0x71, 0x75, 0x61, 0x72, 0x65, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x01, 0x52, + 0x0c, 0x73, 0x75, 0x6d, 0x4f, 0x66, 0x53, 0x71, 0x75, 0x61, 0x72, 0x65, 0x73, 0x12, 0x14, 0x0a, + 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x01, 0x52, 0x05, 0x63, 0x6f, + 0x75, 0x6e, 0x74, 0x22, 0x4b, 0x0a, 0x12, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, 0x65, + 0x73, 0x75, 0x6c, 0x74, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, + 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, + 0x75, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, + 0x22, 0xc9, 0x02, 0x0a, 0x0b, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x73, + 0x12, 0x39, 0x0a, 0x09, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, + 0x6e, 0x67, 0x2e, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x44, 0x61, 0x74, 0x61, + 0x52, 0x09, 0x6c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x74, + 0x69, 0x6d, 0x65, 0x5f, 0x65, 0x6c, 0x61, 0x70, 0x73, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x01, 0x52, 0x0b, 0x74, 0x69, 0x6d, 0x65, 0x45, 0x6c, 0x61, 0x70, 0x73, 0x65, 0x64, 0x12, 0x1b, + 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x75, 0x73, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x01, 0x52, 0x08, 0x74, 0x69, 0x6d, 0x65, 0x55, 0x73, 0x65, 0x72, 0x12, 0x1f, 0x0a, 0x0b, 0x74, + 0x69, 0x6d, 0x65, 0x5f, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x18, 0x04, 0x20, 0x01, 0x28, 0x01, + 0x52, 0x0a, 0x74, 0x69, 0x6d, 0x65, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x12, 0x49, 0x0a, 0x0f, + 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x5f, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, + 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, + 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, 0x65, 0x73, 0x75, + 0x6c, 0x74, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x0e, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x12, 0x22, 0x0a, 0x0d, 0x63, 0x71, 0x5f, 0x70, 0x6f, + 0x6c, 0x6c, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b, + 0x63, 0x71, 0x50, 0x6f, 0x6c, 0x6c, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x2f, 0x0a, 0x0a, 0x63, + 0x6f, 0x72, 0x65, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x10, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, 0x74, 0x61, 0x74, + 0x73, 0x52, 0x09, 0x63, 0x6f, 0x72, 0x65, 0x53, 0x74, 0x61, 0x74, 0x73, 0x42, 0x1f, 0x0a, 0x0f, + 0x69, 0x6f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x42, + 0x0a, 0x53, 0x74, 0x61, 0x74, 0x73, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x62, 0x06, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_grpc_testing_stats_proto_rawDescOnce sync.Once + file_grpc_testing_stats_proto_rawDescData = file_grpc_testing_stats_proto_rawDesc +) + +func file_grpc_testing_stats_proto_rawDescGZIP() []byte { + file_grpc_testing_stats_proto_rawDescOnce.Do(func() { + file_grpc_testing_stats_proto_rawDescData = protoimpl.X.CompressGZIP(file_grpc_testing_stats_proto_rawDescData) + }) + return file_grpc_testing_stats_proto_rawDescData +} + +var file_grpc_testing_stats_proto_msgTypes = make([]protoimpl.MessageInfo, 5) +var file_grpc_testing_stats_proto_goTypes = []interface{}{ + (*ServerStats)(nil), // 0: grpc.testing.ServerStats + (*HistogramParams)(nil), // 1: grpc.testing.HistogramParams + (*HistogramData)(nil), // 2: grpc.testing.HistogramData + (*RequestResultCount)(nil), // 3: grpc.testing.RequestResultCount + (*ClientStats)(nil), // 4: grpc.testing.ClientStats + (*core.Stats)(nil), // 5: grpc.core.Stats +} +var file_grpc_testing_stats_proto_depIdxs = []int32{ + 5, // 0: grpc.testing.ServerStats.core_stats:type_name -> grpc.core.Stats + 2, // 1: grpc.testing.ClientStats.latencies:type_name -> grpc.testing.HistogramData + 3, // 2: grpc.testing.ClientStats.request_results:type_name -> grpc.testing.RequestResultCount + 5, // 3: grpc.testing.ClientStats.core_stats:type_name -> grpc.core.Stats + 4, // [4:4] is the sub-list for method output_type + 4, // [4:4] is the sub-list for method input_type + 4, // [4:4] is the sub-list for extension type_name + 4, // [4:4] is the sub-list for extension extendee + 0, // [0:4] is the sub-list for field type_name +} + +func init() { file_grpc_testing_stats_proto_init() } +func file_grpc_testing_stats_proto_init() { + if File_grpc_testing_stats_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_grpc_testing_stats_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ServerStats); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_stats_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*HistogramParams); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_stats_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*HistogramData); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_stats_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RequestResultCount); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_testing_stats_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ClientStats); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_grpc_testing_stats_proto_rawDesc, + NumEnums: 0, + NumMessages: 5, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_grpc_testing_stats_proto_goTypes, + DependencyIndexes: file_grpc_testing_stats_proto_depIdxs, + MessageInfos: file_grpc_testing_stats_proto_msgTypes, + }.Build() + File_grpc_testing_stats_proto = out.File + file_grpc_testing_stats_proto_rawDesc = nil + file_grpc_testing_stats_proto_goTypes = nil + file_grpc_testing_stats_proto_depIdxs = nil +} diff --git a/interop/grpc_testing/test.pb.go b/interop/grpc_testing/test.pb.go index 9b44623c25ae..58305cd3fcc3 100644 --- a/interop/grpc_testing/test.pb.go +++ b/interop/grpc_testing/test.pb.go @@ -1,902 +1,241 @@ +// Copyright 2015-2016 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// An integration test service that covers all the method signature permutations +// of unary/streaming requests/responses. + // Code generated by protoc-gen-go. DO NOT EDIT. -// source: interop/grpc_testing/test.proto +// versions: +// protoc-gen-go v1.31.0 +// protoc v4.22.0 +// source: grpc/testing/test.proto package grpc_testing import ( - fmt "fmt" - proto "github.com/golang/protobuf/proto" - math "math" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" ) -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package - -// The type of payload that should be returned. -type PayloadType int32 - const ( - // Compressable text format. - PayloadType_COMPRESSABLE PayloadType = 0 - // Uncompressable binary format. - PayloadType_UNCOMPRESSABLE PayloadType = 1 - // Randomly chosen from all other formats defined in this enum. - PayloadType_RANDOM PayloadType = 2 + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -var PayloadType_name = map[int32]string{ - 0: "COMPRESSABLE", - 1: "UNCOMPRESSABLE", - 2: "RANDOM", -} - -var PayloadType_value = map[string]int32{ - "COMPRESSABLE": 0, - "UNCOMPRESSABLE": 1, - "RANDOM": 2, -} - -func (x PayloadType) String() string { - return proto.EnumName(PayloadType_name, int32(x)) -} - -func (PayloadType) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_534063719f48d90d, []int{0} -} - -// The type of route that a client took to reach a server w.r.t. gRPCLB. -// The server must fill in "fallback" if it detects that the RPC reached -// the server via the "gRPCLB fallback" path, and "backend" if it detects -// that the RPC reached the server via "gRPCLB backend" path (i.e. if it got -// the address of this server from the gRPCLB server BalanceLoad RPC). Exactly -// how this detection is done is context and server dependant. -type GrpclbRouteType int32 - -const ( - // Server didn't detect the route that a client took to reach it. - GrpclbRouteType_GRPCLB_ROUTE_TYPE_UNKNOWN GrpclbRouteType = 0 - // Indicates that a client reached a server via gRPCLB fallback. - GrpclbRouteType_GRPCLB_ROUTE_TYPE_FALLBACK GrpclbRouteType = 1 - // Indicates that a client reached a server as a gRPCLB-given backend. - GrpclbRouteType_GRPCLB_ROUTE_TYPE_BACKEND GrpclbRouteType = 2 -) - -var GrpclbRouteType_name = map[int32]string{ - 0: "GRPCLB_ROUTE_TYPE_UNKNOWN", - 1: "GRPCLB_ROUTE_TYPE_FALLBACK", - 2: "GRPCLB_ROUTE_TYPE_BACKEND", -} - -var GrpclbRouteType_value = map[string]int32{ - "GRPCLB_ROUTE_TYPE_UNKNOWN": 0, - "GRPCLB_ROUTE_TYPE_FALLBACK": 1, - "GRPCLB_ROUTE_TYPE_BACKEND": 2, -} - -func (x GrpclbRouteType) String() string { - return proto.EnumName(GrpclbRouteType_name, int32(x)) -} - -func (GrpclbRouteType) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_534063719f48d90d, []int{1} -} - -type Empty struct { - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *Empty) Reset() { *m = Empty{} } -func (m *Empty) String() string { return proto.CompactTextString(m) } -func (*Empty) ProtoMessage() {} -func (*Empty) Descriptor() ([]byte, []int) { - return fileDescriptor_534063719f48d90d, []int{0} -} - -func (m *Empty) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Empty.Unmarshal(m, b) -} -func (m *Empty) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Empty.Marshal(b, m, deterministic) -} -func (m *Empty) XXX_Merge(src proto.Message) { - xxx_messageInfo_Empty.Merge(m, src) -} -func (m *Empty) XXX_Size() int { - return xxx_messageInfo_Empty.Size(m) -} -func (m *Empty) XXX_DiscardUnknown() { - xxx_messageInfo_Empty.DiscardUnknown(m) -} - -var xxx_messageInfo_Empty proto.InternalMessageInfo - -// A block of data, to simply increase gRPC message size. -type Payload struct { - // The type of data in body. - Type PayloadType `protobuf:"varint,1,opt,name=type,proto3,enum=grpc.testing.PayloadType" json:"type,omitempty"` - // Primary contents of payload. - Body []byte `protobuf:"bytes,2,opt,name=body,proto3" json:"body,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *Payload) Reset() { *m = Payload{} } -func (m *Payload) String() string { return proto.CompactTextString(m) } -func (*Payload) ProtoMessage() {} -func (*Payload) Descriptor() ([]byte, []int) { - return fileDescriptor_534063719f48d90d, []int{1} -} - -func (m *Payload) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Payload.Unmarshal(m, b) -} -func (m *Payload) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Payload.Marshal(b, m, deterministic) -} -func (m *Payload) XXX_Merge(src proto.Message) { - xxx_messageInfo_Payload.Merge(m, src) -} -func (m *Payload) XXX_Size() int { - return xxx_messageInfo_Payload.Size(m) -} -func (m *Payload) XXX_DiscardUnknown() { - xxx_messageInfo_Payload.DiscardUnknown(m) -} - -var xxx_messageInfo_Payload proto.InternalMessageInfo - -func (m *Payload) GetType() PayloadType { - if m != nil { - return m.Type - } - return PayloadType_COMPRESSABLE -} - -func (m *Payload) GetBody() []byte { - if m != nil { - return m.Body - } - return nil -} - -// A protobuf representation for grpc status. This is used by test -// clients to specify a status that the server should attempt to return. -type EchoStatus struct { - Code int32 `protobuf:"varint,1,opt,name=code,proto3" json:"code,omitempty"` - Message string `protobuf:"bytes,2,opt,name=message,proto3" json:"message,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *EchoStatus) Reset() { *m = EchoStatus{} } -func (m *EchoStatus) String() string { return proto.CompactTextString(m) } -func (*EchoStatus) ProtoMessage() {} -func (*EchoStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_534063719f48d90d, []int{2} -} - -func (m *EchoStatus) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_EchoStatus.Unmarshal(m, b) -} -func (m *EchoStatus) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_EchoStatus.Marshal(b, m, deterministic) -} -func (m *EchoStatus) XXX_Merge(src proto.Message) { - xxx_messageInfo_EchoStatus.Merge(m, src) -} -func (m *EchoStatus) XXX_Size() int { - return xxx_messageInfo_EchoStatus.Size(m) -} -func (m *EchoStatus) XXX_DiscardUnknown() { - xxx_messageInfo_EchoStatus.DiscardUnknown(m) -} - -var xxx_messageInfo_EchoStatus proto.InternalMessageInfo - -func (m *EchoStatus) GetCode() int32 { - if m != nil { - return m.Code - } - return 0 -} - -func (m *EchoStatus) GetMessage() string { - if m != nil { - return m.Message - } - return "" -} - -// Unary request. -type SimpleRequest struct { - // Desired payload type in the response from the server. - // If response_type is RANDOM, server randomly chooses one from other formats. - ResponseType PayloadType `protobuf:"varint,1,opt,name=response_type,json=responseType,proto3,enum=grpc.testing.PayloadType" json:"response_type,omitempty"` - // Desired payload size in the response from the server. - // If response_type is COMPRESSABLE, this denotes the size before compression. - ResponseSize int32 `protobuf:"varint,2,opt,name=response_size,json=responseSize,proto3" json:"response_size,omitempty"` - // Optional input payload sent along with the request. - Payload *Payload `protobuf:"bytes,3,opt,name=payload,proto3" json:"payload,omitempty"` - // Whether SimpleResponse should include username. - FillUsername bool `protobuf:"varint,4,opt,name=fill_username,json=fillUsername,proto3" json:"fill_username,omitempty"` - // Whether SimpleResponse should include OAuth scope. - FillOauthScope bool `protobuf:"varint,5,opt,name=fill_oauth_scope,json=fillOauthScope,proto3" json:"fill_oauth_scope,omitempty"` - // Whether server should return a given status - ResponseStatus *EchoStatus `protobuf:"bytes,7,opt,name=response_status,json=responseStatus,proto3" json:"response_status,omitempty"` - // Whether SimpleResponse should include server_id. - FillServerId bool `protobuf:"varint,9,opt,name=fill_server_id,json=fillServerId,proto3" json:"fill_server_id,omitempty"` - // Whether SimpleResponse should include grpclb_route_type. - FillGrpclbRouteType bool `protobuf:"varint,10,opt,name=fill_grpclb_route_type,json=fillGrpclbRouteType,proto3" json:"fill_grpclb_route_type,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *SimpleRequest) Reset() { *m = SimpleRequest{} } -func (m *SimpleRequest) String() string { return proto.CompactTextString(m) } -func (*SimpleRequest) ProtoMessage() {} -func (*SimpleRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_534063719f48d90d, []int{3} -} - -func (m *SimpleRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_SimpleRequest.Unmarshal(m, b) -} -func (m *SimpleRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_SimpleRequest.Marshal(b, m, deterministic) -} -func (m *SimpleRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_SimpleRequest.Merge(m, src) -} -func (m *SimpleRequest) XXX_Size() int { - return xxx_messageInfo_SimpleRequest.Size(m) -} -func (m *SimpleRequest) XXX_DiscardUnknown() { - xxx_messageInfo_SimpleRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_SimpleRequest proto.InternalMessageInfo - -func (m *SimpleRequest) GetResponseType() PayloadType { - if m != nil { - return m.ResponseType - } - return PayloadType_COMPRESSABLE -} - -func (m *SimpleRequest) GetResponseSize() int32 { - if m != nil { - return m.ResponseSize - } - return 0 -} - -func (m *SimpleRequest) GetPayload() *Payload { - if m != nil { - return m.Payload - } - return nil -} - -func (m *SimpleRequest) GetFillUsername() bool { - if m != nil { - return m.FillUsername - } - return false -} - -func (m *SimpleRequest) GetFillOauthScope() bool { - if m != nil { - return m.FillOauthScope - } - return false -} - -func (m *SimpleRequest) GetResponseStatus() *EchoStatus { - if m != nil { - return m.ResponseStatus - } - return nil -} - -func (m *SimpleRequest) GetFillServerId() bool { - if m != nil { - return m.FillServerId - } - return false -} - -func (m *SimpleRequest) GetFillGrpclbRouteType() bool { - if m != nil { - return m.FillGrpclbRouteType - } - return false -} - -// Unary response, as configured by the request. -type SimpleResponse struct { - // Payload to increase message size. - Payload *Payload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"` - // The user the request came from, for verifying authentication was - // successful when the client expected it. - Username string `protobuf:"bytes,2,opt,name=username,proto3" json:"username,omitempty"` - // OAuth scope. - OauthScope string `protobuf:"bytes,3,opt,name=oauth_scope,json=oauthScope,proto3" json:"oauth_scope,omitempty"` - // Server ID. This must be unique among different server instances, - // but the same across all RPC's made to a particular server instance. - ServerId string `protobuf:"bytes,4,opt,name=server_id,json=serverId,proto3" json:"server_id,omitempty"` - // gRPCLB Path. - GrpclbRouteType GrpclbRouteType `protobuf:"varint,5,opt,name=grpclb_route_type,json=grpclbRouteType,proto3,enum=grpc.testing.GrpclbRouteType" json:"grpclb_route_type,omitempty"` - // Server hostname. - Hostname string `protobuf:"bytes,6,opt,name=hostname,proto3" json:"hostname,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *SimpleResponse) Reset() { *m = SimpleResponse{} } -func (m *SimpleResponse) String() string { return proto.CompactTextString(m) } -func (*SimpleResponse) ProtoMessage() {} -func (*SimpleResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_534063719f48d90d, []int{4} -} - -func (m *SimpleResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_SimpleResponse.Unmarshal(m, b) -} -func (m *SimpleResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_SimpleResponse.Marshal(b, m, deterministic) -} -func (m *SimpleResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_SimpleResponse.Merge(m, src) -} -func (m *SimpleResponse) XXX_Size() int { - return xxx_messageInfo_SimpleResponse.Size(m) -} -func (m *SimpleResponse) XXX_DiscardUnknown() { - xxx_messageInfo_SimpleResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_SimpleResponse proto.InternalMessageInfo - -func (m *SimpleResponse) GetPayload() *Payload { - if m != nil { - return m.Payload - } - return nil -} - -func (m *SimpleResponse) GetUsername() string { - if m != nil { - return m.Username - } - return "" -} - -func (m *SimpleResponse) GetOauthScope() string { - if m != nil { - return m.OauthScope - } - return "" -} - -func (m *SimpleResponse) GetServerId() string { - if m != nil { - return m.ServerId - } - return "" -} - -func (m *SimpleResponse) GetGrpclbRouteType() GrpclbRouteType { - if m != nil { - return m.GrpclbRouteType - } - return GrpclbRouteType_GRPCLB_ROUTE_TYPE_UNKNOWN -} - -func (m *SimpleResponse) GetHostname() string { - if m != nil { - return m.Hostname - } - return "" -} - -// Client-streaming request. -type StreamingInputCallRequest struct { - // Optional input payload sent along with the request. - Payload *Payload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *StreamingInputCallRequest) Reset() { *m = StreamingInputCallRequest{} } -func (m *StreamingInputCallRequest) String() string { return proto.CompactTextString(m) } -func (*StreamingInputCallRequest) ProtoMessage() {} -func (*StreamingInputCallRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_534063719f48d90d, []int{5} -} - -func (m *StreamingInputCallRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_StreamingInputCallRequest.Unmarshal(m, b) -} -func (m *StreamingInputCallRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_StreamingInputCallRequest.Marshal(b, m, deterministic) -} -func (m *StreamingInputCallRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_StreamingInputCallRequest.Merge(m, src) -} -func (m *StreamingInputCallRequest) XXX_Size() int { - return xxx_messageInfo_StreamingInputCallRequest.Size(m) -} -func (m *StreamingInputCallRequest) XXX_DiscardUnknown() { - xxx_messageInfo_StreamingInputCallRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_StreamingInputCallRequest proto.InternalMessageInfo - -func (m *StreamingInputCallRequest) GetPayload() *Payload { - if m != nil { - return m.Payload - } - return nil -} - -// Client-streaming response. -type StreamingInputCallResponse struct { - // Aggregated size of payloads received from the client. - AggregatedPayloadSize int32 `protobuf:"varint,1,opt,name=aggregated_payload_size,json=aggregatedPayloadSize,proto3" json:"aggregated_payload_size,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *StreamingInputCallResponse) Reset() { *m = StreamingInputCallResponse{} } -func (m *StreamingInputCallResponse) String() string { return proto.CompactTextString(m) } -func (*StreamingInputCallResponse) ProtoMessage() {} -func (*StreamingInputCallResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_534063719f48d90d, []int{6} -} - -func (m *StreamingInputCallResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_StreamingInputCallResponse.Unmarshal(m, b) -} -func (m *StreamingInputCallResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_StreamingInputCallResponse.Marshal(b, m, deterministic) -} -func (m *StreamingInputCallResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_StreamingInputCallResponse.Merge(m, src) -} -func (m *StreamingInputCallResponse) XXX_Size() int { - return xxx_messageInfo_StreamingInputCallResponse.Size(m) -} -func (m *StreamingInputCallResponse) XXX_DiscardUnknown() { - xxx_messageInfo_StreamingInputCallResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_StreamingInputCallResponse proto.InternalMessageInfo - -func (m *StreamingInputCallResponse) GetAggregatedPayloadSize() int32 { - if m != nil { - return m.AggregatedPayloadSize - } - return 0 -} - -// Configuration for a particular response. -type ResponseParameters struct { - // Desired payload sizes in responses from the server. - // If response_type is COMPRESSABLE, this denotes the size before compression. - Size int32 `protobuf:"varint,1,opt,name=size,proto3" json:"size,omitempty"` - // Desired interval between consecutive responses in the response stream in - // microseconds. - IntervalUs int32 `protobuf:"varint,2,opt,name=interval_us,json=intervalUs,proto3" json:"interval_us,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *ResponseParameters) Reset() { *m = ResponseParameters{} } -func (m *ResponseParameters) String() string { return proto.CompactTextString(m) } -func (*ResponseParameters) ProtoMessage() {} -func (*ResponseParameters) Descriptor() ([]byte, []int) { - return fileDescriptor_534063719f48d90d, []int{7} -} - -func (m *ResponseParameters) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ResponseParameters.Unmarshal(m, b) -} -func (m *ResponseParameters) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ResponseParameters.Marshal(b, m, deterministic) -} -func (m *ResponseParameters) XXX_Merge(src proto.Message) { - xxx_messageInfo_ResponseParameters.Merge(m, src) -} -func (m *ResponseParameters) XXX_Size() int { - return xxx_messageInfo_ResponseParameters.Size(m) -} -func (m *ResponseParameters) XXX_DiscardUnknown() { - xxx_messageInfo_ResponseParameters.DiscardUnknown(m) -} - -var xxx_messageInfo_ResponseParameters proto.InternalMessageInfo - -func (m *ResponseParameters) GetSize() int32 { - if m != nil { - return m.Size - } - return 0 -} - -func (m *ResponseParameters) GetIntervalUs() int32 { - if m != nil { - return m.IntervalUs - } - return 0 -} - -// Server-streaming request. -type StreamingOutputCallRequest struct { - // Desired payload type in the response from the server. - // If response_type is RANDOM, the payload from each response in the stream - // might be of different types. This is to simulate a mixed type of payload - // stream. - ResponseType PayloadType `protobuf:"varint,1,opt,name=response_type,json=responseType,proto3,enum=grpc.testing.PayloadType" json:"response_type,omitempty"` - // Configuration for each expected response message. - ResponseParameters []*ResponseParameters `protobuf:"bytes,2,rep,name=response_parameters,json=responseParameters,proto3" json:"response_parameters,omitempty"` - // Optional input payload sent along with the request. - Payload *Payload `protobuf:"bytes,3,opt,name=payload,proto3" json:"payload,omitempty"` - // Whether server should return a given status - ResponseStatus *EchoStatus `protobuf:"bytes,7,opt,name=response_status,json=responseStatus,proto3" json:"response_status,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *StreamingOutputCallRequest) Reset() { *m = StreamingOutputCallRequest{} } -func (m *StreamingOutputCallRequest) String() string { return proto.CompactTextString(m) } -func (*StreamingOutputCallRequest) ProtoMessage() {} -func (*StreamingOutputCallRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_534063719f48d90d, []int{8} -} - -func (m *StreamingOutputCallRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_StreamingOutputCallRequest.Unmarshal(m, b) -} -func (m *StreamingOutputCallRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_StreamingOutputCallRequest.Marshal(b, m, deterministic) -} -func (m *StreamingOutputCallRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_StreamingOutputCallRequest.Merge(m, src) -} -func (m *StreamingOutputCallRequest) XXX_Size() int { - return xxx_messageInfo_StreamingOutputCallRequest.Size(m) -} -func (m *StreamingOutputCallRequest) XXX_DiscardUnknown() { - xxx_messageInfo_StreamingOutputCallRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_StreamingOutputCallRequest proto.InternalMessageInfo - -func (m *StreamingOutputCallRequest) GetResponseType() PayloadType { - if m != nil { - return m.ResponseType - } - return PayloadType_COMPRESSABLE -} - -func (m *StreamingOutputCallRequest) GetResponseParameters() []*ResponseParameters { - if m != nil { - return m.ResponseParameters - } - return nil -} - -func (m *StreamingOutputCallRequest) GetPayload() *Payload { - if m != nil { - return m.Payload - } - return nil -} - -func (m *StreamingOutputCallRequest) GetResponseStatus() *EchoStatus { - if m != nil { - return m.ResponseStatus - } - return nil -} - -// Server-streaming response, as configured by the request and parameters. -type StreamingOutputCallResponse struct { - // Payload to increase response size. - Payload *Payload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *StreamingOutputCallResponse) Reset() { *m = StreamingOutputCallResponse{} } -func (m *StreamingOutputCallResponse) String() string { return proto.CompactTextString(m) } -func (*StreamingOutputCallResponse) ProtoMessage() {} -func (*StreamingOutputCallResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_534063719f48d90d, []int{9} -} - -func (m *StreamingOutputCallResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_StreamingOutputCallResponse.Unmarshal(m, b) -} -func (m *StreamingOutputCallResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_StreamingOutputCallResponse.Marshal(b, m, deterministic) -} -func (m *StreamingOutputCallResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_StreamingOutputCallResponse.Merge(m, src) -} -func (m *StreamingOutputCallResponse) XXX_Size() int { - return xxx_messageInfo_StreamingOutputCallResponse.Size(m) -} -func (m *StreamingOutputCallResponse) XXX_DiscardUnknown() { - xxx_messageInfo_StreamingOutputCallResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_StreamingOutputCallResponse proto.InternalMessageInfo - -func (m *StreamingOutputCallResponse) GetPayload() *Payload { - if m != nil { - return m.Payload - } - return nil -} - -type LoadBalancerStatsRequest struct { - // Request stats for the next num_rpcs sent by client. - NumRpcs int32 `protobuf:"varint,1,opt,name=num_rpcs,json=numRpcs,proto3" json:"num_rpcs,omitempty"` - // If num_rpcs have not completed within timeout_sec, return partial results. - TimeoutSec int32 `protobuf:"varint,2,opt,name=timeout_sec,json=timeoutSec,proto3" json:"timeout_sec,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *LoadBalancerStatsRequest) Reset() { *m = LoadBalancerStatsRequest{} } -func (m *LoadBalancerStatsRequest) String() string { return proto.CompactTextString(m) } -func (*LoadBalancerStatsRequest) ProtoMessage() {} -func (*LoadBalancerStatsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_534063719f48d90d, []int{10} -} - -func (m *LoadBalancerStatsRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_LoadBalancerStatsRequest.Unmarshal(m, b) -} -func (m *LoadBalancerStatsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_LoadBalancerStatsRequest.Marshal(b, m, deterministic) -} -func (m *LoadBalancerStatsRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_LoadBalancerStatsRequest.Merge(m, src) -} -func (m *LoadBalancerStatsRequest) XXX_Size() int { - return xxx_messageInfo_LoadBalancerStatsRequest.Size(m) -} -func (m *LoadBalancerStatsRequest) XXX_DiscardUnknown() { - xxx_messageInfo_LoadBalancerStatsRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_LoadBalancerStatsRequest proto.InternalMessageInfo - -func (m *LoadBalancerStatsRequest) GetNumRpcs() int32 { - if m != nil { - return m.NumRpcs - } - return 0 -} - -func (m *LoadBalancerStatsRequest) GetTimeoutSec() int32 { - if m != nil { - return m.TimeoutSec - } - return 0 -} - -type LoadBalancerStatsResponse struct { - // The number of completed RPCs for each peer. - RpcsByPeer map[string]int32 `protobuf:"bytes,1,rep,name=rpcs_by_peer,json=rpcsByPeer,proto3" json:"rpcs_by_peer,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` - // The number of RPCs that failed to record a remote peer. - NumFailures int32 `protobuf:"varint,2,opt,name=num_failures,json=numFailures,proto3" json:"num_failures,omitempty"` - // The number of completed RPCs for each method (UnaryCall or EmptyCall). - RpcsByMethod map[string]*LoadBalancerStatsResponse_RpcsByPeer `protobuf:"bytes,3,rep,name=rpcs_by_method,json=rpcsByMethod,proto3" json:"rpcs_by_method,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *LoadBalancerStatsResponse) Reset() { *m = LoadBalancerStatsResponse{} } -func (m *LoadBalancerStatsResponse) String() string { return proto.CompactTextString(m) } -func (*LoadBalancerStatsResponse) ProtoMessage() {} -func (*LoadBalancerStatsResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_534063719f48d90d, []int{11} -} - -func (m *LoadBalancerStatsResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_LoadBalancerStatsResponse.Unmarshal(m, b) -} -func (m *LoadBalancerStatsResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_LoadBalancerStatsResponse.Marshal(b, m, deterministic) -} -func (m *LoadBalancerStatsResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_LoadBalancerStatsResponse.Merge(m, src) -} -func (m *LoadBalancerStatsResponse) XXX_Size() int { - return xxx_messageInfo_LoadBalancerStatsResponse.Size(m) -} -func (m *LoadBalancerStatsResponse) XXX_DiscardUnknown() { - xxx_messageInfo_LoadBalancerStatsResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_LoadBalancerStatsResponse proto.InternalMessageInfo - -func (m *LoadBalancerStatsResponse) GetRpcsByPeer() map[string]int32 { - if m != nil { - return m.RpcsByPeer - } - return nil -} - -func (m *LoadBalancerStatsResponse) GetNumFailures() int32 { - if m != nil { - return m.NumFailures - } - return 0 -} - -func (m *LoadBalancerStatsResponse) GetRpcsByMethod() map[string]*LoadBalancerStatsResponse_RpcsByPeer { - if m != nil { - return m.RpcsByMethod - } - return nil -} - -type LoadBalancerStatsResponse_RpcsByPeer struct { - // The number of completed RPCs for each peer. - RpcsByPeer map[string]int32 `protobuf:"bytes,1,rep,name=rpcs_by_peer,json=rpcsByPeer,proto3" json:"rpcs_by_peer,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *LoadBalancerStatsResponse_RpcsByPeer) Reset() { *m = LoadBalancerStatsResponse_RpcsByPeer{} } -func (m *LoadBalancerStatsResponse_RpcsByPeer) String() string { return proto.CompactTextString(m) } -func (*LoadBalancerStatsResponse_RpcsByPeer) ProtoMessage() {} -func (*LoadBalancerStatsResponse_RpcsByPeer) Descriptor() ([]byte, []int) { - return fileDescriptor_534063719f48d90d, []int{11, 0} -} - -func (m *LoadBalancerStatsResponse_RpcsByPeer) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_LoadBalancerStatsResponse_RpcsByPeer.Unmarshal(m, b) -} -func (m *LoadBalancerStatsResponse_RpcsByPeer) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_LoadBalancerStatsResponse_RpcsByPeer.Marshal(b, m, deterministic) -} -func (m *LoadBalancerStatsResponse_RpcsByPeer) XXX_Merge(src proto.Message) { - xxx_messageInfo_LoadBalancerStatsResponse_RpcsByPeer.Merge(m, src) -} -func (m *LoadBalancerStatsResponse_RpcsByPeer) XXX_Size() int { - return xxx_messageInfo_LoadBalancerStatsResponse_RpcsByPeer.Size(m) -} -func (m *LoadBalancerStatsResponse_RpcsByPeer) XXX_DiscardUnknown() { - xxx_messageInfo_LoadBalancerStatsResponse_RpcsByPeer.DiscardUnknown(m) -} - -var xxx_messageInfo_LoadBalancerStatsResponse_RpcsByPeer proto.InternalMessageInfo - -func (m *LoadBalancerStatsResponse_RpcsByPeer) GetRpcsByPeer() map[string]int32 { - if m != nil { - return m.RpcsByPeer - } - return nil -} - -func init() { - proto.RegisterEnum("grpc.testing.PayloadType", PayloadType_name, PayloadType_value) - proto.RegisterEnum("grpc.testing.GrpclbRouteType", GrpclbRouteType_name, GrpclbRouteType_value) - proto.RegisterType((*Empty)(nil), "grpc.testing.Empty") - proto.RegisterType((*Payload)(nil), "grpc.testing.Payload") - proto.RegisterType((*EchoStatus)(nil), "grpc.testing.EchoStatus") - proto.RegisterType((*SimpleRequest)(nil), "grpc.testing.SimpleRequest") - proto.RegisterType((*SimpleResponse)(nil), "grpc.testing.SimpleResponse") - proto.RegisterType((*StreamingInputCallRequest)(nil), "grpc.testing.StreamingInputCallRequest") - proto.RegisterType((*StreamingInputCallResponse)(nil), "grpc.testing.StreamingInputCallResponse") - proto.RegisterType((*ResponseParameters)(nil), "grpc.testing.ResponseParameters") - proto.RegisterType((*StreamingOutputCallRequest)(nil), "grpc.testing.StreamingOutputCallRequest") - proto.RegisterType((*StreamingOutputCallResponse)(nil), "grpc.testing.StreamingOutputCallResponse") - proto.RegisterType((*LoadBalancerStatsRequest)(nil), "grpc.testing.LoadBalancerStatsRequest") - proto.RegisterType((*LoadBalancerStatsResponse)(nil), "grpc.testing.LoadBalancerStatsResponse") - proto.RegisterMapType((map[string]*LoadBalancerStatsResponse_RpcsByPeer)(nil), "grpc.testing.LoadBalancerStatsResponse.RpcsByMethodEntry") - proto.RegisterMapType((map[string]int32)(nil), "grpc.testing.LoadBalancerStatsResponse.RpcsByPeerEntry") - proto.RegisterType((*LoadBalancerStatsResponse_RpcsByPeer)(nil), "grpc.testing.LoadBalancerStatsResponse.RpcsByPeer") - proto.RegisterMapType((map[string]int32)(nil), "grpc.testing.LoadBalancerStatsResponse.RpcsByPeer.RpcsByPeerEntry") -} - -func init() { proto.RegisterFile("interop/grpc_testing/test.proto", fileDescriptor_534063719f48d90d) } - -var fileDescriptor_534063719f48d90d = []byte{ - // 1083 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x57, 0xdd, 0x72, 0xdb, 0xc4, - 0x17, 0x8f, 0x1c, 0x3b, 0x8e, 0x8f, 0x5d, 0xc7, 0xd9, 0xb4, 0xff, 0xbf, 0xe2, 0x50, 0x6a, 0x04, - 0x43, 0x4d, 0x99, 0x3a, 0x8c, 0x3b, 0x7c, 0x75, 0xa6, 0x30, 0x71, 0xe2, 0xa4, 0x99, 0x3a, 0xb6, - 0x91, 0x63, 0x98, 0x72, 0xa3, 0xd9, 0xc8, 0x1b, 0x45, 0x83, 0xa4, 0x15, 0xab, 0x55, 0x06, 0xf7, - 0x86, 0x19, 0x1e, 0x81, 0x57, 0xe0, 0x09, 0xb8, 0xe6, 0x6d, 0x78, 0x12, 0x66, 0x57, 0x92, 0x3f, - 0x15, 0x1a, 0x93, 0x81, 0x2b, 0xef, 0x9e, 0xcf, 0xdf, 0xf9, 0x9d, 0x3d, 0xbb, 0x16, 0x3c, 0xb2, - 0x3d, 0x4e, 0x18, 0xf5, 0xf7, 0x2d, 0xe6, 0x9b, 0x06, 0x27, 0x01, 0xb7, 0x3d, 0x6b, 0x5f, 0xfc, - 0x36, 0x7c, 0x46, 0x39, 0x45, 0x25, 0xa1, 0x68, 0xc4, 0x0a, 0x2d, 0x0f, 0xb9, 0xb6, 0xeb, 0xf3, - 0xb1, 0xd6, 0x81, 0x7c, 0x1f, 0x8f, 0x1d, 0x8a, 0x47, 0xe8, 0x29, 0x64, 0xf9, 0xd8, 0x27, 0xaa, - 0x52, 0x53, 0xea, 0xe5, 0xe6, 0x6e, 0x63, 0xd6, 0xa1, 0x11, 0x1b, 0x9d, 0x8f, 0x7d, 0xa2, 0x4b, - 0x33, 0x84, 0x20, 0x7b, 0x41, 0x47, 0x63, 0x35, 0x53, 0x53, 0xea, 0x25, 0x5d, 0xae, 0xb5, 0xe7, - 0x00, 0x6d, 0xf3, 0x8a, 0x0e, 0x38, 0xe6, 0x61, 0x20, 0x2c, 0x4c, 0x3a, 0x8a, 0x02, 0xe6, 0x74, - 0xb9, 0x46, 0x2a, 0xe4, 0x5d, 0x12, 0x04, 0xd8, 0x22, 0xd2, 0xb1, 0xa0, 0x27, 0x5b, 0xed, 0xd7, - 0x75, 0xb8, 0x37, 0xb0, 0x5d, 0xdf, 0x21, 0x3a, 0xf9, 0x31, 0x24, 0x01, 0x47, 0x5f, 0xc1, 0x3d, - 0x46, 0x02, 0x9f, 0x7a, 0x01, 0x31, 0x6e, 0x87, 0xac, 0x94, 0xd8, 0x8b, 0x1d, 0x7a, 0x7f, 0xc6, - 0x3f, 0xb0, 0xdf, 0x44, 0x19, 0x73, 0x53, 0xa3, 0x81, 0xfd, 0x86, 0xa0, 0x7d, 0xc8, 0xfb, 0x51, - 0x04, 0x75, 0xbd, 0xa6, 0xd4, 0x8b, 0xcd, 0x07, 0xa9, 0xe1, 0xf5, 0xc4, 0x4a, 0x44, 0xbd, 0xb4, - 0x1d, 0xc7, 0x08, 0x03, 0xc2, 0x3c, 0xec, 0x12, 0x35, 0x5b, 0x53, 0xea, 0x9b, 0x7a, 0x49, 0x08, - 0x87, 0xb1, 0x0c, 0xd5, 0xa1, 0x22, 0x8d, 0x28, 0x0e, 0xf9, 0x95, 0x11, 0x98, 0xd4, 0x27, 0x6a, - 0x4e, 0xda, 0x95, 0x85, 0xbc, 0x27, 0xc4, 0x03, 0x21, 0x45, 0x07, 0xb0, 0x35, 0x05, 0x29, 0x79, - 0x53, 0xf3, 0x12, 0x87, 0x3a, 0x8f, 0x63, 0xca, 0xab, 0x5e, 0x9e, 0x14, 0x10, 0xf1, 0xfc, 0x01, - 0xc8, 0xa0, 0x46, 0x40, 0xd8, 0x35, 0x61, 0x86, 0x3d, 0x52, 0x0b, 0x53, 0x48, 0x03, 0x29, 0x3c, - 0x1d, 0xa1, 0x67, 0xf0, 0x3f, 0x69, 0x25, 0xa2, 0x3a, 0x17, 0x06, 0xa3, 0x21, 0x8f, 0x69, 0x05, - 0x69, 0xbd, 0x23, 0xb4, 0x27, 0x52, 0xa9, 0x0b, 0x9d, 0xa0, 0x50, 0xfb, 0x25, 0x03, 0xe5, 0xa4, - 0x29, 0x51, 0xce, 0x59, 0xc2, 0x94, 0x5b, 0x11, 0x56, 0x85, 0xcd, 0x09, 0x57, 0x51, 0xcf, 0x27, - 0x7b, 0xf4, 0x08, 0x8a, 0xb3, 0x14, 0xad, 0x4b, 0x35, 0xd0, 0x29, 0x3d, 0x7b, 0x50, 0x98, 0x96, - 0x95, 0x8d, 0xbc, 0x83, 0xa4, 0xa4, 0x53, 0xd8, 0x5e, 0xae, 0x26, 0x27, 0x0f, 0xc9, 0xc3, 0x79, - 0x50, 0x0b, 0x75, 0xe9, 0x5b, 0xd6, 0xbc, 0x40, 0x80, 0xbc, 0xa2, 0x01, 0x97, 0x20, 0x37, 0xa2, - 0x34, 0xc9, 0x5e, 0xeb, 0xc0, 0xee, 0x80, 0x33, 0x82, 0x5d, 0xdb, 0xb3, 0x4e, 0x3d, 0x3f, 0xe4, - 0x87, 0xd8, 0x71, 0x92, 0x43, 0xba, 0x2a, 0x1d, 0xda, 0x39, 0x54, 0xd3, 0xa2, 0xc5, 0xec, 0x7e, - 0x06, 0xff, 0xc7, 0x96, 0xc5, 0x88, 0x85, 0x39, 0x19, 0x19, 0xb1, 0x4f, 0x74, 0x7a, 0xa3, 0x31, - 0x7a, 0x30, 0x55, 0xc7, 0xa1, 0xc5, 0x31, 0xd6, 0x4e, 0x01, 0x25, 0x31, 0xfa, 0x98, 0x61, 0x97, - 0x70, 0xc2, 0xe4, 0x04, 0xce, 0xb8, 0xca, 0xb5, 0xa0, 0x5c, 0xde, 0x15, 0xd7, 0x58, 0x9c, 0xe1, - 0x78, 0x26, 0x20, 0x11, 0x0d, 0x03, 0xed, 0xb7, 0xcc, 0x0c, 0xc2, 0x5e, 0xc8, 0x17, 0x0a, 0xbe, - 0xeb, 0x54, 0x7e, 0x03, 0x3b, 0x13, 0x7f, 0x7f, 0x02, 0x55, 0xcd, 0xd4, 0xd6, 0xeb, 0xc5, 0x66, - 0x6d, 0x3e, 0xca, 0x72, 0x49, 0x3a, 0x62, 0xcb, 0x65, 0xae, 0x3c, 0xc3, 0x77, 0x1f, 0x3a, 0xad, - 0x0b, 0x7b, 0xa9, 0x24, 0xfd, 0xc3, 0x29, 0xd1, 0xbe, 0x05, 0xb5, 0x43, 0xf1, 0xa8, 0x85, 0x1d, - 0xec, 0x99, 0x84, 0x89, 0x2c, 0x41, 0x42, 0xf9, 0x2e, 0x6c, 0x7a, 0xa1, 0x6b, 0x30, 0xdf, 0x0c, - 0xe2, 0x56, 0xe6, 0xbd, 0xd0, 0xd5, 0x7d, 0x33, 0x10, 0xdd, 0xe4, 0xb6, 0x4b, 0x68, 0xc8, 0x8d, - 0x80, 0x98, 0x49, 0x37, 0x63, 0xd1, 0x80, 0x98, 0xda, 0x9f, 0x59, 0xd8, 0x4d, 0x09, 0x1c, 0xc3, - 0x7c, 0x0d, 0x25, 0x11, 0xd5, 0xb8, 0x18, 0x1b, 0x3e, 0x21, 0x4c, 0x55, 0x64, 0x17, 0x3e, 0x9f, - 0xc7, 0x7a, 0xa3, 0x7b, 0x43, 0x40, 0x68, 0x8d, 0xfb, 0x84, 0xb0, 0xb6, 0xc7, 0xd9, 0x58, 0x07, - 0x36, 0x11, 0xa0, 0xf7, 0xa0, 0x24, 0x40, 0x5f, 0x62, 0xdb, 0x09, 0x19, 0x49, 0x0e, 0x5a, 0xd1, - 0x0b, 0xdd, 0xe3, 0x58, 0x84, 0x0c, 0x28, 0x27, 0xd9, 0x5d, 0xc2, 0xaf, 0xa8, 0x68, 0x9f, 0xc8, - 0xff, 0xe5, 0x6a, 0xf9, 0xcf, 0xa4, 0x6f, 0x84, 0xa0, 0xc4, 0x66, 0x44, 0xd5, 0xdf, 0x15, 0x80, - 0x29, 0x46, 0x34, 0x4a, 0xad, 0xb6, 0xb5, 0x7a, 0xb5, 0x7f, 0x57, 0x78, 0xf5, 0x05, 0x6c, 0x2d, - 0xa8, 0x51, 0x05, 0xd6, 0x7f, 0x20, 0x63, 0xd9, 0xbb, 0x82, 0x2e, 0x96, 0xe8, 0x3e, 0xe4, 0xae, - 0xb1, 0x13, 0x26, 0x6f, 0x52, 0xb4, 0x79, 0x9e, 0xf9, 0x42, 0xb9, 0xab, 0x7b, 0x00, 0xdb, 0x4b, - 0xac, 0xa4, 0x04, 0x78, 0x39, 0x1b, 0xa0, 0xd8, 0x6c, 0xae, 0xce, 0xc1, 0x4c, 0xd2, 0x27, 0x5f, - 0x43, 0x71, 0x66, 0xe0, 0x51, 0x05, 0x4a, 0x87, 0xbd, 0xb3, 0xbe, 0xde, 0x1e, 0x0c, 0x0e, 0x5a, - 0x9d, 0x76, 0x65, 0x0d, 0x21, 0x28, 0x0f, 0xbb, 0x73, 0x32, 0x05, 0x01, 0x6c, 0xe8, 0x07, 0xdd, - 0xa3, 0xde, 0x59, 0x25, 0xf3, 0x84, 0xc2, 0xd6, 0xc2, 0x15, 0x8d, 0x1e, 0xc2, 0xee, 0x89, 0xde, - 0x3f, 0xec, 0xb4, 0x0c, 0xbd, 0x37, 0x3c, 0x6f, 0x1b, 0xe7, 0xaf, 0xfb, 0x6d, 0x63, 0xd8, 0x7d, - 0xd5, 0xed, 0x7d, 0xd7, 0xad, 0xac, 0xa1, 0x77, 0xa1, 0xba, 0xac, 0x3e, 0x3e, 0xe8, 0x74, 0x5a, - 0x07, 0x87, 0xaf, 0x2a, 0x4a, 0xba, 0xbb, 0xd0, 0xb5, 0xbb, 0x47, 0x95, 0x4c, 0xf3, 0x8f, 0x2c, - 0x14, 0xcf, 0x49, 0xc0, 0xc5, 0xf3, 0x68, 0x9b, 0x04, 0x7d, 0x0a, 0x05, 0xf9, 0x87, 0x48, 0x0c, - 0x31, 0xda, 0x59, 0xb8, 0x05, 0x84, 0xa2, 0x9a, 0x26, 0x44, 0xc7, 0x50, 0x18, 0x7a, 0x98, 0x45, - 0x6e, 0x7b, 0xf3, 0x16, 0x73, 0x7f, 0x66, 0xaa, 0xef, 0xa4, 0x2b, 0xe3, 0x39, 0x74, 0x60, 0x27, - 0xe5, 0x36, 0x41, 0xf5, 0x05, 0xa7, 0x1b, 0x6f, 0xe5, 0xea, 0x47, 0xb7, 0xb0, 0x8c, 0x72, 0x7d, - 0xa2, 0x20, 0x1b, 0xd0, 0xf2, 0x13, 0x84, 0x1e, 0xdf, 0x10, 0x62, 0xf1, 0xc9, 0xab, 0xd6, 0xdf, - 0x6e, 0x18, 0xa5, 0xaa, 0x8b, 0x54, 0xe5, 0xe3, 0xd0, 0x71, 0x8e, 0x42, 0xdf, 0x21, 0x3f, 0xfd, - 0x6b, 0x35, 0xd5, 0x15, 0x59, 0x55, 0xf9, 0x25, 0x76, 0x2e, 0xff, 0x83, 0x54, 0xcd, 0x21, 0xdc, - 0x1f, 0x7a, 0xb2, 0x83, 0x2e, 0xf1, 0x38, 0x19, 0x25, 0xa7, 0xe8, 0x05, 0x6c, 0xcf, 0xc9, 0x57, - 0x3b, 0x4d, 0xcd, 0x9f, 0x53, 0xde, 0x80, 0x24, 0xb4, 0x09, 0xe5, 0x13, 0xc2, 0x0f, 0x1d, 0x9b, - 0x78, 0x5c, 0x2a, 0xd0, 0x87, 0x6f, 0x9d, 0xd9, 0xa8, 0xb6, 0xc7, 0xb7, 0x9c, 0x6d, 0x6d, 0xad, - 0xf5, 0xf4, 0xfb, 0x8f, 0x2d, 0x4a, 0x2d, 0x87, 0x34, 0x2c, 0xea, 0x60, 0xcf, 0x6a, 0x50, 0x66, - 0xc9, 0x2f, 0x8a, 0xfd, 0xb4, 0xcf, 0x8b, 0x8b, 0x0d, 0xf9, 0x69, 0xf1, 0xec, 0xaf, 0x00, 0x00, - 0x00, 0xff, 0xff, 0x8c, 0x0b, 0x8c, 0x16, 0x7d, 0x0c, 0x00, 0x00, +var File_grpc_testing_test_proto protoreflect.FileDescriptor + +var file_grpc_testing_test_proto_rawDesc = []byte{ + 0x0a, 0x17, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x74, + 0x65, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x67, 0x72, 0x70, 0x63, 0x2e, + 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x1a, 0x18, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x74, 0x65, + 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x1a, 0x1b, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, + 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x32, 0xcb, + 0x05, 0x0a, 0x0b, 0x54, 0x65, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x35, + 0x0a, 0x09, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x43, 0x61, 0x6c, 0x6c, 0x12, 0x13, 0x2e, 0x67, 0x72, + 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, + 0x1a, 0x13, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, + 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x46, 0x0a, 0x09, 0x55, 0x6e, 0x61, 0x72, 0x79, 0x43, 0x61, + 0x6c, 0x6c, 0x12, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, + 0x67, 0x2e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, + 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4f, 0x0a, + 0x12, 0x43, 0x61, 0x63, 0x68, 0x65, 0x61, 0x62, 0x6c, 0x65, 0x55, 0x6e, 0x61, 0x72, 0x79, 0x43, + 0x61, 0x6c, 0x6c, 0x12, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, + 0x6e, 0x67, 0x2e, 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, + 0x53, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x6c, + 0x0a, 0x13, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f, 0x75, 0x74, 0x70, 0x75, + 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x12, 0x28, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, + 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f, 0x75, + 0x74, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x29, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, + 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x43, 0x61, + 0x6c, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x12, 0x69, 0x0a, 0x12, + 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x43, 0x61, + 0x6c, 0x6c, 0x12, 0x27, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, + 0x67, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x70, 0x75, 0x74, + 0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x67, 0x72, + 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, + 0x6d, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x12, 0x69, 0x0a, 0x0e, 0x46, 0x75, 0x6c, 0x6c, 0x44, + 0x75, 0x70, 0x6c, 0x65, 0x78, 0x43, 0x61, 0x6c, 0x6c, 0x12, 0x28, 0x2e, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, + 0x6e, 0x67, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, + 0x6e, 0x67, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f, 0x75, 0x74, 0x70, + 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, + 0x30, 0x01, 0x12, 0x69, 0x0a, 0x0e, 0x48, 0x61, 0x6c, 0x66, 0x44, 0x75, 0x70, 0x6c, 0x65, 0x78, + 0x43, 0x61, 0x6c, 0x6c, 0x12, 0x28, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, + 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f, 0x75, 0x74, + 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, + 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x74, + 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x43, 0x61, 0x6c, + 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x12, 0x3d, 0x0a, + 0x11, 0x55, 0x6e, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x65, 0x64, 0x43, 0x61, + 0x6c, 0x6c, 0x12, 0x13, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, + 0x67, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x13, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, + 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x32, 0x55, 0x0a, 0x14, + 0x55, 0x6e, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x65, 0x64, 0x53, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x12, 0x3d, 0x0a, 0x11, 0x55, 0x6e, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, + 0x65, 0x6e, 0x74, 0x65, 0x64, 0x43, 0x61, 0x6c, 0x6c, 0x12, 0x13, 0x2e, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x13, + 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x45, 0x6d, + 0x70, 0x74, 0x79, 0x32, 0x89, 0x01, 0x0a, 0x10, 0x52, 0x65, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, + 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x3b, 0x0a, 0x05, 0x53, 0x74, 0x61, 0x72, + 0x74, 0x12, 0x1d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, + 0x2e, 0x52, 0x65, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x73, + 0x1a, 0x13, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, + 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x38, 0x0a, 0x04, 0x53, 0x74, 0x6f, 0x70, 0x12, 0x13, 0x2e, + 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x45, 0x6d, 0x70, + 0x74, 0x79, 0x1a, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, + 0x67, 0x2e, 0x52, 0x65, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x32, + 0x86, 0x02, 0x0a, 0x18, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, + 0x53, 0x74, 0x61, 0x74, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x63, 0x0a, 0x0e, + 0x47, 0x65, 0x74, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, 0x26, + 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4c, 0x6f, + 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, + 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, + 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x00, 0x12, 0x84, 0x01, 0x0a, 0x19, 0x47, 0x65, 0x74, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x41, + 0x63, 0x63, 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, + 0x31, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x4c, + 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x41, 0x63, 0x63, 0x75, 0x6d, + 0x75, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x32, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, + 0x67, 0x2e, 0x4c, 0x6f, 0x61, 0x64, 0x42, 0x61, 0x6c, 0x61, 0x6e, 0x63, 0x65, 0x72, 0x41, 0x63, + 0x63, 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74, 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x32, 0x3f, 0x0a, 0x0b, 0x48, 0x6f, 0x6f, 0x6b, + 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x30, 0x0a, 0x04, 0x48, 0x6f, 0x6f, 0x6b, 0x12, + 0x13, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x45, + 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x13, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, + 0x69, 0x6e, 0x67, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x32, 0xd5, 0x01, 0x0a, 0x16, 0x58, 0x64, + 0x73, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x48, 0x65, 0x61, 0x6c, 0x74, 0x68, 0x53, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x12, 0x36, 0x0a, 0x0a, 0x53, 0x65, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, + 0x6e, 0x67, 0x12, 0x13, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, + 0x67, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x13, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, + 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x39, 0x0a, 0x0d, + 0x53, 0x65, 0x74, 0x4e, 0x6f, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x12, 0x13, 0x2e, + 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x45, 0x6d, 0x70, + 0x74, 0x79, 0x1a, 0x13, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, + 0x67, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x48, 0x0a, 0x0f, 0x53, 0x65, 0x6e, 0x64, 0x48, + 0x6f, 0x6f, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x2e, 0x67, 0x72, 0x70, + 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x48, 0x6f, 0x6f, 0x6b, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, + 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x48, 0x6f, 0x6f, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x32, 0x7b, 0x0a, 0x1f, 0x58, 0x64, 0x73, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x43, 0x6c, + 0x69, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x53, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x12, 0x58, 0x0a, 0x09, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, + 0x65, 0x12, 0x24, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, + 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, + 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x75, 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x1d, + 0x0a, 0x1b, 0x69, 0x6f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, + 0x67, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x62, 0x06, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var file_grpc_testing_test_proto_goTypes = []interface{}{ + (*Empty)(nil), // 0: grpc.testing.Empty + (*SimpleRequest)(nil), // 1: grpc.testing.SimpleRequest + (*StreamingOutputCallRequest)(nil), // 2: grpc.testing.StreamingOutputCallRequest + (*StreamingInputCallRequest)(nil), // 3: grpc.testing.StreamingInputCallRequest + (*ReconnectParams)(nil), // 4: grpc.testing.ReconnectParams + (*LoadBalancerStatsRequest)(nil), // 5: grpc.testing.LoadBalancerStatsRequest + (*LoadBalancerAccumulatedStatsRequest)(nil), // 6: grpc.testing.LoadBalancerAccumulatedStatsRequest + (*HookRequest)(nil), // 7: grpc.testing.HookRequest + (*ClientConfigureRequest)(nil), // 8: grpc.testing.ClientConfigureRequest + (*SimpleResponse)(nil), // 9: grpc.testing.SimpleResponse + (*StreamingOutputCallResponse)(nil), // 10: grpc.testing.StreamingOutputCallResponse + (*StreamingInputCallResponse)(nil), // 11: grpc.testing.StreamingInputCallResponse + (*ReconnectInfo)(nil), // 12: grpc.testing.ReconnectInfo + (*LoadBalancerStatsResponse)(nil), // 13: grpc.testing.LoadBalancerStatsResponse + (*LoadBalancerAccumulatedStatsResponse)(nil), // 14: grpc.testing.LoadBalancerAccumulatedStatsResponse + (*HookResponse)(nil), // 15: grpc.testing.HookResponse + (*ClientConfigureResponse)(nil), // 16: grpc.testing.ClientConfigureResponse +} +var file_grpc_testing_test_proto_depIdxs = []int32{ + 0, // 0: grpc.testing.TestService.EmptyCall:input_type -> grpc.testing.Empty + 1, // 1: grpc.testing.TestService.UnaryCall:input_type -> grpc.testing.SimpleRequest + 1, // 2: grpc.testing.TestService.CacheableUnaryCall:input_type -> grpc.testing.SimpleRequest + 2, // 3: grpc.testing.TestService.StreamingOutputCall:input_type -> grpc.testing.StreamingOutputCallRequest + 3, // 4: grpc.testing.TestService.StreamingInputCall:input_type -> grpc.testing.StreamingInputCallRequest + 2, // 5: grpc.testing.TestService.FullDuplexCall:input_type -> grpc.testing.StreamingOutputCallRequest + 2, // 6: grpc.testing.TestService.HalfDuplexCall:input_type -> grpc.testing.StreamingOutputCallRequest + 0, // 7: grpc.testing.TestService.UnimplementedCall:input_type -> grpc.testing.Empty + 0, // 8: grpc.testing.UnimplementedService.UnimplementedCall:input_type -> grpc.testing.Empty + 4, // 9: grpc.testing.ReconnectService.Start:input_type -> grpc.testing.ReconnectParams + 0, // 10: grpc.testing.ReconnectService.Stop:input_type -> grpc.testing.Empty + 5, // 11: grpc.testing.LoadBalancerStatsService.GetClientStats:input_type -> grpc.testing.LoadBalancerStatsRequest + 6, // 12: grpc.testing.LoadBalancerStatsService.GetClientAccumulatedStats:input_type -> grpc.testing.LoadBalancerAccumulatedStatsRequest + 0, // 13: grpc.testing.HookService.Hook:input_type -> grpc.testing.Empty + 0, // 14: grpc.testing.XdsUpdateHealthService.SetServing:input_type -> grpc.testing.Empty + 0, // 15: grpc.testing.XdsUpdateHealthService.SetNotServing:input_type -> grpc.testing.Empty + 7, // 16: grpc.testing.XdsUpdateHealthService.SendHookRequest:input_type -> grpc.testing.HookRequest + 8, // 17: grpc.testing.XdsUpdateClientConfigureService.Configure:input_type -> grpc.testing.ClientConfigureRequest + 0, // 18: grpc.testing.TestService.EmptyCall:output_type -> grpc.testing.Empty + 9, // 19: grpc.testing.TestService.UnaryCall:output_type -> grpc.testing.SimpleResponse + 9, // 20: grpc.testing.TestService.CacheableUnaryCall:output_type -> grpc.testing.SimpleResponse + 10, // 21: grpc.testing.TestService.StreamingOutputCall:output_type -> grpc.testing.StreamingOutputCallResponse + 11, // 22: grpc.testing.TestService.StreamingInputCall:output_type -> grpc.testing.StreamingInputCallResponse + 10, // 23: grpc.testing.TestService.FullDuplexCall:output_type -> grpc.testing.StreamingOutputCallResponse + 10, // 24: grpc.testing.TestService.HalfDuplexCall:output_type -> grpc.testing.StreamingOutputCallResponse + 0, // 25: grpc.testing.TestService.UnimplementedCall:output_type -> grpc.testing.Empty + 0, // 26: grpc.testing.UnimplementedService.UnimplementedCall:output_type -> grpc.testing.Empty + 0, // 27: grpc.testing.ReconnectService.Start:output_type -> grpc.testing.Empty + 12, // 28: grpc.testing.ReconnectService.Stop:output_type -> grpc.testing.ReconnectInfo + 13, // 29: grpc.testing.LoadBalancerStatsService.GetClientStats:output_type -> grpc.testing.LoadBalancerStatsResponse + 14, // 30: grpc.testing.LoadBalancerStatsService.GetClientAccumulatedStats:output_type -> grpc.testing.LoadBalancerAccumulatedStatsResponse + 0, // 31: grpc.testing.HookService.Hook:output_type -> grpc.testing.Empty + 0, // 32: grpc.testing.XdsUpdateHealthService.SetServing:output_type -> grpc.testing.Empty + 0, // 33: grpc.testing.XdsUpdateHealthService.SetNotServing:output_type -> grpc.testing.Empty + 15, // 34: grpc.testing.XdsUpdateHealthService.SendHookRequest:output_type -> grpc.testing.HookResponse + 16, // 35: grpc.testing.XdsUpdateClientConfigureService.Configure:output_type -> grpc.testing.ClientConfigureResponse + 18, // [18:36] is the sub-list for method output_type + 0, // [0:18] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_grpc_testing_test_proto_init() } +func file_grpc_testing_test_proto_init() { + if File_grpc_testing_test_proto != nil { + return + } + file_grpc_testing_empty_proto_init() + file_grpc_testing_messages_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_grpc_testing_test_proto_rawDesc, + NumEnums: 0, + NumMessages: 0, + NumExtensions: 0, + NumServices: 7, + }, + GoTypes: file_grpc_testing_test_proto_goTypes, + DependencyIndexes: file_grpc_testing_test_proto_depIdxs, + }.Build() + File_grpc_testing_test_proto = out.File + file_grpc_testing_test_proto_rawDesc = nil + file_grpc_testing_test_proto_goTypes = nil + file_grpc_testing_test_proto_depIdxs = nil } diff --git a/interop/grpc_testing/test.proto b/interop/grpc_testing/test.proto deleted file mode 100644 index 21451047e513..000000000000 --- a/interop/grpc_testing/test.proto +++ /dev/null @@ -1,235 +0,0 @@ -// Copyright 2017 gRPC authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// An integration test service that covers all the method signature permutations -// of unary/streaming requests/responses. -syntax = "proto3"; - -option go_package = "google.golang.org/grpc/interop/grpc_testing"; - -package grpc.testing; - -message Empty {} - -// The type of payload that should be returned. -enum PayloadType { - // Compressable text format. - COMPRESSABLE = 0; - - // Uncompressable binary format. - UNCOMPRESSABLE = 1; - - // Randomly chosen from all other formats defined in this enum. - RANDOM = 2; -} - -// A block of data, to simply increase gRPC message size. -message Payload { - // The type of data in body. - PayloadType type = 1; - // Primary contents of payload. - bytes body = 2; -} - -// A protobuf representation for grpc status. This is used by test -// clients to specify a status that the server should attempt to return. -message EchoStatus { - int32 code = 1; - string message = 2; -} - -// The type of route that a client took to reach a server w.r.t. gRPCLB. -// The server must fill in "fallback" if it detects that the RPC reached -// the server via the "gRPCLB fallback" path, and "backend" if it detects -// that the RPC reached the server via "gRPCLB backend" path (i.e. if it got -// the address of this server from the gRPCLB server BalanceLoad RPC). Exactly -// how this detection is done is context and server dependant. -enum GrpclbRouteType { - // Server didn't detect the route that a client took to reach it. - GRPCLB_ROUTE_TYPE_UNKNOWN = 0; - // Indicates that a client reached a server via gRPCLB fallback. - GRPCLB_ROUTE_TYPE_FALLBACK = 1; - // Indicates that a client reached a server as a gRPCLB-given backend. - GRPCLB_ROUTE_TYPE_BACKEND = 2; -} - -// Unary request. -message SimpleRequest { - // Desired payload type in the response from the server. - // If response_type is RANDOM, server randomly chooses one from other formats. - PayloadType response_type = 1; - - // Desired payload size in the response from the server. - // If response_type is COMPRESSABLE, this denotes the size before compression. - int32 response_size = 2; - - // Optional input payload sent along with the request. - Payload payload = 3; - - // Whether SimpleResponse should include username. - bool fill_username = 4; - - // Whether SimpleResponse should include OAuth scope. - bool fill_oauth_scope = 5; - - // Whether server should return a given status - EchoStatus response_status = 7; - - // Whether SimpleResponse should include server_id. - bool fill_server_id = 9; - - // Whether SimpleResponse should include grpclb_route_type. - bool fill_grpclb_route_type = 10; -} - -// Unary response, as configured by the request. -message SimpleResponse { - // Payload to increase message size. - Payload payload = 1; - - // The user the request came from, for verifying authentication was - // successful when the client expected it. - string username = 2; - - // OAuth scope. - string oauth_scope = 3; - - // Server ID. This must be unique among different server instances, - // but the same across all RPC's made to a particular server instance. - string server_id = 4; - - // gRPCLB Path. - GrpclbRouteType grpclb_route_type = 5; - - // Server hostname. - string hostname = 6; -} - -// Client-streaming request. -message StreamingInputCallRequest { - // Optional input payload sent along with the request. - Payload payload = 1; - - // Not expecting any payload from the response. -} - -// Client-streaming response. -message StreamingInputCallResponse { - // Aggregated size of payloads received from the client. - int32 aggregated_payload_size = 1; -} - -// Configuration for a particular response. -message ResponseParameters { - // Desired payload sizes in responses from the server. - // If response_type is COMPRESSABLE, this denotes the size before compression. - int32 size = 1; - - // Desired interval between consecutive responses in the response stream in - // microseconds. - int32 interval_us = 2; -} - -// Server-streaming request. -message StreamingOutputCallRequest { - // Desired payload type in the response from the server. - // If response_type is RANDOM, the payload from each response in the stream - // might be of different types. This is to simulate a mixed type of payload - // stream. - PayloadType response_type = 1; - - // Configuration for each expected response message. - repeated ResponseParameters response_parameters = 2; - - // Optional input payload sent along with the request. - Payload payload = 3; - - // Whether server should return a given status - EchoStatus response_status = 7; -} - -// Server-streaming response, as configured by the request and parameters. -message StreamingOutputCallResponse { - // Payload to increase response size. - Payload payload = 1; -} - -// A simple service to test the various types of RPCs and experiment with -// performance with various types of payload. -service TestService { - // One empty request followed by one empty response. - rpc EmptyCall(Empty) returns (Empty); - - // One request followed by one response. - // The server returns the client payload as-is. - rpc UnaryCall(SimpleRequest) returns (SimpleResponse); - - // One request followed by a sequence of responses (streamed download). - // The server returns the payload with client desired type and sizes. - rpc StreamingOutputCall(StreamingOutputCallRequest) - returns (stream StreamingOutputCallResponse); - - // A sequence of requests followed by one response (streamed upload). - // The server returns the aggregated size of client payload as the result. - rpc StreamingInputCall(stream StreamingInputCallRequest) - returns (StreamingInputCallResponse); - - // A sequence of requests with each request served by the server immediately. - // As one request could lead to multiple responses, this interface - // demonstrates the idea of full duplexing. - rpc FullDuplexCall(stream StreamingOutputCallRequest) - returns (stream StreamingOutputCallResponse); - - // A sequence of requests followed by a sequence of responses. - // The server buffers all the client requests and then serves them in order. A - // stream of responses are returned to the client when the server starts with - // first request. - rpc HalfDuplexCall(stream StreamingOutputCallRequest) - returns (stream StreamingOutputCallResponse); -} - -// A simple service NOT implemented at servers so clients can test for -// that case. -service UnimplementedService { - // A call that no server should implement - rpc UnimplementedCall(grpc.testing.Empty) returns (grpc.testing.Empty); -} - -message LoadBalancerStatsRequest { - // Request stats for the next num_rpcs sent by client. - int32 num_rpcs = 1; - // If num_rpcs have not completed within timeout_sec, return partial results. - int32 timeout_sec = 2; -} - -message LoadBalancerStatsResponse { - message RpcsByPeer { - // The number of completed RPCs for each peer. - map rpcs_by_peer = 1; - } - - // The number of completed RPCs for each peer. - map rpcs_by_peer = 1; - // The number of RPCs that failed to record a remote peer. - int32 num_failures = 2; - // The number of completed RPCs for each method (UnaryCall or EmptyCall). - map rpcs_by_method = 3; -} - -// A service used to obtain stats for verifying LB behavior. -service LoadBalancerStatsService { - // Gets the backend distribution for RPCs sent by a test client. - rpc GetClientStats(LoadBalancerStatsRequest) - returns (LoadBalancerStatsResponse) {} -} diff --git a/interop/grpc_testing/test_grpc.pb.go b/interop/grpc_testing/test_grpc.pb.go index d71f6d2a46b9..2d0181a7f34e 100644 --- a/interop/grpc_testing/test_grpc.pb.go +++ b/interop/grpc_testing/test_grpc.pb.go @@ -1,4 +1,25 @@ +// Copyright 2015-2016 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// An integration test service that covers all the method signature permutations +// of unary/streaming requests/responses. + // Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.3.0 +// - protoc v4.22.0 +// source: grpc/testing/test.proto package grpc_testing @@ -11,8 +32,20 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. const _ = grpc.SupportPackageIsVersion7 +const ( + TestService_EmptyCall_FullMethodName = "/grpc.testing.TestService/EmptyCall" + TestService_UnaryCall_FullMethodName = "/grpc.testing.TestService/UnaryCall" + TestService_CacheableUnaryCall_FullMethodName = "/grpc.testing.TestService/CacheableUnaryCall" + TestService_StreamingOutputCall_FullMethodName = "/grpc.testing.TestService/StreamingOutputCall" + TestService_StreamingInputCall_FullMethodName = "/grpc.testing.TestService/StreamingInputCall" + TestService_FullDuplexCall_FullMethodName = "/grpc.testing.TestService/FullDuplexCall" + TestService_HalfDuplexCall_FullMethodName = "/grpc.testing.TestService/HalfDuplexCall" + TestService_UnimplementedCall_FullMethodName = "/grpc.testing.TestService/UnimplementedCall" +) + // TestServiceClient is the client API for TestService service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. @@ -20,8 +53,11 @@ type TestServiceClient interface { // One empty request followed by one empty response. EmptyCall(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error) // One request followed by one response. - // The server returns the client payload as-is. UnaryCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (*SimpleResponse, error) + // One request followed by one response. Response has cache control + // headers set such that a caching HTTP proxy (such as GFE) can + // satisfy subsequent requests. + CacheableUnaryCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (*SimpleResponse, error) // One request followed by a sequence of responses (streamed download). // The server returns the payload with client desired type and sizes. StreamingOutputCall(ctx context.Context, in *StreamingOutputCallRequest, opts ...grpc.CallOption) (TestService_StreamingOutputCallClient, error) @@ -37,6 +73,9 @@ type TestServiceClient interface { // stream of responses are returned to the client when the server starts with // first request. HalfDuplexCall(ctx context.Context, opts ...grpc.CallOption) (TestService_HalfDuplexCallClient, error) + // The test server will not implement this method. It will be used + // to test the behavior when clients call unimplemented methods. + UnimplementedCall(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error) } type testServiceClient struct { @@ -47,39 +86,35 @@ func NewTestServiceClient(cc grpc.ClientConnInterface) TestServiceClient { return &testServiceClient{cc} } -var testServiceEmptyCallStreamDesc = &grpc.StreamDesc{ - StreamName: "EmptyCall", -} - func (c *testServiceClient) EmptyCall(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error) { out := new(Empty) - err := c.cc.Invoke(ctx, "/grpc.testing.TestService/EmptyCall", in, out, opts...) + err := c.cc.Invoke(ctx, TestService_EmptyCall_FullMethodName, in, out, opts...) if err != nil { return nil, err } return out, nil } -var testServiceUnaryCallStreamDesc = &grpc.StreamDesc{ - StreamName: "UnaryCall", -} - func (c *testServiceClient) UnaryCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (*SimpleResponse, error) { out := new(SimpleResponse) - err := c.cc.Invoke(ctx, "/grpc.testing.TestService/UnaryCall", in, out, opts...) + err := c.cc.Invoke(ctx, TestService_UnaryCall_FullMethodName, in, out, opts...) if err != nil { return nil, err } return out, nil } -var testServiceStreamingOutputCallStreamDesc = &grpc.StreamDesc{ - StreamName: "StreamingOutputCall", - ServerStreams: true, +func (c *testServiceClient) CacheableUnaryCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (*SimpleResponse, error) { + out := new(SimpleResponse) + err := c.cc.Invoke(ctx, TestService_CacheableUnaryCall_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil } func (c *testServiceClient) StreamingOutputCall(ctx context.Context, in *StreamingOutputCallRequest, opts ...grpc.CallOption) (TestService_StreamingOutputCallClient, error) { - stream, err := c.cc.NewStream(ctx, testServiceStreamingOutputCallStreamDesc, "/grpc.testing.TestService/StreamingOutputCall", opts...) + stream, err := c.cc.NewStream(ctx, &TestService_ServiceDesc.Streams[0], TestService_StreamingOutputCall_FullMethodName, opts...) if err != nil { return nil, err } @@ -110,13 +145,8 @@ func (x *testServiceStreamingOutputCallClient) Recv() (*StreamingOutputCallRespo return m, nil } -var testServiceStreamingInputCallStreamDesc = &grpc.StreamDesc{ - StreamName: "StreamingInputCall", - ClientStreams: true, -} - func (c *testServiceClient) StreamingInputCall(ctx context.Context, opts ...grpc.CallOption) (TestService_StreamingInputCallClient, error) { - stream, err := c.cc.NewStream(ctx, testServiceStreamingInputCallStreamDesc, "/grpc.testing.TestService/StreamingInputCall", opts...) + stream, err := c.cc.NewStream(ctx, &TestService_ServiceDesc.Streams[1], TestService_StreamingInputCall_FullMethodName, opts...) if err != nil { return nil, err } @@ -149,14 +179,8 @@ func (x *testServiceStreamingInputCallClient) CloseAndRecv() (*StreamingInputCal return m, nil } -var testServiceFullDuplexCallStreamDesc = &grpc.StreamDesc{ - StreamName: "FullDuplexCall", - ServerStreams: true, - ClientStreams: true, -} - func (c *testServiceClient) FullDuplexCall(ctx context.Context, opts ...grpc.CallOption) (TestService_FullDuplexCallClient, error) { - stream, err := c.cc.NewStream(ctx, testServiceFullDuplexCallStreamDesc, "/grpc.testing.TestService/FullDuplexCall", opts...) + stream, err := c.cc.NewStream(ctx, &TestService_ServiceDesc.Streams[2], TestService_FullDuplexCall_FullMethodName, opts...) if err != nil { return nil, err } @@ -186,14 +210,8 @@ func (x *testServiceFullDuplexCallClient) Recv() (*StreamingOutputCallResponse, return m, nil } -var testServiceHalfDuplexCallStreamDesc = &grpc.StreamDesc{ - StreamName: "HalfDuplexCall", - ServerStreams: true, - ClientStreams: true, -} - func (c *testServiceClient) HalfDuplexCall(ctx context.Context, opts ...grpc.CallOption) (TestService_HalfDuplexCallClient, error) { - stream, err := c.cc.NewStream(ctx, testServiceHalfDuplexCallStreamDesc, "/grpc.testing.TestService/HalfDuplexCall", opts...) + stream, err := c.cc.NewStream(ctx, &TestService_ServiceDesc.Streams[3], TestService_HalfDuplexCall_FullMethodName, opts...) if err != nil { return nil, err } @@ -223,100 +241,149 @@ func (x *testServiceHalfDuplexCallClient) Recv() (*StreamingOutputCallResponse, return m, nil } -// TestServiceService is the service API for TestService service. -// Fields should be assigned to their respective handler implementations only before -// RegisterTestServiceService is called. Any unassigned fields will result in the -// handler for that method returning an Unimplemented error. -type TestServiceService struct { +func (c *testServiceClient) UnimplementedCall(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error) { + out := new(Empty) + err := c.cc.Invoke(ctx, TestService_UnimplementedCall_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// TestServiceServer is the server API for TestService service. +// All implementations must embed UnimplementedTestServiceServer +// for forward compatibility +type TestServiceServer interface { // One empty request followed by one empty response. - EmptyCall func(context.Context, *Empty) (*Empty, error) + EmptyCall(context.Context, *Empty) (*Empty, error) // One request followed by one response. - // The server returns the client payload as-is. - UnaryCall func(context.Context, *SimpleRequest) (*SimpleResponse, error) + UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error) + // One request followed by one response. Response has cache control + // headers set such that a caching HTTP proxy (such as GFE) can + // satisfy subsequent requests. + CacheableUnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error) // One request followed by a sequence of responses (streamed download). // The server returns the payload with client desired type and sizes. - StreamingOutputCall func(*StreamingOutputCallRequest, TestService_StreamingOutputCallServer) error + StreamingOutputCall(*StreamingOutputCallRequest, TestService_StreamingOutputCallServer) error // A sequence of requests followed by one response (streamed upload). // The server returns the aggregated size of client payload as the result. - StreamingInputCall func(TestService_StreamingInputCallServer) error + StreamingInputCall(TestService_StreamingInputCallServer) error // A sequence of requests with each request served by the server immediately. // As one request could lead to multiple responses, this interface // demonstrates the idea of full duplexing. - FullDuplexCall func(TestService_FullDuplexCallServer) error + FullDuplexCall(TestService_FullDuplexCallServer) error // A sequence of requests followed by a sequence of responses. // The server buffers all the client requests and then serves them in order. A // stream of responses are returned to the client when the server starts with // first request. - HalfDuplexCall func(TestService_HalfDuplexCallServer) error + HalfDuplexCall(TestService_HalfDuplexCallServer) error + // The test server will not implement this method. It will be used + // to test the behavior when clients call unimplemented methods. + UnimplementedCall(context.Context, *Empty) (*Empty, error) + mustEmbedUnimplementedTestServiceServer() } -func (s *TestServiceService) emptyCall(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - if s.EmptyCall == nil { - return nil, status.Errorf(codes.Unimplemented, "method EmptyCall not implemented") - } +// UnimplementedTestServiceServer must be embedded to have forward compatible implementations. +type UnimplementedTestServiceServer struct { +} + +func (UnimplementedTestServiceServer) EmptyCall(context.Context, *Empty) (*Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method EmptyCall not implemented") +} +func (UnimplementedTestServiceServer) UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method UnaryCall not implemented") +} +func (UnimplementedTestServiceServer) CacheableUnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method CacheableUnaryCall not implemented") +} +func (UnimplementedTestServiceServer) StreamingOutputCall(*StreamingOutputCallRequest, TestService_StreamingOutputCallServer) error { + return status.Errorf(codes.Unimplemented, "method StreamingOutputCall not implemented") +} +func (UnimplementedTestServiceServer) StreamingInputCall(TestService_StreamingInputCallServer) error { + return status.Errorf(codes.Unimplemented, "method StreamingInputCall not implemented") +} +func (UnimplementedTestServiceServer) FullDuplexCall(TestService_FullDuplexCallServer) error { + return status.Errorf(codes.Unimplemented, "method FullDuplexCall not implemented") +} +func (UnimplementedTestServiceServer) HalfDuplexCall(TestService_HalfDuplexCallServer) error { + return status.Errorf(codes.Unimplemented, "method HalfDuplexCall not implemented") +} +func (UnimplementedTestServiceServer) UnimplementedCall(context.Context, *Empty) (*Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method UnimplementedCall not implemented") +} +func (UnimplementedTestServiceServer) mustEmbedUnimplementedTestServiceServer() {} + +// UnsafeTestServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to TestServiceServer will +// result in compilation errors. +type UnsafeTestServiceServer interface { + mustEmbedUnimplementedTestServiceServer() +} + +func RegisterTestServiceServer(s grpc.ServiceRegistrar, srv TestServiceServer) { + s.RegisterService(&TestService_ServiceDesc, srv) +} + +func _TestService_EmptyCall_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(Empty) if err := dec(in); err != nil { return nil, err } if interceptor == nil { - return s.EmptyCall(ctx, in) + return srv.(TestServiceServer).EmptyCall(ctx, in) } info := &grpc.UnaryServerInfo{ - Server: s, - FullMethod: "/grpc.testing.TestService/EmptyCall", + Server: srv, + FullMethod: TestService_EmptyCall_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return s.EmptyCall(ctx, req.(*Empty)) + return srv.(TestServiceServer).EmptyCall(ctx, req.(*Empty)) } return interceptor(ctx, in, info, handler) } -func (s *TestServiceService) unaryCall(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - if s.UnaryCall == nil { - return nil, status.Errorf(codes.Unimplemented, "method UnaryCall not implemented") - } + +func _TestService_UnaryCall_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(SimpleRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { - return s.UnaryCall(ctx, in) + return srv.(TestServiceServer).UnaryCall(ctx, in) } info := &grpc.UnaryServerInfo{ - Server: s, - FullMethod: "/grpc.testing.TestService/UnaryCall", + Server: srv, + FullMethod: TestService_UnaryCall_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return s.UnaryCall(ctx, req.(*SimpleRequest)) + return srv.(TestServiceServer).UnaryCall(ctx, req.(*SimpleRequest)) } return interceptor(ctx, in, info, handler) } -func (s *TestServiceService) streamingOutputCall(_ interface{}, stream grpc.ServerStream) error { - if s.StreamingOutputCall == nil { - return status.Errorf(codes.Unimplemented, "method StreamingOutputCall not implemented") + +func _TestService_CacheableUnaryCall_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(SimpleRequest) + if err := dec(in); err != nil { + return nil, err } - m := new(StreamingOutputCallRequest) - if err := stream.RecvMsg(m); err != nil { - return err + if interceptor == nil { + return srv.(TestServiceServer).CacheableUnaryCall(ctx, in) } - return s.StreamingOutputCall(m, &testServiceStreamingOutputCallServer{stream}) -} -func (s *TestServiceService) streamingInputCall(_ interface{}, stream grpc.ServerStream) error { - if s.StreamingInputCall == nil { - return status.Errorf(codes.Unimplemented, "method StreamingInputCall not implemented") + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: TestService_CacheableUnaryCall_FullMethodName, } - return s.StreamingInputCall(&testServiceStreamingInputCallServer{stream}) -} -func (s *TestServiceService) fullDuplexCall(_ interface{}, stream grpc.ServerStream) error { - if s.FullDuplexCall == nil { - return status.Errorf(codes.Unimplemented, "method FullDuplexCall not implemented") + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(TestServiceServer).CacheableUnaryCall(ctx, req.(*SimpleRequest)) } - return s.FullDuplexCall(&testServiceFullDuplexCallServer{stream}) + return interceptor(ctx, in, info, handler) } -func (s *TestServiceService) halfDuplexCall(_ interface{}, stream grpc.ServerStream) error { - if s.HalfDuplexCall == nil { - return status.Errorf(codes.Unimplemented, "method HalfDuplexCall not implemented") + +func _TestService_StreamingOutputCall_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(StreamingOutputCallRequest) + if err := stream.RecvMsg(m); err != nil { + return err } - return s.HalfDuplexCall(&testServiceHalfDuplexCallServer{stream}) + return srv.(TestServiceServer).StreamingOutputCall(m, &testServiceStreamingOutputCallServer{stream}) } type TestService_StreamingOutputCallServer interface { @@ -332,6 +399,10 @@ func (x *testServiceStreamingOutputCallServer) Send(m *StreamingOutputCallRespon return x.ServerStream.SendMsg(m) } +func _TestService_StreamingInputCall_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(TestServiceServer).StreamingInputCall(&testServiceStreamingInputCallServer{stream}) +} + type TestService_StreamingInputCallServer interface { SendAndClose(*StreamingInputCallResponse) error Recv() (*StreamingInputCallRequest, error) @@ -354,6 +425,10 @@ func (x *testServiceStreamingInputCallServer) Recv() (*StreamingInputCallRequest return m, nil } +func _TestService_FullDuplexCall_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(TestServiceServer).FullDuplexCall(&testServiceFullDuplexCallServer{stream}) +} + type TestService_FullDuplexCallServer interface { Send(*StreamingOutputCallResponse) error Recv() (*StreamingOutputCallRequest, error) @@ -376,6 +451,10 @@ func (x *testServiceFullDuplexCallServer) Recv() (*StreamingOutputCallRequest, e return m, nil } +func _TestService_HalfDuplexCall_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(TestServiceServer).HalfDuplexCall(&testServiceHalfDuplexCallServer{stream}) +} + type TestService_HalfDuplexCallServer interface { Send(*StreamingOutputCallResponse) error Recv() (*StreamingOutputCallRequest, error) @@ -398,118 +477,79 @@ func (x *testServiceHalfDuplexCallServer) Recv() (*StreamingOutputCallRequest, e return m, nil } -// RegisterTestServiceService registers a service implementation with a gRPC server. -func RegisterTestServiceService(s grpc.ServiceRegistrar, srv *TestServiceService) { - sd := grpc.ServiceDesc{ - ServiceName: "grpc.testing.TestService", - Methods: []grpc.MethodDesc{ - { - MethodName: "EmptyCall", - Handler: srv.emptyCall, - }, - { - MethodName: "UnaryCall", - Handler: srv.unaryCall, - }, +func _TestService_UnimplementedCall_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Empty) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(TestServiceServer).UnimplementedCall(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: TestService_UnimplementedCall_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(TestServiceServer).UnimplementedCall(ctx, req.(*Empty)) + } + return interceptor(ctx, in, info, handler) +} + +// TestService_ServiceDesc is the grpc.ServiceDesc for TestService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var TestService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.testing.TestService", + HandlerType: (*TestServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "EmptyCall", + Handler: _TestService_EmptyCall_Handler, }, - Streams: []grpc.StreamDesc{ - { - StreamName: "StreamingOutputCall", - Handler: srv.streamingOutputCall, - ServerStreams: true, - }, - { - StreamName: "StreamingInputCall", - Handler: srv.streamingInputCall, - ClientStreams: true, - }, - { - StreamName: "FullDuplexCall", - Handler: srv.fullDuplexCall, - ServerStreams: true, - ClientStreams: true, - }, - { - StreamName: "HalfDuplexCall", - Handler: srv.halfDuplexCall, - ServerStreams: true, - ClientStreams: true, - }, + { + MethodName: "UnaryCall", + Handler: _TestService_UnaryCall_Handler, }, - Metadata: "interop/grpc_testing/test.proto", - } - - s.RegisterService(&sd, nil) -} - -// NewTestServiceService creates a new TestServiceService containing the -// implemented methods of the TestService service in s. Any unimplemented -// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client. -// This includes situations where the method handler is misspelled or has the wrong -// signature. For this reason, this function should be used with great care and -// is not recommended to be used by most users. -func NewTestServiceService(s interface{}) *TestServiceService { - ns := &TestServiceService{} - if h, ok := s.(interface { - EmptyCall(context.Context, *Empty) (*Empty, error) - }); ok { - ns.EmptyCall = h.EmptyCall - } - if h, ok := s.(interface { - UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error) - }); ok { - ns.UnaryCall = h.UnaryCall - } - if h, ok := s.(interface { - StreamingOutputCall(*StreamingOutputCallRequest, TestService_StreamingOutputCallServer) error - }); ok { - ns.StreamingOutputCall = h.StreamingOutputCall - } - if h, ok := s.(interface { - StreamingInputCall(TestService_StreamingInputCallServer) error - }); ok { - ns.StreamingInputCall = h.StreamingInputCall - } - if h, ok := s.(interface { - FullDuplexCall(TestService_FullDuplexCallServer) error - }); ok { - ns.FullDuplexCall = h.FullDuplexCall - } - if h, ok := s.(interface { - HalfDuplexCall(TestService_HalfDuplexCallServer) error - }); ok { - ns.HalfDuplexCall = h.HalfDuplexCall - } - return ns -} - -// UnstableTestServiceService is the service API for TestService service. -// New methods may be added to this interface if they are added to the service -// definition, which is not a backward-compatible change. For this reason, -// use of this type is not recommended. -type UnstableTestServiceService interface { - // One empty request followed by one empty response. - EmptyCall(context.Context, *Empty) (*Empty, error) - // One request followed by one response. - // The server returns the client payload as-is. - UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error) - // One request followed by a sequence of responses (streamed download). - // The server returns the payload with client desired type and sizes. - StreamingOutputCall(*StreamingOutputCallRequest, TestService_StreamingOutputCallServer) error - // A sequence of requests followed by one response (streamed upload). - // The server returns the aggregated size of client payload as the result. - StreamingInputCall(TestService_StreamingInputCallServer) error - // A sequence of requests with each request served by the server immediately. - // As one request could lead to multiple responses, this interface - // demonstrates the idea of full duplexing. - FullDuplexCall(TestService_FullDuplexCallServer) error - // A sequence of requests followed by a sequence of responses. - // The server buffers all the client requests and then serves them in order. A - // stream of responses are returned to the client when the server starts with - // first request. - HalfDuplexCall(TestService_HalfDuplexCallServer) error + { + MethodName: "CacheableUnaryCall", + Handler: _TestService_CacheableUnaryCall_Handler, + }, + { + MethodName: "UnimplementedCall", + Handler: _TestService_UnimplementedCall_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "StreamingOutputCall", + Handler: _TestService_StreamingOutputCall_Handler, + ServerStreams: true, + }, + { + StreamName: "StreamingInputCall", + Handler: _TestService_StreamingInputCall_Handler, + ClientStreams: true, + }, + { + StreamName: "FullDuplexCall", + Handler: _TestService_FullDuplexCall_Handler, + ServerStreams: true, + ClientStreams: true, + }, + { + StreamName: "HalfDuplexCall", + Handler: _TestService_HalfDuplexCall_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "grpc/testing/test.proto", } +const ( + UnimplementedService_UnimplementedCall_FullMethodName = "/grpc.testing.UnimplementedService/UnimplementedCall" +) + // UnimplementedServiceClient is the client API for UnimplementedService service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. @@ -526,97 +566,218 @@ func NewUnimplementedServiceClient(cc grpc.ClientConnInterface) UnimplementedSer return &unimplementedServiceClient{cc} } -var unimplementedServiceUnimplementedCallStreamDesc = &grpc.StreamDesc{ - StreamName: "UnimplementedCall", -} - func (c *unimplementedServiceClient) UnimplementedCall(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error) { out := new(Empty) - err := c.cc.Invoke(ctx, "/grpc.testing.UnimplementedService/UnimplementedCall", in, out, opts...) + err := c.cc.Invoke(ctx, UnimplementedService_UnimplementedCall_FullMethodName, in, out, opts...) if err != nil { return nil, err } return out, nil } -// UnimplementedServiceService is the service API for UnimplementedService service. -// Fields should be assigned to their respective handler implementations only before -// RegisterUnimplementedServiceService is called. Any unassigned fields will result in the -// handler for that method returning an Unimplemented error. -type UnimplementedServiceService struct { +// UnimplementedServiceServer is the server API for UnimplementedService service. +// All implementations must embed UnimplementedUnimplementedServiceServer +// for forward compatibility +type UnimplementedServiceServer interface { // A call that no server should implement - UnimplementedCall func(context.Context, *Empty) (*Empty, error) + UnimplementedCall(context.Context, *Empty) (*Empty, error) + mustEmbedUnimplementedUnimplementedServiceServer() } -func (s *UnimplementedServiceService) unimplementedCall(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - if s.UnimplementedCall == nil { - return nil, status.Errorf(codes.Unimplemented, "method UnimplementedCall not implemented") - } +// UnimplementedUnimplementedServiceServer must be embedded to have forward compatible implementations. +type UnimplementedUnimplementedServiceServer struct { +} + +func (UnimplementedUnimplementedServiceServer) UnimplementedCall(context.Context, *Empty) (*Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method UnimplementedCall not implemented") +} +func (UnimplementedUnimplementedServiceServer) mustEmbedUnimplementedUnimplementedServiceServer() {} + +// UnsafeUnimplementedServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to UnimplementedServiceServer will +// result in compilation errors. +type UnsafeUnimplementedServiceServer interface { + mustEmbedUnimplementedUnimplementedServiceServer() +} + +func RegisterUnimplementedServiceServer(s grpc.ServiceRegistrar, srv UnimplementedServiceServer) { + s.RegisterService(&UnimplementedService_ServiceDesc, srv) +} + +func _UnimplementedService_UnimplementedCall_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(Empty) if err := dec(in); err != nil { return nil, err } if interceptor == nil { - return s.UnimplementedCall(ctx, in) + return srv.(UnimplementedServiceServer).UnimplementedCall(ctx, in) } info := &grpc.UnaryServerInfo{ - Server: s, - FullMethod: "/grpc.testing.UnimplementedService/UnimplementedCall", + Server: srv, + FullMethod: UnimplementedService_UnimplementedCall_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return s.UnimplementedCall(ctx, req.(*Empty)) + return srv.(UnimplementedServiceServer).UnimplementedCall(ctx, req.(*Empty)) } return interceptor(ctx, in, info, handler) } -// RegisterUnimplementedServiceService registers a service implementation with a gRPC server. -func RegisterUnimplementedServiceService(s grpc.ServiceRegistrar, srv *UnimplementedServiceService) { - sd := grpc.ServiceDesc{ - ServiceName: "grpc.testing.UnimplementedService", - Methods: []grpc.MethodDesc{ - { - MethodName: "UnimplementedCall", - Handler: srv.unimplementedCall, - }, +// UnimplementedService_ServiceDesc is the grpc.ServiceDesc for UnimplementedService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var UnimplementedService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.testing.UnimplementedService", + HandlerType: (*UnimplementedServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "UnimplementedCall", + Handler: _UnimplementedService_UnimplementedCall_Handler, }, - Streams: []grpc.StreamDesc{}, - Metadata: "interop/grpc_testing/test.proto", + }, + Streams: []grpc.StreamDesc{}, + Metadata: "grpc/testing/test.proto", +} + +const ( + ReconnectService_Start_FullMethodName = "/grpc.testing.ReconnectService/Start" + ReconnectService_Stop_FullMethodName = "/grpc.testing.ReconnectService/Stop" +) + +// ReconnectServiceClient is the client API for ReconnectService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type ReconnectServiceClient interface { + Start(ctx context.Context, in *ReconnectParams, opts ...grpc.CallOption) (*Empty, error) + Stop(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*ReconnectInfo, error) +} + +type reconnectServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewReconnectServiceClient(cc grpc.ClientConnInterface) ReconnectServiceClient { + return &reconnectServiceClient{cc} +} + +func (c *reconnectServiceClient) Start(ctx context.Context, in *ReconnectParams, opts ...grpc.CallOption) (*Empty, error) { + out := new(Empty) + err := c.cc.Invoke(ctx, ReconnectService_Start_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *reconnectServiceClient) Stop(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*ReconnectInfo, error) { + out := new(ReconnectInfo) + err := c.cc.Invoke(ctx, ReconnectService_Stop_FullMethodName, in, out, opts...) + if err != nil { + return nil, err } + return out, nil +} + +// ReconnectServiceServer is the server API for ReconnectService service. +// All implementations must embed UnimplementedReconnectServiceServer +// for forward compatibility +type ReconnectServiceServer interface { + Start(context.Context, *ReconnectParams) (*Empty, error) + Stop(context.Context, *Empty) (*ReconnectInfo, error) + mustEmbedUnimplementedReconnectServiceServer() +} + +// UnimplementedReconnectServiceServer must be embedded to have forward compatible implementations. +type UnimplementedReconnectServiceServer struct { +} + +func (UnimplementedReconnectServiceServer) Start(context.Context, *ReconnectParams) (*Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method Start not implemented") +} +func (UnimplementedReconnectServiceServer) Stop(context.Context, *Empty) (*ReconnectInfo, error) { + return nil, status.Errorf(codes.Unimplemented, "method Stop not implemented") +} +func (UnimplementedReconnectServiceServer) mustEmbedUnimplementedReconnectServiceServer() {} - s.RegisterService(&sd, nil) +// UnsafeReconnectServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to ReconnectServiceServer will +// result in compilation errors. +type UnsafeReconnectServiceServer interface { + mustEmbedUnimplementedReconnectServiceServer() } -// NewUnimplementedServiceService creates a new UnimplementedServiceService containing the -// implemented methods of the UnimplementedService service in s. Any unimplemented -// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client. -// This includes situations where the method handler is misspelled or has the wrong -// signature. For this reason, this function should be used with great care and -// is not recommended to be used by most users. -func NewUnimplementedServiceService(s interface{}) *UnimplementedServiceService { - ns := &UnimplementedServiceService{} - if h, ok := s.(interface { - UnimplementedCall(context.Context, *Empty) (*Empty, error) - }); ok { - ns.UnimplementedCall = h.UnimplementedCall +func RegisterReconnectServiceServer(s grpc.ServiceRegistrar, srv ReconnectServiceServer) { + s.RegisterService(&ReconnectService_ServiceDesc, srv) +} + +func _ReconnectService_Start_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ReconnectParams) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ReconnectServiceServer).Start(ctx, in) } - return ns + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ReconnectService_Start_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ReconnectServiceServer).Start(ctx, req.(*ReconnectParams)) + } + return interceptor(ctx, in, info, handler) } -// UnstableUnimplementedServiceService is the service API for UnimplementedService service. -// New methods may be added to this interface if they are added to the service -// definition, which is not a backward-compatible change. For this reason, -// use of this type is not recommended. -type UnstableUnimplementedServiceService interface { - // A call that no server should implement - UnimplementedCall(context.Context, *Empty) (*Empty, error) +func _ReconnectService_Stop_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Empty) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ReconnectServiceServer).Stop(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ReconnectService_Stop_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ReconnectServiceServer).Stop(ctx, req.(*Empty)) + } + return interceptor(ctx, in, info, handler) } +// ReconnectService_ServiceDesc is the grpc.ServiceDesc for ReconnectService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var ReconnectService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.testing.ReconnectService", + HandlerType: (*ReconnectServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Start", + Handler: _ReconnectService_Start_Handler, + }, + { + MethodName: "Stop", + Handler: _ReconnectService_Stop_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "grpc/testing/test.proto", +} + +const ( + LoadBalancerStatsService_GetClientStats_FullMethodName = "/grpc.testing.LoadBalancerStatsService/GetClientStats" + LoadBalancerStatsService_GetClientAccumulatedStats_FullMethodName = "/grpc.testing.LoadBalancerStatsService/GetClientAccumulatedStats" +) + // LoadBalancerStatsServiceClient is the client API for LoadBalancerStatsService service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. type LoadBalancerStatsServiceClient interface { // Gets the backend distribution for RPCs sent by a test client. GetClientStats(ctx context.Context, in *LoadBalancerStatsRequest, opts ...grpc.CallOption) (*LoadBalancerStatsResponse, error) + // Gets the accumulated stats for RPCs sent by a test client. + GetClientAccumulatedStats(ctx context.Context, in *LoadBalancerAccumulatedStatsRequest, opts ...grpc.CallOption) (*LoadBalancerAccumulatedStatsResponse, error) } type loadBalancerStatsServiceClient struct { @@ -627,87 +788,459 @@ func NewLoadBalancerStatsServiceClient(cc grpc.ClientConnInterface) LoadBalancer return &loadBalancerStatsServiceClient{cc} } -var loadBalancerStatsServiceGetClientStatsStreamDesc = &grpc.StreamDesc{ - StreamName: "GetClientStats", -} - func (c *loadBalancerStatsServiceClient) GetClientStats(ctx context.Context, in *LoadBalancerStatsRequest, opts ...grpc.CallOption) (*LoadBalancerStatsResponse, error) { out := new(LoadBalancerStatsResponse) - err := c.cc.Invoke(ctx, "/grpc.testing.LoadBalancerStatsService/GetClientStats", in, out, opts...) + err := c.cc.Invoke(ctx, LoadBalancerStatsService_GetClientStats_FullMethodName, in, out, opts...) if err != nil { return nil, err } return out, nil } -// LoadBalancerStatsServiceService is the service API for LoadBalancerStatsService service. -// Fields should be assigned to their respective handler implementations only before -// RegisterLoadBalancerStatsServiceService is called. Any unassigned fields will result in the -// handler for that method returning an Unimplemented error. -type LoadBalancerStatsServiceService struct { +func (c *loadBalancerStatsServiceClient) GetClientAccumulatedStats(ctx context.Context, in *LoadBalancerAccumulatedStatsRequest, opts ...grpc.CallOption) (*LoadBalancerAccumulatedStatsResponse, error) { + out := new(LoadBalancerAccumulatedStatsResponse) + err := c.cc.Invoke(ctx, LoadBalancerStatsService_GetClientAccumulatedStats_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// LoadBalancerStatsServiceServer is the server API for LoadBalancerStatsService service. +// All implementations must embed UnimplementedLoadBalancerStatsServiceServer +// for forward compatibility +type LoadBalancerStatsServiceServer interface { // Gets the backend distribution for RPCs sent by a test client. - GetClientStats func(context.Context, *LoadBalancerStatsRequest) (*LoadBalancerStatsResponse, error) + GetClientStats(context.Context, *LoadBalancerStatsRequest) (*LoadBalancerStatsResponse, error) + // Gets the accumulated stats for RPCs sent by a test client. + GetClientAccumulatedStats(context.Context, *LoadBalancerAccumulatedStatsRequest) (*LoadBalancerAccumulatedStatsResponse, error) + mustEmbedUnimplementedLoadBalancerStatsServiceServer() } -func (s *LoadBalancerStatsServiceService) getClientStats(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - if s.GetClientStats == nil { - return nil, status.Errorf(codes.Unimplemented, "method GetClientStats not implemented") - } +// UnimplementedLoadBalancerStatsServiceServer must be embedded to have forward compatible implementations. +type UnimplementedLoadBalancerStatsServiceServer struct { +} + +func (UnimplementedLoadBalancerStatsServiceServer) GetClientStats(context.Context, *LoadBalancerStatsRequest) (*LoadBalancerStatsResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetClientStats not implemented") +} +func (UnimplementedLoadBalancerStatsServiceServer) GetClientAccumulatedStats(context.Context, *LoadBalancerAccumulatedStatsRequest) (*LoadBalancerAccumulatedStatsResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetClientAccumulatedStats not implemented") +} +func (UnimplementedLoadBalancerStatsServiceServer) mustEmbedUnimplementedLoadBalancerStatsServiceServer() { +} + +// UnsafeLoadBalancerStatsServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to LoadBalancerStatsServiceServer will +// result in compilation errors. +type UnsafeLoadBalancerStatsServiceServer interface { + mustEmbedUnimplementedLoadBalancerStatsServiceServer() +} + +func RegisterLoadBalancerStatsServiceServer(s grpc.ServiceRegistrar, srv LoadBalancerStatsServiceServer) { + s.RegisterService(&LoadBalancerStatsService_ServiceDesc, srv) +} + +func _LoadBalancerStatsService_GetClientStats_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(LoadBalancerStatsRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { - return s.GetClientStats(ctx, in) + return srv.(LoadBalancerStatsServiceServer).GetClientStats(ctx, in) } info := &grpc.UnaryServerInfo{ - Server: s, - FullMethod: "/grpc.testing.LoadBalancerStatsService/GetClientStats", + Server: srv, + FullMethod: LoadBalancerStatsService_GetClientStats_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return s.GetClientStats(ctx, req.(*LoadBalancerStatsRequest)) + return srv.(LoadBalancerStatsServiceServer).GetClientStats(ctx, req.(*LoadBalancerStatsRequest)) } return interceptor(ctx, in, info, handler) } -// RegisterLoadBalancerStatsServiceService registers a service implementation with a gRPC server. -func RegisterLoadBalancerStatsServiceService(s grpc.ServiceRegistrar, srv *LoadBalancerStatsServiceService) { - sd := grpc.ServiceDesc{ - ServiceName: "grpc.testing.LoadBalancerStatsService", - Methods: []grpc.MethodDesc{ - { - MethodName: "GetClientStats", - Handler: srv.getClientStats, - }, +func _LoadBalancerStatsService_GetClientAccumulatedStats_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(LoadBalancerAccumulatedStatsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(LoadBalancerStatsServiceServer).GetClientAccumulatedStats(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: LoadBalancerStatsService_GetClientAccumulatedStats_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(LoadBalancerStatsServiceServer).GetClientAccumulatedStats(ctx, req.(*LoadBalancerAccumulatedStatsRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// LoadBalancerStatsService_ServiceDesc is the grpc.ServiceDesc for LoadBalancerStatsService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var LoadBalancerStatsService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.testing.LoadBalancerStatsService", + HandlerType: (*LoadBalancerStatsServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "GetClientStats", + Handler: _LoadBalancerStatsService_GetClientStats_Handler, + }, + { + MethodName: "GetClientAccumulatedStats", + Handler: _LoadBalancerStatsService_GetClientAccumulatedStats_Handler, }, - Streams: []grpc.StreamDesc{}, - Metadata: "interop/grpc_testing/test.proto", + }, + Streams: []grpc.StreamDesc{}, + Metadata: "grpc/testing/test.proto", +} + +const ( + HookService_Hook_FullMethodName = "/grpc.testing.HookService/Hook" +) + +// HookServiceClient is the client API for HookService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type HookServiceClient interface { + Hook(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error) +} + +type hookServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewHookServiceClient(cc grpc.ClientConnInterface) HookServiceClient { + return &hookServiceClient{cc} +} + +func (c *hookServiceClient) Hook(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error) { + out := new(Empty) + err := c.cc.Invoke(ctx, HookService_Hook_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// HookServiceServer is the server API for HookService service. +// All implementations must embed UnimplementedHookServiceServer +// for forward compatibility +type HookServiceServer interface { + Hook(context.Context, *Empty) (*Empty, error) + mustEmbedUnimplementedHookServiceServer() +} + +// UnimplementedHookServiceServer must be embedded to have forward compatible implementations. +type UnimplementedHookServiceServer struct { +} + +func (UnimplementedHookServiceServer) Hook(context.Context, *Empty) (*Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method Hook not implemented") +} +func (UnimplementedHookServiceServer) mustEmbedUnimplementedHookServiceServer() {} + +// UnsafeHookServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to HookServiceServer will +// result in compilation errors. +type UnsafeHookServiceServer interface { + mustEmbedUnimplementedHookServiceServer() +} + +func RegisterHookServiceServer(s grpc.ServiceRegistrar, srv HookServiceServer) { + s.RegisterService(&HookService_ServiceDesc, srv) +} + +func _HookService_Hook_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Empty) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(HookServiceServer).Hook(ctx, in) } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: HookService_Hook_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(HookServiceServer).Hook(ctx, req.(*Empty)) + } + return interceptor(ctx, in, info, handler) +} - s.RegisterService(&sd, nil) +// HookService_ServiceDesc is the grpc.ServiceDesc for HookService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var HookService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.testing.HookService", + HandlerType: (*HookServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Hook", + Handler: _HookService_Hook_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "grpc/testing/test.proto", } -// NewLoadBalancerStatsServiceService creates a new LoadBalancerStatsServiceService containing the -// implemented methods of the LoadBalancerStatsService service in s. Any unimplemented -// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client. -// This includes situations where the method handler is misspelled or has the wrong -// signature. For this reason, this function should be used with great care and -// is not recommended to be used by most users. -func NewLoadBalancerStatsServiceService(s interface{}) *LoadBalancerStatsServiceService { - ns := &LoadBalancerStatsServiceService{} - if h, ok := s.(interface { - GetClientStats(context.Context, *LoadBalancerStatsRequest) (*LoadBalancerStatsResponse, error) - }); ok { - ns.GetClientStats = h.GetClientStats +const ( + XdsUpdateHealthService_SetServing_FullMethodName = "/grpc.testing.XdsUpdateHealthService/SetServing" + XdsUpdateHealthService_SetNotServing_FullMethodName = "/grpc.testing.XdsUpdateHealthService/SetNotServing" + XdsUpdateHealthService_SendHookRequest_FullMethodName = "/grpc.testing.XdsUpdateHealthService/SendHookRequest" +) + +// XdsUpdateHealthServiceClient is the client API for XdsUpdateHealthService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type XdsUpdateHealthServiceClient interface { + SetServing(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error) + SetNotServing(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error) + SendHookRequest(ctx context.Context, in *HookRequest, opts ...grpc.CallOption) (*HookResponse, error) +} + +type xdsUpdateHealthServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewXdsUpdateHealthServiceClient(cc grpc.ClientConnInterface) XdsUpdateHealthServiceClient { + return &xdsUpdateHealthServiceClient{cc} +} + +func (c *xdsUpdateHealthServiceClient) SetServing(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error) { + out := new(Empty) + err := c.cc.Invoke(ctx, XdsUpdateHealthService_SetServing_FullMethodName, in, out, opts...) + if err != nil { + return nil, err } - return ns + return out, nil } -// UnstableLoadBalancerStatsServiceService is the service API for LoadBalancerStatsService service. -// New methods may be added to this interface if they are added to the service -// definition, which is not a backward-compatible change. For this reason, -// use of this type is not recommended. -type UnstableLoadBalancerStatsServiceService interface { - // Gets the backend distribution for RPCs sent by a test client. - GetClientStats(context.Context, *LoadBalancerStatsRequest) (*LoadBalancerStatsResponse, error) +func (c *xdsUpdateHealthServiceClient) SetNotServing(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error) { + out := new(Empty) + err := c.cc.Invoke(ctx, XdsUpdateHealthService_SetNotServing_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *xdsUpdateHealthServiceClient) SendHookRequest(ctx context.Context, in *HookRequest, opts ...grpc.CallOption) (*HookResponse, error) { + out := new(HookResponse) + err := c.cc.Invoke(ctx, XdsUpdateHealthService_SendHookRequest_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// XdsUpdateHealthServiceServer is the server API for XdsUpdateHealthService service. +// All implementations must embed UnimplementedXdsUpdateHealthServiceServer +// for forward compatibility +type XdsUpdateHealthServiceServer interface { + SetServing(context.Context, *Empty) (*Empty, error) + SetNotServing(context.Context, *Empty) (*Empty, error) + SendHookRequest(context.Context, *HookRequest) (*HookResponse, error) + mustEmbedUnimplementedXdsUpdateHealthServiceServer() +} + +// UnimplementedXdsUpdateHealthServiceServer must be embedded to have forward compatible implementations. +type UnimplementedXdsUpdateHealthServiceServer struct { +} + +func (UnimplementedXdsUpdateHealthServiceServer) SetServing(context.Context, *Empty) (*Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method SetServing not implemented") +} +func (UnimplementedXdsUpdateHealthServiceServer) SetNotServing(context.Context, *Empty) (*Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method SetNotServing not implemented") +} +func (UnimplementedXdsUpdateHealthServiceServer) SendHookRequest(context.Context, *HookRequest) (*HookResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method SendHookRequest not implemented") +} +func (UnimplementedXdsUpdateHealthServiceServer) mustEmbedUnimplementedXdsUpdateHealthServiceServer() { +} + +// UnsafeXdsUpdateHealthServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to XdsUpdateHealthServiceServer will +// result in compilation errors. +type UnsafeXdsUpdateHealthServiceServer interface { + mustEmbedUnimplementedXdsUpdateHealthServiceServer() +} + +func RegisterXdsUpdateHealthServiceServer(s grpc.ServiceRegistrar, srv XdsUpdateHealthServiceServer) { + s.RegisterService(&XdsUpdateHealthService_ServiceDesc, srv) +} + +func _XdsUpdateHealthService_SetServing_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Empty) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(XdsUpdateHealthServiceServer).SetServing(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: XdsUpdateHealthService_SetServing_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(XdsUpdateHealthServiceServer).SetServing(ctx, req.(*Empty)) + } + return interceptor(ctx, in, info, handler) +} + +func _XdsUpdateHealthService_SetNotServing_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Empty) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(XdsUpdateHealthServiceServer).SetNotServing(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: XdsUpdateHealthService_SetNotServing_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(XdsUpdateHealthServiceServer).SetNotServing(ctx, req.(*Empty)) + } + return interceptor(ctx, in, info, handler) +} + +func _XdsUpdateHealthService_SendHookRequest_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(HookRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(XdsUpdateHealthServiceServer).SendHookRequest(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: XdsUpdateHealthService_SendHookRequest_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(XdsUpdateHealthServiceServer).SendHookRequest(ctx, req.(*HookRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// XdsUpdateHealthService_ServiceDesc is the grpc.ServiceDesc for XdsUpdateHealthService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var XdsUpdateHealthService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.testing.XdsUpdateHealthService", + HandlerType: (*XdsUpdateHealthServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "SetServing", + Handler: _XdsUpdateHealthService_SetServing_Handler, + }, + { + MethodName: "SetNotServing", + Handler: _XdsUpdateHealthService_SetNotServing_Handler, + }, + { + MethodName: "SendHookRequest", + Handler: _XdsUpdateHealthService_SendHookRequest_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "grpc/testing/test.proto", +} + +const ( + XdsUpdateClientConfigureService_Configure_FullMethodName = "/grpc.testing.XdsUpdateClientConfigureService/Configure" +) + +// XdsUpdateClientConfigureServiceClient is the client API for XdsUpdateClientConfigureService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type XdsUpdateClientConfigureServiceClient interface { + // Update the tes client's configuration. + Configure(ctx context.Context, in *ClientConfigureRequest, opts ...grpc.CallOption) (*ClientConfigureResponse, error) +} + +type xdsUpdateClientConfigureServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewXdsUpdateClientConfigureServiceClient(cc grpc.ClientConnInterface) XdsUpdateClientConfigureServiceClient { + return &xdsUpdateClientConfigureServiceClient{cc} +} + +func (c *xdsUpdateClientConfigureServiceClient) Configure(ctx context.Context, in *ClientConfigureRequest, opts ...grpc.CallOption) (*ClientConfigureResponse, error) { + out := new(ClientConfigureResponse) + err := c.cc.Invoke(ctx, XdsUpdateClientConfigureService_Configure_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// XdsUpdateClientConfigureServiceServer is the server API for XdsUpdateClientConfigureService service. +// All implementations must embed UnimplementedXdsUpdateClientConfigureServiceServer +// for forward compatibility +type XdsUpdateClientConfigureServiceServer interface { + // Update the tes client's configuration. + Configure(context.Context, *ClientConfigureRequest) (*ClientConfigureResponse, error) + mustEmbedUnimplementedXdsUpdateClientConfigureServiceServer() +} + +// UnimplementedXdsUpdateClientConfigureServiceServer must be embedded to have forward compatible implementations. +type UnimplementedXdsUpdateClientConfigureServiceServer struct { +} + +func (UnimplementedXdsUpdateClientConfigureServiceServer) Configure(context.Context, *ClientConfigureRequest) (*ClientConfigureResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method Configure not implemented") +} +func (UnimplementedXdsUpdateClientConfigureServiceServer) mustEmbedUnimplementedXdsUpdateClientConfigureServiceServer() { +} + +// UnsafeXdsUpdateClientConfigureServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to XdsUpdateClientConfigureServiceServer will +// result in compilation errors. +type UnsafeXdsUpdateClientConfigureServiceServer interface { + mustEmbedUnimplementedXdsUpdateClientConfigureServiceServer() +} + +func RegisterXdsUpdateClientConfigureServiceServer(s grpc.ServiceRegistrar, srv XdsUpdateClientConfigureServiceServer) { + s.RegisterService(&XdsUpdateClientConfigureService_ServiceDesc, srv) +} + +func _XdsUpdateClientConfigureService_Configure_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ClientConfigureRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(XdsUpdateClientConfigureServiceServer).Configure(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: XdsUpdateClientConfigureService_Configure_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(XdsUpdateClientConfigureServiceServer).Configure(ctx, req.(*ClientConfigureRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// XdsUpdateClientConfigureService_ServiceDesc is the grpc.ServiceDesc for XdsUpdateClientConfigureService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var XdsUpdateClientConfigureService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.testing.XdsUpdateClientConfigureService", + HandlerType: (*XdsUpdateClientConfigureServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Configure", + Handler: _XdsUpdateClientConfigureService_Configure_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "grpc/testing/test.proto", } diff --git a/interop/grpc_testing/worker_service.pb.go b/interop/grpc_testing/worker_service.pb.go new file mode 100644 index 000000000000..1feccc9f7654 --- /dev/null +++ b/interop/grpc_testing/worker_service.pb.go @@ -0,0 +1,118 @@ +// Copyright 2015 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// An integration test service that covers all the method signature permutations +// of unary/streaming requests/responses. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.31.0 +// protoc v4.22.0 +// source: grpc/testing/worker_service.proto + +package grpc_testing + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +var File_grpc_testing_worker_service_proto protoreflect.FileDescriptor + +var file_grpc_testing_worker_service_proto_rawDesc = []byte{ + 0x0a, 0x21, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x77, + 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, + 0x67, 0x1a, 0x1a, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, + 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x32, 0x97, 0x02, + 0x0a, 0x0d, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, + 0x45, 0x0a, 0x09, 0x52, 0x75, 0x6e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x18, 0x2e, 0x67, + 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x65, 0x72, 0x76, + 0x65, 0x72, 0x41, 0x72, 0x67, 0x73, 0x1a, 0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, + 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x28, 0x01, 0x30, 0x01, 0x12, 0x45, 0x0a, 0x09, 0x52, 0x75, 0x6e, 0x43, 0x6c, 0x69, + 0x65, 0x6e, 0x74, 0x12, 0x18, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, + 0x6e, 0x67, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x41, 0x72, 0x67, 0x73, 0x1a, 0x1a, 0x2e, + 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43, 0x6c, 0x69, + 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x28, 0x01, 0x30, 0x01, 0x12, 0x42, 0x0a, + 0x09, 0x43, 0x6f, 0x72, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x19, 0x2e, 0x67, 0x72, 0x70, + 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43, 0x6f, 0x72, 0x65, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, + 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x43, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x34, 0x0a, 0x0a, 0x51, 0x75, 0x69, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x12, + 0x12, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x56, + 0x6f, 0x69, 0x64, 0x1a, 0x12, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, + 0x6e, 0x67, 0x2e, 0x56, 0x6f, 0x69, 0x64, 0x42, 0x27, 0x0a, 0x0f, 0x69, 0x6f, 0x2e, 0x67, 0x72, + 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x42, 0x12, 0x57, 0x6f, 0x72, 0x6b, + 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, + 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var file_grpc_testing_worker_service_proto_goTypes = []interface{}{ + (*ServerArgs)(nil), // 0: grpc.testing.ServerArgs + (*ClientArgs)(nil), // 1: grpc.testing.ClientArgs + (*CoreRequest)(nil), // 2: grpc.testing.CoreRequest + (*Void)(nil), // 3: grpc.testing.Void + (*ServerStatus)(nil), // 4: grpc.testing.ServerStatus + (*ClientStatus)(nil), // 5: grpc.testing.ClientStatus + (*CoreResponse)(nil), // 6: grpc.testing.CoreResponse +} +var file_grpc_testing_worker_service_proto_depIdxs = []int32{ + 0, // 0: grpc.testing.WorkerService.RunServer:input_type -> grpc.testing.ServerArgs + 1, // 1: grpc.testing.WorkerService.RunClient:input_type -> grpc.testing.ClientArgs + 2, // 2: grpc.testing.WorkerService.CoreCount:input_type -> grpc.testing.CoreRequest + 3, // 3: grpc.testing.WorkerService.QuitWorker:input_type -> grpc.testing.Void + 4, // 4: grpc.testing.WorkerService.RunServer:output_type -> grpc.testing.ServerStatus + 5, // 5: grpc.testing.WorkerService.RunClient:output_type -> grpc.testing.ClientStatus + 6, // 6: grpc.testing.WorkerService.CoreCount:output_type -> grpc.testing.CoreResponse + 3, // 7: grpc.testing.WorkerService.QuitWorker:output_type -> grpc.testing.Void + 4, // [4:8] is the sub-list for method output_type + 0, // [0:4] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_grpc_testing_worker_service_proto_init() } +func file_grpc_testing_worker_service_proto_init() { + if File_grpc_testing_worker_service_proto != nil { + return + } + file_grpc_testing_control_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_grpc_testing_worker_service_proto_rawDesc, + NumEnums: 0, + NumMessages: 0, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_grpc_testing_worker_service_proto_goTypes, + DependencyIndexes: file_grpc_testing_worker_service_proto_depIdxs, + }.Build() + File_grpc_testing_worker_service_proto = out.File + file_grpc_testing_worker_service_proto_rawDesc = nil + file_grpc_testing_worker_service_proto_goTypes = nil + file_grpc_testing_worker_service_proto_depIdxs = nil +} diff --git a/interop/grpc_testing/worker_service_grpc.pb.go b/interop/grpc_testing/worker_service_grpc.pb.go new file mode 100644 index 000000000000..1de7f09f841a --- /dev/null +++ b/interop/grpc_testing/worker_service_grpc.pb.go @@ -0,0 +1,330 @@ +// Copyright 2015 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// An integration test service that covers all the method signature permutations +// of unary/streaming requests/responses. + +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.3.0 +// - protoc v4.22.0 +// source: grpc/testing/worker_service.proto + +package grpc_testing + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 + +const ( + WorkerService_RunServer_FullMethodName = "/grpc.testing.WorkerService/RunServer" + WorkerService_RunClient_FullMethodName = "/grpc.testing.WorkerService/RunClient" + WorkerService_CoreCount_FullMethodName = "/grpc.testing.WorkerService/CoreCount" + WorkerService_QuitWorker_FullMethodName = "/grpc.testing.WorkerService/QuitWorker" +) + +// WorkerServiceClient is the client API for WorkerService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type WorkerServiceClient interface { + // Start server with specified workload. + // First request sent specifies the ServerConfig followed by ServerStatus + // response. After that, a "Mark" can be sent anytime to request the latest + // stats. Closing the stream will initiate shutdown of the test server + // and once the shutdown has finished, the OK status is sent to terminate + // this RPC. + RunServer(ctx context.Context, opts ...grpc.CallOption) (WorkerService_RunServerClient, error) + // Start client with specified workload. + // First request sent specifies the ClientConfig followed by ClientStatus + // response. After that, a "Mark" can be sent anytime to request the latest + // stats. Closing the stream will initiate shutdown of the test client + // and once the shutdown has finished, the OK status is sent to terminate + // this RPC. + RunClient(ctx context.Context, opts ...grpc.CallOption) (WorkerService_RunClientClient, error) + // Just return the core count - unary call + CoreCount(ctx context.Context, in *CoreRequest, opts ...grpc.CallOption) (*CoreResponse, error) + // Quit this worker + QuitWorker(ctx context.Context, in *Void, opts ...grpc.CallOption) (*Void, error) +} + +type workerServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewWorkerServiceClient(cc grpc.ClientConnInterface) WorkerServiceClient { + return &workerServiceClient{cc} +} + +func (c *workerServiceClient) RunServer(ctx context.Context, opts ...grpc.CallOption) (WorkerService_RunServerClient, error) { + stream, err := c.cc.NewStream(ctx, &WorkerService_ServiceDesc.Streams[0], WorkerService_RunServer_FullMethodName, opts...) + if err != nil { + return nil, err + } + x := &workerServiceRunServerClient{stream} + return x, nil +} + +type WorkerService_RunServerClient interface { + Send(*ServerArgs) error + Recv() (*ServerStatus, error) + grpc.ClientStream +} + +type workerServiceRunServerClient struct { + grpc.ClientStream +} + +func (x *workerServiceRunServerClient) Send(m *ServerArgs) error { + return x.ClientStream.SendMsg(m) +} + +func (x *workerServiceRunServerClient) Recv() (*ServerStatus, error) { + m := new(ServerStatus) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *workerServiceClient) RunClient(ctx context.Context, opts ...grpc.CallOption) (WorkerService_RunClientClient, error) { + stream, err := c.cc.NewStream(ctx, &WorkerService_ServiceDesc.Streams[1], WorkerService_RunClient_FullMethodName, opts...) + if err != nil { + return nil, err + } + x := &workerServiceRunClientClient{stream} + return x, nil +} + +type WorkerService_RunClientClient interface { + Send(*ClientArgs) error + Recv() (*ClientStatus, error) + grpc.ClientStream +} + +type workerServiceRunClientClient struct { + grpc.ClientStream +} + +func (x *workerServiceRunClientClient) Send(m *ClientArgs) error { + return x.ClientStream.SendMsg(m) +} + +func (x *workerServiceRunClientClient) Recv() (*ClientStatus, error) { + m := new(ClientStatus) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *workerServiceClient) CoreCount(ctx context.Context, in *CoreRequest, opts ...grpc.CallOption) (*CoreResponse, error) { + out := new(CoreResponse) + err := c.cc.Invoke(ctx, WorkerService_CoreCount_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *workerServiceClient) QuitWorker(ctx context.Context, in *Void, opts ...grpc.CallOption) (*Void, error) { + out := new(Void) + err := c.cc.Invoke(ctx, WorkerService_QuitWorker_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// WorkerServiceServer is the server API for WorkerService service. +// All implementations must embed UnimplementedWorkerServiceServer +// for forward compatibility +type WorkerServiceServer interface { + // Start server with specified workload. + // First request sent specifies the ServerConfig followed by ServerStatus + // response. After that, a "Mark" can be sent anytime to request the latest + // stats. Closing the stream will initiate shutdown of the test server + // and once the shutdown has finished, the OK status is sent to terminate + // this RPC. + RunServer(WorkerService_RunServerServer) error + // Start client with specified workload. + // First request sent specifies the ClientConfig followed by ClientStatus + // response. After that, a "Mark" can be sent anytime to request the latest + // stats. Closing the stream will initiate shutdown of the test client + // and once the shutdown has finished, the OK status is sent to terminate + // this RPC. + RunClient(WorkerService_RunClientServer) error + // Just return the core count - unary call + CoreCount(context.Context, *CoreRequest) (*CoreResponse, error) + // Quit this worker + QuitWorker(context.Context, *Void) (*Void, error) + mustEmbedUnimplementedWorkerServiceServer() +} + +// UnimplementedWorkerServiceServer must be embedded to have forward compatible implementations. +type UnimplementedWorkerServiceServer struct { +} + +func (UnimplementedWorkerServiceServer) RunServer(WorkerService_RunServerServer) error { + return status.Errorf(codes.Unimplemented, "method RunServer not implemented") +} +func (UnimplementedWorkerServiceServer) RunClient(WorkerService_RunClientServer) error { + return status.Errorf(codes.Unimplemented, "method RunClient not implemented") +} +func (UnimplementedWorkerServiceServer) CoreCount(context.Context, *CoreRequest) (*CoreResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method CoreCount not implemented") +} +func (UnimplementedWorkerServiceServer) QuitWorker(context.Context, *Void) (*Void, error) { + return nil, status.Errorf(codes.Unimplemented, "method QuitWorker not implemented") +} +func (UnimplementedWorkerServiceServer) mustEmbedUnimplementedWorkerServiceServer() {} + +// UnsafeWorkerServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to WorkerServiceServer will +// result in compilation errors. +type UnsafeWorkerServiceServer interface { + mustEmbedUnimplementedWorkerServiceServer() +} + +func RegisterWorkerServiceServer(s grpc.ServiceRegistrar, srv WorkerServiceServer) { + s.RegisterService(&WorkerService_ServiceDesc, srv) +} + +func _WorkerService_RunServer_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(WorkerServiceServer).RunServer(&workerServiceRunServerServer{stream}) +} + +type WorkerService_RunServerServer interface { + Send(*ServerStatus) error + Recv() (*ServerArgs, error) + grpc.ServerStream +} + +type workerServiceRunServerServer struct { + grpc.ServerStream +} + +func (x *workerServiceRunServerServer) Send(m *ServerStatus) error { + return x.ServerStream.SendMsg(m) +} + +func (x *workerServiceRunServerServer) Recv() (*ServerArgs, error) { + m := new(ServerArgs) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func _WorkerService_RunClient_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(WorkerServiceServer).RunClient(&workerServiceRunClientServer{stream}) +} + +type WorkerService_RunClientServer interface { + Send(*ClientStatus) error + Recv() (*ClientArgs, error) + grpc.ServerStream +} + +type workerServiceRunClientServer struct { + grpc.ServerStream +} + +func (x *workerServiceRunClientServer) Send(m *ClientStatus) error { + return x.ServerStream.SendMsg(m) +} + +func (x *workerServiceRunClientServer) Recv() (*ClientArgs, error) { + m := new(ClientArgs) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func _WorkerService_CoreCount_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CoreRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(WorkerServiceServer).CoreCount(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: WorkerService_CoreCount_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(WorkerServiceServer).CoreCount(ctx, req.(*CoreRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _WorkerService_QuitWorker_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Void) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(WorkerServiceServer).QuitWorker(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: WorkerService_QuitWorker_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(WorkerServiceServer).QuitWorker(ctx, req.(*Void)) + } + return interceptor(ctx, in, info, handler) +} + +// WorkerService_ServiceDesc is the grpc.ServiceDesc for WorkerService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var WorkerService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.testing.WorkerService", + HandlerType: (*WorkerServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "CoreCount", + Handler: _WorkerService_CoreCount_Handler, + }, + { + MethodName: "QuitWorker", + Handler: _WorkerService_QuitWorker_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "RunServer", + Handler: _WorkerService_RunServer_Handler, + ServerStreams: true, + ClientStreams: true, + }, + { + StreamName: "RunClient", + Handler: _WorkerService_RunClient_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "grpc/testing/worker_service.proto", +} diff --git a/interop/grpclb_fallback/client.go b/interop/grpclb_fallback/client_linux.go similarity index 62% rename from interop/grpclb_fallback/client.go rename to interop/grpclb_fallback/client_linux.go index 853c9a4353f1..b1cfde71134e 100644 --- a/interop/grpclb_fallback/client.go +++ b/interop/grpclb_fallback/client_linux.go @@ -1,5 +1,3 @@ -// +build linux - /* * * Copyright 2019 gRPC authors. @@ -37,25 +35,26 @@ import ( "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/alts" "google.golang.org/grpc/credentials/google" + _ "google.golang.org/grpc/xds/googledirectpath" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" testpb "google.golang.org/grpc/interop/grpc_testing" ) var ( - customCredentialsType = flag.String("custom_credentials_type", "", "Client creds to use") - serverURI = flag.String("server_uri", "dns:///staging-grpc-directpath-fallback-test.googleapis.com:443", "The server host name") - unrouteLBAndBackendAddrsCmd = flag.String("unroute_lb_and_backend_addrs_cmd", "", "Command to make LB and backend address unroutable") - blackholeLBAndBackendAddrsCmd = flag.String("blackhole_lb_and_backend_addrs_cmd", "", "Command to make LB and backend addresses blackholed") - testCase = flag.String("test_case", "", + customCredentialsType = flag.String("custom_credentials_type", "", "Client creds to use") + serverURI = flag.String("server_uri", "dns:///staging-grpc-directpath-fallback-test.googleapis.com:443", "The server host name") + induceFallbackCmd = flag.String("induce_fallback_cmd", "", "Command to induce fallback e.g. by making certain addresses unroutable") + fallbackDeadlineSeconds = flag.Int("fallback_deadline_seconds", 1, "How long to wait for fallback to happen after induce_fallback_cmd") + testCase = flag.String("test_case", "", `Configure different test cases. Valid options are: - fast_fallback_before_startup : LB/backend connections fail fast before RPC's have been made; - fast_fallback_after_startup : LB/backend connections fail fast after RPC's have been made; - slow_fallback_before_startup : LB/backend connections black hole before RPC's have been made; - slow_fallback_after_startup : LB/backend connections black hole after RPC's have been made;`) + fallback_before_startup : LB/backend connections fail before RPC's have been made; + fallback_after_startup : LB/backend connections fail after RPC's have been made;`) infoLog = log.New(os.Stderr, "INFO: ", log.Ldate|log.Ltime|log.Lshortfile) errorLog = log.New(os.Stderr, "ERROR: ", log.Ldate|log.Ltime|log.Lshortfile) ) -func doRPCAndGetPath(client testpb.TestServiceClient, timeout time.Duration) testpb.GrpclbRouteType { +func doRPCAndGetPath(client testgrpc.TestServiceClient, timeout time.Duration) testpb.GrpclbRouteType { infoLog.Printf("doRPCAndGetPath timeout:%v\n", timeout) ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() @@ -98,7 +97,6 @@ func dialTCPUserTimeout(ctx context.Context, addr string) (net.Conn, error) { func createTestConn() *grpc.ClientConn { opts := []grpc.DialOption{ grpc.WithContextDialer(dialTCPUserTimeout), - grpc.WithBlock(), } switch *customCredentialsType { case "tls": @@ -128,11 +126,11 @@ func runCmd(command string) { } } -func waitForFallbackAndDoRPCs(client testpb.TestServiceClient, fallbackDeadline time.Time) { +func waitForFallbackAndDoRPCs(client testgrpc.TestServiceClient, fallbackDeadline time.Time) { fallbackRetryCount := 0 fellBack := false for time.Now().Before(fallbackDeadline) { - g := doRPCAndGetPath(client, 1*time.Second) + g := doRPCAndGetPath(client, 20*time.Second) if g == testpb.GrpclbRouteType_GRPCLB_ROUTE_TYPE_FALLBACK { infoLog.Println("Made one successul RPC to a fallback. Now expect the same for the rest.") fellBack = true @@ -155,69 +153,39 @@ func waitForFallbackAndDoRPCs(client testpb.TestServiceClient, fallbackDeadline } } -func doFastFallbackBeforeStartup() { - runCmd(*unrouteLBAndBackendAddrsCmd) - fallbackDeadline := time.Now().Add(5 * time.Second) - conn := createTestConn() - defer conn.Close() - client := testpb.NewTestServiceClient(conn) - waitForFallbackAndDoRPCs(client, fallbackDeadline) -} - -func doSlowFallbackBeforeStartup() { - runCmd(*blackholeLBAndBackendAddrsCmd) - fallbackDeadline := time.Now().Add(20 * time.Second) - conn := createTestConn() - defer conn.Close() - client := testpb.NewTestServiceClient(conn) - waitForFallbackAndDoRPCs(client, fallbackDeadline) -} - -func doFastFallbackAfterStartup() { +func doFallbackBeforeStartup() { + runCmd(*induceFallbackCmd) + fallbackDeadline := time.Now().Add(time.Duration(*fallbackDeadlineSeconds) * time.Second) conn := createTestConn() defer conn.Close() - client := testpb.NewTestServiceClient(conn) - if g := doRPCAndGetPath(client, 20*time.Second); g != testpb.GrpclbRouteType_GRPCLB_ROUTE_TYPE_BACKEND { - errorLog.Fatalf("Expected RPC to take grpclb route type BACKEND. Got: %v", g) - } - runCmd(*unrouteLBAndBackendAddrsCmd) - fallbackDeadline := time.Now().Add(40 * time.Second) + client := testgrpc.NewTestServiceClient(conn) waitForFallbackAndDoRPCs(client, fallbackDeadline) } -func doSlowFallbackAfterStartup() { +func doFallbackAfterStartup() { conn := createTestConn() defer conn.Close() - client := testpb.NewTestServiceClient(conn) + client := testgrpc.NewTestServiceClient(conn) if g := doRPCAndGetPath(client, 20*time.Second); g != testpb.GrpclbRouteType_GRPCLB_ROUTE_TYPE_BACKEND { errorLog.Fatalf("Expected RPC to take grpclb route type BACKEND. Got: %v", g) } - runCmd(*blackholeLBAndBackendAddrsCmd) - fallbackDeadline := time.Now().Add(40 * time.Second) + runCmd(*induceFallbackCmd) + fallbackDeadline := time.Now().Add(time.Duration(*fallbackDeadlineSeconds) * time.Second) waitForFallbackAndDoRPCs(client, fallbackDeadline) } func main() { flag.Parse() - if len(*unrouteLBAndBackendAddrsCmd) == 0 { - errorLog.Fatalf("--unroute_lb_and_backend_addrs_cmd unset") - } - if len(*blackholeLBAndBackendAddrsCmd) == 0 { - errorLog.Fatalf("--blackhole_lb_and_backend_addrs_cmd unset") + if len(*induceFallbackCmd) == 0 { + errorLog.Fatalf("--induce_fallback_cmd unset") } switch *testCase { - case "fast_fallback_before_startup": - doFastFallbackBeforeStartup() - log.Printf("FastFallbackBeforeStartup done!\n") - case "fast_fallback_after_startup": - doFastFallbackAfterStartup() - log.Printf("FastFallbackAfterStartup done!\n") - case "slow_fallback_before_startup": - doSlowFallbackBeforeStartup() - log.Printf("SlowFallbackBeforeStartup done!\n") - case "slow_fallback_after_startup": - doSlowFallbackAfterStartup() - log.Printf("SlowFallbackAfterStartup done!\n") + case "fallback_before_startup": + doFallbackBeforeStartup() + log.Printf("FallbackBeforeStartup done!\n") + case "fallback_after_startup": + doFallbackAfterStartup() + log.Printf("FallbackAfterStartup done!\n") default: errorLog.Fatalf("Unsupported test case: %v", *testCase) } diff --git a/interop/http2/negative_http2_client.go b/interop/http2/negative_http2_client.go index 8ead7f49c6b4..b8c1d522009e 100644 --- a/interop/http2/negative_http2_client.go +++ b/interop/http2/negative_http2_client.go @@ -33,10 +33,13 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/grpclog" "google.golang.org/grpc/interop" - testpb "google.golang.org/grpc/interop/grpc_testing" "google.golang.org/grpc/status" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" ) var ( @@ -66,7 +69,7 @@ func largeSimpleRequest() *testpb.SimpleRequest { } // sends two unary calls. The server asserts that the calls use different connections. -func goaway(tc testpb.TestServiceClient) { +func goaway(tc testgrpc.TestServiceClient) { interop.DoLargeUnaryCall(tc) // sleep to ensure that the client has time to recv the GOAWAY. // TODO(ncteisen): make this less hacky. @@ -74,45 +77,45 @@ func goaway(tc testpb.TestServiceClient) { interop.DoLargeUnaryCall(tc) } -func rstAfterHeader(tc testpb.TestServiceClient) { +func rstAfterHeader(tc testgrpc.TestServiceClient) { req := largeSimpleRequest() reply, err := tc.UnaryCall(context.Background(), req) if reply != nil { - logger.Fatalf("Client received reply despite server sending rst stream after header") + logger.Fatal("Client received reply despite server sending rst stream after header") } if status.Code(err) != codes.Internal { logger.Fatalf("%v.UnaryCall() = _, %v, want _, %v", tc, status.Code(err), codes.Internal) } } -func rstDuringData(tc testpb.TestServiceClient) { +func rstDuringData(tc testgrpc.TestServiceClient) { req := largeSimpleRequest() reply, err := tc.UnaryCall(context.Background(), req) if reply != nil { - logger.Fatalf("Client received reply despite server sending rst stream during data") + logger.Fatal("Client received reply despite server sending rst stream during data") } if status.Code(err) != codes.Unknown { logger.Fatalf("%v.UnaryCall() = _, %v, want _, %v", tc, status.Code(err), codes.Unknown) } } -func rstAfterData(tc testpb.TestServiceClient) { +func rstAfterData(tc testgrpc.TestServiceClient) { req := largeSimpleRequest() reply, err := tc.UnaryCall(context.Background(), req) if reply != nil { - logger.Fatalf("Client received reply despite server sending rst stream after data") + logger.Fatal("Client received reply despite server sending rst stream after data") } if status.Code(err) != codes.Internal { logger.Fatalf("%v.UnaryCall() = _, %v, want _, %v", tc, status.Code(err), codes.Internal) } } -func ping(tc testpb.TestServiceClient) { +func ping(tc testgrpc.TestServiceClient) { // The server will assert that every ping it sends was ACK-ed by the client. interop.DoLargeUnaryCall(tc) } -func maxStreams(tc testpb.TestServiceClient) { +func maxStreams(tc testgrpc.TestServiceClient) { interop.DoLargeUnaryCall(tc) var wg sync.WaitGroup for i := 0; i < 15; i++ { @@ -129,13 +132,13 @@ func main() { flag.Parse() serverAddr := net.JoinHostPort(*serverHost, strconv.Itoa(*serverPort)) var opts []grpc.DialOption - opts = append(opts, grpc.WithInsecure()) + opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) conn, err := grpc.Dial(serverAddr, opts...) if err != nil { logger.Fatalf("Fail to dial: %v", err) } defer conn.Close() - tc := testpb.NewTestServiceClient(conn) + tc := testgrpc.NewTestServiceClient(conn) switch *testCase { case "goaway": goaway(tc) diff --git a/interop/interop_test.sh b/interop/interop_test.sh index 5aeaa2aa10a8..7fc290a12c6b 100755 --- a/interop/interop_test.sh +++ b/interop/interop_test.sh @@ -36,13 +36,33 @@ clean () { } fail () { - echo "$(tput setaf 1) $1 $(tput sgr 0)" + echo "$(tput setaf 1) $(date): $1 $(tput sgr 0)" clean exit 1 } pass () { - echo "$(tput setaf 2) $1 $(tput sgr 0)" + echo "$(tput setaf 2) $(date): $1 $(tput sgr 0)" +} + +withTimeout () { + timer=$1 + shift + + # Run command in the background. + cmd=$(printf '%q ' "$@") + eval "$cmd" & + wpid=$! + # Kill after $timer seconds. + sleep $timer && kill $wpid & + kpid=$! + # Wait for the background thread. + wait $wpid + res=$? + # Kill the killer pid in case it's still running. + kill $kpid || true + wait $kpid || true + return $res } # Don't run some tests that need a special environment: @@ -70,24 +90,40 @@ CASES=( "custom_metadata" "unimplemented_method" "unimplemented_service" + "orca_per_rpc" + "orca_oob" ) # Build server +echo "$(tput setaf 4) $(date): building server $(tput sgr 0)" if ! go build -o /dev/null ./interop/server; then fail "failed to build server" else pass "successfully built server" fi +# Build client +echo "$(tput setaf 4) $(date): building client $(tput sgr 0)" +if ! go build -o /dev/null ./interop/client; then + fail "failed to build client" +else + pass "successfully built client" +fi + # Start server SERVER_LOG="$(mktemp)" -go run ./interop/server --use_tls &> $SERVER_LOG & +GRPC_GO_LOG_SEVERITY_LEVEL=info go run ./interop/server --use_tls &> $SERVER_LOG & for case in ${CASES[@]}; do - echo "$(tput setaf 4) testing: ${case} $(tput sgr 0)" + echo "$(tput setaf 4) $(date): testing: ${case} $(tput sgr 0)" CLIENT_LOG="$(mktemp)" - if ! timeout 20 go run ./interop/client --use_tls --server_host_override=foo.test.google.fr --use_test_ca --test_case="${case}" &> $CLIENT_LOG; then + if ! GRPC_GO_LOG_SEVERITY_LEVEL=info withTimeout 20 go run ./interop/client \ + --use_tls \ + --server_host_override=foo.test.google.fr \ + --use_test_ca --test_case="${case}" \ + --service_config_json='{ "loadBalancingConfig": [{ "test_backend_metrics_load_balancer": {} }]}' \ + &> $CLIENT_LOG; then fail "FAIL: test case ${case} got server log: $(cat $SERVER_LOG) @@ -95,7 +131,7 @@ for case in ${CASES[@]}; do $(cat $CLIENT_LOG) " else - pass "PASS: test case ${case}" + pass "PASS: test case ${case}" fi done diff --git a/interop/observability/Dockerfile b/interop/observability/Dockerfile new file mode 100644 index 000000000000..d969e72abd44 --- /dev/null +++ b/interop/observability/Dockerfile @@ -0,0 +1,53 @@ +# Copyright 2022 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# +# Stage 1: Build the interop test client and server +# + +FROM golang:1.21-bullseye as build + +WORKDIR /grpc-go +COPY . . + +WORKDIR /grpc-go/interop/observability +RUN go build -o server/ server/server.go && \ + go build -o client/ client/client.go + + + +# +# Stage 2: +# +# - Copy only the necessary files to reduce Docker image size. +# - Have an ENTRYPOINT script which will launch the interop test client or server +# with the given parameters. +# + +FROM golang:1.21-bullseye + +ENV GRPC_GO_LOG_SEVERITY_LEVEL info +ENV GRPC_GO_LOG_VERBOSITY_LEVEL 2 + +WORKDIR /grpc-go/interop/observability/server +COPY --from=build /grpc-go/interop/observability/server/server . + +WORKDIR /grpc-go/interop/observability/client +COPY --from=build /grpc-go/interop/observability/client/client . + +WORKDIR /grpc-go/interop/observability +COPY --from=build /grpc-go/interop/observability/run.sh . + +ENTRYPOINT ["/grpc-go/interop/observability/run.sh"] diff --git a/interop/observability/build_docker.sh b/interop/observability/build_docker.sh new file mode 100755 index 000000000000..ed7a1811e923 --- /dev/null +++ b/interop/observability/build_docker.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# Copyright 2022 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex +cd "$(dirname "$0")"/../.. + +# Environment Variables: +# +# TAG_NAME: the docker image tag name +# + +echo Building ${TAG_NAME} + +docker build --no-cache -t ${TAG_NAME} -f ./interop/observability/Dockerfile . diff --git a/interop/observability/client/client.go b/interop/observability/client/client.go new file mode 100644 index 000000000000..d8cf72fa76c9 --- /dev/null +++ b/interop/observability/client/client.go @@ -0,0 +1,78 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package main + +import ( + "context" + "flag" + "log" + "net" + "strconv" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/gcp/observability" + "google.golang.org/grpc/interop" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" +) + +var ( + serverHost = flag.String("server_host", "localhost", "The server host name") + serverPort = flag.Int("server_port", 10000, "The server port number") + testCase = flag.String("test_case", "large_unary", "The action to perform") + numTimes = flag.Int("num_times", 1, "Number of times to run the test case") +) + +func main() { + err := observability.Start(context.Background()) + if err != nil { + log.Fatalf("observability start failed: %v", err) + } + defer observability.End() + flag.Parse() + serverAddr := *serverHost + if *serverPort != 0 { + serverAddr = net.JoinHostPort(*serverHost, strconv.Itoa(*serverPort)) + } + conn, err := grpc.Dial(serverAddr, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + log.Fatalf("Fail to dial: %v", err) + } + defer conn.Close() + tc := testgrpc.NewTestServiceClient(conn) + for i := 0; i < *numTimes; i++ { + if *testCase == "ping_pong" { + interop.DoPingPong(tc) + } else if *testCase == "large_unary" { + interop.DoLargeUnaryCall(tc) + } else if *testCase == "custom_metadata" { + interop.DoCustomMetadata(tc) + } else { + log.Fatalf("Invalid test case: %s", *testCase) + } + } + // TODO(stanleycheung): remove this once the observability exporter plugin is able to + // gracefully flush observability data to cloud at shutdown + // TODO(stanleycheung): see if we can reduce the number 65 + const exporterSleepDuration = 65 * time.Second + log.Printf("Sleeping %v before closing...", exporterSleepDuration) + time.Sleep(exporterSleepDuration) +} diff --git a/interop/observability/go.mod b/interop/observability/go.mod new file mode 100644 index 000000000000..4de7ebb8995a --- /dev/null +++ b/interop/observability/go.mod @@ -0,0 +1,52 @@ +module google.golang.org/grpc/interop/observability + +go 1.19 + +require ( + google.golang.org/grpc v1.56.2 + google.golang.org/grpc/gcp/observability v0.0.0-20230214181353-f4feddb37523 +) + +require ( + cloud.google.com/go v0.110.4 // indirect + cloud.google.com/go/compute v1.21.0 // indirect + cloud.google.com/go/compute/metadata v0.2.3 // indirect + cloud.google.com/go/logging v1.7.0 // indirect + cloud.google.com/go/longrunning v0.5.1 // indirect + cloud.google.com/go/monitoring v1.15.1 // indirect + cloud.google.com/go/trace v1.10.1 // indirect + contrib.go.opencensus.io/exporter/stackdriver v0.13.12 // indirect + github.com/aws/aws-sdk-go v1.44.162 // indirect + github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect + github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 // indirect + github.com/envoyproxy/protoc-gen-validate v1.0.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/go-cmp v0.5.9 // indirect + github.com/google/s2a-go v0.1.4 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect + github.com/googleapis/gax-go/v2 v2.11.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/prometheus/prometheus v2.5.0+incompatible // indirect + go.opencensus.io v0.24.0 // indirect + golang.org/x/crypto v0.11.0 // indirect + golang.org/x/net v0.12.0 // indirect + golang.org/x/oauth2 v0.10.0 // indirect + golang.org/x/sync v0.3.0 // indirect + golang.org/x/sys v0.10.0 // indirect + golang.org/x/text v0.11.0 // indirect + google.golang.org/api v0.126.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect + google.golang.org/grpc/stats/opencensus v1.0.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect +) + +replace google.golang.org/grpc => ../.. + +replace google.golang.org/grpc/gcp/observability => ../../gcp/observability + +replace google.golang.org/grpc/stats/opencensus => ../../stats/opencensus diff --git a/interop/observability/go.sum b/interop/observability/go.sum new file mode 100644 index 000000000000..bc17bc016f8b --- /dev/null +++ b/interop/observability/go.sum @@ -0,0 +1,1714 @@ +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= +cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= +cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= +cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= +cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= +cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= +cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= +cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= +cloud.google.com/go v0.100.1/go.mod h1:fs4QogzfH5n2pBXBP9vRiU+eCny7lD2vmFZy79Iuw1U= +cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= +cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc= +cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34hIU= +cloud.google.com/go v0.104.0/go.mod h1:OO6xxXdJyvuJPcEPBLN9BJPD+jep5G1+2U5B5gkRYtA= +cloud.google.com/go v0.105.0/go.mod h1:PrLgOJNe5nfE9UMxKxgXj4mD3voiP+YQ6gdt6KMFOKM= +cloud.google.com/go v0.107.0/go.mod h1:wpc2eNrD7hXUTy8EKS10jkxpZBjASrORK7goS+3YX2I= +cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= +cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVquxiw= +cloud.google.com/go v0.110.4 h1:1JYyxKMN9hd5dR2MYTPWkGUgcoxVVhg0LKNKEo0qvmk= +cloud.google.com/go v0.110.4/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= +cloud.google.com/go/accessapproval v1.4.0/go.mod h1:zybIuC3KpDOvotz59lFe5qxRZx6C75OtwbisN56xYB4= +cloud.google.com/go/accessapproval v1.5.0/go.mod h1:HFy3tuiGvMdcd/u+Cu5b9NkO1pEICJ46IR82PoUdplw= +cloud.google.com/go/accessapproval v1.6.0/go.mod h1:R0EiYnwV5fsRFiKZkPHr6mwyk2wxUJ30nL4j2pcFY2E= +cloud.google.com/go/accessapproval v1.7.1/go.mod h1:JYczztsHRMK7NTXb6Xw+dwbs/WnOJxbo/2mTI+Kgg68= +cloud.google.com/go/accesscontextmanager v1.3.0/go.mod h1:TgCBehyr5gNMz7ZaH9xubp+CE8dkrszb4oK9CWyvD4o= +cloud.google.com/go/accesscontextmanager v1.4.0/go.mod h1:/Kjh7BBu/Gh83sv+K60vN9QE5NJcd80sU33vIe2IFPE= +cloud.google.com/go/accesscontextmanager v1.6.0/go.mod h1:8XCvZWfYw3K/ji0iVnp+6pu7huxoQTLmxAbVjbloTtM= +cloud.google.com/go/accesscontextmanager v1.7.0/go.mod h1:CEGLewx8dwa33aDAZQujl7Dx+uYhS0eay198wB/VumQ= +cloud.google.com/go/accesscontextmanager v1.8.0/go.mod h1:uI+AI/r1oyWK99NN8cQ3UK76AMelMzgZCvJfsi2c+ps= +cloud.google.com/go/accesscontextmanager v1.8.1/go.mod h1:JFJHfvuaTC+++1iL1coPiG1eu5D24db2wXCDWDjIrxo= +cloud.google.com/go/aiplatform v1.22.0/go.mod h1:ig5Nct50bZlzV6NvKaTwmplLLddFx0YReh9WfTO5jKw= +cloud.google.com/go/aiplatform v1.24.0/go.mod h1:67UUvRBKG6GTayHKV8DBv2RtR1t93YRu5B1P3x99mYY= +cloud.google.com/go/aiplatform v1.27.0/go.mod h1:Bvxqtl40l0WImSb04d0hXFU7gDOiq9jQmorivIiWcKg= +cloud.google.com/go/aiplatform v1.35.0/go.mod h1:7MFT/vCaOyZT/4IIFfxH4ErVg/4ku6lKv3w0+tFTgXQ= +cloud.google.com/go/aiplatform v1.36.1/go.mod h1:WTm12vJRPARNvJ+v6P52RDHCNe4AhvjcIZ/9/RRHy/k= +cloud.google.com/go/aiplatform v1.37.0/go.mod h1:IU2Cv29Lv9oCn/9LkFiiuKfwrRTq+QQMbW+hPCxJGZw= +cloud.google.com/go/aiplatform v1.45.0/go.mod h1:Iu2Q7sC7QGhXUeOhAj/oCK9a+ULz1O4AotZiqjQ8MYA= +cloud.google.com/go/analytics v0.11.0/go.mod h1:DjEWCu41bVbYcKyvlws9Er60YE4a//bK6mnhWvQeFNI= +cloud.google.com/go/analytics v0.12.0/go.mod h1:gkfj9h6XRf9+TS4bmuhPEShsh3hH8PAZzm/41OOhQd4= +cloud.google.com/go/analytics v0.17.0/go.mod h1:WXFa3WSym4IZ+JiKmavYdJwGG/CvpqiqczmL59bTD9M= +cloud.google.com/go/analytics v0.18.0/go.mod h1:ZkeHGQlcIPkw0R/GW+boWHhCOR43xz9RN/jn7WcqfIE= +cloud.google.com/go/analytics v0.19.0/go.mod h1:k8liqf5/HCnOUkbawNtrWWc+UAzyDlW89doe8TtoDsE= +cloud.google.com/go/analytics v0.21.2/go.mod h1:U8dcUtmDmjrmUTnnnRnI4m6zKn/yaA5N9RlEkYFHpQo= +cloud.google.com/go/apigateway v1.3.0/go.mod h1:89Z8Bhpmxu6AmUxuVRg/ECRGReEdiP3vQtk4Z1J9rJk= +cloud.google.com/go/apigateway v1.4.0/go.mod h1:pHVY9MKGaH9PQ3pJ4YLzoj6U5FUDeDFBllIz7WmzJoc= +cloud.google.com/go/apigateway v1.5.0/go.mod h1:GpnZR3Q4rR7LVu5951qfXPJCHquZt02jf7xQx7kpqN8= +cloud.google.com/go/apigateway v1.6.1/go.mod h1:ufAS3wpbRjqfZrzpvLC2oh0MFlpRJm2E/ts25yyqmXA= +cloud.google.com/go/apigeeconnect v1.3.0/go.mod h1:G/AwXFAKo0gIXkPTVfZDd2qA1TxBXJ3MgMRBQkIi9jc= +cloud.google.com/go/apigeeconnect v1.4.0/go.mod h1:kV4NwOKqjvt2JYR0AoIWo2QGfoRtn/pkS3QlHp0Ni04= +cloud.google.com/go/apigeeconnect v1.5.0/go.mod h1:KFaCqvBRU6idyhSNyn3vlHXc8VMDJdRmwDF6JyFRqZ8= +cloud.google.com/go/apigeeconnect v1.6.1/go.mod h1:C4awq7x0JpLtrlQCr8AzVIzAaYgngRqWf9S5Uhg+wWs= +cloud.google.com/go/apigeeregistry v0.4.0/go.mod h1:EUG4PGcsZvxOXAdyEghIdXwAEi/4MEaoqLMLDMIwKXY= +cloud.google.com/go/apigeeregistry v0.5.0/go.mod h1:YR5+s0BVNZfVOUkMa5pAR2xGd0A473vA5M7j247o1wM= +cloud.google.com/go/apigeeregistry v0.6.0/go.mod h1:BFNzW7yQVLZ3yj0TKcwzb8n25CFBri51GVGOEUcgQsc= +cloud.google.com/go/apigeeregistry v0.7.1/go.mod h1:1XgyjZye4Mqtw7T9TsY4NW10U7BojBvG4RMD+vRDrIw= +cloud.google.com/go/apikeys v0.4.0/go.mod h1:XATS/yqZbaBK0HOssf+ALHp8jAlNHUgyfprvNcBIszU= +cloud.google.com/go/apikeys v0.5.0/go.mod h1:5aQfwY4D+ewMMWScd3hm2en3hCj+BROlyrt3ytS7KLI= +cloud.google.com/go/apikeys v0.6.0/go.mod h1:kbpXu5upyiAlGkKrJgQl8A0rKNNJ7dQ377pdroRSSi8= +cloud.google.com/go/appengine v1.4.0/go.mod h1:CS2NhuBuDXM9f+qscZ6V86m1MIIqPj3WC/UoEuR1Sno= +cloud.google.com/go/appengine v1.5.0/go.mod h1:TfasSozdkFI0zeoxW3PTBLiNqRmzraodCWatWI9Dmak= +cloud.google.com/go/appengine v1.6.0/go.mod h1:hg6i0J/BD2cKmDJbaFSYHFyZkgBEfQrDg/X0V5fJn84= +cloud.google.com/go/appengine v1.7.0/go.mod h1:eZqpbHFCqRGa2aCdope7eC0SWLV1j0neb/QnMJVWx6A= +cloud.google.com/go/appengine v1.7.1/go.mod h1:IHLToyb/3fKutRysUlFO0BPt5j7RiQ45nrzEJmKTo6E= +cloud.google.com/go/appengine v1.8.1/go.mod h1:6NJXGLVhZCN9aQ/AEDvmfzKEfoYBlfB80/BHiKVputY= +cloud.google.com/go/area120 v0.5.0/go.mod h1:DE/n4mp+iqVyvxHN41Vf1CR602GiHQjFPusMFW6bGR4= +cloud.google.com/go/area120 v0.6.0/go.mod h1:39yFJqWVgm0UZqWTOdqkLhjoC7uFfgXRC8g/ZegeAh0= +cloud.google.com/go/area120 v0.7.0/go.mod h1:a3+8EUD1SX5RUcCs3MY5YasiO1z6yLiNLRiFrykbynY= +cloud.google.com/go/area120 v0.7.1/go.mod h1:j84i4E1RboTWjKtZVWXPqvK5VHQFJRF2c1Nm69pWm9k= +cloud.google.com/go/area120 v0.8.1/go.mod h1:BVfZpGpB7KFVNxPiQBuHkX6Ed0rS51xIgmGyjrAfzsg= +cloud.google.com/go/artifactregistry v1.6.0/go.mod h1:IYt0oBPSAGYj/kprzsBjZ/4LnG/zOcHyFHjWPCi6SAQ= +cloud.google.com/go/artifactregistry v1.7.0/go.mod h1:mqTOFOnGZx8EtSqK/ZWcsm/4U8B77rbcLP6ruDU2Ixk= +cloud.google.com/go/artifactregistry v1.8.0/go.mod h1:w3GQXkJX8hiKN0v+at4b0qotwijQbYUqF2GWkZzAhC0= +cloud.google.com/go/artifactregistry v1.9.0/go.mod h1:2K2RqvA2CYvAeARHRkLDhMDJ3OXy26h3XW+3/Jh2uYc= +cloud.google.com/go/artifactregistry v1.11.1/go.mod h1:lLYghw+Itq9SONbCa1YWBoWs1nOucMH0pwXN1rOBZFI= +cloud.google.com/go/artifactregistry v1.11.2/go.mod h1:nLZns771ZGAwVLzTX/7Al6R9ehma4WUEhZGWV6CeQNQ= +cloud.google.com/go/artifactregistry v1.12.0/go.mod h1:o6P3MIvtzTOnmvGagO9v/rOjjA0HmhJ+/6KAXrmYDCI= +cloud.google.com/go/artifactregistry v1.13.0/go.mod h1:uy/LNfoOIivepGhooAUpL1i30Hgee3Cu0l4VTWHUC08= +cloud.google.com/go/artifactregistry v1.14.1/go.mod h1:nxVdG19jTaSTu7yA7+VbWL346r3rIdkZ142BSQqhn5E= +cloud.google.com/go/asset v1.5.0/go.mod h1:5mfs8UvcM5wHhqtSv8J1CtxxaQq3AdBxxQi2jGW/K4o= +cloud.google.com/go/asset v1.7.0/go.mod h1:YbENsRK4+xTiL+Ofoj5Ckf+O17kJtgp3Y3nn4uzZz5s= +cloud.google.com/go/asset v1.8.0/go.mod h1:mUNGKhiqIdbr8X7KNayoYvyc4HbbFO9URsjbytpUaW0= +cloud.google.com/go/asset v1.9.0/go.mod h1:83MOE6jEJBMqFKadM9NLRcs80Gdw76qGuHn8m3h8oHQ= +cloud.google.com/go/asset v1.10.0/go.mod h1:pLz7uokL80qKhzKr4xXGvBQXnzHn5evJAEAtZiIb0wY= +cloud.google.com/go/asset v1.11.1/go.mod h1:fSwLhbRvC9p9CXQHJ3BgFeQNM4c9x10lqlrdEUYXlJo= +cloud.google.com/go/asset v1.12.0/go.mod h1:h9/sFOa4eDIyKmH6QMpm4eUK3pDojWnUhTgJlk762Hg= +cloud.google.com/go/asset v1.13.0/go.mod h1:WQAMyYek/b7NBpYq/K4KJWcRqzoalEsxz/t/dTk4THw= +cloud.google.com/go/asset v1.14.1/go.mod h1:4bEJ3dnHCqWCDbWJ/6Vn7GVI9LerSi7Rfdi03hd+WTQ= +cloud.google.com/go/assuredworkloads v1.5.0/go.mod h1:n8HOZ6pff6re5KYfBXcFvSViQjDwxFkAkmUFffJRbbY= +cloud.google.com/go/assuredworkloads v1.6.0/go.mod h1:yo2YOk37Yc89Rsd5QMVECvjaMKymF9OP+QXWlKXUkXw= +cloud.google.com/go/assuredworkloads v1.7.0/go.mod h1:z/736/oNmtGAyU47reJgGN+KVoYoxeLBoj4XkKYscNI= +cloud.google.com/go/assuredworkloads v1.8.0/go.mod h1:AsX2cqyNCOvEQC8RMPnoc0yEarXQk6WEKkxYfL6kGIo= +cloud.google.com/go/assuredworkloads v1.9.0/go.mod h1:kFuI1P78bplYtT77Tb1hi0FMxM0vVpRC7VVoJC3ZoT0= +cloud.google.com/go/assuredworkloads v1.10.0/go.mod h1:kwdUQuXcedVdsIaKgKTp9t0UJkE5+PAVNhdQm4ZVq2E= +cloud.google.com/go/assuredworkloads v1.11.1/go.mod h1:+F04I52Pgn5nmPG36CWFtxmav6+7Q+c5QyJoL18Lry0= +cloud.google.com/go/automl v1.5.0/go.mod h1:34EjfoFGMZ5sgJ9EoLsRtdPSNZLcfflJR39VbVNS2M0= +cloud.google.com/go/automl v1.6.0/go.mod h1:ugf8a6Fx+zP0D59WLhqgTDsQI9w07o64uf/Is3Nh5p8= +cloud.google.com/go/automl v1.7.0/go.mod h1:RL9MYCCsJEOmt0Wf3z9uzG0a7adTT1fe+aObgSpkCt8= +cloud.google.com/go/automl v1.8.0/go.mod h1:xWx7G/aPEe/NP+qzYXktoBSDfjO+vnKMGgsApGJJquM= +cloud.google.com/go/automl v1.12.0/go.mod h1:tWDcHDp86aMIuHmyvjuKeeHEGq76lD7ZqfGLN6B0NuU= +cloud.google.com/go/automl v1.13.1/go.mod h1:1aowgAHWYZU27MybSCFiukPO7xnyawv7pt3zK4bheQE= +cloud.google.com/go/baremetalsolution v0.3.0/go.mod h1:XOrocE+pvK1xFfleEnShBlNAXf+j5blPPxrhjKgnIFc= +cloud.google.com/go/baremetalsolution v0.4.0/go.mod h1:BymplhAadOO/eBa7KewQ0Ppg4A4Wplbn+PsFKRLo0uI= +cloud.google.com/go/baremetalsolution v0.5.0/go.mod h1:dXGxEkmR9BMwxhzBhV0AioD0ULBmuLZI8CdwalUxuss= +cloud.google.com/go/batch v0.3.0/go.mod h1:TR18ZoAekj1GuirsUsR1ZTKN3FC/4UDnScjT8NXImFE= +cloud.google.com/go/batch v0.4.0/go.mod h1:WZkHnP43R/QCGQsZ+0JyG4i79ranE2u8xvjq/9+STPE= +cloud.google.com/go/batch v0.7.0/go.mod h1:vLZN95s6teRUqRQ4s3RLDsH8PvboqBK+rn1oevL159g= +cloud.google.com/go/beyondcorp v0.2.0/go.mod h1:TB7Bd+EEtcw9PCPQhCJtJGjk/7TC6ckmnSFS+xwTfm4= +cloud.google.com/go/beyondcorp v0.3.0/go.mod h1:E5U5lcrcXMsCuoDNyGrpyTm/hn7ne941Jz2vmksAxW8= +cloud.google.com/go/beyondcorp v0.4.0/go.mod h1:3ApA0mbhHx6YImmuubf5pyW8srKnCEPON32/5hj+RmM= +cloud.google.com/go/beyondcorp v0.5.0/go.mod h1:uFqj9X+dSfrheVp7ssLTaRHd2EHqSL4QZmH4e8WXGGU= +cloud.google.com/go/beyondcorp v0.6.1/go.mod h1:YhxDWw946SCbmcWo3fAhw3V4XZMSpQ/VYfcKGAEU8/4= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/bigquery v1.42.0/go.mod h1:8dRTJxhtG+vwBKzE5OseQn/hiydoQN3EedCaOdYmxRA= +cloud.google.com/go/bigquery v1.43.0/go.mod h1:ZMQcXHsl+xmU1z36G2jNGZmKp9zNY5BUua5wDgmNCfw= +cloud.google.com/go/bigquery v1.44.0/go.mod h1:0Y33VqXTEsbamHJvJHdFmtqHvMIY28aK1+dFsvaChGc= +cloud.google.com/go/bigquery v1.47.0/go.mod h1:sA9XOgy0A8vQK9+MWhEQTY6Tix87M/ZurWFIxmF9I/E= +cloud.google.com/go/bigquery v1.48.0/go.mod h1:QAwSz+ipNgfL5jxiaK7weyOhzdoAy1zFm0Nf1fysJac= +cloud.google.com/go/bigquery v1.49.0/go.mod h1:Sv8hMmTFFYBlt/ftw2uN6dFdQPzBlREY9yBh7Oy7/4Q= +cloud.google.com/go/bigquery v1.50.0/go.mod h1:YrleYEh2pSEbgTBZYMJ5SuSr0ML3ypjRB1zgf7pvQLU= +cloud.google.com/go/bigquery v1.52.0/go.mod h1:3b/iXjRQGU4nKa87cXeg6/gogLjO8C6PmuM8i5Bi/u4= +cloud.google.com/go/billing v1.4.0/go.mod h1:g9IdKBEFlItS8bTtlrZdVLWSSdSyFUZKXNS02zKMOZY= +cloud.google.com/go/billing v1.5.0/go.mod h1:mztb1tBc3QekhjSgmpf/CV4LzWXLzCArwpLmP2Gm88s= +cloud.google.com/go/billing v1.6.0/go.mod h1:WoXzguj+BeHXPbKfNWkqVtDdzORazmCjraY+vrxcyvI= +cloud.google.com/go/billing v1.7.0/go.mod h1:q457N3Hbj9lYwwRbnlD7vUpyjq6u5U1RAOArInEiD5Y= +cloud.google.com/go/billing v1.12.0/go.mod h1:yKrZio/eu+okO/2McZEbch17O5CB5NpZhhXG6Z766ss= +cloud.google.com/go/billing v1.13.0/go.mod h1:7kB2W9Xf98hP9Sr12KfECgfGclsH3CQR0R08tnRlRbc= +cloud.google.com/go/billing v1.16.0/go.mod h1:y8vx09JSSJG02k5QxbycNRrN7FGZB6F3CAcgum7jvGA= +cloud.google.com/go/binaryauthorization v1.1.0/go.mod h1:xwnoWu3Y84jbuHa0zd526MJYmtnVXn0syOjaJgy4+dM= +cloud.google.com/go/binaryauthorization v1.2.0/go.mod h1:86WKkJHtRcv5ViNABtYMhhNWRrD1Vpi//uKEy7aYEfI= +cloud.google.com/go/binaryauthorization v1.3.0/go.mod h1:lRZbKgjDIIQvzYQS1p99A7/U1JqvqeZg0wiI5tp6tg0= +cloud.google.com/go/binaryauthorization v1.4.0/go.mod h1:tsSPQrBd77VLplV70GUhBf/Zm3FsKmgSqgm4UmiDItk= +cloud.google.com/go/binaryauthorization v1.5.0/go.mod h1:OSe4OU1nN/VswXKRBmciKpo9LulY41gch5c68htf3/Q= +cloud.google.com/go/binaryauthorization v1.6.1/go.mod h1:TKt4pa8xhowwffiBmbrbcxijJRZED4zrqnwZ1lKH51U= +cloud.google.com/go/certificatemanager v1.3.0/go.mod h1:n6twGDvcUBFu9uBgt4eYvvf3sQ6My8jADcOVwHmzadg= +cloud.google.com/go/certificatemanager v1.4.0/go.mod h1:vowpercVFyqs8ABSmrdV+GiFf2H/ch3KyudYQEMM590= +cloud.google.com/go/certificatemanager v1.6.0/go.mod h1:3Hh64rCKjRAX8dXgRAyOcY5vQ/fE1sh8o+Mdd6KPgY8= +cloud.google.com/go/certificatemanager v1.7.1/go.mod h1:iW8J3nG6SaRYImIa+wXQ0g8IgoofDFRp5UMzaNk1UqI= +cloud.google.com/go/channel v1.8.0/go.mod h1:W5SwCXDJsq/rg3tn3oG0LOxpAo6IMxNa09ngphpSlnk= +cloud.google.com/go/channel v1.9.0/go.mod h1:jcu05W0my9Vx4mt3/rEHpfxc9eKi9XwsdDL8yBMbKUk= +cloud.google.com/go/channel v1.11.0/go.mod h1:IdtI0uWGqhEeatSB62VOoJ8FSUhJ9/+iGkJVqp74CGE= +cloud.google.com/go/channel v1.12.0/go.mod h1:VkxCGKASi4Cq7TbXxlaBezonAYpp1GCnKMY6tnMQnLU= +cloud.google.com/go/channel v1.16.0/go.mod h1:eN/q1PFSl5gyu0dYdmxNXscY/4Fi7ABmeHCJNf/oHmc= +cloud.google.com/go/cloudbuild v1.3.0/go.mod h1:WequR4ULxlqvMsjDEEEFnOG5ZSRSgWOywXYDb1vPE6U= +cloud.google.com/go/cloudbuild v1.4.0/go.mod h1:5Qwa40LHiOXmz3386FrjrYM93rM/hdRr7b53sySrTqA= +cloud.google.com/go/cloudbuild v1.6.0/go.mod h1:UIbc/w9QCbH12xX+ezUsgblrWv+Cv4Tw83GiSMHOn9M= +cloud.google.com/go/cloudbuild v1.7.0/go.mod h1:zb5tWh2XI6lR9zQmsm1VRA+7OCuve5d8S+zJUul8KTg= +cloud.google.com/go/cloudbuild v1.9.0/go.mod h1:qK1d7s4QlO0VwfYn5YuClDGg2hfmLZEb4wQGAbIgL1s= +cloud.google.com/go/cloudbuild v1.10.1/go.mod h1:lyJg7v97SUIPq4RC2sGsz/9tNczhyv2AjML/ci4ulzU= +cloud.google.com/go/clouddms v1.3.0/go.mod h1:oK6XsCDdW4Ib3jCCBugx+gVjevp2TMXFtgxvPSee3OM= +cloud.google.com/go/clouddms v1.4.0/go.mod h1:Eh7sUGCC+aKry14O1NRljhjyrr0NFC0G2cjwX0cByRk= +cloud.google.com/go/clouddms v1.5.0/go.mod h1:QSxQnhikCLUw13iAbffF2CZxAER3xDGNHjsTAkQJcQA= +cloud.google.com/go/clouddms v1.6.1/go.mod h1:Ygo1vL52Ov4TBZQquhz5fiw2CQ58gvu+PlS6PVXCpZI= +cloud.google.com/go/cloudtasks v1.5.0/go.mod h1:fD92REy1x5woxkKEkLdvavGnPJGEn8Uic9nWuLzqCpY= +cloud.google.com/go/cloudtasks v1.6.0/go.mod h1:C6Io+sxuke9/KNRkbQpihnW93SWDU3uXt92nu85HkYI= +cloud.google.com/go/cloudtasks v1.7.0/go.mod h1:ImsfdYWwlWNJbdgPIIGJWC+gemEGTBK/SunNQQNCAb4= +cloud.google.com/go/cloudtasks v1.8.0/go.mod h1:gQXUIwCSOI4yPVK7DgTVFiiP0ZW/eQkydWzwVMdHxrI= +cloud.google.com/go/cloudtasks v1.9.0/go.mod h1:w+EyLsVkLWHcOaqNEyvcKAsWp9p29dL6uL9Nst1cI7Y= +cloud.google.com/go/cloudtasks v1.10.0/go.mod h1:NDSoTLkZ3+vExFEWu2UJV1arUyzVDAiZtdWcsUyNwBs= +cloud.google.com/go/cloudtasks v1.11.1/go.mod h1:a9udmnou9KO2iulGscKR0qBYjreuX8oHwpmFsKspEvM= +cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= +cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= +cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= +cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s= +cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= +cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= +cloud.google.com/go/compute v1.10.0/go.mod h1:ER5CLbMxl90o2jtNbGSbtfOpQKR0t15FOtRsugnLrlU= +cloud.google.com/go/compute v1.12.0/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= +cloud.google.com/go/compute v1.12.1/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= +cloud.google.com/go/compute v1.13.0/go.mod h1:5aPTS0cUNMIc1CE546K+Th6weJUNQErARyZtRXDJ8GE= +cloud.google.com/go/compute v1.14.0/go.mod h1:YfLtxrj9sU4Yxv+sXzZkyPjEyPBZfXHUvjxega5vAdo= +cloud.google.com/go/compute v1.18.0/go.mod h1:1X7yHxec2Ga+Ss6jPyjxRxpu2uu7PLgsOVXvgU0yacs= +cloud.google.com/go/compute v1.19.0/go.mod h1:rikpw2y+UMidAe9tISo04EHNOIf42RLYF/q8Bs93scU= +cloud.google.com/go/compute v1.19.3/go.mod h1:qxvISKp/gYnXkSAD1ppcSOveRAmzxicEv/JlizULFrI= +cloud.google.com/go/compute v1.20.1/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= +cloud.google.com/go/compute v1.21.0 h1:JNBsyXVoOoNJtTQcnEY5uYpZIbeCTYIeDe0Xh1bySMk= +cloud.google.com/go/compute v1.21.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= +cloud.google.com/go/compute/metadata v0.1.0/go.mod h1:Z1VN+bulIf6bt4P/C37K4DyZYZEXYonfTBHHFPO/4UU= +cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= +cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +cloud.google.com/go/contactcenterinsights v1.3.0/go.mod h1:Eu2oemoePuEFc/xKFPjbTuPSj0fYJcPls9TFlPNnHHY= +cloud.google.com/go/contactcenterinsights v1.4.0/go.mod h1:L2YzkGbPsv+vMQMCADxJoT9YiTTnSEd6fEvCeHTYVck= +cloud.google.com/go/contactcenterinsights v1.6.0/go.mod h1:IIDlT6CLcDoyv79kDv8iWxMSTZhLxSCofVV5W6YFM/w= +cloud.google.com/go/contactcenterinsights v1.9.1/go.mod h1:bsg/R7zGLYMVxFFzfh9ooLTruLRCG9fnzhH9KznHhbM= +cloud.google.com/go/container v1.6.0/go.mod h1:Xazp7GjJSeUYo688S+6J5V+n/t+G5sKBTFkKNudGRxg= +cloud.google.com/go/container v1.7.0/go.mod h1:Dp5AHtmothHGX3DwwIHPgq45Y8KmNsgN3amoYfxVkLo= +cloud.google.com/go/container v1.13.1/go.mod h1:6wgbMPeQRw9rSnKBCAJXnds3Pzj03C4JHamr8asWKy4= +cloud.google.com/go/container v1.14.0/go.mod h1:3AoJMPhHfLDxLvrlVWaK57IXzaPnLaZq63WX59aQBfM= +cloud.google.com/go/container v1.15.0/go.mod h1:ft+9S0WGjAyjDggg5S06DXj+fHJICWg8L7isCQe9pQA= +cloud.google.com/go/container v1.22.1/go.mod h1:lTNExE2R7f+DLbAN+rJiKTisauFCaoDq6NURZ83eVH4= +cloud.google.com/go/containeranalysis v0.5.1/go.mod h1:1D92jd8gRR/c0fGMlymRgxWD3Qw9C1ff6/T7mLgVL8I= +cloud.google.com/go/containeranalysis v0.6.0/go.mod h1:HEJoiEIu+lEXM+k7+qLCci0h33lX3ZqoYFdmPcoO7s4= +cloud.google.com/go/containeranalysis v0.7.0/go.mod h1:9aUL+/vZ55P2CXfuZjS4UjQ9AgXoSw8Ts6lemfmxBxI= +cloud.google.com/go/containeranalysis v0.9.0/go.mod h1:orbOANbwk5Ejoom+s+DUCTTJ7IBdBQJDcSylAx/on9s= +cloud.google.com/go/containeranalysis v0.10.1/go.mod h1:Ya2jiILITMY68ZLPaogjmOMNkwsDrWBSTyBubGXO7j0= +cloud.google.com/go/datacatalog v1.3.0/go.mod h1:g9svFY6tuR+j+hrTw3J2dNcmI0dzmSiyOzm8kpLq0a0= +cloud.google.com/go/datacatalog v1.5.0/go.mod h1:M7GPLNQeLfWqeIm3iuiruhPzkt65+Bx8dAKvScX8jvs= +cloud.google.com/go/datacatalog v1.6.0/go.mod h1:+aEyF8JKg+uXcIdAmmaMUmZ3q1b/lKLtXCmXdnc0lbc= +cloud.google.com/go/datacatalog v1.7.0/go.mod h1:9mEl4AuDYWw81UGc41HonIHH7/sn52H0/tc8f8ZbZIE= +cloud.google.com/go/datacatalog v1.8.0/go.mod h1:KYuoVOv9BM8EYz/4eMFxrr4DUKhGIOXxZoKYF5wdISM= +cloud.google.com/go/datacatalog v1.8.1/go.mod h1:RJ58z4rMp3gvETA465Vg+ag8BGgBdnRPEMMSTr5Uv+M= +cloud.google.com/go/datacatalog v1.12.0/go.mod h1:CWae8rFkfp6LzLumKOnmVh4+Zle4A3NXLzVJ1d1mRm0= +cloud.google.com/go/datacatalog v1.13.0/go.mod h1:E4Rj9a5ZtAxcQJlEBTLgMTphfP11/lNaAshpoBgemX8= +cloud.google.com/go/datacatalog v1.14.0/go.mod h1:h0PrGtlihoutNMp/uvwhawLQ9+c63Kz65UFqh49Yo+E= +cloud.google.com/go/datacatalog v1.14.1/go.mod h1:d2CevwTG4yedZilwe+v3E3ZBDRMobQfSG/a6cCCN5R4= +cloud.google.com/go/dataflow v0.6.0/go.mod h1:9QwV89cGoxjjSR9/r7eFDqqjtvbKxAK2BaYU6PVk9UM= +cloud.google.com/go/dataflow v0.7.0/go.mod h1:PX526vb4ijFMesO1o202EaUmouZKBpjHsTlCtB4parQ= +cloud.google.com/go/dataflow v0.8.0/go.mod h1:Rcf5YgTKPtQyYz8bLYhFoIV/vP39eL7fWNcSOyFfLJE= +cloud.google.com/go/dataflow v0.9.1/go.mod h1:Wp7s32QjYuQDWqJPFFlnBKhkAtiFpMTdg00qGbnIHVw= +cloud.google.com/go/dataform v0.3.0/go.mod h1:cj8uNliRlHpa6L3yVhDOBrUXH+BPAO1+KFMQQNSThKo= +cloud.google.com/go/dataform v0.4.0/go.mod h1:fwV6Y4Ty2yIFL89huYlEkwUPtS7YZinZbzzj5S9FzCE= +cloud.google.com/go/dataform v0.5.0/go.mod h1:GFUYRe8IBa2hcomWplodVmUx/iTL0FrsauObOM3Ipr0= +cloud.google.com/go/dataform v0.6.0/go.mod h1:QPflImQy33e29VuapFdf19oPbE4aYTJxr31OAPV+ulA= +cloud.google.com/go/dataform v0.7.0/go.mod h1:7NulqnVozfHvWUBpMDfKMUESr+85aJsC/2O0o3jWPDE= +cloud.google.com/go/dataform v0.8.1/go.mod h1:3BhPSiw8xmppbgzeBbmDvmSWlwouuJkXsXsb8UBih9M= +cloud.google.com/go/datafusion v1.4.0/go.mod h1:1Zb6VN+W6ALo85cXnM1IKiPw+yQMKMhB9TsTSRDo/38= +cloud.google.com/go/datafusion v1.5.0/go.mod h1:Kz+l1FGHB0J+4XF2fud96WMmRiq/wj8N9u007vyXZ2w= +cloud.google.com/go/datafusion v1.6.0/go.mod h1:WBsMF8F1RhSXvVM8rCV3AeyWVxcC2xY6vith3iw3S+8= +cloud.google.com/go/datafusion v1.7.1/go.mod h1:KpoTBbFmoToDExJUso/fcCiguGDk7MEzOWXUsJo0wsI= +cloud.google.com/go/datalabeling v0.5.0/go.mod h1:TGcJ0G2NzcsXSE/97yWjIZO0bXj0KbVlINXMG9ud42I= +cloud.google.com/go/datalabeling v0.6.0/go.mod h1:WqdISuk/+WIGeMkpw/1q7bK/tFEZxsrFJOJdY2bXvTQ= +cloud.google.com/go/datalabeling v0.7.0/go.mod h1:WPQb1y08RJbmpM3ww0CSUAGweL0SxByuW2E+FU+wXcM= +cloud.google.com/go/datalabeling v0.8.1/go.mod h1:XS62LBSVPbYR54GfYQsPXZjTW8UxCK2fkDciSrpRFdY= +cloud.google.com/go/dataplex v1.3.0/go.mod h1:hQuRtDg+fCiFgC8j0zV222HvzFQdRd+SVX8gdmFcZzA= +cloud.google.com/go/dataplex v1.4.0/go.mod h1:X51GfLXEMVJ6UN47ESVqvlsRplbLhcsAt0kZCCKsU0A= +cloud.google.com/go/dataplex v1.5.2/go.mod h1:cVMgQHsmfRoI5KFYq4JtIBEUbYwc3c7tXmIDhRmNNVQ= +cloud.google.com/go/dataplex v1.6.0/go.mod h1:bMsomC/aEJOSpHXdFKFGQ1b0TDPIeL28nJObeO1ppRs= +cloud.google.com/go/dataplex v1.8.1/go.mod h1:7TyrDT6BCdI8/38Uvp0/ZxBslOslP2X2MPDucliyvSE= +cloud.google.com/go/dataproc v1.7.0/go.mod h1:CKAlMjII9H90RXaMpSxQ8EU6dQx6iAYNPcYPOkSbi8s= +cloud.google.com/go/dataproc v1.8.0/go.mod h1:5OW+zNAH0pMpw14JVrPONsxMQYMBqJuzORhIBfBn9uI= +cloud.google.com/go/dataproc v1.12.0/go.mod h1:zrF3aX0uV3ikkMz6z4uBbIKyhRITnxvr4i3IjKsKrw4= +cloud.google.com/go/dataqna v0.5.0/go.mod h1:90Hyk596ft3zUQ8NkFfvICSIfHFh1Bc7C4cK3vbhkeo= +cloud.google.com/go/dataqna v0.6.0/go.mod h1:1lqNpM7rqNLVgWBJyk5NF6Uen2PHym0jtVJonplVsDA= +cloud.google.com/go/dataqna v0.7.0/go.mod h1:Lx9OcIIeqCrw1a6KdO3/5KMP1wAmTc0slZWwP12Qq3c= +cloud.google.com/go/dataqna v0.8.1/go.mod h1:zxZM0Bl6liMePWsHA8RMGAfmTG34vJMapbHAxQ5+WA8= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/datastore v1.10.0/go.mod h1:PC5UzAmDEkAmkfaknstTYbNpgE49HAgW2J1gcgUfmdM= +cloud.google.com/go/datastore v1.11.0/go.mod h1:TvGxBIHCS50u8jzG+AW/ppf87v1of8nwzFNgEZU1D3c= +cloud.google.com/go/datastore v1.12.0/go.mod h1:KjdB88W897MRITkvWWJrg2OUtrR5XVj1EoLgSp6/N70= +cloud.google.com/go/datastore v1.12.1/go.mod h1:KjdB88W897MRITkvWWJrg2OUtrR5XVj1EoLgSp6/N70= +cloud.google.com/go/datastream v1.2.0/go.mod h1:i/uTP8/fZwgATHS/XFu0TcNUhuA0twZxxQ3EyCUQMwo= +cloud.google.com/go/datastream v1.3.0/go.mod h1:cqlOX8xlyYF/uxhiKn6Hbv6WjwPPuI9W2M9SAXwaLLQ= +cloud.google.com/go/datastream v1.4.0/go.mod h1:h9dpzScPhDTs5noEMQVWP8Wx8AFBRyS0s8KWPx/9r0g= +cloud.google.com/go/datastream v1.5.0/go.mod h1:6TZMMNPwjUqZHBKPQ1wwXpb0d5VDVPl2/XoS5yi88q4= +cloud.google.com/go/datastream v1.6.0/go.mod h1:6LQSuswqLa7S4rPAOZFVjHIG3wJIjZcZrw8JDEDJuIs= +cloud.google.com/go/datastream v1.7.0/go.mod h1:uxVRMm2elUSPuh65IbZpzJNMbuzkcvu5CjMqVIUHrww= +cloud.google.com/go/datastream v1.9.1/go.mod h1:hqnmr8kdUBmrnk65k5wNRoHSCYksvpdZIcZIEl8h43Q= +cloud.google.com/go/deploy v1.4.0/go.mod h1:5Xghikd4VrmMLNaF6FiRFDlHb59VM59YoDQnOUdsH/c= +cloud.google.com/go/deploy v1.5.0/go.mod h1:ffgdD0B89tToyW/U/D2eL0jN2+IEV/3EMuXHA0l4r+s= +cloud.google.com/go/deploy v1.6.0/go.mod h1:f9PTHehG/DjCom3QH0cntOVRm93uGBDt2vKzAPwpXQI= +cloud.google.com/go/deploy v1.8.0/go.mod h1:z3myEJnA/2wnB4sgjqdMfgxCA0EqC3RBTNcVPs93mtQ= +cloud.google.com/go/deploy v1.11.0/go.mod h1:tKuSUV5pXbn67KiubiUNUejqLs4f5cxxiCNCeyl0F2g= +cloud.google.com/go/dialogflow v1.15.0/go.mod h1:HbHDWs33WOGJgn6rfzBW1Kv807BE3O1+xGbn59zZWI4= +cloud.google.com/go/dialogflow v1.16.1/go.mod h1:po6LlzGfK+smoSmTBnbkIZY2w8ffjz/RcGSS+sh1el0= +cloud.google.com/go/dialogflow v1.17.0/go.mod h1:YNP09C/kXA1aZdBgC/VtXX74G/TKn7XVCcVumTflA+8= +cloud.google.com/go/dialogflow v1.18.0/go.mod h1:trO7Zu5YdyEuR+BhSNOqJezyFQ3aUzz0njv7sMx/iek= +cloud.google.com/go/dialogflow v1.19.0/go.mod h1:JVmlG1TwykZDtxtTXujec4tQ+D8SBFMoosgy+6Gn0s0= +cloud.google.com/go/dialogflow v1.29.0/go.mod h1:b+2bzMe+k1s9V+F2jbJwpHPzrnIyHihAdRFMtn2WXuM= +cloud.google.com/go/dialogflow v1.31.0/go.mod h1:cuoUccuL1Z+HADhyIA7dci3N5zUssgpBJmCzI6fNRB4= +cloud.google.com/go/dialogflow v1.32.0/go.mod h1:jG9TRJl8CKrDhMEcvfcfFkkpp8ZhgPz3sBGmAUYJ2qE= +cloud.google.com/go/dialogflow v1.38.0/go.mod h1:L7jnH+JL2mtmdChzAIcXQHXMvQkE3U4hTaNltEuxXn4= +cloud.google.com/go/dlp v1.6.0/go.mod h1:9eyB2xIhpU0sVwUixfBubDoRwP+GjeUoxxeueZmqvmM= +cloud.google.com/go/dlp v1.7.0/go.mod h1:68ak9vCiMBjbasxeVD17hVPxDEck+ExiHavX8kiHG+Q= +cloud.google.com/go/dlp v1.9.0/go.mod h1:qdgmqgTyReTz5/YNSSuueR8pl7hO0o9bQ39ZhtgkWp4= +cloud.google.com/go/dlp v1.10.1/go.mod h1:IM8BWz1iJd8njcNcG0+Kyd9OPnqnRNkDV8j42VT5KOI= +cloud.google.com/go/documentai v1.7.0/go.mod h1:lJvftZB5NRiFSX4moiye1SMxHx0Bc3x1+p9e/RfXYiU= +cloud.google.com/go/documentai v1.8.0/go.mod h1:xGHNEB7CtsnySCNrCFdCyyMz44RhFEEX2Q7UD0c5IhU= +cloud.google.com/go/documentai v1.9.0/go.mod h1:FS5485S8R00U10GhgBC0aNGrJxBP8ZVpEeJ7PQDZd6k= +cloud.google.com/go/documentai v1.10.0/go.mod h1:vod47hKQIPeCfN2QS/jULIvQTugbmdc0ZvxxfQY1bg4= +cloud.google.com/go/documentai v1.16.0/go.mod h1:o0o0DLTEZ+YnJZ+J4wNfTxmDVyrkzFvttBXXtYRMHkM= +cloud.google.com/go/documentai v1.18.0/go.mod h1:F6CK6iUH8J81FehpskRmhLq/3VlwQvb7TvwOceQ2tbs= +cloud.google.com/go/documentai v1.20.0/go.mod h1:yJkInoMcK0qNAEdRnqY/D5asy73tnPe88I1YTZT+a8E= +cloud.google.com/go/domains v0.6.0/go.mod h1:T9Rz3GasrpYk6mEGHh4rymIhjlnIuB4ofT1wTxDeT4Y= +cloud.google.com/go/domains v0.7.0/go.mod h1:PtZeqS1xjnXuRPKE/88Iru/LdfoRyEHYA9nFQf4UKpg= +cloud.google.com/go/domains v0.8.0/go.mod h1:M9i3MMDzGFXsydri9/vW+EWz9sWb4I6WyHqdlAk0idE= +cloud.google.com/go/domains v0.9.1/go.mod h1:aOp1c0MbejQQ2Pjf1iJvnVyT+z6R6s8pX66KaCSDYfE= +cloud.google.com/go/edgecontainer v0.1.0/go.mod h1:WgkZ9tp10bFxqO8BLPqv2LlfmQF1X8lZqwW4r1BTajk= +cloud.google.com/go/edgecontainer v0.2.0/go.mod h1:RTmLijy+lGpQ7BXuTDa4C4ssxyXT34NIuHIgKuP4s5w= +cloud.google.com/go/edgecontainer v0.3.0/go.mod h1:FLDpP4nykgwwIfcLt6zInhprzw0lEi2P1fjO6Ie0qbc= +cloud.google.com/go/edgecontainer v1.0.0/go.mod h1:cttArqZpBB2q58W/upSG++ooo6EsblxDIolxa3jSjbY= +cloud.google.com/go/edgecontainer v1.1.1/go.mod h1:O5bYcS//7MELQZs3+7mabRqoWQhXCzenBu0R8bz2rwk= +cloud.google.com/go/errorreporting v0.3.0/go.mod h1:xsP2yaAp+OAW4OIm60An2bbLpqIhKXdWR/tawvl7QzU= +cloud.google.com/go/essentialcontacts v1.3.0/go.mod h1:r+OnHa5jfj90qIfZDO/VztSFqbQan7HV75p8sA+mdGI= +cloud.google.com/go/essentialcontacts v1.4.0/go.mod h1:8tRldvHYsmnBCHdFpvU+GL75oWiBKl80BiqlFh9tp+8= +cloud.google.com/go/essentialcontacts v1.5.0/go.mod h1:ay29Z4zODTuwliK7SnX8E86aUF2CTzdNtvv42niCX0M= +cloud.google.com/go/essentialcontacts v1.6.2/go.mod h1:T2tB6tX+TRak7i88Fb2N9Ok3PvY3UNbUsMag9/BARh4= +cloud.google.com/go/eventarc v1.7.0/go.mod h1:6ctpF3zTnaQCxUjHUdcfgcA1A2T309+omHZth7gDfmc= +cloud.google.com/go/eventarc v1.8.0/go.mod h1:imbzxkyAU4ubfsaKYdQg04WS1NvncblHEup4kvF+4gw= +cloud.google.com/go/eventarc v1.10.0/go.mod h1:u3R35tmZ9HvswGRBnF48IlYgYeBcPUCjkr4BTdem2Kw= +cloud.google.com/go/eventarc v1.11.0/go.mod h1:PyUjsUKPWoRBCHeOxZd/lbOOjahV41icXyUY5kSTvVY= +cloud.google.com/go/eventarc v1.12.1/go.mod h1:mAFCW6lukH5+IZjkvrEss+jmt2kOdYlN8aMx3sRJiAI= +cloud.google.com/go/filestore v1.3.0/go.mod h1:+qbvHGvXU1HaKX2nD0WEPo92TP/8AQuCVEBXNY9z0+w= +cloud.google.com/go/filestore v1.4.0/go.mod h1:PaG5oDfo9r224f8OYXURtAsY+Fbyq/bLYoINEK8XQAI= +cloud.google.com/go/filestore v1.5.0/go.mod h1:FqBXDWBp4YLHqRnVGveOkHDf8svj9r5+mUDLupOWEDs= +cloud.google.com/go/filestore v1.6.0/go.mod h1:di5unNuss/qfZTw2U9nhFqo8/ZDSc466dre85Kydllg= +cloud.google.com/go/filestore v1.7.1/go.mod h1:y10jsorq40JJnjR/lQ8AfFbbcGlw3g+Dp8oN7i7FjV4= +cloud.google.com/go/firestore v1.9.0/go.mod h1:HMkjKHNTtRyZNiMzu7YAsLr9K3X2udY2AMwDaMEQiiE= +cloud.google.com/go/firestore v1.11.0/go.mod h1:b38dKhgzlmNNGTNZZwe7ZRFEuRab1Hay3/DBsIGKKy4= +cloud.google.com/go/functions v1.6.0/go.mod h1:3H1UA3qiIPRWD7PeZKLvHZ9SaQhR26XIJcC0A5GbvAk= +cloud.google.com/go/functions v1.7.0/go.mod h1:+d+QBcWM+RsrgZfV9xo6KfA1GlzJfxcfZcRPEhDDfzg= +cloud.google.com/go/functions v1.8.0/go.mod h1:RTZ4/HsQjIqIYP9a9YPbU+QFoQsAlYgrwOXJWHn1POY= +cloud.google.com/go/functions v1.9.0/go.mod h1:Y+Dz8yGguzO3PpIjhLTbnqV1CWmgQ5UwtlpzoyquQ08= +cloud.google.com/go/functions v1.10.0/go.mod h1:0D3hEOe3DbEvCXtYOZHQZmD+SzYsi1YbI7dGvHfldXw= +cloud.google.com/go/functions v1.12.0/go.mod h1:AXWGrF3e2C/5ehvwYo/GH6O5s09tOPksiKhz+hH8WkA= +cloud.google.com/go/functions v1.13.0/go.mod h1:EU4O007sQm6Ef/PwRsI8N2umygGqPBS/IZQKBQBcJ3c= +cloud.google.com/go/functions v1.15.1/go.mod h1:P5yNWUTkyU+LvW/S9O6V+V423VZooALQlqoXdoPz5AE= +cloud.google.com/go/gaming v1.5.0/go.mod h1:ol7rGcxP/qHTRQE/RO4bxkXq+Fix0j6D4LFPzYTIrDM= +cloud.google.com/go/gaming v1.6.0/go.mod h1:YMU1GEvA39Qt3zWGyAVA9bpYz/yAhTvaQ1t2sK4KPUA= +cloud.google.com/go/gaming v1.7.0/go.mod h1:LrB8U7MHdGgFG851iHAfqUdLcKBdQ55hzXy9xBJz0+w= +cloud.google.com/go/gaming v1.8.0/go.mod h1:xAqjS8b7jAVW0KFYeRUxngo9My3f33kFmua++Pi+ggM= +cloud.google.com/go/gaming v1.9.0/go.mod h1:Fc7kEmCObylSWLO334NcO+O9QMDyz+TKC4v1D7X+Bc0= +cloud.google.com/go/gaming v1.10.1/go.mod h1:XQQvtfP8Rb9Rxnxm5wFVpAp9zCQkJi2bLIb7iHGwB3s= +cloud.google.com/go/gkebackup v0.2.0/go.mod h1:XKvv/4LfG829/B8B7xRkk8zRrOEbKtEam6yNfuQNH60= +cloud.google.com/go/gkebackup v0.3.0/go.mod h1:n/E671i1aOQvUxT541aTkCwExO/bTer2HDlj4TsBRAo= +cloud.google.com/go/gkebackup v0.4.0/go.mod h1:byAyBGUwYGEEww7xsbnUTBHIYcOPy/PgUWUtOeRm9Vg= +cloud.google.com/go/gkeconnect v0.5.0/go.mod h1:c5lsNAg5EwAy7fkqX/+goqFsU1Da/jQFqArp+wGNr/o= +cloud.google.com/go/gkeconnect v0.6.0/go.mod h1:Mln67KyU/sHJEBY8kFZ0xTeyPtzbq9StAVvEULYK16A= +cloud.google.com/go/gkeconnect v0.7.0/go.mod h1:SNfmVqPkaEi3bF/B3CNZOAYPYdg7sU+obZ+QTky2Myw= +cloud.google.com/go/gkeconnect v0.8.1/go.mod h1:KWiK1g9sDLZqhxB2xEuPV8V9NYzrqTUmQR9shJHpOZw= +cloud.google.com/go/gkehub v0.9.0/go.mod h1:WYHN6WG8w9bXU0hqNxt8rm5uxnk8IH+lPY9J2TV7BK0= +cloud.google.com/go/gkehub v0.10.0/go.mod h1:UIPwxI0DsrpsVoWpLB0stwKCP+WFVG9+y977wO+hBH0= +cloud.google.com/go/gkehub v0.11.0/go.mod h1:JOWHlmN+GHyIbuWQPl47/C2RFhnFKH38jH9Ascu3n0E= +cloud.google.com/go/gkehub v0.12.0/go.mod h1:djiIwwzTTBrF5NaXCGv3mf7klpEMcST17VBTVVDcuaw= +cloud.google.com/go/gkehub v0.14.1/go.mod h1:VEXKIJZ2avzrbd7u+zeMtW00Y8ddk/4V9511C9CQGTY= +cloud.google.com/go/gkemulticloud v0.3.0/go.mod h1:7orzy7O0S+5kq95e4Hpn7RysVA7dPs8W/GgfUtsPbrA= +cloud.google.com/go/gkemulticloud v0.4.0/go.mod h1:E9gxVBnseLWCk24ch+P9+B2CoDFJZTyIgLKSalC7tuI= +cloud.google.com/go/gkemulticloud v0.5.0/go.mod h1:W0JDkiyi3Tqh0TJr//y19wyb1yf8llHVto2Htf2Ja3Y= +cloud.google.com/go/gkemulticloud v0.6.1/go.mod h1:kbZ3HKyTsiwqKX7Yw56+wUGwwNZViRnxWK2DVknXWfw= +cloud.google.com/go/grafeas v0.2.0/go.mod h1:KhxgtF2hb0P191HlY5besjYm6MqTSTj3LSI+M+ByZHc= +cloud.google.com/go/grafeas v0.3.0/go.mod h1:P7hgN24EyONOTMyeJH6DxG4zD7fwiYa5Q6GUgyFSOU8= +cloud.google.com/go/gsuiteaddons v1.3.0/go.mod h1:EUNK/J1lZEZO8yPtykKxLXI6JSVN2rg9bN8SXOa0bgM= +cloud.google.com/go/gsuiteaddons v1.4.0/go.mod h1:rZK5I8hht7u7HxFQcFei0+AtfS9uSushomRlg+3ua1o= +cloud.google.com/go/gsuiteaddons v1.5.0/go.mod h1:TFCClYLd64Eaa12sFVmUyG62tk4mdIsI7pAnSXRkcFo= +cloud.google.com/go/gsuiteaddons v1.6.1/go.mod h1:CodrdOqRZcLp5WOwejHWYBjZvfY0kOphkAKpF/3qdZY= +cloud.google.com/go/iam v0.1.0/go.mod h1:vcUNEa0pEm0qRVpmWepWaFMIAI8/hjB9mO8rNCJtF6c= +cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= +cloud.google.com/go/iam v0.5.0/go.mod h1:wPU9Vt0P4UmCux7mqtRu6jcpPAb74cP1fh50J3QpkUc= +cloud.google.com/go/iam v0.6.0/go.mod h1:+1AH33ueBne5MzYccyMHtEKqLE4/kJOibtffMHDMFMc= +cloud.google.com/go/iam v0.7.0/go.mod h1:H5Br8wRaDGNc8XP3keLc4unfUUZeyH3Sfl9XpQEYOeg= +cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGESjkE= +cloud.google.com/go/iam v0.11.0/go.mod h1:9PiLDanza5D+oWFZiH1uG+RnRCfEGKoyl6yo4cgWZGY= +cloud.google.com/go/iam v0.12.0/go.mod h1:knyHGviacl11zrtZUoDuYpDgLjvr28sLQaG0YB2GYAY= +cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= +cloud.google.com/go/iam v1.0.1/go.mod h1:yR3tmSL8BcZB4bxByRv2jkSIahVmCtfKZwLYGBalRE8= +cloud.google.com/go/iam v1.1.0/go.mod h1:nxdHjaKfCr7fNYx/HJMM8LgiMugmveWlkatear5gVyk= +cloud.google.com/go/iam v1.1.1 h1:lW7fzj15aVIXYHREOqjRBV9PsH0Z6u8Y46a1YGvQP4Y= +cloud.google.com/go/iam v1.1.1/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU= +cloud.google.com/go/iap v1.4.0/go.mod h1:RGFwRJdihTINIe4wZ2iCP0zF/qu18ZwyKxrhMhygBEc= +cloud.google.com/go/iap v1.5.0/go.mod h1:UH/CGgKd4KyohZL5Pt0jSKE4m3FR51qg6FKQ/z/Ix9A= +cloud.google.com/go/iap v1.6.0/go.mod h1:NSuvI9C/j7UdjGjIde7t7HBz+QTwBcapPE07+sSRcLk= +cloud.google.com/go/iap v1.7.0/go.mod h1:beqQx56T9O1G1yNPph+spKpNibDlYIiIixiqsQXxLIo= +cloud.google.com/go/iap v1.7.1/go.mod h1:WapEwPc7ZxGt2jFGB/C/bm+hP0Y6NXzOYGjpPnmMS74= +cloud.google.com/go/iap v1.8.1/go.mod h1:sJCbeqg3mvWLqjZNsI6dfAtbbV1DL2Rl7e1mTyXYREQ= +cloud.google.com/go/ids v1.1.0/go.mod h1:WIuwCaYVOzHIj2OhN9HAwvW+DBdmUAdcWlFxRl+KubM= +cloud.google.com/go/ids v1.2.0/go.mod h1:5WXvp4n25S0rA/mQWAg1YEEBBq6/s+7ml1RDCW1IrcY= +cloud.google.com/go/ids v1.3.0/go.mod h1:JBdTYwANikFKaDP6LtW5JAi4gubs57SVNQjemdt6xV4= +cloud.google.com/go/ids v1.4.1/go.mod h1:np41ed8YMU8zOgv53MMMoCntLTn2lF+SUzlM+O3u/jw= +cloud.google.com/go/iot v1.3.0/go.mod h1:r7RGh2B61+B8oz0AGE+J72AhA0G7tdXItODWsaA2oLs= +cloud.google.com/go/iot v1.4.0/go.mod h1:dIDxPOn0UvNDUMD8Ger7FIaTuvMkj+aGk94RPP0iV+g= +cloud.google.com/go/iot v1.5.0/go.mod h1:mpz5259PDl3XJthEmh9+ap0affn/MqNSP4My77Qql9o= +cloud.google.com/go/iot v1.6.0/go.mod h1:IqdAsmE2cTYYNO1Fvjfzo9po179rAtJeVGUvkLN3rLE= +cloud.google.com/go/iot v1.7.1/go.mod h1:46Mgw7ev1k9KqK1ao0ayW9h0lI+3hxeanz+L1zmbbbk= +cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxsnnOA= +cloud.google.com/go/kms v1.5.0/go.mod h1:QJS2YY0eJGBg3mnDfuaCyLauWwBJiHRboYxJ++1xJNg= +cloud.google.com/go/kms v1.6.0/go.mod h1:Jjy850yySiasBUDi6KFUwUv2n1+o7QZFyuUJg6OgjA0= +cloud.google.com/go/kms v1.8.0/go.mod h1:4xFEhYFqvW+4VMELtZyxomGSYtSQKzM178ylFW4jMAg= +cloud.google.com/go/kms v1.9.0/go.mod h1:qb1tPTgfF9RQP8e1wq4cLFErVuTJv7UsSC915J8dh3w= +cloud.google.com/go/kms v1.10.0/go.mod h1:ng3KTUtQQU9bPX3+QGLsflZIHlkbn8amFAMY63m8d24= +cloud.google.com/go/kms v1.10.1/go.mod h1:rIWk/TryCkR59GMC3YtHtXeLzd634lBbKenvyySAyYI= +cloud.google.com/go/kms v1.11.0/go.mod h1:hwdiYC0xjnWsKQQCQQmIQnS9asjYVSK6jtXm+zFqXLM= +cloud.google.com/go/kms v1.12.1/go.mod h1:c9J991h5DTl+kg7gi3MYomh12YEENGrf48ee/N/2CDM= +cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic= +cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= +cloud.google.com/go/language v1.7.0/go.mod h1:DJ6dYN/W+SQOjF8e1hLQXMF21AkH2w9wiPzPCJa2MIE= +cloud.google.com/go/language v1.8.0/go.mod h1:qYPVHf7SPoNNiCL2Dr0FfEFNil1qi3pQEyygwpgVKB8= +cloud.google.com/go/language v1.9.0/go.mod h1:Ns15WooPM5Ad/5no/0n81yUetis74g3zrbeJBE+ptUY= +cloud.google.com/go/language v1.10.1/go.mod h1:CPp94nsdVNiQEt1CNjF5WkTcisLiHPyIbMhvR8H2AW0= +cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8= +cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6t/iPhY2Tyfu08= +cloud.google.com/go/lifesciences v0.8.0/go.mod h1:lFxiEOMqII6XggGbOnKiyZ7IBwoIqA84ClvoezaA/bo= +cloud.google.com/go/lifesciences v0.9.1/go.mod h1:hACAOd1fFbCGLr/+weUKRAJas82Y4vrL3O5326N//Wc= +cloud.google.com/go/logging v1.6.1/go.mod h1:5ZO0mHHbvm8gEmeEUHrmDlTDSu5imF6MUP9OfilNXBw= +cloud.google.com/go/logging v1.7.0 h1:CJYxlNNNNAMkHp9em/YEXcfJg+rPDg7YfwoRpMU+t5I= +cloud.google.com/go/logging v1.7.0/go.mod h1:3xjP2CjkM3ZkO73aj4ASA5wRPGGCRrPIAeNqVNkzY8M= +cloud.google.com/go/longrunning v0.1.1/go.mod h1:UUFxuDWkv22EuY93jjmDMFT5GPQKeFVJBIF6QlTqdsE= +cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI0YSmW3iIlLJc= +cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo= +cloud.google.com/go/longrunning v0.4.2/go.mod h1:OHrnaYyLUV6oqwh0xiS7e5sLQhP1m0QU9R+WhGDMgIQ= +cloud.google.com/go/longrunning v0.5.0/go.mod h1:0JNuqRShmscVAhIACGtskSAWtqtOoPkwP0YF1oVEchc= +cloud.google.com/go/longrunning v0.5.1 h1:Fr7TXftcqTudoyRJa113hyaqlGdiBQkp0Gq7tErFDWI= +cloud.google.com/go/longrunning v0.5.1/go.mod h1:spvimkwdz6SPWKEt/XBij79E9fiTkHSQl/fRUUQJYJc= +cloud.google.com/go/managedidentities v1.3.0/go.mod h1:UzlW3cBOiPrzucO5qWkNkh0w33KFtBJU281hacNvsdE= +cloud.google.com/go/managedidentities v1.4.0/go.mod h1:NWSBYbEMgqmbZsLIyKvxrYbtqOsxY1ZrGM+9RgDqInM= +cloud.google.com/go/managedidentities v1.5.0/go.mod h1:+dWcZ0JlUmpuxpIDfyP5pP5y0bLdRwOS4Lp7gMni/LA= +cloud.google.com/go/managedidentities v1.6.1/go.mod h1:h/irGhTN2SkZ64F43tfGPMbHnypMbu4RB3yl8YcuEak= +cloud.google.com/go/maps v0.1.0/go.mod h1:BQM97WGyfw9FWEmQMpZ5T6cpovXXSd1cGmFma94eubI= +cloud.google.com/go/maps v0.6.0/go.mod h1:o6DAMMfb+aINHz/p/jbcY+mYeXBoZoxTfdSQ8VAJaCw= +cloud.google.com/go/maps v0.7.0/go.mod h1:3GnvVl3cqeSvgMcpRlQidXsPYuDGQ8naBis7MVzpXsY= +cloud.google.com/go/mediatranslation v0.5.0/go.mod h1:jGPUhGTybqsPQn91pNXw0xVHfuJ3leR1wj37oU3y1f4= +cloud.google.com/go/mediatranslation v0.6.0/go.mod h1:hHdBCTYNigsBxshbznuIMFNe5QXEowAuNmmC7h8pu5w= +cloud.google.com/go/mediatranslation v0.7.0/go.mod h1:LCnB/gZr90ONOIQLgSXagp8XUW1ODs2UmUMvcgMfI2I= +cloud.google.com/go/mediatranslation v0.8.1/go.mod h1:L/7hBdEYbYHQJhX2sldtTO5SZZ1C1vkapubj0T2aGig= +cloud.google.com/go/memcache v1.4.0/go.mod h1:rTOfiGZtJX1AaFUrOgsMHX5kAzaTQ8azHiuDoTPzNsE= +cloud.google.com/go/memcache v1.5.0/go.mod h1:dk3fCK7dVo0cUU2c36jKb4VqKPS22BTkf81Xq617aWM= +cloud.google.com/go/memcache v1.6.0/go.mod h1:XS5xB0eQZdHtTuTF9Hf8eJkKtR3pVRCcvJwtm68T3rA= +cloud.google.com/go/memcache v1.7.0/go.mod h1:ywMKfjWhNtkQTxrWxCkCFkoPjLHPW6A7WOTVI8xy3LY= +cloud.google.com/go/memcache v1.9.0/go.mod h1:8oEyzXCu+zo9RzlEaEjHl4KkgjlNDaXbCQeQWlzNFJM= +cloud.google.com/go/memcache v1.10.1/go.mod h1:47YRQIarv4I3QS5+hoETgKO40InqzLP6kpNLvyXuyaA= +cloud.google.com/go/metastore v1.5.0/go.mod h1:2ZNrDcQwghfdtCwJ33nM0+GrBGlVuh8rakL3vdPY3XY= +cloud.google.com/go/metastore v1.6.0/go.mod h1:6cyQTls8CWXzk45G55x57DVQ9gWg7RiH65+YgPsNh9s= +cloud.google.com/go/metastore v1.7.0/go.mod h1:s45D0B4IlsINu87/AsWiEVYbLaIMeUSoxlKKDqBGFS8= +cloud.google.com/go/metastore v1.8.0/go.mod h1:zHiMc4ZUpBiM7twCIFQmJ9JMEkDSyZS9U12uf7wHqSI= +cloud.google.com/go/metastore v1.10.0/go.mod h1:fPEnH3g4JJAk+gMRnrAnoqyv2lpUCqJPWOodSaf45Eo= +cloud.google.com/go/metastore v1.11.1/go.mod h1:uZuSo80U3Wd4zi6C22ZZliOUJ3XeM/MlYi/z5OAOWRA= +cloud.google.com/go/monitoring v1.1.0/go.mod h1:L81pzz7HKn14QCMaCs6NTQkdBnE87TElyanS95vIcl4= +cloud.google.com/go/monitoring v1.7.0/go.mod h1:HpYse6kkGo//7p6sT0wsIC6IBDET0RhIsnmlA53dvEk= +cloud.google.com/go/monitoring v1.8.0/go.mod h1:E7PtoMJ1kQXWxPjB6mv2fhC5/15jInuulFdYYtlcvT4= +cloud.google.com/go/monitoring v1.12.0/go.mod h1:yx8Jj2fZNEkL/GYZyTLS4ZtZEZN8WtDEiEqG4kLK50w= +cloud.google.com/go/monitoring v1.13.0/go.mod h1:k2yMBAB1H9JT/QETjNkgdCGD9bPF712XiLTVr+cBrpw= +cloud.google.com/go/monitoring v1.15.1 h1:65JhLMd+JiYnXr6j5Z63dUYCuOg770p8a/VC+gil/58= +cloud.google.com/go/monitoring v1.15.1/go.mod h1:lADlSAlFdbqQuwwpaImhsJXu1QSdd3ojypXrFSMr2rM= +cloud.google.com/go/networkconnectivity v1.4.0/go.mod h1:nOl7YL8odKyAOtzNX73/M5/mGZgqqMeryi6UPZTk/rA= +cloud.google.com/go/networkconnectivity v1.5.0/go.mod h1:3GzqJx7uhtlM3kln0+x5wyFvuVH1pIBJjhCpjzSt75o= +cloud.google.com/go/networkconnectivity v1.6.0/go.mod h1:OJOoEXW+0LAxHh89nXd64uGG+FbQoeH8DtxCHVOMlaM= +cloud.google.com/go/networkconnectivity v1.7.0/go.mod h1:RMuSbkdbPwNMQjB5HBWD5MpTBnNm39iAVpC3TmsExt8= +cloud.google.com/go/networkconnectivity v1.10.0/go.mod h1:UP4O4sWXJG13AqrTdQCD9TnLGEbtNRqjuaaA7bNjF5E= +cloud.google.com/go/networkconnectivity v1.11.0/go.mod h1:iWmDD4QF16VCDLXUqvyspJjIEtBR/4zq5hwnY2X3scM= +cloud.google.com/go/networkconnectivity v1.12.1/go.mod h1:PelxSWYM7Sh9/guf8CFhi6vIqf19Ir/sbfZRUwXh92E= +cloud.google.com/go/networkmanagement v1.4.0/go.mod h1:Q9mdLLRn60AsOrPc8rs8iNV6OHXaGcDdsIQe1ohekq8= +cloud.google.com/go/networkmanagement v1.5.0/go.mod h1:ZnOeZ/evzUdUsnvRt792H0uYEnHQEMaz+REhhzJRcf4= +cloud.google.com/go/networkmanagement v1.6.0/go.mod h1:5pKPqyXjB/sgtvB5xqOemumoQNB7y95Q7S+4rjSOPYY= +cloud.google.com/go/networkmanagement v1.8.0/go.mod h1:Ho/BUGmtyEqrttTgWEe7m+8vDdK74ibQc+Be0q7Fof0= +cloud.google.com/go/networksecurity v0.5.0/go.mod h1:xS6fOCoqpVC5zx15Z/MqkfDwH4+m/61A3ODiDV1xmiQ= +cloud.google.com/go/networksecurity v0.6.0/go.mod h1:Q5fjhTr9WMI5mbpRYEbiexTzROf7ZbDzvzCrNl14nyU= +cloud.google.com/go/networksecurity v0.7.0/go.mod h1:mAnzoxx/8TBSyXEeESMy9OOYwo1v+gZ5eMRnsT5bC8k= +cloud.google.com/go/networksecurity v0.8.0/go.mod h1:B78DkqsxFG5zRSVuwYFRZ9Xz8IcQ5iECsNrPn74hKHU= +cloud.google.com/go/networksecurity v0.9.1/go.mod h1:MCMdxOKQ30wsBI1eI659f9kEp4wuuAueoC9AJKSPWZQ= +cloud.google.com/go/notebooks v1.2.0/go.mod h1:9+wtppMfVPUeJ8fIWPOq1UnATHISkGXGqTkxeieQ6UY= +cloud.google.com/go/notebooks v1.3.0/go.mod h1:bFR5lj07DtCPC7YAAJ//vHskFBxA5JzYlH68kXVdk34= +cloud.google.com/go/notebooks v1.4.0/go.mod h1:4QPMngcwmgb6uw7Po99B2xv5ufVoIQ7nOGDyL4P8AgA= +cloud.google.com/go/notebooks v1.5.0/go.mod h1:q8mwhnP9aR8Hpfnrc5iN5IBhrXUy8S2vuYs+kBJ/gu0= +cloud.google.com/go/notebooks v1.7.0/go.mod h1:PVlaDGfJgj1fl1S3dUwhFMXFgfYGhYQt2164xOMONmE= +cloud.google.com/go/notebooks v1.8.0/go.mod h1:Lq6dYKOYOWUCTvw5t2q1gp1lAp0zxAxRycayS0iJcqQ= +cloud.google.com/go/notebooks v1.9.1/go.mod h1:zqG9/gk05JrzgBt4ghLzEepPHNwE5jgPcHZRKhlC1A8= +cloud.google.com/go/optimization v1.1.0/go.mod h1:5po+wfvX5AQlPznyVEZjGJTMr4+CAkJf2XSTQOOl9l4= +cloud.google.com/go/optimization v1.2.0/go.mod h1:Lr7SOHdRDENsh+WXVmQhQTrzdu9ybg0NecjHidBq6xs= +cloud.google.com/go/optimization v1.3.1/go.mod h1:IvUSefKiwd1a5p0RgHDbWCIbDFgKuEdB+fPPuP0IDLI= +cloud.google.com/go/optimization v1.4.1/go.mod h1:j64vZQP7h9bO49m2rVaTVoNM0vEBEN5eKPUPbZyXOrk= +cloud.google.com/go/orchestration v1.3.0/go.mod h1:Sj5tq/JpWiB//X/q3Ngwdl5K7B7Y0KZ7bfv0wL6fqVA= +cloud.google.com/go/orchestration v1.4.0/go.mod h1:6W5NLFWs2TlniBphAViZEVhrXRSMgUGDfW7vrWKvsBk= +cloud.google.com/go/orchestration v1.6.0/go.mod h1:M62Bevp7pkxStDfFfTuCOaXgaaqRAga1yKyoMtEoWPQ= +cloud.google.com/go/orchestration v1.8.1/go.mod h1:4sluRF3wgbYVRqz7zJ1/EUNc90TTprliq9477fGobD8= +cloud.google.com/go/orgpolicy v1.4.0/go.mod h1:xrSLIV4RePWmP9P3tBl8S93lTmlAxjm06NSm2UTmKvE= +cloud.google.com/go/orgpolicy v1.5.0/go.mod h1:hZEc5q3wzwXJaKrsx5+Ewg0u1LxJ51nNFlext7Tanwc= +cloud.google.com/go/orgpolicy v1.10.0/go.mod h1:w1fo8b7rRqlXlIJbVhOMPrwVljyuW5mqssvBtU18ONc= +cloud.google.com/go/orgpolicy v1.11.0/go.mod h1:2RK748+FtVvnfuynxBzdnyu7sygtoZa1za/0ZfpOs1M= +cloud.google.com/go/orgpolicy v1.11.1/go.mod h1:8+E3jQcpZJQliP+zaFfayC2Pg5bmhuLK755wKhIIUCE= +cloud.google.com/go/osconfig v1.7.0/go.mod h1:oVHeCeZELfJP7XLxcBGTMBvRO+1nQ5tFG9VQTmYS2Fs= +cloud.google.com/go/osconfig v1.8.0/go.mod h1:EQqZLu5w5XA7eKizepumcvWx+m8mJUhEwiPqWiZeEdg= +cloud.google.com/go/osconfig v1.9.0/go.mod h1:Yx+IeIZJ3bdWmzbQU4fxNl8xsZ4amB+dygAwFPlvnNo= +cloud.google.com/go/osconfig v1.10.0/go.mod h1:uMhCzqC5I8zfD9zDEAfvgVhDS8oIjySWh+l4WK6GnWw= +cloud.google.com/go/osconfig v1.11.0/go.mod h1:aDICxrur2ogRd9zY5ytBLV89KEgT2MKB2L/n6x1ooPw= +cloud.google.com/go/osconfig v1.12.0/go.mod h1:8f/PaYzoS3JMVfdfTubkowZYGmAhUCjjwnjqWI7NVBc= +cloud.google.com/go/osconfig v1.12.1/go.mod h1:4CjBxND0gswz2gfYRCUoUzCm9zCABp91EeTtWXyz0tE= +cloud.google.com/go/oslogin v1.4.0/go.mod h1:YdgMXWRaElXz/lDk1Na6Fh5orF7gvmJ0FGLIs9LId4E= +cloud.google.com/go/oslogin v1.5.0/go.mod h1:D260Qj11W2qx/HVF29zBg+0fd6YCSjSqLUkY/qEenQU= +cloud.google.com/go/oslogin v1.6.0/go.mod h1:zOJ1O3+dTU8WPlGEkFSh7qeHPPSoxrcMbbK1Nm2iX70= +cloud.google.com/go/oslogin v1.7.0/go.mod h1:e04SN0xO1UNJ1M5GP0vzVBFicIe4O53FOfcixIqTyXo= +cloud.google.com/go/oslogin v1.9.0/go.mod h1:HNavntnH8nzrn8JCTT5fj18FuJLFJc4NaZJtBnQtKFs= +cloud.google.com/go/oslogin v1.10.1/go.mod h1:x692z7yAue5nE7CsSnoG0aaMbNoRJRXO4sn73R+ZqAs= +cloud.google.com/go/phishingprotection v0.5.0/go.mod h1:Y3HZknsK9bc9dMi+oE8Bim0lczMU6hrX0UpADuMefr0= +cloud.google.com/go/phishingprotection v0.6.0/go.mod h1:9Y3LBLgy0kDTcYET8ZH3bq/7qni15yVUoAxiFxnlSUA= +cloud.google.com/go/phishingprotection v0.7.0/go.mod h1:8qJI4QKHoda/sb/7/YmMQ2omRLSLYSu9bU0EKCNI+Lk= +cloud.google.com/go/phishingprotection v0.8.1/go.mod h1:AxonW7GovcA8qdEk13NfHq9hNx5KPtfxXNeUxTDxB6I= +cloud.google.com/go/policytroubleshooter v1.3.0/go.mod h1:qy0+VwANja+kKrjlQuOzmlvscn4RNsAc0e15GGqfMxg= +cloud.google.com/go/policytroubleshooter v1.4.0/go.mod h1:DZT4BcRw3QoO8ota9xw/LKtPa8lKeCByYeKTIf/vxdE= +cloud.google.com/go/policytroubleshooter v1.5.0/go.mod h1:Rz1WfV+1oIpPdN2VvvuboLVRsB1Hclg3CKQ53j9l8vw= +cloud.google.com/go/policytroubleshooter v1.6.0/go.mod h1:zYqaPTsmfvpjm5ULxAyD/lINQxJ0DDsnWOP/GZ7xzBc= +cloud.google.com/go/policytroubleshooter v1.7.1/go.mod h1:0NaT5v3Ag1M7U5r0GfDCpUFkWd9YqpubBWsQlhanRv0= +cloud.google.com/go/privatecatalog v0.5.0/go.mod h1:XgosMUvvPyxDjAVNDYxJ7wBW8//hLDDYmnsNcMGq1K0= +cloud.google.com/go/privatecatalog v0.6.0/go.mod h1:i/fbkZR0hLN29eEWiiwue8Pb+GforiEIBnV9yrRUOKI= +cloud.google.com/go/privatecatalog v0.7.0/go.mod h1:2s5ssIFO69F5csTXcwBP7NPFTZvps26xGzvQ2PQaBYg= +cloud.google.com/go/privatecatalog v0.8.0/go.mod h1:nQ6pfaegeDAq/Q5lrfCQzQLhubPiZhSaNhIgfJlnIXs= +cloud.google.com/go/privatecatalog v0.9.1/go.mod h1:0XlDXW2unJXdf9zFz968Hp35gl/bhF4twwpXZAW50JA= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/pubsub v1.26.0/go.mod h1:QgBH3U/jdJy/ftjPhTkyXNj543Tin1pRYcdcPRnFIRI= +cloud.google.com/go/pubsub v1.27.1/go.mod h1:hQN39ymbV9geqBnfQq6Xf63yNhUAhv9CZhzp5O6qsW0= +cloud.google.com/go/pubsub v1.28.0/go.mod h1:vuXFpwaVoIPQMGXqRyUQigu/AX1S3IWugR9xznmcXX8= +cloud.google.com/go/pubsub v1.30.0/go.mod h1:qWi1OPS0B+b5L+Sg6Gmc9zD1Y+HaM0MdUr7LsupY1P4= +cloud.google.com/go/pubsub v1.32.0/go.mod h1:f+w71I33OMyxf9VpMVcZbnG5KSUkCOUHYpFd5U1GdRc= +cloud.google.com/go/pubsublite v1.5.0/go.mod h1:xapqNQ1CuLfGi23Yda/9l4bBCKz/wC3KIJ5gKcxveZg= +cloud.google.com/go/pubsublite v1.6.0/go.mod h1:1eFCS0U11xlOuMFV/0iBqw3zP12kddMeCbj/F3FSj9k= +cloud.google.com/go/pubsublite v1.7.0/go.mod h1:8hVMwRXfDfvGm3fahVbtDbiLePT3gpoiJYJY+vxWxVM= +cloud.google.com/go/pubsublite v1.8.1/go.mod h1:fOLdU4f5xldK4RGJrBMm+J7zMWNj/k4PxwEZXy39QS0= +cloud.google.com/go/recaptchaenterprise v1.3.1/go.mod h1:OdD+q+y4XGeAlxRaMn1Y7/GveP6zmq76byL6tjPE7d4= +cloud.google.com/go/recaptchaenterprise/v2 v2.1.0/go.mod h1:w9yVqajwroDNTfGuhmOjPDN//rZGySaf6PtFVcSCa7o= +cloud.google.com/go/recaptchaenterprise/v2 v2.2.0/go.mod h1:/Zu5jisWGeERrd5HnlS3EUGb/D335f9k51B/FVil0jk= +cloud.google.com/go/recaptchaenterprise/v2 v2.3.0/go.mod h1:O9LwGCjrhGHBQET5CA7dd5NwwNQUErSgEDit1DLNTdo= +cloud.google.com/go/recaptchaenterprise/v2 v2.4.0/go.mod h1:Am3LHfOuBstrLrNCBrlI5sbwx9LBg3te2N6hGvHn2mE= +cloud.google.com/go/recaptchaenterprise/v2 v2.5.0/go.mod h1:O8LzcHXN3rz0j+LBC91jrwI3R+1ZSZEWrfL7XHgNo9U= +cloud.google.com/go/recaptchaenterprise/v2 v2.6.0/go.mod h1:RPauz9jeLtB3JVzg6nCbe12qNoaa8pXc4d/YukAmcnA= +cloud.google.com/go/recaptchaenterprise/v2 v2.7.0/go.mod h1:19wVj/fs5RtYtynAPJdDTb69oW0vNHYDBTbB4NvMD9c= +cloud.google.com/go/recaptchaenterprise/v2 v2.7.2/go.mod h1:kR0KjsJS7Jt1YSyWFkseQ756D45kaYNTlDPPaRAvDBU= +cloud.google.com/go/recommendationengine v0.5.0/go.mod h1:E5756pJcVFeVgaQv3WNpImkFP8a+RptV6dDLGPILjvg= +cloud.google.com/go/recommendationengine v0.6.0/go.mod h1:08mq2umu9oIqc7tDy8sx+MNJdLG0fUi3vaSVbztHgJ4= +cloud.google.com/go/recommendationengine v0.7.0/go.mod h1:1reUcE3GIu6MeBz/h5xZJqNLuuVjNg1lmWMPyjatzac= +cloud.google.com/go/recommendationengine v0.8.1/go.mod h1:MrZihWwtFYWDzE6Hz5nKcNz3gLizXVIDI/o3G1DLcrE= +cloud.google.com/go/recommender v1.5.0/go.mod h1:jdoeiBIVrJe9gQjwd759ecLJbxCDED4A6p+mqoqDvTg= +cloud.google.com/go/recommender v1.6.0/go.mod h1:+yETpm25mcoiECKh9DEScGzIRyDKpZ0cEhWGo+8bo+c= +cloud.google.com/go/recommender v1.7.0/go.mod h1:XLHs/W+T8olwlGOgfQenXBTbIseGclClff6lhFVe9Bs= +cloud.google.com/go/recommender v1.8.0/go.mod h1:PkjXrTT05BFKwxaUxQmtIlrtj0kph108r02ZZQ5FE70= +cloud.google.com/go/recommender v1.9.0/go.mod h1:PnSsnZY7q+VL1uax2JWkt/UegHssxjUVVCrX52CuEmQ= +cloud.google.com/go/recommender v1.10.1/go.mod h1:XFvrE4Suqn5Cq0Lf+mCP6oBHD/yRMA8XxP5sb7Q7gpA= +cloud.google.com/go/redis v1.7.0/go.mod h1:V3x5Jq1jzUcg+UNsRvdmsfuFnit1cfe3Z/PGyq/lm4Y= +cloud.google.com/go/redis v1.8.0/go.mod h1:Fm2szCDavWzBk2cDKxrkmWBqoCiL1+Ctwq7EyqBCA/A= +cloud.google.com/go/redis v1.9.0/go.mod h1:HMYQuajvb2D0LvMgZmLDZW8V5aOC/WxstZHiy4g8OiA= +cloud.google.com/go/redis v1.10.0/go.mod h1:ThJf3mMBQtW18JzGgh41/Wld6vnDDc/F/F35UolRZPM= +cloud.google.com/go/redis v1.11.0/go.mod h1:/X6eicana+BWcUda5PpwZC48o37SiFVTFSs0fWAJ7uQ= +cloud.google.com/go/redis v1.13.1/go.mod h1:VP7DGLpE91M6bcsDdMuyCm2hIpB6Vp2hI090Mfd1tcg= +cloud.google.com/go/resourcemanager v1.3.0/go.mod h1:bAtrTjZQFJkiWTPDb1WBjzvc6/kifjj4QBYuKCCoqKA= +cloud.google.com/go/resourcemanager v1.4.0/go.mod h1:MwxuzkumyTX7/a3n37gmsT3py7LIXwrShilPh3P1tR0= +cloud.google.com/go/resourcemanager v1.5.0/go.mod h1:eQoXNAiAvCf5PXxWxXjhKQoTMaUSNrEfg+6qdf/wots= +cloud.google.com/go/resourcemanager v1.6.0/go.mod h1:YcpXGRs8fDzcUl1Xw8uOVmI8JEadvhRIkoXXUNVYcVo= +cloud.google.com/go/resourcemanager v1.7.0/go.mod h1:HlD3m6+bwhzj9XCouqmeiGuni95NTrExfhoSrkC/3EI= +cloud.google.com/go/resourcemanager v1.9.1/go.mod h1:dVCuosgrh1tINZ/RwBufr8lULmWGOkPS8gL5gqyjdT8= +cloud.google.com/go/resourcesettings v1.3.0/go.mod h1:lzew8VfESA5DQ8gdlHwMrqZs1S9V87v3oCnKCWoOuQU= +cloud.google.com/go/resourcesettings v1.4.0/go.mod h1:ldiH9IJpcrlC3VSuCGvjR5of/ezRrOxFtpJoJo5SmXg= +cloud.google.com/go/resourcesettings v1.5.0/go.mod h1:+xJF7QSG6undsQDfsCJyqWXyBwUoJLhetkRMDRnIoXA= +cloud.google.com/go/resourcesettings v1.6.1/go.mod h1:M7mk9PIZrC5Fgsu1kZJci6mpgN8o0IUzVx3eJU3y4Jw= +cloud.google.com/go/retail v1.8.0/go.mod h1:QblKS8waDmNUhghY2TI9O3JLlFk8jybHeV4BF19FrE4= +cloud.google.com/go/retail v1.9.0/go.mod h1:g6jb6mKuCS1QKnH/dpu7isX253absFl6iE92nHwlBUY= +cloud.google.com/go/retail v1.10.0/go.mod h1:2gDk9HsL4HMS4oZwz6daui2/jmKvqShXKQuB2RZ+cCc= +cloud.google.com/go/retail v1.11.0/go.mod h1:MBLk1NaWPmh6iVFSz9MeKG/Psyd7TAgm6y/9L2B4x9Y= +cloud.google.com/go/retail v1.12.0/go.mod h1:UMkelN/0Z8XvKymXFbD4EhFJlYKRx1FGhQkVPU5kF14= +cloud.google.com/go/retail v1.14.1/go.mod h1:y3Wv3Vr2k54dLNIrCzenyKG8g8dhvhncT2NcNjb/6gE= +cloud.google.com/go/run v0.2.0/go.mod h1:CNtKsTA1sDcnqqIFR3Pb5Tq0usWxJJvsWOCPldRU3Do= +cloud.google.com/go/run v0.3.0/go.mod h1:TuyY1+taHxTjrD0ZFk2iAR+xyOXEA0ztb7U3UNA0zBo= +cloud.google.com/go/run v0.8.0/go.mod h1:VniEnuBwqjigv0A7ONfQUaEItaiCRVujlMqerPPiktM= +cloud.google.com/go/run v0.9.0/go.mod h1:Wwu+/vvg8Y+JUApMwEDfVfhetv30hCG4ZwDR/IXl2Qg= +cloud.google.com/go/scheduler v1.4.0/go.mod h1:drcJBmxF3aqZJRhmkHQ9b3uSSpQoltBPGPxGAWROx6s= +cloud.google.com/go/scheduler v1.5.0/go.mod h1:ri073ym49NW3AfT6DZi21vLZrG07GXr5p3H1KxN5QlI= +cloud.google.com/go/scheduler v1.6.0/go.mod h1:SgeKVM7MIwPn3BqtcBntpLyrIJftQISRrYB5ZtT+KOk= +cloud.google.com/go/scheduler v1.7.0/go.mod h1:jyCiBqWW956uBjjPMMuX09n3x37mtyPJegEWKxRsn44= +cloud.google.com/go/scheduler v1.8.0/go.mod h1:TCET+Y5Gp1YgHT8py4nlg2Sew8nUHMqcpousDgXJVQc= +cloud.google.com/go/scheduler v1.9.0/go.mod h1:yexg5t+KSmqu+njTIh3b7oYPheFtBWGcbVUYF1GGMIc= +cloud.google.com/go/scheduler v1.10.1/go.mod h1:R63Ldltd47Bs4gnhQkmNDse5w8gBRrhObZ54PxgR2Oo= +cloud.google.com/go/secretmanager v1.6.0/go.mod h1:awVa/OXF6IiyaU1wQ34inzQNc4ISIDIrId8qE5QGgKA= +cloud.google.com/go/secretmanager v1.8.0/go.mod h1:hnVgi/bN5MYHd3Gt0SPuTPPp5ENina1/LxM+2W9U9J4= +cloud.google.com/go/secretmanager v1.9.0/go.mod h1:b71qH2l1yHmWQHt9LC80akm86mX8AL6X1MA01dW8ht4= +cloud.google.com/go/secretmanager v1.10.0/go.mod h1:MfnrdvKMPNra9aZtQFvBcvRU54hbPD8/HayQdlUgJpU= +cloud.google.com/go/secretmanager v1.11.1/go.mod h1:znq9JlXgTNdBeQk9TBW/FnR/W4uChEKGeqQWAJ8SXFw= +cloud.google.com/go/security v1.5.0/go.mod h1:lgxGdyOKKjHL4YG3/YwIL2zLqMFCKs0UbQwgyZmfJl4= +cloud.google.com/go/security v1.7.0/go.mod h1:mZklORHl6Bg7CNnnjLH//0UlAlaXqiG7Lb9PsPXLfD0= +cloud.google.com/go/security v1.8.0/go.mod h1:hAQOwgmaHhztFhiQ41CjDODdWP0+AE1B3sX4OFlq+GU= +cloud.google.com/go/security v1.9.0/go.mod h1:6Ta1bO8LXI89nZnmnsZGp9lVoVWXqsVbIq/t9dzI+2Q= +cloud.google.com/go/security v1.10.0/go.mod h1:QtOMZByJVlibUT2h9afNDWRZ1G96gVywH8T5GUSb9IA= +cloud.google.com/go/security v1.12.0/go.mod h1:rV6EhrpbNHrrxqlvW0BWAIawFWq3X90SduMJdFwtLB8= +cloud.google.com/go/security v1.13.0/go.mod h1:Q1Nvxl1PAgmeW0y3HTt54JYIvUdtcpYKVfIB8AOMZ+0= +cloud.google.com/go/security v1.15.1/go.mod h1:MvTnnbsWnehoizHi09zoiZob0iCHVcL4AUBj76h9fXA= +cloud.google.com/go/securitycenter v1.13.0/go.mod h1:cv5qNAqjY84FCN6Y9z28WlkKXyWsgLO832YiWwkCWcU= +cloud.google.com/go/securitycenter v1.14.0/go.mod h1:gZLAhtyKv85n52XYWt6RmeBdydyxfPeTrpToDPw4Auc= +cloud.google.com/go/securitycenter v1.15.0/go.mod h1:PeKJ0t8MoFmmXLXWm41JidyzI3PJjd8sXWaVqg43WWk= +cloud.google.com/go/securitycenter v1.16.0/go.mod h1:Q9GMaLQFUD+5ZTabrbujNWLtSLZIZF7SAR0wWECrjdk= +cloud.google.com/go/securitycenter v1.18.1/go.mod h1:0/25gAzCM/9OL9vVx4ChPeM/+DlfGQJDwBy/UC8AKK0= +cloud.google.com/go/securitycenter v1.19.0/go.mod h1:LVLmSg8ZkkyaNy4u7HCIshAngSQ8EcIRREP3xBnyfag= +cloud.google.com/go/securitycenter v1.23.0/go.mod h1:8pwQ4n+Y9WCWM278R8W3nF65QtY172h4S8aXyI9/hsQ= +cloud.google.com/go/servicecontrol v1.4.0/go.mod h1:o0hUSJ1TXJAmi/7fLJAedOovnujSEvjKCAFNXPQ1RaU= +cloud.google.com/go/servicecontrol v1.5.0/go.mod h1:qM0CnXHhyqKVuiZnGKrIurvVImCs8gmqWsDoqe9sU1s= +cloud.google.com/go/servicecontrol v1.10.0/go.mod h1:pQvyvSRh7YzUF2efw7H87V92mxU8FnFDawMClGCNuAA= +cloud.google.com/go/servicecontrol v1.11.0/go.mod h1:kFmTzYzTUIuZs0ycVqRHNaNhgR+UMUpw9n02l/pY+mc= +cloud.google.com/go/servicecontrol v1.11.1/go.mod h1:aSnNNlwEFBY+PWGQ2DoM0JJ/QUXqV5/ZD9DOLB7SnUk= +cloud.google.com/go/servicedirectory v1.4.0/go.mod h1:gH1MUaZCgtP7qQiI+F+A+OpeKF/HQWgtAddhTbhL2bs= +cloud.google.com/go/servicedirectory v1.5.0/go.mod h1:QMKFL0NUySbpZJ1UZs3oFAmdvVxhhxB6eJ/Vlp73dfg= +cloud.google.com/go/servicedirectory v1.6.0/go.mod h1:pUlbnWsLH9c13yGkxCmfumWEPjsRs1RlmJ4pqiNjVL4= +cloud.google.com/go/servicedirectory v1.7.0/go.mod h1:5p/U5oyvgYGYejufvxhgwjL8UVXjkuw7q5XcG10wx1U= +cloud.google.com/go/servicedirectory v1.8.0/go.mod h1:srXodfhY1GFIPvltunswqXpVxFPpZjf8nkKQT7XcXaY= +cloud.google.com/go/servicedirectory v1.9.0/go.mod h1:29je5JjiygNYlmsGz8k6o+OZ8vd4f//bQLtvzkPPT/s= +cloud.google.com/go/servicedirectory v1.10.1/go.mod h1:Xv0YVH8s4pVOwfM/1eMTl0XJ6bzIOSLDt8f8eLaGOxQ= +cloud.google.com/go/servicemanagement v1.4.0/go.mod h1:d8t8MDbezI7Z2R1O/wu8oTggo3BI2GKYbdG4y/SJTco= +cloud.google.com/go/servicemanagement v1.5.0/go.mod h1:XGaCRe57kfqu4+lRxaFEAuqmjzF0r+gWHjWqKqBvKFo= +cloud.google.com/go/servicemanagement v1.6.0/go.mod h1:aWns7EeeCOtGEX4OvZUWCCJONRZeFKiptqKf1D0l/Jc= +cloud.google.com/go/servicemanagement v1.8.0/go.mod h1:MSS2TDlIEQD/fzsSGfCdJItQveu9NXnUniTrq/L8LK4= +cloud.google.com/go/serviceusage v1.3.0/go.mod h1:Hya1cozXM4SeSKTAgGXgj97GlqUvF5JaoXacR1JTP/E= +cloud.google.com/go/serviceusage v1.4.0/go.mod h1:SB4yxXSaYVuUBYUml6qklyONXNLt83U0Rb+CXyhjEeU= +cloud.google.com/go/serviceusage v1.5.0/go.mod h1:w8U1JvqUqwJNPEOTQjrMHkw3IaIFLoLsPLvsE3xueec= +cloud.google.com/go/serviceusage v1.6.0/go.mod h1:R5wwQcbOWsyuOfbP9tGdAnCAc6B9DRwPG1xtWMDeuPA= +cloud.google.com/go/shell v1.3.0/go.mod h1:VZ9HmRjZBsjLGXusm7K5Q5lzzByZmJHf1d0IWHEN5X4= +cloud.google.com/go/shell v1.4.0/go.mod h1:HDxPzZf3GkDdhExzD/gs8Grqk+dmYcEjGShZgYa9URw= +cloud.google.com/go/shell v1.6.0/go.mod h1:oHO8QACS90luWgxP3N9iZVuEiSF84zNyLytb+qE2f9A= +cloud.google.com/go/shell v1.7.1/go.mod h1:u1RaM+huXFaTojTbW4g9P5emOrrmLE69KrxqQahKn4g= +cloud.google.com/go/spanner v1.41.0/go.mod h1:MLYDBJR/dY4Wt7ZaMIQ7rXOTLjYrmxLE/5ve9vFfWos= +cloud.google.com/go/spanner v1.44.0/go.mod h1:G8XIgYdOK+Fbcpbs7p2fiprDw4CaZX63whnSMLVBxjk= +cloud.google.com/go/spanner v1.45.0/go.mod h1:FIws5LowYz8YAE1J8fOS7DJup8ff7xJeetWEo5REA2M= +cloud.google.com/go/spanner v1.47.0/go.mod h1:IXsJwVW2j4UKs0eYDqodab6HgGuA1bViSqW4uH9lfUI= +cloud.google.com/go/speech v1.6.0/go.mod h1:79tcr4FHCimOp56lwC01xnt/WPJZc4v3gzyT7FoBkCM= +cloud.google.com/go/speech v1.7.0/go.mod h1:KptqL+BAQIhMsj1kOP2la5DSEEerPDuOP/2mmkhHhZQ= +cloud.google.com/go/speech v1.8.0/go.mod h1:9bYIl1/tjsAnMgKGHKmBZzXKEkGgtU+MpdDPTE9f7y0= +cloud.google.com/go/speech v1.9.0/go.mod h1:xQ0jTcmnRFFM2RfX/U+rk6FQNUF6DQlydUSyoooSpco= +cloud.google.com/go/speech v1.14.1/go.mod h1:gEosVRPJ9waG7zqqnsHpYTOoAS4KouMRLDFMekpJ0J0= +cloud.google.com/go/speech v1.15.0/go.mod h1:y6oH7GhqCaZANH7+Oe0BhgIogsNInLlz542tg3VqeYI= +cloud.google.com/go/speech v1.17.1/go.mod h1:8rVNzU43tQvxDaGvqOhpDqgkJTFowBpDvCJ14kGlJYo= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= +cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= +cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc= +cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= +cloud.google.com/go/storage v1.28.1/go.mod h1:Qnisd4CqDdo6BGs2AD5LLnEsmSQ80wQ5ogcBBKhU86Y= +cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4= +cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7BiccwkR7+P7gN8E= +cloud.google.com/go/storagetransfer v1.5.0/go.mod h1:dxNzUopWy7RQevYFHewchb29POFv3/AaBgnhqzqiK0w= +cloud.google.com/go/storagetransfer v1.6.0/go.mod h1:y77xm4CQV/ZhFZH75PLEXY0ROiS7Gh6pSKrM8dJyg6I= +cloud.google.com/go/storagetransfer v1.7.0/go.mod h1:8Giuj1QNb1kfLAiWM1bN6dHzfdlDAVC9rv9abHot2W4= +cloud.google.com/go/storagetransfer v1.8.0/go.mod h1:JpegsHHU1eXg7lMHkvf+KE5XDJ7EQu0GwNJbbVGanEw= +cloud.google.com/go/storagetransfer v1.10.0/go.mod h1:DM4sTlSmGiNczmV6iZyceIh2dbs+7z2Ayg6YAiQlYfA= +cloud.google.com/go/talent v1.1.0/go.mod h1:Vl4pt9jiHKvOgF9KoZo6Kob9oV4lwd/ZD5Cto54zDRw= +cloud.google.com/go/talent v1.2.0/go.mod h1:MoNF9bhFQbiJ6eFD3uSsg0uBALw4n4gaCaEjBw9zo8g= +cloud.google.com/go/talent v1.3.0/go.mod h1:CmcxwJ/PKfRgd1pBjQgU6W3YBwiewmUzQYH5HHmSCmM= +cloud.google.com/go/talent v1.4.0/go.mod h1:ezFtAgVuRf8jRsvyE6EwmbTK5LKciD4KVnHuDEFmOOA= +cloud.google.com/go/talent v1.5.0/go.mod h1:G+ODMj9bsasAEJkQSzO2uHQWXHHXUomArjWQQYkqK6c= +cloud.google.com/go/talent v1.6.2/go.mod h1:CbGvmKCG61mkdjcqTcLOkb2ZN1SrQI8MDyma2l7VD24= +cloud.google.com/go/texttospeech v1.4.0/go.mod h1:FX8HQHA6sEpJ7rCMSfXuzBcysDAuWusNNNvN9FELDd8= +cloud.google.com/go/texttospeech v1.5.0/go.mod h1:oKPLhR4n4ZdQqWKURdwxMy0uiTS1xU161C8W57Wkea4= +cloud.google.com/go/texttospeech v1.6.0/go.mod h1:YmwmFT8pj1aBblQOI3TfKmwibnsfvhIBzPXcW4EBovc= +cloud.google.com/go/texttospeech v1.7.1/go.mod h1:m7QfG5IXxeneGqTapXNxv2ItxP/FS0hCZBwXYqucgSk= +cloud.google.com/go/tpu v1.3.0/go.mod h1:aJIManG0o20tfDQlRIej44FcwGGl/cD0oiRyMKG19IQ= +cloud.google.com/go/tpu v1.4.0/go.mod h1:mjZaX8p0VBgllCzF6wcU2ovUXN9TONFLd7iz227X2Xg= +cloud.google.com/go/tpu v1.5.0/go.mod h1:8zVo1rYDFuW2l4yZVY0R0fb/v44xLh3llq7RuV61fPM= +cloud.google.com/go/tpu v1.6.1/go.mod h1:sOdcHVIgDEEOKuqUoi6Fq53MKHJAtOwtz0GuKsWSH3E= +cloud.google.com/go/trace v1.0.0/go.mod h1:4iErSByzxkyHWzzlAj63/Gmjz0NH1ASqhJguHpGcr6A= +cloud.google.com/go/trace v1.3.0/go.mod h1:FFUE83d9Ca57C+K8rDl/Ih8LwOzWIV1krKgxg6N0G28= +cloud.google.com/go/trace v1.4.0/go.mod h1:UG0v8UBqzusp+z63o7FK74SdFE+AXpCLdFb1rshXG+Y= +cloud.google.com/go/trace v1.8.0/go.mod h1:zH7vcsbAhklH8hWFig58HvxcxyQbaIqMarMg9hn5ECA= +cloud.google.com/go/trace v1.9.0/go.mod h1:lOQqpE5IaWY0Ixg7/r2SjixMuc6lfTFeO4QGM4dQWOk= +cloud.google.com/go/trace v1.10.1 h1:EwGdOLCNfYOOPtgqo+D2sDLZmRCEO1AagRTJCU6ztdg= +cloud.google.com/go/trace v1.10.1/go.mod h1:gbtL94KE5AJLH3y+WVpfWILmqgc6dXcqgNXdOPAQTYk= +cloud.google.com/go/translate v1.3.0/go.mod h1:gzMUwRjvOqj5i69y/LYLd8RrNQk+hOmIXTi9+nb3Djs= +cloud.google.com/go/translate v1.4.0/go.mod h1:06Dn/ppvLD6WvA5Rhdp029IX2Mi3Mn7fpMRLPvXT5Wg= +cloud.google.com/go/translate v1.5.0/go.mod h1:29YDSYveqqpA1CQFD7NQuP49xymq17RXNaUDdc0mNu0= +cloud.google.com/go/translate v1.6.0/go.mod h1:lMGRudH1pu7I3n3PETiOB2507gf3HnfLV8qlkHZEyos= +cloud.google.com/go/translate v1.7.0/go.mod h1:lMGRudH1pu7I3n3PETiOB2507gf3HnfLV8qlkHZEyos= +cloud.google.com/go/translate v1.8.1/go.mod h1:d1ZH5aaOA0CNhWeXeC8ujd4tdCFw8XoNWRljklu5RHs= +cloud.google.com/go/video v1.8.0/go.mod h1:sTzKFc0bUSByE8Yoh8X0mn8bMymItVGPfTuUBUyRgxk= +cloud.google.com/go/video v1.9.0/go.mod h1:0RhNKFRF5v92f8dQt0yhaHrEuH95m068JYOvLZYnJSw= +cloud.google.com/go/video v1.12.0/go.mod h1:MLQew95eTuaNDEGriQdcYn0dTwf9oWiA4uYebxM5kdg= +cloud.google.com/go/video v1.13.0/go.mod h1:ulzkYlYgCp15N2AokzKjy7MQ9ejuynOJdf1tR5lGthk= +cloud.google.com/go/video v1.14.0/go.mod h1:SkgaXwT+lIIAKqWAJfktHT/RbgjSuY6DobxEp0C5yTQ= +cloud.google.com/go/video v1.15.0/go.mod h1:SkgaXwT+lIIAKqWAJfktHT/RbgjSuY6DobxEp0C5yTQ= +cloud.google.com/go/video v1.17.1/go.mod h1:9qmqPqw/Ib2tLqaeHgtakU+l5TcJxCJbhFXM7UJjVzU= +cloud.google.com/go/videointelligence v1.6.0/go.mod h1:w0DIDlVRKtwPCn/C4iwZIJdvC69yInhW0cfi+p546uU= +cloud.google.com/go/videointelligence v1.7.0/go.mod h1:k8pI/1wAhjznARtVT9U1llUaFNPh7muw8QyOUpavru4= +cloud.google.com/go/videointelligence v1.8.0/go.mod h1:dIcCn4gVDdS7yte/w+koiXn5dWVplOZkE+xwG9FgK+M= +cloud.google.com/go/videointelligence v1.9.0/go.mod h1:29lVRMPDYHikk3v8EdPSaL8Ku+eMzDljjuvRs105XoU= +cloud.google.com/go/videointelligence v1.10.0/go.mod h1:LHZngX1liVtUhZvi2uNS0VQuOzNi2TkY1OakiuoUOjU= +cloud.google.com/go/videointelligence v1.11.1/go.mod h1:76xn/8InyQHarjTWsBR058SmlPCwQjgcvoW0aZykOvo= +cloud.google.com/go/vision v1.2.0/go.mod h1:SmNwgObm5DpFBme2xpyOyasvBc1aPdjvMk2bBk0tKD0= +cloud.google.com/go/vision/v2 v2.2.0/go.mod h1:uCdV4PpN1S0jyCyq8sIM42v2Y6zOLkZs+4R9LrGYwFo= +cloud.google.com/go/vision/v2 v2.3.0/go.mod h1:UO61abBx9QRMFkNBbf1D8B1LXdS2cGiiCRx0vSpZoUo= +cloud.google.com/go/vision/v2 v2.4.0/go.mod h1:VtI579ll9RpVTrdKdkMzckdnwMyX2JILb+MhPqRbPsY= +cloud.google.com/go/vision/v2 v2.5.0/go.mod h1:MmaezXOOE+IWa+cS7OhRRLK2cNv1ZL98zhqFFZaaH2E= +cloud.google.com/go/vision/v2 v2.6.0/go.mod h1:158Hes0MvOS9Z/bDMSFpjwsUrZ5fPrdwuyyvKSGAGMY= +cloud.google.com/go/vision/v2 v2.7.0/go.mod h1:H89VysHy21avemp6xcf9b9JvZHVehWbET0uT/bcuY/0= +cloud.google.com/go/vision/v2 v2.7.2/go.mod h1:jKa8oSYBWhYiXarHPvP4USxYANYUEdEsQrloLjrSwJU= +cloud.google.com/go/vmmigration v1.2.0/go.mod h1:IRf0o7myyWFSmVR1ItrBSFLFD/rJkfDCUTO4vLlJvsE= +cloud.google.com/go/vmmigration v1.3.0/go.mod h1:oGJ6ZgGPQOFdjHuocGcLqX4lc98YQ7Ygq8YQwHh9A7g= +cloud.google.com/go/vmmigration v1.5.0/go.mod h1:E4YQ8q7/4W9gobHjQg4JJSgXXSgY21nA5r8swQV+Xxc= +cloud.google.com/go/vmmigration v1.6.0/go.mod h1:bopQ/g4z+8qXzichC7GW1w2MjbErL54rk3/C843CjfY= +cloud.google.com/go/vmmigration v1.7.1/go.mod h1:WD+5z7a/IpZ5bKK//YmT9E047AD+rjycCAvyMxGJbro= +cloud.google.com/go/vmwareengine v0.1.0/go.mod h1:RsdNEf/8UDvKllXhMz5J40XxDrNJNN4sagiox+OI208= +cloud.google.com/go/vmwareengine v0.2.2/go.mod h1:sKdctNJxb3KLZkE/6Oui94iw/xs9PRNC2wnNLXsHvH8= +cloud.google.com/go/vmwareengine v0.3.0/go.mod h1:wvoyMvNWdIzxMYSpH/R7y2h5h3WFkx6d+1TIsP39WGY= +cloud.google.com/go/vmwareengine v0.4.1/go.mod h1:Px64x+BvjPZwWuc4HdmVhoygcXqEkGHXoa7uyfTgSI0= +cloud.google.com/go/vpcaccess v1.4.0/go.mod h1:aQHVbTWDYUR1EbTApSVvMq1EnT57ppDmQzZ3imqIk4w= +cloud.google.com/go/vpcaccess v1.5.0/go.mod h1:drmg4HLk9NkZpGfCmZ3Tz0Bwnm2+DKqViEpeEpOq0m8= +cloud.google.com/go/vpcaccess v1.6.0/go.mod h1:wX2ILaNhe7TlVa4vC5xce1bCnqE3AeH27RV31lnmZes= +cloud.google.com/go/vpcaccess v1.7.1/go.mod h1:FogoD46/ZU+JUBX9D606X21EnxiszYi2tArQwLY4SXs= +cloud.google.com/go/webrisk v1.4.0/go.mod h1:Hn8X6Zr+ziE2aNd8SliSDWpEnSS1u4R9+xXZmFiHmGE= +cloud.google.com/go/webrisk v1.5.0/go.mod h1:iPG6fr52Tv7sGk0H6qUFzmL3HHZev1htXuWDEEsqMTg= +cloud.google.com/go/webrisk v1.6.0/go.mod h1:65sW9V9rOosnc9ZY7A7jsy1zoHS5W9IAXv6dGqhMQMc= +cloud.google.com/go/webrisk v1.7.0/go.mod h1:mVMHgEYH0r337nmt1JyLthzMr6YxwN1aAIEc2fTcq7A= +cloud.google.com/go/webrisk v1.8.0/go.mod h1:oJPDuamzHXgUc+b8SiHRcVInZQuybnvEW72PqTc7sSg= +cloud.google.com/go/webrisk v1.9.1/go.mod h1:4GCmXKcOa2BZcZPn6DCEvE7HypmEJcJkr4mtM+sqYPc= +cloud.google.com/go/websecurityscanner v1.3.0/go.mod h1:uImdKm2wyeXQevQJXeh8Uun/Ym1VqworNDlBXQevGMo= +cloud.google.com/go/websecurityscanner v1.4.0/go.mod h1:ebit/Fp0a+FWu5j4JOmJEV8S8CzdTkAS77oDsiSqYWQ= +cloud.google.com/go/websecurityscanner v1.5.0/go.mod h1:Y6xdCPy81yi0SQnDY1xdNTNpfY1oAgXUlcfN3B3eSng= +cloud.google.com/go/websecurityscanner v1.6.1/go.mod h1:Njgaw3rttgRHXzwCB8kgCYqv5/rGpFCsBOvPbYgszpg= +cloud.google.com/go/workflows v1.6.0/go.mod h1:6t9F5h/unJz41YqfBmqSASJSXccBLtD1Vwf+KmJENM0= +cloud.google.com/go/workflows v1.7.0/go.mod h1:JhSrZuVZWuiDfKEFxU0/F1PQjmpnpcoISEXH2bcHC3M= +cloud.google.com/go/workflows v1.8.0/go.mod h1:ysGhmEajwZxGn1OhGOGKsTXc5PyxOc0vfKf5Af+to4M= +cloud.google.com/go/workflows v1.9.0/go.mod h1:ZGkj1aFIOd9c8Gerkjjq7OW7I5+l6cSvT3ujaO/WwSA= +cloud.google.com/go/workflows v1.10.0/go.mod h1:fZ8LmRmZQWacon9UCX1r/g/DfAXx5VcPALq2CxzdePw= +cloud.google.com/go/workflows v1.11.1/go.mod h1:Z+t10G1wF7h8LgdY/EmRcQY8ptBD/nvofaL6FqlET6g= +contrib.go.opencensus.io/exporter/stackdriver v0.13.12 h1:bjBKzIf7/TAkxd7L2utGaLM78bmUWlCval5K9UeElbY= +contrib.go.opencensus.io/exporter/stackdriver v0.13.12/go.mod h1:mmxnWlrvrFdpiOHOhxBaVi1rkc0WOqhgfknj4Yg0SeQ= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zumjgTw83q2ge/PI+yyw8= +git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3pc= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= +github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm/4RlzPXRlREEwqTHAN3T56Bv2ITsFT3gY= +github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19/go.mod h1:T13YZdzov6OU0A1+RfKZiZN9ca6VeKdBdyDV+BY97Tk= +github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= +github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b/go.mod h1:1KcenG0jGWcpt8ov532z81sp/kMMUG485J2InIOyADM= +github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/apache/arrow/go/v10 v10.0.1/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0= +github.com/apache/arrow/go/v11 v11.0.0/go.mod h1:Eg5OsL5H+e299f7u5ssuXsuHQVEGC4xei5aX110hRiI= +github.com/apache/arrow/go/v12 v12.0.0/go.mod h1:d+tV/eHZZ7Dz7RPrFKtPK02tpr+c9/PEd/zm8mDS9Vg= +github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= +github.com/aws/aws-sdk-go v1.37.0/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= +github.com/aws/aws-sdk-go v1.44.162 h1:hKAd+X+/BLxVMzH+4zKxbQcQQGrk2UhFX0OTu1Mhon8= +github.com/aws/aws-sdk-go v1.44.162/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= +github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g= +github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20230310173818-32f1caf87195/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20230428030218-4003588d1b74/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= +github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= +github.com/envoyproxy/go-control-plane v0.10.3/go.mod h1:fJJn/j26vwOu972OllsvAgJJM//w9BV6Fxbg2LuVd34= +github.com/envoyproxy/go-control-plane v0.11.0/go.mod h1:VnHyVMpzcLvCFt9yUz1UnCwHLhwx1WguiVDV7pTG/tI= +github.com/envoyproxy/go-control-plane v0.11.1/go.mod h1:uhMcXKCQMEJHiAb0w+YGefQLaTEw+YhGluxZkrTmD0g= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v0.6.7/go.mod h1:dyJXwwfPK2VSqiB9Klm1J6romD608Ba7Hij42vrOBCo= +github.com/envoyproxy/protoc-gen-validate v0.9.1/go.mod h1:OKNgG7TCp5pF4d6XftA0++PMirau2/yoOwVac3AbF2w= +github.com/envoyproxy/protoc-gen-validate v0.10.0/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss= +github.com/envoyproxy/protoc-gen-validate v1.0.1/go.mod h1:0vj8bNkYbSTNS2PIyH87KZaeN4x9zpL9Qt8fQC7d+vs= +github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA= +github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= +github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-fonts/dejavu v0.1.0/go.mod h1:4Wt4I4OU2Nq9asgDCteaAaWZOV24E+0/Pwo0gppep4g= +github.com/go-fonts/latin-modern v0.2.0/go.mod h1:rQVLdDMK+mK1xscDwsqM5J8U2jrRa3T0ecnM9pNujks= +github.com/go-fonts/liberation v0.1.1/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY= +github.com/go-fonts/liberation v0.2.0/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY= +github.com/go-fonts/stix v0.1.0/go.mod h1:w/c1f0ldAUlJmLBvlbkvVXLAD+tAMqobIIQpmnUIzUY= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2CSIqUrmQPqA0gdRIlnLEY0gK5JGjh37zN5U= +github.com/go-latex/latex v0.0.0-20210823091927-c0d11ff05a81/go.mod h1:SX0U8uGpxhq9o2S/CELCSUxEWWAuoCUcVCQWv7G2OCk= +github.com/go-pdf/fpdf v0.5.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= +github.com/go-pdf/fpdf v0.6.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= +github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/flatbuffers v2.0.8+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/s2a-go v0.1.0/go.mod h1:OJpEgntRZo8ugHpF9hkoLJbS5dSI20XZeXJ9JVywLlM= +github.com/google/s2a-go v0.1.3/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= +github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= +github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= +github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= +github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= +github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= +github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k= +github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= +github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= +github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= +github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= +github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= +github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo= +github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY= +github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= +github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= +github.com/googleapis/gax-go/v2 v2.8.0/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= +github.com/googleapis/gax-go/v2 v2.10.0/go.mod h1:4UOEnMCrxsSqQ940WnTiD6qJ63le2ev3xfyagutxiPw= +github.com/googleapis/gax-go/v2 v2.11.0 h1:9V9PWXEsWnPpQhu/PeQIkS4eGzMlTLGgt80cUUI8Ki4= +github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI= +github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= +github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU0GSB0f8Nhgmxx0VIRUvaC0w= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= +github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lyft/protoc-gen-star v0.6.0/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= +github.com/lyft/protoc-gen-star v0.6.1/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= +github.com/lyft/protoc-gen-star/v2 v2.0.1/go.mod h1:RcCdONR2ScXaYnQC5tUzxzlpA3WVYF7/opLeUgcQs/o= +github.com/lyft/protoc-gen-star/v2 v2.0.3/go.mod h1:amey7yeodaJhXSbf/TlLvWiqQfLOSpEk//mLlc+axEk= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= +github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2dXMnm1mY= +github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= +github.com/phpdave11/gofpdi v1.0.13/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= +github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= +github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= +github.com/prometheus/prometheus v2.5.0+incompatible h1:7QPitgO2kOFG8ecuRn9O/4L9+10He72rVRJvMXrE9Hg= +github.com/prometheus/prometheus v2.5.0+incompatible/go.mod h1:oAIUtOny2rjMX0OWN5vPR5/q/twIROJvdqnQKDdil/s= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w= +github.com/ruudk/golang-pdf417 v0.0.0-20201230142125-a7e3863a1245/go.mod h1:pQAZKsJ8yyVxGRWYNEm9oFB8ieLgKFnamEyDmSA0BRk= +github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= +github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= +github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= +go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= +golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= +golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= +golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= +golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3/go.mod h1:NOZ3BPKG0ec/BKJQgnvsSFpcKLM5xXVWnvZS97DWHgE= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20220827204233-334a2380cb91/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= +golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20190910094157-69e4b8554b2a/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200430140353-33d19683fad8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200618115811-c13761719519/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20201208152932-35266b937fa6/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20210216034530-4410531fe030/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20210607152325-775e3b0c77b9/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= +golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= +golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= +golang.org/x/image v0.0.0-20220302094943-723b81ca9867/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.0.0-20221012135044-0b7e1fb9d458/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= +golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= +golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= +golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= +golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.0.0-20221006150949-b44042a4b9c1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= +golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= +golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= +golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= +golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= +golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210304124612-50617c2ba197/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210917161153-d61c044b1678/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220829200755-d48e67d00261/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= +golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= +golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= +golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= +golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= +golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= +gonum.org/v1/gonum v0.9.3/go.mod h1:TZumC3NeyVQskjXqmyWt4S3bINhy7B4eYwW69EbyX+0= +gonum.org/v1/gonum v0.11.0/go.mod h1:fSG4YDCxxUZQJ7rKsQrj0gMOg00Il0Z96/qMA4bVQhA= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= +gonum.org/v1/plot v0.9.0/go.mod h1:3Pcqqmp6RHvJI72kgb8fThyUnav364FOsdDo2aGW5lY= +gonum.org/v1/plot v0.10.1/go.mod h1:VZW5OlhkL1mysU9vaqNHnsy86inf6Ot+jB3r+BczCEo= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= +google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= +google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= +google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= +google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= +google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= +google.golang.org/api v0.58.0/go.mod h1:cAbP2FsxoGVNwtgNAmmn3y5G1TWAiVYRmg4yku3lv+E= +google.golang.org/api v0.59.0/go.mod h1:sT2boj7M9YJxZzgeZqXogmhfmRWDtPzT31xkieUbuZU= +google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= +google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= +google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g= +google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA= +google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8= +google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= +google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= +google.golang.org/api v0.77.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= +google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw= +google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg= +google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o= +google.golang.org/api v0.85.0/go.mod h1:AqZf8Ep9uZ2pyTvgL+x0D3Zt0eoT9b5E8fmzfu6FO2g= +google.golang.org/api v0.90.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= +google.golang.org/api v0.93.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= +google.golang.org/api v0.95.0/go.mod h1:eADj+UBuxkh5zlrSntJghuNeg8HwQ1w5lTKkuqaETEI= +google.golang.org/api v0.96.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= +google.golang.org/api v0.97.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= +google.golang.org/api v0.98.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= +google.golang.org/api v0.99.0/go.mod h1:1YOf74vkVndF7pG6hIHuINsM7eWwpVTAfNMNiL91A08= +google.golang.org/api v0.100.0/go.mod h1:ZE3Z2+ZOr87Rx7dqFsdRQkRBk36kDtp/h+QpHbB7a70= +google.golang.org/api v0.102.0/go.mod h1:3VFl6/fzoA+qNuS1N1/VfXY4LjoXN/wzeIp7TweWwGo= +google.golang.org/api v0.103.0/go.mod h1:hGtW6nK1AC+d9si/UBhw8Xli+QMOf6xyNAyJw4qU9w0= +google.golang.org/api v0.106.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= +google.golang.org/api v0.107.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= +google.golang.org/api v0.108.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= +google.golang.org/api v0.110.0/go.mod h1:7FC4Vvx1Mooxh8C5HWjzZHcavuS2f6pmJpZx60ca7iI= +google.golang.org/api v0.111.0/go.mod h1:qtFHvU9mhgTJegR31csQ+rwxyUTHOKFqCKWp1J0fdw0= +google.golang.org/api v0.114.0/go.mod h1:ifYI2ZsFK6/uGddGfAD5BMxlnkBqCmqHSDUVi45N5Yg= +google.golang.org/api v0.118.0/go.mod h1:76TtD3vkgmZ66zZzp72bUUklpmQmKlhh6sYtIjYK+5E= +google.golang.org/api v0.122.0/go.mod h1:gcitW0lvnyWjSp9nKxAbdHKIZ6vF4aajGueeslZOyms= +google.golang.org/api v0.124.0/go.mod h1:xu2HQurE5gi/3t1aFCvhPD781p0a3p11sdunTJ2BlP4= +google.golang.org/api v0.125.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw= +google.golang.org/api v0.126.0 h1:q4GJq+cAdMAC7XP7njvQ4tvohGLiSlytuL4BQxbIZ+o= +google.golang.org/api v0.126.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210329143202-679c6ae281ee/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= +google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210917145530-b395a37504d4/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210921142501-181ce0d877f6/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211008145708-270636b82663/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211018162055-cf77aa76bad2/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= +google.golang.org/genproto v0.0.0-20220329172620-7be39ac1afc7/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220628213854-d9e0b6570c03/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220722212130-b98a9ff5e252/go.mod h1:GkXuJDJ6aQ7lnJcRF+SJVgFdQhypqgl3LB1C9vabdRE= +google.golang.org/genproto v0.0.0-20220801145646-83ce21fca29f/go.mod h1:iHe1svFLAZg9VWz891+QbRMwUv9O/1Ww+/mngYeThbc= +google.golang.org/genproto v0.0.0-20220815135757-37a418bb8959/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220817144833-d7fd3f11b9b1/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220829144015-23454907ede3/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220829175752-36a9c930ecbf/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220913154956-18f8339a66a5/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220914142337-ca0e39ece12f/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220915135415-7fd63a7952de/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220916172020-2692e8806bfa/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220919141832-68c03719ef51/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220920201722-2b89144ce006/go.mod h1:ht8XFiar2npT/g4vkk7O0WYS1sHOHbdujxbEp7CJWbw= +google.golang.org/genproto v0.0.0-20220926165614-551eb538f295/go.mod h1:woMGP53BroOrRY3xTxlbr8Y3eB/nzAvvFM83q7kG2OI= +google.golang.org/genproto v0.0.0-20220926220553-6981cbe3cfce/go.mod h1:woMGP53BroOrRY3xTxlbr8Y3eB/nzAvvFM83q7kG2OI= +google.golang.org/genproto v0.0.0-20221010155953-15ba04fc1c0e/go.mod h1:3526vdqwhZAwq4wsRUaVG555sVgsNmIjRtO7t/JH29U= +google.golang.org/genproto v0.0.0-20221014173430-6e2ab493f96b/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= +google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= +google.golang.org/genproto v0.0.0-20221024153911-1573dae28c9c/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= +google.golang.org/genproto v0.0.0-20221024183307-1bc688fe9f3e/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= +google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c/go.mod h1:CGI5F/G+E5bKwmfYo09AXuVN4dD894kIKUFmVbP2/Fo= +google.golang.org/genproto v0.0.0-20221109142239-94d6d90a7d66/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221114212237-e4508ebdbee1/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221117204609-8f9c96812029/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221201204527-e3fa12d562f3/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd/go.mod h1:cTsE614GARnxrLsqKREzmNYJACSWWpAWdNMwnD7c2BE= +google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230112194545-e10362b5ecf9/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230113154510-dbe35b8444a5/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230123190316-2c411cf9d197/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230124163310-31e0e69b6fc2/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230125152338-dcaf20b6aeaa/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230127162408-596548ed4efa/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230209215440-0dfe4f8abfcc/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230216225411-c8e22ba71e44/go.mod h1:8B0gmkoRebU8ukX6HP+4wrVQUY1+6PkQ44BSyIlflHA= +google.golang.org/genproto v0.0.0-20230222225845-10f96fb3dbec/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw= +google.golang.org/genproto v0.0.0-20230223222841-637eb2293923/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw= +google.golang.org/genproto v0.0.0-20230303212802-e74f57abe488/go.mod h1:TvhZT5f700eVlTNwND1xoEZQeWTB2RY/65kplwl/bFA= +google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= +google.golang.org/genproto v0.0.0-20230320184635-7606e756e683/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= +google.golang.org/genproto v0.0.0-20230323212658-478b75c54725/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/genproto v0.0.0-20230330154414-c0448cd141ea/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= +google.golang.org/genproto v0.0.0-20230525234025-438c736192d0/go.mod h1:9ExIQyXL5hZrHzQceCwuSYwZZ5QZBazOcprJ5rgs3lY= +google.golang.org/genproto v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:zqTuNwFlFRsw5zIts5VnzLQxSRqh+CGOTVMlYbY0Eyk= +google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= +google.golang.org/genproto v0.0.0-20230629202037-9506855d4529/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= +google.golang.org/genproto v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:O9kGHb51iE/nOGvQaDUuadVYqovW56s5emA88lQnj6Y= +google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 h1:Z0hjGZePRE0ZBWotvtrwxFNrNE9CUAGtplaDK5NNI/g= +google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98/go.mod h1:S7mY02OqCJTD0E1OiQy1F72PWFB4bZJ87cAtLPYgDR0= +google.golang.org/genproto/googleapis/api v0.0.0-20230525234020-1aefcd67740a/go.mod h1:ts19tUU+Z0ZShN1y3aPyq2+O3d5FUNNgT6FtOzmrNn8= +google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto/googleapis/api v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto/googleapis/api v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:mPBs5jNgx2GuQGvFwUvVKqtn6HsUw9nP64BedgvqEsQ= +google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 h1:FmF5cCW94Ij59cfpoLiwTgodWmm60eEV0CjlsVg2fuw= +google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= +google.golang.org/genproto/googleapis/bytestream v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:ylj+BE99M198VPbBh6A8d9n3w8fChvyLK3wwBOjXBFA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234015-3fc162c6f38a/go.mod h1:xURIpW9ES5+/GZhnV6beoEtxQrnkRGIfP5VQG2tCBLc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:8mL13HKkDa+IuJ8yruA3ci0q+0vsUz4m//+ottjwS5o= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 h1:bVf09lpb+OJbByTj913DRJioFFAjf/ZGxEz7MajTp2U= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= +lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= +lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= +modernc.org/cc/v3 v3.36.0/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= +modernc.org/cc/v3 v3.36.2/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= +modernc.org/cc/v3 v3.36.3/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= +modernc.org/cc/v3 v3.37.0/go.mod h1:vtL+3mdHx/wcj3iEGz84rQa8vEqR6XM84v5Lcvfph20= +modernc.org/cc/v3 v3.40.0/go.mod h1:/bTg4dnWkSXowUO6ssQKnOV0yMVxDYNIsIrzqTFDGH0= +modernc.org/ccgo/v3 v3.0.0-20220428102840-41399a37e894/go.mod h1:eI31LL8EwEBKPpNpA4bU1/i+sKOwOrQy8D87zWUcRZc= +modernc.org/ccgo/v3 v3.0.0-20220430103911-bc99d88307be/go.mod h1:bwdAnOoaIt8Ax9YdWGjxWsdkPcZyRPHqrOvJxaKAKGw= +modernc.org/ccgo/v3 v3.0.0-20220904174949-82d86e1b6d56/go.mod h1:YSXjPL62P2AMSxBphRHPn7IkzhVHqkvOnRKAKh+W6ZI= +modernc.org/ccgo/v3 v3.16.4/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ= +modernc.org/ccgo/v3 v3.16.6/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ= +modernc.org/ccgo/v3 v3.16.8/go.mod h1:zNjwkizS+fIFDrDjIAgBSCLkWbJuHF+ar3QRn+Z9aws= +modernc.org/ccgo/v3 v3.16.9/go.mod h1:zNMzC9A9xeNUepy6KuZBbugn3c0Mc9TeiJO4lgvkJDo= +modernc.org/ccgo/v3 v3.16.13-0.20221017192402-261537637ce8/go.mod h1:fUB3Vn0nVPReA+7IG7yZDfjv1TMWjhQP8gCxrFAtL5g= +modernc.org/ccgo/v3 v3.16.13/go.mod h1:2Quk+5YgpImhPjv2Qsob1DnZ/4som1lJTodubIcoUkY= +modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ= +modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM= +modernc.org/libc v0.0.0-20220428101251-2d5f3daf273b/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA= +modernc.org/libc v1.16.0/go.mod h1:N4LD6DBE9cf+Dzf9buBlzVJndKr/iJHG97vGLHYnb5A= +modernc.org/libc v1.16.1/go.mod h1:JjJE0eu4yeK7tab2n4S1w8tlWd9MxXLRzheaRnAKymU= +modernc.org/libc v1.16.17/go.mod h1:hYIV5VZczAmGZAnG15Vdngn5HSF5cSkbvfz2B7GRuVU= +modernc.org/libc v1.16.19/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA= +modernc.org/libc v1.17.0/go.mod h1:XsgLldpP4aWlPlsjqKRdHPqCxCjISdHfM/yeWC5GyW0= +modernc.org/libc v1.17.1/go.mod h1:FZ23b+8LjxZs7XtFMbSzL/EhPxNbfZbErxEHc7cbD9s= +modernc.org/libc v1.17.4/go.mod h1:WNg2ZH56rDEwdropAJeZPQkXmDwh+JCA1s/htl6r2fA= +modernc.org/libc v1.18.0/go.mod h1:vj6zehR5bfc98ipowQOM2nIDUZnVew/wNC/2tOGS+q0= +modernc.org/libc v1.20.3/go.mod h1:ZRfIaEkgrYgZDl6pa4W39HgN5G/yDW+NRmNKZBDFrk0= +modernc.org/libc v1.21.4/go.mod h1:przBsL5RDOZajTVslkugzLBj1evTue36jEomFQOoYuI= +modernc.org/libc v1.22.2/go.mod h1:uvQavJ1pZ0hIoC/jfqNoMLURIMhKzINIWypNM17puug= +modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/mathutil v1.4.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/memory v1.1.1/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw= +modernc.org/memory v1.2.0/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw= +modernc.org/memory v1.2.1/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= +modernc.org/memory v1.3.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= +modernc.org/memory v1.4.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= +modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= +modernc.org/opt v0.1.1/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/sqlite v1.18.1/go.mod h1:6ho+Gow7oX5V+OiOQ6Tr4xeqbx13UZ6t+Fw9IRUG4d4= +modernc.org/sqlite v1.18.2/go.mod h1:kvrTLEWgxUcHa2GfHBQtanR1H9ht3hTJNtKpzH9k1u0= +modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw= +modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw= +modernc.org/tcl v1.13.1/go.mod h1:XOLfOwzhkljL4itZkK6T72ckMgvj0BDsnKNdZVUOecw= +modernc.org/tcl v1.13.2/go.mod h1:7CLiGIPo1M8Rv1Mitpv5akc2+8fxUd2y2UzC/MfMzy0= +modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +modernc.org/token v1.0.1/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +modernc.org/z v1.5.1/go.mod h1:eWFB510QWW5Th9YGZT81s+LwvaAs3Q2yr4sP0rmLkv8= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/interop/observability/run.sh b/interop/observability/run.sh new file mode 100755 index 000000000000..b0494668bf84 --- /dev/null +++ b/interop/observability/run.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# Copyright 2022 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex +cd "$(dirname "$0")"/../.. + +if [ "$1" = "server" ] ; then + /grpc-go/interop/observability/server/server "${@:2}" + +elif [ "$1" = "client" ] ; then + /grpc-go/interop/observability/client/client "${@:2}" + +else + echo "Invalid action: $1. Usage:" + echo " $ .../run.sh [server|client] --server_host= --server_port= ..." + exit 1 +fi diff --git a/interop/observability/server/server.go b/interop/observability/server/server.go new file mode 100644 index 000000000000..7efab04bba3f --- /dev/null +++ b/interop/observability/server/server.go @@ -0,0 +1,55 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package main + +import ( + "context" + "flag" + "fmt" + "log" + "net" + + "google.golang.org/grpc" + "google.golang.org/grpc/gcp/observability" + "google.golang.org/grpc/interop" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" +) + +var ( + port = flag.Int("port", 10000, "The server port") +) + +func main() { + err := observability.Start(context.Background()) + if err != nil { + log.Fatalf("observability start failed: %v", err) + } + defer observability.End() + flag.Parse() + lis, err := net.Listen("tcp", fmt.Sprintf(":%d", *port)) + if err != nil { + log.Fatalf("failed to listen: %v", err) + } + server := grpc.NewServer() + defer server.Stop() + testgrpc.RegisterTestServiceServer(server, interop.NewTestServer()) + log.Printf("Observability interop server listening on %v", lis.Addr()) + server.Serve(lis) +} diff --git a/interop/orcalb.go b/interop/orcalb.go new file mode 100644 index 000000000000..de87c8828815 --- /dev/null +++ b/interop/orcalb.go @@ -0,0 +1,169 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package interop + +import ( + "context" + "fmt" + "sync" + "time" + + v3orcapb "github.com/cncf/xds/go/xds/data/orca/v3" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/base" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/orca" +) + +func init() { + balancer.Register(orcabb{}) +} + +type orcabb struct{} + +func (orcabb) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { + return &orcab{cc: cc} +} + +func (orcabb) Name() string { + return "test_backend_metrics_load_balancer" +} + +type orcab struct { + cc balancer.ClientConn + sc balancer.SubConn + cancelWatch func() + + reportMu sync.Mutex + report *v3orcapb.OrcaLoadReport +} + +func (o *orcab) UpdateClientConnState(s balancer.ClientConnState) error { + if o.sc != nil { + o.sc.UpdateAddresses(s.ResolverState.Addresses) + return nil + } + + if len(s.ResolverState.Addresses) == 0 { + o.ResolverError(fmt.Errorf("produced no addresses")) + return fmt.Errorf("resolver produced no addresses") + } + var err error + o.sc, err = o.cc.NewSubConn(s.ResolverState.Addresses, balancer.NewSubConnOptions{StateListener: o.updateSubConnState}) + if err != nil { + o.cc.UpdateState(balancer.State{ConnectivityState: connectivity.TransientFailure, Picker: base.NewErrPicker(fmt.Errorf("error creating subconn: %v", err))}) + return nil + } + o.cancelWatch = orca.RegisterOOBListener(o.sc, o, orca.OOBListenerOptions{ReportInterval: time.Second}) + o.sc.Connect() + o.cc.UpdateState(balancer.State{ConnectivityState: connectivity.Connecting, Picker: base.NewErrPicker(balancer.ErrNoSubConnAvailable)}) + return nil +} + +func (o *orcab) ResolverError(err error) { + if o.sc == nil { + o.cc.UpdateState(balancer.State{ConnectivityState: connectivity.TransientFailure, Picker: base.NewErrPicker(fmt.Errorf("resolver error: %v", err))}) + } +} + +func (o *orcab) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { + logger.Errorf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, state) +} + +func (o *orcab) updateSubConnState(state balancer.SubConnState) { + switch state.ConnectivityState { + case connectivity.Ready: + o.cc.UpdateState(balancer.State{ConnectivityState: connectivity.Ready, Picker: &orcaPicker{o: o}}) + case connectivity.TransientFailure: + o.cc.UpdateState(balancer.State{ConnectivityState: connectivity.TransientFailure, Picker: base.NewErrPicker(fmt.Errorf("all subchannels in transient failure: %v", state.ConnectionError))}) + case connectivity.Connecting: + // Ignore; picker already set to "connecting". + case connectivity.Idle: + o.sc.Connect() + o.cc.UpdateState(balancer.State{ConnectivityState: connectivity.Connecting, Picker: base.NewErrPicker(balancer.ErrNoSubConnAvailable)}) + case connectivity.Shutdown: + // Ignore; we are closing but handle that in Close instead. + } +} + +func (o *orcab) Close() { + o.cancelWatch() +} + +func (o *orcab) OnLoadReport(r *v3orcapb.OrcaLoadReport) { + o.reportMu.Lock() + defer o.reportMu.Unlock() + logger.Infof("received OOB load report: %v", r) + o.report = r +} + +type orcaPicker struct { + o *orcab +} + +func (p *orcaPicker) Pick(info balancer.PickInfo) (balancer.PickResult, error) { + doneCB := func(di balancer.DoneInfo) { + if lr, _ := di.ServerLoad.(*v3orcapb.OrcaLoadReport); lr != nil && + (lr.CpuUtilization != 0 || lr.MemUtilization != 0 || len(lr.Utilization) > 0 || len(lr.RequestCost) > 0) { + // Since all RPCs will respond with a load report due to the + // presence of the DialOption, we need to inspect every field and + // use the out-of-band report instead if all are unset/zero. + setContextCMR(info.Ctx, lr) + } else { + p.o.reportMu.Lock() + defer p.o.reportMu.Unlock() + if lr := p.o.report; lr != nil { + setContextCMR(info.Ctx, lr) + } + } + } + return balancer.PickResult{SubConn: p.o.sc, Done: doneCB}, nil +} + +func setContextCMR(ctx context.Context, lr *v3orcapb.OrcaLoadReport) { + if r := orcaResultFromContext(ctx); r != nil { + *r = lr + } +} + +type orcaKey string + +var orcaCtxKey = orcaKey("orcaResult") + +// contextWithORCAResult sets a key in ctx with a pointer to an ORCA load +// report that is to be filled in by the "test_backend_metrics_load_balancer" +// LB policy's Picker's Done callback. +// +// If a per-call load report is provided from the server for the call, result +// will be filled with that, otherwise the most recent OOB load report is used. +// If no OOB report has been received, result is not modified. +func contextWithORCAResult(ctx context.Context, result **v3orcapb.OrcaLoadReport) context.Context { + return context.WithValue(ctx, orcaCtxKey, result) +} + +// orcaResultFromContext returns the ORCA load report stored in the context. +// The LB policy uses this to communicate the load report back to the interop +// client application. +func orcaResultFromContext(ctx context.Context) **v3orcapb.OrcaLoadReport { + v := ctx.Value(orcaCtxKey) + if v == nil { + return nil + } + return v.(**v3orcapb.OrcaLoadReport) +} diff --git a/interop/server/server.go b/interop/server/server.go index 662fb2963090..67fbc3119963 100644 --- a/interop/server/server.go +++ b/interop/server/server.go @@ -17,20 +17,28 @@ */ // Binary server is an interop server. +// +// See interop test case descriptions [here]. +// +// [here]: https://github.com/grpc/grpc/blob/master/doc/interop-test-descriptions.md package main import ( "flag" "net" "strconv" + "time" "google.golang.org/grpc" "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/alts" "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal" "google.golang.org/grpc/interop" - testpb "google.golang.org/grpc/interop/grpc_testing" + "google.golang.org/grpc/orca" "google.golang.org/grpc/testdata" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" ) var ( @@ -47,14 +55,15 @@ var ( func main() { flag.Parse() if *useTLS && *useALTS { - logger.Fatalf("use_tls and use_alts cannot be both set to true") + logger.Fatal("-use_tls and -use_alts cannot be both set to true") } p := strconv.Itoa(*port) lis, err := net.Listen("tcp", ":"+p) if err != nil { logger.Fatalf("failed to listen: %v", err) } - var opts []grpc.ServerOption + logger.Infof("interop server listening on %v", lis.Addr()) + opts := []grpc.ServerOption{orca.CallMetricsServerOption(nil)} if *useTLS { if *certFile == "" { *certFile = testdata.Path("server1.pem") @@ -64,7 +73,7 @@ func main() { } creds, err := credentials.NewServerTLSFromFile(*certFile, *keyFile) if err != nil { - logger.Fatalf("Failed to generate credentials %v", err) + logger.Fatalf("Failed to generate credentials: %v", err) } opts = append(opts, grpc.Creds(creds)) } else if *useALTS { @@ -76,6 +85,13 @@ func main() { opts = append(opts, grpc.Creds(altsTC)) } server := grpc.NewServer(opts...) - testpb.RegisterTestServiceService(server, interop.NewTestServer()) + metricsRecorder := orca.NewServerMetricsRecorder() + sopts := orca.ServiceOptions{ + MinReportingInterval: time.Second, + ServerMetricsProvider: metricsRecorder, + } + internal.ORCAAllowAnyMinReportingInterval.(func(*orca.ServiceOptions))(&sopts) + orca.Register(server, sopts) + testgrpc.RegisterTestServiceServer(server, interop.NewTestServer(interop.NewTestServerOptions{MetricsRecorder: metricsRecorder})) server.Serve(lis) } diff --git a/interop/test_utils.go b/interop/test_utils.go index 7a42116e74b3..7b597cae40eb 100644 --- a/interop/test_utils.go +++ b/interop/test_utils.go @@ -17,25 +17,37 @@ */ // Package interop contains functions used by interop client/server. +// +// See interop test case descriptions [here]. +// +// [here]: https://github.com/grpc/grpc/blob/master/doc/interop-test-descriptions.md package interop import ( + "bytes" "context" "fmt" "io" - "io/ioutil" + "os" "strings" + "sync" "time" "github.com/golang/protobuf/proto" "golang.org/x/oauth2" "golang.org/x/oauth2/google" "google.golang.org/grpc" + "google.golang.org/grpc/benchmark/stats" "google.golang.org/grpc/codes" "google.golang.org/grpc/grpclog" - testpb "google.golang.org/grpc/interop/grpc_testing" "google.golang.org/grpc/metadata" + "google.golang.org/grpc/orca" + "google.golang.org/grpc/peer" "google.golang.org/grpc/status" + + v3orcapb "github.com/cncf/xds/go/xds/data/orca/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" ) var ( @@ -57,8 +69,6 @@ func ClientNewPayload(t testpb.PayloadType, size int) *testpb.Payload { body := make([]byte, size) switch t { case testpb.PayloadType_COMPRESSABLE: - case testpb.PayloadType_UNCOMPRESSABLE: - logger.Fatalf("PayloadType UNCOMPRESSABLE is not supported") default: logger.Fatalf("Unsupported payload type: %d", t) } @@ -69,7 +79,7 @@ func ClientNewPayload(t testpb.PayloadType, size int) *testpb.Payload { } // DoEmptyUnaryCall performs a unary RPC with empty request and response messages. -func DoEmptyUnaryCall(tc testpb.TestServiceClient, args ...grpc.CallOption) { +func DoEmptyUnaryCall(tc testgrpc.TestServiceClient, args ...grpc.CallOption) { reply, err := tc.EmptyCall(context.Background(), &testpb.Empty{}, args...) if err != nil { logger.Fatal("/TestService/EmptyCall RPC failed: ", err) @@ -80,7 +90,7 @@ func DoEmptyUnaryCall(tc testpb.TestServiceClient, args ...grpc.CallOption) { } // DoLargeUnaryCall performs a unary RPC with large payload in the request and response. -func DoLargeUnaryCall(tc testpb.TestServiceClient, args ...grpc.CallOption) { +func DoLargeUnaryCall(tc testgrpc.TestServiceClient, args ...grpc.CallOption) { pl := ClientNewPayload(testpb.PayloadType_COMPRESSABLE, largeReqSize) req := &testpb.SimpleRequest{ ResponseType: testpb.PayloadType_COMPRESSABLE, @@ -99,7 +109,7 @@ func DoLargeUnaryCall(tc testpb.TestServiceClient, args ...grpc.CallOption) { } // DoClientStreaming performs a client streaming RPC. -func DoClientStreaming(tc testpb.TestServiceClient, args ...grpc.CallOption) { +func DoClientStreaming(tc testgrpc.TestServiceClient, args ...grpc.CallOption) { stream, err := tc.StreamingInputCall(context.Background(), args...) if err != nil { logger.Fatalf("%v.StreamingInputCall(_) = _, %v", tc, err) @@ -125,7 +135,7 @@ func DoClientStreaming(tc testpb.TestServiceClient, args ...grpc.CallOption) { } // DoServerStreaming performs a server streaming RPC. -func DoServerStreaming(tc testpb.TestServiceClient, args ...grpc.CallOption) { +func DoServerStreaming(tc testgrpc.TestServiceClient, args ...grpc.CallOption) { respParam := make([]*testpb.ResponseParameters, len(respSizes)) for i, s := range respSizes { respParam[i] = &testpb.ResponseParameters{ @@ -169,7 +179,7 @@ func DoServerStreaming(tc testpb.TestServiceClient, args ...grpc.CallOption) { } // DoPingPong performs ping-pong style bi-directional streaming RPC. -func DoPingPong(tc testpb.TestServiceClient, args ...grpc.CallOption) { +func DoPingPong(tc testgrpc.TestServiceClient, args ...grpc.CallOption) { stream, err := tc.FullDuplexCall(context.Background(), args...) if err != nil { logger.Fatalf("%v.FullDuplexCall(_) = _, %v", tc, err) @@ -213,7 +223,7 @@ func DoPingPong(tc testpb.TestServiceClient, args ...grpc.CallOption) { } // DoEmptyStream sets up a bi-directional streaming with zero message. -func DoEmptyStream(tc testpb.TestServiceClient, args ...grpc.CallOption) { +func DoEmptyStream(tc testgrpc.TestServiceClient, args ...grpc.CallOption) { stream, err := tc.FullDuplexCall(context.Background(), args...) if err != nil { logger.Fatalf("%v.FullDuplexCall(_) = _, %v", tc, err) @@ -227,7 +237,7 @@ func DoEmptyStream(tc testpb.TestServiceClient, args ...grpc.CallOption) { } // DoTimeoutOnSleepingServer performs an RPC on a sleep server which causes RPC timeout. -func DoTimeoutOnSleepingServer(tc testpb.TestServiceClient, args ...grpc.CallOption) { +func DoTimeoutOnSleepingServer(tc testgrpc.TestServiceClient, args ...grpc.CallOption) { ctx, cancel := context.WithTimeout(context.Background(), 1*time.Millisecond) defer cancel() stream, err := tc.FullDuplexCall(ctx, args...) @@ -251,7 +261,7 @@ func DoTimeoutOnSleepingServer(tc testpb.TestServiceClient, args ...grpc.CallOpt } // DoComputeEngineCreds performs a unary RPC with compute engine auth. -func DoComputeEngineCreds(tc testpb.TestServiceClient, serviceAccount, oauthScope string) { +func DoComputeEngineCreds(tc testgrpc.TestServiceClient, serviceAccount, oauthScope string) { pl := ClientNewPayload(testpb.PayloadType_COMPRESSABLE, largeReqSize) req := &testpb.SimpleRequest{ ResponseType: testpb.PayloadType_COMPRESSABLE, @@ -275,7 +285,7 @@ func DoComputeEngineCreds(tc testpb.TestServiceClient, serviceAccount, oauthScop } func getServiceAccountJSONKey(keyFile string) []byte { - jsonKey, err := ioutil.ReadFile(keyFile) + jsonKey, err := os.ReadFile(keyFile) if err != nil { logger.Fatalf("Failed to read the service account key file: %v", err) } @@ -283,7 +293,7 @@ func getServiceAccountJSONKey(keyFile string) []byte { } // DoServiceAccountCreds performs a unary RPC with service account auth. -func DoServiceAccountCreds(tc testpb.TestServiceClient, serviceAccountKeyFile, oauthScope string) { +func DoServiceAccountCreds(tc testgrpc.TestServiceClient, serviceAccountKeyFile, oauthScope string) { pl := ClientNewPayload(testpb.PayloadType_COMPRESSABLE, largeReqSize) req := &testpb.SimpleRequest{ ResponseType: testpb.PayloadType_COMPRESSABLE, @@ -308,7 +318,7 @@ func DoServiceAccountCreds(tc testpb.TestServiceClient, serviceAccountKeyFile, o } // DoJWTTokenCreds performs a unary RPC with JWT token auth. -func DoJWTTokenCreds(tc testpb.TestServiceClient, serviceAccountKeyFile string) { +func DoJWTTokenCreds(tc testgrpc.TestServiceClient, serviceAccountKeyFile string) { pl := ClientNewPayload(testpb.PayloadType_COMPRESSABLE, largeReqSize) req := &testpb.SimpleRequest{ ResponseType: testpb.PayloadType_COMPRESSABLE, @@ -342,7 +352,7 @@ func GetToken(serviceAccountKeyFile string, oauthScope string) *oauth2.Token { } // DoOauth2TokenCreds performs a unary RPC with OAUTH2 token auth. -func DoOauth2TokenCreds(tc testpb.TestServiceClient, serviceAccountKeyFile, oauthScope string) { +func DoOauth2TokenCreds(tc testgrpc.TestServiceClient, serviceAccountKeyFile, oauthScope string) { pl := ClientNewPayload(testpb.PayloadType_COMPRESSABLE, largeReqSize) req := &testpb.SimpleRequest{ ResponseType: testpb.PayloadType_COMPRESSABLE, @@ -367,7 +377,7 @@ func DoOauth2TokenCreds(tc testpb.TestServiceClient, serviceAccountKeyFile, oaut } // DoPerRPCCreds performs a unary RPC with per RPC OAUTH2 token. -func DoPerRPCCreds(tc testpb.TestServiceClient, serviceAccountKeyFile, oauthScope string) { +func DoPerRPCCreds(tc testgrpc.TestServiceClient, serviceAccountKeyFile, oauthScope string) { jsonKey := getServiceAccountJSONKey(serviceAccountKeyFile) pl := ClientNewPayload(testpb.PayloadType_COMPRESSABLE, largeReqSize) req := &testpb.SimpleRequest{ @@ -395,7 +405,7 @@ func DoPerRPCCreds(tc testpb.TestServiceClient, serviceAccountKeyFile, oauthScop } // DoGoogleDefaultCredentials performs an unary RPC with google default credentials -func DoGoogleDefaultCredentials(tc testpb.TestServiceClient, defaultServiceAccount string) { +func DoGoogleDefaultCredentials(tc testgrpc.TestServiceClient, defaultServiceAccount string) { pl := ClientNewPayload(testpb.PayloadType_COMPRESSABLE, largeReqSize) req := &testpb.SimpleRequest{ ResponseType: testpb.PayloadType_COMPRESSABLE, @@ -414,7 +424,7 @@ func DoGoogleDefaultCredentials(tc testpb.TestServiceClient, defaultServiceAccou } // DoComputeEngineChannelCredentials performs an unary RPC with compute engine channel credentials -func DoComputeEngineChannelCredentials(tc testpb.TestServiceClient, defaultServiceAccount string) { +func DoComputeEngineChannelCredentials(tc testgrpc.TestServiceClient, defaultServiceAccount string) { pl := ClientNewPayload(testpb.PayloadType_COMPRESSABLE, largeReqSize) req := &testpb.SimpleRequest{ ResponseType: testpb.PayloadType_COMPRESSABLE, @@ -438,7 +448,7 @@ var testMetadata = metadata.MD{ } // DoCancelAfterBegin cancels the RPC after metadata has been sent but before payloads are sent. -func DoCancelAfterBegin(tc testpb.TestServiceClient, args ...grpc.CallOption) { +func DoCancelAfterBegin(tc testgrpc.TestServiceClient, args ...grpc.CallOption) { ctx, cancel := context.WithCancel(metadata.NewOutgoingContext(context.Background(), testMetadata)) stream, err := tc.StreamingInputCall(ctx, args...) if err != nil { @@ -452,7 +462,7 @@ func DoCancelAfterBegin(tc testpb.TestServiceClient, args ...grpc.CallOption) { } // DoCancelAfterFirstResponse cancels the RPC after receiving the first message from the server. -func DoCancelAfterFirstResponse(tc testpb.TestServiceClient, args ...grpc.CallOption) { +func DoCancelAfterFirstResponse(tc testgrpc.TestServiceClient, args ...grpc.CallOption) { ctx, cancel := context.WithCancel(context.Background()) stream, err := tc.FullDuplexCall(ctx, args...) if err != nil { @@ -506,7 +516,7 @@ func validateMetadata(header, trailer metadata.MD) { } // DoCustomMetadata checks that metadata is echoed back to the client. -func DoCustomMetadata(tc testpb.TestServiceClient, args ...grpc.CallOption) { +func DoCustomMetadata(tc testgrpc.TestServiceClient, args ...grpc.CallOption) { // Testing with UnaryCall. pl := ClientNewPayload(testpb.PayloadType_COMPRESSABLE, 1) req := &testpb.SimpleRequest{ @@ -568,7 +578,7 @@ func DoCustomMetadata(tc testpb.TestServiceClient, args ...grpc.CallOption) { } // DoStatusCodeAndMessage checks that the status code is propagated back to the client. -func DoStatusCodeAndMessage(tc testpb.TestServiceClient, args ...grpc.CallOption) { +func DoStatusCodeAndMessage(tc testgrpc.TestServiceClient, args ...grpc.CallOption) { var code int32 = 2 msg := "test status message" expectedErr := status.Error(codes.Code(code), msg) @@ -604,7 +614,7 @@ func DoStatusCodeAndMessage(tc testpb.TestServiceClient, args ...grpc.CallOption // DoSpecialStatusMessage verifies Unicode and whitespace is correctly processed // in status message. -func DoSpecialStatusMessage(tc testpb.TestServiceClient, args ...grpc.CallOption) { +func DoSpecialStatusMessage(tc testgrpc.TestServiceClient, args ...grpc.CallOption) { const ( code int32 = 2 msg string = "\t\ntest with whitespace\r\nand Unicode BMP ☺ and non-BMP 😈\t\n" @@ -624,7 +634,7 @@ func DoSpecialStatusMessage(tc testpb.TestServiceClient, args ...grpc.CallOption } // DoUnimplementedService attempts to call a method from an unimplemented service. -func DoUnimplementedService(tc testpb.UnimplementedServiceClient) { +func DoUnimplementedService(tc testgrpc.UnimplementedServiceClient) { _, err := tc.UnimplementedCall(context.Background(), &testpb.Empty{}) if status.Code(err) != codes.Unimplemented { logger.Fatalf("%v.UnimplementedCall() = _, %v, want _, %v", tc, status.Code(err), codes.Unimplemented) @@ -641,7 +651,7 @@ func DoUnimplementedMethod(cc *grpc.ClientConn) { // DoPickFirstUnary runs multiple RPCs (rpcCount) and checks that all requests // are sent to the same backend. -func DoPickFirstUnary(tc testpb.TestServiceClient) { +func DoPickFirstUnary(tc testgrpc.TestServiceClient) { const rpcCount = 100 pl := ClientNewPayload(testpb.PayloadType_COMPRESSABLE, 1) @@ -651,7 +661,8 @@ func DoPickFirstUnary(tc testpb.TestServiceClient) { Payload: pl, FillServerId: true, } - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + // TODO(mohanli): Revert the timeout back to 10s once TD migrates to xdstp. + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() var serverID string for i := 0; i < rpcCount; i++ { @@ -673,11 +684,121 @@ func DoPickFirstUnary(tc testpb.TestServiceClient) { } } -type testServer struct{} +func doOneSoakIteration(ctx context.Context, tc testgrpc.TestServiceClient, resetChannel bool, serverAddr string, soakRequestSize int, soakResponseSize int, dopts []grpc.DialOption, copts []grpc.CallOption) (latency time.Duration, err error) { + start := time.Now() + client := tc + if resetChannel { + var conn *grpc.ClientConn + conn, err = grpc.Dial(serverAddr, dopts...) + if err != nil { + return + } + defer conn.Close() + client = testgrpc.NewTestServiceClient(conn) + } + // per test spec, don't include channel shutdown in latency measurement + defer func() { latency = time.Since(start) }() + // do a large-unary RPC + pl := ClientNewPayload(testpb.PayloadType_COMPRESSABLE, soakRequestSize) + req := &testpb.SimpleRequest{ + ResponseType: testpb.PayloadType_COMPRESSABLE, + ResponseSize: int32(soakResponseSize), + Payload: pl, + } + var reply *testpb.SimpleResponse + reply, err = client.UnaryCall(ctx, req, copts...) + if err != nil { + err = fmt.Errorf("/TestService/UnaryCall RPC failed: %s", err) + return + } + t := reply.GetPayload().GetType() + s := len(reply.GetPayload().GetBody()) + if t != testpb.PayloadType_COMPRESSABLE || s != soakResponseSize { + err = fmt.Errorf("got the reply with type %d len %d; want %d, %d", t, s, testpb.PayloadType_COMPRESSABLE, soakResponseSize) + return + } + return +} -// NewTestServer creates a test server for test service. -func NewTestServer() *testpb.TestServiceService { - return testpb.NewTestServiceService(testpb.UnstableTestServiceService(&testServer{})) +// DoSoakTest runs large unary RPCs in a loop for a configurable number of times, with configurable failure thresholds. +// If resetChannel is false, then each RPC will be performed on tc. Otherwise, each RPC will be performed on a new +// stub that is created with the provided server address and dial options. +// TODO(mohanli-ml): Create SoakTestOptions as a parameter for this method. +func DoSoakTest(tc testgrpc.TestServiceClient, serverAddr string, dopts []grpc.DialOption, resetChannel bool, soakIterations int, maxFailures int, soakRequestSize int, soakResponseSize int, perIterationMaxAcceptableLatency time.Duration, minTimeBetweenRPCs time.Duration, overallDeadline time.Time) { + start := time.Now() + ctx, cancel := context.WithDeadline(context.Background(), overallDeadline) + defer cancel() + iterationsDone := 0 + totalFailures := 0 + hopts := stats.HistogramOptions{ + NumBuckets: 20, + GrowthFactor: 1, + BaseBucketSize: 1, + MinValue: 0, + } + h := stats.NewHistogram(hopts) + for i := 0; i < soakIterations; i++ { + if time.Now().After(overallDeadline) { + break + } + earliestNextStart := time.After(minTimeBetweenRPCs) + iterationsDone++ + var p peer.Peer + latency, err := doOneSoakIteration(ctx, tc, resetChannel, serverAddr, soakRequestSize, soakResponseSize, dopts, []grpc.CallOption{grpc.Peer(&p)}) + latencyMs := int64(latency / time.Millisecond) + h.Add(latencyMs) + if err != nil { + totalFailures++ + addrStr := "nil" + if p.Addr != nil { + addrStr = p.Addr.String() + } + fmt.Fprintf(os.Stderr, "soak iteration: %d elapsed_ms: %d peer: %s server_uri: %s failed: %s\n", i, latencyMs, addrStr, serverAddr, err) + <-earliestNextStart + continue + } + if latency > perIterationMaxAcceptableLatency { + totalFailures++ + fmt.Fprintf(os.Stderr, "soak iteration: %d elapsed_ms: %d peer: %s server_uri: %s exceeds max acceptable latency: %d\n", i, latencyMs, p.Addr.String(), serverAddr, perIterationMaxAcceptableLatency.Milliseconds()) + <-earliestNextStart + continue + } + fmt.Fprintf(os.Stderr, "soak iteration: %d elapsed_ms: %d peer: %s server_uri: %s succeeded\n", i, latencyMs, p.Addr.String(), serverAddr) + <-earliestNextStart + } + var b bytes.Buffer + h.Print(&b) + fmt.Fprintf(os.Stderr, "(server_uri: %s) histogram of per-iteration latencies in milliseconds: %s\n", serverAddr, b.String()) + fmt.Fprintf(os.Stderr, "(server_uri: %s) soak test ran: %d / %d iterations. total failures: %d. max failures threshold: %d. See breakdown above for which iterations succeeded, failed, and why for more info.\n", serverAddr, iterationsDone, soakIterations, totalFailures, maxFailures) + if iterationsDone < soakIterations { + logger.Fatalf("(server_uri: %s) soak test consumed all %f seconds of time and quit early, only having ran %d out of desired %d iterations.", serverAddr, overallDeadline.Sub(start).Seconds(), iterationsDone, soakIterations) + } + if totalFailures > maxFailures { + logger.Fatalf("(server_uri: %s) soak test total failures: %d exceeds max failures threshold: %d.", serverAddr, totalFailures, maxFailures) + } +} + +type testServer struct { + testgrpc.UnimplementedTestServiceServer + + orcaMu sync.Mutex + metricsRecorder orca.ServerMetricsRecorder +} + +// NewTestServerOptions contains options that control the behavior of the test +// server returned by NewTestServer. +type NewTestServerOptions struct { + MetricsRecorder orca.ServerMetricsRecorder +} + +// NewTestServer creates a test server for test service. opts carries optional +// settings and does not need to be provided. If multiple opts are provided, +// only the first one is used. +func NewTestServer(opts ...NewTestServerOptions) testgrpc.TestServiceServer { + if len(opts) > 0 { + return &testServer{metricsRecorder: opts[0].MetricsRecorder} + } + return &testServer{} } func (s *testServer) EmptyCall(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { @@ -691,8 +812,6 @@ func serverNewPayload(t testpb.PayloadType, size int32) (*testpb.Payload, error) body := make([]byte, size) switch t { case testpb.PayloadType_COMPRESSABLE: - case testpb.PayloadType_UNCOMPRESSABLE: - return nil, fmt.Errorf("payloadType UNCOMPRESSABLE is not supported") default: return nil, fmt.Errorf("unsupported payload type: %d", t) } @@ -721,12 +840,30 @@ func (s *testServer) UnaryCall(ctx context.Context, in *testpb.SimpleRequest) (* if err != nil { return nil, err } + if r, orcaData := orca.CallMetricsRecorderFromContext(ctx), in.GetOrcaPerQueryReport(); r != nil && orcaData != nil { + // Transfer the request's per-Call ORCA data to the call metrics + // recorder in the context, if present. + setORCAMetrics(r, orcaData) + } return &testpb.SimpleResponse{ Payload: pl, }, nil } -func (s *testServer) StreamingOutputCall(args *testpb.StreamingOutputCallRequest, stream testpb.TestService_StreamingOutputCallServer) error { +func setORCAMetrics(r orca.ServerMetricsRecorder, orcaData *testpb.TestOrcaReport) { + r.SetCPUUtilization(orcaData.CpuUtilization) + r.SetMemoryUtilization(orcaData.MemoryUtilization) + if rq, ok := r.(orca.CallMetricsRecorder); ok { + for k, v := range orcaData.RequestCost { + rq.SetRequestCost(k, v) + } + } + for k, v := range orcaData.Utilization { + r.SetNamedUtilization(k, v) + } +} + +func (s *testServer) StreamingOutputCall(args *testpb.StreamingOutputCallRequest, stream testgrpc.TestService_StreamingOutputCallServer) error { cs := args.GetResponseParameters() for _, c := range cs { if us := c.GetIntervalUs(); us > 0 { @@ -745,7 +882,7 @@ func (s *testServer) StreamingOutputCall(args *testpb.StreamingOutputCallRequest return nil } -func (s *testServer) StreamingInputCall(stream testpb.TestService_StreamingInputCallServer) error { +func (s *testServer) StreamingInputCall(stream testgrpc.TestService_StreamingInputCallServer) error { var sum int for { in, err := stream.Recv() @@ -762,7 +899,7 @@ func (s *testServer) StreamingInputCall(stream testpb.TestService_StreamingInput } } -func (s *testServer) FullDuplexCall(stream testpb.TestService_FullDuplexCallServer) error { +func (s *testServer) FullDuplexCall(stream testgrpc.TestService_FullDuplexCallServer) error { if md, ok := metadata.FromIncomingContext(stream.Context()); ok { if initialMetadata, ok := md[initialMetadataKey]; ok { header := metadata.Pairs(initialMetadataKey, initialMetadata[0]) @@ -773,6 +910,7 @@ func (s *testServer) FullDuplexCall(stream testpb.TestService_FullDuplexCallServ stream.SetTrailer(trailer) } } + hasORCALock := false for { in, err := stream.Recv() if err == io.EOF { @@ -786,6 +924,18 @@ func (s *testServer) FullDuplexCall(stream testpb.TestService_FullDuplexCallServ if st != nil && st.Code != 0 { return status.Error(codes.Code(st.Code), st.Message) } + + if r, orcaData := s.metricsRecorder, in.GetOrcaOobReport(); r != nil && orcaData != nil { + // Transfer the request's OOB ORCA data to the server metrics recorder + // in the server, if present. + if !hasORCALock { + s.orcaMu.Lock() + defer s.orcaMu.Unlock() + hasORCALock = true + } + setORCAMetrics(r, orcaData) + } + cs := in.GetResponseParameters() for _, c := range cs { if us := c.GetIntervalUs(); us > 0 { @@ -804,7 +954,7 @@ func (s *testServer) FullDuplexCall(stream testpb.TestService_FullDuplexCallServ } } -func (s *testServer) HalfDuplexCall(stream testpb.TestService_HalfDuplexCallServer) error { +func (s *testServer) HalfDuplexCall(stream testgrpc.TestService_HalfDuplexCallServer) error { var msgBuf []*testpb.StreamingOutputCallRequest for { in, err := stream.Recv() @@ -836,3 +986,102 @@ func (s *testServer) HalfDuplexCall(stream testpb.TestService_HalfDuplexCallServ } return nil } + +// DoORCAPerRPCTest performs a unary RPC that enables ORCA per-call reporting +// and verifies the load report sent back to the LB policy's Done callback. +func DoORCAPerRPCTest(tc testgrpc.TestServiceClient) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + orcaRes := &v3orcapb.OrcaLoadReport{} + _, err := tc.UnaryCall(contextWithORCAResult(ctx, &orcaRes), &testpb.SimpleRequest{ + OrcaPerQueryReport: &testpb.TestOrcaReport{ + CpuUtilization: 0.8210, + MemoryUtilization: 0.5847, + RequestCost: map[string]float64{"cost": 3456.32}, + Utilization: map[string]float64{"util": 0.30499}, + }, + }) + if err != nil { + logger.Fatalf("/TestService/UnaryCall RPC failed: ", err) + } + want := &v3orcapb.OrcaLoadReport{ + CpuUtilization: 0.8210, + MemUtilization: 0.5847, + RequestCost: map[string]float64{"cost": 3456.32}, + Utilization: map[string]float64{"util": 0.30499}, + } + if !proto.Equal(orcaRes, want) { + logger.Fatalf("/TestService/UnaryCall RPC received ORCA load report %+v; want %+v", orcaRes, want) + } +} + +// DoORCAOOBTest performs a streaming RPC that enables ORCA OOB reporting and +// verifies the load report sent to the LB policy's OOB listener. +func DoORCAOOBTest(tc testgrpc.TestServiceClient) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + stream, err := tc.FullDuplexCall(ctx) + if err != nil { + logger.Fatalf("/TestService/FullDuplexCall received error starting stream: %v", err) + } + err = stream.Send(&testpb.StreamingOutputCallRequest{ + OrcaOobReport: &testpb.TestOrcaReport{ + CpuUtilization: 0.8210, + MemoryUtilization: 0.5847, + Utilization: map[string]float64{"util": 0.30499}, + }, + ResponseParameters: []*testpb.ResponseParameters{{Size: 1}}, + }) + if err != nil { + logger.Fatalf("/TestService/FullDuplexCall received error sending: %v", err) + } + _, err = stream.Recv() + if err != nil { + logger.Fatalf("/TestService/FullDuplexCall received error receiving: %v", err) + } + + want := &v3orcapb.OrcaLoadReport{ + CpuUtilization: 0.8210, + MemUtilization: 0.5847, + Utilization: map[string]float64{"util": 0.30499}, + } + checkORCAMetrics(ctx, tc, want) + + err = stream.Send(&testpb.StreamingOutputCallRequest{ + OrcaOobReport: &testpb.TestOrcaReport{ + CpuUtilization: 0.29309, + MemoryUtilization: 0.2, + Utilization: map[string]float64{"util": 0.2039}, + }, + ResponseParameters: []*testpb.ResponseParameters{{Size: 1}}, + }) + if err != nil { + logger.Fatalf("/TestService/FullDuplexCall received error sending: %v", err) + } + _, err = stream.Recv() + if err != nil { + logger.Fatalf("/TestService/FullDuplexCall received error receiving: %v", err) + } + + want = &v3orcapb.OrcaLoadReport{ + CpuUtilization: 0.29309, + MemUtilization: 0.2, + Utilization: map[string]float64{"util": 0.2039}, + } + checkORCAMetrics(ctx, tc, want) +} + +func checkORCAMetrics(ctx context.Context, tc testgrpc.TestServiceClient, want *v3orcapb.OrcaLoadReport) { + for ctx.Err() == nil { + orcaRes := &v3orcapb.OrcaLoadReport{} + if _, err := tc.UnaryCall(contextWithORCAResult(ctx, &orcaRes), &testpb.SimpleRequest{}); err != nil { + logger.Fatalf("/TestService/UnaryCall RPC failed: ", err) + } + if proto.Equal(orcaRes, want) { + return + } + logger.Infof("/TestService/UnaryCall RPC received ORCA load report %+v; want %+v", orcaRes, want) + time.Sleep(time.Second) + } + logger.Fatalf("timed out waiting for expected ORCA load report") +} diff --git a/interop/xds/client/Dockerfile b/interop/xds/client/Dockerfile new file mode 100644 index 000000000000..5b6e3f61c5e1 --- /dev/null +++ b/interop/xds/client/Dockerfile @@ -0,0 +1,37 @@ +# Copyright 2021 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Dockerfile for building the xDS interop client. To build the image, run the +# following command from grpc-go directory: +# docker build -t -f interop/xds/client/Dockerfile . + +FROM golang:1.21-alpine as build + +# Make a grpc-go directory and copy the repo into it. +WORKDIR /go/src/grpc-go +COPY . . + +# Build a static binary without cgo so that we can copy just the binary in the +# final image, and can get rid of Go compiler and gRPC-Go dependencies. +RUN go build -tags osusergo,netgo interop/xds/client/client.go + +# Second stage of the build which copies over only the client binary and skips +# the Go compiler and gRPC repo from the earlier stage. This significantly +# reduces the docker image size. +FROM alpine +COPY --from=build /go/src/grpc-go/client . +ENV GRPC_GO_LOG_VERBOSITY_LEVEL=2 +ENV GRPC_GO_LOG_SEVERITY_LEVEL="info" +ENV GRPC_GO_LOG_FORMATTER="json" +ENTRYPOINT ["./client"] diff --git a/interop/xds/client/client.go b/interop/xds/client/client.go index 466219063881..ff03428e1105 100644 --- a/interop/xds/client/client.go +++ b/interop/xds/client/client.go @@ -31,13 +31,25 @@ import ( "time" "google.golang.org/grpc" + "google.golang.org/grpc/admin" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/credentials/xds" "google.golang.org/grpc/grpclog" - testpb "google.golang.org/grpc/interop/grpc_testing" "google.golang.org/grpc/metadata" "google.golang.org/grpc/peer" + "google.golang.org/grpc/reflection" + "google.golang.org/grpc/status" _ "google.golang.org/grpc/xds" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + _ "google.golang.org/grpc/interop/xds" // to register Custom LB. ) +func init() { + rpcCfgs.Store([]*rpcConfig{{typ: unaryCall}}) +} + type statsWatcherKey struct { startID int32 endID int32 @@ -54,7 +66,7 @@ type statsWatcher struct { rpcsByPeer map[string]int32 rpcsByType map[string]map[string]int32 numFailures int32 - remainingRpcs int32 + remainingRPCs int32 chanHosts chan *rpcInfo } @@ -67,27 +79,120 @@ func (watcher *statsWatcher) buildResp() *testpb.LoadBalancerStatsResponse { } return &testpb.LoadBalancerStatsResponse{ - NumFailures: watcher.numFailures + watcher.remainingRpcs, + NumFailures: watcher.numFailures + watcher.remainingRPCs, RpcsByPeer: watcher.rpcsByPeer, RpcsByMethod: rpcsByType, } } +type accumulatedStats struct { + mu sync.Mutex + numRPCsStartedByMethod map[string]int32 + numRPCsSucceededByMethod map[string]int32 + numRPCsFailedByMethod map[string]int32 + rpcStatusByMethod map[string]map[int32]int32 +} + +func convertRPCName(in string) string { + switch in { + case unaryCall: + return testpb.ClientConfigureRequest_UNARY_CALL.String() + case emptyCall: + return testpb.ClientConfigureRequest_EMPTY_CALL.String() + } + logger.Warningf("unrecognized rpc type: %s", in) + return in +} + +// copyStatsMap makes a copy of the map. +func copyStatsMap(originalMap map[string]int32) map[string]int32 { + newMap := make(map[string]int32, len(originalMap)) + for k, v := range originalMap { + newMap[k] = v + } + return newMap +} + +// copyStatsIntMap makes a copy of the map. +func copyStatsIntMap(originalMap map[int32]int32) map[int32]int32 { + newMap := make(map[int32]int32, len(originalMap)) + for k, v := range originalMap { + newMap[k] = v + } + return newMap +} + +func (as *accumulatedStats) makeStatsMap() map[string]*testpb.LoadBalancerAccumulatedStatsResponse_MethodStats { + m := make(map[string]*testpb.LoadBalancerAccumulatedStatsResponse_MethodStats) + for k, v := range as.numRPCsStartedByMethod { + m[k] = &testpb.LoadBalancerAccumulatedStatsResponse_MethodStats{RpcsStarted: v} + } + for k, v := range as.rpcStatusByMethod { + if m[k] == nil { + m[k] = &testpb.LoadBalancerAccumulatedStatsResponse_MethodStats{} + } + m[k].Result = copyStatsIntMap(v) + } + return m +} + +func (as *accumulatedStats) buildResp() *testpb.LoadBalancerAccumulatedStatsResponse { + as.mu.Lock() + defer as.mu.Unlock() + return &testpb.LoadBalancerAccumulatedStatsResponse{ + NumRpcsStartedByMethod: copyStatsMap(as.numRPCsStartedByMethod), + NumRpcsSucceededByMethod: copyStatsMap(as.numRPCsSucceededByMethod), + NumRpcsFailedByMethod: copyStatsMap(as.numRPCsFailedByMethod), + StatsPerMethod: as.makeStatsMap(), + } +} + +func (as *accumulatedStats) startRPC(rpcType string) { + as.mu.Lock() + defer as.mu.Unlock() + as.numRPCsStartedByMethod[convertRPCName(rpcType)]++ +} + +func (as *accumulatedStats) finishRPC(rpcType string, err error) { + as.mu.Lock() + defer as.mu.Unlock() + name := convertRPCName(rpcType) + if as.rpcStatusByMethod[name] == nil { + as.rpcStatusByMethod[name] = make(map[int32]int32) + } + as.rpcStatusByMethod[name][int32(status.Convert(err).Code())]++ + if err != nil { + as.numRPCsFailedByMethod[name]++ + return + } + as.numRPCsSucceededByMethod[name]++ +} + var ( failOnFailedRPC = flag.Bool("fail_on_failed_rpc", false, "Fail client if any RPCs fail after first success") numChannels = flag.Int("num_channels", 1, "Num of channels") printResponse = flag.Bool("print_response", false, "Write RPC response to stdout") qps = flag.Int("qps", 1, "QPS per channel, for each type of RPC") - rpc = flag.String("rpc", "UnaryCall", "Types of RPCs to make, ',' separated string. RPCs can be EmptyCall or UnaryCall") - rpcMetadata = flag.String("metadata", "", "The metadata to send with RPC, in format EmptyCall:key1:value1,UnaryCall:key2:value2") + rpc = flag.String("rpc", "UnaryCall", "Types of RPCs to make, ',' separated string. RPCs can be EmptyCall or UnaryCall. Deprecated: Use Configure RPC to XdsUpdateClientConfigureServiceServer instead.") + rpcMetadata = flag.String("metadata", "", "The metadata to send with RPC, in format EmptyCall:key1:value1,UnaryCall:key2:value2. Deprecated: Use Configure RPC to XdsUpdateClientConfigureServiceServer instead.") rpcTimeout = flag.Duration("rpc_timeout", 20*time.Second, "Per RPC timeout") server = flag.String("server", "localhost:8080", "Address of server to connect to") statsPort = flag.Int("stats_port", 8081, "Port to expose peer distribution stats service") + secureMode = flag.Bool("secure_mode", false, "If true, retrieve security configuration from the management server. Else, use insecure credentials.") + + rpcCfgs atomic.Value mu sync.Mutex currentRequestID int32 watchers = make(map[statsWatcherKey]*statsWatcher) + accStats = accumulatedStats{ + numRPCsStartedByMethod: make(map[string]int32), + numRPCsSucceededByMethod: make(map[string]int32), + numRPCsFailedByMethod: make(map[string]int32), + rpcStatusByMethod: make(map[string]map[int32]int32), + } + // 0 or 1 representing an RPC has succeeded. Use hasRPCSucceeded and // setRPCSucceeded to access in a safe manner. rpcSucceeded uint32 @@ -95,6 +200,10 @@ var ( logger = grpclog.Component("interop") ) +type statsService struct { + testgrpc.UnimplementedLoadBalancerStatsServiceServer +} + func hasRPCSucceeded() bool { return atomic.LoadUint32(&rpcSucceeded) > 0 } @@ -107,7 +216,7 @@ func setRPCSucceeded() { // and return the distribution of remote peers. This is essentially a clientside // LB reporting mechanism that is designed to be queried by an external test // driver when verifying that the client is distributing RPCs as expected. -func getClientStats(ctx context.Context, in *testpb.LoadBalancerStatsRequest) (*testpb.LoadBalancerStatsResponse, error) { +func (s *statsService) GetClientStats(ctx context.Context, in *testpb.LoadBalancerStatsRequest) (*testpb.LoadBalancerStatsResponse, error) { mu.Lock() watcherKey := statsWatcherKey{currentRequestID, currentRequestID + in.GetNumRpcs()} watcher, ok := watchers[watcherKey] @@ -116,7 +225,7 @@ func getClientStats(ctx context.Context, in *testpb.LoadBalancerStatsRequest) (* rpcsByPeer: make(map[string]int32), rpcsByType: make(map[string]map[string]int32), numFailures: 0, - remainingRpcs: in.GetNumRpcs(), + remainingRPCs: in.GetNumRpcs(), chanHosts: make(chan *rpcInfo), } watchers[watcherKey] = watcher @@ -148,8 +257,8 @@ func getClientStats(ctx context.Context, in *testpb.LoadBalancerStatsRequest) (* } else { watcher.numFailures++ } - watcher.remainingRpcs-- - if watcher.remainingRpcs == 0 { + watcher.remainingRPCs-- + if watcher.remainingRPCs == 0 { return watcher.buildResp(), nil } case <-ctx.Done(): @@ -159,17 +268,60 @@ func getClientStats(ctx context.Context, in *testpb.LoadBalancerStatsRequest) (* } } +func (s *statsService) GetClientAccumulatedStats(ctx context.Context, in *testpb.LoadBalancerAccumulatedStatsRequest) (*testpb.LoadBalancerAccumulatedStatsResponse, error) { + return accStats.buildResp(), nil +} + +type configureService struct { + testgrpc.UnimplementedXdsUpdateClientConfigureServiceServer +} + +func (s *configureService) Configure(ctx context.Context, in *testpb.ClientConfigureRequest) (*testpb.ClientConfigureResponse, error) { + rpcsToMD := make(map[testpb.ClientConfigureRequest_RpcType][]string) + for _, typ := range in.GetTypes() { + rpcsToMD[typ] = nil + } + for _, md := range in.GetMetadata() { + typ := md.GetType() + strs, ok := rpcsToMD[typ] + if !ok { + continue + } + rpcsToMD[typ] = append(strs, md.GetKey(), md.GetValue()) + } + cfgs := make([]*rpcConfig, 0, len(rpcsToMD)) + for typ, md := range rpcsToMD { + var rpcType string + switch typ { + case testpb.ClientConfigureRequest_UNARY_CALL: + rpcType = unaryCall + case testpb.ClientConfigureRequest_EMPTY_CALL: + rpcType = emptyCall + default: + return nil, fmt.Errorf("unsupported RPC type: %v", typ) + } + cfgs = append(cfgs, &rpcConfig{ + typ: rpcType, + md: metadata.Pairs(md...), + timeout: in.GetTimeoutSec(), + }) + } + rpcCfgs.Store(cfgs) + return &testpb.ClientConfigureResponse{}, nil +} + const ( unaryCall string = "UnaryCall" emptyCall string = "EmptyCall" ) -func parseRPCTypes(rpcStr string) (ret []string) { +func parseRPCTypes(rpcStr string) []string { if len(rpcStr) == 0 { return []string{unaryCall} } rpcs := strings.Split(rpcStr, ",") + ret := make([]string, 0, len(rpcStr)) for _, r := range rpcs { switch r { case unaryCall, emptyCall: @@ -179,16 +331,18 @@ func parseRPCTypes(rpcStr string) (ret []string) { log.Fatalf("unsupported RPC type: %v", r) } } - return + return ret } type rpcConfig struct { - typ string - md metadata.MD + typ string + md metadata.MD + timeout int32 } // parseRPCMetadata turns EmptyCall:key1:value1 into -// {typ: emptyCall, md: {key1:value1}}. +// +// {typ: emptyCall, md: {key1:value1}}. func parseRPCMetadata(rpcMetadataStr string, rpcs []string) []*rpcConfig { rpcMetadataSplit := strings.Split(rpcMetadataStr, ",") rpcsToMD := make(map[string][]string) @@ -214,7 +368,7 @@ func parseRPCMetadata(rpcMetadataStr string, rpcs []string) []*rpcConfig { func main() { flag.Parse() - rpcCfgs := parseRPCMetadata(*rpcMetadata, parseRPCTypes(*rpc)) + rpcCfgs.Store(parseRPCMetadata(*rpcMetadata, parseRPCTypes(*rpc))) lis, err := net.Listen("tcp", fmt.Sprintf(":%d", *statsPort)) if err != nil { @@ -222,25 +376,45 @@ func main() { } s := grpc.NewServer() defer s.Stop() - testpb.RegisterLoadBalancerStatsServiceService(s, &testpb.LoadBalancerStatsServiceService{GetClientStats: getClientStats}) + testgrpc.RegisterLoadBalancerStatsServiceServer(s, &statsService{}) + testgrpc.RegisterXdsUpdateClientConfigureServiceServer(s, &configureService{}) + reflection.Register(s) + cleanup, err := admin.Register(s) + if err != nil { + logger.Fatalf("Failed to register admin: %v", err) + } + defer cleanup() go s.Serve(lis) - clients := make([]testpb.TestServiceClient, *numChannels) + creds := insecure.NewCredentials() + if *secureMode { + var err error + creds, err = xds.NewClientCredentials(xds.ClientOptions{FallbackCreds: insecure.NewCredentials()}) + if err != nil { + logger.Fatalf("Failed to create xDS credentials: %v", err) + } + } + + clients := make([]testgrpc.TestServiceClient, *numChannels) for i := 0; i < *numChannels; i++ { - conn, err := grpc.DialContext(context.Background(), *server, grpc.WithInsecure()) + conn, err := grpc.Dial(*server, grpc.WithTransportCredentials(creds)) if err != nil { logger.Fatalf("Fail to dial: %v", err) } defer conn.Close() - clients[i] = testpb.NewTestServiceClient(conn) + clients[i] = testgrpc.NewTestServiceClient(conn) } ticker := time.NewTicker(time.Second / time.Duration(*qps**numChannels)) defer ticker.Stop() - sendRPCs(clients, rpcCfgs, ticker) + sendRPCs(clients, ticker) } -func makeOneRPC(c testpb.TestServiceClient, cfg *rpcConfig) (*peer.Peer, *rpcInfo, error) { - ctx, cancel := context.WithTimeout(context.Background(), *rpcTimeout) +func makeOneRPC(c testgrpc.TestServiceClient, cfg *rpcConfig) (*peer.Peer, *rpcInfo, error) { + timeout := *rpcTimeout + if cfg.timeout != 0 { + timeout = time.Duration(cfg.timeout) * time.Second + } + ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() if len(cfg.md) != 0 { @@ -253,6 +427,7 @@ func makeOneRPC(c testpb.TestServiceClient, cfg *rpcConfig) (*peer.Peer, *rpcInf header metadata.MD err error ) + accStats.startRPC(cfg.typ) switch cfg.typ { case unaryCall: var resp *testpb.SimpleResponse @@ -265,6 +440,7 @@ func makeOneRPC(c testpb.TestServiceClient, cfg *rpcConfig) (*peer.Peer, *rpcInf case emptyCall: _, err = c.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&p), grpc.Header(&header)) } + accStats.finishRPC(cfg.typ, err) if err != nil { return nil, nil, err } @@ -276,26 +452,28 @@ func makeOneRPC(c testpb.TestServiceClient, cfg *rpcConfig) (*peer.Peer, *rpcInf return &p, &info, err } -func sendRPCs(clients []testpb.TestServiceClient, cfgs []*rpcConfig, ticker *time.Ticker) { +func sendRPCs(clients []testgrpc.TestServiceClient, ticker *time.Ticker) { var i int for range ticker.C { - go func(i int) { - // Get and increment request ID, and save a list of watchers that - // are interested in this RPC. - mu.Lock() - savedRequestID := currentRequestID - currentRequestID++ - savedWatchers := []*statsWatcher{} - for key, value := range watchers { - if key.startID <= savedRequestID && savedRequestID < key.endID { - savedWatchers = append(savedWatchers, value) - } + // Get and increment request ID, and save a list of watchers that are + // interested in this RPC. + mu.Lock() + savedRequestID := currentRequestID + currentRequestID++ + savedWatchers := []*statsWatcher{} + for key, value := range watchers { + if key.startID <= savedRequestID && savedRequestID < key.endID { + savedWatchers = append(savedWatchers, value) } - mu.Unlock() + } + mu.Unlock() - c := clients[i] + // Get the RPC metadata configurations from the Configure RPC. + cfgs := rpcCfgs.Load().([]*rpcConfig) - for _, cfg := range cfgs { + c := clients[i] + for _, cfg := range cfgs { + go func(cfg *rpcConfig) { p, info, err := makeOneRPC(c, cfg) for _, watcher := range savedWatchers { @@ -321,8 +499,8 @@ func sendRPCs(clients []testpb.TestServiceClient, cfgs []*rpcConfig, ticker *tim fmt.Printf("RPC %q, failed with %v\n", cfg.typ, err) } } - } - }(i) + }(cfg) + } i = (i + 1) % len(clients) } } diff --git a/interop/xds/custom_lb.go b/interop/xds/custom_lb.go new file mode 100644 index 000000000000..a08d82554008 --- /dev/null +++ b/interop/xds/custom_lb.go @@ -0,0 +1,140 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package xds contains various xds interop helpers for usage in interop tests. +package xds + +import ( + "encoding/json" + "fmt" + "sync" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/roundrobin" + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/serviceconfig" +) + +func init() { + balancer.Register(rpcBehaviorBB{}) +} + +const name = "test.RpcBehaviorLoadBalancer" + +type rpcBehaviorBB struct{} + +func (rpcBehaviorBB) Build(cc balancer.ClientConn, bOpts balancer.BuildOptions) balancer.Balancer { + b := &rpcBehaviorLB{ + ClientConn: cc, + } + // round_robin child to complete balancer tree with a usable leaf policy and + // have RPCs actually work. + builder := balancer.Get(roundrobin.Name) + if builder == nil { + // Shouldn't happen, defensive programming. Registered from import of + // roundrobin package. + return nil + } + rr := builder.Build(b, bOpts) + if rr == nil { + // Shouldn't happen, defensive programming. + return nil + } + b.Balancer = rr + return b +} + +func (rpcBehaviorBB) ParseConfig(s json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + lbCfg := &lbConfig{} + if err := json.Unmarshal(s, lbCfg); err != nil { + return nil, fmt.Errorf("rpc-behavior-lb: unable to marshal lbConfig: %s, error: %v", string(s), err) + } + return lbCfg, nil + +} + +func (rpcBehaviorBB) Name() string { + return name +} + +type lbConfig struct { + serviceconfig.LoadBalancingConfig `json:"-"` + RPCBehavior string `json:"rpcBehavior,omitempty"` +} + +// rpcBehaviorLB is a load balancer that wraps a round robin balancer and +// appends the rpc-behavior metadata field to any metadata in pick results based +// on what is specified in configuration. +type rpcBehaviorLB struct { + // embed a ClientConn to wrap only UpdateState() operation + balancer.ClientConn + // embed a Balancer to wrap only UpdateClientConnState() operation + balancer.Balancer + + mu sync.Mutex + cfg *lbConfig +} + +func (b *rpcBehaviorLB) UpdateClientConnState(s balancer.ClientConnState) error { + lbCfg, ok := s.BalancerConfig.(*lbConfig) + if !ok { + return fmt.Errorf("test.RpcBehaviorLoadBalancer:received config with unexpected type %T: %s", s.BalancerConfig, pretty.ToJSON(s.BalancerConfig)) + } + b.mu.Lock() + b.cfg = lbCfg + b.mu.Unlock() + return b.Balancer.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: s.ResolverState, + }) +} + +func (b *rpcBehaviorLB) UpdateState(state balancer.State) { + b.mu.Lock() + rpcBehavior := b.cfg.RPCBehavior + b.mu.Unlock() + + b.ClientConn.UpdateState(balancer.State{ + ConnectivityState: state.ConnectivityState, + Picker: newRPCBehaviorPicker(state.Picker, rpcBehavior), + }) +} + +// rpcBehaviorPicker wraps a picker and adds the rpc-behavior metadata field +// into the child pick result's metadata. +type rpcBehaviorPicker struct { + childPicker balancer.Picker + rpcBehavior string +} + +// Pick appends the rpc-behavior metadata entry to the pick result of the child. +func (p *rpcBehaviorPicker) Pick(info balancer.PickInfo) (balancer.PickResult, error) { + pr, err := p.childPicker.Pick(info) + if err != nil { + return balancer.PickResult{}, err + } + pr.Metadata = metadata.Join(pr.Metadata, metadata.Pairs("rpc-behavior", p.rpcBehavior)) + return pr, nil +} + +func newRPCBehaviorPicker(childPicker balancer.Picker, rpcBehavior string) *rpcBehaviorPicker { + return &rpcBehaviorPicker{ + childPicker: childPicker, + rpcBehavior: rpcBehavior, + } +} diff --git a/interop/xds/custom_lb_test.go b/interop/xds/custom_lb_test.go new file mode 100644 index 000000000000..fc3a7f71c5c9 --- /dev/null +++ b/interop/xds/custom_lb_test.go @@ -0,0 +1,135 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/serviceconfig" +) + +var defaultTestTimeout = 5 * time.Second + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// TestCustomLB tests the Custom LB for the interop client. It configures the +// custom lb as the top level Load Balancing policy of the channel, then asserts +// it can successfully make an RPC and also that the rpc behavior the Custom LB +// is configured with makes it's way to the server in metadata. +func (s) TestCustomLB(t *testing.T) { + errCh := testutils.NewChannel() + // Setup a backend which verifies the expected rpc-behavior metadata is + // present in the request. + backend := &stubserver.StubServer{ + UnaryCallF: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + errCh.Send(errors.New("failed to receive metadata")) + return &testpb.SimpleResponse{}, nil + } + rpcBMD := md.Get("rpc-behavior") + if len(rpcBMD) != 1 { + errCh.Send(errors.New("only one value received for metadata key rpc-behavior")) + return &testpb.SimpleResponse{}, nil + } + wantVal := "error-code-0" + if rpcBMD[0] != wantVal { + errCh.Send(fmt.Errorf("metadata val for key \"rpc-behavior\": got val %v, want val %v", rpcBMD[0], wantVal)) + return &testpb.SimpleResponse{}, nil + } + // Success. + errCh.Send(nil) + return &testpb.SimpleResponse{}, nil + }, + } + if err := backend.StartServer(); err != nil { + t.Fatalf("Failed to start backend: %v", err) + } + t.Logf("Started good TestService backend at: %q", backend.Address) + defer backend.Stop() + + lbCfgJSON := `{ + "loadBalancingConfig": [ + { + "test.RpcBehaviorLoadBalancer": { + "rpcBehavior": "error-code-0" + } + } + ] + }` + + sc := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(lbCfgJSON) + mr := manual.NewBuilderWithScheme("customlb-e2e") + defer mr.Close() + mr.InitialState(resolver.State{ + Addresses: []resolver.Address{ + {Addr: backend.Address}, + }, + ServiceConfig: sc, + }) + + cc, err := grpc.Dial(mr.Scheme()+":///", grpc.WithResolvers(mr), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) + } + defer cc.Close() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testServiceClient := testgrpc.NewTestServiceClient(cc) + + // Make a Unary RPC. This RPC should be successful due to the round_robin + // leaf balancer. Also, the custom load balancer should inject the + // "rpc-behavior" string it is configured with into the metadata sent to + // server. + if _, err := testServiceClient.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + + val, err := errCh.Receive(ctx) + if err != nil { + t.Fatalf("error receiving from errCh: %v", err) + } + + // Should receive nil on the error channel which implies backend verified it + // correctly received the correct "rpc-behavior" metadata. + if err, ok := val.(error); ok { + t.Fatalf("error in backend verifications on metadata received: %v", err) + } +} diff --git a/interop/xds/server/Dockerfile b/interop/xds/server/Dockerfile new file mode 100644 index 000000000000..f7d1cf0ff022 --- /dev/null +++ b/interop/xds/server/Dockerfile @@ -0,0 +1,37 @@ +# Copyright 2021 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Dockerfile for building the xDS interop server. To build the image, run the +# following command from grpc-go directory: +# docker build -t -f interop/xds/server/Dockerfile . + +FROM golang:1.21-alpine as build + +# Make a grpc-go directory and copy the repo into it. +WORKDIR /go/src/grpc-go +COPY . . + +# Build a static binary without cgo so that we can copy just the binary in the +# final image, and can get rid of the Go compiler and gRPC-Go dependencies. +RUN go build -tags osusergo,netgo interop/xds/server/server.go + +# Second stage of the build which copies over only the client binary and skips +# the Go compiler and gRPC repo from the earlier stage. This significantly +# reduces the docker image size. +FROM alpine +COPY --from=build /go/src/grpc-go/server . +ENV GRPC_GO_LOG_VERBOSITY_LEVEL=2 +ENV GRPC_GO_LOG_SEVERITY_LEVEL="info" +ENV GRPC_GO_LOG_FORMATTER="json" +ENTRYPOINT ["./server"] diff --git a/interop/xds/server/server.go b/interop/xds/server/server.go index 8d2a62487a88..5f59b61f303c 100644 --- a/interop/xds/server/server.go +++ b/interop/xds/server/server.go @@ -1,6 +1,6 @@ /* * - * Copyright 2020 gRPC authors. + * Copyright 2021 gRPC authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,32 +16,47 @@ * */ -// Binary server for xDS interop tests. +// Binary server is the server used for xDS interop tests. package main import ( "context" "flag" + "fmt" "log" "net" "os" - "strconv" "google.golang.org/grpc" + "google.golang.org/grpc/admin" + "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/grpclog" - testpb "google.golang.org/grpc/interop/grpc_testing" + "google.golang.org/grpc/health" "google.golang.org/grpc/metadata" + "google.golang.org/grpc/reflection" + "google.golang.org/grpc/xds" + + xdscreds "google.golang.org/grpc/credentials/xds" + healthgrpc "google.golang.org/grpc/health/grpc_health_v1" + healthpb "google.golang.org/grpc/health/grpc_health_v1" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" ) var ( - port = flag.Int("port", 8080, "The server port") - serverID = flag.String("server_id", "go_server", "Server ID included in response") - hostname = getHostname() + port = flag.Int("port", 8080, "Listening port for test service") + maintenancePort = flag.Int("maintenance_port", 8081, "Listening port for maintenance services like health, reflection, channelz etc when -secure_mode is true. When -secure_mode is false, all these services will be registered on -port") + serverID = flag.String("server_id", "go_server", "Server ID included in response") + secureMode = flag.Bool("secure_mode", false, "If true, retrieve security configuration from the management server. Else, use insecure credentials.") + hostNameOverride = flag.String("host_name_override", "", "If set, use this as the hostname instead of the real hostname") logger = grpclog.Component("interop") ) func getHostname() string { + if *hostNameOverride != "" { + return *hostNameOverride + } hostname, err := os.Hostname() if err != nil { log.Fatalf("failed to get hostname: %v", err) @@ -49,24 +64,130 @@ func getHostname() string { return hostname } -func emptyCall(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { - grpc.SetHeader(ctx, metadata.Pairs("hostname", hostname)) +// testServiceImpl provides an implementation of the TestService defined in +// grpc.testing package. +type testServiceImpl struct { + testgrpc.UnimplementedTestServiceServer + hostname string + serverID string +} + +func (s *testServiceImpl) EmptyCall(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + grpc.SetHeader(ctx, metadata.Pairs("hostname", s.hostname)) return &testpb.Empty{}, nil } -func unaryCall(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { - grpc.SetHeader(ctx, metadata.Pairs("hostname", hostname)) - return &testpb.SimpleResponse{ServerId: *serverID, Hostname: hostname}, nil +func (s *testServiceImpl) UnaryCall(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + grpc.SetHeader(ctx, metadata.Pairs("hostname", s.hostname)) + return &testpb.SimpleResponse{ServerId: s.serverID, Hostname: s.hostname}, nil +} + +// xdsUpdateHealthServiceImpl provides an implementation of the +// XdsUpdateHealthService defined in grpc.testing package. +type xdsUpdateHealthServiceImpl struct { + testgrpc.UnimplementedXdsUpdateHealthServiceServer + healthServer *health.Server +} + +func (x *xdsUpdateHealthServiceImpl) SetServing(_ context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + x.healthServer.SetServingStatus("", healthpb.HealthCheckResponse_SERVING) + return &testpb.Empty{}, nil + +} + +func (x *xdsUpdateHealthServiceImpl) SetNotServing(_ context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + x.healthServer.SetServingStatus("", healthpb.HealthCheckResponse_NOT_SERVING) + return &testpb.Empty{}, nil +} + +func xdsServingModeCallback(addr net.Addr, args xds.ServingModeChangeArgs) { + logger.Infof("Serving mode callback for xDS server at %q invoked with mode: %q, err: %v", addr.String(), args.Mode, args.Err) } func main() { flag.Parse() - p := strconv.Itoa(*port) - lis, err := net.Listen("tcp", ":"+p) + + if *secureMode && *port == *maintenancePort { + logger.Fatal("-port and -maintenance_port must be different when -secure_mode is set") + } + + testService := &testServiceImpl{hostname: getHostname(), serverID: *serverID} + healthServer := health.NewServer() + updateHealthService := &xdsUpdateHealthServiceImpl{healthServer: healthServer} + + // If -secure_mode is not set, expose all services on -port with a regular + // gRPC server. + if !*secureMode { + addr := fmt.Sprintf(":%d", *port) + lis, err := net.Listen("tcp4", addr) + if err != nil { + logger.Fatalf("net.Listen(%s) failed: %v", addr, err) + } + + server := grpc.NewServer() + testgrpc.RegisterTestServiceServer(server, testService) + healthServer.SetServingStatus("", healthpb.HealthCheckResponse_SERVING) + healthgrpc.RegisterHealthServer(server, healthServer) + testgrpc.RegisterXdsUpdateHealthServiceServer(server, updateHealthService) + reflection.Register(server) + cleanup, err := admin.Register(server) + if err != nil { + logger.Fatalf("Failed to register admin services: %v", err) + } + defer cleanup() + if err := server.Serve(lis); err != nil { + logger.Errorf("Serve() failed: %v", err) + } + return + } + + // Create a listener on -port to expose the test service. + addr := fmt.Sprintf(":%d", *port) + testLis, err := net.Listen("tcp4", addr) + if err != nil { + logger.Fatalf("net.Listen(%s) failed: %v", addr, err) + } + + // Create server-side xDS credentials with a plaintext fallback. + creds, err := xdscreds.NewServerCredentials(xdscreds.ServerOptions{FallbackCreds: insecure.NewCredentials()}) + if err != nil { + logger.Fatalf("Failed to create xDS credentials: %v", err) + } + + // Create an xDS enabled gRPC server, register the test service + // implementation and start serving. + testServer, err := xds.NewGRPCServer(grpc.Creds(creds), xds.ServingModeCallback(xdsServingModeCallback)) + if err != nil { + logger.Fatal("Failed to create an xDS enabled gRPC server: %v", err) + } + testgrpc.RegisterTestServiceServer(testServer, testService) + go func() { + if err := testServer.Serve(testLis); err != nil { + logger.Errorf("test server Serve() failed: %v", err) + } + }() + defer testServer.Stop() + + // Create a listener on -maintenance_port to expose other services. + addr = fmt.Sprintf(":%d", *maintenancePort) + maintenanceLis, err := net.Listen("tcp4", addr) if err != nil { - logger.Fatalf("failed to listen: %v", err) + logger.Fatalf("net.Listen(%s) failed: %v", addr, err) + } + + // Create a regular gRPC server and register the maintenance services on + // it and start serving. + maintenanceServer := grpc.NewServer() + healthServer.SetServingStatus("", healthpb.HealthCheckResponse_SERVING) + healthgrpc.RegisterHealthServer(maintenanceServer, healthServer) + testgrpc.RegisterXdsUpdateHealthServiceServer(maintenanceServer, updateHealthService) + reflection.Register(maintenanceServer) + cleanup, err := admin.Register(maintenanceServer) + if err != nil { + logger.Fatalf("Failed to register admin services: %v", err) + } + defer cleanup() + if err := maintenanceServer.Serve(maintenanceLis); err != nil { + logger.Errorf("maintenance server Serve() failed: %v", err) } - s := grpc.NewServer() - testpb.RegisterTestServiceService(s, &testpb.TestServiceService{EmptyCall: emptyCall, UnaryCall: unaryCall}) - s.Serve(lis) } diff --git a/interop/xds_federation/client.go b/interop/xds_federation/client.go new file mode 100644 index 000000000000..eee5ba747af3 --- /dev/null +++ b/interop/xds_federation/client.go @@ -0,0 +1,128 @@ +/* + * + * Copyright 2014 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Binary client is an interop client. +package main + +import ( + "flag" + "strings" + "sync" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/google" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/interop" + + _ "google.golang.org/grpc/balancer/grpclb" // Register the grpclb load balancing policy. + _ "google.golang.org/grpc/balancer/rls" // Register the RLS load balancing policy. + _ "google.golang.org/grpc/xds/googledirectpath" // Register xDS resolver required for c2p directpath. + + testgrpc "google.golang.org/grpc/interop/grpc_testing" +) + +const ( + computeEngineCredsName = "compute_engine_channel_creds" + insecureCredsName = "INSECURE_CREDENTIALS" +) + +var ( + serverURIs = flag.String("server_uris", "", "Comma-separated list of sever URIs to make RPCs to") + credentialsTypes = flag.String("credentials_types", "", "Comma-separated list of credentials, each entry is used for the server of the corresponding index in server_uris. Supported values: compute_engine_channel_creds, INSECURE_CREDENTIALS") + soakIterations = flag.Int("soak_iterations", 10, "The number of iterations to use for the two soak tests: rpc_soak and channel_soak") + soakMaxFailures = flag.Int("soak_max_failures", 0, "The number of iterations in soak tests that are allowed to fail (either due to non-OK status code or exceeding the per-iteration max acceptable latency).") + soakPerIterationMaxAcceptableLatencyMs = flag.Int("soak_per_iteration_max_acceptable_latency_ms", 1000, "The number of milliseconds a single iteration in the two soak tests (rpc_soak and channel_soak) should take.") + soakOverallTimeoutSeconds = flag.Int("soak_overall_timeout_seconds", 10, "The overall number of seconds after which a soak test should stop and fail, if the desired number of iterations have not yet completed.") + soakMinTimeMsBetweenRPCs = flag.Int("soak_min_time_ms_between_rpcs", 0, "The minimum time in milliseconds between consecutive RPCs in a soak test (rpc_soak or channel_soak), useful for limiting QPS") + soakRequestSize = flag.Int("soak_request_size", 271828, "The request size in a soak RPC. The default value is set based on the interop large unary test case.") + soakResponseSize = flag.Int("soak_response_size", 314159, "The response size in a soak RPC. The default value is set based on the interop large unary test case.") + testCase = flag.String("test_case", "rpc_soak", + `Configure different test cases. Valid options are: + rpc_soak: sends --soak_iterations large_unary RPCs; + channel_soak: sends --soak_iterations RPCs, rebuilding the channel each time`) + + logger = grpclog.Component("interop") +) + +type clientConfig struct { + tc testgrpc.TestServiceClient + opts []grpc.DialOption + uri string +} + +func main() { + flag.Parse() + // validate flags + uris := strings.Split(*serverURIs, ",") + creds := strings.Split(*credentialsTypes, ",") + if len(uris) != len(creds) { + logger.Fatalf("Number of entries in --server_uris (%d) != number of entries in --credentials_types (%d)", len(uris), len(creds)) + } + for _, c := range creds { + if c != computeEngineCredsName && c != insecureCredsName { + logger.Fatalf("Unsupported credentials type: %v", c) + } + } + var resetChannel bool + switch *testCase { + case "rpc_soak": + resetChannel = false + case "channel_soak": + resetChannel = true + default: + logger.Fatal("Unsupported test case: ", *testCase) + } + + // create clients as specified in flags + var clients []clientConfig + for i := range uris { + var opts []grpc.DialOption + switch creds[i] { + case computeEngineCredsName: + opts = append(opts, grpc.WithCredentialsBundle(google.NewComputeEngineCredentials())) + case insecureCredsName: + opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) + } + cc, err := grpc.Dial(uris[i], opts...) + if err != nil { + logger.Fatalf("Fail to dial %v: %v", uris[i], err) + } + defer cc.Close() + clients = append(clients, clientConfig{ + tc: testgrpc.NewTestServiceClient(cc), + opts: opts, + uri: uris[i], + }) + } + + // run soak tests with the different clients + logger.Infof("Clients running with test case %q", *testCase) + var wg sync.WaitGroup + for i := range clients { + wg.Add(1) + go func(c clientConfig) { + interop.DoSoakTest(c.tc, c.uri, c.opts, resetChannel, *soakIterations, *soakMaxFailures, *soakRequestSize, *soakResponseSize, time.Duration(*soakPerIterationMaxAcceptableLatencyMs)*time.Millisecond, time.Duration(*soakMinTimeMsBetweenRPCs)*time.Millisecond, time.Now().Add(time.Duration(*soakOverallTimeoutSeconds)*time.Second)) + logger.Infof("%s test done for server: %s", *testCase, c.uri) + wg.Done() + }(clients[i]) + } + wg.Wait() + logger.Infoln("All clients done!") +} diff --git a/interop/xds_federation/xds_federation b/interop/xds_federation/xds_federation new file mode 100755 index 000000000000..5901b7c48de6 Binary files /dev/null and b/interop/xds_federation/xds_federation differ diff --git a/metadata/metadata.go b/metadata/metadata.go index cf6d1b94781c..a2cdcaf12a87 100644 --- a/metadata/metadata.go +++ b/metadata/metadata.go @@ -41,16 +41,17 @@ type MD map[string][]string // New creates an MD from a given key-value map. // // Only the following ASCII characters are allowed in keys: -// - digits: 0-9 -// - uppercase letters: A-Z (normalized to lower) -// - lowercase letters: a-z -// - special characters: -_. +// - digits: 0-9 +// - uppercase letters: A-Z (normalized to lower) +// - lowercase letters: a-z +// - special characters: -_. +// // Uppercase letters are automatically converted to lowercase. // // Keys beginning with "grpc-" are reserved for grpc-internal use only and may // result in errors if set in metadata. func New(m map[string]string) MD { - md := MD{} + md := make(MD, len(m)) for k, val := range m { key := strings.ToLower(k) md[key] = append(md[key], val) @@ -62,10 +63,11 @@ func New(m map[string]string) MD { // Pairs panics if len(kv) is odd. // // Only the following ASCII characters are allowed in keys: -// - digits: 0-9 -// - uppercase letters: A-Z (normalized to lower) -// - lowercase letters: a-z -// - special characters: -_. +// - digits: 0-9 +// - uppercase letters: A-Z (normalized to lower) +// - lowercase letters: a-z +// - special characters: -_. +// // Uppercase letters are automatically converted to lowercase. // // Keys beginning with "grpc-" are reserved for grpc-internal use only and may @@ -74,14 +76,10 @@ func Pairs(kv ...string) MD { if len(kv)%2 == 1 { panic(fmt.Sprintf("metadata: Pairs got the odd number of input pairs for metadata: %d", len(kv))) } - md := MD{} - var key string - for i, s := range kv { - if i%2 == 0 { - key = strings.ToLower(s) - continue - } - md[key] = append(md[key], s) + md := make(MD, len(kv)/2) + for i := 0; i < len(kv); i += 2 { + key := strings.ToLower(kv[i]) + md[key] = append(md[key], kv[i+1]) } return md } @@ -93,16 +91,24 @@ func (md MD) Len() int { // Copy returns a copy of md. func (md MD) Copy() MD { - return Join(md) + out := make(MD, len(md)) + for k, v := range md { + out[k] = copyOf(v) + } + return out } // Get obtains the values for a given key. +// +// k is converted to lowercase before searching in md. func (md MD) Get(k string) []string { k = strings.ToLower(k) return md[k] } // Set sets the value of a given key with a slice of values. +// +// k is converted to lowercase before storing in md. func (md MD) Set(k string, vals ...string) { if len(vals) == 0 { return @@ -111,7 +117,10 @@ func (md MD) Set(k string, vals ...string) { md[k] = vals } -// Append adds the values to key k, not overwriting what was already stored at that key. +// Append adds the values to key k, not overwriting what was already stored at +// that key. +// +// k is converted to lowercase before storing in md. func (md MD) Append(k string, vals ...string) { if len(vals) == 0 { return @@ -120,9 +129,17 @@ func (md MD) Append(k string, vals ...string) { md[k] = append(md[k], vals...) } +// Delete removes the values for a given key k which is converted to lowercase +// before removing it from md. +func (md MD) Delete(k string) { + k = strings.ToLower(k) + delete(md, k) +} + // Join joins any number of mds into a single MD. -// The order of values for each key is determined by the order in which -// the mds containing those values are presented to Join. +// +// The order of values for each key is determined by the order in which the mds +// containing those values are presented to Join. func Join(mds ...MD) MD { out := MD{} for _, md := range mds { @@ -149,8 +166,8 @@ func NewOutgoingContext(ctx context.Context, md MD) context.Context { } // AppendToOutgoingContext returns a new context with the provided kv merged -// with any existing metadata in the context. Please refer to the -// documentation of Pairs for a description of kv. +// with any existing metadata in the context. Please refer to the documentation +// of Pairs for a description of kv. func AppendToOutgoingContext(ctx context.Context, kv ...string) context.Context { if len(kv)%2 == 1 { panic(fmt.Sprintf("metadata: AppendToOutgoingContext got an odd number of input pairs for metadata: %d", len(kv))) @@ -158,25 +175,76 @@ func AppendToOutgoingContext(ctx context.Context, kv ...string) context.Context md, _ := ctx.Value(mdOutgoingKey{}).(rawMD) added := make([][]string, len(md.added)+1) copy(added, md.added) - added[len(added)-1] = make([]string, len(kv)) - copy(added[len(added)-1], kv) + kvCopy := make([]string, 0, len(kv)) + for i := 0; i < len(kv); i += 2 { + kvCopy = append(kvCopy, strings.ToLower(kv[i]), kv[i+1]) + } + added[len(added)-1] = kvCopy return context.WithValue(ctx, mdOutgoingKey{}, rawMD{md: md.md, added: added}) } -// FromIncomingContext returns the incoming metadata in ctx if it exists. The -// returned MD should not be modified. Writing to it may cause races. -// Modification should be made to copies of the returned MD. -func FromIncomingContext(ctx context.Context) (md MD, ok bool) { - md, ok = ctx.Value(mdIncomingKey{}).(MD) - return +// FromIncomingContext returns the incoming metadata in ctx if it exists. +// +// All keys in the returned MD are lowercase. +func FromIncomingContext(ctx context.Context) (MD, bool) { + md, ok := ctx.Value(mdIncomingKey{}).(MD) + if !ok { + return nil, false + } + out := make(MD, len(md)) + for k, v := range md { + // We need to manually convert all keys to lower case, because MD is a + // map, and there's no guarantee that the MD attached to the context is + // created using our helper functions. + key := strings.ToLower(k) + out[key] = copyOf(v) + } + return out, true +} + +// ValueFromIncomingContext returns the metadata value corresponding to the metadata +// key from the incoming metadata if it exists. Key must be lower-case. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. +func ValueFromIncomingContext(ctx context.Context, key string) []string { + md, ok := ctx.Value(mdIncomingKey{}).(MD) + if !ok { + return nil + } + + if v, ok := md[key]; ok { + return copyOf(v) + } + for k, v := range md { + // We need to manually convert all keys to lower case, because MD is a + // map, and there's no guarantee that the MD attached to the context is + // created using our helper functions. + if strings.ToLower(k) == key { + return copyOf(v) + } + } + return nil +} + +// the returned slice must not be modified in place +func copyOf(v []string) []string { + vals := make([]string, len(v)) + copy(vals, v) + return vals } -// FromOutgoingContextRaw returns the un-merged, intermediary contents -// of rawMD. Remember to perform strings.ToLower on the keys. The returned -// MD should not be modified. Writing to it may cause races. Modification -// should be made to copies of the returned MD. +// FromOutgoingContextRaw returns the un-merged, intermediary contents of rawMD. +// +// Remember to perform strings.ToLower on the keys, for both the returned MD (MD +// is a map, there's no guarantee it's created using our helper functions) and +// the extra kv pairs (AppendToOutgoingContext doesn't turn them into +// lowercase). // -// This is intended for gRPC-internal use ONLY. +// This is intended for gRPC-internal use ONLY. Users should use +// FromOutgoingContext instead. func FromOutgoingContextRaw(ctx context.Context) (MD, [][]string, bool) { raw, ok := ctx.Value(mdOutgoingKey{}).(rawMD) if !ok { @@ -186,21 +254,39 @@ func FromOutgoingContextRaw(ctx context.Context) (MD, [][]string, bool) { return raw.md, raw.added, true } -// FromOutgoingContext returns the outgoing metadata in ctx if it exists. The -// returned MD should not be modified. Writing to it may cause races. -// Modification should be made to copies of the returned MD. +// FromOutgoingContext returns the outgoing metadata in ctx if it exists. +// +// All keys in the returned MD are lowercase. func FromOutgoingContext(ctx context.Context) (MD, bool) { raw, ok := ctx.Value(mdOutgoingKey{}).(rawMD) if !ok { return nil, false } - mds := make([]MD, 0, len(raw.added)+1) - mds = append(mds, raw.md) - for _, vv := range raw.added { - mds = append(mds, Pairs(vv...)) + mdSize := len(raw.md) + for i := range raw.added { + mdSize += len(raw.added[i]) / 2 + } + + out := make(MD, mdSize) + for k, v := range raw.md { + // We need to manually convert all keys to lower case, because MD is a + // map, and there's no guarantee that the MD attached to the context is + // created using our helper functions. + key := strings.ToLower(k) + out[key] = copyOf(v) + } + for _, added := range raw.added { + if len(added)%2 == 1 { + panic(fmt.Sprintf("metadata: FromOutgoingContext got an odd number of input pairs for metadata: %d", len(added))) + } + + for i := 0; i < len(added); i += 2 { + key := strings.ToLower(added[i]) + out[key] = append(out[key], added[i+1]) + } } - return Join(mds...), ok + return out, ok } type rawMD struct { diff --git a/metadata/metadata_test.go b/metadata/metadata_test.go index 84845d5b1278..9277f2d6c84f 100644 --- a/metadata/metadata_test.go +++ b/metadata/metadata_test.go @@ -23,10 +23,13 @@ import ( "reflect" "strconv" "testing" + "time" "google.golang.org/grpc/internal/grpctest" ) +const defaultTestTimeout = 10 * time.Second + type s struct { grpctest.Tester } @@ -166,9 +169,77 @@ func (s) TestAppend(t *testing.T) { } } +func (s) TestDelete(t *testing.T) { + for _, test := range []struct { + md MD + deleteKey string + want MD + }{ + { + md: Pairs("My-Optional-Header", "42"), + deleteKey: "My-Optional-Header", + want: Pairs(), + }, + { + md: Pairs("My-Optional-Header", "42"), + deleteKey: "Other-Key", + want: Pairs("my-optional-header", "42"), + }, + { + md: Pairs("My-Optional-Header", "42"), + deleteKey: "my-OptIoNal-HeAder", + want: Pairs(), + }, + } { + test.md.Delete(test.deleteKey) + if !reflect.DeepEqual(test.md, test.want) { + t.Errorf("value of metadata is %v, want %v", test.md, test.want) + } + } +} + +func (s) TestValueFromIncomingContext(t *testing.T) { + md := Pairs( + "X-My-Header-1", "42", + "X-My-Header-2", "43-1", + "X-My-Header-2", "43-2", + "x-my-header-3", "44", + ) + ctx := NewIncomingContext(context.Background(), md) + + for _, test := range []struct { + key string + want []string + }{ + { + key: "x-my-header-1", + want: []string{"42"}, + }, + { + key: "x-my-header-2", + want: []string{"43-1", "43-2"}, + }, + { + key: "x-my-header-3", + want: []string{"44"}, + }, + { + key: "x-unknown", + want: nil, + }, + } { + v := ValueFromIncomingContext(ctx, test.key) + if !reflect.DeepEqual(v, test.want) { + t.Errorf("value of metadata is %v, want %v", v, test.want) + } + } +} + func (s) TestAppendToOutgoingContext(t *testing.T) { // Pre-existing metadata - ctx := NewOutgoingContext(context.Background(), Pairs("k1", "v1", "k2", "v2")) + tCtx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctx := NewOutgoingContext(tCtx, Pairs("k1", "v1", "k2", "v2")) ctx = AppendToOutgoingContext(ctx, "k1", "v3") ctx = AppendToOutgoingContext(ctx, "k1", "v4") md, ok := FromOutgoingContext(ctx) @@ -181,7 +252,7 @@ func (s) TestAppendToOutgoingContext(t *testing.T) { } // No existing metadata - ctx = AppendToOutgoingContext(context.Background(), "k1", "v1") + ctx = AppendToOutgoingContext(tCtx, "k1", "v1") md, ok = FromOutgoingContext(ctx) if !ok { t.Errorf("Expected MD to exist in ctx, but got none") @@ -193,7 +264,8 @@ func (s) TestAppendToOutgoingContext(t *testing.T) { } func (s) TestAppendToOutgoingContext_Repeated(t *testing.T) { - ctx := context.Background() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() for i := 0; i < 100; i = i + 2 { ctx1 := AppendToOutgoingContext(ctx, "k", strconv.Itoa(i)) @@ -213,7 +285,9 @@ func (s) TestAppendToOutgoingContext_Repeated(t *testing.T) { func (s) TestAppendToOutgoingContext_FromKVSlice(t *testing.T) { const k, v = "a", "b" kv := []string{k, v} - ctx := AppendToOutgoingContext(context.Background(), kv...) + tCtx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctx := AppendToOutgoingContext(tCtx, kv...) md, _ := FromOutgoingContext(ctx) if md[k][0] != v { t.Fatalf("md[%q] = %q; want %q", k, md[k], v) @@ -230,7 +304,8 @@ func Benchmark_AddingMetadata_ContextManipulationApproach(b *testing.B) { // TODO: Add in N=1-100 tests once Go1.6 support is removed. const num = 10 for n := 0; n < b.N; n++ { - ctx := context.Background() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() for i := 0; i < num; i++ { md, _ := FromOutgoingContext(ctx) NewOutgoingContext(ctx, Join(Pairs("k1", "v1", "k2", "v2"), md)) @@ -241,8 +316,9 @@ func Benchmark_AddingMetadata_ContextManipulationApproach(b *testing.B) { // Newer/faster approach to adding metadata to context func BenchmarkAppendToOutgoingContext(b *testing.B) { const num = 10 + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() for n := 0; n < b.N; n++ { - ctx := context.Background() for i := 0; i < num; i++ { ctx = AppendToOutgoingContext(ctx, "k1", "v1", "k2", "v2") } @@ -250,7 +326,8 @@ func BenchmarkAppendToOutgoingContext(b *testing.B) { } func BenchmarkFromOutgoingContext(b *testing.B) { - ctx := context.Background() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() ctx = NewOutgoingContext(ctx, MD{"k3": {"v3", "v4"}}) ctx = AppendToOutgoingContext(ctx, "k1", "v1", "k2", "v2") @@ -258,3 +335,21 @@ func BenchmarkFromOutgoingContext(b *testing.B) { FromOutgoingContext(ctx) } } + +func BenchmarkFromIncomingContext(b *testing.B) { + md := Pairs("X-My-Header-1", "42") + ctx := NewIncomingContext(context.Background(), md) + b.ResetTimer() + for n := 0; n < b.N; n++ { + FromIncomingContext(ctx) + } +} + +func BenchmarkValueFromIncomingContext(b *testing.B) { + md := Pairs("X-My-Header-1", "42") + ctx := NewIncomingContext(context.Background(), md) + b.ResetTimer() + for n := 0; n < b.N; n++ { + ValueFromIncomingContext(ctx, "x-my-header-1") + } +} diff --git a/orca/call_metrics.go b/orca/call_metrics.go new file mode 100644 index 000000000000..157dad49c657 --- /dev/null +++ b/orca/call_metrics.go @@ -0,0 +1,196 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package orca + +import ( + "context" + "sync" + + "google.golang.org/grpc" + grpcinternal "google.golang.org/grpc/internal" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/orca/internal" + "google.golang.org/protobuf/proto" +) + +// CallMetricsRecorder allows a service method handler to record per-RPC +// metrics. It contains all utilization-based metrics from +// ServerMetricsRecorder as well as additional request cost metrics. +type CallMetricsRecorder interface { + ServerMetricsRecorder + + // SetRequestCost sets the relevant server metric. + SetRequestCost(name string, val float64) + // DeleteRequestCost deletes the relevant server metric to prevent it + // from being sent. + DeleteRequestCost(name string) + + // SetNamedMetric sets the relevant server metric. + SetNamedMetric(name string, val float64) + // DeleteNamedMetric deletes the relevant server metric to prevent it + // from being sent. + DeleteNamedMetric(name string) +} + +type callMetricsRecorderCtxKey struct{} + +// CallMetricsRecorderFromContext returns the RPC-specific custom metrics +// recorder embedded in the provided RPC context. +// +// Returns nil if no custom metrics recorder is found in the provided context, +// which will be the case when custom metrics reporting is not enabled. +func CallMetricsRecorderFromContext(ctx context.Context) CallMetricsRecorder { + rw, ok := ctx.Value(callMetricsRecorderCtxKey{}).(*recorderWrapper) + if !ok { + return nil + } + return rw.recorder() +} + +// recorderWrapper is a wrapper around a CallMetricsRecorder to ensure that +// concurrent calls to CallMetricsRecorderFromContext() results in only one +// allocation of the underlying metrics recorder, while also allowing for lazy +// initialization of the recorder itself. +type recorderWrapper struct { + once sync.Once + r CallMetricsRecorder + smp ServerMetricsProvider +} + +func (rw *recorderWrapper) recorder() CallMetricsRecorder { + rw.once.Do(func() { + rw.r = newServerMetricsRecorder() + }) + return rw.r +} + +// setTrailerMetadata adds a trailer metadata entry with key being set to +// `internal.TrailerMetadataKey` and value being set to the binary-encoded +// orca.OrcaLoadReport protobuf message. +// +// This function is called from the unary and streaming interceptors defined +// above. Any errors encountered here are not propagated to the caller because +// they are ignored there. Hence we simply log any errors encountered here at +// warning level, and return nothing. +func (rw *recorderWrapper) setTrailerMetadata(ctx context.Context) { + var sm *ServerMetrics + if rw.smp != nil { + sm = rw.smp.ServerMetrics() + sm.merge(rw.r.ServerMetrics()) + } else { + sm = rw.r.ServerMetrics() + } + + b, err := proto.Marshal(sm.toLoadReportProto()) + if err != nil { + logger.Warningf("Failed to marshal load report: %v", err) + return + } + if err := grpc.SetTrailer(ctx, metadata.Pairs(internal.TrailerMetadataKey, string(b))); err != nil { + logger.Warningf("Failed to set trailer metadata: %v", err) + } +} + +var joinServerOptions = grpcinternal.JoinServerOptions.(func(...grpc.ServerOption) grpc.ServerOption) + +// CallMetricsServerOption returns a server option which enables the reporting +// of per-RPC custom backend metrics for unary and streaming RPCs. +// +// Server applications interested in injecting custom backend metrics should +// pass the server option returned from this function as the first argument to +// grpc.NewServer(). +// +// Subsequently, server RPC handlers can retrieve a reference to the RPC +// specific custom metrics recorder [CallMetricsRecorder] to be used, via a call +// to CallMetricsRecorderFromContext(), and inject custom metrics at any time +// during the RPC lifecycle. +// +// The injected custom metrics will be sent as part of trailer metadata, as a +// binary-encoded [ORCA LoadReport] protobuf message, with the metadata key +// being set be "endpoint-load-metrics-bin". +// +// If a non-nil ServerMetricsProvider is provided, the gRPC server will +// transmit the metrics it provides, overwritten by any per-RPC metrics given +// to the CallMetricsRecorder. A ServerMetricsProvider is typically obtained +// by calling NewServerMetricsRecorder. +// +// [ORCA LoadReport]: https://github.com/cncf/xds/blob/main/xds/data/orca/v3/orca_load_report.proto#L15 +func CallMetricsServerOption(smp ServerMetricsProvider) grpc.ServerOption { + return joinServerOptions(grpc.ChainUnaryInterceptor(unaryInt(smp)), grpc.ChainStreamInterceptor(streamInt(smp))) +} + +func unaryInt(smp ServerMetricsProvider) func(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + return func(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + // We don't allocate the metric recorder here. It will be allocated the + // first time the user calls CallMetricsRecorderFromContext(). + rw := &recorderWrapper{smp: smp} + ctxWithRecorder := newContextWithRecorderWrapper(ctx, rw) + + resp, err := handler(ctxWithRecorder, req) + + // It is safe to access the underlying metric recorder inside the wrapper at + // this point, as the user's RPC handler is done executing, and therefore + // there will be no more calls to CallMetricsRecorderFromContext(), which is + // where the metric recorder is lazy allocated. + if rw.r != nil { + rw.setTrailerMetadata(ctx) + } + return resp, err + } +} + +func streamInt(smp ServerMetricsProvider) func(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + return func(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + // We don't allocate the metric recorder here. It will be allocated the + // first time the user calls CallMetricsRecorderFromContext(). + rw := &recorderWrapper{smp: smp} + ws := &wrappedStream{ + ServerStream: ss, + ctx: newContextWithRecorderWrapper(ss.Context(), rw), + } + + err := handler(srv, ws) + + // It is safe to access the underlying metric recorder inside the wrapper at + // this point, as the user's RPC handler is done executing, and therefore + // there will be no more calls to CallMetricsRecorderFromContext(), which is + // where the metric recorder is lazy allocated. + if rw.r != nil { + rw.setTrailerMetadata(ss.Context()) + } + return err + } +} + +func newContextWithRecorderWrapper(ctx context.Context, r *recorderWrapper) context.Context { + return context.WithValue(ctx, callMetricsRecorderCtxKey{}, r) +} + +// wrappedStream wraps the grpc.ServerStream received by the streaming +// interceptor. Overrides only the Context() method to return a context which +// contains a reference to the CallMetricsRecorder corresponding to this +// stream. +type wrappedStream struct { + grpc.ServerStream + ctx context.Context +} + +func (w *wrappedStream) Context() context.Context { + return w.ctx +} diff --git a/orca/call_metrics_test.go b/orca/call_metrics_test.go new file mode 100644 index 000000000000..474ac710b55b --- /dev/null +++ b/orca/call_metrics_test.go @@ -0,0 +1,287 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package orca_test + +import ( + "context" + "errors" + "io" + "testing" + + "github.com/golang/protobuf/proto" + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/orca" + "google.golang.org/grpc/orca/internal" + + v3orcapb "github.com/cncf/xds/go/xds/data/orca/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +// TestE2ECallMetricsUnary tests the injection of custom backend metrics from +// the server application for a unary RPC, and verifies that expected load +// reports are received at the client. +func (s) TestE2ECallMetricsUnary(t *testing.T) { + tests := []struct { + desc string + injectMetrics bool + wantProto *v3orcapb.OrcaLoadReport + }{ + { + desc: "with custom backend metrics", + injectMetrics: true, + wantProto: &v3orcapb.OrcaLoadReport{ + CpuUtilization: 1.0, + MemUtilization: 0.9, + RequestCost: map[string]float64{"queryCost": 25.0}, + Utilization: map[string]float64{"queueSize": 0.75}, + }, + }, + { + desc: "with no custom backend metrics", + injectMetrics: false, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + // A server option to enable reporting of per-call backend metrics. + smr := orca.NewServerMetricsRecorder() + callMetricsServerOption := orca.CallMetricsServerOption(smr) + smr.SetCPUUtilization(1.0) + + // An interceptor to injects custom backend metrics, added only when + // the injectMetrics field in the test is set. + injectingInterceptor := func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp any, err error) { + recorder := orca.CallMetricsRecorderFromContext(ctx) + if recorder == nil { + err := errors.New("Failed to retrieve per-RPC custom metrics recorder from the RPC context") + t.Error(err) + return nil, err + } + recorder.SetMemoryUtilization(0.9) + // This value will be overwritten by a write to the same metric + // from the server handler. + recorder.SetNamedUtilization("queueSize", 1.0) + return handler(ctx, req) + } + + // A stub server whose unary handler injects custom metrics, if the + // injectMetrics field in the test is set. It overwrites one of the + // values injected above, by the interceptor. + srv := stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { + if !test.injectMetrics { + return &testpb.Empty{}, nil + } + recorder := orca.CallMetricsRecorderFromContext(ctx) + if recorder == nil { + err := errors.New("Failed to retrieve per-RPC custom metrics recorder from the RPC context") + t.Error(err) + return nil, err + } + recorder.SetRequestCost("queryCost", 25.0) + recorder.SetNamedUtilization("queueSize", 0.75) + return &testpb.Empty{}, nil + }, + } + + // Start the stub server with the appropriate server options. + sopts := []grpc.ServerOption{callMetricsServerOption} + if test.injectMetrics { + sopts = append(sopts, grpc.ChainUnaryInterceptor(injectingInterceptor)) + } + if err := srv.StartServer(sopts...); err != nil { + t.Fatalf("Failed to start server: %v", err) + } + defer srv.Stop() + + // Dial the stub server. + cc, err := grpc.Dial(srv.Address, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.Dial(%s) failed: %v", srv.Address, err) + } + defer cc.Close() + + // Make a unary RPC and expect the trailer metadata to contain the custom + // backend metrics as an ORCA LoadReport protobuf message. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + client := testgrpc.NewTestServiceClient(cc) + trailer := metadata.MD{} + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Trailer(&trailer)); err != nil { + t.Fatalf("EmptyCall failed: %v", err) + } + + gotProto, err := internal.ToLoadReport(trailer) + if err != nil { + t.Fatalf("When retrieving load report, got error: %v, want: ", err) + } + if test.wantProto != nil && !cmp.Equal(gotProto, test.wantProto, cmp.Comparer(proto.Equal)) { + t.Fatalf("Received load report in trailer: %s, want: %s", pretty.ToJSON(gotProto), pretty.ToJSON(test.wantProto)) + } + }) + } +} + +// TestE2ECallMetricsStreaming tests the injection of custom backend metrics +// from the server application for a streaming RPC, and verifies that expected +// load reports are received at the client. +func (s) TestE2ECallMetricsStreaming(t *testing.T) { + tests := []struct { + desc string + injectMetrics bool + wantProto *v3orcapb.OrcaLoadReport + }{ + { + desc: "with custom backend metrics", + injectMetrics: true, + wantProto: &v3orcapb.OrcaLoadReport{ + CpuUtilization: 1.0, + MemUtilization: 0.5, + RequestCost: map[string]float64{"queryCost": 0.25}, + Utilization: map[string]float64{"queueSize": 0.75}, + }, + }, + { + desc: "with no custom backend metrics", + injectMetrics: false, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + // A server option to enable reporting of per-call backend metrics. + smr := orca.NewServerMetricsRecorder() + callMetricsServerOption := orca.CallMetricsServerOption(smr) + smr.SetCPUUtilization(1.0) + + // An interceptor which injects custom backend metrics, added only + // when the injectMetrics field in the test is set. + injectingInterceptor := func(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + recorder := orca.CallMetricsRecorderFromContext(ss.Context()) + if recorder == nil { + err := errors.New("Failed to retrieve per-RPC custom metrics recorder from the RPC context") + t.Error(err) + return err + } + recorder.SetMemoryUtilization(0.5) + // This value will be overwritten by a write to the same metric + // from the server handler. + recorder.SetNamedUtilization("queueSize", 1.0) + return handler(srv, ss) + } + + // A stub server whose streaming handler injects custom metrics, if + // the injectMetrics field in the test is set. It overwrites one of + // the values injected above, by the interceptor. + srv := stubserver.StubServer{ + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + if test.injectMetrics { + recorder := orca.CallMetricsRecorderFromContext(stream.Context()) + if recorder == nil { + err := errors.New("Failed to retrieve per-RPC custom metrics recorder from the RPC context") + t.Error(err) + return err + } + recorder.SetRequestCost("queryCost", 0.25) + recorder.SetNamedUtilization("queueSize", 0.75) + } + + // Streaming implementation replies with a dummy response until the + // client closes the stream (in which case it will see an io.EOF), + // or an error occurs while reading/writing messages. + for { + _, err := stream.Recv() + if err == io.EOF { + return nil + } + if err != nil { + return err + } + payload := &testpb.Payload{Body: make([]byte, 32)} + if err := stream.Send(&testpb.StreamingOutputCallResponse{Payload: payload}); err != nil { + return err + } + } + }, + } + + // Start the stub server with the appropriate server options. + sopts := []grpc.ServerOption{callMetricsServerOption} + if test.injectMetrics { + sopts = append(sopts, grpc.ChainStreamInterceptor(injectingInterceptor)) + } + if err := srv.StartServer(sopts...); err != nil { + t.Fatalf("Failed to start server: %v", err) + } + defer srv.Stop() + + // Dial the stub server. + cc, err := grpc.Dial(srv.Address, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.Dial(%s) failed: %v", srv.Address, err) + } + defer cc.Close() + + // Start the full duplex streaming RPC. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + tc := testgrpc.NewTestServiceClient(cc) + stream, err := tc.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("FullDuplexCall failed: %v", err) + } + + // Send one request to the server. + payload := &testpb.Payload{Body: make([]byte, 32)} + req := &testpb.StreamingOutputCallRequest{Payload: payload} + if err := stream.Send(req); err != nil { + t.Fatalf("stream.Send() failed: %v", err) + } + // Read one reply from the server. + if _, err := stream.Recv(); err != nil { + t.Fatalf("stream.Recv() failed: %v", err) + } + // Close the sending side. + if err := stream.CloseSend(); err != nil { + t.Fatalf("stream.CloseSend() failed: %v", err) + } + // Make sure it is safe to read the trailer. + for { + if _, err := stream.Recv(); err != nil { + break + } + } + + gotProto, err := internal.ToLoadReport(stream.Trailer()) + if err != nil { + t.Fatalf("When retrieving load report, got error: %v, want: ", err) + } + if test.wantProto != nil && !cmp.Equal(gotProto, test.wantProto, cmp.Comparer(proto.Equal)) { + t.Fatalf("Received load report in trailer: %s, want: %s", pretty.ToJSON(gotProto), pretty.ToJSON(test.wantProto)) + } + }) + } +} diff --git a/orca/internal/internal.go b/orca/internal/internal.go new file mode 100644 index 000000000000..d1425c3e7164 --- /dev/null +++ b/orca/internal/internal.go @@ -0,0 +1,71 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package internal contains orca-internal code, for testing purposes and to +// avoid polluting the godoc of the top-level orca package. +package internal + +import ( + "errors" + "fmt" + + ibackoff "google.golang.org/grpc/internal/backoff" + "google.golang.org/grpc/metadata" + "google.golang.org/protobuf/proto" + + v3orcapb "github.com/cncf/xds/go/xds/data/orca/v3" +) + +// AllowAnyMinReportingInterval prevents clamping of the MinReportingInterval +// configured via ServiceOptions, to a minimum of 30s. +// +// For testing purposes only. +var AllowAnyMinReportingInterval any // func(*ServiceOptions) + +// DefaultBackoffFunc is used by the producer to control its backoff behavior. +// +// For testing purposes only. +var DefaultBackoffFunc = ibackoff.DefaultExponential.Backoff + +// TrailerMetadataKey is the key in which the per-call backend metrics are +// transmitted. +const TrailerMetadataKey = "endpoint-load-metrics-bin" + +// ToLoadReport unmarshals a binary encoded [ORCA LoadReport] protobuf message +// from md and returns the corresponding struct. The load report is expected to +// be stored as the value for key "endpoint-load-metrics-bin". +// +// If no load report was found in the provided metadata, if multiple load +// reports are found, or if the load report found cannot be parsed, an error is +// returned. +// +// [ORCA LoadReport]: (https://github.com/cncf/xds/blob/main/xds/data/orca/v3/orca_load_report.proto#L15) +func ToLoadReport(md metadata.MD) (*v3orcapb.OrcaLoadReport, error) { + vs := md.Get(TrailerMetadataKey) + if len(vs) == 0 { + return nil, nil + } + if len(vs) != 1 { + return nil, errors.New("multiple orca load reports found in provided metadata") + } + ret := new(v3orcapb.OrcaLoadReport) + if err := proto.Unmarshal([]byte(vs[0]), ret); err != nil { + return nil, fmt.Errorf("failed to unmarshal load report found in metadata: %v", err) + } + return ret, nil +} diff --git a/orca/orca.go b/orca/orca.go new file mode 100644 index 000000000000..d0cb3720c8ac --- /dev/null +++ b/orca/orca.go @@ -0,0 +1,57 @@ +/* + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package orca implements Open Request Cost Aggregation, which is an open +// standard for request cost aggregation and reporting by backends and the +// corresponding aggregation of such reports by L7 load balancers (such as +// Envoy) on the data plane. In a proxyless world with gRPC enabled +// applications, aggregation of such reports will be done by the gRPC client. +// +// # Experimental +// +// Notice: All APIs is this package are EXPERIMENTAL and may be changed or +// removed in a later release. +package orca + +import ( + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal/balancerload" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/orca/internal" +) + +var logger = grpclog.Component("orca-backend-metrics") + +// loadParser implements the Parser interface defined in `internal/balancerload` +// package. This interface is used by the client stream to parse load reports +// sent by the server in trailer metadata. The parsed loads are then sent to +// balancers via balancer.DoneInfo. +// +// The grpc package cannot directly call toLoadReport() as that would cause an +// import cycle. Hence this roundabout method is used. +type loadParser struct{} + +func (loadParser) Parse(md metadata.MD) any { + lr, err := internal.ToLoadReport(md) + if err != nil { + logger.Infof("Parse failed: %v", err) + } + return lr +} + +func init() { + balancerload.SetParser(loadParser{}) +} diff --git a/orca/orca_test.go b/orca/orca_test.go new file mode 100644 index 000000000000..4f85e7b01592 --- /dev/null +++ b/orca/orca_test.go @@ -0,0 +1,102 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package orca_test + +import ( + "testing" + "time" + + "github.com/golang/protobuf/proto" + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/orca/internal" + + v3orcapb "github.com/cncf/xds/go/xds/data/orca/v3" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +const defaultTestTimeout = 5 * time.Second + +func (s) TestToLoadReport(t *testing.T) { + goodReport := &v3orcapb.OrcaLoadReport{ + CpuUtilization: 1.0, + MemUtilization: 50.0, + RequestCost: map[string]float64{"queryCost": 25.0}, + Utilization: map[string]float64{"queueSize": 75.0}, + } + tests := []struct { + name string + md metadata.MD + want *v3orcapb.OrcaLoadReport + wantErr bool + }{ + { + name: "no load report in metadata", + md: metadata.MD{}, + wantErr: false, + }, + { + name: "badly marshaled load report", + md: func() metadata.MD { + return metadata.Pairs("endpoint-load-metrics-bin", string("foo-bar")) + }(), + wantErr: true, + }, + { + name: "multiple load reports", + md: func() metadata.MD { + b, _ := proto.Marshal(goodReport) + return metadata.Pairs("endpoint-load-metrics-bin", string(b), "endpoint-load-metrics-bin", string(b)) + }(), + wantErr: true, + }, + { + name: "good load report", + md: func() metadata.MD { + b, _ := proto.Marshal(goodReport) + return metadata.Pairs("endpoint-load-metrics-bin", string(b)) + }(), + want: goodReport, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := internal.ToLoadReport(test.md) + if (err != nil) != test.wantErr { + t.Fatalf("orca.ToLoadReport(%v) = %v, wantErr: %v", test.md, err, test.wantErr) + } + if test.wantErr { + return + } + if !cmp.Equal(got, test.want, cmp.Comparer(proto.Equal)) { + t.Fatalf("Extracted load report from metadata: %s, want: %s", pretty.ToJSON(got), pretty.ToJSON(test.want)) + } + }) + } +} diff --git a/orca/producer.go b/orca/producer.go new file mode 100644 index 000000000000..2d58725547fc --- /dev/null +++ b/orca/producer.go @@ -0,0 +1,241 @@ +/* + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package orca + +import ( + "context" + "sync" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/orca/internal" + "google.golang.org/grpc/status" + + v3orcapb "github.com/cncf/xds/go/xds/data/orca/v3" + v3orcaservicegrpc "github.com/cncf/xds/go/xds/service/orca/v3" + v3orcaservicepb "github.com/cncf/xds/go/xds/service/orca/v3" + "google.golang.org/protobuf/types/known/durationpb" +) + +type producerBuilder struct{} + +// Build constructs and returns a producer and its cleanup function +func (*producerBuilder) Build(cci any) (balancer.Producer, func()) { + p := &producer{ + client: v3orcaservicegrpc.NewOpenRcaServiceClient(cci.(grpc.ClientConnInterface)), + intervals: make(map[time.Duration]int), + listeners: make(map[OOBListener]struct{}), + backoff: internal.DefaultBackoffFunc, + } + return p, func() { + <-p.stopped + } +} + +var producerBuilderSingleton = &producerBuilder{} + +// OOBListener is used to receive out-of-band load reports as they arrive. +type OOBListener interface { + // OnLoadReport is called when a load report is received. + OnLoadReport(*v3orcapb.OrcaLoadReport) +} + +// OOBListenerOptions contains options to control how an OOBListener is called. +type OOBListenerOptions struct { + // ReportInterval specifies how often to request the server to provide a + // load report. May be provided less frequently if the server requires a + // longer interval, or may be provided more frequently if another + // subscriber requests a shorter interval. + ReportInterval time.Duration +} + +// RegisterOOBListener registers an out-of-band load report listener on sc. +// Any OOBListener may only be registered once per subchannel at a time. The +// returned stop function must be called when no longer needed. Do not +// register a single OOBListener more than once per SubConn. +func RegisterOOBListener(sc balancer.SubConn, l OOBListener, opts OOBListenerOptions) (stop func()) { + pr, close := sc.GetOrBuildProducer(producerBuilderSingleton) + p := pr.(*producer) + + p.registerListener(l, opts.ReportInterval) + + // TODO: When we can register for SubConn state updates, automatically call + // stop() on SHUTDOWN. + + // If stop is called multiple times, prevent it from having any effect on + // subsequent calls. + return grpcsync.OnceFunc(func() { + p.unregisterListener(l, opts.ReportInterval) + close() + }) +} + +type producer struct { + client v3orcaservicegrpc.OpenRcaServiceClient + + // backoff is called between stream attempts to determine how long to delay + // to avoid overloading a server experiencing problems. The attempt count + // is incremented when stream errors occur and is reset when the stream + // reports a result. + backoff func(int) time.Duration + + mu sync.Mutex + intervals map[time.Duration]int // map from interval time to count of listeners requesting that time + listeners map[OOBListener]struct{} // set of registered listeners + minInterval time.Duration + stop func() // stops the current run goroutine + stopped chan struct{} // closed when the run goroutine exits +} + +// registerListener adds the listener and its requested report interval to the +// producer. +func (p *producer) registerListener(l OOBListener, interval time.Duration) { + p.mu.Lock() + defer p.mu.Unlock() + + p.listeners[l] = struct{}{} + p.intervals[interval]++ + if len(p.listeners) == 1 || interval < p.minInterval { + p.minInterval = interval + p.updateRunLocked() + } +} + +// registerListener removes the listener and its requested report interval to +// the producer. +func (p *producer) unregisterListener(l OOBListener, interval time.Duration) { + p.mu.Lock() + defer p.mu.Unlock() + + delete(p.listeners, l) + p.intervals[interval]-- + if p.intervals[interval] == 0 { + delete(p.intervals, interval) + + if p.minInterval == interval { + p.recomputeMinInterval() + p.updateRunLocked() + } + } +} + +// recomputeMinInterval sets p.minInterval to the minimum key's value in +// p.intervals. +func (p *producer) recomputeMinInterval() { + first := true + for interval := range p.intervals { + if first || interval < p.minInterval { + p.minInterval = interval + first = false + } + } +} + +// updateRunLocked is called whenever the run goroutine needs to be started / +// stopped / restarted due to: 1. the initial listener being registered, 2. the +// final listener being unregistered, or 3. the minimum registered interval +// changing. +func (p *producer) updateRunLocked() { + if p.stop != nil { + p.stop() + p.stop = nil + } + if len(p.listeners) > 0 { + var ctx context.Context + ctx, p.stop = context.WithCancel(context.Background()) + p.stopped = make(chan struct{}) + go p.run(ctx, p.stopped, p.minInterval) + } +} + +// run manages the ORCA OOB stream on the subchannel. +func (p *producer) run(ctx context.Context, done chan struct{}, interval time.Duration) { + defer close(done) + + backoffAttempt := 0 + backoffTimer := time.NewTimer(0) + for ctx.Err() == nil { + select { + case <-backoffTimer.C: + case <-ctx.Done(): + return + } + + resetBackoff, err := p.runStream(ctx, interval) + + if resetBackoff { + backoffTimer.Reset(0) + backoffAttempt = 0 + } else { + backoffTimer.Reset(p.backoff(backoffAttempt)) + backoffAttempt++ + } + + switch { + case err == nil: + // No error was encountered; restart the stream. + case ctx.Err() != nil: + // Producer was stopped; exit immediately and without logging an + // error. + return + case status.Code(err) == codes.Unimplemented: + // Unimplemented; do not retry. + logger.Error("Server doesn't support ORCA OOB load reporting protocol; not listening for load reports.") + return + case status.Code(err) == codes.Unavailable, status.Code(err) == codes.Canceled: + // TODO: these codes should ideally log an error, too, but for now + // we receive them when shutting down the ClientConn (Unavailable + // if the stream hasn't started yet, and Canceled if it happens + // mid-stream). Once we can determine the state or ensure the + // producer is stopped before the stream ends, we can log an error + // when it's not a natural shutdown. + default: + // Log all other errors. + logger.Error("Received unexpected stream error:", err) + } + } +} + +// runStream runs a single stream on the subchannel and returns the resulting +// error, if any, and whether or not the run loop should reset the backoff +// timer to zero or advance it. +func (p *producer) runStream(ctx context.Context, interval time.Duration) (resetBackoff bool, err error) { + streamCtx, cancel := context.WithCancel(ctx) + defer cancel() + stream, err := p.client.StreamCoreMetrics(streamCtx, &v3orcaservicepb.OrcaLoadReportRequest{ + ReportInterval: durationpb.New(interval), + }) + if err != nil { + return false, err + } + + for { + report, err := stream.Recv() + if err != nil { + return resetBackoff, err + } + resetBackoff = true + p.mu.Lock() + for l := range p.listeners { + l.OnLoadReport(report) + } + p.mu.Unlock() + } +} diff --git a/orca/producer_test.go b/orca/producer_test.go new file mode 100644 index 000000000000..ecaf57e0631e --- /dev/null +++ b/orca/producer_test.go @@ -0,0 +1,547 @@ +/* + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package orca_test + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + "github.com/golang/protobuf/proto" + "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/roundrobin" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/orca" + "google.golang.org/grpc/orca/internal" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/status" + + v3orcapb "github.com/cncf/xds/go/xds/data/orca/v3" + v3orcaservicegrpc "github.com/cncf/xds/go/xds/service/orca/v3" + v3orcaservicepb "github.com/cncf/xds/go/xds/service/orca/v3" +) + +// customLBB wraps a round robin LB policy but provides a ClientConn wrapper to +// add an ORCA OOB report producer for all created SubConns. +type customLBB struct{} + +func (customLBB) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { + return balancer.Get(roundrobin.Name).Build(&ccWrapper{ClientConn: cc}, opts) +} + +func (customLBB) Name() string { return "customLB" } + +func init() { + balancer.Register(customLBB{}) +} + +type ccWrapper struct { + balancer.ClientConn +} + +func (w *ccWrapper) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) { + if len(addrs) != 1 { + panic(fmt.Sprintf("got addrs=%v; want len(addrs) == 1", addrs)) + } + sc, err := w.ClientConn.NewSubConn(addrs, opts) + if err != nil { + return sc, err + } + l := getListenerInfo(addrs[0]) + l.listener.cleanup = orca.RegisterOOBListener(sc, l.listener, l.opts) + l.sc = sc + return sc, nil +} + +// listenerInfo is stored in an address's attributes to allow ORCA +// listeners to be registered on subconns created for that address. +type listenerInfo struct { + listener *testOOBListener + opts orca.OOBListenerOptions + sc balancer.SubConn // Set by the LB policy +} + +type listenerInfoKey struct{} + +func setListenerInfo(addr resolver.Address, l *listenerInfo) resolver.Address { + addr.Attributes = addr.Attributes.WithValue(listenerInfoKey{}, l) + return addr +} + +func getListenerInfo(addr resolver.Address) *listenerInfo { + return addr.Attributes.Value(listenerInfoKey{}).(*listenerInfo) +} + +// testOOBListener is a simple listener that pushes load reports to a channel. +type testOOBListener struct { + cleanup func() + loadReportCh chan *v3orcapb.OrcaLoadReport +} + +func newTestOOBListener() *testOOBListener { + return &testOOBListener{cleanup: func() {}, loadReportCh: make(chan *v3orcapb.OrcaLoadReport)} +} + +func (t *testOOBListener) Stop() { t.cleanup() } + +func (t *testOOBListener) OnLoadReport(r *v3orcapb.OrcaLoadReport) { + t.loadReportCh <- r +} + +// TestProducer is a basic, end-to-end style test of an LB policy with an +// OOBListener communicating with a server with an ORCA service. +func (s) TestProducer(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Use a fixed backoff for stream recreation. + oldBackoff := internal.DefaultBackoffFunc + internal.DefaultBackoffFunc = func(int) time.Duration { return 10 * time.Millisecond } + defer func() { internal.DefaultBackoffFunc = oldBackoff }() + + // Initialize listener for our ORCA server. + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatal(err) + } + + // Register the OpenRCAService with a very short metrics reporting interval. + const shortReportingInterval = 50 * time.Millisecond + smr := orca.NewServerMetricsRecorder() + opts := orca.ServiceOptions{MinReportingInterval: shortReportingInterval, ServerMetricsProvider: smr} + internal.AllowAnyMinReportingInterval.(func(*orca.ServiceOptions))(&opts) + s := grpc.NewServer() + if err := orca.Register(s, opts); err != nil { + t.Fatalf("orca.Register failed: %v", err) + } + go s.Serve(lis) + defer s.Stop() + + // Create our client with an OOB listener in the LB policy it selects. + r := manual.NewBuilderWithScheme("whatever") + oobLis := newTestOOBListener() + + lisOpts := orca.OOBListenerOptions{ReportInterval: 50 * time.Millisecond} + li := &listenerInfo{listener: oobLis, opts: lisOpts} + addr := setListenerInfo(resolver.Address{Addr: lis.Addr().String()}, li) + r.InitialState(resolver.State{Addresses: []resolver.Address{addr}}) + cc, err := grpc.Dial("whatever:///whatever", grpc.WithDefaultServiceConfig(`{"loadBalancingConfig": [{"customLB":{}}]}`), grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.Dial failed: %v", err) + } + defer cc.Close() + + // Ensure the OOB listener is stopped before the client is closed to avoid + // a potential irrelevant error in the logs. + defer oobLis.Stop() + + // Set a few metrics and wait for them on the client side. + smr.SetCPUUtilization(10) + smr.SetMemoryUtilization(0.1) + smr.SetNamedUtilization("bob", 0.555) + loadReportWant := &v3orcapb.OrcaLoadReport{ + CpuUtilization: 10, + MemUtilization: 0.1, + Utilization: map[string]float64{"bob": 0.555}, + } + +testReport: + for { + select { + case r := <-oobLis.loadReportCh: + t.Log("Load report received: ", r) + if proto.Equal(r, loadReportWant) { + // Success! + break testReport + } + case <-ctx.Done(): + t.Fatalf("timed out waiting for load report: %v", loadReportWant) + } + } + + // Change and add metrics and wait for them on the client side. + smr.SetCPUUtilization(0.5) + smr.SetMemoryUtilization(0.2) + smr.SetNamedUtilization("mary", 0.321) + loadReportWant = &v3orcapb.OrcaLoadReport{ + CpuUtilization: 0.5, + MemUtilization: 0.2, + Utilization: map[string]float64{"bob": 0.555, "mary": 0.321}, + } + + for { + select { + case r := <-oobLis.loadReportCh: + t.Log("Load report received: ", r) + if proto.Equal(r, loadReportWant) { + // Success! + return + } + case <-ctx.Done(): + t.Fatalf("timed out waiting for load report: %v", loadReportWant) + } + } +} + +// fakeORCAService is a simple implementation of an ORCA service that pushes +// requests it receives from clients to a channel and sends responses from a +// channel back. This allows tests to verify the client is sending requests +// and processing responses properly. +type fakeORCAService struct { + v3orcaservicegrpc.UnimplementedOpenRcaServiceServer + + reqCh chan *v3orcaservicepb.OrcaLoadReportRequest + respCh chan any // either *v3orcapb.OrcaLoadReport or error +} + +func newFakeORCAService() *fakeORCAService { + return &fakeORCAService{ + reqCh: make(chan *v3orcaservicepb.OrcaLoadReportRequest), + respCh: make(chan any), + } +} + +func (f *fakeORCAService) close() { + close(f.respCh) +} + +func (f *fakeORCAService) StreamCoreMetrics(req *v3orcaservicepb.OrcaLoadReportRequest, stream v3orcaservicegrpc.OpenRcaService_StreamCoreMetricsServer) error { + f.reqCh <- req + for resp := range f.respCh { + if err, ok := resp.(error); ok { + return err + } + if err := stream.Send(resp.(*v3orcapb.OrcaLoadReport)); err != nil { + // In the event that a stream error occurs, a new stream will have + // been created that was waiting for this response message. Push + // it back onto the channel and return. + // + // This happens because we range over respCh. If we changed to + // instead select on respCh + stream.Context(), the same situation + // could still occur due to a race between noticing the two events, + // so such a workaround would still be needed to prevent flakiness. + f.respCh <- resp + return err + } + } + return nil +} + +// TestProducerBackoff verifies that the ORCA producer applies the proper +// backoff after stream failures. +func (s) TestProducerBackoff(t *testing.T) { + grpctest.TLogger.ExpectErrorN("injected error", 4) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Provide a convenient way to expect backoff calls and return a minimal + // value. + const backoffShouldNotBeCalled = 9999 // Use to assert backoff function is not called. + const backoffAllowAny = -1 // Use to ignore any backoff calls. + expectedBackoff := backoffAllowAny + oldBackoff := internal.DefaultBackoffFunc + internal.DefaultBackoffFunc = func(got int) time.Duration { + if expectedBackoff == backoffShouldNotBeCalled { + t.Errorf("Unexpected backoff call; parameter = %v", got) + } else if expectedBackoff != backoffAllowAny { + if got != expectedBackoff { + t.Errorf("Unexpected backoff received; got %v want %v", got, expectedBackoff) + } + } + return time.Millisecond + } + defer func() { internal.DefaultBackoffFunc = oldBackoff }() + + // Initialize listener for our ORCA server. + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatal(err) + } + + // Register our fake ORCA service. + s := grpc.NewServer() + fake := newFakeORCAService() + defer fake.close() + v3orcaservicegrpc.RegisterOpenRcaServiceServer(s, fake) + go s.Serve(lis) + defer s.Stop() + + // Define the report interval and a function to wait for it to be sent to + // the server. + const reportInterval = 123 * time.Second + awaitRequest := func(interval time.Duration) { + select { + case req := <-fake.reqCh: + if got := req.GetReportInterval().AsDuration(); got != interval { + t.Errorf("Unexpected report interval; got %v want %v", got, interval) + } + case <-ctx.Done(): + t.Fatalf("Did not receive client request") + } + } + + // Create our client with an OOB listener in the LB policy it selects. + r := manual.NewBuilderWithScheme("whatever") + oobLis := newTestOOBListener() + + lisOpts := orca.OOBListenerOptions{ReportInterval: reportInterval} + li := &listenerInfo{listener: oobLis, opts: lisOpts} + r.InitialState(resolver.State{Addresses: []resolver.Address{setListenerInfo(resolver.Address{Addr: lis.Addr().String()}, li)}}) + cc, err := grpc.Dial("whatever:///whatever", grpc.WithDefaultServiceConfig(`{"loadBalancingConfig": [{"customLB":{}}]}`), grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.Dial failed: %v", err) + } + defer cc.Close() + + // Ensure the OOB listener is stopped before the client is closed to avoid + // a potential irrelevant error in the logs. + defer oobLis.Stop() + + // Define a load report to send and expect the client to see. + loadReportWant := &v3orcapb.OrcaLoadReport{ + CpuUtilization: 10, + MemUtilization: 0.1, + Utilization: map[string]float64{"bob": 0.555}, + } + + // Unblock the fake. + awaitRequest(reportInterval) + fake.respCh <- loadReportWant + select { + case r := <-oobLis.loadReportCh: + t.Log("Load report received: ", r) + if proto.Equal(r, loadReportWant) { + // Success! + break + } + case <-ctx.Done(): + t.Fatalf("timed out waiting for load report: %v", loadReportWant) + } + + // The next request should be immediate, since there was a message + // received. + expectedBackoff = backoffShouldNotBeCalled + fake.respCh <- status.Errorf(codes.Internal, "injected error") + awaitRequest(reportInterval) + + // The next requests will need to backoff. + expectedBackoff = 0 + fake.respCh <- status.Errorf(codes.Internal, "injected error") + awaitRequest(reportInterval) + expectedBackoff = 1 + fake.respCh <- status.Errorf(codes.Internal, "injected error") + awaitRequest(reportInterval) + expectedBackoff = 2 + fake.respCh <- status.Errorf(codes.Internal, "injected error") + awaitRequest(reportInterval) + // The next request should be immediate, since there was a message + // received. + expectedBackoff = backoffShouldNotBeCalled + + // Send another valid response and wait for it on the client. + fake.respCh <- loadReportWant + select { + case r := <-oobLis.loadReportCh: + t.Log("Load report received: ", r) + if proto.Equal(r, loadReportWant) { + // Success! + break + } + case <-ctx.Done(): + t.Fatalf("timed out waiting for load report: %v", loadReportWant) + } +} + +// TestProducerMultipleListeners tests that multiple listeners works as +// expected in a producer: requesting the proper interval and delivering the +// update to all listeners. +func (s) TestProducerMultipleListeners(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Provide a convenient way to expect backoff calls and return a minimal + // value. + oldBackoff := internal.DefaultBackoffFunc + internal.DefaultBackoffFunc = func(got int) time.Duration { + return time.Millisecond + } + defer func() { internal.DefaultBackoffFunc = oldBackoff }() + + // Initialize listener for our ORCA server. + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatal(err) + } + + // Register our fake ORCA service. + s := grpc.NewServer() + fake := newFakeORCAService() + defer fake.close() + v3orcaservicegrpc.RegisterOpenRcaServiceServer(s, fake) + go s.Serve(lis) + defer s.Stop() + + // Define the report interval and a function to wait for it to be sent to + // the server. + const reportInterval1 = 123 * time.Second + const reportInterval2 = 234 * time.Second + const reportInterval3 = 56 * time.Second + awaitRequest := func(interval time.Duration) { + select { + case req := <-fake.reqCh: + if got := req.GetReportInterval().AsDuration(); got != interval { + t.Errorf("Unexpected report interval; got %v want %v", got, interval) + } + case <-ctx.Done(): + t.Fatalf("Did not receive client request") + } + } + + // Create our client with an OOB listener in the LB policy it selects. + r := manual.NewBuilderWithScheme("whatever") + oobLis1 := newTestOOBListener() + lisOpts1 := orca.OOBListenerOptions{ReportInterval: reportInterval1} + li := &listenerInfo{listener: oobLis1, opts: lisOpts1} + r.InitialState(resolver.State{Addresses: []resolver.Address{setListenerInfo(resolver.Address{Addr: lis.Addr().String()}, li)}}) + cc, err := grpc.Dial("whatever:///whatever", grpc.WithDefaultServiceConfig(`{"loadBalancingConfig": [{"customLB":{}}]}`), grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.Dial failed: %v", err) + } + defer cc.Close() + + // Ensure the OOB listener is stopped before the client is closed to avoid + // a potential irrelevant error in the logs. + defer oobLis1.Stop() + + oobLis2 := newTestOOBListener() + lisOpts2 := orca.OOBListenerOptions{ReportInterval: reportInterval2} + + oobLis3 := newTestOOBListener() + lisOpts3 := orca.OOBListenerOptions{ReportInterval: reportInterval3} + + // Define a load report to send and expect the client to see. + loadReportWant := &v3orcapb.OrcaLoadReport{ + CpuUtilization: 10, + MemUtilization: 0.1, + Utilization: map[string]float64{"bob": 0.555}, + } + + // Receive reports and update counts for the three listeners. + var reportsMu sync.Mutex + var reportsReceived1, reportsReceived2, reportsReceived3 int + go func() { + for { + select { + case r := <-oobLis1.loadReportCh: + t.Log("Load report 1 received: ", r) + if !proto.Equal(r, loadReportWant) { + t.Errorf("Unexpected report received: %+v", r) + } + reportsMu.Lock() + reportsReceived1++ + reportsMu.Unlock() + case r := <-oobLis2.loadReportCh: + t.Log("Load report 2 received: ", r) + if !proto.Equal(r, loadReportWant) { + t.Errorf("Unexpected report received: %+v", r) + } + reportsMu.Lock() + reportsReceived2++ + reportsMu.Unlock() + case r := <-oobLis3.loadReportCh: + t.Log("Load report 3 received: ", r) + if !proto.Equal(r, loadReportWant) { + t.Errorf("Unexpected report received: %+v", r) + } + reportsMu.Lock() + reportsReceived3++ + reportsMu.Unlock() + case <-ctx.Done(): + // Test has ended; exit + return + } + } + }() + + // checkReports is a helper function to check the report counts for the three listeners. + checkReports := func(r1, r2, r3 int) { + t.Helper() + for ctx.Err() == nil { + reportsMu.Lock() + if r1 == reportsReceived1 && r2 == reportsReceived2 && r3 == reportsReceived3 { + // Success! + reportsMu.Unlock() + return + } + if reportsReceived1 > r1 || reportsReceived2 > r2 || reportsReceived3 > r3 { + reportsMu.Unlock() + t.Fatalf("received excess reports. got %v %v %v; want %v %v %v", reportsReceived1, reportsReceived2, reportsReceived3, r1, r2, r3) + return + } + reportsMu.Unlock() + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("timed out waiting for reports received. got %v %v %v; want %v %v %v", reportsReceived1, reportsReceived2, reportsReceived3, r1, r2, r3) + } + + // Only 1 listener; expect reportInterval1 to be used and expect the report + // to be sent to the listener. + awaitRequest(reportInterval1) + fake.respCh <- loadReportWant + checkReports(1, 0, 0) + + // Register listener 2 with a less frequent interval; no need to recreate + // stream. Report should go to both listeners. + oobLis2.cleanup = orca.RegisterOOBListener(li.sc, oobLis2, lisOpts2) + fake.respCh <- loadReportWant + checkReports(2, 1, 0) + + // Register listener 3 with a more frequent interval; stream is recreated + // with this interval. The next report will go to all three listeners. + oobLis3.cleanup = orca.RegisterOOBListener(li.sc, oobLis3, lisOpts3) + awaitRequest(reportInterval3) + fake.respCh <- loadReportWant + checkReports(3, 2, 1) + + // Another report without a change in listeners should go to all three listeners. + fake.respCh <- loadReportWant + checkReports(4, 3, 2) + + // Stop listener 2. This does not affect the interval as listener 3 is + // still the shortest. The next update goes to listeners 1 and 3. + oobLis2.Stop() + fake.respCh <- loadReportWant + checkReports(5, 3, 3) + + // Stop listener 3. This makes the interval longer. Reports should only + // go to listener 1 now. + oobLis3.Stop() + awaitRequest(reportInterval1) + fake.respCh <- loadReportWant + checkReports(6, 3, 3) + // Another report without a change in listeners should go to the first listener. + fake.respCh <- loadReportWant + checkReports(7, 3, 3) +} diff --git a/orca/server_metrics.go b/orca/server_metrics.go new file mode 100644 index 000000000000..f2cdb9b0b26f --- /dev/null +++ b/orca/server_metrics.go @@ -0,0 +1,351 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package orca + +import ( + "sync" + + v3orcapb "github.com/cncf/xds/go/xds/data/orca/v3" +) + +// ServerMetrics is the data returned from a server to a client to describe the +// current state of the server and/or the cost of a request when used per-call. +type ServerMetrics struct { + CPUUtilization float64 // CPU utilization: [0, inf); unset=-1 + MemUtilization float64 // Memory utilization: [0, 1.0]; unset=-1 + AppUtilization float64 // Application utilization: [0, inf); unset=-1 + QPS float64 // queries per second: [0, inf); unset=-1 + EPS float64 // errors per second: [0, inf); unset=-1 + + // The following maps must never be nil. + + Utilization map[string]float64 // Custom fields: [0, 1.0] + RequestCost map[string]float64 // Custom fields: [0, inf); not sent OOB + NamedMetrics map[string]float64 // Custom fields: [0, inf); not sent OOB +} + +// toLoadReportProto dumps sm as an OrcaLoadReport proto. +func (sm *ServerMetrics) toLoadReportProto() *v3orcapb.OrcaLoadReport { + ret := &v3orcapb.OrcaLoadReport{ + Utilization: sm.Utilization, + RequestCost: sm.RequestCost, + NamedMetrics: sm.NamedMetrics, + } + if sm.CPUUtilization != -1 { + ret.CpuUtilization = sm.CPUUtilization + } + if sm.MemUtilization != -1 { + ret.MemUtilization = sm.MemUtilization + } + if sm.AppUtilization != -1 { + ret.ApplicationUtilization = sm.AppUtilization + } + if sm.QPS != -1 { + ret.RpsFractional = sm.QPS + } + if sm.EPS != -1 { + ret.Eps = sm.EPS + } + return ret +} + +// merge merges o into sm, overwriting any values present in both. +func (sm *ServerMetrics) merge(o *ServerMetrics) { + mergeMap(sm.Utilization, o.Utilization) + mergeMap(sm.RequestCost, o.RequestCost) + mergeMap(sm.NamedMetrics, o.NamedMetrics) + if o.CPUUtilization != -1 { + sm.CPUUtilization = o.CPUUtilization + } + if o.MemUtilization != -1 { + sm.MemUtilization = o.MemUtilization + } + if o.AppUtilization != -1 { + sm.AppUtilization = o.AppUtilization + } + if o.QPS != -1 { + sm.QPS = o.QPS + } + if o.EPS != -1 { + sm.EPS = o.EPS + } +} + +func mergeMap(a, b map[string]float64) { + for k, v := range b { + a[k] = v + } +} + +// ServerMetricsRecorder allows for recording and providing out of band server +// metrics. +type ServerMetricsRecorder interface { + ServerMetricsProvider + + // SetCPUUtilization sets the CPU utilization server metric. Must be + // greater than zero. + SetCPUUtilization(float64) + // DeleteCPUUtilization deletes the CPU utilization server metric to + // prevent it from being sent. + DeleteCPUUtilization() + + // SetMemoryUtilization sets the memory utilization server metric. Must be + // in the range [0, 1]. + SetMemoryUtilization(float64) + // DeleteMemoryUtilization deletes the memory utiliztion server metric to + // prevent it from being sent. + DeleteMemoryUtilization() + + // SetApplicationUtilization sets the application utilization server + // metric. Must be greater than zero. + SetApplicationUtilization(float64) + // DeleteApplicationUtilization deletes the application utilization server + // metric to prevent it from being sent. + DeleteApplicationUtilization() + + // SetQPS sets the Queries Per Second server metric. Must be greater than + // zero. + SetQPS(float64) + // DeleteQPS deletes the Queries Per Second server metric to prevent it + // from being sent. + DeleteQPS() + + // SetEPS sets the Errors Per Second server metric. Must be greater than + // zero. + SetEPS(float64) + // DeleteEPS deletes the Errors Per Second server metric to prevent it from + // being sent. + DeleteEPS() + + // SetNamedUtilization sets the named utilization server metric for the + // name provided. val must be in the range [0, 1]. + SetNamedUtilization(name string, val float64) + // DeleteNamedUtilization deletes the named utilization server metric for + // the name provided to prevent it from being sent. + DeleteNamedUtilization(name string) +} + +type serverMetricsRecorder struct { + mu sync.Mutex // protects state + state *ServerMetrics // the current metrics +} + +// NewServerMetricsRecorder returns an in-memory store for ServerMetrics and +// allows for safe setting and retrieving of ServerMetrics. Also implements +// ServerMetricsProvider for use with NewService. +func NewServerMetricsRecorder() ServerMetricsRecorder { + return newServerMetricsRecorder() +} + +func newServerMetricsRecorder() *serverMetricsRecorder { + return &serverMetricsRecorder{ + state: &ServerMetrics{ + CPUUtilization: -1, + MemUtilization: -1, + AppUtilization: -1, + QPS: -1, + EPS: -1, + Utilization: make(map[string]float64), + RequestCost: make(map[string]float64), + NamedMetrics: make(map[string]float64), + }, + } +} + +// ServerMetrics returns a copy of the current ServerMetrics. +func (s *serverMetricsRecorder) ServerMetrics() *ServerMetrics { + s.mu.Lock() + defer s.mu.Unlock() + return &ServerMetrics{ + CPUUtilization: s.state.CPUUtilization, + MemUtilization: s.state.MemUtilization, + AppUtilization: s.state.AppUtilization, + QPS: s.state.QPS, + EPS: s.state.EPS, + Utilization: copyMap(s.state.Utilization), + RequestCost: copyMap(s.state.RequestCost), + NamedMetrics: copyMap(s.state.NamedMetrics), + } +} + +func copyMap(m map[string]float64) map[string]float64 { + ret := make(map[string]float64, len(m)) + for k, v := range m { + ret[k] = v + } + return ret +} + +// SetCPUUtilization records a measurement for the CPU utilization metric. +func (s *serverMetricsRecorder) SetCPUUtilization(val float64) { + if val < 0 { + if logger.V(2) { + logger.Infof("Ignoring CPU Utilization value out of range: %v", val) + } + return + } + s.mu.Lock() + defer s.mu.Unlock() + s.state.CPUUtilization = val +} + +// DeleteCPUUtilization deletes the relevant server metric to prevent it from +// being sent. +func (s *serverMetricsRecorder) DeleteCPUUtilization() { + s.mu.Lock() + defer s.mu.Unlock() + s.state.CPUUtilization = -1 +} + +// SetMemoryUtilization records a measurement for the memory utilization metric. +func (s *serverMetricsRecorder) SetMemoryUtilization(val float64) { + if val < 0 || val > 1 { + if logger.V(2) { + logger.Infof("Ignoring Memory Utilization value out of range: %v", val) + } + return + } + s.mu.Lock() + defer s.mu.Unlock() + s.state.MemUtilization = val +} + +// DeleteMemoryUtilization deletes the relevant server metric to prevent it +// from being sent. +func (s *serverMetricsRecorder) DeleteMemoryUtilization() { + s.mu.Lock() + defer s.mu.Unlock() + s.state.MemUtilization = -1 +} + +// SetApplicationUtilization records a measurement for a generic utilization +// metric. +func (s *serverMetricsRecorder) SetApplicationUtilization(val float64) { + if val < 0 { + if logger.V(2) { + logger.Infof("Ignoring Application Utilization value out of range: %v", val) + } + return + } + s.mu.Lock() + defer s.mu.Unlock() + s.state.AppUtilization = val +} + +// DeleteApplicationUtilization deletes the relevant server metric to prevent +// it from being sent. +func (s *serverMetricsRecorder) DeleteApplicationUtilization() { + s.mu.Lock() + defer s.mu.Unlock() + s.state.AppUtilization = -1 +} + +// SetQPS records a measurement for the QPS metric. +func (s *serverMetricsRecorder) SetQPS(val float64) { + if val < 0 { + if logger.V(2) { + logger.Infof("Ignoring QPS value out of range: %v", val) + } + return + } + s.mu.Lock() + defer s.mu.Unlock() + s.state.QPS = val +} + +// DeleteQPS deletes the relevant server metric to prevent it from being sent. +func (s *serverMetricsRecorder) DeleteQPS() { + s.mu.Lock() + defer s.mu.Unlock() + s.state.QPS = -1 +} + +// SetEPS records a measurement for the EPS metric. +func (s *serverMetricsRecorder) SetEPS(val float64) { + if val < 0 { + if logger.V(2) { + logger.Infof("Ignoring EPS value out of range: %v", val) + } + return + } + s.mu.Lock() + defer s.mu.Unlock() + s.state.EPS = val +} + +// DeleteEPS deletes the relevant server metric to prevent it from being sent. +func (s *serverMetricsRecorder) DeleteEPS() { + s.mu.Lock() + defer s.mu.Unlock() + s.state.EPS = -1 +} + +// SetNamedUtilization records a measurement for a utilization metric uniquely +// identifiable by name. +func (s *serverMetricsRecorder) SetNamedUtilization(name string, val float64) { + if val < 0 || val > 1 { + if logger.V(2) { + logger.Infof("Ignoring Named Utilization value out of range: %v", val) + } + return + } + s.mu.Lock() + defer s.mu.Unlock() + s.state.Utilization[name] = val +} + +// DeleteNamedUtilization deletes any previously recorded measurement for a +// utilization metric uniquely identifiable by name. +func (s *serverMetricsRecorder) DeleteNamedUtilization(name string) { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.state.Utilization, name) +} + +// SetRequestCost records a measurement for a utilization metric uniquely +// identifiable by name. +func (s *serverMetricsRecorder) SetRequestCost(name string, val float64) { + s.mu.Lock() + defer s.mu.Unlock() + s.state.RequestCost[name] = val +} + +// DeleteRequestCost deletes any previously recorded measurement for a +// utilization metric uniquely identifiable by name. +func (s *serverMetricsRecorder) DeleteRequestCost(name string) { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.state.RequestCost, name) +} + +// SetNamedMetric records a measurement for a utilization metric uniquely +// identifiable by name. +func (s *serverMetricsRecorder) SetNamedMetric(name string, val float64) { + s.mu.Lock() + defer s.mu.Unlock() + s.state.NamedMetrics[name] = val +} + +// DeleteNamedMetric deletes any previously recorded measurement for a +// utilization metric uniquely identifiable by name. +func (s *serverMetricsRecorder) DeleteNamedMetric(name string) { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.state.NamedMetrics, name) +} diff --git a/orca/server_metrics_test.go b/orca/server_metrics_test.go new file mode 100644 index 000000000000..ecc80d0e584b --- /dev/null +++ b/orca/server_metrics_test.go @@ -0,0 +1,175 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package orca + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/internal/grpctest" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func (s) TestServerMetrics_Setters(t *testing.T) { + smr := NewServerMetricsRecorder() + + smr.SetCPUUtilization(0.1) + smr.SetMemoryUtilization(0.2) + smr.SetApplicationUtilization(0.3) + smr.SetQPS(0.4) + smr.SetEPS(0.5) + smr.SetNamedUtilization("x", 0.6) + + want := &ServerMetrics{ + CPUUtilization: 0.1, + MemUtilization: 0.2, + AppUtilization: 0.3, + QPS: 0.4, + EPS: 0.5, + Utilization: map[string]float64{"x": 0.6}, + NamedMetrics: map[string]float64{}, + RequestCost: map[string]float64{}, + } + + got := smr.ServerMetrics() + if d := cmp.Diff(got, want); d != "" { + t.Fatalf("unexpected server metrics: -got +want: %v", d) + } +} + +func (s) TestServerMetrics_Deleters(t *testing.T) { + smr := NewServerMetricsRecorder() + + smr.SetCPUUtilization(0.1) + smr.SetMemoryUtilization(0.2) + smr.SetApplicationUtilization(0.3) + smr.SetQPS(0.4) + smr.SetEPS(0.5) + smr.SetNamedUtilization("x", 0.6) + smr.SetNamedUtilization("y", 0.7) + + // Now delete everything except named_utilization "y". + smr.DeleteCPUUtilization() + smr.DeleteMemoryUtilization() + smr.DeleteApplicationUtilization() + smr.DeleteQPS() + smr.DeleteEPS() + smr.DeleteNamedUtilization("x") + + want := &ServerMetrics{ + CPUUtilization: -1, + MemUtilization: -1, + AppUtilization: -1, + QPS: -1, + EPS: -1, + Utilization: map[string]float64{"y": 0.7}, + NamedMetrics: map[string]float64{}, + RequestCost: map[string]float64{}, + } + + got := smr.ServerMetrics() + if d := cmp.Diff(got, want); d != "" { + t.Fatalf("unexpected server metrics: -got +want: %v", d) + } +} + +func (s) TestServerMetrics_Setters_Range(t *testing.T) { + smr := NewServerMetricsRecorder() + + smr.SetCPUUtilization(0.1) + smr.SetMemoryUtilization(0.2) + smr.SetApplicationUtilization(0.3) + smr.SetQPS(0.4) + smr.SetEPS(0.5) + smr.SetNamedUtilization("x", 0.6) + + // Negatives for all these fields should be ignored. + smr.SetCPUUtilization(-2) + smr.SetMemoryUtilization(-3) + smr.SetApplicationUtilization(-4) + smr.SetQPS(-0.1) + smr.SetEPS(-0.6) + smr.SetNamedUtilization("x", -2) + + // Memory and named utilizations over 1 are ignored. + smr.SetMemoryUtilization(1.1) + smr.SetNamedUtilization("x", 1.1) + + want := &ServerMetrics{ + CPUUtilization: 0.1, + MemUtilization: 0.2, + AppUtilization: 0.3, + QPS: 0.4, + EPS: 0.5, + Utilization: map[string]float64{"x": 0.6}, + NamedMetrics: map[string]float64{}, + RequestCost: map[string]float64{}, + } + + got := smr.ServerMetrics() + if d := cmp.Diff(got, want); d != "" { + t.Fatalf("unexpected server metrics: -got +want: %v", d) + } +} + +func (s) TestServerMetrics_Merge(t *testing.T) { + sm1 := &ServerMetrics{ + CPUUtilization: 0.1, + MemUtilization: 0.2, + AppUtilization: 0.3, + QPS: -1, + EPS: 0, + Utilization: map[string]float64{"x": 0.6}, + NamedMetrics: map[string]float64{"y": 0.2}, + RequestCost: map[string]float64{"a": 0.1}, + } + + sm2 := &ServerMetrics{ + CPUUtilization: -1, + AppUtilization: 0, + QPS: 0.9, + EPS: 20, + Utilization: map[string]float64{"x": 0.5, "y": 0.4}, + NamedMetrics: map[string]float64{"x": 0.1}, + RequestCost: map[string]float64{"a": 0.2}, + } + + want := &ServerMetrics{ + CPUUtilization: 0.1, + MemUtilization: 0, + AppUtilization: 0, + QPS: 0.9, + EPS: 20, + Utilization: map[string]float64{"x": 0.5, "y": 0.4}, + NamedMetrics: map[string]float64{"x": 0.1, "y": 0.2}, + RequestCost: map[string]float64{"a": 0.2}, + } + + sm1.merge(sm2) + if d := cmp.Diff(sm1, want); d != "" { + t.Fatalf("unexpected server metrics: -got +want: %v", d) + } +} diff --git a/orca/service.go b/orca/service.go new file mode 100644 index 000000000000..7461a6b05a1a --- /dev/null +++ b/orca/service.go @@ -0,0 +1,163 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package orca + +import ( + "fmt" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal" + ointernal "google.golang.org/grpc/orca/internal" + "google.golang.org/grpc/status" + + v3orcaservicegrpc "github.com/cncf/xds/go/xds/service/orca/v3" + v3orcaservicepb "github.com/cncf/xds/go/xds/service/orca/v3" +) + +func init() { + ointernal.AllowAnyMinReportingInterval = func(so *ServiceOptions) { + so.allowAnyMinReportingInterval = true + } + internal.ORCAAllowAnyMinReportingInterval = ointernal.AllowAnyMinReportingInterval +} + +// minReportingInterval is the absolute minimum value supported for +// out-of-band metrics reporting from the ORCA service implementation +// provided by the orca package. +const minReportingInterval = 30 * time.Second + +// Service provides an implementation of the OpenRcaService as defined in the +// [ORCA] service protos. Instances of this type must be created via calls to +// Register() or NewService(). +// +// Server applications can use the SetXxx() and DeleteXxx() methods to record +// measurements corresponding to backend metrics, which eventually get pushed to +// clients who have initiated the SteamCoreMetrics streaming RPC. +// +// [ORCA]: https://github.com/cncf/xds/blob/main/xds/service/orca/v3/orca.proto +type Service struct { + v3orcaservicegrpc.UnimplementedOpenRcaServiceServer + + // Minimum reporting interval, as configured by the user, or the default. + minReportingInterval time.Duration + + smProvider ServerMetricsProvider +} + +// ServiceOptions contains options to configure the ORCA service implementation. +type ServiceOptions struct { + // ServerMetricsProvider is the provider to be used by the service for + // reporting OOB server metrics to clients. Typically obtained via + // NewServerMetricsRecorder. This field is required. + ServerMetricsProvider ServerMetricsProvider + + // MinReportingInterval sets the lower bound for how often out-of-band + // metrics are reported on the streaming RPC initiated by the client. If + // unspecified, negative or less than the default value of 30s, the default + // is used. Clients may request a higher value as part of the + // StreamCoreMetrics streaming RPC. + MinReportingInterval time.Duration + + // Allow a minReportingInterval which is less than the default of 30s. + // Used for testing purposes only. + allowAnyMinReportingInterval bool +} + +// A ServerMetricsProvider provides ServerMetrics upon request. +type ServerMetricsProvider interface { + // ServerMetrics returns the current set of server metrics. It should + // return a read-only, immutable copy of the data that is active at the + // time of the call. + ServerMetrics() *ServerMetrics +} + +// NewService creates a new ORCA service implementation configured using the +// provided options. +func NewService(opts ServiceOptions) (*Service, error) { + // The default minimum supported reporting interval value can be overridden + // for testing purposes through the orca internal package. + if opts.ServerMetricsProvider == nil { + return nil, fmt.Errorf("ServerMetricsProvider not specified") + } + if !opts.allowAnyMinReportingInterval { + if opts.MinReportingInterval < 0 || opts.MinReportingInterval < minReportingInterval { + opts.MinReportingInterval = minReportingInterval + } + } + service := &Service{ + minReportingInterval: opts.MinReportingInterval, + smProvider: opts.ServerMetricsProvider, + } + return service, nil +} + +// Register creates a new ORCA service implementation configured using the +// provided options and registers the same on the provided grpc Server. +func Register(s *grpc.Server, opts ServiceOptions) error { + // TODO(https://github.com/cncf/xds/issues/41): replace *grpc.Server with + // grpc.ServiceRegistrar when possible. + service, err := NewService(opts) + if err != nil { + return err + } + v3orcaservicegrpc.RegisterOpenRcaServiceServer(s, service) + return nil +} + +// determineReportingInterval determines the reporting interval for out-of-band +// metrics. If the reporting interval is not specified in the request, or is +// negative or is less than the configured minimum (via +// ServiceOptions.MinReportingInterval), the latter is used. Else the value from +// the incoming request is used. +func (s *Service) determineReportingInterval(req *v3orcaservicepb.OrcaLoadReportRequest) time.Duration { + if req.GetReportInterval() == nil { + return s.minReportingInterval + } + dur := req.GetReportInterval().AsDuration() + if dur < s.minReportingInterval { + logger.Warningf("Received reporting interval %q is less than configured minimum: %v. Using minimum", dur, s.minReportingInterval) + return s.minReportingInterval + } + return dur +} + +func (s *Service) sendMetricsResponse(stream v3orcaservicegrpc.OpenRcaService_StreamCoreMetricsServer) error { + return stream.Send(s.smProvider.ServerMetrics().toLoadReportProto()) +} + +// StreamCoreMetrics streams custom backend metrics injected by the server +// application. +func (s *Service) StreamCoreMetrics(req *v3orcaservicepb.OrcaLoadReportRequest, stream v3orcaservicegrpc.OpenRcaService_StreamCoreMetricsServer) error { + ticker := time.NewTicker(s.determineReportingInterval(req)) + defer ticker.Stop() + + for { + if err := s.sendMetricsResponse(stream); err != nil { + return err + } + // Send a response containing the currently recorded metrics + select { + case <-stream.Context().Done(): + return status.Error(codes.Canceled, "Stream has ended.") + case <-ticker.C: + } + } +} diff --git a/orca/service_test.go b/orca/service_test.go new file mode 100644 index 000000000000..9c4defbe266b --- /dev/null +++ b/orca/service_test.go @@ -0,0 +1,197 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package orca_test + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + "github.com/golang/protobuf/proto" + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/orca" + "google.golang.org/grpc/orca/internal" + "google.golang.org/protobuf/types/known/durationpb" + + v3orcapb "github.com/cncf/xds/go/xds/data/orca/v3" + v3orcaservicegrpc "github.com/cncf/xds/go/xds/service/orca/v3" + v3orcaservicepb "github.com/cncf/xds/go/xds/service/orca/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +const requestsMetricKey = "test-service-requests" + +// An implementation of grpc_testing.TestService for the purpose of this test. +// We cannot use the StubServer approach here because we need to register the +// OpenRCAService as well on the same gRPC server. +type testServiceImpl struct { + mu sync.Mutex + requests int64 + + testgrpc.TestServiceServer + smr orca.ServerMetricsRecorder +} + +func (t *testServiceImpl) UnaryCall(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + t.mu.Lock() + t.requests++ + t.mu.Unlock() + + t.smr.SetNamedUtilization(requestsMetricKey, float64(t.requests)*0.01) + t.smr.SetCPUUtilization(50.0) + t.smr.SetMemoryUtilization(0.9) + t.smr.SetApplicationUtilization(1.2) + return &testpb.SimpleResponse{}, nil +} + +func (t *testServiceImpl) EmptyCall(context.Context, *testpb.Empty) (*testpb.Empty, error) { + t.smr.DeleteNamedUtilization(requestsMetricKey) + t.smr.SetCPUUtilization(0) + t.smr.SetMemoryUtilization(0) + t.smr.DeleteApplicationUtilization() + return &testpb.Empty{}, nil +} + +// TestE2E_CustomBackendMetrics_OutOfBand tests the injection of out-of-band +// custom backend metrics from the server application, and verifies that +// expected load reports are received at the client. +// +// TODO: Change this test to use the client API, when ready, to read the +// out-of-band metrics pushed by the server. +func (s) TestE2E_CustomBackendMetrics_OutOfBand(t *testing.T) { + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatal(err) + } + + // Override the min reporting interval in the internal package. + const shortReportingInterval = 10 * time.Millisecond + smr := orca.NewServerMetricsRecorder() + opts := orca.ServiceOptions{MinReportingInterval: shortReportingInterval, ServerMetricsProvider: smr} + internal.AllowAnyMinReportingInterval.(func(*orca.ServiceOptions))(&opts) + + // Register the OpenRCAService with a very short metrics reporting interval. + s := grpc.NewServer() + if err := orca.Register(s, opts); err != nil { + t.Fatalf("orca.EnableOutOfBandMetricsReportingForTesting() failed: %v", err) + } + + // Register the test service implementation on the same grpc server, and start serving. + testgrpc.RegisterTestServiceServer(s, &testServiceImpl{smr: smr}) + go s.Serve(lis) + defer s.Stop() + t.Logf("Started gRPC server at %s...", lis.Addr().String()) + + // Dial the test server. + cc, err := grpc.Dial(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.Dial(%s) failed: %v", lis.Addr().String(), err) + } + defer cc.Close() + + // Spawn a goroutine which sends 20 unary RPCs to the test server. This + // will trigger the injection of custom backend metrics from the + // testServiceImpl. + const numRequests = 20 + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testStub := testgrpc.NewTestServiceClient(cc) + errCh := make(chan error, 1) + go func() { + for i := 0; i < numRequests; i++ { + if _, err := testStub.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil { + errCh <- fmt.Errorf("UnaryCall failed: %v", err) + return + } + time.Sleep(time.Millisecond) + } + errCh <- nil + }() + + // Start the server streaming RPC to receive custom backend metrics. + oobStub := v3orcaservicegrpc.NewOpenRcaServiceClient(cc) + stream, err := oobStub.StreamCoreMetrics(ctx, &v3orcaservicepb.OrcaLoadReportRequest{ReportInterval: durationpb.New(shortReportingInterval)}) + if err != nil { + t.Fatalf("Failed to create a stream for out-of-band metrics") + } + + // Wait for the server to push metrics which indicate the completion of all + // the unary RPCs made from the above goroutine. + for { + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for out-of-band custom backend metrics to match expected values") + case err := <-errCh: + if err != nil { + t.Fatal(err) + } + default: + } + + wantProto := &v3orcapb.OrcaLoadReport{ + CpuUtilization: 50.0, + MemUtilization: 0.9, + ApplicationUtilization: 1.2, + Utilization: map[string]float64{requestsMetricKey: numRequests * 0.01}, + } + gotProto, err := stream.Recv() + if err != nil { + t.Fatalf("Recv() failed: %v", err) + } + if !cmp.Equal(gotProto, wantProto, cmp.Comparer(proto.Equal)) { + t.Logf("Received load report from stream: %s, want: %s", pretty.ToJSON(gotProto), pretty.ToJSON(wantProto)) + continue + } + // This means that we received the metrics which we expected. + break + } + + // The EmptyCall RPC is expected to delete earlier injected metrics. + if _, err := testStub.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall failed: %v", err) + } + // Wait for the server to push empty metrics which indicate the processing + // of the above EmptyCall RPC. + for { + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for out-of-band custom backend metrics to match expected values") + default: + } + + wantProto := &v3orcapb.OrcaLoadReport{} + gotProto, err := stream.Recv() + if err != nil { + t.Fatalf("Recv() failed: %v", err) + } + if !cmp.Equal(gotProto, wantProto, cmp.Comparer(proto.Equal)) { + t.Logf("Received load report from stream: %s, want: %s", pretty.ToJSON(gotProto), pretty.ToJSON(wantProto)) + continue + } + // This means that we received the metrics which we expected. + break + } +} diff --git a/picker_wrapper.go b/picker_wrapper.go index a58174b6f436..236837f4157c 100644 --- a/picker_wrapper.go +++ b/picker_wrapper.go @@ -26,27 +26,38 @@ import ( "google.golang.org/grpc/balancer" "google.golang.org/grpc/codes" "google.golang.org/grpc/internal/channelz" + istatus "google.golang.org/grpc/internal/status" "google.golang.org/grpc/internal/transport" + "google.golang.org/grpc/stats" "google.golang.org/grpc/status" ) // pickerWrapper is a wrapper of balancer.Picker. It blocks on certain pick // actions and unblock when there's a picker update. type pickerWrapper struct { - mu sync.Mutex - done bool - blockingCh chan struct{} - picker balancer.Picker + mu sync.Mutex + done bool + idle bool + blockingCh chan struct{} + picker balancer.Picker + statsHandlers []stats.Handler // to record blocking picker calls } -func newPickerWrapper() *pickerWrapper { - return &pickerWrapper{blockingCh: make(chan struct{})} +func newPickerWrapper(statsHandlers []stats.Handler) *pickerWrapper { + return &pickerWrapper{ + blockingCh: make(chan struct{}), + statsHandlers: statsHandlers, + } } // updatePicker is called by UpdateBalancerState. It unblocks all blocked pick. func (pw *pickerWrapper) updatePicker(p balancer.Picker) { pw.mu.Lock() - if pw.done { + if pw.done || pw.idle { + // There is a small window where a picker update from the LB policy can + // race with the channel going to idle mode. If the picker is idle here, + // it is because the channel asked it to do so, and therefore it is sage + // to ignore the update from the LB policy. pw.mu.Unlock() return } @@ -57,12 +68,16 @@ func (pw *pickerWrapper) updatePicker(p balancer.Picker) { pw.mu.Unlock() } -func doneChannelzWrapper(acw *acBalancerWrapper, done func(balancer.DoneInfo)) func(balancer.DoneInfo) { - acw.mu.Lock() - ac := acw.ac - acw.mu.Unlock() +// doneChannelzWrapper performs the following: +// - increments the calls started channelz counter +// - wraps the done function in the passed in result to increment the calls +// failed or calls succeeded channelz counter before invoking the actual +// done function. +func doneChannelzWrapper(acbw *acBalancerWrapper, result *balancer.PickResult) { + ac := acbw.ac ac.incrCallsStarted() - return func(b balancer.DoneInfo) { + done := result.Done + result.Done = func(b balancer.DoneInfo) { if b.Err != nil && b.Err != io.EOF { ac.incrCallsFailed() } else { @@ -81,15 +96,16 @@ func doneChannelzWrapper(acw *acBalancerWrapper, done func(balancer.DoneInfo)) f // - the current picker returns other errors and failfast is false. // - the subConn returned by the current picker is not READY // When one of these situations happens, pick blocks until the picker gets updated. -func (pw *pickerWrapper) pick(ctx context.Context, failfast bool, info balancer.PickInfo) (transport.ClientTransport, func(balancer.DoneInfo), error) { +func (pw *pickerWrapper) pick(ctx context.Context, failfast bool, info balancer.PickInfo) (transport.ClientTransport, balancer.PickResult, error) { var ch chan struct{} var lastPickErr error + for { pw.mu.Lock() if pw.done { pw.mu.Unlock() - return nil, nil, ErrClientConnClosing + return nil, balancer.PickResult{}, ErrClientConnClosing } if pw.picker == nil { @@ -110,28 +126,45 @@ func (pw *pickerWrapper) pick(ctx context.Context, failfast bool, info balancer. } switch ctx.Err() { case context.DeadlineExceeded: - return nil, nil, status.Error(codes.DeadlineExceeded, errStr) + return nil, balancer.PickResult{}, status.Error(codes.DeadlineExceeded, errStr) case context.Canceled: - return nil, nil, status.Error(codes.Canceled, errStr) + return nil, balancer.PickResult{}, status.Error(codes.Canceled, errStr) } case <-ch: } continue } + // If the channel is set, it means that the pick call had to wait for a + // new picker at some point. Either it's the first iteration and this + // function received the first picker, or a picker errored with + // ErrNoSubConnAvailable or errored with failfast set to false, which + // will trigger a continue to the next iteration. In the first case this + // conditional will hit if this call had to block (the channel is set). + // In the second case, the only way it will get to this conditional is + // if there is a new picker. + if ch != nil { + for _, sh := range pw.statsHandlers { + sh.HandleRPC(ctx, &stats.PickerUpdated{}) + } + } + ch = pw.blockingCh p := pw.picker pw.mu.Unlock() pickResult, err := p.Pick(info) - if err != nil { if err == balancer.ErrNoSubConnAvailable { continue } - if _, ok := status.FromError(err); ok { + if st, ok := status.FromError(err); ok { // Status error: end the RPC unconditionally with this status. - return nil, nil, err + // First restrict the code to the list allowed by gRFC A54. + if istatus.IsRestrictedControlPlaneCode(st) { + err = status.Errorf(codes.Internal, "received picker error with illegal status: %v", err) + } + return nil, balancer.PickResult{}, dropError{error: err} } // For all other errors, wait for ready RPCs should block and other // RPCs should fail with unavailable. @@ -139,19 +172,20 @@ func (pw *pickerWrapper) pick(ctx context.Context, failfast bool, info balancer. lastPickErr = err continue } - return nil, nil, status.Error(codes.Unavailable, err.Error()) + return nil, balancer.PickResult{}, status.Error(codes.Unavailable, err.Error()) } - acw, ok := pickResult.SubConn.(*acBalancerWrapper) + acbw, ok := pickResult.SubConn.(*acBalancerWrapper) if !ok { - logger.Error("subconn returned from pick is not *acBalancerWrapper") + logger.Errorf("subconn returned from pick is type %T, not *acBalancerWrapper", pickResult.SubConn) continue } - if t, ok := acw.getAddrConn().getReadyTransport(); ok { + if t := acbw.ac.getReadyTransport(); t != nil { if channelz.IsOn() { - return t, doneChannelzWrapper(acw, pickResult.Done), nil + doneChannelzWrapper(acbw, &pickResult) + return t, pickResult, nil } - return t, pickResult.Done, nil + return t, pickResult, nil } if pickResult.Done != nil { // Calling done with nil error, no bytes sent and no bytes received. @@ -175,3 +209,28 @@ func (pw *pickerWrapper) close() { pw.done = true close(pw.blockingCh) } + +func (pw *pickerWrapper) enterIdleMode() { + pw.mu.Lock() + defer pw.mu.Unlock() + if pw.done { + return + } + pw.idle = true +} + +func (pw *pickerWrapper) exitIdleMode() { + pw.mu.Lock() + defer pw.mu.Unlock() + if pw.done { + return + } + pw.blockingCh = make(chan struct{}) + pw.idle = false +} + +// dropError is a wrapper error that indicates the LB policy wishes to drop the +// RPC and not retry it. +type dropError struct { + error +} diff --git a/picker_wrapper_test.go b/picker_wrapper_test.go index 5f786b28580e..ba99a06b0f8c 100644 --- a/picker_wrapper_test.go +++ b/picker_wrapper_test.go @@ -66,7 +66,7 @@ func (p *testingPicker) Pick(info balancer.PickInfo) (balancer.PickResult, error } func (s) TestBlockingPickTimeout(t *testing.T) { - bp := newPickerWrapper() + bp := newPickerWrapper(nil) ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond) defer cancel() if _, _, err := bp.pick(ctx, true, balancer.PickInfo{}); status.Code(err) != codes.DeadlineExceeded { @@ -75,7 +75,7 @@ func (s) TestBlockingPickTimeout(t *testing.T) { } func (s) TestBlockingPick(t *testing.T) { - bp := newPickerWrapper() + bp := newPickerWrapper(nil) // All goroutines should block because picker is nil in bp. var finishedCount uint64 for i := goroutineCount; i > 0; i-- { @@ -94,10 +94,10 @@ func (s) TestBlockingPick(t *testing.T) { } func (s) TestBlockingPickNoSubAvailable(t *testing.T) { - bp := newPickerWrapper() + bp := newPickerWrapper(nil) var finishedCount uint64 bp.updatePicker(&testingPicker{err: balancer.ErrNoSubConnAvailable, maxCalled: goroutineCount}) - // All goroutines should block because picker returns no sc available. + // All goroutines should block because picker returns no subConn available. for i := goroutineCount; i > 0; i-- { go func() { if tr, _, err := bp.pick(context.Background(), true, balancer.PickInfo{}); err != nil || tr != testT { @@ -114,7 +114,7 @@ func (s) TestBlockingPickNoSubAvailable(t *testing.T) { } func (s) TestBlockingPickTransientWaitforready(t *testing.T) { - bp := newPickerWrapper() + bp := newPickerWrapper(nil) bp.updatePicker(&testingPicker{err: balancer.ErrTransientFailure, maxCalled: goroutineCount}) var finishedCount uint64 // All goroutines should block because picker returns transientFailure and @@ -135,10 +135,10 @@ func (s) TestBlockingPickTransientWaitforready(t *testing.T) { } func (s) TestBlockingPickSCNotReady(t *testing.T) { - bp := newPickerWrapper() + bp := newPickerWrapper(nil) bp.updatePicker(&testingPicker{sc: testSCNotReady, maxCalled: goroutineCount}) var finishedCount uint64 - // All goroutines should block because sc is not ready. + // All goroutines should block because subConn is not ready. for i := goroutineCount; i > 0; i-- { go func() { if tr, _, err := bp.pick(context.Background(), true, balancer.PickInfo{}); err != nil || tr != testT { diff --git a/pickfirst.go b/pickfirst.go index 56e33f6c76b7..2e9cf66b4afc 100644 --- a/pickfirst.go +++ b/pickfirst.go @@ -19,15 +19,25 @@ package grpc import ( + "encoding/json" "errors" "fmt" "google.golang.org/grpc/balancer" "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/internal/envconfig" + internalgrpclog "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/grpcrand" + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" ) -// PickFirstBalancerName is the name of the pick_first balancer. -const PickFirstBalancerName = "pick_first" +const ( + // PickFirstBalancerName is the name of the pick_first balancer. + PickFirstBalancerName = "pick_first" + logPrefix = "[pick-first-lb %p] " +) func newPickfirstBuilder() balancer.Builder { return &pickfirstBuilder{} @@ -36,101 +46,218 @@ func newPickfirstBuilder() balancer.Builder { type pickfirstBuilder struct{} func (*pickfirstBuilder) Build(cc balancer.ClientConn, opt balancer.BuildOptions) balancer.Balancer { - return &pickfirstBalancer{cc: cc} + b := &pickfirstBalancer{cc: cc} + b.logger = internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(logPrefix, b)) + return b } func (*pickfirstBuilder) Name() string { return PickFirstBalancerName } +type pfConfig struct { + serviceconfig.LoadBalancingConfig `json:"-"` + + // If set to true, instructs the LB policy to shuffle the order of the list + // of addresses received from the name resolver before attempting to + // connect to them. + ShuffleAddressList bool `json:"shuffleAddressList"` +} + +func (*pickfirstBuilder) ParseConfig(js json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + if !envconfig.PickFirstLBConfig { + // Prior to supporting loadbalancing configuration, the pick_first LB + // policy did not implement the balancer.ConfigParser interface. This + // meant that if a non-empty configuration was passed to it, the service + // config unmarshaling code would throw a warning log, but would + // continue using the pick_first LB policy. The code below ensures the + // same behavior is retained if the env var is not set. + if string(js) != "{}" { + logger.Warningf("Ignoring non-empty balancer configuration %q for the pick_first LB policy", string(js)) + } + return nil, nil + } + + var cfg pfConfig + if err := json.Unmarshal(js, &cfg); err != nil { + return nil, fmt.Errorf("pickfirst: unable to unmarshal LB policy config: %s, error: %v", string(js), err) + } + return cfg, nil +} + type pickfirstBalancer struct { - state connectivity.State - cc balancer.ClientConn - sc balancer.SubConn + logger *internalgrpclog.PrefixLogger + state connectivity.State + cc balancer.ClientConn + subConn balancer.SubConn } func (b *pickfirstBalancer) ResolverError(err error) { - switch b.state { - case connectivity.TransientFailure, connectivity.Idle, connectivity.Connecting: - // Set a failing picker if we don't have a good picker. - b.cc.UpdateState(balancer.State{ConnectivityState: connectivity.TransientFailure, - Picker: &picker{err: fmt.Errorf("name resolver error: %v", err)}, - }) + if b.logger.V(2) { + b.logger.Infof("Received error from the name resolver: %v", err) } - if logger.V(2) { - logger.Infof("pickfirstBalancer: ResolverError called with error %v", err) + if b.subConn == nil { + b.state = connectivity.TransientFailure } + + if b.state != connectivity.TransientFailure { + // The picker will not change since the balancer does not currently + // report an error. + return + } + b.cc.UpdateState(balancer.State{ + ConnectivityState: connectivity.TransientFailure, + Picker: &picker{err: fmt.Errorf("name resolver error: %v", err)}, + }) } -func (b *pickfirstBalancer) UpdateClientConnState(cs balancer.ClientConnState) error { - if len(cs.ResolverState.Addresses) == 0 { +func (b *pickfirstBalancer) UpdateClientConnState(state balancer.ClientConnState) error { + addrs := state.ResolverState.Addresses + if len(addrs) == 0 { + // The resolver reported an empty address list. Treat it like an error by + // calling b.ResolverError. + if b.subConn != nil { + // Shut down the old subConn. All addresses were removed, so it is + // no longer valid. + b.subConn.Shutdown() + b.subConn = nil + } b.ResolverError(errors.New("produced zero addresses")) return balancer.ErrBadResolverState } - if b.sc == nil { - var err error - b.sc, err = b.cc.NewSubConn(cs.ResolverState.Addresses, balancer.NewSubConnOptions{}) - if err != nil { - if logger.V(2) { - logger.Errorf("pickfirstBalancer: failed to NewSubConn: %v", err) - } - b.state = connectivity.TransientFailure - b.cc.UpdateState(balancer.State{ConnectivityState: connectivity.TransientFailure, - Picker: &picker{err: fmt.Errorf("error creating connection: %v", err)}, - }) - return balancer.ErrBadResolverState + + // We don't have to guard this block with the env var because ParseConfig + // already does so. + cfg, ok := state.BalancerConfig.(pfConfig) + if state.BalancerConfig != nil && !ok { + return fmt.Errorf("pickfirst: received illegal BalancerConfig (type %T): %v", state.BalancerConfig, state.BalancerConfig) + } + if cfg.ShuffleAddressList { + addrs = append([]resolver.Address{}, addrs...) + grpcrand.Shuffle(len(addrs), func(i, j int) { addrs[i], addrs[j] = addrs[j], addrs[i] }) + } + + if b.logger.V(2) { + b.logger.Infof("Received new config %s, resolver state %s", pretty.ToJSON(cfg), pretty.ToJSON(state.ResolverState)) + } + + if b.subConn != nil { + b.cc.UpdateAddresses(b.subConn, addrs) + return nil + } + + var subConn balancer.SubConn + subConn, err := b.cc.NewSubConn(addrs, balancer.NewSubConnOptions{ + StateListener: func(state balancer.SubConnState) { + b.updateSubConnState(subConn, state) + }, + }) + if err != nil { + if b.logger.V(2) { + b.logger.Infof("Failed to create new SubConn: %v", err) } - b.state = connectivity.Idle - b.cc.UpdateState(balancer.State{ConnectivityState: connectivity.Idle, Picker: &picker{result: balancer.PickResult{SubConn: b.sc}}}) - b.sc.Connect() - } else { - b.sc.UpdateAddresses(cs.ResolverState.Addresses) - b.sc.Connect() + b.state = connectivity.TransientFailure + b.cc.UpdateState(balancer.State{ + ConnectivityState: connectivity.TransientFailure, + Picker: &picker{err: fmt.Errorf("error creating connection: %v", err)}, + }) + return balancer.ErrBadResolverState } + b.subConn = subConn + b.state = connectivity.Idle + b.cc.UpdateState(balancer.State{ + ConnectivityState: connectivity.Connecting, + Picker: &picker{err: balancer.ErrNoSubConnAvailable}, + }) + b.subConn.Connect() return nil } -func (b *pickfirstBalancer) UpdateSubConnState(sc balancer.SubConn, s balancer.SubConnState) { - if logger.V(2) { - logger.Infof("pickfirstBalancer: UpdateSubConnState: %p, %v", sc, s) +// UpdateSubConnState is unused as a StateListener is always registered when +// creating SubConns. +func (b *pickfirstBalancer) UpdateSubConnState(subConn balancer.SubConn, state balancer.SubConnState) { + b.logger.Errorf("UpdateSubConnState(%v, %+v) called unexpectedly", subConn, state) +} + +func (b *pickfirstBalancer) updateSubConnState(subConn balancer.SubConn, state balancer.SubConnState) { + if b.logger.V(2) { + b.logger.Infof("Received SubConn state update: %p, %+v", subConn, state) } - if b.sc != sc { - if logger.V(2) { - logger.Infof("pickfirstBalancer: ignored state change because sc is not recognized") + if b.subConn != subConn { + if b.logger.V(2) { + b.logger.Infof("Ignored state change because subConn is not recognized") } return } - b.state = s.ConnectivityState - if s.ConnectivityState == connectivity.Shutdown { - b.sc = nil + if state.ConnectivityState == connectivity.Shutdown { + b.subConn = nil return } - switch s.ConnectivityState { - case connectivity.Ready, connectivity.Idle: - b.cc.UpdateState(balancer.State{ConnectivityState: s.ConnectivityState, Picker: &picker{result: balancer.PickResult{SubConn: sc}}}) + switch state.ConnectivityState { + case connectivity.Ready: + b.cc.UpdateState(balancer.State{ + ConnectivityState: state.ConnectivityState, + Picker: &picker{result: balancer.PickResult{SubConn: subConn}}, + }) case connectivity.Connecting: - b.cc.UpdateState(balancer.State{ConnectivityState: s.ConnectivityState, Picker: &picker{err: balancer.ErrNoSubConnAvailable}}) + if b.state == connectivity.TransientFailure { + // We stay in TransientFailure until we are Ready. See A62. + return + } + b.cc.UpdateState(balancer.State{ + ConnectivityState: state.ConnectivityState, + Picker: &picker{err: balancer.ErrNoSubConnAvailable}, + }) + case connectivity.Idle: + if b.state == connectivity.TransientFailure { + // We stay in TransientFailure until we are Ready. Also kick the + // subConn out of Idle into Connecting. See A62. + b.subConn.Connect() + return + } + b.cc.UpdateState(balancer.State{ + ConnectivityState: state.ConnectivityState, + Picker: &idlePicker{subConn: subConn}, + }) case connectivity.TransientFailure: b.cc.UpdateState(balancer.State{ - ConnectivityState: s.ConnectivityState, - Picker: &picker{err: s.ConnectionError}, + ConnectivityState: state.ConnectivityState, + Picker: &picker{err: state.ConnectionError}, }) } + b.state = state.ConnectivityState } func (b *pickfirstBalancer) Close() { } +func (b *pickfirstBalancer) ExitIdle() { + if b.subConn != nil && b.state == connectivity.Idle { + b.subConn.Connect() + } +} + type picker struct { result balancer.PickResult err error } -func (p *picker) Pick(info balancer.PickInfo) (balancer.PickResult, error) { +func (p *picker) Pick(balancer.PickInfo) (balancer.PickResult, error) { return p.result, p.err } +// idlePicker is used when the SubConn is IDLE and kicks the SubConn into +// CONNECTING when Pick is called. +type idlePicker struct { + subConn balancer.SubConn +} + +func (i *idlePicker) Pick(balancer.PickInfo) (balancer.PickResult, error) { + i.subConn.Connect() + return balancer.PickResult{}, balancer.ErrNoSubConnAvailable +} + func init() { balancer.Register(newPickfirstBuilder()) } diff --git a/pickfirst_test.go b/pickfirst_test.go deleted file mode 100644 index 9ece7844a355..000000000000 --- a/pickfirst_test.go +++ /dev/null @@ -1,348 +0,0 @@ -/* - * - * Copyright 2017 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package grpc - -import ( - "context" - "math" - "sync" - "testing" - "time" - - "google.golang.org/grpc/codes" - "google.golang.org/grpc/resolver" - "google.golang.org/grpc/resolver/manual" - "google.golang.org/grpc/status" -) - -func errorDesc(err error) string { - if s, ok := status.FromError(err); ok { - return s.Message() - } - return err.Error() -} - -func (s) TestOneBackendPickfirst(t *testing.T) { - r := manual.NewBuilderWithScheme("whatever") - - numServers := 1 - servers, scleanup := startServers(t, numServers, math.MaxInt32) - defer scleanup() - - cc, err := Dial(r.Scheme()+":///test.server", - WithInsecure(), - WithResolvers(r), - WithCodec(testCodec{})) - if err != nil { - t.Fatalf("failed to dial: %v", err) - } - defer cc.Close() - // The first RPC should fail because there's no address. - ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond) - defer cancel() - req := "port" - var reply string - if err := cc.Invoke(ctx, "/foo/bar", &req, &reply); err == nil || status.Code(err) != codes.DeadlineExceeded { - t.Fatalf("EmptyCall() = _, %v, want _, DeadlineExceeded", err) - } - - r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: servers[0].addr}}}) - // The second RPC should succeed. - for i := 0; i < 1000; i++ { - if err = cc.Invoke(context.Background(), "/foo/bar", &req, &reply); err != nil && errorDesc(err) == servers[0].port { - return - } - time.Sleep(time.Millisecond) - } - t.Fatalf("EmptyCall() = _, %v, want _, %v", err, servers[0].port) -} - -func (s) TestBackendsPickfirst(t *testing.T) { - r := manual.NewBuilderWithScheme("whatever") - - numServers := 2 - servers, scleanup := startServers(t, numServers, math.MaxInt32) - defer scleanup() - - cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithResolvers(r), WithCodec(testCodec{})) - if err != nil { - t.Fatalf("failed to dial: %v", err) - } - defer cc.Close() - // The first RPC should fail because there's no address. - ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond) - defer cancel() - req := "port" - var reply string - if err := cc.Invoke(ctx, "/foo/bar", &req, &reply); err == nil || status.Code(err) != codes.DeadlineExceeded { - t.Fatalf("EmptyCall() = _, %v, want _, DeadlineExceeded", err) - } - - r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: servers[0].addr}, {Addr: servers[1].addr}}}) - // The second RPC should succeed with the first server. - for i := 0; i < 1000; i++ { - if err = cc.Invoke(context.Background(), "/foo/bar", &req, &reply); err != nil && errorDesc(err) == servers[0].port { - return - } - time.Sleep(time.Millisecond) - } - t.Fatalf("EmptyCall() = _, %v, want _, %v", err, servers[0].port) -} - -func (s) TestNewAddressWhileBlockingPickfirst(t *testing.T) { - r := manual.NewBuilderWithScheme("whatever") - - numServers := 1 - servers, scleanup := startServers(t, numServers, math.MaxInt32) - defer scleanup() - - cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithResolvers(r), WithCodec(testCodec{})) - if err != nil { - t.Fatalf("failed to dial: %v", err) - } - defer cc.Close() - // The first RPC should fail because there's no address. - ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond) - defer cancel() - req := "port" - var reply string - if err := cc.Invoke(ctx, "/foo/bar", &req, &reply); err == nil || status.Code(err) != codes.DeadlineExceeded { - t.Fatalf("EmptyCall() = _, %v, want _, DeadlineExceeded", err) - } - - var wg sync.WaitGroup - for i := 0; i < 3; i++ { - wg.Add(1) - go func() { - defer wg.Done() - // This RPC blocks until NewAddress is called. - cc.Invoke(context.Background(), "/foo/bar", &req, &reply) - }() - } - time.Sleep(50 * time.Millisecond) - r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: servers[0].addr}}}) - wg.Wait() -} - -func (s) TestCloseWithPendingRPCPickfirst(t *testing.T) { - r := manual.NewBuilderWithScheme("whatever") - - numServers := 1 - _, scleanup := startServers(t, numServers, math.MaxInt32) - defer scleanup() - - cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithResolvers(r), WithCodec(testCodec{})) - if err != nil { - t.Fatalf("failed to dial: %v", err) - } - defer cc.Close() - // The first RPC should fail because there's no address. - ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond) - defer cancel() - req := "port" - var reply string - if err := cc.Invoke(ctx, "/foo/bar", &req, &reply); err == nil || status.Code(err) != codes.DeadlineExceeded { - t.Fatalf("EmptyCall() = _, %v, want _, DeadlineExceeded", err) - } - - var wg sync.WaitGroup - for i := 0; i < 3; i++ { - wg.Add(1) - go func() { - defer wg.Done() - // This RPC blocks until NewAddress is called. - cc.Invoke(context.Background(), "/foo/bar", &req, &reply) - }() - } - time.Sleep(50 * time.Millisecond) - cc.Close() - wg.Wait() -} - -func (s) TestOneServerDownPickfirst(t *testing.T) { - r := manual.NewBuilderWithScheme("whatever") - - numServers := 2 - servers, scleanup := startServers(t, numServers, math.MaxInt32) - defer scleanup() - - cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithResolvers(r), WithCodec(testCodec{})) - if err != nil { - t.Fatalf("failed to dial: %v", err) - } - defer cc.Close() - // The first RPC should fail because there's no address. - ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond) - defer cancel() - req := "port" - var reply string - if err := cc.Invoke(ctx, "/foo/bar", &req, &reply); err == nil || status.Code(err) != codes.DeadlineExceeded { - t.Fatalf("EmptyCall() = _, %v, want _, DeadlineExceeded", err) - } - - r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: servers[0].addr}, {Addr: servers[1].addr}}}) - // The second RPC should succeed with the first server. - for i := 0; i < 1000; i++ { - if err = cc.Invoke(context.Background(), "/foo/bar", &req, &reply); err != nil && errorDesc(err) == servers[0].port { - break - } - time.Sleep(time.Millisecond) - } - - servers[0].stop() - for i := 0; i < 1000; i++ { - if err = cc.Invoke(context.Background(), "/foo/bar", &req, &reply); err != nil && errorDesc(err) == servers[1].port { - return - } - time.Sleep(time.Millisecond) - } - t.Fatalf("EmptyCall() = _, %v, want _, %v", err, servers[0].port) -} - -func (s) TestAllServersDownPickfirst(t *testing.T) { - r := manual.NewBuilderWithScheme("whatever") - - numServers := 2 - servers, scleanup := startServers(t, numServers, math.MaxInt32) - defer scleanup() - - cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithResolvers(r), WithCodec(testCodec{})) - if err != nil { - t.Fatalf("failed to dial: %v", err) - } - defer cc.Close() - // The first RPC should fail because there's no address. - ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond) - defer cancel() - req := "port" - var reply string - if err := cc.Invoke(ctx, "/foo/bar", &req, &reply); err == nil || status.Code(err) != codes.DeadlineExceeded { - t.Fatalf("EmptyCall() = _, %v, want _, DeadlineExceeded", err) - } - - r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: servers[0].addr}, {Addr: servers[1].addr}}}) - // The second RPC should succeed with the first server. - for i := 0; i < 1000; i++ { - if err = cc.Invoke(context.Background(), "/foo/bar", &req, &reply); err != nil && errorDesc(err) == servers[0].port { - break - } - time.Sleep(time.Millisecond) - } - - for i := 0; i < numServers; i++ { - servers[i].stop() - } - for i := 0; i < 1000; i++ { - if err = cc.Invoke(context.Background(), "/foo/bar", &req, &reply); status.Code(err) == codes.Unavailable { - return - } - time.Sleep(time.Millisecond) - } - t.Fatalf("EmptyCall() = _, %v, want _, error with code unavailable", err) -} - -func (s) TestAddressesRemovedPickfirst(t *testing.T) { - r := manual.NewBuilderWithScheme("whatever") - - numServers := 3 - servers, scleanup := startServers(t, numServers, math.MaxInt32) - defer scleanup() - - cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithResolvers(r), WithCodec(testCodec{})) - if err != nil { - t.Fatalf("failed to dial: %v", err) - } - defer cc.Close() - // The first RPC should fail because there's no address. - ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond) - defer cancel() - req := "port" - var reply string - if err := cc.Invoke(ctx, "/foo/bar", &req, &reply); err == nil || status.Code(err) != codes.DeadlineExceeded { - t.Fatalf("EmptyCall() = _, %v, want _, DeadlineExceeded", err) - } - - r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: servers[0].addr}, {Addr: servers[1].addr}, {Addr: servers[2].addr}}}) - for i := 0; i < 1000; i++ { - if err = cc.Invoke(context.Background(), "/foo/bar", &req, &reply); err != nil && errorDesc(err) == servers[0].port { - break - } - time.Sleep(time.Millisecond) - } - for i := 0; i < 20; i++ { - if err := cc.Invoke(context.Background(), "/foo/bar", &req, &reply); err == nil || errorDesc(err) != servers[0].port { - t.Fatalf("Index %d: Invoke(_, _, _, _, _) = %v, want %s", 0, err, servers[0].port) - } - time.Sleep(10 * time.Millisecond) - } - - // Remove server[0]. - r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: servers[1].addr}, {Addr: servers[2].addr}}}) - for i := 0; i < 1000; i++ { - if err = cc.Invoke(context.Background(), "/foo/bar", &req, &reply); err != nil && errorDesc(err) == servers[1].port { - break - } - time.Sleep(time.Millisecond) - } - for i := 0; i < 20; i++ { - if err := cc.Invoke(context.Background(), "/foo/bar", &req, &reply); err == nil || errorDesc(err) != servers[1].port { - t.Fatalf("Index %d: Invoke(_, _, _, _, _) = %v, want %s", 1, err, servers[1].port) - } - time.Sleep(10 * time.Millisecond) - } - - // Append server[0], nothing should change. - r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: servers[1].addr}, {Addr: servers[2].addr}, {Addr: servers[0].addr}}}) - for i := 0; i < 20; i++ { - if err := cc.Invoke(context.Background(), "/foo/bar", &req, &reply); err == nil || errorDesc(err) != servers[1].port { - t.Fatalf("Index %d: Invoke(_, _, _, _, _) = %v, want %s", 1, err, servers[1].port) - } - time.Sleep(10 * time.Millisecond) - } - - // Remove server[1]. - r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: servers[2].addr}, {Addr: servers[0].addr}}}) - for i := 0; i < 1000; i++ { - if err = cc.Invoke(context.Background(), "/foo/bar", &req, &reply); err != nil && errorDesc(err) == servers[2].port { - break - } - time.Sleep(time.Millisecond) - } - for i := 0; i < 20; i++ { - if err := cc.Invoke(context.Background(), "/foo/bar", &req, &reply); err == nil || errorDesc(err) != servers[2].port { - t.Fatalf("Index %d: Invoke(_, _, _, _, _) = %v, want %s", 2, err, servers[2].port) - } - time.Sleep(10 * time.Millisecond) - } - - // Remove server[2]. - r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: servers[0].addr}}}) - for i := 0; i < 1000; i++ { - if err = cc.Invoke(context.Background(), "/foo/bar", &req, &reply); err != nil && errorDesc(err) == servers[0].port { - break - } - time.Sleep(time.Millisecond) - } - for i := 0; i < 20; i++ { - if err := cc.Invoke(context.Background(), "/foo/bar", &req, &reply); err == nil || errorDesc(err) != servers[0].port { - t.Fatalf("Index %d: Invoke(_, _, _, _, _) = %v, want %s", 0, err, servers[0].port) - } - time.Sleep(10 * time.Millisecond) - } -} diff --git a/preloader.go b/preloader.go index 76acbbcc93b9..73bd63364335 100644 --- a/preloader.go +++ b/preloader.go @@ -25,7 +25,10 @@ import ( // PreparedMsg is responsible for creating a Marshalled and Compressed object. // -// This API is EXPERIMENTAL. +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. type PreparedMsg struct { // Struct for preparing msg before sending them encodedData []byte @@ -34,7 +37,7 @@ type PreparedMsg struct { } // Encode marshalls and compresses the message using the codec and compressor for the stream. -func (p *PreparedMsg) Encode(s Stream, msg interface{}) error { +func (p *PreparedMsg) Encode(s Stream, msg any) error { ctx := s.Context() rpcInfo, ok := rpcInfoFromContext(ctx) if !ok { diff --git a/profiling/cmd/remote.go b/profiling/cmd/remote.go index b6adfd6a6bef..71c21c332b74 100644 --- a/profiling/cmd/remote.go +++ b/profiling/cmd/remote.go @@ -26,6 +26,7 @@ import ( "time" "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" ppb "google.golang.org/grpc/profiling/proto" ) @@ -78,7 +79,7 @@ func remoteCommand() error { } logger.Infof("dialing %s", *flagAddress) - cc, err := grpc.Dial(*flagAddress, grpc.WithInsecure()) + cc, err := grpc.Dial(*flagAddress, grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { logger.Errorf("cannot dial %s: %v", *flagAddress, err) return err diff --git a/profiling/profiling.go b/profiling/profiling.go index 6adcc1fac68a..869054ea794a 100644 --- a/profiling/profiling.go +++ b/profiling/profiling.go @@ -18,7 +18,10 @@ // Package profiling exposes methods to manage profiling within gRPC. // -// This package and all its methods are EXPERIMENTAL. +// # Experimental +// +// Notice: This package is EXPERIMENTAL and may be changed or removed in a +// later release. package profiling import ( diff --git a/profiling/proto/service.pb.go b/profiling/proto/service.pb.go index 90f02824ef69..c92c22a0b2b6 100644 --- a/profiling/proto/service.pb.go +++ b/profiling/proto/service.pb.go @@ -1,174 +1,214 @@ +// Copyright 2019 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.31.0 +// protoc v4.22.0 // source: profiling/proto/service.proto package proto import ( - context "context" - fmt "fmt" - proto "github.com/golang/protobuf/proto" - grpc "google.golang.org/grpc" - codes "google.golang.org/grpc/codes" - status "google.golang.org/grpc/status" - math "math" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" ) -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) // EnableRequest defines the fields in a /Profiling/Enable method request to // toggle profiling on and off within a gRPC program. type EnableRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // Setting this to true will enable profiling. Setting this to false will // disable profiling. - Enabled bool `protobuf:"varint,1,opt,name=enabled,proto3" json:"enabled,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Enabled bool `protobuf:"varint,1,opt,name=enabled,proto3" json:"enabled,omitempty"` } -func (m *EnableRequest) Reset() { *m = EnableRequest{} } -func (m *EnableRequest) String() string { return proto.CompactTextString(m) } -func (*EnableRequest) ProtoMessage() {} -func (*EnableRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_e1ab2aa17b47c6fb, []int{0} +func (x *EnableRequest) Reset() { + *x = EnableRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_profiling_proto_service_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *EnableRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_EnableRequest.Unmarshal(m, b) -} -func (m *EnableRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_EnableRequest.Marshal(b, m, deterministic) +func (x *EnableRequest) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *EnableRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_EnableRequest.Merge(m, src) -} -func (m *EnableRequest) XXX_Size() int { - return xxx_messageInfo_EnableRequest.Size(m) -} -func (m *EnableRequest) XXX_DiscardUnknown() { - xxx_messageInfo_EnableRequest.DiscardUnknown(m) + +func (*EnableRequest) ProtoMessage() {} + +func (x *EnableRequest) ProtoReflect() protoreflect.Message { + mi := &file_profiling_proto_service_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_EnableRequest proto.InternalMessageInfo +// Deprecated: Use EnableRequest.ProtoReflect.Descriptor instead. +func (*EnableRequest) Descriptor() ([]byte, []int) { + return file_profiling_proto_service_proto_rawDescGZIP(), []int{0} +} -func (m *EnableRequest) GetEnabled() bool { - if m != nil { - return m.Enabled +func (x *EnableRequest) GetEnabled() bool { + if x != nil { + return x.Enabled } return false } // EnableResponse defines the fields in a /Profiling/Enable method response. type EnableResponse struct { - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields } -func (m *EnableResponse) Reset() { *m = EnableResponse{} } -func (m *EnableResponse) String() string { return proto.CompactTextString(m) } -func (*EnableResponse) ProtoMessage() {} -func (*EnableResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_e1ab2aa17b47c6fb, []int{1} +func (x *EnableResponse) Reset() { + *x = EnableResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_profiling_proto_service_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *EnableResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_EnableResponse.Unmarshal(m, b) -} -func (m *EnableResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_EnableResponse.Marshal(b, m, deterministic) +func (x *EnableResponse) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *EnableResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_EnableResponse.Merge(m, src) -} -func (m *EnableResponse) XXX_Size() int { - return xxx_messageInfo_EnableResponse.Size(m) -} -func (m *EnableResponse) XXX_DiscardUnknown() { - xxx_messageInfo_EnableResponse.DiscardUnknown(m) + +func (*EnableResponse) ProtoMessage() {} + +func (x *EnableResponse) ProtoReflect() protoreflect.Message { + mi := &file_profiling_proto_service_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_EnableResponse proto.InternalMessageInfo +// Deprecated: Use EnableResponse.ProtoReflect.Descriptor instead. +func (*EnableResponse) Descriptor() ([]byte, []int) { + return file_profiling_proto_service_proto_rawDescGZIP(), []int{1} +} // GetStreamStatsRequest defines the fields in a /Profiling/GetStreamStats // method request to retrieve stream-level stats in a gRPC client/server. type GetStreamStatsRequest struct { - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields } -func (m *GetStreamStatsRequest) Reset() { *m = GetStreamStatsRequest{} } -func (m *GetStreamStatsRequest) String() string { return proto.CompactTextString(m) } -func (*GetStreamStatsRequest) ProtoMessage() {} -func (*GetStreamStatsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_e1ab2aa17b47c6fb, []int{2} +func (x *GetStreamStatsRequest) Reset() { + *x = GetStreamStatsRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_profiling_proto_service_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *GetStreamStatsRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_GetStreamStatsRequest.Unmarshal(m, b) -} -func (m *GetStreamStatsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_GetStreamStatsRequest.Marshal(b, m, deterministic) +func (x *GetStreamStatsRequest) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *GetStreamStatsRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_GetStreamStatsRequest.Merge(m, src) -} -func (m *GetStreamStatsRequest) XXX_Size() int { - return xxx_messageInfo_GetStreamStatsRequest.Size(m) -} -func (m *GetStreamStatsRequest) XXX_DiscardUnknown() { - xxx_messageInfo_GetStreamStatsRequest.DiscardUnknown(m) + +func (*GetStreamStatsRequest) ProtoMessage() {} + +func (x *GetStreamStatsRequest) ProtoReflect() protoreflect.Message { + mi := &file_profiling_proto_service_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_GetStreamStatsRequest proto.InternalMessageInfo +// Deprecated: Use GetStreamStatsRequest.ProtoReflect.Descriptor instead. +func (*GetStreamStatsRequest) Descriptor() ([]byte, []int) { + return file_profiling_proto_service_proto_rawDescGZIP(), []int{2} +} // GetStreamStatsResponse defines the fields in a /Profiling/GetStreamStats // method response. type GetStreamStatsResponse struct { - StreamStats []*Stat `protobuf:"bytes,1,rep,name=stream_stats,json=streamStats,proto3" json:"stream_stats,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields -func (m *GetStreamStatsResponse) Reset() { *m = GetStreamStatsResponse{} } -func (m *GetStreamStatsResponse) String() string { return proto.CompactTextString(m) } -func (*GetStreamStatsResponse) ProtoMessage() {} -func (*GetStreamStatsResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_e1ab2aa17b47c6fb, []int{3} + StreamStats []*Stat `protobuf:"bytes,1,rep,name=stream_stats,json=streamStats,proto3" json:"stream_stats,omitempty"` } -func (m *GetStreamStatsResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_GetStreamStatsResponse.Unmarshal(m, b) -} -func (m *GetStreamStatsResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_GetStreamStatsResponse.Marshal(b, m, deterministic) -} -func (m *GetStreamStatsResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_GetStreamStatsResponse.Merge(m, src) +func (x *GetStreamStatsResponse) Reset() { + *x = GetStreamStatsResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_profiling_proto_service_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *GetStreamStatsResponse) XXX_Size() int { - return xxx_messageInfo_GetStreamStatsResponse.Size(m) + +func (x *GetStreamStatsResponse) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *GetStreamStatsResponse) XXX_DiscardUnknown() { - xxx_messageInfo_GetStreamStatsResponse.DiscardUnknown(m) + +func (*GetStreamStatsResponse) ProtoMessage() {} + +func (x *GetStreamStatsResponse) ProtoReflect() protoreflect.Message { + mi := &file_profiling_proto_service_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_GetStreamStatsResponse proto.InternalMessageInfo +// Deprecated: Use GetStreamStatsResponse.ProtoReflect.Descriptor instead. +func (*GetStreamStatsResponse) Descriptor() ([]byte, []int) { + return file_profiling_proto_service_proto_rawDescGZIP(), []int{3} +} -func (m *GetStreamStatsResponse) GetStreamStats() []*Stat { - if m != nil { - return m.StreamStats +func (x *GetStreamStatsResponse) GetStreamStats() []*Stat { + if x != nil { + return x.StreamStats } return nil } @@ -177,6 +217,10 @@ func (m *GetStreamStatsResponse) GetStreamStats() []*Stat { // gRPC that's being profiled. It includes a tag and some additional metadata // to identify itself. type Timer struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // tags is a comma-separated list of strings used to tag a timer. Tags string `protobuf:"bytes,1,opt,name=tags,proto3" json:"tags,omitempty"` // begin_sec and begin_nsec are the start epoch second and nanosecond, @@ -190,75 +234,79 @@ type Timer struct { EndSec int64 `protobuf:"varint,4,opt,name=end_sec,json=endSec,proto3" json:"end_sec,omitempty"` EndNsec int32 `protobuf:"varint,5,opt,name=end_nsec,json=endNsec,proto3" json:"end_nsec,omitempty"` // go_id is the goroutine ID of the component being profiled. - GoId int64 `protobuf:"varint,6,opt,name=go_id,json=goId,proto3" json:"go_id,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + GoId int64 `protobuf:"varint,6,opt,name=go_id,json=goId,proto3" json:"go_id,omitempty"` } -func (m *Timer) Reset() { *m = Timer{} } -func (m *Timer) String() string { return proto.CompactTextString(m) } -func (*Timer) ProtoMessage() {} -func (*Timer) Descriptor() ([]byte, []int) { - return fileDescriptor_e1ab2aa17b47c6fb, []int{4} +func (x *Timer) Reset() { + *x = Timer{} + if protoimpl.UnsafeEnabled { + mi := &file_profiling_proto_service_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *Timer) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Timer.Unmarshal(m, b) +func (x *Timer) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *Timer) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Timer.Marshal(b, m, deterministic) -} -func (m *Timer) XXX_Merge(src proto.Message) { - xxx_messageInfo_Timer.Merge(m, src) -} -func (m *Timer) XXX_Size() int { - return xxx_messageInfo_Timer.Size(m) -} -func (m *Timer) XXX_DiscardUnknown() { - xxx_messageInfo_Timer.DiscardUnknown(m) + +func (*Timer) ProtoMessage() {} + +func (x *Timer) ProtoReflect() protoreflect.Message { + mi := &file_profiling_proto_service_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_Timer proto.InternalMessageInfo +// Deprecated: Use Timer.ProtoReflect.Descriptor instead. +func (*Timer) Descriptor() ([]byte, []int) { + return file_profiling_proto_service_proto_rawDescGZIP(), []int{4} +} -func (m *Timer) GetTags() string { - if m != nil { - return m.Tags +func (x *Timer) GetTags() string { + if x != nil { + return x.Tags } return "" } -func (m *Timer) GetBeginSec() int64 { - if m != nil { - return m.BeginSec +func (x *Timer) GetBeginSec() int64 { + if x != nil { + return x.BeginSec } return 0 } -func (m *Timer) GetBeginNsec() int32 { - if m != nil { - return m.BeginNsec +func (x *Timer) GetBeginNsec() int32 { + if x != nil { + return x.BeginNsec } return 0 } -func (m *Timer) GetEndSec() int64 { - if m != nil { - return m.EndSec +func (x *Timer) GetEndSec() int64 { + if x != nil { + return x.EndSec } return 0 } -func (m *Timer) GetEndNsec() int32 { - if m != nil { - return m.EndNsec +func (x *Timer) GetEndNsec() int32 { + if x != nil { + return x.EndNsec } return 0 } -func (m *Timer) GetGoId() int64 { - if m != nil { - return m.GoId +func (x *Timer) GetGoId() int64 { + if x != nil { + return x.GoId } return 0 } @@ -266,6 +314,10 @@ func (m *Timer) GetGoId() int64 { // A Stat is a collection of Timers along with some additional // metadata to tag and identify itself. type Stat struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // tags is a comma-separated list of strings used to categorize a stat. Tags string `protobuf:"bytes,1,opt,name=tags,proto3" json:"tags,omitempty"` // timers is an array of Timers, each representing a different @@ -275,216 +327,247 @@ type Stat struct { // undefined encoding format. For example, the Stats returned by the // /Profiling/GetStreamStats service use the metadata field to encode the // connection ID and the stream ID of each query. - Metadata []byte `protobuf:"bytes,3,opt,name=metadata,proto3" json:"metadata,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Metadata []byte `protobuf:"bytes,3,opt,name=metadata,proto3" json:"metadata,omitempty"` } -func (m *Stat) Reset() { *m = Stat{} } -func (m *Stat) String() string { return proto.CompactTextString(m) } -func (*Stat) ProtoMessage() {} -func (*Stat) Descriptor() ([]byte, []int) { - return fileDescriptor_e1ab2aa17b47c6fb, []int{5} +func (x *Stat) Reset() { + *x = Stat{} + if protoimpl.UnsafeEnabled { + mi := &file_profiling_proto_service_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *Stat) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Stat.Unmarshal(m, b) -} -func (m *Stat) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Stat.Marshal(b, m, deterministic) -} -func (m *Stat) XXX_Merge(src proto.Message) { - xxx_messageInfo_Stat.Merge(m, src) -} -func (m *Stat) XXX_Size() int { - return xxx_messageInfo_Stat.Size(m) -} -func (m *Stat) XXX_DiscardUnknown() { - xxx_messageInfo_Stat.DiscardUnknown(m) +func (x *Stat) String() string { + return protoimpl.X.MessageStringOf(x) } -var xxx_messageInfo_Stat proto.InternalMessageInfo +func (*Stat) ProtoMessage() {} -func (m *Stat) GetTags() string { - if m != nil { - return m.Tags +func (x *Stat) ProtoReflect() protoreflect.Message { + mi := &file_profiling_proto_service_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms } - return "" + return mi.MessageOf(x) } -func (m *Stat) GetTimers() []*Timer { - if m != nil { - return m.Timers - } - return nil +// Deprecated: Use Stat.ProtoReflect.Descriptor instead. +func (*Stat) Descriptor() ([]byte, []int) { + return file_profiling_proto_service_proto_rawDescGZIP(), []int{5} } -func (m *Stat) GetMetadata() []byte { - if m != nil { - return m.Metadata +func (x *Stat) GetTags() string { + if x != nil { + return x.Tags } - return nil -} - -func init() { - proto.RegisterType((*EnableRequest)(nil), "grpc.go.profiling.v1alpha.EnableRequest") - proto.RegisterType((*EnableResponse)(nil), "grpc.go.profiling.v1alpha.EnableResponse") - proto.RegisterType((*GetStreamStatsRequest)(nil), "grpc.go.profiling.v1alpha.GetStreamStatsRequest") - proto.RegisterType((*GetStreamStatsResponse)(nil), "grpc.go.profiling.v1alpha.GetStreamStatsResponse") - proto.RegisterType((*Timer)(nil), "grpc.go.profiling.v1alpha.Timer") - proto.RegisterType((*Stat)(nil), "grpc.go.profiling.v1alpha.Stat") -} - -func init() { proto.RegisterFile("profiling/proto/service.proto", fileDescriptor_e1ab2aa17b47c6fb) } - -var fileDescriptor_e1ab2aa17b47c6fb = []byte{ - // 392 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x92, 0x41, 0xab, 0xd3, 0x40, - 0x10, 0xc7, 0xc9, 0x6b, 0x12, 0xd3, 0x79, 0xcf, 0x87, 0xac, 0xe8, 0xcb, 0xab, 0x14, 0x43, 0x0e, - 0x92, 0x5e, 0x36, 0xb6, 0x5e, 0x3c, 0x17, 0x44, 0xbc, 0x88, 0x6c, 0x3d, 0x89, 0x52, 0xb6, 0xc9, - 0xb8, 0x06, 0xd2, 0x6c, 0xcc, 0x6e, 0xfb, 0x79, 0xfc, 0x6a, 0x7e, 0x13, 0xd9, 0x49, 0x5b, 0x69, - 0xa9, 0xc5, 0x77, 0x4a, 0x66, 0xe6, 0xff, 0x9b, 0xfd, 0x0f, 0x33, 0x30, 0x6e, 0x3b, 0xfd, 0xbd, - 0xaa, 0xab, 0x46, 0xe5, 0x6d, 0xa7, 0xad, 0xce, 0x0d, 0x76, 0xdb, 0xaa, 0x40, 0x4e, 0x11, 0xbb, - 0x57, 0x5d, 0x5b, 0x70, 0xa5, 0xf9, 0x41, 0xc6, 0xb7, 0x53, 0x59, 0xb7, 0x3f, 0x64, 0x3a, 0x81, - 0xc7, 0xef, 0x1a, 0xb9, 0xaa, 0x51, 0xe0, 0xcf, 0x0d, 0x1a, 0xcb, 0x62, 0x78, 0x84, 0x94, 0x28, - 0x63, 0x2f, 0xf1, 0xb2, 0x48, 0xec, 0xc3, 0xf4, 0x09, 0xdc, 0xee, 0xa5, 0xa6, 0xd5, 0x8d, 0xc1, - 0xf4, 0x0e, 0x9e, 0xbd, 0x47, 0xbb, 0xb0, 0x1d, 0xca, 0xf5, 0xc2, 0x4a, 0x6b, 0x76, 0x4d, 0xd2, - 0xaf, 0xf0, 0xfc, 0xb4, 0xd0, 0x23, 0x6c, 0x0e, 0x37, 0x86, 0xd2, 0x4b, 0xe3, 0xf2, 0xb1, 0x97, - 0x0c, 0xb2, 0xeb, 0xd9, 0x4b, 0xfe, 0x4f, 0x87, 0xdc, 0xf1, 0xe2, 0xda, 0xfc, 0xed, 0x95, 0xfe, - 0xf2, 0x20, 0xf8, 0x5c, 0xad, 0xb1, 0x63, 0x0c, 0x7c, 0x2b, 0x95, 0x21, 0xa7, 0x43, 0x41, 0xff, - 0xec, 0x05, 0x0c, 0x57, 0xa8, 0xaa, 0x66, 0x69, 0xb0, 0x88, 0xaf, 0x12, 0x2f, 0x1b, 0x88, 0x88, - 0x12, 0x0b, 0x2c, 0xd8, 0x18, 0xa0, 0x2f, 0x36, 0xae, 0x3a, 0x48, 0xbc, 0x2c, 0x10, 0xbd, 0xfc, - 0xa3, 0xc1, 0x82, 0xdd, 0xb9, 0xe1, 0x4b, 0x22, 0x7d, 0x22, 0x43, 0x6c, 0x4a, 0xc7, 0xdd, 0x43, - 0xe4, 0x0a, 0x44, 0x05, 0x44, 0x39, 0x21, 0x31, 0x4f, 0x21, 0x50, 0x7a, 0x59, 0x95, 0x71, 0x48, - 0x84, 0xaf, 0xf4, 0x87, 0x32, 0x6d, 0xc1, 0x77, 0x5e, 0xcf, 0x1a, 0x7c, 0x0b, 0xa1, 0x75, 0xee, - 0x4d, 0x7c, 0x45, 0xc3, 0x27, 0x17, 0x86, 0xa7, 0x31, 0xc5, 0x4e, 0xcf, 0x46, 0x10, 0xad, 0xd1, - 0xca, 0x52, 0x5a, 0x49, 0xde, 0x6f, 0xc4, 0x21, 0x9e, 0xfd, 0xf6, 0x60, 0xf8, 0x69, 0xcf, 0xb3, - 0x6f, 0x10, 0xf6, 0xbb, 0x62, 0xd9, 0x85, 0xee, 0x47, 0x9b, 0x1f, 0x4d, 0xfe, 0x43, 0xb9, 0xdb, - 0xe2, 0x06, 0x6e, 0x8f, 0xf7, 0xcb, 0x5e, 0x5f, 0x80, 0xcf, 0xde, 0xc8, 0x68, 0xfa, 0x00, 0xa2, - 0x7f, 0x76, 0x9e, 0x7d, 0x79, 0xa5, 0xb4, 0x56, 0x35, 0x72, 0xa5, 0x6b, 0xd9, 0x28, 0xae, 0x3b, - 0x95, 0xbb, 0x2e, 0xf9, 0xc9, 0xfd, 0xaf, 0x42, 0xfa, 0xbc, 0xf9, 0x13, 0x00, 0x00, 0xff, 0xff, - 0x5d, 0x47, 0x09, 0xa9, 0x19, 0x03, 0x00, 0x00, -} - -// Reference imports to suppress errors if they are not otherwise used. -var _ context.Context -var _ grpc.ClientConnInterface - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the grpc package it is being compiled against. -const _ = grpc.SupportPackageIsVersion6 - -// ProfilingClient is the client API for Profiling service. -// -// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. -type ProfilingClient interface { - // Enable allows users to toggle profiling on and off remotely. - Enable(ctx context.Context, in *EnableRequest, opts ...grpc.CallOption) (*EnableResponse, error) - // GetStreamStats is used to retrieve an array of stream-level stats from a - // gRPC client/server. - GetStreamStats(ctx context.Context, in *GetStreamStatsRequest, opts ...grpc.CallOption) (*GetStreamStatsResponse, error) -} - -type profilingClient struct { - cc grpc.ClientConnInterface -} - -func NewProfilingClient(cc grpc.ClientConnInterface) ProfilingClient { - return &profilingClient{cc} + return "" } -func (c *profilingClient) Enable(ctx context.Context, in *EnableRequest, opts ...grpc.CallOption) (*EnableResponse, error) { - out := new(EnableResponse) - err := c.cc.Invoke(ctx, "/grpc.go.profiling.v1alpha.Profiling/Enable", in, out, opts...) - if err != nil { - return nil, err +func (x *Stat) GetTimers() []*Timer { + if x != nil { + return x.Timers } - return out, nil + return nil } -func (c *profilingClient) GetStreamStats(ctx context.Context, in *GetStreamStatsRequest, opts ...grpc.CallOption) (*GetStreamStatsResponse, error) { - out := new(GetStreamStatsResponse) - err := c.cc.Invoke(ctx, "/grpc.go.profiling.v1alpha.Profiling/GetStreamStats", in, out, opts...) - if err != nil { - return nil, err +func (x *Stat) GetMetadata() []byte { + if x != nil { + return x.Metadata } - return out, nil -} - -// ProfilingServer is the server API for Profiling service. -type ProfilingServer interface { - // Enable allows users to toggle profiling on and off remotely. - Enable(context.Context, *EnableRequest) (*EnableResponse, error) - // GetStreamStats is used to retrieve an array of stream-level stats from a - // gRPC client/server. - GetStreamStats(context.Context, *GetStreamStatsRequest) (*GetStreamStatsResponse, error) -} - -// UnimplementedProfilingServer can be embedded to have forward compatible implementations. -type UnimplementedProfilingServer struct { -} - -func (*UnimplementedProfilingServer) Enable(ctx context.Context, req *EnableRequest) (*EnableResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method Enable not implemented") -} -func (*UnimplementedProfilingServer) GetStreamStats(ctx context.Context, req *GetStreamStatsRequest) (*GetStreamStatsResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method GetStreamStats not implemented") -} - -func RegisterProfilingServer(s *grpc.Server, srv ProfilingServer) { - s.RegisterService(&_Profiling_serviceDesc, srv) + return nil } -func _Profiling_Enable_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(EnableRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(ProfilingServer).Enable(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/grpc.go.profiling.v1alpha.Profiling/Enable", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(ProfilingServer).Enable(ctx, req.(*EnableRequest)) - } - return interceptor(ctx, in, info, handler) -} +var File_profiling_proto_service_proto protoreflect.FileDescriptor + +var file_profiling_proto_service_proto_rawDesc = []byte{ + 0x0a, 0x1d, 0x70, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x69, 0x6e, 0x67, 0x2f, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, + 0x19, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x69, + 0x6e, 0x67, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x22, 0x29, 0x0a, 0x0d, 0x45, 0x6e, + 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x65, + 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x65, 0x6e, + 0x61, 0x62, 0x6c, 0x65, 0x64, 0x22, 0x10, 0x0a, 0x0e, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x17, 0x0a, 0x15, 0x47, 0x65, 0x74, 0x53, 0x74, + 0x72, 0x65, 0x61, 0x6d, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x22, 0x5c, 0x0a, 0x16, 0x47, 0x65, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x53, 0x74, 0x61, + 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x42, 0x0a, 0x0c, 0x73, 0x74, + 0x72, 0x65, 0x61, 0x6d, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x1f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x66, 0x69, + 0x6c, 0x69, 0x6e, 0x67, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x2e, 0x53, 0x74, 0x61, + 0x74, 0x52, 0x0b, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x53, 0x74, 0x61, 0x74, 0x73, 0x22, 0xa0, + 0x01, 0x0a, 0x05, 0x54, 0x69, 0x6d, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x61, 0x67, 0x73, 0x12, 0x1b, 0x0a, 0x09, + 0x62, 0x65, 0x67, 0x69, 0x6e, 0x5f, 0x73, 0x65, 0x63, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, + 0x08, 0x62, 0x65, 0x67, 0x69, 0x6e, 0x53, 0x65, 0x63, 0x12, 0x1d, 0x0a, 0x0a, 0x62, 0x65, 0x67, + 0x69, 0x6e, 0x5f, 0x6e, 0x73, 0x65, 0x63, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, 0x62, + 0x65, 0x67, 0x69, 0x6e, 0x4e, 0x73, 0x65, 0x63, 0x12, 0x17, 0x0a, 0x07, 0x65, 0x6e, 0x64, 0x5f, + 0x73, 0x65, 0x63, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x65, 0x6e, 0x64, 0x53, 0x65, + 0x63, 0x12, 0x19, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x5f, 0x6e, 0x73, 0x65, 0x63, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x05, 0x52, 0x07, 0x65, 0x6e, 0x64, 0x4e, 0x73, 0x65, 0x63, 0x12, 0x13, 0x0a, 0x05, + 0x67, 0x6f, 0x5f, 0x69, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x67, 0x6f, 0x49, + 0x64, 0x22, 0x70, 0x0a, 0x04, 0x53, 0x74, 0x61, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x61, 0x67, + 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x61, 0x67, 0x73, 0x12, 0x38, 0x0a, + 0x06, 0x74, 0x69, 0x6d, 0x65, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, + 0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x69, 0x6e, + 0x67, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x72, 0x52, + 0x06, 0x74, 0x69, 0x6d, 0x65, 0x72, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, + 0x61, 0x74, 0x61, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, + 0x61, 0x74, 0x61, 0x32, 0xe1, 0x01, 0x0a, 0x09, 0x50, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x69, 0x6e, + 0x67, 0x12, 0x5d, 0x0a, 0x06, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x28, 0x2e, 0x67, 0x72, + 0x70, 0x63, 0x2e, 0x67, 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x69, 0x6e, 0x67, 0x2e, + 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x2e, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x6f, 0x2e, + 0x70, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x69, 0x6e, 0x67, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, + 0x61, 0x2e, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x75, 0x0a, 0x0e, 0x47, 0x65, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x53, 0x74, 0x61, + 0x74, 0x73, 0x12, 0x30, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x6f, 0x2e, 0x70, 0x72, 0x6f, + 0x66, 0x69, 0x6c, 0x69, 0x6e, 0x67, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x2e, 0x47, + 0x65, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x31, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x67, 0x6f, 0x2e, 0x70, + 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x69, 0x6e, 0x67, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, + 0x2e, 0x47, 0x65, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x53, 0x74, 0x61, 0x74, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x28, 0x5a, 0x26, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x65, 0x2e, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x72, 0x70, + 0x63, 0x2f, 0x70, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x69, 0x6e, 0x67, 0x2f, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_profiling_proto_service_proto_rawDescOnce sync.Once + file_profiling_proto_service_proto_rawDescData = file_profiling_proto_service_proto_rawDesc +) -func _Profiling_GetStreamStats_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(GetStreamStatsRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(ProfilingServer).GetStreamStats(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/grpc.go.profiling.v1alpha.Profiling/GetStreamStats", +func file_profiling_proto_service_proto_rawDescGZIP() []byte { + file_profiling_proto_service_proto_rawDescOnce.Do(func() { + file_profiling_proto_service_proto_rawDescData = protoimpl.X.CompressGZIP(file_profiling_proto_service_proto_rawDescData) + }) + return file_profiling_proto_service_proto_rawDescData +} + +var file_profiling_proto_service_proto_msgTypes = make([]protoimpl.MessageInfo, 6) +var file_profiling_proto_service_proto_goTypes = []interface{}{ + (*EnableRequest)(nil), // 0: grpc.go.profiling.v1alpha.EnableRequest + (*EnableResponse)(nil), // 1: grpc.go.profiling.v1alpha.EnableResponse + (*GetStreamStatsRequest)(nil), // 2: grpc.go.profiling.v1alpha.GetStreamStatsRequest + (*GetStreamStatsResponse)(nil), // 3: grpc.go.profiling.v1alpha.GetStreamStatsResponse + (*Timer)(nil), // 4: grpc.go.profiling.v1alpha.Timer + (*Stat)(nil), // 5: grpc.go.profiling.v1alpha.Stat +} +var file_profiling_proto_service_proto_depIdxs = []int32{ + 5, // 0: grpc.go.profiling.v1alpha.GetStreamStatsResponse.stream_stats:type_name -> grpc.go.profiling.v1alpha.Stat + 4, // 1: grpc.go.profiling.v1alpha.Stat.timers:type_name -> grpc.go.profiling.v1alpha.Timer + 0, // 2: grpc.go.profiling.v1alpha.Profiling.Enable:input_type -> grpc.go.profiling.v1alpha.EnableRequest + 2, // 3: grpc.go.profiling.v1alpha.Profiling.GetStreamStats:input_type -> grpc.go.profiling.v1alpha.GetStreamStatsRequest + 1, // 4: grpc.go.profiling.v1alpha.Profiling.Enable:output_type -> grpc.go.profiling.v1alpha.EnableResponse + 3, // 5: grpc.go.profiling.v1alpha.Profiling.GetStreamStats:output_type -> grpc.go.profiling.v1alpha.GetStreamStatsResponse + 4, // [4:6] is the sub-list for method output_type + 2, // [2:4] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name +} + +func init() { file_profiling_proto_service_proto_init() } +func file_profiling_proto_service_proto_init() { + if File_profiling_proto_service_proto != nil { + return } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(ProfilingServer).GetStreamStats(ctx, req.(*GetStreamStatsRequest)) + if !protoimpl.UnsafeEnabled { + file_profiling_proto_service_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*EnableRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_profiling_proto_service_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*EnableResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_profiling_proto_service_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetStreamStatsRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_profiling_proto_service_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetStreamStatsResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_profiling_proto_service_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Timer); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_profiling_proto_service_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Stat); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } - return interceptor(ctx, in, info, handler) -} - -var _Profiling_serviceDesc = grpc.ServiceDesc{ - ServiceName: "grpc.go.profiling.v1alpha.Profiling", - HandlerType: (*ProfilingServer)(nil), - Methods: []grpc.MethodDesc{ - { - MethodName: "Enable", - Handler: _Profiling_Enable_Handler, - }, - { - MethodName: "GetStreamStats", - Handler: _Profiling_GetStreamStats_Handler, + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_profiling_proto_service_proto_rawDesc, + NumEnums: 0, + NumMessages: 6, + NumExtensions: 0, + NumServices: 1, }, - }, - Streams: []grpc.StreamDesc{}, - Metadata: "profiling/proto/service.proto", + GoTypes: file_profiling_proto_service_proto_goTypes, + DependencyIndexes: file_profiling_proto_service_proto_depIdxs, + MessageInfos: file_profiling_proto_service_proto_msgTypes, + }.Build() + File_profiling_proto_service_proto = out.File + file_profiling_proto_service_proto_rawDesc = nil + file_profiling_proto_service_proto_goTypes = nil + file_profiling_proto_service_proto_depIdxs = nil } diff --git a/profiling/proto/service_grpc.pb.go b/profiling/proto/service_grpc.pb.go index e32cbb405db4..5d696a26f924 100644 --- a/profiling/proto/service_grpc.pb.go +++ b/profiling/proto/service_grpc.pb.go @@ -1,4 +1,22 @@ +// Copyright 2019 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.3.0 +// - protoc v4.22.0 +// source: profiling/proto/service.proto package proto @@ -11,111 +29,136 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. const _ = grpc.SupportPackageIsVersion7 -// ProfilingService is the service API for Profiling service. -// Fields should be assigned to their respective handler implementations only before -// RegisterProfilingService is called. Any unassigned fields will result in the -// handler for that method returning an Unimplemented error. -type ProfilingService struct { +const ( + Profiling_Enable_FullMethodName = "/grpc.go.profiling.v1alpha.Profiling/Enable" + Profiling_GetStreamStats_FullMethodName = "/grpc.go.profiling.v1alpha.Profiling/GetStreamStats" +) + +// ProfilingClient is the client API for Profiling service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type ProfilingClient interface { // Enable allows users to toggle profiling on and off remotely. - Enable func(context.Context, *EnableRequest) (*EnableResponse, error) + Enable(ctx context.Context, in *EnableRequest, opts ...grpc.CallOption) (*EnableResponse, error) // GetStreamStats is used to retrieve an array of stream-level stats from a // gRPC client/server. - GetStreamStats func(context.Context, *GetStreamStatsRequest) (*GetStreamStatsResponse, error) + GetStreamStats(ctx context.Context, in *GetStreamStatsRequest, opts ...grpc.CallOption) (*GetStreamStatsResponse, error) +} + +type profilingClient struct { + cc grpc.ClientConnInterface +} + +func NewProfilingClient(cc grpc.ClientConnInterface) ProfilingClient { + return &profilingClient{cc} +} + +func (c *profilingClient) Enable(ctx context.Context, in *EnableRequest, opts ...grpc.CallOption) (*EnableResponse, error) { + out := new(EnableResponse) + err := c.cc.Invoke(ctx, Profiling_Enable_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil } -func (s *ProfilingService) enable(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - if s.Enable == nil { - return nil, status.Errorf(codes.Unimplemented, "method Enable not implemented") +func (c *profilingClient) GetStreamStats(ctx context.Context, in *GetStreamStatsRequest, opts ...grpc.CallOption) (*GetStreamStatsResponse, error) { + out := new(GetStreamStatsResponse) + err := c.cc.Invoke(ctx, Profiling_GetStreamStats_FullMethodName, in, out, opts...) + if err != nil { + return nil, err } + return out, nil +} + +// ProfilingServer is the server API for Profiling service. +// All implementations should embed UnimplementedProfilingServer +// for forward compatibility +type ProfilingServer interface { + // Enable allows users to toggle profiling on and off remotely. + Enable(context.Context, *EnableRequest) (*EnableResponse, error) + // GetStreamStats is used to retrieve an array of stream-level stats from a + // gRPC client/server. + GetStreamStats(context.Context, *GetStreamStatsRequest) (*GetStreamStatsResponse, error) +} + +// UnimplementedProfilingServer should be embedded to have forward compatible implementations. +type UnimplementedProfilingServer struct { +} + +func (UnimplementedProfilingServer) Enable(context.Context, *EnableRequest) (*EnableResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method Enable not implemented") +} +func (UnimplementedProfilingServer) GetStreamStats(context.Context, *GetStreamStatsRequest) (*GetStreamStatsResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetStreamStats not implemented") +} + +// UnsafeProfilingServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to ProfilingServer will +// result in compilation errors. +type UnsafeProfilingServer interface { + mustEmbedUnimplementedProfilingServer() +} + +func RegisterProfilingServer(s grpc.ServiceRegistrar, srv ProfilingServer) { + s.RegisterService(&Profiling_ServiceDesc, srv) +} + +func _Profiling_Enable_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(EnableRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { - return s.Enable(ctx, in) + return srv.(ProfilingServer).Enable(ctx, in) } info := &grpc.UnaryServerInfo{ - Server: s, - FullMethod: "/grpc.go.profiling.v1alpha.Profiling/Enable", + Server: srv, + FullMethod: Profiling_Enable_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return s.Enable(ctx, req.(*EnableRequest)) + return srv.(ProfilingServer).Enable(ctx, req.(*EnableRequest)) } return interceptor(ctx, in, info, handler) } -func (s *ProfilingService) getStreamStats(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - if s.GetStreamStats == nil { - return nil, status.Errorf(codes.Unimplemented, "method GetStreamStats not implemented") - } + +func _Profiling_GetStreamStats_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(GetStreamStatsRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { - return s.GetStreamStats(ctx, in) + return srv.(ProfilingServer).GetStreamStats(ctx, in) } info := &grpc.UnaryServerInfo{ - Server: s, - FullMethod: "/grpc.go.profiling.v1alpha.Profiling/GetStreamStats", + Server: srv, + FullMethod: Profiling_GetStreamStats_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return s.GetStreamStats(ctx, req.(*GetStreamStatsRequest)) + return srv.(ProfilingServer).GetStreamStats(ctx, req.(*GetStreamStatsRequest)) } return interceptor(ctx, in, info, handler) } -// RegisterProfilingService registers a service implementation with a gRPC server. -func RegisterProfilingService(s grpc.ServiceRegistrar, srv *ProfilingService) { - sd := grpc.ServiceDesc{ - ServiceName: "grpc.go.profiling.v1alpha.Profiling", - Methods: []grpc.MethodDesc{ - { - MethodName: "Enable", - Handler: srv.enable, - }, - { - MethodName: "GetStreamStats", - Handler: srv.getStreamStats, - }, +// Profiling_ServiceDesc is the grpc.ServiceDesc for Profiling service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var Profiling_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.go.profiling.v1alpha.Profiling", + HandlerType: (*ProfilingServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Enable", + Handler: _Profiling_Enable_Handler, }, - Streams: []grpc.StreamDesc{}, - Metadata: "profiling/proto/service.proto", - } - - s.RegisterService(&sd, nil) -} - -// NewProfilingService creates a new ProfilingService containing the -// implemented methods of the Profiling service in s. Any unimplemented -// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client. -// This includes situations where the method handler is misspelled or has the wrong -// signature. For this reason, this function should be used with great care and -// is not recommended to be used by most users. -func NewProfilingService(s interface{}) *ProfilingService { - ns := &ProfilingService{} - if h, ok := s.(interface { - Enable(context.Context, *EnableRequest) (*EnableResponse, error) - }); ok { - ns.Enable = h.Enable - } - if h, ok := s.(interface { - GetStreamStats(context.Context, *GetStreamStatsRequest) (*GetStreamStatsResponse, error) - }); ok { - ns.GetStreamStats = h.GetStreamStats - } - return ns -} - -// UnstableProfilingService is the service API for Profiling service. -// New methods may be added to this interface if they are added to the service -// definition, which is not a backward-compatible change. For this reason, -// use of this type is not recommended. -type UnstableProfilingService interface { - // Enable allows users to toggle profiling on and off remotely. - Enable(context.Context, *EnableRequest) (*EnableResponse, error) - // GetStreamStats is used to retrieve an array of stream-level stats from a - // gRPC client/server. - GetStreamStats(context.Context, *GetStreamStatsRequest) (*GetStreamStatsResponse, error) + { + MethodName: "GetStreamStats", + Handler: _Profiling_GetStreamStats_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "profiling/proto/service.proto", } diff --git a/profiling/service/service.go b/profiling/service/service.go index e2ce8926d6c5..c0234987392c 100644 --- a/profiling/service/service.go +++ b/profiling/service/service.go @@ -21,7 +21,10 @@ // queried by a client to remotely manage the gRPC profiling behaviour of an // application. // -// This package and all its methods are EXPERIMENTAL. +// # Experimental +// +// Notice: This package is EXPERIMENTAL and may be changed or removed in a +// later release. package service import ( diff --git a/reflection/adapt.go b/reflection/adapt.go new file mode 100644 index 000000000000..33b907a36da4 --- /dev/null +++ b/reflection/adapt.go @@ -0,0 +1,187 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package reflection + +import ( + v1reflectiongrpc "google.golang.org/grpc/reflection/grpc_reflection_v1" + v1reflectionpb "google.golang.org/grpc/reflection/grpc_reflection_v1" + v1alphareflectiongrpc "google.golang.org/grpc/reflection/grpc_reflection_v1alpha" + v1alphareflectionpb "google.golang.org/grpc/reflection/grpc_reflection_v1alpha" +) + +// asV1Alpha returns an implementation of the v1alpha version of the reflection +// interface that delegates all calls to the given v1 version. +func asV1Alpha(svr v1reflectiongrpc.ServerReflectionServer) v1alphareflectiongrpc.ServerReflectionServer { + return v1AlphaServerImpl{svr: svr} +} + +type v1AlphaServerImpl struct { + svr v1reflectiongrpc.ServerReflectionServer +} + +func (s v1AlphaServerImpl) ServerReflectionInfo(stream v1alphareflectiongrpc.ServerReflection_ServerReflectionInfoServer) error { + return s.svr.ServerReflectionInfo(v1AlphaServerStreamAdapter{stream}) +} + +type v1AlphaServerStreamAdapter struct { + v1alphareflectiongrpc.ServerReflection_ServerReflectionInfoServer +} + +func (s v1AlphaServerStreamAdapter) Send(response *v1reflectionpb.ServerReflectionResponse) error { + return s.ServerReflection_ServerReflectionInfoServer.Send(v1ToV1AlphaResponse(response)) +} + +func (s v1AlphaServerStreamAdapter) Recv() (*v1reflectionpb.ServerReflectionRequest, error) { + resp, err := s.ServerReflection_ServerReflectionInfoServer.Recv() + if err != nil { + return nil, err + } + return v1AlphaToV1Request(resp), nil +} + +func v1ToV1AlphaResponse(v1 *v1reflectionpb.ServerReflectionResponse) *v1alphareflectionpb.ServerReflectionResponse { + var v1alpha v1alphareflectionpb.ServerReflectionResponse + v1alpha.ValidHost = v1.ValidHost + if v1.OriginalRequest != nil { + v1alpha.OriginalRequest = v1ToV1AlphaRequest(v1.OriginalRequest) + } + switch mr := v1.MessageResponse.(type) { + case *v1reflectionpb.ServerReflectionResponse_FileDescriptorResponse: + if mr != nil { + v1alpha.MessageResponse = &v1alphareflectionpb.ServerReflectionResponse_FileDescriptorResponse{ + FileDescriptorResponse: &v1alphareflectionpb.FileDescriptorResponse{ + FileDescriptorProto: mr.FileDescriptorResponse.GetFileDescriptorProto(), + }, + } + } + case *v1reflectionpb.ServerReflectionResponse_AllExtensionNumbersResponse: + if mr != nil { + v1alpha.MessageResponse = &v1alphareflectionpb.ServerReflectionResponse_AllExtensionNumbersResponse{ + AllExtensionNumbersResponse: &v1alphareflectionpb.ExtensionNumberResponse{ + BaseTypeName: mr.AllExtensionNumbersResponse.GetBaseTypeName(), + ExtensionNumber: mr.AllExtensionNumbersResponse.GetExtensionNumber(), + }, + } + } + case *v1reflectionpb.ServerReflectionResponse_ListServicesResponse: + if mr != nil { + svcs := make([]*v1alphareflectionpb.ServiceResponse, len(mr.ListServicesResponse.GetService())) + for i, svc := range mr.ListServicesResponse.GetService() { + svcs[i] = &v1alphareflectionpb.ServiceResponse{ + Name: svc.GetName(), + } + } + v1alpha.MessageResponse = &v1alphareflectionpb.ServerReflectionResponse_ListServicesResponse{ + ListServicesResponse: &v1alphareflectionpb.ListServiceResponse{ + Service: svcs, + }, + } + } + case *v1reflectionpb.ServerReflectionResponse_ErrorResponse: + if mr != nil { + v1alpha.MessageResponse = &v1alphareflectionpb.ServerReflectionResponse_ErrorResponse{ + ErrorResponse: &v1alphareflectionpb.ErrorResponse{ + ErrorCode: mr.ErrorResponse.GetErrorCode(), + ErrorMessage: mr.ErrorResponse.GetErrorMessage(), + }, + } + } + default: + // no value set + } + return &v1alpha +} + +func v1AlphaToV1Request(v1alpha *v1alphareflectionpb.ServerReflectionRequest) *v1reflectionpb.ServerReflectionRequest { + var v1 v1reflectionpb.ServerReflectionRequest + v1.Host = v1alpha.Host + switch mr := v1alpha.MessageRequest.(type) { + case *v1alphareflectionpb.ServerReflectionRequest_FileByFilename: + v1.MessageRequest = &v1reflectionpb.ServerReflectionRequest_FileByFilename{ + FileByFilename: mr.FileByFilename, + } + case *v1alphareflectionpb.ServerReflectionRequest_FileContainingSymbol: + v1.MessageRequest = &v1reflectionpb.ServerReflectionRequest_FileContainingSymbol{ + FileContainingSymbol: mr.FileContainingSymbol, + } + case *v1alphareflectionpb.ServerReflectionRequest_FileContainingExtension: + if mr.FileContainingExtension != nil { + v1.MessageRequest = &v1reflectionpb.ServerReflectionRequest_FileContainingExtension{ + FileContainingExtension: &v1reflectionpb.ExtensionRequest{ + ContainingType: mr.FileContainingExtension.GetContainingType(), + ExtensionNumber: mr.FileContainingExtension.GetExtensionNumber(), + }, + } + } + case *v1alphareflectionpb.ServerReflectionRequest_AllExtensionNumbersOfType: + v1.MessageRequest = &v1reflectionpb.ServerReflectionRequest_AllExtensionNumbersOfType{ + AllExtensionNumbersOfType: mr.AllExtensionNumbersOfType, + } + case *v1alphareflectionpb.ServerReflectionRequest_ListServices: + v1.MessageRequest = &v1reflectionpb.ServerReflectionRequest_ListServices{ + ListServices: mr.ListServices, + } + default: + // no value set + } + return &v1 +} + +func v1ToV1AlphaRequest(v1 *v1reflectionpb.ServerReflectionRequest) *v1alphareflectionpb.ServerReflectionRequest { + var v1alpha v1alphareflectionpb.ServerReflectionRequest + v1alpha.Host = v1.Host + switch mr := v1.MessageRequest.(type) { + case *v1reflectionpb.ServerReflectionRequest_FileByFilename: + if mr != nil { + v1alpha.MessageRequest = &v1alphareflectionpb.ServerReflectionRequest_FileByFilename{ + FileByFilename: mr.FileByFilename, + } + } + case *v1reflectionpb.ServerReflectionRequest_FileContainingSymbol: + if mr != nil { + v1alpha.MessageRequest = &v1alphareflectionpb.ServerReflectionRequest_FileContainingSymbol{ + FileContainingSymbol: mr.FileContainingSymbol, + } + } + case *v1reflectionpb.ServerReflectionRequest_FileContainingExtension: + if mr != nil { + v1alpha.MessageRequest = &v1alphareflectionpb.ServerReflectionRequest_FileContainingExtension{ + FileContainingExtension: &v1alphareflectionpb.ExtensionRequest{ + ContainingType: mr.FileContainingExtension.GetContainingType(), + ExtensionNumber: mr.FileContainingExtension.GetExtensionNumber(), + }, + } + } + case *v1reflectionpb.ServerReflectionRequest_AllExtensionNumbersOfType: + if mr != nil { + v1alpha.MessageRequest = &v1alphareflectionpb.ServerReflectionRequest_AllExtensionNumbersOfType{ + AllExtensionNumbersOfType: mr.AllExtensionNumbersOfType, + } + } + case *v1reflectionpb.ServerReflectionRequest_ListServices: + if mr != nil { + v1alpha.MessageRequest = &v1alphareflectionpb.ServerReflectionRequest_ListServices{ + ListServices: mr.ListServices, + } + } + default: + // no value set + } + return &v1alpha +} diff --git a/reflection/grpc_reflection_v1/reflection.pb.go b/reflection/grpc_reflection_v1/reflection.pb.go new file mode 100644 index 000000000000..6f5c786b211c --- /dev/null +++ b/reflection/grpc_reflection_v1/reflection.pb.go @@ -0,0 +1,953 @@ +// Copyright 2016 The gRPC Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Service exported by server reflection. A more complete description of how +// server reflection works can be found at +// https://github.com/grpc/grpc/blob/master/doc/server-reflection.md +// +// The canonical version of this proto can be found at +// https://github.com/grpc/grpc-proto/blob/master/grpc/reflection/v1/reflection.proto + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.31.0 +// protoc v4.22.0 +// source: grpc/reflection/v1/reflection.proto + +package grpc_reflection_v1 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// The message sent by the client when calling ServerReflectionInfo method. +type ServerReflectionRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Host string `protobuf:"bytes,1,opt,name=host,proto3" json:"host,omitempty"` + // To use reflection service, the client should set one of the following + // fields in message_request. The server distinguishes requests by their + // defined field and then handles them using corresponding methods. + // + // Types that are assignable to MessageRequest: + // + // *ServerReflectionRequest_FileByFilename + // *ServerReflectionRequest_FileContainingSymbol + // *ServerReflectionRequest_FileContainingExtension + // *ServerReflectionRequest_AllExtensionNumbersOfType + // *ServerReflectionRequest_ListServices + MessageRequest isServerReflectionRequest_MessageRequest `protobuf_oneof:"message_request"` +} + +func (x *ServerReflectionRequest) Reset() { + *x = ServerReflectionRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_reflection_v1_reflection_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ServerReflectionRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ServerReflectionRequest) ProtoMessage() {} + +func (x *ServerReflectionRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_reflection_v1_reflection_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ServerReflectionRequest.ProtoReflect.Descriptor instead. +func (*ServerReflectionRequest) Descriptor() ([]byte, []int) { + return file_grpc_reflection_v1_reflection_proto_rawDescGZIP(), []int{0} +} + +func (x *ServerReflectionRequest) GetHost() string { + if x != nil { + return x.Host + } + return "" +} + +func (m *ServerReflectionRequest) GetMessageRequest() isServerReflectionRequest_MessageRequest { + if m != nil { + return m.MessageRequest + } + return nil +} + +func (x *ServerReflectionRequest) GetFileByFilename() string { + if x, ok := x.GetMessageRequest().(*ServerReflectionRequest_FileByFilename); ok { + return x.FileByFilename + } + return "" +} + +func (x *ServerReflectionRequest) GetFileContainingSymbol() string { + if x, ok := x.GetMessageRequest().(*ServerReflectionRequest_FileContainingSymbol); ok { + return x.FileContainingSymbol + } + return "" +} + +func (x *ServerReflectionRequest) GetFileContainingExtension() *ExtensionRequest { + if x, ok := x.GetMessageRequest().(*ServerReflectionRequest_FileContainingExtension); ok { + return x.FileContainingExtension + } + return nil +} + +func (x *ServerReflectionRequest) GetAllExtensionNumbersOfType() string { + if x, ok := x.GetMessageRequest().(*ServerReflectionRequest_AllExtensionNumbersOfType); ok { + return x.AllExtensionNumbersOfType + } + return "" +} + +func (x *ServerReflectionRequest) GetListServices() string { + if x, ok := x.GetMessageRequest().(*ServerReflectionRequest_ListServices); ok { + return x.ListServices + } + return "" +} + +type isServerReflectionRequest_MessageRequest interface { + isServerReflectionRequest_MessageRequest() +} + +type ServerReflectionRequest_FileByFilename struct { + // Find a proto file by the file name. + FileByFilename string `protobuf:"bytes,3,opt,name=file_by_filename,json=fileByFilename,proto3,oneof"` +} + +type ServerReflectionRequest_FileContainingSymbol struct { + // Find the proto file that declares the given fully-qualified symbol name. + // This field should be a fully-qualified symbol name + // (e.g. .[.] or .). + FileContainingSymbol string `protobuf:"bytes,4,opt,name=file_containing_symbol,json=fileContainingSymbol,proto3,oneof"` +} + +type ServerReflectionRequest_FileContainingExtension struct { + // Find the proto file which defines an extension extending the given + // message type with the given field number. + FileContainingExtension *ExtensionRequest `protobuf:"bytes,5,opt,name=file_containing_extension,json=fileContainingExtension,proto3,oneof"` +} + +type ServerReflectionRequest_AllExtensionNumbersOfType struct { + // Finds the tag numbers used by all known extensions of the given message + // type, and appends them to ExtensionNumberResponse in an undefined order. + // Its corresponding method is best-effort: it's not guaranteed that the + // reflection service will implement this method, and it's not guaranteed + // that this method will provide all extensions. Returns + // StatusCode::UNIMPLEMENTED if it's not implemented. + // This field should be a fully-qualified type name. The format is + // . + AllExtensionNumbersOfType string `protobuf:"bytes,6,opt,name=all_extension_numbers_of_type,json=allExtensionNumbersOfType,proto3,oneof"` +} + +type ServerReflectionRequest_ListServices struct { + // List the full names of registered services. The content will not be + // checked. + ListServices string `protobuf:"bytes,7,opt,name=list_services,json=listServices,proto3,oneof"` +} + +func (*ServerReflectionRequest_FileByFilename) isServerReflectionRequest_MessageRequest() {} + +func (*ServerReflectionRequest_FileContainingSymbol) isServerReflectionRequest_MessageRequest() {} + +func (*ServerReflectionRequest_FileContainingExtension) isServerReflectionRequest_MessageRequest() {} + +func (*ServerReflectionRequest_AllExtensionNumbersOfType) isServerReflectionRequest_MessageRequest() { +} + +func (*ServerReflectionRequest_ListServices) isServerReflectionRequest_MessageRequest() {} + +// The type name and extension number sent by the client when requesting +// file_containing_extension. +type ExtensionRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Fully-qualified type name. The format should be . + ContainingType string `protobuf:"bytes,1,opt,name=containing_type,json=containingType,proto3" json:"containing_type,omitempty"` + ExtensionNumber int32 `protobuf:"varint,2,opt,name=extension_number,json=extensionNumber,proto3" json:"extension_number,omitempty"` +} + +func (x *ExtensionRequest) Reset() { + *x = ExtensionRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_reflection_v1_reflection_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ExtensionRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ExtensionRequest) ProtoMessage() {} + +func (x *ExtensionRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_reflection_v1_reflection_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ExtensionRequest.ProtoReflect.Descriptor instead. +func (*ExtensionRequest) Descriptor() ([]byte, []int) { + return file_grpc_reflection_v1_reflection_proto_rawDescGZIP(), []int{1} +} + +func (x *ExtensionRequest) GetContainingType() string { + if x != nil { + return x.ContainingType + } + return "" +} + +func (x *ExtensionRequest) GetExtensionNumber() int32 { + if x != nil { + return x.ExtensionNumber + } + return 0 +} + +// The message sent by the server to answer ServerReflectionInfo method. +type ServerReflectionResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + ValidHost string `protobuf:"bytes,1,opt,name=valid_host,json=validHost,proto3" json:"valid_host,omitempty"` + OriginalRequest *ServerReflectionRequest `protobuf:"bytes,2,opt,name=original_request,json=originalRequest,proto3" json:"original_request,omitempty"` + // The server sets one of the following fields according to the message_request + // in the request. + // + // Types that are assignable to MessageResponse: + // + // *ServerReflectionResponse_FileDescriptorResponse + // *ServerReflectionResponse_AllExtensionNumbersResponse + // *ServerReflectionResponse_ListServicesResponse + // *ServerReflectionResponse_ErrorResponse + MessageResponse isServerReflectionResponse_MessageResponse `protobuf_oneof:"message_response"` +} + +func (x *ServerReflectionResponse) Reset() { + *x = ServerReflectionResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_reflection_v1_reflection_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ServerReflectionResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ServerReflectionResponse) ProtoMessage() {} + +func (x *ServerReflectionResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_reflection_v1_reflection_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ServerReflectionResponse.ProtoReflect.Descriptor instead. +func (*ServerReflectionResponse) Descriptor() ([]byte, []int) { + return file_grpc_reflection_v1_reflection_proto_rawDescGZIP(), []int{2} +} + +func (x *ServerReflectionResponse) GetValidHost() string { + if x != nil { + return x.ValidHost + } + return "" +} + +func (x *ServerReflectionResponse) GetOriginalRequest() *ServerReflectionRequest { + if x != nil { + return x.OriginalRequest + } + return nil +} + +func (m *ServerReflectionResponse) GetMessageResponse() isServerReflectionResponse_MessageResponse { + if m != nil { + return m.MessageResponse + } + return nil +} + +func (x *ServerReflectionResponse) GetFileDescriptorResponse() *FileDescriptorResponse { + if x, ok := x.GetMessageResponse().(*ServerReflectionResponse_FileDescriptorResponse); ok { + return x.FileDescriptorResponse + } + return nil +} + +func (x *ServerReflectionResponse) GetAllExtensionNumbersResponse() *ExtensionNumberResponse { + if x, ok := x.GetMessageResponse().(*ServerReflectionResponse_AllExtensionNumbersResponse); ok { + return x.AllExtensionNumbersResponse + } + return nil +} + +func (x *ServerReflectionResponse) GetListServicesResponse() *ListServiceResponse { + if x, ok := x.GetMessageResponse().(*ServerReflectionResponse_ListServicesResponse); ok { + return x.ListServicesResponse + } + return nil +} + +func (x *ServerReflectionResponse) GetErrorResponse() *ErrorResponse { + if x, ok := x.GetMessageResponse().(*ServerReflectionResponse_ErrorResponse); ok { + return x.ErrorResponse + } + return nil +} + +type isServerReflectionResponse_MessageResponse interface { + isServerReflectionResponse_MessageResponse() +} + +type ServerReflectionResponse_FileDescriptorResponse struct { + // This message is used to answer file_by_filename, file_containing_symbol, + // file_containing_extension requests with transitive dependencies. + // As the repeated label is not allowed in oneof fields, we use a + // FileDescriptorResponse message to encapsulate the repeated fields. + // The reflection service is allowed to avoid sending FileDescriptorProtos + // that were previously sent in response to earlier requests in the stream. + FileDescriptorResponse *FileDescriptorResponse `protobuf:"bytes,4,opt,name=file_descriptor_response,json=fileDescriptorResponse,proto3,oneof"` +} + +type ServerReflectionResponse_AllExtensionNumbersResponse struct { + // This message is used to answer all_extension_numbers_of_type requests. + AllExtensionNumbersResponse *ExtensionNumberResponse `protobuf:"bytes,5,opt,name=all_extension_numbers_response,json=allExtensionNumbersResponse,proto3,oneof"` +} + +type ServerReflectionResponse_ListServicesResponse struct { + // This message is used to answer list_services requests. + ListServicesResponse *ListServiceResponse `protobuf:"bytes,6,opt,name=list_services_response,json=listServicesResponse,proto3,oneof"` +} + +type ServerReflectionResponse_ErrorResponse struct { + // This message is used when an error occurs. + ErrorResponse *ErrorResponse `protobuf:"bytes,7,opt,name=error_response,json=errorResponse,proto3,oneof"` +} + +func (*ServerReflectionResponse_FileDescriptorResponse) isServerReflectionResponse_MessageResponse() { +} + +func (*ServerReflectionResponse_AllExtensionNumbersResponse) isServerReflectionResponse_MessageResponse() { +} + +func (*ServerReflectionResponse_ListServicesResponse) isServerReflectionResponse_MessageResponse() {} + +func (*ServerReflectionResponse_ErrorResponse) isServerReflectionResponse_MessageResponse() {} + +// Serialized FileDescriptorProto messages sent by the server answering +// a file_by_filename, file_containing_symbol, or file_containing_extension +// request. +type FileDescriptorResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Serialized FileDescriptorProto messages. We avoid taking a dependency on + // descriptor.proto, which uses proto2 only features, by making them opaque + // bytes instead. + FileDescriptorProto [][]byte `protobuf:"bytes,1,rep,name=file_descriptor_proto,json=fileDescriptorProto,proto3" json:"file_descriptor_proto,omitempty"` +} + +func (x *FileDescriptorResponse) Reset() { + *x = FileDescriptorResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_reflection_v1_reflection_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *FileDescriptorResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FileDescriptorResponse) ProtoMessage() {} + +func (x *FileDescriptorResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_reflection_v1_reflection_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FileDescriptorResponse.ProtoReflect.Descriptor instead. +func (*FileDescriptorResponse) Descriptor() ([]byte, []int) { + return file_grpc_reflection_v1_reflection_proto_rawDescGZIP(), []int{3} +} + +func (x *FileDescriptorResponse) GetFileDescriptorProto() [][]byte { + if x != nil { + return x.FileDescriptorProto + } + return nil +} + +// A list of extension numbers sent by the server answering +// all_extension_numbers_of_type request. +type ExtensionNumberResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Full name of the base type, including the package name. The format + // is . + BaseTypeName string `protobuf:"bytes,1,opt,name=base_type_name,json=baseTypeName,proto3" json:"base_type_name,omitempty"` + ExtensionNumber []int32 `protobuf:"varint,2,rep,packed,name=extension_number,json=extensionNumber,proto3" json:"extension_number,omitempty"` +} + +func (x *ExtensionNumberResponse) Reset() { + *x = ExtensionNumberResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_reflection_v1_reflection_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ExtensionNumberResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ExtensionNumberResponse) ProtoMessage() {} + +func (x *ExtensionNumberResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_reflection_v1_reflection_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ExtensionNumberResponse.ProtoReflect.Descriptor instead. +func (*ExtensionNumberResponse) Descriptor() ([]byte, []int) { + return file_grpc_reflection_v1_reflection_proto_rawDescGZIP(), []int{4} +} + +func (x *ExtensionNumberResponse) GetBaseTypeName() string { + if x != nil { + return x.BaseTypeName + } + return "" +} + +func (x *ExtensionNumberResponse) GetExtensionNumber() []int32 { + if x != nil { + return x.ExtensionNumber + } + return nil +} + +// A list of ServiceResponse sent by the server answering list_services request. +type ListServiceResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // The information of each service may be expanded in the future, so we use + // ServiceResponse message to encapsulate it. + Service []*ServiceResponse `protobuf:"bytes,1,rep,name=service,proto3" json:"service,omitempty"` +} + +func (x *ListServiceResponse) Reset() { + *x = ListServiceResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_reflection_v1_reflection_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ListServiceResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListServiceResponse) ProtoMessage() {} + +func (x *ListServiceResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_reflection_v1_reflection_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListServiceResponse.ProtoReflect.Descriptor instead. +func (*ListServiceResponse) Descriptor() ([]byte, []int) { + return file_grpc_reflection_v1_reflection_proto_rawDescGZIP(), []int{5} +} + +func (x *ListServiceResponse) GetService() []*ServiceResponse { + if x != nil { + return x.Service + } + return nil +} + +// The information of a single service used by ListServiceResponse to answer +// list_services request. +type ServiceResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Full name of a registered service, including its package name. The format + // is . + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` +} + +func (x *ServiceResponse) Reset() { + *x = ServiceResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_reflection_v1_reflection_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ServiceResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ServiceResponse) ProtoMessage() {} + +func (x *ServiceResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_reflection_v1_reflection_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ServiceResponse.ProtoReflect.Descriptor instead. +func (*ServiceResponse) Descriptor() ([]byte, []int) { + return file_grpc_reflection_v1_reflection_proto_rawDescGZIP(), []int{6} +} + +func (x *ServiceResponse) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +// The error code and error message sent by the server when an error occurs. +type ErrorResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // This field uses the error codes defined in grpc::StatusCode. + ErrorCode int32 `protobuf:"varint,1,opt,name=error_code,json=errorCode,proto3" json:"error_code,omitempty"` + ErrorMessage string `protobuf:"bytes,2,opt,name=error_message,json=errorMessage,proto3" json:"error_message,omitempty"` +} + +func (x *ErrorResponse) Reset() { + *x = ErrorResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_reflection_v1_reflection_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ErrorResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ErrorResponse) ProtoMessage() {} + +func (x *ErrorResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_reflection_v1_reflection_proto_msgTypes[7] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ErrorResponse.ProtoReflect.Descriptor instead. +func (*ErrorResponse) Descriptor() ([]byte, []int) { + return file_grpc_reflection_v1_reflection_proto_rawDescGZIP(), []int{7} +} + +func (x *ErrorResponse) GetErrorCode() int32 { + if x != nil { + return x.ErrorCode + } + return 0 +} + +func (x *ErrorResponse) GetErrorMessage() string { + if x != nil { + return x.ErrorMessage + } + return "" +} + +var File_grpc_reflection_v1_reflection_proto protoreflect.FileDescriptor + +var file_grpc_reflection_v1_reflection_proto_rawDesc = []byte{ + 0x0a, 0x23, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x72, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x2f, 0x76, 0x31, 0x2f, 0x72, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x12, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x72, 0x65, 0x66, 0x6c, + 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x22, 0xf3, 0x02, 0x0a, 0x17, 0x53, 0x65, + 0x72, 0x76, 0x65, 0x72, 0x52, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x12, 0x2a, 0x0a, 0x10, 0x66, 0x69, 0x6c, + 0x65, 0x5f, 0x62, 0x79, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0e, 0x66, 0x69, 0x6c, 0x65, 0x42, 0x79, 0x46, 0x69, 0x6c, + 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x36, 0x0a, 0x16, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x63, 0x6f, + 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x69, 0x6e, 0x67, 0x5f, 0x73, 0x79, 0x6d, 0x62, 0x6f, 0x6c, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x14, 0x66, 0x69, 0x6c, 0x65, 0x43, 0x6f, 0x6e, + 0x74, 0x61, 0x69, 0x6e, 0x69, 0x6e, 0x67, 0x53, 0x79, 0x6d, 0x62, 0x6f, 0x6c, 0x12, 0x62, 0x0a, + 0x19, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x69, 0x6e, 0x67, + 0x5f, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x24, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x72, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x00, 0x52, 0x17, 0x66, 0x69, 0x6c, 0x65, 0x43, 0x6f, + 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x69, 0x6e, 0x67, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, + 0x6e, 0x12, 0x42, 0x0a, 0x1d, 0x61, 0x6c, 0x6c, 0x5f, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, + 0x6f, 0x6e, 0x5f, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x73, 0x5f, 0x6f, 0x66, 0x5f, 0x74, 0x79, + 0x70, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x19, 0x61, 0x6c, 0x6c, 0x45, + 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x73, 0x4f, + 0x66, 0x54, 0x79, 0x70, 0x65, 0x12, 0x25, 0x0a, 0x0d, 0x6c, 0x69, 0x73, 0x74, 0x5f, 0x73, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0c, + 0x6c, 0x69, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x42, 0x11, 0x0a, 0x0f, + 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, + 0x66, 0x0a, 0x10, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x12, 0x27, 0x0a, 0x0f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x69, 0x6e, + 0x67, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x63, 0x6f, + 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x69, 0x6e, 0x67, 0x54, 0x79, 0x70, 0x65, 0x12, 0x29, 0x0a, 0x10, + 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0f, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, + 0x6e, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x22, 0xae, 0x04, 0x0a, 0x18, 0x53, 0x65, 0x72, 0x76, + 0x65, 0x72, 0x52, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x5f, 0x68, 0x6f, + 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x48, + 0x6f, 0x73, 0x74, 0x12, 0x56, 0x0a, 0x10, 0x6f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x61, 0x6c, 0x5f, + 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2b, 0x2e, + 0x67, 0x72, 0x70, 0x63, 0x2e, 0x72, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2e, + 0x76, 0x31, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, 0x0f, 0x6f, 0x72, 0x69, 0x67, + 0x69, 0x6e, 0x61, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x66, 0x0a, 0x18, 0x66, + 0x69, 0x6c, 0x65, 0x5f, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x5f, 0x72, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, + 0x67, 0x72, 0x70, 0x63, 0x2e, 0x72, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2e, + 0x76, 0x31, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, + 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x00, 0x52, 0x16, 0x66, 0x69, 0x6c, + 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x72, 0x0a, 0x1e, 0x61, 0x6c, 0x6c, 0x5f, 0x65, 0x78, 0x74, 0x65, 0x6e, + 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x73, 0x5f, 0x72, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x67, 0x72, + 0x70, 0x63, 0x2e, 0x72, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, + 0x2e, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x00, 0x52, 0x1b, 0x61, 0x6c, 0x6c, 0x45, + 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5f, 0x0a, 0x16, 0x6c, 0x69, 0x73, 0x74, 0x5f, + 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x5f, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x72, + 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, + 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x48, 0x00, 0x52, 0x14, 0x6c, 0x69, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4a, 0x0a, 0x0e, 0x65, 0x72, 0x72, 0x6f, + 0x72, 0x5f, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x21, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x72, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x48, 0x00, 0x52, 0x0d, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x12, 0x0a, 0x10, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x5f, + 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x4c, 0x0a, 0x16, 0x46, 0x69, 0x6c, 0x65, + 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x32, 0x0a, 0x15, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x64, 0x65, 0x73, 0x63, 0x72, + 0x69, 0x70, 0x74, 0x6f, 0x72, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x18, 0x01, 0x20, 0x03, 0x28, + 0x0c, 0x52, 0x13, 0x66, 0x69, 0x6c, 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, + 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x6a, 0x0a, 0x17, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, + 0x69, 0x6f, 0x6e, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x24, 0x0a, 0x0e, 0x62, 0x61, 0x73, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x5f, 0x6e, + 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x62, 0x61, 0x73, 0x65, 0x54, + 0x79, 0x70, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x29, 0x0a, 0x10, 0x65, 0x78, 0x74, 0x65, 0x6e, + 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x18, 0x02, 0x20, 0x03, 0x28, + 0x05, 0x52, 0x0f, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x4e, 0x75, 0x6d, 0x62, + 0x65, 0x72, 0x22, 0x54, 0x0a, 0x13, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3d, 0x0a, 0x07, 0x73, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x67, 0x72, 0x70, + 0x63, 0x2e, 0x72, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, + 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x52, + 0x07, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x22, 0x25, 0x0a, 0x0f, 0x53, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, + 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, + 0x53, 0x0a, 0x0d, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x1d, 0x0a, 0x0a, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x43, 0x6f, 0x64, 0x65, 0x12, + 0x23, 0x0a, 0x0d, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x4d, 0x65, 0x73, + 0x73, 0x61, 0x67, 0x65, 0x32, 0x89, 0x01, 0x0a, 0x10, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, + 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x75, 0x0a, 0x14, 0x53, 0x65, 0x72, + 0x76, 0x65, 0x72, 0x52, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, + 0x6f, 0x12, 0x2b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x72, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x65, 0x66, + 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, + 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x72, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x65, 0x66, 0x6c, 0x65, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, + 0x42, 0x66, 0x0a, 0x15, 0x69, 0x6f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x72, 0x65, 0x66, 0x6c, + 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x42, 0x15, 0x53, 0x65, 0x72, 0x76, 0x65, + 0x72, 0x52, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x74, 0x6f, + 0x50, 0x01, 0x5a, 0x34, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67, 0x6f, 0x6c, 0x61, 0x6e, + 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x72, 0x65, 0x66, 0x6c, 0x65, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x72, 0x65, 0x66, 0x6c, 0x65, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_grpc_reflection_v1_reflection_proto_rawDescOnce sync.Once + file_grpc_reflection_v1_reflection_proto_rawDescData = file_grpc_reflection_v1_reflection_proto_rawDesc +) + +func file_grpc_reflection_v1_reflection_proto_rawDescGZIP() []byte { + file_grpc_reflection_v1_reflection_proto_rawDescOnce.Do(func() { + file_grpc_reflection_v1_reflection_proto_rawDescData = protoimpl.X.CompressGZIP(file_grpc_reflection_v1_reflection_proto_rawDescData) + }) + return file_grpc_reflection_v1_reflection_proto_rawDescData +} + +var file_grpc_reflection_v1_reflection_proto_msgTypes = make([]protoimpl.MessageInfo, 8) +var file_grpc_reflection_v1_reflection_proto_goTypes = []interface{}{ + (*ServerReflectionRequest)(nil), // 0: grpc.reflection.v1.ServerReflectionRequest + (*ExtensionRequest)(nil), // 1: grpc.reflection.v1.ExtensionRequest + (*ServerReflectionResponse)(nil), // 2: grpc.reflection.v1.ServerReflectionResponse + (*FileDescriptorResponse)(nil), // 3: grpc.reflection.v1.FileDescriptorResponse + (*ExtensionNumberResponse)(nil), // 4: grpc.reflection.v1.ExtensionNumberResponse + (*ListServiceResponse)(nil), // 5: grpc.reflection.v1.ListServiceResponse + (*ServiceResponse)(nil), // 6: grpc.reflection.v1.ServiceResponse + (*ErrorResponse)(nil), // 7: grpc.reflection.v1.ErrorResponse +} +var file_grpc_reflection_v1_reflection_proto_depIdxs = []int32{ + 1, // 0: grpc.reflection.v1.ServerReflectionRequest.file_containing_extension:type_name -> grpc.reflection.v1.ExtensionRequest + 0, // 1: grpc.reflection.v1.ServerReflectionResponse.original_request:type_name -> grpc.reflection.v1.ServerReflectionRequest + 3, // 2: grpc.reflection.v1.ServerReflectionResponse.file_descriptor_response:type_name -> grpc.reflection.v1.FileDescriptorResponse + 4, // 3: grpc.reflection.v1.ServerReflectionResponse.all_extension_numbers_response:type_name -> grpc.reflection.v1.ExtensionNumberResponse + 5, // 4: grpc.reflection.v1.ServerReflectionResponse.list_services_response:type_name -> grpc.reflection.v1.ListServiceResponse + 7, // 5: grpc.reflection.v1.ServerReflectionResponse.error_response:type_name -> grpc.reflection.v1.ErrorResponse + 6, // 6: grpc.reflection.v1.ListServiceResponse.service:type_name -> grpc.reflection.v1.ServiceResponse + 0, // 7: grpc.reflection.v1.ServerReflection.ServerReflectionInfo:input_type -> grpc.reflection.v1.ServerReflectionRequest + 2, // 8: grpc.reflection.v1.ServerReflection.ServerReflectionInfo:output_type -> grpc.reflection.v1.ServerReflectionResponse + 8, // [8:9] is the sub-list for method output_type + 7, // [7:8] is the sub-list for method input_type + 7, // [7:7] is the sub-list for extension type_name + 7, // [7:7] is the sub-list for extension extendee + 0, // [0:7] is the sub-list for field type_name +} + +func init() { file_grpc_reflection_v1_reflection_proto_init() } +func file_grpc_reflection_v1_reflection_proto_init() { + if File_grpc_reflection_v1_reflection_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_grpc_reflection_v1_reflection_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ServerReflectionRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_reflection_v1_reflection_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ExtensionRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_reflection_v1_reflection_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ServerReflectionResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_reflection_v1_reflection_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*FileDescriptorResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_reflection_v1_reflection_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ExtensionNumberResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_reflection_v1_reflection_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ListServiceResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_reflection_v1_reflection_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ServiceResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_reflection_v1_reflection_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ErrorResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + file_grpc_reflection_v1_reflection_proto_msgTypes[0].OneofWrappers = []interface{}{ + (*ServerReflectionRequest_FileByFilename)(nil), + (*ServerReflectionRequest_FileContainingSymbol)(nil), + (*ServerReflectionRequest_FileContainingExtension)(nil), + (*ServerReflectionRequest_AllExtensionNumbersOfType)(nil), + (*ServerReflectionRequest_ListServices)(nil), + } + file_grpc_reflection_v1_reflection_proto_msgTypes[2].OneofWrappers = []interface{}{ + (*ServerReflectionResponse_FileDescriptorResponse)(nil), + (*ServerReflectionResponse_AllExtensionNumbersResponse)(nil), + (*ServerReflectionResponse_ListServicesResponse)(nil), + (*ServerReflectionResponse_ErrorResponse)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_grpc_reflection_v1_reflection_proto_rawDesc, + NumEnums: 0, + NumMessages: 8, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_grpc_reflection_v1_reflection_proto_goTypes, + DependencyIndexes: file_grpc_reflection_v1_reflection_proto_depIdxs, + MessageInfos: file_grpc_reflection_v1_reflection_proto_msgTypes, + }.Build() + File_grpc_reflection_v1_reflection_proto = out.File + file_grpc_reflection_v1_reflection_proto_rawDesc = nil + file_grpc_reflection_v1_reflection_proto_goTypes = nil + file_grpc_reflection_v1_reflection_proto_depIdxs = nil +} diff --git a/reflection/grpc_reflection_v1/reflection_grpc.pb.go b/reflection/grpc_reflection_v1/reflection_grpc.pb.go new file mode 100644 index 000000000000..62b56a8be0e6 --- /dev/null +++ b/reflection/grpc_reflection_v1/reflection_grpc.pb.go @@ -0,0 +1,164 @@ +// Copyright 2016 The gRPC Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Service exported by server reflection. A more complete description of how +// server reflection works can be found at +// https://github.com/grpc/grpc/blob/master/doc/server-reflection.md +// +// The canonical version of this proto can be found at +// https://github.com/grpc/grpc-proto/blob/master/grpc/reflection/v1/reflection.proto + +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.3.0 +// - protoc v4.22.0 +// source: grpc/reflection/v1/reflection.proto + +package grpc_reflection_v1 + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 + +const ( + ServerReflection_ServerReflectionInfo_FullMethodName = "/grpc.reflection.v1.ServerReflection/ServerReflectionInfo" +) + +// ServerReflectionClient is the client API for ServerReflection service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type ServerReflectionClient interface { + // The reflection service is structured as a bidirectional stream, ensuring + // all related requests go to a single server. + ServerReflectionInfo(ctx context.Context, opts ...grpc.CallOption) (ServerReflection_ServerReflectionInfoClient, error) +} + +type serverReflectionClient struct { + cc grpc.ClientConnInterface +} + +func NewServerReflectionClient(cc grpc.ClientConnInterface) ServerReflectionClient { + return &serverReflectionClient{cc} +} + +func (c *serverReflectionClient) ServerReflectionInfo(ctx context.Context, opts ...grpc.CallOption) (ServerReflection_ServerReflectionInfoClient, error) { + stream, err := c.cc.NewStream(ctx, &ServerReflection_ServiceDesc.Streams[0], ServerReflection_ServerReflectionInfo_FullMethodName, opts...) + if err != nil { + return nil, err + } + x := &serverReflectionServerReflectionInfoClient{stream} + return x, nil +} + +type ServerReflection_ServerReflectionInfoClient interface { + Send(*ServerReflectionRequest) error + Recv() (*ServerReflectionResponse, error) + grpc.ClientStream +} + +type serverReflectionServerReflectionInfoClient struct { + grpc.ClientStream +} + +func (x *serverReflectionServerReflectionInfoClient) Send(m *ServerReflectionRequest) error { + return x.ClientStream.SendMsg(m) +} + +func (x *serverReflectionServerReflectionInfoClient) Recv() (*ServerReflectionResponse, error) { + m := new(ServerReflectionResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +// ServerReflectionServer is the server API for ServerReflection service. +// All implementations should embed UnimplementedServerReflectionServer +// for forward compatibility +type ServerReflectionServer interface { + // The reflection service is structured as a bidirectional stream, ensuring + // all related requests go to a single server. + ServerReflectionInfo(ServerReflection_ServerReflectionInfoServer) error +} + +// UnimplementedServerReflectionServer should be embedded to have forward compatible implementations. +type UnimplementedServerReflectionServer struct { +} + +func (UnimplementedServerReflectionServer) ServerReflectionInfo(ServerReflection_ServerReflectionInfoServer) error { + return status.Errorf(codes.Unimplemented, "method ServerReflectionInfo not implemented") +} + +// UnsafeServerReflectionServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to ServerReflectionServer will +// result in compilation errors. +type UnsafeServerReflectionServer interface { + mustEmbedUnimplementedServerReflectionServer() +} + +func RegisterServerReflectionServer(s grpc.ServiceRegistrar, srv ServerReflectionServer) { + s.RegisterService(&ServerReflection_ServiceDesc, srv) +} + +func _ServerReflection_ServerReflectionInfo_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(ServerReflectionServer).ServerReflectionInfo(&serverReflectionServerReflectionInfoServer{stream}) +} + +type ServerReflection_ServerReflectionInfoServer interface { + Send(*ServerReflectionResponse) error + Recv() (*ServerReflectionRequest, error) + grpc.ServerStream +} + +type serverReflectionServerReflectionInfoServer struct { + grpc.ServerStream +} + +func (x *serverReflectionServerReflectionInfoServer) Send(m *ServerReflectionResponse) error { + return x.ServerStream.SendMsg(m) +} + +func (x *serverReflectionServerReflectionInfoServer) Recv() (*ServerReflectionRequest, error) { + m := new(ServerReflectionRequest) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +// ServerReflection_ServiceDesc is the grpc.ServiceDesc for ServerReflection service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var ServerReflection_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.reflection.v1.ServerReflection", + HandlerType: (*ServerReflectionServer)(nil), + Methods: []grpc.MethodDesc{}, + Streams: []grpc.StreamDesc{ + { + StreamName: "ServerReflectionInfo", + Handler: _ServerReflection_ServerReflectionInfo_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "grpc/reflection/v1/reflection.proto", +} diff --git a/reflection/grpc_reflection_v1alpha/reflection.pb.go b/reflection/grpc_reflection_v1alpha/reflection.pb.go index 900bd6c05c78..69fbfb621ec0 100644 --- a/reflection/grpc_reflection_v1alpha/reflection.pb.go +++ b/reflection/grpc_reflection_v1alpha/reflection.pb.go @@ -1,76 +1,150 @@ +// Copyright 2016 The gRPC Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// Service exported by server reflection + +// Warning: this entire file is deprecated. Use this instead: +// https://github.com/grpc/grpc-proto/blob/master/grpc/reflection/v1/reflection.proto + // Code generated by protoc-gen-go. DO NOT EDIT. -// source: reflection/grpc_reflection_v1alpha/reflection.proto +// versions: +// protoc-gen-go v1.31.0 +// protoc v4.22.0 +// grpc/reflection/v1alpha/reflection.proto is a deprecated file. package grpc_reflection_v1alpha import ( - context "context" - fmt "fmt" - proto "github.com/golang/protobuf/proto" - grpc "google.golang.org/grpc" - codes "google.golang.org/grpc/codes" - status "google.golang.org/grpc/status" - math "math" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" ) -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) // The message sent by the client when calling ServerReflectionInfo method. +// +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. type ServerReflectionRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. Host string `protobuf:"bytes,1,opt,name=host,proto3" json:"host,omitempty"` // To use reflection service, the client should set one of the following // fields in message_request. The server distinguishes requests by their // defined field and then handles them using corresponding methods. // - // Types that are valid to be assigned to MessageRequest: + // Types that are assignable to MessageRequest: + // // *ServerReflectionRequest_FileByFilename // *ServerReflectionRequest_FileContainingSymbol // *ServerReflectionRequest_FileContainingExtension // *ServerReflectionRequest_AllExtensionNumbersOfType // *ServerReflectionRequest_ListServices - MessageRequest isServerReflectionRequest_MessageRequest `protobuf_oneof:"message_request"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + MessageRequest isServerReflectionRequest_MessageRequest `protobuf_oneof:"message_request"` +} + +func (x *ServerReflectionRequest) Reset() { + *x = ServerReflectionRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_reflection_v1alpha_reflection_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ServerReflectionRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ServerReflectionRequest) ProtoMessage() {} + +func (x *ServerReflectionRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_reflection_v1alpha_reflection_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -func (m *ServerReflectionRequest) Reset() { *m = ServerReflectionRequest{} } -func (m *ServerReflectionRequest) String() string { return proto.CompactTextString(m) } -func (*ServerReflectionRequest) ProtoMessage() {} +// Deprecated: Use ServerReflectionRequest.ProtoReflect.Descriptor instead. func (*ServerReflectionRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_e8cf9f2921ad6c95, []int{0} + return file_grpc_reflection_v1alpha_reflection_proto_rawDescGZIP(), []int{0} } -func (m *ServerReflectionRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ServerReflectionRequest.Unmarshal(m, b) +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +func (x *ServerReflectionRequest) GetHost() string { + if x != nil { + return x.Host + } + return "" } -func (m *ServerReflectionRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ServerReflectionRequest.Marshal(b, m, deterministic) + +func (m *ServerReflectionRequest) GetMessageRequest() isServerReflectionRequest_MessageRequest { + if m != nil { + return m.MessageRequest + } + return nil } -func (m *ServerReflectionRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_ServerReflectionRequest.Merge(m, src) + +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +func (x *ServerReflectionRequest) GetFileByFilename() string { + if x, ok := x.GetMessageRequest().(*ServerReflectionRequest_FileByFilename); ok { + return x.FileByFilename + } + return "" } -func (m *ServerReflectionRequest) XXX_Size() int { - return xxx_messageInfo_ServerReflectionRequest.Size(m) + +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +func (x *ServerReflectionRequest) GetFileContainingSymbol() string { + if x, ok := x.GetMessageRequest().(*ServerReflectionRequest_FileContainingSymbol); ok { + return x.FileContainingSymbol + } + return "" } -func (m *ServerReflectionRequest) XXX_DiscardUnknown() { - xxx_messageInfo_ServerReflectionRequest.DiscardUnknown(m) + +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +func (x *ServerReflectionRequest) GetFileContainingExtension() *ExtensionRequest { + if x, ok := x.GetMessageRequest().(*ServerReflectionRequest_FileContainingExtension); ok { + return x.FileContainingExtension + } + return nil } -var xxx_messageInfo_ServerReflectionRequest proto.InternalMessageInfo +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +func (x *ServerReflectionRequest) GetAllExtensionNumbersOfType() string { + if x, ok := x.GetMessageRequest().(*ServerReflectionRequest_AllExtensionNumbersOfType); ok { + return x.AllExtensionNumbersOfType + } + return "" +} -func (m *ServerReflectionRequest) GetHost() string { - if m != nil { - return m.Host +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +func (x *ServerReflectionRequest) GetListServices() string { + if x, ok := x.GetMessageRequest().(*ServerReflectionRequest_ListServices); ok { + return x.ListServices } return "" } @@ -80,22 +154,48 @@ type isServerReflectionRequest_MessageRequest interface { } type ServerReflectionRequest_FileByFilename struct { + // Find a proto file by the file name. + // + // Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. FileByFilename string `protobuf:"bytes,3,opt,name=file_by_filename,json=fileByFilename,proto3,oneof"` } type ServerReflectionRequest_FileContainingSymbol struct { + // Find the proto file that declares the given fully-qualified symbol name. + // This field should be a fully-qualified symbol name + // (e.g. .[.] or .). + // + // Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. FileContainingSymbol string `protobuf:"bytes,4,opt,name=file_containing_symbol,json=fileContainingSymbol,proto3,oneof"` } type ServerReflectionRequest_FileContainingExtension struct { + // Find the proto file which defines an extension extending the given + // message type with the given field number. + // + // Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. FileContainingExtension *ExtensionRequest `protobuf:"bytes,5,opt,name=file_containing_extension,json=fileContainingExtension,proto3,oneof"` } type ServerReflectionRequest_AllExtensionNumbersOfType struct { + // Finds the tag numbers used by all known extensions of extendee_type, and + // appends them to ExtensionNumberResponse in an undefined order. + // Its corresponding method is best-effort: it's not guaranteed that the + // reflection service will implement this method, and it's not guaranteed + // that this method will provide all extensions. Returns + // StatusCode::UNIMPLEMENTED if it's not implemented. + // This field should be a fully-qualified type name. The format is + // . + // + // Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. AllExtensionNumbersOfType string `protobuf:"bytes,6,opt,name=all_extension_numbers_of_type,json=allExtensionNumbersOfType,proto3,oneof"` } type ServerReflectionRequest_ListServices struct { + // List the full names of registered services. The content will not be + // checked. + // + // Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. ListServices string `protobuf:"bytes,7,opt,name=list_services,json=listServices,proto3,oneof"` } @@ -110,162 +210,178 @@ func (*ServerReflectionRequest_AllExtensionNumbersOfType) isServerReflectionRequ func (*ServerReflectionRequest_ListServices) isServerReflectionRequest_MessageRequest() {} -func (m *ServerReflectionRequest) GetMessageRequest() isServerReflectionRequest_MessageRequest { - if m != nil { - return m.MessageRequest - } - return nil -} - -func (m *ServerReflectionRequest) GetFileByFilename() string { - if x, ok := m.GetMessageRequest().(*ServerReflectionRequest_FileByFilename); ok { - return x.FileByFilename - } - return "" -} - -func (m *ServerReflectionRequest) GetFileContainingSymbol() string { - if x, ok := m.GetMessageRequest().(*ServerReflectionRequest_FileContainingSymbol); ok { - return x.FileContainingSymbol - } - return "" -} +// The type name and extension number sent by the client when requesting +// file_containing_extension. +// +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +type ExtensionRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields -func (m *ServerReflectionRequest) GetFileContainingExtension() *ExtensionRequest { - if x, ok := m.GetMessageRequest().(*ServerReflectionRequest_FileContainingExtension); ok { - return x.FileContainingExtension + // Fully-qualified type name. The format should be . + // + // Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. + ContainingType string `protobuf:"bytes,1,opt,name=containing_type,json=containingType,proto3" json:"containing_type,omitempty"` + // Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. + ExtensionNumber int32 `protobuf:"varint,2,opt,name=extension_number,json=extensionNumber,proto3" json:"extension_number,omitempty"` +} + +func (x *ExtensionRequest) Reset() { + *x = ExtensionRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_reflection_v1alpha_reflection_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } - return nil } -func (m *ServerReflectionRequest) GetAllExtensionNumbersOfType() string { - if x, ok := m.GetMessageRequest().(*ServerReflectionRequest_AllExtensionNumbersOfType); ok { - return x.AllExtensionNumbersOfType - } - return "" +func (x *ExtensionRequest) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *ServerReflectionRequest) GetListServices() string { - if x, ok := m.GetMessageRequest().(*ServerReflectionRequest_ListServices); ok { - return x.ListServices - } - return "" -} +func (*ExtensionRequest) ProtoMessage() {} -// XXX_OneofWrappers is for the internal use of the proto package. -func (*ServerReflectionRequest) XXX_OneofWrappers() []interface{} { - return []interface{}{ - (*ServerReflectionRequest_FileByFilename)(nil), - (*ServerReflectionRequest_FileContainingSymbol)(nil), - (*ServerReflectionRequest_FileContainingExtension)(nil), - (*ServerReflectionRequest_AllExtensionNumbersOfType)(nil), - (*ServerReflectionRequest_ListServices)(nil), +func (x *ExtensionRequest) ProtoReflect() protoreflect.Message { + mi := &file_grpc_reflection_v1alpha_reflection_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms } + return mi.MessageOf(x) } -// The type name and extension number sent by the client when requesting -// file_containing_extension. -type ExtensionRequest struct { - // Fully-qualified type name. The format should be . - ContainingType string `protobuf:"bytes,1,opt,name=containing_type,json=containingType,proto3" json:"containing_type,omitempty"` - ExtensionNumber int32 `protobuf:"varint,2,opt,name=extension_number,json=extensionNumber,proto3" json:"extension_number,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *ExtensionRequest) Reset() { *m = ExtensionRequest{} } -func (m *ExtensionRequest) String() string { return proto.CompactTextString(m) } -func (*ExtensionRequest) ProtoMessage() {} +// Deprecated: Use ExtensionRequest.ProtoReflect.Descriptor instead. func (*ExtensionRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_e8cf9f2921ad6c95, []int{1} + return file_grpc_reflection_v1alpha_reflection_proto_rawDescGZIP(), []int{1} } -func (m *ExtensionRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ExtensionRequest.Unmarshal(m, b) -} -func (m *ExtensionRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ExtensionRequest.Marshal(b, m, deterministic) -} -func (m *ExtensionRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_ExtensionRequest.Merge(m, src) -} -func (m *ExtensionRequest) XXX_Size() int { - return xxx_messageInfo_ExtensionRequest.Size(m) -} -func (m *ExtensionRequest) XXX_DiscardUnknown() { - xxx_messageInfo_ExtensionRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_ExtensionRequest proto.InternalMessageInfo - -func (m *ExtensionRequest) GetContainingType() string { - if m != nil { - return m.ContainingType +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +func (x *ExtensionRequest) GetContainingType() string { + if x != nil { + return x.ContainingType } return "" } -func (m *ExtensionRequest) GetExtensionNumber() int32 { - if m != nil { - return m.ExtensionNumber +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +func (x *ExtensionRequest) GetExtensionNumber() int32 { + if x != nil { + return x.ExtensionNumber } return 0 } // The message sent by the server to answer ServerReflectionInfo method. +// +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. type ServerReflectionResponse struct { - ValidHost string `protobuf:"bytes,1,opt,name=valid_host,json=validHost,proto3" json:"valid_host,omitempty"` + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. + ValidHost string `protobuf:"bytes,1,opt,name=valid_host,json=validHost,proto3" json:"valid_host,omitempty"` + // Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. OriginalRequest *ServerReflectionRequest `protobuf:"bytes,2,opt,name=original_request,json=originalRequest,proto3" json:"original_request,omitempty"` - // The server sets one of the following fields according to the - // message_request in the request. + // The server set one of the following fields according to the message_request + // in the request. + // + // Types that are assignable to MessageResponse: // - // Types that are valid to be assigned to MessageResponse: // *ServerReflectionResponse_FileDescriptorResponse // *ServerReflectionResponse_AllExtensionNumbersResponse // *ServerReflectionResponse_ListServicesResponse // *ServerReflectionResponse_ErrorResponse - MessageResponse isServerReflectionResponse_MessageResponse `protobuf_oneof:"message_response"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + MessageResponse isServerReflectionResponse_MessageResponse `protobuf_oneof:"message_response"` } -func (m *ServerReflectionResponse) Reset() { *m = ServerReflectionResponse{} } -func (m *ServerReflectionResponse) String() string { return proto.CompactTextString(m) } -func (*ServerReflectionResponse) ProtoMessage() {} -func (*ServerReflectionResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_e8cf9f2921ad6c95, []int{2} +func (x *ServerReflectionResponse) Reset() { + *x = ServerReflectionResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_reflection_v1alpha_reflection_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *ServerReflectionResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ServerReflectionResponse.Unmarshal(m, b) +func (x *ServerReflectionResponse) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *ServerReflectionResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ServerReflectionResponse.Marshal(b, m, deterministic) -} -func (m *ServerReflectionResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_ServerReflectionResponse.Merge(m, src) + +func (*ServerReflectionResponse) ProtoMessage() {} + +func (x *ServerReflectionResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_reflection_v1alpha_reflection_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -func (m *ServerReflectionResponse) XXX_Size() int { - return xxx_messageInfo_ServerReflectionResponse.Size(m) + +// Deprecated: Use ServerReflectionResponse.ProtoReflect.Descriptor instead. +func (*ServerReflectionResponse) Descriptor() ([]byte, []int) { + return file_grpc_reflection_v1alpha_reflection_proto_rawDescGZIP(), []int{2} } -func (m *ServerReflectionResponse) XXX_DiscardUnknown() { - xxx_messageInfo_ServerReflectionResponse.DiscardUnknown(m) + +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +func (x *ServerReflectionResponse) GetValidHost() string { + if x != nil { + return x.ValidHost + } + return "" } -var xxx_messageInfo_ServerReflectionResponse proto.InternalMessageInfo +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +func (x *ServerReflectionResponse) GetOriginalRequest() *ServerReflectionRequest { + if x != nil { + return x.OriginalRequest + } + return nil +} -func (m *ServerReflectionResponse) GetValidHost() string { +func (m *ServerReflectionResponse) GetMessageResponse() isServerReflectionResponse_MessageResponse { if m != nil { - return m.ValidHost + return m.MessageResponse } - return "" + return nil } -func (m *ServerReflectionResponse) GetOriginalRequest() *ServerReflectionRequest { - if m != nil { - return m.OriginalRequest +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +func (x *ServerReflectionResponse) GetFileDescriptorResponse() *FileDescriptorResponse { + if x, ok := x.GetMessageResponse().(*ServerReflectionResponse_FileDescriptorResponse); ok { + return x.FileDescriptorResponse + } + return nil +} + +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +func (x *ServerReflectionResponse) GetAllExtensionNumbersResponse() *ExtensionNumberResponse { + if x, ok := x.GetMessageResponse().(*ServerReflectionResponse_AllExtensionNumbersResponse); ok { + return x.AllExtensionNumbersResponse + } + return nil +} + +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +func (x *ServerReflectionResponse) GetListServicesResponse() *ListServiceResponse { + if x, ok := x.GetMessageResponse().(*ServerReflectionResponse_ListServicesResponse); ok { + return x.ListServicesResponse + } + return nil +} + +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +func (x *ServerReflectionResponse) GetErrorResponse() *ErrorResponse { + if x, ok := x.GetMessageResponse().(*ServerReflectionResponse_ErrorResponse); ok { + return x.ErrorResponse } return nil } @@ -275,18 +391,35 @@ type isServerReflectionResponse_MessageResponse interface { } type ServerReflectionResponse_FileDescriptorResponse struct { + // This message is used to answer file_by_filename, file_containing_symbol, + // file_containing_extension requests with transitive dependencies. As + // the repeated label is not allowed in oneof fields, we use a + // FileDescriptorResponse message to encapsulate the repeated fields. + // The reflection service is allowed to avoid sending FileDescriptorProtos + // that were previously sent in response to earlier requests in the stream. + // + // Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. FileDescriptorResponse *FileDescriptorResponse `protobuf:"bytes,4,opt,name=file_descriptor_response,json=fileDescriptorResponse,proto3,oneof"` } type ServerReflectionResponse_AllExtensionNumbersResponse struct { + // This message is used to answer all_extension_numbers_of_type requst. + // + // Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. AllExtensionNumbersResponse *ExtensionNumberResponse `protobuf:"bytes,5,opt,name=all_extension_numbers_response,json=allExtensionNumbersResponse,proto3,oneof"` } type ServerReflectionResponse_ListServicesResponse struct { + // This message is used to answer list_services request. + // + // Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. ListServicesResponse *ListServiceResponse `protobuf:"bytes,6,opt,name=list_services_response,json=listServicesResponse,proto3,oneof"` } type ServerReflectionResponse_ErrorResponse struct { + // This message is used when an error occurs. + // + // Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. ErrorResponse *ErrorResponse `protobuf:"bytes,7,opt,name=error_response,json=errorResponse,proto3,oneof"` } @@ -300,455 +433,596 @@ func (*ServerReflectionResponse_ListServicesResponse) isServerReflectionResponse func (*ServerReflectionResponse_ErrorResponse) isServerReflectionResponse_MessageResponse() {} -func (m *ServerReflectionResponse) GetMessageResponse() isServerReflectionResponse_MessageResponse { - if m != nil { - return m.MessageResponse - } - return nil -} - -func (m *ServerReflectionResponse) GetFileDescriptorResponse() *FileDescriptorResponse { - if x, ok := m.GetMessageResponse().(*ServerReflectionResponse_FileDescriptorResponse); ok { - return x.FileDescriptorResponse - } - return nil -} - -func (m *ServerReflectionResponse) GetAllExtensionNumbersResponse() *ExtensionNumberResponse { - if x, ok := m.GetMessageResponse().(*ServerReflectionResponse_AllExtensionNumbersResponse); ok { - return x.AllExtensionNumbersResponse - } - return nil -} - -func (m *ServerReflectionResponse) GetListServicesResponse() *ListServiceResponse { - if x, ok := m.GetMessageResponse().(*ServerReflectionResponse_ListServicesResponse); ok { - return x.ListServicesResponse - } - return nil -} - -func (m *ServerReflectionResponse) GetErrorResponse() *ErrorResponse { - if x, ok := m.GetMessageResponse().(*ServerReflectionResponse_ErrorResponse); ok { - return x.ErrorResponse - } - return nil -} - -// XXX_OneofWrappers is for the internal use of the proto package. -func (*ServerReflectionResponse) XXX_OneofWrappers() []interface{} { - return []interface{}{ - (*ServerReflectionResponse_FileDescriptorResponse)(nil), - (*ServerReflectionResponse_AllExtensionNumbersResponse)(nil), - (*ServerReflectionResponse_ListServicesResponse)(nil), - (*ServerReflectionResponse_ErrorResponse)(nil), - } -} - // Serialized FileDescriptorProto messages sent by the server answering // a file_by_filename, file_containing_symbol, or file_containing_extension // request. +// +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. type FileDescriptorResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // Serialized FileDescriptorProto messages. We avoid taking a dependency on // descriptor.proto, which uses proto2 only features, by making them opaque // bytes instead. - FileDescriptorProto [][]byte `protobuf:"bytes,1,rep,name=file_descriptor_proto,json=fileDescriptorProto,proto3" json:"file_descriptor_proto,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + // + // Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. + FileDescriptorProto [][]byte `protobuf:"bytes,1,rep,name=file_descriptor_proto,json=fileDescriptorProto,proto3" json:"file_descriptor_proto,omitempty"` } -func (m *FileDescriptorResponse) Reset() { *m = FileDescriptorResponse{} } -func (m *FileDescriptorResponse) String() string { return proto.CompactTextString(m) } -func (*FileDescriptorResponse) ProtoMessage() {} -func (*FileDescriptorResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_e8cf9f2921ad6c95, []int{3} +func (x *FileDescriptorResponse) Reset() { + *x = FileDescriptorResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_reflection_v1alpha_reflection_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *FileDescriptorResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_FileDescriptorResponse.Unmarshal(m, b) -} -func (m *FileDescriptorResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_FileDescriptorResponse.Marshal(b, m, deterministic) +func (x *FileDescriptorResponse) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *FileDescriptorResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_FileDescriptorResponse.Merge(m, src) -} -func (m *FileDescriptorResponse) XXX_Size() int { - return xxx_messageInfo_FileDescriptorResponse.Size(m) -} -func (m *FileDescriptorResponse) XXX_DiscardUnknown() { - xxx_messageInfo_FileDescriptorResponse.DiscardUnknown(m) + +func (*FileDescriptorResponse) ProtoMessage() {} + +func (x *FileDescriptorResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_reflection_v1alpha_reflection_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_FileDescriptorResponse proto.InternalMessageInfo +// Deprecated: Use FileDescriptorResponse.ProtoReflect.Descriptor instead. +func (*FileDescriptorResponse) Descriptor() ([]byte, []int) { + return file_grpc_reflection_v1alpha_reflection_proto_rawDescGZIP(), []int{3} +} -func (m *FileDescriptorResponse) GetFileDescriptorProto() [][]byte { - if m != nil { - return m.FileDescriptorProto +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +func (x *FileDescriptorResponse) GetFileDescriptorProto() [][]byte { + if x != nil { + return x.FileDescriptorProto } return nil } // A list of extension numbers sent by the server answering // all_extension_numbers_of_type request. +// +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. type ExtensionNumberResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // Full name of the base type, including the package name. The format // is . - BaseTypeName string `protobuf:"bytes,1,opt,name=base_type_name,json=baseTypeName,proto3" json:"base_type_name,omitempty"` - ExtensionNumber []int32 `protobuf:"varint,2,rep,packed,name=extension_number,json=extensionNumber,proto3" json:"extension_number,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + // + // Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. + BaseTypeName string `protobuf:"bytes,1,opt,name=base_type_name,json=baseTypeName,proto3" json:"base_type_name,omitempty"` + // Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. + ExtensionNumber []int32 `protobuf:"varint,2,rep,packed,name=extension_number,json=extensionNumber,proto3" json:"extension_number,omitempty"` +} + +func (x *ExtensionNumberResponse) Reset() { + *x = ExtensionNumberResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_reflection_v1alpha_reflection_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *ExtensionNumberResponse) Reset() { *m = ExtensionNumberResponse{} } -func (m *ExtensionNumberResponse) String() string { return proto.CompactTextString(m) } -func (*ExtensionNumberResponse) ProtoMessage() {} -func (*ExtensionNumberResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_e8cf9f2921ad6c95, []int{4} +func (x *ExtensionNumberResponse) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *ExtensionNumberResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ExtensionNumberResponse.Unmarshal(m, b) -} -func (m *ExtensionNumberResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ExtensionNumberResponse.Marshal(b, m, deterministic) -} -func (m *ExtensionNumberResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_ExtensionNumberResponse.Merge(m, src) -} -func (m *ExtensionNumberResponse) XXX_Size() int { - return xxx_messageInfo_ExtensionNumberResponse.Size(m) -} -func (m *ExtensionNumberResponse) XXX_DiscardUnknown() { - xxx_messageInfo_ExtensionNumberResponse.DiscardUnknown(m) +func (*ExtensionNumberResponse) ProtoMessage() {} + +func (x *ExtensionNumberResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_reflection_v1alpha_reflection_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_ExtensionNumberResponse proto.InternalMessageInfo +// Deprecated: Use ExtensionNumberResponse.ProtoReflect.Descriptor instead. +func (*ExtensionNumberResponse) Descriptor() ([]byte, []int) { + return file_grpc_reflection_v1alpha_reflection_proto_rawDescGZIP(), []int{4} +} -func (m *ExtensionNumberResponse) GetBaseTypeName() string { - if m != nil { - return m.BaseTypeName +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +func (x *ExtensionNumberResponse) GetBaseTypeName() string { + if x != nil { + return x.BaseTypeName } return "" } -func (m *ExtensionNumberResponse) GetExtensionNumber() []int32 { - if m != nil { - return m.ExtensionNumber +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +func (x *ExtensionNumberResponse) GetExtensionNumber() []int32 { + if x != nil { + return x.ExtensionNumber } return nil } // A list of ServiceResponse sent by the server answering list_services request. +// +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. type ListServiceResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // The information of each service may be expanded in the future, so we use // ServiceResponse message to encapsulate it. - Service []*ServiceResponse `protobuf:"bytes,1,rep,name=service,proto3" json:"service,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + // + // Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. + Service []*ServiceResponse `protobuf:"bytes,1,rep,name=service,proto3" json:"service,omitempty"` } -func (m *ListServiceResponse) Reset() { *m = ListServiceResponse{} } -func (m *ListServiceResponse) String() string { return proto.CompactTextString(m) } -func (*ListServiceResponse) ProtoMessage() {} -func (*ListServiceResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_e8cf9f2921ad6c95, []int{5} +func (x *ListServiceResponse) Reset() { + *x = ListServiceResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_reflection_v1alpha_reflection_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *ListServiceResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ListServiceResponse.Unmarshal(m, b) -} -func (m *ListServiceResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ListServiceResponse.Marshal(b, m, deterministic) -} -func (m *ListServiceResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_ListServiceResponse.Merge(m, src) +func (x *ListServiceResponse) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *ListServiceResponse) XXX_Size() int { - return xxx_messageInfo_ListServiceResponse.Size(m) -} -func (m *ListServiceResponse) XXX_DiscardUnknown() { - xxx_messageInfo_ListServiceResponse.DiscardUnknown(m) + +func (*ListServiceResponse) ProtoMessage() {} + +func (x *ListServiceResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_reflection_v1alpha_reflection_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_ListServiceResponse proto.InternalMessageInfo +// Deprecated: Use ListServiceResponse.ProtoReflect.Descriptor instead. +func (*ListServiceResponse) Descriptor() ([]byte, []int) { + return file_grpc_reflection_v1alpha_reflection_proto_rawDescGZIP(), []int{5} +} -func (m *ListServiceResponse) GetService() []*ServiceResponse { - if m != nil { - return m.Service +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +func (x *ListServiceResponse) GetService() []*ServiceResponse { + if x != nil { + return x.Service } return nil } // The information of a single service used by ListServiceResponse to answer // list_services request. +// +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. type ServiceResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + // Full name of a registered service, including its package name. The format // is . - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *ServiceResponse) Reset() { *m = ServiceResponse{} } -func (m *ServiceResponse) String() string { return proto.CompactTextString(m) } -func (*ServiceResponse) ProtoMessage() {} -func (*ServiceResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_e8cf9f2921ad6c95, []int{6} -} - -func (m *ServiceResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ServiceResponse.Unmarshal(m, b) -} -func (m *ServiceResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ServiceResponse.Marshal(b, m, deterministic) -} -func (m *ServiceResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_ServiceResponse.Merge(m, src) -} -func (m *ServiceResponse) XXX_Size() int { - return xxx_messageInfo_ServiceResponse.Size(m) -} -func (m *ServiceResponse) XXX_DiscardUnknown() { - xxx_messageInfo_ServiceResponse.DiscardUnknown(m) + // + // Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` } -var xxx_messageInfo_ServiceResponse proto.InternalMessageInfo - -func (m *ServiceResponse) GetName() string { - if m != nil { - return m.Name +func (x *ServiceResponse) Reset() { + *x = ServiceResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_reflection_v1alpha_reflection_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } - return "" } -// The error code and error message sent by the server when an error occurs. -type ErrorResponse struct { - // This field uses the error codes defined in grpc::StatusCode. - ErrorCode int32 `protobuf:"varint,1,opt,name=error_code,json=errorCode,proto3" json:"error_code,omitempty"` - ErrorMessage string `protobuf:"bytes,2,opt,name=error_message,json=errorMessage,proto3" json:"error_message,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` +func (x *ServiceResponse) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *ErrorResponse) Reset() { *m = ErrorResponse{} } -func (m *ErrorResponse) String() string { return proto.CompactTextString(m) } -func (*ErrorResponse) ProtoMessage() {} -func (*ErrorResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_e8cf9f2921ad6c95, []int{7} -} +func (*ServiceResponse) ProtoMessage() {} -func (m *ErrorResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ErrorResponse.Unmarshal(m, b) -} -func (m *ErrorResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ErrorResponse.Marshal(b, m, deterministic) -} -func (m *ErrorResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_ErrorResponse.Merge(m, src) -} -func (m *ErrorResponse) XXX_Size() int { - return xxx_messageInfo_ErrorResponse.Size(m) -} -func (m *ErrorResponse) XXX_DiscardUnknown() { - xxx_messageInfo_ErrorResponse.DiscardUnknown(m) +func (x *ServiceResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_reflection_v1alpha_reflection_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_ErrorResponse proto.InternalMessageInfo - -func (m *ErrorResponse) GetErrorCode() int32 { - if m != nil { - return m.ErrorCode - } - return 0 +// Deprecated: Use ServiceResponse.ProtoReflect.Descriptor instead. +func (*ServiceResponse) Descriptor() ([]byte, []int) { + return file_grpc_reflection_v1alpha_reflection_proto_rawDescGZIP(), []int{6} } -func (m *ErrorResponse) GetErrorMessage() string { - if m != nil { - return m.ErrorMessage +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +func (x *ServiceResponse) GetName() string { + if x != nil { + return x.Name } return "" } -func init() { - proto.RegisterType((*ServerReflectionRequest)(nil), "grpc.reflection.v1alpha.ServerReflectionRequest") - proto.RegisterType((*ExtensionRequest)(nil), "grpc.reflection.v1alpha.ExtensionRequest") - proto.RegisterType((*ServerReflectionResponse)(nil), "grpc.reflection.v1alpha.ServerReflectionResponse") - proto.RegisterType((*FileDescriptorResponse)(nil), "grpc.reflection.v1alpha.FileDescriptorResponse") - proto.RegisterType((*ExtensionNumberResponse)(nil), "grpc.reflection.v1alpha.ExtensionNumberResponse") - proto.RegisterType((*ListServiceResponse)(nil), "grpc.reflection.v1alpha.ListServiceResponse") - proto.RegisterType((*ServiceResponse)(nil), "grpc.reflection.v1alpha.ServiceResponse") - proto.RegisterType((*ErrorResponse)(nil), "grpc.reflection.v1alpha.ErrorResponse") -} - -func init() { - proto.RegisterFile("reflection/grpc_reflection_v1alpha/reflection.proto", fileDescriptor_e8cf9f2921ad6c95) -} - -var fileDescriptor_e8cf9f2921ad6c95 = []byte{ - // 686 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x55, 0x41, 0x6f, 0xd3, 0x4c, - 0x10, 0xad, 0xdb, 0xa4, 0x55, 0x26, 0x69, 0x92, 0x6f, 0xdb, 0xaf, 0x71, 0x41, 0x45, 0x91, 0xa1, - 0x90, 0x22, 0x94, 0xb4, 0xa9, 0x84, 0x84, 0xb8, 0xa5, 0x80, 0x82, 0x54, 0x5a, 0xe4, 0x70, 0x01, - 0x0e, 0x2b, 0x27, 0x99, 0xb8, 0x06, 0xc7, 0x6b, 0x76, 0xdd, 0x40, 0x4e, 0xfc, 0x08, 0x7e, 0x14, - 0x7f, 0x89, 0x23, 0xda, 0xb5, 0x63, 0x3b, 0x6e, 0x4c, 0xd5, 0x53, 0x9c, 0x37, 0x33, 0xfb, 0x66, - 0xf6, 0xbd, 0xb1, 0xe1, 0x94, 0xe3, 0xc4, 0xc5, 0x51, 0xe0, 0x30, 0xaf, 0x63, 0x73, 0x7f, 0x44, - 0x93, 0xff, 0x74, 0x76, 0x62, 0xb9, 0xfe, 0x95, 0xd5, 0x49, 0xa0, 0xb6, 0xcf, 0x59, 0xc0, 0x48, - 0x43, 0x66, 0xb6, 0x53, 0x70, 0x94, 0x69, 0xfc, 0x59, 0x87, 0xc6, 0x00, 0xf9, 0x0c, 0xb9, 0x19, - 0x07, 0x4d, 0xfc, 0x76, 0x8d, 0x22, 0x20, 0x04, 0x0a, 0x57, 0x4c, 0x04, 0xba, 0xd6, 0xd4, 0x5a, - 0x25, 0x53, 0x3d, 0x93, 0xa7, 0x50, 0x9f, 0x38, 0x2e, 0xd2, 0xe1, 0x9c, 0xca, 0x5f, 0xcf, 0x9a, - 0xa2, 0xbe, 0x21, 0xe3, 0xfd, 0x35, 0xb3, 0x2a, 0x91, 0xde, 0xfc, 0x4d, 0x84, 0x93, 0xe7, 0xb0, - 0xa7, 0x72, 0x47, 0xcc, 0x0b, 0x2c, 0xc7, 0x73, 0x3c, 0x9b, 0x8a, 0xf9, 0x74, 0xc8, 0x5c, 0xbd, - 0x10, 0x55, 0xec, 0xca, 0xf8, 0x59, 0x1c, 0x1e, 0xa8, 0x28, 0xb1, 0x61, 0x3f, 0x5b, 0x87, 0x3f, - 0x02, 0xf4, 0x84, 0xc3, 0x3c, 0xbd, 0xd8, 0xd4, 0x5a, 0xe5, 0xee, 0x51, 0x3b, 0x67, 0xa0, 0xf6, - 0xeb, 0x45, 0x66, 0x34, 0x45, 0x7f, 0xcd, 0x6c, 0x2c, 0xb3, 0xc4, 0x19, 0xa4, 0x07, 0x07, 0x96, - 0xeb, 0x26, 0x87, 0x53, 0xef, 0x7a, 0x3a, 0x44, 0x2e, 0x28, 0x9b, 0xd0, 0x60, 0xee, 0xa3, 0xbe, - 0x19, 0xf5, 0xb9, 0x6f, 0xb9, 0x6e, 0x5c, 0x76, 0x11, 0x26, 0x5d, 0x4e, 0x3e, 0xcc, 0x7d, 0x24, - 0x87, 0xb0, 0xed, 0x3a, 0x22, 0xa0, 0x02, 0xf9, 0xcc, 0x19, 0xa1, 0xd0, 0xb7, 0xa2, 0x9a, 0x8a, - 0x84, 0x07, 0x11, 0xda, 0xfb, 0x0f, 0x6a, 0x53, 0x14, 0xc2, 0xb2, 0x91, 0xf2, 0xb0, 0x31, 0x63, - 0x02, 0xf5, 0x6c, 0xb3, 0xe4, 0x09, 0xd4, 0x52, 0x53, 0xab, 0x1e, 0xc2, 0xdb, 0xaf, 0x26, 0xb0, - 0xa2, 0x3d, 0x82, 0x7a, 0xb6, 0x6d, 0x7d, 0xbd, 0xa9, 0xb5, 0x8a, 0x66, 0x0d, 0x97, 0x1b, 0x35, - 0x7e, 0x17, 0x40, 0xbf, 0x29, 0xb1, 0xf0, 0x99, 0x27, 0x90, 0x1c, 0x00, 0xcc, 0x2c, 0xd7, 0x19, - 0xd3, 0x94, 0xd2, 0x25, 0x85, 0xf4, 0xa5, 0xdc, 0x9f, 0xa1, 0xce, 0xb8, 0x63, 0x3b, 0x9e, 0xe5, - 0x2e, 0xfa, 0x56, 0x34, 0xe5, 0xee, 0x71, 0xae, 0x02, 0x39, 0x76, 0x32, 0x6b, 0x8b, 0x93, 0x16, - 0xc3, 0x7e, 0x05, 0x5d, 0xe9, 0x3c, 0x46, 0x31, 0xe2, 0x8e, 0x1f, 0x30, 0x4e, 0x79, 0xd4, 0x97, - 0x72, 0x48, 0xb9, 0xdb, 0xc9, 0x25, 0x91, 0x26, 0x7b, 0x15, 0xd7, 0x2d, 0xc6, 0xe9, 0xaf, 0x99, - 0xca, 0x72, 0x37, 0x23, 0xe4, 0x3b, 0x3c, 0x58, 0xad, 0x75, 0x4c, 0x59, 0xbc, 0x65, 0xae, 0x8c, - 0x01, 0x52, 0x9c, 0xf7, 0x57, 0xd8, 0x23, 0x26, 0x1e, 0xc3, 0xde, 0x92, 0x41, 0x12, 0xc2, 0x4d, - 0x45, 0xf8, 0x2c, 0x97, 0xf0, 0x3c, 0x31, 0x50, 0x8a, 0x6c, 0x37, 0xed, 0xab, 0x98, 0xe5, 0x12, - 0xaa, 0xc8, 0x79, 0xfa, 0x06, 0xb7, 0xd4, 0xe9, 0x8f, 0xf3, 0xc7, 0x91, 0xe9, 0xa9, 0x73, 0xb7, - 0x31, 0x0d, 0xf4, 0x08, 0xd4, 0x13, 0xc3, 0x86, 0x98, 0x71, 0x0e, 0x7b, 0xab, 0xef, 0x9d, 0x74, - 0xe1, 0xff, 0xac, 0x94, 0xea, 0xc5, 0xa3, 0x6b, 0xcd, 0x8d, 0x56, 0xc5, 0xdc, 0x59, 0x16, 0xe5, - 0xbd, 0x0c, 0x19, 0x5f, 0xa0, 0x91, 0x73, 0xa5, 0xe4, 0x11, 0x54, 0x87, 0x96, 0x40, 0xb5, 0x00, - 0x54, 0xbd, 0x63, 0x42, 0x67, 0x56, 0x24, 0x2a, 0xfd, 0x7f, 0x21, 0xdf, 0x2f, 0xab, 0x77, 0x60, - 0x63, 0xd5, 0x0e, 0x7c, 0x84, 0x9d, 0x15, 0xb7, 0x49, 0x7a, 0xb0, 0x15, 0xc9, 0xa2, 0x1a, 0x2d, - 0x77, 0x5b, 0xff, 0x74, 0x75, 0xaa, 0xd4, 0x5c, 0x14, 0x1a, 0x87, 0x50, 0xcb, 0x1e, 0x4b, 0xa0, - 0x90, 0x6a, 0x5a, 0x3d, 0x1b, 0x03, 0xd8, 0x5e, 0xba, 0x71, 0xb9, 0x79, 0xa1, 0x62, 0x23, 0x36, - 0x0e, 0x53, 0x8b, 0x66, 0x49, 0x21, 0x67, 0x6c, 0x8c, 0xe4, 0x21, 0x84, 0x82, 0xd0, 0x48, 0x05, - 0xb5, 0x76, 0x25, 0xb3, 0xa2, 0xc0, 0x77, 0x21, 0xd6, 0xfd, 0xa5, 0x41, 0x3d, 0xbb, 0x6e, 0xe4, - 0x27, 0xec, 0x66, 0xb1, 0xb7, 0xde, 0x84, 0x91, 0x3b, 0x6f, 0xec, 0xbd, 0x93, 0x3b, 0x54, 0x84, - 0x53, 0xb5, 0xb4, 0x63, 0xad, 0xf7, 0xf2, 0xd3, 0x0b, 0x9b, 0x31, 0xdb, 0xc5, 0xb6, 0xcd, 0x5c, - 0xcb, 0xb3, 0xdb, 0x8c, 0xdb, 0xea, 0x53, 0xd5, 0xb9, 0xfd, 0xd3, 0x35, 0xdc, 0x54, 0xbe, 0x39, - 0xfd, 0x1b, 0x00, 0x00, 0xff, 0xff, 0x6c, 0x74, 0x3a, 0x67, 0xe7, 0x06, 0x00, 0x00, -} - -// Reference imports to suppress errors if they are not otherwise used. -var _ context.Context -var _ grpc.ClientConnInterface - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the grpc package it is being compiled against. -const _ = grpc.SupportPackageIsVersion6 - -// ServerReflectionClient is the client API for ServerReflection service. +// The error code and error message sent by the server when an error occurs. // -// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. -type ServerReflectionClient interface { - // The reflection service is structured as a bidirectional stream, ensuring - // all related requests go to a single server. - ServerReflectionInfo(ctx context.Context, opts ...grpc.CallOption) (ServerReflection_ServerReflectionInfoClient, error) -} - -type serverReflectionClient struct { - cc grpc.ClientConnInterface -} - -func NewServerReflectionClient(cc grpc.ClientConnInterface) ServerReflectionClient { - return &serverReflectionClient{cc} -} +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +type ErrorResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields -func (c *serverReflectionClient) ServerReflectionInfo(ctx context.Context, opts ...grpc.CallOption) (ServerReflection_ServerReflectionInfoClient, error) { - stream, err := c.cc.NewStream(ctx, &_ServerReflection_serviceDesc.Streams[0], "/grpc.reflection.v1alpha.ServerReflection/ServerReflectionInfo", opts...) - if err != nil { - return nil, err + // This field uses the error codes defined in grpc::StatusCode. + // + // Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. + ErrorCode int32 `protobuf:"varint,1,opt,name=error_code,json=errorCode,proto3" json:"error_code,omitempty"` + // Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. + ErrorMessage string `protobuf:"bytes,2,opt,name=error_message,json=errorMessage,proto3" json:"error_message,omitempty"` +} + +func (x *ErrorResponse) Reset() { + *x = ErrorResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_grpc_reflection_v1alpha_reflection_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } - x := &serverReflectionServerReflectionInfoClient{stream} - return x, nil } -type ServerReflection_ServerReflectionInfoClient interface { - Send(*ServerReflectionRequest) error - Recv() (*ServerReflectionResponse, error) - grpc.ClientStream +func (x *ErrorResponse) String() string { + return protoimpl.X.MessageStringOf(x) } -type serverReflectionServerReflectionInfoClient struct { - grpc.ClientStream -} +func (*ErrorResponse) ProtoMessage() {} -func (x *serverReflectionServerReflectionInfoClient) Send(m *ServerReflectionRequest) error { - return x.ClientStream.SendMsg(m) -} - -func (x *serverReflectionServerReflectionInfoClient) Recv() (*ServerReflectionResponse, error) { - m := new(ServerReflectionResponse) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err +func (x *ErrorResponse) ProtoReflect() protoreflect.Message { + mi := &file_grpc_reflection_v1alpha_reflection_proto_msgTypes[7] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms } - return m, nil -} - -// ServerReflectionServer is the server API for ServerReflection service. -type ServerReflectionServer interface { - // The reflection service is structured as a bidirectional stream, ensuring - // all related requests go to a single server. - ServerReflectionInfo(ServerReflection_ServerReflectionInfoServer) error + return mi.MessageOf(x) } -// UnimplementedServerReflectionServer can be embedded to have forward compatible implementations. -type UnimplementedServerReflectionServer struct { -} - -func (*UnimplementedServerReflectionServer) ServerReflectionInfo(srv ServerReflection_ServerReflectionInfoServer) error { - return status.Errorf(codes.Unimplemented, "method ServerReflectionInfo not implemented") -} - -func RegisterServerReflectionServer(s *grpc.Server, srv ServerReflectionServer) { - s.RegisterService(&_ServerReflection_serviceDesc, srv) -} - -func _ServerReflection_ServerReflectionInfo_Handler(srv interface{}, stream grpc.ServerStream) error { - return srv.(ServerReflectionServer).ServerReflectionInfo(&serverReflectionServerReflectionInfoServer{stream}) +// Deprecated: Use ErrorResponse.ProtoReflect.Descriptor instead. +func (*ErrorResponse) Descriptor() ([]byte, []int) { + return file_grpc_reflection_v1alpha_reflection_proto_rawDescGZIP(), []int{7} } -type ServerReflection_ServerReflectionInfoServer interface { - Send(*ServerReflectionResponse) error - Recv() (*ServerReflectionRequest, error) - grpc.ServerStream +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +func (x *ErrorResponse) GetErrorCode() int32 { + if x != nil { + return x.ErrorCode + } + return 0 } -type serverReflectionServerReflectionInfoServer struct { - grpc.ServerStream +// Deprecated: The entire proto file grpc/reflection/v1alpha/reflection.proto is marked as deprecated. +func (x *ErrorResponse) GetErrorMessage() string { + if x != nil { + return x.ErrorMessage + } + return "" } -func (x *serverReflectionServerReflectionInfoServer) Send(m *ServerReflectionResponse) error { - return x.ServerStream.SendMsg(m) -} +var File_grpc_reflection_v1alpha_reflection_proto protoreflect.FileDescriptor + +var file_grpc_reflection_v1alpha_reflection_proto_rawDesc = []byte{ + 0x0a, 0x28, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x72, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x2f, 0x72, 0x65, 0x66, 0x6c, 0x65, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x17, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x72, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x61, 0x6c, + 0x70, 0x68, 0x61, 0x22, 0xf8, 0x02, 0x0a, 0x17, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x65, + 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x12, 0x0a, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, + 0x6f, 0x73, 0x74, 0x12, 0x2a, 0x0a, 0x10, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x62, 0x79, 0x5f, 0x66, + 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, + 0x0e, 0x66, 0x69, 0x6c, 0x65, 0x42, 0x79, 0x46, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x12, + 0x36, 0x0a, 0x16, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x69, + 0x6e, 0x67, 0x5f, 0x73, 0x79, 0x6d, 0x62, 0x6f, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x48, + 0x00, 0x52, 0x14, 0x66, 0x69, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x69, 0x6e, + 0x67, 0x53, 0x79, 0x6d, 0x62, 0x6f, 0x6c, 0x12, 0x67, 0x0a, 0x19, 0x66, 0x69, 0x6c, 0x65, 0x5f, + 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x69, 0x6e, 0x67, 0x5f, 0x65, 0x78, 0x74, 0x65, 0x6e, + 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x67, 0x72, 0x70, + 0x63, 0x2e, 0x72, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x61, + 0x6c, 0x70, 0x68, 0x61, 0x2e, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x00, 0x52, 0x17, 0x66, 0x69, 0x6c, 0x65, 0x43, 0x6f, 0x6e, + 0x74, 0x61, 0x69, 0x6e, 0x69, 0x6e, 0x67, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, + 0x12, 0x42, 0x0a, 0x1d, 0x61, 0x6c, 0x6c, 0x5f, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, + 0x6e, 0x5f, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x73, 0x5f, 0x6f, 0x66, 0x5f, 0x74, 0x79, 0x70, + 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x19, 0x61, 0x6c, 0x6c, 0x45, 0x78, + 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x73, 0x4f, 0x66, + 0x54, 0x79, 0x70, 0x65, 0x12, 0x25, 0x0a, 0x0d, 0x6c, 0x69, 0x73, 0x74, 0x5f, 0x73, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0c, 0x6c, + 0x69, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x42, 0x11, 0x0a, 0x0f, 0x6d, + 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x66, + 0x0a, 0x10, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x27, 0x0a, 0x0f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x69, 0x6e, 0x67, + 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x63, 0x6f, 0x6e, + 0x74, 0x61, 0x69, 0x6e, 0x69, 0x6e, 0x67, 0x54, 0x79, 0x70, 0x65, 0x12, 0x29, 0x0a, 0x10, 0x65, + 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0f, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, + 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x22, 0xc7, 0x04, 0x0a, 0x18, 0x53, 0x65, 0x72, 0x76, 0x65, + 0x72, 0x52, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x5f, 0x68, 0x6f, 0x73, + 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x48, 0x6f, + 0x73, 0x74, 0x12, 0x5b, 0x0a, 0x10, 0x6f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x61, 0x6c, 0x5f, 0x72, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x67, + 0x72, 0x70, 0x63, 0x2e, 0x72, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, + 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x65, 0x66, + 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, 0x0f, + 0x6f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x61, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x6b, 0x0a, 0x18, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, + 0x6f, 0x72, 0x5f, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x2f, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x72, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, + 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x2e, 0x46, 0x69, 0x6c, 0x65, + 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x48, 0x00, 0x52, 0x16, 0x66, 0x69, 0x6c, 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, + 0x70, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x77, 0x0a, 0x1e, + 0x61, 0x6c, 0x6c, 0x5f, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x6e, 0x75, + 0x6d, 0x62, 0x65, 0x72, 0x73, 0x5f, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x72, 0x65, 0x66, 0x6c, + 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x2e, 0x45, + 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x00, 0x52, 0x1b, 0x61, 0x6c, 0x6c, 0x45, 0x78, 0x74, + 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x64, 0x0a, 0x16, 0x6c, 0x69, 0x73, 0x74, 0x5f, 0x73, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x5f, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x18, + 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x72, 0x65, 0x66, + 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x2e, + 0x4c, 0x69, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x48, 0x00, 0x52, 0x14, 0x6c, 0x69, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4f, 0x0a, 0x0e, 0x65, + 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x18, 0x07, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x72, 0x65, 0x66, 0x6c, 0x65, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x2e, 0x45, 0x72, + 0x72, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x00, 0x52, 0x0d, 0x65, + 0x72, 0x72, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x12, 0x0a, 0x10, + 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x5f, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x4c, 0x0a, 0x16, 0x46, 0x69, 0x6c, 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, + 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x32, 0x0a, 0x15, 0x66, 0x69, + 0x6c, 0x65, 0x5f, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x5f, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x13, 0x66, 0x69, 0x6c, 0x65, 0x44, + 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x6a, + 0x0a, 0x17, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x4e, 0x75, 0x6d, 0x62, 0x65, + 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x24, 0x0a, 0x0e, 0x62, 0x61, 0x73, + 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0c, 0x62, 0x61, 0x73, 0x65, 0x54, 0x79, 0x70, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, + 0x29, 0x0a, 0x10, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x6e, 0x75, 0x6d, + 0x62, 0x65, 0x72, 0x18, 0x02, 0x20, 0x03, 0x28, 0x05, 0x52, 0x0f, 0x65, 0x78, 0x74, 0x65, 0x6e, + 0x73, 0x69, 0x6f, 0x6e, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x22, 0x59, 0x0a, 0x13, 0x4c, 0x69, + 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x42, 0x0a, 0x07, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x72, 0x65, 0x66, 0x6c, 0x65, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x2e, 0x53, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x52, 0x07, 0x73, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x22, 0x25, 0x0a, 0x0f, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x53, 0x0a, 0x0d, + 0x45, 0x72, 0x72, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1d, 0x0a, + 0x0a, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x05, 0x52, 0x09, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x23, 0x0a, 0x0d, + 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0c, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, + 0x65, 0x32, 0x93, 0x01, 0x0a, 0x10, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x65, 0x66, 0x6c, + 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x7f, 0x0a, 0x14, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, + 0x52, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x30, + 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x72, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, + 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x31, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x72, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x65, + 0x72, 0x52, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x42, 0x73, 0x0a, 0x1a, 0x69, 0x6f, 0x2e, 0x67, 0x72, + 0x70, 0x63, 0x2e, 0x72, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, + 0x61, 0x6c, 0x70, 0x68, 0x61, 0x42, 0x15, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x65, 0x66, + 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x39, + 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e, 0x6f, 0x72, + 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x72, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x72, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x5f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0xb8, 0x01, 0x01, 0x62, 0x06, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_grpc_reflection_v1alpha_reflection_proto_rawDescOnce sync.Once + file_grpc_reflection_v1alpha_reflection_proto_rawDescData = file_grpc_reflection_v1alpha_reflection_proto_rawDesc +) -func (x *serverReflectionServerReflectionInfoServer) Recv() (*ServerReflectionRequest, error) { - m := new(ServerReflectionRequest) - if err := x.ServerStream.RecvMsg(m); err != nil { - return nil, err +func file_grpc_reflection_v1alpha_reflection_proto_rawDescGZIP() []byte { + file_grpc_reflection_v1alpha_reflection_proto_rawDescOnce.Do(func() { + file_grpc_reflection_v1alpha_reflection_proto_rawDescData = protoimpl.X.CompressGZIP(file_grpc_reflection_v1alpha_reflection_proto_rawDescData) + }) + return file_grpc_reflection_v1alpha_reflection_proto_rawDescData +} + +var file_grpc_reflection_v1alpha_reflection_proto_msgTypes = make([]protoimpl.MessageInfo, 8) +var file_grpc_reflection_v1alpha_reflection_proto_goTypes = []interface{}{ + (*ServerReflectionRequest)(nil), // 0: grpc.reflection.v1alpha.ServerReflectionRequest + (*ExtensionRequest)(nil), // 1: grpc.reflection.v1alpha.ExtensionRequest + (*ServerReflectionResponse)(nil), // 2: grpc.reflection.v1alpha.ServerReflectionResponse + (*FileDescriptorResponse)(nil), // 3: grpc.reflection.v1alpha.FileDescriptorResponse + (*ExtensionNumberResponse)(nil), // 4: grpc.reflection.v1alpha.ExtensionNumberResponse + (*ListServiceResponse)(nil), // 5: grpc.reflection.v1alpha.ListServiceResponse + (*ServiceResponse)(nil), // 6: grpc.reflection.v1alpha.ServiceResponse + (*ErrorResponse)(nil), // 7: grpc.reflection.v1alpha.ErrorResponse +} +var file_grpc_reflection_v1alpha_reflection_proto_depIdxs = []int32{ + 1, // 0: grpc.reflection.v1alpha.ServerReflectionRequest.file_containing_extension:type_name -> grpc.reflection.v1alpha.ExtensionRequest + 0, // 1: grpc.reflection.v1alpha.ServerReflectionResponse.original_request:type_name -> grpc.reflection.v1alpha.ServerReflectionRequest + 3, // 2: grpc.reflection.v1alpha.ServerReflectionResponse.file_descriptor_response:type_name -> grpc.reflection.v1alpha.FileDescriptorResponse + 4, // 3: grpc.reflection.v1alpha.ServerReflectionResponse.all_extension_numbers_response:type_name -> grpc.reflection.v1alpha.ExtensionNumberResponse + 5, // 4: grpc.reflection.v1alpha.ServerReflectionResponse.list_services_response:type_name -> grpc.reflection.v1alpha.ListServiceResponse + 7, // 5: grpc.reflection.v1alpha.ServerReflectionResponse.error_response:type_name -> grpc.reflection.v1alpha.ErrorResponse + 6, // 6: grpc.reflection.v1alpha.ListServiceResponse.service:type_name -> grpc.reflection.v1alpha.ServiceResponse + 0, // 7: grpc.reflection.v1alpha.ServerReflection.ServerReflectionInfo:input_type -> grpc.reflection.v1alpha.ServerReflectionRequest + 2, // 8: grpc.reflection.v1alpha.ServerReflection.ServerReflectionInfo:output_type -> grpc.reflection.v1alpha.ServerReflectionResponse + 8, // [8:9] is the sub-list for method output_type + 7, // [7:8] is the sub-list for method input_type + 7, // [7:7] is the sub-list for extension type_name + 7, // [7:7] is the sub-list for extension extendee + 0, // [0:7] is the sub-list for field type_name +} + +func init() { file_grpc_reflection_v1alpha_reflection_proto_init() } +func file_grpc_reflection_v1alpha_reflection_proto_init() { + if File_grpc_reflection_v1alpha_reflection_proto != nil { + return } - return m, nil -} - -var _ServerReflection_serviceDesc = grpc.ServiceDesc{ - ServiceName: "grpc.reflection.v1alpha.ServerReflection", - HandlerType: (*ServerReflectionServer)(nil), - Methods: []grpc.MethodDesc{}, - Streams: []grpc.StreamDesc{ - { - StreamName: "ServerReflectionInfo", - Handler: _ServerReflection_ServerReflectionInfo_Handler, - ServerStreams: true, - ClientStreams: true, + if !protoimpl.UnsafeEnabled { + file_grpc_reflection_v1alpha_reflection_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ServerReflectionRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_reflection_v1alpha_reflection_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ExtensionRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_reflection_v1alpha_reflection_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ServerReflectionResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_reflection_v1alpha_reflection_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*FileDescriptorResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_reflection_v1alpha_reflection_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ExtensionNumberResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_reflection_v1alpha_reflection_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ListServiceResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_reflection_v1alpha_reflection_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ServiceResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_grpc_reflection_v1alpha_reflection_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ErrorResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + file_grpc_reflection_v1alpha_reflection_proto_msgTypes[0].OneofWrappers = []interface{}{ + (*ServerReflectionRequest_FileByFilename)(nil), + (*ServerReflectionRequest_FileContainingSymbol)(nil), + (*ServerReflectionRequest_FileContainingExtension)(nil), + (*ServerReflectionRequest_AllExtensionNumbersOfType)(nil), + (*ServerReflectionRequest_ListServices)(nil), + } + file_grpc_reflection_v1alpha_reflection_proto_msgTypes[2].OneofWrappers = []interface{}{ + (*ServerReflectionResponse_FileDescriptorResponse)(nil), + (*ServerReflectionResponse_AllExtensionNumbersResponse)(nil), + (*ServerReflectionResponse_ListServicesResponse)(nil), + (*ServerReflectionResponse_ErrorResponse)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_grpc_reflection_v1alpha_reflection_proto_rawDesc, + NumEnums: 0, + NumMessages: 8, + NumExtensions: 0, + NumServices: 1, }, - }, - Metadata: "reflection/grpc_reflection_v1alpha/reflection.proto", + GoTypes: file_grpc_reflection_v1alpha_reflection_proto_goTypes, + DependencyIndexes: file_grpc_reflection_v1alpha_reflection_proto_depIdxs, + MessageInfos: file_grpc_reflection_v1alpha_reflection_proto_msgTypes, + }.Build() + File_grpc_reflection_v1alpha_reflection_proto = out.File + file_grpc_reflection_v1alpha_reflection_proto_rawDesc = nil + file_grpc_reflection_v1alpha_reflection_proto_goTypes = nil + file_grpc_reflection_v1alpha_reflection_proto_depIdxs = nil } diff --git a/reflection/grpc_reflection_v1alpha/reflection.proto b/reflection/grpc_reflection_v1alpha/reflection.proto deleted file mode 100644 index ee2b82c0a5b3..000000000000 --- a/reflection/grpc_reflection_v1alpha/reflection.proto +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright 2016 gRPC authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Service exported by server reflection - -syntax = "proto3"; - -option go_package = "google.golang.org/grpc/reflection/grpc_reflection_v1alpha"; - -package grpc.reflection.v1alpha; - -service ServerReflection { - // The reflection service is structured as a bidirectional stream, ensuring - // all related requests go to a single server. - rpc ServerReflectionInfo(stream ServerReflectionRequest) - returns (stream ServerReflectionResponse); -} - -// The message sent by the client when calling ServerReflectionInfo method. -message ServerReflectionRequest { - string host = 1; - // To use reflection service, the client should set one of the following - // fields in message_request. The server distinguishes requests by their - // defined field and then handles them using corresponding methods. - oneof message_request { - // Find a proto file by the file name. - string file_by_filename = 3; - - // Find the proto file that declares the given fully-qualified symbol name. - // This field should be a fully-qualified symbol name - // (e.g. .[.] or .). - string file_containing_symbol = 4; - - // Find the proto file which defines an extension extending the given - // message type with the given field number. - ExtensionRequest file_containing_extension = 5; - - // Finds the tag numbers used by all known extensions of extendee_type, and - // appends them to ExtensionNumberResponse in an undefined order. - // Its corresponding method is best-effort: it's not guaranteed that the - // reflection service will implement this method, and it's not guaranteed - // that this method will provide all extensions. Returns - // StatusCode::UNIMPLEMENTED if it's not implemented. - // This field should be a fully-qualified type name. The format is - // . - string all_extension_numbers_of_type = 6; - - // List the full names of registered services. The content will not be - // checked. - string list_services = 7; - } -} - -// The type name and extension number sent by the client when requesting -// file_containing_extension. -message ExtensionRequest { - // Fully-qualified type name. The format should be . - string containing_type = 1; - int32 extension_number = 2; -} - -// The message sent by the server to answer ServerReflectionInfo method. -message ServerReflectionResponse { - string valid_host = 1; - ServerReflectionRequest original_request = 2; - // The server sets one of the following fields according to the - // message_request in the request. - oneof message_response { - // This message is used to answer file_by_filename, file_containing_symbol, - // file_containing_extension requests with transitive dependencies. - // As the repeated label is not allowed in oneof fields, we use a - // FileDescriptorResponse message to encapsulate the repeated fields. - // The reflection service is allowed to avoid sending FileDescriptorProtos - // that were previously sent in response to earlier requests in the stream. - FileDescriptorResponse file_descriptor_response = 4; - - // This message is used to answer all_extension_numbers_of_type requests. - ExtensionNumberResponse all_extension_numbers_response = 5; - - // This message is used to answer list_services requests. - ListServiceResponse list_services_response = 6; - - // This message is used when an error occurs. - ErrorResponse error_response = 7; - } -} - -// Serialized FileDescriptorProto messages sent by the server answering -// a file_by_filename, file_containing_symbol, or file_containing_extension -// request. -message FileDescriptorResponse { - // Serialized FileDescriptorProto messages. We avoid taking a dependency on - // descriptor.proto, which uses proto2 only features, by making them opaque - // bytes instead. - repeated bytes file_descriptor_proto = 1; -} - -// A list of extension numbers sent by the server answering -// all_extension_numbers_of_type request. -message ExtensionNumberResponse { - // Full name of the base type, including the package name. The format - // is . - string base_type_name = 1; - repeated int32 extension_number = 2; -} - -// A list of ServiceResponse sent by the server answering list_services request. -message ListServiceResponse { - // The information of each service may be expanded in the future, so we use - // ServiceResponse message to encapsulate it. - repeated ServiceResponse service = 1; -} - -// The information of a single service used by ListServiceResponse to answer -// list_services request. -message ServiceResponse { - // Full name of a registered service, including its package name. The format - // is . - string name = 1; -} - -// The error code and error message sent by the server when an error occurs. -message ErrorResponse { - // This field uses the error codes defined in grpc::StatusCode. - int32 error_code = 1; - string error_message = 2; -} diff --git a/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go b/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go index 95f9d0cb62d9..367a029be6b3 100644 --- a/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go +++ b/reflection/grpc_reflection_v1alpha/reflection_grpc.pb.go @@ -1,8 +1,31 @@ +// Copyright 2016 The gRPC Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// Service exported by server reflection + +// Warning: this entire file is deprecated. Use this instead: +// https://github.com/grpc/grpc-proto/blob/master/grpc/reflection/v1/reflection.proto + // Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.3.0 +// - protoc v4.22.0 +// grpc/reflection/v1alpha/reflection.proto is a deprecated file. package grpc_reflection_v1alpha import ( + context "context" grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" @@ -10,66 +33,129 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. const _ = grpc.SupportPackageIsVersion7 -// ServerReflectionService is the service API for ServerReflection service. -// Fields should be assigned to their respective handler implementations only before -// RegisterServerReflectionService is called. Any unassigned fields will result in the -// handler for that method returning an Unimplemented error. -type ServerReflectionService struct { +const ( + ServerReflection_ServerReflectionInfo_FullMethodName = "/grpc.reflection.v1alpha.ServerReflection/ServerReflectionInfo" +) + +// ServerReflectionClient is the client API for ServerReflection service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type ServerReflectionClient interface { // The reflection service is structured as a bidirectional stream, ensuring // all related requests go to a single server. - ServerReflectionInfo func(ServerReflection_ServerReflectionInfoServer) error + ServerReflectionInfo(ctx context.Context, opts ...grpc.CallOption) (ServerReflection_ServerReflectionInfoClient, error) } -func (s *ServerReflectionService) serverReflectionInfo(_ interface{}, stream grpc.ServerStream) error { - if s.ServerReflectionInfo == nil { - return status.Errorf(codes.Unimplemented, "method ServerReflectionInfo not implemented") - } - return s.ServerReflectionInfo(&serverReflectionServerReflectionInfoServer{stream}) -} - -// RegisterServerReflectionService registers a service implementation with a gRPC server. -func RegisterServerReflectionService(s grpc.ServiceRegistrar, srv *ServerReflectionService) { - sd := grpc.ServiceDesc{ - ServiceName: "grpc.reflection.v1alpha.ServerReflection", - Methods: []grpc.MethodDesc{}, - Streams: []grpc.StreamDesc{ - { - StreamName: "ServerReflectionInfo", - Handler: srv.serverReflectionInfo, - ServerStreams: true, - ClientStreams: true, - }, - }, - Metadata: "reflection/grpc_reflection_v1alpha/reflection.proto", +type serverReflectionClient struct { + cc grpc.ClientConnInterface +} + +func NewServerReflectionClient(cc grpc.ClientConnInterface) ServerReflectionClient { + return &serverReflectionClient{cc} +} + +func (c *serverReflectionClient) ServerReflectionInfo(ctx context.Context, opts ...grpc.CallOption) (ServerReflection_ServerReflectionInfoClient, error) { + stream, err := c.cc.NewStream(ctx, &ServerReflection_ServiceDesc.Streams[0], ServerReflection_ServerReflectionInfo_FullMethodName, opts...) + if err != nil { + return nil, err } + x := &serverReflectionServerReflectionInfoClient{stream} + return x, nil +} + +type ServerReflection_ServerReflectionInfoClient interface { + Send(*ServerReflectionRequest) error + Recv() (*ServerReflectionResponse, error) + grpc.ClientStream +} + +type serverReflectionServerReflectionInfoClient struct { + grpc.ClientStream +} - s.RegisterService(&sd, nil) -} - -// NewServerReflectionService creates a new ServerReflectionService containing the -// implemented methods of the ServerReflection service in s. Any unimplemented -// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client. -// This includes situations where the method handler is misspelled or has the wrong -// signature. For this reason, this function should be used with great care and -// is not recommended to be used by most users. -func NewServerReflectionService(s interface{}) *ServerReflectionService { - ns := &ServerReflectionService{} - if h, ok := s.(interface { - ServerReflectionInfo(ServerReflection_ServerReflectionInfoServer) error - }); ok { - ns.ServerReflectionInfo = h.ServerReflectionInfo +func (x *serverReflectionServerReflectionInfoClient) Send(m *ServerReflectionRequest) error { + return x.ClientStream.SendMsg(m) +} + +func (x *serverReflectionServerReflectionInfoClient) Recv() (*ServerReflectionResponse, error) { + m := new(ServerReflectionResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err } - return ns + return m, nil } -// UnstableServerReflectionService is the service API for ServerReflection service. -// New methods may be added to this interface if they are added to the service -// definition, which is not a backward-compatible change. For this reason, -// use of this type is not recommended. -type UnstableServerReflectionService interface { +// ServerReflectionServer is the server API for ServerReflection service. +// All implementations should embed UnimplementedServerReflectionServer +// for forward compatibility +type ServerReflectionServer interface { // The reflection service is structured as a bidirectional stream, ensuring // all related requests go to a single server. ServerReflectionInfo(ServerReflection_ServerReflectionInfoServer) error } + +// UnimplementedServerReflectionServer should be embedded to have forward compatible implementations. +type UnimplementedServerReflectionServer struct { +} + +func (UnimplementedServerReflectionServer) ServerReflectionInfo(ServerReflection_ServerReflectionInfoServer) error { + return status.Errorf(codes.Unimplemented, "method ServerReflectionInfo not implemented") +} + +// UnsafeServerReflectionServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to ServerReflectionServer will +// result in compilation errors. +type UnsafeServerReflectionServer interface { + mustEmbedUnimplementedServerReflectionServer() +} + +func RegisterServerReflectionServer(s grpc.ServiceRegistrar, srv ServerReflectionServer) { + s.RegisterService(&ServerReflection_ServiceDesc, srv) +} + +func _ServerReflection_ServerReflectionInfo_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(ServerReflectionServer).ServerReflectionInfo(&serverReflectionServerReflectionInfoServer{stream}) +} + +type ServerReflection_ServerReflectionInfoServer interface { + Send(*ServerReflectionResponse) error + Recv() (*ServerReflectionRequest, error) + grpc.ServerStream +} + +type serverReflectionServerReflectionInfoServer struct { + grpc.ServerStream +} + +func (x *serverReflectionServerReflectionInfoServer) Send(m *ServerReflectionResponse) error { + return x.ServerStream.SendMsg(m) +} + +func (x *serverReflectionServerReflectionInfoServer) Recv() (*ServerReflectionRequest, error) { + m := new(ServerReflectionRequest) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +// ServerReflection_ServiceDesc is the grpc.ServiceDesc for ServerReflection service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var ServerReflection_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.reflection.v1alpha.ServerReflection", + HandlerType: (*ServerReflectionServer)(nil), + Methods: []grpc.MethodDesc{}, + Streams: []grpc.StreamDesc{ + { + StreamName: "ServerReflectionInfo", + Handler: _ServerReflection_ServerReflectionInfo_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "grpc/reflection/v1alpha/reflection.proto", +} diff --git a/reflection/grpc_testing/proto2.pb.go b/reflection/grpc_testing/proto2.pb.go index d34adb82a85e..8c1002bade14 100644 --- a/reflection/grpc_testing/proto2.pb.go +++ b/reflection/grpc_testing/proto2.pb.go @@ -1,90 +1,162 @@ +// Copyright 2017 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.31.0 +// protoc v4.22.0 // source: reflection/grpc_testing/proto2.proto package grpc_testing import ( - fmt "fmt" - proto "github.com/golang/protobuf/proto" - math "math" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" ) -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) type ToBeExtended struct { - Foo *int32 `protobuf:"varint,1,req,name=foo" json:"foo,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - proto.XXX_InternalExtensions `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + extensionFields protoimpl.ExtensionFields -func (m *ToBeExtended) Reset() { *m = ToBeExtended{} } -func (m *ToBeExtended) String() string { return proto.CompactTextString(m) } -func (*ToBeExtended) ProtoMessage() {} -func (*ToBeExtended) Descriptor() ([]byte, []int) { - return fileDescriptor_dddbb2c1ebdcf2b8, []int{0} + Foo *int32 `protobuf:"varint,1,req,name=foo" json:"foo,omitempty"` } -var extRange_ToBeExtended = []proto.ExtensionRange{ - {Start: 10, End: 30}, +func (x *ToBeExtended) Reset() { + *x = ToBeExtended{} + if protoimpl.UnsafeEnabled { + mi := &file_reflection_grpc_testing_proto2_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (*ToBeExtended) ExtensionRangeArray() []proto.ExtensionRange { - return extRange_ToBeExtended +func (x *ToBeExtended) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *ToBeExtended) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ToBeExtended.Unmarshal(m, b) -} -func (m *ToBeExtended) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ToBeExtended.Marshal(b, m, deterministic) -} -func (m *ToBeExtended) XXX_Merge(src proto.Message) { - xxx_messageInfo_ToBeExtended.Merge(m, src) -} -func (m *ToBeExtended) XXX_Size() int { - return xxx_messageInfo_ToBeExtended.Size(m) -} -func (m *ToBeExtended) XXX_DiscardUnknown() { - xxx_messageInfo_ToBeExtended.DiscardUnknown(m) +func (*ToBeExtended) ProtoMessage() {} + +func (x *ToBeExtended) ProtoReflect() protoreflect.Message { + mi := &file_reflection_grpc_testing_proto2_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_ToBeExtended proto.InternalMessageInfo +// Deprecated: Use ToBeExtended.ProtoReflect.Descriptor instead. +func (*ToBeExtended) Descriptor() ([]byte, []int) { + return file_reflection_grpc_testing_proto2_proto_rawDescGZIP(), []int{0} +} -func (m *ToBeExtended) GetFoo() int32 { - if m != nil && m.Foo != nil { - return *m.Foo +func (x *ToBeExtended) GetFoo() int32 { + if x != nil && x.Foo != nil { + return *x.Foo } return 0 } -func init() { - proto.RegisterType((*ToBeExtended)(nil), "grpc.testing.ToBeExtended") +var File_reflection_grpc_testing_proto2_proto protoreflect.FileDescriptor + +var file_reflection_grpc_testing_proto2_proto_rawDesc = []byte{ + 0x0a, 0x24, 0x72, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x67, 0x72, 0x70, + 0x63, 0x5f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x32, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, + 0x74, 0x69, 0x6e, 0x67, 0x22, 0x26, 0x0a, 0x0c, 0x54, 0x6f, 0x42, 0x65, 0x45, 0x78, 0x74, 0x65, + 0x6e, 0x64, 0x65, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x66, 0x6f, 0x6f, 0x18, 0x01, 0x20, 0x02, 0x28, + 0x05, 0x52, 0x03, 0x66, 0x6f, 0x6f, 0x2a, 0x04, 0x08, 0x0a, 0x10, 0x1f, 0x42, 0x30, 0x5a, 0x2e, + 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e, 0x6f, 0x72, + 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x72, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, } -func init() { - proto.RegisterFile("reflection/grpc_testing/proto2.proto", fileDescriptor_dddbb2c1ebdcf2b8) +var ( + file_reflection_grpc_testing_proto2_proto_rawDescOnce sync.Once + file_reflection_grpc_testing_proto2_proto_rawDescData = file_reflection_grpc_testing_proto2_proto_rawDesc +) + +func file_reflection_grpc_testing_proto2_proto_rawDescGZIP() []byte { + file_reflection_grpc_testing_proto2_proto_rawDescOnce.Do(func() { + file_reflection_grpc_testing_proto2_proto_rawDescData = protoimpl.X.CompressGZIP(file_reflection_grpc_testing_proto2_proto_rawDescData) + }) + return file_reflection_grpc_testing_proto2_proto_rawDescData } -var fileDescriptor_dddbb2c1ebdcf2b8 = []byte{ - // 130 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x52, 0x29, 0x4a, 0x4d, 0xcb, - 0x49, 0x4d, 0x2e, 0xc9, 0xcc, 0xcf, 0xd3, 0x4f, 0x2f, 0x2a, 0x48, 0x8e, 0x2f, 0x49, 0x2d, 0x2e, - 0xc9, 0xcc, 0x4b, 0xd7, 0x2f, 0x28, 0xca, 0x2f, 0xc9, 0x37, 0xd2, 0x03, 0x53, 0x42, 0x3c, 0x20, - 0x29, 0x3d, 0xa8, 0x94, 0x92, 0x1a, 0x17, 0x4f, 0x48, 0xbe, 0x53, 0xaa, 0x6b, 0x45, 0x49, 0x6a, - 0x5e, 0x4a, 0x6a, 0x8a, 0x90, 0x00, 0x17, 0x73, 0x5a, 0x7e, 0xbe, 0x04, 0xa3, 0x02, 0x93, 0x06, - 0x6b, 0x10, 0x88, 0xa9, 0xc5, 0xc2, 0xc1, 0x25, 0x20, 0xef, 0x64, 0x10, 0xa5, 0x97, 0x9e, 0x9f, - 0x9f, 0x9e, 0x93, 0xaa, 0x97, 0x9e, 0x9f, 0x93, 0x98, 0x97, 0xae, 0x97, 0x5f, 0x94, 0x0e, 0xb6, - 0x44, 0x1f, 0x87, 0xa5, 0x80, 0x00, 0x00, 0x00, 0xff, 0xff, 0x7f, 0x05, 0x50, 0x64, 0x8e, 0x00, - 0x00, 0x00, +var file_reflection_grpc_testing_proto2_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_reflection_grpc_testing_proto2_proto_goTypes = []interface{}{ + (*ToBeExtended)(nil), // 0: grpc.testing.ToBeExtended +} +var file_reflection_grpc_testing_proto2_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_reflection_grpc_testing_proto2_proto_init() } +func file_reflection_grpc_testing_proto2_proto_init() { + if File_reflection_grpc_testing_proto2_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_reflection_grpc_testing_proto2_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ToBeExtended); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + case 3: + return &v.extensionFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_reflection_grpc_testing_proto2_proto_rawDesc, + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_reflection_grpc_testing_proto2_proto_goTypes, + DependencyIndexes: file_reflection_grpc_testing_proto2_proto_depIdxs, + MessageInfos: file_reflection_grpc_testing_proto2_proto_msgTypes, + }.Build() + File_reflection_grpc_testing_proto2_proto = out.File + file_reflection_grpc_testing_proto2_proto_rawDesc = nil + file_reflection_grpc_testing_proto2_proto_goTypes = nil + file_reflection_grpc_testing_proto2_proto_depIdxs = nil } diff --git a/reflection/grpc_testing/proto2_ext.pb.go b/reflection/grpc_testing/proto2_ext.pb.go index 631b5146d56c..1de098af55fb 100644 --- a/reflection/grpc_testing/proto2_ext.pb.go +++ b/reflection/grpc_testing/proto2_ext.pb.go @@ -1,116 +1,223 @@ +// Copyright 2017 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.31.0 +// protoc v4.22.0 // source: reflection/grpc_testing/proto2_ext.proto package grpc_testing import ( - fmt "fmt" - proto "github.com/golang/protobuf/proto" - math "math" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" ) -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) type Extension struct { - Whatzit *int32 `protobuf:"varint,1,opt,name=whatzit" json:"whatzit,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields -func (m *Extension) Reset() { *m = Extension{} } -func (m *Extension) String() string { return proto.CompactTextString(m) } -func (*Extension) ProtoMessage() {} -func (*Extension) Descriptor() ([]byte, []int) { - return fileDescriptor_071dc827b8673a0c, []int{0} + Whatzit *int32 `protobuf:"varint,1,opt,name=whatzit" json:"whatzit,omitempty"` } -func (m *Extension) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Extension.Unmarshal(m, b) -} -func (m *Extension) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Extension.Marshal(b, m, deterministic) -} -func (m *Extension) XXX_Merge(src proto.Message) { - xxx_messageInfo_Extension.Merge(m, src) +func (x *Extension) Reset() { + *x = Extension{} + if protoimpl.UnsafeEnabled { + mi := &file_reflection_grpc_testing_proto2_ext_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *Extension) XXX_Size() int { - return xxx_messageInfo_Extension.Size(m) + +func (x *Extension) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *Extension) XXX_DiscardUnknown() { - xxx_messageInfo_Extension.DiscardUnknown(m) + +func (*Extension) ProtoMessage() {} + +func (x *Extension) ProtoReflect() protoreflect.Message { + mi := &file_reflection_grpc_testing_proto2_ext_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_Extension proto.InternalMessageInfo +// Deprecated: Use Extension.ProtoReflect.Descriptor instead. +func (*Extension) Descriptor() ([]byte, []int) { + return file_reflection_grpc_testing_proto2_ext_proto_rawDescGZIP(), []int{0} +} -func (m *Extension) GetWhatzit() int32 { - if m != nil && m.Whatzit != nil { - return *m.Whatzit +func (x *Extension) GetWhatzit() int32 { + if x != nil && x.Whatzit != nil { + return *x.Whatzit } return 0 } -var E_Foo = &proto.ExtensionDesc{ - ExtendedType: (*ToBeExtended)(nil), - ExtensionType: (*int32)(nil), - Field: 13, - Name: "grpc.testing.foo", - Tag: "varint,13,opt,name=foo", - Filename: "reflection/grpc_testing/proto2_ext.proto", +var file_reflection_grpc_testing_proto2_ext_proto_extTypes = []protoimpl.ExtensionInfo{ + { + ExtendedType: (*ToBeExtended)(nil), + ExtensionType: (*int32)(nil), + Field: 13, + Name: "grpc.testing.foo", + Tag: "varint,13,opt,name=foo", + Filename: "reflection/grpc_testing/proto2_ext.proto", + }, + { + ExtendedType: (*ToBeExtended)(nil), + ExtensionType: (*Extension)(nil), + Field: 17, + Name: "grpc.testing.bar", + Tag: "bytes,17,opt,name=bar", + Filename: "reflection/grpc_testing/proto2_ext.proto", + }, + { + ExtendedType: (*ToBeExtended)(nil), + ExtensionType: (*SearchRequest)(nil), + Field: 19, + Name: "grpc.testing.baz", + Tag: "bytes,19,opt,name=baz", + Filename: "reflection/grpc_testing/proto2_ext.proto", + }, } -var E_Bar = &proto.ExtensionDesc{ - ExtendedType: (*ToBeExtended)(nil), - ExtensionType: (*Extension)(nil), - Field: 17, - Name: "grpc.testing.bar", - Tag: "bytes,17,opt,name=bar", - Filename: "reflection/grpc_testing/proto2_ext.proto", -} +// Extension fields to ToBeExtended. +var ( + // optional int32 foo = 13; + E_Foo = &file_reflection_grpc_testing_proto2_ext_proto_extTypes[0] + // optional grpc.testing.Extension bar = 17; + E_Bar = &file_reflection_grpc_testing_proto2_ext_proto_extTypes[1] + // optional grpc.testing.SearchRequest baz = 19; + E_Baz = &file_reflection_grpc_testing_proto2_ext_proto_extTypes[2] +) + +var File_reflection_grpc_testing_proto2_ext_proto protoreflect.FileDescriptor -var E_Baz = &proto.ExtensionDesc{ - ExtendedType: (*ToBeExtended)(nil), - ExtensionType: (*SearchRequest)(nil), - Field: 19, - Name: "grpc.testing.baz", - Tag: "bytes,19,opt,name=baz", - Filename: "reflection/grpc_testing/proto2_ext.proto", +var file_reflection_grpc_testing_proto2_ext_proto_rawDesc = []byte{ + 0x0a, 0x28, 0x72, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x67, 0x72, 0x70, + 0x63, 0x5f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x32, + 0x5f, 0x65, 0x78, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x1a, 0x24, 0x72, 0x65, 0x66, 0x6c, 0x65, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, + 0x67, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x32, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x22, + 0x72, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, + 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x22, 0x25, 0x0a, 0x09, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x12, + 0x18, 0x0a, 0x07, 0x77, 0x68, 0x61, 0x74, 0x7a, 0x69, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, + 0x52, 0x07, 0x77, 0x68, 0x61, 0x74, 0x7a, 0x69, 0x74, 0x3a, 0x2c, 0x0a, 0x03, 0x66, 0x6f, 0x6f, + 0x12, 0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, + 0x54, 0x6f, 0x42, 0x65, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x64, 0x65, 0x64, 0x18, 0x0d, 0x20, 0x01, + 0x28, 0x05, 0x52, 0x03, 0x66, 0x6f, 0x6f, 0x3a, 0x45, 0x0a, 0x03, 0x62, 0x61, 0x72, 0x12, 0x1a, + 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x54, 0x6f, + 0x42, 0x65, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x64, 0x65, 0x64, 0x18, 0x11, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x17, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, + 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x03, 0x62, 0x61, 0x72, 0x3a, 0x49, + 0x0a, 0x03, 0x62, 0x61, 0x7a, 0x12, 0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, + 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x54, 0x6f, 0x42, 0x65, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x64, 0x65, + 0x64, 0x18, 0x13, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, + 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x52, 0x03, 0x62, 0x61, 0x7a, 0x42, 0x30, 0x5a, 0x2e, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2e, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x67, + 0x72, 0x70, 0x63, 0x2f, 0x72, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x67, + 0x72, 0x70, 0x63, 0x5f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, } -func init() { - proto.RegisterType((*Extension)(nil), "grpc.testing.Extension") - proto.RegisterExtension(E_Foo) - proto.RegisterExtension(E_Bar) - proto.RegisterExtension(E_Baz) +var ( + file_reflection_grpc_testing_proto2_ext_proto_rawDescOnce sync.Once + file_reflection_grpc_testing_proto2_ext_proto_rawDescData = file_reflection_grpc_testing_proto2_ext_proto_rawDesc +) + +func file_reflection_grpc_testing_proto2_ext_proto_rawDescGZIP() []byte { + file_reflection_grpc_testing_proto2_ext_proto_rawDescOnce.Do(func() { + file_reflection_grpc_testing_proto2_ext_proto_rawDescData = protoimpl.X.CompressGZIP(file_reflection_grpc_testing_proto2_ext_proto_rawDescData) + }) + return file_reflection_grpc_testing_proto2_ext_proto_rawDescData } -func init() { - proto.RegisterFile("reflection/grpc_testing/proto2_ext.proto", fileDescriptor_071dc827b8673a0c) +var file_reflection_grpc_testing_proto2_ext_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_reflection_grpc_testing_proto2_ext_proto_goTypes = []interface{}{ + (*Extension)(nil), // 0: grpc.testing.Extension + (*ToBeExtended)(nil), // 1: grpc.testing.ToBeExtended + (*SearchRequest)(nil), // 2: grpc.testing.SearchRequest +} +var file_reflection_grpc_testing_proto2_ext_proto_depIdxs = []int32{ + 1, // 0: grpc.testing.foo:extendee -> grpc.testing.ToBeExtended + 1, // 1: grpc.testing.bar:extendee -> grpc.testing.ToBeExtended + 1, // 2: grpc.testing.baz:extendee -> grpc.testing.ToBeExtended + 0, // 3: grpc.testing.bar:type_name -> grpc.testing.Extension + 2, // 4: grpc.testing.baz:type_name -> grpc.testing.SearchRequest + 5, // [5:5] is the sub-list for method output_type + 5, // [5:5] is the sub-list for method input_type + 3, // [3:5] is the sub-list for extension type_name + 0, // [0:3] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name } -var fileDescriptor_071dc827b8673a0c = []byte{ - // 224 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x90, 0xc1, 0x4b, 0x85, 0x40, - 0x10, 0xc6, 0x91, 0x47, 0x44, 0x5b, 0x1d, 0xb2, 0x43, 0x62, 0x97, 0x87, 0x14, 0x78, 0x88, 0x35, - 0x3c, 0x7a, 0x14, 0x3c, 0x74, 0xb5, 0x4e, 0x5d, 0x64, 0xd3, 0x71, 0x5d, 0x90, 0x1d, 0x5b, 0x27, - 0x12, 0xff, 0xfa, 0xd8, 0xb5, 0xe2, 0x79, 0xd0, 0xd3, 0xee, 0x30, 0xbf, 0xdf, 0xc7, 0xf0, 0xb1, - 0xd8, 0x40, 0xdb, 0x43, 0x4d, 0x0a, 0x75, 0x22, 0xcd, 0x50, 0x57, 0x04, 0x23, 0x29, 0x2d, 0x93, - 0xc1, 0x20, 0x61, 0x5a, 0xc1, 0x44, 0xdc, 0x7d, 0xfd, 0x2b, 0xbb, 0xe6, 0xbf, 0xeb, 0xf0, 0x61, - 0xdf, 0x5b, 0x9c, 0x30, 0xda, 0xa2, 0xec, 0xbb, 0x30, 0xd1, 0x23, 0xbb, 0x28, 0x26, 0x02, 0x3d, - 0x2a, 0xd4, 0x7e, 0xc0, 0xce, 0xbf, 0x3b, 0x41, 0xb3, 0xa2, 0xc0, 0x3b, 0x7a, 0xf1, 0x59, 0xf9, - 0x37, 0x66, 0x4f, 0xec, 0xd0, 0x22, 0xfa, 0x21, 0x3f, 0x3d, 0x83, 0xbf, 0x61, 0x0e, 0xce, 0x6e, - 0xa0, 0x09, 0xae, 0x9d, 0x61, 0xb1, 0xac, 0x60, 0x87, 0x0f, 0x61, 0x76, 0xe9, 0x9b, 0xa3, 0x17, - 0x5f, 0xa6, 0x77, 0x6b, 0xe2, 0xff, 0x92, 0xd2, 0xfa, 0xd9, 0x8b, 0x8d, 0x99, 0x77, 0x63, 0x6e, - 0x5d, 0xcc, 0xfd, 0x9a, 0x78, 0x05, 0x61, 0xea, 0xae, 0x84, 0xcf, 0x2f, 0x18, 0xc9, 0x46, 0xcd, - 0xf9, 0xf3, 0x3b, 0x97, 0x88, 0xb2, 0x07, 0x2e, 0xb1, 0x17, 0x5a, 0x72, 0x34, 0xd2, 0x75, 0x92, - 0x6c, 0x74, 0xf4, 0x13, 0x00, 0x00, 0xff, 0xff, 0x9a, 0x14, 0xcf, 0xc9, 0x9b, 0x01, 0x00, 0x00, +func init() { file_reflection_grpc_testing_proto2_ext_proto_init() } +func file_reflection_grpc_testing_proto2_ext_proto_init() { + if File_reflection_grpc_testing_proto2_ext_proto != nil { + return + } + file_reflection_grpc_testing_proto2_proto_init() + file_reflection_grpc_testing_test_proto_init() + if !protoimpl.UnsafeEnabled { + file_reflection_grpc_testing_proto2_ext_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Extension); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_reflection_grpc_testing_proto2_ext_proto_rawDesc, + NumEnums: 0, + NumMessages: 1, + NumExtensions: 3, + NumServices: 0, + }, + GoTypes: file_reflection_grpc_testing_proto2_ext_proto_goTypes, + DependencyIndexes: file_reflection_grpc_testing_proto2_ext_proto_depIdxs, + MessageInfos: file_reflection_grpc_testing_proto2_ext_proto_msgTypes, + ExtensionInfos: file_reflection_grpc_testing_proto2_ext_proto_extTypes, + }.Build() + File_reflection_grpc_testing_proto2_ext_proto = out.File + file_reflection_grpc_testing_proto2_ext_proto_rawDesc = nil + file_reflection_grpc_testing_proto2_ext_proto_goTypes = nil + file_reflection_grpc_testing_proto2_ext_proto_depIdxs = nil } diff --git a/reflection/grpc_testing/proto2_ext2.pb.go b/reflection/grpc_testing/proto2_ext2.pb.go index ad2fef3addd3..3173329bb6ab 100644 --- a/reflection/grpc_testing/proto2_ext2.pb.go +++ b/reflection/grpc_testing/proto2_ext2.pb.go @@ -1,105 +1,204 @@ +// Copyright 2017 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.31.0 +// protoc v4.22.0 // source: reflection/grpc_testing/proto2_ext2.proto package grpc_testing import ( - fmt "fmt" - proto "github.com/golang/protobuf/proto" - math "math" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" ) -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) type AnotherExtension struct { - Whatchamacallit *int32 `protobuf:"varint,1,opt,name=whatchamacallit" json:"whatchamacallit,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields -func (m *AnotherExtension) Reset() { *m = AnotherExtension{} } -func (m *AnotherExtension) String() string { return proto.CompactTextString(m) } -func (*AnotherExtension) ProtoMessage() {} -func (*AnotherExtension) Descriptor() ([]byte, []int) { - return fileDescriptor_ead6f7bd8a66fb18, []int{0} + Whatchamacallit *int32 `protobuf:"varint,1,opt,name=whatchamacallit" json:"whatchamacallit,omitempty"` } -func (m *AnotherExtension) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_AnotherExtension.Unmarshal(m, b) -} -func (m *AnotherExtension) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_AnotherExtension.Marshal(b, m, deterministic) -} -func (m *AnotherExtension) XXX_Merge(src proto.Message) { - xxx_messageInfo_AnotherExtension.Merge(m, src) +func (x *AnotherExtension) Reset() { + *x = AnotherExtension{} + if protoimpl.UnsafeEnabled { + mi := &file_reflection_grpc_testing_proto2_ext2_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *AnotherExtension) XXX_Size() int { - return xxx_messageInfo_AnotherExtension.Size(m) + +func (x *AnotherExtension) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *AnotherExtension) XXX_DiscardUnknown() { - xxx_messageInfo_AnotherExtension.DiscardUnknown(m) + +func (*AnotherExtension) ProtoMessage() {} + +func (x *AnotherExtension) ProtoReflect() protoreflect.Message { + mi := &file_reflection_grpc_testing_proto2_ext2_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_AnotherExtension proto.InternalMessageInfo +// Deprecated: Use AnotherExtension.ProtoReflect.Descriptor instead. +func (*AnotherExtension) Descriptor() ([]byte, []int) { + return file_reflection_grpc_testing_proto2_ext2_proto_rawDescGZIP(), []int{0} +} -func (m *AnotherExtension) GetWhatchamacallit() int32 { - if m != nil && m.Whatchamacallit != nil { - return *m.Whatchamacallit +func (x *AnotherExtension) GetWhatchamacallit() int32 { + if x != nil && x.Whatchamacallit != nil { + return *x.Whatchamacallit } return 0 } -var E_Frob = &proto.ExtensionDesc{ - ExtendedType: (*ToBeExtended)(nil), - ExtensionType: (*string)(nil), - Field: 23, - Name: "grpc.testing.frob", - Tag: "bytes,23,opt,name=frob", - Filename: "reflection/grpc_testing/proto2_ext2.proto", +var file_reflection_grpc_testing_proto2_ext2_proto_extTypes = []protoimpl.ExtensionInfo{ + { + ExtendedType: (*ToBeExtended)(nil), + ExtensionType: (*string)(nil), + Field: 23, + Name: "grpc.testing.frob", + Tag: "bytes,23,opt,name=frob", + Filename: "reflection/grpc_testing/proto2_ext2.proto", + }, + { + ExtendedType: (*ToBeExtended)(nil), + ExtensionType: (*AnotherExtension)(nil), + Field: 29, + Name: "grpc.testing.nitz", + Tag: "bytes,29,opt,name=nitz", + Filename: "reflection/grpc_testing/proto2_ext2.proto", + }, } -var E_Nitz = &proto.ExtensionDesc{ - ExtendedType: (*ToBeExtended)(nil), - ExtensionType: (*AnotherExtension)(nil), - Field: 29, - Name: "grpc.testing.nitz", - Tag: "bytes,29,opt,name=nitz", - Filename: "reflection/grpc_testing/proto2_ext2.proto", +// Extension fields to ToBeExtended. +var ( + // optional string frob = 23; + E_Frob = &file_reflection_grpc_testing_proto2_ext2_proto_extTypes[0] + // optional grpc.testing.AnotherExtension nitz = 29; + E_Nitz = &file_reflection_grpc_testing_proto2_ext2_proto_extTypes[1] +) + +var File_reflection_grpc_testing_proto2_ext2_proto protoreflect.FileDescriptor + +var file_reflection_grpc_testing_proto2_ext2_proto_rawDesc = []byte{ + 0x0a, 0x29, 0x72, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x67, 0x72, 0x70, + 0x63, 0x5f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x32, + 0x5f, 0x65, 0x78, 0x74, 0x32, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x67, 0x72, 0x70, + 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x1a, 0x24, 0x72, 0x65, 0x66, 0x6c, 0x65, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x74, 0x65, 0x73, 0x74, 0x69, + 0x6e, 0x67, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x32, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, + 0x3c, 0x0a, 0x10, 0x41, 0x6e, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, + 0x69, 0x6f, 0x6e, 0x12, 0x28, 0x0a, 0x0f, 0x77, 0x68, 0x61, 0x74, 0x63, 0x68, 0x61, 0x6d, 0x61, + 0x63, 0x61, 0x6c, 0x6c, 0x69, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0f, 0x77, 0x68, + 0x61, 0x74, 0x63, 0x68, 0x61, 0x6d, 0x61, 0x63, 0x61, 0x6c, 0x6c, 0x69, 0x74, 0x3a, 0x2e, 0x0a, + 0x04, 0x66, 0x72, 0x6f, 0x62, 0x12, 0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, + 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x54, 0x6f, 0x42, 0x65, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x64, 0x65, + 0x64, 0x18, 0x17, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x66, 0x72, 0x6f, 0x62, 0x3a, 0x4e, 0x0a, + 0x04, 0x6e, 0x69, 0x74, 0x7a, 0x12, 0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, + 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x54, 0x6f, 0x42, 0x65, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x64, 0x65, + 0x64, 0x18, 0x1d, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, + 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x41, 0x6e, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x45, 0x78, + 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x04, 0x6e, 0x69, 0x74, 0x7a, 0x42, 0x30, 0x5a, + 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e, 0x6f, + 0x72, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x72, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, } -func init() { - proto.RegisterType((*AnotherExtension)(nil), "grpc.testing.AnotherExtension") - proto.RegisterExtension(E_Frob) - proto.RegisterExtension(E_Nitz) +var ( + file_reflection_grpc_testing_proto2_ext2_proto_rawDescOnce sync.Once + file_reflection_grpc_testing_proto2_ext2_proto_rawDescData = file_reflection_grpc_testing_proto2_ext2_proto_rawDesc +) + +func file_reflection_grpc_testing_proto2_ext2_proto_rawDescGZIP() []byte { + file_reflection_grpc_testing_proto2_ext2_proto_rawDescOnce.Do(func() { + file_reflection_grpc_testing_proto2_ext2_proto_rawDescData = protoimpl.X.CompressGZIP(file_reflection_grpc_testing_proto2_ext2_proto_rawDescData) + }) + return file_reflection_grpc_testing_proto2_ext2_proto_rawDescData } -func init() { - proto.RegisterFile("reflection/grpc_testing/proto2_ext2.proto", fileDescriptor_ead6f7bd8a66fb18) +var file_reflection_grpc_testing_proto2_ext2_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_reflection_grpc_testing_proto2_ext2_proto_goTypes = []interface{}{ + (*AnotherExtension)(nil), // 0: grpc.testing.AnotherExtension + (*ToBeExtended)(nil), // 1: grpc.testing.ToBeExtended +} +var file_reflection_grpc_testing_proto2_ext2_proto_depIdxs = []int32{ + 1, // 0: grpc.testing.frob:extendee -> grpc.testing.ToBeExtended + 1, // 1: grpc.testing.nitz:extendee -> grpc.testing.ToBeExtended + 0, // 2: grpc.testing.nitz:type_name -> grpc.testing.AnotherExtension + 3, // [3:3] is the sub-list for method output_type + 3, // [3:3] is the sub-list for method input_type + 2, // [2:3] is the sub-list for extension type_name + 0, // [0:2] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name } -var fileDescriptor_ead6f7bd8a66fb18 = []byte{ - // 208 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x8f, 0x31, 0x4b, 0xc7, 0x30, - 0x10, 0xc5, 0x09, 0xfc, 0x1d, 0x8c, 0x82, 0xd2, 0xc5, 0x52, 0x50, 0x8a, 0x38, 0xc4, 0xe5, 0x2a, - 0x1d, 0x8b, 0x8b, 0x05, 0x57, 0x87, 0xe2, 0xe4, 0x52, 0x62, 0x7a, 0x4d, 0x03, 0x31, 0x57, 0xd2, - 0x03, 0x8b, 0x9f, 0x5e, 0x8c, 0x0e, 0x5a, 0xb0, 0xdb, 0x71, 0xef, 0xf7, 0xde, 0xdd, 0x93, 0xb7, - 0x11, 0x47, 0x8f, 0x86, 0x1d, 0x85, 0xca, 0xc6, 0xd9, 0xf4, 0x8c, 0x0b, 0xbb, 0x60, 0xab, 0x39, - 0x12, 0x53, 0xdd, 0xe3, 0xca, 0x35, 0xa4, 0x39, 0x3b, 0xfd, 0xd2, 0xe1, 0x47, 0x2f, 0x6e, 0xf6, - 0x8d, 0xdf, 0x9e, 0xeb, 0x7b, 0x79, 0xfe, 0x10, 0x88, 0x27, 0x8c, 0x8f, 0x2b, 0x63, 0x58, 0x1c, - 0x85, 0x4c, 0xc9, 0xb3, 0xf7, 0x49, 0xb3, 0x99, 0xf4, 0x9b, 0x36, 0xda, 0x7b, 0xc7, 0xb9, 0x28, - 0x85, 0x3a, 0xea, 0xb6, 0xeb, 0x06, 0xe4, 0x61, 0x8c, 0xf4, 0x9a, 0x15, 0xf0, 0xfb, 0x34, 0x3c, - 0x53, 0x8b, 0x29, 0x6e, 0xc0, 0x21, 0xbf, 0x28, 0x85, 0x3a, 0xee, 0x12, 0xd7, 0x3c, 0xc9, 0x43, - 0x70, 0xfc, 0xb1, 0xcb, 0x5f, 0x96, 0x42, 0x9d, 0xd4, 0x57, 0x7f, 0x89, 0xed, 0x8f, 0x5d, 0xca, - 0x69, 0xef, 0x5e, 0xc0, 0x12, 0x59, 0x8f, 0x60, 0xc9, 0xeb, 0x60, 0x81, 0xa2, 0x4d, 0x65, 0xab, - 0x7f, 0xca, 0x7f, 0x06, 0x00, 0x00, 0xff, 0xff, 0x18, 0x74, 0xc1, 0x53, 0x4f, 0x01, 0x00, 0x00, +func init() { file_reflection_grpc_testing_proto2_ext2_proto_init() } +func file_reflection_grpc_testing_proto2_ext2_proto_init() { + if File_reflection_grpc_testing_proto2_ext2_proto != nil { + return + } + file_reflection_grpc_testing_proto2_proto_init() + if !protoimpl.UnsafeEnabled { + file_reflection_grpc_testing_proto2_ext2_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*AnotherExtension); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_reflection_grpc_testing_proto2_ext2_proto_rawDesc, + NumEnums: 0, + NumMessages: 1, + NumExtensions: 2, + NumServices: 0, + }, + GoTypes: file_reflection_grpc_testing_proto2_ext2_proto_goTypes, + DependencyIndexes: file_reflection_grpc_testing_proto2_ext2_proto_depIdxs, + MessageInfos: file_reflection_grpc_testing_proto2_ext2_proto_msgTypes, + ExtensionInfos: file_reflection_grpc_testing_proto2_ext2_proto_extTypes, + }.Build() + File_reflection_grpc_testing_proto2_ext2_proto = out.File + file_reflection_grpc_testing_proto2_ext2_proto_rawDesc = nil + file_reflection_grpc_testing_proto2_ext2_proto_goTypes = nil + file_reflection_grpc_testing_proto2_ext2_proto_depIdxs = nil } diff --git a/reflection/grpc_testing/test.pb.go b/reflection/grpc_testing/test.pb.go index 464bf39dd5c4..17a69b304d1f 100644 --- a/reflection/grpc_testing/test.pb.go +++ b/reflection/grpc_testing/test.pb.go @@ -1,185 +1,321 @@ +// Copyright 2017 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.31.0 +// protoc v4.22.0 // source: reflection/grpc_testing/test.proto package grpc_testing import ( - fmt "fmt" - proto "github.com/golang/protobuf/proto" - math "math" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" ) -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) type SearchResponse struct { - Results []*SearchResponse_Result `protobuf:"bytes,1,rep,name=results,proto3" json:"results,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields -func (m *SearchResponse) Reset() { *m = SearchResponse{} } -func (m *SearchResponse) String() string { return proto.CompactTextString(m) } -func (*SearchResponse) ProtoMessage() {} -func (*SearchResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_b179ea967ba71047, []int{0} + Results []*SearchResponse_Result `protobuf:"bytes,1,rep,name=results,proto3" json:"results,omitempty"` } -func (m *SearchResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_SearchResponse.Unmarshal(m, b) +func (x *SearchResponse) Reset() { + *x = SearchResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_reflection_grpc_testing_test_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *SearchResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_SearchResponse.Marshal(b, m, deterministic) + +func (x *SearchResponse) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *SearchResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_SearchResponse.Merge(m, src) + +func (*SearchResponse) ProtoMessage() {} + +func (x *SearchResponse) ProtoReflect() protoreflect.Message { + mi := &file_reflection_grpc_testing_test_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -func (m *SearchResponse) XXX_Size() int { - return xxx_messageInfo_SearchResponse.Size(m) + +// Deprecated: Use SearchResponse.ProtoReflect.Descriptor instead. +func (*SearchResponse) Descriptor() ([]byte, []int) { + return file_reflection_grpc_testing_test_proto_rawDescGZIP(), []int{0} } -func (m *SearchResponse) XXX_DiscardUnknown() { - xxx_messageInfo_SearchResponse.DiscardUnknown(m) + +func (x *SearchResponse) GetResults() []*SearchResponse_Result { + if x != nil { + return x.Results + } + return nil } -var xxx_messageInfo_SearchResponse proto.InternalMessageInfo +type SearchRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields -func (m *SearchResponse) GetResults() []*SearchResponse_Result { - if m != nil { - return m.Results + Query string `protobuf:"bytes,1,opt,name=query,proto3" json:"query,omitempty"` +} + +func (x *SearchRequest) Reset() { + *x = SearchRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_reflection_grpc_testing_test_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } - return nil } -type SearchResponse_Result struct { - Url string `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"` - Title string `protobuf:"bytes,2,opt,name=title,proto3" json:"title,omitempty"` - Snippets []string `protobuf:"bytes,3,rep,name=snippets,proto3" json:"snippets,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` +func (x *SearchRequest) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *SearchResponse_Result) Reset() { *m = SearchResponse_Result{} } -func (m *SearchResponse_Result) String() string { return proto.CompactTextString(m) } -func (*SearchResponse_Result) ProtoMessage() {} -func (*SearchResponse_Result) Descriptor() ([]byte, []int) { - return fileDescriptor_b179ea967ba71047, []int{0, 0} +func (*SearchRequest) ProtoMessage() {} + +func (x *SearchRequest) ProtoReflect() protoreflect.Message { + mi := &file_reflection_grpc_testing_test_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SearchRequest.ProtoReflect.Descriptor instead. +func (*SearchRequest) Descriptor() ([]byte, []int) { + return file_reflection_grpc_testing_test_proto_rawDescGZIP(), []int{1} } -func (m *SearchResponse_Result) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_SearchResponse_Result.Unmarshal(m, b) +func (x *SearchRequest) GetQuery() string { + if x != nil { + return x.Query + } + return "" } -func (m *SearchResponse_Result) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_SearchResponse_Result.Marshal(b, m, deterministic) + +type SearchResponse_Result struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Url string `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"` + Title string `protobuf:"bytes,2,opt,name=title,proto3" json:"title,omitempty"` + Snippets []string `protobuf:"bytes,3,rep,name=snippets,proto3" json:"snippets,omitempty"` } -func (m *SearchResponse_Result) XXX_Merge(src proto.Message) { - xxx_messageInfo_SearchResponse_Result.Merge(m, src) + +func (x *SearchResponse_Result) Reset() { + *x = SearchResponse_Result{} + if protoimpl.UnsafeEnabled { + mi := &file_reflection_grpc_testing_test_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *SearchResponse_Result) XXX_Size() int { - return xxx_messageInfo_SearchResponse_Result.Size(m) + +func (x *SearchResponse_Result) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *SearchResponse_Result) XXX_DiscardUnknown() { - xxx_messageInfo_SearchResponse_Result.DiscardUnknown(m) + +func (*SearchResponse_Result) ProtoMessage() {} + +func (x *SearchResponse_Result) ProtoReflect() protoreflect.Message { + mi := &file_reflection_grpc_testing_test_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_SearchResponse_Result proto.InternalMessageInfo +// Deprecated: Use SearchResponse_Result.ProtoReflect.Descriptor instead. +func (*SearchResponse_Result) Descriptor() ([]byte, []int) { + return file_reflection_grpc_testing_test_proto_rawDescGZIP(), []int{0, 0} +} -func (m *SearchResponse_Result) GetUrl() string { - if m != nil { - return m.Url +func (x *SearchResponse_Result) GetUrl() string { + if x != nil { + return x.Url } return "" } -func (m *SearchResponse_Result) GetTitle() string { - if m != nil { - return m.Title +func (x *SearchResponse_Result) GetTitle() string { + if x != nil { + return x.Title } return "" } -func (m *SearchResponse_Result) GetSnippets() []string { - if m != nil { - return m.Snippets +func (x *SearchResponse_Result) GetSnippets() []string { + if x != nil { + return x.Snippets } return nil } -type SearchRequest struct { - Query string `protobuf:"bytes,1,opt,name=query,proto3" json:"query,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} +var File_reflection_grpc_testing_test_proto protoreflect.FileDescriptor -func (m *SearchRequest) Reset() { *m = SearchRequest{} } -func (m *SearchRequest) String() string { return proto.CompactTextString(m) } -func (*SearchRequest) ProtoMessage() {} -func (*SearchRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_b179ea967ba71047, []int{1} +var file_reflection_grpc_testing_test_proto_rawDesc = []byte{ + 0x0a, 0x22, 0x72, 0x65, 0x66, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x67, 0x72, 0x70, + 0x63, 0x5f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, + 0x6e, 0x67, 0x22, 0x9d, 0x01, 0x0a, 0x0e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3d, 0x0a, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, + 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, + 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x07, 0x72, 0x65, 0x73, + 0x75, 0x6c, 0x74, 0x73, 0x1a, 0x4c, 0x0a, 0x06, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x10, + 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, + 0x12, 0x14, 0x0a, 0x05, 0x74, 0x69, 0x74, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x05, 0x74, 0x69, 0x74, 0x6c, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x73, 0x6e, 0x69, 0x70, 0x70, 0x65, + 0x74, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x73, 0x6e, 0x69, 0x70, 0x70, 0x65, + 0x74, 0x73, 0x22, 0x25, 0x0a, 0x0d, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x32, 0xa6, 0x01, 0x0a, 0x0d, 0x53, 0x65, + 0x61, 0x72, 0x63, 0x68, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x43, 0x0a, 0x06, 0x53, + 0x65, 0x61, 0x72, 0x63, 0x68, 0x12, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, + 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, + 0x67, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x50, 0x0a, 0x0f, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x53, 0x65, 0x61, + 0x72, 0x63, 0x68, 0x12, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, + 0x6e, 0x67, 0x2e, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x1c, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, + 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, + 0x30, 0x01, 0x42, 0x30, 0x5a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67, 0x6f, 0x6c, + 0x61, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x72, 0x65, 0x66, + 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x74, 0x65, 0x73, + 0x74, 0x69, 0x6e, 0x67, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } -func (m *SearchRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_SearchRequest.Unmarshal(m, b) -} -func (m *SearchRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_SearchRequest.Marshal(b, m, deterministic) -} -func (m *SearchRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_SearchRequest.Merge(m, src) +var ( + file_reflection_grpc_testing_test_proto_rawDescOnce sync.Once + file_reflection_grpc_testing_test_proto_rawDescData = file_reflection_grpc_testing_test_proto_rawDesc +) + +func file_reflection_grpc_testing_test_proto_rawDescGZIP() []byte { + file_reflection_grpc_testing_test_proto_rawDescOnce.Do(func() { + file_reflection_grpc_testing_test_proto_rawDescData = protoimpl.X.CompressGZIP(file_reflection_grpc_testing_test_proto_rawDescData) + }) + return file_reflection_grpc_testing_test_proto_rawDescData } -func (m *SearchRequest) XXX_Size() int { - return xxx_messageInfo_SearchRequest.Size(m) + +var file_reflection_grpc_testing_test_proto_msgTypes = make([]protoimpl.MessageInfo, 3) +var file_reflection_grpc_testing_test_proto_goTypes = []interface{}{ + (*SearchResponse)(nil), // 0: grpc.testing.SearchResponse + (*SearchRequest)(nil), // 1: grpc.testing.SearchRequest + (*SearchResponse_Result)(nil), // 2: grpc.testing.SearchResponse.Result } -func (m *SearchRequest) XXX_DiscardUnknown() { - xxx_messageInfo_SearchRequest.DiscardUnknown(m) +var file_reflection_grpc_testing_test_proto_depIdxs = []int32{ + 2, // 0: grpc.testing.SearchResponse.results:type_name -> grpc.testing.SearchResponse.Result + 1, // 1: grpc.testing.SearchService.Search:input_type -> grpc.testing.SearchRequest + 1, // 2: grpc.testing.SearchService.StreamingSearch:input_type -> grpc.testing.SearchRequest + 0, // 3: grpc.testing.SearchService.Search:output_type -> grpc.testing.SearchResponse + 0, // 4: grpc.testing.SearchService.StreamingSearch:output_type -> grpc.testing.SearchResponse + 3, // [3:5] is the sub-list for method output_type + 1, // [1:3] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name } -var xxx_messageInfo_SearchRequest proto.InternalMessageInfo - -func (m *SearchRequest) GetQuery() string { - if m != nil { - return m.Query +func init() { file_reflection_grpc_testing_test_proto_init() } +func file_reflection_grpc_testing_test_proto_init() { + if File_reflection_grpc_testing_test_proto != nil { + return } - return "" -} - -func init() { - proto.RegisterType((*SearchResponse)(nil), "grpc.testing.SearchResponse") - proto.RegisterType((*SearchResponse_Result)(nil), "grpc.testing.SearchResponse.Result") - proto.RegisterType((*SearchRequest)(nil), "grpc.testing.SearchRequest") -} - -func init() { - proto.RegisterFile("reflection/grpc_testing/test.proto", fileDescriptor_b179ea967ba71047) -} - -var fileDescriptor_b179ea967ba71047 = []byte{ - // 267 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x51, 0x3f, 0x4b, 0xfd, 0x30, - 0x14, 0x25, 0xbf, 0xf2, 0xab, 0xbe, 0xeb, 0x5f, 0x82, 0x43, 0xa9, 0x0e, 0xa5, 0x22, 0x74, 0x4a, - 0x1f, 0xcf, 0xd9, 0x45, 0x57, 0x07, 0x69, 0x37, 0x17, 0xa9, 0xe5, 0x1a, 0x03, 0x31, 0xe9, 0xbb, - 0xb9, 0x15, 0xfc, 0x30, 0xae, 0x7e, 0x4e, 0x69, 0xfb, 0x2a, 0x0a, 0xea, 0xe2, 0x94, 0x9c, 0x73, - 0xcf, 0x39, 0xc9, 0xe5, 0x40, 0x4e, 0xf8, 0x60, 0xb1, 0x65, 0xe3, 0x5d, 0xa9, 0xa9, 0x6b, 0xef, - 0x18, 0x03, 0x1b, 0xa7, 0xcb, 0xe1, 0x54, 0x1d, 0x79, 0xf6, 0x72, 0x77, 0x18, 0xa8, 0xcd, 0x20, - 0x7f, 0x15, 0xb0, 0x5f, 0x63, 0x43, 0xed, 0x63, 0x85, 0xa1, 0xf3, 0x2e, 0xa0, 0xbc, 0x80, 0x2d, - 0xc2, 0xd0, 0x5b, 0x0e, 0x89, 0xc8, 0xa2, 0x62, 0x67, 0x75, 0xaa, 0x3e, 0x5b, 0xd4, 0x57, 0xb9, - 0xaa, 0x46, 0x6d, 0x35, 0x7b, 0xd2, 0x6b, 0x88, 0x27, 0x4a, 0x1e, 0x42, 0xd4, 0x93, 0x4d, 0x44, - 0x26, 0x8a, 0x45, 0x35, 0x5c, 0xe5, 0x11, 0xfc, 0x67, 0xc3, 0x16, 0x93, 0x7f, 0x23, 0x37, 0x01, - 0x99, 0xc2, 0x76, 0x70, 0xa6, 0xeb, 0x90, 0x43, 0x12, 0x65, 0x51, 0xb1, 0xa8, 0x3e, 0x70, 0x7e, - 0x06, 0x7b, 0xf3, 0x7b, 0xeb, 0x1e, 0x03, 0x0f, 0x11, 0xeb, 0x1e, 0xe9, 0x65, 0x13, 0x3b, 0x81, - 0xd5, 0x9b, 0x98, 0x75, 0x35, 0xd2, 0xb3, 0x69, 0x51, 0x5e, 0x41, 0x3c, 0x11, 0xf2, 0xf8, 0xfb, - 0xef, 0x8f, 0x71, 0xe9, 0xc9, 0x6f, 0xbb, 0xc9, 0x1b, 0x38, 0xa8, 0x99, 0xb0, 0x79, 0x32, 0x4e, - 0xff, 0x39, 0xad, 0x10, 0x4b, 0x71, 0xb9, 0xbc, 0x55, 0xda, 0x7b, 0x6d, 0x51, 0x69, 0x6f, 0x1b, - 0xa7, 0x95, 0x27, 0x3d, 0x56, 0x55, 0xfe, 0x50, 0xdd, 0x7d, 0x3c, 0xd6, 0x76, 0xfe, 0x1e, 0x00, - 0x00, 0xff, 0xff, 0x38, 0x42, 0x3b, 0xd2, 0xdc, 0x01, 0x00, 0x00, + if !protoimpl.UnsafeEnabled { + file_reflection_grpc_testing_test_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SearchResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_reflection_grpc_testing_test_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SearchRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_reflection_grpc_testing_test_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SearchResponse_Result); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_reflection_grpc_testing_test_proto_rawDesc, + NumEnums: 0, + NumMessages: 3, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_reflection_grpc_testing_test_proto_goTypes, + DependencyIndexes: file_reflection_grpc_testing_test_proto_depIdxs, + MessageInfos: file_reflection_grpc_testing_test_proto_msgTypes, + }.Build() + File_reflection_grpc_testing_test_proto = out.File + file_reflection_grpc_testing_test_proto_rawDesc = nil + file_reflection_grpc_testing_test_proto_goTypes = nil + file_reflection_grpc_testing_test_proto_depIdxs = nil } diff --git a/reflection/grpc_testing/test_grpc.pb.go b/reflection/grpc_testing/test_grpc.pb.go index 285cdbc85ace..d130f6f3dae3 100644 --- a/reflection/grpc_testing/test_grpc.pb.go +++ b/reflection/grpc_testing/test_grpc.pb.go @@ -1,4 +1,22 @@ +// Copyright 2017 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.3.0 +// - protoc v4.22.0 +// source: reflection/grpc_testing/test.proto package grpc_testing @@ -11,8 +29,14 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. const _ = grpc.SupportPackageIsVersion7 +const ( + SearchService_Search_FullMethodName = "/grpc.testing.SearchService/Search" + SearchService_StreamingSearch_FullMethodName = "/grpc.testing.SearchService/StreamingSearch" +) + // SearchServiceClient is the client API for SearchService service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. @@ -29,27 +53,17 @@ func NewSearchServiceClient(cc grpc.ClientConnInterface) SearchServiceClient { return &searchServiceClient{cc} } -var searchServiceSearchStreamDesc = &grpc.StreamDesc{ - StreamName: "Search", -} - func (c *searchServiceClient) Search(ctx context.Context, in *SearchRequest, opts ...grpc.CallOption) (*SearchResponse, error) { out := new(SearchResponse) - err := c.cc.Invoke(ctx, "/grpc.testing.SearchService/Search", in, out, opts...) + err := c.cc.Invoke(ctx, SearchService_Search_FullMethodName, in, out, opts...) if err != nil { return nil, err } return out, nil } -var searchServiceStreamingSearchStreamDesc = &grpc.StreamDesc{ - StreamName: "StreamingSearch", - ServerStreams: true, - ClientStreams: true, -} - func (c *searchServiceClient) StreamingSearch(ctx context.Context, opts ...grpc.CallOption) (SearchService_StreamingSearchClient, error) { - stream, err := c.cc.NewStream(ctx, searchServiceStreamingSearchStreamDesc, "/grpc.testing.SearchService/StreamingSearch", opts...) + stream, err := c.cc.NewStream(ctx, &SearchService_ServiceDesc.Streams[0], SearchService_StreamingSearch_FullMethodName, opts...) if err != nil { return nil, err } @@ -79,40 +93,58 @@ func (x *searchServiceStreamingSearchClient) Recv() (*SearchResponse, error) { return m, nil } -// SearchServiceService is the service API for SearchService service. -// Fields should be assigned to their respective handler implementations only before -// RegisterSearchServiceService is called. Any unassigned fields will result in the -// handler for that method returning an Unimplemented error. -type SearchServiceService struct { - Search func(context.Context, *SearchRequest) (*SearchResponse, error) - StreamingSearch func(SearchService_StreamingSearchServer) error +// SearchServiceServer is the server API for SearchService service. +// All implementations must embed UnimplementedSearchServiceServer +// for forward compatibility +type SearchServiceServer interface { + Search(context.Context, *SearchRequest) (*SearchResponse, error) + StreamingSearch(SearchService_StreamingSearchServer) error + mustEmbedUnimplementedSearchServiceServer() +} + +// UnimplementedSearchServiceServer must be embedded to have forward compatible implementations. +type UnimplementedSearchServiceServer struct { } -func (s *SearchServiceService) search(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - if s.Search == nil { - return nil, status.Errorf(codes.Unimplemented, "method Search not implemented") - } +func (UnimplementedSearchServiceServer) Search(context.Context, *SearchRequest) (*SearchResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method Search not implemented") +} +func (UnimplementedSearchServiceServer) StreamingSearch(SearchService_StreamingSearchServer) error { + return status.Errorf(codes.Unimplemented, "method StreamingSearch not implemented") +} +func (UnimplementedSearchServiceServer) mustEmbedUnimplementedSearchServiceServer() {} + +// UnsafeSearchServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to SearchServiceServer will +// result in compilation errors. +type UnsafeSearchServiceServer interface { + mustEmbedUnimplementedSearchServiceServer() +} + +func RegisterSearchServiceServer(s grpc.ServiceRegistrar, srv SearchServiceServer) { + s.RegisterService(&SearchService_ServiceDesc, srv) +} + +func _SearchService_Search_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(SearchRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { - return s.Search(ctx, in) + return srv.(SearchServiceServer).Search(ctx, in) } info := &grpc.UnaryServerInfo{ - Server: s, - FullMethod: "/grpc.testing.SearchService/Search", + Server: srv, + FullMethod: SearchService_Search_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return s.Search(ctx, req.(*SearchRequest)) + return srv.(SearchServiceServer).Search(ctx, req.(*SearchRequest)) } return interceptor(ctx, in, info, handler) } -func (s *SearchServiceService) streamingSearch(_ interface{}, stream grpc.ServerStream) error { - if s.StreamingSearch == nil { - return status.Errorf(codes.Unimplemented, "method StreamingSearch not implemented") - } - return s.StreamingSearch(&searchServiceStreamingSearchServer{stream}) + +func _SearchService_StreamingSearch_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(SearchServiceServer).StreamingSearch(&searchServiceStreamingSearchServer{stream}) } type SearchService_StreamingSearchServer interface { @@ -137,56 +169,25 @@ func (x *searchServiceStreamingSearchServer) Recv() (*SearchRequest, error) { return m, nil } -// RegisterSearchServiceService registers a service implementation with a gRPC server. -func RegisterSearchServiceService(s grpc.ServiceRegistrar, srv *SearchServiceService) { - sd := grpc.ServiceDesc{ - ServiceName: "grpc.testing.SearchService", - Methods: []grpc.MethodDesc{ - { - MethodName: "Search", - Handler: srv.search, - }, +// SearchService_ServiceDesc is the grpc.ServiceDesc for SearchService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var SearchService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.testing.SearchService", + HandlerType: (*SearchServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Search", + Handler: _SearchService_Search_Handler, }, - Streams: []grpc.StreamDesc{ - { - StreamName: "StreamingSearch", - Handler: srv.streamingSearch, - ServerStreams: true, - ClientStreams: true, - }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "StreamingSearch", + Handler: _SearchService_StreamingSearch_Handler, + ServerStreams: true, + ClientStreams: true, }, - Metadata: "reflection/grpc_testing/test.proto", - } - - s.RegisterService(&sd, nil) -} - -// NewSearchServiceService creates a new SearchServiceService containing the -// implemented methods of the SearchService service in s. Any unimplemented -// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client. -// This includes situations where the method handler is misspelled or has the wrong -// signature. For this reason, this function should be used with great care and -// is not recommended to be used by most users. -func NewSearchServiceService(s interface{}) *SearchServiceService { - ns := &SearchServiceService{} - if h, ok := s.(interface { - Search(context.Context, *SearchRequest) (*SearchResponse, error) - }); ok { - ns.Search = h.Search - } - if h, ok := s.(interface { - StreamingSearch(SearchService_StreamingSearchServer) error - }); ok { - ns.StreamingSearch = h.StreamingSearch - } - return ns -} - -// UnstableSearchServiceService is the service API for SearchService service. -// New methods may be added to this interface if they are added to the service -// definition, which is not a backward-compatible change. For this reason, -// use of this type is not recommended. -type UnstableSearchServiceService interface { - Search(context.Context, *SearchRequest) (*SearchResponse, error) - StreamingSearch(SearchService_StreamingSearchServer) error + }, + Metadata: "reflection/grpc_testing/test.proto", } diff --git a/reflection/grpc_testing_not_regenerate/README.md b/reflection/grpc_testing_not_regenerate/README.md new file mode 100644 index 000000000000..7f29ff61a537 --- /dev/null +++ b/reflection/grpc_testing_not_regenerate/README.md @@ -0,0 +1,5 @@ +testv3.pb.go is generated with an older version of codegen, to test reflection behavior with `grpc.SupportPackageIsVersion3`. DO NOT REGENERATE! + +testv3.pb.go is manually edited to replace `"golang.org/x/net/context"` with `"context"`. + +dynamic.pb.go is generated with the latest protoc and manually edited to remove everything except the descriptor bytes var, which is renamed and exported. \ No newline at end of file diff --git a/reflection/grpc_testing_not_regenerate/dynamic.pb.go b/reflection/grpc_testing_not_regenerate/dynamic.pb.go new file mode 100644 index 000000000000..35e4f02478b2 --- /dev/null +++ b/reflection/grpc_testing_not_regenerate/dynamic.pb.go @@ -0,0 +1,35 @@ +/* + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc_testing_not_regenerate + +// FileDynamicProtoRawDesc is the descriptor for dynamic.proto, see README.md. +var FileDynamicProtoRawDesc = []byte{ + 0x0a, 0x0d, 0x64, 0x79, 0x6e, 0x61, 0x6d, 0x69, 0x63, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, + 0x0c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x22, 0x0c, 0x0a, + 0x0a, 0x44, 0x79, 0x6e, 0x61, 0x6d, 0x69, 0x63, 0x52, 0x65, 0x73, 0x22, 0x0c, 0x0a, 0x0a, 0x44, + 0x79, 0x6e, 0x61, 0x6d, 0x69, 0x63, 0x52, 0x65, 0x71, 0x32, 0x57, 0x0a, 0x0e, 0x44, 0x79, 0x6e, + 0x61, 0x6d, 0x69, 0x63, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x45, 0x0a, 0x0f, 0x44, + 0x79, 0x6e, 0x61, 0x6d, 0x69, 0x63, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x31, 0x12, 0x18, + 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x44, 0x79, + 0x6e, 0x61, 0x6d, 0x69, 0x63, 0x52, 0x65, 0x71, 0x1a, 0x18, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, + 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x44, 0x79, 0x6e, 0x61, 0x6d, 0x69, 0x63, 0x52, + 0x65, 0x73, 0x42, 0x2c, 0x5a, 0x2a, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67, 0x6f, 0x6c, + 0x61, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x72, 0x65, 0x66, + 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x64, 0x61, 0x74, 0x61, + 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} diff --git a/reflection/grpc_testing_not_regenerate/dynamic.proto b/reflection/grpc_testing_not_regenerate/dynamic.proto new file mode 100644 index 000000000000..5eeba0892336 --- /dev/null +++ b/reflection/grpc_testing_not_regenerate/dynamic.proto @@ -0,0 +1,33 @@ +/* + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +syntax = "proto3"; + +option go_package = "google.golang.org/grpc/reflection/grpc_testing_not_regenerate"; + +package grpc.testing; + +message DynamicRes {} + +message DynamicReq {} + +// DynamicService is used to test reflection on dynamically constructed protocol +// buffer messages. +service DynamicService { + // DynamicMessage1 is a test RPC for dynamically constructed protobufs. + rpc DynamicMessage1(DynamicReq) returns (DynamicRes); +} \ No newline at end of file diff --git a/reflection/grpc_testingv3/testv3.pb.go b/reflection/grpc_testing_not_regenerate/testv3.pb.go similarity index 95% rename from reflection/grpc_testingv3/testv3.pb.go rename to reflection/grpc_testing_not_regenerate/testv3.pb.go index d7a69e546ead..8a690963ec10 100644 --- a/reflection/grpc_testingv3/testv3.pb.go +++ b/reflection/grpc_testing_not_regenerate/testv3.pb.go @@ -1,26 +1,44 @@ +/* + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + // Code generated by protoc-gen-go. // source: testv3.proto // DO NOT EDIT! /* -Package grpc_testingv3 is a generated protocol buffer package. +Package grpc_testing_not_regenerate is a generated protocol buffer package. It is generated from these files: + testv3.proto It has these top-level messages: + SearchResponseV3 SearchRequestV3 */ -package grpc_testingv3 - -import proto "github.com/golang/protobuf/proto" -import fmt "fmt" -import math "math" +package grpc_testing_not_regenerate import ( context "context" + fmt "fmt" + math "math" + proto "github.com/golang/protobuf/proto" grpc "google.golang.org/grpc" ) diff --git a/reflection/grpc_testing_not_regenerate/testv3.proto b/reflection/grpc_testing_not_regenerate/testv3.proto new file mode 100644 index 000000000000..44f93ba8b076 --- /dev/null +++ b/reflection/grpc_testing_not_regenerate/testv3.proto @@ -0,0 +1,58 @@ +/* + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +syntax = "proto3"; + +option go_package = "google.golang.org/grpc/reflection/grpc_testing_not_regenerate"; + +package grpc.testingv3; + +message SearchResponseV3 { + message Result { + string url = 1; + string title = 2; + repeated string snippets = 3; + message Value { + oneof val { + string str = 1; + int64 int = 2; + double real = 3; + } + } + map metadata = 4; + } + enum State { + UNKNOWN = 0; + FRESH = 1; + STALE = 2; + } + repeated Result results = 1; + State state = 2; +} + +message SearchRequestV3 { + string query = 1; +} + +// SearchServiceV3 is used to test grpc server reflection. +service SearchServiceV3 { + // Search is a unary RPC. + rpc Search(SearchRequestV3) returns (SearchResponseV3); + + // StreamingSearch is a streaming RPC. + rpc StreamingSearch(stream SearchRequestV3) returns (stream SearchResponseV3); +} diff --git a/reflection/grpc_testingv3/README.md b/reflection/grpc_testingv3/README.md deleted file mode 100644 index 83d58756a440..000000000000 --- a/reflection/grpc_testingv3/README.md +++ /dev/null @@ -1,3 +0,0 @@ -The pb.go is genenated with an older version of codegen, to test reflection behavior with `grpc.SupportPackageIsVersion3`. DO NOT REGENERATE! - -pb.go is manually edited to replace `"golang.org/x/net/context"` with `"context"`. diff --git a/reflection/grpc_testingv3/testv3.proto b/reflection/grpc_testingv3/testv3.proto deleted file mode 100644 index 38a615a90d91..000000000000 --- a/reflection/grpc_testingv3/testv3.proto +++ /dev/null @@ -1,37 +0,0 @@ -syntax = "proto3"; - -option go_package = "google.golang.org/grpc/reflection/grpc_testingv3"; - -package grpc.testingv3; - -message SearchResponseV3 { - message Result { - string url = 1; - string title = 2; - repeated string snippets = 3; - message Value { - oneof val { - string str = 1; - int64 int = 2; - double real = 3; - } - } - map metadata = 4; - } - enum State { - UNKNOWN = 0; - FRESH = 1; - STALE = 2; - } - repeated Result results = 1; - State state = 2; -} - -message SearchRequestV3 { - string query = 1; -} - -service SearchServiceV3 { - rpc Search(SearchRequestV3) returns (SearchResponseV3); - rpc StreamingSearch(stream SearchRequestV3) returns (stream SearchResponseV3); -} diff --git a/reflection/serverreflection.go b/reflection/serverreflection.go index 7b6dd414a275..76dae09d8886 100644 --- a/reflection/serverreflection.go +++ b/reflection/serverreflection.go @@ -23,6 +23,7 @@ The service implemented is defined in: https://github.com/grpc/grpc/blob/master/src/proto/grpc/reflection/v1alpha/reflection.proto. To register server reflection on a gRPC server: + import "google.golang.org/grpc/reflection" s := grpc.NewServer() @@ -32,328 +33,228 @@ To register server reflection on a gRPC server: reflection.Register(s) s.Serve(lis) - */ package reflection // import "google.golang.org/grpc/reflection" import ( - "bytes" - "compress/gzip" - "fmt" "io" - "io/ioutil" - "reflect" "sort" - "sync" - "github.com/golang/protobuf/proto" - dpb "github.com/golang/protobuf/protoc-gen-go/descriptor" "google.golang.org/grpc" "google.golang.org/grpc/codes" - rpb "google.golang.org/grpc/reflection/grpc_reflection_v1alpha" "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/reflect/protodesc" + "google.golang.org/protobuf/reflect/protoreflect" + "google.golang.org/protobuf/reflect/protoregistry" + + v1reflectiongrpc "google.golang.org/grpc/reflection/grpc_reflection_v1" + v1reflectionpb "google.golang.org/grpc/reflection/grpc_reflection_v1" + v1alphareflectiongrpc "google.golang.org/grpc/reflection/grpc_reflection_v1alpha" ) -type serverReflectionServer struct { - rpb.UnimplementedServerReflectionServer - s *grpc.Server - - initSymbols sync.Once - serviceNames []string - symbols map[string]*dpb.FileDescriptorProto // map of fully-qualified names to files -} - -// Register registers the server reflection service on the given gRPC server. -func Register(s *grpc.Server) { - rpb.RegisterServerReflectionServer(s, &serverReflectionServer{ - s: s, - }) -} - -// protoMessage is used for type assertion on proto messages. -// Generated proto message implements function Descriptor(), but Descriptor() -// is not part of interface proto.Message. This interface is needed to -// call Descriptor(). -type protoMessage interface { - Descriptor() ([]byte, []int) -} - -func (s *serverReflectionServer) getSymbols() (svcNames []string, symbolIndex map[string]*dpb.FileDescriptorProto) { - s.initSymbols.Do(func() { - serviceInfo := s.s.GetServiceInfo() - - s.symbols = map[string]*dpb.FileDescriptorProto{} - s.serviceNames = make([]string, 0, len(serviceInfo)) - processed := map[string]struct{}{} - for svc, info := range serviceInfo { - s.serviceNames = append(s.serviceNames, svc) - fdenc, ok := parseMetadata(info.Metadata) - if !ok { - continue - } - fd, err := decodeFileDesc(fdenc) - if err != nil { - continue - } - s.processFile(fd, processed) - } - sort.Strings(s.serviceNames) - }) - - return s.serviceNames, s.symbols -} - -func (s *serverReflectionServer) processFile(fd *dpb.FileDescriptorProto, processed map[string]struct{}) { - filename := fd.GetName() - if _, ok := processed[filename]; ok { - return - } - processed[filename] = struct{}{} - - prefix := fd.GetPackage() - - for _, msg := range fd.MessageType { - s.processMessage(fd, prefix, msg) - } - for _, en := range fd.EnumType { - s.processEnum(fd, prefix, en) - } - for _, ext := range fd.Extension { - s.processField(fd, prefix, ext) - } - for _, svc := range fd.Service { - svcName := fqn(prefix, svc.GetName()) - s.symbols[svcName] = fd - for _, meth := range svc.Method { - name := fqn(svcName, meth.GetName()) - s.symbols[name] = fd - } - } - - for _, dep := range fd.Dependency { - fdenc := proto.FileDescriptor(dep) - fdDep, err := decodeFileDesc(fdenc) - if err != nil { - continue - } - s.processFile(fdDep, processed) - } +// GRPCServer is the interface provided by a gRPC server. It is implemented by +// *grpc.Server, but could also be implemented by other concrete types. It acts +// as a registry, for accumulating the services exposed by the server. +type GRPCServer interface { + grpc.ServiceRegistrar + ServiceInfoProvider } -func (s *serverReflectionServer) processMessage(fd *dpb.FileDescriptorProto, prefix string, msg *dpb.DescriptorProto) { - msgName := fqn(prefix, msg.GetName()) - s.symbols[msgName] = fd +var _ GRPCServer = (*grpc.Server)(nil) - for _, nested := range msg.NestedType { - s.processMessage(fd, msgName, nested) - } - for _, en := range msg.EnumType { - s.processEnum(fd, msgName, en) - } - for _, ext := range msg.Extension { - s.processField(fd, msgName, ext) - } - for _, fld := range msg.Field { - s.processField(fd, msgName, fld) - } - for _, oneof := range msg.OneofDecl { - oneofName := fqn(msgName, oneof.GetName()) - s.symbols[oneofName] = fd - } -} - -func (s *serverReflectionServer) processEnum(fd *dpb.FileDescriptorProto, prefix string, en *dpb.EnumDescriptorProto) { - enName := fqn(prefix, en.GetName()) - s.symbols[enName] = fd - - for _, val := range en.Value { - valName := fqn(enName, val.GetName()) - s.symbols[valName] = fd - } -} - -func (s *serverReflectionServer) processField(fd *dpb.FileDescriptorProto, prefix string, fld *dpb.FieldDescriptorProto) { - fldName := fqn(prefix, fld.GetName()) - s.symbols[fldName] = fd -} - -func fqn(prefix, name string) string { - if prefix == "" { - return name - } - return prefix + "." + name -} - -// fileDescForType gets the file descriptor for the given type. -// The given type should be a proto message. -func (s *serverReflectionServer) fileDescForType(st reflect.Type) (*dpb.FileDescriptorProto, error) { - m, ok := reflect.Zero(reflect.PtrTo(st)).Interface().(protoMessage) - if !ok { - return nil, fmt.Errorf("failed to create message from type: %v", st) - } - enc, _ := m.Descriptor() - - return decodeFileDesc(enc) +// Register registers the server reflection service on the given gRPC server. +// Both the v1 and v1alpha versions are registered. +func Register(s GRPCServer) { + svr := NewServerV1(ServerOptions{Services: s}) + v1alphareflectiongrpc.RegisterServerReflectionServer(s, asV1Alpha(svr)) + v1reflectiongrpc.RegisterServerReflectionServer(s, svr) } -// decodeFileDesc does decompression and unmarshalling on the given -// file descriptor byte slice. -func decodeFileDesc(enc []byte) (*dpb.FileDescriptorProto, error) { - raw, err := decompress(enc) - if err != nil { - return nil, fmt.Errorf("failed to decompress enc: %v", err) - } - - fd := new(dpb.FileDescriptorProto) - if err := proto.Unmarshal(raw, fd); err != nil { - return nil, fmt.Errorf("bad descriptor: %v", err) - } - return fd, nil +// RegisterV1 registers only the v1 version of the server reflection service +// on the given gRPC server. Many clients may only support v1alpha so most +// users should use Register instead, at least until clients have upgraded. +func RegisterV1(s GRPCServer) { + svr := NewServerV1(ServerOptions{Services: s}) + v1reflectiongrpc.RegisterServerReflectionServer(s, svr) } -// decompress does gzip decompression. -func decompress(b []byte) ([]byte, error) { - r, err := gzip.NewReader(bytes.NewReader(b)) - if err != nil { - return nil, fmt.Errorf("bad gzipped descriptor: %v", err) - } - out, err := ioutil.ReadAll(r) - if err != nil { - return nil, fmt.Errorf("bad gzipped descriptor: %v", err) - } - return out, nil +// ServiceInfoProvider is an interface used to retrieve metadata about the +// services to expose. +// +// The reflection service is only interested in the service names, but the +// signature is this way so that *grpc.Server implements it. So it is okay +// for a custom implementation to return zero values for the +// grpc.ServiceInfo values in the map. +// +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. +type ServiceInfoProvider interface { + GetServiceInfo() map[string]grpc.ServiceInfo } -func typeForName(name string) (reflect.Type, error) { - pt := proto.MessageType(name) - if pt == nil { - return nil, fmt.Errorf("unknown type: %q", name) - } - st := pt.Elem() - - return st, nil +// ExtensionResolver is the interface used to query details about extensions. +// This interface is satisfied by protoregistry.GlobalTypes. +// +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. +type ExtensionResolver interface { + protoregistry.ExtensionTypeResolver + RangeExtensionsByMessage(message protoreflect.FullName, f func(protoreflect.ExtensionType) bool) } -func fileDescContainingExtension(st reflect.Type, ext int32) (*dpb.FileDescriptorProto, error) { - m, ok := reflect.Zero(reflect.PtrTo(st)).Interface().(proto.Message) - if !ok { - return nil, fmt.Errorf("failed to create message from type: %v", st) - } - - var extDesc *proto.ExtensionDesc - for id, desc := range proto.RegisteredExtensions(m) { - if id == ext { - extDesc = desc - break - } - } - - if extDesc == nil { - return nil, fmt.Errorf("failed to find registered extension for extension number %v", ext) - } - - return decodeFileDesc(proto.FileDescriptor(extDesc.Filename)) +// ServerOptions represents the options used to construct a reflection server. +// +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. +type ServerOptions struct { + // The source of advertised RPC services. If not specified, the reflection + // server will report an empty list when asked to list services. + // + // This value will typically be a *grpc.Server. But the set of advertised + // services can be customized by wrapping a *grpc.Server or using an + // alternate implementation that returns a custom set of service names. + Services ServiceInfoProvider + // Optional resolver used to load descriptors. If not specified, + // protoregistry.GlobalFiles will be used. + DescriptorResolver protodesc.Resolver + // Optional resolver used to query for known extensions. If not specified, + // protoregistry.GlobalTypes will be used. + ExtensionResolver ExtensionResolver } -func (s *serverReflectionServer) allExtensionNumbersForType(st reflect.Type) ([]int32, error) { - m, ok := reflect.Zero(reflect.PtrTo(st)).Interface().(proto.Message) - if !ok { - return nil, fmt.Errorf("failed to create message from type: %v", st) - } - - exts := proto.RegisteredExtensions(m) - out := make([]int32, 0, len(exts)) - for id := range exts { - out = append(out, id) - } - return out, nil +// NewServer returns a reflection server implementation using the given options. +// This can be used to customize behavior of the reflection service. Most usages +// should prefer to use Register instead. For backwards compatibility reasons, +// this returns the v1alpha version of the reflection server. For a v1 version +// of the reflection server, see NewServerV1. +// +// # Experimental +// +// Notice: This function is EXPERIMENTAL and may be changed or removed in a +// later release. +func NewServer(opts ServerOptions) v1alphareflectiongrpc.ServerReflectionServer { + return asV1Alpha(NewServerV1(opts)) } -// fileDescEncodingByFilename finds the file descriptor for given filename, -// does marshalling on it and returns the marshalled result. -func (s *serverReflectionServer) fileDescEncodingByFilename(name string) ([]byte, error) { - enc := proto.FileDescriptor(name) - if enc == nil { - return nil, fmt.Errorf("unknown file: %v", name) - } - fd, err := decodeFileDesc(enc) - if err != nil { - return nil, err +// NewServerV1 returns a reflection server implementation using the given options. +// This can be used to customize behavior of the reflection service. Most usages +// should prefer to use Register instead. +// +// # Experimental +// +// Notice: This function is EXPERIMENTAL and may be changed or removed in a +// later release. +func NewServerV1(opts ServerOptions) v1reflectiongrpc.ServerReflectionServer { + if opts.DescriptorResolver == nil { + opts.DescriptorResolver = protoregistry.GlobalFiles + } + if opts.ExtensionResolver == nil { + opts.ExtensionResolver = protoregistry.GlobalTypes + } + return &serverReflectionServer{ + s: opts.Services, + descResolver: opts.DescriptorResolver, + extResolver: opts.ExtensionResolver, } - return proto.Marshal(fd) } -// parseMetadata finds the file descriptor bytes specified meta. -// For SupportPackageIsVersion4, m is the name of the proto file, we -// call proto.FileDescriptor to get the byte slice. -// For SupportPackageIsVersion3, m is a byte slice itself. -func parseMetadata(meta interface{}) ([]byte, bool) { - // Check if meta is the file name. - if fileNameForMeta, ok := meta.(string); ok { - return proto.FileDescriptor(fileNameForMeta), true - } - - // Check if meta is the byte slice. - if enc, ok := meta.([]byte); ok { - return enc, true - } - - return nil, false +type serverReflectionServer struct { + v1alphareflectiongrpc.UnimplementedServerReflectionServer + s ServiceInfoProvider + descResolver protodesc.Resolver + extResolver ExtensionResolver } -// fileDescEncodingContainingSymbol finds the file descriptor containing the given symbol, -// does marshalling on it and returns the marshalled result. -// The given symbol can be a type, a service or a method. -func (s *serverReflectionServer) fileDescEncodingContainingSymbol(name string) ([]byte, error) { - _, symbols := s.getSymbols() - fd := symbols[name] - if fd == nil { - // Check if it's a type name that was not present in the - // transitive dependencies of the registered services. - if st, err := typeForName(name); err == nil { - fd, err = s.fileDescForType(st) +// fileDescWithDependencies returns a slice of serialized fileDescriptors in +// wire format ([]byte). The fileDescriptors will include fd and all the +// transitive dependencies of fd with names not in sentFileDescriptors. +func (s *serverReflectionServer) fileDescWithDependencies(fd protoreflect.FileDescriptor, sentFileDescriptors map[string]bool) ([][]byte, error) { + var r [][]byte + queue := []protoreflect.FileDescriptor{fd} + for len(queue) > 0 { + currentfd := queue[0] + queue = queue[1:] + if sent := sentFileDescriptors[currentfd.Path()]; len(r) == 0 || !sent { + sentFileDescriptors[currentfd.Path()] = true + fdProto := protodesc.ToFileDescriptorProto(currentfd) + currentfdEncoded, err := proto.Marshal(fdProto) if err != nil { return nil, err } + r = append(r, currentfdEncoded) + } + for i := 0; i < currentfd.Imports().Len(); i++ { + queue = append(queue, currentfd.Imports().Get(i)) } } - - if fd == nil { - return nil, fmt.Errorf("unknown symbol: %v", name) - } - - return proto.Marshal(fd) + return r, nil } -// fileDescEncodingContainingExtension finds the file descriptor containing given extension, -// does marshalling on it and returns the marshalled result. -func (s *serverReflectionServer) fileDescEncodingContainingExtension(typeName string, extNum int32) ([]byte, error) { - st, err := typeForName(typeName) +// fileDescEncodingContainingSymbol finds the file descriptor containing the +// given symbol, finds all of its previously unsent transitive dependencies, +// does marshalling on them, and returns the marshalled result. The given symbol +// can be a type, a service or a method. +func (s *serverReflectionServer) fileDescEncodingContainingSymbol(name string, sentFileDescriptors map[string]bool) ([][]byte, error) { + d, err := s.descResolver.FindDescriptorByName(protoreflect.FullName(name)) if err != nil { return nil, err } - fd, err := fileDescContainingExtension(st, extNum) + return s.fileDescWithDependencies(d.ParentFile(), sentFileDescriptors) +} + +// fileDescEncodingContainingExtension finds the file descriptor containing +// given extension, finds all of its previously unsent transitive dependencies, +// does marshalling on them, and returns the marshalled result. +func (s *serverReflectionServer) fileDescEncodingContainingExtension(typeName string, extNum int32, sentFileDescriptors map[string]bool) ([][]byte, error) { + xt, err := s.extResolver.FindExtensionByNumber(protoreflect.FullName(typeName), protoreflect.FieldNumber(extNum)) if err != nil { return nil, err } - return proto.Marshal(fd) + return s.fileDescWithDependencies(xt.TypeDescriptor().ParentFile(), sentFileDescriptors) } // allExtensionNumbersForTypeName returns all extension numbers for the given type. func (s *serverReflectionServer) allExtensionNumbersForTypeName(name string) ([]int32, error) { - st, err := typeForName(name) - if err != nil { - return nil, err + var numbers []int32 + s.extResolver.RangeExtensionsByMessage(protoreflect.FullName(name), func(xt protoreflect.ExtensionType) bool { + numbers = append(numbers, int32(xt.TypeDescriptor().Number())) + return true + }) + sort.Slice(numbers, func(i, j int) bool { + return numbers[i] < numbers[j] + }) + if len(numbers) == 0 { + // maybe return an error if given type name is not known + if _, err := s.descResolver.FindDescriptorByName(protoreflect.FullName(name)); err != nil { + return nil, err + } } - extNums, err := s.allExtensionNumbersForType(st) - if err != nil { - return nil, err + return numbers, nil +} + +// listServices returns the names of services this server exposes. +func (s *serverReflectionServer) listServices() []*v1reflectionpb.ServiceResponse { + serviceInfo := s.s.GetServiceInfo() + resp := make([]*v1reflectionpb.ServiceResponse, 0, len(serviceInfo)) + for svc := range serviceInfo { + resp = append(resp, &v1reflectionpb.ServiceResponse{Name: svc}) } - return extNums, nil + sort.Slice(resp, func(i, j int) bool { + return resp[i].Name < resp[j].Name + }) + return resp } // ServerReflectionInfo is the reflection service handler. -func (s *serverReflectionServer) ServerReflectionInfo(stream rpb.ServerReflection_ServerReflectionInfoServer) error { +func (s *serverReflectionServer) ServerReflectionInfo(stream v1reflectiongrpc.ServerReflection_ServerReflectionInfoServer) error { + sentFileDescriptors := make(map[string]bool) for { in, err := stream.Recv() if err == io.EOF { @@ -363,83 +264,80 @@ func (s *serverReflectionServer) ServerReflectionInfo(stream rpb.ServerReflectio return err } - out := &rpb.ServerReflectionResponse{ + out := &v1reflectionpb.ServerReflectionResponse{ ValidHost: in.Host, OriginalRequest: in, } switch req := in.MessageRequest.(type) { - case *rpb.ServerReflectionRequest_FileByFilename: - b, err := s.fileDescEncodingByFilename(req.FileByFilename) + case *v1reflectionpb.ServerReflectionRequest_FileByFilename: + var b [][]byte + fd, err := s.descResolver.FindFileByPath(req.FileByFilename) + if err == nil { + b, err = s.fileDescWithDependencies(fd, sentFileDescriptors) + } if err != nil { - out.MessageResponse = &rpb.ServerReflectionResponse_ErrorResponse{ - ErrorResponse: &rpb.ErrorResponse{ + out.MessageResponse = &v1reflectionpb.ServerReflectionResponse_ErrorResponse{ + ErrorResponse: &v1reflectionpb.ErrorResponse{ ErrorCode: int32(codes.NotFound), ErrorMessage: err.Error(), }, } } else { - out.MessageResponse = &rpb.ServerReflectionResponse_FileDescriptorResponse{ - FileDescriptorResponse: &rpb.FileDescriptorResponse{FileDescriptorProto: [][]byte{b}}, + out.MessageResponse = &v1reflectionpb.ServerReflectionResponse_FileDescriptorResponse{ + FileDescriptorResponse: &v1reflectionpb.FileDescriptorResponse{FileDescriptorProto: b}, } } - case *rpb.ServerReflectionRequest_FileContainingSymbol: - b, err := s.fileDescEncodingContainingSymbol(req.FileContainingSymbol) + case *v1reflectionpb.ServerReflectionRequest_FileContainingSymbol: + b, err := s.fileDescEncodingContainingSymbol(req.FileContainingSymbol, sentFileDescriptors) if err != nil { - out.MessageResponse = &rpb.ServerReflectionResponse_ErrorResponse{ - ErrorResponse: &rpb.ErrorResponse{ + out.MessageResponse = &v1reflectionpb.ServerReflectionResponse_ErrorResponse{ + ErrorResponse: &v1reflectionpb.ErrorResponse{ ErrorCode: int32(codes.NotFound), ErrorMessage: err.Error(), }, } } else { - out.MessageResponse = &rpb.ServerReflectionResponse_FileDescriptorResponse{ - FileDescriptorResponse: &rpb.FileDescriptorResponse{FileDescriptorProto: [][]byte{b}}, + out.MessageResponse = &v1reflectionpb.ServerReflectionResponse_FileDescriptorResponse{ + FileDescriptorResponse: &v1reflectionpb.FileDescriptorResponse{FileDescriptorProto: b}, } } - case *rpb.ServerReflectionRequest_FileContainingExtension: + case *v1reflectionpb.ServerReflectionRequest_FileContainingExtension: typeName := req.FileContainingExtension.ContainingType extNum := req.FileContainingExtension.ExtensionNumber - b, err := s.fileDescEncodingContainingExtension(typeName, extNum) + b, err := s.fileDescEncodingContainingExtension(typeName, extNum, sentFileDescriptors) if err != nil { - out.MessageResponse = &rpb.ServerReflectionResponse_ErrorResponse{ - ErrorResponse: &rpb.ErrorResponse{ + out.MessageResponse = &v1reflectionpb.ServerReflectionResponse_ErrorResponse{ + ErrorResponse: &v1reflectionpb.ErrorResponse{ ErrorCode: int32(codes.NotFound), ErrorMessage: err.Error(), }, } } else { - out.MessageResponse = &rpb.ServerReflectionResponse_FileDescriptorResponse{ - FileDescriptorResponse: &rpb.FileDescriptorResponse{FileDescriptorProto: [][]byte{b}}, + out.MessageResponse = &v1reflectionpb.ServerReflectionResponse_FileDescriptorResponse{ + FileDescriptorResponse: &v1reflectionpb.FileDescriptorResponse{FileDescriptorProto: b}, } } - case *rpb.ServerReflectionRequest_AllExtensionNumbersOfType: + case *v1reflectionpb.ServerReflectionRequest_AllExtensionNumbersOfType: extNums, err := s.allExtensionNumbersForTypeName(req.AllExtensionNumbersOfType) if err != nil { - out.MessageResponse = &rpb.ServerReflectionResponse_ErrorResponse{ - ErrorResponse: &rpb.ErrorResponse{ + out.MessageResponse = &v1reflectionpb.ServerReflectionResponse_ErrorResponse{ + ErrorResponse: &v1reflectionpb.ErrorResponse{ ErrorCode: int32(codes.NotFound), ErrorMessage: err.Error(), }, } } else { - out.MessageResponse = &rpb.ServerReflectionResponse_AllExtensionNumbersResponse{ - AllExtensionNumbersResponse: &rpb.ExtensionNumberResponse{ + out.MessageResponse = &v1reflectionpb.ServerReflectionResponse_AllExtensionNumbersResponse{ + AllExtensionNumbersResponse: &v1reflectionpb.ExtensionNumberResponse{ BaseTypeName: req.AllExtensionNumbersOfType, ExtensionNumber: extNums, }, } } - case *rpb.ServerReflectionRequest_ListServices: - svcNames, _ := s.getSymbols() - serviceResponses := make([]*rpb.ServiceResponse, len(svcNames)) - for i, n := range svcNames { - serviceResponses[i] = &rpb.ServiceResponse{ - Name: n, - } - } - out.MessageResponse = &rpb.ServerReflectionResponse_ListServicesResponse{ - ListServicesResponse: &rpb.ListServiceResponse{ - Service: serviceResponses, + case *v1reflectionpb.ServerReflectionRequest_ListServices: + out.MessageResponse = &v1reflectionpb.ServerReflectionResponse_ListServicesResponse{ + ListServicesResponse: &v1reflectionpb.ListServiceResponse{ + Service: s.listServices(), }, } default: diff --git a/reflection/serverreflection_test.go b/reflection/serverreflection_test.go index db5fce2d8939..b17fa25d3d1c 100644 --- a/reflection/serverreflection_test.go +++ b/reflection/serverreflection_test.go @@ -25,32 +25,45 @@ import ( "reflect" "sort" "testing" + "time" - "github.com/golang/protobuf/proto" - dpb "github.com/golang/protobuf/protoc-gen-go/descriptor" "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/internal/grpctest" - rpb "google.golang.org/grpc/reflection/grpc_reflection_v1alpha" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/reflect/protodesc" + "google.golang.org/protobuf/reflect/protoreflect" + "google.golang.org/protobuf/reflect/protoregistry" + "google.golang.org/protobuf/types/descriptorpb" + "google.golang.org/protobuf/types/dynamicpb" + + v1reflectiongrpc "google.golang.org/grpc/reflection/grpc_reflection_v1" + v1reflectionpb "google.golang.org/grpc/reflection/grpc_reflection_v1" + v1alphareflectiongrpc "google.golang.org/grpc/reflection/grpc_reflection_v1alpha" + v1alphareflectionpb "google.golang.org/grpc/reflection/grpc_reflection_v1alpha" pb "google.golang.org/grpc/reflection/grpc_testing" - pbv3 "google.golang.org/grpc/reflection/grpc_testingv3" + pbv3 "google.golang.org/grpc/reflection/grpc_testing_not_regenerate" ) var ( - s = &serverReflectionServer{} + s = NewServerV1(ServerOptions{}).(*serverReflectionServer) // fileDescriptor of each test proto file. - fdTest *dpb.FileDescriptorProto - fdTestv3 *dpb.FileDescriptorProto - fdProto2 *dpb.FileDescriptorProto - fdProto2Ext *dpb.FileDescriptorProto - fdProto2Ext2 *dpb.FileDescriptorProto + fdProto2Ext *descriptorpb.FileDescriptorProto + fdProto2Ext2 *descriptorpb.FileDescriptorProto + fdDynamic *descriptorpb.FileDescriptorProto + // reflection descriptors. + fdDynamicFile protoreflect.FileDescriptor // fileDescriptor marshalled. fdTestByte []byte fdTestv3Byte []byte fdProto2Byte []byte fdProto2ExtByte []byte fdProto2Ext2Byte []byte + fdDynamicByte []byte ) +const defaultTestTimeout = 10 * time.Second + type x struct { grpctest.Tester } @@ -59,85 +72,78 @@ func Test(t *testing.T) { grpctest.RunSubTests(t, x{}) } -func loadFileDesc(filename string) (*dpb.FileDescriptorProto, []byte) { - enc := proto.FileDescriptor(filename) - if enc == nil { - panic(fmt.Sprintf("failed to find fd for file: %v", filename)) - } - fd, err := decodeFileDesc(enc) +func loadFileDesc(filename string) (*descriptorpb.FileDescriptorProto, []byte) { + fd, err := protoregistry.GlobalFiles.FindFileByPath(filename) if err != nil { - panic(fmt.Sprintf("failed to decode enc: %v", err)) + panic(err) } - b, err := proto.Marshal(fd) + fdProto := protodesc.ToFileDescriptorProto(fd) + b, err := proto.Marshal(fdProto) if err != nil { panic(fmt.Sprintf("failed to marshal fd: %v", err)) } - return fd, b + return fdProto, b } -func init() { - fdTest, fdTestByte = loadFileDesc("reflection/grpc_testing/test.proto") - fdTestv3, fdTestv3Byte = loadFileDesc("testv3.proto") - fdProto2, fdProto2Byte = loadFileDesc("reflection/grpc_testing/proto2.proto") - fdProto2Ext, fdProto2ExtByte = loadFileDesc("reflection/grpc_testing/proto2_ext.proto") - fdProto2Ext2, fdProto2Ext2Byte = loadFileDesc("reflection/grpc_testing/proto2_ext2.proto") -} +func loadFileDescDynamic(b []byte) (*descriptorpb.FileDescriptorProto, protoreflect.FileDescriptor, []byte) { + m := new(descriptorpb.FileDescriptorProto) + if err := proto.Unmarshal(b, m); err != nil { + panic("failed to unmarshal dynamic proto raw descriptor") + } -func (x) TestFileDescForType(t *testing.T) { - for _, test := range []struct { - st reflect.Type - wantFd *dpb.FileDescriptorProto - }{ - {reflect.TypeOf(pb.SearchResponse_Result{}), fdTest}, - {reflect.TypeOf(pb.ToBeExtended{}), fdProto2}, - } { - fd, err := s.fileDescForType(test.st) - if err != nil || !proto.Equal(fd, test.wantFd) { - t.Errorf("fileDescForType(%q) = %q, %v, want %q, ", test.st, fd, err, test.wantFd) - } + fd, err := protodesc.NewFile(m, nil) + if err != nil { + panic(err) } -} -func (x) TestTypeForName(t *testing.T) { - for _, test := range []struct { - name string - want reflect.Type - }{ - {"grpc.testing.SearchResponse", reflect.TypeOf(pb.SearchResponse{})}, - } { - r, err := typeForName(test.name) - if err != nil || r != test.want { - t.Errorf("typeForName(%q) = %q, %v, want %q, ", test.name, r, err, test.want) - } + err = protoregistry.GlobalFiles.RegisterFile(fd) + if err != nil { + panic(err) } -} -func (x) TestTypeForNameNotFound(t *testing.T) { - for _, test := range []string{ - "grpc.testing.not_exiting", - } { - _, err := typeForName(test) - if err == nil { - t.Errorf("typeForName(%q) = _, %v, want _, ", test, err) + for i := 0; i < fd.Messages().Len(); i++ { + m := fd.Messages().Get(i) + if err := protoregistry.GlobalTypes.RegisterMessage(dynamicpb.NewMessageType(m)); err != nil { + panic(err) } } + + return m, fd, b +} + +func init() { + _, fdTestByte = loadFileDesc("reflection/grpc_testing/test.proto") + _, fdTestv3Byte = loadFileDesc("testv3.proto") + _, fdProto2Byte = loadFileDesc("reflection/grpc_testing/proto2.proto") + fdProto2Ext, fdProto2ExtByte = loadFileDesc("reflection/grpc_testing/proto2_ext.proto") + fdProto2Ext2, fdProto2Ext2Byte = loadFileDesc("reflection/grpc_testing/proto2_ext2.proto") + fdDynamic, fdDynamicFile, fdDynamicByte = loadFileDescDynamic(pbv3.FileDynamicProtoRawDesc) } func (x) TestFileDescContainingExtension(t *testing.T) { for _, test := range []struct { - st reflect.Type + st string extNum int32 - want *dpb.FileDescriptorProto + want *descriptorpb.FileDescriptorProto }{ - {reflect.TypeOf(pb.ToBeExtended{}), 13, fdProto2Ext}, - {reflect.TypeOf(pb.ToBeExtended{}), 17, fdProto2Ext}, - {reflect.TypeOf(pb.ToBeExtended{}), 19, fdProto2Ext}, - {reflect.TypeOf(pb.ToBeExtended{}), 23, fdProto2Ext2}, - {reflect.TypeOf(pb.ToBeExtended{}), 29, fdProto2Ext2}, + {"grpc.testing.ToBeExtended", 13, fdProto2Ext}, + {"grpc.testing.ToBeExtended", 17, fdProto2Ext}, + {"grpc.testing.ToBeExtended", 19, fdProto2Ext}, + {"grpc.testing.ToBeExtended", 23, fdProto2Ext2}, + {"grpc.testing.ToBeExtended", 29, fdProto2Ext2}, } { - fd, err := fileDescContainingExtension(test.st, test.extNum) - if err != nil || !proto.Equal(fd, test.want) { - t.Errorf("fileDescContainingExtension(%q) = %q, %v, want %q, ", test.st, fd, err, test.want) + fd, err := s.fileDescEncodingContainingExtension(test.st, test.extNum, map[string]bool{}) + if err != nil { + t.Errorf("fileDescContainingExtension(%q) return error: %v", test.st, err) + continue + } + var actualFd descriptorpb.FileDescriptorProto + if err := proto.Unmarshal(fd[0], &actualFd); err != nil { + t.Errorf("fileDescContainingExtension(%q) return invalid bytes: %v", test.st, err) + continue + } + if !proto.Equal(&actualFd, test.want) { + t.Errorf("fileDescContainingExtension(%q) returned %q, but wanted %q", test.st, &actualFd, test.want) } } } @@ -149,14 +155,14 @@ func (s intArray) Len() int { return len(s) } func (s intArray) Swap(i, j int) { s[i], s[j] = s[j], s[i] } func (s intArray) Less(i, j int) bool { return s[i] < s[j] } -func (x) TestAllExtensionNumbersForType(t *testing.T) { +func (x) TestAllExtensionNumbersForTypeName(t *testing.T) { for _, test := range []struct { - st reflect.Type + st string want []int32 }{ - {reflect.TypeOf(pb.ToBeExtended{}), []int32{13, 17, 19, 23, 29}}, + {"grpc.testing.ToBeExtended", []int32{13, 17, 19, 23, 29}}, } { - r, err := s.allExtensionNumbersForType(test.st) + r, err := s.allExtensionNumbersForTypeName(test.st) sort.Sort(intArray(r)) if err != nil || !reflect.DeepEqual(r, test.want) { t.Errorf("allExtensionNumbersForType(%q) = %v, %v, want %v, ", test.st, r, err, test.want) @@ -166,9 +172,9 @@ func (x) TestAllExtensionNumbersForType(t *testing.T) { // Do end2end tests. -type server struct{} - -var _ pb.UnstableSearchServiceService = (*server)(nil) +type server struct { + pb.UnimplementedSearchServiceServer +} func (s *server) Search(ctx context.Context, in *pb.SearchRequest) (*pb.SearchResponse, error) { return &pb.SearchResponse{}, nil @@ -195,39 +201,97 @@ func (x) TestReflectionEnd2end(t *testing.T) { t.Fatalf("failed to listen: %v", err) } s := grpc.NewServer() - pb.RegisterSearchServiceService(s, pb.NewSearchServiceService(&server{})) + pb.RegisterSearchServiceServer(s, &server{}) pbv3.RegisterSearchServiceV3Server(s, &serverV3{}) + + registerDynamicProto(s, fdDynamic, fdDynamicFile) + // Register reflection service on s. Register(s) go s.Serve(lis) + t.Cleanup(s.Stop) // Create client. - conn, err := grpc.Dial(lis.Addr().String(), grpc.WithInsecure()) + conn, err := grpc.Dial(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { t.Fatalf("cannot connect to server: %v", err) } defer conn.Close() - c := rpb.NewServerReflectionClient(conn) - stream, err := c.ServerReflectionInfo(context.Background(), grpc.WaitForReady(true)) - if err != nil { - t.Fatalf("cannot get ServerReflectionInfo: %v", err) + clientV1 := v1reflectiongrpc.NewServerReflectionClient(conn) + clientV1Alpha := v1alphareflectiongrpc.NewServerReflectionClient(conn) + testCases := []struct { + name string + client v1reflectiongrpc.ServerReflectionClient + }{ + { + name: "v1", + client: clientV1, + }, + { + name: "v1alpha", + client: v1AlphaClientAdapter{stub: clientV1Alpha}, + }, } + for _, testCase := range testCases { + c := testCase.client + t.Run(testCase.name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + stream, err := c.ServerReflectionInfo(ctx, grpc.WaitForReady(true)) + if err != nil { + t.Fatalf("cannot get ServerReflectionInfo: %v", err) + } - testFileByFilename(t, stream) - testFileByFilenameError(t, stream) - testFileContainingSymbol(t, stream) - testFileContainingSymbolError(t, stream) - testFileContainingExtension(t, stream) - testFileContainingExtensionError(t, stream) - testAllExtensionNumbersOfType(t, stream) - testAllExtensionNumbersOfTypeError(t, stream) - testListServices(t, stream) - - s.Stop() + testFileByFilenameTransitiveClosure(t, stream, true) + testFileByFilenameTransitiveClosure(t, stream, false) + testFileByFilename(t, stream) + testFileByFilenameError(t, stream) + testFileContainingSymbol(t, stream) + testFileContainingSymbolError(t, stream) + testFileContainingExtension(t, stream) + testFileContainingExtensionError(t, stream) + testAllExtensionNumbersOfType(t, stream) + testAllExtensionNumbersOfTypeError(t, stream) + testListServices(t, stream) + }) + } } -func testFileByFilename(t *testing.T, stream rpb.ServerReflection_ServerReflectionInfoClient) { +func testFileByFilenameTransitiveClosure(t *testing.T, stream v1reflectiongrpc.ServerReflection_ServerReflectionInfoClient, expectClosure bool) { + filename := "reflection/grpc_testing/proto2_ext2.proto" + if err := stream.Send(&v1reflectionpb.ServerReflectionRequest{ + MessageRequest: &v1reflectionpb.ServerReflectionRequest_FileByFilename{ + FileByFilename: filename, + }, + }); err != nil { + t.Fatalf("failed to send request: %v", err) + } + r, err := stream.Recv() + if err != nil { + // io.EOF is not ok. + t.Fatalf("failed to recv response: %v", err) + } + switch r.MessageResponse.(type) { + case *v1reflectionpb.ServerReflectionResponse_FileDescriptorResponse: + if !reflect.DeepEqual(r.GetFileDescriptorResponse().FileDescriptorProto[0], fdProto2Ext2Byte) { + t.Errorf("FileByFilename(%v)\nreceived: %q,\nwant: %q", filename, r.GetFileDescriptorResponse().FileDescriptorProto[0], fdProto2Ext2Byte) + } + if expectClosure { + if len(r.GetFileDescriptorResponse().FileDescriptorProto) != 2 { + t.Errorf("FileByFilename(%v) returned %v file descriptors, expected 2", filename, len(r.GetFileDescriptorResponse().FileDescriptorProto)) + } else if !reflect.DeepEqual(r.GetFileDescriptorResponse().FileDescriptorProto[1], fdProto2Byte) { + t.Errorf("FileByFilename(%v)\nreceived: %q,\nwant: %q", filename, r.GetFileDescriptorResponse().FileDescriptorProto[1], fdProto2Byte) + } + } else if len(r.GetFileDescriptorResponse().FileDescriptorProto) != 1 { + t.Errorf("FileByFilename(%v) returned %v file descriptors, expected 1", filename, len(r.GetFileDescriptorResponse().FileDescriptorProto)) + } + default: + t.Errorf("FileByFilename(%v) = %v, want type ", filename, r.MessageResponse) + } +} + +func testFileByFilename(t *testing.T, stream v1reflectiongrpc.ServerReflection_ServerReflectionInfoClient) { for _, test := range []struct { filename string want []byte @@ -235,9 +299,10 @@ func testFileByFilename(t *testing.T, stream rpb.ServerReflection_ServerReflecti {"reflection/grpc_testing/test.proto", fdTestByte}, {"reflection/grpc_testing/proto2.proto", fdProto2Byte}, {"reflection/grpc_testing/proto2_ext.proto", fdProto2ExtByte}, + {"dynamic.proto", fdDynamicByte}, } { - if err := stream.Send(&rpb.ServerReflectionRequest{ - MessageRequest: &rpb.ServerReflectionRequest_FileByFilename{ + if err := stream.Send(&v1reflectionpb.ServerReflectionRequest{ + MessageRequest: &v1reflectionpb.ServerReflectionRequest_FileByFilename{ FileByFilename: test.filename, }, }); err != nil { @@ -250,7 +315,7 @@ func testFileByFilename(t *testing.T, stream rpb.ServerReflection_ServerReflecti } switch r.MessageResponse.(type) { - case *rpb.ServerReflectionResponse_FileDescriptorResponse: + case *v1reflectionpb.ServerReflectionResponse_FileDescriptorResponse: if !reflect.DeepEqual(r.GetFileDescriptorResponse().FileDescriptorProto[0], test.want) { t.Errorf("FileByFilename(%v)\nreceived: %q,\nwant: %q", test.filename, r.GetFileDescriptorResponse().FileDescriptorProto[0], test.want) } @@ -260,14 +325,14 @@ func testFileByFilename(t *testing.T, stream rpb.ServerReflection_ServerReflecti } } -func testFileByFilenameError(t *testing.T, stream rpb.ServerReflection_ServerReflectionInfoClient) { +func testFileByFilenameError(t *testing.T, stream v1reflectiongrpc.ServerReflection_ServerReflectionInfoClient) { for _, test := range []string{ "test.poto", "proo2.proto", "proto2_et.proto", } { - if err := stream.Send(&rpb.ServerReflectionRequest{ - MessageRequest: &rpb.ServerReflectionRequest_FileByFilename{ + if err := stream.Send(&v1reflectionpb.ServerReflectionRequest{ + MessageRequest: &v1reflectionpb.ServerReflectionRequest_FileByFilename{ FileByFilename: test, }, }); err != nil { @@ -280,14 +345,14 @@ func testFileByFilenameError(t *testing.T, stream rpb.ServerReflection_ServerRef } switch r.MessageResponse.(type) { - case *rpb.ServerReflectionResponse_ErrorResponse: + case *v1reflectionpb.ServerReflectionResponse_ErrorResponse: default: t.Errorf("FileByFilename(%v) = %v, want type ", test, r.MessageResponse) } } } -func testFileContainingSymbol(t *testing.T, stream rpb.ServerReflection_ServerReflectionInfoClient) { +func testFileContainingSymbol(t *testing.T, stream v1reflectiongrpc.ServerReflection_ServerReflectionInfoClient) { for _, test := range []struct { symbol string want []byte @@ -307,10 +372,14 @@ func testFileContainingSymbol(t *testing.T, stream rpb.ServerReflection_ServerRe {"grpc.testingv3.SearchResponseV3.Result.Value.val", fdTestv3Byte}, {"grpc.testingv3.SearchResponseV3.Result.Value.str", fdTestv3Byte}, {"grpc.testingv3.SearchResponseV3.State", fdTestv3Byte}, - {"grpc.testingv3.SearchResponseV3.State.FRESH", fdTestv3Byte}, + {"grpc.testingv3.SearchResponseV3.FRESH", fdTestv3Byte}, + // Test dynamic symbols + {"grpc.testing.DynamicService", fdDynamicByte}, + {"grpc.testing.DynamicReq", fdDynamicByte}, + {"grpc.testing.DynamicRes", fdDynamicByte}, } { - if err := stream.Send(&rpb.ServerReflectionRequest{ - MessageRequest: &rpb.ServerReflectionRequest_FileContainingSymbol{ + if err := stream.Send(&v1reflectionpb.ServerReflectionRequest{ + MessageRequest: &v1reflectionpb.ServerReflectionRequest_FileContainingSymbol{ FileContainingSymbol: test.symbol, }, }); err != nil { @@ -323,7 +392,7 @@ func testFileContainingSymbol(t *testing.T, stream rpb.ServerReflection_ServerRe } switch r.MessageResponse.(type) { - case *rpb.ServerReflectionResponse_FileDescriptorResponse: + case *v1reflectionpb.ServerReflectionResponse_FileDescriptorResponse: if !reflect.DeepEqual(r.GetFileDescriptorResponse().FileDescriptorProto[0], test.want) { t.Errorf("FileContainingSymbol(%v)\nreceived: %q,\nwant: %q", test.symbol, r.GetFileDescriptorResponse().FileDescriptorProto[0], test.want) } @@ -333,15 +402,15 @@ func testFileContainingSymbol(t *testing.T, stream rpb.ServerReflection_ServerRe } } -func testFileContainingSymbolError(t *testing.T, stream rpb.ServerReflection_ServerReflectionInfoClient) { +func testFileContainingSymbolError(t *testing.T, stream v1reflectiongrpc.ServerReflection_ServerReflectionInfoClient) { for _, test := range []string{ "grpc.testing.SerchService", "grpc.testing.SearchService.SearchE", "grpc.tesing.SearchResponse", "gpc.testing.ToBeExtended", } { - if err := stream.Send(&rpb.ServerReflectionRequest{ - MessageRequest: &rpb.ServerReflectionRequest_FileContainingSymbol{ + if err := stream.Send(&v1reflectionpb.ServerReflectionRequest{ + MessageRequest: &v1reflectionpb.ServerReflectionRequest_FileContainingSymbol{ FileContainingSymbol: test, }, }); err != nil { @@ -354,14 +423,14 @@ func testFileContainingSymbolError(t *testing.T, stream rpb.ServerReflection_Ser } switch r.MessageResponse.(type) { - case *rpb.ServerReflectionResponse_ErrorResponse: + case *v1reflectionpb.ServerReflectionResponse_ErrorResponse: default: t.Errorf("FileContainingSymbol(%v) = %v, want type ", test, r.MessageResponse) } } } -func testFileContainingExtension(t *testing.T, stream rpb.ServerReflection_ServerReflectionInfoClient) { +func testFileContainingExtension(t *testing.T, stream v1reflectiongrpc.ServerReflection_ServerReflectionInfoClient) { for _, test := range []struct { typeName string extNum int32 @@ -373,9 +442,9 @@ func testFileContainingExtension(t *testing.T, stream rpb.ServerReflection_Serve {"grpc.testing.ToBeExtended", 23, fdProto2Ext2Byte}, {"grpc.testing.ToBeExtended", 29, fdProto2Ext2Byte}, } { - if err := stream.Send(&rpb.ServerReflectionRequest{ - MessageRequest: &rpb.ServerReflectionRequest_FileContainingExtension{ - FileContainingExtension: &rpb.ExtensionRequest{ + if err := stream.Send(&v1reflectionpb.ServerReflectionRequest{ + MessageRequest: &v1reflectionpb.ServerReflectionRequest_FileContainingExtension{ + FileContainingExtension: &v1reflectionpb.ExtensionRequest{ ContainingType: test.typeName, ExtensionNumber: test.extNum, }, @@ -390,7 +459,7 @@ func testFileContainingExtension(t *testing.T, stream rpb.ServerReflection_Serve } switch r.MessageResponse.(type) { - case *rpb.ServerReflectionResponse_FileDescriptorResponse: + case *v1reflectionpb.ServerReflectionResponse_FileDescriptorResponse: if !reflect.DeepEqual(r.GetFileDescriptorResponse().FileDescriptorProto[0], test.want) { t.Errorf("FileContainingExtension(%v, %v)\nreceived: %q,\nwant: %q", test.typeName, test.extNum, r.GetFileDescriptorResponse().FileDescriptorProto[0], test.want) } @@ -400,7 +469,7 @@ func testFileContainingExtension(t *testing.T, stream rpb.ServerReflection_Serve } } -func testFileContainingExtensionError(t *testing.T, stream rpb.ServerReflection_ServerReflectionInfoClient) { +func testFileContainingExtensionError(t *testing.T, stream v1reflectiongrpc.ServerReflection_ServerReflectionInfoClient) { for _, test := range []struct { typeName string extNum int32 @@ -408,9 +477,9 @@ func testFileContainingExtensionError(t *testing.T, stream rpb.ServerReflection_ {"grpc.testing.ToBExtended", 17}, {"grpc.testing.ToBeExtended", 15}, } { - if err := stream.Send(&rpb.ServerReflectionRequest{ - MessageRequest: &rpb.ServerReflectionRequest_FileContainingExtension{ - FileContainingExtension: &rpb.ExtensionRequest{ + if err := stream.Send(&v1reflectionpb.ServerReflectionRequest{ + MessageRequest: &v1reflectionpb.ServerReflectionRequest_FileContainingExtension{ + FileContainingExtension: &v1reflectionpb.ExtensionRequest{ ContainingType: test.typeName, ExtensionNumber: test.extNum, }, @@ -425,22 +494,23 @@ func testFileContainingExtensionError(t *testing.T, stream rpb.ServerReflection_ } switch r.MessageResponse.(type) { - case *rpb.ServerReflectionResponse_ErrorResponse: + case *v1reflectionpb.ServerReflectionResponse_ErrorResponse: default: t.Errorf("FileContainingExtension(%v, %v) = %v, want type ", test.typeName, test.extNum, r.MessageResponse) } } } -func testAllExtensionNumbersOfType(t *testing.T, stream rpb.ServerReflection_ServerReflectionInfoClient) { +func testAllExtensionNumbersOfType(t *testing.T, stream v1reflectiongrpc.ServerReflection_ServerReflectionInfoClient) { for _, test := range []struct { typeName string want []int32 }{ {"grpc.testing.ToBeExtended", []int32{13, 17, 19, 23, 29}}, + {"grpc.testing.DynamicReq", nil}, } { - if err := stream.Send(&rpb.ServerReflectionRequest{ - MessageRequest: &rpb.ServerReflectionRequest_AllExtensionNumbersOfType{ + if err := stream.Send(&v1reflectionpb.ServerReflectionRequest{ + MessageRequest: &v1reflectionpb.ServerReflectionRequest_AllExtensionNumbersOfType{ AllExtensionNumbersOfType: test.typeName, }, }); err != nil { @@ -453,7 +523,7 @@ func testAllExtensionNumbersOfType(t *testing.T, stream rpb.ServerReflection_Ser } switch r.MessageResponse.(type) { - case *rpb.ServerReflectionResponse_AllExtensionNumbersResponse: + case *v1reflectionpb.ServerReflectionResponse_AllExtensionNumbersResponse: extNum := r.GetAllExtensionNumbersResponse().ExtensionNumber sort.Sort(intArray(extNum)) if r.GetAllExtensionNumbersResponse().BaseTypeName != test.typeName || @@ -466,12 +536,12 @@ func testAllExtensionNumbersOfType(t *testing.T, stream rpb.ServerReflection_Ser } } -func testAllExtensionNumbersOfTypeError(t *testing.T, stream rpb.ServerReflection_ServerReflectionInfoClient) { +func testAllExtensionNumbersOfTypeError(t *testing.T, stream v1reflectiongrpc.ServerReflection_ServerReflectionInfoClient) { for _, test := range []string{ "grpc.testing.ToBeExtendedE", } { - if err := stream.Send(&rpb.ServerReflectionRequest{ - MessageRequest: &rpb.ServerReflectionRequest_AllExtensionNumbersOfType{ + if err := stream.Send(&v1reflectionpb.ServerReflectionRequest{ + MessageRequest: &v1reflectionpb.ServerReflectionRequest_AllExtensionNumbersOfType{ AllExtensionNumbersOfType: test, }, }); err != nil { @@ -484,16 +554,16 @@ func testAllExtensionNumbersOfTypeError(t *testing.T, stream rpb.ServerReflectio } switch r.MessageResponse.(type) { - case *rpb.ServerReflectionResponse_ErrorResponse: + case *v1reflectionpb.ServerReflectionResponse_ErrorResponse: default: t.Errorf("AllExtensionNumbersOfType(%v) = %v, want type ", test, r.MessageResponse) } } } -func testListServices(t *testing.T, stream rpb.ServerReflection_ServerReflectionInfoClient) { - if err := stream.Send(&rpb.ServerReflectionRequest{ - MessageRequest: &rpb.ServerReflectionRequest_ListServices{}, +func testListServices(t *testing.T, stream v1reflectiongrpc.ServerReflection_ServerReflectionInfoClient) { + if err := stream.Send(&v1reflectionpb.ServerReflectionRequest{ + MessageRequest: &v1reflectionpb.ServerReflectionRequest_ListServices{}, }); err != nil { t.Fatalf("failed to send request: %v", err) } @@ -504,12 +574,14 @@ func testListServices(t *testing.T, stream rpb.ServerReflection_ServerReflection } switch r.MessageResponse.(type) { - case *rpb.ServerReflectionResponse_ListServicesResponse: + case *v1reflectionpb.ServerReflectionResponse_ListServicesResponse: services := r.GetListServicesResponse().Service want := []string{ "grpc.testingv3.SearchServiceV3", "grpc.testing.SearchService", + "grpc.reflection.v1.ServerReflection", "grpc.reflection.v1alpha.ServerReflection", + "grpc.testing.DynamicService", } // Compare service names in response with want. if len(services) != len(want) { @@ -530,3 +602,107 @@ func testListServices(t *testing.T, stream rpb.ServerReflection_ServerReflection t.Errorf("ListServices = %v, want type ", r.MessageResponse) } } + +func registerDynamicProto(srv *grpc.Server, fdp *descriptorpb.FileDescriptorProto, fd protoreflect.FileDescriptor) { + type emptyInterface any + + for i := 0; i < fd.Services().Len(); i++ { + s := fd.Services().Get(i) + + sd := &grpc.ServiceDesc{ + ServiceName: string(s.FullName()), + HandlerType: (*emptyInterface)(nil), + Metadata: fdp.GetName(), + } + + for j := 0; j < s.Methods().Len(); j++ { + m := s.Methods().Get(j) + sd.Methods = append(sd.Methods, grpc.MethodDesc{ + MethodName: string(m.Name()), + }) + } + + srv.RegisterService(sd, struct{}{}) + } +} + +type v1AlphaClientAdapter struct { + stub v1alphareflectiongrpc.ServerReflectionClient +} + +func (v v1AlphaClientAdapter) ServerReflectionInfo(ctx context.Context, opts ...grpc.CallOption) (v1reflectiongrpc.ServerReflection_ServerReflectionInfoClient, error) { + stream, err := v.stub.ServerReflectionInfo(ctx, opts...) + if err != nil { + return nil, err + } + return v1AlphaClientStreamAdapter{stream}, nil +} + +type v1AlphaClientStreamAdapter struct { + v1alphareflectiongrpc.ServerReflection_ServerReflectionInfoClient +} + +func (s v1AlphaClientStreamAdapter) Send(request *v1reflectionpb.ServerReflectionRequest) error { + return s.ServerReflection_ServerReflectionInfoClient.Send(v1ToV1AlphaRequest(request)) +} + +func (s v1AlphaClientStreamAdapter) Recv() (*v1reflectionpb.ServerReflectionResponse, error) { + resp, err := s.ServerReflection_ServerReflectionInfoClient.Recv() + if err != nil { + return nil, err + } + return v1AlphaToV1Response(resp), nil +} + +func v1AlphaToV1Response(v1alpha *v1alphareflectionpb.ServerReflectionResponse) *v1reflectionpb.ServerReflectionResponse { + var v1 v1reflectionpb.ServerReflectionResponse + v1.ValidHost = v1alpha.ValidHost + if v1alpha.OriginalRequest != nil { + v1.OriginalRequest = v1AlphaToV1Request(v1alpha.OriginalRequest) + } + switch mr := v1alpha.MessageResponse.(type) { + case *v1alphareflectionpb.ServerReflectionResponse_FileDescriptorResponse: + if mr != nil { + v1.MessageResponse = &v1reflectionpb.ServerReflectionResponse_FileDescriptorResponse{ + FileDescriptorResponse: &v1reflectionpb.FileDescriptorResponse{ + FileDescriptorProto: mr.FileDescriptorResponse.GetFileDescriptorProto(), + }, + } + } + case *v1alphareflectionpb.ServerReflectionResponse_AllExtensionNumbersResponse: + if mr != nil { + v1.MessageResponse = &v1reflectionpb.ServerReflectionResponse_AllExtensionNumbersResponse{ + AllExtensionNumbersResponse: &v1reflectionpb.ExtensionNumberResponse{ + BaseTypeName: mr.AllExtensionNumbersResponse.GetBaseTypeName(), + ExtensionNumber: mr.AllExtensionNumbersResponse.GetExtensionNumber(), + }, + } + } + case *v1alphareflectionpb.ServerReflectionResponse_ListServicesResponse: + if mr != nil { + svcs := make([]*v1reflectionpb.ServiceResponse, len(mr.ListServicesResponse.GetService())) + for i, svc := range mr.ListServicesResponse.GetService() { + svcs[i] = &v1reflectionpb.ServiceResponse{ + Name: svc.GetName(), + } + } + v1.MessageResponse = &v1reflectionpb.ServerReflectionResponse_ListServicesResponse{ + ListServicesResponse: &v1reflectionpb.ListServiceResponse{ + Service: svcs, + }, + } + } + case *v1alphareflectionpb.ServerReflectionResponse_ErrorResponse: + if mr != nil { + v1.MessageResponse = &v1reflectionpb.ServerReflectionResponse_ErrorResponse{ + ErrorResponse: &v1reflectionpb.ErrorResponse{ + ErrorCode: mr.ErrorResponse.GetErrorCode(), + ErrorMessage: mr.ErrorResponse.GetErrorMessage(), + }, + } + } + default: + // no value set + } + return &v1 +} diff --git a/regenerate.sh b/regenerate.sh index 750e3500e529..a6f26c8ab0f0 100755 --- a/regenerate.sh +++ b/regenerate.sh @@ -26,8 +26,13 @@ export GOBIN=${WORKDIR}/bin export PATH=${GOBIN}:${PATH} mkdir -p ${GOBIN} -echo "go install github.com/golang/protobuf/protoc-gen-go" -(cd test/tools && go install github.com/golang/protobuf/protoc-gen-go) +echo "remove existing generated files" +# grpc_testing_not_regenerate/*.pb.go is not re-generated, +# see grpc_testing_not_regenerate/README.md for details. +rm -f $(find . -name '*.pb.go' | grep -v 'grpc_testing_not_regenerate') + +echo "go install google.golang.org/protobuf/cmd/protoc-gen-go" +(cd test/tools && go install google.golang.org/protobuf/cmd/protoc-gen-go) echo "go install cmd/protoc-gen-go-grpc" (cd cmd/protoc-gen-go-grpc && go install .) @@ -35,92 +40,84 @@ echo "go install cmd/protoc-gen-go-grpc" echo "git clone https://github.com/grpc/grpc-proto" git clone --quiet https://github.com/grpc/grpc-proto ${WORKDIR}/grpc-proto +echo "git clone https://github.com/protocolbuffers/protobuf" +git clone --quiet https://github.com/protocolbuffers/protobuf ${WORKDIR}/protobuf + +# Pull in code.proto as a proto dependency mkdir -p ${WORKDIR}/googleapis/google/rpc echo "curl https://raw.githubusercontent.com/googleapis/googleapis/master/google/rpc/code.proto" curl --silent https://raw.githubusercontent.com/googleapis/googleapis/master/google/rpc/code.proto > ${WORKDIR}/googleapis/google/rpc/code.proto -# Pull in the following repos to build the MeshCA config proto. -ENVOY_API_REPOS=( - "https://github.com/envoyproxy/data-plane-api" - "https://github.com/cncf/udpa" - "https://github.com/envoyproxy/protoc-gen-validate" -) -for repo in ${ENVOY_API_REPOS[@]}; do - dirname=$(basename ${repo}) - mkdir -p ${WORKDIR}/${dirname} - echo "git clone ${repo}" - git clone --quiet ${repo} ${WORKDIR}/${dirname} -done - -# Pull in the MeshCA service proto. -mkdir -p ${WORKDIR}/istio/istio/google/security/meshca/v1 -echo "curl https://raw.githubusercontent.com/istio/istio/master/security/proto/providers/google/meshca.proto" -curl --silent https://raw.githubusercontent.com/istio/istio/master/security/proto/providers/google/meshca.proto > ${WORKDIR}/istio/istio/google/security/meshca/v1/meshca.proto - mkdir -p ${WORKDIR}/out -# Generates legacy gRPC Server symbols in addition to the newer Service symbols +# Generates sources without the embed requirement LEGACY_SOURCES=( - ${WORKDIR}/googleapis/google/rpc/code.proto ${WORKDIR}/grpc-proto/grpc/binlog/v1/binarylog.proto ${WORKDIR}/grpc-proto/grpc/channelz/v1/channelz.proto - ${WORKDIR}/grpc-proto/grpc/gcp/altscontext.proto - ${WORKDIR}/grpc-proto/grpc/gcp/handshaker.proto - ${WORKDIR}/grpc-proto/grpc/gcp/transport_security_common.proto ${WORKDIR}/grpc-proto/grpc/health/v1/health.proto ${WORKDIR}/grpc-proto/grpc/lb/v1/load_balancer.proto - ${WORKDIR}/grpc-proto/grpc/lookup/v1/rls.proto - ${WORKDIR}/grpc-proto/grpc/lookup/v1/rls_config.proto - ${WORKDIR}/grpc-proto/grpc/service_config/service_config.proto - ${WORKDIR}/grpc-proto/grpc/tls/provider/meshca/experimental/config.proto - ${WORKDIR}/istio/istio/google/security/meshca/v1/meshca.proto profiling/proto/service.proto - reflection/grpc_reflection_v1alpha/reflection.proto + ${WORKDIR}/grpc-proto/grpc/reflection/v1alpha/reflection.proto + ${WORKDIR}/grpc-proto/grpc/reflection/v1/reflection.proto ) # Generates only the new gRPC Service symbols SOURCES=( $(git ls-files --exclude-standard --cached --others "*.proto" | grep -v '^\(profiling/proto/service.proto\|reflection/grpc_reflection_v1alpha/reflection.proto\)$') + ${WORKDIR}/grpc-proto/grpc/gcp/altscontext.proto + ${WORKDIR}/grpc-proto/grpc/gcp/handshaker.proto + ${WORKDIR}/grpc-proto/grpc/gcp/transport_security_common.proto + ${WORKDIR}/grpc-proto/grpc/lookup/v1/rls.proto + ${WORKDIR}/grpc-proto/grpc/lookup/v1/rls_config.proto + ${WORKDIR}/grpc-proto/grpc/testing/*.proto + ${WORKDIR}/grpc-proto/grpc/core/*.proto ) # These options of the form 'Mfoo.proto=bar' instruct the codegen to use an # import path of 'bar' in the generated code when 'foo.proto' is imported in # one of the sources. -OPTS=Mgrpc/service_config/service_config.proto=/internal/proto/grpc_service_config,\ -Menvoy/config/core/v3/config_source.proto=github.com/envoyproxy/go-control-plane/envoy/config/core/v3 +# +# Note that the protos listed here are all for testing purposes. All protos to +# be used externally should have a go_package option (and they don't need to be +# listed here). +OPTS=Mgrpc/core/stats.proto=google.golang.org/grpc/interop/grpc_testing/core,\ +Mgrpc/testing/benchmark_service.proto=google.golang.org/grpc/interop/grpc_testing,\ +Mgrpc/testing/stats.proto=google.golang.org/grpc/interop/grpc_testing,\ +Mgrpc/testing/report_qps_scenario_service.proto=google.golang.org/grpc/interop/grpc_testing,\ +Mgrpc/testing/messages.proto=google.golang.org/grpc/interop/grpc_testing,\ +Mgrpc/testing/worker_service.proto=google.golang.org/grpc/interop/grpc_testing,\ +Mgrpc/testing/control.proto=google.golang.org/grpc/interop/grpc_testing,\ +Mgrpc/testing/test.proto=google.golang.org/grpc/interop/grpc_testing,\ +Mgrpc/testing/payloads.proto=google.golang.org/grpc/interop/grpc_testing,\ +Mgrpc/testing/empty.proto=google.golang.org/grpc/interop/grpc_testing for src in ${SOURCES[@]}; do echo "protoc ${src}" - protoc --go_out=${OPTS}:${WORKDIR}/out --go-grpc_out=${OPTS}:${WORKDIR}/out ${src} + protoc --go_out=${OPTS}:${WORKDIR}/out --go-grpc_out=${OPTS}:${WORKDIR}/out \ + -I"." \ + -I${WORKDIR}/grpc-proto \ + -I${WORKDIR}/googleapis \ + -I${WORKDIR}/protobuf/src \ + ${src} done for src in ${LEGACY_SOURCES[@]}; do echo "protoc ${src}" - protoc --go_out=${OPTS},plugins=grpc:${WORKDIR}/out --go-grpc_out=${OPTS},migration_mode=true:${WORKDIR}/out \ + protoc --go_out=${OPTS}:${WORKDIR}/out --go-grpc_out=${OPTS},require_unimplemented_servers=false:${WORKDIR}/out \ -I"." \ -I${WORKDIR}/grpc-proto \ -I${WORKDIR}/googleapis \ - -I${WORKDIR}/data-plane-api \ - -I${WORKDIR}/udpa \ - -I${WORKDIR}/protoc-gen-validate \ - -I${WORKDIR}/istio \ + -I${WORKDIR}/protobuf/src \ ${src} done # The go_package option in grpc/lookup/v1/rls.proto doesn't match the # current location. Move it into the right place. -mkdir -p ${WORKDIR}/out/google.golang.org/grpc/balancer/rls/internal/proto/grpc_lookup_v1 -mv ${WORKDIR}/out/google.golang.org/grpc/lookup/grpc_lookup_v1/* ${WORKDIR}/out/google.golang.org/grpc/balancer/rls/internal/proto/grpc_lookup_v1 - -# grpc_testingv3/testv3.pb.go is not re-generated because it was -# intentionally generated by an older version of protoc-gen-go. -rm ${WORKDIR}/out/google.golang.org/grpc/reflection/grpc_testingv3/*.pb.go - -# grpc/service_config/service_config.proto does not have a go_package option. -mv ${WORKDIR}/out/grpc/service_config/service_config.pb.go internal/proto/grpc_service_config +mkdir -p ${WORKDIR}/out/google.golang.org/grpc/internal/proto/grpc_lookup_v1 +mv ${WORKDIR}/out/google.golang.org/grpc/lookup/grpc_lookup_v1/* ${WORKDIR}/out/google.golang.org/grpc/internal/proto/grpc_lookup_v1 -# istio/google/security/meshca/v1/meshca.proto does not have a go_package option. -mkdir -p ${WORKDIR}/out/google.golang.org/grpc/credentials/tls/certprovider/meshca/internal/v1/ -mv ${WORKDIR}/out/istio/google/security/meshca/v1/* ${WORKDIR}/out/google.golang.org/grpc/credentials/tls/certprovider/meshca/internal/v1/ +# grpc_testing_not_regenerate/*.pb.go are not re-generated, +# see grpc_testing_not_regenerate/README.md for details. +rm ${WORKDIR}/out/google.golang.org/grpc/reflection/grpc_testing_not_regenerate/*.pb.go cp -R ${WORKDIR}/out/google.golang.org/grpc/* . diff --git a/resolver/manual/manual.go b/resolver/manual/manual.go index e141f96a62df..e6b0f14cd941 100644 --- a/resolver/manual/manual.go +++ b/resolver/manual/manual.go @@ -21,8 +21,7 @@ package manual import ( - "strconv" - "time" + "sync" "google.golang.org/grpc/resolver" ) @@ -30,21 +29,36 @@ import ( // NewBuilderWithScheme creates a new test resolver builder with the given scheme. func NewBuilderWithScheme(scheme string) *Resolver { return &Resolver{ - ResolveNowCallback: func(resolver.ResolveNowOptions) {}, - scheme: scheme, + BuildCallback: func(resolver.Target, resolver.ClientConn, resolver.BuildOptions) {}, + UpdateStateCallback: func(error) {}, + ResolveNowCallback: func(resolver.ResolveNowOptions) {}, + CloseCallback: func() {}, + scheme: scheme, } } // Resolver is also a resolver builder. // It's build() function always returns itself. type Resolver struct { + // BuildCallback is called when the Build method is called. Must not be + // nil. Must not be changed after the resolver may be built. + BuildCallback func(resolver.Target, resolver.ClientConn, resolver.BuildOptions) + // UpdateStateCallback is called when the UpdateState method is called on + // the resolver. The value passed as argument to this callback is the value + // returned by the resolver.ClientConn. Must not be nil. Must not be + // changed after the resolver may be built. + UpdateStateCallback func(err error) // ResolveNowCallback is called when the ResolveNow method is called on the // resolver. Must not be nil. Must not be changed after the resolver may // be built. ResolveNowCallback func(resolver.ResolveNowOptions) - scheme string + // CloseCallback is called when the Close method is called. Must not be + // nil. Must not be changed after the resolver may be built. + CloseCallback func() + scheme string // Fields actually belong to the resolver. + mu sync.Mutex // Guards access to CC. CC resolver.ClientConn bootstrapState *resolver.State } @@ -57,7 +71,10 @@ func (r *Resolver) InitialState(s resolver.State) { // Build returns itself for Resolver, because it's both a builder and a resolver. func (r *Resolver) Build(target resolver.Target, cc resolver.ClientConn, opts resolver.BuildOptions) (resolver.Resolver, error) { + r.mu.Lock() r.CC = cc + r.mu.Unlock() + r.BuildCallback(target, cc, opts) if r.bootstrapState != nil { r.UpdateState(*r.bootstrapState) } @@ -75,19 +92,21 @@ func (r *Resolver) ResolveNow(o resolver.ResolveNowOptions) { } // Close is a noop for Resolver. -func (*Resolver) Close() {} +func (r *Resolver) Close() { + r.CloseCallback() +} // UpdateState calls CC.UpdateState. func (r *Resolver) UpdateState(s resolver.State) { - r.CC.UpdateState(s) + r.mu.Lock() + err := r.CC.UpdateState(s) + r.mu.Unlock() + r.UpdateStateCallback(err) } -// GenerateAndRegisterManualResolver generates a random scheme and a Resolver -// with it. It also registers this Resolver. -// It returns the Resolver and a cleanup function to unregister it. -func GenerateAndRegisterManualResolver() (*Resolver, func()) { - scheme := strconv.FormatInt(time.Now().UnixNano(), 36) - r := NewBuilderWithScheme(scheme) - resolver.Register(r) - return r, func() { resolver.UnregisterForTesting(scheme) } +// ReportError calls CC.ReportError. +func (r *Resolver) ReportError(err error) { + r.mu.Lock() + r.CC.ReportError(err) + r.mu.Unlock() } diff --git a/resolver/map.go b/resolver/map.go new file mode 100644 index 000000000000..804be887de0a --- /dev/null +++ b/resolver/map.go @@ -0,0 +1,138 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package resolver + +type addressMapEntry struct { + addr Address + value any +} + +// AddressMap is a map of addresses to arbitrary values taking into account +// Attributes. BalancerAttributes are ignored, as are Metadata and Type. +// Multiple accesses may not be performed concurrently. Must be created via +// NewAddressMap; do not construct directly. +type AddressMap struct { + // The underlying map is keyed by an Address with fields that we don't care + // about being set to their zero values. The only fields that we care about + // are `Addr`, `ServerName` and `Attributes`. Since we need to be able to + // distinguish between addresses with same `Addr` and `ServerName`, but + // different `Attributes`, we cannot store the `Attributes` in the map key. + // + // The comparison operation for structs work as follows: + // Struct values are comparable if all their fields are comparable. Two + // struct values are equal if their corresponding non-blank fields are equal. + // + // The value type of the map contains a slice of addresses which match the key + // in their `Addr` and `ServerName` fields and contain the corresponding value + // associated with them. + m map[Address]addressMapEntryList +} + +func toMapKey(addr *Address) Address { + return Address{Addr: addr.Addr, ServerName: addr.ServerName} +} + +type addressMapEntryList []*addressMapEntry + +// NewAddressMap creates a new AddressMap. +func NewAddressMap() *AddressMap { + return &AddressMap{m: make(map[Address]addressMapEntryList)} +} + +// find returns the index of addr in the addressMapEntry slice, or -1 if not +// present. +func (l addressMapEntryList) find(addr Address) int { + for i, entry := range l { + // Attributes are the only thing to match on here, since `Addr` and + // `ServerName` are already equal. + if entry.addr.Attributes.Equal(addr.Attributes) { + return i + } + } + return -1 +} + +// Get returns the value for the address in the map, if present. +func (a *AddressMap) Get(addr Address) (value any, ok bool) { + addrKey := toMapKey(&addr) + entryList := a.m[addrKey] + if entry := entryList.find(addr); entry != -1 { + return entryList[entry].value, true + } + return nil, false +} + +// Set updates or adds the value to the address in the map. +func (a *AddressMap) Set(addr Address, value any) { + addrKey := toMapKey(&addr) + entryList := a.m[addrKey] + if entry := entryList.find(addr); entry != -1 { + entryList[entry].value = value + return + } + a.m[addrKey] = append(entryList, &addressMapEntry{addr: addr, value: value}) +} + +// Delete removes addr from the map. +func (a *AddressMap) Delete(addr Address) { + addrKey := toMapKey(&addr) + entryList := a.m[addrKey] + entry := entryList.find(addr) + if entry == -1 { + return + } + if len(entryList) == 1 { + entryList = nil + } else { + copy(entryList[entry:], entryList[entry+1:]) + entryList = entryList[:len(entryList)-1] + } + a.m[addrKey] = entryList +} + +// Len returns the number of entries in the map. +func (a *AddressMap) Len() int { + ret := 0 + for _, entryList := range a.m { + ret += len(entryList) + } + return ret +} + +// Keys returns a slice of all current map keys. +func (a *AddressMap) Keys() []Address { + ret := make([]Address, 0, a.Len()) + for _, entryList := range a.m { + for _, entry := range entryList { + ret = append(ret, entry.addr) + } + } + return ret +} + +// Values returns a slice of all current map values. +func (a *AddressMap) Values() []any { + ret := make([]any, 0, a.Len()) + for _, entryList := range a.m { + for _, entry := range entryList { + ret = append(ret, entry.value) + } + } + return ret +} diff --git a/resolver/map_test.go b/resolver/map_test.go new file mode 100644 index 000000000000..0b0ac1667902 --- /dev/null +++ b/resolver/map_test.go @@ -0,0 +1,163 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package resolver + +import ( + "fmt" + "sort" + "testing" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/attributes" +) + +// Note: each address is different from addr1 by one value. addr7 matches +// addr1, since the only difference is BalancerAttributes, which are not +// compared. +var ( + addr1 = Address{Addr: "a1", Attributes: attributes.New("a1", 3), ServerName: "s1"} + addr2 = Address{Addr: "a2", Attributes: attributes.New("a1", 3), ServerName: "s1"} + addr3 = Address{Addr: "a1", Attributes: attributes.New("a2", 3), ServerName: "s1"} + addr4 = Address{Addr: "a1", Attributes: attributes.New("a1", 2), ServerName: "s1"} + addr5 = Address{Addr: "a1", Attributes: attributes.New("a1", "3"), ServerName: "s1"} + addr6 = Address{Addr: "a1", Attributes: attributes.New("a1", 3), ServerName: "s2"} + addr7 = Address{Addr: "a1", Attributes: attributes.New("a1", 3), ServerName: "s1", BalancerAttributes: attributes.New("xx", 3)} +) + +func (s) TestAddressMap_Length(t *testing.T) { + addrMap := NewAddressMap() + if got := addrMap.Len(); got != 0 { + t.Fatalf("addrMap.Len() = %v; want 0", got) + } + for i := 0; i < 10; i++ { + addrMap.Set(addr1, nil) + if got, want := addrMap.Len(), 1; got != want { + t.Fatalf("addrMap.Len() = %v; want %v", got, want) + } + addrMap.Set(addr7, nil) // aliases addr1 + } + for i := 0; i < 10; i++ { + addrMap.Set(addr2, nil) + if got, want := addrMap.Len(), 2; got != want { + t.Fatalf("addrMap.Len() = %v; want %v", got, want) + } + } +} + +func (s) TestAddressMap_Get(t *testing.T) { + addrMap := NewAddressMap() + addrMap.Set(addr1, 1) + + if got, ok := addrMap.Get(addr2); ok || got != nil { + t.Fatalf("addrMap.Get(addr1) = %v, %v; want nil, false", got, ok) + } + + addrMap.Set(addr2, 2) + addrMap.Set(addr3, 3) + addrMap.Set(addr4, 4) + addrMap.Set(addr5, 5) + addrMap.Set(addr6, 6) + addrMap.Set(addr7, 7) // aliases addr1 + if got, ok := addrMap.Get(addr1); !ok || got.(int) != 7 { + t.Fatalf("addrMap.Get(addr1) = %v, %v; want %v, true", got, ok, 7) + } + if got, ok := addrMap.Get(addr2); !ok || got.(int) != 2 { + t.Fatalf("addrMap.Get(addr2) = %v, %v; want %v, true", got, ok, 2) + } + if got, ok := addrMap.Get(addr3); !ok || got.(int) != 3 { + t.Fatalf("addrMap.Get(addr3) = %v, %v; want %v, true", got, ok, 3) + } + if got, ok := addrMap.Get(addr4); !ok || got.(int) != 4 { + t.Fatalf("addrMap.Get(addr4) = %v, %v; want %v, true", got, ok, 4) + } + if got, ok := addrMap.Get(addr5); !ok || got.(int) != 5 { + t.Fatalf("addrMap.Get(addr5) = %v, %v; want %v, true", got, ok, 5) + } + if got, ok := addrMap.Get(addr6); !ok || got.(int) != 6 { + t.Fatalf("addrMap.Get(addr6) = %v, %v; want %v, true", got, ok, 6) + } + if got, ok := addrMap.Get(addr7); !ok || got.(int) != 7 { + t.Fatalf("addrMap.Get(addr7) = %v, %v; want %v, true", got, ok, 7) + } +} + +func (s) TestAddressMap_Delete(t *testing.T) { + addrMap := NewAddressMap() + addrMap.Set(addr1, 1) + addrMap.Set(addr2, 2) + if got, want := addrMap.Len(), 2; got != want { + t.Fatalf("addrMap.Len() = %v; want %v", got, want) + } + addrMap.Delete(addr3) + addrMap.Delete(addr4) + addrMap.Delete(addr5) + addrMap.Delete(addr6) + addrMap.Delete(addr7) // aliases addr1 + if got, ok := addrMap.Get(addr1); ok || got != nil { + t.Fatalf("addrMap.Get(addr1) = %v, %v; want nil, false", got, ok) + } + if got, ok := addrMap.Get(addr7); ok || got != nil { + t.Fatalf("addrMap.Get(addr7) = %v, %v; want nil, false", got, ok) + } + if got, ok := addrMap.Get(addr2); !ok || got.(int) != 2 { + t.Fatalf("addrMap.Get(addr2) = %v, %v; want %v, true", got, ok, 2) + } +} + +func (s) TestAddressMap_Keys(t *testing.T) { + addrMap := NewAddressMap() + addrMap.Set(addr1, 1) + addrMap.Set(addr2, 2) + addrMap.Set(addr3, 3) + addrMap.Set(addr4, 4) + addrMap.Set(addr5, 5) + addrMap.Set(addr6, 6) + addrMap.Set(addr7, 7) // aliases addr1 + + want := []Address{addr1, addr2, addr3, addr4, addr5, addr6} + got := addrMap.Keys() + if d := cmp.Diff(want, got, cmp.Transformer("sort", func(in []Address) []Address { + out := append([]Address(nil), in...) + sort.Slice(out, func(i, j int) bool { return fmt.Sprint(out[i]) < fmt.Sprint(out[j]) }) + return out + })); d != "" { + t.Fatalf("addrMap.Keys returned unexpected elements (-want, +got):\n%v", d) + } +} + +func (s) TestAddressMap_Values(t *testing.T) { + addrMap := NewAddressMap() + addrMap.Set(addr1, 1) + addrMap.Set(addr2, 2) + addrMap.Set(addr3, 3) + addrMap.Set(addr4, 4) + addrMap.Set(addr5, 5) + addrMap.Set(addr6, 6) + addrMap.Set(addr7, 7) // aliases addr1 + + want := []int{2, 3, 4, 5, 6, 7} + var got []int + for _, v := range addrMap.Values() { + got = append(got, v.(int)) + } + sort.Ints(got) + if diff := cmp.Diff(want, got); diff != "" { + t.Fatalf("addrMap.Values returned unexpected elements (-want, +got):\n%v", diff) + } +} diff --git a/resolver/resolver.go b/resolver/resolver.go index 379275a2d9b4..11384e228e54 100644 --- a/resolver/resolver.go +++ b/resolver/resolver.go @@ -22,7 +22,10 @@ package resolver import ( "context" + "fmt" "net" + "net/url" + "strings" "google.golang.org/grpc/attributes" "google.golang.org/grpc/credentials" @@ -38,8 +41,9 @@ var ( // TODO(bar) install dns resolver in init(){}. -// Register registers the resolver builder to the resolver map. b.Scheme will be -// used as the scheme registered with this builder. +// Register registers the resolver builder to the resolver map. b.Scheme will +// be used as the scheme registered with this builder. The registry is case +// sensitive, and schemes should not contain any uppercase characters. // // NOTE: this function must only be called during initialization time (i.e. in // an init() function), and is not thread-safe. If multiple Resolvers are @@ -73,27 +77,12 @@ func GetDefaultScheme() string { return defaultScheme } -// AddressType indicates the address type returned by name resolution. -// -// Deprecated: use Attributes in Address instead. -type AddressType uint8 - -const ( - // Backend indicates the address is for a backend server. - // - // Deprecated: use Attributes in Address instead. - Backend AddressType = iota - // GRPCLB indicates the address is for a grpclb load balancer. - // - // Deprecated: to select the GRPCLB load balancing policy, use a service - // config with a corresponding loadBalancingConfig. To supply balancer - // addresses to the GRPCLB load balancing policy, set State.Attributes - // using balancer/grpclb/state.Set. - GRPCLB -) - // Address represents a server the client connects to. -// This is the EXPERIMENTAL API and may be changed or extended in the future. +// +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. type Address struct { // Addr is the server address on which a connection will be established. Addr string @@ -103,28 +92,56 @@ type Address struct { // the address, instead of the hostname from the Dial target string. In most cases, // this should not be set. // - // If Type is GRPCLB, ServerName should be the name of the remote load - // balancer, not the name of the backend. - // // WARNING: ServerName must only be populated with trusted values. It // is insecure to populate it with data from untrusted inputs since untrusted // values could be used to bypass the authority checks performed by TLS. ServerName string // Attributes contains arbitrary data about this address intended for - // consumption by the load balancing policy. + // consumption by the SubConn. Attributes *attributes.Attributes - // Type is the type of this address. + // BalancerAttributes contains arbitrary data about this address intended + // for consumption by the LB policy. These attributes do not affect SubConn + // creation, connection establishment, handshaking, etc. // - // Deprecated: use Attributes instead. - Type AddressType + // Deprecated: when an Address is inside an Endpoint, this field should not + // be used, and it will eventually be removed entirely. + BalancerAttributes *attributes.Attributes // Metadata is the information associated with Addr, which may be used // to make load balancing decision. // // Deprecated: use Attributes instead. - Metadata interface{} + Metadata any +} + +// Equal returns whether a and o are identical. Metadata is compared directly, +// not with any recursive introspection. +// +// This method compares all fields of the address. When used to tell apart +// addresses during subchannel creation or connection establishment, it might be +// more appropriate for the caller to implement custom equality logic. +func (a Address) Equal(o Address) bool { + return a.Addr == o.Addr && a.ServerName == o.ServerName && + a.Attributes.Equal(o.Attributes) && + a.BalancerAttributes.Equal(o.BalancerAttributes) && + a.Metadata == o.Metadata +} + +// String returns JSON formatted string representation of the address. +func (a Address) String() string { + var sb strings.Builder + sb.WriteString(fmt.Sprintf("{Addr: %q, ", a.Addr)) + sb.WriteString(fmt.Sprintf("ServerName: %q, ", a.ServerName)) + if a.Attributes != nil { + sb.WriteString(fmt.Sprintf("Attributes: %v, ", a.Attributes.String())) + } + if a.BalancerAttributes != nil { + sb.WriteString(fmt.Sprintf("BalancerAttributes: %v", a.BalancerAttributes.String())) + } + sb.WriteString("}") + return sb.String() } // BuildOptions includes additional information for the builder to create @@ -153,11 +170,37 @@ type BuildOptions struct { Dialer func(context.Context, string) (net.Conn, error) } +// An Endpoint is one network endpoint, or server, which may have multiple +// addresses with which it can be accessed. +type Endpoint struct { + // Addresses contains a list of addresses used to access this endpoint. + Addresses []Address + + // Attributes contains arbitrary data about this endpoint intended for + // consumption by the LB policy. + Attributes *attributes.Attributes +} + // State contains the current Resolver state relevant to the ClientConn. type State struct { // Addresses is the latest set of resolved addresses for the target. + // + // If a resolver sets Addresses but does not set Endpoints, one Endpoint + // will be created for each Address before the State is passed to the LB + // policy. The BalancerAttributes of each entry in Addresses will be set + // in Endpoints.Attributes, and be cleared in the Endpoint's Address's + // BalancerAttributes. + // + // Soon, Addresses will be deprecated and replaced fully by Endpoints. Addresses []Address + // Endpoints is the latest set of resolved endpoints for the target. + // + // If a resolver produces a State containing Endpoints but not Addresses, + // it must take care to ensure the LB policies it selects will support + // Endpoints. + Endpoints []Endpoint + // ServiceConfig contains the result from parsing the latest service // config. If it is nil, it indicates no service config is present or the // resolver does not provide service configs. @@ -177,7 +220,16 @@ type State struct { // gRPC to add new methods to this interface. type ClientConn interface { // UpdateState updates the state of the ClientConn appropriately. - UpdateState(State) + // + // If an error is returned, the resolver should try to resolve the + // target again. The resolver should use a backoff timer to prevent + // overloading the server with requests. If a resolver is certain that + // reresolving will not change the result, e.g. because it is + // a watch-based resolver, returned errors can be ignored. + // + // If the resolved State is the same as the last reported one, calling + // UpdateState can be omitted. + UpdateState(State) error // ReportError notifies the ClientConn that the Resolver encountered an // error. The ClientConn will notify the load balancer and begin calling // ResolveNow on the Resolver with exponential backoff. @@ -200,25 +252,38 @@ type ClientConn interface { // Target represents a target for gRPC, as specified in: // https://github.com/grpc/grpc/blob/master/doc/naming.md. -// It is parsed from the target string that gets passed into Dial or DialContext by the user. And -// grpc passes it to the resolver and the balancer. -// -// If the target follows the naming spec, and the parsed scheme is registered with grpc, we will -// parse the target string according to the spec. e.g. "dns://some_authority/foo.bar" will be parsed -// into &Target{Scheme: "dns", Authority: "some_authority", Endpoint: "foo.bar"} +// It is parsed from the target string that gets passed into Dial or DialContext +// by the user. And gRPC passes it to the resolver and the balancer. // -// If the target does not contain a scheme, we will apply the default scheme, and set the Target to -// be the full target string. e.g. "foo.bar" will be parsed into -// &Target{Scheme: resolver.GetDefaultScheme(), Endpoint: "foo.bar"}. -// -// If the parsed scheme is not registered (i.e. no corresponding resolver available to resolve the -// endpoint), we set the Scheme to be the default scheme, and set the Endpoint to be the full target -// string. e.g. target string "unknown_scheme://authority/endpoint" will be parsed into -// &Target{Scheme: resolver.GetDefaultScheme(), Endpoint: "unknown_scheme://authority/endpoint"}. +// If the target follows the naming spec, and the parsed scheme is registered +// with gRPC, we will parse the target string according to the spec. If the +// target does not contain a scheme or if the parsed scheme is not registered +// (i.e. no corresponding resolver available to resolve the endpoint), we will +// apply the default scheme, and will attempt to reparse it. type Target struct { - Scheme string - Authority string - Endpoint string + // URL contains the parsed dial target with an optional default scheme added + // to it if the original dial target contained no scheme or contained an + // unregistered scheme. Any query params specified in the original dial + // target can be accessed from here. + URL url.URL +} + +// Endpoint retrieves endpoint without leading "/" from either `URL.Path` +// or `URL.Opaque`. The latter is used when the former is empty. +func (t Target) Endpoint() string { + endpoint := t.URL.Path + if endpoint == "" { + endpoint = t.URL.Opaque + } + // For targets of the form "[scheme]://[authority]/endpoint, the endpoint + // value returned from url.Parse() contains a leading "/". Although this is + // in accordance with RFC 3986, we do not want to break existing resolver + // implementations which expect the endpoint without the leading "/". So, we + // end up stripping the leading "/" here. But this will result in an + // incorrect parsing for something like "unix:///path/to/socket". Since we + // own the "unix" resolver, we can workaround in the unix resolver by using + // the `URL` field. + return strings.TrimPrefix(endpoint, "/") } // Builder creates a resolver that will be used to watch name resolution updates. @@ -228,8 +293,10 @@ type Builder interface { // gRPC dial calls Build synchronously, and fails if the returned error is // not nil. Build(target Target, cc ClientConn, opts BuildOptions) (Resolver, error) - // Scheme returns the scheme supported by this resolver. - // Scheme is defined at https://github.com/grpc/grpc/blob/master/doc/naming.md. + // Scheme returns the scheme supported by this resolver. Scheme is defined + // at https://github.com/grpc/grpc/blob/master/doc/naming.md. The returned + // string should not contain uppercase characters, as they will not match + // the parsed target's scheme as defined in RFC 3986. Scheme() string } @@ -247,10 +314,3 @@ type Resolver interface { // Close closes the resolver. Close() } - -// UnregisterForTesting removes the resolver builder with the given scheme from the -// resolver map. -// This function is for testing only. -func UnregisterForTesting(scheme string) { - delete(m, scheme) -} diff --git a/resolver/resolver_test.go b/resolver/resolver_test.go new file mode 100644 index 000000000000..8d061f9b66d2 --- /dev/null +++ b/resolver/resolver_test.go @@ -0,0 +1,33 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package resolver + +import ( + "testing" + + "google.golang.org/grpc/internal/grpctest" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} diff --git a/resolver_conn_wrapper.go b/resolver_conn_wrapper.go index 265002a75e00..d68330560848 100644 --- a/resolver_conn_wrapper.go +++ b/resolver_conn_wrapper.go @@ -19,184 +19,212 @@ package grpc import ( - "fmt" + "context" "strings" "sync" - "time" "google.golang.org/grpc/balancer" - "google.golang.org/grpc/credentials" "google.golang.org/grpc/internal/channelz" "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/pretty" "google.golang.org/grpc/resolver" "google.golang.org/grpc/serviceconfig" ) +// resolverStateUpdater wraps the single method used by ccResolverWrapper to +// report a state update from the actual resolver implementation. +type resolverStateUpdater interface { + updateResolverState(s resolver.State, err error) error +} + // ccResolverWrapper is a wrapper on top of cc for resolvers. // It implements resolver.ClientConn interface. type ccResolverWrapper struct { - cc *ClientConn - resolverMu sync.Mutex - resolver resolver.Resolver - done *grpcsync.Event - curState resolver.State - - pollingMu sync.Mutex - polling chan struct{} + // The following fields are initialized when the wrapper is created and are + // read-only afterwards, and therefore can be accessed without a mutex. + cc resolverStateUpdater + channelzID *channelz.Identifier + ignoreServiceConfig bool + opts ccResolverWrapperOpts + serializer *grpcsync.CallbackSerializer // To serialize all incoming calls. + serializerCancel context.CancelFunc // To close the serializer, accessed only from close(). + + // All incoming (resolver --> gRPC) calls are guaranteed to execute in a + // mutually exclusive manner as they are scheduled on the serializer. + // Fields accessed *only* in these serializer callbacks, can therefore be + // accessed without a mutex. + curState resolver.State + + // mu guards access to the below fields. + mu sync.Mutex + closed bool + resolver resolver.Resolver // Accessed only from outgoing calls. +} + +// ccResolverWrapperOpts wraps the arguments to be passed when creating a new +// ccResolverWrapper. +type ccResolverWrapperOpts struct { + target resolver.Target // User specified dial target to resolve. + builder resolver.Builder // Resolver builder to use. + bOpts resolver.BuildOptions // Resolver build options to use. + channelzID *channelz.Identifier // Channelz identifier for the channel. } // newCCResolverWrapper uses the resolver.Builder to build a Resolver and // returns a ccResolverWrapper object which wraps the newly built resolver. -func newCCResolverWrapper(cc *ClientConn, rb resolver.Builder) (*ccResolverWrapper, error) { +func newCCResolverWrapper(cc resolverStateUpdater, opts ccResolverWrapperOpts) (*ccResolverWrapper, error) { + ctx, cancel := context.WithCancel(context.Background()) ccr := &ccResolverWrapper{ - cc: cc, - done: grpcsync.NewEvent(), - } - - var credsClone credentials.TransportCredentials - if creds := cc.dopts.copts.TransportCredentials; creds != nil { - credsClone = creds.Clone() - } - rbo := resolver.BuildOptions{ - DisableServiceConfig: cc.dopts.disableServiceConfig, - DialCreds: credsClone, - CredsBundle: cc.dopts.copts.CredsBundle, - Dialer: cc.dopts.copts.Dialer, - } - - var err error - // We need to hold the lock here while we assign to the ccr.resolver field - // to guard against a data race caused by the following code path, - // rb.Build-->ccr.ReportError-->ccr.poll-->ccr.resolveNow, would end up - // accessing ccr.resolver which is being assigned here. - ccr.resolverMu.Lock() - defer ccr.resolverMu.Unlock() - ccr.resolver, err = rb.Build(cc.parsedTarget, ccr, rbo) + cc: cc, + channelzID: opts.channelzID, + ignoreServiceConfig: opts.bOpts.DisableServiceConfig, + opts: opts, + serializer: grpcsync.NewCallbackSerializer(ctx), + serializerCancel: cancel, + } + + // Cannot hold the lock at build time because the resolver can send an + // update or error inline and these incoming calls grab the lock to schedule + // a callback in the serializer. + r, err := opts.builder.Build(opts.target, ccr, opts.bOpts) if err != nil { + cancel() return nil, err } + + // Any error reported by the resolver at build time that leads to a + // re-resolution request from the balancer is dropped by grpc until we + // return from this function. So, we don't have to handle pending resolveNow + // requests here. + ccr.mu.Lock() + ccr.resolver = r + ccr.mu.Unlock() + return ccr, nil } func (ccr *ccResolverWrapper) resolveNow(o resolver.ResolveNowOptions) { - ccr.resolverMu.Lock() - if !ccr.done.HasFired() { - ccr.resolver.ResolveNow(o) + ccr.mu.Lock() + defer ccr.mu.Unlock() + + // ccr.resolver field is set only after the call to Build() returns. But in + // the process of building, the resolver may send an error update which when + // propagated to the balancer may result in a re-resolution request. + if ccr.closed || ccr.resolver == nil { + return } - ccr.resolverMu.Unlock() + ccr.resolver.ResolveNow(o) } func (ccr *ccResolverWrapper) close() { - ccr.resolverMu.Lock() - ccr.resolver.Close() - ccr.done.Fire() - ccr.resolverMu.Unlock() -} - -// poll begins or ends asynchronous polling of the resolver based on whether -// err is ErrBadResolverState. -func (ccr *ccResolverWrapper) poll(err error) { - ccr.pollingMu.Lock() - defer ccr.pollingMu.Unlock() - if err != balancer.ErrBadResolverState { - // stop polling - if ccr.polling != nil { - close(ccr.polling) - ccr.polling = nil - } - return - } - if ccr.polling != nil { - // already polling + ccr.mu.Lock() + if ccr.closed { + ccr.mu.Unlock() return } - p := make(chan struct{}) - ccr.polling = p - go func() { - for i := 0; ; i++ { - ccr.resolveNow(resolver.ResolveNowOptions{}) - t := time.NewTimer(ccr.cc.dopts.resolveNowBackoff(i)) - select { - case <-p: - t.Stop() - return - case <-ccr.done.Done(): - // Resolver has been closed. - t.Stop() - return - case <-t.C: - select { - case <-p: - return - default: - } - // Timer expired; re-resolve. - } - } - }() + + channelz.Info(logger, ccr.channelzID, "Closing the name resolver") + + // Close the serializer to ensure that no more calls from the resolver are + // handled, before actually closing the resolver. + ccr.serializerCancel() + ccr.closed = true + r := ccr.resolver + ccr.mu.Unlock() + + // Give enqueued callbacks a chance to finish. + <-ccr.serializer.Done() + + // Spawn a goroutine to close the resolver (since it may block trying to + // cleanup all allocated resources) and return early. + go r.Close() } -func (ccr *ccResolverWrapper) UpdateState(s resolver.State) { - if ccr.done.HasFired() { - return +// serializerScheduleLocked is a convenience method to schedule a function to be +// run on the serializer while holding ccr.mu. +func (ccr *ccResolverWrapper) serializerScheduleLocked(f func(context.Context)) { + ccr.mu.Lock() + ccr.serializer.Schedule(f) + ccr.mu.Unlock() +} + +// UpdateState is called by resolver implementations to report new state to gRPC +// which includes addresses and service config. +func (ccr *ccResolverWrapper) UpdateState(s resolver.State) error { + errCh := make(chan error, 1) + if s.Endpoints == nil { + s.Endpoints = make([]resolver.Endpoint, 0, len(s.Addresses)) + for _, a := range s.Addresses { + ep := resolver.Endpoint{Addresses: []resolver.Address{a}, Attributes: a.BalancerAttributes} + ep.Addresses[0].BalancerAttributes = nil + s.Endpoints = append(s.Endpoints, ep) + } } - channelz.Infof(logger, ccr.cc.channelzID, "ccResolverWrapper: sending update to cc: %v", s) - if channelz.IsOn() { + ok := ccr.serializer.Schedule(func(context.Context) { ccr.addChannelzTraceEvent(s) + ccr.curState = s + if err := ccr.cc.updateResolverState(ccr.curState, nil); err == balancer.ErrBadResolverState { + errCh <- balancer.ErrBadResolverState + return + } + errCh <- nil + }) + if !ok { + // The only time when Schedule() fail to add the callback to the + // serializer is when the serializer is closed, and this happens only + // when the resolver wrapper is closed. + return nil } - ccr.curState = s - ccr.poll(ccr.cc.updateResolverState(ccr.curState, nil)) + return <-errCh } +// ReportError is called by resolver implementations to report errors +// encountered during name resolution to gRPC. func (ccr *ccResolverWrapper) ReportError(err error) { - if ccr.done.HasFired() { - return - } - channelz.Warningf(logger, ccr.cc.channelzID, "ccResolverWrapper: reporting error to cc: %v", err) - ccr.poll(ccr.cc.updateResolverState(resolver.State{}, err)) + ccr.serializerScheduleLocked(func(_ context.Context) { + channelz.Warningf(logger, ccr.channelzID, "ccResolverWrapper: reporting error to cc: %v", err) + ccr.cc.updateResolverState(resolver.State{}, err) + }) } -// NewAddress is called by the resolver implementation to send addresses to gRPC. +// NewAddress is called by the resolver implementation to send addresses to +// gRPC. func (ccr *ccResolverWrapper) NewAddress(addrs []resolver.Address) { - if ccr.done.HasFired() { - return - } - channelz.Infof(logger, ccr.cc.channelzID, "ccResolverWrapper: sending new addresses to cc: %v", addrs) - if channelz.IsOn() { + ccr.serializerScheduleLocked(func(_ context.Context) { ccr.addChannelzTraceEvent(resolver.State{Addresses: addrs, ServiceConfig: ccr.curState.ServiceConfig}) - } - ccr.curState.Addresses = addrs - ccr.poll(ccr.cc.updateResolverState(ccr.curState, nil)) + ccr.curState.Addresses = addrs + ccr.cc.updateResolverState(ccr.curState, nil) + }) } // NewServiceConfig is called by the resolver implementation to send service // configs to gRPC. func (ccr *ccResolverWrapper) NewServiceConfig(sc string) { - if ccr.done.HasFired() { - return - } - channelz.Infof(logger, ccr.cc.channelzID, "ccResolverWrapper: got new service config: %v", sc) - if ccr.cc.dopts.disableServiceConfig { - channelz.Info(logger, ccr.cc.channelzID, "Service config lookups disabled; ignoring config") - return - } - scpr := parseServiceConfig(sc) - if scpr.Err != nil { - channelz.Warningf(logger, ccr.cc.channelzID, "ccResolverWrapper: error parsing service config: %v", scpr.Err) - ccr.poll(balancer.ErrBadResolverState) - return - } - if channelz.IsOn() { + ccr.serializerScheduleLocked(func(_ context.Context) { + channelz.Infof(logger, ccr.channelzID, "ccResolverWrapper: got new service config: %s", sc) + if ccr.ignoreServiceConfig { + channelz.Info(logger, ccr.channelzID, "Service config lookups disabled; ignoring config") + return + } + scpr := parseServiceConfig(sc) + if scpr.Err != nil { + channelz.Warningf(logger, ccr.channelzID, "ccResolverWrapper: error parsing service config: %v", scpr.Err) + return + } ccr.addChannelzTraceEvent(resolver.State{Addresses: ccr.curState.Addresses, ServiceConfig: scpr}) - } - ccr.curState.ServiceConfig = scpr - ccr.poll(ccr.cc.updateResolverState(ccr.curState, nil)) + ccr.curState.ServiceConfig = scpr + ccr.cc.updateResolverState(ccr.curState, nil) + }) } +// ParseServiceConfig is called by resolver implementations to parse a JSON +// representation of the service config. func (ccr *ccResolverWrapper) ParseServiceConfig(scJSON string) *serviceconfig.ParseResult { return parseServiceConfig(scJSON) } +// addChannelzTraceEvent adds a channelz trace event containing the new +// state received from resolver implementations. func (ccr *ccResolverWrapper) addChannelzTraceEvent(s resolver.State) { var updates []string var oldSC, newSC *ServiceConfig @@ -215,8 +243,5 @@ func (ccr *ccResolverWrapper) addChannelzTraceEvent(s resolver.State) { } else if len(ccr.curState.Addresses) == 0 && len(s.Addresses) > 0 { updates = append(updates, "resolver returned new addresses") } - channelz.AddTraceEvent(logger, ccr.cc.channelzID, 0, &channelz.TraceEventDesc{ - Desc: fmt.Sprintf("Resolver state updated: %+v (%v)", s, strings.Join(updates, "; ")), - Severity: channelz.CtINFO, - }) + channelz.Infof(logger, ccr.channelzID, "Resolver state updated: %s (%v)", pretty.ToJSON(s), strings.Join(updates, "; ")) } diff --git a/resolver_conn_wrapper_test.go b/resolver_conn_wrapper_test.go deleted file mode 100644 index e125976a5359..000000000000 --- a/resolver_conn_wrapper_test.go +++ /dev/null @@ -1,214 +0,0 @@ -/* - * - * Copyright 2017 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package grpc - -import ( - "context" - "errors" - "fmt" - "net" - "strings" - "testing" - "time" - - "google.golang.org/grpc/balancer" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/internal/balancer/stub" - "google.golang.org/grpc/resolver" - "google.golang.org/grpc/resolver/manual" - "google.golang.org/grpc/serviceconfig" - "google.golang.org/grpc/status" -) - -// The target string with unknown scheme should be kept unchanged and passed to -// the dialer. -func (s) TestDialParseTargetUnknownScheme(t *testing.T) { - for _, test := range []struct { - targetStr string - want string - }{ - {"/unix/socket/address", "/unix/socket/address"}, - - // Special test for "unix:///". - {"unix:///unix/socket/address", "unix:///unix/socket/address"}, - - // For known scheme. - {"passthrough://a.server.com/google.com", "google.com"}, - } { - dialStrCh := make(chan string, 1) - cc, err := Dial(test.targetStr, WithInsecure(), WithDialer(func(addr string, _ time.Duration) (net.Conn, error) { - select { - case dialStrCh <- addr: - default: - } - return nil, fmt.Errorf("test dialer, always error") - })) - if err != nil { - t.Fatalf("Failed to create ClientConn: %v", err) - } - got := <-dialStrCh - cc.Close() - if got != test.want { - t.Errorf("Dial(%q), dialer got %q, want %q", test.targetStr, got, test.want) - } - } -} - -func testResolverErrorPolling(t *testing.T, badUpdate func(*manual.Resolver), goodUpdate func(*manual.Resolver), dopts ...DialOption) { - boIter := make(chan int) - resolverBackoff := func(v int) time.Duration { - boIter <- v - return 0 - } - - r := manual.NewBuilderWithScheme("whatever") - rn := make(chan struct{}) - defer func() { close(rn) }() - r.ResolveNowCallback = func(resolver.ResolveNowOptions) { rn <- struct{}{} } - - defaultDialOptions := []DialOption{ - WithInsecure(), - WithResolvers(r), - withResolveNowBackoff(resolverBackoff), - } - cc, err := Dial(r.Scheme()+":///test.server", append(defaultDialOptions, dopts...)...) - if err != nil { - t.Fatalf("Dial(_, _) = _, %v; want _, nil", err) - } - defer cc.Close() - badUpdate(r) - - panicAfter := time.AfterFunc(5*time.Second, func() { panic("timed out polling resolver") }) - defer panicAfter.Stop() - - // Ensure ResolveNow is called, then Backoff with the right parameter, several times - for i := 0; i < 7; i++ { - <-rn - if v := <-boIter; v != i { - t.Errorf("Backoff call %v uses value %v", i, v) - } - } - - // UpdateState will block if ResolveNow is being called (which blocks on - // rn), so call it in a goroutine. - goodUpdate(r) - - // Wait awhile to ensure ResolveNow and Backoff stop being called when the - // state is OK (i.e. polling was cancelled). - for { - t := time.NewTimer(50 * time.Millisecond) - select { - case <-rn: - // ClientConn is still calling ResolveNow - <-boIter - time.Sleep(5 * time.Millisecond) - continue - case <-t.C: - // ClientConn stopped calling ResolveNow; success - } - break - } -} - -const happyBalancerName = "happy balancer" - -func init() { - // Register a balancer that never returns an error from - // UpdateClientConnState, and doesn't do anything else either. - bf := stub.BalancerFuncs{ - UpdateClientConnState: func(*stub.BalancerData, balancer.ClientConnState) error { - return nil - }, - } - stub.Register(happyBalancerName, bf) -} - -// TestResolverErrorPolling injects resolver errors and verifies ResolveNow is -// called with the appropriate backoff strategy being consulted between -// ResolveNow calls. -func (s) TestResolverErrorPolling(t *testing.T) { - testResolverErrorPolling(t, func(r *manual.Resolver) { - r.CC.ReportError(errors.New("res err")) - }, func(r *manual.Resolver) { - // UpdateState will block if ResolveNow is being called (which blocks on - // rn), so call it in a goroutine. - go r.CC.UpdateState(resolver.State{}) - }, - WithDefaultServiceConfig(fmt.Sprintf(`{ "loadBalancingConfig": [{"%v": {}}] }`, happyBalancerName))) -} - -// TestServiceConfigErrorPolling injects a service config error and verifies -// ResolveNow is called with the appropriate backoff strategy being consulted -// between ResolveNow calls. -func (s) TestServiceConfigErrorPolling(t *testing.T) { - testResolverErrorPolling(t, func(r *manual.Resolver) { - badsc := r.CC.ParseServiceConfig("bad config") - r.UpdateState(resolver.State{ServiceConfig: badsc}) - }, func(r *manual.Resolver) { - // UpdateState will block if ResolveNow is being called (which blocks on - // rn), so call it in a goroutine. - go r.CC.UpdateState(resolver.State{}) - }, - WithDefaultServiceConfig(fmt.Sprintf(`{ "loadBalancingConfig": [{"%v": {}}] }`, happyBalancerName))) -} - -// TestResolverErrorInBuild makes the resolver.Builder call into the ClientConn -// during the Build call. We use two separate mutexes in the code which make -// sure there is no data race in this code path, and also that there is no -// deadlock. -func (s) TestResolverErrorInBuild(t *testing.T) { - r := manual.NewBuilderWithScheme("whatever") - r.InitialState(resolver.State{ServiceConfig: &serviceconfig.ParseResult{Err: errors.New("resolver build err")}}) - - cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithResolvers(r)) - if err != nil { - t.Fatalf("Dial(_, _) = _, %v; want _, nil", err) - } - defer cc.Close() - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - var dummy int - const wantMsg = "error parsing service config" - const wantCode = codes.Unavailable - if err := cc.Invoke(ctx, "/foo/bar", &dummy, &dummy); status.Code(err) != wantCode || !strings.Contains(status.Convert(err).Message(), wantMsg) { - t.Fatalf("cc.Invoke(_, _, _, _) = %v; want status.Code()==%v, status.Message() contains %q", err, wantCode, wantMsg) - } -} - -func (s) TestServiceConfigErrorRPC(t *testing.T) { - r := manual.NewBuilderWithScheme("whatever") - - cc, err := Dial(r.Scheme()+":///test.server", WithInsecure(), WithResolvers(r)) - if err != nil { - t.Fatalf("Dial(_, _) = _, %v; want _, nil", err) - } - defer cc.Close() - badsc := r.CC.ParseServiceConfig("bad config") - r.UpdateState(resolver.State{ServiceConfig: badsc}) - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - var dummy int - const wantMsg = "error parsing service config" - const wantCode = codes.Unavailable - if err := cc.Invoke(ctx, "/foo/bar", &dummy, &dummy); status.Code(err) != wantCode || !strings.Contains(status.Convert(err).Message(), wantMsg) { - t.Fatalf("cc.Invoke(_, _, _, _) = %v; want status.Code()==%v, status.Message() contains %q", err, wantCode, wantMsg) - } -} diff --git a/resolver_test.go b/resolver_test.go new file mode 100644 index 000000000000..bde92f7b0bf4 --- /dev/null +++ b/resolver_test.go @@ -0,0 +1,144 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import ( + "context" + "fmt" + "net" + "testing" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/attributes" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/balancer/stub" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" +) + +type wrapResolverBuilder struct { + resolver.Builder + scheme string +} + +func (w *wrapResolverBuilder) Scheme() string { + return w.scheme +} + +func init() { + resolver.Register(&wrapResolverBuilder{Builder: resolver.Get("passthrough"), scheme: "casetest"}) + resolver.Register(&wrapResolverBuilder{Builder: resolver.Get("dns"), scheme: "caseTest"}) +} + +func (s) TestResolverCaseSensitivity(t *testing.T) { + // This should find the "casetest" resolver instead of the "caseTest" + // resolver, even though the latter was registered later. "casetest" is + // "passthrough" and "caseTest" is "dns". With "passthrough" the dialer + // should see the target's address directly, but "dns" would be converted + // into a loopback IP (v4 or v6) address. + target := "caseTest:///localhost:1234" + addrCh := make(chan string, 1) + customDialer := func(ctx context.Context, addr string) (net.Conn, error) { + select { + case addrCh <- addr: + default: + } + return nil, fmt.Errorf("not dialing with custom dialer") + } + + cc, err := Dial(target, WithContextDialer(customDialer), WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Unexpected Dial(%q) error: %v", target, err) + } + cc.Connect() + if got, want := <-addrCh, "localhost:1234"; got != want { + cc.Close() + t.Fatalf("Dialer got address %q; wanted %q", got, want) + } + cc.Close() + + // Clear addrCh for future use. + select { + case <-addrCh: + default: + } + + res := &wrapResolverBuilder{Builder: resolver.Get("dns"), scheme: "caseTest2"} + // This should not find the injected resolver due to the case not matching. + // This results in "passthrough" being used with the address as the whole + // target. + target = "caseTest2:///localhost:1234" + cc, err = Dial(target, WithContextDialer(customDialer), WithResolvers(res), WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Unexpected Dial(%q) error: %v", target, err) + } + cc.Connect() + if got, want := <-addrCh, target; got != want { + cc.Close() + t.Fatalf("Dialer got address %q; wanted %q", got, want) + } + cc.Close() +} + +// TestResolverAddressesToEndpoints ensures one Endpoint is created for each +// entry in resolver.State.Addresses automatically. +func (s) TestResolverAddressesToEndpoints(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + const scheme = "testresolveraddressestoendpoints" + r := manual.NewBuilderWithScheme(scheme) + + stateCh := make(chan balancer.ClientConnState, 1) + bf := stub.BalancerFuncs{ + UpdateClientConnState: func(_ *stub.BalancerData, ccs balancer.ClientConnState) error { + stateCh <- ccs + return nil + }, + } + balancerName := "stub-balancer-" + scheme + stub.Register(balancerName, bf) + + a1 := attributes.New("x", "y") + a2 := attributes.New("a", "b") + r.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: "addr1", BalancerAttributes: a1}, {Addr: "addr2", BalancerAttributes: a2}}}) + + cc, err := Dial(r.Scheme()+":///", + WithTransportCredentials(insecure.NewCredentials()), + WithResolvers(r), + WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, balancerName))) + if err != nil { + t.Fatalf("Unexpected error dialing: %v", err) + } + defer cc.Close() + + select { + case got := <-stateCh: + want := []resolver.Endpoint{ + {Addresses: []resolver.Address{{Addr: "addr1"}}, Attributes: a1}, + {Addresses: []resolver.Address{{Addr: "addr2"}}, Attributes: a2}, + } + if diff := cmp.Diff(got.ResolverState.Endpoints, want); diff != "" { + t.Errorf("Did not receive expected endpoints. Diff (-got +want):\n%v", diff) + } + case <-ctx.Done(): + t.Fatalf("timed out waiting for endpoints") + } +} diff --git a/rpc_util.go b/rpc_util.go index d4870ba4a996..b7723aa09cbb 100644 --- a/rpc_util.go +++ b/rpc_util.go @@ -25,9 +25,7 @@ import ( "encoding/binary" "fmt" "io" - "io/ioutil" "math" - "net/url" "strings" "sync" "time" @@ -77,8 +75,8 @@ func NewGZIPCompressorWithLevel(level int) (Compressor, error) { } return &gzipCompressor{ pool: sync.Pool{ - New: func() interface{} { - w, err := gzip.NewWriterLevel(ioutil.Discard, level) + New: func() any { + w, err := gzip.NewWriterLevel(io.Discard, level) if err != nil { panic(err) } @@ -144,7 +142,7 @@ func (d *gzipDecompressor) Do(r io.Reader) ([]byte, error) { z.Close() d.pool.Put(z) }() - return ioutil.ReadAll(z) + return io.ReadAll(z) } func (d *gzipDecompressor) Type() string { @@ -161,6 +159,7 @@ type callInfo struct { contentSubtype string codec baseCodec maxRetryRPCBufferSize int + onFinish []func(err error) } func defaultCallInfo() *callInfo { @@ -198,7 +197,11 @@ func Header(md *metadata.MD) CallOption { // HeaderCallOption is a CallOption for collecting response header metadata. // The metadata field will be populated *after* the RPC completes. -// This is an EXPERIMENTAL API. +// +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. type HeaderCallOption struct { HeaderAddr *metadata.MD } @@ -216,7 +219,11 @@ func Trailer(md *metadata.MD) CallOption { // TrailerCallOption is a CallOption for collecting response trailer metadata. // The metadata field will be populated *after* the RPC completes. -// This is an EXPERIMENTAL API. +// +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. type TrailerCallOption struct { TrailerAddr *metadata.MD } @@ -234,7 +241,11 @@ func Peer(p *peer.Peer) CallOption { // PeerCallOption is a CallOption for collecting the identity of the remote // peer. The peer field will be populated *after* the RPC completes. -// This is an EXPERIMENTAL API. +// +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. type PeerCallOption struct { PeerAddr *peer.Peer } @@ -247,7 +258,8 @@ func (o PeerCallOption) after(c *callInfo, attempt *csAttempt) { } // WaitForReady configures the action to take when an RPC is attempted on broken -// connections or unreachable servers. If waitForReady is false, the RPC will fail +// connections or unreachable servers. If waitForReady is false and the +// connection is in the TRANSIENT_FAILURE state, the RPC will fail // immediately. Otherwise, the RPC client will block the call until a // connection is available (or the call is canceled or times out) and will // retry the call if it fails due to a transient error. gRPC will not retry if @@ -269,7 +281,11 @@ func FailFast(failFast bool) CallOption { // FailFastCallOption is a CallOption for indicating whether an RPC should fail // fast or not. -// This is an EXPERIMENTAL API. +// +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. type FailFastCallOption struct { FailFast bool } @@ -280,15 +296,55 @@ func (o FailFastCallOption) before(c *callInfo) error { } func (o FailFastCallOption) after(c *callInfo, attempt *csAttempt) {} +// OnFinish returns a CallOption that configures a callback to be called when +// the call completes. The error passed to the callback is the status of the +// RPC, and may be nil. The onFinish callback provided will only be called once +// by gRPC. This is mainly used to be used by streaming interceptors, to be +// notified when the RPC completes along with information about the status of +// the RPC. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. +func OnFinish(onFinish func(err error)) CallOption { + return OnFinishCallOption{ + OnFinish: onFinish, + } +} + +// OnFinishCallOption is CallOption that indicates a callback to be called when +// the call completes. +// +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. +type OnFinishCallOption struct { + OnFinish func(error) +} + +func (o OnFinishCallOption) before(c *callInfo) error { + c.onFinish = append(c.onFinish, o.OnFinish) + return nil +} + +func (o OnFinishCallOption) after(c *callInfo, attempt *csAttempt) {} + // MaxCallRecvMsgSize returns a CallOption which sets the maximum message size -// in bytes the client can receive. +// in bytes the client can receive. If this is not set, gRPC uses the default +// 4MB. func MaxCallRecvMsgSize(bytes int) CallOption { return MaxRecvMsgSizeCallOption{MaxRecvMsgSize: bytes} } // MaxRecvMsgSizeCallOption is a CallOption that indicates the maximum message // size in bytes the client can receive. -// This is an EXPERIMENTAL API. +// +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. type MaxRecvMsgSizeCallOption struct { MaxRecvMsgSize int } @@ -300,14 +356,19 @@ func (o MaxRecvMsgSizeCallOption) before(c *callInfo) error { func (o MaxRecvMsgSizeCallOption) after(c *callInfo, attempt *csAttempt) {} // MaxCallSendMsgSize returns a CallOption which sets the maximum message size -// in bytes the client can send. +// in bytes the client can send. If this is not set, gRPC uses the default +// `math.MaxInt32`. func MaxCallSendMsgSize(bytes int) CallOption { return MaxSendMsgSizeCallOption{MaxSendMsgSize: bytes} } // MaxSendMsgSizeCallOption is a CallOption that indicates the maximum message // size in bytes the client can send. -// This is an EXPERIMENTAL API. +// +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. type MaxSendMsgSizeCallOption struct { MaxSendMsgSize int } @@ -326,7 +387,11 @@ func PerRPCCredentials(creds credentials.PerRPCCredentials) CallOption { // PerRPCCredsCallOption is a CallOption that indicates the per-RPC // credentials to use for the call. -// This is an EXPERIMENTAL API. +// +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. type PerRPCCredsCallOption struct { Creds credentials.PerRPCCredentials } @@ -341,13 +406,20 @@ func (o PerRPCCredsCallOption) after(c *callInfo, attempt *csAttempt) {} // sending the request. If WithCompressor is also set, UseCompressor has // higher priority. // -// This API is EXPERIMENTAL. +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. func UseCompressor(name string) CallOption { return CompressorCallOption{CompressorType: name} } // CompressorCallOption is a CallOption that indicates the compressor to use. -// This is an EXPERIMENTAL API. +// +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. type CompressorCallOption struct { CompressorType string } @@ -380,7 +452,11 @@ func CallContentSubtype(contentSubtype string) CallOption { // ContentSubtypeCallOption is a CallOption that indicates the content-subtype // used for marshaling messages. -// This is an EXPERIMENTAL API. +// +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. type ContentSubtypeCallOption struct { ContentSubtype string } @@ -391,9 +467,10 @@ func (o ContentSubtypeCallOption) before(c *callInfo) error { } func (o ContentSubtypeCallOption) after(c *callInfo, attempt *csAttempt) {} -// ForceCodec returns a CallOption that will set the given Codec to be -// used for all request and response messages for a call. The result of calling -// String() will be used as the content-subtype in a case-insensitive manner. +// ForceCodec returns a CallOption that will set codec to be used for all +// request and response messages for a call. The result of calling Name() will +// be used as the content-subtype after converting to lowercase, unless +// CallContentSubtype is also used. // // See Content-Type on // https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests for @@ -404,7 +481,10 @@ func (o ContentSubtypeCallOption) after(c *callInfo, attempt *csAttempt) {} // This function is provided for advanced users; prefer to use only // CallContentSubtype to select a registered codec instead. // -// This is an EXPERIMENTAL API. +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. func ForceCodec(codec encoding.Codec) CallOption { return ForceCodecCallOption{Codec: codec} } @@ -412,7 +492,10 @@ func ForceCodec(codec encoding.Codec) CallOption { // ForceCodecCallOption is a CallOption that indicates the codec used for // marshaling messages. // -// This is an EXPERIMENTAL API. +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. type ForceCodecCallOption struct { Codec encoding.Codec } @@ -434,7 +517,10 @@ func CallCustomCodec(codec Codec) CallOption { // CustomCodecCallOption is a CallOption that indicates the codec used for // marshaling messages. // -// This is an EXPERIMENTAL API. +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. type CustomCodecCallOption struct { Codec Codec } @@ -448,14 +534,21 @@ func (o CustomCodecCallOption) after(c *callInfo, attempt *csAttempt) {} // MaxRetryRPCBufferSize returns a CallOption that limits the amount of memory // used for buffering this RPC's requests for retry purposes. // -// This API is EXPERIMENTAL. +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. func MaxRetryRPCBufferSize(bytes int) CallOption { return MaxRetryRPCBufferSizeCallOption{bytes} } // MaxRetryRPCBufferSizeCallOption is a CallOption indicating the amount of // memory to be used for caching this RPC for retry purposes. -// This is an EXPERIMENTAL API. +// +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. type MaxRetryRPCBufferSizeCallOption struct { MaxRetryRPCBufferSize int } @@ -484,6 +577,9 @@ type parser struct { // The header of a gRPC message. Find more detail at // https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md header [5]byte + + // recvBufferPool is the pool of shared receive buffers. + recvBufferPool SharedBufferPool } // recvMsg reads a complete gRPC message from the stream. @@ -492,10 +588,11 @@ type parser struct { // format. The caller owns the returned msg memory. // // If there is an error, possible values are: -// * io.EOF, when no messages remain -// * io.ErrUnexpectedEOF -// * of type transport.ConnectionError -// * an error from the status package +// - io.EOF, when no messages remain +// - io.ErrUnexpectedEOF +// - of type transport.ConnectionError +// - an error from the status package +// // No other error values or types must be returned, which also means // that the underlying io.Reader must not return an incompatible // error. @@ -516,9 +613,7 @@ func (p *parser) recvMsg(maxReceiveMessageSize int) (pf payloadFormat, msg []byt if int(length) > maxReceiveMessageSize { return 0, nil, status.Errorf(codes.ResourceExhausted, "grpc: received message larger than max (%d vs. %d)", length, maxReceiveMessageSize) } - // TODO(bradfitz,zhaoq): garbage. reuse buffer after proto decoding instead - // of making it for each message: - msg = make([]byte, int(length)) + msg = p.recvBufferPool.Get(int(length)) if _, err := p.r.Read(msg); err != nil { if err == io.EOF { err = io.ErrUnexpectedEOF @@ -531,7 +626,7 @@ func (p *parser) recvMsg(maxReceiveMessageSize int) (pf payloadFormat, msg []byt // encode serializes msg and returns a buffer containing the message, or an // error if it is too large to be transmitted by grpc. If msg is nil, it // generates an empty message. -func encode(c baseCodec, msg interface{}) ([]byte, error) { +func encode(c baseCodec, msg any) ([]byte, error) { if msg == nil { // NOTE: typed nils will not be caught by this check return nil, nil } @@ -598,14 +693,15 @@ func msgHeader(data, compData []byte) (hdr []byte, payload []byte) { return hdr, data } -func outPayload(client bool, msg interface{}, data, payload []byte, t time.Time) *stats.OutPayload { +func outPayload(client bool, msg any, data, payload []byte, t time.Time) *stats.OutPayload { return &stats.OutPayload{ - Client: client, - Payload: msg, - Data: data, - Length: len(data), - WireLength: len(payload) + headerLen, - SentTime: t, + Client: client, + Payload: msg, + Data: data, + Length: len(data), + WireLength: len(payload) + headerLen, + CompressedLength: len(payload), + SentTime: t, } } @@ -626,17 +722,17 @@ func checkRecvPayload(pf payloadFormat, recvCompress string, haveCompressor bool } type payloadInfo struct { - wireLength int // The compressed length got from wire. + compressedLength int // The compressed length got from wire. uncompressedBytes []byte } func recvAndDecompress(p *parser, s *transport.Stream, dc Decompressor, maxReceiveMessageSize int, payInfo *payloadInfo, compressor encoding.Compressor) ([]byte, error) { - pf, d, err := p.recvMsg(maxReceiveMessageSize) + pf, buf, err := p.recvMsg(maxReceiveMessageSize) if err != nil { return nil, err } if payInfo != nil { - payInfo.wireLength = len(d) + payInfo.compressedLength = len(buf) } if st := checkRecvPayload(pf, s.RecvCompress(), compressor != nil || dc != nil); st != nil { @@ -648,23 +744,21 @@ func recvAndDecompress(p *parser, s *transport.Stream, dc Decompressor, maxRecei // To match legacy behavior, if the decompressor is set by WithDecompressor or RPCDecompressor, // use this decompressor as the default. if dc != nil { - d, err = dc.Do(bytes.NewReader(d)) - size = len(d) + buf, err = dc.Do(bytes.NewReader(buf)) + size = len(buf) } else { - d, size, err = decompress(compressor, d, maxReceiveMessageSize) + buf, size, err = decompress(compressor, buf, maxReceiveMessageSize) } if err != nil { - return nil, status.Errorf(codes.Internal, "grpc: failed to decompress the received message %v", err) + return nil, status.Errorf(codes.Internal, "grpc: failed to decompress the received message: %v", err) + } + if size > maxReceiveMessageSize { + // TODO: Revisit the error code. Currently keep it consistent with java + // implementation. + return nil, status.Errorf(codes.ResourceExhausted, "grpc: received message after decompression larger than max (%d vs. %d)", size, maxReceiveMessageSize) } - } else { - size = len(d) - } - if size > maxReceiveMessageSize { - // TODO: Revisit the error code. Currently keep it consistent with java - // implementation. - return nil, status.Errorf(codes.ResourceExhausted, "grpc: received message larger than max (%d vs. %d)", size, maxReceiveMessageSize) } - return d, nil + return buf, nil } // Using compressor, decompress d, returning data and size. @@ -691,23 +785,25 @@ func decompress(compressor encoding.Compressor, d []byte, maxReceiveMessageSize } // Read from LimitReader with limit max+1. So if the underlying // reader is over limit, the result will be bigger than max. - d, err = ioutil.ReadAll(io.LimitReader(dcReader, int64(maxReceiveMessageSize)+1)) + d, err = io.ReadAll(io.LimitReader(dcReader, int64(maxReceiveMessageSize)+1)) return d, len(d), err } // For the two compressor parameters, both should not be set, but if they are, // dc takes precedence over compressor. // TODO(dfawley): wrap the old compressor/decompressor using the new API? -func recv(p *parser, c baseCodec, s *transport.Stream, dc Decompressor, m interface{}, maxReceiveMessageSize int, payInfo *payloadInfo, compressor encoding.Compressor) error { - d, err := recvAndDecompress(p, s, dc, maxReceiveMessageSize, payInfo, compressor) +func recv(p *parser, c baseCodec, s *transport.Stream, dc Decompressor, m any, maxReceiveMessageSize int, payInfo *payloadInfo, compressor encoding.Compressor) error { + buf, err := recvAndDecompress(p, s, dc, maxReceiveMessageSize, payInfo, compressor) if err != nil { return err } - if err := c.Unmarshal(d, m); err != nil { - return status.Errorf(codes.Internal, "grpc: failed to unmarshal the received message %v", err) + if err := c.Unmarshal(buf, m); err != nil { + return status.Errorf(codes.Internal, "grpc: failed to unmarshal the received message: %v", err) } if payInfo != nil { - payInfo.uncompressedBytes = d + payInfo.uncompressedBytes = buf + } else { + p.recvBufferPool.Put(&buf) } return nil } @@ -767,39 +863,54 @@ func ErrorDesc(err error) string { // Errorf returns nil if c is OK. // // Deprecated: use status.Errorf instead. -func Errorf(c codes.Code, format string, a ...interface{}) error { +func Errorf(c codes.Code, format string, a ...any) error { return status.Errorf(c, format, a...) } +var errContextCanceled = status.Error(codes.Canceled, context.Canceled.Error()) +var errContextDeadline = status.Error(codes.DeadlineExceeded, context.DeadlineExceeded.Error()) + // toRPCErr converts an error into an error from the status package. func toRPCErr(err error) error { - if err == nil || err == io.EOF { + switch err { + case nil, io.EOF: return err - } - if err == io.ErrUnexpectedEOF { + case context.DeadlineExceeded: + return errContextDeadline + case context.Canceled: + return errContextCanceled + case io.ErrUnexpectedEOF: return status.Error(codes.Internal, err.Error()) } - if _, ok := status.FromError(err); ok { - return err - } + switch e := err.(type) { case transport.ConnectionError: return status.Error(codes.Unavailable, e.Desc) - default: - switch err { - case context.DeadlineExceeded: - return status.Error(codes.DeadlineExceeded, err.Error()) - case context.Canceled: - return status.Error(codes.Canceled, err.Error()) - } + case *transport.NewStreamError: + return toRPCErr(e.Err) } + + if _, ok := status.FromError(err); ok { + return err + } + return status.Error(codes.Unknown, err.Error()) } // setCallInfoCodec should only be called after CallOptions have been applied. func setCallInfoCodec(c *callInfo) error { if c.codec != nil { - // codec was already set by a CallOption; use it. + // codec was already set by a CallOption; use it, but set the content + // subtype if it is not set. + if c.contentSubtype == "" { + // c.codec is a baseCodec to hide the difference between grpc.Codec and + // encoding.Codec (Name vs. String method name). We only support + // setting content subtype from encoding.Codec to avoid a behavior + // change with the deprecated version. + if ec, ok := c.codec.(encoding.Codec); ok { + c.contentSubtype = strings.ToLower(ec.Name()) + } + } return nil } @@ -817,40 +928,6 @@ func setCallInfoCodec(c *callInfo) error { return nil } -// parseDialTarget returns the network and address to pass to dialer -func parseDialTarget(target string) (net string, addr string) { - net = "tcp" - - m1 := strings.Index(target, ":") - m2 := strings.Index(target, ":/") - - // handle unix:addr which will fail with url.Parse - if m1 >= 0 && m2 < 0 { - if n := target[0:m1]; n == "unix" { - net = n - addr = target[m1+1:] - return net, addr - } - } - if m2 >= 0 { - t, err := url.Parse(target) - if err != nil { - return net, target - } - scheme := t.Scheme - addr = t.Path - if scheme == "unix" { - net = scheme - if addr == "" { - addr = t.Host - } - return net, addr - } - } - - return net, target -} - // channelzData is used to store channelz related data for ClientConn, addrConn and Server. // These fields cannot be embedded in the original structs (e.g. ClientConn), since to do atomic // operation on int64 variable on 32-bit machine, user is responsible to enforce memory alignment. @@ -868,8 +945,7 @@ type channelzData struct { // buffer files to ensure compatibility with the gRPC version used. The latest // support package version is 7. // -// Older versions are kept for compatibility. They may be removed if -// compatibility cannot be maintained. +// Older versions are kept for compatibility. // // These constants should not be referenced from any other code. const ( diff --git a/rpc_util_test.go b/rpc_util_test.go index 2449c23815ec..84f2348655b9 100644 --- a/rpc_util_test.go +++ b/rpc_util_test.go @@ -65,7 +65,7 @@ func (s) TestSimpleParsing(t *testing.T) { {append([]byte{0, 1, 0, 0, 0}, bigMsg...), nil, bigMsg, compressionNone}, } { buf := fullReader{bytes.NewReader(test.p)} - parser := &parser{r: buf} + parser := &parser{r: buf, recvBufferPool: nopBufferPool{}} pt, b, err := parser.recvMsg(math.MaxInt32) if err != test.err || !bytes.Equal(b, test.b) || pt != test.pt { t.Fatalf("parser{%v}.recvMsg(_) = %v, %v, %v\nwant %v, %v, %v", test.p, pt, b, err, test.pt, test.b, test.err) @@ -77,7 +77,7 @@ func (s) TestMultipleParsing(t *testing.T) { // Set a byte stream consists of 3 messages with their headers. p := []byte{0, 0, 0, 0, 1, 'a', 0, 0, 0, 0, 2, 'b', 'c', 0, 0, 0, 0, 1, 'd'} b := fullReader{bytes.NewReader(p)} - parser := &parser{r: b} + parser := &parser{r: b, recvBufferPool: nopBufferPool{}} wantRecvs := []struct { pt payloadFormat @@ -191,27 +191,6 @@ func (s) TestToRPCErr(t *testing.T) { } } -func (s) TestParseDialTarget(t *testing.T) { - for _, test := range []struct { - target, wantNet, wantAddr string - }{ - {"unix:etcd:0", "unix", "etcd:0"}, - {"unix:///tmp/unix-3", "unix", "/tmp/unix-3"}, - {"unix://domain", "unix", "domain"}, - {"unix://etcd:0", "unix", "etcd:0"}, - {"unix:///etcd:0", "unix", "/etcd:0"}, - {"passthrough://unix://domain", "tcp", "passthrough://unix://domain"}, - {"https://google.com:443", "tcp", "https://google.com:443"}, - {"dns:///google.com", "tcp", "dns:///google.com"}, - {"/unix/socket/address", "tcp", "/unix/socket/address"}, - } { - gotNet, gotAddr := parseDialTarget(test.target) - if gotNet != test.wantNet || gotAddr != test.wantAddr { - t.Errorf("parseDialTarget(%q) = %s, %s want %s, %s", test.target, gotNet, gotAddr, test.wantNet, test.wantAddr) - } - } -} - // bmEncode benchmarks encoding a Protocol Buffer message containing mSize // bytes. func bmEncode(b *testing.B, mSize int) { diff --git a/security/advancedtls/advancedtls.go b/security/advancedtls/advancedtls.go index c21ff753aab6..4b5d1f4825c9 100644 --- a/security/advancedtls/advancedtls.go +++ b/security/advancedtls/advancedtls.go @@ -27,10 +27,11 @@ import ( "crypto/x509" "fmt" "net" - "syscall" + "reflect" "time" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/tls/certprovider" credinternal "google.golang.org/grpc/internal/credentials" ) @@ -79,21 +80,67 @@ type GetRootCAsResults struct { TrustCerts *x509.CertPool } -// RootCertificateOptions contains a field and a function for obtaining root -// trust certificates. -// It is used by both ClientOptions and ServerOptions. -// If users want to use default verification, but did not provide a valid -// RootCertificateOptions, we use the system default trust certificates. +// RootCertificateOptions contains options to obtain root trust certificates +// for both the client and the server. +// At most one option could be set. If none of them are set, we +// use the system default trust certificates. type RootCertificateOptions struct { - // If field RootCACerts is set, field GetRootCAs will be ignored. RootCACerts - // will be used every time when verifying the peer certificates, without - // performing root certificate reloading. + // If RootCACerts is set, it will be used every time when verifying + // the peer certificates, without performing root certificate reloading. RootCACerts *x509.CertPool - // If GetRootCAs is set and RootCACerts is nil, GetRootCAs will be invoked - // every time asked to check certificates sent from the server when a new - // connection is established. - // This is known as root CA certificate reloading. - GetRootCAs func(params *GetRootCAsParams) (*GetRootCAsResults, error) + // If GetRootCertificates is set, it will be invoked to obtain root certs for + // every new connection. + GetRootCertificates func(params *GetRootCAsParams) (*GetRootCAsResults, error) + // If RootProvider is set, we will use the root certs from the Provider's + // KeyMaterial() call in the new connections. The Provider must have initial + // credentials if specified. Otherwise, KeyMaterial() will block forever. + RootProvider certprovider.Provider +} + +// nonNilFieldCount returns the number of set fields in RootCertificateOptions. +func (o RootCertificateOptions) nonNilFieldCount() int { + cnt := 0 + rv := reflect.ValueOf(o) + for i := 0; i < rv.NumField(); i++ { + if !rv.Field(i).IsNil() { + cnt++ + } + } + return cnt +} + +// IdentityCertificateOptions contains options to obtain identity certificates +// for both the client and the server. +// At most one option could be set. +type IdentityCertificateOptions struct { + // If Certificates is set, it will be used every time when needed to present + //identity certificates, without performing identity certificate reloading. + Certificates []tls.Certificate + // If GetIdentityCertificatesForClient is set, it will be invoked to obtain + // identity certs for every new connection. + // This field MUST be set on client side. + GetIdentityCertificatesForClient func(*tls.CertificateRequestInfo) (*tls.Certificate, error) + // If GetIdentityCertificatesForServer is set, it will be invoked to obtain + // identity certs for every new connection. + // This field MUST be set on server side. + GetIdentityCertificatesForServer func(*tls.ClientHelloInfo) ([]*tls.Certificate, error) + // If IdentityProvider is set, we will use the identity certs from the + // Provider's KeyMaterial() call in the new connections. The Provider must + // have initial credentials if specified. Otherwise, KeyMaterial() will block + // forever. + IdentityProvider certprovider.Provider +} + +// nonNilFieldCount returns the number of set fields in IdentityCertificateOptions. +func (o IdentityCertificateOptions) nonNilFieldCount() int { + cnt := 0 + rv := reflect.ValueOf(o) + for i := 0; i < rv.NumField(); i++ { + if !rv.Field(i).IsNil() { + cnt++ + } + } + return cnt } // VerificationType is the enum type that represents different levels of @@ -115,27 +162,11 @@ const ( SkipVerification ) -// ClientOptions contains all the fields and functions needed to be filled by -// the client. -// General rules for certificate setting on client side: -// Certificates or GetClientCertificate indicates the certificates sent from -// the client to the server to prove client's identities. The rules for setting -// these two fields are: -// If requiring mutual authentication on server side: -// Either Certificates or GetClientCertificate must be set; the other will -// be ignored. -// Otherwise: -// Nothing needed(the two fields will be ignored). +// ClientOptions contains the fields needed to be filled by the client. type ClientOptions struct { - // If field Certificates is set, field GetClientCertificate will be ignored. - // The client will use Certificates every time when asked for a certificate, - // without performing certificate reloading. - Certificates []tls.Certificate - // If GetClientCertificate is set and Certificates is nil, the client will - // invoke this function every time asked to present certificates to the - // server when a new connection is established. This is known as peer - // certificate reloading. - GetClientCertificate func(*tls.CertificateRequestInfo) (*tls.Certificate, error) + // IdentityOptions is OPTIONAL on client side. This field only needs to be + // set if mutual authentication is required on server side. + IdentityOptions IdentityCertificateOptions // VerifyPeer is a custom verification check after certificate signature // check. // If this is set, we will perform this customized check after doing the @@ -145,87 +176,148 @@ type ClientOptions struct { // it will override the virtual host name of authority (e.g. :authority // header field) in requests. ServerNameOverride string - // RootCertificateOptions is REQUIRED to be correctly set on client side. - RootCertificateOptions + // RootOptions is OPTIONAL on client side. If not set, we will try to use the + // default trust certificates in users' OS system. + RootOptions RootCertificateOptions // VType is the verification type on the client side. VType VerificationType + // RevocationConfig is the configurations for certificate revocation checks. + // It could be nil if such checks are not needed. + RevocationConfig *RevocationConfig + // MinVersion contains the minimum TLS version that is acceptable. + // By default, TLS 1.2 is currently used as the minimum when acting as a + // client, and TLS 1.0 when acting as a server. TLS 1.0 is the minimum + // supported by this package, both as a client and as a server. + MinVersion uint16 + // MaxVersion contains the maximum TLS version that is acceptable. + // By default, the maximum version supported by this package is used, + // which is currently TLS 1.3. + MaxVersion uint16 } -// ServerOptions contains all the fields and functions needed to be filled by -// the client. -// General rules for certificate setting on server side: -// Certificates or GetClientCertificate indicates the certificates sent from -// the server to the client to prove server's identities. The rules for setting -// these two fields are: -// Either Certificates or GetCertificates must be set; the other will be ignored. +// ServerOptions contains the fields needed to be filled by the server. type ServerOptions struct { - // If field Certificates is set, field GetClientCertificate will be ignored. - // The server will use Certificates every time when asked for a certificate, - // without performing certificate reloading. - Certificates []tls.Certificate - // If GetClientCertificate is set and Certificates is nil, the server will - // invoke this function every time asked to present certificates to the - // client when a new connection is established. This is known as peer - // certificate reloading. - GetCertificates func(*tls.ClientHelloInfo) ([]*tls.Certificate, error) + // IdentityOptions is REQUIRED on server side. + IdentityOptions IdentityCertificateOptions // VerifyPeer is a custom verification check after certificate signature // check. // If this is set, we will perform this customized check after doing the // normal check(s) indicated by setting VType. VerifyPeer CustomVerificationFunc - // RootCertificateOptions is only required when mutual TLS is - // enabled(RequireClientCert is true). - RootCertificateOptions + // RootOptions is OPTIONAL on server side. This field only needs to be set if + // mutual authentication is required(RequireClientCert is true). + RootOptions RootCertificateOptions // If the server want the client to send certificates. RequireClientCert bool // VType is the verification type on the server side. VType VerificationType + // RevocationConfig is the configurations for certificate revocation checks. + // It could be nil if such checks are not needed. + RevocationConfig *RevocationConfig + // MinVersion contains the minimum TLS version that is acceptable. + // By default, TLS 1.2 is currently used as the minimum when acting as a + // client, and TLS 1.0 when acting as a server. TLS 1.0 is the minimum + // supported by this package, both as a client and as a server. + MinVersion uint16 + // MaxVersion contains the maximum TLS version that is acceptable. + // By default, the maximum version supported by this package is used, + // which is currently TLS 1.3. + MaxVersion uint16 } func (o *ClientOptions) config() (*tls.Config, error) { if o.VType == SkipVerification && o.VerifyPeer == nil { - return nil, fmt.Errorf( - "client needs to provide custom verification mechanism if choose to skip default verification") + return nil, fmt.Errorf("client needs to provide custom verification mechanism if choose to skip default verification") } - rootCAs := o.RootCACerts - if o.VType != SkipVerification && o.RootCACerts == nil && o.GetRootCAs == nil { - // Set rootCAs to system default. - systemRootCAs, err := x509.SystemCertPool() - if err != nil { - return nil, err - } - rootCAs = systemRootCAs + // Make sure users didn't specify more than one fields in + // RootCertificateOptions and IdentityCertificateOptions. + if num := o.RootOptions.nonNilFieldCount(); num > 1 { + return nil, fmt.Errorf("at most one field in RootCertificateOptions could be specified") + } + if num := o.IdentityOptions.nonNilFieldCount(); num > 1 { + return nil, fmt.Errorf("at most one field in IdentityCertificateOptions could be specified") + } + if o.IdentityOptions.GetIdentityCertificatesForServer != nil { + return nil, fmt.Errorf("GetIdentityCertificatesForServer cannot be specified on the client side") + } + if o.MinVersion > o.MaxVersion { + return nil, fmt.Errorf("the minimum TLS version is larger than the maximum TLS version") } - // We have to set InsecureSkipVerify to true to skip the default checks and - // use the verification function we built from buildVerifyFunc. config := &tls.Config{ - ServerName: o.ServerNameOverride, - Certificates: o.Certificates, - GetClientCertificate: o.GetClientCertificate, - InsecureSkipVerify: true, + ServerName: o.ServerNameOverride, + // We have to set InsecureSkipVerify to true to skip the default checks and + // use the verification function we built from buildVerifyFunc. + InsecureSkipVerify: true, + MinVersion: o.MinVersion, + MaxVersion: o.MaxVersion, + } + // Propagate root-certificate-related fields in tls.Config. + switch { + case o.RootOptions.RootCACerts != nil: + config.RootCAs = o.RootOptions.RootCACerts + case o.RootOptions.GetRootCertificates != nil: + // In cases when users provide GetRootCertificates callback, since this + // callback is not contained in tls.Config, we have nothing to set here. + // We will invoke the callback in ClientHandshake. + case o.RootOptions.RootProvider != nil: + o.RootOptions.GetRootCertificates = func(*GetRootCAsParams) (*GetRootCAsResults, error) { + km, err := o.RootOptions.RootProvider.KeyMaterial(context.Background()) + if err != nil { + return nil, err + } + return &GetRootCAsResults{TrustCerts: km.Roots}, nil + } + default: + // No root certificate options specified by user. Use the certificates + // stored in system default path as the last resort. + if o.VType != SkipVerification { + systemRootCAs, err := x509.SystemCertPool() + if err != nil { + return nil, err + } + config.RootCAs = systemRootCAs + } } - if rootCAs != nil { - config.RootCAs = rootCAs + // Propagate identity-certificate-related fields in tls.Config. + switch { + case o.IdentityOptions.Certificates != nil: + config.Certificates = o.IdentityOptions.Certificates + case o.IdentityOptions.GetIdentityCertificatesForClient != nil: + config.GetClientCertificate = o.IdentityOptions.GetIdentityCertificatesForClient + case o.IdentityOptions.IdentityProvider != nil: + config.GetClientCertificate = func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { + km, err := o.IdentityOptions.IdentityProvider.KeyMaterial(context.Background()) + if err != nil { + return nil, err + } + if len(km.Certs) != 1 { + return nil, fmt.Errorf("there should always be only one identity cert chain on the client side in IdentityProvider") + } + return &km.Certs[0], nil + } + default: + // It's fine for users to not specify identity certificate options here. } return config, nil } func (o *ServerOptions) config() (*tls.Config, error) { - if o.Certificates == nil && o.GetCertificates == nil { - return nil, fmt.Errorf("either Certificates or GetCertificates must be specified") - } if o.RequireClientCert && o.VType == SkipVerification && o.VerifyPeer == nil { - return nil, fmt.Errorf( - "server needs to provide custom verification mechanism if choose to skip default verification, but require client certificate(s)") + return nil, fmt.Errorf("server needs to provide custom verification mechanism if choose to skip default verification, but require client certificate(s)") } - clientCAs := o.RootCACerts - if o.VType != SkipVerification && o.RootCACerts == nil && o.GetRootCAs == nil && o.RequireClientCert { - // Set clientCAs to system default. - systemRootCAs, err := x509.SystemCertPool() - if err != nil { - return nil, err - } - clientCAs = systemRootCAs + // Make sure users didn't specify more than one fields in + // RootCertificateOptions and IdentityCertificateOptions. + if num := o.RootOptions.nonNilFieldCount(); num > 1 { + return nil, fmt.Errorf("at most one field in RootCertificateOptions could be specified") + } + if num := o.IdentityOptions.nonNilFieldCount(); num > 1 { + return nil, fmt.Errorf("at most one field in IdentityCertificateOptions could be specified") + } + if o.IdentityOptions.GetIdentityCertificatesForClient != nil { + return nil, fmt.Errorf("GetIdentityCertificatesForClient cannot be specified on the server side") + } + if o.MinVersion > o.MaxVersion { + return nil, fmt.Errorf("the minimum TLS version is larger than the maximum TLS version") } clientAuth := tls.NoClientCert if o.RequireClientCert { @@ -235,18 +327,62 @@ func (o *ServerOptions) config() (*tls.Config, error) { clientAuth = tls.RequireAnyClientCert } config := &tls.Config{ - ClientAuth: clientAuth, - Certificates: o.Certificates, + ClientAuth: clientAuth, + MinVersion: o.MinVersion, + MaxVersion: o.MaxVersion, + } + // Propagate root-certificate-related fields in tls.Config. + switch { + case o.RootOptions.RootCACerts != nil: + config.ClientCAs = o.RootOptions.RootCACerts + case o.RootOptions.GetRootCertificates != nil: + // In cases when users provide GetRootCertificates callback, since this + // callback is not contained in tls.Config, we have nothing to set here. + // We will invoke the callback in ServerHandshake. + case o.RootOptions.RootProvider != nil: + o.RootOptions.GetRootCertificates = func(*GetRootCAsParams) (*GetRootCAsResults, error) { + km, err := o.RootOptions.RootProvider.KeyMaterial(context.Background()) + if err != nil { + return nil, err + } + return &GetRootCAsResults{TrustCerts: km.Roots}, nil + } + default: + // No root certificate options specified by user. Use the certificates + // stored in system default path as the last resort. + if o.VType != SkipVerification && o.RequireClientCert { + systemRootCAs, err := x509.SystemCertPool() + if err != nil { + return nil, err + } + config.ClientCAs = systemRootCAs + } } - if o.GetCertificates != nil { - // GetCertificate is only able to perform SNI logic for go1.10 and above. - // It will return the first certificate in o.GetCertificates for go1.9. + // Propagate identity-certificate-related fields in tls.Config. + switch { + case o.IdentityOptions.Certificates != nil: + config.Certificates = o.IdentityOptions.Certificates + case o.IdentityOptions.GetIdentityCertificatesForServer != nil: config.GetCertificate = func(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) { return buildGetCertificates(clientHello, o) } - } - if clientCAs != nil { - config.ClientCAs = clientCAs + case o.IdentityOptions.IdentityProvider != nil: + o.IdentityOptions.GetIdentityCertificatesForServer = func(*tls.ClientHelloInfo) ([]*tls.Certificate, error) { + km, err := o.IdentityOptions.IdentityProvider.KeyMaterial(context.Background()) + if err != nil { + return nil, err + } + var certChains []*tls.Certificate + for i := 0; i < len(km.Certs); i++ { + certChains = append(certChains, &km.Certs[i]) + } + return certChains, nil + } + config.GetCertificate = func(clientHello *tls.ClientHelloInfo) (*tls.Certificate, error) { + return buildGetCertificates(clientHello, o) + } + default: + return nil, fmt.Errorf("needs to specify at least one field in IdentityCertificateOptions") } return config, nil } @@ -254,11 +390,12 @@ func (o *ServerOptions) config() (*tls.Config, error) { // advancedTLSCreds is the credentials required for authenticating a connection // using TLS. type advancedTLSCreds struct { - config *tls.Config - verifyFunc CustomVerificationFunc - getRootCAs func(params *GetRootCAsParams) (*GetRootCAsResults, error) - isClient bool - vType VerificationType + config *tls.Config + verifyFunc CustomVerificationFunc + getRootCAs func(params *GetRootCAsParams) (*GetRootCAsResults, error) + isClient bool + vType VerificationType + revocationConfig *RevocationConfig } func (c advancedTLSCreds) Info() credentials.ProtocolInfo { @@ -271,7 +408,7 @@ func (c advancedTLSCreds) Info() credentials.ProtocolInfo { func (c *advancedTLSCreds) ClientHandshake(ctx context.Context, authority string, rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) { // Use local cfg to avoid clobbering ServerName if using multiple endpoints. - cfg := cloneTLSConfig(c.config) + cfg := credinternal.CloneTLSConfig(c.config) // We return the full authority name to users if ServerName is empty without // stripping the trailing port. if cfg.ServerName == "" { @@ -301,11 +438,11 @@ func (c *advancedTLSCreds) ClientHandshake(ctx context.Context, authority string }, } info.SPIFFEID = credinternal.SPIFFEIDFromState(conn.ConnectionState()) - return WrapSyscallConn(rawConn, conn), info, nil + return credinternal.WrapSyscallConn(rawConn, conn), info, nil } func (c *advancedTLSCreds) ServerHandshake(rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) { - cfg := cloneTLSConfig(c.config) + cfg := credinternal.CloneTLSConfig(c.config) cfg.VerifyPeerCertificate = buildVerifyFunc(c, "", rawConn) conn := tls.Server(rawConn, cfg) if err := conn.Handshake(); err != nil { @@ -319,12 +456,12 @@ func (c *advancedTLSCreds) ServerHandshake(rawConn net.Conn) (net.Conn, credenti }, } info.SPIFFEID = credinternal.SPIFFEIDFromState(conn.ConnectionState()) - return WrapSyscallConn(rawConn, conn), info, nil + return credinternal.WrapSyscallConn(rawConn, conn), info, nil } func (c *advancedTLSCreds) Clone() credentials.TransportCredentials { return &advancedTLSCreds{ - config: cloneTLSConfig(c.config), + config: credinternal.CloneTLSConfig(c.config), verifyFunc: c.verifyFunc, getRootCAs: c.getRootCAs, isClient: c.isClient, @@ -340,15 +477,23 @@ func (c *advancedTLSCreds) OverrideServerName(serverNameOverride string) error { // and possibly custom verification check. // We have to build our own verification function here because current // tls module: -// 1. does not have a good support on root cert reloading. -// 2. will ignore basic certificate check when setting InsecureSkipVerify -// to true. +// 1. does not have a good support on root cert reloading. +// 2. will ignore basic certificate check when setting InsecureSkipVerify +// to true. func buildVerifyFunc(c *advancedTLSCreds, serverName string, rawConn net.Conn) func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error { return func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error { chains := verifiedChains var leafCert *x509.Certificate + rawCertList := make([]*x509.Certificate, len(rawCerts)) + for i, asn1Data := range rawCerts { + cert, err := x509.ParseCertificate(asn1Data) + if err != nil { + return err + } + rawCertList[i] = cert + } if c.vType == CertAndHostVerification || c.vType == CertVerification { // perform possible trust credential reloading and certificate check rootCAs := c.config.RootCAs @@ -367,14 +512,6 @@ func buildVerifyFunc(c *advancedTLSCreds, rootCAs = results.TrustCerts } // Verify peers' certificates against RootCAs and get verifiedChains. - certs := make([]*x509.Certificate, len(rawCerts)) - for i, asn1Data := range rawCerts { - cert, err := x509.ParseCertificate(asn1Data) - if err != nil { - return err - } - certs[i] = cert - } keyUsages := []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth} if !c.isClient { keyUsages = []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth} @@ -385,19 +522,35 @@ func buildVerifyFunc(c *advancedTLSCreds, Intermediates: x509.NewCertPool(), KeyUsages: keyUsages, } - for _, cert := range certs[1:] { + for _, cert := range rawCertList[1:] { opts.Intermediates.AddCert(cert) } // Perform default hostname check if specified. if c.isClient && c.vType == CertAndHostVerification && serverName != "" { - opts.DNSName = serverName + parsedName, _, err := net.SplitHostPort(serverName) + if err != nil { + // If the serverName had no host port or if the serverName cannot be + // parsed, use it as-is. + parsedName = serverName + } + opts.DNSName = parsedName } var err error - chains, err = certs[0].Verify(opts) + chains, err = rawCertList[0].Verify(opts) if err != nil { return err } - leafCert = certs[0] + leafCert = rawCertList[0] + } + // Perform certificate revocation check if specified. + if c.revocationConfig != nil { + verifiedChains := chains + if verifiedChains == nil { + verifiedChains = [][]*x509.Certificate{rawCertList} + } + if err := CheckChainRevocation(verifiedChains, *c.revocationConfig); err != nil { + return err + } } // Perform custom verification check if specified. if c.verifyFunc != nil { @@ -421,13 +574,14 @@ func NewClientCreds(o *ClientOptions) (credentials.TransportCredentials, error) return nil, err } tc := &advancedTLSCreds{ - config: conf, - isClient: true, - getRootCAs: o.GetRootCAs, - verifyFunc: o.VerifyPeer, - vType: o.VType, - } - tc.config.NextProtos = appendH2ToNextProtos(tc.config.NextProtos) + config: conf, + isClient: true, + getRootCAs: o.RootOptions.GetRootCertificates, + verifyFunc: o.VerifyPeer, + vType: o.VType, + revocationConfig: o.RevocationConfig, + } + tc.config.NextProtos = credinternal.AppendH2ToNextProtos(tc.config.NextProtos) return tc, nil } @@ -439,97 +593,13 @@ func NewServerCreds(o *ServerOptions) (credentials.TransportCredentials, error) return nil, err } tc := &advancedTLSCreds{ - config: conf, - isClient: false, - getRootCAs: o.GetRootCAs, - verifyFunc: o.VerifyPeer, - vType: o.VType, - } - tc.config.NextProtos = appendH2ToNextProtos(tc.config.NextProtos) + config: conf, + isClient: false, + getRootCAs: o.RootOptions.GetRootCertificates, + verifyFunc: o.VerifyPeer, + vType: o.VType, + revocationConfig: o.RevocationConfig, + } + tc.config.NextProtos = credinternal.AppendH2ToNextProtos(tc.config.NextProtos) return tc, nil } - -// TODO(ZhenLian): The code below are duplicates with gRPC-Go under -// credentials/internal. Consider refactoring in the future. -const alpnProtoStrH2 = "h2" - -func appendH2ToNextProtos(ps []string) []string { - for _, p := range ps { - if p == alpnProtoStrH2 { - return ps - } - } - ret := make([]string, 0, len(ps)+1) - ret = append(ret, ps...) - return append(ret, alpnProtoStrH2) -} - -// We give syscall.Conn a new name here since syscall.Conn and net.Conn used -// below have the same names. -type sysConn = syscall.Conn - -// syscallConn keeps reference of rawConn to support syscall.Conn for channelz. -// SyscallConn() (the method in interface syscall.Conn) is explicitly -// implemented on this type, -// -// Interface syscall.Conn is implemented by most net.Conn implementations (e.g. -// TCPConn, UnixConn), but is not part of net.Conn interface. So wrapper conns -// that embed net.Conn don't implement syscall.Conn. (Side note: tls.Conn -// doesn't embed net.Conn, so even if syscall.Conn is part of net.Conn, it won't -// help here). -type syscallConn struct { - net.Conn - // sysConn is a type alias of syscall.Conn. It's necessary because the name - // `Conn` collides with `net.Conn`. - sysConn -} - -// WrapSyscallConn tries to wrap rawConn and newConn into a net.Conn that -// implements syscall.Conn. rawConn will be used to support syscall, and newConn -// will be used for read/write. -// -// This function returns newConn if rawConn doesn't implement syscall.Conn. -func WrapSyscallConn(rawConn, newConn net.Conn) net.Conn { - sysConn, ok := rawConn.(syscall.Conn) - if !ok { - return newConn - } - return &syscallConn{ - Conn: newConn, - sysConn: sysConn, - } -} - -func cloneTLSConfig(cfg *tls.Config) *tls.Config { - if cfg == nil { - return &tls.Config{} - } - return cfg.Clone() -} - -// buildGetCertificates returns the certificate that matches the SNI field -// for the given ClientHelloInfo, defaulting to the first element of o.GetCertificates. -func buildGetCertificates(clientHello *tls.ClientHelloInfo, o *ServerOptions) (*tls.Certificate, error) { - if o.GetCertificates == nil { - return nil, fmt.Errorf("function GetCertificates must be specified") - } - certificates, err := o.GetCertificates(clientHello) - if err != nil { - return nil, err - } - if len(certificates) == 0 { - return nil, fmt.Errorf("no certificates configured") - } - // If users pass in only one certificate, return that certificate. - if len(certificates) == 1 { - return certificates[0], nil - } - // Choose the SNI certificate using SupportsCertificate. - for _, cert := range certificates { - if err := clientHello.SupportsCertificate(cert); err == nil { - return cert, nil - } - } - // If nothing matches, return the first certificate. - return certificates[0], nil -} diff --git a/security/advancedtls/advancedtls_integration_test.go b/security/advancedtls/advancedtls_integration_test.go index 195da18385cb..3659497fd5d2 100644 --- a/security/advancedtls/advancedtls_integration_test.go +++ b/security/advancedtls/advancedtls_integration_test.go @@ -24,19 +24,27 @@ import ( "crypto/x509" "fmt" "net" + "os" "sync" "testing" "time" "google.golang.org/grpc" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/tls/certprovider" + "google.golang.org/grpc/credentials/tls/certprovider/pemfile" pb "google.golang.org/grpc/examples/helloworld/helloworld" + "google.golang.org/grpc/security/advancedtls/internal/testutils" "google.golang.org/grpc/security/advancedtls/testdata" ) -var ( - address = "localhost:50051" - port = ":50051" +const ( + // Default timeout for normal connections. + defaultTestTimeout = 5 * time.Second + // Intervals that set to monitor the credential updates. + credRefreshingInterval = 200 * time.Millisecond + // Time we wait for the credential updates to be picked up. + sleepInterval = 400 * time.Millisecond ) // stageInfo contains a stage number indicating the current phase of each @@ -67,77 +75,17 @@ func (s *stageInfo) reset() { s.stage = 0 } -// certStore contains all the certificates used in the integration tests. -type certStore struct { - // clientPeer1 is the certificate sent by client to prove its identity. - // It is trusted by serverTrust1. - clientPeer1 tls.Certificate - // clientPeer2 is the certificate sent by client to prove its identity. - // It is trusted by serverTrust2. - clientPeer2 tls.Certificate - // serverPeer1 is the certificate sent by server to prove its identity. - // It is trusted by clientTrust1. - serverPeer1 tls.Certificate - // serverPeer2 is the certificate sent by server to prove its identity. - // It is trusted by clientTrust2. - serverPeer2 tls.Certificate - clientTrust1 *x509.CertPool - clientTrust2 *x509.CertPool - serverTrust1 *x509.CertPool - serverTrust2 *x509.CertPool +type greeterServer struct { + pb.UnimplementedGreeterServer } -// loadCerts function is used to load test certificates at the beginning of -// each integration test. -func (cs *certStore) loadCerts() error { - var err error - cs.clientPeer1, err = tls.LoadX509KeyPair(testdata.Path("client_cert_1.pem"), - testdata.Path("client_key_1.pem")) - if err != nil { - return err - } - cs.clientPeer2, err = tls.LoadX509KeyPair(testdata.Path("client_cert_2.pem"), - testdata.Path("client_key_2.pem")) - if err != nil { - return err - } - cs.serverPeer1, err = tls.LoadX509KeyPair(testdata.Path("server_cert_1.pem"), - testdata.Path("server_key_1.pem")) - if err != nil { - return err - } - cs.serverPeer2, err = tls.LoadX509KeyPair(testdata.Path("server_cert_2.pem"), - testdata.Path("server_key_2.pem")) - if err != nil { - return err - } - cs.clientTrust1, err = readTrustCert(testdata.Path("client_trust_cert_1.pem")) - if err != nil { - return err - } - cs.clientTrust2, err = readTrustCert(testdata.Path("client_trust_cert_2.pem")) - if err != nil { - return err - } - cs.serverTrust1, err = readTrustCert(testdata.Path("server_trust_cert_1.pem")) - if err != nil { - return err - } - cs.serverTrust2, err = readTrustCert(testdata.Path("server_trust_cert_2.pem")) - if err != nil { - return err - } - return nil -} - -// serverImpl is used to implement pb.GreeterServer. -type serverImpl struct{} - -// SayHello is a simple implementation of pb.GreeterServer. -func (s *serverImpl) SayHello(ctx context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) { +// sayHello is a simple implementation of the pb.GreeterServer SayHello method. +func (greeterServer) SayHello(ctx context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) { return &pb.HelloReply{Message: "Hello " + in.Name}, nil } +// TODO(ZhenLian): remove shouldFail to the function signature to provider +// tests. func callAndVerify(msg string, client pb.GreeterClient, shouldFail bool) error { ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() @@ -148,7 +96,9 @@ func callAndVerify(msg string, client pb.GreeterClient, shouldFail bool) error { return nil } -func callAndVerifyWithClientConn(connCtx context.Context, msg string, creds credentials.TransportCredentials, shouldFail bool) (*grpc.ClientConn, pb.GreeterClient, error) { +// TODO(ZhenLian): remove shouldFail and add ...DialOption to the function +// signature to provider cleaner tests. +func callAndVerifyWithClientConn(connCtx context.Context, address string, msg string, creds credentials.TransportCredentials, shouldFail bool) (*grpc.ClientConn, pb.GreeterClient, error) { var conn *grpc.ClientConn var err error // If we want the test to fail, we establish a non-blocking connection to @@ -182,10 +132,9 @@ func callAndVerifyWithClientConn(connCtx context.Context, msg string, creds cred // (could be change the client's trust certificate, or change custom // verification function, etc) func (s) TestEnd2End(t *testing.T) { - cs := &certStore{} - err := cs.loadCerts() - if err != nil { - t.Fatalf("failed to load certs: %v", err) + cs := &testutils.CertStore{} + if err := cs.LoadCerts(); err != nil { + t.Fatalf("cs.LoadCerts() failed, err: %v", err) } stage := &stageInfo{} for _, test := range []struct { @@ -205,38 +154,38 @@ func (s) TestEnd2End(t *testing.T) { }{ // Test Scenarios: // At initialization(stage = 0), client will be initialized with cert - // clientPeer1 and clientTrust1, server with serverPeer1 and serverTrust1. - // The mutual authentication works at the beginning, since clientPeer1 is - // trusted by serverTrust1, and serverPeer1 by clientTrust1. - // At stage 1, client changes clientPeer1 to clientPeer2. Since clientPeer2 - // is not trusted by serverTrust1, following rpc calls are expected to + // ClientCert1 and ClientTrust1, server with ServerCert1 and ServerTrust1. + // The mutual authentication works at the beginning, since ClientCert1 is + // trusted by ServerTrust1, and ServerCert1 by ClientTrust1. + // At stage 1, client changes ClientCert1 to ClientCert2. Since ClientCert2 + // is not trusted by ServerTrust1, following rpc calls are expected to // fail, while the previous rpc calls are still good because those are // already authenticated. - // At stage 2, the server changes serverTrust1 to serverTrust2, and we - // should see it again accepts the connection, since clientPeer2 is trusted - // by serverTrust2. + // At stage 2, the server changes ServerTrust1 to ServerTrust2, and we + // should see it again accepts the connection, since ClientCert2 is trusted + // by ServerTrust2. { - desc: "TestClientPeerCertReloadServerTrustCertReload", + desc: "test the reloading feature for client identity callback and server trust callback", clientGetCert: func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { switch stage.read() { case 0: - return &cs.clientPeer1, nil + return &cs.ClientCert1, nil default: - return &cs.clientPeer2, nil + return &cs.ClientCert2, nil } }, - clientRoot: cs.clientTrust1, + clientRoot: cs.ClientTrust1, clientVerifyFunc: func(params *VerificationFuncParams) (*VerificationResults, error) { return &VerificationResults{}, nil }, clientVType: CertVerification, - serverCert: []tls.Certificate{cs.serverPeer1}, + serverCert: []tls.Certificate{cs.ServerCert1}, serverGetRoot: func(params *GetRootCAsParams) (*GetRootCAsResults, error) { switch stage.read() { case 0, 1: - return &GetRootCAsResults{TrustCerts: cs.serverTrust1}, nil + return &GetRootCAsResults{TrustCerts: cs.ServerTrust1}, nil default: - return &GetRootCAsResults{TrustCerts: cs.serverTrust2}, nil + return &GetRootCAsResults{TrustCerts: cs.ServerTrust2}, nil } }, serverVerifyFunc: func(params *VerificationFuncParams) (*VerificationResults, error) { @@ -246,25 +195,25 @@ func (s) TestEnd2End(t *testing.T) { }, // Test Scenarios: // At initialization(stage = 0), client will be initialized with cert - // clientPeer1 and clientTrust1, server with serverPeer1 and serverTrust1. - // The mutual authentication works at the beginning, since clientPeer1 is - // trusted by serverTrust1, and serverPeer1 by clientTrust1. - // At stage 1, server changes serverPeer1 to serverPeer2. Since serverPeer2 - // is not trusted by clientTrust1, following rpc calls are expected to + // ClientCert1 and ClientTrust1, server with ServerCert1 and ServerTrust1. + // The mutual authentication works at the beginning, since ClientCert1 is + // trusted by ServerTrust1, and ServerCert1 by ClientTrust1. + // At stage 1, server changes ServerCert1 to ServerCert2. Since ServerCert2 + // is not trusted by ClientTrust1, following rpc calls are expected to // fail, while the previous rpc calls are still good because those are // already authenticated. - // At stage 2, the client changes clientTrust1 to clientTrust2, and we - // should see it again accepts the connection, since serverPeer2 is trusted - // by clientTrust2. + // At stage 2, the client changes ClientTrust1 to ClientTrust2, and we + // should see it again accepts the connection, since ServerCert2 is trusted + // by ClientTrust2. { - desc: "TestServerPeerCertReloadClientTrustCertReload", - clientCert: []tls.Certificate{cs.clientPeer1}, + desc: "test the reloading feature for server identity callback and client trust callback", + clientCert: []tls.Certificate{cs.ClientCert1}, clientGetRoot: func(params *GetRootCAsParams) (*GetRootCAsResults, error) { switch stage.read() { case 0, 1: - return &GetRootCAsResults{TrustCerts: cs.clientTrust1}, nil + return &GetRootCAsResults{TrustCerts: cs.ClientTrust1}, nil default: - return &GetRootCAsResults{TrustCerts: cs.clientTrust2}, nil + return &GetRootCAsResults{TrustCerts: cs.ClientTrust2}, nil } }, clientVerifyFunc: func(params *VerificationFuncParams) (*VerificationResults, error) { @@ -274,12 +223,12 @@ func (s) TestEnd2End(t *testing.T) { serverGetCert: func(*tls.ClientHelloInfo) ([]*tls.Certificate, error) { switch stage.read() { case 0: - return []*tls.Certificate{&cs.serverPeer1}, nil + return []*tls.Certificate{&cs.ServerCert1}, nil default: - return []*tls.Certificate{&cs.serverPeer2}, nil + return []*tls.Certificate{&cs.ServerCert2}, nil } }, - serverRoot: cs.serverTrust1, + serverRoot: cs.ServerTrust1, serverVerifyFunc: func(params *VerificationFuncParams) (*VerificationResults, error) { return &VerificationResults{}, nil }, @@ -287,26 +236,26 @@ func (s) TestEnd2End(t *testing.T) { }, // Test Scenarios: // At initialization(stage = 0), client will be initialized with cert - // clientPeer1 and clientTrust1, server with serverPeer1 and serverTrust1. - // The mutual authentication works at the beginning, since clientPeer1 - // trusted by serverTrust1, serverPeer1 by clientTrust1, and also the - // custom verification check allows the CommonName on serverPeer1. - // At stage 1, server changes serverPeer1 to serverPeer2, and client - // changes clientTrust1 to clientTrust2. Although serverPeer2 is trusted by - // clientTrust2, our authorization check only accepts serverPeer1, and + // ClientCert1 and ClientTrust1, server with ServerCert1 and ServerTrust1. + // The mutual authentication works at the beginning, since ClientCert1 + // trusted by ServerTrust1, ServerCert1 by ClientTrust1, and also the + // custom verification check allows the CommonName on ServerCert1. + // At stage 1, server changes ServerCert1 to ServerCert2, and client + // changes ClientTrust1 to ClientTrust2. Although ServerCert2 is trusted by + // ClientTrust2, our authorization check only accepts ServerCert1, and // hence the following calls should fail. Previous connections should // not be affected. // At stage 2, the client changes authorization check to only accept - // serverPeer2. Now we should see the connection becomes normal again. + // ServerCert2. Now we should see the connection becomes normal again. { - desc: "TestClientCustomVerification", - clientCert: []tls.Certificate{cs.clientPeer1}, + desc: "test client custom verification", + clientCert: []tls.Certificate{cs.ClientCert1}, clientGetRoot: func(params *GetRootCAsParams) (*GetRootCAsResults, error) { switch stage.read() { case 0: - return &GetRootCAsResults{TrustCerts: cs.clientTrust1}, nil + return &GetRootCAsResults{TrustCerts: cs.ClientTrust1}, nil default: - return &GetRootCAsResults{TrustCerts: cs.clientTrust2}, nil + return &GetRootCAsResults{TrustCerts: cs.ClientTrust2}, nil } }, clientVerifyFunc: func(params *VerificationFuncParams) (*VerificationResults, error) { @@ -320,12 +269,12 @@ func (s) TestEnd2End(t *testing.T) { authzCheck := false switch stage.read() { case 0, 1: - // foo.bar.com is the common name on serverPeer1 + // foo.bar.com is the common name on ServerCert1 if cert.Subject.CommonName == "foo.bar.com" { authzCheck = true } default: - // foo.bar.server2.com is the common name on serverPeer2 + // foo.bar.server2.com is the common name on ServerCert2 if cert.Subject.CommonName == "foo.bar.server2.com" { authzCheck = true } @@ -339,12 +288,12 @@ func (s) TestEnd2End(t *testing.T) { serverGetCert: func(*tls.ClientHelloInfo) ([]*tls.Certificate, error) { switch stage.read() { case 0: - return []*tls.Certificate{&cs.serverPeer1}, nil + return []*tls.Certificate{&cs.ServerCert1}, nil default: - return []*tls.Certificate{&cs.serverPeer2}, nil + return []*tls.Certificate{&cs.ServerCert2}, nil } }, - serverRoot: cs.serverTrust1, + serverRoot: cs.ServerTrust1, serverVerifyFunc: func(params *VerificationFuncParams) (*VerificationResults, error) { return &VerificationResults{}, nil }, @@ -352,25 +301,25 @@ func (s) TestEnd2End(t *testing.T) { }, // Test Scenarios: // At initialization(stage = 0), client will be initialized with cert - // clientPeer1 and clientTrust1, server with serverPeer1 and serverTrust1. - // The mutual authentication works at the beginning, since clientPeer1 - // trusted by serverTrust1, serverPeer1 by clientTrust1, and also the + // ClientCert1 and ClientTrust1, server with ServerCert1 and ServerTrust1. + // The mutual authentication works at the beginning, since ClientCert1 + // trusted by ServerTrust1, ServerCert1 by ClientTrust1, and also the // custom verification check on server side allows all connections. - // At stage 1, server disallows the the connections by setting custom + // At stage 1, server disallows the connections by setting custom // verification check. The following calls should fail. Previous // connections should not be affected. // At stage 2, server allows all the connections again and the // authentications should go back to normal. { desc: "TestServerCustomVerification", - clientCert: []tls.Certificate{cs.clientPeer1}, - clientRoot: cs.clientTrust1, + clientCert: []tls.Certificate{cs.ClientCert1}, + clientRoot: cs.ClientTrust1, clientVerifyFunc: func(params *VerificationFuncParams) (*VerificationResults, error) { return &VerificationResults{}, nil }, clientVType: CertVerification, - serverCert: []tls.Certificate{cs.serverPeer1}, - serverRoot: cs.serverTrust1, + serverCert: []tls.Certificate{cs.ServerCert1}, + serverRoot: cs.ServerTrust1, serverVerifyFunc: func(params *VerificationFuncParams) (*VerificationResults, error) { switch stage.read() { case 0, 2: @@ -388,11 +337,13 @@ func (s) TestEnd2End(t *testing.T) { t.Run(test.desc, func(t *testing.T) { // Start a server using ServerOptions in another goroutine. serverOptions := &ServerOptions{ - Certificates: test.serverCert, - GetCertificates: test.serverGetCert, - RootCertificateOptions: RootCertificateOptions{ - RootCACerts: test.serverRoot, - GetRootCAs: test.serverGetRoot, + IdentityOptions: IdentityCertificateOptions{ + Certificates: test.serverCert, + GetIdentityCertificatesForServer: test.serverGetCert, + }, + RootOptions: RootCertificateOptions{ + RootCACerts: test.serverRoot, + GetRootCertificates: test.serverGetRoot, }, RequireClientCert: true, VerifyPeer: test.serverVerifyFunc, @@ -404,32 +355,35 @@ func (s) TestEnd2End(t *testing.T) { } s := grpc.NewServer(grpc.Creds(serverTLSCreds)) defer s.Stop() - lis, err := net.Listen("tcp", port) + lis, err := net.Listen("tcp", "localhost:0") if err != nil { t.Fatalf("failed to listen: %v", err) } defer lis.Close() - pb.RegisterGreeterServer(s, &serverImpl{}) + addr := fmt.Sprintf("localhost:%v", lis.Addr().(*net.TCPAddr).Port) + pb.RegisterGreeterServer(s, greeterServer{}) go s.Serve(lis) clientOptions := &ClientOptions{ - Certificates: test.clientCert, - GetClientCertificate: test.clientGetCert, - VerifyPeer: test.clientVerifyFunc, - RootCertificateOptions: RootCertificateOptions{ - RootCACerts: test.clientRoot, - GetRootCAs: test.clientGetRoot, + IdentityOptions: IdentityCertificateOptions{ + Certificates: test.clientCert, + GetIdentityCertificatesForClient: test.clientGetCert, + }, + VerifyPeer: test.clientVerifyFunc, + RootOptions: RootCertificateOptions{ + RootCACerts: test.clientRoot, + GetRootCertificates: test.clientGetRoot, }, VType: test.clientVType, } clientTLSCreds, err := NewClientCreds(clientOptions) if err != nil { - t.Fatalf("clientTLSCreds failed to create") + t.Fatalf("clientTLSCreds failed to create: %v", err) } // ------------------------Scenario 1------------------------------------ // stage = 0, initial connection should succeed - ctx1, cancel1 := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel1() - conn, greetClient, err := callAndVerifyWithClientConn(ctx1, "rpc call 1", clientTLSCreds, false) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + conn, greetClient, err := callAndVerifyWithClientConn(ctx, addr, "rpc call 1", clientTLSCreds, false) if err != nil { t.Fatal(err) } @@ -444,20 +398,19 @@ func (s) TestEnd2End(t *testing.T) { } // ------------------------Scenario 3------------------------------------ // stage = 1, new connection should fail - ctx2, cancel2 := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel2() - conn2, greetClient, err := callAndVerifyWithClientConn(ctx2, "rpc call 3", clientTLSCreds, true) + ctx2, cancel2 := context.WithTimeout(context.Background(), defaultTestTimeout) + conn2, _, err := callAndVerifyWithClientConn(ctx2, addr, "rpc call 3", clientTLSCreds, true) if err != nil { t.Fatal(err) } defer conn2.Close() - //// -------------------------------------------------------------------- + // Immediately cancel the context so the dialing won't drag the entire timeout still it stops. + cancel2() + // ---------------------------------------------------------------------- stage.increase() // ------------------------Scenario 4------------------------------------ // stage = 2, new connection should succeed - ctx3, cancel3 := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel3() - conn3, greetClient, err := callAndVerifyWithClientConn(ctx3, "rpc call 4", clientTLSCreds, false) + conn3, _, err := callAndVerifyWithClientConn(ctx, addr, "rpc call 4", clientTLSCreds, false) if err != nil { t.Fatal(err) } @@ -467,3 +420,534 @@ func (s) TestEnd2End(t *testing.T) { }) } } + +type tmpCredsFiles struct { + clientCertTmp *os.File + clientKeyTmp *os.File + clientTrustTmp *os.File + serverCertTmp *os.File + serverKeyTmp *os.File + serverTrustTmp *os.File +} + +// Create temp files that are used to hold credentials. +func createTmpFiles() (*tmpCredsFiles, error) { + tmpFiles := &tmpCredsFiles{} + var err error + tmpFiles.clientCertTmp, err = os.CreateTemp(os.TempDir(), "pre-") + if err != nil { + return nil, err + } + tmpFiles.clientKeyTmp, err = os.CreateTemp(os.TempDir(), "pre-") + if err != nil { + return nil, err + } + tmpFiles.clientTrustTmp, err = os.CreateTemp(os.TempDir(), "pre-") + if err != nil { + return nil, err + } + tmpFiles.serverCertTmp, err = os.CreateTemp(os.TempDir(), "pre-") + if err != nil { + return nil, err + } + tmpFiles.serverKeyTmp, err = os.CreateTemp(os.TempDir(), "pre-") + if err != nil { + return nil, err + } + tmpFiles.serverTrustTmp, err = os.CreateTemp(os.TempDir(), "pre-") + if err != nil { + return nil, err + } + return tmpFiles, nil +} + +// Copy the credential contents to the temporary files. +func (tmpFiles *tmpCredsFiles) copyCredsToTmpFiles() error { + if err := copyFileContents(testdata.Path("client_cert_1.pem"), tmpFiles.clientCertTmp.Name()); err != nil { + return err + } + if err := copyFileContents(testdata.Path("client_key_1.pem"), tmpFiles.clientKeyTmp.Name()); err != nil { + return err + } + if err := copyFileContents(testdata.Path("client_trust_cert_1.pem"), tmpFiles.clientTrustTmp.Name()); err != nil { + return err + } + if err := copyFileContents(testdata.Path("server_cert_1.pem"), tmpFiles.serverCertTmp.Name()); err != nil { + return err + } + if err := copyFileContents(testdata.Path("server_key_1.pem"), tmpFiles.serverKeyTmp.Name()); err != nil { + return err + } + if err := copyFileContents(testdata.Path("server_trust_cert_1.pem"), tmpFiles.serverTrustTmp.Name()); err != nil { + return err + } + return nil +} + +func (tmpFiles *tmpCredsFiles) removeFiles() { + os.Remove(tmpFiles.clientCertTmp.Name()) + os.Remove(tmpFiles.clientKeyTmp.Name()) + os.Remove(tmpFiles.clientTrustTmp.Name()) + os.Remove(tmpFiles.serverCertTmp.Name()) + os.Remove(tmpFiles.serverKeyTmp.Name()) + os.Remove(tmpFiles.serverTrustTmp.Name()) +} + +func copyFileContents(sourceFile, destinationFile string) error { + input, err := os.ReadFile(sourceFile) + if err != nil { + return err + } + err = os.WriteFile(destinationFile, input, 0644) + if err != nil { + return err + } + return nil +} + +// Create PEMFileProvider(s) watching the content changes of temporary +// files. +func createProviders(tmpFiles *tmpCredsFiles) (certprovider.Provider, certprovider.Provider, certprovider.Provider, certprovider.Provider, error) { + clientIdentityOptions := pemfile.Options{ + CertFile: tmpFiles.clientCertTmp.Name(), + KeyFile: tmpFiles.clientKeyTmp.Name(), + RefreshDuration: credRefreshingInterval, + } + clientIdentityProvider, err := pemfile.NewProvider(clientIdentityOptions) + if err != nil { + return nil, nil, nil, nil, err + } + clientRootOptions := pemfile.Options{ + RootFile: tmpFiles.clientTrustTmp.Name(), + RefreshDuration: credRefreshingInterval, + } + clientRootProvider, err := pemfile.NewProvider(clientRootOptions) + if err != nil { + return nil, nil, nil, nil, err + } + serverIdentityOptions := pemfile.Options{ + CertFile: tmpFiles.serverCertTmp.Name(), + KeyFile: tmpFiles.serverKeyTmp.Name(), + RefreshDuration: credRefreshingInterval, + } + serverIdentityProvider, err := pemfile.NewProvider(serverIdentityOptions) + if err != nil { + return nil, nil, nil, nil, err + } + serverRootOptions := pemfile.Options{ + RootFile: tmpFiles.serverTrustTmp.Name(), + RefreshDuration: credRefreshingInterval, + } + serverRootProvider, err := pemfile.NewProvider(serverRootOptions) + if err != nil { + return nil, nil, nil, nil, err + } + return clientIdentityProvider, clientRootProvider, serverIdentityProvider, serverRootProvider, nil +} + +// In order to test advanced TLS provider features, we used temporary files to +// hold credential data, and copy the contents under testdata/ to these tmp +// files. +// Initially, we establish a good connection with providers watching contents +// from tmp files. +// Next, we change the identity certs that IdentityProvider is watching. Since +// the identity key is not changed, the IdentityProvider should ignore the +// update, and the connection should still be good. +// Then the identity key is changed. This time IdentityProvider should pick +// up the update, and the connection should fail, due to the trust certs on the +// other side is not changed. +// Finally, the trust certs that other-side's RootProvider is watching get +// changed. The connection should go back to normal again. +func (s) TestPEMFileProviderEnd2End(t *testing.T) { + tmpFiles, err := createTmpFiles() + if err != nil { + t.Fatalf("createTmpFiles() failed, error: %v", err) + } + defer tmpFiles.removeFiles() + for _, test := range []struct { + desc string + certUpdateFunc func() + keyUpdateFunc func() + trustCertUpdateFunc func() + }{ + { + desc: "test the reloading feature for clientIdentityProvider and serverTrustProvider", + certUpdateFunc: func() { + err = copyFileContents(testdata.Path("client_cert_2.pem"), tmpFiles.clientCertTmp.Name()) + if err != nil { + t.Fatalf("copyFileContents(%s, %s) failed: %v", testdata.Path("client_cert_2.pem"), tmpFiles.clientCertTmp.Name(), err) + } + }, + keyUpdateFunc: func() { + err = copyFileContents(testdata.Path("client_key_2.pem"), tmpFiles.clientKeyTmp.Name()) + if err != nil { + t.Fatalf("copyFileContents(%s, %s) failed: %v", testdata.Path("client_key_2.pem"), tmpFiles.clientKeyTmp.Name(), err) + } + }, + trustCertUpdateFunc: func() { + err = copyFileContents(testdata.Path("server_trust_cert_2.pem"), tmpFiles.serverTrustTmp.Name()) + if err != nil { + t.Fatalf("copyFileContents(%s, %s) failed: %v", testdata.Path("server_trust_cert_2.pem"), tmpFiles.serverTrustTmp.Name(), err) + } + }, + }, + { + desc: "test the reloading feature for serverIdentityProvider and clientTrustProvider", + certUpdateFunc: func() { + err = copyFileContents(testdata.Path("server_cert_2.pem"), tmpFiles.serverCertTmp.Name()) + if err != nil { + t.Fatalf("copyFileContents(%s, %s) failed: %v", testdata.Path("server_cert_2.pem"), tmpFiles.serverCertTmp.Name(), err) + } + }, + keyUpdateFunc: func() { + err = copyFileContents(testdata.Path("server_key_2.pem"), tmpFiles.serverKeyTmp.Name()) + if err != nil { + t.Fatalf("copyFileContents(%s, %s) failed: %v", testdata.Path("server_key_2.pem"), tmpFiles.serverKeyTmp.Name(), err) + } + }, + trustCertUpdateFunc: func() { + err = copyFileContents(testdata.Path("client_trust_cert_2.pem"), tmpFiles.clientTrustTmp.Name()) + if err != nil { + t.Fatalf("copyFileContents(%s, %s) failed: %v", testdata.Path("client_trust_cert_2.pem"), tmpFiles.clientTrustTmp.Name(), err) + } + }, + }, + } { + test := test + t.Run(test.desc, func(t *testing.T) { + if err := tmpFiles.copyCredsToTmpFiles(); err != nil { + t.Fatalf("tmpFiles.copyCredsToTmpFiles() failed, error: %v", err) + } + clientIdentityProvider, clientRootProvider, serverIdentityProvider, serverRootProvider, err := createProviders(tmpFiles) + if err != nil { + t.Fatalf("createProviders(%v) failed, error: %v", tmpFiles, err) + } + defer clientIdentityProvider.Close() + defer clientRootProvider.Close() + defer serverIdentityProvider.Close() + defer serverRootProvider.Close() + // Start a server and create a client using advancedtls API with Provider. + serverOptions := &ServerOptions{ + IdentityOptions: IdentityCertificateOptions{ + IdentityProvider: serverIdentityProvider, + }, + RootOptions: RootCertificateOptions{ + RootProvider: serverRootProvider, + }, + RequireClientCert: true, + VerifyPeer: func(params *VerificationFuncParams) (*VerificationResults, error) { + return &VerificationResults{}, nil + }, + VType: CertVerification, + } + serverTLSCreds, err := NewServerCreds(serverOptions) + if err != nil { + t.Fatalf("failed to create server creds: %v", err) + } + s := grpc.NewServer(grpc.Creds(serverTLSCreds)) + defer s.Stop() + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("failed to listen: %v", err) + } + defer lis.Close() + addr := fmt.Sprintf("localhost:%v", lis.Addr().(*net.TCPAddr).Port) + pb.RegisterGreeterServer(s, greeterServer{}) + go s.Serve(lis) + clientOptions := &ClientOptions{ + IdentityOptions: IdentityCertificateOptions{ + IdentityProvider: clientIdentityProvider, + }, + VerifyPeer: func(params *VerificationFuncParams) (*VerificationResults, error) { + return &VerificationResults{}, nil + }, + RootOptions: RootCertificateOptions{ + RootProvider: clientRootProvider, + }, + VType: CertVerification, + } + clientTLSCreds, err := NewClientCreds(clientOptions) + if err != nil { + t.Fatalf("clientTLSCreds failed to create, error: %v", err) + } + + // At initialization, the connection should be good. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + conn, greetClient, err := callAndVerifyWithClientConn(ctx, addr, "rpc call 1", clientTLSCreds, false) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + // Make the identity cert change, and wait 1 second for the provider to + // pick up the change. + test.certUpdateFunc() + time.Sleep(sleepInterval) + // The already-established connection should not be affected. + err = callAndVerify("rpc call 2", greetClient, false) + if err != nil { + t.Fatal(err) + } + // New connections should still be good, because the Provider didn't pick + // up the changes due to key-cert mismatch. + conn2, _, err := callAndVerifyWithClientConn(ctx, addr, "rpc call 3", clientTLSCreds, false) + if err != nil { + t.Fatal(err) + } + defer conn2.Close() + // Make the identity key change, and wait 1 second for the provider to + // pick up the change. + test.keyUpdateFunc() + time.Sleep(sleepInterval) + // New connections should fail now, because the Provider picked the + // change, and *_cert_2.pem is not trusted by *_trust_cert_1.pem on the + // other side. + ctx2, cancel2 := context.WithTimeout(context.Background(), defaultTestTimeout) + conn3, _, err := callAndVerifyWithClientConn(ctx2, addr, "rpc call 4", clientTLSCreds, true) + if err != nil { + t.Fatal(err) + } + defer conn3.Close() + // Immediately cancel the context so the dialing won't drag the entire timeout still it stops. + cancel2() + // Make the trust cert change on the other side, and wait 1 second for + // the provider to pick up the change. + test.trustCertUpdateFunc() + time.Sleep(sleepInterval) + // New connections should be good, because the other side is using + // *_trust_cert_2.pem now. + conn4, _, err := callAndVerifyWithClientConn(ctx, addr, "rpc call 5", clientTLSCreds, false) + if err != nil { + t.Fatal(err) + } + defer conn4.Close() + }) + } +} + +func (s) TestDefaultHostNameCheck(t *testing.T) { + cs := &testutils.CertStore{} + if err := cs.LoadCerts(); err != nil { + t.Fatalf("cs.LoadCerts() failed, err: %v", err) + } + for _, test := range []struct { + desc string + clientRoot *x509.CertPool + clientVType VerificationType + serverCert []tls.Certificate + serverVType VerificationType + expectError bool + }{ + // Client side sets vType to CertAndHostVerification, and will do + // default hostname check. Server uses a cert without "localhost" or + // "127.0.0.1" as common name or SAN names, and will hence fail. + { + desc: "Bad default hostname check", + clientRoot: cs.ClientTrust1, + clientVType: CertAndHostVerification, + serverCert: []tls.Certificate{cs.ServerCert1}, + serverVType: CertAndHostVerification, + expectError: true, + }, + // Client side sets vType to CertAndHostVerification, and will do + // default hostname check. Server uses a certificate with "localhost" as + // common name, and will hence pass the default hostname check. + { + desc: "Good default hostname check", + clientRoot: cs.ClientTrust1, + clientVType: CertAndHostVerification, + serverCert: []tls.Certificate{cs.ServerPeerLocalhost1}, + serverVType: CertAndHostVerification, + expectError: false, + }, + } { + test := test + t.Run(test.desc, func(t *testing.T) { + // Start a server using ServerOptions in another goroutine. + serverOptions := &ServerOptions{ + IdentityOptions: IdentityCertificateOptions{ + Certificates: test.serverCert, + }, + RequireClientCert: false, + VType: test.serverVType, + } + serverTLSCreds, err := NewServerCreds(serverOptions) + if err != nil { + t.Fatalf("failed to create server creds: %v", err) + } + s := grpc.NewServer(grpc.Creds(serverTLSCreds)) + defer s.Stop() + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("failed to listen: %v", err) + } + defer lis.Close() + addr := fmt.Sprintf("localhost:%v", lis.Addr().(*net.TCPAddr).Port) + pb.RegisterGreeterServer(s, greeterServer{}) + go s.Serve(lis) + clientOptions := &ClientOptions{ + RootOptions: RootCertificateOptions{ + RootCACerts: test.clientRoot, + }, + VType: test.clientVType, + } + clientTLSCreds, err := NewClientCreds(clientOptions) + if err != nil { + t.Fatalf("clientTLSCreds failed to create: %v", err) + } + shouldFail := false + if test.expectError { + shouldFail = true + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + conn, _, err := callAndVerifyWithClientConn(ctx, addr, "rpc call 1", clientTLSCreds, shouldFail) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + }) + } +} + +func (s) TestTLSVersions(t *testing.T) { + cs := &testutils.CertStore{} + if err := cs.LoadCerts(); err != nil { + t.Fatalf("cs.LoadCerts() failed, err: %v", err) + } + for _, test := range []struct { + desc string + expectError bool + clientMinVersion uint16 + clientMaxVersion uint16 + serverMinVersion uint16 + serverMaxVersion uint16 + }{ + // Client side sets TLS version that is higher than required from the server side. + { + desc: "Client TLS version higher than server", + clientMinVersion: tls.VersionTLS13, + clientMaxVersion: tls.VersionTLS13, + serverMinVersion: tls.VersionTLS12, + serverMaxVersion: tls.VersionTLS12, + expectError: true, + }, + // Server side sets TLS version that is higher than required from the client side. + { + desc: "Server TLS version higher than client", + clientMinVersion: tls.VersionTLS12, + clientMaxVersion: tls.VersionTLS12, + serverMinVersion: tls.VersionTLS13, + serverMaxVersion: tls.VersionTLS13, + expectError: true, + }, + // Client and server set proper TLS versions. + { + desc: "Good TLS version settings", + clientMinVersion: tls.VersionTLS12, + clientMaxVersion: tls.VersionTLS13, + serverMinVersion: tls.VersionTLS12, + serverMaxVersion: tls.VersionTLS13, + expectError: false, + }, + { + desc: "Client 1.2 - 1.3 and server 1.2", + clientMinVersion: tls.VersionTLS12, + clientMaxVersion: tls.VersionTLS13, + serverMinVersion: tls.VersionTLS12, + serverMaxVersion: tls.VersionTLS12, + expectError: false, + }, + { + desc: "Client 1.2 - 1.3 and server 1.1 - 1.2", + clientMinVersion: tls.VersionTLS12, + clientMaxVersion: tls.VersionTLS13, + serverMinVersion: tls.VersionTLS11, + serverMaxVersion: tls.VersionTLS12, + expectError: false, + }, + { + desc: "Client 1.2 - 1.3 and server 1.3", + clientMinVersion: tls.VersionTLS12, + clientMaxVersion: tls.VersionTLS13, + serverMinVersion: tls.VersionTLS13, + serverMaxVersion: tls.VersionTLS13, + expectError: false, + }, + { + desc: "Client 1.2 - 1.2 and server 1.2 - 1.3", + clientMinVersion: tls.VersionTLS12, + clientMaxVersion: tls.VersionTLS12, + serverMinVersion: tls.VersionTLS12, + serverMaxVersion: tls.VersionTLS13, + expectError: false, + }, + { + desc: "Client 1.1 - 1.2 and server 1.2 - 1.3", + clientMinVersion: tls.VersionTLS11, + clientMaxVersion: tls.VersionTLS12, + serverMinVersion: tls.VersionTLS12, + serverMaxVersion: tls.VersionTLS13, + expectError: false, + }, + { + desc: "Client 1.3 and server 1.2 - 1.3", + clientMinVersion: tls.VersionTLS13, + clientMaxVersion: tls.VersionTLS13, + serverMinVersion: tls.VersionTLS12, + serverMaxVersion: tls.VersionTLS13, + expectError: false, + }, + } { + test := test + t.Run(test.desc, func(t *testing.T) { + // Start a server using ServerOptions in another goroutine. + serverOptions := &ServerOptions{ + IdentityOptions: IdentityCertificateOptions{ + Certificates: []tls.Certificate{cs.ServerPeerLocalhost1}, + }, + RequireClientCert: false, + VType: CertAndHostVerification, + MinVersion: test.serverMinVersion, + MaxVersion: test.serverMaxVersion, + } + serverTLSCreds, err := NewServerCreds(serverOptions) + if err != nil { + t.Fatalf("failed to create server creds: %v", err) + } + s := grpc.NewServer(grpc.Creds(serverTLSCreds)) + defer s.Stop() + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("failed to listen: %v", err) + } + defer lis.Close() + addr := fmt.Sprintf("localhost:%v", lis.Addr().(*net.TCPAddr).Port) + pb.RegisterGreeterServer(s, greeterServer{}) + go s.Serve(lis) + clientOptions := &ClientOptions{ + RootOptions: RootCertificateOptions{ + RootCACerts: cs.ClientTrust1, + }, + VType: CertAndHostVerification, + MinVersion: test.clientMinVersion, + MaxVersion: test.clientMaxVersion, + } + clientTLSCreds, err := NewClientCreds(clientOptions) + if err != nil { + t.Fatalf("clientTLSCreds failed to create: %v", err) + } + shouldFail := false + if test.expectError { + shouldFail = true + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + conn, _, err := callAndVerifyWithClientConn(ctx, addr, "rpc call 1", clientTLSCreds, shouldFail) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + }) + } +} diff --git a/security/advancedtls/advancedtls_test.go b/security/advancedtls/advancedtls_test.go index 8800a51a94d6..afad25e7cb4b 100644 --- a/security/advancedtls/advancedtls_test.go +++ b/security/advancedtls/advancedtls_test.go @@ -22,19 +22,16 @@ import ( "context" "crypto/tls" "crypto/x509" - "encoding/pem" "errors" "fmt" - "io/ioutil" - "math/big" "net" - "reflect" - "syscall" "testing" - "github.com/google/go-cmp/cmp" + lru "github.com/hashicorp/golang-lru" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/tls/certprovider" "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/security/advancedtls/internal/testutils" "google.golang.org/grpc/security/advancedtls/testdata" ) @@ -46,14 +43,302 @@ func Test(t *testing.T) { grpctest.RunSubTests(t, s{}) } +type provType int + +const ( + provTypeRoot provType = iota + provTypeIdentity +) + +type fakeProvider struct { + pt provType + isClient bool + wantMultiCert bool + wantError bool +} + +func (f fakeProvider) KeyMaterial(ctx context.Context) (*certprovider.KeyMaterial, error) { + if f.wantError { + return nil, fmt.Errorf("bad fakeProvider") + } + cs := &testutils.CertStore{} + if err := cs.LoadCerts(); err != nil { + return nil, fmt.Errorf("cs.LoadCerts() failed, err: %v", err) + } + if f.pt == provTypeRoot && f.isClient { + return &certprovider.KeyMaterial{Roots: cs.ClientTrust1}, nil + } + if f.pt == provTypeRoot && !f.isClient { + return &certprovider.KeyMaterial{Roots: cs.ServerTrust1}, nil + } + if f.pt == provTypeIdentity && f.isClient { + if f.wantMultiCert { + return &certprovider.KeyMaterial{Certs: []tls.Certificate{cs.ClientCert1, cs.ClientCert2}}, nil + } + return &certprovider.KeyMaterial{Certs: []tls.Certificate{cs.ClientCert1}}, nil + } + if f.wantMultiCert { + return &certprovider.KeyMaterial{Certs: []tls.Certificate{cs.ServerCert1, cs.ServerCert2}}, nil + } + return &certprovider.KeyMaterial{Certs: []tls.Certificate{cs.ServerCert1}}, nil +} + +func (f fakeProvider) Close() {} + +func (s) TestClientOptionsConfigErrorCases(t *testing.T) { + tests := []struct { + desc string + clientVType VerificationType + IdentityOptions IdentityCertificateOptions + RootOptions RootCertificateOptions + MinVersion uint16 + MaxVersion uint16 + }{ + { + desc: "Skip default verification and provide no root credentials", + clientVType: SkipVerification, + }, + { + desc: "More than one fields in RootCertificateOptions is specified", + clientVType: CertVerification, + RootOptions: RootCertificateOptions{ + RootCACerts: x509.NewCertPool(), + RootProvider: fakeProvider{}, + }, + }, + { + desc: "More than one fields in IdentityCertificateOptions is specified", + clientVType: CertVerification, + IdentityOptions: IdentityCertificateOptions{ + GetIdentityCertificatesForClient: func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { + return nil, nil + }, + IdentityProvider: fakeProvider{pt: provTypeIdentity}, + }, + }, + { + desc: "Specify GetIdentityCertificatesForServer", + IdentityOptions: IdentityCertificateOptions{ + GetIdentityCertificatesForServer: func(*tls.ClientHelloInfo) ([]*tls.Certificate, error) { + return nil, nil + }, + }, + }, + { + desc: "Invalid min/max TLS versions", + MinVersion: tls.VersionTLS13, + MaxVersion: tls.VersionTLS12, + }, + } + for _, test := range tests { + test := test + t.Run(test.desc, func(t *testing.T) { + clientOptions := &ClientOptions{ + VType: test.clientVType, + IdentityOptions: test.IdentityOptions, + RootOptions: test.RootOptions, + MinVersion: test.MinVersion, + MaxVersion: test.MaxVersion, + } + _, err := clientOptions.config() + if err == nil { + t.Fatalf("ClientOptions{%v}.config() returns no err, wantErr != nil", clientOptions) + } + }) + } +} + +func (s) TestClientOptionsConfigSuccessCases(t *testing.T) { + tests := []struct { + desc string + clientVType VerificationType + IdentityOptions IdentityCertificateOptions + RootOptions RootCertificateOptions + MinVersion uint16 + MaxVersion uint16 + }{ + { + desc: "Use system default if no fields in RootCertificateOptions is specified", + clientVType: CertVerification, + }, + { + desc: "Good case with mutual TLS", + clientVType: CertVerification, + RootOptions: RootCertificateOptions{ + RootProvider: fakeProvider{}, + }, + IdentityOptions: IdentityCertificateOptions{ + IdentityProvider: fakeProvider{pt: provTypeIdentity}, + }, + MinVersion: tls.VersionTLS12, + MaxVersion: tls.VersionTLS13, + }, + } + for _, test := range tests { + test := test + t.Run(test.desc, func(t *testing.T) { + clientOptions := &ClientOptions{ + VType: test.clientVType, + IdentityOptions: test.IdentityOptions, + RootOptions: test.RootOptions, + MinVersion: test.MinVersion, + MaxVersion: test.MaxVersion, + } + clientConfig, err := clientOptions.config() + if err != nil { + t.Fatalf("ClientOptions{%v}.config() = %v, wantErr == nil", clientOptions, err) + } + // Verify that the system-provided certificates would be used + // when no verification method was set in clientOptions. + if clientOptions.RootOptions.RootCACerts == nil && + clientOptions.RootOptions.GetRootCertificates == nil && clientOptions.RootOptions.RootProvider == nil { + if clientConfig.RootCAs == nil { + t.Fatalf("Failed to assign system-provided certificates on the client side.") + } + } + }) + } +} + +func (s) TestServerOptionsConfigErrorCases(t *testing.T) { + tests := []struct { + desc string + requireClientCert bool + serverVType VerificationType + IdentityOptions IdentityCertificateOptions + RootOptions RootCertificateOptions + MinVersion uint16 + MaxVersion uint16 + }{ + { + desc: "Skip default verification and provide no root credentials", + requireClientCert: true, + serverVType: SkipVerification, + }, + { + desc: "More than one fields in RootCertificateOptions is specified", + requireClientCert: true, + serverVType: CertVerification, + RootOptions: RootCertificateOptions{ + RootCACerts: x509.NewCertPool(), + GetRootCertificates: func(*GetRootCAsParams) (*GetRootCAsResults, error) { + return nil, nil + }, + }, + }, + { + desc: "More than one fields in IdentityCertificateOptions is specified", + serverVType: CertVerification, + IdentityOptions: IdentityCertificateOptions{ + Certificates: []tls.Certificate{}, + IdentityProvider: fakeProvider{pt: provTypeIdentity}, + }, + }, + { + desc: "no field in IdentityCertificateOptions is specified", + serverVType: CertVerification, + }, + { + desc: "Specify GetIdentityCertificatesForClient", + IdentityOptions: IdentityCertificateOptions{ + GetIdentityCertificatesForClient: func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { + return nil, nil + }, + }, + }, + { + desc: "Invalid min/max TLS versions", + MinVersion: tls.VersionTLS13, + MaxVersion: tls.VersionTLS12, + }, + } + for _, test := range tests { + test := test + t.Run(test.desc, func(t *testing.T) { + serverOptions := &ServerOptions{ + VType: test.serverVType, + RequireClientCert: test.requireClientCert, + IdentityOptions: test.IdentityOptions, + RootOptions: test.RootOptions, + MinVersion: test.MinVersion, + MaxVersion: test.MaxVersion, + } + _, err := serverOptions.config() + if err == nil { + t.Fatalf("ServerOptions{%v}.config() returns no err, wantErr != nil", serverOptions) + } + }) + } +} + +func (s) TestServerOptionsConfigSuccessCases(t *testing.T) { + tests := []struct { + desc string + requireClientCert bool + serverVType VerificationType + IdentityOptions IdentityCertificateOptions + RootOptions RootCertificateOptions + MinVersion uint16 + MaxVersion uint16 + }{ + { + desc: "Use system default if no fields in RootCertificateOptions is specified", + requireClientCert: true, + serverVType: CertVerification, + IdentityOptions: IdentityCertificateOptions{ + Certificates: []tls.Certificate{}, + }, + }, + { + desc: "Good case with mutual TLS", + requireClientCert: true, + serverVType: CertVerification, + RootOptions: RootCertificateOptions{ + RootProvider: fakeProvider{}, + }, + IdentityOptions: IdentityCertificateOptions{ + GetIdentityCertificatesForServer: func(*tls.ClientHelloInfo) ([]*tls.Certificate, error) { + return nil, nil + }, + }, + MinVersion: tls.VersionTLS12, + MaxVersion: tls.VersionTLS13, + }, + } + for _, test := range tests { + test := test + t.Run(test.desc, func(t *testing.T) { + serverOptions := &ServerOptions{ + VType: test.serverVType, + RequireClientCert: test.requireClientCert, + IdentityOptions: test.IdentityOptions, + RootOptions: test.RootOptions, + MinVersion: test.MinVersion, + MaxVersion: test.MaxVersion, + } + serverConfig, err := serverOptions.config() + if err != nil { + t.Fatalf("ServerOptions{%v}.config() = %v, wantErr == nil", serverOptions, err) + } + // Verify that the system-provided certificates would be used + // when no verification method was set in serverOptions. + if serverOptions.RootOptions.RootCACerts == nil && + serverOptions.RootOptions.GetRootCertificates == nil && serverOptions.RootOptions.RootProvider == nil { + if serverConfig.ClientCAs == nil { + t.Fatalf("Failed to assign system-provided certificates on the server side.") + } + } + }) + } +} + func (s) TestClientServerHandshake(t *testing.T) { - // ------------------Load Client Trust Cert and Peer Cert------------------- - clientTrustPool, err := readTrustCert(testdata.Path("client_trust_cert_1.pem")) - if err != nil { - t.Fatalf("Client is unable to load trust certs. Error: %v", err) + cs := &testutils.CertStore{} + if err := cs.LoadCerts(); err != nil { + t.Fatalf("cs.LoadCerts() failed, err: %v", err) } getRootCAsForClient := func(params *GetRootCAsParams) (*GetRootCAsResults, error) { - return &GetRootCAsResults{TrustCerts: clientTrustPool}, nil + return &GetRootCAsResults{TrustCerts: cs.ClientTrust1}, nil } clientVerifyFuncGood := func(params *VerificationFuncParams) (*VerificationResults, error) { if params.ServerName == "" { @@ -69,18 +354,8 @@ func (s) TestClientServerHandshake(t *testing.T) { verifyFuncBad := func(params *VerificationFuncParams) (*VerificationResults, error) { return nil, fmt.Errorf("custom verification function failed") } - clientPeerCert, err := tls.LoadX509KeyPair(testdata.Path("client_cert_1.pem"), - testdata.Path("client_key_1.pem")) - if err != nil { - t.Fatalf("Client is unable to parse peer certificates. Error: %v", err) - } - // ------------------Load Server Trust Cert and Peer Cert------------------- - serverTrustPool, err := readTrustCert(testdata.Path("server_trust_cert_1.pem")) - if err != nil { - t.Fatalf("Server is unable to load trust certs. Error: %v", err) - } getRootCAsForServer := func(params *GetRootCAsParams) (*GetRootCAsResults, error) { - return &GetRootCAsResults{TrustCerts: serverTrustPool}, nil + return &GetRootCAsResults{TrustCerts: cs.ServerTrust1}, nil } serverVerifyFunc := func(params *VerificationFuncParams) (*VerificationResults, error) { if params.ServerName != "" { @@ -93,14 +368,13 @@ func (s) TestClientServerHandshake(t *testing.T) { return &VerificationResults{}, nil } - serverPeerCert, err := tls.LoadX509KeyPair(testdata.Path("server_cert_1.pem"), - testdata.Path("server_key_1.pem")) - if err != nil { - t.Fatalf("Server is unable to parse peer certificates. Error: %v", err) - } getRootCAsForServerBad := func(params *GetRootCAsParams) (*GetRootCAsResults, error) { return nil, fmt.Errorf("bad root certificate reloading") } + cache, err := lru.New(5) + if err != nil { + t.Fatalf("lru.New: err = %v", err) + } for _, test := range []struct { desc string clientCert []tls.Certificate @@ -109,7 +383,9 @@ func (s) TestClientServerHandshake(t *testing.T) { clientGetRoot func(params *GetRootCAsParams) (*GetRootCAsResults, error) clientVerifyFunc CustomVerificationFunc clientVType VerificationType - clientExpectCreateError bool + clientRootProvider certprovider.Provider + clientIdentityProvider certprovider.Provider + clientRevocationConfig *RevocationConfig clientExpectHandshakeError bool serverMutualTLS bool serverCert []tls.Certificate @@ -118,23 +394,11 @@ func (s) TestClientServerHandshake(t *testing.T) { serverGetRoot func(params *GetRootCAsParams) (*GetRootCAsResults, error) serverVerifyFunc CustomVerificationFunc serverVType VerificationType + serverRootProvider certprovider.Provider + serverIdentityProvider certprovider.Provider + serverRevocationConfig *RevocationConfig serverExpectError bool }{ - // Client: nil setting - // Server: only set serverCert with mutual TLS off - // Expected Behavior: server side failure - // Reason: if clientRoot, clientGetRoot and verifyFunc is not set, client - // side doesn't provide any verification mechanism. We don't allow this - // even setting vType to SkipVerification. Clients should at least provide - // their own verification logic. - { - desc: "Client has no trust cert; server sends peer cert", - clientVType: SkipVerification, - clientExpectCreateError: true, - serverCert: []tls.Certificate{serverPeerCert}, - serverVType: CertAndHostVerification, - serverExpectError: true, - }, // Client: nil setting except verifyFuncGood // Server: only set serverCert with mutual TLS off // Expected Behavior: success @@ -144,39 +408,9 @@ func (s) TestClientServerHandshake(t *testing.T) { desc: "Client has no trust cert with verifyFuncGood; server sends peer cert", clientVerifyFunc: clientVerifyFuncGood, clientVType: SkipVerification, - serverCert: []tls.Certificate{serverPeerCert}, + serverCert: []tls.Certificate{cs.ServerCert1}, serverVType: CertAndHostVerification, }, - // Client: only set clientRoot - // Server: only set serverCert with mutual TLS off - // Expected Behavior: server side failure and client handshake failure - // Reason: client side sets vType to CertAndHostVerification, and will do - // default hostname check. All the default hostname checks will fail in - // this test suites. - { - desc: "Client has root cert; server sends peer cert", - clientRoot: clientTrustPool, - clientVType: CertAndHostVerification, - clientExpectHandshakeError: true, - serverCert: []tls.Certificate{serverPeerCert}, - serverVType: CertAndHostVerification, - serverExpectError: true, - }, - // Client: only set clientGetRoot - // Server: only set serverCert with mutual TLS off - // Expected Behavior: server side failure and client handshake failure - // Reason: client side sets vType to CertAndHostVerification, and will do - // default hostname check. All the default hostname checks will fail in - // this test suites. - { - desc: "Client sets reload root function; server sends peer cert", - clientGetRoot: getRootCAsForClient, - clientVType: CertAndHostVerification, - clientExpectHandshakeError: true, - serverCert: []tls.Certificate{serverPeerCert}, - serverVType: CertAndHostVerification, - serverExpectError: true, - }, // Client: set clientGetRoot and clientVerifyFunc // Server: only set serverCert with mutual TLS off // Expected Behavior: success @@ -185,7 +419,7 @@ func (s) TestClientServerHandshake(t *testing.T) { clientGetRoot: getRootCAsForClient, clientVerifyFunc: clientVerifyFuncGood, clientVType: CertVerification, - serverCert: []tls.Certificate{serverPeerCert}, + serverCert: []tls.Certificate{cs.ServerCert1}, serverVType: CertAndHostVerification, }, // Client: set clientGetRoot and bad clientVerifyFunc function @@ -198,66 +432,35 @@ func (s) TestClientServerHandshake(t *testing.T) { clientVerifyFunc: verifyFuncBad, clientVType: CertVerification, clientExpectHandshakeError: true, - serverCert: []tls.Certificate{serverPeerCert}, + serverCert: []tls.Certificate{cs.ServerCert1}, serverVType: CertVerification, serverExpectError: true, }, - // Client: set clientGetRoot and clientVerifyFunc - // Server: nil setting - // Expected Behavior: server side failure - // Reason: server side must either set serverCert or serverGetCert - { - desc: "Client sets reload root function with verifyFuncGood; server sets nil", - clientGetRoot: getRootCAsForClient, - clientVerifyFunc: clientVerifyFuncGood, - clientVType: CertVerification, - serverVType: CertVerification, - serverExpectError: true, - }, // Client: set clientGetRoot, clientVerifyFunc and clientCert // Server: set serverRoot and serverCert with mutual TLS on // Expected Behavior: success { desc: "Client sets peer cert, reload root function with verifyFuncGood; server sets peer cert and root cert; mutualTLS", - clientCert: []tls.Certificate{clientPeerCert}, + clientCert: []tls.Certificate{cs.ClientCert1}, clientGetRoot: getRootCAsForClient, clientVerifyFunc: clientVerifyFuncGood, clientVType: CertVerification, serverMutualTLS: true, - serverCert: []tls.Certificate{serverPeerCert}, - serverRoot: serverTrustPool, + serverCert: []tls.Certificate{cs.ServerCert1}, + serverRoot: cs.ServerTrust1, serverVType: CertVerification, }, // Client: set clientGetRoot, clientVerifyFunc and clientCert - // Server: set serverCert, but not setting any of serverRoot, serverGetRoot - // or serverVerifyFunc, with mutual TLS on - // Expected Behavior: server side failure - // Reason: server side needs to provide any verification mechanism when - // mTLS in on, even setting vType to SkipVerification. Servers should at - // least provide their own verification logic. - { - desc: "Client sets peer cert, reload root function with verifyFuncGood; server sets no verification; mutualTLS", - clientCert: []tls.Certificate{clientPeerCert}, - clientGetRoot: getRootCAsForClient, - clientVerifyFunc: clientVerifyFuncGood, - clientVType: CertVerification, - clientExpectHandshakeError: true, - serverMutualTLS: true, - serverCert: []tls.Certificate{serverPeerCert}, - serverVType: SkipVerification, - serverExpectError: true, - }, - // Client: set clientGetRoot, clientVerifyFunc and clientCert // Server: set serverGetRoot and serverCert with mutual TLS on // Expected Behavior: success { desc: "Client sets peer cert, reload root function with verifyFuncGood; Server sets peer cert, reload root function; mutualTLS", - clientCert: []tls.Certificate{clientPeerCert}, + clientCert: []tls.Certificate{cs.ClientCert1}, clientGetRoot: getRootCAsForClient, clientVerifyFunc: clientVerifyFuncGood, clientVType: CertVerification, serverMutualTLS: true, - serverCert: []tls.Certificate{serverPeerCert}, + serverCert: []tls.Certificate{cs.ServerCert1}, serverGetRoot: getRootCAsForServer, serverVType: CertVerification, }, @@ -268,12 +471,12 @@ func (s) TestClientServerHandshake(t *testing.T) { // Reason: server side reloading returns failure { desc: "Client sets peer cert, reload root function with verifyFuncGood; Server sets peer cert, bad reload root function; mutualTLS", - clientCert: []tls.Certificate{clientPeerCert}, + clientCert: []tls.Certificate{cs.ClientCert1}, clientGetRoot: getRootCAsForClient, clientVerifyFunc: clientVerifyFuncGood, clientVType: CertVerification, serverMutualTLS: true, - serverCert: []tls.Certificate{serverPeerCert}, + serverCert: []tls.Certificate{cs.ServerCert1}, serverGetRoot: getRootCAsForServerBad, serverVType: CertVerification, serverExpectError: true, @@ -284,14 +487,14 @@ func (s) TestClientServerHandshake(t *testing.T) { { desc: "Client sets reload peer/root function with verifyFuncGood; Server sets reload peer/root function with verifyFuncGood; mutualTLS", clientGetCert: func(info *tls.CertificateRequestInfo) (*tls.Certificate, error) { - return &clientPeerCert, nil + return &cs.ClientCert1, nil }, clientGetRoot: getRootCAsForClient, clientVerifyFunc: clientVerifyFuncGood, clientVType: CertVerification, serverMutualTLS: true, serverGetCert: func(info *tls.ClientHelloInfo) ([]*tls.Certificate, error) { - return []*tls.Certificate{&serverPeerCert}, nil + return []*tls.Certificate{&cs.ServerCert1}, nil }, serverGetRoot: getRootCAsForServer, serverVerifyFunc: serverVerifyFunc, @@ -305,14 +508,14 @@ func (s) TestClientServerHandshake(t *testing.T) { { desc: "Client sends wrong peer cert; Server sets reload peer/root function with verifyFuncGood; mutualTLS", clientGetCert: func(info *tls.CertificateRequestInfo) (*tls.Certificate, error) { - return &serverPeerCert, nil + return &cs.ServerCert1, nil }, clientGetRoot: getRootCAsForClient, clientVerifyFunc: clientVerifyFuncGood, clientVType: CertVerification, serverMutualTLS: true, serverGetCert: func(info *tls.ClientHelloInfo) ([]*tls.Certificate, error) { - return []*tls.Certificate{&serverPeerCert}, nil + return []*tls.Certificate{&cs.ServerCert1}, nil }, serverGetRoot: getRootCAsForServer, serverVerifyFunc: serverVerifyFunc, @@ -326,7 +529,7 @@ func (s) TestClientServerHandshake(t *testing.T) { { desc: "Client has wrong trust cert; Server sets reload peer/root function with verifyFuncGood; mutualTLS", clientGetCert: func(info *tls.CertificateRequestInfo) (*tls.Certificate, error) { - return &clientPeerCert, nil + return &cs.ClientCert1, nil }, clientGetRoot: getRootCAsForServer, clientVerifyFunc: clientVerifyFuncGood, @@ -334,7 +537,7 @@ func (s) TestClientServerHandshake(t *testing.T) { clientExpectHandshakeError: true, serverMutualTLS: true, serverGetCert: func(info *tls.ClientHelloInfo) ([]*tls.Certificate, error) { - return []*tls.Certificate{&serverPeerCert}, nil + return []*tls.Certificate{&cs.ServerCert1}, nil }, serverGetRoot: getRootCAsForServer, serverVerifyFunc: serverVerifyFunc, @@ -349,14 +552,14 @@ func (s) TestClientServerHandshake(t *testing.T) { { desc: "Client sets reload peer/root function with verifyFuncGood; Server sends wrong peer cert; mutualTLS", clientGetCert: func(info *tls.CertificateRequestInfo) (*tls.Certificate, error) { - return &clientPeerCert, nil + return &cs.ClientCert1, nil }, clientGetRoot: getRootCAsForClient, clientVerifyFunc: clientVerifyFuncGood, clientVType: CertVerification, serverMutualTLS: true, serverGetCert: func(info *tls.ClientHelloInfo) ([]*tls.Certificate, error) { - return []*tls.Certificate{&clientPeerCert}, nil + return []*tls.Certificate{&cs.ClientCert1}, nil }, serverGetRoot: getRootCAsForServer, serverVerifyFunc: serverVerifyFunc, @@ -370,7 +573,7 @@ func (s) TestClientServerHandshake(t *testing.T) { { desc: "Client sets reload peer/root function with verifyFuncGood; Server has wrong trust cert; mutualTLS", clientGetCert: func(info *tls.CertificateRequestInfo) (*tls.Certificate, error) { - return &clientPeerCert, nil + return &cs.ClientCert1, nil }, clientGetRoot: getRootCAsForClient, clientVerifyFunc: clientVerifyFuncGood, @@ -378,7 +581,7 @@ func (s) TestClientServerHandshake(t *testing.T) { clientExpectHandshakeError: true, serverMutualTLS: true, serverGetCert: func(info *tls.ClientHelloInfo) ([]*tls.Certificate, error) { - return []*tls.Certificate{&serverPeerCert}, nil + return []*tls.Certificate{&cs.ServerCert1}, nil }, serverGetRoot: getRootCAsForClient, serverVerifyFunc: serverVerifyFunc, @@ -391,18 +594,116 @@ func (s) TestClientServerHandshake(t *testing.T) { // server custom check fails { desc: "Client sets peer cert, reload root function with verifyFuncGood; Server sets bad custom check; mutualTLS", - clientCert: []tls.Certificate{clientPeerCert}, + clientCert: []tls.Certificate{cs.ClientCert1}, clientGetRoot: getRootCAsForClient, clientVerifyFunc: clientVerifyFuncGood, clientVType: CertVerification, clientExpectHandshakeError: true, serverMutualTLS: true, - serverCert: []tls.Certificate{serverPeerCert}, + serverCert: []tls.Certificate{cs.ServerCert1}, serverGetRoot: getRootCAsForServer, serverVerifyFunc: verifyFuncBad, serverVType: CertVerification, serverExpectError: true, }, + // Client: set a clientIdentityProvider which will get multiple cert chains + // Server: set serverIdentityProvider and serverRootProvider with mutual TLS on + // Expected Behavior: server side failure due to multiple cert chains in + // clientIdentityProvider + { + desc: "Client sets multiple certs in clientIdentityProvider; Server sets root and identity provider; mutualTLS", + clientIdentityProvider: fakeProvider{pt: provTypeIdentity, isClient: true, wantMultiCert: true}, + clientRootProvider: fakeProvider{isClient: true}, + clientVerifyFunc: clientVerifyFuncGood, + clientVType: CertVerification, + serverMutualTLS: true, + serverIdentityProvider: fakeProvider{pt: provTypeIdentity, isClient: false}, + serverRootProvider: fakeProvider{isClient: false}, + serverVType: CertVerification, + serverExpectError: true, + }, + // Client: set a bad clientIdentityProvider + // Server: set serverIdentityProvider and serverRootProvider with mutual TLS on + // Expected Behavior: server side failure due to bad clientIdentityProvider + { + desc: "Client sets bad clientIdentityProvider; Server sets root and identity provider; mutualTLS", + clientIdentityProvider: fakeProvider{pt: provTypeIdentity, isClient: true, wantError: true}, + clientRootProvider: fakeProvider{isClient: true}, + clientVerifyFunc: clientVerifyFuncGood, + clientVType: CertVerification, + serverMutualTLS: true, + serverIdentityProvider: fakeProvider{pt: provTypeIdentity, isClient: false}, + serverRootProvider: fakeProvider{isClient: false}, + serverVType: CertVerification, + serverExpectError: true, + }, + // Client: set clientIdentityProvider and clientRootProvider + // Server: set bad serverRootProvider with mutual TLS on + // Expected Behavior: server side failure due to bad serverRootProvider + { + desc: "Client sets root and identity provider; Server sets bad root provider; mutualTLS", + clientIdentityProvider: fakeProvider{pt: provTypeIdentity, isClient: true}, + clientRootProvider: fakeProvider{isClient: true}, + clientVerifyFunc: clientVerifyFuncGood, + clientVType: CertVerification, + serverMutualTLS: true, + serverIdentityProvider: fakeProvider{pt: provTypeIdentity, isClient: false}, + serverRootProvider: fakeProvider{isClient: false, wantError: true}, + serverVType: CertVerification, + serverExpectError: true, + }, + // Client: set clientIdentityProvider and clientRootProvider + // Server: set serverIdentityProvider and serverRootProvider with mutual TLS on + // Expected Behavior: success + { + desc: "Client sets root and identity provider; Server sets root and identity provider; mutualTLS", + clientIdentityProvider: fakeProvider{pt: provTypeIdentity, isClient: true}, + clientRootProvider: fakeProvider{isClient: true}, + clientVerifyFunc: clientVerifyFuncGood, + clientVType: CertVerification, + serverMutualTLS: true, + serverIdentityProvider: fakeProvider{pt: provTypeIdentity, isClient: false}, + serverRootProvider: fakeProvider{isClient: false}, + serverVType: CertVerification, + }, + // Client: set clientIdentityProvider and clientRootProvider + // Server: set serverIdentityProvider getting multiple cert chains and serverRootProvider with mutual TLS on + // Expected Behavior: success, because server side has SNI + { + desc: "Client sets root and identity provider; Server sets multiple certs in serverIdentityProvider; mutualTLS", + clientIdentityProvider: fakeProvider{pt: provTypeIdentity, isClient: true}, + clientRootProvider: fakeProvider{isClient: true}, + clientVerifyFunc: clientVerifyFuncGood, + clientVType: CertVerification, + serverMutualTLS: true, + serverIdentityProvider: fakeProvider{pt: provTypeIdentity, isClient: false, wantMultiCert: true}, + serverRootProvider: fakeProvider{isClient: false}, + serverVType: CertVerification, + }, + // Client: set valid credentials with the revocation config + // Server: set valid credentials with the revocation config + // Expected Behavior: success, because non of the certificate chains sent in the connection are revoked + { + desc: "Client sets peer cert, reload root function with verifyFuncGood; Server sets peer cert, reload root function; mutualTLS", + clientCert: []tls.Certificate{cs.ClientCert1}, + clientGetRoot: getRootCAsForClient, + clientVerifyFunc: clientVerifyFuncGood, + clientVType: CertVerification, + clientRevocationConfig: &RevocationConfig{ + RootDir: testdata.Path("crl"), + AllowUndetermined: true, + Cache: cache, + }, + serverMutualTLS: true, + serverCert: []tls.Certificate{cs.ServerCert1}, + serverGetRoot: getRootCAsForServer, + serverVType: CertVerification, + serverRevocationConfig: &RevocationConfig{ + RootDir: testdata.Path("crl"), + AllowUndetermined: true, + Cache: cache, + }, + }, } { test := test t.Run(test.desc, func(t *testing.T) { @@ -413,15 +714,20 @@ func (s) TestClientServerHandshake(t *testing.T) { } // Start a server using ServerOptions in another goroutine. serverOptions := &ServerOptions{ - Certificates: test.serverCert, - GetCertificates: test.serverGetCert, - RootCertificateOptions: RootCertificateOptions{ - RootCACerts: test.serverRoot, - GetRootCAs: test.serverGetRoot, + IdentityOptions: IdentityCertificateOptions{ + Certificates: test.serverCert, + GetIdentityCertificatesForServer: test.serverGetCert, + IdentityProvider: test.serverIdentityProvider, + }, + RootOptions: RootCertificateOptions{ + RootCACerts: test.serverRoot, + GetRootCertificates: test.serverGetRoot, + RootProvider: test.serverRootProvider, }, RequireClientCert: test.serverMutualTLS, VerifyPeer: test.serverVerifyFunc, VType: test.serverVType, + RevocationConfig: test.serverRevocationConfig, } go func(done chan credentials.AuthInfo, lis net.Listener, serverOptions *ServerOptions) { serverRawConn, err := lis.Accept() @@ -452,23 +758,23 @@ func (s) TestClientServerHandshake(t *testing.T) { } defer conn.Close() clientOptions := &ClientOptions{ - Certificates: test.clientCert, - GetClientCertificate: test.clientGetCert, - VerifyPeer: test.clientVerifyFunc, - RootCertificateOptions: RootCertificateOptions{ - RootCACerts: test.clientRoot, - GetRootCAs: test.clientGetRoot, + IdentityOptions: IdentityCertificateOptions{ + Certificates: test.clientCert, + GetIdentityCertificatesForClient: test.clientGetCert, + IdentityProvider: test.clientIdentityProvider, }, - VType: test.clientVType, - } - clientTLS, newClientErr := NewClientCreds(clientOptions) - if newClientErr != nil && test.clientExpectCreateError { - return + VerifyPeer: test.clientVerifyFunc, + RootOptions: RootCertificateOptions{ + RootCACerts: test.clientRoot, + GetRootCertificates: test.clientGetRoot, + RootProvider: test.clientRootProvider, + }, + VType: test.clientVType, + RevocationConfig: test.clientRevocationConfig, } - if newClientErr != nil && !test.clientExpectCreateError || - newClientErr == nil && test.clientExpectCreateError { - t.Fatalf("Expect error: %v, but err is %v", - test.clientExpectCreateError, newClientErr) + clientTLS, err := NewClientCreds(clientOptions) + if err != nil { + t.Fatalf("NewClientCreds failed: %v", err) } _, clientAuthInfo, handshakeErr := clientTLS.ClientHandshake(context.Background(), lisAddr, conn) @@ -496,24 +802,6 @@ func (s) TestClientServerHandshake(t *testing.T) { } } -func readTrustCert(fileName string) (*x509.CertPool, error) { - trustData, err := ioutil.ReadFile(fileName) - if err != nil { - return nil, err - } - trustBlock, _ := pem.Decode(trustData) - if trustBlock == nil { - return nil, err - } - trustCert, err := x509.ParseCertificate(trustBlock.Bytes) - if err != nil { - return nil, err - } - trustPool := x509.NewCertPool() - trustPool.AddCert(trustCert) - return trustPool, nil -} - func compare(a1, a2 credentials.AuthInfo) bool { if a1.AuthType() != a2.AuthType() { return false @@ -536,13 +824,13 @@ func compare(a1, a2 credentials.AuthInfo) bool { func (s) TestAdvancedTLSOverrideServerName(t *testing.T) { expectedServerName := "server.name" - clientTrustPool, err := readTrustCert(testdata.Path("client_trust_cert_1.pem")) - if err != nil { - t.Fatalf("Client is unable to load trust certs. Error: %v", err) + cs := &testutils.CertStore{} + if err := cs.LoadCerts(); err != nil { + t.Fatalf("cs.LoadCerts() failed, err: %v", err) } clientOptions := &ClientOptions{ - RootCertificateOptions: RootCertificateOptions{ - RootCACerts: clientTrustPool, + RootOptions: RootCertificateOptions{ + RootCACerts: cs.ClientTrust1, }, ServerNameOverride: expectedServerName, } @@ -556,186 +844,39 @@ func (s) TestAdvancedTLSOverrideServerName(t *testing.T) { } } -func (s) TestTLSClone(t *testing.T) { - expectedServerName := "server.name" - clientTrustPool, err := readTrustCert(testdata.Path("client_trust_cert_1.pem")) - if err != nil { - t.Fatalf("Client is unable to load trust certs. Error: %v", err) - } - clientOptions := &ClientOptions{ - RootCertificateOptions: RootCertificateOptions{ - RootCACerts: clientTrustPool, - }, - ServerNameOverride: expectedServerName, - } - c, err := NewClientCreds(clientOptions) - if err != nil { - t.Fatalf("Failed to create new client: %v", err) - } - cc := c.Clone() - if cc.Info().ServerName != expectedServerName { - t.Fatalf("cc.Info().ServerName = %v, want %v", cc.Info().ServerName, expectedServerName) - } - cc.OverrideServerName("") - if c.Info().ServerName != expectedServerName { - t.Fatalf("Change in clone should not affect the original, "+ - "c.Info().ServerName = %v, want %v", c.Info().ServerName, expectedServerName) - } - -} - -func (s) TestAppendH2ToNextProtos(t *testing.T) { - tests := []struct { - name string - ps []string - want []string - }{ - { - name: "empty", - ps: nil, - want: []string{"h2"}, - }, - { - name: "only h2", - ps: []string{"h2"}, - want: []string{"h2"}, - }, - { - name: "with h2", - ps: []string{"alpn", "h2"}, - want: []string{"alpn", "h2"}, - }, - { - name: "no h2", - ps: []string{"alpn"}, - want: []string{"alpn", "h2"}, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := appendH2ToNextProtos(tt.ps); !reflect.DeepEqual(got, tt.want) { - t.Errorf("appendH2ToNextProtos() = %v, want %v", got, tt.want) - } - }) - } -} - -type nonSyscallConn struct { - net.Conn -} - -func (s) TestWrapSyscallConn(t *testing.T) { - sc := &syscallConn{} - nsc := &nonSyscallConn{} - - wrapConn := WrapSyscallConn(sc, nsc) - if _, ok := wrapConn.(syscall.Conn); !ok { - t.Errorf("returned conn (type %T) doesn't implement syscall.Conn, want implement", - wrapConn) - } -} - -func (s) TestOptionsConfig(t *testing.T) { - serverPeerCert, err := tls.LoadX509KeyPair(testdata.Path("server_cert_1.pem"), - testdata.Path("server_key_1.pem")) - if err != nil { - t.Fatalf("Server is unable to parse peer certificates. Error: %v", err) - } - tests := []struct { - desc string - clientVType VerificationType - serverMutualTLS bool - serverCert []tls.Certificate - serverVType VerificationType - }{ - { - desc: "Client uses system-provided RootCAs; server uses system-provided ClientCAs", - clientVType: CertVerification, - serverMutualTLS: true, - serverCert: []tls.Certificate{serverPeerCert}, - serverVType: CertAndHostVerification, - }, - } - for _, test := range tests { - test := test - t.Run(test.desc, func(t *testing.T) { - serverOptions := &ServerOptions{ - Certificates: test.serverCert, - RequireClientCert: test.serverMutualTLS, - VType: test.serverVType, - } - serverConfig, err := serverOptions.config() - if err != nil { - t.Fatalf("Unable to generate serverConfig. Error: %v", err) - } - // Verify that the system-provided certificates would be used - // when no verification method was set in serverOptions. - if serverOptions.RootCACerts == nil && serverOptions.GetRootCAs == nil && - serverOptions.RequireClientCert && serverConfig.ClientCAs == nil { - t.Fatalf("Failed to assign system-provided certificates on the server side.") - } - clientOptions := &ClientOptions{ - VType: test.clientVType, - } - clientConfig, err := clientOptions.config() - if err != nil { - t.Fatalf("Unable to generate clientConfig. Error: %v", err) - } - // Verify that the system-provided certificates would be used - // when no verification method was set in clientOptions. - if clientOptions.RootCACerts == nil && clientOptions.GetRootCAs == nil && - clientConfig.RootCAs == nil { - t.Fatalf("Failed to assign system-provided certificates on the client side.") - } - }) - } -} - func (s) TestGetCertificatesSNI(t *testing.T) { - // Load server certificates for setting the serverGetCert callback function. - serverCert1, err := tls.LoadX509KeyPair(testdata.Path("server_cert_1.pem"), testdata.Path("server_key_1.pem")) - if err != nil { - t.Fatalf("tls.LoadX509KeyPair(server_cert_1.pem, server_key_1.pem) failed: %v", err) - } - serverCert2, err := tls.LoadX509KeyPair(testdata.Path("server_cert_2.pem"), testdata.Path("server_key_2.pem")) - if err != nil { - t.Fatalf("tls.LoadX509KeyPair(server_cert_2.pem, server_key_2.pem) failed: %v", err) + cs := &testutils.CertStore{} + if err := cs.LoadCerts(); err != nil { + t.Fatalf("cs.LoadCerts() failed, err: %v", err) } - serverCert3, err := tls.LoadX509KeyPair(testdata.Path("server_cert_3.pem"), testdata.Path("server_key_3.pem")) - if err != nil { - t.Fatalf("tls.LoadX509KeyPair(server_cert_3.pem, server_key_3.pem) failed: %v", err) - } - tests := []struct { desc string serverName string - wantCert tls.Certificate + // Use Common Name on the certificate to differentiate if we choose the right cert. The common name on all of the three certs are different. + wantCommonName string }{ { - desc: "Select serverCert1", + desc: "Select ServerCert1", // "foo.bar.com" is the common name on server certificate server_cert_1.pem. - serverName: "foo.bar.com", - wantCert: serverCert1, - }, - { - desc: "Select serverCert2", - // "foo.bar.server2.com" is the common name on server certificate server_cert_2.pem. - serverName: "foo.bar.server2.com", - wantCert: serverCert2, + serverName: "foo.bar.com", + wantCommonName: "foo.bar.com", }, { desc: "Select serverCert3", + // "foo.bar.server3.com" is the common name on server certificate server_cert_3.pem. // "google.com" is one of the DNS names on server certificate server_cert_3.pem. - serverName: "google.com", - wantCert: serverCert3, + serverName: "google.com", + wantCommonName: "foo.bar.server3.com", }, } for _, test := range tests { test := test t.Run(test.desc, func(t *testing.T) { serverOptions := &ServerOptions{ - GetCertificates: func(info *tls.ClientHelloInfo) ([]*tls.Certificate, error) { - return []*tls.Certificate{&serverCert1, &serverCert2, &serverCert3}, nil + IdentityOptions: IdentityCertificateOptions{ + GetIdentityCertificatesForServer: func(info *tls.ClientHelloInfo) ([]*tls.Certificate, error) { + return []*tls.Certificate{&cs.ServerCert1, &cs.ServerCert2, &cs.ServerPeer3}, nil + }, }, } serverConfig, err := serverOptions.config() @@ -754,8 +895,18 @@ func (s) TestGetCertificatesSNI(t *testing.T) { if err != nil { t.Fatalf("serverConfig.GetCertificate(clientHello) failed: %v", err) } - if !cmp.Equal(*gotCertificate, test.wantCert, cmp.AllowUnexported(big.Int{})) { - t.Errorf("GetCertificates() = %v, want %v", *gotCertificate, test.wantCert) + if gotCertificate == nil || len(gotCertificate.Certificate) == 0 { + t.Fatalf("Got nil or empty Certificate after calling serverConfig.GetCertificate.") + } + parsedCert, err := x509.ParseCertificate(gotCertificate.Certificate[0]) + if err != nil { + t.Fatalf("x509.ParseCertificate(%v) failed: %v", gotCertificate.Certificate[0], err) + } + if parsedCert == nil { + t.Fatalf("Got nil Certificate after calling x509.ParseCertificate.") + } + if parsedCert.Subject.CommonName != test.wantCommonName { + t.Errorf("Common name mismatch, got %v, want %v", parsedCert.Subject.CommonName, test.wantCommonName) } }) } diff --git a/security/advancedtls/crl.go b/security/advancedtls/crl.go new file mode 100644 index 000000000000..207c7f81a2ef --- /dev/null +++ b/security/advancedtls/crl.go @@ -0,0 +1,548 @@ +// TODO(@gregorycooke) - Remove when only golang 1.19+ is supported +//go:build go1.19 + +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package advancedtls + +import ( + "bytes" + "crypto/sha1" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/asn1" + "encoding/binary" + "encoding/hex" + "encoding/pem" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "golang.org/x/crypto/cryptobyte" + cbasn1 "golang.org/x/crypto/cryptobyte/asn1" + "google.golang.org/grpc/grpclog" +) + +var grpclogLogger = grpclog.Component("advancedtls") + +// Cache is an interface to cache CRL files. +// The cache implementation must be concurrency safe. +// A fixed size lru cache from golang-lru is recommended. +type Cache interface { + // Add adds a value to the cache. + Add(key, value any) bool + // Get looks up a key's value from the cache. + Get(key any) (value any, ok bool) +} + +// RevocationConfig contains options for CRL lookup. +type RevocationConfig struct { + // RootDir is the directory to search for CRL files. + // Directory format must match OpenSSL X509_LOOKUP_hash_dir(3). + RootDir string + // AllowUndetermined controls if certificate chains with RevocationUndetermined + // revocation status are allowed to complete. + AllowUndetermined bool + // Cache will store CRL files if not nil, otherwise files are reloaded for every lookup. + Cache Cache +} + +// RevocationStatus is the revocation status for a certificate or chain. +type RevocationStatus int + +const ( + // RevocationUndetermined means we couldn't find or verify a CRL for the cert. + RevocationUndetermined RevocationStatus = iota + // RevocationUnrevoked means we found the CRL for the cert and the cert is not revoked. + RevocationUnrevoked + // RevocationRevoked means we found the CRL and the cert is revoked. + RevocationRevoked +) + +func (s RevocationStatus) String() string { + return [...]string{"RevocationUndetermined", "RevocationUnrevoked", "RevocationRevoked"}[s] +} + +// certificateListExt contains a pkix.CertificateList and parsed +// extensions that aren't provided by the golang CRL parser. +type certificateListExt struct { + CertList *x509.RevocationList + // RFC5280, 5.2.1, all conforming CRLs must have a AKID with the ID method. + AuthorityKeyID []byte + RawIssuer []byte +} + +const tagDirectoryName = 4 + +var ( + // RFC5280, 5.2.4 id-ce-deltaCRLIndicator OBJECT IDENTIFIER ::= { id-ce 27 } + oidDeltaCRLIndicator = asn1.ObjectIdentifier{2, 5, 29, 27} + // RFC5280, 5.2.5 id-ce-issuingDistributionPoint OBJECT IDENTIFIER ::= { id-ce 28 } + oidIssuingDistributionPoint = asn1.ObjectIdentifier{2, 5, 29, 28} + // RFC5280, 5.3.3 id-ce-certificateIssuer OBJECT IDENTIFIER ::= { id-ce 29 } + oidCertificateIssuer = asn1.ObjectIdentifier{2, 5, 29, 29} + // RFC5290, 4.2.1.1 id-ce-authorityKeyIdentifier OBJECT IDENTIFIER ::= { id-ce 35 } + oidAuthorityKeyIdentifier = asn1.ObjectIdentifier{2, 5, 29, 35} +) + +// x509NameHash implements the OpenSSL X509_NAME_hash function for hashed directory lookups. +// +// NOTE: due to the behavior of asn1.Marshal, if the original encoding of the RDN sequence +// contains strings which do not use the ASN.1 PrintableString type, the name will not be +// re-encoded using those types, resulting in a hash which does not match that produced +// by OpenSSL. +func x509NameHash(r pkix.RDNSequence) string { + var canonBytes []byte + // First, canonicalize all the strings. + for _, rdnSet := range r { + for i, rdn := range rdnSet { + value, ok := rdn.Value.(string) + if !ok { + continue + } + // OpenSSL trims all whitespace, does a tolower, and removes extra spaces between words. + // Implemented in x509_name_canon in OpenSSL + canonStr := strings.Join(strings.Fields( + strings.TrimSpace(strings.ToLower(value))), " ") + // Then it changes everything to UTF8 strings + rdnSet[i].Value = asn1.RawValue{Tag: asn1.TagUTF8String, Bytes: []byte(canonStr)} + + } + } + + // Finally, OpenSSL drops the initial sequence tag + // so we marshal all the RDNs separately instead of as a group. + for _, canonRdn := range r { + b, err := asn1.Marshal(canonRdn) + if err != nil { + continue + } + canonBytes = append(canonBytes, b...) + } + + issuerHash := sha1.Sum(canonBytes) + // Openssl takes the first 4 bytes and encodes them as a little endian + // uint32 and then uses the hex to make the file name. + // In C++, this would be: + // (((unsigned long)md[0]) | ((unsigned long)md[1] << 8L) | + // ((unsigned long)md[2] << 16L) | ((unsigned long)md[3] << 24L) + // ) & 0xffffffffL; + fileHash := binary.LittleEndian.Uint32(issuerHash[0:4]) + return fmt.Sprintf("%08x", fileHash) +} + +// CheckRevocation checks the connection for revoked certificates based on RFC5280. +// This implementation has the following major limitations: +// - Indirect CRL files are not supported. +// - CRL loading is only supported from directories in the X509_LOOKUP_hash_dir format. +// - OnlySomeReasons is not supported. +// - Delta CRL files are not supported. +// - Certificate CRLDistributionPoint must be URLs, but are then ignored and converted into a file path. +// - CRL checks are done after path building, which goes against RFC4158. +func CheckRevocation(conn tls.ConnectionState, cfg RevocationConfig) error { + return CheckChainRevocation(conn.VerifiedChains, cfg) +} + +// CheckChainRevocation checks the verified certificate chain +// for revoked certificates based on RFC5280. +func CheckChainRevocation(verifiedChains [][]*x509.Certificate, cfg RevocationConfig) error { + // Iterate the verified chains looking for one that is RevocationUnrevoked. + // A single RevocationUnrevoked chain is enough to allow the connection, and a single RevocationRevoked + // chain does not mean the connection should fail. + count := make(map[RevocationStatus]int) + for _, chain := range verifiedChains { + switch checkChain(chain, cfg) { + case RevocationUnrevoked: + // If any chain is RevocationUnrevoked then return no error. + return nil + case RevocationRevoked: + // If this chain is revoked, keep looking for another chain. + count[RevocationRevoked]++ + continue + case RevocationUndetermined: + if cfg.AllowUndetermined { + return nil + } + count[RevocationUndetermined]++ + continue + } + } + return fmt.Errorf("no unrevoked chains found: %v", count) +} + +// checkChain will determine and check all certificates in chain against the CRL +// defined in the certificate with the following rules: +// 1. If any certificate is RevocationRevoked, return RevocationRevoked. +// 2. If any certificate is RevocationUndetermined, return RevocationUndetermined. +// 3. If all certificates are RevocationUnrevoked, return RevocationUnrevoked. +func checkChain(chain []*x509.Certificate, cfg RevocationConfig) RevocationStatus { + chainStatus := RevocationUnrevoked + for _, c := range chain { + switch checkCert(c, chain, cfg) { + case RevocationRevoked: + // Easy case, if a cert in the chain is revoked, the chain is revoked. + return RevocationRevoked + case RevocationUndetermined: + // If we couldn't find the revocation status for a cert, the chain is at best RevocationUndetermined + // keep looking to see if we find a cert in the chain that's RevocationRevoked, + // but return RevocationUndetermined at a minimum. + chainStatus = RevocationUndetermined + case RevocationUnrevoked: + // Continue iterating up the cert chain. + continue + } + } + return chainStatus +} + +func cachedCrl(rawIssuer []byte, cache Cache) (*certificateListExt, bool) { + val, ok := cache.Get(hex.EncodeToString(rawIssuer)) + if !ok { + return nil, false + } + crl, ok := val.(*certificateListExt) + if !ok { + return nil, false + } + // If the CRL is expired, force a reload. + if hasExpired(crl.CertList, time.Now()) { + return nil, false + } + return crl, true +} + +// fetchIssuerCRL fetches and verifies the CRL for rawIssuer from disk or cache if configured in cfg. +func fetchIssuerCRL(rawIssuer []byte, crlVerifyCrt []*x509.Certificate, cfg RevocationConfig) (*certificateListExt, error) { + if cfg.Cache != nil { + if crl, ok := cachedCrl(rawIssuer, cfg.Cache); ok { + return crl, nil + } + } + + crl, err := fetchCRL(rawIssuer, cfg) + if err != nil { + return nil, fmt.Errorf("fetchCRL() failed: %v", err) + } + + if err := verifyCRL(crl, rawIssuer, crlVerifyCrt); err != nil { + return nil, fmt.Errorf("verifyCRL() failed: %v", err) + } + if cfg.Cache != nil { + cfg.Cache.Add(hex.EncodeToString(rawIssuer), crl) + } + return crl, nil +} + +// checkCert checks a single certificate against the CRL defined in the certificate. +// It will fetch and verify the CRL(s) defined in the root directory specified by cfg. +// If we can't load any authoritative CRL files, the status is RevocationUndetermined. +// c is the certificate to check. +// crlVerifyCrt is the group of possible certificates to verify the crl. +func checkCert(c *x509.Certificate, crlVerifyCrt []*x509.Certificate, cfg RevocationConfig) RevocationStatus { + crl, err := fetchIssuerCRL(c.RawIssuer, crlVerifyCrt, cfg) + if err != nil { + // We couldn't load any CRL files for the certificate, so we don't know if it's RevocationUnrevoked or not. + grpclogLogger.Warningf("getIssuerCRL(%v) err = %v", c.Issuer, err) + return RevocationUndetermined + } + revocation, err := checkCertRevocation(c, crl) + if err != nil { + grpclogLogger.Warningf("checkCertRevocation(CRL %v) failed: %v", crl.CertList.Issuer, err) + // We couldn't check the CRL file for some reason, so we don't know if it's RevocationUnrevoked or not. + return RevocationUndetermined + } + // Here we've gotten a CRL that loads and verifies. + // We only handle all-reasons CRL files, so this file + // is authoritative for the certificate. + return revocation +} + +func checkCertRevocation(c *x509.Certificate, crl *certificateListExt) (RevocationStatus, error) { + // Per section 5.3.3 we prime the certificate issuer with the CRL issuer. + // Subsequent entries use the previous entry's issuer. + rawEntryIssuer := crl.RawIssuer + + // Loop through all the revoked certificates. + for _, revCert := range crl.CertList.RevokedCertificates { + // 5.3 Loop through CRL entry extensions for needed information. + for _, ext := range revCert.Extensions { + if oidCertificateIssuer.Equal(ext.Id) { + extIssuer, err := parseCertIssuerExt(ext) + if err != nil { + grpclogLogger.Info(err) + if ext.Critical { + return RevocationUndetermined, err + } + // Since this is a non-critical extension, we can skip it even though + // there was a parsing failure. + continue + } + rawEntryIssuer = extIssuer + } else if ext.Critical { + return RevocationUndetermined, fmt.Errorf("checkCertRevocation: Unhandled critical extension: %v", ext.Id) + } + } + + // If the issuer and serial number appear in the CRL, the certificate is revoked. + if bytes.Equal(c.RawIssuer, rawEntryIssuer) && c.SerialNumber.Cmp(revCert.SerialNumber) == 0 { + // CRL contains the serial, so return revoked. + return RevocationRevoked, nil + } + } + // We did not find the serial in the CRL file that was valid for the cert + // so the certificate is not revoked. + return RevocationUnrevoked, nil +} + +func parseCertIssuerExt(ext pkix.Extension) ([]byte, error) { + // 5.3.3 Certificate Issuer + // CertificateIssuer ::= GeneralNames + // GeneralNames ::= SEQUENCE SIZE (1..MAX) OF GeneralName + var generalNames []asn1.RawValue + if rest, err := asn1.Unmarshal(ext.Value, &generalNames); err != nil || len(rest) != 0 { + return nil, fmt.Errorf("asn1.Unmarshal failed: %v", err) + } + + for _, generalName := range generalNames { + // GeneralName ::= CHOICE { + // otherName [0] OtherName, + // rfc822Name [1] IA5String, + // dNSName [2] IA5String, + // x400Address [3] ORAddress, + // directoryName [4] Name, + // ediPartyName [5] EDIPartyName, + // uniformResourceIdentifier [6] IA5String, + // iPAddress [7] OCTET STRING, + // registeredID [8] OBJECT IDENTIFIER } + if generalName.Tag == tagDirectoryName { + return generalName.Bytes, nil + } + } + // Conforming CRL issuers MUST include in this extension the + // distinguished name (DN) from the issuer field of the certificate that + // corresponds to this CRL entry. + // If we couldn't get a directoryName, we can't reason about this file so cert status is + // RevocationUndetermined. + return nil, errors.New("no DN found in certificate issuer") +} + +// RFC 5280, 4.2.1.1 +type authKeyID struct { + ID []byte `asn1:"optional,tag:0"` +} + +// RFC5280, 5.2.5 +// id-ce-issuingDistributionPoint OBJECT IDENTIFIER ::= { id-ce 28 } + +// IssuingDistributionPoint ::= SEQUENCE { +// distributionPoint [0] DistributionPointName OPTIONAL, +// onlyContainsUserCerts [1] BOOLEAN DEFAULT FALSE, +// onlyContainsCACerts [2] BOOLEAN DEFAULT FALSE, +// onlySomeReasons [3] ReasonFlags OPTIONAL, +// indirectCRL [4] BOOLEAN DEFAULT FALSE, +// onlyContainsAttributeCerts [5] BOOLEAN DEFAULT FALSE } + +// -- at most one of onlyContainsUserCerts, onlyContainsCACerts, +// -- and onlyContainsAttributeCerts may be set to TRUE. +type issuingDistributionPoint struct { + DistributionPoint asn1.RawValue `asn1:"optional,tag:0"` + OnlyContainsUserCerts bool `asn1:"optional,tag:1"` + OnlyContainsCACerts bool `asn1:"optional,tag:2"` + OnlySomeReasons asn1.BitString `asn1:"optional,tag:3"` + IndirectCRL bool `asn1:"optional,tag:4"` + OnlyContainsAttributeCerts bool `asn1:"optional,tag:5"` +} + +// parseCRLExtensions parses the extensions for a CRL +// and checks that they're supported by the parser. +func parseCRLExtensions(c *x509.RevocationList) (*certificateListExt, error) { + if c == nil { + return nil, errors.New("c is nil, expected any value") + } + certList := &certificateListExt{CertList: c} + + for _, ext := range c.Extensions { + switch { + case oidDeltaCRLIndicator.Equal(ext.Id): + return nil, fmt.Errorf("delta CRLs unsupported") + + case oidAuthorityKeyIdentifier.Equal(ext.Id): + var a authKeyID + if rest, err := asn1.Unmarshal(ext.Value, &a); err != nil { + return nil, fmt.Errorf("asn1.Unmarshal failed: %v", err) + } else if len(rest) != 0 { + return nil, errors.New("trailing data after AKID extension") + } + certList.AuthorityKeyID = a.ID + + case oidIssuingDistributionPoint.Equal(ext.Id): + var dp issuingDistributionPoint + if rest, err := asn1.Unmarshal(ext.Value, &dp); err != nil { + return nil, fmt.Errorf("asn1.Unmarshal failed: %v", err) + } else if len(rest) != 0 { + return nil, errors.New("trailing data after IssuingDistributionPoint extension") + } + + if dp.OnlyContainsUserCerts || dp.OnlyContainsCACerts || dp.OnlyContainsAttributeCerts { + return nil, errors.New("CRL only contains some certificate types") + } + if dp.IndirectCRL { + return nil, errors.New("indirect CRLs unsupported") + } + if dp.OnlySomeReasons.BitLength != 0 { + return nil, errors.New("onlySomeReasons unsupported") + } + + case ext.Critical: + return nil, fmt.Errorf("unsupported critical extension: %v", ext.Id) + } + } + + if len(certList.AuthorityKeyID) == 0 { + return nil, errors.New("authority key identifier extension missing") + } + return certList, nil +} + +func fetchCRL(rawIssuer []byte, cfg RevocationConfig) (*certificateListExt, error) { + var parsedCRL *certificateListExt + // 6.3.3 (a) (1) (ii) + // According to X509_LOOKUP_hash_dir the format is issuer_hash.rN where N is an increasing number. + // There are no gaps, so we break when we can't find a file. + for i := 0; ; i++ { + // Unmarshal to RDNSeqence according to http://go/godoc/crypto/x509/pkix/#Name. + var r pkix.RDNSequence + rest, err := asn1.Unmarshal(rawIssuer, &r) + if len(rest) != 0 || err != nil { + return nil, fmt.Errorf("asn1.Unmarshal(Issuer) len(rest) = %d failed: %v", len(rest), err) + } + crlPath := fmt.Sprintf("%s.r%d", filepath.Join(cfg.RootDir, x509NameHash(r)), i) + crlBytes, err := os.ReadFile(crlPath) + if err != nil { + // Break when we can't read a CRL file. + grpclogLogger.Infof("readFile: %v", err) + break + } + + crl, err := parseRevocationList(crlBytes) + if err != nil { + // Parsing errors for a CRL shouldn't happen so fail. + return nil, fmt.Errorf("parseRevocationList(%v) failed: %v", crlPath, err) + } + var certList *certificateListExt + if certList, err = parseCRLExtensions(crl); err != nil { + grpclogLogger.Infof("fetchCRL: unsupported crl %v: %v", crlPath, err) + // Continue to find a supported CRL + continue + } + + rawCRLIssuer, err := extractCRLIssuer(crlBytes) + if err != nil { + return nil, err + } + certList.RawIssuer = rawCRLIssuer + // RFC5280, 6.3.3 (b) Verify the issuer and scope of the complete CRL. + if bytes.Equal(rawIssuer, rawCRLIssuer) { + parsedCRL = certList + // Continue to find the highest number in the .rN suffix. + continue + } + } + + if parsedCRL == nil { + return nil, fmt.Errorf("fetchCrls no CRLs found for issuer") + } + return parsedCRL, nil +} + +func verifyCRL(crl *certificateListExt, rawIssuer []byte, chain []*x509.Certificate) error { + // RFC5280, 6.3.3 (f) Obtain and validateate the certification path for the issuer of the complete CRL + // We intentionally limit our CRLs to be signed with the same certificate path as the certificate + // so we can use the chain from the connection. + + for _, c := range chain { + // Use the key where the subject and KIDs match. + // This departs from RFC4158, 3.5.12 which states that KIDs + // cannot eliminate certificates, but RFC5280, 5.2.1 states that + // "Conforming CRL issuers MUST use the key identifier method, and MUST + // include this extension in all CRLs issued." + // So, this is much simpler than RFC4158 and should be compatible. + if bytes.Equal(c.SubjectKeyId, crl.AuthorityKeyID) && bytes.Equal(c.RawSubject, crl.RawIssuer) { + // RFC5280, 6.3.3 (g) Validate signature. + return crl.CertList.CheckSignatureFrom(c) + } + } + return fmt.Errorf("verifyCRL: No certificates mached CRL issuer (%v)", crl.CertList.Issuer) +} + +// pemType is the type of a PEM encoded CRL. +const pemType string = "X509 CRL" + +var crlPemPrefix = []byte("-----BEGIN X509 CRL") + +func crlPemToDer(crlBytes []byte) []byte { + block, _ := pem.Decode(crlBytes) + if block != nil && block.Type == pemType { + crlBytes = block.Bytes + } + return crlBytes +} + +// extractCRLIssuer extracts the raw ASN.1 encoding of the CRL issuer. Due to the design of +// pkix.CertificateList and pkix.RDNSequence, it is not possible to reliably marshal the +// parsed Issuer to it's original raw encoding. +func extractCRLIssuer(crlBytes []byte) ([]byte, error) { + if bytes.HasPrefix(crlBytes, crlPemPrefix) { + crlBytes = crlPemToDer(crlBytes) + } + der := cryptobyte.String(crlBytes) + var issuer cryptobyte.String + if !der.ReadASN1(&der, cbasn1.SEQUENCE) || + !der.ReadASN1(&der, cbasn1.SEQUENCE) || + !der.SkipOptionalASN1(cbasn1.INTEGER) || + !der.SkipASN1(cbasn1.SEQUENCE) || + !der.ReadASN1Element(&issuer, cbasn1.SEQUENCE) { + return nil, errors.New("extractCRLIssuer: invalid ASN.1 encoding") + } + return issuer, nil +} + +func hasExpired(crl *x509.RevocationList, now time.Time) bool { + return !now.Before(crl.NextUpdate) +} + +// parseRevocationList comes largely from here +// x509.go: +// https://github.com/golang/go/blob/e2f413402527505144beea443078649380e0c545/src/crypto/x509/x509.go#L1669-L1690 +// We must first convert PEM to DER to be able to use the new +// x509.ParseRevocationList instead of the deprecated x509.ParseCRL +func parseRevocationList(crlBytes []byte) (*x509.RevocationList, error) { + if bytes.HasPrefix(crlBytes, crlPemPrefix) { + crlBytes = crlPemToDer(crlBytes) + } + crl, err := x509.ParseRevocationList(crlBytes) + if err != nil { + return nil, err + } + return crl, nil +} diff --git a/security/advancedtls/crl_deprecated.go b/security/advancedtls/crl_deprecated.go new file mode 100644 index 000000000000..fc83f72ebb3f --- /dev/null +++ b/security/advancedtls/crl_deprecated.go @@ -0,0 +1,521 @@ +// TODO(@gregorycooke) - Remove file when only golang 1.19+ is supported +//go:build !go1.19 + +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package advancedtls + +import ( + "bytes" + "crypto/sha1" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/asn1" + "encoding/binary" + "encoding/hex" + "encoding/pem" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "golang.org/x/crypto/cryptobyte" + cbasn1 "golang.org/x/crypto/cryptobyte/asn1" + "google.golang.org/grpc/grpclog" +) + +var grpclogLogger = grpclog.Component("advancedtls") + +// Cache is an interface to cache CRL files. +// The cache implementation must be concurrency safe. +// A fixed size lru cache from golang-lru is recommended. +type Cache interface { + // Add adds a value to the cache. + Add(key, value any) bool + // Get looks up a key's value from the cache. + Get(key any) (value any, ok bool) +} + +// RevocationConfig contains options for CRL lookup. +type RevocationConfig struct { + // RootDir is the directory to search for CRL files. + // Directory format must match OpenSSL X509_LOOKUP_hash_dir(3). + RootDir string + // AllowUndetermined controls if certificate chains with RevocationUndetermined + // revocation status are allowed to complete. + AllowUndetermined bool + // Cache will store CRL files if not nil, otherwise files are reloaded for every lookup. + Cache Cache +} + +// RevocationStatus is the revocation status for a certificate or chain. +type RevocationStatus int + +const ( + // RevocationUndetermined means we couldn't find or verify a CRL for the cert. + RevocationUndetermined RevocationStatus = iota + // RevocationUnrevoked means we found the CRL for the cert and the cert is not revoked. + RevocationUnrevoked + // RevocationRevoked means we found the CRL and the cert is revoked. + RevocationRevoked +) + +func (s RevocationStatus) String() string { + return [...]string{"RevocationUndetermined", "RevocationUnrevoked", "RevocationRevoked"}[s] +} + +// certificateListExt contains a pkix.CertificateList and parsed +// extensions that aren't provided by the golang CRL parser. +type certificateListExt struct { + CertList *pkix.CertificateList + // RFC5280, 5.2.1, all conforming CRLs must have a AKID with the ID method. + AuthorityKeyID []byte + RawIssuer []byte +} + +const tagDirectoryName = 4 + +var ( + // RFC5280, 5.2.4 id-ce-deltaCRLIndicator OBJECT IDENTIFIER ::= { id-ce 27 } + oidDeltaCRLIndicator = asn1.ObjectIdentifier{2, 5, 29, 27} + // RFC5280, 5.2.5 id-ce-issuingDistributionPoint OBJECT IDENTIFIER ::= { id-ce 28 } + oidIssuingDistributionPoint = asn1.ObjectIdentifier{2, 5, 29, 28} + // RFC5280, 5.3.3 id-ce-certificateIssuer OBJECT IDENTIFIER ::= { id-ce 29 } + oidCertificateIssuer = asn1.ObjectIdentifier{2, 5, 29, 29} + // RFC5290, 4.2.1.1 id-ce-authorityKeyIdentifier OBJECT IDENTIFIER ::= { id-ce 35 } + oidAuthorityKeyIdentifier = asn1.ObjectIdentifier{2, 5, 29, 35} +) + +// x509NameHash implements the OpenSSL X509_NAME_hash function for hashed directory lookups. +// +// NOTE: due to the behavior of asn1.Marshal, if the original encoding of the RDN sequence +// contains strings which do not use the ASN.1 PrintableString type, the name will not be +// re-encoded using those types, resulting in a hash which does not match that produced +// by OpenSSL. +func x509NameHash(r pkix.RDNSequence) string { + var canonBytes []byte + // First, canonicalize all the strings. + for _, rdnSet := range r { + for i, rdn := range rdnSet { + value, ok := rdn.Value.(string) + if !ok { + continue + } + // OpenSSL trims all whitespace, does a tolower, and removes extra spaces between words. + // Implemented in x509_name_canon in OpenSSL + canonStr := strings.Join(strings.Fields( + strings.TrimSpace(strings.ToLower(value))), " ") + // Then it changes everything to UTF8 strings + rdnSet[i].Value = asn1.RawValue{Tag: asn1.TagUTF8String, Bytes: []byte(canonStr)} + + } + } + + // Finally, OpenSSL drops the initial sequence tag + // so we marshal all the RDNs separately instead of as a group. + for _, canonRdn := range r { + b, err := asn1.Marshal(canonRdn) + if err != nil { + continue + } + canonBytes = append(canonBytes, b...) + } + + issuerHash := sha1.Sum(canonBytes) + // Openssl takes the first 4 bytes and encodes them as a little endian + // uint32 and then uses the hex to make the file name. + // In C++, this would be: + // (((unsigned long)md[0]) | ((unsigned long)md[1] << 8L) | + // ((unsigned long)md[2] << 16L) | ((unsigned long)md[3] << 24L) + // ) & 0xffffffffL; + fileHash := binary.LittleEndian.Uint32(issuerHash[0:4]) + return fmt.Sprintf("%08x", fileHash) +} + +// CheckRevocation checks the connection for revoked certificates based on RFC5280. +// This implementation has the following major limitations: +// - Indirect CRL files are not supported. +// - CRL loading is only supported from directories in the X509_LOOKUP_hash_dir format. +// - OnlySomeReasons is not supported. +// - Delta CRL files are not supported. +// - Certificate CRLDistributionPoint must be URLs, but are then ignored and converted into a file path. +// - CRL checks are done after path building, which goes against RFC4158. +func CheckRevocation(conn tls.ConnectionState, cfg RevocationConfig) error { + return CheckChainRevocation(conn.VerifiedChains, cfg) +} + +// CheckChainRevocation checks the verified certificate chain +// for revoked certificates based on RFC5280. +func CheckChainRevocation(verifiedChains [][]*x509.Certificate, cfg RevocationConfig) error { + // Iterate the verified chains looking for one that is RevocationUnrevoked. + // A single RevocationUnrevoked chain is enough to allow the connection, and a single RevocationRevoked + // chain does not mean the connection should fail. + count := make(map[RevocationStatus]int) + for _, chain := range verifiedChains { + switch checkChain(chain, cfg) { + case RevocationUnrevoked: + // If any chain is RevocationUnrevoked then return no error. + return nil + case RevocationRevoked: + // If this chain is revoked, keep looking for another chain. + count[RevocationRevoked]++ + continue + case RevocationUndetermined: + if cfg.AllowUndetermined { + return nil + } + count[RevocationUndetermined]++ + continue + } + } + return fmt.Errorf("no unrevoked chains found: %v", count) +} + +// checkChain will determine and check all certificates in chain against the CRL +// defined in the certificate with the following rules: +// 1. If any certificate is RevocationRevoked, return RevocationRevoked. +// 2. If any certificate is RevocationUndetermined, return RevocationUndetermined. +// 3. If all certificates are RevocationUnrevoked, return RevocationUnrevoked. +func checkChain(chain []*x509.Certificate, cfg RevocationConfig) RevocationStatus { + chainStatus := RevocationUnrevoked + for _, c := range chain { + switch checkCert(c, chain, cfg) { + case RevocationRevoked: + // Easy case, if a cert in the chain is revoked, the chain is revoked. + return RevocationRevoked + case RevocationUndetermined: + // If we couldn't find the revocation status for a cert, the chain is at best RevocationUndetermined + // keep looking to see if we find a cert in the chain that's RevocationRevoked, + // but return RevocationUndetermined at a minimum. + chainStatus = RevocationUndetermined + case RevocationUnrevoked: + // Continue iterating up the cert chain. + continue + } + } + return chainStatus +} + +func cachedCrl(rawIssuer []byte, cache Cache) (*certificateListExt, bool) { + val, ok := cache.Get(hex.EncodeToString(rawIssuer)) + if !ok { + return nil, false + } + crl, ok := val.(*certificateListExt) + if !ok { + return nil, false + } + // If the CRL is expired, force a reload. + if crl.CertList.HasExpired(time.Now()) { + return nil, false + } + return crl, true +} + +// fetchIssuerCRL fetches and verifies the CRL for rawIssuer from disk or cache if configured in cfg. +func fetchIssuerCRL(rawIssuer []byte, crlVerifyCrt []*x509.Certificate, cfg RevocationConfig) (*certificateListExt, error) { + if cfg.Cache != nil { + if crl, ok := cachedCrl(rawIssuer, cfg.Cache); ok { + return crl, nil + } + } + + crl, err := fetchCRL(rawIssuer, cfg) + if err != nil { + return nil, fmt.Errorf("fetchCRL() failed: %v", err) + } + + if err := verifyCRL(crl, rawIssuer, crlVerifyCrt); err != nil { + return nil, fmt.Errorf("verifyCRL() failed: %v", err) + } + if cfg.Cache != nil { + cfg.Cache.Add(hex.EncodeToString(rawIssuer), crl) + } + return crl, nil +} + +// checkCert checks a single certificate against the CRL defined in the certificate. +// It will fetch and verify the CRL(s) defined in the root directory specified by cfg. +// If we can't load any authoritative CRL files, the status is RevocationUndetermined. +// c is the certificate to check. +// crlVerifyCrt is the group of possible certificates to verify the crl. +func checkCert(c *x509.Certificate, crlVerifyCrt []*x509.Certificate, cfg RevocationConfig) RevocationStatus { + crl, err := fetchIssuerCRL(c.RawIssuer, crlVerifyCrt, cfg) + if err != nil { + // We couldn't load any CRL files for the certificate, so we don't know if it's RevocationUnrevoked or not. + grpclogLogger.Warningf("getIssuerCRL(%v) err = %v", c.Issuer, err) + return RevocationUndetermined + } + revocation, err := checkCertRevocation(c, crl) + if err != nil { + grpclogLogger.Warningf("checkCertRevocation(CRL %v) failed: %v", crl.CertList.TBSCertList.Issuer, err) + // We couldn't check the CRL file for some reason, so we don't know if it's RevocationUnrevoked or not. + return RevocationUndetermined + } + // Here we've gotten a CRL that loads and verifies. + // We only handle all-reasons CRL files, so this file + // is authoritative for the certificate. + return revocation +} + +func checkCertRevocation(c *x509.Certificate, crl *certificateListExt) (RevocationStatus, error) { + // Per section 5.3.3 we prime the certificate issuer with the CRL issuer. + // Subsequent entries use the previous entry's issuer. + rawEntryIssuer := crl.RawIssuer + + // Loop through all the revoked certificates. + for _, revCert := range crl.CertList.TBSCertList.RevokedCertificates { + // 5.3 Loop through CRL entry extensions for needed information. + for _, ext := range revCert.Extensions { + if oidCertificateIssuer.Equal(ext.Id) { + extIssuer, err := parseCertIssuerExt(ext) + if err != nil { + grpclogLogger.Info(err) + if ext.Critical { + return RevocationUndetermined, err + } + // Since this is a non-critical extension, we can skip it even though + // there was a parsing failure. + continue + } + rawEntryIssuer = extIssuer + } else if ext.Critical { + return RevocationUndetermined, fmt.Errorf("checkCertRevocation: Unhandled critical extension: %v", ext.Id) + } + } + + // If the issuer and serial number appear in the CRL, the certificate is revoked. + if bytes.Equal(c.RawIssuer, rawEntryIssuer) && c.SerialNumber.Cmp(revCert.SerialNumber) == 0 { + // CRL contains the serial, so return revoked. + return RevocationRevoked, nil + } + } + // We did not find the serial in the CRL file that was valid for the cert + // so the certificate is not revoked. + return RevocationUnrevoked, nil +} + +func parseCertIssuerExt(ext pkix.Extension) ([]byte, error) { + // 5.3.3 Certificate Issuer + // CertificateIssuer ::= GeneralNames + // GeneralNames ::= SEQUENCE SIZE (1..MAX) OF GeneralName + var generalNames []asn1.RawValue + if rest, err := asn1.Unmarshal(ext.Value, &generalNames); err != nil || len(rest) != 0 { + return nil, fmt.Errorf("asn1.Unmarshal failed: %v", err) + } + + for _, generalName := range generalNames { + // GeneralName ::= CHOICE { + // otherName [0] OtherName, + // rfc822Name [1] IA5String, + // dNSName [2] IA5String, + // x400Address [3] ORAddress, + // directoryName [4] Name, + // ediPartyName [5] EDIPartyName, + // uniformResourceIdentifier [6] IA5String, + // iPAddress [7] OCTET STRING, + // registeredID [8] OBJECT IDENTIFIER } + if generalName.Tag == tagDirectoryName { + return generalName.Bytes, nil + } + } + // Conforming CRL issuers MUST include in this extension the + // distinguished name (DN) from the issuer field of the certificate that + // corresponds to this CRL entry. + // If we couldn't get a directoryName, we can't reason about this file so cert status is + // RevocationUndetermined. + return nil, errors.New("no DN found in certificate issuer") +} + +// RFC 5280, 4.2.1.1 +type authKeyID struct { + ID []byte `asn1:"optional,tag:0"` +} + +// RFC5280, 5.2.5 +// id-ce-issuingDistributionPoint OBJECT IDENTIFIER ::= { id-ce 28 } + +// IssuingDistributionPoint ::= SEQUENCE { +// distributionPoint [0] DistributionPointName OPTIONAL, +// onlyContainsUserCerts [1] BOOLEAN DEFAULT FALSE, +// onlyContainsCACerts [2] BOOLEAN DEFAULT FALSE, +// onlySomeReasons [3] ReasonFlags OPTIONAL, +// indirectCRL [4] BOOLEAN DEFAULT FALSE, +// onlyContainsAttributeCerts [5] BOOLEAN DEFAULT FALSE } + +// -- at most one of onlyContainsUserCerts, onlyContainsCACerts, +// -- and onlyContainsAttributeCerts may be set to TRUE. +type issuingDistributionPoint struct { + DistributionPoint asn1.RawValue `asn1:"optional,tag:0"` + OnlyContainsUserCerts bool `asn1:"optional,tag:1"` + OnlyContainsCACerts bool `asn1:"optional,tag:2"` + OnlySomeReasons asn1.BitString `asn1:"optional,tag:3"` + IndirectCRL bool `asn1:"optional,tag:4"` + OnlyContainsAttributeCerts bool `asn1:"optional,tag:5"` +} + +// parseCRLExtensions parses the extensions for a CRL +// and checks that they're supported by the parser. +func parseCRLExtensions(c *pkix.CertificateList) (*certificateListExt, error) { + if c == nil { + return nil, errors.New("c is nil, expected any value") + } + certList := &certificateListExt{CertList: c} + + for _, ext := range c.TBSCertList.Extensions { + switch { + case oidDeltaCRLIndicator.Equal(ext.Id): + return nil, fmt.Errorf("delta CRLs unsupported") + + case oidAuthorityKeyIdentifier.Equal(ext.Id): + var a authKeyID + if rest, err := asn1.Unmarshal(ext.Value, &a); err != nil { + return nil, fmt.Errorf("asn1.Unmarshal failed: %v", err) + } else if len(rest) != 0 { + return nil, errors.New("trailing data after AKID extension") + } + certList.AuthorityKeyID = a.ID + + case oidIssuingDistributionPoint.Equal(ext.Id): + var dp issuingDistributionPoint + if rest, err := asn1.Unmarshal(ext.Value, &dp); err != nil { + return nil, fmt.Errorf("asn1.Unmarshal failed: %v", err) + } else if len(rest) != 0 { + return nil, errors.New("trailing data after IssuingDistributionPoint extension") + } + + if dp.OnlyContainsUserCerts || dp.OnlyContainsCACerts || dp.OnlyContainsAttributeCerts { + return nil, errors.New("CRL only contains some certificate types") + } + if dp.IndirectCRL { + return nil, errors.New("indirect CRLs unsupported") + } + if dp.OnlySomeReasons.BitLength != 0 { + return nil, errors.New("onlySomeReasons unsupported") + } + + case ext.Critical: + return nil, fmt.Errorf("unsupported critical extension: %v", ext.Id) + } + } + + if len(certList.AuthorityKeyID) == 0 { + return nil, errors.New("authority key identifier extension missing") + } + return certList, nil +} + +func fetchCRL(rawIssuer []byte, cfg RevocationConfig) (*certificateListExt, error) { + var parsedCRL *certificateListExt + // 6.3.3 (a) (1) (ii) + // According to X509_LOOKUP_hash_dir the format is issuer_hash.rN where N is an increasing number. + // There are no gaps, so we break when we can't find a file. + for i := 0; ; i++ { + // Unmarshal to RDNSeqence according to http://go/godoc/crypto/x509/pkix/#Name. + var r pkix.RDNSequence + rest, err := asn1.Unmarshal(rawIssuer, &r) + if len(rest) != 0 || err != nil { + return nil, fmt.Errorf("asn1.Unmarshal(Issuer) len(rest) = %d failed: %v", len(rest), err) + } + crlPath := fmt.Sprintf("%s.r%d", filepath.Join(cfg.RootDir, x509NameHash(r)), i) + crlBytes, err := os.ReadFile(crlPath) + if err != nil { + // Break when we can't read a CRL file. + grpclogLogger.Infof("readFile: %v", err) + break + } + + crl, err := x509.ParseCRL(crlBytes) + if err != nil { + // Parsing errors for a CRL shouldn't happen so fail. + return nil, fmt.Errorf("x509.ParseCrl(%v) failed: %v", crlPath, err) + } + var certList *certificateListExt + if certList, err = parseCRLExtensions(crl); err != nil { + grpclogLogger.Infof("fetchCRL: unsupported crl %v: %v", crlPath, err) + // Continue to find a supported CRL + continue + } + + rawCRLIssuer, err := extractCRLIssuer(crlBytes) + if err != nil { + return nil, err + } + certList.RawIssuer = rawCRLIssuer + // RFC5280, 6.3.3 (b) Verify the issuer and scope of the complete CRL. + if bytes.Equal(rawIssuer, rawCRLIssuer) { + parsedCRL = certList + // Continue to find the highest number in the .rN suffix. + continue + } + } + + if parsedCRL == nil { + return nil, fmt.Errorf("fetchCrls no CRLs found for issuer") + } + return parsedCRL, nil +} + +func verifyCRL(crl *certificateListExt, rawIssuer []byte, chain []*x509.Certificate) error { + // RFC5280, 6.3.3 (f) Obtain and validateate the certification path for the issuer of the complete CRL + // We intentionally limit our CRLs to be signed with the same certificate path as the certificate + // so we can use the chain from the connection. + + for _, c := range chain { + // Use the key where the subject and KIDs match. + // This departs from RFC4158, 3.5.12 which states that KIDs + // cannot eliminate certificates, but RFC5280, 5.2.1 states that + // "Conforming CRL issuers MUST use the key identifier method, and MUST + // include this extension in all CRLs issued." + // So, this is much simpler than RFC4158 and should be compatible. + if bytes.Equal(c.SubjectKeyId, crl.AuthorityKeyID) && bytes.Equal(c.RawSubject, crl.RawIssuer) { + // RFC5280, 6.3.3 (g) Validate signature. + return c.CheckCRLSignature(crl.CertList) + } + } + return fmt.Errorf("verifyCRL: No certificates mached CRL issuer (%v)", crl.CertList.TBSCertList.Issuer) +} + +var crlPemPrefix = []byte("-----BEGIN X509 CRL") + +// extractCRLIssuer extracts the raw ASN.1 encoding of the CRL issuer. Due to the design of +// pkix.CertificateList and pkix.RDNSequence, it is not possible to reliably marshal the +// parsed Issuer to it's original raw encoding. +func extractCRLIssuer(crlBytes []byte) ([]byte, error) { + if bytes.HasPrefix(crlBytes, crlPemPrefix) { + block, _ := pem.Decode(crlBytes) + if block != nil && block.Type == "X509 CRL" { + crlBytes = block.Bytes + } + } + + der := cryptobyte.String(crlBytes) + var issuer cryptobyte.String + if !der.ReadASN1(&der, cbasn1.SEQUENCE) || + !der.ReadASN1(&der, cbasn1.SEQUENCE) || + !der.SkipOptionalASN1(cbasn1.INTEGER) || + !der.SkipASN1(cbasn1.SEQUENCE) || + !der.ReadASN1Element(&issuer, cbasn1.SEQUENCE) { + return nil, errors.New("extractCRLIssuer: invalid ASN.1 encoding") + } + return issuer, nil +} diff --git a/security/advancedtls/crl_deprecated_test.go b/security/advancedtls/crl_deprecated_test.go new file mode 100644 index 000000000000..20e230cf25e6 --- /dev/null +++ b/security/advancedtls/crl_deprecated_test.go @@ -0,0 +1,775 @@ +// TODO(@gregorycooke) - Remove file when only golang 1.19+ is supported +//go:build !go1.19 + +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package advancedtls + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/asn1" + "encoding/hex" + "encoding/pem" + "fmt" + "math/big" + "net" + "os" + "path" + "strings" + "testing" + "time" + + lru "github.com/hashicorp/golang-lru" + "google.golang.org/grpc/security/advancedtls/testdata" +) + +func TestX509NameHash(t *testing.T) { + nameTests := []struct { + in pkix.Name + out string + }{ + { + in: pkix.Name{ + Country: []string{"US"}, + Organization: []string{"Example"}, + }, + out: "9cdd41ff", + }, + { + in: pkix.Name{ + Country: []string{"us"}, + Organization: []string{"example"}, + }, + out: "9cdd41ff", + }, + { + in: pkix.Name{ + Country: []string{" us"}, + Organization: []string{"example"}, + }, + out: "9cdd41ff", + }, + { + in: pkix.Name{ + Country: []string{"US"}, + Province: []string{"California"}, + Locality: []string{"Mountain View"}, + Organization: []string{"BoringSSL"}, + }, + out: "c24414d9", + }, + { + in: pkix.Name{ + Country: []string{"US"}, + Province: []string{"California"}, + Locality: []string{"Mountain View"}, + Organization: []string{"BoringSSL"}, + }, + out: "c24414d9", + }, + { + in: pkix.Name{ + SerialNumber: "87f4514475ba0a2b", + }, + out: "9dc713cd", + }, + { + in: pkix.Name{ + Country: []string{"US"}, + Province: []string{"California"}, + Locality: []string{"Mountain View"}, + Organization: []string{"Google LLC"}, + OrganizationalUnit: []string{"Production", "campus-sln"}, + CommonName: "Root CA (2021-02-02T07:30:36-08:00)", + }, + out: "0b35a562", + }, + { + in: pkix.Name{ + ExtraNames: []pkix.AttributeTypeAndValue{ + {Type: asn1.ObjectIdentifier{5, 5, 5, 5}, Value: "aaaa"}, + }, + }, + out: "eea339da", + }, + } + for _, tt := range nameTests { + t.Run(tt.in.String(), func(t *testing.T) { + h := x509NameHash(tt.in.ToRDNSequence()) + if h != tt.out { + t.Errorf("x509NameHash(%v): Got %v wanted %v", tt.in, h, tt.out) + } + }) + } +} + +func TestUnsupportedCRLs(t *testing.T) { + crlBytesSomeReasons := []byte(`-----BEGIN X509 CRL----- +MIIEeDCCA2ACAQEwDQYJKoZIhvcNAQELBQAwQjELMAkGA1UEBhMCVVMxHjAcBgNV +BAoTFUdvb2dsZSBUcnVzdCBTZXJ2aWNlczETMBEGA1UEAxMKR1RTIENBIDFPMRcN +MjEwNDI2MTI1OTQxWhcNMjEwNTA2MTE1OTQwWjCCAn0wIgIRAPOOG3L4VLC7CAAA +AABxQgEXDTIxMDQxOTEyMTgxOFowIQIQUK0UwBZkVdQIAAAAAHFCBRcNMjEwNDE5 +MTIxODE4WjAhAhBRIXBJaKoQkQgAAAAAcULHFw0yMTA0MjAxMjE4MTdaMCICEQCv +qQWUq5UxmQgAAAAAcULMFw0yMTA0MjAxMjE4MTdaMCICEQDdv5k1kKwKTQgAAAAA +cUOQFw0yMTA0MjExMjE4MTZaMCICEQDGIEfR8N9sEAgAAAAAcUOWFw0yMTA0MjEx +MjE4MThaMCECEBHgbLXlj5yUCAAAAABxQ/IXDTIxMDQyMTIzMDAyNlowIQIQE1wT +2GGYqKwIAAAAAHFD7xcNMjEwNDIxMjMwMDI5WjAiAhEAo/bSyDjpVtsIAAAAAHFE +txcNMjEwNDIyMjMwMDI3WjAhAhARdCrSrHE0dAgAAAAAcUS/Fw0yMTA0MjIyMzAw +MjhaMCECEHONohfWn3wwCAAAAABxRX8XDTIxMDQyMzIzMDAyOVowIgIRAOYkiUPA +os4vCAAAAABxRYgXDTIxMDQyMzIzMDAyOFowIQIQRNTow5Eg2gEIAAAAAHFGShcN +MjEwNDI0MjMwMDI2WjAhAhBX32dH4/WQ6AgAAAAAcUZNFw0yMTA0MjQyMzAwMjZa +MCICEQDHnUM1vsaP/wgAAAAAcUcQFw0yMTA0MjUyMzAwMjZaMCECEEm5rvmL8sj6 +CAAAAABxRxQXDTIxMDQyNTIzMDAyN1owIQIQW16OQs4YQYkIAAAAAHFIABcNMjEw +NDI2MTI1NDA4WjAhAhAhSohpYsJtDQgAAAAAcUgEFw0yMTA0MjYxMjU0MDlaoGkw +ZzAfBgNVHSMEGDAWgBSY0fhuEOvPm+xgnxiQG6DrfQn9KzALBgNVHRQEBAICBngw +NwYDVR0cAQH/BC0wK6AmoCSGImh0dHA6Ly9jcmwucGtpLmdvb2cvR1RTMU8xY29y +ZS5jcmyBAf8wDQYJKoZIhvcNAQELBQADggEBADPBXbxVxMJ1HC7btXExRUpJHUlU +YbeCZGx6zj5F8pkopbmpV7cpewwhm848Fx4VaFFppZQZd92O08daEC6aEqoug4qF +z6ZrOLzhuKfpW8E93JjgL91v0FYN7iOcT7+ERKCwVEwEkuxszxs7ggW6OJYJNvHh +priIdmcPoiQ3ZrIRH0vE3BfUcNXnKFGATWuDkiRI0I4A5P7NiOf+lAuGZet3/eom +0chgts6sdau10GfeUpHUd4f8e93cS/QeLeG16z7LC8vRLstU3m3vrknpZbdGqSia +97w66mqcnQh9V0swZiEnVLmLufaiuDZJ+6nUzSvLqBlb/ei3T/tKV0BoKJA= +-----END X509 CRL-----`) + + crlBytesIndirect := []byte(`-----BEGIN X509 CRL----- +MIIDGjCCAgICAQEwDQYJKoZIhvcNAQELBQAwdjELMAkGA1UEBhMCVVMxEzARBgNV +BAgTCkNhbGlmb3JuaWExFDASBgNVBAoTC1Rlc3RpbmcgTHRkMSowKAYDVQQLEyFU +ZXN0aW5nIEx0ZCBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkxEDAOBgNVBAMTB1Rlc3Qg +Q0EXDTIxMDExNjAyMjAxNloXDTIxMDEyMDA2MjAxNlowgfIwbAIBAhcNMjEwMTE2 +MDIyMDE2WjBYMAoGA1UdFQQDCgEEMEoGA1UdHQEB/wRAMD6kPDA6MQwwCgYDVQQG +EwNVU0ExDTALBgNVBAcTBGhlcmUxCzAJBgNVBAoTAnVzMQ4wDAYDVQQDEwVUZXN0 +MTAgAgEDFw0yMTAxMTYwMjIwMTZaMAwwCgYDVR0VBAMKAQEwYAIBBBcNMjEwMTE2 +MDIyMDE2WjBMMEoGA1UdHQEB/wRAMD6kPDA6MQwwCgYDVQQGEwNVU0ExDTALBgNV +BAcTBGhlcmUxCzAJBgNVBAoTAnVzMQ4wDAYDVQQDEwVUZXN0MqBjMGEwHwYDVR0j +BBgwFoAURJSDWAOfhGCryBjl8dsQjBitl3swCgYDVR0UBAMCAQEwMgYDVR0cAQH/ +BCgwJqAhoB+GHWh0dHA6Ly9jcmxzLnBraS5nb29nL3Rlc3QuY3JshAH/MA0GCSqG +SIb3DQEBCwUAA4IBAQBVXX67mr2wFPmEWCe6mf/wFnPl3xL6zNOl96YJtsd7ulcS +TEbdJpaUnWFQ23+Tpzdj/lI2aQhTg5Lvii3o+D8C5r/Jc5NhSOtVJJDI/IQLh4pG +NgGdljdbJQIT5D2Z71dgbq1ocxn8DefZIJjO3jp8VnAm7AIMX2tLTySzD2MpMeMq +XmcN4lG1e4nx+xjzp7MySYO42NRY3LkphVzJhu3dRBYhBKViRJxw9hLttChitJpF +6Kh6a0QzrEY/QDJGhE1VrAD2c5g/SKnHPDVoCWo4ACIICi76KQQSIWfIdp4W/SY3 +qsSIp8gfxSyzkJP+Ngkm2DdLjlJQCZ9R0MZP9Xj4 +-----END X509 CRL-----`) + + var tests = []struct { + desc string + in []byte + }{ + { + desc: "some reasons", + in: crlBytesSomeReasons, + }, + { + desc: "indirect", + in: crlBytesIndirect, + }, + } + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + crl, err := x509.ParseCRL(tt.in) + if err != nil { + t.Fatal(err) + } + if _, err := parseCRLExtensions(crl); err == nil { + t.Error("expected error got ok") + } + }) + } +} + +func TestCheckCertRevocation(t *testing.T) { + dummyCrlFile := []byte(`-----BEGIN X509 CRL----- +MIIDGjCCAgICAQEwDQYJKoZIhvcNAQELBQAwdjELMAkGA1UEBhMCVVMxEzARBgNV +BAgTCkNhbGlmb3JuaWExFDASBgNVBAoTC1Rlc3RpbmcgTHRkMSowKAYDVQQLEyFU +ZXN0aW5nIEx0ZCBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkxEDAOBgNVBAMTB1Rlc3Qg +Q0EXDTIxMDExNjAyMjAxNloXDTIxMDEyMDA2MjAxNlowgfIwbAIBAhcNMjEwMTE2 +MDIyMDE2WjBYMAoGA1UdFQQDCgEEMEoGA1UdHQEB/wRAMD6kPDA6MQwwCgYDVQQG +EwNVU0ExDTALBgNVBAcTBGhlcmUxCzAJBgNVBAoTAnVzMQ4wDAYDVQQDEwVUZXN0 +MTAgAgEDFw0yMTAxMTYwMjIwMTZaMAwwCgYDVR0VBAMKAQEwYAIBBBcNMjEwMTE2 +MDIyMDE2WjBMMEoGA1UdHQEB/wRAMD6kPDA6MQwwCgYDVQQGEwNVU0ExDTALBgNV +BAcTBGhlcmUxCzAJBgNVBAoTAnVzMQ4wDAYDVQQDEwVUZXN0MqBjMGEwHwYDVR0j +BBgwFoAURJSDWAOfhGCryBjl8dsQjBitl3swCgYDVR0UBAMCAQEwMgYDVR0cAQH/ +BCgwJqAhoB+GHWh0dHA6Ly9jcmxzLnBraS5nb29nL3Rlc3QuY3JshAH/MA0GCSqG +SIb3DQEBCwUAA4IBAQBVXX67mr2wFPmEWCe6mf/wFnPl3xL6zNOl96YJtsd7ulcS +TEbdJpaUnWFQ23+Tpzdj/lI2aQhTg5Lvii3o+D8C5r/Jc5NhSOtVJJDI/IQLh4pG +NgGdljdbJQIT5D2Z71dgbq1ocxn8DefZIJjO3jp8VnAm7AIMX2tLTySzD2MpMeMq +XmcN4lG1e4nx+xjzp7MySYO42NRY3LkphVzJhu3dRBYhBKViRJxw9hLttChitJpF +6Kh6a0QzrEY/QDJGhE1VrAD2c5g/SKnHPDVoCWo4ACIICi76KQQSIWfIdp4W/SY3 +qsSIp8gfxSyzkJP+Ngkm2DdLjlJQCZ9R0MZP9Xj4 +-----END X509 CRL-----`) + crl, err := x509.ParseCRL(dummyCrlFile) + if err != nil { + t.Fatalf("x509.ParseCRL(dummyCrlFile) failed: %v", err) + } + crlExt := &certificateListExt{CertList: crl} + var crlIssuer pkix.Name + crlIssuer.FillFromRDNSequence(&crl.TBSCertList.Issuer) + + var revocationTests = []struct { + desc string + in x509.Certificate + revoked RevocationStatus + }{ + { + desc: "Single revoked", + in: x509.Certificate{ + Issuer: pkix.Name{ + Country: []string{"USA"}, + Locality: []string{"here"}, + Organization: []string{"us"}, + CommonName: "Test1", + }, + SerialNumber: big.NewInt(2), + CRLDistributionPoints: []string{"test"}, + }, + revoked: RevocationRevoked, + }, + { + desc: "Revoked no entry issuer", + in: x509.Certificate{ + Issuer: pkix.Name{ + Country: []string{"USA"}, + Locality: []string{"here"}, + Organization: []string{"us"}, + CommonName: "Test1", + }, + SerialNumber: big.NewInt(3), + CRLDistributionPoints: []string{"test"}, + }, + revoked: RevocationRevoked, + }, + { + desc: "Revoked new entry issuer", + in: x509.Certificate{ + Issuer: pkix.Name{ + Country: []string{"USA"}, + Locality: []string{"here"}, + Organization: []string{"us"}, + CommonName: "Test2", + }, + SerialNumber: big.NewInt(4), + CRLDistributionPoints: []string{"test"}, + }, + revoked: RevocationRevoked, + }, + { + desc: "Single unrevoked", + in: x509.Certificate{ + Issuer: pkix.Name{ + Country: []string{"USA"}, + Locality: []string{"here"}, + Organization: []string{"us"}, + CommonName: "Test2", + }, + SerialNumber: big.NewInt(1), + CRLDistributionPoints: []string{"test"}, + }, + revoked: RevocationUnrevoked, + }, + { + desc: "Single unrevoked Issuer", + in: x509.Certificate{ + Issuer: crlIssuer, + SerialNumber: big.NewInt(2), + CRLDistributionPoints: []string{"test"}, + }, + revoked: RevocationUnrevoked, + }, + } + + for _, tt := range revocationTests { + rawIssuer, err := asn1.Marshal(tt.in.Issuer.ToRDNSequence()) + if err != nil { + t.Fatalf("asn1.Marshal(%v) failed: %v", tt.in.Issuer.ToRDNSequence(), err) + } + tt.in.RawIssuer = rawIssuer + t.Run(tt.desc, func(t *testing.T) { + rev, err := checkCertRevocation(&tt.in, crlExt) + if err != nil { + t.Errorf("checkCertRevocation(%v) err = %v", tt.in.Issuer, err) + } else if rev != tt.revoked { + t.Errorf("checkCertRevocation(%v(%v)) returned %v wanted %v", + tt.in.Issuer, tt.in.SerialNumber, rev, tt.revoked) + } + }) + } +} + +func makeChain(t *testing.T, name string) []*x509.Certificate { + t.Helper() + + certChain := make([]*x509.Certificate, 0) + + rest, err := os.ReadFile(name) + if err != nil { + t.Fatalf("os.ReadFile(%v) failed %v", name, err) + } + for len(rest) > 0 { + var block *pem.Block + block, rest = pem.Decode(rest) + c, err := x509.ParseCertificate(block.Bytes) + if err != nil { + t.Fatalf("ParseCertificate error %v", err) + } + t.Logf("Parsed Cert sub = %v iss = %v", c.Subject, c.Issuer) + certChain = append(certChain, c) + } + return certChain +} + +func loadCRL(t *testing.T, path string) *certificateListExt { + b, err := os.ReadFile(path) + if err != nil { + t.Fatalf("readFile(%v) failed err = %v", path, err) + } + crl, err := x509.ParseCRL(b) + if err != nil { + t.Fatalf("ParseCrl(%v) failed err = %v", path, err) + } + crlExt, err := parseCRLExtensions(crl) + if err != nil { + t.Fatalf("parseCRLExtensions(%v) failed err = %v", path, err) + } + crlExt.RawIssuer, err = extractCRLIssuer(b) + if err != nil { + t.Fatalf("extractCRLIssuer(%v) failed err= %v", path, err) + } + return crlExt +} + +func TestCachedCRL(t *testing.T) { + cache, err := lru.New(5) + if err != nil { + t.Fatalf("lru.New: err = %v", err) + } + + tests := []struct { + desc string + val any + ok bool + }{ + { + desc: "Valid", + val: &certificateListExt{ + CertList: &pkix.CertificateList{ + TBSCertList: pkix.TBSCertificateList{ + NextUpdate: time.Now().Add(time.Hour), + }, + }}, + ok: true, + }, + { + desc: "Expired", + val: &certificateListExt{ + CertList: &pkix.CertificateList{ + TBSCertList: pkix.TBSCertificateList{ + NextUpdate: time.Now().Add(-time.Hour), + }, + }}, + ok: false, + }, + { + desc: "Wrong Type", + val: "string", + ok: false, + }, + { + desc: "Empty", + val: nil, + ok: false, + }, + } + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + if tt.val != nil { + cache.Add(hex.EncodeToString([]byte(tt.desc)), tt.val) + } + _, ok := cachedCrl([]byte(tt.desc), cache) + if tt.ok != ok { + t.Errorf("Cache ok error expected %v vs %v", tt.ok, ok) + } + }) + } +} + +func TestGetIssuerCRLCache(t *testing.T) { + cache, err := lru.New(5) + if err != nil { + t.Fatalf("lru.New: err = %v", err) + } + + tests := []struct { + desc string + rawIssuer []byte + certs []*x509.Certificate + }{ + { + desc: "Valid", + rawIssuer: makeChain(t, testdata.Path("crl/unrevoked.pem"))[1].RawIssuer, + certs: makeChain(t, testdata.Path("crl/unrevoked.pem")), + }, + { + desc: "Unverified", + rawIssuer: makeChain(t, testdata.Path("crl/unrevoked.pem"))[1].RawIssuer, + }, + { + desc: "Not Found", + rawIssuer: []byte("not_found"), + }, + } + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + cache.Purge() + _, err := fetchIssuerCRL(tt.rawIssuer, tt.certs, RevocationConfig{ + RootDir: testdata.Path("."), + Cache: cache, + }) + if err == nil && cache.Len() == 0 { + t.Error("Verified CRL not added to cache") + } + if err != nil && cache.Len() != 0 { + t.Error("Unverified CRL added to cache") + } + }) + } +} + +func TestVerifyCrl(t *testing.T) { + tampered := loadCRL(t, testdata.Path("crl/1.crl")) + // Change the signature so it won't verify + tampered.CertList.SignatureValue.Bytes[0]++ + + verifyTests := []struct { + desc string + crl *certificateListExt + certs []*x509.Certificate + cert *x509.Certificate + errWant string + }{ + { + desc: "Pass intermediate", + crl: loadCRL(t, testdata.Path("crl/1.crl")), + certs: makeChain(t, testdata.Path("crl/unrevoked.pem")), + cert: makeChain(t, testdata.Path("crl/unrevoked.pem"))[1], + errWant: "", + }, + { + desc: "Pass leaf", + crl: loadCRL(t, testdata.Path("crl/2.crl")), + certs: makeChain(t, testdata.Path("crl/unrevoked.pem")), + cert: makeChain(t, testdata.Path("crl/unrevoked.pem"))[2], + errWant: "", + }, + { + desc: "Fail wrong cert chain", + crl: loadCRL(t, testdata.Path("crl/3.crl")), + certs: makeChain(t, testdata.Path("crl/unrevoked.pem")), + cert: makeChain(t, testdata.Path("crl/revokedInt.pem"))[1], + errWant: "No certificates mached", + }, + { + desc: "Fail no certs", + crl: loadCRL(t, testdata.Path("crl/1.crl")), + certs: []*x509.Certificate{}, + cert: makeChain(t, testdata.Path("crl/unrevoked.pem"))[1], + errWant: "No certificates mached", + }, + { + desc: "Fail Tampered signature", + crl: tampered, + certs: makeChain(t, testdata.Path("crl/unrevoked.pem")), + cert: makeChain(t, testdata.Path("crl/unrevoked.pem"))[1], + errWant: "verification failure", + }, + } + + for _, tt := range verifyTests { + t.Run(tt.desc, func(t *testing.T) { + err := verifyCRL(tt.crl, tt.cert.RawIssuer, tt.certs) + switch { + case tt.errWant == "" && err != nil: + t.Errorf("Valid CRL did not verify err = %v", err) + case tt.errWant != "" && err == nil: + t.Error("Invalid CRL verified") + case tt.errWant != "" && !strings.Contains(err.Error(), tt.errWant): + t.Errorf("fetchIssuerCRL(_, %v, %v, _) = %v; want Contains(%v)", tt.cert.RawIssuer, tt.certs, err, tt.errWant) + } + }) + } +} + +func TestRevokedCert(t *testing.T) { + revokedIntChain := makeChain(t, testdata.Path("crl/revokedInt.pem")) + revokedLeafChain := makeChain(t, testdata.Path("crl/revokedLeaf.pem")) + validChain := makeChain(t, testdata.Path("crl/unrevoked.pem")) + cache, err := lru.New(5) + if err != nil { + t.Fatalf("lru.New: err = %v", err) + } + + var revocationTests = []struct { + desc string + in tls.ConnectionState + revoked bool + allowUndetermined bool + }{ + { + desc: "Single unrevoked", + in: tls.ConnectionState{VerifiedChains: [][]*x509.Certificate{validChain}}, + revoked: false, + }, + { + desc: "Single revoked intermediate", + in: tls.ConnectionState{VerifiedChains: [][]*x509.Certificate{revokedIntChain}}, + revoked: true, + }, + { + desc: "Single revoked leaf", + in: tls.ConnectionState{VerifiedChains: [][]*x509.Certificate{revokedLeafChain}}, + revoked: true, + }, + { + desc: "Multi one revoked", + in: tls.ConnectionState{VerifiedChains: [][]*x509.Certificate{validChain, revokedLeafChain}}, + revoked: false, + }, + { + desc: "Multi revoked", + in: tls.ConnectionState{VerifiedChains: [][]*x509.Certificate{revokedLeafChain, revokedIntChain}}, + revoked: true, + }, + { + desc: "Multi unrevoked", + in: tls.ConnectionState{VerifiedChains: [][]*x509.Certificate{validChain, validChain}}, + revoked: false, + }, + { + desc: "Undetermined revoked", + in: tls.ConnectionState{VerifiedChains: [][]*x509.Certificate{ + {&x509.Certificate{CRLDistributionPoints: []string{"test"}}}, + }}, + revoked: true, + }, + { + desc: "Undetermined allowed", + in: tls.ConnectionState{VerifiedChains: [][]*x509.Certificate{ + {&x509.Certificate{CRLDistributionPoints: []string{"test"}}}, + }}, + revoked: false, + allowUndetermined: true, + }, + } + + for _, tt := range revocationTests { + t.Run(tt.desc, func(t *testing.T) { + err := CheckRevocation(tt.in, RevocationConfig{ + RootDir: testdata.Path("crl"), + AllowUndetermined: tt.allowUndetermined, + Cache: cache, + }) + t.Logf("CheckRevocation err = %v", err) + if tt.revoked && err == nil { + t.Error("Revoked certificate chain was allowed") + } else if !tt.revoked && err != nil { + t.Error("Unrevoked certificate not allowed") + } + }) + } +} + +func setupTLSConn(t *testing.T) (net.Listener, *x509.Certificate, *ecdsa.PrivateKey) { + t.Helper() + templ := x509.Certificate{ + SerialNumber: big.NewInt(5), + BasicConstraintsValid: true, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + IsCA: true, + Subject: pkix.Name{CommonName: "test-cert"}, + KeyUsage: x509.KeyUsageCertSign, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + IPAddresses: []net.IP{net.ParseIP("::1")}, + CRLDistributionPoints: []string{"http://static.corp.google.com/crl/campus-sln/borg"}, + } + + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("ecdsa.GenerateKey failed err = %v", err) + } + rawCert, err := x509.CreateCertificate(rand.Reader, &templ, &templ, key.Public(), key) + if err != nil { + t.Fatalf("x509.CreateCertificate failed err = %v", err) + } + cert, err := x509.ParseCertificate(rawCert) + if err != nil { + t.Fatalf("x509.ParseCertificate failed err = %v", err) + } + + srvCfg := tls.Config{ + Certificates: []tls.Certificate{ + { + Certificate: [][]byte{cert.Raw}, + PrivateKey: key, + }, + }, + } + l, err := tls.Listen("tcp6", "[::1]:0", &srvCfg) + if err != nil { + t.Fatalf("tls.Listen failed err = %v", err) + } + return l, cert, key +} + +// TestVerifyConnection will setup a client/server connection and check revocation in the real TLS dialer +func TestVerifyConnection(t *testing.T) { + lis, cert, key := setupTLSConn(t) + defer func() { + lis.Close() + }() + + var handshakeTests = []struct { + desc string + revoked []pkix.RevokedCertificate + success bool + }{ + { + desc: "Empty CRL", + revoked: []pkix.RevokedCertificate{}, + success: true, + }, + { + desc: "Revoked Cert", + revoked: []pkix.RevokedCertificate{ + { + SerialNumber: cert.SerialNumber, + RevocationTime: time.Now(), + }, + }, + success: false, + }, + } + for _, tt := range handshakeTests { + t.Run(tt.desc, func(t *testing.T) { + // Accept one connection. + go func() { + conn, err := lis.Accept() + if err != nil { + t.Errorf("tls.Accept failed err = %v", err) + } else { + conn.Write([]byte("Hello, World!")) + conn.Close() + } + }() + + dir, err := os.MkdirTemp("", "crl_dir") + if err != nil { + t.Fatalf("os.MkdirTemp failed err = %v", err) + } + defer os.RemoveAll(dir) + + crl, err := cert.CreateCRL(rand.Reader, key, tt.revoked, time.Now(), time.Now().Add(time.Hour)) + if err != nil { + t.Fatalf("templ.CreateCRL failed err = %v", err) + } + + err = os.WriteFile(path.Join(dir, fmt.Sprintf("%s.r0", x509NameHash(cert.Subject.ToRDNSequence()))), crl, 0777) + if err != nil { + t.Fatalf("os.WriteFile failed err = %v", err) + } + + cp := x509.NewCertPool() + cp.AddCert(cert) + cliCfg := tls.Config{ + RootCAs: cp, + VerifyConnection: func(cs tls.ConnectionState) error { + return CheckRevocation(cs, RevocationConfig{RootDir: dir}) + }, + } + conn, err := tls.Dial(lis.Addr().Network(), lis.Addr().String(), &cliCfg) + t.Logf("tls.Dial err = %v", err) + if tt.success && err != nil { + t.Errorf("Expected success got err = %v", err) + } + if !tt.success && err == nil { + t.Error("Expected error, but got success") + } + if err == nil { + conn.Close() + } + }) + } +} + +func TestIssuerNonPrintableString(t *testing.T) { + rawIssuer, err := hex.DecodeString("300c310a300806022a030c023a29") + if err != nil { + t.Fatalf("failed to decode issuer: %s", err) + } + _, err = fetchCRL(rawIssuer, RevocationConfig{RootDir: testdata.Path("crl")}) + if err != nil { + t.Fatalf("fetchCRL failed: %s", err) + } +} + +// TestCRLCacheExpirationReloading tests the basic expiration and reloading of a +// cached CRL. The setup places an empty CRL in the cache, and a corresponding +// CRL with a revocation in the CRL directory. We then validate the certificate +// to verify that the certificate is not revoked. Then, we modify the +// NextUpdate time to be in the past so that when we next check for revocation, +// the existing cache entry should be seen as expired, and the CRL in the +// directory showing `revokedInt.pem` as revoked will be loaded, resulting in +// the check returning `RevocationRevoked`. +func TestCRLCacheExpirationReloading(t *testing.T) { + cache, err := lru.New(5) + if err != nil { + t.Fatalf("Creating cache failed") + } + + var certs = makeChain(t, testdata.Path("crl/revokedInt.pem")) + // Certs[1] has the same issuer as the revoked cert + rawIssuer := certs[1].RawIssuer + + // `3.crl`` revokes `revokedInt.pem` + crl := loadCRL(t, testdata.Path("crl/3.crl")) + // Modify the crl so that the cert is NOT revoked and add it to the cache + crl.CertList.TBSCertList.RevokedCertificates = nil + crl.CertList.TBSCertList.NextUpdate = time.Now().Add(time.Hour) + cache.Add(hex.EncodeToString(rawIssuer), crl) + var cfg = RevocationConfig{RootDir: testdata.Path("crl"), Cache: cache} + revocationStatus := checkChain(certs, cfg) + if revocationStatus != RevocationUnrevoked { + t.Fatalf("Certificate check should be RevocationUnrevoked, was %v", revocationStatus) + } + + // Modify the entry in the cache so that the cache will be refreshed + crl.CertList.TBSCertList.NextUpdate = time.Now() + cache.Add(hex.EncodeToString(rawIssuer), crl) + + revocationStatus = checkChain(certs, cfg) + if revocationStatus != RevocationRevoked { + t.Fatalf("A certificate should have been `RevocationRevoked` but was %v", revocationStatus) + } +} diff --git a/security/advancedtls/crl_test.go b/security/advancedtls/crl_test.go new file mode 100644 index 000000000000..2821f2bd494d --- /dev/null +++ b/security/advancedtls/crl_test.go @@ -0,0 +1,776 @@ +// TODO(@gregorycooke) - Remove when only golang 1.19+ is supported +//go:build go1.19 + +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package advancedtls + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/asn1" + "encoding/hex" + "encoding/pem" + "fmt" + "math/big" + "net" + "os" + "path" + "strings" + "testing" + "time" + + lru "github.com/hashicorp/golang-lru" + "google.golang.org/grpc/security/advancedtls/testdata" +) + +func TestX509NameHash(t *testing.T) { + nameTests := []struct { + in pkix.Name + out string + }{ + { + in: pkix.Name{ + Country: []string{"US"}, + Organization: []string{"Example"}, + }, + out: "9cdd41ff", + }, + { + in: pkix.Name{ + Country: []string{"us"}, + Organization: []string{"example"}, + }, + out: "9cdd41ff", + }, + { + in: pkix.Name{ + Country: []string{" us"}, + Organization: []string{"example"}, + }, + out: "9cdd41ff", + }, + { + in: pkix.Name{ + Country: []string{"US"}, + Province: []string{"California"}, + Locality: []string{"Mountain View"}, + Organization: []string{"BoringSSL"}, + }, + out: "c24414d9", + }, + { + in: pkix.Name{ + Country: []string{"US"}, + Province: []string{"California"}, + Locality: []string{"Mountain View"}, + Organization: []string{"BoringSSL"}, + }, + out: "c24414d9", + }, + { + in: pkix.Name{ + SerialNumber: "87f4514475ba0a2b", + }, + out: "9dc713cd", + }, + { + in: pkix.Name{ + Country: []string{"US"}, + Province: []string{"California"}, + Locality: []string{"Mountain View"}, + Organization: []string{"Google LLC"}, + OrganizationalUnit: []string{"Production", "campus-sln"}, + CommonName: "Root CA (2021-02-02T07:30:36-08:00)", + }, + out: "0b35a562", + }, + { + in: pkix.Name{ + ExtraNames: []pkix.AttributeTypeAndValue{ + {Type: asn1.ObjectIdentifier{5, 5, 5, 5}, Value: "aaaa"}, + }, + }, + out: "eea339da", + }, + } + for _, tt := range nameTests { + t.Run(tt.in.String(), func(t *testing.T) { + h := x509NameHash(tt.in.ToRDNSequence()) + if h != tt.out { + t.Errorf("x509NameHash(%v): Got %v wanted %v", tt.in, h, tt.out) + } + }) + } +} + +func TestUnsupportedCRLs(t *testing.T) { + crlBytesSomeReasons := []byte(`-----BEGIN X509 CRL----- +MIIEeDCCA2ACAQEwDQYJKoZIhvcNAQELBQAwQjELMAkGA1UEBhMCVVMxHjAcBgNV +BAoTFUdvb2dsZSBUcnVzdCBTZXJ2aWNlczETMBEGA1UEAxMKR1RTIENBIDFPMRcN +MjEwNDI2MTI1OTQxWhcNMjEwNTA2MTE1OTQwWjCCAn0wIgIRAPOOG3L4VLC7CAAA +AABxQgEXDTIxMDQxOTEyMTgxOFowIQIQUK0UwBZkVdQIAAAAAHFCBRcNMjEwNDE5 +MTIxODE4WjAhAhBRIXBJaKoQkQgAAAAAcULHFw0yMTA0MjAxMjE4MTdaMCICEQCv +qQWUq5UxmQgAAAAAcULMFw0yMTA0MjAxMjE4MTdaMCICEQDdv5k1kKwKTQgAAAAA +cUOQFw0yMTA0MjExMjE4MTZaMCICEQDGIEfR8N9sEAgAAAAAcUOWFw0yMTA0MjEx +MjE4MThaMCECEBHgbLXlj5yUCAAAAABxQ/IXDTIxMDQyMTIzMDAyNlowIQIQE1wT +2GGYqKwIAAAAAHFD7xcNMjEwNDIxMjMwMDI5WjAiAhEAo/bSyDjpVtsIAAAAAHFE +txcNMjEwNDIyMjMwMDI3WjAhAhARdCrSrHE0dAgAAAAAcUS/Fw0yMTA0MjIyMzAw +MjhaMCECEHONohfWn3wwCAAAAABxRX8XDTIxMDQyMzIzMDAyOVowIgIRAOYkiUPA +os4vCAAAAABxRYgXDTIxMDQyMzIzMDAyOFowIQIQRNTow5Eg2gEIAAAAAHFGShcN +MjEwNDI0MjMwMDI2WjAhAhBX32dH4/WQ6AgAAAAAcUZNFw0yMTA0MjQyMzAwMjZa +MCICEQDHnUM1vsaP/wgAAAAAcUcQFw0yMTA0MjUyMzAwMjZaMCECEEm5rvmL8sj6 +CAAAAABxRxQXDTIxMDQyNTIzMDAyN1owIQIQW16OQs4YQYkIAAAAAHFIABcNMjEw +NDI2MTI1NDA4WjAhAhAhSohpYsJtDQgAAAAAcUgEFw0yMTA0MjYxMjU0MDlaoGkw +ZzAfBgNVHSMEGDAWgBSY0fhuEOvPm+xgnxiQG6DrfQn9KzALBgNVHRQEBAICBngw +NwYDVR0cAQH/BC0wK6AmoCSGImh0dHA6Ly9jcmwucGtpLmdvb2cvR1RTMU8xY29y +ZS5jcmyBAf8wDQYJKoZIhvcNAQELBQADggEBADPBXbxVxMJ1HC7btXExRUpJHUlU +YbeCZGx6zj5F8pkopbmpV7cpewwhm848Fx4VaFFppZQZd92O08daEC6aEqoug4qF +z6ZrOLzhuKfpW8E93JjgL91v0FYN7iOcT7+ERKCwVEwEkuxszxs7ggW6OJYJNvHh +priIdmcPoiQ3ZrIRH0vE3BfUcNXnKFGATWuDkiRI0I4A5P7NiOf+lAuGZet3/eom +0chgts6sdau10GfeUpHUd4f8e93cS/QeLeG16z7LC8vRLstU3m3vrknpZbdGqSia +97w66mqcnQh9V0swZiEnVLmLufaiuDZJ+6nUzSvLqBlb/ei3T/tKV0BoKJA= +-----END X509 CRL-----`) + + crlBytesIndirect := []byte(`-----BEGIN X509 CRL----- +MIIDGjCCAgICAQEwDQYJKoZIhvcNAQELBQAwdjELMAkGA1UEBhMCVVMxEzARBgNV +BAgTCkNhbGlmb3JuaWExFDASBgNVBAoTC1Rlc3RpbmcgTHRkMSowKAYDVQQLEyFU +ZXN0aW5nIEx0ZCBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkxEDAOBgNVBAMTB1Rlc3Qg +Q0EXDTIxMDExNjAyMjAxNloXDTIxMDEyMDA2MjAxNlowgfIwbAIBAhcNMjEwMTE2 +MDIyMDE2WjBYMAoGA1UdFQQDCgEEMEoGA1UdHQEB/wRAMD6kPDA6MQwwCgYDVQQG +EwNVU0ExDTALBgNVBAcTBGhlcmUxCzAJBgNVBAoTAnVzMQ4wDAYDVQQDEwVUZXN0 +MTAgAgEDFw0yMTAxMTYwMjIwMTZaMAwwCgYDVR0VBAMKAQEwYAIBBBcNMjEwMTE2 +MDIyMDE2WjBMMEoGA1UdHQEB/wRAMD6kPDA6MQwwCgYDVQQGEwNVU0ExDTALBgNV +BAcTBGhlcmUxCzAJBgNVBAoTAnVzMQ4wDAYDVQQDEwVUZXN0MqBjMGEwHwYDVR0j +BBgwFoAURJSDWAOfhGCryBjl8dsQjBitl3swCgYDVR0UBAMCAQEwMgYDVR0cAQH/ +BCgwJqAhoB+GHWh0dHA6Ly9jcmxzLnBraS5nb29nL3Rlc3QuY3JshAH/MA0GCSqG +SIb3DQEBCwUAA4IBAQBVXX67mr2wFPmEWCe6mf/wFnPl3xL6zNOl96YJtsd7ulcS +TEbdJpaUnWFQ23+Tpzdj/lI2aQhTg5Lvii3o+D8C5r/Jc5NhSOtVJJDI/IQLh4pG +NgGdljdbJQIT5D2Z71dgbq1ocxn8DefZIJjO3jp8VnAm7AIMX2tLTySzD2MpMeMq +XmcN4lG1e4nx+xjzp7MySYO42NRY3LkphVzJhu3dRBYhBKViRJxw9hLttChitJpF +6Kh6a0QzrEY/QDJGhE1VrAD2c5g/SKnHPDVoCWo4ACIICi76KQQSIWfIdp4W/SY3 +qsSIp8gfxSyzkJP+Ngkm2DdLjlJQCZ9R0MZP9Xj4 +-----END X509 CRL-----`) + + var tests = []struct { + desc string + in []byte + }{ + { + desc: "some reasons", + in: crlBytesSomeReasons, + }, + { + desc: "indirect", + in: crlBytesIndirect, + }, + } + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + crl, err := parseRevocationList(tt.in) + if err != nil { + t.Fatal(err) + } + if _, err := parseCRLExtensions(crl); err == nil { + t.Error("expected error got ok") + } + }) + } +} + +func TestCheckCertRevocation(t *testing.T) { + dummyCrlFile := []byte(`-----BEGIN X509 CRL----- +MIIDGjCCAgICAQEwDQYJKoZIhvcNAQELBQAwdjELMAkGA1UEBhMCVVMxEzARBgNV +BAgTCkNhbGlmb3JuaWExFDASBgNVBAoTC1Rlc3RpbmcgTHRkMSowKAYDVQQLEyFU +ZXN0aW5nIEx0ZCBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkxEDAOBgNVBAMTB1Rlc3Qg +Q0EXDTIxMDExNjAyMjAxNloXDTIxMDEyMDA2MjAxNlowgfIwbAIBAhcNMjEwMTE2 +MDIyMDE2WjBYMAoGA1UdFQQDCgEEMEoGA1UdHQEB/wRAMD6kPDA6MQwwCgYDVQQG +EwNVU0ExDTALBgNVBAcTBGhlcmUxCzAJBgNVBAoTAnVzMQ4wDAYDVQQDEwVUZXN0 +MTAgAgEDFw0yMTAxMTYwMjIwMTZaMAwwCgYDVR0VBAMKAQEwYAIBBBcNMjEwMTE2 +MDIyMDE2WjBMMEoGA1UdHQEB/wRAMD6kPDA6MQwwCgYDVQQGEwNVU0ExDTALBgNV +BAcTBGhlcmUxCzAJBgNVBAoTAnVzMQ4wDAYDVQQDEwVUZXN0MqBjMGEwHwYDVR0j +BBgwFoAURJSDWAOfhGCryBjl8dsQjBitl3swCgYDVR0UBAMCAQEwMgYDVR0cAQH/ +BCgwJqAhoB+GHWh0dHA6Ly9jcmxzLnBraS5nb29nL3Rlc3QuY3JshAH/MA0GCSqG +SIb3DQEBCwUAA4IBAQBVXX67mr2wFPmEWCe6mf/wFnPl3xL6zNOl96YJtsd7ulcS +TEbdJpaUnWFQ23+Tpzdj/lI2aQhTg5Lvii3o+D8C5r/Jc5NhSOtVJJDI/IQLh4pG +NgGdljdbJQIT5D2Z71dgbq1ocxn8DefZIJjO3jp8VnAm7AIMX2tLTySzD2MpMeMq +XmcN4lG1e4nx+xjzp7MySYO42NRY3LkphVzJhu3dRBYhBKViRJxw9hLttChitJpF +6Kh6a0QzrEY/QDJGhE1VrAD2c5g/SKnHPDVoCWo4ACIICi76KQQSIWfIdp4W/SY3 +qsSIp8gfxSyzkJP+Ngkm2DdLjlJQCZ9R0MZP9Xj4 +-----END X509 CRL-----`) + crl, err := parseRevocationList(dummyCrlFile) + if err != nil { + t.Fatalf("parseRevocationList(dummyCrlFile) failed: %v", err) + } + crlExt := &certificateListExt{CertList: crl} + var crlIssuer pkix.Name = crl.Issuer + + var revocationTests = []struct { + desc string + in x509.Certificate + revoked RevocationStatus + }{ + { + desc: "Single revoked", + in: x509.Certificate{ + Issuer: pkix.Name{ + Country: []string{"USA"}, + Locality: []string{"here"}, + Organization: []string{"us"}, + CommonName: "Test1", + }, + SerialNumber: big.NewInt(2), + CRLDistributionPoints: []string{"test"}, + }, + revoked: RevocationRevoked, + }, + { + desc: "Revoked no entry issuer", + in: x509.Certificate{ + Issuer: pkix.Name{ + Country: []string{"USA"}, + Locality: []string{"here"}, + Organization: []string{"us"}, + CommonName: "Test1", + }, + SerialNumber: big.NewInt(3), + CRLDistributionPoints: []string{"test"}, + }, + revoked: RevocationRevoked, + }, + { + desc: "Revoked new entry issuer", + in: x509.Certificate{ + Issuer: pkix.Name{ + Country: []string{"USA"}, + Locality: []string{"here"}, + Organization: []string{"us"}, + CommonName: "Test2", + }, + SerialNumber: big.NewInt(4), + CRLDistributionPoints: []string{"test"}, + }, + revoked: RevocationRevoked, + }, + { + desc: "Single unrevoked", + in: x509.Certificate{ + Issuer: pkix.Name{ + Country: []string{"USA"}, + Locality: []string{"here"}, + Organization: []string{"us"}, + CommonName: "Test2", + }, + SerialNumber: big.NewInt(1), + CRLDistributionPoints: []string{"test"}, + }, + revoked: RevocationUnrevoked, + }, + { + desc: "Single unrevoked Issuer", + in: x509.Certificate{ + Issuer: crlIssuer, + SerialNumber: big.NewInt(2), + CRLDistributionPoints: []string{"test"}, + }, + revoked: RevocationUnrevoked, + }, + } + + for _, tt := range revocationTests { + rawIssuer, err := asn1.Marshal(tt.in.Issuer.ToRDNSequence()) + if err != nil { + t.Fatalf("asn1.Marshal(%v) failed: %v", tt.in.Issuer.ToRDNSequence(), err) + } + tt.in.RawIssuer = rawIssuer + t.Run(tt.desc, func(t *testing.T) { + rev, err := checkCertRevocation(&tt.in, crlExt) + if err != nil { + t.Errorf("checkCertRevocation(%v) err = %v", tt.in.Issuer, err) + } else if rev != tt.revoked { + t.Errorf("checkCertRevocation(%v(%v)) returned %v wanted %v", + tt.in.Issuer, tt.in.SerialNumber, rev, tt.revoked) + } + }) + } +} + +func makeChain(t *testing.T, name string) []*x509.Certificate { + t.Helper() + + certChain := make([]*x509.Certificate, 0) + + rest, err := os.ReadFile(name) + if err != nil { + t.Fatalf("os.ReadFile(%v) failed %v", name, err) + } + for len(rest) > 0 { + var block *pem.Block + block, rest = pem.Decode(rest) + c, err := x509.ParseCertificate(block.Bytes) + if err != nil { + t.Fatalf("ParseCertificate error %v", err) + } + t.Logf("Parsed Cert sub = %v iss = %v", c.Subject, c.Issuer) + certChain = append(certChain, c) + } + return certChain +} + +func loadCRL(t *testing.T, path string) *certificateListExt { + b, err := os.ReadFile(path) + if err != nil { + t.Fatalf("readFile(%v) failed err = %v", path, err) + } + crl, err := parseRevocationList(b) + if err != nil { + t.Fatalf("parseCrl(%v) failed err = %v", path, err) + } + crlExt, err := parseCRLExtensions(crl) + if err != nil { + t.Fatalf("parseCRLExtensions(%v) failed err = %v", path, err) + } + crlExt.RawIssuer, err = extractCRLIssuer(b) + if err != nil { + t.Fatalf("extractCRLIssuer(%v) failed err= %v", path, err) + } + return crlExt +} + +func TestCachedCRL(t *testing.T) { + cache, err := lru.New(5) + if err != nil { + t.Fatalf("lru.New: err = %v", err) + } + + tests := []struct { + desc string + val any + ok bool + }{ + { + desc: "Valid", + val: &certificateListExt{ + CertList: &x509.RevocationList{ + NextUpdate: time.Now().Add(time.Hour), + }}, + ok: true, + }, + { + desc: "Expired", + val: &certificateListExt{ + CertList: &x509.RevocationList{ + NextUpdate: time.Now().Add(-time.Hour), + }}, + ok: false, + }, + { + desc: "Wrong Type", + val: "string", + ok: false, + }, + { + desc: "Empty", + val: nil, + ok: false, + }, + } + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + if tt.val != nil { + cache.Add(hex.EncodeToString([]byte(tt.desc)), tt.val) + } + _, ok := cachedCrl([]byte(tt.desc), cache) + if tt.ok != ok { + t.Errorf("Cache ok error expected %v vs %v", tt.ok, ok) + } + }) + } +} + +func TestGetIssuerCRLCache(t *testing.T) { + cache, err := lru.New(5) + if err != nil { + t.Fatalf("lru.New: err = %v", err) + } + + tests := []struct { + desc string + rawIssuer []byte + certs []*x509.Certificate + }{ + { + desc: "Valid", + rawIssuer: makeChain(t, testdata.Path("crl/unrevoked.pem"))[1].RawIssuer, + certs: makeChain(t, testdata.Path("crl/unrevoked.pem")), + }, + { + desc: "Unverified", + rawIssuer: makeChain(t, testdata.Path("crl/unrevoked.pem"))[1].RawIssuer, + }, + { + desc: "Not Found", + rawIssuer: []byte("not_found"), + }, + } + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + cache.Purge() + _, err := fetchIssuerCRL(tt.rawIssuer, tt.certs, RevocationConfig{ + RootDir: testdata.Path("."), + Cache: cache, + }) + if err == nil && cache.Len() == 0 { + t.Error("Verified CRL not added to cache") + } + if err != nil && cache.Len() != 0 { + t.Error("Unverified CRL added to cache") + } + }) + } +} + +func TestVerifyCrl(t *testing.T) { + tampered := loadCRL(t, testdata.Path("crl/1.crl")) + // Change the signature so it won't verify + tampered.CertList.Signature[0]++ + + verifyTests := []struct { + desc string + crl *certificateListExt + certs []*x509.Certificate + cert *x509.Certificate + errWant string + }{ + { + desc: "Pass intermediate", + crl: loadCRL(t, testdata.Path("crl/1.crl")), + certs: makeChain(t, testdata.Path("crl/unrevoked.pem")), + cert: makeChain(t, testdata.Path("crl/unrevoked.pem"))[1], + errWant: "", + }, + { + desc: "Pass leaf", + crl: loadCRL(t, testdata.Path("crl/2.crl")), + certs: makeChain(t, testdata.Path("crl/unrevoked.pem")), + cert: makeChain(t, testdata.Path("crl/unrevoked.pem"))[2], + errWant: "", + }, + { + desc: "Fail wrong cert chain", + crl: loadCRL(t, testdata.Path("crl/3.crl")), + certs: makeChain(t, testdata.Path("crl/unrevoked.pem")), + cert: makeChain(t, testdata.Path("crl/revokedInt.pem"))[1], + errWant: "No certificates mached", + }, + { + desc: "Fail no certs", + crl: loadCRL(t, testdata.Path("crl/1.crl")), + certs: []*x509.Certificate{}, + cert: makeChain(t, testdata.Path("crl/unrevoked.pem"))[1], + errWant: "No certificates mached", + }, + { + desc: "Fail Tampered signature", + crl: tampered, + certs: makeChain(t, testdata.Path("crl/unrevoked.pem")), + cert: makeChain(t, testdata.Path("crl/unrevoked.pem"))[1], + errWant: "verification failure", + }, + } + + for _, tt := range verifyTests { + t.Run(tt.desc, func(t *testing.T) { + err := verifyCRL(tt.crl, tt.cert.RawIssuer, tt.certs) + switch { + case tt.errWant == "" && err != nil: + t.Errorf("Valid CRL did not verify err = %v", err) + case tt.errWant != "" && err == nil: + t.Error("Invalid CRL verified") + case tt.errWant != "" && !strings.Contains(err.Error(), tt.errWant): + t.Errorf("fetchIssuerCRL(_, %v, %v, _) = %v; want Contains(%v)", tt.cert.RawIssuer, tt.certs, err, tt.errWant) + } + }) + } +} + +func TestRevokedCert(t *testing.T) { + revokedIntChain := makeChain(t, testdata.Path("crl/revokedInt.pem")) + revokedLeafChain := makeChain(t, testdata.Path("crl/revokedLeaf.pem")) + validChain := makeChain(t, testdata.Path("crl/unrevoked.pem")) + cache, err := lru.New(5) + if err != nil { + t.Fatalf("lru.New: err = %v", err) + } + + var revocationTests = []struct { + desc string + in tls.ConnectionState + revoked bool + allowUndetermined bool + }{ + { + desc: "Single unrevoked", + in: tls.ConnectionState{VerifiedChains: [][]*x509.Certificate{validChain}}, + revoked: false, + }, + { + desc: "Single revoked intermediate", + in: tls.ConnectionState{VerifiedChains: [][]*x509.Certificate{revokedIntChain}}, + revoked: true, + }, + { + desc: "Single revoked leaf", + in: tls.ConnectionState{VerifiedChains: [][]*x509.Certificate{revokedLeafChain}}, + revoked: true, + }, + { + desc: "Multi one revoked", + in: tls.ConnectionState{VerifiedChains: [][]*x509.Certificate{validChain, revokedLeafChain}}, + revoked: false, + }, + { + desc: "Multi revoked", + in: tls.ConnectionState{VerifiedChains: [][]*x509.Certificate{revokedLeafChain, revokedIntChain}}, + revoked: true, + }, + { + desc: "Multi unrevoked", + in: tls.ConnectionState{VerifiedChains: [][]*x509.Certificate{validChain, validChain}}, + revoked: false, + }, + { + desc: "Undetermined revoked", + in: tls.ConnectionState{VerifiedChains: [][]*x509.Certificate{ + {&x509.Certificate{CRLDistributionPoints: []string{"test"}}}, + }}, + revoked: true, + }, + { + desc: "Undetermined allowed", + in: tls.ConnectionState{VerifiedChains: [][]*x509.Certificate{ + {&x509.Certificate{CRLDistributionPoints: []string{"test"}}}, + }}, + revoked: false, + allowUndetermined: true, + }, + } + + for _, tt := range revocationTests { + t.Run(tt.desc, func(t *testing.T) { + err := CheckRevocation(tt.in, RevocationConfig{ + RootDir: testdata.Path("crl"), + AllowUndetermined: tt.allowUndetermined, + Cache: cache, + }) + t.Logf("CheckRevocation err = %v", err) + if tt.revoked && err == nil { + t.Error("Revoked certificate chain was allowed") + } else if !tt.revoked && err != nil { + t.Error("Unrevoked certificate not allowed") + } + }) + } +} + +func setupTLSConn(t *testing.T) (net.Listener, *x509.Certificate, *ecdsa.PrivateKey) { + t.Helper() + templ := x509.Certificate{ + SerialNumber: big.NewInt(5), + BasicConstraintsValid: true, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + IsCA: true, + Subject: pkix.Name{CommonName: "test-cert"}, + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + IPAddresses: []net.IP{net.ParseIP("::1")}, + CRLDistributionPoints: []string{"http://static.corp.google.com/crl/campus-sln/borg"}, + } + + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("ecdsa.GenerateKey failed err = %v", err) + } + rawCert, err := x509.CreateCertificate(rand.Reader, &templ, &templ, key.Public(), key) + if err != nil { + t.Fatalf("x509.CreateCertificate failed err = %v", err) + } + cert, err := x509.ParseCertificate(rawCert) + if err != nil { + t.Fatalf("x509.ParseCertificate failed err = %v", err) + } + + srvCfg := tls.Config{ + Certificates: []tls.Certificate{ + { + Certificate: [][]byte{cert.Raw}, + PrivateKey: key, + }, + }, + } + l, err := tls.Listen("tcp6", "[::1]:0", &srvCfg) + if err != nil { + t.Fatalf("tls.Listen failed err = %v", err) + } + return l, cert, key +} + +// TestVerifyConnection will setup a client/server connection and check revocation in the real TLS dialer +func TestVerifyConnection(t *testing.T) { + lis, cert, key := setupTLSConn(t) + defer func() { + lis.Close() + }() + + var handshakeTests = []struct { + desc string + revoked []pkix.RevokedCertificate + success bool + }{ + { + desc: "Empty CRL", + revoked: []pkix.RevokedCertificate{}, + success: true, + }, + { + desc: "Revoked Cert", + revoked: []pkix.RevokedCertificate{ + { + SerialNumber: cert.SerialNumber, + RevocationTime: time.Now(), + }, + }, + success: false, + }, + } + for _, tt := range handshakeTests { + t.Run(tt.desc, func(t *testing.T) { + // Accept one connection. + go func() { + conn, err := lis.Accept() + if err != nil { + t.Errorf("tls.Accept failed err = %v", err) + } else { + conn.Write([]byte("Hello, World!")) + conn.Close() + } + }() + + dir, err := os.MkdirTemp("", "crl_dir") + if err != nil { + t.Fatalf("os.MkdirTemp failed err = %v", err) + } + defer os.RemoveAll(dir) + + template := &x509.RevocationList{ + RevokedCertificates: tt.revoked, + ThisUpdate: time.Now(), + NextUpdate: time.Now().Add(time.Hour), + Number: big.NewInt(1), + } + crl, err := x509.CreateRevocationList(rand.Reader, template, cert, key) + if err != nil { + t.Fatalf("templ.CreateRevocationList failed err = %v", err) + } + + err = os.WriteFile(path.Join(dir, fmt.Sprintf("%s.r0", x509NameHash(cert.Subject.ToRDNSequence()))), crl, 0777) + if err != nil { + t.Fatalf("os.WriteFile failed err = %v", err) + } + + cp := x509.NewCertPool() + cp.AddCert(cert) + cliCfg := tls.Config{ + RootCAs: cp, + VerifyConnection: func(cs tls.ConnectionState) error { + return CheckRevocation(cs, RevocationConfig{RootDir: dir}) + }, + } + conn, err := tls.Dial(lis.Addr().Network(), lis.Addr().String(), &cliCfg) + t.Logf("tls.Dial err = %v", err) + if tt.success && err != nil { + t.Errorf("Expected success got err = %v", err) + } + if !tt.success && err == nil { + t.Error("Expected error, but got success") + } + if err == nil { + conn.Close() + } + }) + } +} + +func TestIssuerNonPrintableString(t *testing.T) { + rawIssuer, err := hex.DecodeString("300c310a300806022a030c023a29") + if err != nil { + t.Fatalf("failed to decode issuer: %s", err) + } + _, err = fetchCRL(rawIssuer, RevocationConfig{RootDir: testdata.Path("crl")}) + if err != nil { + t.Fatalf("fetchCRL failed: %s", err) + } +} + +// TestCRLCacheExpirationReloading tests the basic expiration and reloading of a +// cached CRL. The setup places an empty CRL in the cache, and a corresponding +// CRL with a revocation in the CRL directory. We then validate the certificate +// to verify that the certificate is not revoked. Then, we modify the +// NextUpdate time to be in the past so that when we next check for revocation, +// the existing cache entry should be seen as expired, and the CRL in the +// directory showing `revokedInt.pem` as revoked will be loaded, resulting in +// the check returning `RevocationRevoked`. +func TestCRLCacheExpirationReloading(t *testing.T) { + cache, err := lru.New(5) + if err != nil { + t.Fatalf("Creating cache failed") + } + + var certs = makeChain(t, testdata.Path("crl/revokedInt.pem")) + // Certs[1] has the same issuer as the revoked cert + rawIssuer := certs[1].RawIssuer + + // `3.crl`` revokes `revokedInt.pem` + crl := loadCRL(t, testdata.Path("crl/3.crl")) + // Modify the crl so that the cert is NOT revoked and add it to the cache + crl.CertList.RevokedCertificates = nil + crl.CertList.NextUpdate = time.Now().Add(time.Hour) + cache.Add(hex.EncodeToString(rawIssuer), crl) + var cfg = RevocationConfig{RootDir: testdata.Path("crl"), Cache: cache} + revocationStatus := checkChain(certs, cfg) + if revocationStatus != RevocationUnrevoked { + t.Fatalf("Certificate check should be RevocationUnrevoked, was %v", revocationStatus) + } + + // Modify the entry in the cache so that the cache will be refreshed + crl.CertList.NextUpdate = time.Now() + cache.Add(hex.EncodeToString(rawIssuer), crl) + + revocationStatus = checkChain(certs, cfg) + if revocationStatus != RevocationRevoked { + t.Fatalf("A certificate should have been `RevocationRevoked` but was %v", revocationStatus) + } +} diff --git a/security/advancedtls/examples/credential_reloading_from_files/README.md b/security/advancedtls/examples/credential_reloading_from_files/README.md new file mode 100644 index 000000000000..be9e06e76e4c --- /dev/null +++ b/security/advancedtls/examples/credential_reloading_from_files/README.md @@ -0,0 +1,23 @@ +# Credential Reloading From Files + +Credential reloading is a feature supported in the advancedtls library. +A very common way to achieve this is to reload from files. + +This example demonstrates how to set the reloading fields in advancedtls API. +Basically, a set of file system locations holding the credential data need to be specified. +Once the credential data needs to be updated, users just change the credential data in the file system, and gRPC will pick up the changes automatically. + +A couple of things to note: + 1. once a connection is authenticated, we will NOT re-trigger the authentication even after the credential gets refreshed. + 2. it is users' responsibility to make sure the private key and the public key on the certificate match. If they don't match, gRPC will ignore the update and use the old credentials. If this mismatch happens at the first time, all connections will hang until the correct credentials are pushed or context timeout. + +## Try it +In directory `security/advancedtls/examples`: + +``` +go run server/main.go +``` + +``` +go run client/main.go +``` \ No newline at end of file diff --git a/security/advancedtls/examples/credential_reloading_from_files/client/main.go b/security/advancedtls/examples/credential_reloading_from_files/client/main.go new file mode 100644 index 000000000000..51dcb2a30ab9 --- /dev/null +++ b/security/advancedtls/examples/credential_reloading_from_files/client/main.go @@ -0,0 +1,111 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// The client demonstrates how to use the credential reloading feature in +// advancedtls to make a mTLS connection to the server. +package main + +import ( + "context" + "flag" + "log" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/tls/certprovider/pemfile" + pb "google.golang.org/grpc/examples/helloworld/helloworld" + "google.golang.org/grpc/security/advancedtls" + "google.golang.org/grpc/security/advancedtls/testdata" +) + +var address = "localhost:50051" + +const ( + // Default timeout for normal connections. + defaultTimeout = 2 * time.Second + // Intervals that set to monitor the credential updates. + credRefreshingInterval = 500 * time.Millisecond +) + +func main() { + tmpKeyFile := flag.String("key", "", "temporary key file path") + tmpCertFile := flag.String("cert", "", "temporary cert file path") + flag.Parse() + + if tmpKeyFile == nil || *tmpKeyFile == "" { + log.Fatalf("tmpKeyFile is nil or empty.") + } + if tmpCertFile == nil || *tmpCertFile == "" { + log.Fatalf("tmpCertFile is nil or empty.") + } + + // Initialize credential struct using reloading API. + identityOptions := pemfile.Options{ + CertFile: *tmpCertFile, + KeyFile: *tmpKeyFile, + RefreshDuration: credRefreshingInterval, + } + identityProvider, err := pemfile.NewProvider(identityOptions) + if err != nil { + log.Fatalf("pemfile.NewProvider(%v) failed: %v", identityOptions, err) + } + rootOptions := pemfile.Options{ + RootFile: testdata.Path("client_trust_cert_1.pem"), + RefreshDuration: credRefreshingInterval, + } + rootProvider, err := pemfile.NewProvider(rootOptions) + if err != nil { + log.Fatalf("pemfile.NewProvider(%v) failed: %v", rootOptions, err) + } + options := &advancedtls.ClientOptions{ + IdentityOptions: advancedtls.IdentityCertificateOptions{ + IdentityProvider: identityProvider, + }, + VerifyPeer: func(params *advancedtls.VerificationFuncParams) (*advancedtls.VerificationResults, error) { + return &advancedtls.VerificationResults{}, nil + }, + RootOptions: advancedtls.RootCertificateOptions{ + RootProvider: rootProvider, + }, + VType: advancedtls.CertVerification, + } + clientTLSCreds, err := advancedtls.NewClientCreds(options) + if err != nil { + log.Fatalf("advancedtls.NewClientCreds(%v) failed: %v", options, err) + } + + // Make a connection using the credentials. + conn, err := grpc.Dial(address, grpc.WithTransportCredentials(clientTLSCreds)) + if err != nil { + log.Fatalf("grpc.DialContext to %s failed: %v", address, err) + } + client := pb.NewGreeterClient(conn) + + // Send the requests every 0.5s. The credential is expected to be changed in + // the bash script. We don't cancel the context nor call conn.Close() here, + // since the bash script is expected to close the client goroutine. + for { + ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout) + _, err = client.SayHello(ctx, &pb.HelloRequest{Name: "gRPC"}, grpc.WaitForReady(true)) + if err != nil { + log.Fatalf("client.SayHello failed: %v", err) + } + cancel() + time.Sleep(500 * time.Millisecond) + } +} diff --git a/security/advancedtls/examples/credential_reloading_from_files/server/main.go b/security/advancedtls/examples/credential_reloading_from_files/server/main.go new file mode 100644 index 000000000000..164ef093be74 --- /dev/null +++ b/security/advancedtls/examples/credential_reloading_from_files/server/main.go @@ -0,0 +1,112 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// The server demonstrates how to use the credential reloading feature in +// advancedtls to serve mTLS connections from the client. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "net" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/tls/certprovider/pemfile" + "google.golang.org/grpc/keepalive" + "google.golang.org/grpc/security/advancedtls" + "google.golang.org/grpc/security/advancedtls/testdata" + + pb "google.golang.org/grpc/examples/helloworld/helloworld" +) + +var port = ":50051" + +// Intervals that set to monitor the credential updates. +const credRefreshingInterval = 1 * time.Minute + +type greeterServer struct { + pb.UnimplementedGreeterServer +} + +// sayHello is a simple implementation of the pb.GreeterServer SayHello method. +func (greeterServer) SayHello(ctx context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) { + return &pb.HelloReply{Message: "Hello " + in.Name}, nil +} + +func main() { + flag.Parse() + fmt.Printf("server starting on port %s...\n", port) + + identityOptions := pemfile.Options{ + CertFile: testdata.Path("server_cert_1.pem"), + KeyFile: testdata.Path("server_key_1.pem"), + RefreshDuration: credRefreshingInterval, + } + identityProvider, err := pemfile.NewProvider(identityOptions) + if err != nil { + log.Fatalf("pemfile.NewProvider(%v) failed: %v", identityOptions, err) + } + defer identityProvider.Close() + rootOptions := pemfile.Options{ + RootFile: testdata.Path("server_trust_cert_1.pem"), + RefreshDuration: credRefreshingInterval, + } + rootProvider, err := pemfile.NewProvider(rootOptions) + if err != nil { + log.Fatalf("pemfile.NewProvider(%v) failed: %v", rootOptions, err) + } + defer rootProvider.Close() + + // Start a server and create a client using advancedtls API with Provider. + options := &advancedtls.ServerOptions{ + IdentityOptions: advancedtls.IdentityCertificateOptions{ + IdentityProvider: identityProvider, + }, + RootOptions: advancedtls.RootCertificateOptions{ + RootProvider: rootProvider, + }, + RequireClientCert: true, + VerifyPeer: func(params *advancedtls.VerificationFuncParams) (*advancedtls.VerificationResults, error) { + // This message is to show the certificate under the hood is actually reloaded. + fmt.Printf("Client common name: %s.\n", params.Leaf.Subject.CommonName) + return &advancedtls.VerificationResults{}, nil + }, + VType: advancedtls.CertVerification, + } + serverTLSCreds, err := advancedtls.NewServerCreds(options) + if err != nil { + log.Fatalf("advancedtls.NewServerCreds(%v) failed: %v", options, err) + } + s := grpc.NewServer(grpc.Creds(serverTLSCreds), grpc.KeepaliveParams(keepalive.ServerParameters{ + // Set the max connection time to be 0.5 s to force the client to + // re-establish the connection, and hence re-invoke the verification + // callback. + MaxConnectionAge: 500 * time.Millisecond, + })) + lis, err := net.Listen("tcp", port) + if err != nil { + log.Fatalf("failed to listen: %v", err) + } + pb.RegisterGreeterServer(s, greeterServer{}) + if err := s.Serve(lis); err != nil { + log.Fatalf("failed to serve: %v", err) + } +} diff --git a/security/advancedtls/examples/examples_test.sh b/security/advancedtls/examples/examples_test.sh new file mode 100755 index 000000000000..2b911523d8eb --- /dev/null +++ b/security/advancedtls/examples/examples_test.sh @@ -0,0 +1,114 @@ +#!/bin/bash +# +# Copyright 2020 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set +e + +export TMPDIR=$(mktemp -d) +trap "rm -rf ${TMPDIR}" EXIT + +clean () { + for i in {1..10}; do + jobs -p | xargs -n1 pkill -P + # A simple "wait" just hangs sometimes. Running `jobs` seems to help. + sleep 1 + if jobs | read; then + return + fi + done + echo "$(tput setaf 1) clean failed to kill tests $(tput sgr 0)" + jobs + pstree + rm ${CLIENT_LOG} + rm ${SERVER_LOG} + rm ${KEY_FILE_PATH} + rm ${CERT_FILE_PATH} + exit 1 +} + +fail () { + echo "$(tput setaf 1) $1 $(tput sgr 0)" + clean + exit 1 +} + +pass () { + echo "$(tput setaf 2) $1 $(tput sgr 0)" +} + +EXAMPLES=( + "credential_reloading_from_files" +) + +declare -a EXPECTED_SERVER_OUTPUT=("Client common name: foo.bar.hoo.com" "Client common name: foo.bar.another.client.com") + +cd ./security/advancedtls/examples + +for example in ${EXAMPLES[@]}; do + echo "$(tput setaf 4) testing: ${example} $(tput sgr 0)" + + KEY_FILE_PATH=$(mktemp) + cat ../testdata/client_key_1.pem > ${KEY_FILE_PATH} + + CERT_FILE_PATH=$(mktemp) + cat ../testdata/client_cert_1.pem > ${CERT_FILE_PATH} + + # Build server. + if ! go build -o /dev/null ./${example}/*server/*.go; then + fail "failed to build server" + else + pass "successfully built server" + fi + + # Build client. + if ! go build -o /dev/null ./${example}/*client/*.go; then + fail "failed to build client" + else + pass "successfully built client" + fi + + # Start server. + SERVER_LOG="$(mktemp)" + go run ./$example/*server/*.go &> $SERVER_LOG & + + # Run client binary. + CLIENT_LOG="$(mktemp)" + go run ${example}/*client/*.go -key=${KEY_FILE_PATH} -cert=${CERT_FILE_PATH} &> $CLIENT_LOG & + + # Wait for the client to send some requests using old credentials. + sleep 4s + + # Switch to the new credentials. + cat ../testdata/another_client_key_1.pem > ${KEY_FILE_PATH} + cat ../testdata/another_client_cert_1.pem > ${CERT_FILE_PATH} + + # Wait for the client to send some requests using new credentials. + sleep 4s + + # Check server log for expected output. + for output in "${EXPECTED_SERVER_OUTPUT[@]}"; do + if ! grep -q "$output" $SERVER_LOG; then + fail "server log missing output: $output + got server log: + $(cat $SERVER_LOG) + " + else + pass "server log contains expected output: $output" + fi + done + + clean +done diff --git a/security/advancedtls/examples/go.mod b/security/advancedtls/examples/go.mod new file mode 100644 index 000000000000..51db9fe1a7f1 --- /dev/null +++ b/security/advancedtls/examples/go.mod @@ -0,0 +1,25 @@ +module google.golang.org/grpc/security/advancedtls/examples + +go 1.19 + +require ( + google.golang.org/grpc v1.56.2 + google.golang.org/grpc/examples v0.0.0-20230714201633-919fe359160c + google.golang.org/grpc/security/advancedtls v0.0.0-20230714201633-919fe359160c +) + +require ( + github.com/golang/protobuf v1.5.3 // indirect + golang.org/x/crypto v0.11.0 // indirect + golang.org/x/net v0.12.0 // indirect + golang.org/x/sys v0.10.0 // indirect + golang.org/x/text v0.11.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect + google.golang.org/protobuf v1.31.0 // indirect +) + +replace google.golang.org/grpc => ../../.. + +replace google.golang.org/grpc/examples => ../../../examples + +replace google.golang.org/grpc/security/advancedtls => ../ diff --git a/security/advancedtls/examples/go.sum b/security/advancedtls/examples/go.sum new file mode 100644 index 000000000000..d8c9b788b8ef --- /dev/null +++ b/security/advancedtls/examples/go.sum @@ -0,0 +1,21 @@ +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= +golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= +golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= +golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 h1:bVf09lpb+OJbByTj913DRJioFFAjf/ZGxEz7MajTp2U= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= diff --git a/security/advancedtls/go.mod b/security/advancedtls/go.mod index 58823b03e877..00bb51de9215 100644 --- a/security/advancedtls/go.mod +++ b/security/advancedtls/go.mod @@ -1,12 +1,23 @@ module google.golang.org/grpc/security/advancedtls -go 1.13 +go 1.19 require ( - github.com/google/go-cmp v0.4.0 - golang.org/x/net v0.0.0-20200602114024-627f9648deb9 // indirect - golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 // indirect - golang.org/x/text v0.3.3 // indirect - google.golang.org/grpc v1.31.0 - google.golang.org/grpc/examples v0.0.0-20200731180010-8bec2f5d898f + github.com/hashicorp/golang-lru v0.5.4 + golang.org/x/crypto v0.11.0 + google.golang.org/grpc v1.56.2 + google.golang.org/grpc/examples v0.0.0-20201112215255-90f1b3ee835b ) + +require ( + github.com/golang/protobuf v1.5.3 // indirect + golang.org/x/net v0.12.0 // indirect + golang.org/x/sys v0.10.0 // indirect + golang.org/x/text v0.11.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect + google.golang.org/protobuf v1.31.0 // indirect +) + +replace google.golang.org/grpc => ../../ + +replace google.golang.org/grpc/examples => ../../examples diff --git a/security/advancedtls/go.sum b/security/advancedtls/go.sum index 8283cf07e736..25a57b755b02 100644 --- a/security/advancedtls/go.sum +++ b/security/advancedtls/go.sum @@ -1,86 +1,22 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20200602114024-627f9648deb9 h1:pNX+40auqi2JqRfOP1akLGtYcn15TUbkhwuCO3foqqM= -golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 h1:OjiUf46hAmXblsZdnoSXsEUSKU8r1UEzcL5RVZ4gO9Y= -golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= +golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= +golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200624020401-64a14ca9d1ad h1:uAwc13+y0Y8QZLTYhLCu6lHhnG99ecQU5FYTj8zxAng= -google.golang.org/genproto v0.0.0-20200624020401-64a14ca9d1ad/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0 h1:T7P4R73V3SSDPhH7WW7ATbfViLtmamH0DKrP3f9AuDI= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc/examples v0.0.0-20200731180010-8bec2f5d898f h1:HNNmM2dnxUknBEJDvuCRZcUzhGbhkz00ckV2ha2gFJY= -google.golang.org/grpc/examples v0.0.0-20200731180010-8bec2f5d898f/go.mod h1:TGiSRL2BBv2WqzfsFNWYp/pkWdtf5kbZS/DQ9Ee3mWk= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 h1:bVf09lpb+OJbByTj913DRJioFFAjf/ZGxEz7MajTp2U= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= diff --git a/security/advancedtls/internal/testutils/testutils.go b/security/advancedtls/internal/testutils/testutils.go new file mode 100644 index 000000000000..1bc0dc3bf4e2 --- /dev/null +++ b/security/advancedtls/internal/testutils/testutils.go @@ -0,0 +1,107 @@ +/* + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package testutils contains helper functions for advancedtls. +package testutils + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "os" + + "google.golang.org/grpc/security/advancedtls/testdata" +) + +// CertStore contains all the certificates used in the integration tests. +type CertStore struct { + // ClientCert1 is the certificate sent by client to prove its identity. + // It is trusted by ServerTrust1. + ClientCert1 tls.Certificate + // ClientCert2 is the certificate sent by client to prove its identity. + // It is trusted by ServerTrust2. + ClientCert2 tls.Certificate + // ServerCert1 is the certificate sent by server to prove its identity. + // It is trusted by ClientTrust1. + ServerCert1 tls.Certificate + // ServerCert2 is the certificate sent by server to prove its identity. + // It is trusted by ClientTrust2. + ServerCert2 tls.Certificate + // ServerPeer3 is the certificate sent by server to prove its identity. + ServerPeer3 tls.Certificate + // ServerPeerLocalhost1 is the certificate sent by server to prove its + // identity. It has "localhost" as its common name, and is trusted by + // ClientTrust1. + ServerPeerLocalhost1 tls.Certificate + // ClientTrust1 is the root certificate used on the client side. + ClientTrust1 *x509.CertPool + // ClientTrust2 is the root certificate used on the client side. + ClientTrust2 *x509.CertPool + // ServerTrust1 is the root certificate used on the server side. + ServerTrust1 *x509.CertPool + // ServerTrust2 is the root certificate used on the server side. + ServerTrust2 *x509.CertPool +} + +func readTrustCert(fileName string) (*x509.CertPool, error) { + trustData, err := os.ReadFile(fileName) + if err != nil { + return nil, err + } + trustPool := x509.NewCertPool() + if !trustPool.AppendCertsFromPEM(trustData) { + return nil, fmt.Errorf("error loading trust certificates") + } + return trustPool, nil +} + +// LoadCerts function is used to load test certificates at the beginning of +// each integration test. +func (cs *CertStore) LoadCerts() error { + var err error + if cs.ClientCert1, err = tls.LoadX509KeyPair(testdata.Path("client_cert_1.pem"), testdata.Path("client_key_1.pem")); err != nil { + return err + } + if cs.ClientCert2, err = tls.LoadX509KeyPair(testdata.Path("client_cert_2.pem"), testdata.Path("client_key_2.pem")); err != nil { + return err + } + if cs.ServerCert1, err = tls.LoadX509KeyPair(testdata.Path("server_cert_1.pem"), testdata.Path("server_key_1.pem")); err != nil { + return err + } + if cs.ServerCert2, err = tls.LoadX509KeyPair(testdata.Path("server_cert_2.pem"), testdata.Path("server_key_2.pem")); err != nil { + return err + } + if cs.ServerPeer3, err = tls.LoadX509KeyPair(testdata.Path("server_cert_3.pem"), testdata.Path("server_key_3.pem")); err != nil { + return err + } + if cs.ServerPeerLocalhost1, err = tls.LoadX509KeyPair(testdata.Path("server_cert_localhost_1.pem"), testdata.Path("server_key_localhost_1.pem")); err != nil { + return err + } + if cs.ClientTrust1, err = readTrustCert(testdata.Path("client_trust_cert_1.pem")); err != nil { + return err + } + if cs.ClientTrust2, err = readTrustCert(testdata.Path("client_trust_cert_2.pem")); err != nil { + return err + } + if cs.ServerTrust1, err = readTrustCert(testdata.Path("server_trust_cert_1.pem")); err != nil { + return err + } + if cs.ServerTrust2, err = readTrustCert(testdata.Path("server_trust_cert_2.pem")); err != nil { + return err + } + return nil +} diff --git a/security/advancedtls/sni.go b/security/advancedtls/sni.go new file mode 100644 index 000000000000..3e7befb1f904 --- /dev/null +++ b/security/advancedtls/sni.go @@ -0,0 +1,51 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package advancedtls + +import ( + "crypto/tls" + "fmt" +) + +// buildGetCertificates returns the certificate that matches the SNI field +// for the given ClientHelloInfo, defaulting to the first element of o.GetCertificates. +func buildGetCertificates(clientHello *tls.ClientHelloInfo, o *ServerOptions) (*tls.Certificate, error) { + if o.IdentityOptions.GetIdentityCertificatesForServer == nil { + return nil, fmt.Errorf("function GetCertificates must be specified") + } + certificates, err := o.IdentityOptions.GetIdentityCertificatesForServer(clientHello) + if err != nil { + return nil, err + } + if len(certificates) == 0 { + return nil, fmt.Errorf("no certificates configured") + } + // If users pass in only one certificate, return that certificate. + if len(certificates) == 1 { + return certificates[0], nil + } + // Choose the SNI certificate using SupportsCertificate. + for _, cert := range certificates { + if err := clientHello.SupportsCertificate(cert); err == nil { + return cert, nil + } + } + // If nothing matches, return the first certificate. + return certificates[0], nil +} diff --git a/security/advancedtls/testdata/README.md b/security/advancedtls/testdata/README.md index 48a1c135c44a..12a6c6b1c1f8 100644 --- a/security/advancedtls/testdata/README.md +++ b/security/advancedtls/testdata/README.md @@ -14,29 +14,35 @@ commands we run: $ openssl req -x509 -newkey rsa:4096 -keyout ca_key.pem -out ca_cert.pem -nodes -days $DURATION_DAYS ``` -2. Generate a CSR `csr.pem` using `subject_key.pem`: +2. Generate a private key `subject_key.pem` for the subject: + + ``` + $ openssl genrsa -out subject_key.pem 4096 + ``` + +3. Generate a CSR `csr.pem` using `subject_key.pem`: ``` $ openssl req -new -key subject_key.pem -out csr.pem ``` - -3. Generate a private key `subject_key.pem` for the subject: - + For some cases, we might want to add some extra SAN fields in `subject_cert.pem`. + In those cases, we can create a configuration file(for example, localhost-openssl.cnf), and do the following: ``` - $ openssl genrsa -out subject_key.pem 4096 + $ openssl req -new -key subject_key.pem -out csr.pem -config $CONFIG_FILE_NAME ``` 4. Use `ca_key.pem` and `ca_cert.pem` to sign `csr.pem`, and get a certificate, `subject_cert.pem`, for the subject: - This step requires some additional files and please check out [this answer from StackOverflow](https://stackoverflow.com/a/21340898) for more. + This step requires some additional configuration steps and please check out [this answer from StackOverflow](https://stackoverflow.com/a/21340898) for more. ``` $ openssl ca -config openssl-ca.cnf -policy signing_policy -extensions signing_req -out subject_cert.pem -in csr.pem -keyfile ca_key.pem -cert ca_cert.pem ``` + Please see an example configuration template at `openssl-ca.cnf`. 5. Verify the `subject_cert.pem` is trusted by `ca_cert.pem`: ``` $ openssl verify -verbose -CAfile ca_cert.pem subject_cert.pem - ``` + ``` \ No newline at end of file diff --git a/security/advancedtls/testdata/another_client_cert_1.pem b/security/advancedtls/testdata/another_client_cert_1.pem new file mode 100644 index 000000000000..603f9d772c11 --- /dev/null +++ b/security/advancedtls/testdata/another_client_cert_1.pem @@ -0,0 +1,122 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 1 (0x1) + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=VA, O=Internet Widgits Pty Ltd, CN=foo.bar.hoo.ca.com + Validity + Not Before: Nov 7 17:11:57 2020 GMT + Not After : Mar 25 17:11:57 2048 GMT + Subject: C=US, ST=CA, O=Internet Widgits Pty Ltd, CN=foo.bar.another.client.com + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + RSA Public-Key: (4096 bit) + Modulus: + 00:ec:00:a0:69:4b:11:ae:30:21:15:64:4d:c3:cb: + 94:67:58:44:9a:5e:ca:c3:82:75:eb:6f:8d:b7:33: + ca:e0:69:5f:98:1f:17:e5:7d:04:dd:f9:af:63:3b: + 84:0e:48:2e:23:fc:b4:de:02:64:ad:0c:8f:6b:70: + 15:fe:00:98:47:cb:c3:ff:5c:fb:b4:22:cd:36:18: + 6e:4a:6d:de:0e:7a:90:d8:3d:af:76:1d:17:07:67: + 79:7f:ab:59:af:1f:37:99:b9:46:ea:db:b3:a4:a2: + 8a:e9:b3:41:83:04:71:82:2f:88:51:5d:b3:ba:7d: + 72:d3:ef:97:47:89:b4:d0:78:f1:d6:ef:8a:09:94: + 19:54:ae:08:8c:77:49:2f:6b:c6:c9:56:0c:fa:ec: + 77:76:f0:83:d8:83:d0:4d:b5:f5:d9:e3:12:85:5c: + 64:c5:82:60:73:20:fa:8a:36:00:f4:f2:bc:cf:48: + 08:94:5f:2b:39:88:4c:56:f5:65:30:67:41:78:99: + 7e:26:f5:ab:e7:3d:b0:a3:8c:55:c5:e1:12:39:23: + 00:68:88:c2:b1:43:2f:61:7c:d4:08:35:52:28:5d: + 93:3e:84:c9:8d:0a:37:df:75:06:f4:ae:1e:2a:1d: + e3:f9:0f:26:80:ad:4e:6a:c3:6a:0c:2e:0f:31:0f: + b2:39:b0:98:aa:1e:96:c7:0b:a3:70:6c:35:52:82: + 56:0f:27:e1:07:d5:89:a6:97:58:97:6f:9f:4f:db: + 0e:e6:ef:fd:62:f4:c3:d5:bb:af:ee:f7:5b:26:6d: + 79:c0:46:71:94:d1:6f:ea:2a:61:78:1c:c5:09:4d: + 93:35:bf:6d:1d:26:49:2e:84:9e:8f:48:9c:93:e0: + ef:bb:c7:85:f1:20:2b:8e:1e:da:18:76:db:c6:42: + 4f:87:ed:e9:44:90:ab:99:ea:d7:3a:89:f9:be:b5: + 3a:b8:5b:50:ec:7c:54:ce:d9:ff:9e:94:30:97:25: + e5:ce:13:b4:ee:06:56:39:0b:d4:51:77:a2:e8:3f: + b4:e7:60:2a:03:44:3a:93:ed:8a:f0:d7:90:f7:92: + fe:e9:19:57:37:24:20:f5:9c:64:d0:fc:13:d1:3e: + a5:1d:ea:14:23:8d:f6:ee:c0:5c:f4:27:82:87:90: + 6c:80:6d:ff:6f:1d:ab:2e:83:0f:73:4a:78:3d:40: + 7f:e6:f9:55:b4:ea:e8:83:f7:86:e0:c8:c1:d9:96: + a0:7d:38:08:72:88:d2:c5:90:e3:3e:2a:f9:64:26: + 52:6e:20:79:20:ea:99:1b:75:65:4b:12:93:22:09: + f9:ec:1b:af:74:a4:3c:a3:df:07:d0:50:95:ad:7e: + 55:60:25 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + D5:43:51:8B:A8:4C:84:D0:C8:DE:29:14:1B:15:7A:62:01:ED:FF:EC + X509v3 Authority Key Identifier: + keyid:B4:19:08:1C:FC:10:23:C5:30:86:22:BC:CB:B1:5F:AD:EA:7A:5D:F1 + + X509v3 Basic Constraints: + CA:FALSE + X509v3 Key Usage: + Digital Signature, Key Encipherment + Signature Algorithm: sha256WithRSAEncryption + 61:a0:89:19:3e:e8:3d:35:bf:6e:5d:0c:d0:ec:36:85:d4:27: + 41:6b:21:0e:5e:ed:e9:e2:17:ef:2b:19:5d:1b:4a:52:7f:e5: + 66:db:24:40:4d:cb:f8:9d:92:5e:10:67:68:d0:f8:56:ab:da: + 3c:8c:4f:02:07:9e:22:49:45:de:bd:d2:92:e8:ef:9d:f2:68: + 97:0d:10:43:c7:b7:73:9e:76:14:7b:1d:b5:8e:df:df:7b:0d: + b3:48:36:21:98:67:3f:41:55:c7:26:09:99:fc:bc:92:8c:9a: + ac:64:76:9d:b2:b1:ca:41:e5:59:79:8d:76:f1:b4:bb:2e:28: + 8b:bb:73:c4:83:0e:f9:6e:62:2c:49:d8:00:e5:87:c7:53:1b: + 45:ec:29:a4:1c:20:82:bf:d1:f4:a4:23:98:b2:1e:41:a0:ee: + 1a:0c:7e:bd:00:84:f5:17:05:1e:a6:7b:fd:75:ee:b9:6d:6b: + 31:2a:0d:97:fa:57:86:57:25:44:8c:5e:e9:bc:78:30:13:77: + 70:87:78:e1:7a:fb:26:db:b9:95:d0:04:c9:93:26:a2:96:b5: + 2b:d5:d3:61:22:b3:ca:31:51:15:6d:51:0a:fb:22:6e:ca:16: + 61:a5:91:9c:e0:39:de:cb:ad:fe:23:51:15:34:42:ae:ac:82: + 80:9a:12:f6:f4:a6:8b:65:b5:f3:21:7b:78:ab:e2:53:83:1c: + 08:83:a5:1a:26:f4:a8:f2:cd:19:d1:85:99:99:45:c8:46:35: + 48:c4:fa:44:80:b0:04:67:48:36:c9:5e:44:fa:41:6e:a7:f2: + eb:22:9e:f8:d3:f2:0d:79:1b:33:78:c4:d0:60:e0:93:5f:69: + 6f:cf:f0:df:04:3d:5b:b3:ac:09:30:ae:32:ea:0f:9f:7b:2c: + 69:bf:f3:fd:6d:7c:a2:25:dc:15:82:df:e0:a5:84:64:39:26: + 43:28:0f:cb:2e:7f:fe:9e:c2:7e:20:d3:ca:6d:96:1c:0e:e1: + b1:ac:0d:2e:4d:97:6f:14:39:65:1a:65:9f:13:2d:0a:28:f3: + 2e:6d:30:af:c6:82:cb:33:20:86:17:de:52:0c:9b:a1:fb:2a: + 59:ca:b1:18:b7:56:06:52:46:9c:8d:36:c6:2e:61:f2:02:2b: + ec:72:15:40:3a:1f:e9:ad:ca:5e:19:aa:1a:73:b5:d4:4d:b9: + d5:b2:24:8c:b6:eb:e6:cc:b2:0e:12:c6:26:52:f7:7a:e7:2c: + 0b:1f:bc:24:1c:76:66:bb:2f:1d:ec:26:9e:53:5b:12:0a:e4: + ae:e3:bc:dd:21:a0:aa:cc:e4:9a:1d:8e:f7:16:36:b2:ed:97: + 81:5a:48:22:a8:f6:6a:2d +-----BEGIN CERTIFICATE----- +MIIFkTCCA3mgAwIBAgIBATANBgkqhkiG9w0BAQsFADBaMQswCQYDVQQGEwJVUzEL +MAkGA1UECAwCVkExITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEb +MBkGA1UEAwwSZm9vLmJhci5ob28uY2EuY29tMB4XDTIwMTEwNzE3MTE1N1oXDTQ4 +MDMyNTE3MTE1N1owYjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMSEwHwYDVQQK +DBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQxIzAhBgNVBAMMGmZvby5iYXIuYW5v +dGhlci5jbGllbnQuY29tMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA +7ACgaUsRrjAhFWRNw8uUZ1hEml7Kw4J162+NtzPK4GlfmB8X5X0E3fmvYzuEDkgu +I/y03gJkrQyPa3AV/gCYR8vD/1z7tCLNNhhuSm3eDnqQ2D2vdh0XB2d5f6tZrx83 +mblG6tuzpKKK6bNBgwRxgi+IUV2zun1y0++XR4m00Hjx1u+KCZQZVK4IjHdJL2vG +yVYM+ux3dvCD2IPQTbX12eMShVxkxYJgcyD6ijYA9PK8z0gIlF8rOYhMVvVlMGdB +eJl+JvWr5z2wo4xVxeESOSMAaIjCsUMvYXzUCDVSKF2TPoTJjQo333UG9K4eKh3j ++Q8mgK1OasNqDC4PMQ+yObCYqh6WxwujcGw1UoJWDyfhB9WJppdYl2+fT9sO5u/9 +YvTD1buv7vdbJm15wEZxlNFv6ipheBzFCU2TNb9tHSZJLoSej0ick+Dvu8eF8SAr +jh7aGHbbxkJPh+3pRJCrmerXOon5vrU6uFtQ7HxUztn/npQwlyXlzhO07gZWOQvU +UXei6D+052AqA0Q6k+2K8NeQ95L+6RlXNyQg9Zxk0PwT0T6lHeoUI4327sBc9CeC +h5BsgG3/bx2rLoMPc0p4PUB/5vlVtOrog/eG4MjB2ZagfTgIcojSxZDjPir5ZCZS +biB5IOqZG3VlSxKTIgn57BuvdKQ8o98H0FCVrX5VYCUCAwEAAaNaMFgwHQYDVR0O +BBYEFNVDUYuoTITQyN4pFBsVemIB7f/sMB8GA1UdIwQYMBaAFLQZCBz8ECPFMIYi +vMuxX63qel3xMAkGA1UdEwQCMAAwCwYDVR0PBAQDAgWgMA0GCSqGSIb3DQEBCwUA +A4ICAQBhoIkZPug9Nb9uXQzQ7DaF1CdBayEOXu3p4hfvKxldG0pSf+Vm2yRATcv4 +nZJeEGdo0PhWq9o8jE8CB54iSUXevdKS6O+d8miXDRBDx7dznnYUex21jt/few2z +SDYhmGc/QVXHJgmZ/LySjJqsZHadsrHKQeVZeY128bS7LiiLu3PEgw75bmIsSdgA +5YfHUxtF7CmkHCCCv9H0pCOYsh5BoO4aDH69AIT1FwUepnv9de65bWsxKg2X+leG +VyVEjF7pvHgwE3dwh3jhevsm27mV0ATJkyailrUr1dNhIrPKMVEVbVEK+yJuyhZh +pZGc4Dney63+I1EVNEKurIKAmhL29KaLZbXzIXt4q+JTgxwIg6UaJvSo8s0Z0YWZ +mUXIRjVIxPpEgLAEZ0g2yV5E+kFup/LrIp740/INeRszeMTQYOCTX2lvz/DfBD1b +s6wJMK4y6g+feyxpv/P9bXyiJdwVgt/gpYRkOSZDKA/LLn/+nsJ+INPKbZYcDuGx +rA0uTZdvFDllGmWfEy0KKPMubTCvxoLLMyCGF95SDJuh+ypZyrEYt1YGUkacjTbG +LmHyAivschVAOh/prcpeGaoac7XUTbnVsiSMtuvmzLIOEsYmUvd65ywLH7wkHHZm +uy8d7CaeU1sSCuSu47zdIaCqzOSaHY73Fjay7ZeBWkgiqPZqLQ== +-----END CERTIFICATE----- diff --git a/security/advancedtls/testdata/another_client_key_1.pem b/security/advancedtls/testdata/another_client_key_1.pem new file mode 100644 index 000000000000..ad61ccfeabea --- /dev/null +++ b/security/advancedtls/testdata/another_client_key_1.pem @@ -0,0 +1,51 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIJKQIBAAKCAgEA7ACgaUsRrjAhFWRNw8uUZ1hEml7Kw4J162+NtzPK4GlfmB8X +5X0E3fmvYzuEDkguI/y03gJkrQyPa3AV/gCYR8vD/1z7tCLNNhhuSm3eDnqQ2D2v +dh0XB2d5f6tZrx83mblG6tuzpKKK6bNBgwRxgi+IUV2zun1y0++XR4m00Hjx1u+K +CZQZVK4IjHdJL2vGyVYM+ux3dvCD2IPQTbX12eMShVxkxYJgcyD6ijYA9PK8z0gI +lF8rOYhMVvVlMGdBeJl+JvWr5z2wo4xVxeESOSMAaIjCsUMvYXzUCDVSKF2TPoTJ +jQo333UG9K4eKh3j+Q8mgK1OasNqDC4PMQ+yObCYqh6WxwujcGw1UoJWDyfhB9WJ +ppdYl2+fT9sO5u/9YvTD1buv7vdbJm15wEZxlNFv6ipheBzFCU2TNb9tHSZJLoSe +j0ick+Dvu8eF8SArjh7aGHbbxkJPh+3pRJCrmerXOon5vrU6uFtQ7HxUztn/npQw +lyXlzhO07gZWOQvUUXei6D+052AqA0Q6k+2K8NeQ95L+6RlXNyQg9Zxk0PwT0T6l +HeoUI4327sBc9CeCh5BsgG3/bx2rLoMPc0p4PUB/5vlVtOrog/eG4MjB2ZagfTgI +cojSxZDjPir5ZCZSbiB5IOqZG3VlSxKTIgn57BuvdKQ8o98H0FCVrX5VYCUCAwEA +AQKCAgAzIaOfjHMlMSpJzzSGAjqB9X7Pj1AQ8dgIjV+/3InM+yeJ9tqfjumaCjm0 +nzVqPrs4cszg+NXFJF6CYYNyR8C2dXBeiE/EZHHfkYV7vLgKnQV6xEqapYzSvtl1 +DrPcnD/Yn2q9AaK3PbwpC/xanYDWOuQm9M02z20se9Fj33L8Y+fJsJZQovSmAxq5 +DDMgAhLMlkczqj3r2ApIw65C1/SPI4JkwHLY0/l/mBqQDUlByMGdizbIpqHf0ibw +BDTLOuPVdDP/zuRSsmvt0z7WI4BmPq4c99xuuWavkXMC4EKPmk6Hkg907kzSrjE2 +m+7PIzC8SksGQAYoXXRBdU03TPZI3PaHb/Lyb6TCVK4gmhAF+QsSobOkFsM72QpF +JBdcRgNsy/fZLFsPFH+g4zCRLrRcPrP1ifiqXOrDJ128JD5PqrEBMyXKQMkUB6qq ++0Hdm2kPWhs5tM4XY7X3xIQJ8AmZ/VPSHwecIUIu4SwdBlaDfQAZgTFER0vzweS5 +PPuJXp/bLW/gOa7hG7bKwmMTHR9ge+i/aMN1JEY6t2pY8AGH9RiH0WRcgiIC6eev +YTmpb8dd4GhatqueKN48UpzCK6CRmvLeBINAyOLybV4RM9cQb6g2sWTc0sgQR4tL +QMB0XAug6QOD7pXFa36JCCrkDDDojjzPaOLCCVtY03si6Yps1QKCAQEA9sKM7nr1 +QoZjo1C7KAu35l1vHVZoXPXm4z8UVNzHqRs+Bd8cut1nNm/kDE5ur5ceGkbSqGs2 +ktqGF/nZYVpUJBKyIXVO4wamUeSFdojllUkXJKOyT+qav1J9/FtOUaONRuc8u452 +HkBclijh4fRkkuM2SROvM7+5YGhQFR/CZN/RXenOD+7wPWa3/3cmV7fZff3dQjDb +7K7vzq/BQVsbfjVSzd3Zwuo0teFz0AEWWt2LE455v/gYP9laIRWxGbIxpRgIyu2e +M/CwRRQT1SOJ9dwGGYls2yTHTfD2hYJnw0fXXo7YyMntMhRazKvQllZF0CdpHZO2 +OBk7UxwNwnMLnwKCAQEA9Nb0RUAR2gDoDQSWoMrRZ3cXCbA1efzP82A/TcppB1En +qqPwuoDd+nTH6KuXaZJYzbGSADyHWzMUaokWz28wC/2ffOhIxSdbU5su/20QfuIN +6rb9BZv6NGjh35nH8ftfyx5tW26iV7mcveig2ixQLg62xTHB8p0FJM3QRIflZ3OK +FWCC/+vZVyiAtGL6xS3XKPHukSeD5zOFKcF3GB9KdSbwED7bdxQ06HGfoWOgCmYb +FnlUWaUZl/wIePYxUSjAmGLkctwH2x83MiRcvPLGI/lW3xzZFHxNj+K9/aJPAbyY +JvZmBeezXphzrU3GjStzrwwd9SQpM4TDkDMIGBk9uwKCAQEAg/KcMZmGNEBwXw/4 +Q/2gJIqps+JUhADpqXI9iPNVwFNU4wbe8f0aB73lD7+Q6EvCSQK9+lj6IaTAN2ne +l3QZsgBdSA7WVAdmQDwWMcAaI62ltm3iF2G3xb5yp9KbGoR+Mv/LNe+DscFwwMqz +noN0lCbzDDh+YwmOMsMUr3cAF7im17UB/vshc3PNx8kKs7UXk4uAGLjPoMwaZ0cL +68qv9NjGolaS7usVrHwV1Y//SC9XAuoYqFIdhWbQDwjuXnMuoL0tVnWhNtzpJMcL +o9kRGGrCyDz3/Ga6PC8xY0rL+VwdCe8QdK2lLDY+J1toejs/sYKhbrNhqLW1R0el +A+lIuQKCAQAWkoKup7t9l7vNB3FDna80lLwg/ofPmUkqrOLpLxIDxK2dg8O7zgmo +/382qispZn6daBOHxgzMkab+M2lQ8nVBhb5ga6HZ20kGKjZpAgsVR430563oCHtG +vaylSq4uVvh753A5j7eT0t7qeznpI1C5Dk43W+D/lw5UWE0tJEI4CWTfl6g8I+hD +qs5C0yU/bHx7n+JYq4XzmMJcGSP7q1bX+iEDvmfJUKmYDHGlFWQ50TQKHGF0ak4z +vt6hGEFvtAwdgHCDTlnDD9us2cFbAh7WTjR+GVDCHLuh2kudyIr0JAj6/phlTvkw +bWmsvpDhjvH5X2qboRvTThghgTLr1dflAoIBAQDn4g3+8Aqf7LTR5EbU64BU7j5R +1CddpJDBafV+lmhkO5oFPFXh72tJu+P1zIRBhCwWqBGCZZQK0LUzezrenyGJH+dk +ZlEr3axKfiv9WKRe1AB6lJSxtOgPy9Kzb5pMFEXipqFQ70i49MjRjJ6WuMRMznug +q/dkVq7q7K+4Dv+/PDHyXols5LsMTNs8bypS0wCubklAk12NxRdeHu4JXJPjtdyS +dQvQj3oheh4km5Q7EqxPnMRvxg2FK7RNwvZ/Q70dCXb8jh7u4Ty4sq5tK4RGyAkL +Zfbng79N0J/+kvRFczaVIvCwSbfexPx/BtGsA1HnfAb2ADug8RlhEwHM9E8G +-----END RSA PRIVATE KEY----- diff --git a/security/advancedtls/testdata/crl/0b35a562.r0 b/security/advancedtls/testdata/crl/0b35a562.r0 new file mode 120000 index 000000000000..1a84eabdfc72 --- /dev/null +++ b/security/advancedtls/testdata/crl/0b35a562.r0 @@ -0,0 +1 @@ +5.crl \ No newline at end of file diff --git a/security/advancedtls/testdata/crl/0b35a562.r1 b/security/advancedtls/testdata/crl/0b35a562.r1 new file mode 120000 index 000000000000..6e6f10978918 --- /dev/null +++ b/security/advancedtls/testdata/crl/0b35a562.r1 @@ -0,0 +1 @@ +1.crl \ No newline at end of file diff --git a/security/advancedtls/testdata/crl/1.crl b/security/advancedtls/testdata/crl/1.crl new file mode 100644 index 000000000000..5b12ded4a66f --- /dev/null +++ b/security/advancedtls/testdata/crl/1.crl @@ -0,0 +1,10 @@ +-----BEGIN X509 CRL----- +MIIBYDCCAQYCAQEwCgYIKoZIzj0EAwIwgaUxCzAJBgNVBAYTAlVTMRMwEQYDVQQI +EwpDYWxpZm9ybmlhMRYwFAYDVQQHEw1Nb3VudGFpbiBWaWV3MRMwEQYDVQQKEwpH +b29nbGUgTExDMSYwEQYDVQQLEwpQcm9kdWN0aW9uMBEGA1UECxMKY2FtcHVzLXNs +bjEsMCoGA1UEAxMjUm9vdCBDQSAoMjAyMS0wMi0wMlQwNzozMDozNi0wODowMCkX +DTIxMDIwMjE1MzAzNloXDTIxMDIwOTE1MzAzNlqgLzAtMB8GA1UdIwQYMBaAFPQN +tnCIBcG4ReQgoVi0kPgTROseMAoGA1UdFAQDAgEAMAoGCCqGSM49BAMCA0gAMEUC +IQDB9WEPBPHEo5xjCv8CT9okockJJnkLDOus6FypVLqj5QIgYw9/PYLwb41/Uc+4 +LLTAsfdDWh7xBJmqvVQglMoJOEc= +-----END X509 CRL----- diff --git a/security/advancedtls/testdata/crl/1ab871c8.r0 b/security/advancedtls/testdata/crl/1ab871c8.r0 new file mode 120000 index 000000000000..f2cd877e7edb --- /dev/null +++ b/security/advancedtls/testdata/crl/1ab871c8.r0 @@ -0,0 +1 @@ +2.crl \ No newline at end of file diff --git a/security/advancedtls/testdata/crl/2.crl b/security/advancedtls/testdata/crl/2.crl new file mode 100644 index 000000000000..5ca9afd71419 --- /dev/null +++ b/security/advancedtls/testdata/crl/2.crl @@ -0,0 +1,10 @@ +-----BEGIN X509 CRL----- +MIIBYDCCAQYCAQEwCgYIKoZIzj0EAwIwgaUxCzAJBgNVBAYTAlVTMRMwEQYDVQQI +EwpDYWxpZm9ybmlhMRYwFAYDVQQHEw1Nb3VudGFpbiBWaWV3MRMwEQYDVQQKEwpH +b29nbGUgTExDMSYwEQYDVQQLEwpQcm9kdWN0aW9uMBEGA1UECxMKY2FtcHVzLXNs +bjEsMCoGA1UEAxMjbm9kZSBDQSAoMjAyMS0wMi0wMlQwNzozMDozNi0wODowMCkX +DTIxMDIwMjE1MzAzNloXDTIxMDIwOTE1MzAzNlqgLzAtMB8GA1UdIwQYMBaAFBjo +V5Jnk/gp1k7fmWwkvTk/cF/IMAoGA1UdFAQDAgEAMAoGCCqGSM49BAMCA0gAMEUC +IQDgjA1Vj/pNFtNRL0vFEdapmFoArHM2+rn4IiP8jYLsCAIgAj2KEHbbtJ3zl5XP +WVW6ZyW7r3wIX+Bt3vLJWPrQtf8= +-----END X509 CRL----- diff --git a/security/advancedtls/testdata/crl/2f11f022.r0 b/security/advancedtls/testdata/crl/2f11f022.r0 new file mode 100644 index 000000000000..e570f17ee2a5 --- /dev/null +++ b/security/advancedtls/testdata/crl/2f11f022.r0 @@ -0,0 +1,7 @@ +-----BEGIN X509 CRL----- +MIHnMFICAQEwDQYJKoZIhvcNAQEMBQAwDDEKMAgGAioDDAI6KRcNMDkxMTEwMjMw +MDAwWhcNMDkxMTExMDAwMDAwWqASMBAwDgYDVR0jBAcwBYADAQIDMA0GCSqGSIb3 +DQEBDAUAA4GBAMl2sjOjtOQ+OCsRyjM0IvqTn7lmdGJMvpYAym367JBamJPCbYrL +MifCjCA1ra7gG0MweZbpm4SG2YLakwi1/B+XhApQ5VVv5SwDn6Yy5zr9ePLEF7Iy +sP86e9s5XfOusLTW+Spre8q1vi7pJrRvUxhJGuUuLoM6Uhvh65ViilDJ +-----END X509 CRL----- diff --git a/security/advancedtls/testdata/crl/3.crl b/security/advancedtls/testdata/crl/3.crl new file mode 100644 index 000000000000..d37ad2247f59 --- /dev/null +++ b/security/advancedtls/testdata/crl/3.crl @@ -0,0 +1,11 @@ +-----BEGIN X509 CRL----- +MIIBiDCCAS8CAQEwCgYIKoZIzj0EAwIwgaUxCzAJBgNVBAYTAlVTMRMwEQYDVQQI +EwpDYWxpZm9ybmlhMRYwFAYDVQQHEw1Nb3VudGFpbiBWaWV3MRMwEQYDVQQKEwpH +b29nbGUgTExDMSYwEQYDVQQLEwpQcm9kdWN0aW9uMBEGA1UECxMKY2FtcHVzLXNs +bjEsMCoGA1UEAxMjUm9vdCBDQSAoMjAyMS0wMi0wMlQwNzozMTo1NC0wODowMCkX +DTIxMDIwMjE1MzE1NFoXDTIxMDIwOTE1MzE1NFowJzAlAhQAroEYW855BRqTrlov +5cBCGvkutxcNMjEwMjAyMTUzMTU0WqAvMC0wHwYDVR0jBBgwFoAUeq/TQ959KbWk +/um08jSTXogXpWUwCgYDVR0UBAMCAQEwCgYIKoZIzj0EAwIDRwAwRAIgaSOIhJDg +wOLYlbXkmxW0cqy/AfOUNYbz5D/8/FfvhosCICftg7Vzlu0Nh83jikyjy+wtkiJt +ZYNvGFQ3Sp2L3A9e +-----END X509 CRL----- diff --git a/security/advancedtls/testdata/crl/4.crl b/security/advancedtls/testdata/crl/4.crl new file mode 100644 index 000000000000..d4ee6f7cf186 --- /dev/null +++ b/security/advancedtls/testdata/crl/4.crl @@ -0,0 +1,10 @@ +-----BEGIN X509 CRL----- +MIIBYDCCAQYCAQEwCgYIKoZIzj0EAwIwgaUxCzAJBgNVBAYTAlVTMRMwEQYDVQQI +EwpDYWxpZm9ybmlhMRYwFAYDVQQHEw1Nb3VudGFpbiBWaWV3MRMwEQYDVQQKEwpH +b29nbGUgTExDMSYwEQYDVQQLEwpQcm9kdWN0aW9uMBEGA1UECxMKY2FtcHVzLXNs +bjEsMCoGA1UEAxMjbm9kZSBDQSAoMjAyMS0wMi0wMlQwNzozMTo1NC0wODowMCkX +DTIxMDIwMjE1MzE1NFoXDTIxMDIwOTE1MzE1NFqgLzAtMB8GA1UdIwQYMBaAFIVn +8tIFgZpIdhomgYJ2c5ULLzpSMAoGA1UdFAQDAgEAMAoGCCqGSM49BAMCA0gAMEUC +ICupTvOqgAyRa1nn7+Pe/1vvlJPAQ8gUfTQsQ6XX3v6oAiEA08B2PsK6aTEwzjry +pXqhlUNZFzgaXrVVQuEJbyJ1qoU= +-----END X509 CRL----- diff --git a/security/advancedtls/testdata/crl/5.crl b/security/advancedtls/testdata/crl/5.crl new file mode 100644 index 000000000000..d1c24f0f25a3 --- /dev/null +++ b/security/advancedtls/testdata/crl/5.crl @@ -0,0 +1,10 @@ +-----BEGIN X509 CRL----- +MIIBXzCCAQYCAQEwCgYIKoZIzj0EAwIwgaUxCzAJBgNVBAYTAlVTMRMwEQYDVQQI +EwpDYWxpZm9ybmlhMRYwFAYDVQQHEw1Nb3VudGFpbiBWaWV3MRMwEQYDVQQKEwpH +b29nbGUgTExDMSYwEQYDVQQLEwpQcm9kdWN0aW9uMBEGA1UECxMKY2FtcHVzLXNs +bjEsMCoGA1UEAxMjUm9vdCBDQSAoMjAyMS0wMi0wMlQwNzozMjo1Ny0wODowMCkX +DTIxMDIwMjE1MzI1N1oXDTIxMDIwOTE1MzI1N1qgLzAtMB8GA1UdIwQYMBaAFN+g +xTAtSTlb5Qqvrbp4rZtsaNzqMAoGA1UdFAQDAgEAMAoGCCqGSM49BAMCA0cAMEQC +IHrRKjieY7w7gxvpkJAdszPZBlaSSp/c9wILutBTy7SyAiAwhaHfgas89iRfaBs2 +EhGIeK39A+kSzqu6qEQBHpK36g== +-----END X509 CRL----- diff --git a/security/advancedtls/testdata/crl/6.crl b/security/advancedtls/testdata/crl/6.crl new file mode 100644 index 000000000000..87ef378f6aba --- /dev/null +++ b/security/advancedtls/testdata/crl/6.crl @@ -0,0 +1,11 @@ +-----BEGIN X509 CRL----- +MIIBiDCCAS8CAQEwCgYIKoZIzj0EAwIwgaUxCzAJBgNVBAYTAlVTMRMwEQYDVQQI +EwpDYWxpZm9ybmlhMRYwFAYDVQQHEw1Nb3VudGFpbiBWaWV3MRMwEQYDVQQKEwpH +b29nbGUgTExDMSYwEQYDVQQLEwpQcm9kdWN0aW9uMBEGA1UECxMKY2FtcHVzLXNs +bjEsMCoGA1UEAxMjbm9kZSBDQSAoMjAyMS0wMi0wMlQwNzozMjo1Ny0wODowMCkX +DTIxMDIwMjE1MzI1N1oXDTIxMDIwOTE1MzI1N1owJzAlAhQAxSe/pGmyvzN7mxm5 +6ZJTYUXYuhcNMjEwMjAyMTUzMjU3WqAvMC0wHwYDVR0jBBgwFoAUpZ30UJXB4lI9 +j2SzodCtRFckrRcwCgYDVR0UBAMCAQEwCgYIKoZIzj0EAwIDRwAwRAIgRg3u7t3b +oyV5FhMuGGzWnfIwnKclpT8imnp8tEN253sCIFUY7DjiDohwu4Zup3bWs1OaZ3q3 +cm+j0H/oe8zzCAgp +-----END X509 CRL----- diff --git a/security/advancedtls/testdata/crl/71eac5a2.r0 b/security/advancedtls/testdata/crl/71eac5a2.r0 new file mode 120000 index 000000000000..9f37924cae0c --- /dev/null +++ b/security/advancedtls/testdata/crl/71eac5a2.r0 @@ -0,0 +1 @@ +4.crl \ No newline at end of file diff --git a/security/advancedtls/testdata/crl/7a1799af.r0 b/security/advancedtls/testdata/crl/7a1799af.r0 new file mode 120000 index 000000000000..f34df5b59c01 --- /dev/null +++ b/security/advancedtls/testdata/crl/7a1799af.r0 @@ -0,0 +1 @@ +3.crl \ No newline at end of file diff --git a/security/advancedtls/testdata/crl/8828a7e6.r0 b/security/advancedtls/testdata/crl/8828a7e6.r0 new file mode 120000 index 000000000000..70bead214cc3 --- /dev/null +++ b/security/advancedtls/testdata/crl/8828a7e6.r0 @@ -0,0 +1 @@ +6.crl \ No newline at end of file diff --git a/security/advancedtls/testdata/crl/README.md b/security/advancedtls/testdata/crl/README.md new file mode 100644 index 000000000000..00cb09c31928 --- /dev/null +++ b/security/advancedtls/testdata/crl/README.md @@ -0,0 +1,48 @@ +# CRL Test Data + +This directory contains cert chains and CRL files for revocation testing. + +To print the chain, use a command like, + +```shell +openssl crl2pkcs7 -nocrl -certfile security/crl/x509/client/testdata/revokedLeaf.pem | openssl pkcs7 -print_certs -text -noout +``` + +The crl file symlinks are generated with `openssl rehash` + +## unrevoked.pem + +A certificate chain with CRL files and unrevoked certs + +* Subject: C=US, ST=California, L=Mountain View, O=Google LLC, OU=Production, + OU=campus-sln, CN=Root CA (2021-02-02T07:30:36-08:00) + * 1.crl + +NOTE: 1.crl file is symlinked with 5.crl to simulate two issuers that hash to +the same value to test that loading multiple files works. + +* Subject: C=US, ST=California, L=Mountain View, O=Google LLC, OU=Production, + OU=campus-sln, CN=node CA (2021-02-02T07:30:36-08:00) + * 2.crl + +## revokedInt.pem + +Certificate chain where the intermediate is revoked + +* Subject: C=US, ST=California, L=Mountain View, O=Google LLC, OU=Production, + OU=campus-sln, CN=Root CA (2021-02-02T07:31:54-08:00) + * 3.crl +* Subject: C=US, ST=California, L=Mountain View, O=Google LLC, OU=Production, + OU=campus-sln, CN=node CA (2021-02-02T07:31:54-08:00) + * 4.crl + +## revokedLeaf.pem + +Certificate chain where the leaf is revoked + +* Subject: C=US, ST=California, L=Mountain View, O=Google LLC, OU=Production, + OU=campus-sln, CN=Root CA (2021-02-02T07:32:57-08:00) + * 5.crl +* Subject: C=US, ST=California, L=Mountain View, O=Google LLC, OU=Production, + OU=campus-sln, CN=node CA (2021-02-02T07:32:57-08:00) + * 6.crl diff --git a/security/advancedtls/testdata/crl/deee447d.r0 b/security/advancedtls/testdata/crl/deee447d.r0 new file mode 120000 index 000000000000..1a84eabdfc72 --- /dev/null +++ b/security/advancedtls/testdata/crl/deee447d.r0 @@ -0,0 +1 @@ +5.crl \ No newline at end of file diff --git a/security/advancedtls/testdata/crl/revokedInt.pem b/security/advancedtls/testdata/crl/revokedInt.pem new file mode 100644 index 000000000000..8b7282ff8221 --- /dev/null +++ b/security/advancedtls/testdata/crl/revokedInt.pem @@ -0,0 +1,58 @@ +-----BEGIN CERTIFICATE----- +MIIDAzCCAqmgAwIBAgITAWjKwm2dNQvkO62Jgyr5rAvVQzAKBggqhkjOPQQDAjCB +pTELMAkGA1UEBhMCVVMxEzARBgNVBAgTCkNhbGlmb3JuaWExFjAUBgNVBAcTDU1v +dW50YWluIFZpZXcxEzARBgNVBAoTCkdvb2dsZSBMTEMxJjARBgNVBAsTClByb2R1 +Y3Rpb24wEQYDVQQLEwpjYW1wdXMtc2xuMSwwKgYDVQQDEyNSb290IENBICgyMDIx +LTAyLTAyVDA3OjMxOjU0LTA4OjAwKTAgFw0yMTAyMDIxNTMxNTRaGA85OTk5MTIz +MTIzNTk1OVowgaUxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYw +FAYDVQQHEw1Nb3VudGFpbiBWaWV3MRMwEQYDVQQKEwpHb29nbGUgTExDMSYwEQYD +VQQLEwpQcm9kdWN0aW9uMBEGA1UECxMKY2FtcHVzLXNsbjEsMCoGA1UEAxMjUm9v +dCBDQSAoMjAyMS0wMi0wMlQwNzozMTo1NC0wODowMCkwWTATBgcqhkjOPQIBBggq +hkjOPQMBBwNCAAQhA0/puhTtSxbVVHseVhL2z7QhpPyJs5Q4beKi7tpaYRDmVn6p +Phh+jbRzg8Qj4gKI/Q1rrdm4rKer63LHpdWdo4GzMIGwMA4GA1UdDwEB/wQEAwIB +BjAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDwYDVR0TAQH/BAUwAwEB +/zAdBgNVHQ4EFgQUeq/TQ959KbWk/um08jSTXogXpWUwHwYDVR0jBBgwFoAUeq/T +Q959KbWk/um08jSTXogXpWUwLgYDVR0RBCcwJYYjc3BpZmZlOi8vY2FtcHVzLXNs +bi5wcm9kLmdvb2dsZS5jb20wCgYIKoZIzj0EAwIDSAAwRQIgOSQZvyDPQwVOWnpF +zWvI+DS2yXIj/2T2EOvJz2XgcK4CIQCL0mh/+DxLiO4zzbInKr0mxpGSxSeZCUk7 +1ZF7AeLlbw== +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIDizCCAzKgAwIBAgIUAK6BGFvOeQUak65aL+XAQhr5LrcwCgYIKoZIzj0EAwIw +gaUxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQHEw1N +b3VudGFpbiBWaWV3MRMwEQYDVQQKEwpHb29nbGUgTExDMSYwEQYDVQQLEwpQcm9k +dWN0aW9uMBEGA1UECxMKY2FtcHVzLXNsbjEsMCoGA1UEAxMjUm9vdCBDQSAoMjAy +MS0wMi0wMlQwNzozMTo1NC0wODowMCkwIBcNMjEwMjAyMTUzMTU0WhgPOTk5OTEy +MzEyMzU5NTlaMIGlMQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEW +MBQGA1UEBxMNTW91bnRhaW4gVmlldzETMBEGA1UEChMKR29vZ2xlIExMQzEmMBEG +A1UECxMKUHJvZHVjdGlvbjARBgNVBAsTCmNhbXB1cy1zbG4xLDAqBgNVBAMTI25v +ZGUgQ0EgKDIwMjEtMDItMDJUMDc6MzE6NTQtMDg6MDApMFkwEwYHKoZIzj0CAQYI +KoZIzj0DAQcDQgAEye6UOlBos8Q3FFBiLahD9BaLTA18bO4MTPyv35T3lppvxD5X +U/AnEllOnx5OMtMjMBbIQjSkMbiQ9xNXoSqB6aOCATowggE2MA4GA1UdDwEB/wQE +AwIBBjAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDwYDVR0TAQH/BAUw +AwEB/zAdBgNVHQ4EFgQUhWfy0gWBmkh2GiaBgnZzlQsvOlIwHwYDVR0jBBgwFoAU +eq/TQ959KbWk/um08jSTXogXpWUwMwYDVR0RBCwwKoYoc3BpZmZlOi8vbm9kZS5j +YW1wdXMtc2xuLnByb2QuZ29vZ2xlLmNvbTA7BgNVHR4BAf8EMTAvoC0wK4YpY3Nj +cy10ZWFtLm5vZGUuY2FtcHVzLXNsbi5wcm9kLmdvb2dsZS5jb20wQgYDVR0fBDsw +OTA3oDWgM4YxaHR0cDovL3N0YXRpYy5jb3JwLmdvb2dsZS5jb20vY3JsL2NhbXB1 +cy1zbG4vbm9kZTAKBggqhkjOPQQDAgNHADBEAiA79rPu6ZO1/0qB6RxL7jVz1200 +UTo8ioB4itbTzMnJqAIgJqp/Rc8OhpsfzQX8XnIIkl+SewT+tOxJT1MHVNMlVhc= +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIC0DCCAnWgAwIBAgITXQ2c/C27OGqk4Pbu+MNJlOtpYTAKBggqhkjOPQQDAjCB +pTELMAkGA1UEBhMCVVMxEzARBgNVBAgTCkNhbGlmb3JuaWExFjAUBgNVBAcTDU1v +dW50YWluIFZpZXcxEzARBgNVBAoTCkdvb2dsZSBMTEMxJjARBgNVBAsTClByb2R1 +Y3Rpb24wEQYDVQQLEwpjYW1wdXMtc2xuMSwwKgYDVQQDEyNub2RlIENBICgyMDIx +LTAyLTAyVDA3OjMxOjU0LTA4OjAwKTAgFw0yMTAyMDIxNTMxNTRaGA85OTk5MTIz +MTIzNTk1OVowADBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABN2/1le5d3hS/piw +hrNMHjd7gPEjzXwtuXQTzdV+aaeOf3ldnC6OnEF/bggym9MldQSJZLXPYSaoj430 +Vu5PRNejggEkMIIBIDAOBgNVHQ8BAf8EBAMCB4AwHQYDVR0lBBYwFAYIKwYBBQUH +AwIGCCsGAQUFBwMBMB0GA1UdDgQWBBTEewP3JgrJPekWWGGjChVqaMhaqTAfBgNV +HSMEGDAWgBSFZ/LSBYGaSHYaJoGCdnOVCy86UjBrBgNVHREBAf8EYTBfghZqemFi +MTIucHJvZC5nb29nbGUuY29thkVzcGlmZmU6Ly9jc2NzLXRlYW0ubm9kZS5jYW1w +dXMtc2xuLnByb2QuZ29vZ2xlLmNvbS9yb2xlL2JvcmctYWRtaW4tY28wQgYDVR0f +BDswOTA3oDWgM4YxaHR0cDovL3N0YXRpYy5jb3JwLmdvb2dsZS5jb20vY3JsL2Nh +bXB1cy1zbG4vbm9kZTAKBggqhkjOPQQDAgNJADBGAiEA9w4qp3nHpXo+6d7mZc69 +QoALfP5ynfBCArt8bAlToo8CIQCgc/lTfl2BtBko+7h/w6pKxLeuoQkvCL5gHFyK +LXE6vA== +-----END CERTIFICATE----- diff --git a/security/advancedtls/testdata/crl/revokedLeaf.pem b/security/advancedtls/testdata/crl/revokedLeaf.pem new file mode 100644 index 000000000000..b7541abf6214 --- /dev/null +++ b/security/advancedtls/testdata/crl/revokedLeaf.pem @@ -0,0 +1,59 @@ +-----BEGIN CERTIFICATE----- +MIIDAzCCAqmgAwIBAgITTwodm6C4ZabFVUVa5yBw0TbzJTAKBggqhkjOPQQDAjCB +pTELMAkGA1UEBhMCVVMxEzARBgNVBAgTCkNhbGlmb3JuaWExFjAUBgNVBAcTDU1v +dW50YWluIFZpZXcxEzARBgNVBAoTCkdvb2dsZSBMTEMxJjARBgNVBAsTClByb2R1 +Y3Rpb24wEQYDVQQLEwpjYW1wdXMtc2xuMSwwKgYDVQQDEyNSb290IENBICgyMDIx +LTAyLTAyVDA3OjMyOjU3LTA4OjAwKTAgFw0yMTAyMDIxNTMyNTdaGA85OTk5MTIz +MTIzNTk1OVowgaUxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYw +FAYDVQQHEw1Nb3VudGFpbiBWaWV3MRMwEQYDVQQKEwpHb29nbGUgTExDMSYwEQYD +VQQLEwpQcm9kdWN0aW9uMBEGA1UECxMKY2FtcHVzLXNsbjEsMCoGA1UEAxMjUm9v +dCBDQSAoMjAyMS0wMi0wMlQwNzozMjo1Ny0wODowMCkwWTATBgcqhkjOPQIBBggq +hkjOPQMBBwNCAARoZnzQWvAoyhvCLA2cFIK17khSaA9aA+flS5X9fLRt4RsfPCx3 +kim7wYKQSmBhQdc1UM4h3969r1c1Fvsh2H9qo4GzMIGwMA4GA1UdDwEB/wQEAwIB +BjAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDwYDVR0TAQH/BAUwAwEB +/zAdBgNVHQ4EFgQU36DFMC1JOVvlCq+tunitm2xo3OowHwYDVR0jBBgwFoAU36DF +MC1JOVvlCq+tunitm2xo3OowLgYDVR0RBCcwJYYjc3BpZmZlOi8vY2FtcHVzLXNs +bi5wcm9kLmdvb2dsZS5jb20wCgYIKoZIzj0EAwIDSAAwRQIgN7S9dQOQzNih92ag +7c5uQxuz+M6wnxWj/uwGQIIghRUCIQD2UDH6kkRSYQuyP0oN7XYO3XFjmZ2Yer6m +1ZS8fyWYYA== +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIDjTCCAzKgAwIBAgIUAOmArBu9gihLTlqP3W7Et0UoocEwCgYIKoZIzj0EAwIw +gaUxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQHEw1N +b3VudGFpbiBWaWV3MRMwEQYDVQQKEwpHb29nbGUgTExDMSYwEQYDVQQLEwpQcm9k +dWN0aW9uMBEGA1UECxMKY2FtcHVzLXNsbjEsMCoGA1UEAxMjUm9vdCBDQSAoMjAy +MS0wMi0wMlQwNzozMjo1Ny0wODowMCkwIBcNMjEwMjAyMTUzMjU3WhgPOTk5OTEy +MzEyMzU5NTlaMIGlMQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEW +MBQGA1UEBxMNTW91bnRhaW4gVmlldzETMBEGA1UEChMKR29vZ2xlIExMQzEmMBEG +A1UECxMKUHJvZHVjdGlvbjARBgNVBAsTCmNhbXB1cy1zbG4xLDAqBgNVBAMTI25v +ZGUgQ0EgKDIwMjEtMDItMDJUMDc6MzI6NTctMDg6MDApMFkwEwYHKoZIzj0CAQYI +KoZIzj0DAQcDQgAEfrgVEVQfSEFeCF1/FGeW7oq0yxecenT1BESfj4Z0zJ8p7P9W +bj1o6Rn6dUNlEhGrx7E3/4NFJ0cL1BSNGHkjiqOCATowggE2MA4GA1UdDwEB/wQE +AwIBBjAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDwYDVR0TAQH/BAUw +AwEB/zAdBgNVHQ4EFgQUpZ30UJXB4lI9j2SzodCtRFckrRcwHwYDVR0jBBgwFoAU +36DFMC1JOVvlCq+tunitm2xo3OowMwYDVR0RBCwwKoYoc3BpZmZlOi8vbm9kZS5j +YW1wdXMtc2xuLnByb2QuZ29vZ2xlLmNvbTA7BgNVHR4BAf8EMTAvoC0wK4YpY3Nj +cy10ZWFtLm5vZGUuY2FtcHVzLXNsbi5wcm9kLmdvb2dsZS5jb20wQgYDVR0fBDsw +OTA3oDWgM4YxaHR0cDovL3N0YXRpYy5jb3JwLmdvb2dsZS5jb20vY3JsL2NhbXB1 +cy1zbG4vbm9kZTAKBggqhkjOPQQDAgNJADBGAiEAnuONgMqmbBlj4ibw5BgDtZUM +pboACSFJtEOJu4Yqjt0CIQDI5193J4wUcAY0BK0vO9rRfbNOIc+4ke9ieBDPSuhm +mA== +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIICzzCCAnagAwIBAgIUAMUnv6Rpsr8ze5sZuemSU2FF2LowCgYIKoZIzj0EAwIw +gaUxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQHEw1N +b3VudGFpbiBWaWV3MRMwEQYDVQQKEwpHb29nbGUgTExDMSYwEQYDVQQLEwpQcm9k +dWN0aW9uMBEGA1UECxMKY2FtcHVzLXNsbjEsMCoGA1UEAxMjbm9kZSBDQSAoMjAy +MS0wMi0wMlQwNzozMjo1Ny0wODowMCkwIBcNMjEwMjAyMTUzMjU3WhgPOTk5OTEy +MzEyMzU5NTlaMAAwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAASCmYiIHUux5WFz +S0ksJzAPL7YTEh5o5MdXgLPB/WM6x9sVsQDSYU0PF5qc9vPNhkQzGBW79dkBnxhW +AGJkFr1Po4IBJDCCASAwDgYDVR0PAQH/BAQDAgeAMB0GA1UdJQQWMBQGCCsGAQUF +BwMCBggrBgEFBQcDATAdBgNVHQ4EFgQUCR1CGEdlks0qcxCExO0rP1B/Z7UwHwYD +VR0jBBgwFoAUpZ30UJXB4lI9j2SzodCtRFckrRcwawYDVR0RAQH/BGEwX4IWanph +YjEyLnByb2QuZ29vZ2xlLmNvbYZFc3BpZmZlOi8vY3Njcy10ZWFtLm5vZGUuY2Ft +cHVzLXNsbi5wcm9kLmdvb2dsZS5jb20vcm9sZS9ib3JnLWFkbWluLWNvMEIGA1Ud +HwQ7MDkwN6A1oDOGMWh0dHA6Ly9zdGF0aWMuY29ycC5nb29nbGUuY29tL2NybC9j +YW1wdXMtc2xuL25vZGUwCgYIKoZIzj0EAwIDRwAwRAIgK9vQYNoL8HlEwWv89ioG +aQ1+8swq6Bo/5mJBrdVLvY8CIGxo6M9vJkPdObmetWNC+lmKuZDoqJWI0AAmBT2J +mR2r +-----END CERTIFICATE----- diff --git a/security/advancedtls/testdata/crl/unrevoked.pem b/security/advancedtls/testdata/crl/unrevoked.pem new file mode 100644 index 000000000000..5c5fc58a7a5e --- /dev/null +++ b/security/advancedtls/testdata/crl/unrevoked.pem @@ -0,0 +1,58 @@ +-----BEGIN CERTIFICATE----- +MIIDBDCCAqqgAwIBAgIUALy864QhnkTdceLH52k2XVOe8IQwCgYIKoZIzj0EAwIw +gaUxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQHEw1N +b3VudGFpbiBWaWV3MRMwEQYDVQQKEwpHb29nbGUgTExDMSYwEQYDVQQLEwpQcm9k +dWN0aW9uMBEGA1UECxMKY2FtcHVzLXNsbjEsMCoGA1UEAxMjUm9vdCBDQSAoMjAy +MS0wMi0wMlQwNzozMDozNi0wODowMCkwIBcNMjEwMjAyMTUzMDM2WhgPOTk5OTEy +MzEyMzU5NTlaMIGlMQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEW +MBQGA1UEBxMNTW91bnRhaW4gVmlldzETMBEGA1UEChMKR29vZ2xlIExMQzEmMBEG +A1UECxMKUHJvZHVjdGlvbjARBgNVBAsTCmNhbXB1cy1zbG4xLDAqBgNVBAMTI1Jv +b3QgQ0EgKDIwMjEtMDItMDJUMDc6MzA6MzYtMDg6MDApMFkwEwYHKoZIzj0CAQYI +KoZIzj0DAQcDQgAEYv/JS5hQ5kIgdKqYZWTKCO/6gloHAmIb1G8lmY0oXLXYNHQ4 +qHN7/pPtlcHQp0WK/hM8IGvgOUDoynA8mj0H9KOBszCBsDAOBgNVHQ8BAf8EBAMC +AQYwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMA8GA1UdEwEB/wQFMAMB +Af8wHQYDVR0OBBYEFPQNtnCIBcG4ReQgoVi0kPgTROseMB8GA1UdIwQYMBaAFPQN +tnCIBcG4ReQgoVi0kPgTROseMC4GA1UdEQQnMCWGI3NwaWZmZTovL2NhbXB1cy1z +bG4ucHJvZC5nb29nbGUuY29tMAoGCCqGSM49BAMCA0gAMEUCIQDwBn20DB4X/7Uk +Q5BR8JxQYUPxOfvuedjfeA8bPvQ2FwIgOEWa0cXJs1JxarILJeCXtdXvBgu6LEGQ +3Pk/bgz8Gek= +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIDizCCAzKgAwIBAgIUAM/6RKQ7Vke0i4xp5LaAqV73cmIwCgYIKoZIzj0EAwIw +gaUxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQHEw1N +b3VudGFpbiBWaWV3MRMwEQYDVQQKEwpHb29nbGUgTExDMSYwEQYDVQQLEwpQcm9k +dWN0aW9uMBEGA1UECxMKY2FtcHVzLXNsbjEsMCoGA1UEAxMjUm9vdCBDQSAoMjAy +MS0wMi0wMlQwNzozMDozNi0wODowMCkwIBcNMjEwMjAyMTUzMDM2WhgPOTk5OTEy +MzEyMzU5NTlaMIGlMQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEW +MBQGA1UEBxMNTW91bnRhaW4gVmlldzETMBEGA1UEChMKR29vZ2xlIExMQzEmMBEG +A1UECxMKUHJvZHVjdGlvbjARBgNVBAsTCmNhbXB1cy1zbG4xLDAqBgNVBAMTI25v +ZGUgQ0EgKDIwMjEtMDItMDJUMDc6MzA6MzYtMDg6MDApMFkwEwYHKoZIzj0CAQYI +KoZIzj0DAQcDQgAEllnhxmMYiUPUgRGmenbnm10gXpM94zHx3D1/HumPs6arjYuT +Zlhx81XL+g4bu4HII2qcGdP+Hqj/MMFNDI9z4aOCATowggE2MA4GA1UdDwEB/wQE +AwIBBjAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDwYDVR0TAQH/BAUw +AwEB/zAdBgNVHQ4EFgQUGOhXkmeT+CnWTt+ZbCS9OT9wX8gwHwYDVR0jBBgwFoAU +9A22cIgFwbhF5CChWLSQ+BNE6x4wMwYDVR0RBCwwKoYoc3BpZmZlOi8vbm9kZS5j +YW1wdXMtc2xuLnByb2QuZ29vZ2xlLmNvbTA7BgNVHR4BAf8EMTAvoC0wK4YpY3Nj +cy10ZWFtLm5vZGUuY2FtcHVzLXNsbi5wcm9kLmdvb2dsZS5jb20wQgYDVR0fBDsw +OTA3oDWgM4YxaHR0cDovL3N0YXRpYy5jb3JwLmdvb2dsZS5jb20vY3JsL2NhbXB1 +cy1zbG4vbm9kZTAKBggqhkjOPQQDAgNHADBEAiA86egqPw0qyapAeMGbHxrmYZYa +i5ARQsSKRmQixgYizQIgW+2iRWN6Kbqt4WcwpmGv/xDckdRXakF5Ign/WUDO5u4= +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIICzzCCAnWgAwIBAgITYjjKfYZUKQNUjNyF+hLDGpHJKTAKBggqhkjOPQQDAjCB +pTELMAkGA1UEBhMCVVMxEzARBgNVBAgTCkNhbGlmb3JuaWExFjAUBgNVBAcTDU1v +dW50YWluIFZpZXcxEzARBgNVBAoTCkdvb2dsZSBMTEMxJjARBgNVBAsTClByb2R1 +Y3Rpb24wEQYDVQQLEwpjYW1wdXMtc2xuMSwwKgYDVQQDEyNub2RlIENBICgyMDIx +LTAyLTAyVDA3OjMwOjM2LTA4OjAwKTAgFw0yMTAyMDIxNTMwMzZaGA85OTk5MTIz +MTIzNTk1OVowADBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABD4r4+nCgZExYF8v +CLvGn0lY/cmam8mAkJDXRN2Ja2t+JwaTOptPmbbXft+1NTk5gCg5wB+FJCnaV3I/ +HaxEhBWjggEkMIIBIDAOBgNVHQ8BAf8EBAMCB4AwHQYDVR0lBBYwFAYIKwYBBQUH +AwIGCCsGAQUFBwMBMB0GA1UdDgQWBBTTCjXX1Txjc00tBg/5cFzpeCSKuDAfBgNV +HSMEGDAWgBQY6FeSZ5P4KdZO35lsJL05P3BfyDBrBgNVHREBAf8EYTBfghZqemFi +MTIucHJvZC5nb29nbGUuY29thkVzcGlmZmU6Ly9jc2NzLXRlYW0ubm9kZS5jYW1w +dXMtc2xuLnByb2QuZ29vZ2xlLmNvbS9yb2xlL2JvcmctYWRtaW4tY28wQgYDVR0f +BDswOTA3oDWgM4YxaHR0cDovL3N0YXRpYy5jb3JwLmdvb2dsZS5jb20vY3JsL2Nh +bXB1cy1zbG4vbm9kZTAKBggqhkjOPQQDAgNIADBFAiBq3URViNyMLpvzZHC1Y+4L ++35guyIJfjHu08P3S8/xswIhAJtWSQ1ZtozdOzGxg7GfUo4hR+5SP6rBTgIqXEfq +48fW +-----END CERTIFICATE----- diff --git a/security/advancedtls/testdata/localhost-openssl.cnf b/security/advancedtls/testdata/localhost-openssl.cnf new file mode 100644 index 000000000000..5f46206d2889 --- /dev/null +++ b/security/advancedtls/testdata/localhost-openssl.cnf @@ -0,0 +1,23 @@ +[req] +distinguished_name = req_distinguished_name +req_extensions = v3_req + +[req_distinguished_name] +countryName = Country Name (2 letter code) +countryName_default = US +stateOrProvinceName = State or Province Name (full name) +stateOrProvinceName_default = Illinois +localityName = Locality Name (eg, city) +localityName_default = Chicago +organizationName = Organization Name (eg, company) +organizationName_default = Example, Co. +commonName = Common Name (eg, YOUR name) +commonName_max = 64 + +[v3_req] +basicConstraints = CA:FALSE +keyUsage = nonRepudiation, digitalSignature, keyEncipherment +subjectAltName = @alt_names + +[alt_names] +DNS.1 = localhost diff --git a/security/advancedtls/testdata/openssl-ca.cnf b/security/advancedtls/testdata/openssl-ca.cnf new file mode 100644 index 000000000000..196a50c26471 --- /dev/null +++ b/security/advancedtls/testdata/openssl-ca.cnf @@ -0,0 +1,87 @@ +base_dir = . +certificate = $base_dir/cacert.pem # The CA certifcate +private_key = $base_dir/cakey.pem # The CA private key +new_certs_dir = $base_dir # Location for new certs after signing +database = $base_dir/index.txt # Database index file +serial = $base_dir/serial.txt # The current serial number + +unique_subject = no # Set to 'no' to allow creation of + # several certificates with same subject. + +HOME = . +RANDFILE = $ENV::HOME/.rnd + +#################################################################### +[ ca ] +default_ca = CA_default # The default ca section + +[ CA_default ] + +default_days = 10000 # How long to certify for +default_crl_days = 30 # How long before next CRL +default_md = sha256 # Use public key default MD +preserve = no # Keep passed DN ordering + +x509_extensions = ca_extensions # The extensions to add to the cert + +email_in_dn = no # Don't concat the email in the DN +copy_extensions = copy # Required to copy SANs from CSR to cert + +#################################################################### +[ req ] +default_bits = 4096 +default_keyfile = cakey.pem +distinguished_name = ca_distinguished_name +x509_extensions = ca_extensions +string_mask = utf8only + +#################################################################### +[ ca_distinguished_name ] +countryName = Country Name (2 letter code) +countryName_default = US + +stateOrProvinceName = State or Province Name (full name) +stateOrProvinceName_default = Maryland + +localityName = Locality Name (eg, city) +localityName_default = Baltimore + +organizationName = Organization Name (eg, company) +organizationName_default = Test CA, Limited + +organizationalUnitName = Organizational Unit (eg, division) +organizationalUnitName_default = Server Research Department + +commonName = Common Name (e.g. server FQDN or YOUR name) +commonName_default = Test CA + +emailAddress = Email Address +emailAddress_default = test@example.com + +#################################################################### +[ ca_extensions ] + +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid:always, issuer +basicConstraints = critical, CA:true +keyUsage = keyCertSign, cRLSign + + + + +#################################################################### +[ signing_policy ] +countryName = optional +stateOrProvinceName = optional +localityName = optional +organizationName = optional +organizationalUnitName = optional +commonName = supplied +emailAddress = optional + +#################################################################### +[ signing_req ] +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid,issuer +basicConstraints = CA:FALSE +keyUsage = digitalSignature, keyEncipherment diff --git a/security/advancedtls/testdata/server_cert_localhost_1.pem b/security/advancedtls/testdata/server_cert_localhost_1.pem new file mode 100644 index 000000000000..b6364f23ac9b --- /dev/null +++ b/security/advancedtls/testdata/server_cert_localhost_1.pem @@ -0,0 +1,124 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 5 (0x5) + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=US, ST=CA, L=SVL, O=Internet Widgits Pty Ltd + Validity + Not Before: Dec 15 18:05:59 2020 GMT + Not After : May 2 18:05:59 2048 GMT + Subject: C=US, ST=Illinois, L=Chicago, O=Example, Co., CN=localhost + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + RSA Public-Key: (4096 bit) + Modulus: + 00:d3:4d:01:08:9d:80:de:15:fa:0a:52:ed:2c:f6: + 4b:9d:c5:67:2d:b6:41:33:d7:5b:b4:fd:f0:a8:5c: + dd:8a:8d:1f:2d:12:5d:47:88:09:d4:96:ee:63:10: + f9:9d:9a:6c:34:c4:ec:a9:0d:95:6c:48:bc:17:d0: + 77:53:93:f8:44:8a:0b:0b:a2:4d:1d:53:f7:55:a7: + ed:6a:35:ad:1d:af:79:bd:d5:c6:5b:96:24:9e:4d: + d8:e9:21:93:e1:93:3b:5d:c4:e3:90:1a:36:d0:f7: + bb:8c:22:e9:d2:f8:29:19:bf:24:c6:52:21:0e:d6: + 3a:73:48:ba:e7:81:34:ba:23:21:57:c3:51:0e:59: + 51:9f:a6:07:56:17:c4:aa:1b:7b:6c:36:6b:ab:32: + 5e:2e:f7:70:dc:eb:f3:da:3b:39:4e:a4:8c:bf:40: + 72:ef:00:1e:46:c9:07:6a:93:4c:36:b7:c3:2e:7c: + c5:c1:85:9f:6b:4d:2d:fd:3c:9b:9a:cf:52:b2:fa: + ba:f8:ce:cc:8b:dc:57:7b:ed:37:69:c9:80:dc:a6: + 2c:a7:e4:4b:8c:77:cb:2e:28:65:64:61:a4:c8:33: + d9:f8:7d:b7:7c:4f:b7:f4:07:5a:89:ae:2a:59:8c: + ac:00:c7:ce:b0:d0:9b:cd:4d:83:39:55:bb:72:0f: + 62:d4:5f:8b:c3:c7:e2:79:95:53:eb:a0:26:0d:52: + 7b:4d:40:56:66:2b:55:67:f5:1a:c9:e8:a8:49:bd: + e7:e4:31:9a:e1:8d:80:f2:cc:ab:3d:70:f6:fe:75: + cb:aa:1b:12:d0:6f:d3:c8:df:f1:bd:ce:2b:21:42: + e5:2c:bd:c6:c8:c4:bb:4a:3d:92:48:0d:49:a8:96: + c0:51:ed:30:64:81:dd:ce:05:d4:ff:07:87:59:0b: + 13:41:8e:1d:58:e4:47:0c:00:97:12:e4:67:94:24: + 67:ef:ed:1e:85:89:df:85:78:10:1f:7f:b2:e8:af: + e7:0f:c7:ec:aa:01:67:b6:0a:9a:23:83:90:1d:0a: + 2a:37:80:c3:26:d7:f1:24:29:b3:d8:37:7c:5c:f8: + e3:08:96:1a:34:2d:fa:ff:75:e8:70:25:e2:ab:51: + 9d:f7:8a:23:52:89:02:c8:71:ea:cd:a2:89:b6:eb: + 6e:c9:fd:fb:dc:63:e8:f9:db:d7:57:d2:0c:fc:56: + 9c:16:a8:67:fe:17:88:07:e8:f8:c5:7a:33:72:ad: + b4:5a:43:5b:49:be:79:8b:00:18:83:f7:19:00:07: + 94:2c:38:96:61:a7:55:a8:30:db:7e:07:f6:c5:a3: + 65:1e:93:2d:6b:5e:d9:a6:0f:fa:c2:19:9b:59:4c: + 41:ba:07 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + F3:DC:6A:5B:B7:CE:E9:E1:4D:3E:C4:AE:B7:8E:39:E3:6D:CA:AF:C7 + X509v3 Authority Key Identifier: + keyid:5A:A5:DA:B1:99:D4:E5:0E:E6:1E:94:EA:FF:FC:62:E2:ED:09:F1:06 + + X509v3 Basic Constraints: + CA:FALSE + X509v3 Key Usage: + Digital Signature, Key Encipherment + X509v3 Subject Alternative Name: + DNS:localhost + Signature Algorithm: sha256WithRSAEncryption + 54:13:3d:55:d3:4b:d8:85:f0:54:a8:33:5c:a1:9f:87:79:31: + 34:7e:52:b9:46:ea:97:43:fd:0a:9b:ae:2c:8b:74:cd:51:d5: + 63:b3:b0:b4:19:a1:da:73:b4:e2:47:0c:d9:33:4a:c6:aa:27: + 41:bc:4d:15:74:42:59:eb:41:8e:28:49:f0:55:89:13:f8:f0: + 35:1f:e9:b2:ae:48:79:e7:15:a2:aa:59:e0:fd:91:c8:7f:ba: + aa:a4:2e:77:2a:e4:62:fe:c2:83:51:dc:58:83:f8:7f:b2:47: + 68:8b:1f:9d:9b:12:25:f2:0f:7d:06:a4:9a:be:a3:af:2d:27: + 32:4a:20:fe:5f:98:d0:5d:6a:10:c9:04:49:54:26:60:20:6e: + 38:f2:91:a4:24:e7:ed:d2:e4:aa:88:7e:9c:6d:0a:1e:30:97: + a9:9c:45:35:09:fc:45:6c:32:d0:db:79:04:fa:03:7c:36:e5: + 27:72:1d:77:a1:d4:12:86:53:bf:93:4e:f7:da:7a:a1:4b:5b: + 42:e4:6b:9b:a5:58:f2:ef:30:2e:a6:6f:f7:63:fd:16:b0:35: + 81:99:3d:41:dc:da:34:46:ea:1f:02:cc:5e:dc:67:1c:62:68: + 76:45:5b:ea:f3:2f:a5:0f:2e:b8:d0:df:ad:8f:3b:03:b4:36: + 21:d4:4d:99:09:76:22:a4:61:b4:59:ca:8b:42:5d:64:2a:9a: + 4f:7f:67:65:38:4a:c6:e7:8b:ff:36:0d:42:4b:60:03:77:99: + 71:1f:ac:f4:83:bb:06:5c:22:fa:ed:4c:c5:37:22:22:11:85: + 1a:ab:06:e3:14:c8:71:bf:79:1f:b4:22:64:ea:65:5e:99:c5: + ca:3a:1e:d2:22:96:24:df:65:b1:42:7e:72:21:6b:de:49:cd: + 09:45:4d:3e:ff:45:22:f4:01:d5:09:5b:dd:22:cc:25:8f:b2: + 5e:b3:35:f8:b3:6d:6e:2b:bd:98:f2:eb:5b:0e:58:31:f3:90: + ec:d9:41:9d:e4:4d:e7:6a:ff:32:63:d1:45:f0:fd:a6:7d:34: + 16:e4:c8:0c:79:39:5d:46:e1:28:f1:13:49:a6:02:07:a2:ca: + de:ae:4a:3a:f7:0b:e4:27:49:33:c8:29:2f:cd:6f:0e:5a:be: + e0:5d:c0:98:7e:15:97:ea:a5:ba:84:3b:ab:8d:aa:81:d7:7f: + df:7c:b9:e7:11:6b:7f:82:6e:62:57:d9:ba:25:60:1d:93:49: + eb:48:91:88:2c:22:74:dc:50:cf:6c:44:bd:04:84:fb:97:53: + 57:62:56:8b:b1:5c:a2:06:ef:c4:01:21:d6:73:4f:58:79:60: + 75:09:53:7d:cb:8c:a2:86 +-----BEGIN CERTIFICATE----- +MIIFkzCCA3ugAwIBAgIBBTANBgkqhkiG9w0BAQsFADBLMQswCQYDVQQGEwJVUzEL +MAkGA1UECAwCQ0ExDDAKBgNVBAcMA1NWTDEhMB8GA1UECgwYSW50ZXJuZXQgV2lk +Z2l0cyBQdHkgTHRkMB4XDTIwMTIxNTE4MDU1OVoXDTQ4MDUwMjE4MDU1OVowXTEL +MAkGA1UEBhMCVVMxETAPBgNVBAgMCElsbGlub2lzMRAwDgYDVQQHDAdDaGljYWdv +MRUwEwYDVQQKDAxFeGFtcGxlLCBDby4xEjAQBgNVBAMMCWxvY2FsaG9zdDCCAiIw +DQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBANNNAQidgN4V+gpS7Sz2S53FZy22 +QTPXW7T98Khc3YqNHy0SXUeICdSW7mMQ+Z2abDTE7KkNlWxIvBfQd1OT+ESKCwui +TR1T91Wn7Wo1rR2veb3VxluWJJ5N2Okhk+GTO13E45AaNtD3u4wi6dL4KRm/JMZS +IQ7WOnNIuueBNLojIVfDUQ5ZUZ+mB1YXxKobe2w2a6syXi73cNzr89o7OU6kjL9A +cu8AHkbJB2qTTDa3wy58xcGFn2tNLf08m5rPUrL6uvjOzIvcV3vtN2nJgNymLKfk +S4x3yy4oZWRhpMgz2fh9t3xPt/QHWomuKlmMrADHzrDQm81NgzlVu3IPYtRfi8PH +4nmVU+ugJg1Se01AVmYrVWf1GsnoqEm95+QxmuGNgPLMqz1w9v51y6obEtBv08jf +8b3OKyFC5Sy9xsjEu0o9kkgNSaiWwFHtMGSB3c4F1P8Hh1kLE0GOHVjkRwwAlxLk +Z5QkZ+/tHoWJ34V4EB9/suiv5w/H7KoBZ7YKmiODkB0KKjeAwybX8SQps9g3fFz4 +4wiWGjQt+v916HAl4qtRnfeKI1KJAshx6s2iibbrbsn9+9xj6Pnb11fSDPxWnBao +Z/4XiAfo+MV6M3KttFpDW0m+eYsAGIP3GQAHlCw4lmGnVagw234H9sWjZR6TLWte +2aYP+sIZm1lMQboHAgMBAAGjcDBuMB0GA1UdDgQWBBTz3Gpbt87p4U0+xK63jjnj +bcqvxzAfBgNVHSMEGDAWgBRapdqxmdTlDuYelOr//GLi7QnxBjAJBgNVHRMEAjAA +MAsGA1UdDwQEAwIFoDAUBgNVHREEDTALgglsb2NhbGhvc3QwDQYJKoZIhvcNAQEL +BQADggIBAFQTPVXTS9iF8FSoM1yhn4d5MTR+UrlG6pdD/QqbriyLdM1R1WOzsLQZ +odpztOJHDNkzSsaqJ0G8TRV0QlnrQY4oSfBViRP48DUf6bKuSHnnFaKqWeD9kch/ +uqqkLncq5GL+woNR3FiD+H+yR2iLH52bEiXyD30GpJq+o68tJzJKIP5fmNBdahDJ +BElUJmAgbjjykaQk5+3S5KqIfpxtCh4wl6mcRTUJ/EVsMtDbeQT6A3w25SdyHXeh +1BKGU7+TTvfaeqFLW0Lka5ulWPLvMC6mb/dj/RawNYGZPUHc2jRG6h8CzF7cZxxi +aHZFW+rzL6UPLrjQ362POwO0NiHUTZkJdiKkYbRZyotCXWQqmk9/Z2U4Ssbni/82 +DUJLYAN3mXEfrPSDuwZcIvrtTMU3IiIRhRqrBuMUyHG/eR+0ImTqZV6Zxco6HtIi +liTfZbFCfnIha95JzQlFTT7/RSL0AdUJW90izCWPsl6zNfizbW4rvZjy61sOWDHz +kOzZQZ3kTedq/zJj0UXw/aZ9NBbkyAx5OV1G4SjxE0mmAgeiyt6uSjr3C+QnSTPI +KS/Nbw5avuBdwJh+FZfqpbqEO6uNqoHXf998uecRa3+CbmJX2bolYB2TSetIkYgs +InTcUM9sRL0EhPuXU1diVouxXKIG78QBIdZzT1h5YHUJU33LjKKG +-----END CERTIFICATE----- diff --git a/security/advancedtls/testdata/server_key_localhost_1.pem b/security/advancedtls/testdata/server_key_localhost_1.pem new file mode 100644 index 000000000000..1185eff04ba4 --- /dev/null +++ b/security/advancedtls/testdata/server_key_localhost_1.pem @@ -0,0 +1,51 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIJKAIBAAKCAgEA000BCJ2A3hX6ClLtLPZLncVnLbZBM9dbtP3wqFzdio0fLRJd +R4gJ1JbuYxD5nZpsNMTsqQ2VbEi8F9B3U5P4RIoLC6JNHVP3VaftajWtHa95vdXG +W5Yknk3Y6SGT4ZM7XcTjkBo20Pe7jCLp0vgpGb8kxlIhDtY6c0i654E0uiMhV8NR +DllRn6YHVhfEqht7bDZrqzJeLvdw3Ovz2js5TqSMv0By7wAeRskHapNMNrfDLnzF +wYWfa00t/Tybms9Ssvq6+M7Mi9xXe+03acmA3KYsp+RLjHfLLihlZGGkyDPZ+H23 +fE+39Adaia4qWYysAMfOsNCbzU2DOVW7cg9i1F+Lw8fieZVT66AmDVJ7TUBWZitV +Z/UayeioSb3n5DGa4Y2A8syrPXD2/nXLqhsS0G/TyN/xvc4rIULlLL3GyMS7Sj2S +SA1JqJbAUe0wZIHdzgXU/weHWQsTQY4dWORHDACXEuRnlCRn7+0ehYnfhXgQH3+y +6K/nD8fsqgFntgqaI4OQHQoqN4DDJtfxJCmz2Dd8XPjjCJYaNC36/3XocCXiq1Gd +94ojUokCyHHqzaKJtutuyf373GPo+dvXV9IM/FacFqhn/heIB+j4xXozcq20WkNb +Sb55iwAYg/cZAAeULDiWYadVqDDbfgf2xaNlHpMta17Zpg/6whmbWUxBugcCAwEA +AQKCAgBswO5uQ7qnE8Kc+6+M+7tRmd+QHIUUrJxL3IO39AwmmpnYNeKCxZbhr0lE +/eCr6GYXBuAT5qTolcsRqr8v6jHW/QHQXBm6pZPgp0y/5J6Ub9OGDHhKfU2dmM2y +uBCIAqKEkajaa1OZXFhQOUwFxKpK0SGZXX4cR9DPszhXnR3JS/mGVUXrz7b+J5MR +EaysLPbqbFwgQg1NuReC7YKV6POG8ZRrfz1om7P5lNBXXzbT1uMDkz6pax/xN0kb +VM118Y1MB1aiZrXKqn7wjth9fzPu3SyQwSTNSH7v4+TDtKn+TQm8JuCAf/tbA0nr +IRQ1AP0qbayJPuVh1qpaoTCX9SlU29QwahuXmjqZzbtXlre64Psfmx6fGpTrFVY4 +Irhd4If/VwtbHCK6hU4c8GsIori+5rgquKBQgawQgDXCcF4671RJlwQnYm9YT5cb +q84wgBfGbXODHV1A+qJL3J+K/YzwxZKi7w9hXE+MsBoDiiUPekTBZA3GkFRCXwpN +nnb/BpnHnf3Fvy+BcmjzlzaPJZ1mwLJJEH0/R820U59S5m2fh7Gwy9+ZZsSNsJVW +fGaMZBGhOgAiBLd7zl7If4Eza14U499P8Rl8+4RiaMjnQC8xyI5eb49DtKcPdDp9 +d+ZQiXTQeFWriCeXTQlEV8uUCpRosfUtJC7xjljQWHeNeMpnAQKCAQEA7u13pfYd +XH4BpO+wlRwjb4ADc6wLWFOm0nySNB3fzDrOix0vAGvm7sjtIGVYjcHAbjG2oflH +siqgFaT5iQUdvo5AGZU9rp6f4qctJQuSuti4eXE1P+YSGyNALISE49lpi/z3lwJ7 +K3zxT0FywS2pvHEMc3+s+T0otik285/TJ7yg9d5UTS3TAgLzkan6wzGCAos935Ok +0/pJXWk3jjmc7q9cMGPDbX2kb67j4BuqbDbiCNTdfL2BOErwZw6+lissp8TPVNf2 +03Hj24GQGO/A9H8U47M2bcY63UgcflmBwKvXVSnCVpYl9UjY/A4j45w2zjAX/lm1 +3mjTBiwQGVR1hwKCAQEA4mYt2SuyS6AkdpANUrzISS0FvLTy/Qb3iU7iBkSBqShe +XV/a520uTJ+5v0m6ifGsafrjKJ0Rb3GYAug/mzWItTuRArIH5hTNd+FaAeC9XBEv +dvUnAnffvocvAnmpQdO/kJLY54rA1KnuIJ30Xy5653LgQLur/b2EG7Fljc9vHqI8 +Emp7sLwPPFzelR+ryTak8GimUHm7psPJryvAeB8ak68TAmlBkLKrdO6oE3O1fJoO +cFbZG3r2KdRPZgBpegskOOgk0GQIG7z3G0auZpiwViRJLTBda1lLLYnVBcT2Qq10 +AQebj1pDC7Op33kZMNtQtrb6bvEdPhEsaWqcfZm3gQKCAQAvgcojlq8539gl2n7q +9yBYoESPcGsFEgT+n0RW1oXUTvEYmiHpXIsbeZokseIMtbS0dHAS/sTxuSYBh78S +LpE+fXxjWdhc6y9xWrpQPl/bhRIRG6Bx5yY8fSLadzMRNv6UliUIwraI7BvzHVla +7eBtFrFaGc3j9PQuXD2P7XyHzyrWGHH8sprdMIcLtJemziZCqTsRRIMmnwKNb0lb +nzsD/pw/Bucp0yyqBEVNH1Mglz0Ucnbjwa566fOpGjZtF4KWjTyIazSp0GB1Gerz ++mAMfWRC7jRpWVwE+byoptV04PY8+cOpgctkXSq/23PpYvtGvitXKLFP2tnyxToi +PzfrAoIBAGk9+HgosOQo2GppAliAu1YQ4MbdEst+bplcmwMw21lIE72yLm9AOLKT +2WPLoTQ4rN5DK0+Y3B8DHhfT4KWE2DzvKLSpD7Tr3KuqjQ2sbDodHwRcZ7rlAJRw +APFUntKj3TwWl0/jF0qEh9aPtqZ8U9O9efN9ijEU5RF+gGfQkqYZ4nTpHQCGG0sD +HNETfOa3SSscapukSw/1mY6ddwYf51nZm6uWRE1AUSW1P1pzgl0evDGKnbgBi+bb +8+DFtkJuZXMyrtJUfdRvHiuGytGUjvwsN/wSrIqXYrQTi3v4GEXcnb1QzQZxfhM1 +fHUOtSAaA0Y8fuQNn3tXvl5umbplN4ECggEBAIPDR2/Igmze9sCqZF06fexBZUcB +j6gsjY7zJYppAO6tniyLkQaOHoCZGvMA2xxcR1+4F+QQvLOkRTPxXHqZgNwze9oK +Y9QLkhsu0Q3A0+C/YC3HGL90HKndRx6pLHVhrBg3vxoTjQpPkpVT6UYnMGnRr5D5 +qluURPPYR56WX76rMud0yDTB/MeNsNjdBn8ukr1LwR40ircFgmy/O+FdR9mpPOVY +3DIDysyzug3IeIoa0QSy7aKHHGd9pTHJ3EA/tfoTCmr9iywKxsgPp1fcSjt8Nfg7 +mVo2qbVoqchrp0tFG2/MoxqiOlRMtWX6N37btlMxifj28BDE63hG53W4bGA= +-----END RSA PRIVATE KEY----- diff --git a/security/authorization/engine/engine.go b/security/authorization/engine/engine.go index d598ff409fa6..3a95e5ec8800 100644 --- a/security/authorization/engine/engine.go +++ b/security/authorization/engine/engine.go @@ -22,10 +22,10 @@ import ( "strconv" pb "github.com/envoyproxy/go-control-plane/envoy/config/rbac/v2" - cel "github.com/google/cel-go/cel" + "github.com/google/cel-go/cel" "github.com/google/cel-go/checker/decls" - types "github.com/google/cel-go/common/types" - interpreter "github.com/google/cel-go/interpreter" + "github.com/google/cel-go/common/types" + "github.com/google/cel-go/interpreter" expr "google.golang.org/genproto/googleapis/api/expr/v1alpha1" "google.golang.org/grpc/grpclog" "google.golang.org/grpc/metadata" @@ -53,12 +53,12 @@ var intAttributeMap = map[string]func(*AuthorizationArgs) (int, error){ // activationImpl is an implementation of interpreter.Activation. // An Activation is the primary mechanism by which a caller supplies input into a CEL program. type activationImpl struct { - dict map[string]interface{} + dict map[string]any } // ResolveName returns a value from the activation by qualified name, or false if the name // could not be found. -func (activation activationImpl) ResolveName(name string) (interface{}, bool) { +func (activation activationImpl) ResolveName(name string) (any, bool) { result, ok := activation.dict[name] return result, ok } @@ -79,7 +79,7 @@ type AuthorizationArgs struct { // newActivation converts AuthorizationArgs into the activation for CEL. func newActivation(args *AuthorizationArgs) interpreter.Activation { // Fill out evaluation map, only adding the attributes that can be extracted. - evalMap := make(map[string]interface{}) + evalMap := make(map[string]any) for key, function := range stringAttributeMap { val, err := function(args) if err == nil { @@ -253,13 +253,16 @@ func getDecision(engine *policyEngine, match bool) Decision { return DecisionDeny } -// Returns the authorization decision of a single policy engine based on activation. -// If any policy matches, the decision matches the engine's action, and the first -// matching policy name will be returned. -// Else if any policy is missing attributes, the decision is unknown, and the list of -// policy names that can't be evaluated due to missing attributes will be returned. -// Else, the decision is the opposite of the engine's action, i.e. an ALLOW engine -// will return DecisionDeny, and vice versa. +// Returns the authorization decision of a single policy engine based on +// activation. If any policy matches, the decision matches the engine's +// action, and the first matching policy name will be returned. +// +// Else if any policy is missing attributes, the decision is unknown, and the +// list of policy names that can't be evaluated due to missing attributes will +// be returned. +// +// Else, the decision is the opposite of the engine's action, i.e. an ALLOW +// engine will return DecisionDeny, and vice versa. func (engine *policyEngine) evaluate(activation interpreter.Activation) (Decision, []string) { unknownPolicyNames := []string{} for policyName, program := range engine.programs { @@ -324,6 +327,7 @@ func NewAuthorizationEngine(allow, deny *pb.RBAC) (*AuthorizationEngine, error) decls.NewVar("destination.address", decls.String), decls.NewVar("destination.port", decls.Int), decls.NewVar("connection.uri_san_peer_certificate", decls.String), + decls.NewVar("source.principal", decls.String), ), ) if err != nil { diff --git a/security/authorization/engine/engine_test.go b/security/authorization/engine/engine_test.go index a87a7f53d858..181c2564ba15 100644 --- a/security/authorization/engine/engine_test.go +++ b/security/authorization/engine/engine_test.go @@ -17,33 +17,47 @@ package engine import ( + "context" "reflect" "sort" "testing" pb "github.com/envoyproxy/go-control-plane/envoy/config/rbac/v2" - cel "github.com/google/cel-go/cel" + "github.com/google/cel-go/cel" + "github.com/google/cel-go/checker/decls" "github.com/google/cel-go/common/types" "github.com/google/cel-go/common/types/ref" - interpreter "github.com/google/cel-go/interpreter" + "github.com/google/cel-go/interpreter" + "github.com/google/go-cmp/cmp" + expr "google.golang.org/genproto/googleapis/api/expr/v1alpha1" "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/peer" "google.golang.org/grpc/status" ) -type programMock struct { +type s struct { + grpctest.Tester +} + +type fakeProgram struct { out ref.Val err error } -func (mock programMock) Eval(vars interface{}) (ref.Val, *cel.EvalDetails, error) { - return mock.out, nil, mock.err +func (fake fakeProgram) Eval(vars any) (ref.Val, *cel.EvalDetails, error) { + return fake.out, nil, fake.err +} + +func (fake fakeProgram) ContextEval(ctx context.Context, vars any) (ref.Val, *cel.EvalDetails, error) { + return fake.Eval(vars) } type valMock struct { - val interface{} + val any } -func (mock valMock) ConvertToNative(typeDesc reflect.Type) (interface{}, error) { +func (mock valMock) ConvertToNative(typeDesc reflect.Type) (any, error) { return nil, nil } @@ -62,16 +76,28 @@ func (mock valMock) Type() ref.Type { return nil } -func (mock valMock) Value() interface{} { +func (mock valMock) Value() any { return mock.val } +type addrMock struct { + addr string +} + +func (mock addrMock) Network() string { + return "tcp" +} + +func (mock addrMock) String() string { + return mock.addr +} + var ( emptyActivation = interpreter.EmptyActivation() - unsuccessfulProgram = programMock{out: nil, err: status.Errorf(codes.InvalidArgument, "Unsuccessful program evaluation")} - errProgram = programMock{out: valMock{"missing attributes"}, err: status.Errorf(codes.InvalidArgument, "Successful program evaluation to an error result -- missing attributes")} - trueProgram = programMock{out: valMock{true}, err: nil} - falseProgram = programMock{out: valMock{false}, err: nil} + unsuccessfulProgram = fakeProgram{out: nil, err: status.Errorf(codes.InvalidArgument, "Unsuccessful program evaluation")} + errProgram = fakeProgram{out: valMock{"missing attributes"}, err: status.Errorf(codes.InvalidArgument, "Successful program evaluation to an error result -- missing attributes")} + trueProgram = fakeProgram{out: valMock{true}, err: nil} + falseProgram = fakeProgram{out: valMock{false}, err: nil} allowMatchEngine = &policyEngine{action: pb.RBAC_ALLOW, programs: map[string]cel.Program{ "allow match policy1": unsuccessfulProgram, @@ -92,41 +118,37 @@ var ( }} ) -func TestNewAuthorizationEngine(t *testing.T) { +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func (s) TestNewAuthorizationEngine(t *testing.T) { tests := map[string]struct { allow *pb.RBAC deny *pb.RBAC - wantErr string + wantErr bool errStr string }{ "too few rbacs": { - allow: nil, - deny: nil, - wantErr: "at least one of allow, deny must be non-nil", + wantErr: true, errStr: "Expected error: at least one of allow, deny must be non-nil", }, "one rbac allow": { - allow: &pb.RBAC{Action: pb.RBAC_ALLOW}, - deny: nil, - wantErr: "", - errStr: "Expected 1 ALLOW RBAC to be successful", + allow: &pb.RBAC{Action: pb.RBAC_ALLOW}, + errStr: "Expected 1 ALLOW RBAC to be successful", }, "one rbac deny": { - allow: nil, - deny: &pb.RBAC{Action: pb.RBAC_DENY}, - wantErr: "", - errStr: "Expected 1 DENY RBAC to be successful", + deny: &pb.RBAC{Action: pb.RBAC_DENY}, + errStr: "Expected 1 DENY RBAC to be successful", }, "two rbacs": { - allow: &pb.RBAC{Action: pb.RBAC_ALLOW}, - deny: &pb.RBAC{Action: pb.RBAC_DENY}, - wantErr: "", - errStr: "Expected 2 RBACs (DENY + ALLOW) to be successful", + allow: &pb.RBAC{Action: pb.RBAC_ALLOW}, + deny: &pb.RBAC{Action: pb.RBAC_DENY}, + errStr: "Expected 2 RBACs (DENY + ALLOW) to be successful", }, "wrong rbac actions": { allow: &pb.RBAC{Action: pb.RBAC_DENY}, - deny: nil, - wantErr: "allow must have action ALLOW, deny must have action DENY", + wantErr: true, errStr: "Expected error: allow must have action ALLOW, deny must have action DENY", }, } @@ -134,17 +156,14 @@ func TestNewAuthorizationEngine(t *testing.T) { for name, tc := range tests { t.Run(name, func(t *testing.T) { _, gotErr := NewAuthorizationEngine(tc.allow, tc.deny) - if tc.wantErr == "" && gotErr == nil { - return - } - if gotErr == nil || gotErr.Error() != tc.wantErr { - t.Errorf(tc.errStr) + if (gotErr != nil) != tc.wantErr { + t.Fatal(tc.errStr) } }) } } -func TestGetDecision(t *testing.T) { +func (s) TestGetDecision(t *testing.T) { tests := map[string]struct { engine *policyEngine match bool @@ -157,7 +176,6 @@ func TestGetDecision(t *testing.T) { }, "ALLOW engine fail": { engine: &policyEngine{action: pb.RBAC_ALLOW, programs: map[string]cel.Program{}}, - match: false, want: DecisionDeny, }, "DENY engine match": { @@ -167,7 +185,6 @@ func TestGetDecision(t *testing.T) { }, "DENY engine fail": { engine: &policyEngine{action: pb.RBAC_DENY, programs: map[string]cel.Program{}}, - match: false, want: DecisionAllow, }, } @@ -175,13 +192,13 @@ func TestGetDecision(t *testing.T) { for name, tc := range tests { t.Run(name, func(t *testing.T) { if got := getDecision(tc.engine, tc.match); got != tc.want { - t.Errorf("Expected %v, instead got %v", tc.want, got) + t.Fatalf("getDecision(%v, %v) = (%v), want (%v)", tc.engine, tc.match, got, tc.want) } }) } } -func TestPolicyEngineEvaluate(t *testing.T) { +func (s) TestPolicyEngineEvaluate(t *testing.T) { tests := map[string]struct { engine *policyEngine activation interpreter.Activation @@ -218,43 +235,38 @@ func TestPolicyEngineEvaluate(t *testing.T) { t.Run(name, func(t *testing.T) { gotDecision, gotPolicyNames := tc.engine.evaluate(tc.activation) sort.Strings(gotPolicyNames) - if gotDecision != tc.wantDecision || !reflect.DeepEqual(gotPolicyNames, tc.wantPolicyNames) { - t.Errorf("Expected (%v, %v), instead got (%v, %v)", tc.wantDecision, tc.wantPolicyNames, gotDecision, gotPolicyNames) + if gotDecision != tc.wantDecision || !cmp.Equal(gotPolicyNames, tc.wantPolicyNames) { + t.Fatalf("policyEngine.evaluate(%v, %v) = (%v, %v), want (%v, %v)", tc.engine, tc.activation, gotDecision, gotPolicyNames, tc.wantDecision, tc.wantPolicyNames) } }) } } -func TestAuthorizationEngineEvaluate(t *testing.T) { +func (s) TestAuthorizationEngineEvaluate(t *testing.T) { tests := map[string]struct { engine *AuthorizationEngine authArgs *AuthorizationArgs wantAuthDecision *AuthorizationDecision - wantErr error }{ "allow match": { engine: &AuthorizationEngine{allow: allowMatchEngine}, authArgs: &AuthorizationArgs{}, wantAuthDecision: &AuthorizationDecision{decision: DecisionAllow, policyNames: []string{"allow match policy2"}}, - wantErr: nil, }, "deny fail": { engine: &AuthorizationEngine{deny: denyFailEngine}, authArgs: &AuthorizationArgs{}, wantAuthDecision: &AuthorizationDecision{decision: DecisionAllow, policyNames: []string{}}, - wantErr: nil, }, "first engine unknown": { engine: &AuthorizationEngine{allow: allowMatchEngine, deny: denyUnknownEngine}, authArgs: &AuthorizationArgs{}, wantAuthDecision: &AuthorizationDecision{decision: DecisionUnknown, policyNames: []string{"deny unknown policy2", "deny unknown policy3"}}, - wantErr: nil, }, "second engine match": { engine: &AuthorizationEngine{allow: allowMatchEngine, deny: denyFailEngine}, authArgs: &AuthorizationArgs{}, wantAuthDecision: &AuthorizationDecision{decision: DecisionAllow, policyNames: []string{"allow match policy2"}}, - wantErr: nil, }, } @@ -262,10 +274,145 @@ func TestAuthorizationEngineEvaluate(t *testing.T) { t.Run(name, func(t *testing.T) { gotAuthDecision, gotErr := tc.engine.Evaluate(tc.authArgs) sort.Strings(gotAuthDecision.policyNames) - if tc.wantErr != nil && (gotErr == nil || gotErr.Error() != tc.wantErr.Error()) { - t.Errorf("Expected error to be %v, instead got %v", tc.wantErr, gotErr) - } else if tc.wantErr == nil && (gotErr != nil || gotAuthDecision.decision != tc.wantAuthDecision.decision || !reflect.DeepEqual(gotAuthDecision.policyNames, tc.wantAuthDecision.policyNames)) { - t.Errorf("Expected authorization decision to be (%v, %v), instead got (%v, %v)", tc.wantAuthDecision.decision, tc.wantAuthDecision.policyNames, gotAuthDecision.decision, gotAuthDecision.policyNames) + if gotErr != nil || gotAuthDecision.decision != tc.wantAuthDecision.decision || !cmp.Equal(gotAuthDecision.policyNames, tc.wantAuthDecision.policyNames) { + t.Fatalf("AuthorizationEngine.Evaluate(%v, %v) = (%v, %v), want (%v, %v)", tc.engine, tc.authArgs, gotAuthDecision, gotErr, tc.wantAuthDecision, nil) + } + }) + } +} + +func (s) TestIntegration(t *testing.T) { + declarations := []*expr.Decl{ + decls.NewVar("request.url_path", decls.String), + decls.NewVar("request.host", decls.String), + decls.NewVar("request.method", decls.String), + decls.NewVar("request.headers", decls.NewMapType(decls.String, decls.String)), + decls.NewVar("source.address", decls.String), + decls.NewVar("source.port", decls.Int), + decls.NewVar("destination.address", decls.String), + decls.NewVar("destination.port", decls.Int), + decls.NewVar("connection.uri_san_peer_certificate", decls.String), + decls.NewVar("source.principal", decls.String), + } + + tests := map[string]struct { + allow *pb.RBAC + deny *pb.RBAC + authArgs *AuthorizationArgs + wantAuthDecision *AuthorizationDecision + }{ + "ALLOW engine: DecisionAllow": { + allow: &pb.RBAC{Action: pb.RBAC_ALLOW, Policies: map[string]*pb.Policy{ + "url_path starts with": {Condition: compileStringToExpr("request.url_path.startsWith('/pkg.service/test')", declarations)}, + }}, + authArgs: &AuthorizationArgs{fullMethod: "/pkg.service/test/method"}, + wantAuthDecision: &AuthorizationDecision{decision: DecisionAllow, policyNames: []string{"url_path starts with"}}, + }, + "ALLOW engine: DecisionUnknown": { + allow: &pb.RBAC{Action: pb.RBAC_ALLOW, Policies: map[string]*pb.Policy{ + "url_path and uri_san_peer_certificate": {Condition: compileStringToExpr("request.url_path == '/pkg.service/test' && connection.uri_san_peer_certificate == 'cluster/ns/default/sa/admin'", declarations)}, + "source port": {Condition: compileStringToExpr("source.port == 8080", declarations)}, + }}, + authArgs: &AuthorizationArgs{peerInfo: &peer.Peer{Addr: addrMock{addr: "192.0.2.1:25"}}, fullMethod: "/pkg.service/test"}, + wantAuthDecision: &AuthorizationDecision{decision: DecisionUnknown, policyNames: []string{"url_path and uri_san_peer_certificate"}}, + }, + "ALLOW engine: DecisionDeny": { + allow: &pb.RBAC{Action: pb.RBAC_ALLOW, Policies: map[string]*pb.Policy{ + "url_path": {Condition: compileStringToExpr("request.url_path == '/pkg.service/test'", declarations)}, + }}, + authArgs: &AuthorizationArgs{fullMethod: "/pkg.service/test/method"}, + wantAuthDecision: &AuthorizationDecision{decision: DecisionDeny, policyNames: []string{}}, + }, + "DENY engine: DecisionAllow": { + deny: &pb.RBAC{Action: pb.RBAC_DENY, Policies: map[string]*pb.Policy{ + "url_path and uri_san_peer_certificate": {Condition: compileStringToExpr("request.url_path == '/pkg.service/test' && connection.uri_san_peer_certificate == 'cluster/ns/default/sa/admin'", declarations)}, + }}, + authArgs: &AuthorizationArgs{fullMethod: "/pkg.service/test/method"}, + wantAuthDecision: &AuthorizationDecision{decision: DecisionAllow, policyNames: []string{}}, + }, + "DENY engine: DecisionUnknown": { + deny: &pb.RBAC{Action: pb.RBAC_DENY, Policies: map[string]*pb.Policy{ + "destination address": {Condition: compileStringToExpr("destination.address == '192.0.3.1'", declarations)}, + "source port": {Condition: compileStringToExpr("source.port == 8080", declarations)}, + }}, + authArgs: &AuthorizationArgs{peerInfo: &peer.Peer{Addr: addrMock{addr: "192.0.2.1:25"}}, fullMethod: "/pkg.service/test"}, + wantAuthDecision: &AuthorizationDecision{decision: DecisionUnknown, policyNames: []string{"destination address"}}, + }, + "DENY engine: DecisionDeny": { + deny: &pb.RBAC{Action: pb.RBAC_DENY, Policies: map[string]*pb.Policy{ + "destination address": {Condition: compileStringToExpr("destination.address == '192.0.3.1'", declarations)}, + "source address or source port": {Condition: compileStringToExpr("source.address == '192.0.4.1' || source.port == 8080", declarations)}, + }}, + authArgs: &AuthorizationArgs{peerInfo: &peer.Peer{Addr: addrMock{addr: "192.0.2.1:8080"}}, fullMethod: "/pkg.service/test"}, + wantAuthDecision: &AuthorizationDecision{decision: DecisionDeny, policyNames: []string{"source address or source port"}}, + }, + "DENY ALLOW engine: DecisionDeny from DENY policy": { + allow: &pb.RBAC{Action: pb.RBAC_ALLOW, Policies: map[string]*pb.Policy{ + "url_path starts with": {Condition: compileStringToExpr("request.url_path.startsWith('/pkg.service/test')", declarations)}, + }}, + deny: &pb.RBAC{Action: pb.RBAC_DENY, Policies: map[string]*pb.Policy{ + "destination address": {Condition: compileStringToExpr("destination.address == '192.0.3.1'", declarations)}, + "source address or source port": {Condition: compileStringToExpr("source.address == '192.0.4.1' || source.port == 8080", declarations)}, + }}, + authArgs: &AuthorizationArgs{peerInfo: &peer.Peer{Addr: addrMock{addr: "192.0.2.1:8080"}}, fullMethod: "/pkg.service/test"}, + wantAuthDecision: &AuthorizationDecision{decision: DecisionDeny, policyNames: []string{"source address or source port"}}, + }, + "DENY ALLOW engine: DecisionUnknown from DENY policy": { + allow: &pb.RBAC{Action: pb.RBAC_ALLOW, Policies: map[string]*pb.Policy{ + "url_path starts with": {Condition: compileStringToExpr("request.url_path.startsWith('/pkg.service/test')", declarations)}, + }}, + deny: &pb.RBAC{Action: pb.RBAC_DENY, Policies: map[string]*pb.Policy{ + "destination address": {Condition: compileStringToExpr("destination.address == '192.0.3.1'", declarations)}, + "source port and destination port": {Condition: compileStringToExpr("source.port == 8080 && destination.port == 1234", declarations)}, + }}, + authArgs: &AuthorizationArgs{peerInfo: &peer.Peer{Addr: addrMock{addr: "192.0.2.1:8080"}}, fullMethod: "/pkg.service/test/method"}, + wantAuthDecision: &AuthorizationDecision{decision: DecisionUnknown, policyNames: []string{"destination address", "source port and destination port"}}, + }, + "DENY ALLOW engine: DecisionAllow from ALLOW policy": { + allow: &pb.RBAC{Action: pb.RBAC_ALLOW, Policies: map[string]*pb.Policy{ + "method or url_path starts with": {Condition: compileStringToExpr("request.method == 'POST' || request.url_path.startsWith('/pkg.service/test')", declarations)}, + }}, + deny: &pb.RBAC{Action: pb.RBAC_DENY, Policies: map[string]*pb.Policy{ + "source address": {Condition: compileStringToExpr("source.address == '192.0.3.1'", declarations)}, + "source port and url_path": {Condition: compileStringToExpr("source.port == 8080 && request.url_path == 'pkg.service/test'", declarations)}, + }}, + authArgs: &AuthorizationArgs{peerInfo: &peer.Peer{Addr: addrMock{addr: "192.0.2.1:8080"}}, fullMethod: "/pkg.service/test/method"}, + wantAuthDecision: &AuthorizationDecision{decision: DecisionAllow, policyNames: []string{"method or url_path starts with"}}, + }, + "DENY ALLOW engine: DecisionUnknown from ALLOW policy": { + allow: &pb.RBAC{Action: pb.RBAC_ALLOW, Policies: map[string]*pb.Policy{ + "url_path starts with and method": {Condition: compileStringToExpr("request.url_path.startsWith('/pkg.service/test') && request.method == 'POST'", declarations)}, + }}, + deny: &pb.RBAC{Action: pb.RBAC_DENY, Policies: map[string]*pb.Policy{ + "source address": {Condition: compileStringToExpr("source.address == '192.0.3.1'", declarations)}, + "source port and url_path": {Condition: compileStringToExpr("source.port == 8080 && request.url_path == 'pkg.service/test'", declarations)}, + }}, + authArgs: &AuthorizationArgs{peerInfo: &peer.Peer{Addr: addrMock{addr: "192.0.2.1:8080"}}, fullMethod: "/pkg.service/test/method"}, + wantAuthDecision: &AuthorizationDecision{decision: DecisionUnknown, policyNames: []string{"url_path starts with and method"}}, + }, + "DENY ALLOW engine: DecisionDeny from ALLOW policy": { + allow: &pb.RBAC{Action: pb.RBAC_ALLOW, Policies: map[string]*pb.Policy{ + "url_path starts with and source port": {Condition: compileStringToExpr("request.url_path.startsWith('/pkg.service/test') && source.port == 1234", declarations)}, + }}, + deny: &pb.RBAC{Action: pb.RBAC_DENY, Policies: map[string]*pb.Policy{ + "source address": {Condition: compileStringToExpr("source.address == '192.0.3.1'", declarations)}, + "source port and url_path": {Condition: compileStringToExpr("source.port == 8080 && request.url_path == 'pkg.service/test'", declarations)}, + }}, + authArgs: &AuthorizationArgs{peerInfo: &peer.Peer{Addr: addrMock{addr: "192.0.2.1:8080"}}, fullMethod: "/pkg.service/test/method"}, + wantAuthDecision: &AuthorizationDecision{decision: DecisionDeny, policyNames: []string{}}, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + engine, err := NewAuthorizationEngine(tc.allow, tc.deny) + if err != nil { + t.Fatalf("Error constructing authorization engine: %v", err) + } + gotAuthDecision, gotErr := engine.Evaluate(tc.authArgs) + sort.Strings(gotAuthDecision.policyNames) + if gotErr != nil || gotAuthDecision.decision != tc.wantAuthDecision.decision || !cmp.Equal(gotAuthDecision.policyNames, tc.wantAuthDecision.policyNames) { + t.Fatalf("NewAuthorizationEngine(%v, %v).Evaluate(%v) = (%v, %v), want (%v, %v)", tc.allow, tc.deny, tc.authArgs, gotAuthDecision, gotErr, tc.wantAuthDecision, nil) } }) } diff --git a/security/authorization/util/util.go b/security/authorization/engine/util.go similarity index 86% rename from security/authorization/util/util.go rename to security/authorization/engine/util.go index 8aadad3862ba..3d1690ac2bff 100644 --- a/security/authorization/util/util.go +++ b/security/authorization/engine/util.go @@ -61,3 +61,11 @@ func compileStringToCheckedExpr(expr string, declarations []*expr.Decl) (*expr.C } return checkedExpr, nil } + +func compileStringToExpr(expr string, declarations []*expr.Decl) *expr.Expr { + checkedExpr, err := compileStringToCheckedExpr(expr, declarations) + if err != nil { + logger.Fatalf("error encountered when compiling string to expression: %v", err) + } + return checkedExpr.Expr +} diff --git a/security/authorization/util/util_test.go b/security/authorization/engine/util_test.go similarity index 82% rename from security/authorization/util/util_test.go rename to security/authorization/engine/util_test.go index 981a00864d58..d991b7a3833d 100644 --- a/security/authorization/util/util_test.go +++ b/security/authorization/engine/util_test.go @@ -22,20 +22,11 @@ import ( "testing" expr "google.golang.org/genproto/googleapis/api/expr/v1alpha1" - "google.golang.org/grpc/internal/grpctest" "github.com/google/cel-go/cel" "github.com/google/cel-go/checker/decls" ) -type s struct { - grpctest.Tester -} - -func Test(t *testing.T) { - grpctest.RunSubTests(t, s{}) -} - func (s) TestStringConvert(t *testing.T) { declarations := []*expr.Decl{ decls.NewIdent("request.url_path", decls.String, nil), @@ -52,44 +43,44 @@ func (s) TestStringConvert(t *testing.T) { wantParsingError bool wantEvalError bool expr string - authzArgs map[string]interface{} + authzArgs map[string]any }{ { desc: "single primitive match", wantEvalOutcome: true, expr: "request.url_path.startsWith('/pkg.service/test')", - authzArgs: map[string]interface{}{"request.url_path": "/pkg.service/test"}, + authzArgs: map[string]any{"request.url_path": "/pkg.service/test"}, }, { desc: "single compare match", wantEvalOutcome: true, expr: "connection.uri_san_peer_certificate == 'cluster/ns/default/sa/admin'", - authzArgs: map[string]interface{}{"connection.uri_san_peer_certificate": "cluster/ns/default/sa/admin"}, + authzArgs: map[string]any{"connection.uri_san_peer_certificate": "cluster/ns/default/sa/admin"}, }, { desc: "single primitive no match", wantEvalOutcome: false, expr: "request.url_path.startsWith('/pkg.service/test')", - authzArgs: map[string]interface{}{"request.url_path": "/source/pkg.service/test"}, + authzArgs: map[string]any{"request.url_path": "/source/pkg.service/test"}, }, { desc: "primitive and compare match", wantEvalOutcome: true, expr: "request.url_path == '/pkg.service/test' && connection.uri_san_peer_certificate == 'cluster/ns/default/sa/admin'", - authzArgs: map[string]interface{}{"request.url_path": "/pkg.service/test", + authzArgs: map[string]any{"request.url_path": "/pkg.service/test", "connection.uri_san_peer_certificate": "cluster/ns/default/sa/admin"}, }, { desc: "parse error field not present in environment", wantParsingError: true, expr: "request.source_path.startsWith('/pkg.service/test')", - authzArgs: map[string]interface{}{"request.url_path": "/pkg.service/test"}, + authzArgs: map[string]any{"request.url_path": "/pkg.service/test"}, }, { desc: "eval error argument not included in environment", wantEvalError: true, expr: "request.url_path.startsWith('/pkg.service/test')", - authzArgs: map[string]interface{}{"request.source_path": "/pkg.service/test"}, + authzArgs: map[string]any{"request.source_path": "/pkg.service/test"}, }, } { test := test diff --git a/security/authorization/go.mod b/security/authorization/go.mod index 075cbe9158c0..ec4ab2d11d87 100644 --- a/security/authorization/go.mod +++ b/security/authorization/go.mod @@ -1,11 +1,23 @@ module google.golang.org/grpc/security/authorization -go 1.12 +go 1.19 require ( - github.com/envoyproxy/go-control-plane v0.9.5 - github.com/google/cel-go v0.5.1 - google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 - google.golang.org/grpc v1.31.0 - google.golang.org/protobuf v1.25.0 + github.com/envoyproxy/go-control-plane v0.11.1 + github.com/google/cel-go v0.16.0 + github.com/google/go-cmp v0.5.9 + google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 + google.golang.org/grpc v1.56.2 + google.golang.org/protobuf v1.31.0 +) + +require ( + github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230321174746-8dcc6526cfb1 // indirect + github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 // indirect + github.com/envoyproxy/protoc-gen-validate v1.0.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/stoewer/go-strcase v1.3.0 // indirect + golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 // indirect + golang.org/x/text v0.11.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect ) diff --git a/security/authorization/go.sum b/security/authorization/go.sum index a953711e01e6..a4e37a004eb9 100644 --- a/security/authorization/go.sum +++ b/security/authorization/go.sum @@ -1,104 +1,92 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/antlr/antlr4 v0.0.0-20200503195918-621b933c7a7f h1:0cEys61Sr2hUBEXfNV8eyQP01oZuBgoMeHunebPirK8= -github.com/antlr/antlr4 v0.0.0-20200503195918-621b933c7a7f/go.mod h1:T7PbCXFs94rrTttyxjbyT5+/1V8T2TYDejxUfHJjw1Y= +github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230321174746-8dcc6526cfb1 h1:X8MJ0fnN5FPdcGF5Ij2/OW+HgiJrRg3AfHAx1PJtIzM= +github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230321174746-8dcc6526cfb1/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20200313221541-5f7e5dd04533 h1:8wZizuKuZVu5COB7EsBYxBQz8nRcXXn5d4Gt91eJLvU= -github.com/cncf/udpa/go v0.0.0-20200313221541-5f7e5dd04533/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= +github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.5 h1:lRJIqDD8yjV1YyPRqecMdytjDLs2fTXq363aCib5xPU= -github.com/envoyproxy/go-control-plane v0.9.5/go.mod h1:OXl5to++W0ctG+EHWTFUjiypVxC/Y4VLc/KFU+al13s= -github.com/envoyproxy/protoc-gen-validate v0.1.0 h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A= +github.com/envoyproxy/go-control-plane v0.11.1 h1:wSUXTlLfiAQRWs2F+p+EKOY9rUyis1MyGqJ2DIk5HpM= +github.com/envoyproxy/go-control-plane v0.11.1/go.mod h1:uhMcXKCQMEJHiAb0w+YGefQLaTEw+YhGluxZkrTmD0g= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= +github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA= +github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4 h1:87PNWwrRvUSnqS4dlcBU/ftvOIBep4sYuBLlh6rX2wk= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1 h1:ZFgWrT+bLgsYPirOnRfKLYJLvssAegOj/hgyMFdJZe0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/google/cel-go v0.5.1 h1:oDsbtAwlwFPEcC8dMoRWNuVzWJUDeDZeHjoet9rXjTs= -github.com/google/cel-go v0.5.1/go.mod h1:9SvtVVTtZV4DTB1/RuAD1D2HhuqEIdmZEE/r/lrFyKE= -github.com/google/cel-spec v0.4.0/go.mod h1:2pBM5cU4UKjbPDXBgwWkiwBsVgnxknuEJ7C5TDWwORQ= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/cel-go v0.16.0 h1:DG9YQ8nFCFXAs/FDDwBxmL1tpKNrdlGUM9U3537bX/Y= +github.com/google/cel-go v0.16.0/go.mod h1:HXZKzB0LXqer5lHHgfWAnlYwJaQBDKMjxjulNQzhwhY= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= +github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 h1:MGwJjxBy0HJshjDNfLsYO8xppfqWlA5ZT9OhtUUhTNw= +golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 h1:uYVVQ9WP/Ds2ROhcaGPeIdVq0RIXVLwsHlnvJ+cT1So= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20200305110556-506484158171 h1:xes2Q2k+d/+YNXVw0FpZkIDJiaux4OVrRKXRAzH6A0U= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 h1:FmF5cCW94Ij59cfpoLiwTgodWmm60eEV0CjlsVg2fuw= +google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 h1:bVf09lpb+OJbByTj913DRJioFFAjf/ZGxEz7MajTp2U= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1 h1:wdKvqQk7IttEw92GoRyKG2IDrUIpgpj6H6m81yfeMW0= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.31.0 h1:T7P4R73V3SSDPhH7WW7ATbfViLtmamH0DKrP3f9AuDI= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/grpc v1.56.2 h1:fVRFRnXvU+x6C4IlHZewvJOVHoOv1TUuQyoRsYnB4bI= +google.golang.org/grpc v1.56.2/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/server.go b/server.go index 33fc3240bcc6..244123c6c5a8 100644 --- a/server.go +++ b/server.go @@ -40,10 +40,11 @@ import ( "google.golang.org/grpc/encoding" "google.golang.org/grpc/encoding/proto" "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal" "google.golang.org/grpc/internal/binarylog" "google.golang.org/grpc/internal/channelz" - "google.golang.org/grpc/internal/grpcrand" "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/grpcutil" "google.golang.org/grpc/internal/transport" "google.golang.org/grpc/keepalive" "google.golang.org/grpc/metadata" @@ -56,12 +57,36 @@ import ( const ( defaultServerMaxReceiveMessageSize = 1024 * 1024 * 4 defaultServerMaxSendMessageSize = math.MaxInt32 + + // Server transports are tracked in a map which is keyed on listener + // address. For regular gRPC traffic, connections are accepted in Serve() + // through a call to Accept(), and we use the actual listener address as key + // when we add it to the map. But for connections received through + // ServeHTTP(), we do not have a listener and hence use this dummy value. + listenerAddressForServeHTTP = "listenerAddressForServeHTTP" ) +func init() { + internal.GetServerCredentials = func(srv *Server) credentials.TransportCredentials { + return srv.opts.creds + } + internal.DrainServerTransports = func(srv *Server, addr string) { + srv.drainServerTransports(addr) + } + internal.AddGlobalServerOptions = func(opt ...ServerOption) { + globalServerOptions = append(globalServerOptions, opt...) + } + internal.ClearGlobalServerOptions = func() { + globalServerOptions = nil + } + internal.BinaryLogger = binaryLogger + internal.JoinServerOptions = newJoinServerOption +} + var statusOK = status.New(codes.OK, "") var logger = grpclog.Component("core") -type methodHandler func(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor UnaryServerInterceptor) (interface{}, error) +type methodHandler func(srv any, ctx context.Context, dec func(any) error, interceptor UnaryServerInterceptor) (any, error) // MethodDesc represents an RPC service's method specification. type MethodDesc struct { @@ -74,20 +99,20 @@ type ServiceDesc struct { ServiceName string // The pointer to the service interface. Used to check whether the user // provided implementation satisfies the interface requirements. - HandlerType interface{} + HandlerType any Methods []MethodDesc Streams []StreamDesc - Metadata interface{} + Metadata any } // serviceInfo wraps information about a service. It is very similar to // ServiceDesc and is constructed from it for internal purposes. type serviceInfo struct { // Contains the implementation for the methods in this service. - serviceImpl interface{} + serviceImpl any methods map[string]*MethodDesc streams map[string]*StreamDesc - mdata interface{} + mdata any } type serverWorkerData struct { @@ -100,9 +125,12 @@ type serverWorkerData struct { type Server struct { opts serverOptions - mu sync.Mutex // guards following - lis map[net.Listener]bool - conns map[transport.ServerTransport]bool + mu sync.Mutex // guards following + lis map[net.Listener]bool + // conns contains all active server transports. It is a map keyed on a + // listener address with the value being the set of active transports + // belonging to that listener. + conns map[string]map[transport.ServerTransport]bool serve bool drain bool cv *sync.Cond // signaled when connections close for GracefulStop @@ -114,10 +142,10 @@ type Server struct { channelzRemoveOnce sync.Once serveWG sync.WaitGroup // counts active Serve goroutines for GracefulStop - channelzID int64 // channelz unique identification number + channelzID *channelz.Identifier czData *channelzData - serverWorkerChannels []chan *serverWorkerData + serverWorkerChannel chan *serverWorkerData } type serverOptions struct { @@ -129,8 +157,9 @@ type serverOptions struct { streamInt StreamServerInterceptor chainUnaryInts []UnaryServerInterceptor chainStreamInts []StreamServerInterceptor + binaryLogger binarylog.Logger inTapHandle tap.ServerInHandle - statsHandler stats.Handler + statsHandlers []stats.Handler maxConcurrentStreams uint32 maxReceiveMessageSize int maxSendMessageSize int @@ -141,10 +170,12 @@ type serverOptions struct { initialConnWindowSize int32 writeBufferSize int readBufferSize int + sharedWriteBuffer bool connectionTimeout time.Duration maxHeaderListSize *uint32 headerTableSize *uint32 numServerWorkers uint32 + recvBufferPool SharedBufferPool } var defaultServerOptions = serverOptions{ @@ -153,7 +184,9 @@ var defaultServerOptions = serverOptions{ connectionTimeout: 120 * time.Second, writeBufferSize: defaultWriteBufSize, readBufferSize: defaultReadBufSize, + recvBufferPool: nopBufferPool{}, } +var globalServerOptions []ServerOption // A ServerOption sets options such as credentials, codec and keepalive parameters, etc. type ServerOption interface { @@ -163,7 +196,10 @@ type ServerOption interface { // EmptyServerOption does not alter the server configuration. It can be embedded // in another structure to build custom server options. // -// This API is EXPERIMENTAL. +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. type EmptyServerOption struct{} func (EmptyServerOption) apply(*serverOptions) {} @@ -184,10 +220,41 @@ func newFuncServerOption(f func(*serverOptions)) *funcServerOption { } } -// WriteBufferSize determines how much data can be batched before doing a write on the wire. -// The corresponding memory allocation for this buffer will be twice the size to keep syscalls low. -// The default value for this buffer is 32KB. -// Zero will disable the write buffer such that each write will be on underlying connection. +// joinServerOption provides a way to combine arbitrary number of server +// options into one. +type joinServerOption struct { + opts []ServerOption +} + +func (mdo *joinServerOption) apply(do *serverOptions) { + for _, opt := range mdo.opts { + opt.apply(do) + } +} + +func newJoinServerOption(opts ...ServerOption) ServerOption { + return &joinServerOption{opts: opts} +} + +// SharedWriteBuffer allows reusing per-connection transport write buffer. +// If this option is set to true every connection will release the buffer after +// flushing the data on the wire. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. +func SharedWriteBuffer(val bool) ServerOption { + return newFuncServerOption(func(o *serverOptions) { + o.sharedWriteBuffer = val + }) +} + +// WriteBufferSize determines how much data can be batched before doing a write +// on the wire. The corresponding memory allocation for this buffer will be +// twice the size to keep syscalls low. The default value for this buffer is +// 32KB. Zero or negative values will disable the write buffer such that each +// write will be on underlying connection. // Note: A Send call may not directly translate to a write. func WriteBufferSize(s int) ServerOption { return newFuncServerOption(func(o *serverOptions) { @@ -195,11 +262,10 @@ func WriteBufferSize(s int) ServerOption { }) } -// ReadBufferSize lets you set the size of read buffer, this determines how much data can be read at most -// for one read syscall. -// The default value for this buffer is 32KB. -// Zero will disable read buffer for a connection so data framer can access the underlying -// conn directly. +// ReadBufferSize lets you set the size of read buffer, this determines how much +// data can be read at most for one read syscall. The default value for this +// buffer is 32KB. Zero or negative values will disable read buffer for a +// connection so data framer can access the underlying conn directly. func ReadBufferSize(s int) ServerOption { return newFuncServerOption(func(o *serverOptions) { o.readBufferSize = s @@ -224,9 +290,9 @@ func InitialConnWindowSize(s int32) ServerOption { // KeepaliveParams returns a ServerOption that sets keepalive and max-age parameters for the server. func KeepaliveParams(kp keepalive.ServerParameters) ServerOption { - if kp.Time > 0 && kp.Time < time.Second { + if kp.Time > 0 && kp.Time < internal.KeepaliveMinServerPingTime { logger.Warning("Adjusting keepalive ping interval to minimum period of 1s") - kp.Time = time.Second + kp.Time = internal.KeepaliveMinServerPingTime } return newFuncServerOption(func(o *serverOptions) { @@ -256,6 +322,35 @@ func CustomCodec(codec Codec) ServerOption { }) } +// ForceServerCodec returns a ServerOption that sets a codec for message +// marshaling and unmarshaling. +// +// This will override any lookups by content-subtype for Codecs registered +// with RegisterCodec. +// +// See Content-Type on +// https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests for +// more details. Also see the documentation on RegisterCodec and +// CallContentSubtype for more details on the interaction between encoding.Codec +// and content-subtype. +// +// This function is provided for advanced users; prefer to register codecs +// using encoding.RegisterCodec. +// The server will automatically use registered codecs based on the incoming +// requests' headers. See also +// https://github.com/grpc/grpc-go/blob/master/Documentation/encoding.md#using-a-codec. +// Will be supported throughout 1.x. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. +func ForceServerCodec(codec encoding.Codec) ServerOption { + return newFuncServerOption(func(o *serverOptions) { + o.codec = codec + }) +} + // RPCCompressor returns a ServerOption that sets a compressor for outbound // messages. For backward compatibility, all outbound messages will be sent // using this compressor, regardless of incoming message compression. By @@ -366,6 +461,11 @@ func ChainStreamInterceptor(interceptors ...StreamServerInterceptor) ServerOptio // InTapHandle returns a ServerOption that sets the tap handle for all the server // transport to be created. Only one can be installed. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. func InTapHandle(h tap.ServerInHandle) ServerOption { return newFuncServerOption(func(o *serverOptions) { if o.inTapHandle != nil { @@ -378,7 +478,21 @@ func InTapHandle(h tap.ServerInHandle) ServerOption { // StatsHandler returns a ServerOption that sets the stats handler for the server. func StatsHandler(h stats.Handler) ServerOption { return newFuncServerOption(func(o *serverOptions) { - o.statsHandler = h + if h == nil { + logger.Error("ignoring nil parameter in grpc.StatsHandler ServerOption") + // Do not allow a nil stats handler, which would otherwise cause + // panics. + return + } + o.statsHandlers = append(o.statsHandlers, h) + }) +} + +// binaryLogger returns a ServerOption that can set the binary logger for the +// server. +func binaryLogger(bl binarylog.Logger) ServerOption { + return newFuncServerOption(func(o *serverOptions) { + o.binaryLogger = bl }) } @@ -405,7 +519,10 @@ func UnknownServiceHandler(streamHandler StreamHandler) ServerOption { // new connections. If this is not set, the default is 120 seconds. A zero or // negative value will result in an immediate timeout. // -// This API is EXPERIMENTAL. +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. func ConnectionTimeout(d time.Duration) ServerOption { return newFuncServerOption(func(o *serverOptions) { o.connectionTimeout = d @@ -423,7 +540,10 @@ func MaxHeaderListSize(s uint32) ServerOption { // HeaderTableSize returns a ServerOption that sets the size of dynamic // header table for stream. // -// This API is EXPERIMENTAL. +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. func HeaderTableSize(s uint32) ServerOption { return newFuncServerOption(func(o *serverOptions) { o.headerTableSize = &s @@ -435,7 +555,10 @@ func HeaderTableSize(s uint32) ServerOption { // zero (default) will disable workers and spawn a new goroutine for each // stream. // -// This API is EXPERIMENTAL. +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. func NumStreamWorkers(numServerWorkers uint32) ServerOption { // TODO: If/when this API gets stabilized (i.e. stream workers become the // only way streams are processed), change the behavior of the zero value to @@ -446,6 +569,27 @@ func NumStreamWorkers(numServerWorkers uint32) ServerOption { }) } +// RecvBufferPool returns a ServerOption that configures the server +// to use the provided shared buffer pool for parsing incoming messages. Depending +// on the application's workload, this could result in reduced memory allocation. +// +// If you are unsure about how to implement a memory pool but want to utilize one, +// begin with grpc.NewSharedBufferPool. +// +// Note: The shared buffer pool feature will not be active if any of the following +// options are used: StatsHandler, EnableTracing, or binary logging. In such +// cases, the shared buffer pool will be ignored. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. +func RecvBufferPool(bufferPool SharedBufferPool) ServerOption { + return newFuncServerOption(func(o *serverOptions) { + o.recvBufferPool = bufferPool + }) +} + // serverWorkerResetThreshold defines how often the stack must be reset. Every // N requests, by spawning a new goroutine in its place, a worker can reset its // stack so that large stacks don't live in memory forever. 2^16 should allow @@ -454,53 +598,54 @@ func NumStreamWorkers(numServerWorkers uint32) ServerOption { const serverWorkerResetThreshold = 1 << 16 // serverWorkers blocks on a *transport.Stream channel forever and waits for -// data to be fed by serveStreams. This allows different requests to be +// data to be fed by serveStreams. This allows multiple requests to be // processed by the same goroutine, removing the need for expensive stack // re-allocations (see the runtime.morestack problem [1]). // // [1] https://github.com/golang/go/issues/18138 -func (s *Server) serverWorker(ch chan *serverWorkerData) { - // To make sure all server workers don't reset at the same time, choose a - // random number of iterations before resetting. - threshold := serverWorkerResetThreshold + grpcrand.Intn(serverWorkerResetThreshold) - for completed := 0; completed < threshold; completed++ { - data, ok := <-ch +func (s *Server) serverWorker() { + for completed := 0; completed < serverWorkerResetThreshold; completed++ { + data, ok := <-s.serverWorkerChannel if !ok { return } - s.handleStream(data.st, data.stream, s.traceInfo(data.st, data.stream)) - data.wg.Done() + s.handleSingleStream(data) } - go s.serverWorker(ch) + go s.serverWorker() +} + +func (s *Server) handleSingleStream(data *serverWorkerData) { + defer data.wg.Done() + s.handleStream(data.st, data.stream, s.traceInfo(data.st, data.stream)) } -// initServerWorkers creates worker goroutines and channels to process incoming +// initServerWorkers creates worker goroutines and a channel to process incoming // connections to reduce the time spent overall on runtime.morestack. func (s *Server) initServerWorkers() { - s.serverWorkerChannels = make([]chan *serverWorkerData, s.opts.numServerWorkers) + s.serverWorkerChannel = make(chan *serverWorkerData) for i := uint32(0); i < s.opts.numServerWorkers; i++ { - s.serverWorkerChannels[i] = make(chan *serverWorkerData) - go s.serverWorker(s.serverWorkerChannels[i]) + go s.serverWorker() } } func (s *Server) stopServerWorkers() { - for i := uint32(0); i < s.opts.numServerWorkers; i++ { - close(s.serverWorkerChannels[i]) - } + close(s.serverWorkerChannel) } // NewServer creates a gRPC server which has no service registered and has not // started to accept requests yet. func NewServer(opt ...ServerOption) *Server { opts := defaultServerOptions + for _, o := range globalServerOptions { + o.apply(&opts) + } for _, o := range opt { o.apply(&opts) } s := &Server{ lis: make(map[net.Listener]bool), opts: opts, - conns: make(map[transport.ServerTransport]bool), + conns: make(map[string]map[transport.ServerTransport]bool), services: make(map[string]*serviceInfo), quit: grpcsync.NewEvent(), done: grpcsync.NewEvent(), @@ -518,15 +663,14 @@ func NewServer(opt ...ServerOption) *Server { s.initServerWorkers() } - if channelz.IsOn() { - s.channelzID = channelz.RegisterServer(&channelzServer{s}, "") - } + s.channelzID = channelz.RegisterServer(&channelzServer{s}, "") + channelz.Info(logger, s.channelzID, "Server created") return s } // printf records an event in s's event log, unless s has been stopped. // REQUIRES s.mu is held. -func (s *Server) printf(format string, a ...interface{}) { +func (s *Server) printf(format string, a ...any) { if s.events != nil { s.events.Printf(format, a...) } @@ -534,7 +678,7 @@ func (s *Server) printf(format string, a ...interface{}) { // errorf records an error in s's event log, unless s has been stopped. // REQUIRES s.mu is held. -func (s *Server) errorf(format string, a ...interface{}) { +func (s *Server) errorf(format string, a ...any) { if s.events != nil { s.events.Errorf(format, a...) } @@ -549,14 +693,14 @@ type ServiceRegistrar interface { // once the server has started serving. // desc describes the service and its methods and handlers. impl is the // service implementation which is passed to the method handlers. - RegisterService(desc *ServiceDesc, impl interface{}) + RegisterService(desc *ServiceDesc, impl any) } // RegisterService registers a service and its implementation to the gRPC // server. It is called from the IDL generated code. This must be called before // invoking Serve. If ss is non-nil (for legacy code), its type is checked to // ensure it implements sd.HandlerType. -func (s *Server) RegisterService(sd *ServiceDesc, ss interface{}) { +func (s *Server) RegisterService(sd *ServiceDesc, ss any) { if ss != nil { ht := reflect.TypeOf(sd.HandlerType).Elem() st := reflect.TypeOf(ss) @@ -567,7 +711,7 @@ func (s *Server) RegisterService(sd *ServiceDesc, ss interface{}) { s.register(sd, ss) } -func (s *Server) register(sd *ServiceDesc, ss interface{}) { +func (s *Server) register(sd *ServiceDesc, ss any) { s.mu.Lock() defer s.mu.Unlock() s.printf("RegisterService(%q)", sd.ServiceName) @@ -608,7 +752,7 @@ type MethodInfo struct { type ServiceInfo struct { Methods []MethodInfo // Metadata is the metadata specified in ServiceDesc when registering service. - Metadata interface{} + Metadata any } // GetServiceInfo returns a map from service names to ServiceInfo. @@ -644,16 +788,9 @@ func (s *Server) GetServiceInfo() map[string]ServiceInfo { // the server being stopped. var ErrServerStopped = errors.New("grpc: the server has been stopped") -func (s *Server) useTransportAuthenticator(rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) { - if s.opts.creds == nil { - return rawConn, nil, nil - } - return s.opts.creds.ServerHandshake(rawConn) -} - type listenSocket struct { net.Listener - channelzID int64 + channelzID *channelz.Identifier } func (l *listenSocket) ChannelzMetric() *channelz.SocketInternalMetric { @@ -665,9 +802,8 @@ func (l *listenSocket) ChannelzMetric() *channelz.SocketInternalMetric { func (l *listenSocket) Close() error { err := l.Listener.Close() - if channelz.IsOn() { - channelz.RemoveEntry(l.channelzID) - } + channelz.RemoveEntry(l.channelzID) + channelz.Info(logger, l.channelzID, "ListenSocket deleted") return err } @@ -700,11 +836,6 @@ func (s *Server) Serve(lis net.Listener) error { ls := &listenSocket{Listener: lis} s.lis[ls] = true - if channelz.IsOn() { - ls.channelzID = channelz.RegisterListenSocket(ls, s.channelzID, lis.Addr().String()) - } - s.mu.Unlock() - defer func() { s.mu.Lock() if s.lis != nil && s.lis[ls] { @@ -714,8 +845,16 @@ func (s *Server) Serve(lis net.Listener) error { s.mu.Unlock() }() - var tempDelay time.Duration // how long to sleep on accept failure + var err error + ls.channelzID, err = channelz.RegisterListenSocket(ls, s.channelzID, lis.Addr().String()) + if err != nil { + s.mu.Unlock() + return err + } + s.mu.Unlock() + channelz.Info(logger, ls.channelzID, "ListenSocket created") + var tempDelay time.Duration // how long to sleep on accept failure for { rawConn, err := lis.Accept() if err != nil { @@ -759,7 +898,7 @@ func (s *Server) Serve(lis net.Listener) error { // s.conns before this conn can be added. s.serveWG.Add(1) go func() { - s.handleRawConn(rawConn) + s.handleRawConn(lis.Addr().String(), rawConn) s.serveWG.Done() }() } @@ -767,68 +906,72 @@ func (s *Server) Serve(lis net.Listener) error { // handleRawConn forks a goroutine to handle a just-accepted connection that // has not had any I/O performed on it yet. -func (s *Server) handleRawConn(rawConn net.Conn) { +func (s *Server) handleRawConn(lisAddr string, rawConn net.Conn) { if s.quit.HasFired() { rawConn.Close() return } rawConn.SetDeadline(time.Now().Add(s.opts.connectionTimeout)) - conn, authInfo, err := s.useTransportAuthenticator(rawConn) - if err != nil { - // ErrConnDispatched means that the connection was dispatched away from - // gRPC; those connections should be left open. - if err != credentials.ErrConnDispatched { - s.mu.Lock() - s.errorf("ServerHandshake(%q) failed: %v", rawConn.RemoteAddr(), err) - s.mu.Unlock() - channelz.Warningf(logger, s.channelzID, "grpc: Server.Serve failed to complete security handshake from %q: %v", rawConn.RemoteAddr(), err) - rawConn.Close() - } - rawConn.SetDeadline(time.Time{}) - return - } // Finish handshaking (HTTP2) - st := s.newHTTP2Transport(conn, authInfo) + st := s.newHTTP2Transport(rawConn) + rawConn.SetDeadline(time.Time{}) if st == nil { return } - rawConn.SetDeadline(time.Time{}) - if !s.addConn(st) { + if !s.addConn(lisAddr, st) { return } go func() { s.serveStreams(st) - s.removeConn(st) + s.removeConn(lisAddr, st) }() } +func (s *Server) drainServerTransports(addr string) { + s.mu.Lock() + conns := s.conns[addr] + for st := range conns { + st.Drain("") + } + s.mu.Unlock() +} + // newHTTP2Transport sets up a http/2 transport (using the // gRPC http2 server transport in transport/http2_server.go). -func (s *Server) newHTTP2Transport(c net.Conn, authInfo credentials.AuthInfo) transport.ServerTransport { +func (s *Server) newHTTP2Transport(c net.Conn) transport.ServerTransport { config := &transport.ServerConfig{ MaxStreams: s.opts.maxConcurrentStreams, - AuthInfo: authInfo, + ConnectionTimeout: s.opts.connectionTimeout, + Credentials: s.opts.creds, InTapHandle: s.opts.inTapHandle, - StatsHandler: s.opts.statsHandler, + StatsHandlers: s.opts.statsHandlers, KeepaliveParams: s.opts.keepaliveParams, KeepalivePolicy: s.opts.keepalivePolicy, InitialWindowSize: s.opts.initialWindowSize, InitialConnWindowSize: s.opts.initialConnWindowSize, WriteBufferSize: s.opts.writeBufferSize, ReadBufferSize: s.opts.readBufferSize, + SharedWriteBuffer: s.opts.sharedWriteBuffer, ChannelzParentID: s.channelzID, MaxHeaderListSize: s.opts.maxHeaderListSize, HeaderTableSize: s.opts.headerTableSize, } - st, err := transport.NewServerTransport("http2", c, config) + st, err := transport.NewServerTransport(c, config) if err != nil { s.mu.Lock() s.errorf("NewServerTransport(%q) failed: %v", c.RemoteAddr(), err) s.mu.Unlock() - c.Close() - channelz.Warning(logger, s.channelzID, "grpc: Server.Serve failed to create ServerTransport: ", err) + // ErrConnDispatched means that the connection was dispatched away from + // gRPC; those connections should be left open. + if err != credentials.ErrConnDispatched { + // Don't log on ErrConnDispatched and io.EOF to prevent log spam. + if err != io.EOF { + channelz.Info(logger, s.channelzID, "grpc: Server.Serve failed to create ServerTransport: ", err) + } + c.Close() + } return nil } @@ -836,29 +979,24 @@ func (s *Server) newHTTP2Transport(c net.Conn, authInfo credentials.AuthInfo) tr } func (s *Server) serveStreams(st transport.ServerTransport) { - defer st.Close() + defer st.Close(errors.New("finished serving streams for the server transport")) var wg sync.WaitGroup - var roundRobinCounter uint32 st.HandleStreams(func(stream *transport.Stream) { wg.Add(1) if s.opts.numServerWorkers > 0 { data := &serverWorkerData{st: st, wg: &wg, stream: stream} select { - case s.serverWorkerChannels[atomic.AddUint32(&roundRobinCounter, 1)%s.opts.numServerWorkers] <- data: + case s.serverWorkerChannel <- data: + return default: // If all stream workers are busy, fallback to the default code path. - go func() { - s.handleStream(st, stream, s.traceInfo(st, stream)) - wg.Done() - }() } - } else { - go func() { - defer wg.Done() - s.handleStream(st, stream, s.traceInfo(st, stream)) - }() } + go func() { + defer wg.Done() + s.handleStream(st, stream, s.traceInfo(st, stream)) + }() }, func(ctx context.Context, method string) context.Context { if !EnableTracing { return ctx @@ -883,28 +1021,33 @@ var _ http.Handler = (*Server)(nil) // To share one port (such as 443 for https) between gRPC and an // existing http.Handler, use a root http.Handler such as: // -// if r.ProtoMajor == 2 && strings.HasPrefix( -// r.Header.Get("Content-Type"), "application/grpc") { -// grpcServer.ServeHTTP(w, r) -// } else { -// yourMux.ServeHTTP(w, r) -// } +// if r.ProtoMajor == 2 && strings.HasPrefix( +// r.Header.Get("Content-Type"), "application/grpc") { +// grpcServer.ServeHTTP(w, r) +// } else { +// yourMux.ServeHTTP(w, r) +// } // // Note that ServeHTTP uses Go's HTTP/2 server implementation which is totally // separate from grpc-go's HTTP/2 server. Performance and features may vary // between the two paths. ServeHTTP does not support some gRPC features -// available through grpc-go's HTTP/2 server, and it is currently EXPERIMENTAL -// and subject to change. +// available through grpc-go's HTTP/2 server. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { - st, err := transport.NewServerHandlerTransport(w, r, s.opts.statsHandler) + st, err := transport.NewServerHandlerTransport(w, r, s.opts.statsHandlers) if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + // Errors returned from transport.NewServerHandlerTransport have + // already been written to w. return } - if !s.addConn(st) { + if !s.addConn(listenerAddressForServeHTTP, st) { return } - defer s.removeConn(st) + defer s.removeConn(listenerAddressForServeHTTP, st) s.serveStreams(st) } @@ -932,27 +1075,40 @@ func (s *Server) traceInfo(st transport.ServerTransport, stream *transport.Strea return trInfo } -func (s *Server) addConn(st transport.ServerTransport) bool { +func (s *Server) addConn(addr string, st transport.ServerTransport) bool { s.mu.Lock() defer s.mu.Unlock() if s.conns == nil { - st.Close() + st.Close(errors.New("Server.addConn called when server has already been stopped")) return false } if s.drain { // Transport added after we drained our existing conns: drain it // immediately. - st.Drain() + st.Drain("") } - s.conns[st] = true + + if s.conns[addr] == nil { + // Create a map entry if this is the first connection on this listener. + s.conns[addr] = make(map[transport.ServerTransport]bool) + } + s.conns[addr][st] = true return true } -func (s *Server) removeConn(st transport.ServerTransport) { +func (s *Server) removeConn(addr string, st transport.ServerTransport) { s.mu.Lock() defer s.mu.Unlock() - if s.conns != nil { - delete(s.conns, st) + + conns := s.conns[addr] + if conns != nil { + delete(conns, st) + if len(conns) == 0 { + // If the last connection for this address is being removed, also + // remove the map entry corresponding to the address. This is used + // in GracefulStop() when waiting for all connections to be closed. + delete(s.conns, addr) + } s.cv.Broadcast() } } @@ -979,7 +1135,7 @@ func (s *Server) incrCallsFailed() { atomic.AddInt64(&s.czData.callsFailed, 1) } -func (s *Server) sendResponse(t transport.ServerTransport, stream *transport.Stream, msg interface{}, cp Compressor, opts *transport.Options, comp encoding.Compressor) error { +func (s *Server) sendResponse(t transport.ServerTransport, stream *transport.Stream, msg any, cp Compressor, opts *transport.Options, comp encoding.Compressor) error { data, err := encode(s.getCodec(stream.ContentSubtype()), msg) if err != nil { channelz.Error(logger, s.channelzID, "grpc: server failed to encode response: ", err) @@ -996,8 +1152,10 @@ func (s *Server) sendResponse(t transport.ServerTransport, stream *transport.Str return status.Errorf(codes.ResourceExhausted, "grpc: trying to send message larger than max (%d vs. %d)", len(payload), s.opts.maxSendMessageSize) } err = t.Write(stream, hdr, payload, opts) - if err == nil && s.opts.statsHandler != nil { - s.opts.statsHandler.HandleRPC(stream.Context(), outPayload(false, msg, data, payload, time.Now())) + if err == nil { + for _, sh := range s.opts.statsHandlers { + sh.HandleRPC(stream.Context(), outPayload(false, msg, data, payload, time.Now())) + } } return err } @@ -1017,36 +1175,40 @@ func chainUnaryServerInterceptors(s *Server) { } else if len(interceptors) == 1 { chainedInt = interceptors[0] } else { - chainedInt = func(ctx context.Context, req interface{}, info *UnaryServerInfo, handler UnaryHandler) (interface{}, error) { - return interceptors[0](ctx, req, info, getChainUnaryHandler(interceptors, 0, info, handler)) - } + chainedInt = chainUnaryInterceptors(interceptors) } s.opts.unaryInt = chainedInt } -// getChainUnaryHandler recursively generate the chained UnaryHandler +func chainUnaryInterceptors(interceptors []UnaryServerInterceptor) UnaryServerInterceptor { + return func(ctx context.Context, req any, info *UnaryServerInfo, handler UnaryHandler) (any, error) { + return interceptors[0](ctx, req, info, getChainUnaryHandler(interceptors, 0, info, handler)) + } +} + func getChainUnaryHandler(interceptors []UnaryServerInterceptor, curr int, info *UnaryServerInfo, finalHandler UnaryHandler) UnaryHandler { if curr == len(interceptors)-1 { return finalHandler } - - return func(ctx context.Context, req interface{}) (interface{}, error) { + return func(ctx context.Context, req any) (any, error) { return interceptors[curr+1](ctx, req, info, getChainUnaryHandler(interceptors, curr+1, info, finalHandler)) } } func (s *Server) processUnaryRPC(t transport.ServerTransport, stream *transport.Stream, info *serviceInfo, md *MethodDesc, trInfo *traceInfo) (err error) { - sh := s.opts.statsHandler - if sh != nil || trInfo != nil || channelz.IsOn() { + shs := s.opts.statsHandlers + if len(shs) != 0 || trInfo != nil || channelz.IsOn() { if channelz.IsOn() { s.incrCallsStarted() } var statsBegin *stats.Begin - if sh != nil { + for _, sh := range shs { beginTime := time.Now() statsBegin = &stats.Begin{ - BeginTime: beginTime, + BeginTime: beginTime, + IsClientStream: false, + IsServerStream: false, } sh.HandleRPC(stream.Context(), statsBegin) } @@ -1066,13 +1228,13 @@ func (s *Server) processUnaryRPC(t transport.ServerTransport, stream *transport. defer func() { if trInfo != nil { if err != nil && err != io.EOF { - trInfo.tr.LazyLog(&fmtStringer{"%v", []interface{}{err}}, true) + trInfo.tr.LazyLog(&fmtStringer{"%v", []any{err}}, true) trInfo.tr.SetError() } trInfo.tr.Finish() } - if sh != nil { + for _, sh := range shs { end := &stats.End{ BeginTime: statsBegin.BeginTime, EndTime: time.Now(), @@ -1092,9 +1254,16 @@ func (s *Server) processUnaryRPC(t transport.ServerTransport, stream *transport. } }() } - - binlog := binarylog.GetMethodLogger(stream.Method()) - if binlog != nil { + var binlogs []binarylog.MethodLogger + if ml := binarylog.GetMethodLogger(stream.Method()); ml != nil { + binlogs = append(binlogs, ml) + } + if s.opts.binaryLogger != nil { + if ml := s.opts.binaryLogger.GetMethodLogger(stream.Method()); ml != nil { + binlogs = append(binlogs, ml) + } + } + if len(binlogs) != 0 { ctx := stream.Context() md, _ := metadata.FromIncomingContext(ctx) logEntry := &binarylog.ClientHeader{ @@ -1114,7 +1283,9 @@ func (s *Server) processUnaryRPC(t transport.ServerTransport, stream *transport. if peer, ok := peer.FromContext(ctx); ok { logEntry.PeerAddr = peer.Addr } - binlog.Log(logEntry) + for _, binlog := range binlogs { + binlog.Log(ctx, logEntry) + } } // comp and cp are used for compression. decomp and dc are used for @@ -1124,6 +1295,7 @@ func (s *Server) processUnaryRPC(t transport.ServerTransport, stream *transport. var comp, decomp encoding.Compressor var cp Compressor var dc Decompressor + var sendCompressorName string // If dc is set and matches the stream's compression, use it. Otherwise, try // to find a matching registered compressor for decomp. @@ -1144,46 +1316,56 @@ func (s *Server) processUnaryRPC(t transport.ServerTransport, stream *transport. // NOTE: this needs to be ahead of all handling, https://github.com/grpc/grpc-go/issues/686. if s.opts.cp != nil { cp = s.opts.cp - stream.SetSendCompress(cp.Type()) + sendCompressorName = cp.Type() } else if rc := stream.RecvCompress(); rc != "" && rc != encoding.Identity { // Legacy compressor not specified; attempt to respond with same encoding. comp = encoding.GetCompressor(rc) if comp != nil { - stream.SetSendCompress(rc) + sendCompressorName = comp.Name() + } + } + + if sendCompressorName != "" { + if err := stream.SetSendCompress(sendCompressorName); err != nil { + return status.Errorf(codes.Internal, "grpc: failed to set send compressor: %v", err) } } var payInfo *payloadInfo - if sh != nil || binlog != nil { + if len(shs) != 0 || len(binlogs) != 0 { payInfo = &payloadInfo{} } - d, err := recvAndDecompress(&parser{r: stream}, stream, dc, s.opts.maxReceiveMessageSize, payInfo, decomp) + d, err := recvAndDecompress(&parser{r: stream, recvBufferPool: s.opts.recvBufferPool}, stream, dc, s.opts.maxReceiveMessageSize, payInfo, decomp) if err != nil { if e := t.WriteStatus(stream, status.Convert(err)); e != nil { - channelz.Warningf(logger, s.channelzID, "grpc: Server.processUnaryRPC failed to write status %v", e) + channelz.Warningf(logger, s.channelzID, "grpc: Server.processUnaryRPC failed to write status: %v", e) } return err } if channelz.IsOn() { t.IncrMsgRecv() } - df := func(v interface{}) error { + df := func(v any) error { if err := s.getCodec(stream.ContentSubtype()).Unmarshal(d, v); err != nil { return status.Errorf(codes.Internal, "grpc: error unmarshalling request: %v", err) } - if sh != nil { + for _, sh := range shs { sh.HandleRPC(stream.Context(), &stats.InPayload{ - RecvTime: time.Now(), - Payload: v, - WireLength: payInfo.wireLength, - Data: d, - Length: len(d), + RecvTime: time.Now(), + Payload: v, + Length: len(d), + WireLength: payInfo.compressedLength + headerLen, + CompressedLength: payInfo.compressedLength, + Data: d, }) } - if binlog != nil { - binlog.Log(&binarylog.ClientMessage{ + if len(binlogs) != 0 { + cm := &binarylog.ClientMessage{ Message: d, - }) + } + for _, binlog := range binlogs { + binlog.Log(stream.Context(), cm) + } } if trInfo != nil { trInfo.tr.LazyLog(&payload{sent: false, msg: v}, true) @@ -1195,9 +1377,10 @@ func (s *Server) processUnaryRPC(t transport.ServerTransport, stream *transport. if appErr != nil { appStatus, ok := status.FromError(appErr) if !ok { - // Convert appErr if it is not a grpc status error. - appErr = status.Error(codes.Unknown, appErr.Error()) - appStatus, _ = status.FromError(appErr) + // Convert non-status application error to a status error with code + // Unknown, but handle context errors specifically. + appStatus = status.FromContextError(appErr) + appErr = appStatus.Err() } if trInfo != nil { trInfo.tr.LazyLog(stringer(appStatus.Message()), true) @@ -1206,18 +1389,24 @@ func (s *Server) processUnaryRPC(t transport.ServerTransport, stream *transport. if e := t.WriteStatus(stream, appStatus); e != nil { channelz.Warningf(logger, s.channelzID, "grpc: Server.processUnaryRPC failed to write status: %v", e) } - if binlog != nil { + if len(binlogs) != 0 { if h, _ := stream.Header(); h.Len() > 0 { // Only log serverHeader if there was header. Otherwise it can // be trailer only. - binlog.Log(&binarylog.ServerHeader{ + sh := &binarylog.ServerHeader{ Header: h, - }) + } + for _, binlog := range binlogs { + binlog.Log(stream.Context(), sh) + } } - binlog.Log(&binarylog.ServerTrailer{ + st := &binarylog.ServerTrailer{ Trailer: stream.Trailer(), Err: appErr, - }) + } + for _, binlog := range binlogs { + binlog.Log(stream.Context(), st) + } } return appErr } @@ -1226,6 +1415,11 @@ func (s *Server) processUnaryRPC(t transport.ServerTransport, stream *transport. } opts := &transport.Options{Last: true} + // Server handler could have set new compressor by calling SetSendCompressor. + // In case it is set, we need to use it for compressing outbound message. + if stream.SendCompress() != sendCompressorName { + comp = encoding.GetCompressor(stream.SendCompress()) + } if err := s.sendResponse(t, stream, reply, cp, opts, comp); err != nil { if err == io.EOF { // The entire stream is done (for unary RPC only). @@ -1243,26 +1437,34 @@ func (s *Server) processUnaryRPC(t transport.ServerTransport, stream *transport. panic(fmt.Sprintf("grpc: Unexpected error (%T) from sendResponse: %v", st, st)) } } - if binlog != nil { + if len(binlogs) != 0 { h, _ := stream.Header() - binlog.Log(&binarylog.ServerHeader{ + sh := &binarylog.ServerHeader{ Header: h, - }) - binlog.Log(&binarylog.ServerTrailer{ + } + st := &binarylog.ServerTrailer{ Trailer: stream.Trailer(), Err: appErr, - }) + } + for _, binlog := range binlogs { + binlog.Log(stream.Context(), sh) + binlog.Log(stream.Context(), st) + } } return err } - if binlog != nil { + if len(binlogs) != 0 { h, _ := stream.Header() - binlog.Log(&binarylog.ServerHeader{ + sh := &binarylog.ServerHeader{ Header: h, - }) - binlog.Log(&binarylog.ServerMessage{ + } + sm := &binarylog.ServerMessage{ Message: reply, - }) + } + for _, binlog := range binlogs { + binlog.Log(stream.Context(), sh) + binlog.Log(stream.Context(), sm) + } } if channelz.IsOn() { t.IncrMsgSent() @@ -1273,14 +1475,16 @@ func (s *Server) processUnaryRPC(t transport.ServerTransport, stream *transport. // TODO: Should we be logging if writing status failed here, like above? // Should the logging be in WriteStatus? Should we ignore the WriteStatus // error or allow the stats handler to see it? - err = t.WriteStatus(stream, statusOK) - if binlog != nil { - binlog.Log(&binarylog.ServerTrailer{ + if len(binlogs) != 0 { + st := &binarylog.ServerTrailer{ Trailer: stream.Trailer(), Err: appErr, - }) + } + for _, binlog := range binlogs { + binlog.Log(stream.Context(), st) + } } - return err + return t.WriteStatus(stream, statusOK) } // chainStreamServerInterceptors chains all stream server interceptors into one. @@ -1298,22 +1502,24 @@ func chainStreamServerInterceptors(s *Server) { } else if len(interceptors) == 1 { chainedInt = interceptors[0] } else { - chainedInt = func(srv interface{}, ss ServerStream, info *StreamServerInfo, handler StreamHandler) error { - return interceptors[0](srv, ss, info, getChainStreamHandler(interceptors, 0, info, handler)) - } + chainedInt = chainStreamInterceptors(interceptors) } s.opts.streamInt = chainedInt } -// getChainStreamHandler recursively generate the chained StreamHandler +func chainStreamInterceptors(interceptors []StreamServerInterceptor) StreamServerInterceptor { + return func(srv any, ss ServerStream, info *StreamServerInfo, handler StreamHandler) error { + return interceptors[0](srv, ss, info, getChainStreamHandler(interceptors, 0, info, handler)) + } +} + func getChainStreamHandler(interceptors []StreamServerInterceptor, curr int, info *StreamServerInfo, finalHandler StreamHandler) StreamHandler { if curr == len(interceptors)-1 { return finalHandler } - - return func(srv interface{}, ss ServerStream) error { - return interceptors[curr+1](srv, ss, info, getChainStreamHandler(interceptors, curr+1, info, finalHandler)) + return func(srv any, stream ServerStream) error { + return interceptors[curr+1](srv, stream, info, getChainStreamHandler(interceptors, curr+1, info, finalHandler)) } } @@ -1321,35 +1527,39 @@ func (s *Server) processStreamingRPC(t transport.ServerTransport, stream *transp if channelz.IsOn() { s.incrCallsStarted() } - sh := s.opts.statsHandler + shs := s.opts.statsHandlers var statsBegin *stats.Begin - if sh != nil { + if len(shs) != 0 { beginTime := time.Now() statsBegin = &stats.Begin{ - BeginTime: beginTime, + BeginTime: beginTime, + IsClientStream: sd.ClientStreams, + IsServerStream: sd.ServerStreams, + } + for _, sh := range shs { + sh.HandleRPC(stream.Context(), statsBegin) } - sh.HandleRPC(stream.Context(), statsBegin) } ctx := NewContextWithServerTransportStream(stream.Context(), stream) ss := &serverStream{ ctx: ctx, t: t, s: stream, - p: &parser{r: stream}, + p: &parser{r: stream, recvBufferPool: s.opts.recvBufferPool}, codec: s.getCodec(stream.ContentSubtype()), maxReceiveMessageSize: s.opts.maxReceiveMessageSize, maxSendMessageSize: s.opts.maxSendMessageSize, trInfo: trInfo, - statsHandler: sh, + statsHandler: shs, } - if sh != nil || trInfo != nil || channelz.IsOn() { + if len(shs) != 0 || trInfo != nil || channelz.IsOn() { // See comment in processUnaryRPC on defers. defer func() { if trInfo != nil { ss.mu.Lock() if err != nil && err != io.EOF { - ss.trInfo.tr.LazyLog(&fmtStringer{"%v", []interface{}{err}}, true) + ss.trInfo.tr.LazyLog(&fmtStringer{"%v", []any{err}}, true) ss.trInfo.tr.SetError() } ss.trInfo.tr.Finish() @@ -1357,7 +1567,7 @@ func (s *Server) processStreamingRPC(t transport.ServerTransport, stream *transp ss.mu.Unlock() } - if sh != nil { + if len(shs) != 0 { end := &stats.End{ BeginTime: statsBegin.BeginTime, EndTime: time.Now(), @@ -1365,7 +1575,9 @@ func (s *Server) processStreamingRPC(t transport.ServerTransport, stream *transp if err != nil && err != io.EOF { end.Error = toRPCErr(err) } - sh.HandleRPC(stream.Context(), end) + for _, sh := range shs { + sh.HandleRPC(stream.Context(), end) + } } if channelz.IsOn() { @@ -1378,8 +1590,15 @@ func (s *Server) processStreamingRPC(t transport.ServerTransport, stream *transp }() } - ss.binlog = binarylog.GetMethodLogger(stream.Method()) - if ss.binlog != nil { + if ml := binarylog.GetMethodLogger(stream.Method()); ml != nil { + ss.binlogs = append(ss.binlogs, ml) + } + if s.opts.binaryLogger != nil { + if ml := s.opts.binaryLogger.GetMethodLogger(stream.Method()); ml != nil { + ss.binlogs = append(ss.binlogs, ml) + } + } + if len(ss.binlogs) != 0 { md, _ := metadata.FromIncomingContext(ctx) logEntry := &binarylog.ClientHeader{ Header: md, @@ -1398,7 +1617,9 @@ func (s *Server) processStreamingRPC(t transport.ServerTransport, stream *transp if peer, ok := peer.FromContext(ss.Context()); ok { logEntry.PeerAddr = peer.Addr } - ss.binlog.Log(logEntry) + for _, binlog := range ss.binlogs { + binlog.Log(stream.Context(), logEntry) + } } // If dc is set and matches the stream's compression, use it. Otherwise, try @@ -1420,20 +1641,28 @@ func (s *Server) processStreamingRPC(t transport.ServerTransport, stream *transp // NOTE: this needs to be ahead of all handling, https://github.com/grpc/grpc-go/issues/686. if s.opts.cp != nil { ss.cp = s.opts.cp - stream.SetSendCompress(s.opts.cp.Type()) + ss.sendCompressorName = s.opts.cp.Type() } else if rc := stream.RecvCompress(); rc != "" && rc != encoding.Identity { // Legacy compressor not specified; attempt to respond with same encoding. ss.comp = encoding.GetCompressor(rc) if ss.comp != nil { - stream.SetSendCompress(rc) + ss.sendCompressorName = rc + } + } + + if ss.sendCompressorName != "" { + if err := stream.SetSendCompress(ss.sendCompressorName); err != nil { + return status.Errorf(codes.Internal, "grpc: failed to set send compressor: %v", err) } } + ss.ctx = newContextWithRPCInfo(ss.ctx, false, ss.codec, ss.cp, ss.comp) + if trInfo != nil { trInfo.tr.LazyLog(&trInfo.firstLine, false) } var appErr error - var server interface{} + var server any if info != nil { server = info.serviceImpl } @@ -1450,7 +1679,9 @@ func (s *Server) processStreamingRPC(t transport.ServerTransport, stream *transp if appErr != nil { appStatus, ok := status.FromError(appErr) if !ok { - appStatus = status.New(codes.Unknown, appErr.Error()) + // Convert non-status application error to a status error with code + // Unknown, but handle context errors specifically. + appStatus = status.FromContextError(appErr) appErr = appStatus.Err() } if trInfo != nil { @@ -1459,13 +1690,16 @@ func (s *Server) processStreamingRPC(t transport.ServerTransport, stream *transp ss.trInfo.tr.SetError() ss.mu.Unlock() } - t.WriteStatus(ss.s, appStatus) - if ss.binlog != nil { - ss.binlog.Log(&binarylog.ServerTrailer{ + if len(ss.binlogs) != 0 { + st := &binarylog.ServerTrailer{ Trailer: ss.s.Trailer(), Err: appErr, - }) + } + for _, binlog := range ss.binlogs { + binlog.Log(stream.Context(), st) + } } + t.WriteStatus(ss.s, appStatus) // TODO: Should we log an error from WriteStatus here and below? return appErr } @@ -1474,14 +1708,16 @@ func (s *Server) processStreamingRPC(t transport.ServerTransport, stream *transp ss.trInfo.tr.LazyLog(stringer("OK"), false) ss.mu.Unlock() } - err = t.WriteStatus(ss.s, statusOK) - if ss.binlog != nil { - ss.binlog.Log(&binarylog.ServerTrailer{ + if len(ss.binlogs) != 0 { + st := &binarylog.ServerTrailer{ Trailer: ss.s.Trailer(), Err: appErr, - }) + } + for _, binlog := range ss.binlogs { + binlog.Log(stream.Context(), st) + } } - return err + return t.WriteStatus(ss.s, statusOK) } func (s *Server) handleStream(t transport.ServerTransport, stream *transport.Stream, trInfo *traceInfo) { @@ -1492,13 +1728,13 @@ func (s *Server) handleStream(t transport.ServerTransport, stream *transport.Str pos := strings.LastIndex(sm, "/") if pos == -1 { if trInfo != nil { - trInfo.tr.LazyLog(&fmtStringer{"Malformed method name %q", []interface{}{sm}}, true) + trInfo.tr.LazyLog(&fmtStringer{"Malformed method name %q", []any{sm}}, true) trInfo.tr.SetError() } errDesc := fmt.Sprintf("malformed method name: %q", stream.Method()) - if err := t.WriteStatus(stream, status.New(codes.ResourceExhausted, errDesc)); err != nil { + if err := t.WriteStatus(stream, status.New(codes.Unimplemented, errDesc)); err != nil { if trInfo != nil { - trInfo.tr.LazyLog(&fmtStringer{"%v", []interface{}{err}}, true) + trInfo.tr.LazyLog(&fmtStringer{"%v", []any{err}}, true) trInfo.tr.SetError() } channelz.Warningf(logger, s.channelzID, "grpc: Server.handleStream failed to write status: %v", err) @@ -1539,7 +1775,7 @@ func (s *Server) handleStream(t transport.ServerTransport, stream *transport.Str } if err := t.WriteStatus(stream, status.New(codes.Unimplemented, errDesc)); err != nil { if trInfo != nil { - trInfo.tr.LazyLog(&fmtStringer{"%v", []interface{}{err}}, true) + trInfo.tr.LazyLog(&fmtStringer{"%v", []any{err}}, true) trInfo.tr.SetError() } channelz.Warningf(logger, s.channelzID, "grpc: Server.handleStream failed to write status: %v", err) @@ -1555,7 +1791,10 @@ type streamKey struct{} // NewContextWithServerTransportStream creates a new context from ctx and // attaches stream to it. // -// This API is EXPERIMENTAL. +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. func NewContextWithServerTransportStream(ctx context.Context, stream ServerTransportStream) context.Context { return context.WithValue(ctx, streamKey{}, stream) } @@ -1567,7 +1806,10 @@ func NewContextWithServerTransportStream(ctx context.Context, stream ServerTrans // // See also NewContextWithServerTransportStream. // -// This API is EXPERIMENTAL. +// # Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. type ServerTransportStream interface { Method() string SetHeader(md metadata.MD) error @@ -1579,7 +1821,10 @@ type ServerTransportStream interface { // ctx. Returns nil if the given context has no stream associated with it // (which implies it is not an RPC invocation context). // -// This API is EXPERIMENTAL. +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. func ServerTransportStreamFromContext(ctx context.Context) ServerTransportStream { s, _ := ctx.Value(streamKey{}).(ServerTransportStream) return s @@ -1598,16 +1843,12 @@ func (s *Server) Stop() { s.done.Fire() }() - s.channelzRemoveOnce.Do(func() { - if channelz.IsOn() { - channelz.RemoveEntry(s.channelzID) - } - }) + s.channelzRemoveOnce.Do(func() { channelz.RemoveEntry(s.channelzID) }) s.mu.Lock() listeners := s.lis s.lis = nil - st := s.conns + conns := s.conns s.conns = nil // interrupt GracefulStop if Stop and GracefulStop are called concurrently. s.cv.Broadcast() @@ -1616,8 +1857,10 @@ func (s *Server) Stop() { for lis := range listeners { lis.Close() } - for c := range st { - c.Close() + for _, cs := range conns { + for st := range cs { + st.Close(errors.New("Server.Stop called")) + } } if s.opts.numServerWorkers > 0 { s.stopServerWorkers() @@ -1638,11 +1881,7 @@ func (s *Server) GracefulStop() { s.quit.Fire() defer s.done.Fire() - s.channelzRemoveOnce.Do(func() { - if channelz.IsOn() { - channelz.RemoveEntry(s.channelzID) - } - }) + s.channelzRemoveOnce.Do(func() { channelz.RemoveEntry(s.channelzID) }) s.mu.Lock() if s.conns == nil { s.mu.Unlock() @@ -1654,8 +1893,10 @@ func (s *Server) GracefulStop() { } s.lis = nil if !s.drain { - for st := range s.conns { - st.Drain() + for _, conns := range s.conns { + for st := range conns { + st.Drain("graceful_stop") + } } s.drain = true } @@ -1693,12 +1934,26 @@ func (s *Server) getCodec(contentSubtype string) baseCodec { return codec } -// SetHeader sets the header metadata. -// When called multiple times, all the provided metadata will be merged. -// All the metadata will be sent out when one of the following happens: -// - grpc.SendHeader() is called; -// - The first response is sent out; -// - An RPC status is sent out (error or success). +// SetHeader sets the header metadata to be sent from the server to the client. +// The context provided must be the context passed to the server's handler. +// +// Streaming RPCs should prefer the SetHeader method of the ServerStream. +// +// When called multiple times, all the provided metadata will be merged. All +// the metadata will be sent out when one of the following happens: +// +// - grpc.SendHeader is called, or for streaming handlers, stream.SendHeader. +// - The first response message is sent. For unary handlers, this occurs when +// the handler returns; for streaming handlers, this can happen when stream's +// SendMsg method is called. +// - An RPC status is sent out (error or success). This occurs when the handler +// returns. +// +// SetHeader will fail if called after any of the events above. +// +// The error returned is compatible with the status package. However, the +// status code will often not match the RPC status as seen by the client +// application, and therefore, should not be relied upon for this purpose. func SetHeader(ctx context.Context, md metadata.MD) error { if md.Len() == 0 { return nil @@ -1710,8 +1965,14 @@ func SetHeader(ctx context.Context, md metadata.MD) error { return stream.SetHeader(md) } -// SendHeader sends header metadata. It may be called at most once. -// The provided md and headers set by SetHeader() will be sent. +// SendHeader sends header metadata. It may be called at most once, and may not +// be called after any event that causes headers to be sent (see SetHeader for +// a complete list). The provided md and headers set by SetHeader() will be +// sent. +// +// The error returned is compatible with the status package. However, the +// status code will often not match the RPC status as seen by the client +// application, and therefore, should not be relied upon for this purpose. func SendHeader(ctx context.Context, md metadata.MD) error { stream := ServerTransportStreamFromContext(ctx) if stream == nil { @@ -1723,8 +1984,66 @@ func SendHeader(ctx context.Context, md metadata.MD) error { return nil } +// SetSendCompressor sets a compressor for outbound messages from the server. +// It must not be called after any event that causes headers to be sent +// (see ServerStream.SetHeader for the complete list). Provided compressor is +// used when below conditions are met: +// +// - compressor is registered via encoding.RegisterCompressor +// - compressor name must exist in the client advertised compressor names +// sent in grpc-accept-encoding header. Use ClientSupportedCompressors to +// get client supported compressor names. +// +// The context provided must be the context passed to the server's handler. +// It must be noted that compressor name encoding.Identity disables the +// outbound compression. +// By default, server messages will be sent using the same compressor with +// which request messages were sent. +// +// It is not safe to call SetSendCompressor concurrently with SendHeader and +// SendMsg. +// +// # Experimental +// +// Notice: This function is EXPERIMENTAL and may be changed or removed in a +// later release. +func SetSendCompressor(ctx context.Context, name string) error { + stream, ok := ServerTransportStreamFromContext(ctx).(*transport.Stream) + if !ok || stream == nil { + return fmt.Errorf("failed to fetch the stream from the given context") + } + + if err := validateSendCompressor(name, stream.ClientAdvertisedCompressors()); err != nil { + return fmt.Errorf("unable to set send compressor: %w", err) + } + + return stream.SetSendCompress(name) +} + +// ClientSupportedCompressors returns compressor names advertised by the client +// via grpc-accept-encoding header. +// +// The context provided must be the context passed to the server's handler. +// +// # Experimental +// +// Notice: This function is EXPERIMENTAL and may be changed or removed in a +// later release. +func ClientSupportedCompressors(ctx context.Context) ([]string, error) { + stream, ok := ServerTransportStreamFromContext(ctx).(*transport.Stream) + if !ok || stream == nil { + return nil, fmt.Errorf("failed to fetch the stream from the given context %v", ctx) + } + + return strings.Split(stream.ClientAdvertisedCompressors(), ","), nil +} + // SetTrailer sets the trailer metadata that will be sent when an RPC returns. // When called more than once, all the provided metadata will be merged. +// +// The error returned is compatible with the status package. However, the +// status code will often not match the RPC status as seen by the client +// application, and therefore, should not be relied upon for this purpose. func SetTrailer(ctx context.Context, md metadata.MD) error { if md.Len() == 0 { return nil @@ -1753,3 +2072,22 @@ type channelzServer struct { func (c *channelzServer) ChannelzMetric() *channelz.ServerInternalMetric { return c.s.channelzMetric() } + +// validateSendCompressor returns an error when given compressor name cannot be +// handled by the server or the client based on the advertised compressors. +func validateSendCompressor(name, clientCompressors string) error { + if name == encoding.Identity { + return nil + } + + if !grpcutil.IsCompressorNameRegistered(name) { + return fmt.Errorf("compressor not registered %q", name) + } + + for _, c := range strings.Split(clientCompressors, ",") { + if c == name { + return nil // found match + } + } + return fmt.Errorf("client does not support compressor %q", name) +} diff --git a/server_test.go b/server_test.go index fcfde30706c3..0048a0f74156 100644 --- a/server_test.go +++ b/server_test.go @@ -22,17 +22,27 @@ import ( "context" "net" "reflect" + "strconv" "strings" "testing" "time" + "github.com/google/go-cmp/cmp" "google.golang.org/grpc/internal/transport" + "google.golang.org/grpc/status" ) -type emptyServiceServer interface{} +type emptyServiceServer any type testServer struct{} +func errorDesc(err error) string { + if s, ok := status.FromError(err); ok { + return s.Message() + } + return err.Error() +} + func (s) TestStopBeforeServe(t *testing.T) { lis, err := net.Listen("tcp", "localhost:0") if err != nil { @@ -121,6 +131,34 @@ func (s) TestGetServiceInfo(t *testing.T) { } } +func (s) TestRetryChainedInterceptor(t *testing.T) { + var records []int + i1 := func(ctx context.Context, req any, info *UnaryServerInfo, handler UnaryHandler) (resp any, err error) { + records = append(records, 1) + // call handler twice to simulate a retry here. + handler(ctx, req) + return handler(ctx, req) + } + i2 := func(ctx context.Context, req any, info *UnaryServerInfo, handler UnaryHandler) (resp any, err error) { + records = append(records, 2) + return handler(ctx, req) + } + i3 := func(ctx context.Context, req any, info *UnaryServerInfo, handler UnaryHandler) (resp any, err error) { + records = append(records, 3) + return handler(ctx, req) + } + + ii := chainUnaryInterceptors([]UnaryServerInterceptor{i1, i2, i3}) + + handler := func(ctx context.Context, req any) (any, error) { + return nil, nil + } + ii(context.Background(), nil, nil, handler) + if !cmp.Equal(records, []int{1, 2, 3, 2, 3}) { + t.Fatalf("retry failed on chained interceptors: %v", records) + } +} + func (s) TestStreamContext(t *testing.T) { expectedStream := &transport.Stream{} ctx := NewContextWithServerTransportStream(context.Background(), expectedStream) @@ -130,3 +168,59 @@ func (s) TestStreamContext(t *testing.T) { t.Fatalf("GetStreamFromContext(%v) = %v, %t, want: %v, true", ctx, stream, ok, expectedStream) } } + +func BenchmarkChainUnaryInterceptor(b *testing.B) { + for _, n := range []int{1, 3, 5, 10} { + n := n + b.Run(strconv.Itoa(n), func(b *testing.B) { + interceptors := make([]UnaryServerInterceptor, 0, n) + for i := 0; i < n; i++ { + interceptors = append(interceptors, func( + ctx context.Context, req any, info *UnaryServerInfo, handler UnaryHandler, + ) (any, error) { + return handler(ctx, req) + }) + } + + s := NewServer(ChainUnaryInterceptor(interceptors...)) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err := s.opts.unaryInt(context.Background(), nil, nil, + func(ctx context.Context, req any) (any, error) { + return nil, nil + }, + ); err != nil { + b.Fatal(err) + } + } + }) + } +} + +func BenchmarkChainStreamInterceptor(b *testing.B) { + for _, n := range []int{1, 3, 5, 10} { + n := n + b.Run(strconv.Itoa(n), func(b *testing.B) { + interceptors := make([]StreamServerInterceptor, 0, n) + for i := 0; i < n; i++ { + interceptors = append(interceptors, func( + srv any, ss ServerStream, info *StreamServerInfo, handler StreamHandler, + ) error { + return handler(srv, ss) + }) + } + + s := NewServer(ChainStreamInterceptor(interceptors...)) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if err := s.opts.streamInt(nil, nil, nil, func(srv any, stream ServerStream) error { + return nil + }); err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/service_config.go b/service_config.go index 5e434ca7f354..0df11fc09882 100644 --- a/service_config.go +++ b/service_config.go @@ -23,8 +23,6 @@ import ( "errors" "fmt" "reflect" - "strconv" - "strings" "time" "google.golang.org/grpc/codes" @@ -41,29 +39,7 @@ const maxInt = int(^uint(0) >> 1) // Deprecated: Users should not use this struct. Service config should be received // through name resolver, as specified here // https://github.com/grpc/grpc/blob/master/doc/service_config.md -type MethodConfig struct { - // WaitForReady indicates whether RPCs sent to this method should wait until - // the connection is ready by default (!failfast). The value specified via the - // gRPC client API will override the value set here. - WaitForReady *bool - // Timeout is the default timeout for RPCs sent to this method. The actual - // deadline used will be the minimum of the value specified here and the value - // set by the application via the gRPC client API. If either one is not set, - // then the other will be used. If neither is set, then the RPC has no deadline. - Timeout *time.Duration - // MaxReqSize is the maximum allowed payload size for an individual request in a - // stream (client->server) in bytes. The size which is measured is the serialized - // payload after per-message compression (but before stream compression) in bytes. - // The actual value used is the minimum of the value specified here and the value set - // by the application via the gRPC client API. If either one is not set, then the other - // will be used. If neither is set, then the built-in default is used. - MaxReqSize *int - // MaxRespSize is the maximum allowed payload size for an individual response in a - // stream (server->client) in bytes. - MaxRespSize *int - // RetryPolicy configures retry options for the method. - retryPolicy *retryPolicy -} +type MethodConfig = internalserviceconfig.MethodConfig type lbConfig struct { name string @@ -79,10 +55,9 @@ type lbConfig struct { type ServiceConfig struct { serviceconfig.Config - // LB is the load balancer the service providers recommends. The balancer - // specified via grpc.WithBalancerName will override this. This is deprecated; - // lbConfigs is preferred. If lbConfig and LB are both present, lbConfig - // will be used. + // LB is the load balancer the service providers recommends. This is + // deprecated; lbConfigs is preferred. If lbConfig and LB are both present, + // lbConfig will be used. LB *string // lbConfig is the service config's load balancing configuration. If @@ -127,38 +102,10 @@ type healthCheckConfig struct { ServiceName string } -// retryPolicy defines the go-native version of the retry policy defined by the -// service config here: -// https://github.com/grpc/proposal/blob/master/A6-client-retries.md#integration-with-service-config -type retryPolicy struct { - // MaxAttempts is the maximum number of attempts, including the original RPC. - // - // This field is required and must be two or greater. - maxAttempts int - - // Exponential backoff parameters. The initial retry attempt will occur at - // random(0, initialBackoff). In general, the nth attempt will occur at - // random(0, - // min(initialBackoff*backoffMultiplier**(n-1), maxBackoff)). - // - // These fields are required and must be greater than zero. - initialBackoff time.Duration - maxBackoff time.Duration - backoffMultiplier float64 - - // The set of status codes which may be retried. - // - // Status codes are specified as strings, e.g., "UNAVAILABLE". - // - // This field is required and must be non-empty. - // Note: a set is used to store this for easy lookup. - retryableStatusCodes map[codes.Code]bool -} - type jsonRetryPolicy struct { MaxAttempts int - InitialBackoff string - MaxBackoff string + InitialBackoff internalserviceconfig.Duration + MaxBackoff internalserviceconfig.Duration BackoffMultiplier float64 RetryableStatusCodes []codes.Code } @@ -180,50 +127,6 @@ type retryThrottlingPolicy struct { TokenRatio float64 } -func parseDuration(s *string) (*time.Duration, error) { - if s == nil { - return nil, nil - } - if !strings.HasSuffix(*s, "s") { - return nil, fmt.Errorf("malformed duration %q", *s) - } - ss := strings.SplitN((*s)[:len(*s)-1], ".", 3) - if len(ss) > 2 { - return nil, fmt.Errorf("malformed duration %q", *s) - } - // hasDigits is set if either the whole or fractional part of the number is - // present, since both are optional but one is required. - hasDigits := false - var d time.Duration - if len(ss[0]) > 0 { - i, err := strconv.ParseInt(ss[0], 10, 32) - if err != nil { - return nil, fmt.Errorf("malformed duration %q: %v", *s, err) - } - d = time.Duration(i) * time.Second - hasDigits = true - } - if len(ss) == 2 && len(ss[1]) > 0 { - if len(ss[1]) > 9 { - return nil, fmt.Errorf("malformed duration %q", *s) - } - f, err := strconv.ParseInt(ss[1], 10, 64) - if err != nil { - return nil, fmt.Errorf("malformed duration %q: %v", *s, err) - } - for i := 9; i > len(ss[1]); i-- { - f *= 10 - } - d += time.Duration(f) - hasDigits = true - } - if !hasDigits { - return nil, fmt.Errorf("malformed duration %q", *s) - } - - return &d, nil -} - type jsonName struct { Service string Method string @@ -252,7 +155,7 @@ func (j jsonName) generatePath() (string, error) { type jsonMC struct { Name *[]jsonName WaitForReady *bool - Timeout *string + Timeout *internalserviceconfig.Duration MaxRequestMessageBytes *int64 MaxResponseMessageBytes *int64 RetryPolicy *jsonRetryPolicy @@ -268,7 +171,7 @@ type jsonSC struct { } func init() { - internal.ParseServiceConfigForTesting = parseServiceConfig + internal.ParseServiceConfig = parseServiceConfig } func parseServiceConfig(js string) *serviceconfig.ParseResult { if len(js) == 0 { @@ -277,7 +180,7 @@ func parseServiceConfig(js string) *serviceconfig.ParseResult { var rsc jsonSC err := json.Unmarshal([]byte(js), &rsc) if err != nil { - logger.Warningf("grpc: parseServiceConfig error unmarshaling %s due to %v", js, err) + logger.Warningf("grpc: unmarshaling service config %s: %v", js, err) return &serviceconfig.ParseResult{Err: err} } sc := ServiceConfig{ @@ -303,18 +206,13 @@ func parseServiceConfig(js string) *serviceconfig.ParseResult { if m.Name == nil { continue } - d, err := parseDuration(m.Timeout) - if err != nil { - logger.Warningf("grpc: parseServiceConfig error unmarshaling %s due to %v", js, err) - return &serviceconfig.ParseResult{Err: err} - } mc := MethodConfig{ WaitForReady: m.WaitForReady, - Timeout: d, + Timeout: (*time.Duration)(m.Timeout), } - if mc.retryPolicy, err = convertRetryPolicy(m.RetryPolicy); err != nil { - logger.Warningf("grpc: parseServiceConfig error unmarshaling %s due to %v", js, err) + if mc.RetryPolicy, err = convertRetryPolicy(m.RetryPolicy); err != nil { + logger.Warningf("grpc: unmarshaling service config %s: %v", js, err) return &serviceconfig.ParseResult{Err: err} } if m.MaxRequestMessageBytes != nil { @@ -334,13 +232,13 @@ func parseServiceConfig(js string) *serviceconfig.ParseResult { for i, n := range *m.Name { path, err := n.generatePath() if err != nil { - logger.Warningf("grpc: parseServiceConfig error unmarshaling %s due to methodConfig[%d]: %v", js, i, err) + logger.Warningf("grpc: error unmarshaling service config %s due to methodConfig[%d]: %v", js, i, err) return &serviceconfig.ParseResult{Err: err} } if _, ok := paths[path]; ok { err = errDuplicatedName - logger.Warningf("grpc: parseServiceConfig error unmarshaling %s due to methodConfig[%d]: %v", js, i, err) + logger.Warningf("grpc: error unmarshaling service config %s due to methodConfig[%d]: %v", js, i, err) return &serviceconfig.ParseResult{Err: err} } paths[path] = struct{}{} @@ -359,41 +257,33 @@ func parseServiceConfig(js string) *serviceconfig.ParseResult { return &serviceconfig.ParseResult{Config: &sc} } -func convertRetryPolicy(jrp *jsonRetryPolicy) (p *retryPolicy, err error) { +func convertRetryPolicy(jrp *jsonRetryPolicy) (p *internalserviceconfig.RetryPolicy, err error) { if jrp == nil { return nil, nil } - ib, err := parseDuration(&jrp.InitialBackoff) - if err != nil { - return nil, err - } - mb, err := parseDuration(&jrp.MaxBackoff) - if err != nil { - return nil, err - } if jrp.MaxAttempts <= 1 || - *ib <= 0 || - *mb <= 0 || + jrp.InitialBackoff <= 0 || + jrp.MaxBackoff <= 0 || jrp.BackoffMultiplier <= 0 || len(jrp.RetryableStatusCodes) == 0 { logger.Warningf("grpc: ignoring retry policy %v due to illegal configuration", jrp) return nil, nil } - rp := &retryPolicy{ - maxAttempts: jrp.MaxAttempts, - initialBackoff: *ib, - maxBackoff: *mb, - backoffMultiplier: jrp.BackoffMultiplier, - retryableStatusCodes: make(map[codes.Code]bool), + rp := &internalserviceconfig.RetryPolicy{ + MaxAttempts: jrp.MaxAttempts, + InitialBackoff: time.Duration(jrp.InitialBackoff), + MaxBackoff: time.Duration(jrp.MaxBackoff), + BackoffMultiplier: jrp.BackoffMultiplier, + RetryableStatusCodes: make(map[codes.Code]bool), } - if rp.maxAttempts > 5 { + if rp.MaxAttempts > 5 { // TODO(retry): Make the max maxAttempts configurable. - rp.maxAttempts = 5 + rp.MaxAttempts = 5 } for _, code := range jrp.RetryableStatusCodes { - rp.retryableStatusCodes[code] = true + rp.RetryableStatusCodes[code] = true } return rp, nil } @@ -431,6 +321,9 @@ func init() { // // If any of them is NOT *ServiceConfig, return false. func equalServiceConfig(a, b serviceconfig.Config) bool { + if a == nil && b == nil { + return true + } aa, ok := a.(*ServiceConfig) if !ok { return false diff --git a/service_config_test.go b/service_config_test.go index b3c6988e8d97..90ed40a68021 100644 --- a/service_config_test.go +++ b/service_config_test.go @@ -20,8 +20,6 @@ package grpc import ( "encoding/json" - "fmt" - "math" "reflect" "testing" "time" @@ -449,55 +447,6 @@ func (s) TestParseMethodConfigDuplicatedName(t *testing.T) { }) } -func (s) TestParseDuration(t *testing.T) { - testCases := []struct { - s *string - want *time.Duration - err bool - }{ - {s: nil, want: nil}, - {s: newString("1s"), want: newDuration(time.Second)}, - {s: newString("-1s"), want: newDuration(-time.Second)}, - {s: newString("1.1s"), want: newDuration(1100 * time.Millisecond)}, - {s: newString("1.s"), want: newDuration(time.Second)}, - {s: newString("1.0s"), want: newDuration(time.Second)}, - {s: newString(".002s"), want: newDuration(2 * time.Millisecond)}, - {s: newString(".002000s"), want: newDuration(2 * time.Millisecond)}, - {s: newString("0.003s"), want: newDuration(3 * time.Millisecond)}, - {s: newString("0.000004s"), want: newDuration(4 * time.Microsecond)}, - {s: newString("5000.000000009s"), want: newDuration(5000*time.Second + 9*time.Nanosecond)}, - {s: newString("4999.999999999s"), want: newDuration(5000*time.Second - time.Nanosecond)}, - {s: newString("1"), err: true}, - {s: newString("s"), err: true}, - {s: newString(".s"), err: true}, - {s: newString("1 s"), err: true}, - {s: newString(" 1s"), err: true}, - {s: newString("1ms"), err: true}, - {s: newString("1.1.1s"), err: true}, - {s: newString("Xs"), err: true}, - {s: newString("as"), err: true}, - {s: newString(".0000000001s"), err: true}, - {s: newString(fmt.Sprint(math.MaxInt32) + "s"), want: newDuration(math.MaxInt32 * time.Second)}, - {s: newString(fmt.Sprint(int64(math.MaxInt32)+1) + "s"), err: true}, - } - for _, tc := range testCases { - got, err := parseDuration(tc.s) - if tc.err != (err != nil) || - (got == nil) != (tc.want == nil) || - (got != nil && *got != *tc.want) { - wantErr := "" - if tc.err { - wantErr = "" - } - s := "" - if tc.s != nil { - s = `&"` + *tc.s + `"` - } - t.Errorf("parseDuration(%v) = %v, %v; want %v, %v", s, got, err, tc.want, wantErr) - } - } -} - func newBool(b bool) *bool { return &b } diff --git a/serviceconfig/serviceconfig.go b/serviceconfig/serviceconfig.go index 187c304421cf..35e7a20a04ba 100644 --- a/serviceconfig/serviceconfig.go +++ b/serviceconfig/serviceconfig.go @@ -19,7 +19,10 @@ // Package serviceconfig defines types and methods for operating on gRPC // service configs. // -// This package is EXPERIMENTAL. +// # Experimental +// +// Notice: This package is EXPERIMENTAL and may be changed or removed in a +// later release. package serviceconfig // Config represents an opaque data structure holding a service config. diff --git a/shared_buffer_pool.go b/shared_buffer_pool.go new file mode 100644 index 000000000000..48a64cfe8e25 --- /dev/null +++ b/shared_buffer_pool.go @@ -0,0 +1,154 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import "sync" + +// SharedBufferPool is a pool of buffers that can be shared, resulting in +// decreased memory allocation. Currently, in gRPC-go, it is only utilized +// for parsing incoming messages. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. +type SharedBufferPool interface { + // Get returns a buffer with specified length from the pool. + // + // The returned byte slice may be not zero initialized. + Get(length int) []byte + + // Put returns a buffer to the pool. + Put(*[]byte) +} + +// NewSharedBufferPool creates a simple SharedBufferPool with buckets +// of different sizes to optimize memory usage. This prevents the pool from +// wasting large amounts of memory, even when handling messages of varying sizes. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. +func NewSharedBufferPool() SharedBufferPool { + return &simpleSharedBufferPool{ + pools: [poolArraySize]simpleSharedBufferChildPool{ + newBytesPool(level0PoolMaxSize), + newBytesPool(level1PoolMaxSize), + newBytesPool(level2PoolMaxSize), + newBytesPool(level3PoolMaxSize), + newBytesPool(level4PoolMaxSize), + newBytesPool(0), + }, + } +} + +// simpleSharedBufferPool is a simple implementation of SharedBufferPool. +type simpleSharedBufferPool struct { + pools [poolArraySize]simpleSharedBufferChildPool +} + +func (p *simpleSharedBufferPool) Get(size int) []byte { + return p.pools[p.poolIdx(size)].Get(size) +} + +func (p *simpleSharedBufferPool) Put(bs *[]byte) { + p.pools[p.poolIdx(cap(*bs))].Put(bs) +} + +func (p *simpleSharedBufferPool) poolIdx(size int) int { + switch { + case size <= level0PoolMaxSize: + return level0PoolIdx + case size <= level1PoolMaxSize: + return level1PoolIdx + case size <= level2PoolMaxSize: + return level2PoolIdx + case size <= level3PoolMaxSize: + return level3PoolIdx + case size <= level4PoolMaxSize: + return level4PoolIdx + default: + return levelMaxPoolIdx + } +} + +const ( + level0PoolMaxSize = 16 // 16 B + level1PoolMaxSize = level0PoolMaxSize * 16 // 256 B + level2PoolMaxSize = level1PoolMaxSize * 16 // 4 KB + level3PoolMaxSize = level2PoolMaxSize * 16 // 64 KB + level4PoolMaxSize = level3PoolMaxSize * 16 // 1 MB +) + +const ( + level0PoolIdx = iota + level1PoolIdx + level2PoolIdx + level3PoolIdx + level4PoolIdx + levelMaxPoolIdx + poolArraySize +) + +type simpleSharedBufferChildPool interface { + Get(size int) []byte + Put(any) +} + +type bufferPool struct { + sync.Pool + + defaultSize int +} + +func (p *bufferPool) Get(size int) []byte { + bs := p.Pool.Get().(*[]byte) + + if cap(*bs) < size { + p.Pool.Put(bs) + + return make([]byte, size) + } + + return (*bs)[:size] +} + +func newBytesPool(size int) simpleSharedBufferChildPool { + return &bufferPool{ + Pool: sync.Pool{ + New: func() any { + bs := make([]byte, size) + return &bs + }, + }, + defaultSize: size, + } +} + +// nopBufferPool is a buffer pool just makes new buffer without pooling. +type nopBufferPool struct { +} + +func (nopBufferPool) Get(length int) []byte { + return make([]byte, length) +} + +func (nopBufferPool) Put(*[]byte) { +} diff --git a/shared_buffer_pool_test.go b/shared_buffer_pool_test.go new file mode 100644 index 000000000000..f5ed7c8314f1 --- /dev/null +++ b/shared_buffer_pool_test.go @@ -0,0 +1,48 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc + +import "testing" + +func (s) TestSharedBufferPool(t *testing.T) { + pools := []SharedBufferPool{ + nopBufferPool{}, + NewSharedBufferPool(), + } + + lengths := []int{ + level4PoolMaxSize + 1, + level4PoolMaxSize, + level3PoolMaxSize, + level2PoolMaxSize, + level1PoolMaxSize, + level0PoolMaxSize, + } + + for _, p := range pools { + for _, l := range lengths { + bs := p.Get(l) + if len(bs) != l { + t.Fatalf("Expected buffer of length %d, got %d", l, len(bs)) + } + + p.Put(&bs) + } + } +} diff --git a/stats/grpc_testing/test.pb.go b/stats/grpc_testing/test.pb.go deleted file mode 100644 index df3796ebbd30..000000000000 --- a/stats/grpc_testing/test.pb.go +++ /dev/null @@ -1,125 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. -// source: stats/grpc_testing/test.proto - -package grpc_testing - -import ( - fmt "fmt" - proto "github.com/golang/protobuf/proto" - math "math" -) - -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package - -type SimpleRequest struct { - Id int32 `protobuf:"varint,2,opt,name=id,proto3" json:"id,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *SimpleRequest) Reset() { *m = SimpleRequest{} } -func (m *SimpleRequest) String() string { return proto.CompactTextString(m) } -func (*SimpleRequest) ProtoMessage() {} -func (*SimpleRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_d7a50f7a8e9e8e09, []int{0} -} - -func (m *SimpleRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_SimpleRequest.Unmarshal(m, b) -} -func (m *SimpleRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_SimpleRequest.Marshal(b, m, deterministic) -} -func (m *SimpleRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_SimpleRequest.Merge(m, src) -} -func (m *SimpleRequest) XXX_Size() int { - return xxx_messageInfo_SimpleRequest.Size(m) -} -func (m *SimpleRequest) XXX_DiscardUnknown() { - xxx_messageInfo_SimpleRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_SimpleRequest proto.InternalMessageInfo - -func (m *SimpleRequest) GetId() int32 { - if m != nil { - return m.Id - } - return 0 -} - -type SimpleResponse struct { - Id int32 `protobuf:"varint,3,opt,name=id,proto3" json:"id,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *SimpleResponse) Reset() { *m = SimpleResponse{} } -func (m *SimpleResponse) String() string { return proto.CompactTextString(m) } -func (*SimpleResponse) ProtoMessage() {} -func (*SimpleResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_d7a50f7a8e9e8e09, []int{1} -} - -func (m *SimpleResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_SimpleResponse.Unmarshal(m, b) -} -func (m *SimpleResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_SimpleResponse.Marshal(b, m, deterministic) -} -func (m *SimpleResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_SimpleResponse.Merge(m, src) -} -func (m *SimpleResponse) XXX_Size() int { - return xxx_messageInfo_SimpleResponse.Size(m) -} -func (m *SimpleResponse) XXX_DiscardUnknown() { - xxx_messageInfo_SimpleResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_SimpleResponse proto.InternalMessageInfo - -func (m *SimpleResponse) GetId() int32 { - if m != nil { - return m.Id - } - return 0 -} - -func init() { - proto.RegisterType((*SimpleRequest)(nil), "grpc.testing.SimpleRequest") - proto.RegisterType((*SimpleResponse)(nil), "grpc.testing.SimpleResponse") -} - -func init() { proto.RegisterFile("stats/grpc_testing/test.proto", fileDescriptor_d7a50f7a8e9e8e09) } - -var fileDescriptor_d7a50f7a8e9e8e09 = []byte{ - // 233 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x91, 0xcf, 0x4a, 0x03, 0x31, - 0x10, 0xc6, 0x49, 0x44, 0xc1, 0x51, 0x17, 0xc9, 0x49, 0xfc, 0x83, 0xa5, 0xa7, 0x8a, 0x90, 0x2d, - 0xfa, 0x06, 0x56, 0x7a, 0x2d, 0x74, 0xf5, 0xe2, 0x45, 0x62, 0x3b, 0x84, 0xc0, 0x34, 0x89, 0xc9, - 0xac, 0xe8, 0x1b, 0xfa, 0x58, 0x92, 0xb2, 0x0b, 0x8a, 0xde, 0xd6, 0xd3, 0x1c, 0xbe, 0x8f, 0xdf, - 0xc7, 0xf0, 0x83, 0x8b, 0xcc, 0x86, 0x73, 0x6d, 0x53, 0x5c, 0x3d, 0x33, 0x66, 0x76, 0xde, 0xd6, - 0xe5, 0xea, 0x98, 0x02, 0x07, 0x75, 0x58, 0x02, 0xdd, 0x05, 0xe3, 0x4b, 0x38, 0x6a, 0xdc, 0x26, - 0x12, 0x2e, 0xf1, 0xb5, 0xc5, 0xcc, 0xaa, 0x02, 0xe9, 0xd6, 0x27, 0x72, 0x24, 0x26, 0xbb, 0x4b, - 0xe9, 0xd6, 0xe3, 0x11, 0x54, 0x7d, 0x21, 0xc7, 0xe0, 0x33, 0x76, 0x8d, 0x9d, 0xbe, 0x71, 0xf3, - 0x29, 0xe1, 0xe0, 0x01, 0x33, 0x37, 0x98, 0xde, 0xdc, 0x0a, 0xd5, 0x1c, 0xf6, 0x1f, 0xbd, 0x49, - 0x1f, 0x33, 0x43, 0xa4, 0xce, 0xf4, 0xf7, 0x39, 0xfd, 0x63, 0xeb, 0xf4, 0xfc, 0xef, 0xb0, 0xdb, - 0x59, 0x40, 0x35, 0x6f, 0x89, 0xee, 0xdb, 0x48, 0xf8, 0x3e, 0x10, 0x36, 0x11, 0x53, 0xa1, 0x16, - 0x70, 0x3c, 0x23, 0x87, 0x9e, 0x1b, 0x4e, 0x68, 0x36, 0x83, 0x91, 0x05, 0x58, 0x9e, 0xc6, 0xf4, - 0x2f, 0xc0, 0xa9, 0xb8, 0xbb, 0x7e, 0xba, 0xb2, 0x21, 0x58, 0x42, 0x6d, 0x03, 0x19, 0x6f, 0x75, - 0x48, 0x76, 0x2b, 0xb2, 0xfe, 0xed, 0xf4, 0x65, 0x6f, 0xeb, 0xf3, 0xf6, 0x2b, 0x00, 0x00, 0xff, - 0xff, 0x7c, 0x26, 0xce, 0x3c, 0xf0, 0x01, 0x00, 0x00, -} diff --git a/stats/grpc_testing/test.proto b/stats/grpc_testing/test.proto deleted file mode 100644 index 2716363260a6..000000000000 --- a/stats/grpc_testing/test.proto +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2017 gRPC authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto3"; - -option go_package = "google.golang.org/grpc/stats/grpc_testing"; - -package grpc.testing; - -message SimpleRequest { - int32 id = 2; -} - -message SimpleResponse { - int32 id = 3; -} - -// A simple test service. -service TestService { - // One request followed by one response. - // The server returns the client id as-is. - rpc UnaryCall(SimpleRequest) returns (SimpleResponse); - - // A sequence of requests with each request served by the server immediately. - // As one request could lead to multiple responses, this interface - // demonstrates the idea of full duplexing. - rpc FullDuplexCall(stream SimpleRequest) returns (stream SimpleResponse); - - // Client stream - rpc ClientStreamCall(stream SimpleRequest) returns (SimpleResponse); - - // Server stream - rpc ServerStreamCall(SimpleRequest) returns (stream SimpleResponse); -} diff --git a/stats/grpc_testing/test_grpc.pb.go b/stats/grpc_testing/test_grpc.pb.go deleted file mode 100644 index 1fff30c631d6..000000000000 --- a/stats/grpc_testing/test_grpc.pb.go +++ /dev/null @@ -1,366 +0,0 @@ -// Code generated by protoc-gen-go-grpc. DO NOT EDIT. - -package grpc_testing - -import ( - context "context" - grpc "google.golang.org/grpc" - codes "google.golang.org/grpc/codes" - status "google.golang.org/grpc/status" -) - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the grpc package it is being compiled against. -const _ = grpc.SupportPackageIsVersion7 - -// TestServiceClient is the client API for TestService service. -// -// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. -type TestServiceClient interface { - // One request followed by one response. - // The server returns the client id as-is. - UnaryCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (*SimpleResponse, error) - // A sequence of requests with each request served by the server immediately. - // As one request could lead to multiple responses, this interface - // demonstrates the idea of full duplexing. - FullDuplexCall(ctx context.Context, opts ...grpc.CallOption) (TestService_FullDuplexCallClient, error) - // Client stream - ClientStreamCall(ctx context.Context, opts ...grpc.CallOption) (TestService_ClientStreamCallClient, error) - // Server stream - ServerStreamCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (TestService_ServerStreamCallClient, error) -} - -type testServiceClient struct { - cc grpc.ClientConnInterface -} - -func NewTestServiceClient(cc grpc.ClientConnInterface) TestServiceClient { - return &testServiceClient{cc} -} - -var testServiceUnaryCallStreamDesc = &grpc.StreamDesc{ - StreamName: "UnaryCall", -} - -func (c *testServiceClient) UnaryCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (*SimpleResponse, error) { - out := new(SimpleResponse) - err := c.cc.Invoke(ctx, "/grpc.testing.TestService/UnaryCall", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -var testServiceFullDuplexCallStreamDesc = &grpc.StreamDesc{ - StreamName: "FullDuplexCall", - ServerStreams: true, - ClientStreams: true, -} - -func (c *testServiceClient) FullDuplexCall(ctx context.Context, opts ...grpc.CallOption) (TestService_FullDuplexCallClient, error) { - stream, err := c.cc.NewStream(ctx, testServiceFullDuplexCallStreamDesc, "/grpc.testing.TestService/FullDuplexCall", opts...) - if err != nil { - return nil, err - } - x := &testServiceFullDuplexCallClient{stream} - return x, nil -} - -type TestService_FullDuplexCallClient interface { - Send(*SimpleRequest) error - Recv() (*SimpleResponse, error) - grpc.ClientStream -} - -type testServiceFullDuplexCallClient struct { - grpc.ClientStream -} - -func (x *testServiceFullDuplexCallClient) Send(m *SimpleRequest) error { - return x.ClientStream.SendMsg(m) -} - -func (x *testServiceFullDuplexCallClient) Recv() (*SimpleResponse, error) { - m := new(SimpleResponse) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -var testServiceClientStreamCallStreamDesc = &grpc.StreamDesc{ - StreamName: "ClientStreamCall", - ClientStreams: true, -} - -func (c *testServiceClient) ClientStreamCall(ctx context.Context, opts ...grpc.CallOption) (TestService_ClientStreamCallClient, error) { - stream, err := c.cc.NewStream(ctx, testServiceClientStreamCallStreamDesc, "/grpc.testing.TestService/ClientStreamCall", opts...) - if err != nil { - return nil, err - } - x := &testServiceClientStreamCallClient{stream} - return x, nil -} - -type TestService_ClientStreamCallClient interface { - Send(*SimpleRequest) error - CloseAndRecv() (*SimpleResponse, error) - grpc.ClientStream -} - -type testServiceClientStreamCallClient struct { - grpc.ClientStream -} - -func (x *testServiceClientStreamCallClient) Send(m *SimpleRequest) error { - return x.ClientStream.SendMsg(m) -} - -func (x *testServiceClientStreamCallClient) CloseAndRecv() (*SimpleResponse, error) { - if err := x.ClientStream.CloseSend(); err != nil { - return nil, err - } - m := new(SimpleResponse) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -var testServiceServerStreamCallStreamDesc = &grpc.StreamDesc{ - StreamName: "ServerStreamCall", - ServerStreams: true, -} - -func (c *testServiceClient) ServerStreamCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (TestService_ServerStreamCallClient, error) { - stream, err := c.cc.NewStream(ctx, testServiceServerStreamCallStreamDesc, "/grpc.testing.TestService/ServerStreamCall", opts...) - if err != nil { - return nil, err - } - x := &testServiceServerStreamCallClient{stream} - if err := x.ClientStream.SendMsg(in); err != nil { - return nil, err - } - if err := x.ClientStream.CloseSend(); err != nil { - return nil, err - } - return x, nil -} - -type TestService_ServerStreamCallClient interface { - Recv() (*SimpleResponse, error) - grpc.ClientStream -} - -type testServiceServerStreamCallClient struct { - grpc.ClientStream -} - -func (x *testServiceServerStreamCallClient) Recv() (*SimpleResponse, error) { - m := new(SimpleResponse) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -// TestServiceService is the service API for TestService service. -// Fields should be assigned to their respective handler implementations only before -// RegisterTestServiceService is called. Any unassigned fields will result in the -// handler for that method returning an Unimplemented error. -type TestServiceService struct { - // One request followed by one response. - // The server returns the client id as-is. - UnaryCall func(context.Context, *SimpleRequest) (*SimpleResponse, error) - // A sequence of requests with each request served by the server immediately. - // As one request could lead to multiple responses, this interface - // demonstrates the idea of full duplexing. - FullDuplexCall func(TestService_FullDuplexCallServer) error - // Client stream - ClientStreamCall func(TestService_ClientStreamCallServer) error - // Server stream - ServerStreamCall func(*SimpleRequest, TestService_ServerStreamCallServer) error -} - -func (s *TestServiceService) unaryCall(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - if s.UnaryCall == nil { - return nil, status.Errorf(codes.Unimplemented, "method UnaryCall not implemented") - } - in := new(SimpleRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return s.UnaryCall(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: s, - FullMethod: "/grpc.testing.TestService/UnaryCall", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return s.UnaryCall(ctx, req.(*SimpleRequest)) - } - return interceptor(ctx, in, info, handler) -} -func (s *TestServiceService) fullDuplexCall(_ interface{}, stream grpc.ServerStream) error { - if s.FullDuplexCall == nil { - return status.Errorf(codes.Unimplemented, "method FullDuplexCall not implemented") - } - return s.FullDuplexCall(&testServiceFullDuplexCallServer{stream}) -} -func (s *TestServiceService) clientStreamCall(_ interface{}, stream grpc.ServerStream) error { - if s.ClientStreamCall == nil { - return status.Errorf(codes.Unimplemented, "method ClientStreamCall not implemented") - } - return s.ClientStreamCall(&testServiceClientStreamCallServer{stream}) -} -func (s *TestServiceService) serverStreamCall(_ interface{}, stream grpc.ServerStream) error { - if s.ServerStreamCall == nil { - return status.Errorf(codes.Unimplemented, "method ServerStreamCall not implemented") - } - m := new(SimpleRequest) - if err := stream.RecvMsg(m); err != nil { - return err - } - return s.ServerStreamCall(m, &testServiceServerStreamCallServer{stream}) -} - -type TestService_FullDuplexCallServer interface { - Send(*SimpleResponse) error - Recv() (*SimpleRequest, error) - grpc.ServerStream -} - -type testServiceFullDuplexCallServer struct { - grpc.ServerStream -} - -func (x *testServiceFullDuplexCallServer) Send(m *SimpleResponse) error { - return x.ServerStream.SendMsg(m) -} - -func (x *testServiceFullDuplexCallServer) Recv() (*SimpleRequest, error) { - m := new(SimpleRequest) - if err := x.ServerStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -type TestService_ClientStreamCallServer interface { - SendAndClose(*SimpleResponse) error - Recv() (*SimpleRequest, error) - grpc.ServerStream -} - -type testServiceClientStreamCallServer struct { - grpc.ServerStream -} - -func (x *testServiceClientStreamCallServer) SendAndClose(m *SimpleResponse) error { - return x.ServerStream.SendMsg(m) -} - -func (x *testServiceClientStreamCallServer) Recv() (*SimpleRequest, error) { - m := new(SimpleRequest) - if err := x.ServerStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -type TestService_ServerStreamCallServer interface { - Send(*SimpleResponse) error - grpc.ServerStream -} - -type testServiceServerStreamCallServer struct { - grpc.ServerStream -} - -func (x *testServiceServerStreamCallServer) Send(m *SimpleResponse) error { - return x.ServerStream.SendMsg(m) -} - -// RegisterTestServiceService registers a service implementation with a gRPC server. -func RegisterTestServiceService(s grpc.ServiceRegistrar, srv *TestServiceService) { - sd := grpc.ServiceDesc{ - ServiceName: "grpc.testing.TestService", - Methods: []grpc.MethodDesc{ - { - MethodName: "UnaryCall", - Handler: srv.unaryCall, - }, - }, - Streams: []grpc.StreamDesc{ - { - StreamName: "FullDuplexCall", - Handler: srv.fullDuplexCall, - ServerStreams: true, - ClientStreams: true, - }, - { - StreamName: "ClientStreamCall", - Handler: srv.clientStreamCall, - ClientStreams: true, - }, - { - StreamName: "ServerStreamCall", - Handler: srv.serverStreamCall, - ServerStreams: true, - }, - }, - Metadata: "stats/grpc_testing/test.proto", - } - - s.RegisterService(&sd, nil) -} - -// NewTestServiceService creates a new TestServiceService containing the -// implemented methods of the TestService service in s. Any unimplemented -// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client. -// This includes situations where the method handler is misspelled or has the wrong -// signature. For this reason, this function should be used with great care and -// is not recommended to be used by most users. -func NewTestServiceService(s interface{}) *TestServiceService { - ns := &TestServiceService{} - if h, ok := s.(interface { - UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error) - }); ok { - ns.UnaryCall = h.UnaryCall - } - if h, ok := s.(interface { - FullDuplexCall(TestService_FullDuplexCallServer) error - }); ok { - ns.FullDuplexCall = h.FullDuplexCall - } - if h, ok := s.(interface { - ClientStreamCall(TestService_ClientStreamCallServer) error - }); ok { - ns.ClientStreamCall = h.ClientStreamCall - } - if h, ok := s.(interface { - ServerStreamCall(*SimpleRequest, TestService_ServerStreamCallServer) error - }); ok { - ns.ServerStreamCall = h.ServerStreamCall - } - return ns -} - -// UnstableTestServiceService is the service API for TestService service. -// New methods may be added to this interface if they are added to the service -// definition, which is not a backward-compatible change. For this reason, -// use of this type is not recommended. -type UnstableTestServiceService interface { - // One request followed by one response. - // The server returns the client id as-is. - UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error) - // A sequence of requests with each request served by the server immediately. - // As one request could lead to multiple responses, this interface - // demonstrates the idea of full duplexing. - FullDuplexCall(TestService_FullDuplexCallServer) error - // Client stream - ClientStreamCall(TestService_ClientStreamCallServer) error - // Server stream - ServerStreamCall(*SimpleRequest, TestService_ServerStreamCallServer) error -} diff --git a/stats/opencensus/client_metrics.go b/stats/opencensus/client_metrics.go new file mode 100644 index 000000000000..4d45845f8c56 --- /dev/null +++ b/stats/opencensus/client_metrics.go @@ -0,0 +1,150 @@ +/* + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opencensus + +import ( + "go.opencensus.io/stats" + "go.opencensus.io/stats/view" + "go.opencensus.io/tag" +) + +var ( + keyClientMethod = tag.MustNewKey("grpc_client_method") + keyClientStatus = tag.MustNewKey("grpc_client_status") +) + +// Measures, which are recorded by client stats handler: Note that due to the +// nature of how stats handlers are called on gRPC's client side, the per rpc +// unit is actually per attempt throughout this definition file. +var ( + clientSentMessagesPerRPC = stats.Int64("grpc.io/client/sent_messages_per_rpc", "Number of messages sent in the RPC (always 1 for non-streaming RPCs).", stats.UnitDimensionless) + clientSentBytesPerRPC = stats.Int64("grpc.io/client/sent_bytes_per_rpc", "Total bytes sent across all request messages per RPC.", stats.UnitBytes) + clientSentCompressedBytesPerRPC = stats.Int64("grpc.io/client/sent_compressed_message_bytes_per_rpc", "Total compressed bytes sent across all request messages per RPC.", stats.UnitBytes) + clientReceivedMessagesPerRPC = stats.Int64("grpc.io/client/received_messages_per_rpc", "Number of response messages received per RPC (always 1 for non-streaming RPCs).", stats.UnitDimensionless) + clientReceivedBytesPerRPC = stats.Int64("grpc.io/client/received_bytes_per_rpc", "Total bytes received across all response messages per RPC.", stats.UnitBytes) + clientReceivedCompressedBytesPerRPC = stats.Int64("grpc.io/client/received_compressed_message_bytes_per_rpc", "Total compressed bytes received across all response messages per RPC.", stats.UnitBytes) + clientRoundtripLatency = stats.Float64("grpc.io/client/roundtrip_latency", "Time between first byte of request sent to last byte of response received, or terminal error.", stats.UnitMilliseconds) + clientStartedRPCs = stats.Int64("grpc.io/client/started_rpcs", "The total number of client RPCs ever opened, including those that have not completed.", stats.UnitDimensionless) + clientServerLatency = stats.Float64("grpc.io/client/server_latency", `Propagated from the server and should have the same value as "grpc.io/server/latency".`, stats.UnitMilliseconds) + // Per call measure: + clientAPILatency = stats.Float64("grpc.io/client/api_latency", "The end-to-end time the gRPC library takes to complete an RPC from the application’s perspective", stats.UnitMilliseconds) +) + +var ( + // ClientSentMessagesPerRPCView is the distribution of sent messages per + // RPC, keyed on method. + ClientSentMessagesPerRPCView = &view.View{ + Measure: clientSentMessagesPerRPC, + Name: "grpc.io/client/sent_messages_per_rpc", + Description: "Distribution of sent messages per RPC, by method.", + TagKeys: []tag.Key{keyClientMethod}, + Aggregation: countDistribution, + } + // ClientReceivedMessagesPerRPCView is the distribution of received messages + // per RPC, keyed on method. + ClientReceivedMessagesPerRPCView = &view.View{ + Measure: clientReceivedMessagesPerRPC, + Name: "grpc.io/client/received_messages_per_rpc", + Description: "Distribution of received messages per RPC, by method.", + TagKeys: []tag.Key{keyClientMethod}, + Aggregation: countDistribution, + } + // ClientSentBytesPerRPCView is the distribution of sent bytes per RPC, + // keyed on method. + ClientSentBytesPerRPCView = &view.View{ + Measure: clientSentBytesPerRPC, + Name: "grpc.io/client/sent_bytes_per_rpc", + Description: "Distribution of sent bytes per RPC, by method.", + TagKeys: []tag.Key{keyClientMethod}, + Aggregation: bytesDistribution, + } + // ClientSentCompressedMessageBytesPerRPCView is the distribution of + // compressed sent message bytes per RPC, keyed on method. + ClientSentCompressedMessageBytesPerRPCView = &view.View{ + Measure: clientSentCompressedBytesPerRPC, + Name: "grpc.io/client/sent_compressed_message_bytes_per_rpc", + Description: "Distribution of sent compressed message bytes per RPC, by method.", + TagKeys: []tag.Key{keyClientMethod}, + Aggregation: bytesDistribution, + } + // ClientReceivedBytesPerRPCView is the distribution of received bytes per + // RPC, keyed on method. + ClientReceivedBytesPerRPCView = &view.View{ + Measure: clientReceivedBytesPerRPC, + Name: "grpc.io/client/received_bytes_per_rpc", + Description: "Distribution of received bytes per RPC, by method.", + TagKeys: []tag.Key{keyClientMethod}, + Aggregation: bytesDistribution, + } + // ClientReceivedCompressedMessageBytesPerRPCView is the distribution of + // compressed received message bytes per RPC, keyed on method. + ClientReceivedCompressedMessageBytesPerRPCView = &view.View{ + Measure: clientReceivedCompressedBytesPerRPC, + Name: "grpc.io/client/received_compressed_message_bytes_per_rpc", + Description: "Distribution of received compressed message bytes per RPC, by method.", + TagKeys: []tag.Key{keyClientMethod}, + Aggregation: bytesDistribution, + } + // ClientStartedRPCsView is the count of opened RPCs, keyed on method. + ClientStartedRPCsView = &view.View{ + Measure: clientStartedRPCs, + Name: "grpc.io/client/started_rpcs", + Description: "Number of opened client RPCs, by method.", + TagKeys: []tag.Key{keyClientMethod}, + Aggregation: view.Count(), + } + // ClientCompletedRPCsView is the count of completed RPCs, keyed on method + // and status. + ClientCompletedRPCsView = &view.View{ + Measure: clientRoundtripLatency, + Name: "grpc.io/client/completed_rpcs", + Description: "Number of completed RPCs by method and status.", + TagKeys: []tag.Key{keyClientMethod, keyClientStatus}, + Aggregation: view.Count(), + } + // ClientRoundtripLatencyView is the distribution of round-trip latency in + // milliseconds per RPC, keyed on method. + ClientRoundtripLatencyView = &view.View{ + Measure: clientRoundtripLatency, + Name: "grpc.io/client/roundtrip_latency", + Description: "Distribution of round-trip latency, by method.", + TagKeys: []tag.Key{keyClientMethod}, + Aggregation: millisecondsDistribution, + } + + // The following metric is per call: + + // ClientAPILatencyView is the distribution of client api latency for the + // full RPC call, keyed on method and status. + ClientAPILatencyView = &view.View{ + Measure: clientAPILatency, + Name: "grpc.io/client/api_latency", + Description: "Distribution of client api latency, by method and status", + TagKeys: []tag.Key{keyClientMethod, keyClientStatus}, + Aggregation: millisecondsDistribution, + } +) + +// DefaultClientViews is the set of client views which are considered the +// minimum required to monitor client side performance. +var DefaultClientViews = []*view.View{ + ClientSentBytesPerRPCView, + ClientReceivedBytesPerRPCView, + ClientRoundtripLatencyView, + ClientCompletedRPCsView, + ClientStartedRPCsView, +} diff --git a/stats/opencensus/e2e_test.go b/stats/opencensus/e2e_test.go new file mode 100644 index 000000000000..3b3c2bbbd908 --- /dev/null +++ b/stats/opencensus/e2e_test.go @@ -0,0 +1,1619 @@ +/* + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opencensus + +import ( + "context" + "errors" + "fmt" + "io" + "reflect" + "sort" + "sync" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "go.opencensus.io/stats/view" + "go.opencensus.io/tag" + "go.opencensus.io/trace" + + "google.golang.org/grpc" + "google.golang.org/grpc/encoding/gzip" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/leakcheck" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func init() { + // OpenCensus, once included in binary, will spawn a global goroutine + // recorder that is not controllable by application. + // https://github.com/census-instrumentation/opencensus-go/issues/1191 + leakcheck.RegisterIgnoreGoroutine("go.opencensus.io/stats/view.(*worker).start") +} + +var defaultTestTimeout = 5 * time.Second + +type fakeExporter struct { + t *testing.T + + mu sync.RWMutex + seenViews map[string]*viewInformation + seenSpans []spanInformation +} + +// viewInformation is information Exported from the view package through +// ExportView relevant to testing, i.e. a reasonably non flaky expectation of +// desired emissions to Exporter. +type viewInformation struct { + aggType view.AggType + aggBuckets []float64 + desc string + tagKeys []tag.Key + rows []*view.Row +} + +func (fe *fakeExporter) ExportView(vd *view.Data) { + fe.mu.Lock() + defer fe.mu.Unlock() + fe.seenViews[vd.View.Name] = &viewInformation{ + aggType: vd.View.Aggregation.Type, + aggBuckets: vd.View.Aggregation.Buckets, + desc: vd.View.Description, + tagKeys: vd.View.TagKeys, + rows: vd.Rows, + } +} + +// compareRows compares rows with respect to the information desired to test. +// Both the tags representing the rows and also the data of the row are tested +// for equality. Rows are in nondeterministic order when ExportView is called, +// but handled inside this function by sorting. +func compareRows(rows []*view.Row, rows2 []*view.Row) bool { + if len(rows) != len(rows2) { + return false + } + // Sort both rows according to the same rule. This is to take away non + // determinism in the row ordering passed to the Exporter, while keeping the + // row data. + sort.Slice(rows, func(i, j int) bool { + return rows[i].String() > rows[j].String() + }) + + sort.Slice(rows2, func(i, j int) bool { + return rows2[i].String() > rows2[j].String() + }) + + for i, row := range rows { + if !cmp.Equal(row.Tags, rows2[i].Tags, cmp.Comparer(func(a tag.Key, b tag.Key) bool { + return a.Name() == b.Name() + })) { + return false + } + if !compareData(row.Data, rows2[i].Data) { + return false + } + } + return true +} + +// compareData returns whether the two aggregation data's are equal to each +// other with respect to parts of the data desired for correct emission. The +// function first makes sure the two types of aggregation data are the same, and +// then checks the equality for the respective aggregation data type. +func compareData(ad view.AggregationData, ad2 view.AggregationData) bool { + if ad == nil && ad2 == nil { + return true + } + if ad == nil || ad2 == nil { + return false + } + if reflect.TypeOf(ad) != reflect.TypeOf(ad2) { + return false + } + switch ad1 := ad.(type) { + case *view.DistributionData: + dd2 := ad2.(*view.DistributionData) + // Count and Count Per Buckets are reasonable for correctness, + // especially since we verify equality of bucket endpoints elsewhere. + if ad1.Count != dd2.Count { + return false + } + for i, count := range ad1.CountPerBucket { + if count != dd2.CountPerBucket[i] { + return false + } + } + case *view.CountData: + cd2 := ad2.(*view.CountData) + return ad1.Value == cd2.Value + + // gRPC open census plugin does not have these next two types of aggregation + // data types present, for now just check for type equality between the two + // aggregation data points (done above). + // case *view.SumData + // case *view.LastValueData: + } + return true +} + +func (vi *viewInformation) Equal(vi2 *viewInformation) bool { + if vi == nil && vi2 == nil { + return true + } + if vi == nil || vi2 == nil { + return false + } + if vi.aggType != vi2.aggType { + return false + } + if !cmp.Equal(vi.aggBuckets, vi2.aggBuckets) { + return false + } + if vi.desc != vi2.desc { + return false + } + if !cmp.Equal(vi.tagKeys, vi2.tagKeys, cmp.Comparer(func(a tag.Key, b tag.Key) bool { + return a.Name() == b.Name() + })) { + return false + } + if !compareRows(vi.rows, vi2.rows) { + return false + } + return true +} + +// distributionDataLatencyCount checks if the view information contains the +// desired distrubtion latency total count that falls in buckets of 5 seconds or +// less. This must be called with non nil view information that is aggregated +// with distribution data. Returns a nil error if correct count information +// found, non nil error if correct information not found. +func distributionDataLatencyCount(vi *viewInformation, countWant int64, wantTags [][]tag.Tag) error { + var totalCount int64 + var largestIndexWithFive int + for i, bucket := range vi.aggBuckets { + // Distribution for latency is measured in milliseconds, so 5 * 1000 = + // 5000. + if bucket > 5000 { + largestIndexWithFive = i + break + } + } + // Sort rows by string name. This is to take away non determinism in the row + // ordering passed to the Exporter, while keeping the row data. + sort.Slice(vi.rows, func(i, j int) bool { + return vi.rows[i].String() > vi.rows[j].String() + }) + // Iterating through rows sums up data points for all methods. In this case, + // a data point for the unary and for the streaming RPC. + for i, row := range vi.rows { + // The method names corresponding to unary and streaming call should + // have the leading slash removed. + if diff := cmp.Diff(row.Tags, wantTags[i], cmp.Comparer(func(a tag.Key, b tag.Key) bool { + return a.Name() == b.Name() + })); diff != "" { + return fmt.Errorf("wrong tag keys for unary method -got, +want: %v", diff) + } + // This could potentially have an extra measurement in buckets above 5s, + // but that's fine. Count of buckets that could contain up to 5s is a + // good enough assertion. + for i, count := range row.Data.(*view.DistributionData).CountPerBucket { + if i >= largestIndexWithFive { + break + } + totalCount = totalCount + count + } + } + if totalCount != countWant { + return fmt.Errorf("wrong total count for counts under 5: %v, wantCount: %v", totalCount, countWant) + } + return nil +} + +// waitForServerCompletedRPCs waits until both Unary and Streaming metric rows +// appear, in two separate rows, for server completed RPC's view. Returns an +// error if the Unary and Streaming metric are not found within the passed +// context's timeout. +func waitForServerCompletedRPCs(ctx context.Context) error { + for ; ctx.Err() == nil; <-time.After(time.Millisecond) { + rows, err := view.RetrieveData("grpc.io/server/completed_rpcs") + if err != nil { + continue + } + unaryFound := false + streamingFound := false + for _, row := range rows { + for _, tag := range row.Tags { + if tag.Value == "grpc.testing.TestService/UnaryCall" { + unaryFound = true + break + } else if tag.Value == "grpc.testing.TestService/FullDuplexCall" { + streamingFound = true + break + } + } + if unaryFound && streamingFound { + return nil + } + } + } + return fmt.Errorf("timeout when waiting for Unary and Streaming rows to be present for \"grpc.io/server/completed_rpcs\"") +} + +// TestAllMetricsOneFunction tests emitted metrics from gRPC. It registers all +// the metrics provided by this package. It then configures a system with a gRPC +// Client and gRPC server with the OpenCensus Dial and Server Option configured, +// and makes a Unary RPC and a Streaming RPC. These two RPCs should cause +// certain emissions for each registered metric through the OpenCensus View +// package. +func (s) TestAllMetricsOneFunction(t *testing.T) { + allViews := []*view.View{ + ClientStartedRPCsView, + ServerStartedRPCsView, + ClientCompletedRPCsView, + ServerCompletedRPCsView, + ClientSentBytesPerRPCView, + ClientSentCompressedMessageBytesPerRPCView, + ServerSentBytesPerRPCView, + ServerSentCompressedMessageBytesPerRPCView, + ClientReceivedBytesPerRPCView, + ClientReceivedCompressedMessageBytesPerRPCView, + ServerReceivedBytesPerRPCView, + ServerReceivedCompressedMessageBytesPerRPCView, + ClientSentMessagesPerRPCView, + ServerSentMessagesPerRPCView, + ClientReceivedMessagesPerRPCView, + ServerReceivedMessagesPerRPCView, + ClientRoundtripLatencyView, + ServerLatencyView, + ClientAPILatencyView, + } + view.Register(allViews...) + // Unregister unconditionally in this defer to correctly cleanup globals in + // error conditions. + defer view.Unregister(allViews...) + fe := &fakeExporter{ + t: t, + seenViews: make(map[string]*viewInformation), + } + view.RegisterExporter(fe) + defer view.UnregisterExporter(fe) + + ss := &stubserver.StubServer{ + UnaryCallF: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{Payload: &testpb.Payload{ + Body: make([]byte, 10000), + }}, nil + }, + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + for { + _, err := stream.Recv() + if err == io.EOF { + return nil + } + } + }, + } + if err := ss.Start([]grpc.ServerOption{ServerOption(TraceOptions{})}, DialOption(TraceOptions{})); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + // Make two RPC's, a unary RPC and a streaming RPC. These should cause + // certain metrics to be emitted. + if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{Payload: &testpb.Payload{ + Body: make([]byte, 10000), + }}, grpc.UseCompressor(gzip.Name)); err != nil { + t.Fatalf("Unexpected error from UnaryCall: %v", err) + } + stream, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("ss.Client.FullDuplexCall failed: %f", err) + } + + stream.CloseSend() + if _, err = stream.Recv(); err != io.EOF { + t.Fatalf("unexpected error: %v, expected an EOF error", err) + } + + cmtk := tag.MustNewKey("grpc_client_method") + smtk := tag.MustNewKey("grpc_server_method") + cstk := tag.MustNewKey("grpc_client_status") + sstk := tag.MustNewKey("grpc_server_status") + wantMetrics := []struct { + metric *view.View + wantVI *viewInformation + wantTags [][]tag.Tag // for non determinstic (i.e. latency) metrics. First dimension represents rows. + }{ + { + metric: ClientStartedRPCsView, + wantVI: &viewInformation{ + aggType: view.AggTypeCount, + aggBuckets: []float64{}, + desc: "Number of opened client RPCs, by method.", + tagKeys: []tag.Key{ + cmtk, + }, + + rows: []*view.Row{ + { + Tags: []tag.Tag{ + { + Key: cmtk, + Value: "grpc.testing.TestService/UnaryCall", + }, + }, + Data: &view.CountData{ + Value: 1, + }, + }, + { + Tags: []tag.Tag{ + { + Key: cmtk, + Value: "grpc.testing.TestService/FullDuplexCall", + }, + }, + Data: &view.CountData{ + Value: 1, + }, + }, + }, + }, + }, + { + metric: ServerStartedRPCsView, + wantVI: &viewInformation{ + aggType: view.AggTypeCount, + aggBuckets: []float64{}, + desc: "Number of opened server RPCs, by method.", + tagKeys: []tag.Key{ + smtk, + }, + rows: []*view.Row{ + { + Tags: []tag.Tag{ + { + Key: smtk, + Value: "grpc.testing.TestService/UnaryCall", + }, + }, + Data: &view.CountData{ + Value: 1, + }, + }, + { + Tags: []tag.Tag{ + { + Key: smtk, + Value: "grpc.testing.TestService/FullDuplexCall", + }, + }, + Data: &view.CountData{ + Value: 1, + }, + }, + }, + }, + }, + { + metric: ClientCompletedRPCsView, + wantVI: &viewInformation{ + aggType: view.AggTypeCount, + aggBuckets: []float64{}, + desc: "Number of completed RPCs by method and status.", + tagKeys: []tag.Key{ + cmtk, + cstk, + }, + rows: []*view.Row{ + { + Tags: []tag.Tag{ + { + Key: cmtk, + Value: "grpc.testing.TestService/UnaryCall", + }, + { + Key: cstk, + Value: "OK", + }, + }, + Data: &view.CountData{ + Value: 1, + }, + }, + { + Tags: []tag.Tag{ + { + Key: cmtk, + Value: "grpc.testing.TestService/FullDuplexCall", + }, + { + Key: cstk, + Value: "OK", + }, + }, + Data: &view.CountData{ + Value: 1, + }, + }, + }, + }, + }, + { + metric: ServerCompletedRPCsView, + wantVI: &viewInformation{ + aggType: view.AggTypeCount, + aggBuckets: []float64{}, + desc: "Number of completed RPCs by method and status.", + tagKeys: []tag.Key{ + smtk, + sstk, + }, + rows: []*view.Row{ + { + Tags: []tag.Tag{ + { + Key: smtk, + Value: "grpc.testing.TestService/UnaryCall", + }, + { + Key: sstk, + Value: "OK", + }, + }, + Data: &view.CountData{ + Value: 1, + }, + }, + { + Tags: []tag.Tag{ + { + Key: smtk, + Value: "grpc.testing.TestService/FullDuplexCall", + }, + { + Key: sstk, + Value: "OK", + }, + }, + Data: &view.CountData{ + Value: 1, + }, + }, + }, + }, + }, + { + metric: ClientSentBytesPerRPCView, + wantVI: &viewInformation{ + aggType: view.AggTypeDistribution, + aggBuckets: bytesDistributionBounds, + desc: "Distribution of sent bytes per RPC, by method.", + tagKeys: []tag.Key{ + cmtk, + }, + rows: []*view.Row{ + { + Tags: []tag.Tag{ + { + Key: cmtk, + Value: "grpc.testing.TestService/UnaryCall", + }, + }, + Data: &view.DistributionData{ + Count: 1, + CountPerBucket: []int64{0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + }, + { + Tags: []tag.Tag{ + { + Key: cmtk, + Value: "grpc.testing.TestService/FullDuplexCall", + }, + }, + Data: &view.DistributionData{ + Count: 1, + CountPerBucket: []int64{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + }, + }, + }, + }, + { + metric: ClientSentCompressedMessageBytesPerRPCView, + wantVI: &viewInformation{ + aggType: view.AggTypeDistribution, + aggBuckets: bytesDistributionBounds, + desc: "Distribution of sent compressed message bytes per RPC, by method.", + tagKeys: []tag.Key{ + cmtk, + }, + rows: []*view.Row{ + { + Tags: []tag.Tag{ + { + Key: cmtk, + Value: "grpc.testing.TestService/UnaryCall", + }, + }, + Data: &view.DistributionData{ + Count: 1, + CountPerBucket: []int64{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + }, + { + Tags: []tag.Tag{ + { + Key: cmtk, + Value: "grpc.testing.TestService/FullDuplexCall", + }, + }, + Data: &view.DistributionData{ + Count: 1, + CountPerBucket: []int64{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + }, + }, + }, + }, + { + metric: ServerSentBytesPerRPCView, + wantVI: &viewInformation{ + aggType: view.AggTypeDistribution, + aggBuckets: bytesDistributionBounds, + desc: "Distribution of sent bytes per RPC, by method.", + tagKeys: []tag.Key{ + smtk, + }, + rows: []*view.Row{ + { + Tags: []tag.Tag{ + { + Key: smtk, + Value: "grpc.testing.TestService/UnaryCall", + }, + }, + Data: &view.DistributionData{ + Count: 1, + CountPerBucket: []int64{0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + }, + { + Tags: []tag.Tag{ + { + Key: smtk, + Value: "grpc.testing.TestService/FullDuplexCall", + }, + }, + Data: &view.DistributionData{ + Count: 1, + CountPerBucket: []int64{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + }, + }, + }, + }, + { + metric: ServerSentCompressedMessageBytesPerRPCView, + wantVI: &viewInformation{ + aggType: view.AggTypeDistribution, + aggBuckets: bytesDistributionBounds, + desc: "Distribution of sent compressed message bytes per RPC, by method.", + tagKeys: []tag.Key{ + smtk, + }, + rows: []*view.Row{ + { + Tags: []tag.Tag{ + { + Key: smtk, + Value: "grpc.testing.TestService/UnaryCall", + }, + }, + Data: &view.DistributionData{ + Count: 1, + CountPerBucket: []int64{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + }, + { + Tags: []tag.Tag{ + { + Key: smtk, + Value: "grpc.testing.TestService/FullDuplexCall", + }, + }, + Data: &view.DistributionData{ + Count: 1, + CountPerBucket: []int64{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + }, + }, + }, + }, + { + metric: ClientReceivedBytesPerRPCView, + wantVI: &viewInformation{ + aggType: view.AggTypeDistribution, + aggBuckets: bytesDistributionBounds, + desc: "Distribution of received bytes per RPC, by method.", + tagKeys: []tag.Key{ + cmtk, + }, + rows: []*view.Row{ + { + Tags: []tag.Tag{ + { + Key: cmtk, + Value: "grpc.testing.TestService/UnaryCall", + }, + }, + Data: &view.DistributionData{ + Count: 1, + CountPerBucket: []int64{0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + }, + { + Tags: []tag.Tag{ + { + Key: cmtk, + Value: "grpc.testing.TestService/FullDuplexCall", + }, + }, + Data: &view.DistributionData{ + Count: 1, + CountPerBucket: []int64{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + }, + }, + }, + }, + { + metric: ClientReceivedCompressedMessageBytesPerRPCView, + wantVI: &viewInformation{ + aggType: view.AggTypeDistribution, + aggBuckets: bytesDistributionBounds, + desc: "Distribution of received compressed message bytes per RPC, by method.", + tagKeys: []tag.Key{ + cmtk, + }, + rows: []*view.Row{ + { + Tags: []tag.Tag{ + { + Key: cmtk, + Value: "grpc.testing.TestService/UnaryCall", + }, + }, + Data: &view.DistributionData{ + Count: 1, + CountPerBucket: []int64{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + }, + { + Tags: []tag.Tag{ + { + Key: cmtk, + Value: "grpc.testing.TestService/FullDuplexCall", + }, + }, + Data: &view.DistributionData{ + Count: 1, + CountPerBucket: []int64{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + }, + }, + }, + }, + { + metric: ServerReceivedBytesPerRPCView, + wantVI: &viewInformation{ + aggType: view.AggTypeDistribution, + aggBuckets: bytesDistributionBounds, + desc: "Distribution of received bytes per RPC, by method.", + tagKeys: []tag.Key{ + smtk, + }, + rows: []*view.Row{ + { + Tags: []tag.Tag{ + { + Key: smtk, + Value: "grpc.testing.TestService/UnaryCall", + }, + }, + Data: &view.DistributionData{ + Count: 1, + CountPerBucket: []int64{0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + }, + { + Tags: []tag.Tag{ + { + Key: smtk, + Value: "grpc.testing.TestService/FullDuplexCall", + }, + }, + Data: &view.DistributionData{ + Count: 1, + CountPerBucket: []int64{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + }, + }, + }, + }, + { + metric: ServerReceivedCompressedMessageBytesPerRPCView, + wantVI: &viewInformation{ + aggType: view.AggTypeDistribution, + aggBuckets: bytesDistributionBounds, + desc: "Distribution of received compressed message bytes per RPC, by method.", + tagKeys: []tag.Key{ + smtk, + }, + rows: []*view.Row{ + { + Tags: []tag.Tag{ + { + Key: smtk, + Value: "grpc.testing.TestService/UnaryCall", + }, + }, + Data: &view.DistributionData{ + Count: 1, + CountPerBucket: []int64{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + }, + { + Tags: []tag.Tag{ + { + Key: smtk, + Value: "grpc.testing.TestService/FullDuplexCall", + }, + }, + Data: &view.DistributionData{ + Count: 1, + CountPerBucket: []int64{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + }, + }, + }, + }, + { + metric: ClientSentMessagesPerRPCView, + wantVI: &viewInformation{ + aggType: view.AggTypeDistribution, + aggBuckets: countDistributionBounds, + desc: "Distribution of sent messages per RPC, by method.", + tagKeys: []tag.Key{ + cmtk, + }, + rows: []*view.Row{ + { + Tags: []tag.Tag{ + { + Key: cmtk, + Value: "grpc.testing.TestService/UnaryCall", + }, + }, + Data: &view.DistributionData{ + Count: 1, + CountPerBucket: []int64{0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + }, + { + Tags: []tag.Tag{ + { + Key: cmtk, + Value: "grpc.testing.TestService/FullDuplexCall", + }, + }, + Data: &view.DistributionData{ + Count: 1, + CountPerBucket: []int64{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + }, + }, + }, + }, + { + metric: ServerSentMessagesPerRPCView, + wantVI: &viewInformation{ + aggType: view.AggTypeDistribution, + aggBuckets: countDistributionBounds, + desc: "Distribution of sent messages per RPC, by method.", + tagKeys: []tag.Key{ + smtk, + }, + rows: []*view.Row{ + { + Tags: []tag.Tag{ + { + Key: smtk, + Value: "grpc.testing.TestService/UnaryCall", + }, + }, + Data: &view.DistributionData{ + Count: 1, + CountPerBucket: []int64{0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + }, + { + Tags: []tag.Tag{ + { + Key: smtk, + Value: "grpc.testing.TestService/FullDuplexCall", + }, + }, + Data: &view.DistributionData{ + Count: 1, + CountPerBucket: []int64{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + }, + }, + }, + }, + { + metric: ClientReceivedMessagesPerRPCView, + wantVI: &viewInformation{ + aggType: view.AggTypeDistribution, + aggBuckets: countDistributionBounds, + desc: "Distribution of received messages per RPC, by method.", + tagKeys: []tag.Key{ + cmtk, + }, + rows: []*view.Row{ + { + Tags: []tag.Tag{ + { + Key: cmtk, + Value: "grpc.testing.TestService/UnaryCall", + }, + }, + Data: &view.DistributionData{ + Count: 1, + CountPerBucket: []int64{0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + }, + { + Tags: []tag.Tag{ + { + Key: cmtk, + Value: "grpc.testing.TestService/FullDuplexCall", + }, + }, + Data: &view.DistributionData{ + Count: 1, + CountPerBucket: []int64{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + }, + }, + }, + }, + { + metric: ServerReceivedMessagesPerRPCView, + wantVI: &viewInformation{ + aggType: view.AggTypeDistribution, + aggBuckets: countDistributionBounds, + desc: "Distribution of received messages per RPC, by method.", + tagKeys: []tag.Key{ + smtk, + }, + rows: []*view.Row{ + { + Tags: []tag.Tag{ + { + Key: smtk, + Value: "grpc.testing.TestService/UnaryCall", + }, + }, + Data: &view.DistributionData{ + Count: 1, + CountPerBucket: []int64{0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + }, + { + Tags: []tag.Tag{ + { + Key: smtk, + Value: "grpc.testing.TestService/FullDuplexCall", + }, + }, + Data: &view.DistributionData{ + Count: 1, + CountPerBucket: []int64{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + }, + }, + }, + }, + { + metric: ClientRoundtripLatencyView, + wantTags: [][]tag.Tag{ + { + { + Key: cmtk, + Value: "grpc.testing.TestService/UnaryCall", + }, + }, + { + { + Key: cmtk, + Value: "grpc.testing.TestService/FullDuplexCall", + }, + }, + }, + }, + { + metric: ServerLatencyView, + wantTags: [][]tag.Tag{ + { + { + Key: smtk, + Value: "grpc.testing.TestService/UnaryCall", + }, + }, + { + { + Key: smtk, + Value: "grpc.testing.TestService/FullDuplexCall", + }, + }, + }, + }, + // Per call metrics: + { + metric: ClientAPILatencyView, + wantTags: [][]tag.Tag{ + { + { + Key: cmtk, + Value: "grpc.testing.TestService/UnaryCall", + }, + { + Key: cstk, + Value: "OK", + }, + }, + { + { + Key: cmtk, + Value: "grpc.testing.TestService/FullDuplexCall", + }, + { + Key: cstk, + Value: "OK", + }, + }, + }, + }, + } + // Server Side stats.End call happens asynchronously for both Unary and + // Streaming calls with respect to the RPC returning client side. Thus, add + // a sync point at the global view package level for these two rows to be + // recorded, which will be synchronously uploaded to exporters right after. + if err := waitForServerCompletedRPCs(ctx); err != nil { + t.Fatal(err) + } + view.Unregister(allViews...) + // Assert the expected emissions for each metric match the expected + // emissions. + for _, wantMetric := range wantMetrics { + metricName := wantMetric.metric.Name + var vi *viewInformation + if vi = fe.seenViews[metricName]; vi == nil { + t.Fatalf("couldn't find %v in the views exported, never collected", metricName) + } + + // For latency metrics, there is a lot of non determinism about + // the exact milliseconds of RPCs that finish. Thus, rather than + // declare the exact data you want, make sure the latency + // measurement points for the two RPCs above fall within buckets + // that fall into less than 5 seconds, which is the rpc timeout. + if metricName == "grpc.io/client/roundtrip_latency" || metricName == "grpc.io/server/server_latency" || metricName == "grpc.io/client/api_latency" { + // RPCs have a context timeout of 5s, so all the recorded + // measurements (one per RPC - two total) should fall within 5 + // second buckets. + if err := distributionDataLatencyCount(vi, 2, wantMetric.wantTags); err != nil { + t.Fatalf("Invalid OpenCensus export view data for metric %v: %v", metricName, err) + } + continue + } + if diff := cmp.Diff(vi, wantMetric.wantVI); diff != "" { + t.Fatalf("got unexpected viewInformation for metric %v, diff (-got, +want): %v", metricName, diff) + } + // Note that this test only fatals with one error if a metric fails. + // This is fine, as all are expected to pass so if a single one fails + // you can figure it out and iterate as needed. + } +} + +// TestOpenCensusTags tests this instrumentation code's ability to propagate +// OpenCensus tags across the wire. It also tests the server stats handler's +// functionality of adding the server method tag for the application to see. The +// test makes an Unary RPC without a tag map and with a tag map, and expects to +// see a tag map at the application layer with server method tag in the first +// case, and a tag map at the application layer with the populated tag map plus +// server method tag in second case. +func (s) TestOpenCensusTags(t *testing.T) { + // This stub servers functions represent the application layer server side. + // This is the intended feature being tested: that open census tags + // populated at the client side application layer end up at the server side + // application layer with the server method tag key in addition to the map + // populated at the client side application layer if populated. + tmCh := testutils.NewChannel() + ss := &stubserver.StubServer{ + UnaryCallF: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + // Do the sends of the tag maps for assertions in this main testing + // goroutine. Do the receives and assertions in a forked goroutine. + if tm := tag.FromContext(ctx); tm != nil { + tmCh.Send(tm) + } else { + tmCh.Send(errors.New("no tag map received server side")) + } + return &testpb.SimpleResponse{}, nil + }, + } + if err := ss.Start([]grpc.ServerOption{ServerOption(TraceOptions{})}, DialOption(TraceOptions{})); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + key1 := tag.MustNewKey("key 1") + wg := sync.WaitGroup{} + wg.Add(1) + readerErrCh := testutils.NewChannel() + // Spawn a goroutine to receive and validation two tag maps received by the + // server application code. + go func() { + defer wg.Done() + unaryCallMethodName := "grpc.testing.TestService/UnaryCall" + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + // Attempt to receive the tag map from the first RPC. + if tm, err := tmCh.Receive(ctx); err == nil { + tagMap, ok := tm.(*tag.Map) + // Shouldn't happen, this test sends only *tag.Map type on channel. + if !ok { + readerErrCh.Send(fmt.Errorf("received wrong type from channel: %T", tm)) + } + // keyServerMethod should be present in this tag map received server + // side. + val, ok := tagMap.Value(keyServerMethod) + if !ok { + readerErrCh.Send(fmt.Errorf("no key: %v present in OpenCensus tag map", keyServerMethod.Name())) + } + if val != unaryCallMethodName { + readerErrCh.Send(fmt.Errorf("serverMethod receieved: %v, want server method: %v", val, unaryCallMethodName)) + } + } else { + readerErrCh.Send(fmt.Errorf("error while waiting for a tag map: %v", err)) + } + readerErrCh.Send(nil) + + // Attempt to receive the tag map from the second RPC. + if tm, err := tmCh.Receive(ctx); err == nil { + tagMap, ok := tm.(*tag.Map) + // Shouldn't happen, this test sends only *tag.Map type on channel. + if !ok { + readerErrCh.Send(fmt.Errorf("received wrong type from channel: %T", tm)) + } + // key1: "value1" populated in the tag map client side should make + // its way to server. + val, ok := tagMap.Value(key1) + if !ok { + readerErrCh.Send(fmt.Errorf("no key: %v present in OpenCensus tag map", key1.Name())) + } + if val != "value1" { + readerErrCh.Send(fmt.Errorf("key %v received: %v, want server method: %v", key1.Name(), val, unaryCallMethodName)) + } + // keyServerMethod should be appended to tag map as well. + val, ok = tagMap.Value(keyServerMethod) + if !ok { + readerErrCh.Send(fmt.Errorf("no key: %v present in OpenCensus tag map", keyServerMethod.Name())) + } + if val != unaryCallMethodName { + readerErrCh.Send(fmt.Errorf("key: %v received: %v, want server method: %v", keyServerMethod.Name(), val, unaryCallMethodName)) + } + } else { + readerErrCh.Send(fmt.Errorf("error while waiting for second tag map: %v", err)) + } + readerErrCh.Send(nil) + }() + + // Make a unary RPC without populating an OpenCensus tag map. The server + // side should receive an OpenCensus tag map containing only the + // keyServerMethod. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{Payload: &testpb.Payload{}}); err != nil { + t.Fatalf("Unexpected error from UnaryCall: %v", err) + } + + ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + // Should receive a nil error from the readerErrCh, meaning the reader + // goroutine successfully received a tag map with the keyServerMethod + // populated. + if chErr, err := readerErrCh.Receive(ctx); chErr != nil || err != nil { + if err != nil { + t.Fatalf("Should have received something from error channel: %v", err) + } + if chErr != nil { + t.Fatalf("Should have received a nil error from channel, instead received: %v", chErr) + } + } + + tm := &tag.Map{} + ctx = tag.NewContext(ctx, tm) + ctx, err := tag.New(ctx, tag.Upsert(key1, "value1")) + // Setup steps like this can fatal, so easier to do the RPC's and subsequent + // sends of the tag maps of the RPC's in main goroutine and have the + // corresponding receives and assertions in a forked goroutine. + if err != nil { + t.Fatalf("Error creating tag map: %v", err) + } + // Make a unary RPC with a populated OpenCensus tag map. The server side + // should receive an OpenCensus tag map containing this populated tag map + // with the keyServerMethod tag appended to it. + if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{Payload: &testpb.Payload{}}); err != nil { + t.Fatalf("Unexpected error from UnaryCall: %v", err) + } + if chErr, err := readerErrCh.Receive(ctx); chErr != nil || err != nil { + if err != nil { + t.Fatalf("Should have received something from error channel: %v", err) + } + if chErr != nil { + t.Fatalf("Should have received a nil error from channel, instead received: %v", chErr) + } + } + wg.Wait() +} + +// compareSpanContext only checks the equality of the trace options, which +// represent whether the span should be sampled. The other fields are checked +// for presence in later assertions. +func compareSpanContext(sc trace.SpanContext, sc2 trace.SpanContext) bool { + return sc.TraceOptions.IsSampled() == sc2.TraceOptions.IsSampled() +} + +func compareMessageEvents(me []trace.MessageEvent, me2 []trace.MessageEvent) bool { + if len(me) != len(me2) { + return false + } + // Order matters here, message events are deterministic so no flakiness to + // test. + for i, e := range me { + e2 := me2[i] + if e.EventType != e2.EventType { + return false + } + if e.MessageID != e2.MessageID { + return false + } + if e.UncompressedByteSize != e2.UncompressedByteSize { + return false + } + if e.CompressedByteSize != e2.CompressedByteSize { + return false + } + } + return true +} + +// compareLinks compares the type of link received compared to the wanted link. +func compareLinks(ls []trace.Link, ls2 []trace.Link) bool { + if len(ls) != len(ls2) { + return false + } + for i, l := range ls { + l2 := ls2[i] + if l.Type != l2.Type { + return false + } + } + return true +} + +// spanInformation is the information received about the span. This is a subset +// of information that is important to verify that gRPC has knobs over, which +// goes through a stable OpenCensus API with well defined behavior. This keeps +// the robustness of assertions over time. +type spanInformation struct { + // SpanContext either gets pulled off the wire in certain cases server side + // or created. + sc trace.SpanContext + parentSpanID trace.SpanID + spanKind int + name string + message string + messageEvents []trace.MessageEvent + status trace.Status + links []trace.Link + hasRemoteParent bool + childSpanCount int +} + +// validateTraceAndSpanIDs checks for consistent trace ID across the full trace. +// It also asserts each span has a corresponding generated SpanID, and makes +// sure in the case of a server span and a client span, the server span points +// to the client span as its parent. This is assumed to be called with spans +// from the same RPC (thus the same trace). If called with spanInformation slice +// of length 2, it assumes first span is a server span which points to second +// span as parent and second span is a client span. These assertions are +// orthogonal to pure equality assertions, as this data is generated at runtime, +// so can only test relations between IDs (i.e. this part of the data has the +// same ID as this part of the data). +// +// Returns an error in the case of a failing assertion, non nil error otherwise. +func validateTraceAndSpanIDs(sis []spanInformation) error { + var traceID trace.TraceID + for i, si := range sis { + // Trace IDs should all be consistent across every span, since this + // function assumes called with Span from one RPC, which all fall under + // one trace. + if i == 0 { + traceID = si.sc.TraceID + } else { + if !cmp.Equal(si.sc.TraceID, traceID) { + return fmt.Errorf("TraceIDs should all be consistent: %v, %v", si.sc.TraceID, traceID) + } + } + // Due to the span IDs being 8 bytes, the documentation states that it + // is practically a mathematical uncertainty in practice to create two + // colliding IDs. Thus, for a presence check (the ID was actually + // generated, I will simply compare to the zero value, even though a + // zero value is a theoretical possibility of generation). This is + // because in practice, this zero value defined by this test will never + // collide with the generated ID. + if cmp.Equal(si.sc.SpanID, trace.SpanID{}) { + return errors.New("span IDs should be populated from the creation of the span") + } + } + // If the length of spans of an RPC is 2, it means there is a server span + // which exports first and a client span which exports second. Thus, the + // server span should point to the client span as its parent, represented + // by its ID. + if len(sis) == 2 { + if !cmp.Equal(sis[0].parentSpanID, sis[1].sc.SpanID) { + return fmt.Errorf("server span should point to the client span as its parent. parentSpanID: %v, clientSpanID: %v", sis[0].parentSpanID, sis[1].sc.SpanID) + } + } + return nil +} + +// Equal compares the constant data of the exported span information that is +// important for correctness known before runtime. +func (si spanInformation) Equal(si2 spanInformation) bool { + if !compareSpanContext(si.sc, si2.sc) { + return false + } + + if si.spanKind != si2.spanKind { + return false + } + if si.name != si2.name { + return false + } + if si.message != si2.message { + return false + } + // Ignore attribute comparison because Java doesn't even populate any so not + // important for correctness. + if !compareMessageEvents(si.messageEvents, si2.messageEvents) { + return false + } + if !cmp.Equal(si.status, si2.status) { + return false + } + // compare link type as link type child is important. + if !compareLinks(si.links, si2.links) { + return false + } + if si.hasRemoteParent != si2.hasRemoteParent { + return false + } + return si.childSpanCount == si2.childSpanCount +} + +func (fe *fakeExporter) ExportSpan(sd *trace.SpanData) { + fe.mu.Lock() + defer fe.mu.Unlock() + + // Persist the subset of data received that is important for correctness and + // to make various assertions on later. Keep the ordering as ordering of + // spans is deterministic in the context of one RPC. + gotSI := spanInformation{ + sc: sd.SpanContext, + parentSpanID: sd.ParentSpanID, + spanKind: sd.SpanKind, + name: sd.Name, + message: sd.Message, + // annotations - ignore + // attributes - ignore, I just left them in from previous but no spec + // for correctness so no need to test. Java doesn't even have any + // attributes. + messageEvents: sd.MessageEvents, + status: sd.Status, + links: sd.Links, + hasRemoteParent: sd.HasRemoteParent, + childSpanCount: sd.ChildSpanCount, + } + fe.seenSpans = append(fe.seenSpans, gotSI) +} + +// waitForServerSpan waits until a server span appears somewhere in the span +// list in an exporter. Returns an error if no server span found within the +// passed context's timeout. +func waitForServerSpan(ctx context.Context, fe *fakeExporter) error { + for ; ctx.Err() == nil; <-time.After(time.Millisecond) { + fe.mu.Lock() + for _, seenSpan := range fe.seenSpans { + if seenSpan.spanKind == trace.SpanKindServer { + fe.mu.Unlock() + return nil + } + } + fe.mu.Unlock() + } + return fmt.Errorf("timeout when waiting for server span to be present in exporter") +} + +// TestSpan tests emitted spans from gRPC. It configures a system with a gRPC +// Client and gRPC server with the OpenCensus Dial and Server Option configured, +// and makes a Unary RPC and a Streaming RPC. This should cause spans with +// certain information to be emitted from client and server side for each RPC. +func (s) TestSpan(t *testing.T) { + fe := &fakeExporter{ + t: t, + } + trace.RegisterExporter(fe) + defer trace.UnregisterExporter(fe) + + so := TraceOptions{ + TS: trace.ProbabilitySampler(1), + DisableTrace: false, + } + ss := &stubserver.StubServer{ + UnaryCallF: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{}, nil + }, + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + for { + _, err := stream.Recv() + if err == io.EOF { + return nil + } + } + }, + } + if err := ss.Start([]grpc.ServerOption{ServerOption(so)}, DialOption(so)); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Make a Unary RPC. This should cause a span with message events + // corresponding to the request message and response message to be emitted + // both from the client and the server. + if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{Payload: &testpb.Payload{}}); err != nil { + t.Fatalf("Unexpected error from UnaryCall: %v", err) + } + wantSI := []spanInformation{ + { + sc: trace.SpanContext{ + TraceOptions: 1, + }, + name: "Attempt.grpc.testing.TestService.UnaryCall", + messageEvents: []trace.MessageEvent{ + { + EventType: trace.MessageEventTypeSent, + MessageID: 1, // First msg send so 1 (see comment above) + UncompressedByteSize: 2, + CompressedByteSize: 2, + }, + { + EventType: trace.MessageEventTypeRecv, + MessageID: 1, // First msg recv so 1 (see comment above) + }, + }, + hasRemoteParent: false, + }, + { + // Sampling rate of 100 percent, so this should populate every span + // with the information that this span is being sampled. Here and + // every other span emitted in this test. + sc: trace.SpanContext{ + TraceOptions: 1, + }, + spanKind: trace.SpanKindServer, + name: "grpc.testing.TestService.UnaryCall", + // message id - "must be calculated as two different counters + // starting from 1 one for sent messages and one for received + // message. This way we guarantee that the values will be consistent + // between different implementations. In case of unary calls only + // one sent and one received message will be recorded for both + // client and server spans." + messageEvents: []trace.MessageEvent{ + { + EventType: trace.MessageEventTypeRecv, + MessageID: 1, // First msg recv so 1 (see comment above) + UncompressedByteSize: 2, + CompressedByteSize: 2, + }, + { + EventType: trace.MessageEventTypeSent, + MessageID: 1, // First msg send so 1 (see comment above) + }, + }, + links: []trace.Link{ + { + Type: trace.LinkTypeChild, + }, + }, + // For some reason, status isn't populated in the data sent to the + // exporter. This seems wrong, but it didn't send status in old + // instrumentation code, so I'm iffy on it but fine. + hasRemoteParent: true, + }, + { + sc: trace.SpanContext{ + TraceOptions: 1, + }, + spanKind: trace.SpanKindClient, + name: "grpc.testing.TestService.UnaryCall", + hasRemoteParent: false, + childSpanCount: 1, + }, + } + if err := waitForServerSpan(ctx, fe); err != nil { + t.Fatal(err) + } + var spanInfoSort = func(i, j int) bool { + // This will order into attempt span (which has an unset span kind to + // not prepend Sent. to span names in backends), then call span, then + // server span. + return fe.seenSpans[i].spanKind < fe.seenSpans[j].spanKind + } + fe.mu.Lock() + // Sort the underlying seen Spans for cmp.Diff assertions and ID + // relationship assertions. + sort.Slice(fe.seenSpans, spanInfoSort) + if diff := cmp.Diff(fe.seenSpans, wantSI); diff != "" { + fe.mu.Unlock() + t.Fatalf("got unexpected spans, diff (-got, +want): %v", diff) + } + if err := validateTraceAndSpanIDs(fe.seenSpans); err != nil { + fe.mu.Unlock() + t.Fatalf("Error in runtime data assertions: %v", err) + } + if !cmp.Equal(fe.seenSpans[1].parentSpanID, fe.seenSpans[0].sc.SpanID) { + t.Fatalf("server span should point to the client attempt span as its parent. parentSpanID: %v, clientAttemptSpanID: %v", fe.seenSpans[1].parentSpanID, fe.seenSpans[0].sc.SpanID) + } + if !cmp.Equal(fe.seenSpans[0].parentSpanID, fe.seenSpans[2].sc.SpanID) { + t.Fatalf("client attempt span should point to the client call span as its parent. parentSpanID: %v, clientCallSpanID: %v", fe.seenSpans[0].parentSpanID, fe.seenSpans[2].sc.SpanID) + } + + fe.seenSpans = nil + fe.mu.Unlock() + + stream, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("ss.Client.FullDuplexCall failed: %v", err) + } + // Send two messages. This should be recorded in the emitted spans message + // events, with message IDs which increase for each message. + if err := stream.Send(&testpb.StreamingOutputCallRequest{}); err != nil { + t.Fatalf("stream.Send failed: %v", err) + } + if err := stream.Send(&testpb.StreamingOutputCallRequest{}); err != nil { + t.Fatalf("stream.Send failed: %v", err) + } + + stream.CloseSend() + if _, err = stream.Recv(); err != io.EOF { + t.Fatalf("unexpected error: %v, expected an EOF error", err) + } + + wantSI = []spanInformation{ + { + sc: trace.SpanContext{ + TraceOptions: 1, + }, + name: "Attempt.grpc.testing.TestService.FullDuplexCall", + messageEvents: []trace.MessageEvent{ + { + EventType: trace.MessageEventTypeSent, + MessageID: 1, // First msg send so 1 + }, + { + EventType: trace.MessageEventTypeSent, + MessageID: 2, // Second msg send so 2 + }, + }, + hasRemoteParent: false, + }, + { + sc: trace.SpanContext{ + TraceOptions: 1, + }, + spanKind: trace.SpanKindServer, + name: "grpc.testing.TestService.FullDuplexCall", + links: []trace.Link{ + { + Type: trace.LinkTypeChild, + }, + }, + messageEvents: []trace.MessageEvent{ + { + EventType: trace.MessageEventTypeRecv, + MessageID: 1, // First msg recv so 1 + }, + { + EventType: trace.MessageEventTypeRecv, + MessageID: 2, // Second msg recv so 2 + }, + }, + hasRemoteParent: true, + }, + { + sc: trace.SpanContext{ + TraceOptions: 1, + }, + spanKind: trace.SpanKindClient, + name: "grpc.testing.TestService.FullDuplexCall", + hasRemoteParent: false, + childSpanCount: 1, + }, + } + if err := waitForServerSpan(ctx, fe); err != nil { + t.Fatal(err) + } + fe.mu.Lock() + defer fe.mu.Unlock() + // Sort the underlying seen Spans for cmp.Diff assertions and ID + // relationship assertions. + sort.Slice(fe.seenSpans, spanInfoSort) + if diff := cmp.Diff(fe.seenSpans, wantSI); diff != "" { + t.Fatalf("got unexpected spans, diff (-got, +want): %v", diff) + } + if err := validateTraceAndSpanIDs(fe.seenSpans); err != nil { + t.Fatalf("Error in runtime data assertions: %v", err) + } + if !cmp.Equal(fe.seenSpans[1].parentSpanID, fe.seenSpans[0].sc.SpanID) { + t.Fatalf("server span should point to the client attempt span as its parent. parentSpanID: %v, clientAttemptSpanID: %v", fe.seenSpans[1].parentSpanID, fe.seenSpans[0].sc.SpanID) + } + if !cmp.Equal(fe.seenSpans[0].parentSpanID, fe.seenSpans[2].sc.SpanID) { + t.Fatalf("client attempt span should point to the client call span as its parent. parentSpanID: %v, clientCallSpanID: %v", fe.seenSpans[0].parentSpanID, fe.seenSpans[2].sc.SpanID) + } +} diff --git a/stats/opencensus/go.mod b/stats/opencensus/go.mod new file mode 100644 index 000000000000..9f79073d36df --- /dev/null +++ b/stats/opencensus/go.mod @@ -0,0 +1,21 @@ +module google.golang.org/grpc/stats/opencensus + +go 1.19 + +require ( + github.com/google/go-cmp v0.5.9 + go.opencensus.io v0.24.0 + google.golang.org/grpc v1.56.2 +) + +require ( + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.3 // indirect + golang.org/x/net v0.12.0 // indirect + golang.org/x/sys v0.10.0 // indirect + golang.org/x/text v0.11.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect + google.golang.org/protobuf v1.31.0 // indirect +) + +replace google.golang.org/grpc => ../.. diff --git a/stats/opencensus/go.sum b/stats/opencensus/go.sum new file mode 100644 index 000000000000..57133666b6da --- /dev/null +++ b/stats/opencensus/go.sum @@ -0,0 +1,1667 @@ +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= +cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= +cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= +cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= +cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= +cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= +cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= +cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= +cloud.google.com/go v0.100.1/go.mod h1:fs4QogzfH5n2pBXBP9vRiU+eCny7lD2vmFZy79Iuw1U= +cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= +cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc= +cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34hIU= +cloud.google.com/go v0.104.0/go.mod h1:OO6xxXdJyvuJPcEPBLN9BJPD+jep5G1+2U5B5gkRYtA= +cloud.google.com/go v0.105.0/go.mod h1:PrLgOJNe5nfE9UMxKxgXj4mD3voiP+YQ6gdt6KMFOKM= +cloud.google.com/go v0.107.0/go.mod h1:wpc2eNrD7hXUTy8EKS10jkxpZBjASrORK7goS+3YX2I= +cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= +cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVquxiw= +cloud.google.com/go v0.110.4/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= +cloud.google.com/go/accessapproval v1.4.0/go.mod h1:zybIuC3KpDOvotz59lFe5qxRZx6C75OtwbisN56xYB4= +cloud.google.com/go/accessapproval v1.5.0/go.mod h1:HFy3tuiGvMdcd/u+Cu5b9NkO1pEICJ46IR82PoUdplw= +cloud.google.com/go/accessapproval v1.6.0/go.mod h1:R0EiYnwV5fsRFiKZkPHr6mwyk2wxUJ30nL4j2pcFY2E= +cloud.google.com/go/accessapproval v1.7.1/go.mod h1:JYczztsHRMK7NTXb6Xw+dwbs/WnOJxbo/2mTI+Kgg68= +cloud.google.com/go/accesscontextmanager v1.3.0/go.mod h1:TgCBehyr5gNMz7ZaH9xubp+CE8dkrszb4oK9CWyvD4o= +cloud.google.com/go/accesscontextmanager v1.4.0/go.mod h1:/Kjh7BBu/Gh83sv+K60vN9QE5NJcd80sU33vIe2IFPE= +cloud.google.com/go/accesscontextmanager v1.6.0/go.mod h1:8XCvZWfYw3K/ji0iVnp+6pu7huxoQTLmxAbVjbloTtM= +cloud.google.com/go/accesscontextmanager v1.7.0/go.mod h1:CEGLewx8dwa33aDAZQujl7Dx+uYhS0eay198wB/VumQ= +cloud.google.com/go/accesscontextmanager v1.8.0/go.mod h1:uI+AI/r1oyWK99NN8cQ3UK76AMelMzgZCvJfsi2c+ps= +cloud.google.com/go/accesscontextmanager v1.8.1/go.mod h1:JFJHfvuaTC+++1iL1coPiG1eu5D24db2wXCDWDjIrxo= +cloud.google.com/go/aiplatform v1.22.0/go.mod h1:ig5Nct50bZlzV6NvKaTwmplLLddFx0YReh9WfTO5jKw= +cloud.google.com/go/aiplatform v1.24.0/go.mod h1:67UUvRBKG6GTayHKV8DBv2RtR1t93YRu5B1P3x99mYY= +cloud.google.com/go/aiplatform v1.27.0/go.mod h1:Bvxqtl40l0WImSb04d0hXFU7gDOiq9jQmorivIiWcKg= +cloud.google.com/go/aiplatform v1.35.0/go.mod h1:7MFT/vCaOyZT/4IIFfxH4ErVg/4ku6lKv3w0+tFTgXQ= +cloud.google.com/go/aiplatform v1.36.1/go.mod h1:WTm12vJRPARNvJ+v6P52RDHCNe4AhvjcIZ/9/RRHy/k= +cloud.google.com/go/aiplatform v1.37.0/go.mod h1:IU2Cv29Lv9oCn/9LkFiiuKfwrRTq+QQMbW+hPCxJGZw= +cloud.google.com/go/aiplatform v1.45.0/go.mod h1:Iu2Q7sC7QGhXUeOhAj/oCK9a+ULz1O4AotZiqjQ8MYA= +cloud.google.com/go/analytics v0.11.0/go.mod h1:DjEWCu41bVbYcKyvlws9Er60YE4a//bK6mnhWvQeFNI= +cloud.google.com/go/analytics v0.12.0/go.mod h1:gkfj9h6XRf9+TS4bmuhPEShsh3hH8PAZzm/41OOhQd4= +cloud.google.com/go/analytics v0.17.0/go.mod h1:WXFa3WSym4IZ+JiKmavYdJwGG/CvpqiqczmL59bTD9M= +cloud.google.com/go/analytics v0.18.0/go.mod h1:ZkeHGQlcIPkw0R/GW+boWHhCOR43xz9RN/jn7WcqfIE= +cloud.google.com/go/analytics v0.19.0/go.mod h1:k8liqf5/HCnOUkbawNtrWWc+UAzyDlW89doe8TtoDsE= +cloud.google.com/go/analytics v0.21.2/go.mod h1:U8dcUtmDmjrmUTnnnRnI4m6zKn/yaA5N9RlEkYFHpQo= +cloud.google.com/go/apigateway v1.3.0/go.mod h1:89Z8Bhpmxu6AmUxuVRg/ECRGReEdiP3vQtk4Z1J9rJk= +cloud.google.com/go/apigateway v1.4.0/go.mod h1:pHVY9MKGaH9PQ3pJ4YLzoj6U5FUDeDFBllIz7WmzJoc= +cloud.google.com/go/apigateway v1.5.0/go.mod h1:GpnZR3Q4rR7LVu5951qfXPJCHquZt02jf7xQx7kpqN8= +cloud.google.com/go/apigateway v1.6.1/go.mod h1:ufAS3wpbRjqfZrzpvLC2oh0MFlpRJm2E/ts25yyqmXA= +cloud.google.com/go/apigeeconnect v1.3.0/go.mod h1:G/AwXFAKo0gIXkPTVfZDd2qA1TxBXJ3MgMRBQkIi9jc= +cloud.google.com/go/apigeeconnect v1.4.0/go.mod h1:kV4NwOKqjvt2JYR0AoIWo2QGfoRtn/pkS3QlHp0Ni04= +cloud.google.com/go/apigeeconnect v1.5.0/go.mod h1:KFaCqvBRU6idyhSNyn3vlHXc8VMDJdRmwDF6JyFRqZ8= +cloud.google.com/go/apigeeconnect v1.6.1/go.mod h1:C4awq7x0JpLtrlQCr8AzVIzAaYgngRqWf9S5Uhg+wWs= +cloud.google.com/go/apigeeregistry v0.4.0/go.mod h1:EUG4PGcsZvxOXAdyEghIdXwAEi/4MEaoqLMLDMIwKXY= +cloud.google.com/go/apigeeregistry v0.5.0/go.mod h1:YR5+s0BVNZfVOUkMa5pAR2xGd0A473vA5M7j247o1wM= +cloud.google.com/go/apigeeregistry v0.6.0/go.mod h1:BFNzW7yQVLZ3yj0TKcwzb8n25CFBri51GVGOEUcgQsc= +cloud.google.com/go/apigeeregistry v0.7.1/go.mod h1:1XgyjZye4Mqtw7T9TsY4NW10U7BojBvG4RMD+vRDrIw= +cloud.google.com/go/apikeys v0.4.0/go.mod h1:XATS/yqZbaBK0HOssf+ALHp8jAlNHUgyfprvNcBIszU= +cloud.google.com/go/apikeys v0.5.0/go.mod h1:5aQfwY4D+ewMMWScd3hm2en3hCj+BROlyrt3ytS7KLI= +cloud.google.com/go/apikeys v0.6.0/go.mod h1:kbpXu5upyiAlGkKrJgQl8A0rKNNJ7dQ377pdroRSSi8= +cloud.google.com/go/appengine v1.4.0/go.mod h1:CS2NhuBuDXM9f+qscZ6V86m1MIIqPj3WC/UoEuR1Sno= +cloud.google.com/go/appengine v1.5.0/go.mod h1:TfasSozdkFI0zeoxW3PTBLiNqRmzraodCWatWI9Dmak= +cloud.google.com/go/appengine v1.6.0/go.mod h1:hg6i0J/BD2cKmDJbaFSYHFyZkgBEfQrDg/X0V5fJn84= +cloud.google.com/go/appengine v1.7.0/go.mod h1:eZqpbHFCqRGa2aCdope7eC0SWLV1j0neb/QnMJVWx6A= +cloud.google.com/go/appengine v1.7.1/go.mod h1:IHLToyb/3fKutRysUlFO0BPt5j7RiQ45nrzEJmKTo6E= +cloud.google.com/go/appengine v1.8.1/go.mod h1:6NJXGLVhZCN9aQ/AEDvmfzKEfoYBlfB80/BHiKVputY= +cloud.google.com/go/area120 v0.5.0/go.mod h1:DE/n4mp+iqVyvxHN41Vf1CR602GiHQjFPusMFW6bGR4= +cloud.google.com/go/area120 v0.6.0/go.mod h1:39yFJqWVgm0UZqWTOdqkLhjoC7uFfgXRC8g/ZegeAh0= +cloud.google.com/go/area120 v0.7.0/go.mod h1:a3+8EUD1SX5RUcCs3MY5YasiO1z6yLiNLRiFrykbynY= +cloud.google.com/go/area120 v0.7.1/go.mod h1:j84i4E1RboTWjKtZVWXPqvK5VHQFJRF2c1Nm69pWm9k= +cloud.google.com/go/area120 v0.8.1/go.mod h1:BVfZpGpB7KFVNxPiQBuHkX6Ed0rS51xIgmGyjrAfzsg= +cloud.google.com/go/artifactregistry v1.6.0/go.mod h1:IYt0oBPSAGYj/kprzsBjZ/4LnG/zOcHyFHjWPCi6SAQ= +cloud.google.com/go/artifactregistry v1.7.0/go.mod h1:mqTOFOnGZx8EtSqK/ZWcsm/4U8B77rbcLP6ruDU2Ixk= +cloud.google.com/go/artifactregistry v1.8.0/go.mod h1:w3GQXkJX8hiKN0v+at4b0qotwijQbYUqF2GWkZzAhC0= +cloud.google.com/go/artifactregistry v1.9.0/go.mod h1:2K2RqvA2CYvAeARHRkLDhMDJ3OXy26h3XW+3/Jh2uYc= +cloud.google.com/go/artifactregistry v1.11.1/go.mod h1:lLYghw+Itq9SONbCa1YWBoWs1nOucMH0pwXN1rOBZFI= +cloud.google.com/go/artifactregistry v1.11.2/go.mod h1:nLZns771ZGAwVLzTX/7Al6R9ehma4WUEhZGWV6CeQNQ= +cloud.google.com/go/artifactregistry v1.12.0/go.mod h1:o6P3MIvtzTOnmvGagO9v/rOjjA0HmhJ+/6KAXrmYDCI= +cloud.google.com/go/artifactregistry v1.13.0/go.mod h1:uy/LNfoOIivepGhooAUpL1i30Hgee3Cu0l4VTWHUC08= +cloud.google.com/go/artifactregistry v1.14.1/go.mod h1:nxVdG19jTaSTu7yA7+VbWL346r3rIdkZ142BSQqhn5E= +cloud.google.com/go/asset v1.5.0/go.mod h1:5mfs8UvcM5wHhqtSv8J1CtxxaQq3AdBxxQi2jGW/K4o= +cloud.google.com/go/asset v1.7.0/go.mod h1:YbENsRK4+xTiL+Ofoj5Ckf+O17kJtgp3Y3nn4uzZz5s= +cloud.google.com/go/asset v1.8.0/go.mod h1:mUNGKhiqIdbr8X7KNayoYvyc4HbbFO9URsjbytpUaW0= +cloud.google.com/go/asset v1.9.0/go.mod h1:83MOE6jEJBMqFKadM9NLRcs80Gdw76qGuHn8m3h8oHQ= +cloud.google.com/go/asset v1.10.0/go.mod h1:pLz7uokL80qKhzKr4xXGvBQXnzHn5evJAEAtZiIb0wY= +cloud.google.com/go/asset v1.11.1/go.mod h1:fSwLhbRvC9p9CXQHJ3BgFeQNM4c9x10lqlrdEUYXlJo= +cloud.google.com/go/asset v1.12.0/go.mod h1:h9/sFOa4eDIyKmH6QMpm4eUK3pDojWnUhTgJlk762Hg= +cloud.google.com/go/asset v1.13.0/go.mod h1:WQAMyYek/b7NBpYq/K4KJWcRqzoalEsxz/t/dTk4THw= +cloud.google.com/go/asset v1.14.1/go.mod h1:4bEJ3dnHCqWCDbWJ/6Vn7GVI9LerSi7Rfdi03hd+WTQ= +cloud.google.com/go/assuredworkloads v1.5.0/go.mod h1:n8HOZ6pff6re5KYfBXcFvSViQjDwxFkAkmUFffJRbbY= +cloud.google.com/go/assuredworkloads v1.6.0/go.mod h1:yo2YOk37Yc89Rsd5QMVECvjaMKymF9OP+QXWlKXUkXw= +cloud.google.com/go/assuredworkloads v1.7.0/go.mod h1:z/736/oNmtGAyU47reJgGN+KVoYoxeLBoj4XkKYscNI= +cloud.google.com/go/assuredworkloads v1.8.0/go.mod h1:AsX2cqyNCOvEQC8RMPnoc0yEarXQk6WEKkxYfL6kGIo= +cloud.google.com/go/assuredworkloads v1.9.0/go.mod h1:kFuI1P78bplYtT77Tb1hi0FMxM0vVpRC7VVoJC3ZoT0= +cloud.google.com/go/assuredworkloads v1.10.0/go.mod h1:kwdUQuXcedVdsIaKgKTp9t0UJkE5+PAVNhdQm4ZVq2E= +cloud.google.com/go/assuredworkloads v1.11.1/go.mod h1:+F04I52Pgn5nmPG36CWFtxmav6+7Q+c5QyJoL18Lry0= +cloud.google.com/go/automl v1.5.0/go.mod h1:34EjfoFGMZ5sgJ9EoLsRtdPSNZLcfflJR39VbVNS2M0= +cloud.google.com/go/automl v1.6.0/go.mod h1:ugf8a6Fx+zP0D59WLhqgTDsQI9w07o64uf/Is3Nh5p8= +cloud.google.com/go/automl v1.7.0/go.mod h1:RL9MYCCsJEOmt0Wf3z9uzG0a7adTT1fe+aObgSpkCt8= +cloud.google.com/go/automl v1.8.0/go.mod h1:xWx7G/aPEe/NP+qzYXktoBSDfjO+vnKMGgsApGJJquM= +cloud.google.com/go/automl v1.12.0/go.mod h1:tWDcHDp86aMIuHmyvjuKeeHEGq76lD7ZqfGLN6B0NuU= +cloud.google.com/go/automl v1.13.1/go.mod h1:1aowgAHWYZU27MybSCFiukPO7xnyawv7pt3zK4bheQE= +cloud.google.com/go/baremetalsolution v0.3.0/go.mod h1:XOrocE+pvK1xFfleEnShBlNAXf+j5blPPxrhjKgnIFc= +cloud.google.com/go/baremetalsolution v0.4.0/go.mod h1:BymplhAadOO/eBa7KewQ0Ppg4A4Wplbn+PsFKRLo0uI= +cloud.google.com/go/baremetalsolution v0.5.0/go.mod h1:dXGxEkmR9BMwxhzBhV0AioD0ULBmuLZI8CdwalUxuss= +cloud.google.com/go/batch v0.3.0/go.mod h1:TR18ZoAekj1GuirsUsR1ZTKN3FC/4UDnScjT8NXImFE= +cloud.google.com/go/batch v0.4.0/go.mod h1:WZkHnP43R/QCGQsZ+0JyG4i79ranE2u8xvjq/9+STPE= +cloud.google.com/go/batch v0.7.0/go.mod h1:vLZN95s6teRUqRQ4s3RLDsH8PvboqBK+rn1oevL159g= +cloud.google.com/go/beyondcorp v0.2.0/go.mod h1:TB7Bd+EEtcw9PCPQhCJtJGjk/7TC6ckmnSFS+xwTfm4= +cloud.google.com/go/beyondcorp v0.3.0/go.mod h1:E5U5lcrcXMsCuoDNyGrpyTm/hn7ne941Jz2vmksAxW8= +cloud.google.com/go/beyondcorp v0.4.0/go.mod h1:3ApA0mbhHx6YImmuubf5pyW8srKnCEPON32/5hj+RmM= +cloud.google.com/go/beyondcorp v0.5.0/go.mod h1:uFqj9X+dSfrheVp7ssLTaRHd2EHqSL4QZmH4e8WXGGU= +cloud.google.com/go/beyondcorp v0.6.1/go.mod h1:YhxDWw946SCbmcWo3fAhw3V4XZMSpQ/VYfcKGAEU8/4= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/bigquery v1.42.0/go.mod h1:8dRTJxhtG+vwBKzE5OseQn/hiydoQN3EedCaOdYmxRA= +cloud.google.com/go/bigquery v1.43.0/go.mod h1:ZMQcXHsl+xmU1z36G2jNGZmKp9zNY5BUua5wDgmNCfw= +cloud.google.com/go/bigquery v1.44.0/go.mod h1:0Y33VqXTEsbamHJvJHdFmtqHvMIY28aK1+dFsvaChGc= +cloud.google.com/go/bigquery v1.47.0/go.mod h1:sA9XOgy0A8vQK9+MWhEQTY6Tix87M/ZurWFIxmF9I/E= +cloud.google.com/go/bigquery v1.48.0/go.mod h1:QAwSz+ipNgfL5jxiaK7weyOhzdoAy1zFm0Nf1fysJac= +cloud.google.com/go/bigquery v1.49.0/go.mod h1:Sv8hMmTFFYBlt/ftw2uN6dFdQPzBlREY9yBh7Oy7/4Q= +cloud.google.com/go/bigquery v1.50.0/go.mod h1:YrleYEh2pSEbgTBZYMJ5SuSr0ML3ypjRB1zgf7pvQLU= +cloud.google.com/go/bigquery v1.52.0/go.mod h1:3b/iXjRQGU4nKa87cXeg6/gogLjO8C6PmuM8i5Bi/u4= +cloud.google.com/go/billing v1.4.0/go.mod h1:g9IdKBEFlItS8bTtlrZdVLWSSdSyFUZKXNS02zKMOZY= +cloud.google.com/go/billing v1.5.0/go.mod h1:mztb1tBc3QekhjSgmpf/CV4LzWXLzCArwpLmP2Gm88s= +cloud.google.com/go/billing v1.6.0/go.mod h1:WoXzguj+BeHXPbKfNWkqVtDdzORazmCjraY+vrxcyvI= +cloud.google.com/go/billing v1.7.0/go.mod h1:q457N3Hbj9lYwwRbnlD7vUpyjq6u5U1RAOArInEiD5Y= +cloud.google.com/go/billing v1.12.0/go.mod h1:yKrZio/eu+okO/2McZEbch17O5CB5NpZhhXG6Z766ss= +cloud.google.com/go/billing v1.13.0/go.mod h1:7kB2W9Xf98hP9Sr12KfECgfGclsH3CQR0R08tnRlRbc= +cloud.google.com/go/billing v1.16.0/go.mod h1:y8vx09JSSJG02k5QxbycNRrN7FGZB6F3CAcgum7jvGA= +cloud.google.com/go/binaryauthorization v1.1.0/go.mod h1:xwnoWu3Y84jbuHa0zd526MJYmtnVXn0syOjaJgy4+dM= +cloud.google.com/go/binaryauthorization v1.2.0/go.mod h1:86WKkJHtRcv5ViNABtYMhhNWRrD1Vpi//uKEy7aYEfI= +cloud.google.com/go/binaryauthorization v1.3.0/go.mod h1:lRZbKgjDIIQvzYQS1p99A7/U1JqvqeZg0wiI5tp6tg0= +cloud.google.com/go/binaryauthorization v1.4.0/go.mod h1:tsSPQrBd77VLplV70GUhBf/Zm3FsKmgSqgm4UmiDItk= +cloud.google.com/go/binaryauthorization v1.5.0/go.mod h1:OSe4OU1nN/VswXKRBmciKpo9LulY41gch5c68htf3/Q= +cloud.google.com/go/binaryauthorization v1.6.1/go.mod h1:TKt4pa8xhowwffiBmbrbcxijJRZED4zrqnwZ1lKH51U= +cloud.google.com/go/certificatemanager v1.3.0/go.mod h1:n6twGDvcUBFu9uBgt4eYvvf3sQ6My8jADcOVwHmzadg= +cloud.google.com/go/certificatemanager v1.4.0/go.mod h1:vowpercVFyqs8ABSmrdV+GiFf2H/ch3KyudYQEMM590= +cloud.google.com/go/certificatemanager v1.6.0/go.mod h1:3Hh64rCKjRAX8dXgRAyOcY5vQ/fE1sh8o+Mdd6KPgY8= +cloud.google.com/go/certificatemanager v1.7.1/go.mod h1:iW8J3nG6SaRYImIa+wXQ0g8IgoofDFRp5UMzaNk1UqI= +cloud.google.com/go/channel v1.8.0/go.mod h1:W5SwCXDJsq/rg3tn3oG0LOxpAo6IMxNa09ngphpSlnk= +cloud.google.com/go/channel v1.9.0/go.mod h1:jcu05W0my9Vx4mt3/rEHpfxc9eKi9XwsdDL8yBMbKUk= +cloud.google.com/go/channel v1.11.0/go.mod h1:IdtI0uWGqhEeatSB62VOoJ8FSUhJ9/+iGkJVqp74CGE= +cloud.google.com/go/channel v1.12.0/go.mod h1:VkxCGKASi4Cq7TbXxlaBezonAYpp1GCnKMY6tnMQnLU= +cloud.google.com/go/channel v1.16.0/go.mod h1:eN/q1PFSl5gyu0dYdmxNXscY/4Fi7ABmeHCJNf/oHmc= +cloud.google.com/go/cloudbuild v1.3.0/go.mod h1:WequR4ULxlqvMsjDEEEFnOG5ZSRSgWOywXYDb1vPE6U= +cloud.google.com/go/cloudbuild v1.4.0/go.mod h1:5Qwa40LHiOXmz3386FrjrYM93rM/hdRr7b53sySrTqA= +cloud.google.com/go/cloudbuild v1.6.0/go.mod h1:UIbc/w9QCbH12xX+ezUsgblrWv+Cv4Tw83GiSMHOn9M= +cloud.google.com/go/cloudbuild v1.7.0/go.mod h1:zb5tWh2XI6lR9zQmsm1VRA+7OCuve5d8S+zJUul8KTg= +cloud.google.com/go/cloudbuild v1.9.0/go.mod h1:qK1d7s4QlO0VwfYn5YuClDGg2hfmLZEb4wQGAbIgL1s= +cloud.google.com/go/cloudbuild v1.10.1/go.mod h1:lyJg7v97SUIPq4RC2sGsz/9tNczhyv2AjML/ci4ulzU= +cloud.google.com/go/clouddms v1.3.0/go.mod h1:oK6XsCDdW4Ib3jCCBugx+gVjevp2TMXFtgxvPSee3OM= +cloud.google.com/go/clouddms v1.4.0/go.mod h1:Eh7sUGCC+aKry14O1NRljhjyrr0NFC0G2cjwX0cByRk= +cloud.google.com/go/clouddms v1.5.0/go.mod h1:QSxQnhikCLUw13iAbffF2CZxAER3xDGNHjsTAkQJcQA= +cloud.google.com/go/clouddms v1.6.1/go.mod h1:Ygo1vL52Ov4TBZQquhz5fiw2CQ58gvu+PlS6PVXCpZI= +cloud.google.com/go/cloudtasks v1.5.0/go.mod h1:fD92REy1x5woxkKEkLdvavGnPJGEn8Uic9nWuLzqCpY= +cloud.google.com/go/cloudtasks v1.6.0/go.mod h1:C6Io+sxuke9/KNRkbQpihnW93SWDU3uXt92nu85HkYI= +cloud.google.com/go/cloudtasks v1.7.0/go.mod h1:ImsfdYWwlWNJbdgPIIGJWC+gemEGTBK/SunNQQNCAb4= +cloud.google.com/go/cloudtasks v1.8.0/go.mod h1:gQXUIwCSOI4yPVK7DgTVFiiP0ZW/eQkydWzwVMdHxrI= +cloud.google.com/go/cloudtasks v1.9.0/go.mod h1:w+EyLsVkLWHcOaqNEyvcKAsWp9p29dL6uL9Nst1cI7Y= +cloud.google.com/go/cloudtasks v1.10.0/go.mod h1:NDSoTLkZ3+vExFEWu2UJV1arUyzVDAiZtdWcsUyNwBs= +cloud.google.com/go/cloudtasks v1.11.1/go.mod h1:a9udmnou9KO2iulGscKR0qBYjreuX8oHwpmFsKspEvM= +cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= +cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= +cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= +cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s= +cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= +cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= +cloud.google.com/go/compute v1.10.0/go.mod h1:ER5CLbMxl90o2jtNbGSbtfOpQKR0t15FOtRsugnLrlU= +cloud.google.com/go/compute v1.12.0/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= +cloud.google.com/go/compute v1.12.1/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= +cloud.google.com/go/compute v1.13.0/go.mod h1:5aPTS0cUNMIc1CE546K+Th6weJUNQErARyZtRXDJ8GE= +cloud.google.com/go/compute v1.14.0/go.mod h1:YfLtxrj9sU4Yxv+sXzZkyPjEyPBZfXHUvjxega5vAdo= +cloud.google.com/go/compute v1.18.0/go.mod h1:1X7yHxec2Ga+Ss6jPyjxRxpu2uu7PLgsOVXvgU0yacs= +cloud.google.com/go/compute v1.19.0/go.mod h1:rikpw2y+UMidAe9tISo04EHNOIf42RLYF/q8Bs93scU= +cloud.google.com/go/compute v1.19.3/go.mod h1:qxvISKp/gYnXkSAD1ppcSOveRAmzxicEv/JlizULFrI= +cloud.google.com/go/compute v1.20.1/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= +cloud.google.com/go/compute v1.21.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= +cloud.google.com/go/compute/metadata v0.1.0/go.mod h1:Z1VN+bulIf6bt4P/C37K4DyZYZEXYonfTBHHFPO/4UU= +cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= +cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +cloud.google.com/go/contactcenterinsights v1.3.0/go.mod h1:Eu2oemoePuEFc/xKFPjbTuPSj0fYJcPls9TFlPNnHHY= +cloud.google.com/go/contactcenterinsights v1.4.0/go.mod h1:L2YzkGbPsv+vMQMCADxJoT9YiTTnSEd6fEvCeHTYVck= +cloud.google.com/go/contactcenterinsights v1.6.0/go.mod h1:IIDlT6CLcDoyv79kDv8iWxMSTZhLxSCofVV5W6YFM/w= +cloud.google.com/go/contactcenterinsights v1.9.1/go.mod h1:bsg/R7zGLYMVxFFzfh9ooLTruLRCG9fnzhH9KznHhbM= +cloud.google.com/go/container v1.6.0/go.mod h1:Xazp7GjJSeUYo688S+6J5V+n/t+G5sKBTFkKNudGRxg= +cloud.google.com/go/container v1.7.0/go.mod h1:Dp5AHtmothHGX3DwwIHPgq45Y8KmNsgN3amoYfxVkLo= +cloud.google.com/go/container v1.13.1/go.mod h1:6wgbMPeQRw9rSnKBCAJXnds3Pzj03C4JHamr8asWKy4= +cloud.google.com/go/container v1.14.0/go.mod h1:3AoJMPhHfLDxLvrlVWaK57IXzaPnLaZq63WX59aQBfM= +cloud.google.com/go/container v1.15.0/go.mod h1:ft+9S0WGjAyjDggg5S06DXj+fHJICWg8L7isCQe9pQA= +cloud.google.com/go/container v1.22.1/go.mod h1:lTNExE2R7f+DLbAN+rJiKTisauFCaoDq6NURZ83eVH4= +cloud.google.com/go/containeranalysis v0.5.1/go.mod h1:1D92jd8gRR/c0fGMlymRgxWD3Qw9C1ff6/T7mLgVL8I= +cloud.google.com/go/containeranalysis v0.6.0/go.mod h1:HEJoiEIu+lEXM+k7+qLCci0h33lX3ZqoYFdmPcoO7s4= +cloud.google.com/go/containeranalysis v0.7.0/go.mod h1:9aUL+/vZ55P2CXfuZjS4UjQ9AgXoSw8Ts6lemfmxBxI= +cloud.google.com/go/containeranalysis v0.9.0/go.mod h1:orbOANbwk5Ejoom+s+DUCTTJ7IBdBQJDcSylAx/on9s= +cloud.google.com/go/containeranalysis v0.10.1/go.mod h1:Ya2jiILITMY68ZLPaogjmOMNkwsDrWBSTyBubGXO7j0= +cloud.google.com/go/datacatalog v1.3.0/go.mod h1:g9svFY6tuR+j+hrTw3J2dNcmI0dzmSiyOzm8kpLq0a0= +cloud.google.com/go/datacatalog v1.5.0/go.mod h1:M7GPLNQeLfWqeIm3iuiruhPzkt65+Bx8dAKvScX8jvs= +cloud.google.com/go/datacatalog v1.6.0/go.mod h1:+aEyF8JKg+uXcIdAmmaMUmZ3q1b/lKLtXCmXdnc0lbc= +cloud.google.com/go/datacatalog v1.7.0/go.mod h1:9mEl4AuDYWw81UGc41HonIHH7/sn52H0/tc8f8ZbZIE= +cloud.google.com/go/datacatalog v1.8.0/go.mod h1:KYuoVOv9BM8EYz/4eMFxrr4DUKhGIOXxZoKYF5wdISM= +cloud.google.com/go/datacatalog v1.8.1/go.mod h1:RJ58z4rMp3gvETA465Vg+ag8BGgBdnRPEMMSTr5Uv+M= +cloud.google.com/go/datacatalog v1.12.0/go.mod h1:CWae8rFkfp6LzLumKOnmVh4+Zle4A3NXLzVJ1d1mRm0= +cloud.google.com/go/datacatalog v1.13.0/go.mod h1:E4Rj9a5ZtAxcQJlEBTLgMTphfP11/lNaAshpoBgemX8= +cloud.google.com/go/datacatalog v1.14.0/go.mod h1:h0PrGtlihoutNMp/uvwhawLQ9+c63Kz65UFqh49Yo+E= +cloud.google.com/go/datacatalog v1.14.1/go.mod h1:d2CevwTG4yedZilwe+v3E3ZBDRMobQfSG/a6cCCN5R4= +cloud.google.com/go/dataflow v0.6.0/go.mod h1:9QwV89cGoxjjSR9/r7eFDqqjtvbKxAK2BaYU6PVk9UM= +cloud.google.com/go/dataflow v0.7.0/go.mod h1:PX526vb4ijFMesO1o202EaUmouZKBpjHsTlCtB4parQ= +cloud.google.com/go/dataflow v0.8.0/go.mod h1:Rcf5YgTKPtQyYz8bLYhFoIV/vP39eL7fWNcSOyFfLJE= +cloud.google.com/go/dataflow v0.9.1/go.mod h1:Wp7s32QjYuQDWqJPFFlnBKhkAtiFpMTdg00qGbnIHVw= +cloud.google.com/go/dataform v0.3.0/go.mod h1:cj8uNliRlHpa6L3yVhDOBrUXH+BPAO1+KFMQQNSThKo= +cloud.google.com/go/dataform v0.4.0/go.mod h1:fwV6Y4Ty2yIFL89huYlEkwUPtS7YZinZbzzj5S9FzCE= +cloud.google.com/go/dataform v0.5.0/go.mod h1:GFUYRe8IBa2hcomWplodVmUx/iTL0FrsauObOM3Ipr0= +cloud.google.com/go/dataform v0.6.0/go.mod h1:QPflImQy33e29VuapFdf19oPbE4aYTJxr31OAPV+ulA= +cloud.google.com/go/dataform v0.7.0/go.mod h1:7NulqnVozfHvWUBpMDfKMUESr+85aJsC/2O0o3jWPDE= +cloud.google.com/go/dataform v0.8.1/go.mod h1:3BhPSiw8xmppbgzeBbmDvmSWlwouuJkXsXsb8UBih9M= +cloud.google.com/go/datafusion v1.4.0/go.mod h1:1Zb6VN+W6ALo85cXnM1IKiPw+yQMKMhB9TsTSRDo/38= +cloud.google.com/go/datafusion v1.5.0/go.mod h1:Kz+l1FGHB0J+4XF2fud96WMmRiq/wj8N9u007vyXZ2w= +cloud.google.com/go/datafusion v1.6.0/go.mod h1:WBsMF8F1RhSXvVM8rCV3AeyWVxcC2xY6vith3iw3S+8= +cloud.google.com/go/datafusion v1.7.1/go.mod h1:KpoTBbFmoToDExJUso/fcCiguGDk7MEzOWXUsJo0wsI= +cloud.google.com/go/datalabeling v0.5.0/go.mod h1:TGcJ0G2NzcsXSE/97yWjIZO0bXj0KbVlINXMG9ud42I= +cloud.google.com/go/datalabeling v0.6.0/go.mod h1:WqdISuk/+WIGeMkpw/1q7bK/tFEZxsrFJOJdY2bXvTQ= +cloud.google.com/go/datalabeling v0.7.0/go.mod h1:WPQb1y08RJbmpM3ww0CSUAGweL0SxByuW2E+FU+wXcM= +cloud.google.com/go/datalabeling v0.8.1/go.mod h1:XS62LBSVPbYR54GfYQsPXZjTW8UxCK2fkDciSrpRFdY= +cloud.google.com/go/dataplex v1.3.0/go.mod h1:hQuRtDg+fCiFgC8j0zV222HvzFQdRd+SVX8gdmFcZzA= +cloud.google.com/go/dataplex v1.4.0/go.mod h1:X51GfLXEMVJ6UN47ESVqvlsRplbLhcsAt0kZCCKsU0A= +cloud.google.com/go/dataplex v1.5.2/go.mod h1:cVMgQHsmfRoI5KFYq4JtIBEUbYwc3c7tXmIDhRmNNVQ= +cloud.google.com/go/dataplex v1.6.0/go.mod h1:bMsomC/aEJOSpHXdFKFGQ1b0TDPIeL28nJObeO1ppRs= +cloud.google.com/go/dataplex v1.8.1/go.mod h1:7TyrDT6BCdI8/38Uvp0/ZxBslOslP2X2MPDucliyvSE= +cloud.google.com/go/dataproc v1.7.0/go.mod h1:CKAlMjII9H90RXaMpSxQ8EU6dQx6iAYNPcYPOkSbi8s= +cloud.google.com/go/dataproc v1.8.0/go.mod h1:5OW+zNAH0pMpw14JVrPONsxMQYMBqJuzORhIBfBn9uI= +cloud.google.com/go/dataproc v1.12.0/go.mod h1:zrF3aX0uV3ikkMz6z4uBbIKyhRITnxvr4i3IjKsKrw4= +cloud.google.com/go/dataqna v0.5.0/go.mod h1:90Hyk596ft3zUQ8NkFfvICSIfHFh1Bc7C4cK3vbhkeo= +cloud.google.com/go/dataqna v0.6.0/go.mod h1:1lqNpM7rqNLVgWBJyk5NF6Uen2PHym0jtVJonplVsDA= +cloud.google.com/go/dataqna v0.7.0/go.mod h1:Lx9OcIIeqCrw1a6KdO3/5KMP1wAmTc0slZWwP12Qq3c= +cloud.google.com/go/dataqna v0.8.1/go.mod h1:zxZM0Bl6liMePWsHA8RMGAfmTG34vJMapbHAxQ5+WA8= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/datastore v1.10.0/go.mod h1:PC5UzAmDEkAmkfaknstTYbNpgE49HAgW2J1gcgUfmdM= +cloud.google.com/go/datastore v1.11.0/go.mod h1:TvGxBIHCS50u8jzG+AW/ppf87v1of8nwzFNgEZU1D3c= +cloud.google.com/go/datastore v1.12.0/go.mod h1:KjdB88W897MRITkvWWJrg2OUtrR5XVj1EoLgSp6/N70= +cloud.google.com/go/datastore v1.12.1/go.mod h1:KjdB88W897MRITkvWWJrg2OUtrR5XVj1EoLgSp6/N70= +cloud.google.com/go/datastream v1.2.0/go.mod h1:i/uTP8/fZwgATHS/XFu0TcNUhuA0twZxxQ3EyCUQMwo= +cloud.google.com/go/datastream v1.3.0/go.mod h1:cqlOX8xlyYF/uxhiKn6Hbv6WjwPPuI9W2M9SAXwaLLQ= +cloud.google.com/go/datastream v1.4.0/go.mod h1:h9dpzScPhDTs5noEMQVWP8Wx8AFBRyS0s8KWPx/9r0g= +cloud.google.com/go/datastream v1.5.0/go.mod h1:6TZMMNPwjUqZHBKPQ1wwXpb0d5VDVPl2/XoS5yi88q4= +cloud.google.com/go/datastream v1.6.0/go.mod h1:6LQSuswqLa7S4rPAOZFVjHIG3wJIjZcZrw8JDEDJuIs= +cloud.google.com/go/datastream v1.7.0/go.mod h1:uxVRMm2elUSPuh65IbZpzJNMbuzkcvu5CjMqVIUHrww= +cloud.google.com/go/datastream v1.9.1/go.mod h1:hqnmr8kdUBmrnk65k5wNRoHSCYksvpdZIcZIEl8h43Q= +cloud.google.com/go/deploy v1.4.0/go.mod h1:5Xghikd4VrmMLNaF6FiRFDlHb59VM59YoDQnOUdsH/c= +cloud.google.com/go/deploy v1.5.0/go.mod h1:ffgdD0B89tToyW/U/D2eL0jN2+IEV/3EMuXHA0l4r+s= +cloud.google.com/go/deploy v1.6.0/go.mod h1:f9PTHehG/DjCom3QH0cntOVRm93uGBDt2vKzAPwpXQI= +cloud.google.com/go/deploy v1.8.0/go.mod h1:z3myEJnA/2wnB4sgjqdMfgxCA0EqC3RBTNcVPs93mtQ= +cloud.google.com/go/deploy v1.11.0/go.mod h1:tKuSUV5pXbn67KiubiUNUejqLs4f5cxxiCNCeyl0F2g= +cloud.google.com/go/dialogflow v1.15.0/go.mod h1:HbHDWs33WOGJgn6rfzBW1Kv807BE3O1+xGbn59zZWI4= +cloud.google.com/go/dialogflow v1.16.1/go.mod h1:po6LlzGfK+smoSmTBnbkIZY2w8ffjz/RcGSS+sh1el0= +cloud.google.com/go/dialogflow v1.17.0/go.mod h1:YNP09C/kXA1aZdBgC/VtXX74G/TKn7XVCcVumTflA+8= +cloud.google.com/go/dialogflow v1.18.0/go.mod h1:trO7Zu5YdyEuR+BhSNOqJezyFQ3aUzz0njv7sMx/iek= +cloud.google.com/go/dialogflow v1.19.0/go.mod h1:JVmlG1TwykZDtxtTXujec4tQ+D8SBFMoosgy+6Gn0s0= +cloud.google.com/go/dialogflow v1.29.0/go.mod h1:b+2bzMe+k1s9V+F2jbJwpHPzrnIyHihAdRFMtn2WXuM= +cloud.google.com/go/dialogflow v1.31.0/go.mod h1:cuoUccuL1Z+HADhyIA7dci3N5zUssgpBJmCzI6fNRB4= +cloud.google.com/go/dialogflow v1.32.0/go.mod h1:jG9TRJl8CKrDhMEcvfcfFkkpp8ZhgPz3sBGmAUYJ2qE= +cloud.google.com/go/dialogflow v1.38.0/go.mod h1:L7jnH+JL2mtmdChzAIcXQHXMvQkE3U4hTaNltEuxXn4= +cloud.google.com/go/dlp v1.6.0/go.mod h1:9eyB2xIhpU0sVwUixfBubDoRwP+GjeUoxxeueZmqvmM= +cloud.google.com/go/dlp v1.7.0/go.mod h1:68ak9vCiMBjbasxeVD17hVPxDEck+ExiHavX8kiHG+Q= +cloud.google.com/go/dlp v1.9.0/go.mod h1:qdgmqgTyReTz5/YNSSuueR8pl7hO0o9bQ39ZhtgkWp4= +cloud.google.com/go/dlp v1.10.1/go.mod h1:IM8BWz1iJd8njcNcG0+Kyd9OPnqnRNkDV8j42VT5KOI= +cloud.google.com/go/documentai v1.7.0/go.mod h1:lJvftZB5NRiFSX4moiye1SMxHx0Bc3x1+p9e/RfXYiU= +cloud.google.com/go/documentai v1.8.0/go.mod h1:xGHNEB7CtsnySCNrCFdCyyMz44RhFEEX2Q7UD0c5IhU= +cloud.google.com/go/documentai v1.9.0/go.mod h1:FS5485S8R00U10GhgBC0aNGrJxBP8ZVpEeJ7PQDZd6k= +cloud.google.com/go/documentai v1.10.0/go.mod h1:vod47hKQIPeCfN2QS/jULIvQTugbmdc0ZvxxfQY1bg4= +cloud.google.com/go/documentai v1.16.0/go.mod h1:o0o0DLTEZ+YnJZ+J4wNfTxmDVyrkzFvttBXXtYRMHkM= +cloud.google.com/go/documentai v1.18.0/go.mod h1:F6CK6iUH8J81FehpskRmhLq/3VlwQvb7TvwOceQ2tbs= +cloud.google.com/go/documentai v1.20.0/go.mod h1:yJkInoMcK0qNAEdRnqY/D5asy73tnPe88I1YTZT+a8E= +cloud.google.com/go/domains v0.6.0/go.mod h1:T9Rz3GasrpYk6mEGHh4rymIhjlnIuB4ofT1wTxDeT4Y= +cloud.google.com/go/domains v0.7.0/go.mod h1:PtZeqS1xjnXuRPKE/88Iru/LdfoRyEHYA9nFQf4UKpg= +cloud.google.com/go/domains v0.8.0/go.mod h1:M9i3MMDzGFXsydri9/vW+EWz9sWb4I6WyHqdlAk0idE= +cloud.google.com/go/domains v0.9.1/go.mod h1:aOp1c0MbejQQ2Pjf1iJvnVyT+z6R6s8pX66KaCSDYfE= +cloud.google.com/go/edgecontainer v0.1.0/go.mod h1:WgkZ9tp10bFxqO8BLPqv2LlfmQF1X8lZqwW4r1BTajk= +cloud.google.com/go/edgecontainer v0.2.0/go.mod h1:RTmLijy+lGpQ7BXuTDa4C4ssxyXT34NIuHIgKuP4s5w= +cloud.google.com/go/edgecontainer v0.3.0/go.mod h1:FLDpP4nykgwwIfcLt6zInhprzw0lEi2P1fjO6Ie0qbc= +cloud.google.com/go/edgecontainer v1.0.0/go.mod h1:cttArqZpBB2q58W/upSG++ooo6EsblxDIolxa3jSjbY= +cloud.google.com/go/edgecontainer v1.1.1/go.mod h1:O5bYcS//7MELQZs3+7mabRqoWQhXCzenBu0R8bz2rwk= +cloud.google.com/go/errorreporting v0.3.0/go.mod h1:xsP2yaAp+OAW4OIm60An2bbLpqIhKXdWR/tawvl7QzU= +cloud.google.com/go/essentialcontacts v1.3.0/go.mod h1:r+OnHa5jfj90qIfZDO/VztSFqbQan7HV75p8sA+mdGI= +cloud.google.com/go/essentialcontacts v1.4.0/go.mod h1:8tRldvHYsmnBCHdFpvU+GL75oWiBKl80BiqlFh9tp+8= +cloud.google.com/go/essentialcontacts v1.5.0/go.mod h1:ay29Z4zODTuwliK7SnX8E86aUF2CTzdNtvv42niCX0M= +cloud.google.com/go/essentialcontacts v1.6.2/go.mod h1:T2tB6tX+TRak7i88Fb2N9Ok3PvY3UNbUsMag9/BARh4= +cloud.google.com/go/eventarc v1.7.0/go.mod h1:6ctpF3zTnaQCxUjHUdcfgcA1A2T309+omHZth7gDfmc= +cloud.google.com/go/eventarc v1.8.0/go.mod h1:imbzxkyAU4ubfsaKYdQg04WS1NvncblHEup4kvF+4gw= +cloud.google.com/go/eventarc v1.10.0/go.mod h1:u3R35tmZ9HvswGRBnF48IlYgYeBcPUCjkr4BTdem2Kw= +cloud.google.com/go/eventarc v1.11.0/go.mod h1:PyUjsUKPWoRBCHeOxZd/lbOOjahV41icXyUY5kSTvVY= +cloud.google.com/go/eventarc v1.12.1/go.mod h1:mAFCW6lukH5+IZjkvrEss+jmt2kOdYlN8aMx3sRJiAI= +cloud.google.com/go/filestore v1.3.0/go.mod h1:+qbvHGvXU1HaKX2nD0WEPo92TP/8AQuCVEBXNY9z0+w= +cloud.google.com/go/filestore v1.4.0/go.mod h1:PaG5oDfo9r224f8OYXURtAsY+Fbyq/bLYoINEK8XQAI= +cloud.google.com/go/filestore v1.5.0/go.mod h1:FqBXDWBp4YLHqRnVGveOkHDf8svj9r5+mUDLupOWEDs= +cloud.google.com/go/filestore v1.6.0/go.mod h1:di5unNuss/qfZTw2U9nhFqo8/ZDSc466dre85Kydllg= +cloud.google.com/go/filestore v1.7.1/go.mod h1:y10jsorq40JJnjR/lQ8AfFbbcGlw3g+Dp8oN7i7FjV4= +cloud.google.com/go/firestore v1.9.0/go.mod h1:HMkjKHNTtRyZNiMzu7YAsLr9K3X2udY2AMwDaMEQiiE= +cloud.google.com/go/firestore v1.11.0/go.mod h1:b38dKhgzlmNNGTNZZwe7ZRFEuRab1Hay3/DBsIGKKy4= +cloud.google.com/go/functions v1.6.0/go.mod h1:3H1UA3qiIPRWD7PeZKLvHZ9SaQhR26XIJcC0A5GbvAk= +cloud.google.com/go/functions v1.7.0/go.mod h1:+d+QBcWM+RsrgZfV9xo6KfA1GlzJfxcfZcRPEhDDfzg= +cloud.google.com/go/functions v1.8.0/go.mod h1:RTZ4/HsQjIqIYP9a9YPbU+QFoQsAlYgrwOXJWHn1POY= +cloud.google.com/go/functions v1.9.0/go.mod h1:Y+Dz8yGguzO3PpIjhLTbnqV1CWmgQ5UwtlpzoyquQ08= +cloud.google.com/go/functions v1.10.0/go.mod h1:0D3hEOe3DbEvCXtYOZHQZmD+SzYsi1YbI7dGvHfldXw= +cloud.google.com/go/functions v1.12.0/go.mod h1:AXWGrF3e2C/5ehvwYo/GH6O5s09tOPksiKhz+hH8WkA= +cloud.google.com/go/functions v1.13.0/go.mod h1:EU4O007sQm6Ef/PwRsI8N2umygGqPBS/IZQKBQBcJ3c= +cloud.google.com/go/functions v1.15.1/go.mod h1:P5yNWUTkyU+LvW/S9O6V+V423VZooALQlqoXdoPz5AE= +cloud.google.com/go/gaming v1.5.0/go.mod h1:ol7rGcxP/qHTRQE/RO4bxkXq+Fix0j6D4LFPzYTIrDM= +cloud.google.com/go/gaming v1.6.0/go.mod h1:YMU1GEvA39Qt3zWGyAVA9bpYz/yAhTvaQ1t2sK4KPUA= +cloud.google.com/go/gaming v1.7.0/go.mod h1:LrB8U7MHdGgFG851iHAfqUdLcKBdQ55hzXy9xBJz0+w= +cloud.google.com/go/gaming v1.8.0/go.mod h1:xAqjS8b7jAVW0KFYeRUxngo9My3f33kFmua++Pi+ggM= +cloud.google.com/go/gaming v1.9.0/go.mod h1:Fc7kEmCObylSWLO334NcO+O9QMDyz+TKC4v1D7X+Bc0= +cloud.google.com/go/gaming v1.10.1/go.mod h1:XQQvtfP8Rb9Rxnxm5wFVpAp9zCQkJi2bLIb7iHGwB3s= +cloud.google.com/go/gkebackup v0.2.0/go.mod h1:XKvv/4LfG829/B8B7xRkk8zRrOEbKtEam6yNfuQNH60= +cloud.google.com/go/gkebackup v0.3.0/go.mod h1:n/E671i1aOQvUxT541aTkCwExO/bTer2HDlj4TsBRAo= +cloud.google.com/go/gkebackup v0.4.0/go.mod h1:byAyBGUwYGEEww7xsbnUTBHIYcOPy/PgUWUtOeRm9Vg= +cloud.google.com/go/gkeconnect v0.5.0/go.mod h1:c5lsNAg5EwAy7fkqX/+goqFsU1Da/jQFqArp+wGNr/o= +cloud.google.com/go/gkeconnect v0.6.0/go.mod h1:Mln67KyU/sHJEBY8kFZ0xTeyPtzbq9StAVvEULYK16A= +cloud.google.com/go/gkeconnect v0.7.0/go.mod h1:SNfmVqPkaEi3bF/B3CNZOAYPYdg7sU+obZ+QTky2Myw= +cloud.google.com/go/gkeconnect v0.8.1/go.mod h1:KWiK1g9sDLZqhxB2xEuPV8V9NYzrqTUmQR9shJHpOZw= +cloud.google.com/go/gkehub v0.9.0/go.mod h1:WYHN6WG8w9bXU0hqNxt8rm5uxnk8IH+lPY9J2TV7BK0= +cloud.google.com/go/gkehub v0.10.0/go.mod h1:UIPwxI0DsrpsVoWpLB0stwKCP+WFVG9+y977wO+hBH0= +cloud.google.com/go/gkehub v0.11.0/go.mod h1:JOWHlmN+GHyIbuWQPl47/C2RFhnFKH38jH9Ascu3n0E= +cloud.google.com/go/gkehub v0.12.0/go.mod h1:djiIwwzTTBrF5NaXCGv3mf7klpEMcST17VBTVVDcuaw= +cloud.google.com/go/gkehub v0.14.1/go.mod h1:VEXKIJZ2avzrbd7u+zeMtW00Y8ddk/4V9511C9CQGTY= +cloud.google.com/go/gkemulticloud v0.3.0/go.mod h1:7orzy7O0S+5kq95e4Hpn7RysVA7dPs8W/GgfUtsPbrA= +cloud.google.com/go/gkemulticloud v0.4.0/go.mod h1:E9gxVBnseLWCk24ch+P9+B2CoDFJZTyIgLKSalC7tuI= +cloud.google.com/go/gkemulticloud v0.5.0/go.mod h1:W0JDkiyi3Tqh0TJr//y19wyb1yf8llHVto2Htf2Ja3Y= +cloud.google.com/go/gkemulticloud v0.6.1/go.mod h1:kbZ3HKyTsiwqKX7Yw56+wUGwwNZViRnxWK2DVknXWfw= +cloud.google.com/go/grafeas v0.2.0/go.mod h1:KhxgtF2hb0P191HlY5besjYm6MqTSTj3LSI+M+ByZHc= +cloud.google.com/go/grafeas v0.3.0/go.mod h1:P7hgN24EyONOTMyeJH6DxG4zD7fwiYa5Q6GUgyFSOU8= +cloud.google.com/go/gsuiteaddons v1.3.0/go.mod h1:EUNK/J1lZEZO8yPtykKxLXI6JSVN2rg9bN8SXOa0bgM= +cloud.google.com/go/gsuiteaddons v1.4.0/go.mod h1:rZK5I8hht7u7HxFQcFei0+AtfS9uSushomRlg+3ua1o= +cloud.google.com/go/gsuiteaddons v1.5.0/go.mod h1:TFCClYLd64Eaa12sFVmUyG62tk4mdIsI7pAnSXRkcFo= +cloud.google.com/go/gsuiteaddons v1.6.1/go.mod h1:CodrdOqRZcLp5WOwejHWYBjZvfY0kOphkAKpF/3qdZY= +cloud.google.com/go/iam v0.1.0/go.mod h1:vcUNEa0pEm0qRVpmWepWaFMIAI8/hjB9mO8rNCJtF6c= +cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= +cloud.google.com/go/iam v0.5.0/go.mod h1:wPU9Vt0P4UmCux7mqtRu6jcpPAb74cP1fh50J3QpkUc= +cloud.google.com/go/iam v0.6.0/go.mod h1:+1AH33ueBne5MzYccyMHtEKqLE4/kJOibtffMHDMFMc= +cloud.google.com/go/iam v0.7.0/go.mod h1:H5Br8wRaDGNc8XP3keLc4unfUUZeyH3Sfl9XpQEYOeg= +cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGESjkE= +cloud.google.com/go/iam v0.11.0/go.mod h1:9PiLDanza5D+oWFZiH1uG+RnRCfEGKoyl6yo4cgWZGY= +cloud.google.com/go/iam v0.12.0/go.mod h1:knyHGviacl11zrtZUoDuYpDgLjvr28sLQaG0YB2GYAY= +cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= +cloud.google.com/go/iam v1.0.1/go.mod h1:yR3tmSL8BcZB4bxByRv2jkSIahVmCtfKZwLYGBalRE8= +cloud.google.com/go/iam v1.1.0/go.mod h1:nxdHjaKfCr7fNYx/HJMM8LgiMugmveWlkatear5gVyk= +cloud.google.com/go/iam v1.1.1/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU= +cloud.google.com/go/iap v1.4.0/go.mod h1:RGFwRJdihTINIe4wZ2iCP0zF/qu18ZwyKxrhMhygBEc= +cloud.google.com/go/iap v1.5.0/go.mod h1:UH/CGgKd4KyohZL5Pt0jSKE4m3FR51qg6FKQ/z/Ix9A= +cloud.google.com/go/iap v1.6.0/go.mod h1:NSuvI9C/j7UdjGjIde7t7HBz+QTwBcapPE07+sSRcLk= +cloud.google.com/go/iap v1.7.0/go.mod h1:beqQx56T9O1G1yNPph+spKpNibDlYIiIixiqsQXxLIo= +cloud.google.com/go/iap v1.7.1/go.mod h1:WapEwPc7ZxGt2jFGB/C/bm+hP0Y6NXzOYGjpPnmMS74= +cloud.google.com/go/iap v1.8.1/go.mod h1:sJCbeqg3mvWLqjZNsI6dfAtbbV1DL2Rl7e1mTyXYREQ= +cloud.google.com/go/ids v1.1.0/go.mod h1:WIuwCaYVOzHIj2OhN9HAwvW+DBdmUAdcWlFxRl+KubM= +cloud.google.com/go/ids v1.2.0/go.mod h1:5WXvp4n25S0rA/mQWAg1YEEBBq6/s+7ml1RDCW1IrcY= +cloud.google.com/go/ids v1.3.0/go.mod h1:JBdTYwANikFKaDP6LtW5JAi4gubs57SVNQjemdt6xV4= +cloud.google.com/go/ids v1.4.1/go.mod h1:np41ed8YMU8zOgv53MMMoCntLTn2lF+SUzlM+O3u/jw= +cloud.google.com/go/iot v1.3.0/go.mod h1:r7RGh2B61+B8oz0AGE+J72AhA0G7tdXItODWsaA2oLs= +cloud.google.com/go/iot v1.4.0/go.mod h1:dIDxPOn0UvNDUMD8Ger7FIaTuvMkj+aGk94RPP0iV+g= +cloud.google.com/go/iot v1.5.0/go.mod h1:mpz5259PDl3XJthEmh9+ap0affn/MqNSP4My77Qql9o= +cloud.google.com/go/iot v1.6.0/go.mod h1:IqdAsmE2cTYYNO1Fvjfzo9po179rAtJeVGUvkLN3rLE= +cloud.google.com/go/iot v1.7.1/go.mod h1:46Mgw7ev1k9KqK1ao0ayW9h0lI+3hxeanz+L1zmbbbk= +cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxsnnOA= +cloud.google.com/go/kms v1.5.0/go.mod h1:QJS2YY0eJGBg3mnDfuaCyLauWwBJiHRboYxJ++1xJNg= +cloud.google.com/go/kms v1.6.0/go.mod h1:Jjy850yySiasBUDi6KFUwUv2n1+o7QZFyuUJg6OgjA0= +cloud.google.com/go/kms v1.8.0/go.mod h1:4xFEhYFqvW+4VMELtZyxomGSYtSQKzM178ylFW4jMAg= +cloud.google.com/go/kms v1.9.0/go.mod h1:qb1tPTgfF9RQP8e1wq4cLFErVuTJv7UsSC915J8dh3w= +cloud.google.com/go/kms v1.10.0/go.mod h1:ng3KTUtQQU9bPX3+QGLsflZIHlkbn8amFAMY63m8d24= +cloud.google.com/go/kms v1.10.1/go.mod h1:rIWk/TryCkR59GMC3YtHtXeLzd634lBbKenvyySAyYI= +cloud.google.com/go/kms v1.11.0/go.mod h1:hwdiYC0xjnWsKQQCQQmIQnS9asjYVSK6jtXm+zFqXLM= +cloud.google.com/go/kms v1.12.1/go.mod h1:c9J991h5DTl+kg7gi3MYomh12YEENGrf48ee/N/2CDM= +cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic= +cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= +cloud.google.com/go/language v1.7.0/go.mod h1:DJ6dYN/W+SQOjF8e1hLQXMF21AkH2w9wiPzPCJa2MIE= +cloud.google.com/go/language v1.8.0/go.mod h1:qYPVHf7SPoNNiCL2Dr0FfEFNil1qi3pQEyygwpgVKB8= +cloud.google.com/go/language v1.9.0/go.mod h1:Ns15WooPM5Ad/5no/0n81yUetis74g3zrbeJBE+ptUY= +cloud.google.com/go/language v1.10.1/go.mod h1:CPp94nsdVNiQEt1CNjF5WkTcisLiHPyIbMhvR8H2AW0= +cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8= +cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6t/iPhY2Tyfu08= +cloud.google.com/go/lifesciences v0.8.0/go.mod h1:lFxiEOMqII6XggGbOnKiyZ7IBwoIqA84ClvoezaA/bo= +cloud.google.com/go/lifesciences v0.9.1/go.mod h1:hACAOd1fFbCGLr/+weUKRAJas82Y4vrL3O5326N//Wc= +cloud.google.com/go/logging v1.6.1/go.mod h1:5ZO0mHHbvm8gEmeEUHrmDlTDSu5imF6MUP9OfilNXBw= +cloud.google.com/go/logging v1.7.0/go.mod h1:3xjP2CjkM3ZkO73aj4ASA5wRPGGCRrPIAeNqVNkzY8M= +cloud.google.com/go/longrunning v0.1.1/go.mod h1:UUFxuDWkv22EuY93jjmDMFT5GPQKeFVJBIF6QlTqdsE= +cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI0YSmW3iIlLJc= +cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo= +cloud.google.com/go/longrunning v0.4.2/go.mod h1:OHrnaYyLUV6oqwh0xiS7e5sLQhP1m0QU9R+WhGDMgIQ= +cloud.google.com/go/longrunning v0.5.0/go.mod h1:0JNuqRShmscVAhIACGtskSAWtqtOoPkwP0YF1oVEchc= +cloud.google.com/go/longrunning v0.5.1/go.mod h1:spvimkwdz6SPWKEt/XBij79E9fiTkHSQl/fRUUQJYJc= +cloud.google.com/go/managedidentities v1.3.0/go.mod h1:UzlW3cBOiPrzucO5qWkNkh0w33KFtBJU281hacNvsdE= +cloud.google.com/go/managedidentities v1.4.0/go.mod h1:NWSBYbEMgqmbZsLIyKvxrYbtqOsxY1ZrGM+9RgDqInM= +cloud.google.com/go/managedidentities v1.5.0/go.mod h1:+dWcZ0JlUmpuxpIDfyP5pP5y0bLdRwOS4Lp7gMni/LA= +cloud.google.com/go/managedidentities v1.6.1/go.mod h1:h/irGhTN2SkZ64F43tfGPMbHnypMbu4RB3yl8YcuEak= +cloud.google.com/go/maps v0.1.0/go.mod h1:BQM97WGyfw9FWEmQMpZ5T6cpovXXSd1cGmFma94eubI= +cloud.google.com/go/maps v0.6.0/go.mod h1:o6DAMMfb+aINHz/p/jbcY+mYeXBoZoxTfdSQ8VAJaCw= +cloud.google.com/go/maps v0.7.0/go.mod h1:3GnvVl3cqeSvgMcpRlQidXsPYuDGQ8naBis7MVzpXsY= +cloud.google.com/go/mediatranslation v0.5.0/go.mod h1:jGPUhGTybqsPQn91pNXw0xVHfuJ3leR1wj37oU3y1f4= +cloud.google.com/go/mediatranslation v0.6.0/go.mod h1:hHdBCTYNigsBxshbznuIMFNe5QXEowAuNmmC7h8pu5w= +cloud.google.com/go/mediatranslation v0.7.0/go.mod h1:LCnB/gZr90ONOIQLgSXagp8XUW1ODs2UmUMvcgMfI2I= +cloud.google.com/go/mediatranslation v0.8.1/go.mod h1:L/7hBdEYbYHQJhX2sldtTO5SZZ1C1vkapubj0T2aGig= +cloud.google.com/go/memcache v1.4.0/go.mod h1:rTOfiGZtJX1AaFUrOgsMHX5kAzaTQ8azHiuDoTPzNsE= +cloud.google.com/go/memcache v1.5.0/go.mod h1:dk3fCK7dVo0cUU2c36jKb4VqKPS22BTkf81Xq617aWM= +cloud.google.com/go/memcache v1.6.0/go.mod h1:XS5xB0eQZdHtTuTF9Hf8eJkKtR3pVRCcvJwtm68T3rA= +cloud.google.com/go/memcache v1.7.0/go.mod h1:ywMKfjWhNtkQTxrWxCkCFkoPjLHPW6A7WOTVI8xy3LY= +cloud.google.com/go/memcache v1.9.0/go.mod h1:8oEyzXCu+zo9RzlEaEjHl4KkgjlNDaXbCQeQWlzNFJM= +cloud.google.com/go/memcache v1.10.1/go.mod h1:47YRQIarv4I3QS5+hoETgKO40InqzLP6kpNLvyXuyaA= +cloud.google.com/go/metastore v1.5.0/go.mod h1:2ZNrDcQwghfdtCwJ33nM0+GrBGlVuh8rakL3vdPY3XY= +cloud.google.com/go/metastore v1.6.0/go.mod h1:6cyQTls8CWXzk45G55x57DVQ9gWg7RiH65+YgPsNh9s= +cloud.google.com/go/metastore v1.7.0/go.mod h1:s45D0B4IlsINu87/AsWiEVYbLaIMeUSoxlKKDqBGFS8= +cloud.google.com/go/metastore v1.8.0/go.mod h1:zHiMc4ZUpBiM7twCIFQmJ9JMEkDSyZS9U12uf7wHqSI= +cloud.google.com/go/metastore v1.10.0/go.mod h1:fPEnH3g4JJAk+gMRnrAnoqyv2lpUCqJPWOodSaf45Eo= +cloud.google.com/go/metastore v1.11.1/go.mod h1:uZuSo80U3Wd4zi6C22ZZliOUJ3XeM/MlYi/z5OAOWRA= +cloud.google.com/go/monitoring v1.7.0/go.mod h1:HpYse6kkGo//7p6sT0wsIC6IBDET0RhIsnmlA53dvEk= +cloud.google.com/go/monitoring v1.8.0/go.mod h1:E7PtoMJ1kQXWxPjB6mv2fhC5/15jInuulFdYYtlcvT4= +cloud.google.com/go/monitoring v1.12.0/go.mod h1:yx8Jj2fZNEkL/GYZyTLS4ZtZEZN8WtDEiEqG4kLK50w= +cloud.google.com/go/monitoring v1.13.0/go.mod h1:k2yMBAB1H9JT/QETjNkgdCGD9bPF712XiLTVr+cBrpw= +cloud.google.com/go/monitoring v1.15.1/go.mod h1:lADlSAlFdbqQuwwpaImhsJXu1QSdd3ojypXrFSMr2rM= +cloud.google.com/go/networkconnectivity v1.4.0/go.mod h1:nOl7YL8odKyAOtzNX73/M5/mGZgqqMeryi6UPZTk/rA= +cloud.google.com/go/networkconnectivity v1.5.0/go.mod h1:3GzqJx7uhtlM3kln0+x5wyFvuVH1pIBJjhCpjzSt75o= +cloud.google.com/go/networkconnectivity v1.6.0/go.mod h1:OJOoEXW+0LAxHh89nXd64uGG+FbQoeH8DtxCHVOMlaM= +cloud.google.com/go/networkconnectivity v1.7.0/go.mod h1:RMuSbkdbPwNMQjB5HBWD5MpTBnNm39iAVpC3TmsExt8= +cloud.google.com/go/networkconnectivity v1.10.0/go.mod h1:UP4O4sWXJG13AqrTdQCD9TnLGEbtNRqjuaaA7bNjF5E= +cloud.google.com/go/networkconnectivity v1.11.0/go.mod h1:iWmDD4QF16VCDLXUqvyspJjIEtBR/4zq5hwnY2X3scM= +cloud.google.com/go/networkconnectivity v1.12.1/go.mod h1:PelxSWYM7Sh9/guf8CFhi6vIqf19Ir/sbfZRUwXh92E= +cloud.google.com/go/networkmanagement v1.4.0/go.mod h1:Q9mdLLRn60AsOrPc8rs8iNV6OHXaGcDdsIQe1ohekq8= +cloud.google.com/go/networkmanagement v1.5.0/go.mod h1:ZnOeZ/evzUdUsnvRt792H0uYEnHQEMaz+REhhzJRcf4= +cloud.google.com/go/networkmanagement v1.6.0/go.mod h1:5pKPqyXjB/sgtvB5xqOemumoQNB7y95Q7S+4rjSOPYY= +cloud.google.com/go/networkmanagement v1.8.0/go.mod h1:Ho/BUGmtyEqrttTgWEe7m+8vDdK74ibQc+Be0q7Fof0= +cloud.google.com/go/networksecurity v0.5.0/go.mod h1:xS6fOCoqpVC5zx15Z/MqkfDwH4+m/61A3ODiDV1xmiQ= +cloud.google.com/go/networksecurity v0.6.0/go.mod h1:Q5fjhTr9WMI5mbpRYEbiexTzROf7ZbDzvzCrNl14nyU= +cloud.google.com/go/networksecurity v0.7.0/go.mod h1:mAnzoxx/8TBSyXEeESMy9OOYwo1v+gZ5eMRnsT5bC8k= +cloud.google.com/go/networksecurity v0.8.0/go.mod h1:B78DkqsxFG5zRSVuwYFRZ9Xz8IcQ5iECsNrPn74hKHU= +cloud.google.com/go/networksecurity v0.9.1/go.mod h1:MCMdxOKQ30wsBI1eI659f9kEp4wuuAueoC9AJKSPWZQ= +cloud.google.com/go/notebooks v1.2.0/go.mod h1:9+wtppMfVPUeJ8fIWPOq1UnATHISkGXGqTkxeieQ6UY= +cloud.google.com/go/notebooks v1.3.0/go.mod h1:bFR5lj07DtCPC7YAAJ//vHskFBxA5JzYlH68kXVdk34= +cloud.google.com/go/notebooks v1.4.0/go.mod h1:4QPMngcwmgb6uw7Po99B2xv5ufVoIQ7nOGDyL4P8AgA= +cloud.google.com/go/notebooks v1.5.0/go.mod h1:q8mwhnP9aR8Hpfnrc5iN5IBhrXUy8S2vuYs+kBJ/gu0= +cloud.google.com/go/notebooks v1.7.0/go.mod h1:PVlaDGfJgj1fl1S3dUwhFMXFgfYGhYQt2164xOMONmE= +cloud.google.com/go/notebooks v1.8.0/go.mod h1:Lq6dYKOYOWUCTvw5t2q1gp1lAp0zxAxRycayS0iJcqQ= +cloud.google.com/go/notebooks v1.9.1/go.mod h1:zqG9/gk05JrzgBt4ghLzEepPHNwE5jgPcHZRKhlC1A8= +cloud.google.com/go/optimization v1.1.0/go.mod h1:5po+wfvX5AQlPznyVEZjGJTMr4+CAkJf2XSTQOOl9l4= +cloud.google.com/go/optimization v1.2.0/go.mod h1:Lr7SOHdRDENsh+WXVmQhQTrzdu9ybg0NecjHidBq6xs= +cloud.google.com/go/optimization v1.3.1/go.mod h1:IvUSefKiwd1a5p0RgHDbWCIbDFgKuEdB+fPPuP0IDLI= +cloud.google.com/go/optimization v1.4.1/go.mod h1:j64vZQP7h9bO49m2rVaTVoNM0vEBEN5eKPUPbZyXOrk= +cloud.google.com/go/orchestration v1.3.0/go.mod h1:Sj5tq/JpWiB//X/q3Ngwdl5K7B7Y0KZ7bfv0wL6fqVA= +cloud.google.com/go/orchestration v1.4.0/go.mod h1:6W5NLFWs2TlniBphAViZEVhrXRSMgUGDfW7vrWKvsBk= +cloud.google.com/go/orchestration v1.6.0/go.mod h1:M62Bevp7pkxStDfFfTuCOaXgaaqRAga1yKyoMtEoWPQ= +cloud.google.com/go/orchestration v1.8.1/go.mod h1:4sluRF3wgbYVRqz7zJ1/EUNc90TTprliq9477fGobD8= +cloud.google.com/go/orgpolicy v1.4.0/go.mod h1:xrSLIV4RePWmP9P3tBl8S93lTmlAxjm06NSm2UTmKvE= +cloud.google.com/go/orgpolicy v1.5.0/go.mod h1:hZEc5q3wzwXJaKrsx5+Ewg0u1LxJ51nNFlext7Tanwc= +cloud.google.com/go/orgpolicy v1.10.0/go.mod h1:w1fo8b7rRqlXlIJbVhOMPrwVljyuW5mqssvBtU18ONc= +cloud.google.com/go/orgpolicy v1.11.0/go.mod h1:2RK748+FtVvnfuynxBzdnyu7sygtoZa1za/0ZfpOs1M= +cloud.google.com/go/orgpolicy v1.11.1/go.mod h1:8+E3jQcpZJQliP+zaFfayC2Pg5bmhuLK755wKhIIUCE= +cloud.google.com/go/osconfig v1.7.0/go.mod h1:oVHeCeZELfJP7XLxcBGTMBvRO+1nQ5tFG9VQTmYS2Fs= +cloud.google.com/go/osconfig v1.8.0/go.mod h1:EQqZLu5w5XA7eKizepumcvWx+m8mJUhEwiPqWiZeEdg= +cloud.google.com/go/osconfig v1.9.0/go.mod h1:Yx+IeIZJ3bdWmzbQU4fxNl8xsZ4amB+dygAwFPlvnNo= +cloud.google.com/go/osconfig v1.10.0/go.mod h1:uMhCzqC5I8zfD9zDEAfvgVhDS8oIjySWh+l4WK6GnWw= +cloud.google.com/go/osconfig v1.11.0/go.mod h1:aDICxrur2ogRd9zY5ytBLV89KEgT2MKB2L/n6x1ooPw= +cloud.google.com/go/osconfig v1.12.0/go.mod h1:8f/PaYzoS3JMVfdfTubkowZYGmAhUCjjwnjqWI7NVBc= +cloud.google.com/go/osconfig v1.12.1/go.mod h1:4CjBxND0gswz2gfYRCUoUzCm9zCABp91EeTtWXyz0tE= +cloud.google.com/go/oslogin v1.4.0/go.mod h1:YdgMXWRaElXz/lDk1Na6Fh5orF7gvmJ0FGLIs9LId4E= +cloud.google.com/go/oslogin v1.5.0/go.mod h1:D260Qj11W2qx/HVF29zBg+0fd6YCSjSqLUkY/qEenQU= +cloud.google.com/go/oslogin v1.6.0/go.mod h1:zOJ1O3+dTU8WPlGEkFSh7qeHPPSoxrcMbbK1Nm2iX70= +cloud.google.com/go/oslogin v1.7.0/go.mod h1:e04SN0xO1UNJ1M5GP0vzVBFicIe4O53FOfcixIqTyXo= +cloud.google.com/go/oslogin v1.9.0/go.mod h1:HNavntnH8nzrn8JCTT5fj18FuJLFJc4NaZJtBnQtKFs= +cloud.google.com/go/oslogin v1.10.1/go.mod h1:x692z7yAue5nE7CsSnoG0aaMbNoRJRXO4sn73R+ZqAs= +cloud.google.com/go/phishingprotection v0.5.0/go.mod h1:Y3HZknsK9bc9dMi+oE8Bim0lczMU6hrX0UpADuMefr0= +cloud.google.com/go/phishingprotection v0.6.0/go.mod h1:9Y3LBLgy0kDTcYET8ZH3bq/7qni15yVUoAxiFxnlSUA= +cloud.google.com/go/phishingprotection v0.7.0/go.mod h1:8qJI4QKHoda/sb/7/YmMQ2omRLSLYSu9bU0EKCNI+Lk= +cloud.google.com/go/phishingprotection v0.8.1/go.mod h1:AxonW7GovcA8qdEk13NfHq9hNx5KPtfxXNeUxTDxB6I= +cloud.google.com/go/policytroubleshooter v1.3.0/go.mod h1:qy0+VwANja+kKrjlQuOzmlvscn4RNsAc0e15GGqfMxg= +cloud.google.com/go/policytroubleshooter v1.4.0/go.mod h1:DZT4BcRw3QoO8ota9xw/LKtPa8lKeCByYeKTIf/vxdE= +cloud.google.com/go/policytroubleshooter v1.5.0/go.mod h1:Rz1WfV+1oIpPdN2VvvuboLVRsB1Hclg3CKQ53j9l8vw= +cloud.google.com/go/policytroubleshooter v1.6.0/go.mod h1:zYqaPTsmfvpjm5ULxAyD/lINQxJ0DDsnWOP/GZ7xzBc= +cloud.google.com/go/policytroubleshooter v1.7.1/go.mod h1:0NaT5v3Ag1M7U5r0GfDCpUFkWd9YqpubBWsQlhanRv0= +cloud.google.com/go/privatecatalog v0.5.0/go.mod h1:XgosMUvvPyxDjAVNDYxJ7wBW8//hLDDYmnsNcMGq1K0= +cloud.google.com/go/privatecatalog v0.6.0/go.mod h1:i/fbkZR0hLN29eEWiiwue8Pb+GforiEIBnV9yrRUOKI= +cloud.google.com/go/privatecatalog v0.7.0/go.mod h1:2s5ssIFO69F5csTXcwBP7NPFTZvps26xGzvQ2PQaBYg= +cloud.google.com/go/privatecatalog v0.8.0/go.mod h1:nQ6pfaegeDAq/Q5lrfCQzQLhubPiZhSaNhIgfJlnIXs= +cloud.google.com/go/privatecatalog v0.9.1/go.mod h1:0XlDXW2unJXdf9zFz968Hp35gl/bhF4twwpXZAW50JA= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/pubsub v1.26.0/go.mod h1:QgBH3U/jdJy/ftjPhTkyXNj543Tin1pRYcdcPRnFIRI= +cloud.google.com/go/pubsub v1.27.1/go.mod h1:hQN39ymbV9geqBnfQq6Xf63yNhUAhv9CZhzp5O6qsW0= +cloud.google.com/go/pubsub v1.28.0/go.mod h1:vuXFpwaVoIPQMGXqRyUQigu/AX1S3IWugR9xznmcXX8= +cloud.google.com/go/pubsub v1.30.0/go.mod h1:qWi1OPS0B+b5L+Sg6Gmc9zD1Y+HaM0MdUr7LsupY1P4= +cloud.google.com/go/pubsub v1.32.0/go.mod h1:f+w71I33OMyxf9VpMVcZbnG5KSUkCOUHYpFd5U1GdRc= +cloud.google.com/go/pubsublite v1.5.0/go.mod h1:xapqNQ1CuLfGi23Yda/9l4bBCKz/wC3KIJ5gKcxveZg= +cloud.google.com/go/pubsublite v1.6.0/go.mod h1:1eFCS0U11xlOuMFV/0iBqw3zP12kddMeCbj/F3FSj9k= +cloud.google.com/go/pubsublite v1.7.0/go.mod h1:8hVMwRXfDfvGm3fahVbtDbiLePT3gpoiJYJY+vxWxVM= +cloud.google.com/go/pubsublite v1.8.1/go.mod h1:fOLdU4f5xldK4RGJrBMm+J7zMWNj/k4PxwEZXy39QS0= +cloud.google.com/go/recaptchaenterprise v1.3.1/go.mod h1:OdD+q+y4XGeAlxRaMn1Y7/GveP6zmq76byL6tjPE7d4= +cloud.google.com/go/recaptchaenterprise/v2 v2.1.0/go.mod h1:w9yVqajwroDNTfGuhmOjPDN//rZGySaf6PtFVcSCa7o= +cloud.google.com/go/recaptchaenterprise/v2 v2.2.0/go.mod h1:/Zu5jisWGeERrd5HnlS3EUGb/D335f9k51B/FVil0jk= +cloud.google.com/go/recaptchaenterprise/v2 v2.3.0/go.mod h1:O9LwGCjrhGHBQET5CA7dd5NwwNQUErSgEDit1DLNTdo= +cloud.google.com/go/recaptchaenterprise/v2 v2.4.0/go.mod h1:Am3LHfOuBstrLrNCBrlI5sbwx9LBg3te2N6hGvHn2mE= +cloud.google.com/go/recaptchaenterprise/v2 v2.5.0/go.mod h1:O8LzcHXN3rz0j+LBC91jrwI3R+1ZSZEWrfL7XHgNo9U= +cloud.google.com/go/recaptchaenterprise/v2 v2.6.0/go.mod h1:RPauz9jeLtB3JVzg6nCbe12qNoaa8pXc4d/YukAmcnA= +cloud.google.com/go/recaptchaenterprise/v2 v2.7.0/go.mod h1:19wVj/fs5RtYtynAPJdDTb69oW0vNHYDBTbB4NvMD9c= +cloud.google.com/go/recaptchaenterprise/v2 v2.7.2/go.mod h1:kR0KjsJS7Jt1YSyWFkseQ756D45kaYNTlDPPaRAvDBU= +cloud.google.com/go/recommendationengine v0.5.0/go.mod h1:E5756pJcVFeVgaQv3WNpImkFP8a+RptV6dDLGPILjvg= +cloud.google.com/go/recommendationengine v0.6.0/go.mod h1:08mq2umu9oIqc7tDy8sx+MNJdLG0fUi3vaSVbztHgJ4= +cloud.google.com/go/recommendationengine v0.7.0/go.mod h1:1reUcE3GIu6MeBz/h5xZJqNLuuVjNg1lmWMPyjatzac= +cloud.google.com/go/recommendationengine v0.8.1/go.mod h1:MrZihWwtFYWDzE6Hz5nKcNz3gLizXVIDI/o3G1DLcrE= +cloud.google.com/go/recommender v1.5.0/go.mod h1:jdoeiBIVrJe9gQjwd759ecLJbxCDED4A6p+mqoqDvTg= +cloud.google.com/go/recommender v1.6.0/go.mod h1:+yETpm25mcoiECKh9DEScGzIRyDKpZ0cEhWGo+8bo+c= +cloud.google.com/go/recommender v1.7.0/go.mod h1:XLHs/W+T8olwlGOgfQenXBTbIseGclClff6lhFVe9Bs= +cloud.google.com/go/recommender v1.8.0/go.mod h1:PkjXrTT05BFKwxaUxQmtIlrtj0kph108r02ZZQ5FE70= +cloud.google.com/go/recommender v1.9.0/go.mod h1:PnSsnZY7q+VL1uax2JWkt/UegHssxjUVVCrX52CuEmQ= +cloud.google.com/go/recommender v1.10.1/go.mod h1:XFvrE4Suqn5Cq0Lf+mCP6oBHD/yRMA8XxP5sb7Q7gpA= +cloud.google.com/go/redis v1.7.0/go.mod h1:V3x5Jq1jzUcg+UNsRvdmsfuFnit1cfe3Z/PGyq/lm4Y= +cloud.google.com/go/redis v1.8.0/go.mod h1:Fm2szCDavWzBk2cDKxrkmWBqoCiL1+Ctwq7EyqBCA/A= +cloud.google.com/go/redis v1.9.0/go.mod h1:HMYQuajvb2D0LvMgZmLDZW8V5aOC/WxstZHiy4g8OiA= +cloud.google.com/go/redis v1.10.0/go.mod h1:ThJf3mMBQtW18JzGgh41/Wld6vnDDc/F/F35UolRZPM= +cloud.google.com/go/redis v1.11.0/go.mod h1:/X6eicana+BWcUda5PpwZC48o37SiFVTFSs0fWAJ7uQ= +cloud.google.com/go/redis v1.13.1/go.mod h1:VP7DGLpE91M6bcsDdMuyCm2hIpB6Vp2hI090Mfd1tcg= +cloud.google.com/go/resourcemanager v1.3.0/go.mod h1:bAtrTjZQFJkiWTPDb1WBjzvc6/kifjj4QBYuKCCoqKA= +cloud.google.com/go/resourcemanager v1.4.0/go.mod h1:MwxuzkumyTX7/a3n37gmsT3py7LIXwrShilPh3P1tR0= +cloud.google.com/go/resourcemanager v1.5.0/go.mod h1:eQoXNAiAvCf5PXxWxXjhKQoTMaUSNrEfg+6qdf/wots= +cloud.google.com/go/resourcemanager v1.6.0/go.mod h1:YcpXGRs8fDzcUl1Xw8uOVmI8JEadvhRIkoXXUNVYcVo= +cloud.google.com/go/resourcemanager v1.7.0/go.mod h1:HlD3m6+bwhzj9XCouqmeiGuni95NTrExfhoSrkC/3EI= +cloud.google.com/go/resourcemanager v1.9.1/go.mod h1:dVCuosgrh1tINZ/RwBufr8lULmWGOkPS8gL5gqyjdT8= +cloud.google.com/go/resourcesettings v1.3.0/go.mod h1:lzew8VfESA5DQ8gdlHwMrqZs1S9V87v3oCnKCWoOuQU= +cloud.google.com/go/resourcesettings v1.4.0/go.mod h1:ldiH9IJpcrlC3VSuCGvjR5of/ezRrOxFtpJoJo5SmXg= +cloud.google.com/go/resourcesettings v1.5.0/go.mod h1:+xJF7QSG6undsQDfsCJyqWXyBwUoJLhetkRMDRnIoXA= +cloud.google.com/go/resourcesettings v1.6.1/go.mod h1:M7mk9PIZrC5Fgsu1kZJci6mpgN8o0IUzVx3eJU3y4Jw= +cloud.google.com/go/retail v1.8.0/go.mod h1:QblKS8waDmNUhghY2TI9O3JLlFk8jybHeV4BF19FrE4= +cloud.google.com/go/retail v1.9.0/go.mod h1:g6jb6mKuCS1QKnH/dpu7isX253absFl6iE92nHwlBUY= +cloud.google.com/go/retail v1.10.0/go.mod h1:2gDk9HsL4HMS4oZwz6daui2/jmKvqShXKQuB2RZ+cCc= +cloud.google.com/go/retail v1.11.0/go.mod h1:MBLk1NaWPmh6iVFSz9MeKG/Psyd7TAgm6y/9L2B4x9Y= +cloud.google.com/go/retail v1.12.0/go.mod h1:UMkelN/0Z8XvKymXFbD4EhFJlYKRx1FGhQkVPU5kF14= +cloud.google.com/go/retail v1.14.1/go.mod h1:y3Wv3Vr2k54dLNIrCzenyKG8g8dhvhncT2NcNjb/6gE= +cloud.google.com/go/run v0.2.0/go.mod h1:CNtKsTA1sDcnqqIFR3Pb5Tq0usWxJJvsWOCPldRU3Do= +cloud.google.com/go/run v0.3.0/go.mod h1:TuyY1+taHxTjrD0ZFk2iAR+xyOXEA0ztb7U3UNA0zBo= +cloud.google.com/go/run v0.8.0/go.mod h1:VniEnuBwqjigv0A7ONfQUaEItaiCRVujlMqerPPiktM= +cloud.google.com/go/run v0.9.0/go.mod h1:Wwu+/vvg8Y+JUApMwEDfVfhetv30hCG4ZwDR/IXl2Qg= +cloud.google.com/go/scheduler v1.4.0/go.mod h1:drcJBmxF3aqZJRhmkHQ9b3uSSpQoltBPGPxGAWROx6s= +cloud.google.com/go/scheduler v1.5.0/go.mod h1:ri073ym49NW3AfT6DZi21vLZrG07GXr5p3H1KxN5QlI= +cloud.google.com/go/scheduler v1.6.0/go.mod h1:SgeKVM7MIwPn3BqtcBntpLyrIJftQISRrYB5ZtT+KOk= +cloud.google.com/go/scheduler v1.7.0/go.mod h1:jyCiBqWW956uBjjPMMuX09n3x37mtyPJegEWKxRsn44= +cloud.google.com/go/scheduler v1.8.0/go.mod h1:TCET+Y5Gp1YgHT8py4nlg2Sew8nUHMqcpousDgXJVQc= +cloud.google.com/go/scheduler v1.9.0/go.mod h1:yexg5t+KSmqu+njTIh3b7oYPheFtBWGcbVUYF1GGMIc= +cloud.google.com/go/scheduler v1.10.1/go.mod h1:R63Ldltd47Bs4gnhQkmNDse5w8gBRrhObZ54PxgR2Oo= +cloud.google.com/go/secretmanager v1.6.0/go.mod h1:awVa/OXF6IiyaU1wQ34inzQNc4ISIDIrId8qE5QGgKA= +cloud.google.com/go/secretmanager v1.8.0/go.mod h1:hnVgi/bN5MYHd3Gt0SPuTPPp5ENina1/LxM+2W9U9J4= +cloud.google.com/go/secretmanager v1.9.0/go.mod h1:b71qH2l1yHmWQHt9LC80akm86mX8AL6X1MA01dW8ht4= +cloud.google.com/go/secretmanager v1.10.0/go.mod h1:MfnrdvKMPNra9aZtQFvBcvRU54hbPD8/HayQdlUgJpU= +cloud.google.com/go/secretmanager v1.11.1/go.mod h1:znq9JlXgTNdBeQk9TBW/FnR/W4uChEKGeqQWAJ8SXFw= +cloud.google.com/go/security v1.5.0/go.mod h1:lgxGdyOKKjHL4YG3/YwIL2zLqMFCKs0UbQwgyZmfJl4= +cloud.google.com/go/security v1.7.0/go.mod h1:mZklORHl6Bg7CNnnjLH//0UlAlaXqiG7Lb9PsPXLfD0= +cloud.google.com/go/security v1.8.0/go.mod h1:hAQOwgmaHhztFhiQ41CjDODdWP0+AE1B3sX4OFlq+GU= +cloud.google.com/go/security v1.9.0/go.mod h1:6Ta1bO8LXI89nZnmnsZGp9lVoVWXqsVbIq/t9dzI+2Q= +cloud.google.com/go/security v1.10.0/go.mod h1:QtOMZByJVlibUT2h9afNDWRZ1G96gVywH8T5GUSb9IA= +cloud.google.com/go/security v1.12.0/go.mod h1:rV6EhrpbNHrrxqlvW0BWAIawFWq3X90SduMJdFwtLB8= +cloud.google.com/go/security v1.13.0/go.mod h1:Q1Nvxl1PAgmeW0y3HTt54JYIvUdtcpYKVfIB8AOMZ+0= +cloud.google.com/go/security v1.15.1/go.mod h1:MvTnnbsWnehoizHi09zoiZob0iCHVcL4AUBj76h9fXA= +cloud.google.com/go/securitycenter v1.13.0/go.mod h1:cv5qNAqjY84FCN6Y9z28WlkKXyWsgLO832YiWwkCWcU= +cloud.google.com/go/securitycenter v1.14.0/go.mod h1:gZLAhtyKv85n52XYWt6RmeBdydyxfPeTrpToDPw4Auc= +cloud.google.com/go/securitycenter v1.15.0/go.mod h1:PeKJ0t8MoFmmXLXWm41JidyzI3PJjd8sXWaVqg43WWk= +cloud.google.com/go/securitycenter v1.16.0/go.mod h1:Q9GMaLQFUD+5ZTabrbujNWLtSLZIZF7SAR0wWECrjdk= +cloud.google.com/go/securitycenter v1.18.1/go.mod h1:0/25gAzCM/9OL9vVx4ChPeM/+DlfGQJDwBy/UC8AKK0= +cloud.google.com/go/securitycenter v1.19.0/go.mod h1:LVLmSg8ZkkyaNy4u7HCIshAngSQ8EcIRREP3xBnyfag= +cloud.google.com/go/securitycenter v1.23.0/go.mod h1:8pwQ4n+Y9WCWM278R8W3nF65QtY172h4S8aXyI9/hsQ= +cloud.google.com/go/servicecontrol v1.4.0/go.mod h1:o0hUSJ1TXJAmi/7fLJAedOovnujSEvjKCAFNXPQ1RaU= +cloud.google.com/go/servicecontrol v1.5.0/go.mod h1:qM0CnXHhyqKVuiZnGKrIurvVImCs8gmqWsDoqe9sU1s= +cloud.google.com/go/servicecontrol v1.10.0/go.mod h1:pQvyvSRh7YzUF2efw7H87V92mxU8FnFDawMClGCNuAA= +cloud.google.com/go/servicecontrol v1.11.0/go.mod h1:kFmTzYzTUIuZs0ycVqRHNaNhgR+UMUpw9n02l/pY+mc= +cloud.google.com/go/servicecontrol v1.11.1/go.mod h1:aSnNNlwEFBY+PWGQ2DoM0JJ/QUXqV5/ZD9DOLB7SnUk= +cloud.google.com/go/servicedirectory v1.4.0/go.mod h1:gH1MUaZCgtP7qQiI+F+A+OpeKF/HQWgtAddhTbhL2bs= +cloud.google.com/go/servicedirectory v1.5.0/go.mod h1:QMKFL0NUySbpZJ1UZs3oFAmdvVxhhxB6eJ/Vlp73dfg= +cloud.google.com/go/servicedirectory v1.6.0/go.mod h1:pUlbnWsLH9c13yGkxCmfumWEPjsRs1RlmJ4pqiNjVL4= +cloud.google.com/go/servicedirectory v1.7.0/go.mod h1:5p/U5oyvgYGYejufvxhgwjL8UVXjkuw7q5XcG10wx1U= +cloud.google.com/go/servicedirectory v1.8.0/go.mod h1:srXodfhY1GFIPvltunswqXpVxFPpZjf8nkKQT7XcXaY= +cloud.google.com/go/servicedirectory v1.9.0/go.mod h1:29je5JjiygNYlmsGz8k6o+OZ8vd4f//bQLtvzkPPT/s= +cloud.google.com/go/servicedirectory v1.10.1/go.mod h1:Xv0YVH8s4pVOwfM/1eMTl0XJ6bzIOSLDt8f8eLaGOxQ= +cloud.google.com/go/servicemanagement v1.4.0/go.mod h1:d8t8MDbezI7Z2R1O/wu8oTggo3BI2GKYbdG4y/SJTco= +cloud.google.com/go/servicemanagement v1.5.0/go.mod h1:XGaCRe57kfqu4+lRxaFEAuqmjzF0r+gWHjWqKqBvKFo= +cloud.google.com/go/servicemanagement v1.6.0/go.mod h1:aWns7EeeCOtGEX4OvZUWCCJONRZeFKiptqKf1D0l/Jc= +cloud.google.com/go/servicemanagement v1.8.0/go.mod h1:MSS2TDlIEQD/fzsSGfCdJItQveu9NXnUniTrq/L8LK4= +cloud.google.com/go/serviceusage v1.3.0/go.mod h1:Hya1cozXM4SeSKTAgGXgj97GlqUvF5JaoXacR1JTP/E= +cloud.google.com/go/serviceusage v1.4.0/go.mod h1:SB4yxXSaYVuUBYUml6qklyONXNLt83U0Rb+CXyhjEeU= +cloud.google.com/go/serviceusage v1.5.0/go.mod h1:w8U1JvqUqwJNPEOTQjrMHkw3IaIFLoLsPLvsE3xueec= +cloud.google.com/go/serviceusage v1.6.0/go.mod h1:R5wwQcbOWsyuOfbP9tGdAnCAc6B9DRwPG1xtWMDeuPA= +cloud.google.com/go/shell v1.3.0/go.mod h1:VZ9HmRjZBsjLGXusm7K5Q5lzzByZmJHf1d0IWHEN5X4= +cloud.google.com/go/shell v1.4.0/go.mod h1:HDxPzZf3GkDdhExzD/gs8Grqk+dmYcEjGShZgYa9URw= +cloud.google.com/go/shell v1.6.0/go.mod h1:oHO8QACS90luWgxP3N9iZVuEiSF84zNyLytb+qE2f9A= +cloud.google.com/go/shell v1.7.1/go.mod h1:u1RaM+huXFaTojTbW4g9P5emOrrmLE69KrxqQahKn4g= +cloud.google.com/go/spanner v1.41.0/go.mod h1:MLYDBJR/dY4Wt7ZaMIQ7rXOTLjYrmxLE/5ve9vFfWos= +cloud.google.com/go/spanner v1.44.0/go.mod h1:G8XIgYdOK+Fbcpbs7p2fiprDw4CaZX63whnSMLVBxjk= +cloud.google.com/go/spanner v1.45.0/go.mod h1:FIws5LowYz8YAE1J8fOS7DJup8ff7xJeetWEo5REA2M= +cloud.google.com/go/spanner v1.47.0/go.mod h1:IXsJwVW2j4UKs0eYDqodab6HgGuA1bViSqW4uH9lfUI= +cloud.google.com/go/speech v1.6.0/go.mod h1:79tcr4FHCimOp56lwC01xnt/WPJZc4v3gzyT7FoBkCM= +cloud.google.com/go/speech v1.7.0/go.mod h1:KptqL+BAQIhMsj1kOP2la5DSEEerPDuOP/2mmkhHhZQ= +cloud.google.com/go/speech v1.8.0/go.mod h1:9bYIl1/tjsAnMgKGHKmBZzXKEkGgtU+MpdDPTE9f7y0= +cloud.google.com/go/speech v1.9.0/go.mod h1:xQ0jTcmnRFFM2RfX/U+rk6FQNUF6DQlydUSyoooSpco= +cloud.google.com/go/speech v1.14.1/go.mod h1:gEosVRPJ9waG7zqqnsHpYTOoAS4KouMRLDFMekpJ0J0= +cloud.google.com/go/speech v1.15.0/go.mod h1:y6oH7GhqCaZANH7+Oe0BhgIogsNInLlz542tg3VqeYI= +cloud.google.com/go/speech v1.17.1/go.mod h1:8rVNzU43tQvxDaGvqOhpDqgkJTFowBpDvCJ14kGlJYo= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= +cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= +cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc= +cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= +cloud.google.com/go/storage v1.28.1/go.mod h1:Qnisd4CqDdo6BGs2AD5LLnEsmSQ80wQ5ogcBBKhU86Y= +cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4= +cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7BiccwkR7+P7gN8E= +cloud.google.com/go/storagetransfer v1.5.0/go.mod h1:dxNzUopWy7RQevYFHewchb29POFv3/AaBgnhqzqiK0w= +cloud.google.com/go/storagetransfer v1.6.0/go.mod h1:y77xm4CQV/ZhFZH75PLEXY0ROiS7Gh6pSKrM8dJyg6I= +cloud.google.com/go/storagetransfer v1.7.0/go.mod h1:8Giuj1QNb1kfLAiWM1bN6dHzfdlDAVC9rv9abHot2W4= +cloud.google.com/go/storagetransfer v1.8.0/go.mod h1:JpegsHHU1eXg7lMHkvf+KE5XDJ7EQu0GwNJbbVGanEw= +cloud.google.com/go/storagetransfer v1.10.0/go.mod h1:DM4sTlSmGiNczmV6iZyceIh2dbs+7z2Ayg6YAiQlYfA= +cloud.google.com/go/talent v1.1.0/go.mod h1:Vl4pt9jiHKvOgF9KoZo6Kob9oV4lwd/ZD5Cto54zDRw= +cloud.google.com/go/talent v1.2.0/go.mod h1:MoNF9bhFQbiJ6eFD3uSsg0uBALw4n4gaCaEjBw9zo8g= +cloud.google.com/go/talent v1.3.0/go.mod h1:CmcxwJ/PKfRgd1pBjQgU6W3YBwiewmUzQYH5HHmSCmM= +cloud.google.com/go/talent v1.4.0/go.mod h1:ezFtAgVuRf8jRsvyE6EwmbTK5LKciD4KVnHuDEFmOOA= +cloud.google.com/go/talent v1.5.0/go.mod h1:G+ODMj9bsasAEJkQSzO2uHQWXHHXUomArjWQQYkqK6c= +cloud.google.com/go/talent v1.6.2/go.mod h1:CbGvmKCG61mkdjcqTcLOkb2ZN1SrQI8MDyma2l7VD24= +cloud.google.com/go/texttospeech v1.4.0/go.mod h1:FX8HQHA6sEpJ7rCMSfXuzBcysDAuWusNNNvN9FELDd8= +cloud.google.com/go/texttospeech v1.5.0/go.mod h1:oKPLhR4n4ZdQqWKURdwxMy0uiTS1xU161C8W57Wkea4= +cloud.google.com/go/texttospeech v1.6.0/go.mod h1:YmwmFT8pj1aBblQOI3TfKmwibnsfvhIBzPXcW4EBovc= +cloud.google.com/go/texttospeech v1.7.1/go.mod h1:m7QfG5IXxeneGqTapXNxv2ItxP/FS0hCZBwXYqucgSk= +cloud.google.com/go/tpu v1.3.0/go.mod h1:aJIManG0o20tfDQlRIej44FcwGGl/cD0oiRyMKG19IQ= +cloud.google.com/go/tpu v1.4.0/go.mod h1:mjZaX8p0VBgllCzF6wcU2ovUXN9TONFLd7iz227X2Xg= +cloud.google.com/go/tpu v1.5.0/go.mod h1:8zVo1rYDFuW2l4yZVY0R0fb/v44xLh3llq7RuV61fPM= +cloud.google.com/go/tpu v1.6.1/go.mod h1:sOdcHVIgDEEOKuqUoi6Fq53MKHJAtOwtz0GuKsWSH3E= +cloud.google.com/go/trace v1.3.0/go.mod h1:FFUE83d9Ca57C+K8rDl/Ih8LwOzWIV1krKgxg6N0G28= +cloud.google.com/go/trace v1.4.0/go.mod h1:UG0v8UBqzusp+z63o7FK74SdFE+AXpCLdFb1rshXG+Y= +cloud.google.com/go/trace v1.8.0/go.mod h1:zH7vcsbAhklH8hWFig58HvxcxyQbaIqMarMg9hn5ECA= +cloud.google.com/go/trace v1.9.0/go.mod h1:lOQqpE5IaWY0Ixg7/r2SjixMuc6lfTFeO4QGM4dQWOk= +cloud.google.com/go/trace v1.10.1/go.mod h1:gbtL94KE5AJLH3y+WVpfWILmqgc6dXcqgNXdOPAQTYk= +cloud.google.com/go/translate v1.3.0/go.mod h1:gzMUwRjvOqj5i69y/LYLd8RrNQk+hOmIXTi9+nb3Djs= +cloud.google.com/go/translate v1.4.0/go.mod h1:06Dn/ppvLD6WvA5Rhdp029IX2Mi3Mn7fpMRLPvXT5Wg= +cloud.google.com/go/translate v1.5.0/go.mod h1:29YDSYveqqpA1CQFD7NQuP49xymq17RXNaUDdc0mNu0= +cloud.google.com/go/translate v1.6.0/go.mod h1:lMGRudH1pu7I3n3PETiOB2507gf3HnfLV8qlkHZEyos= +cloud.google.com/go/translate v1.7.0/go.mod h1:lMGRudH1pu7I3n3PETiOB2507gf3HnfLV8qlkHZEyos= +cloud.google.com/go/translate v1.8.1/go.mod h1:d1ZH5aaOA0CNhWeXeC8ujd4tdCFw8XoNWRljklu5RHs= +cloud.google.com/go/video v1.8.0/go.mod h1:sTzKFc0bUSByE8Yoh8X0mn8bMymItVGPfTuUBUyRgxk= +cloud.google.com/go/video v1.9.0/go.mod h1:0RhNKFRF5v92f8dQt0yhaHrEuH95m068JYOvLZYnJSw= +cloud.google.com/go/video v1.12.0/go.mod h1:MLQew95eTuaNDEGriQdcYn0dTwf9oWiA4uYebxM5kdg= +cloud.google.com/go/video v1.13.0/go.mod h1:ulzkYlYgCp15N2AokzKjy7MQ9ejuynOJdf1tR5lGthk= +cloud.google.com/go/video v1.14.0/go.mod h1:SkgaXwT+lIIAKqWAJfktHT/RbgjSuY6DobxEp0C5yTQ= +cloud.google.com/go/video v1.15.0/go.mod h1:SkgaXwT+lIIAKqWAJfktHT/RbgjSuY6DobxEp0C5yTQ= +cloud.google.com/go/video v1.17.1/go.mod h1:9qmqPqw/Ib2tLqaeHgtakU+l5TcJxCJbhFXM7UJjVzU= +cloud.google.com/go/videointelligence v1.6.0/go.mod h1:w0DIDlVRKtwPCn/C4iwZIJdvC69yInhW0cfi+p546uU= +cloud.google.com/go/videointelligence v1.7.0/go.mod h1:k8pI/1wAhjznARtVT9U1llUaFNPh7muw8QyOUpavru4= +cloud.google.com/go/videointelligence v1.8.0/go.mod h1:dIcCn4gVDdS7yte/w+koiXn5dWVplOZkE+xwG9FgK+M= +cloud.google.com/go/videointelligence v1.9.0/go.mod h1:29lVRMPDYHikk3v8EdPSaL8Ku+eMzDljjuvRs105XoU= +cloud.google.com/go/videointelligence v1.10.0/go.mod h1:LHZngX1liVtUhZvi2uNS0VQuOzNi2TkY1OakiuoUOjU= +cloud.google.com/go/videointelligence v1.11.1/go.mod h1:76xn/8InyQHarjTWsBR058SmlPCwQjgcvoW0aZykOvo= +cloud.google.com/go/vision v1.2.0/go.mod h1:SmNwgObm5DpFBme2xpyOyasvBc1aPdjvMk2bBk0tKD0= +cloud.google.com/go/vision/v2 v2.2.0/go.mod h1:uCdV4PpN1S0jyCyq8sIM42v2Y6zOLkZs+4R9LrGYwFo= +cloud.google.com/go/vision/v2 v2.3.0/go.mod h1:UO61abBx9QRMFkNBbf1D8B1LXdS2cGiiCRx0vSpZoUo= +cloud.google.com/go/vision/v2 v2.4.0/go.mod h1:VtI579ll9RpVTrdKdkMzckdnwMyX2JILb+MhPqRbPsY= +cloud.google.com/go/vision/v2 v2.5.0/go.mod h1:MmaezXOOE+IWa+cS7OhRRLK2cNv1ZL98zhqFFZaaH2E= +cloud.google.com/go/vision/v2 v2.6.0/go.mod h1:158Hes0MvOS9Z/bDMSFpjwsUrZ5fPrdwuyyvKSGAGMY= +cloud.google.com/go/vision/v2 v2.7.0/go.mod h1:H89VysHy21avemp6xcf9b9JvZHVehWbET0uT/bcuY/0= +cloud.google.com/go/vision/v2 v2.7.2/go.mod h1:jKa8oSYBWhYiXarHPvP4USxYANYUEdEsQrloLjrSwJU= +cloud.google.com/go/vmmigration v1.2.0/go.mod h1:IRf0o7myyWFSmVR1ItrBSFLFD/rJkfDCUTO4vLlJvsE= +cloud.google.com/go/vmmigration v1.3.0/go.mod h1:oGJ6ZgGPQOFdjHuocGcLqX4lc98YQ7Ygq8YQwHh9A7g= +cloud.google.com/go/vmmigration v1.5.0/go.mod h1:E4YQ8q7/4W9gobHjQg4JJSgXXSgY21nA5r8swQV+Xxc= +cloud.google.com/go/vmmigration v1.6.0/go.mod h1:bopQ/g4z+8qXzichC7GW1w2MjbErL54rk3/C843CjfY= +cloud.google.com/go/vmmigration v1.7.1/go.mod h1:WD+5z7a/IpZ5bKK//YmT9E047AD+rjycCAvyMxGJbro= +cloud.google.com/go/vmwareengine v0.1.0/go.mod h1:RsdNEf/8UDvKllXhMz5J40XxDrNJNN4sagiox+OI208= +cloud.google.com/go/vmwareengine v0.2.2/go.mod h1:sKdctNJxb3KLZkE/6Oui94iw/xs9PRNC2wnNLXsHvH8= +cloud.google.com/go/vmwareengine v0.3.0/go.mod h1:wvoyMvNWdIzxMYSpH/R7y2h5h3WFkx6d+1TIsP39WGY= +cloud.google.com/go/vmwareengine v0.4.1/go.mod h1:Px64x+BvjPZwWuc4HdmVhoygcXqEkGHXoa7uyfTgSI0= +cloud.google.com/go/vpcaccess v1.4.0/go.mod h1:aQHVbTWDYUR1EbTApSVvMq1EnT57ppDmQzZ3imqIk4w= +cloud.google.com/go/vpcaccess v1.5.0/go.mod h1:drmg4HLk9NkZpGfCmZ3Tz0Bwnm2+DKqViEpeEpOq0m8= +cloud.google.com/go/vpcaccess v1.6.0/go.mod h1:wX2ILaNhe7TlVa4vC5xce1bCnqE3AeH27RV31lnmZes= +cloud.google.com/go/vpcaccess v1.7.1/go.mod h1:FogoD46/ZU+JUBX9D606X21EnxiszYi2tArQwLY4SXs= +cloud.google.com/go/webrisk v1.4.0/go.mod h1:Hn8X6Zr+ziE2aNd8SliSDWpEnSS1u4R9+xXZmFiHmGE= +cloud.google.com/go/webrisk v1.5.0/go.mod h1:iPG6fr52Tv7sGk0H6qUFzmL3HHZev1htXuWDEEsqMTg= +cloud.google.com/go/webrisk v1.6.0/go.mod h1:65sW9V9rOosnc9ZY7A7jsy1zoHS5W9IAXv6dGqhMQMc= +cloud.google.com/go/webrisk v1.7.0/go.mod h1:mVMHgEYH0r337nmt1JyLthzMr6YxwN1aAIEc2fTcq7A= +cloud.google.com/go/webrisk v1.8.0/go.mod h1:oJPDuamzHXgUc+b8SiHRcVInZQuybnvEW72PqTc7sSg= +cloud.google.com/go/webrisk v1.9.1/go.mod h1:4GCmXKcOa2BZcZPn6DCEvE7HypmEJcJkr4mtM+sqYPc= +cloud.google.com/go/websecurityscanner v1.3.0/go.mod h1:uImdKm2wyeXQevQJXeh8Uun/Ym1VqworNDlBXQevGMo= +cloud.google.com/go/websecurityscanner v1.4.0/go.mod h1:ebit/Fp0a+FWu5j4JOmJEV8S8CzdTkAS77oDsiSqYWQ= +cloud.google.com/go/websecurityscanner v1.5.0/go.mod h1:Y6xdCPy81yi0SQnDY1xdNTNpfY1oAgXUlcfN3B3eSng= +cloud.google.com/go/websecurityscanner v1.6.1/go.mod h1:Njgaw3rttgRHXzwCB8kgCYqv5/rGpFCsBOvPbYgszpg= +cloud.google.com/go/workflows v1.6.0/go.mod h1:6t9F5h/unJz41YqfBmqSASJSXccBLtD1Vwf+KmJENM0= +cloud.google.com/go/workflows v1.7.0/go.mod h1:JhSrZuVZWuiDfKEFxU0/F1PQjmpnpcoISEXH2bcHC3M= +cloud.google.com/go/workflows v1.8.0/go.mod h1:ysGhmEajwZxGn1OhGOGKsTXc5PyxOc0vfKf5Af+to4M= +cloud.google.com/go/workflows v1.9.0/go.mod h1:ZGkj1aFIOd9c8Gerkjjq7OW7I5+l6cSvT3ujaO/WwSA= +cloud.google.com/go/workflows v1.10.0/go.mod h1:fZ8LmRmZQWacon9UCX1r/g/DfAXx5VcPALq2CxzdePw= +cloud.google.com/go/workflows v1.11.1/go.mod h1:Z+t10G1wF7h8LgdY/EmRcQY8ptBD/nvofaL6FqlET6g= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zumjgTw83q2ge/PI+yyw8= +git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3pc= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= +github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm/4RlzPXRlREEwqTHAN3T56Bv2ITsFT3gY= +github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19/go.mod h1:T13YZdzov6OU0A1+RfKZiZN9ca6VeKdBdyDV+BY97Tk= +github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= +github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b/go.mod h1:1KcenG0jGWcpt8ov532z81sp/kMMUG485J2InIOyADM= +github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/apache/arrow/go/v10 v10.0.1/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0= +github.com/apache/arrow/go/v11 v11.0.0/go.mod h1:Eg5OsL5H+e299f7u5ssuXsuHQVEGC4xei5aX110hRiI= +github.com/apache/arrow/go/v12 v12.0.0/go.mod h1:d+tV/eHZZ7Dz7RPrFKtPK02tpr+c9/PEd/zm8mDS9Vg= +github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= +github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= +github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20230310173818-32f1caf87195/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20230428030218-4003588d1b74/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= +github.com/envoyproxy/go-control-plane v0.10.3/go.mod h1:fJJn/j26vwOu972OllsvAgJJM//w9BV6Fxbg2LuVd34= +github.com/envoyproxy/go-control-plane v0.11.0/go.mod h1:VnHyVMpzcLvCFt9yUz1UnCwHLhwx1WguiVDV7pTG/tI= +github.com/envoyproxy/go-control-plane v0.11.1/go.mod h1:uhMcXKCQMEJHiAb0w+YGefQLaTEw+YhGluxZkrTmD0g= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v0.6.7/go.mod h1:dyJXwwfPK2VSqiB9Klm1J6romD608Ba7Hij42vrOBCo= +github.com/envoyproxy/protoc-gen-validate v0.9.1/go.mod h1:OKNgG7TCp5pF4d6XftA0++PMirau2/yoOwVac3AbF2w= +github.com/envoyproxy/protoc-gen-validate v0.10.0/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss= +github.com/envoyproxy/protoc-gen-validate v1.0.1/go.mod h1:0vj8bNkYbSTNS2PIyH87KZaeN4x9zpL9Qt8fQC7d+vs= +github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= +github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-fonts/dejavu v0.1.0/go.mod h1:4Wt4I4OU2Nq9asgDCteaAaWZOV24E+0/Pwo0gppep4g= +github.com/go-fonts/latin-modern v0.2.0/go.mod h1:rQVLdDMK+mK1xscDwsqM5J8U2jrRa3T0ecnM9pNujks= +github.com/go-fonts/liberation v0.1.1/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY= +github.com/go-fonts/liberation v0.2.0/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY= +github.com/go-fonts/stix v0.1.0/go.mod h1:w/c1f0ldAUlJmLBvlbkvVXLAD+tAMqobIIQpmnUIzUY= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2CSIqUrmQPqA0gdRIlnLEY0gK5JGjh37zN5U= +github.com/go-latex/latex v0.0.0-20210823091927-c0d11ff05a81/go.mod h1:SX0U8uGpxhq9o2S/CELCSUxEWWAuoCUcVCQWv7G2OCk= +github.com/go-pdf/fpdf v0.5.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= +github.com/go-pdf/fpdf v0.6.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= +github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/flatbuffers v2.0.8+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/s2a-go v0.1.0/go.mod h1:OJpEgntRZo8ugHpF9hkoLJbS5dSI20XZeXJ9JVywLlM= +github.com/google/s2a-go v0.1.3/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= +github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= +github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= +github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= +github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= +github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= +github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= +github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= +github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= +github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= +github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo= +github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY= +github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= +github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= +github.com/googleapis/gax-go/v2 v2.8.0/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= +github.com/googleapis/gax-go/v2 v2.10.0/go.mod h1:4UOEnMCrxsSqQ940WnTiD6qJ63le2ev3xfyagutxiPw= +github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI= +github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= +github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU0GSB0f8Nhgmxx0VIRUvaC0w= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= +github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lyft/protoc-gen-star v0.6.0/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= +github.com/lyft/protoc-gen-star v0.6.1/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA= +github.com/lyft/protoc-gen-star/v2 v2.0.1/go.mod h1:RcCdONR2ScXaYnQC5tUzxzlpA3WVYF7/opLeUgcQs/o= +github.com/lyft/protoc-gen-star/v2 v2.0.3/go.mod h1:amey7yeodaJhXSbf/TlLvWiqQfLOSpEk//mLlc+axEk= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= +github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2dXMnm1mY= +github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= +github.com/phpdave11/gofpdi v1.0.13/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= +github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= +github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w= +github.com/ruudk/golang-pdf417 v0.0.0-20201230142125-a7e3863a1245/go.mod h1:pQAZKsJ8yyVxGRWYNEm9oFB8ieLgKFnamEyDmSA0BRk= +github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= +github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= +github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= +go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= +golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= +golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= +golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3/go.mod h1:NOZ3BPKG0ec/BKJQgnvsSFpcKLM5xXVWnvZS97DWHgE= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20220827204233-334a2380cb91/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= +golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20190910094157-69e4b8554b2a/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200430140353-33d19683fad8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20200618115811-c13761719519/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20201208152932-35266b937fa6/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20210216034530-4410531fe030/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20210607152325-775e3b0c77b9/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= +golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= +golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= +golang.org/x/image v0.0.0-20220302094943-723b81ca9867/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.0.0-20221012135044-0b7e1fb9d458/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= +golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= +golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= +golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= +golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.0.0-20221006150949-b44042a4b9c1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= +golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= +golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= +golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= +golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210304124612-50617c2ba197/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220829200755-d48e67d00261/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= +golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= +golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= +golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= +golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= +golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= +gonum.org/v1/gonum v0.9.3/go.mod h1:TZumC3NeyVQskjXqmyWt4S3bINhy7B4eYwW69EbyX+0= +gonum.org/v1/gonum v0.11.0/go.mod h1:fSG4YDCxxUZQJ7rKsQrj0gMOg00Il0Z96/qMA4bVQhA= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= +gonum.org/v1/plot v0.9.0/go.mod h1:3Pcqqmp6RHvJI72kgb8fThyUnav364FOsdDo2aGW5lY= +gonum.org/v1/plot v0.10.1/go.mod h1:VZW5OlhkL1mysU9vaqNHnsy86inf6Ot+jB3r+BczCEo= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= +google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= +google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= +google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= +google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= +google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= +google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= +google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= +google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g= +google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA= +google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8= +google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= +google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= +google.golang.org/api v0.77.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= +google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw= +google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg= +google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o= +google.golang.org/api v0.85.0/go.mod h1:AqZf8Ep9uZ2pyTvgL+x0D3Zt0eoT9b5E8fmzfu6FO2g= +google.golang.org/api v0.90.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= +google.golang.org/api v0.93.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= +google.golang.org/api v0.95.0/go.mod h1:eADj+UBuxkh5zlrSntJghuNeg8HwQ1w5lTKkuqaETEI= +google.golang.org/api v0.96.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= +google.golang.org/api v0.97.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= +google.golang.org/api v0.98.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= +google.golang.org/api v0.99.0/go.mod h1:1YOf74vkVndF7pG6hIHuINsM7eWwpVTAfNMNiL91A08= +google.golang.org/api v0.100.0/go.mod h1:ZE3Z2+ZOr87Rx7dqFsdRQkRBk36kDtp/h+QpHbB7a70= +google.golang.org/api v0.102.0/go.mod h1:3VFl6/fzoA+qNuS1N1/VfXY4LjoXN/wzeIp7TweWwGo= +google.golang.org/api v0.103.0/go.mod h1:hGtW6nK1AC+d9si/UBhw8Xli+QMOf6xyNAyJw4qU9w0= +google.golang.org/api v0.106.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= +google.golang.org/api v0.107.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= +google.golang.org/api v0.108.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= +google.golang.org/api v0.110.0/go.mod h1:7FC4Vvx1Mooxh8C5HWjzZHcavuS2f6pmJpZx60ca7iI= +google.golang.org/api v0.111.0/go.mod h1:qtFHvU9mhgTJegR31csQ+rwxyUTHOKFqCKWp1J0fdw0= +google.golang.org/api v0.114.0/go.mod h1:ifYI2ZsFK6/uGddGfAD5BMxlnkBqCmqHSDUVi45N5Yg= +google.golang.org/api v0.118.0/go.mod h1:76TtD3vkgmZ66zZzp72bUUklpmQmKlhh6sYtIjYK+5E= +google.golang.org/api v0.122.0/go.mod h1:gcitW0lvnyWjSp9nKxAbdHKIZ6vF4aajGueeslZOyms= +google.golang.org/api v0.124.0/go.mod h1:xu2HQurE5gi/3t1aFCvhPD781p0a3p11sdunTJ2BlP4= +google.golang.org/api v0.125.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw= +google.golang.org/api v0.126.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210329143202-679c6ae281ee/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= +google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= +google.golang.org/genproto v0.0.0-20220329172620-7be39ac1afc7/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220628213854-d9e0b6570c03/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220722212130-b98a9ff5e252/go.mod h1:GkXuJDJ6aQ7lnJcRF+SJVgFdQhypqgl3LB1C9vabdRE= +google.golang.org/genproto v0.0.0-20220801145646-83ce21fca29f/go.mod h1:iHe1svFLAZg9VWz891+QbRMwUv9O/1Ww+/mngYeThbc= +google.golang.org/genproto v0.0.0-20220815135757-37a418bb8959/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220817144833-d7fd3f11b9b1/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220829144015-23454907ede3/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220829175752-36a9c930ecbf/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= +google.golang.org/genproto v0.0.0-20220913154956-18f8339a66a5/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220914142337-ca0e39ece12f/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220915135415-7fd63a7952de/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220916172020-2692e8806bfa/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220919141832-68c03719ef51/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= +google.golang.org/genproto v0.0.0-20220920201722-2b89144ce006/go.mod h1:ht8XFiar2npT/g4vkk7O0WYS1sHOHbdujxbEp7CJWbw= +google.golang.org/genproto v0.0.0-20220926165614-551eb538f295/go.mod h1:woMGP53BroOrRY3xTxlbr8Y3eB/nzAvvFM83q7kG2OI= +google.golang.org/genproto v0.0.0-20220926220553-6981cbe3cfce/go.mod h1:woMGP53BroOrRY3xTxlbr8Y3eB/nzAvvFM83q7kG2OI= +google.golang.org/genproto v0.0.0-20221010155953-15ba04fc1c0e/go.mod h1:3526vdqwhZAwq4wsRUaVG555sVgsNmIjRtO7t/JH29U= +google.golang.org/genproto v0.0.0-20221014173430-6e2ab493f96b/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= +google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= +google.golang.org/genproto v0.0.0-20221024153911-1573dae28c9c/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= +google.golang.org/genproto v0.0.0-20221024183307-1bc688fe9f3e/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= +google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c/go.mod h1:CGI5F/G+E5bKwmfYo09AXuVN4dD894kIKUFmVbP2/Fo= +google.golang.org/genproto v0.0.0-20221109142239-94d6d90a7d66/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221114212237-e4508ebdbee1/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221117204609-8f9c96812029/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221201204527-e3fa12d562f3/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd/go.mod h1:cTsE614GARnxrLsqKREzmNYJACSWWpAWdNMwnD7c2BE= +google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230112194545-e10362b5ecf9/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230113154510-dbe35b8444a5/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230123190316-2c411cf9d197/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230124163310-31e0e69b6fc2/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230125152338-dcaf20b6aeaa/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230127162408-596548ed4efa/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230209215440-0dfe4f8abfcc/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230216225411-c8e22ba71e44/go.mod h1:8B0gmkoRebU8ukX6HP+4wrVQUY1+6PkQ44BSyIlflHA= +google.golang.org/genproto v0.0.0-20230222225845-10f96fb3dbec/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw= +google.golang.org/genproto v0.0.0-20230223222841-637eb2293923/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw= +google.golang.org/genproto v0.0.0-20230303212802-e74f57abe488/go.mod h1:TvhZT5f700eVlTNwND1xoEZQeWTB2RY/65kplwl/bFA= +google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= +google.golang.org/genproto v0.0.0-20230320184635-7606e756e683/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= +google.golang.org/genproto v0.0.0-20230323212658-478b75c54725/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/genproto v0.0.0-20230330154414-c0448cd141ea/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/genproto v0.0.0-20230403163135-c38d8f061ccd/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= +google.golang.org/genproto v0.0.0-20230525234025-438c736192d0/go.mod h1:9ExIQyXL5hZrHzQceCwuSYwZZ5QZBazOcprJ5rgs3lY= +google.golang.org/genproto v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:zqTuNwFlFRsw5zIts5VnzLQxSRqh+CGOTVMlYbY0Eyk= +google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= +google.golang.org/genproto v0.0.0-20230629202037-9506855d4529/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= +google.golang.org/genproto v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:O9kGHb51iE/nOGvQaDUuadVYqovW56s5emA88lQnj6Y= +google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98/go.mod h1:S7mY02OqCJTD0E1OiQy1F72PWFB4bZJ87cAtLPYgDR0= +google.golang.org/genproto/googleapis/api v0.0.0-20230525234020-1aefcd67740a/go.mod h1:ts19tUU+Z0ZShN1y3aPyq2+O3d5FUNNgT6FtOzmrNn8= +google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto/googleapis/api v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto/googleapis/api v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:mPBs5jNgx2GuQGvFwUvVKqtn6HsUw9nP64BedgvqEsQ= +google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= +google.golang.org/genproto/googleapis/bytestream v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:ylj+BE99M198VPbBh6A8d9n3w8fChvyLK3wwBOjXBFA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234015-3fc162c6f38a/go.mod h1:xURIpW9ES5+/GZhnV6beoEtxQrnkRGIfP5VQG2tCBLc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:8mL13HKkDa+IuJ8yruA3ci0q+0vsUz4m//+ottjwS5o= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 h1:bVf09lpb+OJbByTj913DRJioFFAjf/ZGxEz7MajTp2U= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= +lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= +lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= +modernc.org/cc/v3 v3.36.0/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= +modernc.org/cc/v3 v3.36.2/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= +modernc.org/cc/v3 v3.36.3/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= +modernc.org/cc/v3 v3.37.0/go.mod h1:vtL+3mdHx/wcj3iEGz84rQa8vEqR6XM84v5Lcvfph20= +modernc.org/cc/v3 v3.40.0/go.mod h1:/bTg4dnWkSXowUO6ssQKnOV0yMVxDYNIsIrzqTFDGH0= +modernc.org/ccgo/v3 v3.0.0-20220428102840-41399a37e894/go.mod h1:eI31LL8EwEBKPpNpA4bU1/i+sKOwOrQy8D87zWUcRZc= +modernc.org/ccgo/v3 v3.0.0-20220430103911-bc99d88307be/go.mod h1:bwdAnOoaIt8Ax9YdWGjxWsdkPcZyRPHqrOvJxaKAKGw= +modernc.org/ccgo/v3 v3.0.0-20220904174949-82d86e1b6d56/go.mod h1:YSXjPL62P2AMSxBphRHPn7IkzhVHqkvOnRKAKh+W6ZI= +modernc.org/ccgo/v3 v3.16.4/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ= +modernc.org/ccgo/v3 v3.16.6/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ= +modernc.org/ccgo/v3 v3.16.8/go.mod h1:zNjwkizS+fIFDrDjIAgBSCLkWbJuHF+ar3QRn+Z9aws= +modernc.org/ccgo/v3 v3.16.9/go.mod h1:zNMzC9A9xeNUepy6KuZBbugn3c0Mc9TeiJO4lgvkJDo= +modernc.org/ccgo/v3 v3.16.13-0.20221017192402-261537637ce8/go.mod h1:fUB3Vn0nVPReA+7IG7yZDfjv1TMWjhQP8gCxrFAtL5g= +modernc.org/ccgo/v3 v3.16.13/go.mod h1:2Quk+5YgpImhPjv2Qsob1DnZ/4som1lJTodubIcoUkY= +modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ= +modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM= +modernc.org/libc v0.0.0-20220428101251-2d5f3daf273b/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA= +modernc.org/libc v1.16.0/go.mod h1:N4LD6DBE9cf+Dzf9buBlzVJndKr/iJHG97vGLHYnb5A= +modernc.org/libc v1.16.1/go.mod h1:JjJE0eu4yeK7tab2n4S1w8tlWd9MxXLRzheaRnAKymU= +modernc.org/libc v1.16.17/go.mod h1:hYIV5VZczAmGZAnG15Vdngn5HSF5cSkbvfz2B7GRuVU= +modernc.org/libc v1.16.19/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA= +modernc.org/libc v1.17.0/go.mod h1:XsgLldpP4aWlPlsjqKRdHPqCxCjISdHfM/yeWC5GyW0= +modernc.org/libc v1.17.1/go.mod h1:FZ23b+8LjxZs7XtFMbSzL/EhPxNbfZbErxEHc7cbD9s= +modernc.org/libc v1.17.4/go.mod h1:WNg2ZH56rDEwdropAJeZPQkXmDwh+JCA1s/htl6r2fA= +modernc.org/libc v1.18.0/go.mod h1:vj6zehR5bfc98ipowQOM2nIDUZnVew/wNC/2tOGS+q0= +modernc.org/libc v1.20.3/go.mod h1:ZRfIaEkgrYgZDl6pa4W39HgN5G/yDW+NRmNKZBDFrk0= +modernc.org/libc v1.21.4/go.mod h1:przBsL5RDOZajTVslkugzLBj1evTue36jEomFQOoYuI= +modernc.org/libc v1.22.2/go.mod h1:uvQavJ1pZ0hIoC/jfqNoMLURIMhKzINIWypNM17puug= +modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/mathutil v1.4.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/memory v1.1.1/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw= +modernc.org/memory v1.2.0/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw= +modernc.org/memory v1.2.1/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= +modernc.org/memory v1.3.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= +modernc.org/memory v1.4.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= +modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= +modernc.org/opt v0.1.1/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/sqlite v1.18.1/go.mod h1:6ho+Gow7oX5V+OiOQ6Tr4xeqbx13UZ6t+Fw9IRUG4d4= +modernc.org/sqlite v1.18.2/go.mod h1:kvrTLEWgxUcHa2GfHBQtanR1H9ht3hTJNtKpzH9k1u0= +modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw= +modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw= +modernc.org/tcl v1.13.1/go.mod h1:XOLfOwzhkljL4itZkK6T72ckMgvj0BDsnKNdZVUOecw= +modernc.org/tcl v1.13.2/go.mod h1:7CLiGIPo1M8Rv1Mitpv5akc2+8fxUd2y2UzC/MfMzy0= +modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +modernc.org/token v1.0.1/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +modernc.org/z v1.5.1/go.mod h1:eWFB510QWW5Th9YGZT81s+LwvaAs3Q2yr4sP0rmLkv8= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/stats/opencensus/opencensus.go b/stats/opencensus/opencensus.go new file mode 100644 index 000000000000..f477d0918a37 --- /dev/null +++ b/stats/opencensus/opencensus.go @@ -0,0 +1,251 @@ +/* + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package opencensus implements opencensus instrumentation code for gRPC-Go +// clients and servers. +package opencensus + +import ( + "context" + "strings" + "time" + + ocstats "go.opencensus.io/stats" + "go.opencensus.io/tag" + "go.opencensus.io/trace" + + "google.golang.org/grpc" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/stats" + "google.golang.org/grpc/status" +) + +var ( + joinDialOptions = internal.JoinDialOptions.(func(...grpc.DialOption) grpc.DialOption) +) + +// TraceOptions are the tracing options for opencensus instrumentation. +type TraceOptions struct { + // TS is the Sampler used for tracing. + TS trace.Sampler + // DisableTrace determines whether traces are disabled for an OpenCensus + // Dial or Server option. will overwrite any global option setting. + DisableTrace bool +} + +// DialOption returns a dial option which enables OpenCensus instrumentation +// code for a grpc.ClientConn. +// +// Client applications interested in instrumenting their grpc.ClientConn should +// pass the dial option returned from this function as the first dial option to +// grpc.Dial(). +// +// Using this option will always lead to instrumentation, however in order to +// use the data an exporter must be registered with the OpenCensus trace package +// for traces and the OpenCensus view package for metrics. Client side has +// retries, so a Unary and Streaming Interceptor are registered to handle per +// RPC traces/metrics, and a Stats Handler is registered to handle per RPC +// attempt trace/metrics. These three components registered work together in +// conjunction, and do not work standalone. It is not supported to use this +// alongside another stats handler dial option. +func DialOption(to TraceOptions) grpc.DialOption { + csh := &clientStatsHandler{to: to} + return joinDialOptions(grpc.WithChainUnaryInterceptor(csh.unaryInterceptor), grpc.WithChainStreamInterceptor(csh.streamInterceptor), grpc.WithStatsHandler(csh)) +} + +// ServerOption returns a server option which enables OpenCensus instrumentation +// code for a grpc.Server. +// +// Server applications interested in instrumenting their grpc.Server should +// pass the server option returned from this function as the first argument to +// grpc.NewServer(). +// +// Using this option will always lead to instrumentation, however in order to +// use the data an exporter must be registered with the OpenCensus trace package +// for traces and the OpenCensus view package for metrics. Server side does not +// have retries, so a registered Stats Handler is the only option that is +// returned. It is not supported to use this alongside another stats handler +// server option. +func ServerOption(to TraceOptions) grpc.ServerOption { + return grpc.StatsHandler(&serverStatsHandler{to: to}) +} + +// createCallSpan creates a call span if tracing is enabled, which will be put +// in the context provided if created. +func (csh *clientStatsHandler) createCallSpan(ctx context.Context, method string) (context.Context, *trace.Span) { + var span *trace.Span + if !csh.to.DisableTrace { + mn := strings.Replace(removeLeadingSlash(method), "/", ".", -1) + ctx, span = trace.StartSpan(ctx, mn, trace.WithSampler(csh.to.TS), trace.WithSpanKind(trace.SpanKindClient)) + } + return ctx, span +} + +// perCallTracesAndMetrics records per call spans and metrics. +func perCallTracesAndMetrics(err error, span *trace.Span, startTime time.Time, method string) { + s := status.Convert(err) + if span != nil { + span.SetStatus(trace.Status{Code: int32(s.Code()), Message: s.Message()}) + span.End() + } + callLatency := float64(time.Since(startTime)) / float64(time.Millisecond) + ocstats.RecordWithOptions(context.Background(), + ocstats.WithTags( + tag.Upsert(keyClientMethod, removeLeadingSlash(method)), + tag.Upsert(keyClientStatus, canonicalString(s.Code())), + ), + ocstats.WithMeasurements( + clientAPILatency.M(callLatency), + ), + ) +} + +// unaryInterceptor handles per RPC context management. It also handles per RPC +// tracing and stats by creating a top level call span and recording the latency +// for the full RPC call. +func (csh *clientStatsHandler) unaryInterceptor(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + startTime := time.Now() + ctx, span := csh.createCallSpan(ctx, method) + err := invoker(ctx, method, req, reply, cc, opts...) + perCallTracesAndMetrics(err, span, startTime, method) + return err +} + +// streamInterceptor handles per RPC context management. It also handles per RPC +// tracing and stats by creating a top level call span and recording the latency +// for the full RPC call. +func (csh *clientStatsHandler) streamInterceptor(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) { + startTime := time.Now() + ctx, span := csh.createCallSpan(ctx, method) + callback := func(err error) { + perCallTracesAndMetrics(err, span, startTime, method) + } + opts = append([]grpc.CallOption{grpc.OnFinish(callback)}, opts...) + s, err := streamer(ctx, desc, cc, method, opts...) + if err != nil { + return nil, err + } + return s, nil +} + +type rpcInfo struct { + mi *metricsInfo + ti *traceInfo +} + +type rpcInfoKey struct{} + +func setRPCInfo(ctx context.Context, ri *rpcInfo) context.Context { + return context.WithValue(ctx, rpcInfoKey{}, ri) +} + +// getRPCInfo returns the rpcInfo stored in the context, or nil +// if there isn't one. +func getRPCInfo(ctx context.Context) *rpcInfo { + ri, _ := ctx.Value(rpcInfoKey{}).(*rpcInfo) + return ri +} + +// SpanContextFromContext returns the Span Context about the Span in the +// context. Returns false if no Span in the context. +func SpanContextFromContext(ctx context.Context) (trace.SpanContext, bool) { + ri, ok := ctx.Value(rpcInfoKey{}).(*rpcInfo) + if !ok { + return trace.SpanContext{}, false + } + if ri.ti == nil || ri.ti.span == nil { + return trace.SpanContext{}, false + } + sc := ri.ti.span.SpanContext() + return sc, true +} + +type clientStatsHandler struct { + to TraceOptions +} + +// TagConn exists to satisfy stats.Handler. +func (csh *clientStatsHandler) TagConn(ctx context.Context, _ *stats.ConnTagInfo) context.Context { + return ctx +} + +// HandleConn exists to satisfy stats.Handler. +func (csh *clientStatsHandler) HandleConn(context.Context, stats.ConnStats) {} + +// TagRPC implements per RPC attempt context management. +func (csh *clientStatsHandler) TagRPC(ctx context.Context, rti *stats.RPCTagInfo) context.Context { + ctx, mi := csh.statsTagRPC(ctx, rti) + var ti *traceInfo + if !csh.to.DisableTrace { + ctx, ti = csh.traceTagRPC(ctx, rti) + } + ri := &rpcInfo{ + mi: mi, + ti: ti, + } + return setRPCInfo(ctx, ri) +} + +func (csh *clientStatsHandler) HandleRPC(ctx context.Context, rs stats.RPCStats) { + ri := getRPCInfo(ctx) + if ri == nil { + // Shouldn't happen because TagRPC populates this information. + return + } + recordRPCData(ctx, rs, ri.mi) + if !csh.to.DisableTrace { + populateSpan(ctx, rs, ri.ti) + } +} + +type serverStatsHandler struct { + to TraceOptions +} + +// TagConn exists to satisfy stats.Handler. +func (ssh *serverStatsHandler) TagConn(ctx context.Context, _ *stats.ConnTagInfo) context.Context { + return ctx +} + +// HandleConn exists to satisfy stats.Handler. +func (ssh *serverStatsHandler) HandleConn(context.Context, stats.ConnStats) {} + +// TagRPC implements per RPC context management. +func (ssh *serverStatsHandler) TagRPC(ctx context.Context, rti *stats.RPCTagInfo) context.Context { + ctx, mi := ssh.statsTagRPC(ctx, rti) + var ti *traceInfo + if !ssh.to.DisableTrace { + ctx, ti = ssh.traceTagRPC(ctx, rti) + } + ri := &rpcInfo{ + mi: mi, + ti: ti, + } + return setRPCInfo(ctx, ri) +} + +// HandleRPC implements per RPC tracing and stats implementation. +func (ssh *serverStatsHandler) HandleRPC(ctx context.Context, rs stats.RPCStats) { + ri := getRPCInfo(ctx) + if ri == nil { + // Shouldn't happen because TagRPC populates this information. + return + } + recordRPCData(ctx, rs, ri.mi) + if !ssh.to.DisableTrace { + populateSpan(ctx, rs, ri.ti) + } +} diff --git a/stats/opencensus/server_metrics.go b/stats/opencensus/server_metrics.go new file mode 100644 index 000000000000..4f087c2209a7 --- /dev/null +++ b/stats/opencensus/server_metrics.go @@ -0,0 +1,135 @@ +/* + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opencensus + +import ( + "go.opencensus.io/stats" + "go.opencensus.io/stats/view" + "go.opencensus.io/tag" +) + +var ( + keyServerMethod = tag.MustNewKey("grpc_server_method") + keyServerStatus = tag.MustNewKey("grpc_server_status") +) + +// Measures, which are recorded by server stats handler: Note that on gRPC's +// server side, the per rpc unit is truly per rpc, as there is no concept of a +// rpc attempt server side. +var ( + serverReceivedMessagesPerRPC = stats.Int64("grpc.io/server/received_messages_per_rpc", "Number of messages received in each RPC. Has value 1 for non-streaming RPCs.", stats.UnitDimensionless) // the collection/measurement point of this measure handles the /rpc aspect of it + serverReceivedBytesPerRPC = stats.Int64("grpc.io/server/received_bytes_per_rpc", "Total bytes received across all messages per RPC.", stats.UnitBytes) + serverReceivedCompressedBytesPerRPC = stats.Int64("grpc.io/server/received_compressed_bytes_per_rpc", "Total compressed bytes received across all messages per RPC.", stats.UnitBytes) + serverSentMessagesPerRPC = stats.Int64("grpc.io/server/sent_messages_per_rpc", "Number of messages sent in each RPC. Has value 1 for non-streaming RPCs.", stats.UnitDimensionless) + serverSentBytesPerRPC = stats.Int64("grpc.io/server/sent_bytes_per_rpc", "Total bytes sent in across all response messages per RPC.", stats.UnitBytes) + serverSentCompressedBytesPerRPC = stats.Int64("grpc.io/server/sent_compressed_bytes_per_rpc", "Total compressed bytes sent in across all response messages per RPC.", stats.UnitBytes) + serverStartedRPCs = stats.Int64("grpc.io/server/started_rpcs", "The total number of server RPCs ever opened, including those that have not completed.", stats.UnitDimensionless) + serverLatency = stats.Float64("grpc.io/server/server_latency", "Time between first byte of request received to last byte of response sent, or terminal error.", stats.UnitMilliseconds) +) + +var ( + // ServerSentMessagesPerRPCView is the distribution of sent messages per + // RPC, keyed on method. + ServerSentMessagesPerRPCView = &view.View{ + Name: "grpc.io/server/sent_messages_per_rpc", + Description: "Distribution of sent messages per RPC, by method.", + TagKeys: []tag.Key{keyServerMethod}, + Measure: serverSentMessagesPerRPC, + Aggregation: countDistribution, + } + // ServerReceivedMessagesPerRPCView is the distribution of received messages + // per RPC, keyed on method. + ServerReceivedMessagesPerRPCView = &view.View{ + Name: "grpc.io/server/received_messages_per_rpc", + Description: "Distribution of received messages per RPC, by method.", + TagKeys: []tag.Key{keyServerMethod}, + Measure: serverReceivedMessagesPerRPC, + Aggregation: countDistribution, + } + // ServerSentBytesPerRPCView is the distribution of received bytes per RPC, + // keyed on method. + ServerSentBytesPerRPCView = &view.View{ + Name: "grpc.io/server/sent_bytes_per_rpc", + Description: "Distribution of sent bytes per RPC, by method.", + Measure: serverSentBytesPerRPC, + TagKeys: []tag.Key{keyServerMethod}, + Aggregation: bytesDistribution, + } + // ServerSentCompressedMessageBytesPerRPCView is the distribution of + // received compressed message bytes per RPC, keyed on method. + ServerSentCompressedMessageBytesPerRPCView = &view.View{ + Name: "grpc.io/server/sent_compressed_message_bytes_per_rpc", + Description: "Distribution of sent compressed message bytes per RPC, by method.", + Measure: serverSentCompressedBytesPerRPC, + TagKeys: []tag.Key{keyServerMethod}, + Aggregation: bytesDistribution, + } + // ServerReceivedBytesPerRPCView is the distribution of sent bytes per RPC, + // keyed on method. + ServerReceivedBytesPerRPCView = &view.View{ + Name: "grpc.io/server/received_bytes_per_rpc", + Description: "Distribution of received bytes per RPC, by method.", + Measure: serverReceivedBytesPerRPC, + TagKeys: []tag.Key{keyServerMethod}, + Aggregation: bytesDistribution, + } + // ServerReceivedCompressedMessageBytesPerRPCView is the distribution of + // sent compressed message bytes per RPC, keyed on method. + ServerReceivedCompressedMessageBytesPerRPCView = &view.View{ + Name: "grpc.io/server/received_compressed_message_bytes_per_rpc", + Description: "Distribution of received compressed message bytes per RPC, by method.", + Measure: serverReceivedCompressedBytesPerRPC, + TagKeys: []tag.Key{keyServerMethod}, + Aggregation: bytesDistribution, + } + // ServerStartedRPCsView is the count of opened RPCs, keyed on method. + ServerStartedRPCsView = &view.View{ + Measure: serverStartedRPCs, + Name: "grpc.io/server/started_rpcs", + Description: "Number of opened server RPCs, by method.", + TagKeys: []tag.Key{keyServerMethod}, + Aggregation: view.Count(), + } + // ServerCompletedRPCsView is the count of completed RPCs, keyed on + // method and status. + ServerCompletedRPCsView = &view.View{ + Name: "grpc.io/server/completed_rpcs", + Description: "Number of completed RPCs by method and status.", + TagKeys: []tag.Key{keyServerMethod, keyServerStatus}, + Measure: serverLatency, + Aggregation: view.Count(), + } + // ServerLatencyView is the distribution of server latency in milliseconds + // per RPC, keyed on method. + ServerLatencyView = &view.View{ + Name: "grpc.io/server/server_latency", + Description: "Distribution of server latency in milliseconds, by method.", + TagKeys: []tag.Key{keyServerMethod}, + Measure: serverLatency, + Aggregation: millisecondsDistribution, + } +) + +// DefaultServerViews is the set of server views which are considered the +// minimum required to monitor server side performance. +var DefaultServerViews = []*view.View{ + ServerReceivedBytesPerRPCView, + ServerSentBytesPerRPCView, + ServerLatencyView, + ServerCompletedRPCsView, + ServerStartedRPCsView, +} diff --git a/stats/opencensus/stats.go b/stats/opencensus/stats.go new file mode 100644 index 000000000000..01cd0b9b157d --- /dev/null +++ b/stats/opencensus/stats.go @@ -0,0 +1,222 @@ +/* + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opencensus + +import ( + "context" + "strings" + "sync/atomic" + "time" + + ocstats "go.opencensus.io/stats" + "go.opencensus.io/stats/view" + "go.opencensus.io/tag" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/stats" + "google.golang.org/grpc/status" +) + +var logger = grpclog.Component("opencensus-instrumentation") + +var canonicalString = internal.CanonicalString.(func(codes.Code) string) + +var ( + // bounds separate variable for testing purposes. + bytesDistributionBounds = []float64{1024, 2048, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216, 67108864, 268435456, 1073741824, 4294967296} + bytesDistribution = view.Distribution(bytesDistributionBounds...) + millisecondsDistribution = view.Distribution(0.01, 0.05, 0.1, 0.3, 0.6, 0.8, 1, 2, 3, 4, 5, 6, 8, 10, 13, 16, 20, 25, 30, 40, 50, 65, 80, 100, 130, 160, 200, 250, 300, 400, 500, 650, 800, 1000, 2000, 5000, 10000, 20000, 50000, 100000) + countDistributionBounds = []float64{1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536} + countDistribution = view.Distribution(countDistributionBounds...) +) + +func removeLeadingSlash(mn string) string { + return strings.TrimLeft(mn, "/") +} + +// metricsInfo is data used for recording metrics about the rpc attempt client +// side, and the overall rpc server side. +type metricsInfo struct { + // access these counts atomically for hedging in the future + // number of messages sent from side (client || server) + sentMsgs int64 + // number of bytes sent (within each message) from side (client || server) + sentBytes int64 + // number of bytes after compression (within each message) from side (client || server) + sentCompressedBytes int64 + // number of messages received on side (client || server) + recvMsgs int64 + // number of bytes received (within each message) received on side (client + // || server) + recvBytes int64 + // number of compressed bytes received (within each message) received on + // side (client || server) + recvCompressedBytes int64 + + startTime time.Time + method string +} + +// statsTagRPC creates a recording object to derive measurements from in the +// context, scoping the recordings to per RPC Attempt client side (scope of the +// context). It also populates the gRPC Metadata within the context with any +// opencensus specific tags set by the application in the context, binary +// encoded to send across the wire. +func (csh *clientStatsHandler) statsTagRPC(ctx context.Context, info *stats.RPCTagInfo) (context.Context, *metricsInfo) { + mi := &metricsInfo{ + startTime: time.Now(), + method: info.FullMethodName, + } + + // Populate gRPC Metadata with OpenCensus tag map if set by application. + if tm := tag.FromContext(ctx); tm != nil { + ctx = stats.SetTags(ctx, tag.Encode(tm)) + } + return ctx, mi +} + +// statsTagRPC creates a recording object to derive measurements from in the +// context, scoping the recordings to per RPC server side (scope of the +// context). It also deserializes the opencensus tags set in the context's gRPC +// Metadata, and adds a server method tag to the opencensus tags. +func (ssh *serverStatsHandler) statsTagRPC(ctx context.Context, info *stats.RPCTagInfo) (context.Context, *metricsInfo) { + mi := &metricsInfo{ + startTime: time.Now(), + method: info.FullMethodName, + } + + if tagsBin := stats.Tags(ctx); tagsBin != nil { + if tags, err := tag.Decode(tagsBin); err == nil { + ctx = tag.NewContext(ctx, tags) + } + } + // We can ignore the error here because in the error case, the context + // passed in is returned. If the call errors, the server side application + // layer won't get this key server method information in the tag map, but + // this instrumentation code will function as normal. + ctx, _ = tag.New(ctx, tag.Upsert(keyServerMethod, removeLeadingSlash(info.FullMethodName))) + return ctx, mi +} + +func recordRPCData(ctx context.Context, s stats.RPCStats, mi *metricsInfo) { + if mi == nil { + // Shouldn't happen, as gRPC calls TagRPC which populates the metricsInfo in + // context. + logger.Error("ctx passed into stats handler metrics event handling has no metrics data present") + return + } + switch st := s.(type) { + case *stats.InHeader, *stats.OutHeader, *stats.InTrailer, *stats.OutTrailer: + // Headers and Trailers are not relevant to the measures, as the + // measures concern number of messages and bytes for messages. This + // aligns with flow control. + case *stats.Begin: + recordDataBegin(ctx, mi, st) + case *stats.OutPayload: + recordDataOutPayload(mi, st) + case *stats.InPayload: + recordDataInPayload(mi, st) + case *stats.End: + recordDataEnd(ctx, mi, st) + default: + // Shouldn't happen. gRPC calls into stats handler, and will never not + // be one of the types above. + logger.Errorf("Received unexpected stats type (%T) with data: %v", s, s) + } +} + +// recordDataBegin takes a measurement related to the RPC beginning, +// client/server started RPCs dependent on the caller. +func recordDataBegin(ctx context.Context, mi *metricsInfo, b *stats.Begin) { + if b.Client { + ocstats.RecordWithOptions(ctx, + ocstats.WithTags(tag.Upsert(keyClientMethod, removeLeadingSlash(mi.method))), + ocstats.WithMeasurements(clientStartedRPCs.M(1))) + return + } + ocstats.RecordWithOptions(ctx, + ocstats.WithTags(tag.Upsert(keyServerMethod, removeLeadingSlash(mi.method))), + ocstats.WithMeasurements(serverStartedRPCs.M(1))) +} + +// recordDataOutPayload records the length in bytes of outgoing messages and +// increases total count of sent messages both stored in the RPCs (attempt on +// client side) context for use in taking measurements at RPC end. +func recordDataOutPayload(mi *metricsInfo, op *stats.OutPayload) { + atomic.AddInt64(&mi.sentMsgs, 1) + atomic.AddInt64(&mi.sentBytes, int64(op.Length)) + atomic.AddInt64(&mi.sentCompressedBytes, int64(op.CompressedLength)) +} + +// recordDataInPayload records the length in bytes of incoming messages and +// increases total count of sent messages both stored in the RPCs (attempt on +// client side) context for use in taking measurements at RPC end. +func recordDataInPayload(mi *metricsInfo, ip *stats.InPayload) { + atomic.AddInt64(&mi.recvMsgs, 1) + atomic.AddInt64(&mi.recvBytes, int64(ip.Length)) + atomic.AddInt64(&mi.recvCompressedBytes, int64(ip.CompressedLength)) +} + +// recordDataEnd takes per RPC measurements derived from information derived +// from the lifetime of the RPC (RPC attempt client side). +func recordDataEnd(ctx context.Context, mi *metricsInfo, e *stats.End) { + // latency bounds for distribution data (speced millisecond bounds) have + // fractions, thus need a float. + latency := float64(time.Since(mi.startTime)) / float64(time.Millisecond) + var st string + if e.Error != nil { + s, _ := status.FromError(e.Error) + st = canonicalString(s.Code()) + } else { + st = "OK" + } + + // TODO: Attach trace data through attachments?!?! + + if e.Client { + ocstats.RecordWithOptions(ctx, + ocstats.WithTags( + tag.Upsert(keyClientMethod, removeLeadingSlash(mi.method)), + tag.Upsert(keyClientStatus, st)), + ocstats.WithMeasurements( + clientSentBytesPerRPC.M(atomic.LoadInt64(&mi.sentBytes)), + clientSentCompressedBytesPerRPC.M(atomic.LoadInt64(&mi.sentCompressedBytes)), + clientSentMessagesPerRPC.M(atomic.LoadInt64(&mi.sentMsgs)), + clientReceivedMessagesPerRPC.M(atomic.LoadInt64(&mi.recvMsgs)), + clientReceivedBytesPerRPC.M(atomic.LoadInt64(&mi.recvBytes)), + clientReceivedCompressedBytesPerRPC.M(atomic.LoadInt64(&mi.recvCompressedBytes)), + clientRoundtripLatency.M(latency), + clientServerLatency.M(latency), + )) + return + } + ocstats.RecordWithOptions(ctx, + ocstats.WithTags( + tag.Upsert(keyServerMethod, removeLeadingSlash(mi.method)), + tag.Upsert(keyServerStatus, st), + ), + ocstats.WithMeasurements( + serverSentBytesPerRPC.M(atomic.LoadInt64(&mi.sentBytes)), + serverSentCompressedBytesPerRPC.M(atomic.LoadInt64(&mi.sentCompressedBytes)), + serverSentMessagesPerRPC.M(atomic.LoadInt64(&mi.sentMsgs)), + serverReceivedMessagesPerRPC.M(atomic.LoadInt64(&mi.recvMsgs)), + serverReceivedBytesPerRPC.M(atomic.LoadInt64(&mi.recvBytes)), + serverReceivedCompressedBytesPerRPC.M(atomic.LoadInt64(&mi.recvCompressedBytes)), + serverLatency.M(latency))) +} diff --git a/stats/opencensus/trace.go b/stats/opencensus/trace.go new file mode 100644 index 000000000000..5a5438e1cc39 --- /dev/null +++ b/stats/opencensus/trace.go @@ -0,0 +1,124 @@ +/* + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opencensus + +import ( + "context" + "strings" + "sync/atomic" + + "go.opencensus.io/trace" + "go.opencensus.io/trace/propagation" + + "google.golang.org/grpc/stats" + "google.golang.org/grpc/status" +) + +// traceInfo is data used for recording traces. +type traceInfo struct { + span *trace.Span + countSentMsg uint32 + countRecvMsg uint32 +} + +// traceTagRPC populates context with a new span, and serializes information +// about this span into gRPC Metadata. +func (csh *clientStatsHandler) traceTagRPC(ctx context.Context, rti *stats.RPCTagInfo) (context.Context, *traceInfo) { + // TODO: get consensus on whether this method name of "s.m" is correct. + mn := "Attempt." + strings.Replace(removeLeadingSlash(rti.FullMethodName), "/", ".", -1) + // Returned context is ignored because will populate context with data that + // wraps the span instead. Don't set span kind client on this attempt span + // to prevent backend from prepending span name with "Sent.". + _, span := trace.StartSpan(ctx, mn, trace.WithSampler(csh.to.TS)) + + tcBin := propagation.Binary(span.SpanContext()) + return stats.SetTrace(ctx, tcBin), &traceInfo{ + span: span, + countSentMsg: 0, // msg events scoped to scope of context, per attempt client side + countRecvMsg: 0, + } +} + +// traceTagRPC populates context with new span data, with a parent based on the +// spanContext deserialized from context passed in (wire data in gRPC metadata) +// if present. +func (ssh *serverStatsHandler) traceTagRPC(ctx context.Context, rti *stats.RPCTagInfo) (context.Context, *traceInfo) { + mn := strings.Replace(removeLeadingSlash(rti.FullMethodName), "/", ".", -1) + + var span *trace.Span + if sc, ok := propagation.FromBinary(stats.Trace(ctx)); ok { + // Returned context is ignored because will populate context with data + // that wraps the span instead. + _, span = trace.StartSpanWithRemoteParent(ctx, mn, sc, trace.WithSpanKind(trace.SpanKindServer), trace.WithSampler(ssh.to.TS)) + span.AddLink(trace.Link{TraceID: sc.TraceID, SpanID: sc.SpanID, Type: trace.LinkTypeChild}) + } else { + // Returned context is ignored because will populate context with data + // that wraps the span instead. + _, span = trace.StartSpan(ctx, mn, trace.WithSpanKind(trace.SpanKindServer), trace.WithSampler(ssh.to.TS)) + } + + return ctx, &traceInfo{ + span: span, + countSentMsg: 0, + countRecvMsg: 0, + } +} + +// populateSpan populates span information based on stats passed in (invariants +// of the RPC lifecycle), and also ends span which triggers the span to be +// exported. +func populateSpan(ctx context.Context, rs stats.RPCStats, ti *traceInfo) { + if ti == nil || ti.span == nil { + // Shouldn't happen, tagRPC call comes before this function gets called + // which populates this information. + logger.Error("ctx passed into stats handler tracing event handling has no span present") + return + } + span := ti.span + + switch rs := rs.(type) { + case *stats.Begin: + // Note: Go always added these attributes even though they are not + // defined by the OpenCensus gRPC spec. Thus, they are unimportant for + // correctness. + span.AddAttributes( + trace.BoolAttribute("Client", rs.Client), + trace.BoolAttribute("FailFast", rs.FailFast), + ) + case *stats.PickerUpdated: + span.Annotate(nil, "Delayed LB pick complete") + case *stats.InPayload: + // message id - "must be calculated as two different counters starting + // from one for sent messages and one for received messages." + mi := atomic.AddUint32(&ti.countRecvMsg, 1) + span.AddMessageReceiveEvent(int64(mi), int64(rs.Length), int64(rs.CompressedLength)) + case *stats.OutPayload: + mi := atomic.AddUint32(&ti.countSentMsg, 1) + span.AddMessageSendEvent(int64(mi), int64(rs.Length), int64(rs.CompressedLength)) + case *stats.End: + if rs.Error != nil { + // "The mapping between gRPC canonical codes and OpenCensus codes + // can be found here", which implies 1:1 mapping to gRPC statuses + // (OpenCensus statuses are based off gRPC statuses and a subset). + s := status.Convert(rs.Error) + span.SetStatus(trace.Status{Code: int32(s.Code()), Message: s.Message()}) + } else { + span.SetStatus(trace.Status{Code: trace.StatusCodeOK}) // could get rid of this else conditional and just leave as 0 value, but this makes it explicit + } + span.End() + } +} diff --git a/stats/stats.go b/stats/stats.go index 63e476ee7ff8..4ab70e2d462a 100644 --- a/stats/stats.go +++ b/stats/stats.go @@ -36,15 +36,22 @@ type RPCStats interface { IsClient() bool } -// Begin contains stats when an RPC begins. +// Begin contains stats when an RPC attempt begins. // FailFast is only valid if this Begin is from client side. type Begin struct { // Client is true if this Begin is from client side. Client bool - // BeginTime is the time when the RPC begins. + // BeginTime is the time when the RPC attempt begins. BeginTime time.Time // FailFast indicates if this RPC is failfast. FailFast bool + // IsClientStream indicates whether the RPC is a client streaming RPC. + IsClientStream bool + // IsServerStream indicates whether the RPC is a server streaming RPC. + IsServerStream bool + // IsTransparentRetryAttempt indicates whether this attempt was initiated + // due to transparently retrying a previous attempt. + IsTransparentRetryAttempt bool } // IsClient indicates if the stats information is from client side. @@ -52,18 +59,36 @@ func (s *Begin) IsClient() bool { return s.Client } func (s *Begin) isRPCStats() {} +// PickerUpdated indicates that the LB policy provided a new picker while the +// RPC was waiting for one. +type PickerUpdated struct{} + +// IsClient indicates if the stats information is from client side. Only Client +// Side interfaces with a Picker, thus always returns true. +func (*PickerUpdated) IsClient() bool { return true } + +func (*PickerUpdated) isRPCStats() {} + // InPayload contains the information for an incoming payload. type InPayload struct { // Client is true if this InPayload is from client side. Client bool // Payload is the payload with original type. - Payload interface{} + Payload any // Data is the serialized message payload. Data []byte - // Length is the length of uncompressed data. + + // Length is the size of the uncompressed payload data. Does not include any + // framing (gRPC or HTTP/2). Length int - // WireLength is the length of data on wire (compressed, signed, encrypted). + // CompressedLength is the size of the compressed payload data. Does not + // include any framing (gRPC or HTTP/2). Same as Length if compression not + // enabled. + CompressedLength int + // WireLength is the size of the compressed payload data plus gRPC framing. + // Does not include HTTP/2 framing. WireLength int + // RecvTime is the time when the payload is received. RecvTime time.Time } @@ -119,12 +144,18 @@ type OutPayload struct { // Client is true if this OutPayload is from client side. Client bool // Payload is the payload with original type. - Payload interface{} + Payload any // Data is the serialized message payload. Data []byte - // Length is the length of uncompressed data. + // Length is the size of the uncompressed payload data. Does not include any + // framing (gRPC or HTTP/2). Length int - // WireLength is the length of data on wire (compressed, signed, encrypted). + // CompressedLength is the size of the compressed payload data. Does not + // include any framing (gRPC or HTTP/2). Same as Length if compression not + // enabled. + CompressedLength int + // WireLength is the size of the compressed payload data plus gRPC framing. + // Does not include HTTP/2 framing. WireLength int // SentTime is the time when the payload is sent. SentTime time.Time diff --git a/stats/stats_test.go b/stats/stats_test.go index 628533edbe2c..7dcb53491a26 100644 --- a/stats/stats_test.go +++ b/stats/stats_test.go @@ -30,13 +30,18 @@ import ( "github.com/golang/protobuf/proto" "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/internal/grpctest" "google.golang.org/grpc/metadata" "google.golang.org/grpc/stats" - testpb "google.golang.org/grpc/stats/grpc_testing" "google.golang.org/grpc/status" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" ) +const defaultTestTimeout = 10 * time.Second + type s struct { grpctest.Tester } @@ -73,9 +78,20 @@ var ( errorID int32 = 32202 ) -type testServer struct{} +func idToPayload(id int32) *testpb.Payload { + return &testpb.Payload{Body: []byte{byte(id), byte(id >> 8), byte(id >> 16), byte(id >> 24)}} +} + +func payloadToID(p *testpb.Payload) int32 { + if p == nil || len(p.Body) != 4 { + panic("invalid payload") + } + return int32(p.Body[0]) + int32(p.Body[1])<<8 + int32(p.Body[2])<<16 + int32(p.Body[3])<<24 +} -var _ testpb.UnstableTestServiceService = (*testServer)(nil) +type testServer struct { + testgrpc.UnimplementedTestServiceServer +} func (s *testServer) UnaryCall(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { if err := grpc.SendHeader(ctx, testHeaderMetadata); err != nil { @@ -85,14 +101,14 @@ func (s *testServer) UnaryCall(ctx context.Context, in *testpb.SimpleRequest) (* return nil, status.Errorf(status.Code(err), "grpc.SetTrailer(_, %v) = %v, want ", testTrailerMetadata, err) } - if in.Id == errorID { - return nil, fmt.Errorf("got error id: %v", in.Id) + if id := payloadToID(in.Payload); id == errorID { + return nil, fmt.Errorf("got error id: %v", id) } - return &testpb.SimpleResponse{Id: in.Id}, nil + return &testpb.SimpleResponse{Payload: in.Payload}, nil } -func (s *testServer) FullDuplexCall(stream testpb.TestService_FullDuplexCallServer) error { +func (s *testServer) FullDuplexCall(stream testgrpc.TestService_FullDuplexCallServer) error { if err := stream.SendHeader(testHeaderMetadata); err != nil { return status.Errorf(status.Code(err), "%v.SendHeader(%v) = %v, want %v", stream, testHeaderMetadata, err, nil) } @@ -107,17 +123,17 @@ func (s *testServer) FullDuplexCall(stream testpb.TestService_FullDuplexCallServ return err } - if in.Id == errorID { - return fmt.Errorf("got error id: %v", in.Id) + if id := payloadToID(in.Payload); id == errorID { + return fmt.Errorf("got error id: %v", id) } - if err := stream.Send(&testpb.SimpleResponse{Id: in.Id}); err != nil { + if err := stream.Send(&testpb.StreamingOutputCallResponse{Payload: in.Payload}); err != nil { return err } } } -func (s *testServer) ClientStreamCall(stream testpb.TestService_ClientStreamCallServer) error { +func (s *testServer) StreamingInputCall(stream testgrpc.TestService_StreamingInputCallServer) error { if err := stream.SendHeader(testHeaderMetadata); err != nil { return status.Errorf(status.Code(err), "%v.SendHeader(%v) = %v, want %v", stream, testHeaderMetadata, err, nil) } @@ -126,30 +142,30 @@ func (s *testServer) ClientStreamCall(stream testpb.TestService_ClientStreamCall in, err := stream.Recv() if err == io.EOF { // read done. - return stream.SendAndClose(&testpb.SimpleResponse{Id: int32(0)}) + return stream.SendAndClose(&testpb.StreamingInputCallResponse{AggregatedPayloadSize: 0}) } if err != nil { return err } - if in.Id == errorID { - return fmt.Errorf("got error id: %v", in.Id) + if id := payloadToID(in.Payload); id == errorID { + return fmt.Errorf("got error id: %v", id) } } } -func (s *testServer) ServerStreamCall(in *testpb.SimpleRequest, stream testpb.TestService_ServerStreamCallServer) error { +func (s *testServer) StreamingOutputCall(in *testpb.StreamingOutputCallRequest, stream testgrpc.TestService_StreamingOutputCallServer) error { if err := stream.SendHeader(testHeaderMetadata); err != nil { return status.Errorf(status.Code(err), "%v.SendHeader(%v) = %v, want %v", stream, testHeaderMetadata, err, nil) } stream.SetTrailer(testTrailerMetadata) - if in.Id == errorID { - return fmt.Errorf("got error id: %v", in.Id) + if id := payloadToID(in.Payload); id == errorID { + return fmt.Errorf("got error id: %v", id) } for i := 0; i < 5; i++ { - if err := stream.Send(&testpb.SimpleResponse{Id: in.Id}); err != nil { + if err := stream.Send(&testpb.StreamingOutputCallResponse{Payload: in.Payload}); err != nil { return err } } @@ -160,12 +176,12 @@ func (s *testServer) ServerStreamCall(in *testpb.SimpleRequest, stream testpb.Te // func, modified as needed, and then started with its startServer method. // It should be cleaned up with the tearDown method. type test struct { - t *testing.T - compress string - clientStatsHandler stats.Handler - serverStatsHandler stats.Handler + t *testing.T + compress string + clientStatsHandlers []stats.Handler + serverStatsHandlers []stats.Handler - testService *testpb.TestServiceService // nil means none + testServer testgrpc.TestServiceServer // nil means none // srv and srvAddr are set once startServer is called. srv *grpc.Server srvAddr string @@ -188,20 +204,20 @@ type testConfig struct { // newTest returns a new test using the provided testing.T and // environment. It is returned with default values. Tests should // modify it before calling its startServer and clientConn methods. -func newTest(t *testing.T, tc *testConfig, ch stats.Handler, sh stats.Handler) *test { +func newTest(t *testing.T, tc *testConfig, chs []stats.Handler, shs []stats.Handler) *test { te := &test{ - t: t, - compress: tc.compress, - clientStatsHandler: ch, - serverStatsHandler: sh, + t: t, + compress: tc.compress, + clientStatsHandlers: chs, + serverStatsHandlers: shs, } return te } // startServer starts a gRPC server listening. Callers should defer a // call to te.tearDown to clean up. -func (te *test) startServer(ts *testpb.TestServiceService) { - te.testService = ts +func (te *test) startServer(ts testgrpc.TestServiceServer) { + te.testServer = ts lis, err := net.Listen("tcp", "localhost:0") if err != nil { te.t.Fatalf("Failed to listen: %v", err) @@ -213,13 +229,13 @@ func (te *test) startServer(ts *testpb.TestServiceService) { grpc.RPCDecompressor(grpc.NewGZIPDecompressor()), ) } - if te.serverStatsHandler != nil { - opts = append(opts, grpc.StatsHandler(te.serverStatsHandler)) + for _, sh := range te.serverStatsHandlers { + opts = append(opts, grpc.StatsHandler(sh)) } s := grpc.NewServer(opts...) te.srv = s - if te.testService != nil { - testpb.RegisterTestServiceService(s, te.testService) + if te.testServer != nil { + testgrpc.RegisterTestServiceServer(s, te.testServer) } go s.Serve(lis) @@ -231,7 +247,7 @@ func (te *test) clientConn() *grpc.ClientConn { return te.cc } opts := []grpc.DialOption{ - grpc.WithInsecure(), + grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithBlock(), grpc.WithUserAgent("test/0.0.1"), } @@ -241,8 +257,8 @@ func (te *test) clientConn() *grpc.ClientConn { grpc.WithDecompressor(grpc.NewGZIPDecompressor()), ) } - if te.clientStatsHandler != nil { - opts = append(opts, grpc.WithStatsHandler(te.clientStatsHandler)) + for _, sh := range te.clientStatsHandlers { + opts = append(opts, grpc.WithStatsHandler(sh)) } var err error @@ -275,25 +291,29 @@ func (te *test) doUnaryCall(c *rpcConfig) (*testpb.SimpleRequest, *testpb.Simple req *testpb.SimpleRequest err error ) - tc := testpb.NewTestServiceClient(te.clientConn()) + tc := testgrpc.NewTestServiceClient(te.clientConn()) if c.success { - req = &testpb.SimpleRequest{Id: errorID + 1} + req = &testpb.SimpleRequest{Payload: idToPayload(errorID + 1)} } else { - req = &testpb.SimpleRequest{Id: errorID} + req = &testpb.SimpleRequest{Payload: idToPayload(errorID)} } - ctx := metadata.NewOutgoingContext(context.Background(), testMetadata) - resp, err = tc.UnaryCall(ctx, req, grpc.WaitForReady(!c.failfast)) + + tCtx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + resp, err = tc.UnaryCall(metadata.NewOutgoingContext(tCtx, testMetadata), req, grpc.WaitForReady(!c.failfast)) return req, resp, err } -func (te *test) doFullDuplexCallRoundtrip(c *rpcConfig) ([]*testpb.SimpleRequest, []*testpb.SimpleResponse, error) { +func (te *test) doFullDuplexCallRoundtrip(c *rpcConfig) ([]proto.Message, []proto.Message, error) { var ( - reqs []*testpb.SimpleRequest - resps []*testpb.SimpleResponse + reqs []proto.Message + resps []proto.Message err error ) - tc := testpb.NewTestServiceClient(te.clientConn()) - stream, err := tc.FullDuplexCall(metadata.NewOutgoingContext(context.Background(), testMetadata), grpc.WaitForReady(!c.failfast)) + tc := testgrpc.NewTestServiceClient(te.clientConn()) + tCtx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + stream, err := tc.FullDuplexCall(metadata.NewOutgoingContext(tCtx, testMetadata), grpc.WaitForReady(!c.failfast)) if err != nil { return reqs, resps, err } @@ -302,14 +322,14 @@ func (te *test) doFullDuplexCallRoundtrip(c *rpcConfig) ([]*testpb.SimpleRequest startID = errorID } for i := 0; i < c.count; i++ { - req := &testpb.SimpleRequest{ - Id: int32(i) + startID, + req := &testpb.StreamingOutputCallRequest{ + Payload: idToPayload(int32(i) + startID), } reqs = append(reqs, req) if err = stream.Send(req); err != nil { return reqs, resps, err } - var resp *testpb.SimpleResponse + var resp *testpb.StreamingOutputCallResponse if resp, err = stream.Recv(); err != nil { return reqs, resps, err } @@ -325,14 +345,16 @@ func (te *test) doFullDuplexCallRoundtrip(c *rpcConfig) ([]*testpb.SimpleRequest return reqs, resps, nil } -func (te *test) doClientStreamCall(c *rpcConfig) ([]*testpb.SimpleRequest, *testpb.SimpleResponse, error) { +func (te *test) doClientStreamCall(c *rpcConfig) ([]proto.Message, *testpb.StreamingInputCallResponse, error) { var ( - reqs []*testpb.SimpleRequest - resp *testpb.SimpleResponse + reqs []proto.Message + resp *testpb.StreamingInputCallResponse err error ) - tc := testpb.NewTestServiceClient(te.clientConn()) - stream, err := tc.ClientStreamCall(metadata.NewOutgoingContext(context.Background(), testMetadata), grpc.WaitForReady(!c.failfast)) + tc := testgrpc.NewTestServiceClient(te.clientConn()) + tCtx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + stream, err := tc.StreamingInputCall(metadata.NewOutgoingContext(tCtx, testMetadata), grpc.WaitForReady(!c.failfast)) if err != nil { return reqs, resp, err } @@ -341,8 +363,8 @@ func (te *test) doClientStreamCall(c *rpcConfig) ([]*testpb.SimpleRequest, *test startID = errorID } for i := 0; i < c.count; i++ { - req := &testpb.SimpleRequest{ - Id: int32(i) + startID, + req := &testpb.StreamingInputCallRequest{ + Payload: idToPayload(int32(i) + startID), } reqs = append(reqs, req) if err = stream.Send(req); err != nil { @@ -353,26 +375,28 @@ func (te *test) doClientStreamCall(c *rpcConfig) ([]*testpb.SimpleRequest, *test return reqs, resp, err } -func (te *test) doServerStreamCall(c *rpcConfig) (*testpb.SimpleRequest, []*testpb.SimpleResponse, error) { +func (te *test) doServerStreamCall(c *rpcConfig) (*testpb.StreamingOutputCallRequest, []proto.Message, error) { var ( - req *testpb.SimpleRequest - resps []*testpb.SimpleResponse + req *testpb.StreamingOutputCallRequest + resps []proto.Message err error ) - tc := testpb.NewTestServiceClient(te.clientConn()) + tc := testgrpc.NewTestServiceClient(te.clientConn()) var startID int32 if !c.success { startID = errorID } - req = &testpb.SimpleRequest{Id: startID} - stream, err := tc.ServerStreamCall(metadata.NewOutgoingContext(context.Background(), testMetadata), req, grpc.WaitForReady(!c.failfast)) + req = &testpb.StreamingOutputCallRequest{Payload: idToPayload(startID)} + tCtx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + stream, err := tc.StreamingOutputCall(metadata.NewOutgoingContext(tCtx, testMetadata), req, grpc.WaitForReady(!c.failfast)) if err != nil { return req, resps, err } for { - var resp *testpb.SimpleResponse + var resp *testpb.StreamingOutputCallResponse resp, err := stream.Recv() if err == io.EOF { return req, resps, nil @@ -384,21 +408,23 @@ func (te *test) doServerStreamCall(c *rpcConfig) (*testpb.SimpleRequest, []*test } type expectedData struct { - method string - serverAddr string - compression string - reqIdx int - requests []*testpb.SimpleRequest - respIdx int - responses []*testpb.SimpleResponse - err error - failfast bool + method string + isClientStream bool + isServerStream bool + serverAddr string + compression string + reqIdx int + requests []proto.Message + respIdx int + responses []proto.Message + err error + failfast bool } type gotData struct { ctx context.Context client bool - s interface{} // This could be RPCStats or ConnStats. + s any // This could be RPCStats or ConnStats. } const ( @@ -433,6 +459,12 @@ func checkBegin(t *testing.T, d *gotData, e *expectedData) { t.Fatalf("st.FailFast = %v, want %v", st.FailFast, e.failfast) } } + if st.IsClientStream != e.isClientStream { + t.Fatalf("st.IsClientStream = %v, want %v", st.IsClientStream, e.isClientStream) + } + if st.IsServerStream != e.isServerStream { + t.Fatalf("st.IsServerStream = %v, want %v", st.IsServerStream, e.isServerStream) + } } func checkInHeader(t *testing.T, d *gotData, e *expectedData) { @@ -536,9 +568,9 @@ func checkInPayload(t *testing.T, d *gotData, e *expectedData) { } // Below are sanity checks that WireLength and RecvTime are populated. // TODO: check values of WireLength and RecvTime. - if len(st.Data) > 0 && st.WireLength == 0 { + if len(st.Data) > 0 && st.CompressedLength == 0 { t.Fatalf("st.WireLength = %v with non-empty data, want ", - st.WireLength) + st.CompressedLength) } if st.RecvTime.IsZero() { t.Fatalf("st.ReceivedTime = %v, want ", st.RecvTime) @@ -722,7 +754,7 @@ func checkEnd(t *testing.T, d *gotData, e *expectedData) { } } -func checkConnBegin(t *testing.T, d *gotData, e *expectedData) { +func checkConnBegin(t *testing.T, d *gotData) { var ( ok bool st *stats.ConnBegin @@ -736,7 +768,7 @@ func checkConnBegin(t *testing.T, d *gotData, e *expectedData) { st.IsClient() // TODO remove this. } -func checkConnEnd(t *testing.T, d *gotData, e *expectedData) { +func checkConnEnd(t *testing.T, d *gotData) { var ( ok bool st *stats.ConnEnd @@ -784,9 +816,9 @@ func checkConnStats(t *testing.T, got []*gotData) { t.Fatalf("got %v stats, want even positive number", len(got)) } // The first conn stats must be a ConnBegin. - checkConnBegin(t, got[0], nil) + checkConnBegin(t, got[0]) // The last conn stats must be a ConnEnd. - checkConnEnd(t, got[len(got)-1], nil) + checkConnEnd(t, got[len(got)-1]) } func checkServerStats(t *testing.T, got []*gotData, expect *expectedData, checkFuncs []func(t *testing.T, d *gotData, e *expectedData)) { @@ -814,18 +846,21 @@ func checkServerStats(t *testing.T, got []*gotData, expect *expectedData, checkF func testServerStats(t *testing.T, tc *testConfig, cc *rpcConfig, checkFuncs []func(t *testing.T, d *gotData, e *expectedData)) { h := &statshandler{} - te := newTest(t, tc, nil, h) - te.startServer(testpb.NewTestServiceService(&testServer{})) + te := newTest(t, tc, nil, []stats.Handler{h}) + te.startServer(&testServer{}) defer te.tearDown() var ( - reqs []*testpb.SimpleRequest - resps []*testpb.SimpleResponse + reqs []proto.Message + resps []proto.Message err error method string - req *testpb.SimpleRequest - resp *testpb.SimpleResponse + isClientStream bool + isServerStream bool + + req proto.Message + resp proto.Message e error ) @@ -833,22 +868,26 @@ func testServerStats(t *testing.T, tc *testConfig, cc *rpcConfig, checkFuncs []f case unaryRPC: method = "/grpc.testing.TestService/UnaryCall" req, resp, e = te.doUnaryCall(cc) - reqs = []*testpb.SimpleRequest{req} - resps = []*testpb.SimpleResponse{resp} + reqs = []proto.Message{req} + resps = []proto.Message{resp} err = e case clientStreamRPC: - method = "/grpc.testing.TestService/ClientStreamCall" + method = "/grpc.testing.TestService/StreamingInputCall" reqs, resp, e = te.doClientStreamCall(cc) - resps = []*testpb.SimpleResponse{resp} + resps = []proto.Message{resp} err = e + isClientStream = true case serverStreamRPC: - method = "/grpc.testing.TestService/ServerStreamCall" + method = "/grpc.testing.TestService/StreamingOutputCall" req, resps, e = te.doServerStreamCall(cc) - reqs = []*testpb.SimpleRequest{req} + reqs = []proto.Message{req} err = e + isServerStream = true case fullDuplexStreamRPC: method = "/grpc.testing.TestService/FullDuplexCall" reqs, resps, err = te.doFullDuplexCallRoundtrip(cc) + isClientStream = true + isServerStream = true } if cc.success != (err == nil) { t.Fatalf("cc.success: %v, got error: %v", cc.success, err) @@ -877,12 +916,14 @@ func testServerStats(t *testing.T, tc *testConfig, cc *rpcConfig, checkFuncs []f } expect := &expectedData{ - serverAddr: te.srvAddr, - compression: tc.compress, - method: method, - requests: reqs, - responses: resps, - err: err, + serverAddr: te.srvAddr, + compression: tc.compress, + method: method, + requests: reqs, + responses: resps, + err: err, + isClientStream: isClientStream, + isServerStream: isServerStream, } h.mu.Lock() @@ -1105,40 +1146,47 @@ func checkClientStats(t *testing.T, got []*gotData, expect *expectedData, checkF func testClientStats(t *testing.T, tc *testConfig, cc *rpcConfig, checkFuncs map[int]*checkFuncWithCount) { h := &statshandler{} - te := newTest(t, tc, h, nil) - te.startServer(testpb.NewTestServiceService(&testServer{})) + te := newTest(t, tc, []stats.Handler{h}, nil) + te.startServer(&testServer{}) defer te.tearDown() var ( - reqs []*testpb.SimpleRequest - resps []*testpb.SimpleResponse + reqs []proto.Message + resps []proto.Message method string err error - req *testpb.SimpleRequest - resp *testpb.SimpleResponse + isClientStream bool + isServerStream bool + + req proto.Message + resp proto.Message e error ) switch cc.callType { case unaryRPC: method = "/grpc.testing.TestService/UnaryCall" req, resp, e = te.doUnaryCall(cc) - reqs = []*testpb.SimpleRequest{req} - resps = []*testpb.SimpleResponse{resp} + reqs = []proto.Message{req} + resps = []proto.Message{resp} err = e case clientStreamRPC: - method = "/grpc.testing.TestService/ClientStreamCall" + method = "/grpc.testing.TestService/StreamingInputCall" reqs, resp, e = te.doClientStreamCall(cc) - resps = []*testpb.SimpleResponse{resp} + resps = []proto.Message{resp} err = e + isClientStream = true case serverStreamRPC: - method = "/grpc.testing.TestService/ServerStreamCall" + method = "/grpc.testing.TestService/StreamingOutputCall" req, resps, e = te.doServerStreamCall(cc) - reqs = []*testpb.SimpleRequest{req} + reqs = []proto.Message{req} err = e + isServerStream = true case fullDuplexStreamRPC: method = "/grpc.testing.TestService/FullDuplexCall" reqs, resps, err = te.doFullDuplexCallRoundtrip(cc) + isClientStream = true + isServerStream = true } if cc.success != (err == nil) { t.Fatalf("cc.success: %v, got error: %v", cc.success, err) @@ -1171,13 +1219,15 @@ func testClientStats(t *testing.T, tc *testConfig, cc *rpcConfig, checkFuncs map } expect := &expectedData{ - serverAddr: te.srvAddr, - compression: tc.compress, - method: method, - requests: reqs, - responses: resps, - failfast: cc.failfast, - err: err, + serverAddr: te.srvAddr, + compression: tc.compress, + method: method, + requests: reqs, + responses: resps, + failfast: cc.failfast, + err: err, + isClientStream: isClientStream, + isServerStream: isServerStream, } h.mu.Lock() @@ -1286,7 +1336,9 @@ func (s) TestClientStatsFullDuplexRPCError(t *testing.T) { func (s) TestTags(t *testing.T) { b := []byte{5, 2, 4, 3, 1} - ctx := stats.SetTags(context.Background(), b) + tCtx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctx := stats.SetTags(tCtx, b) if tg := stats.OutgoingTags(ctx); !reflect.DeepEqual(tg, b) { t.Errorf("OutgoingTags(%v) = %v; want %v", ctx, tg, b) } @@ -1294,7 +1346,7 @@ func (s) TestTags(t *testing.T) { t.Errorf("Tags(%v) = %v; want nil", ctx, tg) } - ctx = stats.SetIncomingTags(context.Background(), b) + ctx = stats.SetIncomingTags(tCtx, b) if tg := stats.Tags(ctx); !reflect.DeepEqual(tg, b) { t.Errorf("Tags(%v) = %v; want %v", ctx, tg, b) } @@ -1305,7 +1357,9 @@ func (s) TestTags(t *testing.T) { func (s) TestTrace(t *testing.T) { b := []byte{5, 2, 4, 3, 1} - ctx := stats.SetTrace(context.Background(), b) + tCtx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctx := stats.SetTrace(tCtx, b) if tr := stats.OutgoingTrace(ctx); !reflect.DeepEqual(tr, b) { t.Errorf("OutgoingTrace(%v) = %v; want %v", ctx, tr, b) } @@ -1313,7 +1367,7 @@ func (s) TestTrace(t *testing.T) { t.Errorf("Trace(%v) = %v; want nil", ctx, tr) } - ctx = stats.SetIncomingTrace(context.Background(), b) + ctx = stats.SetIncomingTrace(tCtx, b) if tr := stats.Trace(ctx); !reflect.DeepEqual(tr, b) { t.Errorf("Trace(%v) = %v; want %v", ctx, tr, b) } @@ -1321,3 +1375,95 @@ func (s) TestTrace(t *testing.T) { t.Errorf("OutgoingTrace(%v) = %v; want nil", ctx, tr) } } + +func (s) TestMultipleClientStatsHandler(t *testing.T) { + h := &statshandler{} + tc := &testConfig{compress: ""} + te := newTest(t, tc, []stats.Handler{h, h}, nil) + te.startServer(&testServer{}) + defer te.tearDown() + + cc := &rpcConfig{success: false, failfast: false, callType: unaryRPC} + _, _, err := te.doUnaryCall(cc) + if cc.success != (err == nil) { + t.Fatalf("cc.success: %v, got error: %v", cc.success, err) + } + te.cc.Close() + te.srv.GracefulStop() // Wait for the server to stop. + + for start := time.Now(); time.Since(start) < defaultTestTimeout; { + h.mu.Lock() + if _, ok := h.gotRPC[len(h.gotRPC)-1].s.(*stats.End); ok && len(h.gotRPC) == 12 { + h.mu.Unlock() + break + } + h.mu.Unlock() + time.Sleep(10 * time.Millisecond) + } + + for start := time.Now(); time.Since(start) < defaultTestTimeout; { + h.mu.Lock() + if _, ok := h.gotConn[len(h.gotConn)-1].s.(*stats.ConnEnd); ok && len(h.gotConn) == 4 { + h.mu.Unlock() + break + } + h.mu.Unlock() + time.Sleep(10 * time.Millisecond) + } + + // Each RPC generates 6 stats events on the client-side, times 2 StatsHandler + if len(h.gotRPC) != 12 { + t.Fatalf("h.gotRPC: unexpected amount of RPCStats: %v != %v", len(h.gotRPC), 12) + } + + // Each connection generates 4 conn events on the client-side, times 2 StatsHandler + if len(h.gotConn) != 4 { + t.Fatalf("h.gotConn: unexpected amount of ConnStats: %v != %v", len(h.gotConn), 4) + } +} + +func (s) TestMultipleServerStatsHandler(t *testing.T) { + h := &statshandler{} + tc := &testConfig{compress: ""} + te := newTest(t, tc, nil, []stats.Handler{h, h}) + te.startServer(&testServer{}) + defer te.tearDown() + + cc := &rpcConfig{success: false, failfast: false, callType: unaryRPC} + _, _, err := te.doUnaryCall(cc) + if cc.success != (err == nil) { + t.Fatalf("cc.success: %v, got error: %v", cc.success, err) + } + te.cc.Close() + te.srv.GracefulStop() // Wait for the server to stop. + + for start := time.Now(); time.Since(start) < defaultTestTimeout; { + h.mu.Lock() + if _, ok := h.gotRPC[len(h.gotRPC)-1].s.(*stats.End); ok { + h.mu.Unlock() + break + } + h.mu.Unlock() + time.Sleep(10 * time.Millisecond) + } + + for start := time.Now(); time.Since(start) < defaultTestTimeout; { + h.mu.Lock() + if _, ok := h.gotConn[len(h.gotConn)-1].s.(*stats.ConnEnd); ok { + h.mu.Unlock() + break + } + h.mu.Unlock() + time.Sleep(10 * time.Millisecond) + } + + // Each RPC generates 6 stats events on the server-side, times 2 StatsHandler + if len(h.gotRPC) != 12 { + t.Fatalf("h.gotRPC: unexpected amount of RPCStats: %v != %v", len(h.gotRPC), 12) + } + + // Each connection generates 4 conn events on the server-side, times 2 StatsHandler + if len(h.gotConn) != 4 { + t.Fatalf("h.gotConn: unexpected amount of ConnStats: %v != %v", len(h.gotConn), 4) + } +} diff --git a/status/status.go b/status/status.go index 01e182c306c2..a93360efb847 100644 --- a/status/status.go +++ b/status/status.go @@ -29,6 +29,7 @@ package status import ( "context" + "errors" "fmt" spb "google.golang.org/genproto/googleapis/rpc/status" @@ -49,7 +50,7 @@ func New(c codes.Code, msg string) *Status { } // Newf returns New(c, fmt.Sprintf(format, a...)). -func Newf(c codes.Code, format string, a ...interface{}) *Status { +func Newf(c codes.Code, format string, a ...any) *Status { return New(c, fmt.Sprintf(format, a...)) } @@ -59,7 +60,7 @@ func Error(c codes.Code, msg string) error { } // Errorf returns Error(c, fmt.Sprintf(format, a...)). -func Errorf(c codes.Code, format string, a ...interface{}) error { +func Errorf(c codes.Code, format string, a ...any) error { return Error(c, fmt.Sprintf(format, a...)) } @@ -73,17 +74,54 @@ func FromProto(s *spb.Status) *Status { return status.FromProto(s) } -// FromError returns a Status representing err if it was produced from this -// package or has a method `GRPCStatus() *Status`. Otherwise, ok is false and a -// Status is returned with codes.Unknown and the original error message. +// FromError returns a Status representation of err. +// +// - If err was produced by this package or implements the method `GRPCStatus() +// *Status` and `GRPCStatus()` does not return nil, or if err wraps a type +// satisfying this, the Status from `GRPCStatus()` is returned. For wrapped +// errors, the message returned contains the entire err.Error() text and not +// just the wrapped status. In that case, ok is true. +// +// - If err is nil, a Status is returned with codes.OK and no message, and ok +// is true. +// +// - If err implements the method `GRPCStatus() *Status` and `GRPCStatus()` +// returns nil (which maps to Codes.OK), or if err wraps a type +// satisfying this, a Status is returned with codes.Unknown and err's +// Error() message, and ok is false. +// +// - Otherwise, err is an error not compatible with this package. In this +// case, a Status is returned with codes.Unknown and err's Error() message, +// and ok is false. func FromError(err error) (s *Status, ok bool) { if err == nil { return nil, true } - if se, ok := err.(interface { - GRPCStatus() *Status - }); ok { - return se.GRPCStatus(), true + type grpcstatus interface{ GRPCStatus() *Status } + if gs, ok := err.(grpcstatus); ok { + grpcStatus := gs.GRPCStatus() + if grpcStatus == nil { + // Error has status nil, which maps to codes.OK. There + // is no sensible behavior for this, so we turn it into + // an error with codes.Unknown and discard the existing + // status. + return New(codes.Unknown, err.Error()), false + } + return grpcStatus, true + } + var gs grpcstatus + if errors.As(err, &gs) { + grpcStatus := gs.GRPCStatus() + if grpcStatus == nil { + // Error wraps an error that has status nil, which maps + // to codes.OK. There is no sensible behavior for this, + // so we turn it into an error with codes.Unknown and + // discard the existing status. + return New(codes.Unknown, err.Error()), false + } + p := grpcStatus.Proto() + p.Message = err.Error() + return status.FromProto(p), true } return New(codes.Unknown, err.Error()), false } @@ -95,33 +133,30 @@ func Convert(err error) *Status { return s } -// Code returns the Code of the error if it is a Status error, codes.OK if err -// is nil, or codes.Unknown otherwise. +// Code returns the Code of the error if it is a Status error or if it wraps a +// Status error. If that is not the case, it returns codes.OK if err is nil, or +// codes.Unknown otherwise. func Code(err error) codes.Code { // Don't use FromError to avoid allocation of OK status. if err == nil { return codes.OK } - if se, ok := err.(interface { - GRPCStatus() *Status - }); ok { - return se.GRPCStatus().Code() - } - return codes.Unknown + + return Convert(err).Code() } -// FromContextError converts a context error into a Status. It returns a -// Status with codes.OK if err is nil, or a Status with codes.Unknown if err is -// non-nil and not a context error. +// FromContextError converts a context error or wrapped context error into a +// Status. It returns a Status with codes.OK if err is nil, or a Status with +// codes.Unknown if err is non-nil and not a context error. func FromContextError(err error) *Status { - switch err { - case nil: + if err == nil { return nil - case context.DeadlineExceeded: + } + if errors.Is(err, context.DeadlineExceeded) { return New(codes.DeadlineExceeded, err.Error()) - case context.Canceled: + } + if errors.Is(err, context.Canceled) { return New(codes.Canceled, err.Error()) - default: - return New(codes.Unknown, err.Error()) } + return New(codes.Unknown, err.Error()) } diff --git a/status/status_ext_test.go b/status/status_ext_test.go index 4c1efc56320f..33c8c71a0062 100644 --- a/status/status_ext_test.go +++ b/status/status_ext_test.go @@ -26,7 +26,8 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/internal/grpctest" "google.golang.org/grpc/status" - "google.golang.org/grpc/test/grpc_testing" + + testpb "google.golang.org/grpc/interop/grpc_testing" ) type s struct { @@ -49,7 +50,7 @@ func errWithDetails(t *testing.T, s *status.Status, details ...proto.Message) er func (s) TestErrorIs(t *testing.T) { // Test errors. testErr := status.Error(codes.Internal, "internal server error") - testErrWithDetails := errWithDetails(t, status.New(codes.Internal, "internal server error"), &grpc_testing.Empty{}) + testErrWithDetails := errWithDetails(t, status.New(codes.Internal, "internal server error"), &testpb.Empty{}) // Test cases. testCases := []struct { @@ -62,8 +63,8 @@ func (s) TestErrorIs(t *testing.T) { {err1: testErr, err2: status.Error(codes.Unknown, "internal server error"), want: false}, {err1: testErr, err2: errors.New("non-grpc error"), want: false}, {err1: testErrWithDetails, err2: status.Error(codes.Internal, "internal server error"), want: false}, - {err1: testErrWithDetails, err2: errWithDetails(t, status.New(codes.Internal, "internal server error"), &grpc_testing.Empty{}), want: true}, - {err1: testErrWithDetails, err2: errWithDetails(t, status.New(codes.Internal, "internal server error"), &grpc_testing.Empty{}, &grpc_testing.Empty{}), want: false}, + {err1: testErrWithDetails, err2: errWithDetails(t, status.New(codes.Internal, "internal server error"), &testpb.Empty{}), want: true}, + {err1: testErrWithDetails, err2: errWithDetails(t, status.New(codes.Internal, "internal server error"), &testpb.Empty{}, &testpb.Empty{}), want: false}, } for _, tc := range testCases { diff --git a/status/status_test.go b/status/status_test.go index 839a3c390ede..d21a862f3637 100644 --- a/status/status_test.go +++ b/status/status_test.go @@ -32,6 +32,7 @@ import ( cpb "google.golang.org/genproto/googleapis/rpc/code" epb "google.golang.org/genproto/googleapis/rpc/errdetails" spb "google.golang.org/genproto/googleapis/rpc/status" + "google.golang.org/grpc/codes" "google.golang.org/grpc/internal/grpctest" "google.golang.org/grpc/internal/status" @@ -192,6 +193,97 @@ func (s) TestFromErrorUnknownError(t *testing.T) { } } +func (s) TestFromErrorWrapped(t *testing.T) { + const code, message = codes.Internal, "test description" + err := fmt.Errorf("wrapped error: %w", Error(code, message)) + s, ok := FromError(err) + if !ok || s.Code() != code || s.Message() != err.Error() || s.Err() == nil { + t.Fatalf("FromError(%v) = %v, %v; want , true", err, s, ok, code, message) + } +} + +type customErrorNilStatus struct { +} + +func (c customErrorNilStatus) Error() string { + return "test" +} + +func (c customErrorNilStatus) GRPCStatus() *Status { + return nil +} + +func (s) TestFromErrorImplementsInterfaceReturnsOKStatus(t *testing.T) { + err := customErrorNilStatus{} + s, ok := FromError(err) + if ok || s.Code() != codes.Unknown || s.Message() != err.Error() { + t.Fatalf("FromError(%v) = %v, %v; want , true", err, s, ok, codes.Unknown, err.Error()) + } +} + +func (s) TestFromErrorImplementsInterfaceReturnsOKStatusWrapped(t *testing.T) { + err := fmt.Errorf("wrapping: %w", customErrorNilStatus{}) + s, ok := FromError(err) + if ok || s.Code() != codes.Unknown || s.Message() != err.Error() { + t.Fatalf("FromError(%v) = %v, %v; want , true", err, s, ok, codes.Unknown, err.Error()) + } +} + +func (s) TestFromErrorImplementsInterfaceWrapped(t *testing.T) { + const code, message = codes.Internal, "test description" + err := fmt.Errorf("wrapped error: %w", customError{Code: code, Message: message}) + s, ok := FromError(err) + if !ok || s.Code() != code || s.Message() != err.Error() || s.Err() == nil { + t.Fatalf("FromError(%v) = %v, %v; want , true", err, s, ok, code, message) + } +} + +func (s) TestCode(t *testing.T) { + const code = codes.Internal + err := Error(code, "test description") + if s := Code(err); s != code { + t.Fatalf("Code(%v) = %v; want ", err, s, code) + } +} + +func (s) TestCodeOK(t *testing.T) { + if s, code := Code(nil), codes.OK; s != code { + t.Fatalf("Code(%v) = %v; want ", nil, s, code) + } +} + +func (s) TestCodeImplementsInterface(t *testing.T) { + const code = codes.Internal + err := customError{Code: code, Message: "test description"} + if s := Code(err); s != code { + t.Fatalf("Code(%v) = %v; want ", err, s, code) + } +} + +func (s) TestCodeUnknownError(t *testing.T) { + const code = codes.Unknown + err := errors.New("unknown error") + if s := Code(err); s != code { + t.Fatalf("Code(%v) = %v; want ", err, s, code) + } +} + +func (s) TestCodeWrapped(t *testing.T) { + const code = codes.Internal + err := fmt.Errorf("wrapped: %w", Error(code, "test description")) + if s := Code(err); s != code { + t.Fatalf("Code(%v) = %v; want ", err, s, code) + } +} + +func (s) TestCodeImplementsInterfaceWrapped(t *testing.T) { + const code = codes.Internal + err := fmt.Errorf("wrapped: %w", customError{Code: code, Message: "test description"}) + if s := Code(err); s != code { + t.Fatalf("Code(%v) = %v; want ", err, s, code) + } +} + func (s) TestConvertKnownError(t *testing.T) { code, message := codes.Internal, "test description" err := Error(code, message) @@ -285,7 +377,7 @@ func (s) TestStatus_WithDetails_Fail(t *testing.T) { func (s) TestStatus_ErrorDetails_Fail(t *testing.T) { tests := []struct { s *Status - i []interface{} + i []any }{ { nil, @@ -297,7 +389,7 @@ func (s) TestStatus_ErrorDetails_Fail(t *testing.T) { }, { New(codes.OK, ""), - []interface{}{}, + []any{}, }, { FromProto(&spb.Status{ @@ -314,7 +406,7 @@ func (s) TestStatus_ErrorDetails_Fail(t *testing.T) { }), }, }), - []interface{}{ + []any{ errors.New(`message type url "" is invalid`), &epb.ResourceInfo{ ResourceType: "book", @@ -364,6 +456,8 @@ func (s) TestFromContextError(t *testing.T) { {in: context.DeadlineExceeded, want: New(codes.DeadlineExceeded, context.DeadlineExceeded.Error())}, {in: context.Canceled, want: New(codes.Canceled, context.Canceled.Error())}, {in: errors.New("other"), want: New(codes.Unknown, "other")}, + {in: fmt.Errorf("wrapped: %w", context.DeadlineExceeded), want: New(codes.DeadlineExceeded, "wrapped: "+context.DeadlineExceeded.Error())}, + {in: fmt.Errorf("wrapped: %w", context.Canceled), want: New(codes.Canceled, "wrapped: "+context.Canceled.Error())}, } for _, tc := range testCases { got := FromContextError(tc.in) diff --git a/stream.go b/stream.go index fbc3fb11cb4d..421a41f8854f 100644 --- a/stream.go +++ b/stream.go @@ -31,11 +31,16 @@ import ( "google.golang.org/grpc/balancer" "google.golang.org/grpc/codes" "google.golang.org/grpc/encoding" + "google.golang.org/grpc/internal" "google.golang.org/grpc/internal/balancerload" "google.golang.org/grpc/internal/binarylog" "google.golang.org/grpc/internal/channelz" "google.golang.org/grpc/internal/grpcrand" "google.golang.org/grpc/internal/grpcutil" + imetadata "google.golang.org/grpc/internal/metadata" + iresolver "google.golang.org/grpc/internal/resolver" + "google.golang.org/grpc/internal/serviceconfig" + istatus "google.golang.org/grpc/internal/status" "google.golang.org/grpc/internal/transport" "google.golang.org/grpc/metadata" "google.golang.org/grpc/peer" @@ -44,20 +49,28 @@ import ( ) // StreamHandler defines the handler called by gRPC server to complete the -// execution of a streaming RPC. If a StreamHandler returns an error, it -// should be produced by the status package, or else gRPC will use -// codes.Unknown as the status code and err.Error() as the status message -// of the RPC. -type StreamHandler func(srv interface{}, stream ServerStream) error - -// StreamDesc represents a streaming RPC service's method specification. +// execution of a streaming RPC. +// +// If a StreamHandler returns an error, it should either be produced by the +// status package, or be one of the context errors. Otherwise, gRPC will use +// codes.Unknown as the status code and err.Error() as the status message of the +// RPC. +type StreamHandler func(srv any, stream ServerStream) error + +// StreamDesc represents a streaming RPC service's method specification. Used +// on the server when registering services and on the client when initiating +// new streams. type StreamDesc struct { - StreamName string - Handler StreamHandler - - // At least one of these is true. - ServerStreams bool - ClientStreams bool + // StreamName and Handler are only used when registering handlers on a + // server. + StreamName string // the name of the method excluding the service + Handler StreamHandler // the handler called for the method + + // ServerStreams and ClientStreams are used for registering handlers on a + // server as well as defining RPC behavior when passed to NewClientStream + // and ClientConn.NewStream. At least one must be true. + ServerStreams bool // indicates the server can perform streaming sends + ClientStreams bool // indicates the client can perform streaming sends } // Stream defines the common interface a client or server stream has to satisfy. @@ -67,9 +80,9 @@ type Stream interface { // Deprecated: See ClientStream and ServerStream documentation instead. Context() context.Context // Deprecated: See ClientStream and ServerStream documentation instead. - SendMsg(m interface{}) error + SendMsg(m any) error // Deprecated: See ClientStream and ServerStream documentation instead. - RecvMsg(m interface{}) error + RecvMsg(m any) error } // ClientStream defines the client-side behavior of a streaming RPC. @@ -78,7 +91,9 @@ type Stream interface { // status package. type ClientStream interface { // Header returns the header metadata received from the server if there - // is any. It blocks if the metadata is not ready to read. + // is any. It blocks if the metadata is not ready to read. If the metadata + // is nil and the error is also nil, then the stream was terminated without + // headers, and the status can be discovered by calling RecvMsg. Header() (metadata.MD, error) // Trailer returns the trailer metadata from the server, if there is any. // It must only be called after stream.CloseAndRecv has returned, or @@ -111,7 +126,10 @@ type ClientStream interface { // calling RecvMsg on the same stream at the same time, but it is not safe // to call SendMsg on the same stream in different goroutines. It is also // not safe to call CloseSend concurrently with SendMsg. - SendMsg(m interface{}) error + // + // It is not safe to modify the message after calling SendMsg. Tracing + // libraries and stats handlers may use the message lazily. + SendMsg(m any) error // RecvMsg blocks until it receives a message into m or the stream is // done. It returns io.EOF when the stream completes successfully. On // any other error, the stream is aborted and the error contains the RPC @@ -120,7 +138,7 @@ type ClientStream interface { // It is safe to have a goroutine calling SendMsg and another goroutine // calling RecvMsg on the same stream at the same time, but it is not // safe to call RecvMsg on the same stream in different goroutines. - RecvMsg(m interface{}) error + RecvMsg(m any) error } // NewStream creates a new Stream for the client side. This is typically @@ -129,17 +147,22 @@ type ClientStream interface { // To ensure resources are not leaked due to the stream returned, one of the following // actions must be performed: // -// 1. Call Close on the ClientConn. -// 2. Cancel the context provided. -// 3. Call RecvMsg until a non-nil error is returned. A protobuf-generated -// client-streaming RPC, for instance, might use the helper function -// CloseAndRecv (note that CloseSend does not Recv, therefore is not -// guaranteed to release all resources). -// 4. Receive a non-nil, non-io.EOF error from Header or SendMsg. +// 1. Call Close on the ClientConn. +// 2. Cancel the context provided. +// 3. Call RecvMsg until a non-nil error is returned. A protobuf-generated +// client-streaming RPC, for instance, might use the helper function +// CloseAndRecv (note that CloseSend does not Recv, therefore is not +// guaranteed to release all resources). +// 4. Receive a non-nil, non-io.EOF error from Header or SendMsg. // // If none of the above happen, a goroutine and a context will be leaked, and grpc // will not call the optionally-configured stats handler with a stats.End message. func (cc *ClientConn) NewStream(ctx context.Context, desc *StreamDesc, method string, opts ...CallOption) (ClientStream, error) { + if err := cc.idlenessMgr.OnCallBegin(); err != nil { + return nil, err + } + defer cc.idlenessMgr.OnCallEnd() + // allow interceptor to see all applicable call options, which means those // configured as defaults from dial option as well as per-call options opts = combine(cc.dopts.callOptions, opts) @@ -156,6 +179,20 @@ func NewClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, meth } func newClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, method string, opts ...CallOption) (_ ClientStream, err error) { + if md, added, ok := metadata.FromOutgoingContextRaw(ctx); ok { + // validate md + if err := imetadata.Validate(md); err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + // validate added + for _, kvs := range added { + for i := 0; i < len(kvs); i += 2 { + if err := imetadata.ValidatePair(kvs[i], kvs[i+1]); err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + } + } + } if channelz.IsOn() { cc.incrCallsStarted() defer func() { @@ -164,13 +201,55 @@ func newClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, meth } }() } - c := defaultCallInfo() // Provide an opportunity for the first RPC to see the first service config // provided by the resolver. if err := cc.waitForResolvedAddrs(ctx); err != nil { return nil, err } - mc := cc.GetMethodConfig(method) + + var mc serviceconfig.MethodConfig + var onCommit func() + var newStream = func(ctx context.Context, done func()) (iresolver.ClientStream, error) { + return newClientStreamWithParams(ctx, desc, cc, method, mc, onCommit, done, opts...) + } + + rpcInfo := iresolver.RPCInfo{Context: ctx, Method: method} + rpcConfig, err := cc.safeConfigSelector.SelectConfig(rpcInfo) + if err != nil { + if st, ok := status.FromError(err); ok { + // Restrict the code to the list allowed by gRFC A54. + if istatus.IsRestrictedControlPlaneCode(st) { + err = status.Errorf(codes.Internal, "config selector returned illegal status: %v", err) + } + return nil, err + } + return nil, toRPCErr(err) + } + + if rpcConfig != nil { + if rpcConfig.Context != nil { + ctx = rpcConfig.Context + } + mc = rpcConfig.MethodConfig + onCommit = rpcConfig.OnCommitted + if rpcConfig.Interceptor != nil { + rpcInfo.Context = nil + ns := newStream + newStream = func(ctx context.Context, done func()) (iresolver.ClientStream, error) { + cs, err := rpcConfig.Interceptor.NewStream(ctx, rpcInfo, done, ns) + if err != nil { + return nil, toRPCErr(err) + } + return cs, nil + } + } + } + + return newStream(ctx, func() {}) +} + +func newClientStreamWithParams(ctx context.Context, desc *StreamDesc, cc *ClientConn, method string, mc serviceconfig.MethodConfig, onCommit, doneFunc func(), opts ...CallOption) (_ iresolver.ClientStream, err error) { + c := defaultCallInfo() if mc.WaitForReady != nil { c.failFast = !*mc.WaitForReady } @@ -207,6 +286,7 @@ func newClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, meth Host: cc.authority, Method: method, ContentSubtype: c.contentSubtype, + DoneFunc: doneFunc, } // Set our outgoing compression according to the UseCompressor CallOption, if @@ -230,33 +310,6 @@ func newClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, meth if c.creds != nil { callHdr.Creds = c.creds } - var trInfo *traceInfo - if EnableTracing { - trInfo = &traceInfo{ - tr: trace.New("grpc.Sent."+methodFamily(method), method), - firstLine: firstLine{ - client: true, - }, - } - if deadline, ok := ctx.Deadline(); ok { - trInfo.firstLine.deadline = time.Until(deadline) - } - trInfo.tr.LazyLog(&trInfo.firstLine, false) - ctx = trace.NewContext(ctx, trInfo.tr) - } - ctx = newContextWithRPCInfo(ctx, c.failFast, c.codec, cp, comp) - sh := cc.dopts.copts.StatsHandler - var beginTime time.Time - if sh != nil { - ctx = sh.TagRPC(ctx, &stats.RPCTagInfo{FullMethodName: method, FailFast: c.failFast}) - beginTime = time.Now() - begin := &stats.Begin{ - Client: true, - BeginTime: beginTime, - FailFast: c.failFast, - } - sh.HandleRPC(ctx, begin) - } cs := &clientStream{ callHdr: callHdr, @@ -270,28 +323,41 @@ func newClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, meth cp: cp, comp: comp, cancel: cancel, - beginTime: beginTime, firstAttempt: true, + onCommit: onCommit, } if !cc.dopts.disableRetry { cs.retryThrottler = cc.retryThrottler.Load().(*retryThrottler) } - cs.binlog = binarylog.GetMethodLogger(method) - - // Only this initial attempt has stats/tracing. - // TODO(dfawley): move to newAttempt when per-attempt stats are implemented. - if err := cs.newAttemptLocked(sh, trInfo); err != nil { - cs.finish(err) - return nil, err + if ml := binarylog.GetMethodLogger(method); ml != nil { + cs.binlogs = append(cs.binlogs, ml) + } + if cc.dopts.binaryLogger != nil { + if ml := cc.dopts.binaryLogger.GetMethodLogger(method); ml != nil { + cs.binlogs = append(cs.binlogs, ml) + } } - op := func(a *csAttempt) error { return a.newStream() } + // Pick the transport to use and create a new stream on the transport. + // Assign cs.attempt upon success. + op := func(a *csAttempt) error { + if err := a.getTransport(); err != nil { + return err + } + if err := a.newStream(); err != nil { + return err + } + // Because this operation is always called either here (while creating + // the clientStream) or by the retry code while locked when replaying + // the operation, it is safe to access cs.attempt directly. + cs.attempt = a + return nil + } if err := cs.withRetry(op, func() { cs.bufferForRetryLocked(0, op) }); err != nil { - cs.finish(err) return nil, err } - if cs.binlog != nil { + if len(cs.binlogs) != 0 { md, _ := metadata.FromOutgoingContext(ctx) logEntry := &binarylog.ClientHeader{ OnClientSide: true, @@ -305,7 +371,9 @@ func newClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, meth logEntry.Timeout = 0 } } - cs.binlog.Log(logEntry) + for _, binlog := range cs.binlogs { + binlog.Log(cs.ctx, logEntry) + } } if desc != unaryStreamDesc { @@ -326,63 +394,123 @@ func newClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, meth return cs, nil } -// newAttemptLocked creates a new attempt with a transport. -// If it succeeds, then it replaces clientStream's attempt with this new attempt. -func (cs *clientStream) newAttemptLocked(sh stats.Handler, trInfo *traceInfo) (retErr error) { - newAttempt := &csAttempt{ - cs: cs, - dc: cs.cc.dopts.dc, - statsHandler: sh, - trInfo: trInfo, +// newAttemptLocked creates a new csAttempt without a transport or stream. +func (cs *clientStream) newAttemptLocked(isTransparent bool) (*csAttempt, error) { + if err := cs.ctx.Err(); err != nil { + return nil, toRPCErr(err) } - defer func() { - if retErr != nil { - // This attempt is not set in the clientStream, so it's finish won't - // be called. Call it here for stats and trace in case they are not - // nil. - newAttempt.finish(retErr) + if err := cs.cc.ctx.Err(); err != nil { + return nil, ErrClientConnClosing + } + + ctx := newContextWithRPCInfo(cs.ctx, cs.callInfo.failFast, cs.callInfo.codec, cs.cp, cs.comp) + method := cs.callHdr.Method + var beginTime time.Time + shs := cs.cc.dopts.copts.StatsHandlers + for _, sh := range shs { + ctx = sh.TagRPC(ctx, &stats.RPCTagInfo{FullMethodName: method, FailFast: cs.callInfo.failFast}) + beginTime = time.Now() + begin := &stats.Begin{ + Client: true, + BeginTime: beginTime, + FailFast: cs.callInfo.failFast, + IsClientStream: cs.desc.ClientStreams, + IsServerStream: cs.desc.ServerStreams, + IsTransparentRetryAttempt: isTransparent, } - }() + sh.HandleRPC(ctx, begin) + } - if err := cs.ctx.Err(); err != nil { - return toRPCErr(err) + var trInfo *traceInfo + if EnableTracing { + trInfo = &traceInfo{ + tr: trace.New("grpc.Sent."+methodFamily(method), method), + firstLine: firstLine{ + client: true, + }, + } + if deadline, ok := ctx.Deadline(); ok { + trInfo.firstLine.deadline = time.Until(deadline) + } + trInfo.tr.LazyLog(&trInfo.firstLine, false) + ctx = trace.NewContext(ctx, trInfo.tr) } - ctx := cs.ctx - if cs.cc.parsedTarget.Scheme == "xds" { + if cs.cc.parsedTarget.URL.Scheme == internal.GRPCResolverSchemeExtraMetadata { // Add extra metadata (metadata that will be added by transport) to context // so the balancer can see them. - ctx = grpcutil.WithExtraMetadata(cs.ctx, metadata.Pairs( + ctx = grpcutil.WithExtraMetadata(ctx, metadata.Pairs( "content-type", grpcutil.ContentType(cs.callHdr.ContentSubtype), )) } - t, done, err := cs.cc.getTransport(ctx, cs.callInfo.failFast, cs.callHdr.Method) + + return &csAttempt{ + ctx: ctx, + beginTime: beginTime, + cs: cs, + dc: cs.cc.dopts.dc, + statsHandlers: shs, + trInfo: trInfo, + }, nil +} + +func (a *csAttempt) getTransport() error { + cs := a.cs + + var err error + a.t, a.pickResult, err = cs.cc.getTransport(a.ctx, cs.callInfo.failFast, cs.callHdr.Method) if err != nil { + if de, ok := err.(dropError); ok { + err = de.error + a.drop = true + } return err } - if trInfo != nil { - trInfo.firstLine.SetRemoteAddr(t.RemoteAddr()) + if a.trInfo != nil { + a.trInfo.firstLine.SetRemoteAddr(a.t.RemoteAddr()) } - newAttempt.t = t - newAttempt.done = done - cs.attempt = newAttempt return nil } func (a *csAttempt) newStream() error { cs := a.cs cs.callHdr.PreviousAttempts = cs.numRetries - s, err := a.t.NewStream(cs.ctx, cs.callHdr) + + // Merge metadata stored in PickResult, if any, with existing call metadata. + // It is safe to overwrite the csAttempt's context here, since all state + // maintained in it are local to the attempt. When the attempt has to be + // retried, a new instance of csAttempt will be created. + if a.pickResult.Metadata != nil { + // We currently do not have a function it the metadata package which + // merges given metadata with existing metadata in a context. Existing + // function `AppendToOutgoingContext()` takes a variadic argument of key + // value pairs. + // + // TODO: Make it possible to retrieve key value pairs from metadata.MD + // in a form passable to AppendToOutgoingContext(), or create a version + // of AppendToOutgoingContext() that accepts a metadata.MD. + md, _ := metadata.FromOutgoingContext(a.ctx) + md = metadata.Join(md, a.pickResult.Metadata) + a.ctx = metadata.NewOutgoingContext(a.ctx, md) + } + + s, err := a.t.NewStream(a.ctx, cs.callHdr) if err != nil { - if _, ok := err.(transport.PerformedIOError); ok { - // Return without converting to an RPC error so retry code can - // inspect. + nse, ok := err.(*transport.NewStreamError) + if !ok { + // Unexpected. return err } - return toRPCErr(err) + + if nse.AllowTransparentRetry { + a.allowTransparentRetry = true + } + + // Unwrap and convert error. + return toRPCErr(nse.Err) } - cs.attempt.s = s - cs.attempt.p = &parser{r: s} + a.s = s + a.p = &parser{r: s, recvBufferPool: a.cs.cc.dopts.recvBufferPool} return nil } @@ -400,8 +528,7 @@ type clientStream struct { cancel context.CancelFunc // cancels all attempts - sentLast bool // sent an end stream - beginTime time.Time + sentLast bool // sent an end stream methodConfig *MethodConfig @@ -409,7 +536,7 @@ type clientStream struct { retryThrottler *retryThrottler // The throttler active when the RPC began. - binlog *binarylog.MethodLogger // Binary logger, can be nil. + binlogs []binarylog.MethodLogger // serverHeaderBinlogged is a boolean for whether server header has been // logged. Server header will be logged when the first time one of those // happens: stream.Header(), stream.Recv(). @@ -432,7 +559,8 @@ type clientStream struct { // place where we need to check if the attempt is nil. attempt *csAttempt // TODO(hedging): hedging will have multiple attempts simultaneously. - committed bool // active attempt committed for retry? + committed bool // active attempt committed for retry? + onCommit func() buffer []func(a *csAttempt) error // operations to replay on retry bufferSize int // current size of buffer } @@ -440,11 +568,12 @@ type clientStream struct { // csAttempt implements a single transport stream attempt within a // clientStream. type csAttempt struct { - cs *clientStream - t transport.ClientTransport - s *transport.Stream - p *parser - done func(balancer.DoneInfo) + ctx context.Context + cs *clientStream + t transport.ClientTransport + s *transport.Stream + p *parser + pickResult balancer.PickResult finished bool dc Decompressor @@ -457,10 +586,19 @@ type csAttempt struct { // and cleared when the finish method is called. trInfo *traceInfo - statsHandler stats.Handler + statsHandlers []stats.Handler + beginTime time.Time + + // set for newStream errors that may be transparently retried + allowTransparentRetry bool + // set for pick errors that are returned as a status + drop bool } func (cs *clientStream) commitAttemptLocked() { + if !cs.committed && cs.onCommit != nil { + cs.onCommit() + } cs.committed = true cs.buffer = nil } @@ -472,85 +610,76 @@ func (cs *clientStream) commitAttempt() { } // shouldRetry returns nil if the RPC should be retried; otherwise it returns -// the error that should be returned by the operation. -func (cs *clientStream) shouldRetry(err error) error { - unprocessed := false - if cs.attempt.s == nil { - pioErr, ok := err.(transport.PerformedIOError) - if ok { - // Unwrap error. - err = toRPCErr(pioErr.Err) - } else { - unprocessed = true - } - if !ok && !cs.callInfo.failFast { - // In the event of a non-IO operation error from NewStream, we - // never attempted to write anything to the wire, so we can retry - // indefinitely for non-fail-fast RPCs. - return nil - } +// the error that should be returned by the operation. If the RPC should be +// retried, the bool indicates whether it is being retried transparently. +func (a *csAttempt) shouldRetry(err error) (bool, error) { + cs := a.cs + + if cs.finished || cs.committed || a.drop { + // RPC is finished or committed or was dropped by the picker; cannot retry. + return false, err } - if cs.finished || cs.committed { - // RPC is finished or committed; cannot retry. - return err + if a.s == nil && a.allowTransparentRetry { + return true, nil } // Wait for the trailers. - if cs.attempt.s != nil { - <-cs.attempt.s.Done() - unprocessed = cs.attempt.s.Unprocessed() + unprocessed := false + if a.s != nil { + <-a.s.Done() + unprocessed = a.s.Unprocessed() } if cs.firstAttempt && unprocessed { // First attempt, stream unprocessed: transparently retry. - return nil + return true, nil } if cs.cc.dopts.disableRetry { - return err + return false, err } pushback := 0 hasPushback := false - if cs.attempt.s != nil { - if !cs.attempt.s.TrailersOnly() { - return err + if a.s != nil { + if !a.s.TrailersOnly() { + return false, err } // TODO(retry): Move down if the spec changes to not check server pushback // before considering this a failure for throttling. - sps := cs.attempt.s.Trailer()["grpc-retry-pushback-ms"] + sps := a.s.Trailer()["grpc-retry-pushback-ms"] if len(sps) == 1 { var e error if pushback, e = strconv.Atoi(sps[0]); e != nil || pushback < 0 { channelz.Infof(logger, cs.cc.channelzID, "Server retry pushback specified to abort (%q).", sps[0]) cs.retryThrottler.throttle() // This counts as a failure for throttling. - return err + return false, err } hasPushback = true } else if len(sps) > 1 { channelz.Warningf(logger, cs.cc.channelzID, "Server retry pushback specified multiple values (%q); not retrying.", sps) cs.retryThrottler.throttle() // This counts as a failure for throttling. - return err + return false, err } } var code codes.Code - if cs.attempt.s != nil { - code = cs.attempt.s.Status().Code() + if a.s != nil { + code = a.s.Status().Code() } else { - code = status.Convert(err).Code() + code = status.Code(err) } - rp := cs.methodConfig.retryPolicy - if rp == nil || !rp.retryableStatusCodes[code] { - return err + rp := cs.methodConfig.RetryPolicy + if rp == nil || !rp.RetryableStatusCodes[code] { + return false, err } // Note: the ordering here is important; we count this as a failure // only if the code matched a retryable code. if cs.retryThrottler.throttle() { - return err + return false, err } - if cs.numRetries+1 >= rp.maxAttempts { - return err + if cs.numRetries+1 >= rp.MaxAttempts { + return false, err } var dur time.Duration @@ -558,9 +687,9 @@ func (cs *clientStream) shouldRetry(err error) error { dur = time.Millisecond * time.Duration(pushback) cs.numRetriesSincePushback = 0 } else { - fact := math.Pow(rp.backoffMultiplier, float64(cs.numRetriesSincePushback)) - cur := float64(rp.initialBackoff) * fact - if max := float64(rp.maxBackoff); cur > max { + fact := math.Pow(rp.BackoffMultiplier, float64(cs.numRetriesSincePushback)) + cur := float64(rp.InitialBackoff) * fact + if max := float64(rp.MaxBackoff); cur > max { cur = max } dur = time.Duration(grpcrand.Int63n(int64(cur))) @@ -573,26 +702,32 @@ func (cs *clientStream) shouldRetry(err error) error { select { case <-t.C: cs.numRetries++ - return nil + return false, nil case <-cs.ctx.Done(): t.Stop() - return status.FromContextError(cs.ctx.Err()).Err() + return false, status.FromContextError(cs.ctx.Err()).Err() } } // Returns nil if a retry was performed and succeeded; error otherwise. -func (cs *clientStream) retryLocked(lastErr error) error { +func (cs *clientStream) retryLocked(attempt *csAttempt, lastErr error) error { for { - cs.attempt.finish(lastErr) - if err := cs.shouldRetry(lastErr); err != nil { + attempt.finish(toRPCErr(lastErr)) + isTransparent, err := attempt.shouldRetry(lastErr) + if err != nil { cs.commitAttemptLocked() return err } cs.firstAttempt = false - if err := cs.newAttemptLocked(nil, nil); err != nil { + attempt, err = cs.newAttemptLocked(isTransparent) + if err != nil { + // Only returns error if the clientconn is closed or the context of + // the stream is canceled. return err } - if lastErr = cs.replayBufferLocked(); lastErr == nil { + // Note that the first op in the replay buffer always sets cs.attempt + // if it is able to pick a transport and create a stream. + if lastErr = cs.replayBufferLocked(attempt); lastErr == nil { return nil } } @@ -602,7 +737,10 @@ func (cs *clientStream) Context() context.Context { cs.commitAttempt() // No need to lock before using attempt, since we know it is committed and // cannot change. - return cs.attempt.s.Context() + if cs.attempt.s != nil { + return cs.attempt.s.Context() + } + return cs.ctx } func (cs *clientStream) withRetry(op func(a *csAttempt) error, onSuccess func()) error { @@ -610,7 +748,23 @@ func (cs *clientStream) withRetry(op func(a *csAttempt) error, onSuccess func()) for { if cs.committed { cs.mu.Unlock() - return op(cs.attempt) + // toRPCErr is used in case the error from the attempt comes from + // NewClientStream, which intentionally doesn't return a status + // error to allow for further inspection; all other errors should + // already be status errors. + return toRPCErr(op(cs.attempt)) + } + if len(cs.buffer) == 0 { + // For the first op, which controls creation of the stream and + // assigns cs.attempt, we need to create a new attempt inline + // before executing the first op. On subsequent ops, the attempt + // is created immediately before replaying the ops. + var err error + if cs.attempt, err = cs.newAttemptLocked(false /* isTransparent */); err != nil { + cs.mu.Unlock() + cs.finish(err) + return err + } } a := cs.attempt cs.mu.Unlock() @@ -628,7 +782,7 @@ func (cs *clientStream) withRetry(op func(a *csAttempt) error, onSuccess func()) cs.mu.Unlock() return err } - if err := cs.retryLocked(err); err != nil { + if err := cs.retryLocked(a, err); err != nil { cs.mu.Unlock() return err } @@ -642,12 +796,21 @@ func (cs *clientStream) Header() (metadata.MD, error) { m, err = a.s.Header() return toRPCErr(err) }, cs.commitAttemptLocked) + + if m == nil && err == nil { + // The stream ended with success. Finish the clientStream. + err = io.EOF + } + if err != nil { cs.finish(err) - return nil, err + // Do not return the error. The user should get it by calling Recv(). + return nil, nil } - if cs.binlog != nil && !cs.serverHeaderBinlogged { - // Only log if binary log is on and header has not been logged. + + if len(cs.binlogs) != 0 && !cs.serverHeaderBinlogged && m != nil { + // Only log if binary log is on and header has not been logged, and + // there is actually headers to log. logEntry := &binarylog.ServerHeader{ OnClientSide: true, Header: m, @@ -656,10 +819,13 @@ func (cs *clientStream) Header() (metadata.MD, error) { if peer, ok := peer.FromContext(cs.Context()); ok { logEntry.PeerAddr = peer.Addr } - cs.binlog.Log(logEntry) cs.serverHeaderBinlogged = true + for _, binlog := range cs.binlogs { + binlog.Log(cs.ctx, logEntry) + } } - return m, err + + return m, nil } func (cs *clientStream) Trailer() metadata.MD { @@ -677,10 +843,9 @@ func (cs *clientStream) Trailer() metadata.MD { return cs.attempt.s.Trailer() } -func (cs *clientStream) replayBufferLocked() error { - a := cs.attempt +func (cs *clientStream) replayBufferLocked(attempt *csAttempt) error { for _, f := range cs.buffer { - if err := f(a); err != nil { + if err := f(attempt); err != nil { return err } } @@ -700,7 +865,7 @@ func (cs *clientStream) bufferForRetryLocked(sz int, op func(a *csAttempt) error cs.buffer = append(cs.buffer, op) } -func (cs *clientStream) SendMsg(m interface{}) (err error) { +func (cs *clientStream) SendMsg(m any) (err error) { defer func() { if err != nil && err != io.EOF { // Call finish on the client stream for errors generated by this SendMsg @@ -728,61 +893,46 @@ func (cs *clientStream) SendMsg(m interface{}) (err error) { if len(payload) > *cs.callInfo.maxSendMessageSize { return status.Errorf(codes.ResourceExhausted, "trying to send message larger than max (%d vs. %d)", len(payload), *cs.callInfo.maxSendMessageSize) } - msgBytes := data // Store the pointer before setting to nil. For binary logging. op := func(a *csAttempt) error { - err := a.sendMsg(m, hdr, payload, data) - // nil out the message and uncomp when replaying; they are only needed for - // stats which is disabled for subsequent attempts. - m, data = nil, nil - return err + return a.sendMsg(m, hdr, payload, data) } err = cs.withRetry(op, func() { cs.bufferForRetryLocked(len(hdr)+len(payload), op) }) - if cs.binlog != nil && err == nil { - cs.binlog.Log(&binarylog.ClientMessage{ + if len(cs.binlogs) != 0 && err == nil { + cm := &binarylog.ClientMessage{ OnClientSide: true, - Message: msgBytes, - }) + Message: data, + } + for _, binlog := range cs.binlogs { + binlog.Log(cs.ctx, cm) + } } - return + return err } -func (cs *clientStream) RecvMsg(m interface{}) error { - if cs.binlog != nil && !cs.serverHeaderBinlogged { +func (cs *clientStream) RecvMsg(m any) error { + if len(cs.binlogs) != 0 && !cs.serverHeaderBinlogged { // Call Header() to binary log header if it's not already logged. cs.Header() } var recvInfo *payloadInfo - if cs.binlog != nil { + if len(cs.binlogs) != 0 { recvInfo = &payloadInfo{} } err := cs.withRetry(func(a *csAttempt) error { return a.recvMsg(m, recvInfo) }, cs.commitAttemptLocked) - if cs.binlog != nil && err == nil { - cs.binlog.Log(&binarylog.ServerMessage{ + if len(cs.binlogs) != 0 && err == nil { + sm := &binarylog.ServerMessage{ OnClientSide: true, Message: recvInfo.uncompressedBytes, - }) + } + for _, binlog := range cs.binlogs { + binlog.Log(cs.ctx, sm) + } } if err != nil || !cs.desc.ServerStreams { // err != nil or non-server-streaming indicates end of stream. cs.finish(err) - - if cs.binlog != nil { - // finish will not log Trailer. Log Trailer here. - logEntry := &binarylog.ServerTrailer{ - OnClientSide: true, - Trailer: cs.Trailer(), - Err: err, - } - if logEntry.Err == io.EOF { - logEntry.Err = nil - } - if peer, ok := peer.FromContext(cs.Context()); ok { - logEntry.PeerAddr = peer.Addr - } - cs.binlog.Log(logEntry) - } } return err } @@ -802,10 +952,13 @@ func (cs *clientStream) CloseSend() error { return nil } cs.withRetry(op, func() { cs.bufferForRetryLocked(0, op) }) - if cs.binlog != nil { - cs.binlog.Log(&binarylog.ClientHalfClose{ + if len(cs.binlogs) != 0 { + chc := &binarylog.ClientHalfClose{ OnClientSide: true, - }) + } + for _, binlog := range cs.binlogs { + binlog.Log(cs.ctx, chc) + } } // We never returned an error here for reasons. return nil @@ -822,6 +975,9 @@ func (cs *clientStream) finish(err error) { return } cs.finished = true + for _, onFinish := range cs.callInfo.onFinish { + onFinish(err) + } cs.commitAttemptLocked() if cs.attempt != nil { cs.attempt.finish(err) @@ -832,16 +988,31 @@ func (cs *clientStream) finish(err error) { } } } + cs.mu.Unlock() - // For binary logging. only log cancel in finish (could be caused by RPC ctx - // canceled or ClientConn closed). Trailer will be logged in RecvMsg. - // - // Only one of cancel or trailer needs to be logged. In the cases where - // users don't call RecvMsg, users must have already canceled the RPC. - if cs.binlog != nil && status.Code(err) == codes.Canceled { - cs.binlog.Log(&binarylog.Cancel{ - OnClientSide: true, - }) + // Only one of cancel or trailer needs to be logged. + if len(cs.binlogs) != 0 { + switch err { + case errContextCanceled, errContextDeadline, ErrClientConnClosing: + c := &binarylog.Cancel{ + OnClientSide: true, + } + for _, binlog := range cs.binlogs { + binlog.Log(cs.ctx, c) + } + default: + logEntry := &binarylog.ServerTrailer{ + OnClientSide: true, + Trailer: cs.Trailer(), + Err: err, + } + if peer, ok := peer.FromContext(cs.Context()); ok { + logEntry.PeerAddr = peer.Addr + } + for _, binlog := range cs.binlogs { + binlog.Log(cs.ctx, logEntry) + } + } } if err == nil { cs.retryThrottler.successfulRPC() @@ -856,7 +1027,7 @@ func (cs *clientStream) finish(err error) { cs.cancel() } -func (a *csAttempt) sendMsg(m interface{}, hdr, payld, data []byte) error { +func (a *csAttempt) sendMsg(m any, hdr, payld, data []byte) error { cs := a.cs if a.trInfo != nil { a.mu.Lock() @@ -874,8 +1045,8 @@ func (a *csAttempt) sendMsg(m interface{}, hdr, payld, data []byte) error { } return io.EOF } - if a.statsHandler != nil { - a.statsHandler.HandleRPC(cs.ctx, outPayload(true, m, data, payld, time.Now())) + for _, sh := range a.statsHandlers { + sh.HandleRPC(a.ctx, outPayload(true, m, data, payld, time.Now())) } if channelz.IsOn() { a.t.IncrMsgSent() @@ -883,9 +1054,9 @@ func (a *csAttempt) sendMsg(m interface{}, hdr, payld, data []byte) error { return nil } -func (a *csAttempt) recvMsg(m interface{}, payInfo *payloadInfo) (err error) { +func (a *csAttempt) recvMsg(m any, payInfo *payloadInfo) (err error) { cs := a.cs - if a.statsHandler != nil && payInfo == nil { + if len(a.statsHandlers) != 0 && payInfo == nil { payInfo = &payloadInfo{} } @@ -913,6 +1084,7 @@ func (a *csAttempt) recvMsg(m interface{}, payInfo *payloadInfo) (err error) { } return io.EOF // indicates successful end of stream. } + return toRPCErr(err) } if a.trInfo != nil { @@ -922,15 +1094,16 @@ func (a *csAttempt) recvMsg(m interface{}, payInfo *payloadInfo) (err error) { } a.mu.Unlock() } - if a.statsHandler != nil { - a.statsHandler.HandleRPC(cs.ctx, &stats.InPayload{ + for _, sh := range a.statsHandlers { + sh.HandleRPC(a.ctx, &stats.InPayload{ Client: true, RecvTime: time.Now(), Payload: m, // TODO truncate large payload. - Data: payInfo.uncompressedBytes, - WireLength: payInfo.wireLength, - Length: len(payInfo.uncompressedBytes), + Data: payInfo.uncompressedBytes, + WireLength: payInfo.compressedLength + headerLen, + CompressedLength: payInfo.compressedLength, + Length: len(payInfo.uncompressedBytes), }) } if channelz.IsOn() { @@ -969,12 +1142,12 @@ func (a *csAttempt) finish(err error) { tr = a.s.Trailer() } - if a.done != nil { + if a.pickResult.Done != nil { br := false if a.s != nil { br = a.s.BytesReceived() } - a.done(balancer.DoneInfo{ + a.pickResult.Done(balancer.DoneInfo{ Err: err, Trailer: tr, BytesSent: a.s != nil, @@ -982,15 +1155,15 @@ func (a *csAttempt) finish(err error) { ServerLoad: balancerload.Parse(tr), }) } - if a.statsHandler != nil { + for _, sh := range a.statsHandlers { end := &stats.End{ Client: true, - BeginTime: a.cs.beginTime, + BeginTime: a.beginTime, EndTime: time.Now(), Trailer: tr, Error: err, } - a.statsHandler.HandleRPC(a.cs.ctx, end) + sh.HandleRPC(a.ctx, end) } if a.trInfo != nil && a.trInfo.tr != nil { if err == nil { @@ -1096,17 +1269,22 @@ func newNonRetryClientStream(ctx context.Context, desc *StreamDesc, method strin return nil, err } as.s = s - as.p = &parser{r: s} + as.p = &parser{r: s, recvBufferPool: ac.dopts.recvBufferPool} ac.incrCallsStarted() if desc != unaryStreamDesc { - // Listen on cc and stream contexts to cleanup when the user closes the - // ClientConn or cancels the stream context. In all other cases, an error - // should already be injected into the recv buffer by the transport, which - // the client will eventually receive, and then we will cancel the stream's - // context in clientStream.finish. + // Listen on stream context to cleanup when the stream context is + // canceled. Also listen for the addrConn's context in case the + // addrConn is closed or reconnects to a different address. In all + // other cases, an error should already be injected into the recv + // buffer by the transport, which the client will eventually receive, + // and then we will cancel the stream's context in + // addrConnStream.finish. go func() { + ac.mu.Lock() + acCtx := ac.ctx + ac.mu.Unlock() select { - case <-ac.ctx.Done(): + case <-acCtx.Done(): as.finish(status.Error(codes.Canceled, "grpc: the SubConn is closing")) case <-ctx.Done(): as.finish(toRPCErr(ctx.Err())) @@ -1169,7 +1347,7 @@ func (as *addrConnStream) Context() context.Context { return as.s.Context() } -func (as *addrConnStream) SendMsg(m interface{}) (err error) { +func (as *addrConnStream) SendMsg(m any) (err error) { defer func() { if err != nil && err != io.EOF { // Call finish on the client stream for errors generated by this SendMsg @@ -1214,7 +1392,7 @@ func (as *addrConnStream) SendMsg(m interface{}) (err error) { return nil } -func (as *addrConnStream) RecvMsg(m interface{}) (err error) { +func (as *addrConnStream) RecvMsg(m any) (err error) { defer func() { if err != nil || !as.desc.ServerStreams { // err != nil or non-server-streaming indicates end of stream. @@ -1295,8 +1473,10 @@ func (as *addrConnStream) finish(err error) { // ServerStream defines the server-side behavior of a streaming RPC. // -// All errors returned from ServerStream methods are compatible with the -// status package. +// Errors returned from ServerStream methods are compatible with the status +// package. However, the status code will often not match the RPC status as +// seen by the client application, and therefore, should not be relied upon for +// this purpose. type ServerStream interface { // SetHeader sets the header metadata. It may be called multiple times. // When call multiple times, all the provided metadata will be merged. @@ -1328,7 +1508,10 @@ type ServerStream interface { // It is safe to have a goroutine calling SendMsg and another goroutine // calling RecvMsg on the same stream at the same time, but it is not safe // to call SendMsg on the same stream in different goroutines. - SendMsg(m interface{}) error + // + // It is not safe to modify the message after calling SendMsg. Tracing + // libraries and stats handlers may use the message lazily. + SendMsg(m any) error // RecvMsg blocks until it receives a message into m or the stream is // done. It returns io.EOF when the client has performed a CloseSend. On // any non-EOF error, the stream is aborted and the error contains the @@ -1337,7 +1520,7 @@ type ServerStream interface { // It is safe to have a goroutine calling SendMsg and another goroutine // calling RecvMsg on the same stream at the same time, but it is not // safe to call RecvMsg on the same stream in different goroutines. - RecvMsg(m interface{}) error + RecvMsg(m any) error } // serverStream implements a server side Stream. @@ -1353,13 +1536,15 @@ type serverStream struct { comp encoding.Compressor decomp encoding.Compressor + sendCompressorName string + maxReceiveMessageSize int maxSendMessageSize int trInfo *traceInfo - statsHandler stats.Handler + statsHandler []stats.Handler - binlog *binarylog.MethodLogger + binlogs []binarylog.MethodLogger // serverHeaderBinlogged indicates whether server header has been logged. It // will happen when one of the following two happens: stream.SendHeader(), // stream.Send(). @@ -1379,17 +1564,29 @@ func (ss *serverStream) SetHeader(md metadata.MD) error { if md.Len() == 0 { return nil } + err := imetadata.Validate(md) + if err != nil { + return status.Error(codes.Internal, err.Error()) + } return ss.s.SetHeader(md) } func (ss *serverStream) SendHeader(md metadata.MD) error { - err := ss.t.WriteHeader(ss.s, md) - if ss.binlog != nil && !ss.serverHeaderBinlogged { + err := imetadata.Validate(md) + if err != nil { + return status.Error(codes.Internal, err.Error()) + } + + err = ss.t.WriteHeader(ss.s, md) + if len(ss.binlogs) != 0 && !ss.serverHeaderBinlogged { h, _ := ss.s.Header() - ss.binlog.Log(&binarylog.ServerHeader{ + sh := &binarylog.ServerHeader{ Header: h, - }) + } ss.serverHeaderBinlogged = true + for _, binlog := range ss.binlogs { + binlog.Log(ss.ctx, sh) + } } return err } @@ -1398,10 +1595,13 @@ func (ss *serverStream) SetTrailer(md metadata.MD) { if md.Len() == 0 { return } + if err := imetadata.Validate(md); err != nil { + logger.Errorf("stream: failed to validate md when setting trailer, err: %v", err) + } ss.s.SetTrailer(md) } -func (ss *serverStream) SendMsg(m interface{}) (err error) { +func (ss *serverStream) SendMsg(m any) (err error) { defer func() { if ss.trInfo != nil { ss.mu.Lock() @@ -1409,7 +1609,7 @@ func (ss *serverStream) SendMsg(m interface{}) (err error) { if err == nil { ss.trInfo.tr.LazyLog(&payload{sent: true, msg: m}, true) } else { - ss.trInfo.tr.LazyLog(&fmtStringer{"%v", []interface{}{err}}, true) + ss.trInfo.tr.LazyLog(&fmtStringer{"%v", []any{err}}, true) ss.trInfo.tr.SetError() } } @@ -1430,6 +1630,13 @@ func (ss *serverStream) SendMsg(m interface{}) (err error) { } }() + // Server handler could have set new compressor by calling SetSendCompressor. + // In case it is set, we need to use it for compressing outbound message. + if sendCompressorsName := ss.s.SendCompress(); sendCompressorsName != ss.sendCompressorName { + ss.comp = encoding.GetCompressor(sendCompressorsName) + ss.sendCompressorName = sendCompressorsName + } + // load hdr, payload, data hdr, payload, data, err := prepareMsg(m, ss.codec, ss.cp, ss.comp) if err != nil { @@ -1443,25 +1650,33 @@ func (ss *serverStream) SendMsg(m interface{}) (err error) { if err := ss.t.Write(ss.s, hdr, payload, &transport.Options{Last: false}); err != nil { return toRPCErr(err) } - if ss.binlog != nil { + if len(ss.binlogs) != 0 { if !ss.serverHeaderBinlogged { h, _ := ss.s.Header() - ss.binlog.Log(&binarylog.ServerHeader{ + sh := &binarylog.ServerHeader{ Header: h, - }) + } ss.serverHeaderBinlogged = true + for _, binlog := range ss.binlogs { + binlog.Log(ss.ctx, sh) + } } - ss.binlog.Log(&binarylog.ServerMessage{ + sm := &binarylog.ServerMessage{ Message: data, - }) + } + for _, binlog := range ss.binlogs { + binlog.Log(ss.ctx, sm) + } } - if ss.statsHandler != nil { - ss.statsHandler.HandleRPC(ss.s.Context(), outPayload(false, m, data, payload, time.Now())) + if len(ss.statsHandler) != 0 { + for _, sh := range ss.statsHandler { + sh.HandleRPC(ss.s.Context(), outPayload(false, m, data, payload, time.Now())) + } } return nil } -func (ss *serverStream) RecvMsg(m interface{}) (err error) { +func (ss *serverStream) RecvMsg(m any) (err error) { defer func() { if ss.trInfo != nil { ss.mu.Lock() @@ -1469,7 +1684,7 @@ func (ss *serverStream) RecvMsg(m interface{}) (err error) { if err == nil { ss.trInfo.tr.LazyLog(&payload{sent: false, msg: m}, true) } else if err != io.EOF { - ss.trInfo.tr.LazyLog(&fmtStringer{"%v", []interface{}{err}}, true) + ss.trInfo.tr.LazyLog(&fmtStringer{"%v", []any{err}}, true) ss.trInfo.tr.SetError() } } @@ -1490,13 +1705,16 @@ func (ss *serverStream) RecvMsg(m interface{}) (err error) { } }() var payInfo *payloadInfo - if ss.statsHandler != nil || ss.binlog != nil { + if len(ss.statsHandler) != 0 || len(ss.binlogs) != 0 { payInfo = &payloadInfo{} } if err := recv(ss.p, ss.codec, ss.s, ss.dc, m, ss.maxReceiveMessageSize, payInfo, ss.decomp); err != nil { if err == io.EOF { - if ss.binlog != nil { - ss.binlog.Log(&binarylog.ClientHalfClose{}) + if len(ss.binlogs) != 0 { + chc := &binarylog.ClientHalfClose{} + for _, binlog := range ss.binlogs { + binlog.Log(ss.ctx, chc) + } } return err } @@ -1505,20 +1723,26 @@ func (ss *serverStream) RecvMsg(m interface{}) (err error) { } return toRPCErr(err) } - if ss.statsHandler != nil { - ss.statsHandler.HandleRPC(ss.s.Context(), &stats.InPayload{ - RecvTime: time.Now(), - Payload: m, - // TODO truncate large payload. - Data: payInfo.uncompressedBytes, - WireLength: payInfo.wireLength, - Length: len(payInfo.uncompressedBytes), - }) + if len(ss.statsHandler) != 0 { + for _, sh := range ss.statsHandler { + sh.HandleRPC(ss.s.Context(), &stats.InPayload{ + RecvTime: time.Now(), + Payload: m, + // TODO truncate large payload. + Data: payInfo.uncompressedBytes, + Length: len(payInfo.uncompressedBytes), + WireLength: payInfo.compressedLength + headerLen, + CompressedLength: payInfo.compressedLength, + }) + } } - if ss.binlog != nil { - ss.binlog.Log(&binarylog.ClientMessage{ + if len(ss.binlogs) != 0 { + cm := &binarylog.ClientMessage{ Message: payInfo.uncompressedBytes, - }) + } + for _, binlog := range ss.binlogs { + binlog.Log(ss.ctx, cm) + } } return nil } @@ -1532,7 +1756,7 @@ func MethodFromServerStream(stream ServerStream) (string, bool) { // prepareMsg returns the hdr, payload and data // using the compressors passed or using the // passed preparedmsg -func prepareMsg(m interface{}, codec baseCodec, cp Compressor, comp encoding.Compressor) (hdr, payload, data []byte, err error) { +func prepareMsg(m any, codec baseCodec, cp Compressor, comp encoding.Compressor) (hdr, payload, data []byte, err error) { if preparedMsg, ok := m.(*PreparedMsg); ok { return preparedMsg.hdr, preparedMsg.payload, preparedMsg.encodedData, nil } diff --git a/stream_test.go b/stream_test.go new file mode 100644 index 000000000000..7af066799c16 --- /dev/null +++ b/stream_test.go @@ -0,0 +1,67 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpc_test + +import ( + "context" + "testing" + "time" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/interop/grpc_testing" + "google.golang.org/grpc/status" +) + +const defaultTestTimeout = 10 * time.Second + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func (s) TestStream_Header_TrailersOnly(t *testing.T) { + ss := stubserver.StubServer{ + FullDuplexCallF: func(stream grpc_testing.TestService_FullDuplexCallServer) error { + return status.Errorf(codes.NotFound, "a test error") + }, + } + if err := ss.Start(nil); err != nil { + t.Fatal("Error starting server:", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + s, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatal("Error staring call", err) + } + if md, err := s.Header(); md != nil || err != nil { + t.Fatalf("s.Header() = %v, %v; want nil, nil", md, err) + } + if _, err := s.Recv(); status.Code(err) != codes.NotFound { + t.Fatalf("s.Recv() = _, %v; want _, err.Code()=codes.NotFound", err) + } +} diff --git a/stress/client/main.go b/stress/client/main.go index 0353476d8f39..ef3db7c13864 100644 --- a/stress/client/main.go +++ b/stress/client/main.go @@ -33,12 +33,14 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/grpclog" "google.golang.org/grpc/interop" - testpb "google.golang.org/grpc/interop/grpc_testing" "google.golang.org/grpc/status" - metricspb "google.golang.org/grpc/stress/grpc_testing" "google.golang.org/grpc/testdata" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + metricspb "google.golang.org/grpc/stress/grpc_testing" ) var ( @@ -146,13 +148,12 @@ func (g *gauge) get() int64 { // server implements metrics server functions. type server struct { + metricspb.UnimplementedMetricsServiceServer mutex sync.RWMutex // gauges is a map from /stress_test/server_/channel_/stub_/qps to its qps gauge. gauges map[string]*gauge } -var _ metricspb.UnstableMetricsServiceService = (*server)(nil) - // newMetricsServer returns a new metrics server. func newMetricsServer() *server { return &server{gauges: make(map[string]*gauge)} @@ -203,14 +204,14 @@ func startServer(server *server, port int) { } s := grpc.NewServer() - metricspb.RegisterMetricsServiceService(s, metricspb.NewMetricsServiceService(server)) + metricspb.RegisterMetricsServiceServer(s, server) s.Serve(lis) } // performRPCs uses weightedRandomTestSelector to select test case and runs the tests. func performRPCs(gauge *gauge, conn *grpc.ClientConn, selector *weightedRandomTestSelector, stop <-chan bool) { - client := testpb.NewTestServiceClient(conn) + client := testgrpc.NewTestServiceClient(conn) var numCalls int64 startTime := time.Now() for { @@ -286,14 +287,14 @@ func newConn(address string, useTLS, testCA bool, tlsServerName string) (*grpc.C } creds, err = credentials.NewClientTLSFromFile(*caFile, sn) if err != nil { - logger.Fatalf("Failed to create TLS credentials %v", err) + logger.Fatalf("Failed to create TLS credentials: %v", err) } } else { creds = credentials.NewClientTLSFromCert(nil, sn) } opts = append(opts, grpc.WithTransportCredentials(creds)) } else { - opts = append(opts, grpc.WithInsecure()) + opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) } return grpc.Dial(address, opts...) } diff --git a/stress/grpc_testing/metrics.pb.go b/stress/grpc_testing/metrics.pb.go index 40a0123c44bd..e30eeae322e2 100644 --- a/stress/grpc_testing/metrics.pb.go +++ b/stress/grpc_testing/metrics.pb.go @@ -1,66 +1,124 @@ +// Copyright 2015-2016 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Contains the definitions for a metrics service and the type of metrics +// exposed by the service. +// +// Currently, 'Gauge' (i.e a metric that represents the measured value of +// something at an instant of time) is the only metric type supported by the +// service. + // Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.31.0 +// protoc v4.22.0 // source: stress/grpc_testing/metrics.proto package grpc_testing import ( - fmt "fmt" - proto "github.com/golang/protobuf/proto" - math "math" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" ) -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) // Response message containing the gauge name and value type GaugeResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - // Types that are valid to be assigned to Value: + // Types that are assignable to Value: + // // *GaugeResponse_LongValue // *GaugeResponse_DoubleValue // *GaugeResponse_StringValue - Value isGaugeResponse_Value `protobuf_oneof:"value"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Value isGaugeResponse_Value `protobuf_oneof:"value"` } -func (m *GaugeResponse) Reset() { *m = GaugeResponse{} } -func (m *GaugeResponse) String() string { return proto.CompactTextString(m) } -func (*GaugeResponse) ProtoMessage() {} -func (*GaugeResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_028251bc41da09ab, []int{0} +func (x *GaugeResponse) Reset() { + *x = GaugeResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_stress_grpc_testing_metrics_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GaugeResponse) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *GaugeResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_GaugeResponse.Unmarshal(m, b) +func (*GaugeResponse) ProtoMessage() {} + +func (x *GaugeResponse) ProtoReflect() protoreflect.Message { + mi := &file_stress_grpc_testing_metrics_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -func (m *GaugeResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_GaugeResponse.Marshal(b, m, deterministic) + +// Deprecated: Use GaugeResponse.ProtoReflect.Descriptor instead. +func (*GaugeResponse) Descriptor() ([]byte, []int) { + return file_stress_grpc_testing_metrics_proto_rawDescGZIP(), []int{0} } -func (m *GaugeResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_GaugeResponse.Merge(m, src) + +func (x *GaugeResponse) GetName() string { + if x != nil { + return x.Name + } + return "" } -func (m *GaugeResponse) XXX_Size() int { - return xxx_messageInfo_GaugeResponse.Size(m) + +func (m *GaugeResponse) GetValue() isGaugeResponse_Value { + if m != nil { + return m.Value + } + return nil } -func (m *GaugeResponse) XXX_DiscardUnknown() { - xxx_messageInfo_GaugeResponse.DiscardUnknown(m) + +func (x *GaugeResponse) GetLongValue() int64 { + if x, ok := x.GetValue().(*GaugeResponse_LongValue); ok { + return x.LongValue + } + return 0 } -var xxx_messageInfo_GaugeResponse proto.InternalMessageInfo +func (x *GaugeResponse) GetDoubleValue() float64 { + if x, ok := x.GetValue().(*GaugeResponse_DoubleValue); ok { + return x.DoubleValue + } + return 0 +} -func (m *GaugeResponse) GetName() string { - if m != nil { - return m.Name +func (x *GaugeResponse) GetStringValue() string { + if x, ok := x.GetValue().(*GaugeResponse_StringValue); ok { + return x.StringValue } return "" } @@ -87,140 +145,221 @@ func (*GaugeResponse_DoubleValue) isGaugeResponse_Value() {} func (*GaugeResponse_StringValue) isGaugeResponse_Value() {} -func (m *GaugeResponse) GetValue() isGaugeResponse_Value { - if m != nil { - return m.Value - } - return nil -} +// Request message containing the gauge name +type GaugeRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields -func (m *GaugeResponse) GetLongValue() int64 { - if x, ok := m.GetValue().(*GaugeResponse_LongValue); ok { - return x.LongValue - } - return 0 + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` } -func (m *GaugeResponse) GetDoubleValue() float64 { - if x, ok := m.GetValue().(*GaugeResponse_DoubleValue); ok { - return x.DoubleValue +func (x *GaugeRequest) Reset() { + *x = GaugeRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_stress_grpc_testing_metrics_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } - return 0 } -func (m *GaugeResponse) GetStringValue() string { - if x, ok := m.GetValue().(*GaugeResponse_StringValue); ok { - return x.StringValue - } - return "" +func (x *GaugeRequest) String() string { + return protoimpl.X.MessageStringOf(x) } -// XXX_OneofWrappers is for the internal use of the proto package. -func (*GaugeResponse) XXX_OneofWrappers() []interface{} { - return []interface{}{ - (*GaugeResponse_LongValue)(nil), - (*GaugeResponse_DoubleValue)(nil), - (*GaugeResponse_StringValue)(nil), - } -} +func (*GaugeRequest) ProtoMessage() {} -// Request message containing the gauge name -type GaugeRequest struct { - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` +func (x *GaugeRequest) ProtoReflect() protoreflect.Message { + mi := &file_stress_grpc_testing_metrics_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -func (m *GaugeRequest) Reset() { *m = GaugeRequest{} } -func (m *GaugeRequest) String() string { return proto.CompactTextString(m) } -func (*GaugeRequest) ProtoMessage() {} +// Deprecated: Use GaugeRequest.ProtoReflect.Descriptor instead. func (*GaugeRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_028251bc41da09ab, []int{1} + return file_stress_grpc_testing_metrics_proto_rawDescGZIP(), []int{1} } -func (m *GaugeRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_GaugeRequest.Unmarshal(m, b) -} -func (m *GaugeRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_GaugeRequest.Marshal(b, m, deterministic) -} -func (m *GaugeRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_GaugeRequest.Merge(m, src) -} -func (m *GaugeRequest) XXX_Size() int { - return xxx_messageInfo_GaugeRequest.Size(m) -} -func (m *GaugeRequest) XXX_DiscardUnknown() { - xxx_messageInfo_GaugeRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_GaugeRequest proto.InternalMessageInfo - -func (m *GaugeRequest) GetName() string { - if m != nil { - return m.Name +func (x *GaugeRequest) GetName() string { + if x != nil { + return x.Name } return "" } type EmptyMessage struct { - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields } -func (m *EmptyMessage) Reset() { *m = EmptyMessage{} } -func (m *EmptyMessage) String() string { return proto.CompactTextString(m) } -func (*EmptyMessage) ProtoMessage() {} -func (*EmptyMessage) Descriptor() ([]byte, []int) { - return fileDescriptor_028251bc41da09ab, []int{2} +func (x *EmptyMessage) Reset() { + *x = EmptyMessage{} + if protoimpl.UnsafeEnabled { + mi := &file_stress_grpc_testing_metrics_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *EmptyMessage) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_EmptyMessage.Unmarshal(m, b) -} -func (m *EmptyMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_EmptyMessage.Marshal(b, m, deterministic) -} -func (m *EmptyMessage) XXX_Merge(src proto.Message) { - xxx_messageInfo_EmptyMessage.Merge(m, src) -} -func (m *EmptyMessage) XXX_Size() int { - return xxx_messageInfo_EmptyMessage.Size(m) -} -func (m *EmptyMessage) XXX_DiscardUnknown() { - xxx_messageInfo_EmptyMessage.DiscardUnknown(m) +func (x *EmptyMessage) String() string { + return protoimpl.X.MessageStringOf(x) } -var xxx_messageInfo_EmptyMessage proto.InternalMessageInfo +func (*EmptyMessage) ProtoMessage() {} -func init() { - proto.RegisterType((*GaugeResponse)(nil), "grpc.testing.GaugeResponse") - proto.RegisterType((*GaugeRequest)(nil), "grpc.testing.GaugeRequest") - proto.RegisterType((*EmptyMessage)(nil), "grpc.testing.EmptyMessage") +func (x *EmptyMessage) ProtoReflect() protoreflect.Message { + mi := &file_stress_grpc_testing_metrics_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -func init() { proto.RegisterFile("stress/grpc_testing/metrics.proto", fileDescriptor_028251bc41da09ab) } +// Deprecated: Use EmptyMessage.ProtoReflect.Descriptor instead. +func (*EmptyMessage) Descriptor() ([]byte, []int) { + return file_stress_grpc_testing_metrics_proto_rawDescGZIP(), []int{2} +} + +var File_stress_grpc_testing_metrics_proto protoreflect.FileDescriptor + +var file_stress_grpc_testing_metrics_proto_rawDesc = []byte{ + 0x0a, 0x21, 0x73, 0x74, 0x72, 0x65, 0x73, 0x73, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x74, 0x65, + 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, + 0x67, 0x22, 0x97, 0x01, 0x0a, 0x0d, 0x47, 0x61, 0x75, 0x67, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1f, 0x0a, 0x0a, 0x6c, 0x6f, 0x6e, 0x67, 0x5f, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x48, 0x00, 0x52, 0x09, 0x6c, + 0x6f, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x23, 0x0a, 0x0c, 0x64, 0x6f, 0x75, 0x62, + 0x6c, 0x65, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x01, 0x48, 0x00, + 0x52, 0x0b, 0x64, 0x6f, 0x75, 0x62, 0x6c, 0x65, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x23, 0x0a, + 0x0c, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0b, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, + 0x75, 0x65, 0x42, 0x07, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x22, 0x0a, 0x0c, 0x47, + 0x61, 0x75, 0x67, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, + 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, + 0x0e, 0x0a, 0x0c, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x32, + 0xa0, 0x01, 0x0a, 0x0e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x12, 0x49, 0x0a, 0x0c, 0x47, 0x65, 0x74, 0x41, 0x6c, 0x6c, 0x47, 0x61, 0x75, 0x67, + 0x65, 0x73, 0x12, 0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, + 0x67, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x1b, + 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x61, + 0x75, 0x67, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x12, 0x43, 0x0a, + 0x08, 0x47, 0x65, 0x74, 0x47, 0x61, 0x75, 0x67, 0x65, 0x12, 0x1a, 0x2e, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x61, 0x75, 0x67, 0x65, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x74, 0x65, 0x73, + 0x74, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x61, 0x75, 0x67, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x42, 0x2c, 0x5a, 0x2a, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67, 0x6f, 0x6c, + 0x61, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x73, 0x74, 0x72, + 0x65, 0x73, 0x73, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x5f, 0x74, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x67, + 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_stress_grpc_testing_metrics_proto_rawDescOnce sync.Once + file_stress_grpc_testing_metrics_proto_rawDescData = file_stress_grpc_testing_metrics_proto_rawDesc +) -var fileDescriptor_028251bc41da09ab = []byte{ - // 288 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x91, 0x3f, 0x4f, 0xc3, 0x30, - 0x10, 0xc5, 0x6b, 0x5a, 0xfe, 0xf4, 0x08, 0x1d, 0x3c, 0x55, 0x65, 0x20, 0x84, 0x25, 0x42, 0xc8, - 0x41, 0xf0, 0x09, 0x28, 0x42, 0x29, 0x43, 0x97, 0x20, 0x31, 0xb0, 0x54, 0x69, 0x38, 0x59, 0x91, - 0x9c, 0x38, 0xf8, 0x9c, 0x4a, 0x7c, 0x12, 0x56, 0x3e, 0x2a, 0x8a, 0x13, 0x55, 0x29, 0xaa, 0xba, - 0x59, 0xbf, 0xf7, 0xfc, 0x7c, 0xe7, 0x07, 0xd7, 0x64, 0x0d, 0x12, 0x45, 0xd2, 0x54, 0xd9, 0xca, - 0x22, 0xd9, 0xbc, 0x94, 0x51, 0x81, 0xd6, 0xe4, 0x19, 0x89, 0xca, 0x68, 0xab, 0xb9, 0xd7, 0x68, - 0xa2, 0xd3, 0x82, 0x1f, 0x06, 0x17, 0x71, 0x5a, 0x4b, 0x4c, 0x90, 0x2a, 0x5d, 0x12, 0x72, 0x0e, - 0xa3, 0x32, 0x2d, 0x70, 0xca, 0x7c, 0x16, 0x8e, 0x13, 0x77, 0xe6, 0x57, 0x00, 0x4a, 0x97, 0x72, - 0xb5, 0x49, 0x55, 0x8d, 0xd3, 0x23, 0x9f, 0x85, 0xc3, 0xc5, 0x20, 0x19, 0x37, 0xec, 0xbd, 0x41, - 0xfc, 0x06, 0xbc, 0x4f, 0x5d, 0xaf, 0x15, 0x76, 0x96, 0xa1, 0xcf, 0x42, 0xb6, 0x18, 0x24, 0xe7, - 0x2d, 0xdd, 0x9a, 0xc8, 0x9a, 0x7c, 0x9b, 0x33, 0x6a, 0x5e, 0x68, 0x4c, 0x2d, 0x75, 0xa6, 0xf9, - 0x29, 0x1c, 0x3b, 0x35, 0x08, 0xc0, 0xeb, 0x06, 0xfb, 0xaa, 0x91, 0xec, 0xbe, 0xb9, 0x82, 0x09, - 0x78, 0x2f, 0x45, 0x65, 0xbf, 0x97, 0x48, 0x94, 0x4a, 0x7c, 0xf8, 0x65, 0x30, 0x59, 0xb6, 0xdb, - 0xbe, 0xa1, 0xd9, 0xe4, 0x19, 0xf2, 0x57, 0xf0, 0x62, 0xb4, 0x4f, 0x4a, 0xb9, 0x30, 0xe2, 0x33, - 0xd1, 0xdf, 0x5f, 0xf4, 0xaf, 0xcf, 0x2e, 0x77, 0xb5, 0x9d, 0x7f, 0xb9, 0x67, 0xfc, 0x19, 0xce, - 0x62, 0xb4, 0x8e, 0xfe, 0x8f, 0xe9, 0x4f, 0x7a, 0x30, 0x66, 0x7e, 0xf7, 0x71, 0x2b, 0xb5, 0x96, - 0x0a, 0x85, 0xd4, 0x2a, 0x2d, 0xa5, 0xd0, 0x46, 0xba, 0xba, 0xa2, 0x3d, 0xd5, 0xad, 0x4f, 0x5c, - 0x67, 0x8f, 0x7f, 0x01, 0x00, 0x00, 0xff, 0xff, 0xb8, 0x8c, 0x62, 0x73, 0xd8, 0x01, 0x00, 0x00, +func file_stress_grpc_testing_metrics_proto_rawDescGZIP() []byte { + file_stress_grpc_testing_metrics_proto_rawDescOnce.Do(func() { + file_stress_grpc_testing_metrics_proto_rawDescData = protoimpl.X.CompressGZIP(file_stress_grpc_testing_metrics_proto_rawDescData) + }) + return file_stress_grpc_testing_metrics_proto_rawDescData +} + +var file_stress_grpc_testing_metrics_proto_msgTypes = make([]protoimpl.MessageInfo, 3) +var file_stress_grpc_testing_metrics_proto_goTypes = []interface{}{ + (*GaugeResponse)(nil), // 0: grpc.testing.GaugeResponse + (*GaugeRequest)(nil), // 1: grpc.testing.GaugeRequest + (*EmptyMessage)(nil), // 2: grpc.testing.EmptyMessage +} +var file_stress_grpc_testing_metrics_proto_depIdxs = []int32{ + 2, // 0: grpc.testing.MetricsService.GetAllGauges:input_type -> grpc.testing.EmptyMessage + 1, // 1: grpc.testing.MetricsService.GetGauge:input_type -> grpc.testing.GaugeRequest + 0, // 2: grpc.testing.MetricsService.GetAllGauges:output_type -> grpc.testing.GaugeResponse + 0, // 3: grpc.testing.MetricsService.GetGauge:output_type -> grpc.testing.GaugeResponse + 2, // [2:4] is the sub-list for method output_type + 0, // [0:2] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_stress_grpc_testing_metrics_proto_init() } +func file_stress_grpc_testing_metrics_proto_init() { + if File_stress_grpc_testing_metrics_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_stress_grpc_testing_metrics_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GaugeResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_stress_grpc_testing_metrics_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GaugeRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_stress_grpc_testing_metrics_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*EmptyMessage); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + file_stress_grpc_testing_metrics_proto_msgTypes[0].OneofWrappers = []interface{}{ + (*GaugeResponse_LongValue)(nil), + (*GaugeResponse_DoubleValue)(nil), + (*GaugeResponse_StringValue)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_stress_grpc_testing_metrics_proto_rawDesc, + NumEnums: 0, + NumMessages: 3, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_stress_grpc_testing_metrics_proto_goTypes, + DependencyIndexes: file_stress_grpc_testing_metrics_proto_depIdxs, + MessageInfos: file_stress_grpc_testing_metrics_proto_msgTypes, + }.Build() + File_stress_grpc_testing_metrics_proto = out.File + file_stress_grpc_testing_metrics_proto_rawDesc = nil + file_stress_grpc_testing_metrics_proto_goTypes = nil + file_stress_grpc_testing_metrics_proto_depIdxs = nil } diff --git a/stress/grpc_testing/metrics_grpc.pb.go b/stress/grpc_testing/metrics_grpc.pb.go index 3f0a83584b2d..4e2f985bdf16 100644 --- a/stress/grpc_testing/metrics_grpc.pb.go +++ b/stress/grpc_testing/metrics_grpc.pb.go @@ -1,4 +1,29 @@ +// Copyright 2015-2016 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Contains the definitions for a metrics service and the type of metrics +// exposed by the service. +// +// Currently, 'Gauge' (i.e a metric that represents the measured value of +// something at an instant of time) is the only metric type supported by the +// service. + // Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.3.0 +// - protoc v4.22.0 +// source: stress/grpc_testing/metrics.proto package grpc_testing @@ -11,8 +36,14 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. const _ = grpc.SupportPackageIsVersion7 +const ( + MetricsService_GetAllGauges_FullMethodName = "/grpc.testing.MetricsService/GetAllGauges" + MetricsService_GetGauge_FullMethodName = "/grpc.testing.MetricsService/GetGauge" +) + // MetricsServiceClient is the client API for MetricsService service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. @@ -32,13 +63,8 @@ func NewMetricsServiceClient(cc grpc.ClientConnInterface) MetricsServiceClient { return &metricsServiceClient{cc} } -var metricsServiceGetAllGaugesStreamDesc = &grpc.StreamDesc{ - StreamName: "GetAllGauges", - ServerStreams: true, -} - func (c *metricsServiceClient) GetAllGauges(ctx context.Context, in *EmptyMessage, opts ...grpc.CallOption) (MetricsService_GetAllGaugesClient, error) { - stream, err := c.cc.NewStream(ctx, metricsServiceGetAllGaugesStreamDesc, "/grpc.testing.MetricsService/GetAllGauges", opts...) + stream, err := c.cc.NewStream(ctx, &MetricsService_ServiceDesc.Streams[0], MetricsService_GetAllGauges_FullMethodName, opts...) if err != nil { return nil, err } @@ -69,60 +95,56 @@ func (x *metricsServiceGetAllGaugesClient) Recv() (*GaugeResponse, error) { return m, nil } -var metricsServiceGetGaugeStreamDesc = &grpc.StreamDesc{ - StreamName: "GetGauge", -} - func (c *metricsServiceClient) GetGauge(ctx context.Context, in *GaugeRequest, opts ...grpc.CallOption) (*GaugeResponse, error) { out := new(GaugeResponse) - err := c.cc.Invoke(ctx, "/grpc.testing.MetricsService/GetGauge", in, out, opts...) + err := c.cc.Invoke(ctx, MetricsService_GetGauge_FullMethodName, in, out, opts...) if err != nil { return nil, err } return out, nil } -// MetricsServiceService is the service API for MetricsService service. -// Fields should be assigned to their respective handler implementations only before -// RegisterMetricsServiceService is called. Any unassigned fields will result in the -// handler for that method returning an Unimplemented error. -type MetricsServiceService struct { +// MetricsServiceServer is the server API for MetricsService service. +// All implementations must embed UnimplementedMetricsServiceServer +// for forward compatibility +type MetricsServiceServer interface { // Returns the values of all the gauges that are currently being maintained by // the service - GetAllGauges func(*EmptyMessage, MetricsService_GetAllGaugesServer) error + GetAllGauges(*EmptyMessage, MetricsService_GetAllGaugesServer) error // Returns the value of one gauge - GetGauge func(context.Context, *GaugeRequest) (*GaugeResponse, error) + GetGauge(context.Context, *GaugeRequest) (*GaugeResponse, error) + mustEmbedUnimplementedMetricsServiceServer() } -func (s *MetricsServiceService) getAllGauges(_ interface{}, stream grpc.ServerStream) error { - if s.GetAllGauges == nil { - return status.Errorf(codes.Unimplemented, "method GetAllGauges not implemented") - } +// UnimplementedMetricsServiceServer must be embedded to have forward compatible implementations. +type UnimplementedMetricsServiceServer struct { +} + +func (UnimplementedMetricsServiceServer) GetAllGauges(*EmptyMessage, MetricsService_GetAllGaugesServer) error { + return status.Errorf(codes.Unimplemented, "method GetAllGauges not implemented") +} +func (UnimplementedMetricsServiceServer) GetGauge(context.Context, *GaugeRequest) (*GaugeResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetGauge not implemented") +} +func (UnimplementedMetricsServiceServer) mustEmbedUnimplementedMetricsServiceServer() {} + +// UnsafeMetricsServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to MetricsServiceServer will +// result in compilation errors. +type UnsafeMetricsServiceServer interface { + mustEmbedUnimplementedMetricsServiceServer() +} + +func RegisterMetricsServiceServer(s grpc.ServiceRegistrar, srv MetricsServiceServer) { + s.RegisterService(&MetricsService_ServiceDesc, srv) +} + +func _MetricsService_GetAllGauges_Handler(srv interface{}, stream grpc.ServerStream) error { m := new(EmptyMessage) if err := stream.RecvMsg(m); err != nil { return err } - return s.GetAllGauges(m, &metricsServiceGetAllGaugesServer{stream}) -} -func (s *MetricsServiceService) getGauge(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - if s.GetGauge == nil { - return nil, status.Errorf(codes.Unimplemented, "method GetGauge not implemented") - } - in := new(GaugeRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return s.GetGauge(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: s, - FullMethod: "/grpc.testing.MetricsService/GetGauge", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return s.GetGauge(ctx, req.(*GaugeRequest)) - } - return interceptor(ctx, in, info, handler) + return srv.(MetricsServiceServer).GetAllGauges(m, &metricsServiceGetAllGaugesServer{stream}) } type MetricsService_GetAllGaugesServer interface { @@ -138,58 +160,42 @@ func (x *metricsServiceGetAllGaugesServer) Send(m *GaugeResponse) error { return x.ServerStream.SendMsg(m) } -// RegisterMetricsServiceService registers a service implementation with a gRPC server. -func RegisterMetricsServiceService(s grpc.ServiceRegistrar, srv *MetricsServiceService) { - sd := grpc.ServiceDesc{ - ServiceName: "grpc.testing.MetricsService", - Methods: []grpc.MethodDesc{ - { - MethodName: "GetGauge", - Handler: srv.getGauge, - }, - }, - Streams: []grpc.StreamDesc{ - { - StreamName: "GetAllGauges", - Handler: srv.getAllGauges, - ServerStreams: true, - }, - }, - Metadata: "stress/grpc_testing/metrics.proto", +func _MetricsService_GetGauge_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GaugeRequest) + if err := dec(in); err != nil { + return nil, err } - - s.RegisterService(&sd, nil) -} - -// NewMetricsServiceService creates a new MetricsServiceService containing the -// implemented methods of the MetricsService service in s. Any unimplemented -// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client. -// This includes situations where the method handler is misspelled or has the wrong -// signature. For this reason, this function should be used with great care and -// is not recommended to be used by most users. -func NewMetricsServiceService(s interface{}) *MetricsServiceService { - ns := &MetricsServiceService{} - if h, ok := s.(interface { - GetAllGauges(*EmptyMessage, MetricsService_GetAllGaugesServer) error - }); ok { - ns.GetAllGauges = h.GetAllGauges + if interceptor == nil { + return srv.(MetricsServiceServer).GetGauge(ctx, in) } - if h, ok := s.(interface { - GetGauge(context.Context, *GaugeRequest) (*GaugeResponse, error) - }); ok { - ns.GetGauge = h.GetGauge + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: MetricsService_GetGauge_FullMethodName, } - return ns + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(MetricsServiceServer).GetGauge(ctx, req.(*GaugeRequest)) + } + return interceptor(ctx, in, info, handler) } -// UnstableMetricsServiceService is the service API for MetricsService service. -// New methods may be added to this interface if they are added to the service -// definition, which is not a backward-compatible change. For this reason, -// use of this type is not recommended. -type UnstableMetricsServiceService interface { - // Returns the values of all the gauges that are currently being maintained by - // the service - GetAllGauges(*EmptyMessage, MetricsService_GetAllGaugesServer) error - // Returns the value of one gauge - GetGauge(context.Context, *GaugeRequest) (*GaugeResponse, error) +// MetricsService_ServiceDesc is the grpc.ServiceDesc for MetricsService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var MetricsService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.testing.MetricsService", + HandlerType: (*MetricsServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "GetGauge", + Handler: _MetricsService_GetGauge_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "GetAllGauges", + Handler: _MetricsService_GetAllGauges_Handler, + ServerStreams: true, + }, + }, + Metadata: "stress/grpc_testing/metrics.proto", } diff --git a/stress/metrics_client/main.go b/stress/metrics_client/main.go index ad6db6dd7a19..8948f868dbf3 100644 --- a/stress/metrics_client/main.go +++ b/stress/metrics_client/main.go @@ -26,6 +26,7 @@ import ( "io" "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/grpclog" metricspb "google.golang.org/grpc/stress/grpc_testing" ) @@ -71,10 +72,10 @@ func printMetrics(client metricspb.MetricsServiceClient, totalOnly bool) { func main() { flag.Parse() if *metricsServerAddress == "" { - logger.Fatalf("Metrics server address is empty.") + logger.Fatal("-metrics_server_address is unset") } - conn, err := grpc.Dial(*metricsServerAddress, grpc.WithInsecure()) + conn, err := grpc.Dial(*metricsServerAddress, grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { logger.Fatalf("cannot connect to metrics server: %v", err) } diff --git a/tap/tap.go b/tap/tap.go index 584360f681b8..bfa5dfa40e4d 100644 --- a/tap/tap.go +++ b/tap/tap.go @@ -17,7 +17,12 @@ */ // Package tap defines the function handles which are executed on the transport -// layer of gRPC-Go and related information. Everything here is EXPERIMENTAL. +// layer of gRPC-Go and related information. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. package tap import ( @@ -32,16 +37,16 @@ type Info struct { // TODO: More to be added. } -// ServerInHandle defines the function which runs before a new stream is created -// on the server side. If it returns a non-nil error, the stream will not be -// created and a RST_STREAM will be sent back to the client with REFUSED_STREAM. -// The client will receive an RPC error "code = Unavailable, desc = stream -// terminated by RST_STREAM with error code: REFUSED_STREAM". +// ServerInHandle defines the function which runs before a new stream is +// created on the server side. If it returns a non-nil error, the stream will +// not be created and an error will be returned to the client. If the error +// returned is a status error, that status code and message will be used, +// otherwise PermissionDenied will be the code and err.Error() will be the +// message. // // It's intended to be used in situations where you don't want to waste the -// resources to accept the new stream (e.g. rate-limiting). And the content of -// the error will be ignored and won't be sent back to the client. For other -// general usages, please use interceptors. +// resources to accept the new stream (e.g. rate-limiting). For other general +// usages, please use interceptors. // // Note that it is executed in the per-connection I/O goroutine(s) instead of // per-RPC goroutine. Therefore, users should NOT have any diff --git a/test/authority_test.go b/test/authority_test.go index 6cd5d82eec19..6ee2d3db9d7c 100644 --- a/test/authority_test.go +++ b/test/authority_test.go @@ -1,3 +1,6 @@ +//go:build linux +// +build linux + /* * * Copyright 2020 gRPC authors. @@ -21,84 +24,219 @@ package test import ( "context" "fmt" + "net" "os" + "strings" + "sync" "testing" - "time" + "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/stubserver" "google.golang.org/grpc/metadata" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" "google.golang.org/grpc/status" - testpb "google.golang.org/grpc/test/grpc_testing" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" ) -func runUnixTest(t *testing.T, address, target, expectedAuthority string) { - if err := os.RemoveAll(address); err != nil { - t.Fatalf("Error removing socket file %v: %v\n", address, err) +func authorityChecker(ctx context.Context, expectedAuthority string) (*testpb.Empty, error) { + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return nil, status.Error(codes.InvalidArgument, "failed to parse metadata") } - us := &stubServer{ - emptyCall: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { - md, ok := metadata.FromIncomingContext(ctx) - if !ok { - return nil, status.Error(codes.InvalidArgument, "failed to parse metadata") - } - auths, ok := md[":authority"] - if !ok { - return nil, status.Error(codes.InvalidArgument, "no authority header") - } - if len(auths) < 1 { - return nil, status.Error(codes.InvalidArgument, "no authority header") - } - if auths[0] != expectedAuthority { - return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("invalid authority header %v, expected %v", auths[0], expectedAuthority)) - } - return &testpb.Empty{}, nil + auths, ok := md[":authority"] + if !ok { + return nil, status.Error(codes.InvalidArgument, "no authority header") + } + if len(auths) != 1 { + return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("no authority header, auths = %v", auths)) + } + if auths[0] != expectedAuthority { + return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("invalid authority header %v, expected %v", auths[0], expectedAuthority)) + } + return &testpb.Empty{}, nil +} + +func runUnixTest(t *testing.T, address, target, expectedAuthority string, dialer func(context.Context, string) (net.Conn, error)) { + if !strings.HasPrefix(target, "unix-abstract:") { + if err := os.RemoveAll(address); err != nil { + t.Fatalf("Error removing socket file %v: %v\n", address, err) + } + } + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + return authorityChecker(ctx, expectedAuthority) }, - network: "unix", - address: address, - target: target, + Network: "unix", + Address: address, + Target: target, + } + opts := []grpc.DialOption{} + if dialer != nil { + opts = append(opts, grpc.WithContextDialer(dialer)) } - if err := us.Start(nil); err != nil { + if err := ss.Start(nil, opts...); err != nil { t.Fatalf("Error starting endpoint server: %v", err) - return } - defer us.Stop() - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer ss.Stop() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() - _, err := us.client.EmptyCall(ctx, &testpb.Empty{}) + _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}) if err != nil { t.Errorf("us.client.EmptyCall(_, _) = _, %v; want _, nil", err) } } +type authorityTest struct { + name string + address string + target string + authority string + dialTargetWant string +} + +var authorityTests = []authorityTest{ + { + name: "UnixRelative", + address: "sock.sock", + target: "unix:sock.sock", + authority: "localhost", + dialTargetWant: "unix:sock.sock", + }, + { + name: "UnixAbsolute", + address: "/tmp/sock.sock", + target: "unix:/tmp/sock.sock", + authority: "localhost", + dialTargetWant: "unix:///tmp/sock.sock", + }, + { + name: "UnixAbsoluteAlternate", + address: "/tmp/sock.sock", + target: "unix:///tmp/sock.sock", + authority: "localhost", + dialTargetWant: "unix:///tmp/sock.sock", + }, + { + name: "UnixPassthrough", + address: "/tmp/sock.sock", + target: "passthrough:///unix:///tmp/sock.sock", + authority: "unix:%2F%2F%2Ftmp%2Fsock.sock", + dialTargetWant: "unix:///tmp/sock.sock", + }, + { + name: "UnixAbstract", + address: "@abc efg", + target: "unix-abstract:abc efg", + authority: "localhost", + dialTargetWant: "unix:@abc efg", + }, +} + +// TestUnix does end to end tests with the various supported unix target +// formats, ensuring that the authority is set as expected. func (s) TestUnix(t *testing.T) { - tests := []struct { - name string - address string - target string - authority string - }{ - { - name: "Unix1", - address: "sock.sock", - target: "unix:sock.sock", - authority: "localhost", - }, - { - name: "Unix2", - address: "/tmp/sock.sock", - target: "unix:/tmp/sock.sock", - authority: "localhost", + for _, test := range authorityTests { + t.Run(test.name, func(t *testing.T) { + runUnixTest(t, test.address, test.target, test.authority, nil) + }) + } +} + +// TestUnixCustomDialer does end to end tests with various supported unix target +// formats, ensuring that the target sent to the dialer does NOT have the +// "unix:" prefix stripped. +func (s) TestUnixCustomDialer(t *testing.T) { + for _, test := range authorityTests { + t.Run(test.name+"WithDialer", func(t *testing.T) { + dialer := func(ctx context.Context, address string) (net.Conn, error) { + if address != test.dialTargetWant { + return nil, fmt.Errorf("expected target %v in custom dialer, instead got %v", test.dialTargetWant, address) + } + address = address[len("unix:"):] + return (&net.Dialer{}).DialContext(ctx, "unix", address) + } + runUnixTest(t, test.address, test.target, test.authority, dialer) + }) + } +} + +// TestColonPortAuthority does an end to end test with the target for grpc.Dial +// being ":[port]". Ensures authority is "localhost:[port]". +func (s) TestColonPortAuthority(t *testing.T) { + expectedAuthority := "" + var authorityMu sync.Mutex + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + authorityMu.Lock() + defer authorityMu.Unlock() + return authorityChecker(ctx, expectedAuthority) }, - { - name: "Unix3", - address: "/tmp/sock.sock", - target: "unix:///tmp/sock.sock", - authority: "localhost", + Network: "tcp", + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + _, port, err := net.SplitHostPort(ss.Address) + if err != nil { + t.Fatalf("Failed splitting host from post: %v", err) + } + authorityMu.Lock() + expectedAuthority = "localhost:" + port + authorityMu.Unlock() + // ss.Start dials, but not the ":[port]" target that is being tested here. + // Dial again, with ":[port]" as the target. + // + // Append "localhost" before calling net.Dial, in case net.Dial on certain + // platforms doesn't work well for address without the IP. + cc, err := grpc.Dial(":"+port, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, "tcp", "localhost"+addr) + })) + if err != nil { + t.Fatalf("grpc.Dial(%q) = %v", ss.Target, err) + } + defer cc.Close() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + _, err = testgrpc.NewTestServiceClient(cc).EmptyCall(ctx, &testpb.Empty{}) + if err != nil { + t.Errorf("us.client.EmptyCall(_, _) = _, %v; want _, nil", err) + } +} + +// TestAuthorityReplacedWithResolverAddress tests the scenario where the resolver +// returned address contains a ServerName override. The test verifies that the the +// :authority header value sent to the server as part of the http/2 HEADERS frame +// is set to the value specified in the resolver returned address. +func (s) TestAuthorityReplacedWithResolverAddress(t *testing.T) { + const expectedAuthority = "test.server.name" + + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + return authorityChecker(ctx, expectedAuthority) }, } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - runUnixTest(t, test.address, test.target, test.authority) - }) + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + r := manual.NewBuilderWithScheme("whatever") + r.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: ss.Address, ServerName: expectedAuthority}}}) + cc, err := grpc.Dial(r.Scheme()+":///whatever", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("grpc.Dial(%q) = %v", ss.Address, err) + } + defer cc.Close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err = testgrpc.NewTestServiceClient(cc).EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() rpc failed: %v", err) } } diff --git a/test/balancer_switching_test.go b/test/balancer_switching_test.go new file mode 100644 index 000000000000..5decc4d3b83b --- /dev/null +++ b/test/balancer_switching_test.go @@ -0,0 +1,509 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "context" + "fmt" + "testing" + + "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + grpclbstate "google.golang.org/grpc/balancer/grpclb/state" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/balancer/stub" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils/fakegrpclb" + "google.golang.org/grpc/internal/testutils/pickfirst" + rrutil "google.golang.org/grpc/internal/testutils/roundrobin" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +const ( + loadBalancedServiceName = "foo.bar.service" + loadBalancedServicePort = 443 + wantGRPCLBTraceDesc = `Channel switches to new LB policy "grpclb"` + wantRoundRobinTraceDesc = `Channel switches to new LB policy "round_robin"` + + // This is the number of stub backends set up at the start of each test. The + // first backend is used for the "grpclb" policy and the rest are used for + // other LB policies to test balancer switching. + backendCount = 3 +) + +// setupBackendsAndFakeGRPCLB sets up backendCount number of stub server +// backends and a fake grpclb server for tests which exercise balancer switch +// scenarios involving grpclb. +// +// The fake grpclb server always returns the first of the configured stub +// backends as backend addresses. So, the tests are free to use the other +// backends with other LB policies to verify balancer switching scenarios. +// +// Returns a cleanup function to be invoked by the caller. +func setupBackendsAndFakeGRPCLB(t *testing.T) ([]*stubserver.StubServer, *fakegrpclb.Server, func()) { + backends, backendsCleanup := startBackendsForBalancerSwitch(t) + + lbServer, err := fakegrpclb.NewServer(fakegrpclb.ServerParams{ + LoadBalancedServiceName: loadBalancedServiceName, + LoadBalancedServicePort: loadBalancedServicePort, + BackendAddresses: []string{backends[0].Address}, + }) + if err != nil { + t.Fatalf("failed to create fake grpclb server: %v", err) + } + go func() { + if err := lbServer.Serve(); err != nil { + t.Errorf("fake grpclb Serve() failed: %v", err) + } + }() + + return backends, lbServer, func() { + backendsCleanup() + lbServer.Stop() + } +} + +// startBackendsForBalancerSwitch spins up a bunch of stub server backends +// exposing the TestService. Returns a cleanup function to be invoked by the +// caller. +func startBackendsForBalancerSwitch(t *testing.T) ([]*stubserver.StubServer, func()) { + t.Helper() + + backends := make([]*stubserver.StubServer, backendCount) + for i := 0; i < backendCount; i++ { + backend := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil }, + } + if err := backend.StartServer(); err != nil { + t.Fatalf("Failed to start backend: %v", err) + } + t.Logf("Started TestService backend at: %q", backend.Address) + backends[i] = backend + } + return backends, func() { + for _, b := range backends { + b.Stop() + } + } +} + +// TestBalancerSwitch_Basic tests the basic scenario of switching from one LB +// policy to another, as specified in the service config. +func (s) TestBalancerSwitch_Basic(t *testing.T) { + backends, cleanup := startBackendsForBalancerSwitch(t) + defer cleanup() + addrs := stubBackendsToResolverAddrs(backends) + + r := manual.NewBuilderWithScheme("whatever") + cc, err := grpc.Dial(r.Scheme()+":///test.server", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) + } + defer cc.Close() + + // Push a resolver update without an LB policy in the service config. The + // channel should pick the default LB policy, which is pick_first. + r.UpdateState(resolver.State{Addresses: addrs}) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + // Push a resolver update with the service config specifying "round_robin". + r.UpdateState(resolver.State{ + Addresses: addrs, + ServiceConfig: parseServiceConfig(t, r, rrServiceConfig), + }) + client := testgrpc.NewTestServiceClient(cc) + if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs); err != nil { + t.Fatal(err) + } + + // Push a resolver update with the service config specifying "pick_first". + r.UpdateState(resolver.State{ + Addresses: addrs, + ServiceConfig: parseServiceConfig(t, r, pickFirstServiceConfig), + }) + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } +} + +// TestBalancerSwitch_grpclbToPickFirst tests the scenario where the channel +// starts off "grpclb", switches to "pick_first" and back. +func (s) TestBalancerSwitch_grpclbToPickFirst(t *testing.T) { + backends, lbServer, cleanup := setupBackendsAndFakeGRPCLB(t) + defer cleanup() + + addrs := stubBackendsToResolverAddrs(backends) + r := manual.NewBuilderWithScheme("whatever") + target := fmt.Sprintf("%s:///%s", r.Scheme(), loadBalancedServiceName) + cc, err := grpc.Dial(target, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) + } + defer cc.Close() + + // Push a resolver update with a GRPCLB service config and a single address + // pointing to the grpclb server we created above. This will cause the + // channel to switch to the "grpclb" balancer, which returns a single + // backend address. + grpclbConfig := parseServiceConfig(t, r, `{"loadBalancingPolicy": "grpclb"}`) + state := resolver.State{ServiceConfig: grpclbConfig} + r.UpdateState(grpclbstate.Set(state, &grpclbstate.State{BalancerAddresses: []resolver.Address{{Addr: lbServer.Address()}}})) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + client := testgrpc.NewTestServiceClient(cc) + if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs[0:1]); err != nil { + t.Fatal(err) + } + + // Push a resolver update containing a non-existent grpclb server address. + // This should not lead to a balancer switch. + const nonExistentServer = "non-existent-grpclb-server-address" + r.UpdateState(grpclbstate.Set(state, &grpclbstate.State{BalancerAddresses: []resolver.Address{{Addr: nonExistentServer}}})) + if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs[:1]); err != nil { + t.Fatal(err) + } + + // Push a resolver update containing no grpclb server address. This should + // lead to the channel using the default LB policy which is pick_first. The + // list of addresses pushed as part of this update is different from the one + // returned by the "grpclb" balancer. So, we should see RPCs going to the + // newly configured backends, as part of the balancer switch. + emptyConfig := parseServiceConfig(t, r, `{}`) + r.UpdateState(resolver.State{Addresses: addrs[1:], ServiceConfig: emptyConfig}) + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { + t.Fatal(err) + } +} + +// TestBalancerSwitch_pickFirstToGRPCLB tests the scenario where the channel +// starts off with "pick_first", switches to "grpclb" and back. +func (s) TestBalancerSwitch_pickFirstToGRPCLB(t *testing.T) { + backends, lbServer, cleanup := setupBackendsAndFakeGRPCLB(t) + defer cleanup() + + addrs := stubBackendsToResolverAddrs(backends) + r := manual.NewBuilderWithScheme("whatever") + target := fmt.Sprintf("%s:///%s", r.Scheme(), loadBalancedServiceName) + cc, err := grpc.Dial(target, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) + } + defer cc.Close() + + // Push a resolver update containing no grpclb server address. This should + // lead to the channel using the default LB policy which is pick_first. + r.UpdateState(resolver.State{Addresses: addrs[1:]}) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { + t.Fatal(err) + } + + // Push a resolver update with no service config and a single address pointing + // to the grpclb server we created above. This will cause the channel to + // switch to the "grpclb" balancer, which returns a single backend address. + grpclbConfig := parseServiceConfig(t, r, `{"loadBalancingPolicy": "grpclb"}`) + state := resolver.State{ServiceConfig: grpclbConfig} + r.UpdateState(grpclbstate.Set(state, &grpclbstate.State{BalancerAddresses: []resolver.Address{{Addr: lbServer.Address()}}})) + client := testgrpc.NewTestServiceClient(cc) + if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs[:1]); err != nil { + t.Fatal(err) + } + + // Push a resolver update containing a non-existent grpclb server address. + // This should not lead to a balancer switch. + r.UpdateState(grpclbstate.Set(state, &grpclbstate.State{BalancerAddresses: []resolver.Address{{Addr: "nonExistentServer"}}})) + if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs[:1]); err != nil { + t.Fatal(err) + } + + // Switch to "pick_first" again by sending no grpclb server addresses. + emptyConfig := parseServiceConfig(t, r, `{}`) + r.UpdateState(resolver.State{Addresses: addrs[1:], ServiceConfig: emptyConfig}) + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { + t.Fatal(err) + } +} + +// TestBalancerSwitch_RoundRobinToGRPCLB tests the scenario where the channel +// starts off with "round_robin", switches to "grpclb" and back. +// +// Note that this test uses the deprecated `loadBalancingPolicy` field in the +// service config. +func (s) TestBalancerSwitch_RoundRobinToGRPCLB(t *testing.T) { + backends, lbServer, cleanup := setupBackendsAndFakeGRPCLB(t) + defer cleanup() + + addrs := stubBackendsToResolverAddrs(backends) + r := manual.NewBuilderWithScheme("whatever") + target := fmt.Sprintf("%s:///%s", r.Scheme(), loadBalancedServiceName) + cc, err := grpc.Dial(target, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) + } + defer cc.Close() + + // Note the use of the deprecated `loadBalancingPolicy` field here instead + // of the now recommended `loadBalancingConfig` field. The logic in the + // ClientConn which decides which balancer to switch to looks at the + // following places in the given order of preference: + // - `loadBalancingConfig` field + // - addresses of type grpclb + // - `loadBalancingPolicy` field + // If we use the `loadBalancingPolicy` field, the switch to "grpclb" later on + // in the test will not happen as the ClientConn will continue to use the LB + // policy received in the first update. + scpr := parseServiceConfig(t, r, `{"loadBalancingPolicy": "round_robin"}`) + + // Push a resolver update with the service config specifying "round_robin". + r.UpdateState(resolver.State{Addresses: addrs[1:], ServiceConfig: scpr}) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + client := testgrpc.NewTestServiceClient(cc) + if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs[1:]); err != nil { + t.Fatal(err) + } + + // Push a resolver update with grpclb and a single balancer address + // pointing to the grpclb server we created above. This will cause the + // channel to switch to the "grpclb" balancer, which returns a single + // backend address. + grpclbConfig := parseServiceConfig(t, r, `{"loadBalancingPolicy": "grpclb"}`) + state := resolver.State{ServiceConfig: grpclbConfig} + r.UpdateState(grpclbstate.Set(state, &grpclbstate.State{BalancerAddresses: []resolver.Address{{Addr: lbServer.Address()}}})) + if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs[:1]); err != nil { + t.Fatal(err) + } + + // Switch back to "round_robin". + r.UpdateState(resolver.State{Addresses: addrs[1:], ServiceConfig: scpr}) + if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs[1:]); err != nil { + t.Fatal(err) + } +} + +// TestBalancerSwitch_grpclbNotRegistered tests the scenario where the grpclb +// balancer is not registered. Verifies that the ClientConn fallbacks to the +// default LB policy or the LB policy specified in the service config, and that +// addresses of type "grpclb" are filtered out. +func (s) TestBalancerSwitch_grpclbNotRegistered(t *testing.T) { + // Unregister the grpclb balancer builder for the duration of this test. + grpclbBuilder := balancer.Get("grpclb") + internal.BalancerUnregister(grpclbBuilder.Name()) + defer balancer.Register(grpclbBuilder) + + backends, cleanup := startBackendsForBalancerSwitch(t) + defer cleanup() + addrs := stubBackendsToResolverAddrs(backends) + + r := manual.NewBuilderWithScheme("whatever") + cc, err := grpc.Dial(r.Scheme()+":///test.server", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) + } + defer cc.Close() + + // Push a resolver update which contains a bunch of stub server backends and a + // grpclb server address. The latter should get the ClientConn to try and + // apply the grpclb policy. But since grpclb is not registered, it should + // fallback to the default LB policy which is pick_first. The ClientConn is + // also expected to filter out the grpclb address when sending the addresses + // list fo pick_first. + grpclbAddr := []resolver.Address{{Addr: "non-existent-grpclb-server-address"}} + grpclbConfig := parseServiceConfig(t, r, `{"loadBalancingPolicy": "grpclb"}`) + state := resolver.State{ServiceConfig: grpclbConfig, Addresses: addrs} + r.UpdateState(grpclbstate.Set(state, &grpclbstate.State{BalancerAddresses: grpclbAddr})) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + // Push a resolver update with the same addresses, but with a service config + // specifying "round_robin". The ClientConn is expected to filter out the + // grpclb address when sending the addresses list to round_robin. + r.UpdateState(resolver.State{ + Addresses: addrs, + ServiceConfig: parseServiceConfig(t, r, rrServiceConfig), + }) + client := testgrpc.NewTestServiceClient(cc) + if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs); err != nil { + t.Fatal(err) + } +} + +// TestBalancerSwitch_OldBalancerCallsShutdownInClose tests the scenario where +// the balancer being switched out calls Shutdown() in its Close() +// method. Verifies that this sequence of calls doesn't lead to a deadlock. +func (s) TestBalancerSwitch_OldBalancerCallsShutdownInClose(t *testing.T) { + // Register a stub balancer which calls Shutdown() from its Close(). + scChan := make(chan balancer.SubConn, 1) + uccsCalled := make(chan struct{}, 1) + stub.Register(t.Name(), stub.BalancerFuncs{ + UpdateClientConnState: func(data *stub.BalancerData, ccs balancer.ClientConnState) error { + sc, err := data.ClientConn.NewSubConn(ccs.ResolverState.Addresses, balancer.NewSubConnOptions{}) + if err != nil { + t.Errorf("failed to create subConn: %v", err) + } + scChan <- sc + close(uccsCalled) + return nil + }, + Close: func(data *stub.BalancerData) { + (<-scChan).Shutdown() + }, + }) + + r := manual.NewBuilderWithScheme("whatever") + cc, err := grpc.Dial(r.Scheme()+":///test.server", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) + } + defer cc.Close() + + // Push a resolver update specifying our stub balancer as the LB policy. + scpr := parseServiceConfig(t, r, fmt.Sprintf(`{"loadBalancingPolicy": "%v"}`, t.Name())) + r.UpdateState(resolver.State{ + Addresses: []resolver.Address{{Addr: "dummy-address"}}, + ServiceConfig: scpr, + }) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + select { + case <-ctx.Done(): + t.Fatalf("timeout waiting for UpdateClientConnState to be called: %v", ctx.Err()) + case <-uccsCalled: + } + + // The following service config update will switch balancer from our stub + // balancer to pick_first. The former will be closed, which will call + // sc.Shutdown() inline. + // + // This is to make sure the sc.Shutdown() from Close() doesn't cause a + // deadlock (e.g. trying to grab a mutex while it's already locked). + // + // Do it in a goroutine so this test will fail with a helpful message + // (though the goroutine will still leak). + done := make(chan struct{}) + go func() { + r.UpdateState(resolver.State{ + Addresses: []resolver.Address{{Addr: "dummy-address"}}, + ServiceConfig: parseServiceConfig(t, r, pickFirstServiceConfig), + }) + close(done) + }() + + select { + case <-ctx.Done(): + t.Fatalf("timeout waiting for resolver.UpdateState to finish: %v", ctx.Err()) + case <-done: + } +} + +// TestBalancerSwitch_Graceful tests the graceful switching of LB policies. It +// starts off by configuring "round_robin" on the channel and ensures that RPCs +// are successful. Then, it switches to a stub balancer which does not report a +// picker until instructed by the test do to so. At this point, the test +// verifies that RPCs are still successful using the old balancer. Then the test +// asks the new balancer to report a healthy picker and the test verifies that +// the RPCs get routed using the picker reported by the new balancer. +func (s) TestBalancerSwitch_Graceful(t *testing.T) { + backends, cleanup := startBackendsForBalancerSwitch(t) + defer cleanup() + addrs := stubBackendsToResolverAddrs(backends) + + r := manual.NewBuilderWithScheme("whatever") + cc, err := grpc.Dial(r.Scheme()+":///test.server", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) + } + defer cc.Close() + + // Push a resolver update with the service config specifying "round_robin". + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + r.UpdateState(resolver.State{ + Addresses: addrs[1:], + ServiceConfig: parseServiceConfig(t, r, rrServiceConfig), + }) + client := testgrpc.NewTestServiceClient(cc) + if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs[1:]); err != nil { + t.Fatal(err) + } + + // Register a stub balancer which uses a "pick_first" balancer underneath and + // signals on a channel when it receives ClientConn updates. But it does not + // forward the ccUpdate to the underlying "pick_first" balancer until the test + // asks it to do so. This allows us to test the graceful switch functionality. + // Until the test asks the stub balancer to forward the ccUpdate, RPCs should + // get routed to the old balancer. And once the test gives the go ahead, RPCs + // should get routed to the new balancer. + ccUpdateCh := make(chan struct{}) + waitToProceed := make(chan struct{}) + stub.Register(t.Name(), stub.BalancerFuncs{ + Init: func(bd *stub.BalancerData) { + pf := balancer.Get(grpc.PickFirstBalancerName) + bd.Data = pf.Build(bd.ClientConn, bd.BuildOptions) + }, + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + bal := bd.Data.(balancer.Balancer) + close(ccUpdateCh) + go func() { + <-waitToProceed + bal.UpdateClientConnState(ccs) + }() + return nil + }, + }) + + // Push a resolver update with the service config specifying our stub + // balancer. We should see a trace event for this balancer switch. But RPCs + // should still be routed to the old balancer since our stub balancer does not + // report a ready picker until we ask it to do so. + r.UpdateState(resolver.State{ + Addresses: addrs[:1], + ServiceConfig: r.CC.ParseServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%v": {}}]}`, t.Name())), + }) + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for a ClientConnState update on the new balancer") + case <-ccUpdateCh: + } + if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs[1:]); err != nil { + t.Fatal(err) + } + + // Ask our stub balancer to forward the earlier received ccUpdate to the + // underlying "pick_first" balancer which will result in a healthy picker + // being reported to the channel. RPCs should start using the new balancer. + close(waitToProceed) + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } +} diff --git a/test/balancer_test.go b/test/balancer_test.go index c45a380115b3..0c71da7146d6 100644 --- a/test/balancer_test.go +++ b/test/balancer_test.go @@ -28,24 +28,30 @@ import ( "time" "github.com/google/go-cmp/cmp" + "google.golang.org/grpc" "google.golang.org/grpc/attributes" "google.golang.org/grpc/balancer" - "google.golang.org/grpc/balancer/roundrobin" "google.golang.org/grpc/codes" "google.golang.org/grpc/connectivity" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" "google.golang.org/grpc/internal/balancer/stub" "google.golang.org/grpc/internal/balancerload" "google.golang.org/grpc/internal/grpcsync" "google.golang.org/grpc/internal/grpcutil" + imetadata "google.golang.org/grpc/internal/metadata" + "google.golang.org/grpc/internal/stubserver" "google.golang.org/grpc/internal/testutils" "google.golang.org/grpc/metadata" "google.golang.org/grpc/resolver" "google.golang.org/grpc/resolver/manual" "google.golang.org/grpc/status" - testpb "google.golang.org/grpc/test/grpc_testing" "google.golang.org/grpc/testdata" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" ) const testBalancerName = "testbalancer" @@ -81,31 +87,30 @@ func (b *testBalancer) UpdateClientConnState(state balancer.ClientConnState) err // Only create a subconn at the first time. if b.sc == nil { var err error + b.newSubConnOptions.StateListener = b.updateSubConnState b.sc, err = b.cc.NewSubConn(state.ResolverState.Addresses, b.newSubConnOptions) if err != nil { logger.Errorf("testBalancer: failed to NewSubConn: %v", err) return nil } - b.cc.UpdateState(balancer.State{ConnectivityState: connectivity.Connecting, Picker: &picker{sc: b.sc, bal: b}}) + b.cc.UpdateState(balancer.State{ConnectivityState: connectivity.Connecting, Picker: &picker{err: balancer.ErrNoSubConnAvailable, bal: b}}) b.sc.Connect() } return nil } func (b *testBalancer) UpdateSubConnState(sc balancer.SubConn, s balancer.SubConnState) { - logger.Infof("testBalancer: UpdateSubConnState: %p, %v", sc, s) - if b.sc != sc { - logger.Infof("testBalancer: ignored state change because sc is not recognized") - return - } - if s.ConnectivityState == connectivity.Shutdown { - b.sc = nil - return - } + panic(fmt.Sprintf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, s)) +} + +func (b *testBalancer) updateSubConnState(s balancer.SubConnState) { + logger.Infof("testBalancer: updateSubConnState: %v", s) switch s.ConnectivityState { - case connectivity.Ready, connectivity.Idle: - b.cc.UpdateState(balancer.State{ConnectivityState: s.ConnectivityState, Picker: &picker{sc: sc, bal: b}}) + case connectivity.Ready: + b.cc.UpdateState(balancer.State{ConnectivityState: s.ConnectivityState, Picker: &picker{bal: b}}) + case connectivity.Idle: + b.cc.UpdateState(balancer.State{ConnectivityState: s.ConnectivityState, Picker: &picker{bal: b, idle: true}}) case connectivity.Connecting: b.cc.UpdateState(balancer.State{ConnectivityState: s.ConnectivityState, Picker: &picker{err: balancer.ErrNoSubConnAvailable, bal: b}}) case connectivity.TransientFailure: @@ -115,21 +120,27 @@ func (b *testBalancer) UpdateSubConnState(sc balancer.SubConn, s balancer.SubCon func (b *testBalancer) Close() {} +func (b *testBalancer) ExitIdle() {} + type picker struct { - err error - sc balancer.SubConn - bal *testBalancer + err error + bal *testBalancer + idle bool } func (p *picker) Pick(info balancer.PickInfo) (balancer.PickResult, error) { if p.err != nil { return balancer.PickResult{}, p.err } + if p.idle { + p.bal.sc.Connect() + return balancer.PickResult{}, balancer.ErrNoSubConnAvailable + } extraMD, _ := grpcutil.ExtraMetadata(info.Ctx) info.Ctx = nil // Do not validate context. p.bal.pickInfos = append(p.bal.pickInfos, info) p.bal.pickExtraMDs = append(p.bal.pickExtraMDs, extraMD) - return balancer.PickResult{SubConn: p.sc, Done: func(d balancer.DoneInfo) { p.bal.doneInfo = append(p.bal.doneInfo, d) }}, nil + return balancer.PickResult{SubConn: p.bal.sc, Done: func(d balancer.DoneInfo) { p.bal.doneInfo = append(p.bal.doneInfo, d) }}, nil } func (s) TestCredsBundleFromBalancer(t *testing.T) { @@ -141,7 +152,7 @@ func (s) TestCredsBundleFromBalancer(t *testing.T) { te := newTest(t, env{name: "creds-bundle", network: "tcp", balancer: ""}) te.tapHandle = authHandle te.customDialOptions = []grpc.DialOption{ - grpc.WithBalancerName(testBalancerName), + grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, testBalancerName)), } creds, err := credentials.NewServerTLSFromFile(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem")) if err != nil { @@ -154,8 +165,10 @@ func (s) TestCredsBundleFromBalancer(t *testing.T) { defer te.tearDown() cc := te.clientConn() - tc := testpb.NewTestServiceClient(cc) - if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); err != nil { + tc := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil { t.Fatalf("Test failed. Reason: %v", err) } } @@ -176,31 +189,27 @@ func testPickExtraMetadata(t *testing.T, e env) { ) te.customDialOptions = []grpc.DialOption{ - grpc.WithBalancerName(testBalancerName), + grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, testBalancerName)), grpc.WithUserAgent(testUserAgent), } te.startServer(&testServer{security: e.security}) defer te.tearDown() - // Set resolver to xds to trigger the extra metadata code path. - r := manual.NewBuilderWithScheme("xds") - resolver.Register(r) - defer func() { - resolver.UnregisterForTesting("xds") - }() - r.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: te.srvAddr}}}) - te.resolverScheme = "xds" - cc := te.clientConn() - tc := testpb.NewTestServiceClient(cc) + // Trigger the extra-metadata-adding code path. + defer func(old string) { internal.GRPCResolverSchemeExtraMetadata = old }(internal.GRPCResolverSchemeExtraMetadata) + internal.GRPCResolverSchemeExtraMetadata = "passthrough" - // The RPCs will fail, but we don't care. We just need the pick to happen. - ctx1, cancel1 := context.WithTimeout(context.Background(), time.Second) - defer cancel1() - tc.EmptyCall(ctx1, &testpb.Empty{}) + cc := te.clientConn() + tc := testgrpc.NewTestServiceClient(cc) - ctx2, cancel2 := context.WithTimeout(context.Background(), time.Second) - defer cancel2() - tc.EmptyCall(ctx2, &testpb.Empty{}, grpc.CallContentSubtype(testSubContentType)) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, %v", err, nil) + } + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.CallContentSubtype(testSubContentType)); err != nil { + t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, %v", err, nil) + } want := []metadata.MD{ // First RPC doesn't have sub-content-type. @@ -208,9 +217,8 @@ func testPickExtraMetadata(t *testing.T, e env) { // Second RPC has sub-content-type "proto". {"content-type": []string{"application/grpc+proto"}}, } - - if !cmp.Equal(b.pickExtraMDs, want) { - t.Fatalf("%s", cmp.Diff(b.pickExtraMDs, want)) + if diff := cmp.Diff(want, b.pickExtraMDs); diff != "" { + t.Fatalf("unexpected diff in metadata (-want, +got): %s", diff) } } @@ -225,16 +233,16 @@ func testDoneInfo(t *testing.T, e env) { b := &testBalancer{} balancer.Register(b) te.customDialOptions = []grpc.DialOption{ - grpc.WithBalancerName(testBalancerName), + grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, testBalancerName)), } te.userAgent = failAppUA te.startServer(&testServer{security: e.security}) defer te.tearDown() cc := te.clientConn() - tc := testpb.NewTestServiceClient(cc) + tc := testgrpc.NewTestServiceClient(cc) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() wantErr := detailedError if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); !testutils.StatusErrEqual(err, wantErr) { @@ -276,7 +284,7 @@ const loadMDKey = "X-Endpoint-Load-Metrics-Bin" type testLoadParser struct{} -func (*testLoadParser) Parse(md metadata.MD) interface{} { +func (*testLoadParser) Parse(md metadata.MD) any { vs := md.Get(loadMDKey) if len(vs) == 0 { return nil @@ -289,31 +297,29 @@ func init() { } func (s) TestDoneLoads(t *testing.T) { - for _, e := range listTestEnv() { - testDoneLoads(t, e) - } + testDoneLoads(t) } -func testDoneLoads(t *testing.T, e env) { +func testDoneLoads(t *testing.T) { b := &testBalancer{} balancer.Register(b) const testLoad = "test-load-,-should-be-orca" - ss := &stubServer{ - emptyCall: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { grpc.SetTrailer(ctx, metadata.Pairs(loadMDKey, testLoad)) return &testpb.Empty{}, nil }, } - if err := ss.Start(nil, grpc.WithBalancerName(testBalancerName)); err != nil { + if err := ss.Start(nil, grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, testBalancerName))); err != nil { t.Fatalf("error starting testing server: %v", err) } defer ss.Stop() - tc := testpb.NewTestServiceClient(ss.cc) + tc := testgrpc.NewTestServiceClient(ss.CC) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil { t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, %v", err, nil) @@ -335,94 +341,6 @@ func testDoneLoads(t *testing.T, e env) { } } -const testBalancerKeepAddressesName = "testbalancer-keepingaddresses" - -// testBalancerKeepAddresses keeps the addresses in the builder instead of -// creating SubConns. -// -// It's used to test the addresses balancer gets are correct. -type testBalancerKeepAddresses struct { - addrsChan chan []resolver.Address -} - -func newTestBalancerKeepAddresses() *testBalancerKeepAddresses { - return &testBalancerKeepAddresses{ - addrsChan: make(chan []resolver.Address, 10), - } -} - -func (testBalancerKeepAddresses) ResolverError(err error) { - panic("not implemented") -} - -func (b *testBalancerKeepAddresses) Build(cc balancer.ClientConn, opt balancer.BuildOptions) balancer.Balancer { - return b -} - -func (*testBalancerKeepAddresses) Name() string { - return testBalancerKeepAddressesName -} - -func (b *testBalancerKeepAddresses) UpdateClientConnState(state balancer.ClientConnState) error { - b.addrsChan <- state.ResolverState.Addresses - return nil -} - -func (testBalancerKeepAddresses) UpdateSubConnState(sc balancer.SubConn, s balancer.SubConnState) { - panic("not used") -} - -func (testBalancerKeepAddresses) Close() { -} - -// Make sure that non-grpclb balancers don't get grpclb addresses even if name -// resolver sends them -func (s) TestNonGRPCLBBalancerGetsNoGRPCLBAddress(t *testing.T) { - r := manual.NewBuilderWithScheme("whatever") - - b := newTestBalancerKeepAddresses() - balancer.Register(b) - - cc, err := grpc.Dial(r.Scheme()+":///test.server", grpc.WithInsecure(), grpc.WithResolvers(r), - grpc.WithBalancerName(b.Name())) - if err != nil { - t.Fatalf("failed to dial: %v", err) - } - defer cc.Close() - - grpclbAddresses := []resolver.Address{{ - Addr: "grpc.lb.com", - Type: resolver.GRPCLB, - ServerName: "grpc.lb.com", - }} - - nonGRPCLBAddresses := []resolver.Address{{ - Addr: "localhost", - Type: resolver.Backend, - }} - - r.UpdateState(resolver.State{ - Addresses: nonGRPCLBAddresses, - }) - if got := <-b.addrsChan; !reflect.DeepEqual(got, nonGRPCLBAddresses) { - t.Fatalf("With only backend addresses, balancer got addresses %v, want %v", got, nonGRPCLBAddresses) - } - - r.UpdateState(resolver.State{ - Addresses: grpclbAddresses, - }) - if got := <-b.addrsChan; len(got) != 0 { - t.Fatalf("With only grpclb addresses, balancer got addresses %v, want empty", got) - } - - r.UpdateState(resolver.State{ - Addresses: append(grpclbAddresses, nonGRPCLBAddresses...), - }) - if got := <-b.addrsChan; !reflect.DeepEqual(got, nonGRPCLBAddresses) { - t.Fatalf("With both backend and grpclb addresses, balancer got addresses %v, want %v", got, nonGRPCLBAddresses) - } -} - type aiPicker struct { result balancer.PickResult err error @@ -475,16 +393,18 @@ func (s) TestAddressAttributesInNewSubConn(t *testing.T) { // Only use the first address. attr := attributes.New(testAttrKey, testAttrVal) addrs[0].Attributes = attr - sc, err := bd.ClientConn.NewSubConn([]resolver.Address{addrs[0]}, balancer.NewSubConnOptions{}) + var sc balancer.SubConn + sc, err := bd.ClientConn.NewSubConn([]resolver.Address{addrs[0]}, balancer.NewSubConnOptions{ + StateListener: func(state balancer.SubConnState) { + bd.ClientConn.UpdateState(balancer.State{ConnectivityState: state.ConnectivityState, Picker: &aiPicker{result: balancer.PickResult{SubConn: sc}, err: state.ConnectionError}}) + }, + }) if err != nil { return err } sc.Connect() return nil }, - UpdateSubConnState: func(bd *stub.BalancerData, sc balancer.SubConn, state balancer.SubConnState) { - bd.ClientConn.UpdateState(balancer.State{ConnectivityState: state.ConnectivityState, Picker: &aiPicker{result: balancer.PickResult{SubConn: sc}, err: state.ConnectionError}}) - }, } stub.Register(attrBalancerName, bf) t.Logf("Registered balancer %s...", attrBalancerName) @@ -498,7 +418,7 @@ func (s) TestAddressAttributesInNewSubConn(t *testing.T) { } s := grpc.NewServer() - testpb.RegisterTestServiceService(s, testpb.NewTestServiceService(&testServer{})) + testgrpc.RegisterTestServiceServer(s, &testServer{}) go s.Serve(lis) defer s.Stop() t.Logf("Started gRPC server at %s...", lis.Addr().String()) @@ -514,11 +434,11 @@ func (s) TestAddressAttributesInNewSubConn(t *testing.T) { t.Fatal(err) } defer cc.Close() - tc := testpb.NewTestServiceClient(cc) + tc := testgrpc.NewTestServiceClient(cc) t.Log("Created a ClientConn...") // The first RPC should fail because there's no address. - ctx, cancel := context.WithTimeout(context.Background(), time.Second) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) defer cancel() if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err == nil || status.Code(err) != codes.DeadlineExceeded { t.Fatalf("EmptyCall() = _, %v, want _, DeadlineExceeded", err) @@ -530,7 +450,7 @@ func (s) TestAddressAttributesInNewSubConn(t *testing.T) { t.Logf("Pushing resolver state update: %v through the manual resolver", state) // The second RPC should succeed. - ctx, cancel = context.WithTimeout(context.Background(), time.Second) + ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil { t.Fatalf("EmptyCall() = _, %v, want _, ", err) @@ -543,10 +463,82 @@ func (s) TestAddressAttributesInNewSubConn(t *testing.T) { } } +// TestMetadataInAddressAttributes verifies that the metadata added to +// address.Attributes will be sent with the RPCs. +func (s) TestMetadataInAddressAttributes(t *testing.T) { + const ( + testMDKey = "test-md" + testMDValue = "test-md-value" + mdBalancerName = "metadata-balancer" + ) + + // Register a stub balancer which adds metadata to the first address that it + // receives and then calls NewSubConn on it. + bf := stub.BalancerFuncs{ + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + addrs := ccs.ResolverState.Addresses + if len(addrs) == 0 { + return nil + } + // Only use the first address. + var sc balancer.SubConn + sc, err := bd.ClientConn.NewSubConn([]resolver.Address{ + imetadata.Set(addrs[0], metadata.Pairs(testMDKey, testMDValue)), + }, balancer.NewSubConnOptions{ + StateListener: func(state balancer.SubConnState) { + bd.ClientConn.UpdateState(balancer.State{ConnectivityState: state.ConnectivityState, Picker: &aiPicker{result: balancer.PickResult{SubConn: sc}, err: state.ConnectionError}}) + }, + }) + if err != nil { + return err + } + sc.Connect() + return nil + }, + } + stub.Register(mdBalancerName, bf) + t.Logf("Registered balancer %s...", mdBalancerName) + + testMDChan := make(chan []string, 1) + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + md, ok := metadata.FromIncomingContext(ctx) + if ok { + select { + case testMDChan <- md[testMDKey]: + case <-ctx.Done(): + return nil, ctx.Err() + } + } + return &testpb.Empty{}, nil + }, + } + if err := ss.Start(nil, grpc.WithDefaultServiceConfig( + fmt.Sprintf(`{ "loadBalancingConfig": [{"%v": {}}] }`, mdBalancerName), + )); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + // The RPC should succeed with the expected md. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() = _, %v, want _, ", err) + } + t.Log("Made an RPC which succeeded...") + + // The server should receive the test metadata. + md1 := <-testMDChan + if len(md1) == 0 || md1[0] != testMDValue { + t.Fatalf("got md: %v, want %v", md1, []string{testMDValue}) + } +} + // TestServersSwap creates two servers and verifies the client switches between // them when the name resolver reports the first and then the second. func (s) TestServersSwap(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() // Initialize servers @@ -561,7 +553,7 @@ func (s) TestServersSwap(t *testing.T) { return &testpb.SimpleResponse{Username: username}, nil }, } - testpb.RegisterTestServiceService(s, testpb.NewTestServiceService(ts)) + testgrpc.RegisterTestServiceServer(s, ts) go s.Serve(lis) return lis.Addr().String(), s.Stop } @@ -575,12 +567,12 @@ func (s) TestServersSwap(t *testing.T) { // Initialize client r := manual.NewBuilderWithScheme("whatever") r.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: addr1}}}) - cc, err := grpc.DialContext(ctx, r.Scheme()+":///", grpc.WithInsecure(), grpc.WithResolvers(r)) + cc, err := grpc.DialContext(ctx, r.Scheme()+":///", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) if err != nil { t.Fatalf("Error creating client: %v", err) } defer cc.Close() - client := testpb.NewTestServiceClient(cc) + client := testgrpc.NewTestServiceClient(cc) // Confirm we are connected to the first server if res, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil || res.Username != one { @@ -601,11 +593,8 @@ func (s) TestServersSwap(t *testing.T) { } } -// TestEmptyAddrs verifies client behavior when a working connection is -// removed. In pick first and round-robin, both will continue using the old -// connections. -func (s) TestEmptyAddrs(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) +func (s) TestWaitForReady(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() // Initialize server @@ -621,135 +610,520 @@ func (s) TestEmptyAddrs(t *testing.T) { return &testpb.SimpleResponse{Username: one}, nil }, } - testpb.RegisterTestServiceService(s, testpb.NewTestServiceService(ts)) + testgrpc.RegisterTestServiceServer(s, ts) go s.Serve(lis) - // Initialize pickfirst client - pfr := manual.NewBuilderWithScheme("whatever") - pfrnCalled := grpcsync.NewEvent() - pfr.ResolveNowCallback = func(resolver.ResolveNowOptions) { - pfrnCalled.Fire() - } - pfr.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: lis.Addr().String()}}}) + // Initialize client + r := manual.NewBuilderWithScheme("whatever") - pfcc, err := grpc.DialContext(ctx, pfr.Scheme()+":///", grpc.WithInsecure(), grpc.WithResolvers(pfr)) + cc, err := grpc.DialContext(ctx, r.Scheme()+":///", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) if err != nil { t.Fatalf("Error creating client: %v", err) } - defer pfcc.Close() - pfclient := testpb.NewTestServiceClient(pfcc) + defer cc.Close() + client := testgrpc.NewTestServiceClient(cc) - // Confirm we are connected to the server - if res, err := pfclient.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil || res.Username != one { - t.Fatalf("UnaryCall(_) = %v, %v; want {Username: %q}, nil", res, err, one) - } + // Report an error so non-WFR RPCs will give up early. + r.CC.ReportError(errors.New("fake resolver error")) - // Remove all addresses. - pfr.UpdateState(resolver.State{}) - // Wait for a ResolveNow call on the pick first client's resolver. - <-pfrnCalled.Done() + // Ensure the client is not connected to anything and fails non-WFR RPCs. + if res, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}); status.Code(err) != codes.Unavailable { + t.Fatalf("UnaryCall(_) = %v, %v; want _, Code()=%v", res, err, codes.Unavailable) + } - // Initialize roundrobin client - rrr := manual.NewBuilderWithScheme("whatever") + errChan := make(chan error, 1) + go func() { + if res, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}, grpc.WaitForReady(true)); err != nil || res.Username != one { + errChan <- fmt.Errorf("UnaryCall(_) = %v, %v; want {Username: %q}, nil", res, err, one) + } + close(errChan) + }() - rrrnCalled := grpcsync.NewEvent() - rrr.ResolveNowCallback = func(resolver.ResolveNowOptions) { - rrrnCalled.Fire() + select { + case err := <-errChan: + t.Errorf("unexpected receive from errChan before addresses provided") + t.Fatal(err.Error()) + case <-time.After(5 * time.Millisecond): } - rrr.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: lis.Addr().String()}}}) - rrcc, err := grpc.DialContext(ctx, rrr.Scheme()+":///", grpc.WithInsecure(), grpc.WithResolvers(rrr), - grpc.WithDefaultServiceConfig(fmt.Sprintf(`{ "loadBalancingConfig": [{"%v": {}}] }`, roundrobin.Name))) - if err != nil { - t.Fatalf("Error creating client: %v", err) - } - defer rrcc.Close() - rrclient := testpb.NewTestServiceClient(rrcc) + // Resolve the server. The WFR RPC should unblock and use it. + r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: lis.Addr().String()}}}) - // Confirm we are connected to the server - if res, err := rrclient.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil || res.Username != one { - t.Fatalf("UnaryCall(_) = %v, %v; want {Username: %q}, nil", res, err, one) + if err := <-errChan; err != nil { + t.Fatal(err.Error()) } +} - // Remove all addresses. - rrr.UpdateState(resolver.State{}) - // Wait for a ResolveNow call on the round robin client's resolver. - <-rrrnCalled.Done() +// authorityOverrideTransportCreds returns the configured authority value in its +// Info() method. +type authorityOverrideTransportCreds struct { + credentials.TransportCredentials + authorityOverride string +} - // Confirm several new RPCs succeed on pick first. - for i := 0; i < 10; i++ { - if _, err := pfclient.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil { - t.Fatalf("UnaryCall(_) = _, %v; want _, nil", err) - } - time.Sleep(5 * time.Millisecond) +func (ao *authorityOverrideTransportCreds) ClientHandshake(ctx context.Context, addr string, rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) { + return rawConn, nil, nil +} +func (ao *authorityOverrideTransportCreds) Info() credentials.ProtocolInfo { + return credentials.ProtocolInfo{ServerName: ao.authorityOverride} +} +func (ao *authorityOverrideTransportCreds) Clone() credentials.TransportCredentials { + return &authorityOverrideTransportCreds{authorityOverride: ao.authorityOverride} +} + +// TestAuthorityInBuildOptions tests that the Authority field in +// balancer.BuildOptions is setup correctly from gRPC. +func (s) TestAuthorityInBuildOptions(t *testing.T) { + const dialTarget = "test.server" + + tests := []struct { + name string + dopts []grpc.DialOption + wantAuthority string + }{ + { + name: "authority from dial target", + dopts: []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())}, + wantAuthority: dialTarget, + }, + { + name: "authority from dial option", + dopts: []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithAuthority("authority-override"), + }, + wantAuthority: "authority-override", + }, + { + name: "authority from transport creds", + dopts: []grpc.DialOption{grpc.WithTransportCredentials(&authorityOverrideTransportCreds{authorityOverride: "authority-override-from-transport-creds"})}, + wantAuthority: "authority-override-from-transport-creds", + }, } - // Confirm several new RPCs succeed on round robin. - for i := 0; i < 10; i++ { - if _, err := pfclient.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil { - t.Fatalf("UnaryCall(_) = _, %v; want _, nil", err) - } - time.Sleep(5 * time.Millisecond) + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + authorityCh := make(chan string, 1) + bf := stub.BalancerFuncs{ + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + select { + case authorityCh <- bd.BuildOptions.Authority: + default: + } + + addrs := ccs.ResolverState.Addresses + if len(addrs) == 0 { + return nil + } + + // Only use the first address. + var sc balancer.SubConn + sc, err := bd.ClientConn.NewSubConn([]resolver.Address{addrs[0]}, balancer.NewSubConnOptions{ + StateListener: func(state balancer.SubConnState) { + bd.ClientConn.UpdateState(balancer.State{ConnectivityState: state.ConnectivityState, Picker: &aiPicker{result: balancer.PickResult{SubConn: sc}, err: state.ConnectionError}}) + }, + }) + if err != nil { + return err + } + sc.Connect() + return nil + }, + } + balancerName := "stub-balancer-" + test.name + stub.Register(balancerName, bf) + t.Logf("Registered balancer %s...", balancerName) + + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatal(err) + } + + s := grpc.NewServer() + testgrpc.RegisterTestServiceServer(s, &testServer{}) + go s.Serve(lis) + defer s.Stop() + t.Logf("Started gRPC server at %s...", lis.Addr().String()) + + r := manual.NewBuilderWithScheme("whatever") + t.Logf("Registered manual resolver with scheme %s...", r.Scheme()) + r.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: lis.Addr().String()}}}) + + dopts := append([]grpc.DialOption{ + grpc.WithResolvers(r), + grpc.WithDefaultServiceConfig(fmt.Sprintf(`{ "loadBalancingConfig": [{"%v": {}}] }`, balancerName)), + }, test.dopts...) + cc, err := grpc.Dial(r.Scheme()+":///"+dialTarget, dopts...) + if err != nil { + t.Fatal(err) + } + defer cc.Close() + tc := testgrpc.NewTestServiceClient(cc) + t.Log("Created a ClientConn...") + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() = _, %v, want _, ", err) + } + t.Log("Made an RPC which succeeded...") + + select { + case <-ctx.Done(): + t.Fatal("timeout when waiting for Authority in balancer.BuildOptions") + case gotAuthority := <-authorityCh: + if gotAuthority != test.wantAuthority { + t.Fatalf("Authority in balancer.BuildOptions is %s, want %s", gotAuthority, test.wantAuthority) + } + } + }) } } -func (s) TestWaitForReady(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() +// testCCWrapper wraps a balancer.ClientConn and intercepts UpdateState and +// returns a custom picker which injects arbitrary metadata on a per-call basis. +type testCCWrapper struct { + balancer.ClientConn +} - // Initialize server - lis, err := net.Listen("tcp", "localhost:0") +func (t *testCCWrapper) UpdateState(state balancer.State) { + state.Picker = &wrappedPicker{p: state.Picker} + t.ClientConn.UpdateState(state) +} + +const ( + metadataHeaderInjectedByBalancer = "metadata-header-injected-by-balancer" + metadataHeaderInjectedByApplication = "metadata-header-injected-by-application" + metadataValueInjectedByBalancer = "metadata-value-injected-by-balancer" + metadataValueInjectedByApplication = "metadata-value-injected-by-application" +) + +// wrappedPicker wraps the picker returned by the pick_first +type wrappedPicker struct { + p balancer.Picker +} + +func (wp *wrappedPicker) Pick(info balancer.PickInfo) (balancer.PickResult, error) { + res, err := wp.p.Pick(info) if err != nil { - t.Fatalf("Error while listening. Err: %v", err) + return balancer.PickResult{}, err } - s := grpc.NewServer() - defer s.Stop() - const one = "1" - ts := &funcServer{ - unaryCall: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { - return &testpb.SimpleResponse{Username: one}, nil + + if res.Metadata == nil { + res.Metadata = metadata.Pairs(metadataHeaderInjectedByBalancer, metadataValueInjectedByBalancer) + } else { + res.Metadata.Append(metadataHeaderInjectedByBalancer, metadataValueInjectedByBalancer) + } + return res, nil +} + +// TestMetadataInPickResult tests the scenario where an LB policy inject +// arbitrary metadata on a per-call basis and verifies that the injected +// metadata makes it all the way to the server RPC handler. +func (s) TestMetadataInPickResult(t *testing.T) { + t.Log("Starting test backend...") + mdChan := make(chan metadata.MD, 1) + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + md, _ := metadata.FromIncomingContext(ctx) + select { + case mdChan <- md: + case <-ctx.Done(): + return nil, ctx.Err() + } + return &testpb.Empty{}, nil }, } - testpb.RegisterTestServiceService(s, testpb.NewTestServiceService(ts)) - go s.Serve(lis) + if err := ss.StartServer(); err != nil { + t.Fatalf("Starting test backend: %v", err) + } + defer ss.Stop() + t.Logf("Started test backend at %q", ss.Address) + + // Register a test balancer that contains a pick_first balancer and forwards + // all calls from the ClientConn to it. For state updates from the + // pick_first balancer, it creates a custom picker which injects arbitrary + // metadata on a per-call basis. + stub.Register(t.Name(), stub.BalancerFuncs{ + Init: func(bd *stub.BalancerData) { + cc := &testCCWrapper{ClientConn: bd.ClientConn} + bd.Data = balancer.Get(grpc.PickFirstBalancerName).Build(cc, bd.BuildOptions) + }, + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + bal := bd.Data.(balancer.Balancer) + return bal.UpdateClientConnState(ccs) + }, + }) - // Initialize client + t.Log("Creating ClientConn to test backend...") r := manual.NewBuilderWithScheme("whatever") - - cc, err := grpc.DialContext(ctx, r.Scheme()+":///", grpc.WithInsecure(), grpc.WithResolvers(r)) + r.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: ss.Address}}}) + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(r), + grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, t.Name())), + } + cc, err := grpc.Dial(r.Scheme()+":///test.server", dopts...) if err != nil { - t.Fatalf("Error creating client: %v", err) + t.Fatalf("grpc.Dial(): %v", err) } defer cc.Close() - client := testpb.NewTestServiceClient(cc) + tc := testgrpc.NewTestServiceClient(cc) - // Report an error so non-WFR RPCs will give up early. - r.CC.ReportError(errors.New("fake resolver error")) + t.Log("Making EmptyCall() RPC with custom metadata...") + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + md := metadata.Pairs(metadataHeaderInjectedByApplication, metadataValueInjectedByApplication) + ctx = metadata.NewOutgoingContext(ctx, md) + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() RPC: %v", err) + } + t.Log("EmptyCall() RPC succeeded") - // Ensure the client is not connected to anything and fails non-WFR RPCs. - if res, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}); status.Code(err) != codes.Unavailable { - t.Fatalf("UnaryCall(_) = %v, %v; want _, Code()=%v", res, err, codes.Unavailable) + t.Log("Waiting for custom metadata to be received at the test backend...") + var gotMD metadata.MD + select { + case gotMD = <-mdChan: + case <-ctx.Done(): + t.Fatalf("Timed out waiting for custom metadata to be received at the test backend") } - errChan := make(chan error, 1) + t.Log("Verifying custom metadata added by the client application is received at the test backend...") + wantMDVal := []string{metadataValueInjectedByApplication} + gotMDVal := gotMD.Get(metadataHeaderInjectedByApplication) + if !cmp.Equal(gotMDVal, wantMDVal) { + t.Fatalf("Mismatch in custom metadata received at test backend, got: %v, want %v", gotMDVal, wantMDVal) + } + + t.Log("Verifying custom metadata added by the LB policy is received at the test backend...") + wantMDVal = []string{metadataValueInjectedByBalancer} + gotMDVal = gotMD.Get(metadataHeaderInjectedByBalancer) + if !cmp.Equal(gotMDVal, wantMDVal) { + t.Fatalf("Mismatch in custom metadata received at test backend, got: %v, want %v", gotMDVal, wantMDVal) + } +} + +// producerTestBalancerBuilder and producerTestBalancer start a producer which +// makes an RPC before the subconn is READY, then connects the subconn, and +// pushes the resulting error (expected to be nil) to rpcErrChan. +type producerTestBalancerBuilder struct { + rpcErrChan chan error + ctxChan chan context.Context + connect bool +} + +func (bb *producerTestBalancerBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { + return &producerTestBalancer{cc: cc, rpcErrChan: bb.rpcErrChan, ctxChan: bb.ctxChan, connect: bb.connect} +} + +const producerTestBalancerName = "producer_test_balancer" + +func (bb *producerTestBalancerBuilder) Name() string { return producerTestBalancerName } + +type producerTestBalancer struct { + cc balancer.ClientConn + rpcErrChan chan error + ctxChan chan context.Context + connect bool +} + +func (b *producerTestBalancer) UpdateClientConnState(ccs balancer.ClientConnState) error { + // Create the subconn, but don't connect it. + sc, err := b.cc.NewSubConn(ccs.ResolverState.Addresses, balancer.NewSubConnOptions{}) + if err != nil { + return fmt.Errorf("error creating subconn: %v", err) + } + + // Create the producer. This will call the producer builder's Build + // method, which will try to start an RPC in a goroutine. + p := &testProducerBuilder{start: grpcsync.NewEvent(), rpcErrChan: b.rpcErrChan, ctxChan: b.ctxChan} + sc.GetOrBuildProducer(p) + + // Wait here until the producer is about to perform the RPC, which should + // block until connected. + <-p.start.Done() + + // Ensure the error chan doesn't get anything on it before we connect the + // subconn. + select { + case err := <-b.rpcErrChan: + go func() { b.rpcErrChan <- fmt.Errorf("Got unexpected data on rpcErrChan: %v", err) }() + default: + } + + if b.connect { + // Now we can connect, which will unblock the RPC above. + sc.Connect() + } + + // The stub server requires a READY picker to be reported, to unblock its + // Start method. We won't make RPCs in our test, so a nil picker is okay. + b.cc.UpdateState(balancer.State{ConnectivityState: connectivity.Ready, Picker: nil}) + return nil +} + +func (b *producerTestBalancer) ResolverError(err error) { + panic(fmt.Sprintf("Unexpected resolver error: %v", err)) +} + +func (b *producerTestBalancer) UpdateSubConnState(balancer.SubConn, balancer.SubConnState) {} +func (b *producerTestBalancer) Close() {} + +type testProducerBuilder struct { + start *grpcsync.Event + rpcErrChan chan error + ctxChan chan context.Context +} + +func (b *testProducerBuilder) Build(cci any) (balancer.Producer, func()) { + c := testgrpc.NewTestServiceClient(cci.(grpc.ClientConnInterface)) + // Perform the RPC in a goroutine instead of during build because the + // subchannel's mutex is held here. go func() { - if res, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}, grpc.WaitForReady(true)); err != nil || res.Username != one { - errChan <- fmt.Errorf("UnaryCall(_) = %v, %v; want {Username: %q}, nil", res, err, one) - } - close(errChan) + ctx := <-b.ctxChan + b.start.Fire() + _, err := c.EmptyCall(ctx, &testpb.Empty{}) + b.rpcErrChan <- err }() + return nil, func() {} +} - select { - case err := <-errChan: - t.Errorf("unexpected receive from errChan before addresses provided") - t.Fatal(err.Error()) - case <-time.After(5 * time.Millisecond): +// TestBalancerProducerBlockUntilReady tests that we get no RPC errors from +// producers when subchannels aren't ready. +func (s) TestBalancerProducerBlockUntilReady(t *testing.T) { + // rpcErrChan is given to the LB policy to report the status of the + // producer's one RPC. + ctxChan := make(chan context.Context, 1) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctxChan <- ctx + + rpcErrChan := make(chan error) + balancer.Register(&producerTestBalancerBuilder{rpcErrChan: rpcErrChan, ctxChan: ctxChan, connect: true}) + + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, } - // Resolve the server. The WFR RPC should unblock and use it. - r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: lis.Addr().String()}}}) + // Start the server & client with the test producer LB policy. + svcCfg := fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, producerTestBalancerName) + if err := ss.Start(nil, grpc.WithDefaultServiceConfig(svcCfg)); err != nil { + t.Fatalf("Error starting testing server: %v", err) + } + defer ss.Stop() - if err := <-errChan; err != nil { - t.Fatal(err.Error()) + // Receive the error from the producer's RPC, which should be nil. + if err := <-rpcErrChan; err != nil { + t.Fatalf("Received unexpected error from producer RPC: %v", err) + } +} + +// TestBalancerProducerHonorsContext tests that producers that perform RPC get +// context errors correctly. +func (s) TestBalancerProducerHonorsContext(t *testing.T) { + // rpcErrChan is given to the LB policy to report the status of the + // producer's one RPC. + ctxChan := make(chan context.Context, 1) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + ctxChan <- ctx + + rpcErrChan := make(chan error) + balancer.Register(&producerTestBalancerBuilder{rpcErrChan: rpcErrChan, ctxChan: ctxChan, connect: false}) + + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + } + + // Start the server & client with the test producer LB policy. + svcCfg := fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, producerTestBalancerName) + if err := ss.Start(nil, grpc.WithDefaultServiceConfig(svcCfg)); err != nil { + t.Fatalf("Error starting testing server: %v", err) + } + defer ss.Stop() + + cancel() + + // Receive the error from the producer's RPC, which should be canceled. + if err := <-rpcErrChan; status.Code(err) != codes.Canceled { + t.Fatalf("RPC error: %v; want status.Code(err)=%v", err, codes.Canceled) + } +} + +// TestSubConnShutdown confirms that the Shutdown method on subconns and +// RemoveSubConn method on ClientConn properly initiates subconn shutdown. +func (s) TestSubConnShutdown(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + testCases := []struct { + name string + shutdown func(cc balancer.ClientConn, sc balancer.SubConn) + }{{ + name: "ClientConn.RemoveSubConn", + shutdown: func(cc balancer.ClientConn, sc balancer.SubConn) { + cc.RemoveSubConn(sc) + }, + }, { + name: "SubConn.Shutdown", + shutdown: func(_ balancer.ClientConn, sc balancer.SubConn) { + sc.Shutdown() + }, + }} + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + gotShutdown := grpcsync.NewEvent() + + bf := stub.BalancerFuncs{ + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + var sc balancer.SubConn + opts := balancer.NewSubConnOptions{ + StateListener: func(scs balancer.SubConnState) { + switch scs.ConnectivityState { + case connectivity.Connecting: + // Ignored. + case connectivity.Ready: + tc.shutdown(bd.ClientConn, sc) + case connectivity.Shutdown: + gotShutdown.Fire() + default: + t.Errorf("got unexpected state %q in listener", scs.ConnectivityState) + } + }, + } + sc, err := bd.ClientConn.NewSubConn(ccs.ResolverState.Addresses, opts) + if err != nil { + return err + } + sc.Connect() + // Report the state as READY to unblock ss.Start(), which waits for ready. + bd.ClientConn.UpdateState(balancer.State{ConnectivityState: connectivity.Ready}) + return nil + }, + } + + testBalName := "shutdown-test-balancer-" + tc.name + stub.Register(testBalName, bf) + t.Logf("Registered balancer %s...", testBalName) + + ss := &stubserver.StubServer{} + if err := ss.Start(nil, grpc.WithDefaultServiceConfig( + fmt.Sprintf(`{ "loadBalancingConfig": [{"%v": {}}] }`, testBalName), + )); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + select { + case <-gotShutdown.Done(): + // Success + case <-ctx.Done(): + t.Fatalf("Timed out waiting for gotShutdown to be fired.") + } + }) } } diff --git a/test/bufconn/bufconn.go b/test/bufconn/bufconn.go index 168cdb8578dd..3f77f4876eb8 100644 --- a/test/bufconn/bufconn.go +++ b/test/bufconn/bufconn.go @@ -21,6 +21,7 @@ package bufconn import ( + "context" "fmt" "io" "net" @@ -86,8 +87,17 @@ func (l *Listener) Addr() net.Addr { return addr{} } // providing it the server half of the connection, and returns the client half // of the connection. func (l *Listener) Dial() (net.Conn, error) { + return l.DialContext(context.Background()) +} + +// DialContext creates an in-memory full-duplex network connection, unblocks Accept by +// providing it the server half of the connection, and returns the client half +// of the connection. If ctx is Done, returns ctx.Err() +func (l *Listener) DialContext(ctx context.Context) (net.Conn, error) { p1, p2 := newPipe(l.sz), newPipe(l.sz) select { + case <-ctx.Done(): + return nil, ctx.Err() case <-l.done: return nil, errClosed case l.ch <- &conn{p1, p2}: diff --git a/test/channelz_linux_go110_test.go b/test/channelz_linux_test.go similarity index 87% rename from test/channelz_linux_go110_test.go rename to test/channelz_linux_test.go index dea374bfc08b..d5b691c1d83e 100644 --- a/test/channelz_linux_go110_test.go +++ b/test/channelz_linux_test.go @@ -1,5 +1,3 @@ -// +build linux - /* * * Copyright 2018 gRPC authors. @@ -18,10 +16,6 @@ * */ -// The test in this file should be run in an environment that has go1.10 or later, -// as the function SyscallConn() (required to get socket option) was -// introduced to net.TCPListener in go1.10. - package test import ( @@ -29,7 +23,8 @@ import ( "time" "google.golang.org/grpc/internal/channelz" - testpb "google.golang.org/grpc/test/grpc_testing" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" ) func (s) TestCZSocketMetricsSocketOption(t *testing.T) { @@ -40,13 +35,11 @@ func (s) TestCZSocketMetricsSocketOption(t *testing.T) { } func testCZSocketMetricsSocketOption(t *testing.T, e env) { - czCleanup := channelz.NewChannelzStorage() - defer czCleanupWrapper(czCleanup, t) te := newTest(t, e) te.startServer(&testServer{security: e.security}) defer te.tearDown() cc := te.clientConn() - tc := testpb.NewTestServiceClient(cc) + tc := testgrpc.NewTestServiceClient(cc) doSuccessfulUnaryCall(tc, t) time.Sleep(10 * time.Millisecond) diff --git a/test/channelz_test.go b/test/channelz_test.go index db510d4c6df5..b23acf4bdc1d 100644 --- a/test/channelz_test.go +++ b/test/channelz_test.go @@ -23,34 +23,34 @@ import ( "crypto/tls" "fmt" "net" - "reflect" + "regexp" "strings" "sync" "testing" "time" + "github.com/google/go-cmp/cmp" "golang.org/x/net/http2" "google.golang.org/grpc" _ "google.golang.org/grpc/balancer/grpclb" + grpclbstate "google.golang.org/grpc/balancer/grpclb/state" "google.golang.org/grpc/balancer/roundrobin" "google.golang.org/grpc/codes" "google.golang.org/grpc/connectivity" "google.golang.org/grpc/credentials" "google.golang.org/grpc/internal" "google.golang.org/grpc/internal/channelz" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" "google.golang.org/grpc/keepalive" "google.golang.org/grpc/resolver" "google.golang.org/grpc/resolver/manual" "google.golang.org/grpc/status" - testpb "google.golang.org/grpc/test/grpc_testing" "google.golang.org/grpc/testdata" -) -func czCleanupWrapper(cleanup func() error, t *testing.T) { - if err := cleanup(); err != nil { - t.Error(err) - } -} + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) func verifyResultWithDelay(f func() (bool, error)) error { var ok bool @@ -80,28 +80,27 @@ func (s) TestCZServerRegistrationAndDeletion(t *testing.T) { {total: int(channelz.EntryPerPage), start: 0, max: channelz.EntryPerPage - 1, length: channelz.EntryPerPage - 1, end: false}, } - for _, c := range testcases { - czCleanup := channelz.NewChannelzStorage() - defer czCleanupWrapper(czCleanup, t) + for i, c := range testcases { + // Reset channelz IDs so `start` is valid. + channelz.IDGen.Reset() + e := tcpClearRREnv te := newTest(t, e) te.startServers(&testServer{security: e.security}, c.total) ss, end := channelz.GetServers(c.start, c.max) if int64(len(ss)) != c.length || end != c.end { - t.Fatalf("GetServers(%d) = %+v (len of which: %d), end: %+v, want len(GetServers(%d)) = %d, end: %+v", c.start, ss, len(ss), end, c.start, c.length, c.end) + t.Fatalf("%d: GetServers(%d) = %+v (len of which: %d), end: %+v, want len(GetServers(%d)) = %d, end: %+v", i, c.start, ss, len(ss), end, c.start, c.length, c.end) } te.tearDown() ss, end = channelz.GetServers(c.start, c.max) if len(ss) != 0 || !end { - t.Fatalf("GetServers(0) = %+v (len of which: %d), end: %+v, want len(GetServers(0)) = 0, end: true", ss, len(ss), end) + t.Fatalf("%d: GetServers(0) = %+v (len of which: %d), end: %+v, want len(GetServers(0)) = 0, end: true", i, ss, len(ss), end) } } } func (s) TestCZGetServer(t *testing.T) { - czCleanup := channelz.NewChannelzStorage() - defer czCleanupWrapper(czCleanup, t) e := tcpClearRREnv te := newTest(t, e) te.startServer(&testServer{security: e.security}) @@ -152,8 +151,9 @@ func (s) TestCZTopChannelRegistrationAndDeletion(t *testing.T) { } for _, c := range testcases { - czCleanup := channelz.NewChannelzStorage() - defer czCleanupWrapper(czCleanup, t) + // Reset channelz IDs so `start` is valid. + channelz.IDGen.Reset() + e := tcpClearRREnv te := newTest(t, e) var ccs []*grpc.ClientConn @@ -190,8 +190,6 @@ func (s) TestCZTopChannelRegistrationAndDeletion(t *testing.T) { } func (s) TestCZTopChannelRegistrationAndDeletionWhenDialFail(t *testing.T) { - czCleanup := channelz.NewChannelzStorage() - defer czCleanupWrapper(czCleanup, t) // Make dial fails (due to no transport security specified) _, err := grpc.Dial("fake.addr") if err == nil { @@ -203,17 +201,16 @@ func (s) TestCZTopChannelRegistrationAndDeletionWhenDialFail(t *testing.T) { } func (s) TestCZNestedChannelRegistrationAndDeletion(t *testing.T) { - czCleanup := channelz.NewChannelzStorage() - defer czCleanupWrapper(czCleanup, t) e := tcpClearRREnv // avoid calling API to set balancer type, which will void service config's change of balancer. e.balancer = "" te := newTest(t, e) r := manual.NewBuilderWithScheme("whatever") - resolvedAddrs := []resolver.Address{{Addr: "127.0.0.1:0", Type: resolver.GRPCLB, ServerName: "grpclb.server"}} - r.InitialState(resolver.State{Addresses: resolvedAddrs}) te.resolverScheme = r.Scheme() te.clientConn(grpc.WithResolvers(r)) + resolvedAddrs := []resolver.Address{{Addr: "127.0.0.1:0", ServerName: "grpclb.server"}} + grpclbConfig := parseServiceConfig(t, r, `{"loadBalancingPolicy": "grpclb"}`) + r.UpdateState(grpclbstate.Set(resolver.State{ServiceConfig: grpclbConfig}, &grpclbstate.State{BalancerAddresses: resolvedAddrs})) defer te.tearDown() if err := verifyResultWithDelay(func() (bool, error) { @@ -229,7 +226,10 @@ func (s) TestCZNestedChannelRegistrationAndDeletion(t *testing.T) { t.Fatal(err) } - r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: "127.0.0.1:0"}}, ServiceConfig: parseCfg(r, `{"loadBalancingPolicy": "round_robin"}`)}) + r.UpdateState(resolver.State{ + Addresses: []resolver.Address{{Addr: "127.0.0.1:0"}}, + ServiceConfig: parseServiceConfig(t, r, `{"loadBalancingPolicy": "round_robin"}`), + }) // wait for the shutdown of grpclb balancer if err := verifyResultWithDelay(func() (bool, error) { @@ -247,8 +247,6 @@ func (s) TestCZNestedChannelRegistrationAndDeletion(t *testing.T) { } func (s) TestCZClientSubChannelSocketRegistrationAndDeletion(t *testing.T) { - czCleanup := channelz.NewChannelzStorage() - defer czCleanupWrapper(czCleanup, t) e := tcpClearRREnv num := 3 // number of backends te := newTest(t, e) @@ -335,8 +333,9 @@ func (s) TestCZServerSocketRegistrationAndDeletion(t *testing.T) { } for _, c := range testcases { - czCleanup := channelz.NewChannelzStorage() - defer czCleanupWrapper(czCleanup, t) + // Reset channelz IDs so `start` is valid. + channelz.IDGen.Reset() + e := tcpClearRREnv te := newTest(t, e) te.startServer(&testServer{security: e.security}) @@ -395,8 +394,6 @@ func (s) TestCZServerSocketRegistrationAndDeletion(t *testing.T) { } func (s) TestCZServerListenSocketDeletion(t *testing.T) { - czCleanup := channelz.NewChannelzStorage() - defer czCleanupWrapper(czCleanup, t) s := grpc.NewServer() lis, err := net.Listen("tcp", "localhost:0") if err != nil { @@ -452,13 +449,12 @@ func (s) TestCZRecusivelyDeletionOfEntry(t *testing.T) { // | | // v v // Socket1 Socket2 - czCleanup := channelz.NewChannelzStorage() - defer czCleanupWrapper(czCleanup, t) - topChanID := channelz.RegisterChannel(&dummyChannel{}, 0, "") - subChanID1 := channelz.RegisterSubChannel(&dummyChannel{}, topChanID, "") - subChanID2 := channelz.RegisterSubChannel(&dummyChannel{}, topChanID, "") - sktID1 := channelz.RegisterNormalSocket(&dummySocket{}, subChanID1, "") - sktID2 := channelz.RegisterNormalSocket(&dummySocket{}, subChanID1, "") + + topChanID := channelz.RegisterChannel(&dummyChannel{}, nil, "") + subChanID1, _ := channelz.RegisterSubChannel(&dummyChannel{}, topChanID, "") + subChanID2, _ := channelz.RegisterSubChannel(&dummyChannel{}, topChanID, "") + sktID1, _ := channelz.RegisterNormalSocket(&dummySocket{}, subChanID1, "") + sktID2, _ := channelz.RegisterNormalSocket(&dummySocket{}, subChanID1, "") tcs, _ := channelz.GetTopChannels(0, 0) if tcs == nil || len(tcs) != 1 { @@ -467,7 +463,7 @@ func (s) TestCZRecusivelyDeletionOfEntry(t *testing.T) { if len(tcs[0].SubChans) != 2 { t.Fatalf("There should be two SubChannel entries") } - sc := channelz.GetSubChannel(subChanID1) + sc := channelz.GetSubChannel(subChanID1.Int()) if sc == nil || len(sc.Sockets) != 2 { t.Fatalf("There should be two Socket entries") } @@ -497,8 +493,6 @@ func (s) TestCZRecusivelyDeletionOfEntry(t *testing.T) { } func (s) TestCZChannelMetrics(t *testing.T) { - czCleanup := channelz.NewChannelzStorage() - defer czCleanupWrapper(czCleanup, t) e := tcpClearRREnv num := 3 // number of backends te := newTest(t, e) @@ -513,8 +507,10 @@ func (s) TestCZChannelMetrics(t *testing.T) { te.resolverScheme = r.Scheme() cc := te.clientConn(grpc.WithResolvers(r)) defer te.tearDown() - tc := testpb.NewTestServiceClient(cc) - if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); err != nil { + tc := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil { t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, ", err) } @@ -531,11 +527,11 @@ func (s) TestCZChannelMetrics(t *testing.T) { Payload: largePayload, } - if _, err := tc.UnaryCall(context.Background(), req); err == nil || status.Code(err) != codes.ResourceExhausted { + if _, err := tc.UnaryCall(ctx, req); err == nil || status.Code(err) != codes.ResourceExhausted { t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted) } - stream, err := tc.FullDuplexCall(context.Background()) + stream, err := tc.FullDuplexCall(ctx) if err != nil { t.Fatalf("%v.FullDuplexCall(_) = _, %v, want ", tc, err) } @@ -585,16 +581,16 @@ func (s) TestCZChannelMetrics(t *testing.T) { } func (s) TestCZServerMetrics(t *testing.T) { - czCleanup := channelz.NewChannelzStorage() - defer czCleanupWrapper(czCleanup, t) e := tcpClearRREnv te := newTest(t, e) te.maxServerReceiveMsgSize = newInt(8) te.startServer(&testServer{security: e.security}) defer te.tearDown() cc := te.clientConn() - tc := testpb.NewTestServiceClient(cc) - if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); err != nil { + tc := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil { t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, ", err) } @@ -610,11 +606,11 @@ func (s) TestCZServerMetrics(t *testing.T) { ResponseSize: int32(smallSize), Payload: largePayload, } - if _, err := tc.UnaryCall(context.Background(), req); err == nil || status.Code(err) != codes.ResourceExhausted { + if _, err := tc.UnaryCall(ctx, req); err == nil || status.Code(err) != codes.ResourceExhausted { t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted) } - stream, err := tc.FullDuplexCall(context.Background()) + stream, err := tc.FullDuplexCall(ctx) if err != nil { t.Fatalf("%v.FullDuplexCall(_) = _, %v, want ", tc, err) } @@ -641,7 +637,7 @@ func (s) TestCZServerMetrics(t *testing.T) { } type testServiceClientWrapper struct { - testpb.TestServiceClient + testgrpc.TestServiceClient mu sync.RWMutex streamsCreated int } @@ -666,42 +662,46 @@ func (t *testServiceClientWrapper) UnaryCall(ctx context.Context, in *testpb.Sim return t.TestServiceClient.UnaryCall(ctx, in, opts...) } -func (t *testServiceClientWrapper) StreamingOutputCall(ctx context.Context, in *testpb.StreamingOutputCallRequest, opts ...grpc.CallOption) (testpb.TestService_StreamingOutputCallClient, error) { +func (t *testServiceClientWrapper) StreamingOutputCall(ctx context.Context, in *testpb.StreamingOutputCallRequest, opts ...grpc.CallOption) (testgrpc.TestService_StreamingOutputCallClient, error) { t.mu.Lock() defer t.mu.Unlock() t.streamsCreated++ return t.TestServiceClient.StreamingOutputCall(ctx, in, opts...) } -func (t *testServiceClientWrapper) StreamingInputCall(ctx context.Context, opts ...grpc.CallOption) (testpb.TestService_StreamingInputCallClient, error) { +func (t *testServiceClientWrapper) StreamingInputCall(ctx context.Context, opts ...grpc.CallOption) (testgrpc.TestService_StreamingInputCallClient, error) { t.mu.Lock() defer t.mu.Unlock() t.streamsCreated++ return t.TestServiceClient.StreamingInputCall(ctx, opts...) } -func (t *testServiceClientWrapper) FullDuplexCall(ctx context.Context, opts ...grpc.CallOption) (testpb.TestService_FullDuplexCallClient, error) { +func (t *testServiceClientWrapper) FullDuplexCall(ctx context.Context, opts ...grpc.CallOption) (testgrpc.TestService_FullDuplexCallClient, error) { t.mu.Lock() defer t.mu.Unlock() t.streamsCreated++ return t.TestServiceClient.FullDuplexCall(ctx, opts...) } -func (t *testServiceClientWrapper) HalfDuplexCall(ctx context.Context, opts ...grpc.CallOption) (testpb.TestService_HalfDuplexCallClient, error) { +func (t *testServiceClientWrapper) HalfDuplexCall(ctx context.Context, opts ...grpc.CallOption) (testgrpc.TestService_HalfDuplexCallClient, error) { t.mu.Lock() defer t.mu.Unlock() t.streamsCreated++ return t.TestServiceClient.HalfDuplexCall(ctx, opts...) } -func doSuccessfulUnaryCall(tc testpb.TestServiceClient, t *testing.T) { - if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); err != nil { +func doSuccessfulUnaryCall(tc testgrpc.TestServiceClient, t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil { t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, ", err) } } -func doStreamingInputCallWithLargePayload(tc testpb.TestServiceClient, t *testing.T) { - s, err := tc.StreamingInputCall(context.Background()) +func doStreamingInputCallWithLargePayload(tc testgrpc.TestServiceClient, t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + s, err := tc.StreamingInputCall(ctx) if err != nil { t.Fatalf("TestService/StreamingInputCall(_) = _, %v, want ", err) } @@ -712,7 +712,7 @@ func doStreamingInputCallWithLargePayload(tc testpb.TestServiceClient, t *testin s.Send(&testpb.StreamingInputCallRequest{Payload: payload}) } -func doServerSideFailedUnaryCall(tc testpb.TestServiceClient, t *testing.T) { +func doServerSideFailedUnaryCall(tc testgrpc.TestServiceClient, t *testing.T) { const smallSize = 1 const largeSize = 2000 @@ -725,13 +725,15 @@ func doServerSideFailedUnaryCall(tc testpb.TestServiceClient, t *testing.T) { ResponseSize: int32(smallSize), Payload: largePayload, } - if _, err := tc.UnaryCall(context.Background(), req); err == nil || status.Code(err) != codes.ResourceExhausted { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.UnaryCall(ctx, req); err == nil || status.Code(err) != codes.ResourceExhausted { t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted) } } -func doClientSideInitiatedFailedStream(tc testpb.TestServiceClient, t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) +func doClientSideInitiatedFailedStream(tc testgrpc.TestServiceClient, t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) stream, err := tc.FullDuplexCall(ctx) if err != nil { t.Fatalf("TestService/FullDuplexCall(_) = _, %v, want ", err) @@ -763,8 +765,10 @@ func doClientSideInitiatedFailedStream(tc testpb.TestServiceClient, t *testing.T } // This func is to be used to test client side counting of failed streams. -func doServerSideInitiatedFailedStreamWithRSTStream(tc testpb.TestServiceClient, t *testing.T, l *listenerWrapper) { - stream, err := tc.FullDuplexCall(context.Background()) +func doServerSideInitiatedFailedStreamWithRSTStream(tc testgrpc.TestServiceClient, t *testing.T, l *listenerWrapper) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + stream, err := tc.FullDuplexCall(ctx) if err != nil { t.Fatalf("TestService/FullDuplexCall(_) = _, %v, want ", err) } @@ -801,10 +805,10 @@ func doServerSideInitiatedFailedStreamWithRSTStream(tc testpb.TestServiceClient, } // this func is to be used to test client side counting of failed streams. -func doServerSideInitiatedFailedStreamWithGoAway(tc testpb.TestServiceClient, t *testing.T, l *listenerWrapper) { +func doServerSideInitiatedFailedStreamWithGoAway(ctx context.Context, tc testgrpc.TestServiceClient, t *testing.T, l *listenerWrapper) { // This call is just to keep the transport from shutting down (socket will be deleted // in this case, and we will not be able to get metrics). - s, err := tc.FullDuplexCall(context.Background()) + s, err := tc.FullDuplexCall(ctx) if err != nil { t.Fatalf("TestService/FullDuplexCall(_) = _, %v, want ", err) } @@ -819,7 +823,7 @@ func doServerSideInitiatedFailedStreamWithGoAway(tc testpb.TestServiceClient, t t.Fatalf("s.Recv() failed with error: %v", err) } - s, err = tc.FullDuplexCall(context.Background()) + s, err = tc.FullDuplexCall(ctx) if err != nil { t.Fatalf("TestService/FullDuplexCall(_) = _, %v, want ", err) } @@ -843,20 +847,10 @@ func doServerSideInitiatedFailedStreamWithGoAway(tc testpb.TestServiceClient, t } } -func doIdleCallToInvokeKeepAlive(tc testpb.TestServiceClient, t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - _, err := tc.FullDuplexCall(ctx) - if err != nil { - t.Fatalf("TestService/FullDuplexCall(_) = _, %v, want ", err) - } - // Allow for at least 2 keepalives (1s per ping interval) - time.Sleep(4 * time.Second) - cancel() -} - func (s) TestCZClientSocketMetricsStreamsAndMessagesCount(t *testing.T) { - czCleanup := channelz.NewChannelzStorage() - defer czCleanupWrapper(czCleanup, t) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + e := tcpClearRREnv te := newTest(t, e) te.maxServerReceiveMsgSize = newInt(20) @@ -864,7 +858,7 @@ func (s) TestCZClientSocketMetricsStreamsAndMessagesCount(t *testing.T) { rcw := te.startServerWithConnControl(&testServer{security: e.security}) defer te.tearDown() cc := te.clientConn() - tc := &testServiceClientWrapper{TestServiceClient: testpb.NewTestServiceClient(cc)} + tc := &testServiceClientWrapper{TestServiceClient: testgrpc.NewTestServiceClient(cc)} doSuccessfulUnaryCall(tc, t) var scID, skID int64 @@ -936,7 +930,7 @@ func (s) TestCZClientSocketMetricsStreamsAndMessagesCount(t *testing.T) { t.Fatal(err) } - doServerSideInitiatedFailedStreamWithGoAway(tc, t, rcw) + doServerSideInitiatedFailedStreamWithGoAway(ctx, tc, t, rcw) if err := verifyResultWithDelay(func() (bool, error) { skt := channelz.GetSocket(skID) sktData := skt.SocketData @@ -955,15 +949,13 @@ func (s) TestCZClientSocketMetricsStreamsAndMessagesCount(t *testing.T) { // It is separated from other cases due to setup incompatibly, i.e. max receive // size violation will mask flow control violation. func (s) TestCZClientAndServerSocketMetricsStreamsCountFlowControlRSTStream(t *testing.T) { - czCleanup := channelz.NewChannelzStorage() - defer czCleanupWrapper(czCleanup, t) e := tcpClearRREnv te := newTest(t, e) te.serverInitialWindowSize = 65536 // Avoid overflowing connection level flow control window, which will lead to // transport being closed. te.serverInitialConnWindowSize = 65536 * 2 - ts := &funcServer{fullDuplexCall: func(stream testpb.TestService_FullDuplexCallServer) error { + ts := &stubserver.StubServer{FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { stream.Send(&testpb.StreamingOutputCallResponse{}) <-stream.Context().Done() return status.Errorf(codes.DeadlineExceeded, "deadline exceeded or cancelled") @@ -971,9 +963,9 @@ func (s) TestCZClientAndServerSocketMetricsStreamsCountFlowControlRSTStream(t *t te.startServer(ts) defer te.tearDown() cc, dw := te.clientConnWithConnControl() - tc := &testServiceClientWrapper{TestServiceClient: testpb.NewTestServiceClient(cc)} + tc := &testServiceClientWrapper{TestServiceClient: testgrpc.NewTestServiceClient(cc)} - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) stream, err := tc.FullDuplexCall(ctx) if err != nil { t.Fatalf("TestService/FullDuplexCall(_) = _, %v, want ", err) @@ -1039,8 +1031,6 @@ func (s) TestCZClientAndServerSocketMetricsStreamsCountFlowControlRSTStream(t *t } func (s) TestCZClientAndServerSocketMetricsFlowControl(t *testing.T) { - czCleanup := channelz.NewChannelzStorage() - defer czCleanupWrapper(czCleanup, t) e := tcpClearRREnv te := newTest(t, e) // disable BDP @@ -1051,7 +1041,7 @@ func (s) TestCZClientAndServerSocketMetricsFlowControl(t *testing.T) { te.startServer(&testServer{security: e.security}) defer te.tearDown() cc := te.clientConn() - tc := testpb.NewTestServiceClient(cc) + tc := testgrpc.NewTestServiceClient(cc) for i := 0; i < 10; i++ { doSuccessfulUnaryCall(tc, t) @@ -1152,25 +1142,30 @@ func (s) TestCZClientAndServerSocketMetricsFlowControl(t *testing.T) { } func (s) TestCZClientSocketMetricsKeepAlive(t *testing.T) { - czCleanup := channelz.NewChannelzStorage() - defer czCleanupWrapper(czCleanup, t) + const keepaliveRate = 50 * time.Millisecond defer func(t time.Duration) { internal.KeepaliveMinPingTime = t }(internal.KeepaliveMinPingTime) - internal.KeepaliveMinPingTime = time.Second + internal.KeepaliveMinPingTime = keepaliveRate e := tcpClearRREnv te := newTest(t, e) te.customDialOptions = append(te.customDialOptions, grpc.WithKeepaliveParams( keepalive.ClientParameters{ - Time: time.Second, + Time: keepaliveRate, Timeout: 500 * time.Millisecond, PermitWithoutStream: true, })) te.customServerOptions = append(te.customServerOptions, grpc.KeepaliveEnforcementPolicy( keepalive.EnforcementPolicy{ - MinTime: 500 * time.Millisecond, + MinTime: keepaliveRate, PermitWithoutStream: true, })) te.startServer(&testServer{security: e.security}) - te.clientConn() // Dial the server + cc := te.clientConn() // Dial the server + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + start := time.Now() + // Wait for at least two keepalives to be able to occur. + time.Sleep(2 * keepaliveRate) defer te.tearDown() if err := verifyResultWithDelay(func() (bool, error) { tchan, _ := channelz.GetTopChannels(0, 0) @@ -1195,8 +1190,9 @@ func (s) TestCZClientSocketMetricsKeepAlive(t *testing.T) { break } skt := channelz.GetSocket(id) - if skt.SocketData.KeepAlivesSent != 2 { - return false, fmt.Errorf("there should be 2 KeepAlives sent, not %d", skt.SocketData.KeepAlivesSent) + want := int64(time.Since(start) / keepaliveRate) + if skt.SocketData.KeepAlivesSent != want { + return false, fmt.Errorf("there should be %v KeepAlives sent, not %d", want, skt.SocketData.KeepAlivesSent) } return true, nil }); err != nil { @@ -1205,8 +1201,6 @@ func (s) TestCZClientSocketMetricsKeepAlive(t *testing.T) { } func (s) TestCZServerSocketMetricsStreamsAndMessagesCount(t *testing.T) { - czCleanup := channelz.NewChannelzStorage() - defer czCleanupWrapper(czCleanup, t) e := tcpClearRREnv te := newTest(t, e) te.maxServerReceiveMsgSize = newInt(20) @@ -1214,7 +1208,7 @@ func (s) TestCZServerSocketMetricsStreamsAndMessagesCount(t *testing.T) { te.startServer(&testServer{security: e.security}) defer te.tearDown() cc, _ := te.clientConnWithConnControl() - tc := &testServiceClientWrapper{TestServiceClient: testpb.NewTestServiceClient(cc)} + tc := &testServiceClientWrapper{TestServiceClient: testgrpc.NewTestServiceClient(cc)} var svrID int64 if err := verifyResultWithDelay(func() (bool, error) { @@ -1266,45 +1260,44 @@ func (s) TestCZServerSocketMetricsStreamsAndMessagesCount(t *testing.T) { } func (s) TestCZServerSocketMetricsKeepAlive(t *testing.T) { - czCleanup := channelz.NewChannelzStorage() - defer czCleanupWrapper(czCleanup, t) + defer func(t time.Duration) { internal.KeepaliveMinServerPingTime = t }(internal.KeepaliveMinServerPingTime) + internal.KeepaliveMinServerPingTime = 50 * time.Millisecond + e := tcpClearRREnv te := newTest(t, e) // We setup the server keepalive parameters to send one keepalive every - // second, and verify that the actual number of keepalives is very close to - // the number of seconds elapsed in the test. We had a bug wherein the - // server was sending one keepalive every [Time+Timeout] instead of every - // [Time] period, and since Timeout is configured to a low value here, we - // should be able to verify that the fix works with the above mentioned - // logic. + // 50ms, and verify that the actual number of keepalives is very close to + // Time/50ms. We had a bug wherein the server was sending one keepalive + // every [Time+Timeout] instead of every [Time] period, and since Timeout + // is configured to a high value here, we should be able to verify that the + // fix works with the above mentioned logic. kpOption := grpc.KeepaliveParams(keepalive.ServerParameters{ - Time: time.Second, - Timeout: 100 * time.Millisecond, + Time: 50 * time.Millisecond, + Timeout: 5 * time.Second, }) te.customServerOptions = append(te.customServerOptions, kpOption) te.startServer(&testServer{security: e.security}) defer te.tearDown() cc := te.clientConn() - tc := testpb.NewTestServiceClient(cc) - start := time.Now() - doIdleCallToInvokeKeepAlive(tc, t) - if err := verifyResultWithDelay(func() (bool, error) { - ss, _ := channelz.GetServers(0, 0) - if len(ss) != 1 { - return false, fmt.Errorf("there should be one server, not %d", len(ss)) - } - ns, _ := channelz.GetServerSockets(ss[0].ID, 0, 0) - if len(ns) != 1 { - return false, fmt.Errorf("there should be one server normal socket, not %d", len(ns)) - } - wantKeepalivesCount := int64(time.Since(start).Seconds()) - 1 - if gotKeepalivesCount := ns[0].SocketData.KeepAlivesSent; gotKeepalivesCount != wantKeepalivesCount { - return false, fmt.Errorf("got keepalivesCount: %v, want keepalivesCount: %v", gotKeepalivesCount, wantKeepalivesCount) - } - return true, nil - }); err != nil { - t.Fatal(err) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + + // Allow about 5 pings to happen (250ms/50ms). + time.Sleep(255 * time.Millisecond) + + ss, _ := channelz.GetServers(0, 0) + if len(ss) != 1 { + t.Fatalf("there should be one server, not %d", len(ss)) + } + ns, _ := channelz.GetServerSockets(ss[0].ID, 0, 0) + if len(ns) != 1 { + t.Fatalf("there should be one server normal socket, not %d", len(ns)) + } + const wantMin, wantMax = 3, 7 + if got := ns[0].SocketData.KeepAlivesSent; got < wantMin || got > wantMax { + t.Fatalf("got keepalivesCount: %v, want keepalivesCount: [%v,%v]", got, wantMin, wantMax) } } @@ -1338,8 +1331,6 @@ var cipherSuites = []string{ } func (s) TestCZSocketGetSecurityValueTLS(t *testing.T) { - czCleanup := channelz.NewChannelzStorage() - defer czCleanupWrapper(czCleanup, t) e := tcpTLSRREnv te := newTest(t, e) te.startServer(&testServer{security: e.security}) @@ -1373,7 +1364,7 @@ func (s) TestCZSocketGetSecurityValueTLS(t *testing.T) { if !ok { return false, fmt.Errorf("the SocketData.Security is of type: %T, want: *credentials.TLSChannelzSecurityValue", skt.SocketData.Security) } - if !reflect.DeepEqual(securityVal.RemoteCertificate, cert.Certificate[0]) { + if !cmp.Equal(securityVal.RemoteCertificate, cert.Certificate[0]) { return false, fmt.Errorf("SocketData.Security.RemoteCertificate got: %v, want: %v", securityVal.RemoteCertificate, cert.Certificate[0]) } for _, v := range cipherSuites { @@ -1388,18 +1379,18 @@ func (s) TestCZSocketGetSecurityValueTLS(t *testing.T) { } func (s) TestCZChannelTraceCreationDeletion(t *testing.T) { - czCleanup := channelz.NewChannelzStorage() - defer czCleanupWrapper(czCleanup, t) e := tcpClearRREnv // avoid calling API to set balancer type, which will void service config's change of balancer. e.balancer = "" te := newTest(t, e) r := manual.NewBuilderWithScheme("whatever") - resolvedAddrs := []resolver.Address{{Addr: "127.0.0.1:0", Type: resolver.GRPCLB, ServerName: "grpclb.server"}} - r.InitialState(resolver.State{Addresses: resolvedAddrs}) te.resolverScheme = r.Scheme() te.clientConn(grpc.WithResolvers(r)) + resolvedAddrs := []resolver.Address{{Addr: "127.0.0.1:0", ServerName: "grpclb.server"}} + grpclbConfig := parseServiceConfig(t, r, `{"loadBalancingPolicy": "grpclb"}`) + r.UpdateState(grpclbstate.Set(resolver.State{ServiceConfig: grpclbConfig}, &grpclbstate.State{BalancerAddresses: resolvedAddrs})) defer te.tearDown() + var nestedConn int64 if err := verifyResultWithDelay(func() (bool, error) { tcs, _ := channelz.GetTopChannels(0, 0) @@ -1424,15 +1415,19 @@ func (s) TestCZChannelTraceCreationDeletion(t *testing.T) { if len(ncm.Trace.Events) == 0 { return false, fmt.Errorf("there should be at least one trace event for nested channel not 0") } - if ncm.Trace.Events[0].Desc != "Channel Created" { - return false, fmt.Errorf("the first trace event should be \"Channel Created\", not %q", ncm.Trace.Events[0].Desc) + pattern := `Channel created` + if ok, _ := regexp.MatchString(pattern, ncm.Trace.Events[0].Desc); !ok { + return false, fmt.Errorf("the first trace event should be %q, not %q", pattern, ncm.Trace.Events[0].Desc) } return true, nil }); err != nil { t.Fatal(err) } - r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: "127.0.0.1:0"}}, ServiceConfig: parseCfg(r, `{"loadBalancingPolicy": "round_robin"}`)}) + r.UpdateState(resolver.State{ + Addresses: []resolver.Address{{Addr: "127.0.0.1:0"}}, + ServiceConfig: parseServiceConfig(t, r, `{"loadBalancingPolicy": "round_robin"}`), + }) // wait for the shutdown of grpclb balancer if err := verifyResultWithDelay(func() (bool, error) { @@ -1453,8 +1448,9 @@ func (s) TestCZChannelTraceCreationDeletion(t *testing.T) { if len(ncm.Trace.Events) == 0 { return false, fmt.Errorf("there should be at least one trace event for nested channel not 0") } - if ncm.Trace.Events[len(ncm.Trace.Events)-1].Desc != "Channel Deleted" { - return false, fmt.Errorf("the first trace event should be \"Channel Deleted\", not %q", ncm.Trace.Events[0].Desc) + pattern := `Channel created` + if ok, _ := regexp.MatchString(pattern, ncm.Trace.Events[0].Desc); !ok { + return false, fmt.Errorf("the first trace event should be %q, not %q", pattern, ncm.Trace.Events[0].Desc) } return true, nil }); err != nil { @@ -1463,8 +1459,6 @@ func (s) TestCZChannelTraceCreationDeletion(t *testing.T) { } func (s) TestCZSubChannelTraceCreationDeletion(t *testing.T) { - czCleanup := channelz.NewChannelzStorage() - defer czCleanupWrapper(czCleanup, t) e := tcpClearRREnv te := newTest(t, e) te.startServer(&testServer{security: e.security}) @@ -1502,29 +1496,20 @@ func (s) TestCZSubChannelTraceCreationDeletion(t *testing.T) { if len(scm.Trace.Events) == 0 { return false, fmt.Errorf("there should be at least one trace event for subChannel not 0") } - if scm.Trace.Events[0].Desc != "Subchannel Created" { - return false, fmt.Errorf("the first trace event should be \"Subchannel Created\", not %q", scm.Trace.Events[0].Desc) + pattern := `Subchannel created` + if ok, _ := regexp.MatchString(pattern, scm.Trace.Events[0].Desc); !ok { + return false, fmt.Errorf("the first trace event should be %q, not %q", pattern, scm.Trace.Events[0].Desc) } return true, nil }); err != nil { t.Fatal(err) } - // Wait for ready - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() - for src := te.cc.GetState(); src != connectivity.Ready; src = te.cc.GetState() { - if !te.cc.WaitForStateChange(ctx, src) { - t.Fatalf("timed out waiting for state change. got %v; want %v", src, connectivity.Ready) - } - } + testutils.AwaitState(ctx, t, te.cc, connectivity.Ready) r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: "fake address"}}}) - // Wait for not-ready. - for src := te.cc.GetState(); src == connectivity.Ready; src = te.cc.GetState() { - if !te.cc.WaitForStateChange(ctx, src) { - t.Fatalf("timed out waiting for state change. got %v; want !%v", src, connectivity.Ready) - } - } + testutils.AwaitNotState(ctx, t, te.cc, connectivity.Ready) if err := verifyResultWithDelay(func() (bool, error) { tcs, _ := channelz.GetTopChannels(0, 0) @@ -1544,10 +1529,12 @@ func (s) TestCZSubChannelTraceCreationDeletion(t *testing.T) { if len(scm.Trace.Events) == 0 { return false, fmt.Errorf("there should be at least one trace event for subChannel not 0") } - if got, want := scm.Trace.Events[len(scm.Trace.Events)-1].Desc, "Subchannel Deleted"; got != want { - return false, fmt.Errorf("the last trace event should be %q, not %q", want, got) - } + pattern := `Subchannel deleted` + desc := scm.Trace.Events[len(scm.Trace.Events)-1].Desc + if ok, _ := regexp.MatchString(pattern, desc); !ok { + return false, fmt.Errorf("the last trace event should be %q, not %q", pattern, desc) + } return true, nil }); err != nil { t.Fatal(err) @@ -1555,8 +1542,6 @@ func (s) TestCZSubChannelTraceCreationDeletion(t *testing.T) { } func (s) TestCZChannelAddressResolutionChange(t *testing.T) { - czCleanup := channelz.NewChannelzStorage() - defer czCleanupWrapper(czCleanup, t) e := tcpClearRREnv e.balancer = "" te := newTest(t, e) @@ -1588,12 +1573,15 @@ func (s) TestCZChannelAddressResolutionChange(t *testing.T) { }); err != nil { t.Fatal(err) } - r.UpdateState(resolver.State{Addresses: addrs, ServiceConfig: parseCfg(r, `{"loadBalancingPolicy": "round_robin"}`)}) + r.UpdateState(resolver.State{ + Addresses: addrs, + ServiceConfig: parseServiceConfig(t, r, `{"loadBalancingPolicy": "round_robin"}`), + }) if err := verifyResultWithDelay(func() (bool, error) { cm := channelz.GetChannel(cid) for i := len(cm.Trace.Events) - 1; i >= 0; i-- { - if cm.Trace.Events[i].Desc == fmt.Sprintf("Channel switches to new LB policy %q", roundrobin.Name) { + if strings.Contains(cm.Trace.Events[i].Desc, fmt.Sprintf("Channel switches to new LB policy %q", roundrobin.Name)) { break } if i == 0 { @@ -1605,7 +1593,7 @@ func (s) TestCZChannelAddressResolutionChange(t *testing.T) { t.Fatal(err) } - newSC := parseCfg(r, `{ + newSC := parseServiceConfig(t, r, `{ "methodConfig": [ { "name": [ @@ -1658,8 +1646,6 @@ func (s) TestCZChannelAddressResolutionChange(t *testing.T) { } func (s) TestCZSubChannelPickedNewAddress(t *testing.T) { - czCleanup := channelz.NewChannelzStorage() - defer czCleanupWrapper(czCleanup, t) e := tcpClearRREnv e.balancer = "" te := newTest(t, e) @@ -1673,17 +1659,29 @@ func (s) TestCZSubChannelPickedNewAddress(t *testing.T) { te.resolverScheme = r.Scheme() cc := te.clientConn(grpc.WithResolvers(r)) defer te.tearDown() - tc := testpb.NewTestServiceClient(cc) + tc := testgrpc.NewTestServiceClient(cc) // make sure the connection is up - ctx, cancel := context.WithTimeout(context.Background(), time.Second) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil { t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, ", err) } te.srvs[0].Stop() te.srvs[1].Stop() - // Here, we just wait for all sockets to be up. In the future, if we implement - // IDLE, we may need to make several rpc calls to create the sockets. + // Here, we just wait for all sockets to be up. Make several rpc calls to + // create the sockets since we do not automatically reconnect. + done := make(chan struct{}) + defer close(done) + go func() { + for { + tc.EmptyCall(ctx, &testpb.Empty{}) + select { + case <-time.After(10 * time.Millisecond): + case <-done: + return + } + } + }() if err := verifyResultWithDelay(func() (bool, error) { tcs, _ := channelz.GetTopChannels(0, 0) if len(tcs) != 1 { @@ -1704,7 +1702,7 @@ func (s) TestCZSubChannelPickedNewAddress(t *testing.T) { return false, fmt.Errorf("there should be at least one trace event for subChannel not 0") } for i := len(scm.Trace.Events) - 1; i >= 0; i-- { - if scm.Trace.Events[i].Desc == fmt.Sprintf("Subchannel picks a new address %q to connect", te.srvAddrs[2]) { + if strings.Contains(scm.Trace.Events[i].Desc, fmt.Sprintf("Subchannel picks a new address %q to connect", te.srvAddrs[2])) { break } if i == 0 { @@ -1718,8 +1716,6 @@ func (s) TestCZSubChannelPickedNewAddress(t *testing.T) { } func (s) TestCZSubChannelConnectivityState(t *testing.T) { - czCleanup := channelz.NewChannelzStorage() - defer czCleanupWrapper(czCleanup, t) e := tcpClearRREnv te := newTest(t, e) te.startServer(&testServer{security: e.security}) @@ -1728,16 +1724,16 @@ func (s) TestCZSubChannelConnectivityState(t *testing.T) { te.resolverScheme = r.Scheme() cc := te.clientConn(grpc.WithResolvers(r)) defer te.tearDown() - tc := testpb.NewTestServiceClient(cc) + tc := testgrpc.NewTestServiceClient(cc) // make sure the connection is up - ctx, cancel := context.WithTimeout(context.Background(), time.Second) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil { t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, ", err) } - var subConn int64 te.srv.Stop() + var subConn int64 if err := verifyResultWithDelay(func() (bool, error) { // we need to obtain the SubChannel id before it gets deleted from Channel's children list (due // to effect of r.UpdateState(resolver.State{Addresses:[]resolver.Address{}})) @@ -1752,6 +1748,7 @@ func (s) TestCZSubChannelConnectivityState(t *testing.T) { for k := range tcs[0].SubChans { // get the SubChannel id for further trace inquiry. subConn = k + t.Logf("SubChannel Id is %d", subConn) } } scm := channelz.GetSubChannel(subConn) @@ -1765,8 +1762,10 @@ func (s) TestCZSubChannelConnectivityState(t *testing.T) { return false, fmt.Errorf("there should be at least one trace event for subChannel not 0") } var ready, connecting, transient, shutdown int + t.Log("SubChannel trace events seen so far...") for _, e := range scm.Trace.Events { - if e.Desc == fmt.Sprintf("Subchannel Connectivity change to %v", connectivity.TransientFailure) { + t.Log(e.Desc) + if strings.Contains(e.Desc, fmt.Sprintf("Subchannel Connectivity change to %v", connectivity.TransientFailure)) { transient++ } } @@ -1777,17 +1776,19 @@ func (s) TestCZSubChannelConnectivityState(t *testing.T) { } transient = 0 r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: "fake address"}}}) + t.Log("SubChannel trace events seen so far...") for _, e := range scm.Trace.Events { - if e.Desc == fmt.Sprintf("Subchannel Connectivity change to %v", connectivity.Ready) { + t.Log(e.Desc) + if strings.Contains(e.Desc, fmt.Sprintf("Subchannel Connectivity change to %v", connectivity.Ready)) { ready++ } - if e.Desc == fmt.Sprintf("Subchannel Connectivity change to %v", connectivity.Connecting) { + if strings.Contains(e.Desc, fmt.Sprintf("Subchannel Connectivity change to %v", connectivity.Connecting)) { connecting++ } - if e.Desc == fmt.Sprintf("Subchannel Connectivity change to %v", connectivity.TransientFailure) { + if strings.Contains(e.Desc, fmt.Sprintf("Subchannel Connectivity change to %v", connectivity.TransientFailure)) { transient++ } - if e.Desc == fmt.Sprintf("Subchannel Connectivity change to %v", connectivity.Shutdown) { + if strings.Contains(e.Desc, fmt.Sprintf("Subchannel Connectivity change to %v", connectivity.Shutdown)) { shutdown++ } } @@ -1812,8 +1813,6 @@ func (s) TestCZSubChannelConnectivityState(t *testing.T) { } func (s) TestCZChannelConnectivityState(t *testing.T) { - czCleanup := channelz.NewChannelzStorage() - defer czCleanupWrapper(czCleanup, t) e := tcpClearRREnv te := newTest(t, e) te.startServer(&testServer{security: e.security}) @@ -1822,14 +1821,15 @@ func (s) TestCZChannelConnectivityState(t *testing.T) { te.resolverScheme = r.Scheme() cc := te.clientConn(grpc.WithResolvers(r)) defer te.tearDown() - tc := testpb.NewTestServiceClient(cc) + tc := testgrpc.NewTestServiceClient(cc) // make sure the connection is up - ctx, cancel := context.WithTimeout(context.Background(), time.Second) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil { t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, ", err) } te.srv.Stop() + if err := verifyResultWithDelay(func() (bool, error) { tcs, _ := channelz.GetTopChannels(0, 0) if len(tcs) != 1 { @@ -1837,14 +1837,16 @@ func (s) TestCZChannelConnectivityState(t *testing.T) { } var ready, connecting, transient int + t.Log("Channel trace events seen so far...") for _, e := range tcs[0].Trace.Events { - if e.Desc == fmt.Sprintf("Channel Connectivity change to %v", connectivity.Ready) { + t.Log(e.Desc) + if strings.Contains(e.Desc, fmt.Sprintf("Channel Connectivity change to %v", connectivity.Ready)) { ready++ } - if e.Desc == fmt.Sprintf("Channel Connectivity change to %v", connectivity.Connecting) { + if strings.Contains(e.Desc, fmt.Sprintf("Channel Connectivity change to %v", connectivity.Connecting)) { connecting++ } - if e.Desc == fmt.Sprintf("Channel Connectivity change to %v", connectivity.TransientFailure) { + if strings.Contains(e.Desc, fmt.Sprintf("Channel Connectivity change to %v", connectivity.TransientFailure)) { transient++ } } @@ -1868,20 +1870,17 @@ func (s) TestCZChannelConnectivityState(t *testing.T) { } func (s) TestCZTraceOverwriteChannelDeletion(t *testing.T) { - czCleanup := channelz.NewChannelzStorage() - defer czCleanupWrapper(czCleanup, t) e := tcpClearRREnv - // avoid newTest using WithBalancerName, which would override service - // config's change of balancer below. e.balancer = "" te := newTest(t, e) channelz.SetMaxTraceEntry(1) defer channelz.ResetMaxTraceEntryToDefault() r := manual.NewBuilderWithScheme("whatever") - resolvedAddrs := []resolver.Address{{Addr: "127.0.0.1:0", Type: resolver.GRPCLB, ServerName: "grpclb.server"}} - r.InitialState(resolver.State{Addresses: resolvedAddrs}) te.resolverScheme = r.Scheme() te.clientConn(grpc.WithResolvers(r)) + resolvedAddrs := []resolver.Address{{Addr: "127.0.0.1:0", ServerName: "grpclb.server"}} + grpclbConfig := parseServiceConfig(t, r, `{"loadBalancingPolicy": "grpclb"}`) + r.UpdateState(grpclbstate.Set(resolver.State{ServiceConfig: grpclbConfig}, &grpclbstate.State{BalancerAddresses: resolvedAddrs})) defer te.tearDown() var nestedConn int64 if err := verifyResultWithDelay(func() (bool, error) { @@ -1900,7 +1899,10 @@ func (s) TestCZTraceOverwriteChannelDeletion(t *testing.T) { t.Fatal(err) } - r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: "127.0.0.1:0"}}, ServiceConfig: parseCfg(r, `{"loadBalancingPolicy": "round_robin"}`)}) + r.UpdateState(resolver.State{ + Addresses: []resolver.Address{{Addr: "127.0.0.1:0"}}, + ServiceConfig: parseServiceConfig(t, r, `{"loadBalancingPolicy": "round_robin"}`), + }) // wait for the shutdown of grpclb balancer if err := verifyResultWithDelay(func() (bool, error) { @@ -1918,7 +1920,10 @@ func (s) TestCZTraceOverwriteChannelDeletion(t *testing.T) { // If nested channel deletion is last trace event before the next validation, it will fail, as the top channel will hold a reference to it. // This line forces a trace event on the top channel in that case. - r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: "127.0.0.1:0"}}, ServiceConfig: parseCfg(r, `{"loadBalancingPolicy": "round_robin"}`)}) + r.UpdateState(resolver.State{ + Addresses: []resolver.Address{{Addr: "127.0.0.1:0"}}, + ServiceConfig: parseServiceConfig(t, r, `{"loadBalancingPolicy": "round_robin"}`), + }) // verify that the nested channel no longer exist due to trace referencing it got overwritten. if err := verifyResultWithDelay(func() (bool, error) { @@ -1933,8 +1938,6 @@ func (s) TestCZTraceOverwriteChannelDeletion(t *testing.T) { } func (s) TestCZTraceOverwriteSubChannelDeletion(t *testing.T) { - czCleanup := channelz.NewChannelzStorage() - defer czCleanupWrapper(czCleanup, t) e := tcpClearRREnv te := newTest(t, e) channelz.SetMaxTraceEntry(1) @@ -1964,21 +1967,11 @@ func (s) TestCZTraceOverwriteSubChannelDeletion(t *testing.T) { t.Fatal(err) } - // Wait for ready - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() - for src := te.cc.GetState(); src != connectivity.Ready; src = te.cc.GetState() { - if !te.cc.WaitForStateChange(ctx, src) { - t.Fatalf("timed out waiting for state change. got %v; want %v", src, connectivity.Ready) - } - } + testutils.AwaitState(ctx, t, te.cc, connectivity.Ready) r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: "fake address"}}}) - // Wait for not-ready. - for src := te.cc.GetState(); src == connectivity.Ready; src = te.cc.GetState() { - if !te.cc.WaitForStateChange(ctx, src) { - t.Fatalf("timed out waiting for state change. got %v; want !%v", src, connectivity.Ready) - } - } + testutils.AwaitNotState(ctx, t, te.cc, connectivity.Ready) // verify that the subchannel no longer exist due to trace referencing it got overwritten. if err := verifyResultWithDelay(func() (bool, error) { @@ -1993,8 +1986,6 @@ func (s) TestCZTraceOverwriteSubChannelDeletion(t *testing.T) { } func (s) TestCZTraceTopChannelDeletionTraceClear(t *testing.T) { - czCleanup := channelz.NewChannelzStorage() - defer czCleanupWrapper(czCleanup, t) e := tcpClearRREnv te := newTest(t, e) te.startServer(&testServer{security: e.security}) diff --git a/clientconn_state_transition_test.go b/test/clientconn_state_transition_test.go similarity index 65% rename from clientconn_state_transition_test.go rename to test/clientconn_state_transition_test.go index 0c58131a1c6f..6e9bfb37289d 100644 --- a/clientconn_state_transition_test.go +++ b/test/clientconn_state_transition_test.go @@ -16,24 +16,31 @@ * */ -package grpc +package test import ( "context" + "fmt" "net" "sync" "testing" "time" "golang.org/x/net/http2" + "google.golang.org/grpc" + "google.golang.org/grpc/backoff" "google.golang.org/grpc/balancer" "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/balancer/stub" + "google.golang.org/grpc/internal/grpcsync" "google.golang.org/grpc/internal/testutils" "google.golang.org/grpc/resolver" "google.golang.org/grpc/resolver/manual" ) -const stateRecordingBalancerName = "state_recoding_balancer" +const stateRecordingBalancerName = "state_recording_balancer" var testBalancerBuilder = newStateRecordingBalancerBuilder() @@ -75,7 +82,7 @@ func (s) TestStateTransitions_SingleAddress(t *testing.T) { }, }, { - desc: "When the connection is closed, the client enters TRANSIENT FAILURE.", + desc: "When the connection is closed before the preface is sent, the client enters TRANSIENT FAILURE.", want: []connectivity.State{ connectivity.Connecting, connectivity.TransientFailure, @@ -141,9 +148,6 @@ client enters TRANSIENT FAILURE.`, } func testStateTransitionSingleAddress(t *testing.T, want []connectivity.State, server func(net.Listener) net.Conn) { - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - defer cancel() - pl := testutils.NewPipeListener() defer pl.Close() @@ -156,25 +160,27 @@ func testStateTransitionSingleAddress(t *testing.T, want []connectivity.State, s connMu.Unlock() }() - client, err := DialContext(ctx, - "", - WithInsecure(), - WithBalancerName(stateRecordingBalancerName), - WithDialer(pl.Dialer()), - withBackoff(noBackoff{}), - withMinConnectDeadline(func() time.Duration { return time.Millisecond * 100 })) + client, err := grpc.Dial("", + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, stateRecordingBalancerName)), + grpc.WithDialer(pl.Dialer()), + grpc.WithConnectParams(grpc.ConnectParams{ + Backoff: backoff.Config{}, + MinConnectTimeout: 100 * time.Millisecond, + })) if err != nil { t.Fatal(err) } defer client.Close() - stateNotifications := testBalancerBuilder.nextStateNotifier() - - timeout := time.After(5 * time.Second) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + go testutils.StayConnected(ctx, client) + stateNotifications := testBalancerBuilder.nextStateNotifier() for i := 0; i < len(want); i++ { select { - case <-timeout: + case <-time.After(defaultTestTimeout): t.Fatalf("timed out waiting for state %d (%v) in flow %v", i, want[i], want) case seen := <-stateNotifications: if seen != want[i] { @@ -193,24 +199,16 @@ func testStateTransitionSingleAddress(t *testing.T, want []connectivity.State, s } } -// When a READY connection is closed, the client enters CONNECTING. +// When a READY connection is closed, the client enters IDLE then CONNECTING. func (s) TestStateTransitions_ReadyToConnecting(t *testing.T) { - want := []connectivity.State{ - connectivity.Connecting, - connectivity.Ready, - connectivity.Connecting, - } - - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - defer cancel() - lis, err := net.Listen("tcp", "localhost:0") if err != nil { t.Fatalf("Error while listening. Err: %v", err) } defer lis.Close() - sawReady := make(chan struct{}) + sawReady := make(chan struct{}, 1) + defer close(sawReady) // Launch the server. go func() { @@ -234,23 +232,33 @@ func (s) TestStateTransitions_ReadyToConnecting(t *testing.T) { conn.Close() }() - client, err := DialContext(ctx, lis.Addr().String(), WithInsecure(), WithBalancerName(stateRecordingBalancerName)) + client, err := grpc.Dial(lis.Addr().String(), + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, stateRecordingBalancerName))) if err != nil { t.Fatal(err) } defer client.Close() - stateNotifications := testBalancerBuilder.nextStateNotifier() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + go testutils.StayConnected(ctx, client) - timeout := time.After(5 * time.Second) + stateNotifications := testBalancerBuilder.nextStateNotifier() + want := []connectivity.State{ + connectivity.Connecting, + connectivity.Ready, + connectivity.Idle, + connectivity.Connecting, + } for i := 0; i < len(want); i++ { select { - case <-timeout: + case <-time.After(defaultTestTimeout): t.Fatalf("timed out waiting for state %d (%v) in flow %v", i, want[i], want) case seen := <-stateNotifications: if seen == connectivity.Ready { - close(sawReady) + sawReady <- struct{}{} } if seen != want[i] { t.Fatalf("expected to see %v at position %d in flow %v, got %v", want[i], i, want, seen) @@ -262,14 +270,6 @@ func (s) TestStateTransitions_ReadyToConnecting(t *testing.T) { // When the first connection is closed, the client stays in CONNECTING until it // tries the second address (which succeeds, and then it enters READY). func (s) TestStateTransitions_TriesAllAddrsBeforeTransientFailure(t *testing.T) { - want := []connectivity.State{ - connectivity.Connecting, - connectivity.Ready, - } - - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - defer cancel() - lis1, err := net.Listen("tcp", "localhost:0") if err != nil { t.Fatalf("Error while listening. Err: %v", err) @@ -320,19 +320,25 @@ func (s) TestStateTransitions_TriesAllAddrsBeforeTransientFailure(t *testing.T) {Addr: lis1.Addr().String()}, {Addr: lis2.Addr().String()}, }}) - client, err := DialContext(ctx, "whatever:///this-gets-overwritten", WithInsecure(), WithBalancerName(stateRecordingBalancerName), WithResolvers(rb)) + client, err := grpc.Dial("whatever:///this-gets-overwritten", + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, stateRecordingBalancerName)), + grpc.WithResolvers(rb)) if err != nil { t.Fatal(err) } defer client.Close() stateNotifications := testBalancerBuilder.nextStateNotifier() - - timeout := time.After(5 * time.Second) - + want := []connectivity.State{ + connectivity.Connecting, + connectivity.Ready, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() for i := 0; i < len(want); i++ { select { - case <-timeout: + case <-ctx.Done(): t.Fatalf("timed out waiting for state %d (%v) in flow %v", i, want[i], want) case seen := <-stateNotifications: if seen != want[i] { @@ -341,12 +347,12 @@ func (s) TestStateTransitions_TriesAllAddrsBeforeTransientFailure(t *testing.T) } } select { - case <-timeout: + case <-ctx.Done(): t.Fatal("saw the correct state transitions, but timed out waiting for client to finish interactions with server 1") case <-server1Done: } select { - case <-timeout: + case <-ctx.Done(): t.Fatal("saw the correct state transitions, but timed out waiting for client to finish interactions with server 2") case <-server2Done: } @@ -355,15 +361,6 @@ func (s) TestStateTransitions_TriesAllAddrsBeforeTransientFailure(t *testing.T) // When there are multiple addresses, and we enter READY on one of them, a // later closure should cause the client to enter CONNECTING func (s) TestStateTransitions_MultipleAddrsEntersReady(t *testing.T) { - want := []connectivity.State{ - connectivity.Connecting, - connectivity.Ready, - connectivity.Connecting, - } - - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - defer cancel() - lis1, err := net.Listen("tcp", "localhost:0") if err != nil { t.Fatalf("Error while listening. Err: %v", err) @@ -378,7 +375,8 @@ func (s) TestStateTransitions_MultipleAddrsEntersReady(t *testing.T) { defer lis2.Close() server1Done := make(chan struct{}) - sawReady := make(chan struct{}) + sawReady := make(chan struct{}, 1) + defer close(sawReady) // Launch server 1. go func() { @@ -400,12 +398,6 @@ func (s) TestStateTransitions_MultipleAddrsEntersReady(t *testing.T) { conn.Close() - _, err = lis1.Accept() - if err != nil { - t.Error(err) - return - } - close(server1Done) }() @@ -414,23 +406,33 @@ func (s) TestStateTransitions_MultipleAddrsEntersReady(t *testing.T) { {Addr: lis1.Addr().String()}, {Addr: lis2.Addr().String()}, }}) - client, err := DialContext(ctx, "whatever:///this-gets-overwritten", WithInsecure(), WithBalancerName(stateRecordingBalancerName), WithResolvers(rb)) + client, err := grpc.Dial("whatever:///this-gets-overwritten", + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, stateRecordingBalancerName)), + grpc.WithResolvers(rb)) if err != nil { t.Fatal(err) } defer client.Close() - stateNotifications := testBalancerBuilder.nextStateNotifier() - - timeout := time.After(2 * time.Second) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + go testutils.StayConnected(ctx, client) + stateNotifications := testBalancerBuilder.nextStateNotifier() + want := []connectivity.State{ + connectivity.Connecting, + connectivity.Ready, + connectivity.Idle, + connectivity.Connecting, + } for i := 0; i < len(want); i++ { select { - case <-timeout: + case <-ctx.Done(): t.Fatalf("timed out waiting for state %d (%v) in flow %v", i, want[i], want) case seen := <-stateNotifications: if seen == connectivity.Ready { - close(sawReady) + sawReady <- struct{}{} } if seen != want[i] { t.Fatalf("expected to see %v at position %d in flow %v, got %v", want[i], i, want, seen) @@ -438,26 +440,16 @@ func (s) TestStateTransitions_MultipleAddrsEntersReady(t *testing.T) { } } select { - case <-timeout: + case <-ctx.Done(): t.Fatal("saw the correct state transitions, but timed out waiting for client to finish interactions with server 1") case <-server1Done: } } type stateRecordingBalancer struct { - notifier chan<- connectivity.State balancer.Balancer } -func (b *stateRecordingBalancer) UpdateSubConnState(sc balancer.SubConn, s balancer.SubConnState) { - b.notifier <- s.ConnectivityState - b.Balancer.UpdateSubConnState(sc, s) -} - -func (b *stateRecordingBalancer) ResetNotifier(r chan<- connectivity.State) { - b.notifier = r -} - func (b *stateRecordingBalancer) Close() { b.Balancer.Close() } @@ -481,8 +473,7 @@ func (b *stateRecordingBalancerBuilder) Build(cc balancer.ClientConn, opts balan b.notifier = stateNotifications b.mu.Unlock() return &stateRecordingBalancer{ - notifier: stateNotifications, - Balancer: balancer.Get(PickFirstBalancerName).Build(cc, opts), + Balancer: balancer.Get("pick_first").Build(&stateRecordingCCWrapper{cc, stateNotifications}, opts), } } @@ -494,9 +485,19 @@ func (b *stateRecordingBalancerBuilder) nextStateNotifier() <-chan connectivity. return ret } -type noBackoff struct{} +type stateRecordingCCWrapper struct { + balancer.ClientConn + notifier chan<- connectivity.State +} -func (b noBackoff) Backoff(int) time.Duration { return time.Duration(0) } +func (ccw *stateRecordingCCWrapper) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) { + oldListener := opts.StateListener + opts.StateListener = func(s balancer.SubConnState) { + ccw.notifier <- s.ConnectivityState + oldListener(s) + } + return ccw.ClientConn.NewSubConn(addrs, opts) +} // Keep reading until something causes the connection to die (EOF, server // closed, etc). Useful as a tool for mindlessly keeping the connection @@ -507,3 +508,93 @@ func keepReading(conn net.Conn) { for _, err := conn.Read(buf); err == nil; _, err = conn.Read(buf) { } } + +type funcConnectivityStateSubscriber struct { + onMsg func(connectivity.State) +} + +func (f *funcConnectivityStateSubscriber) OnMessage(msg any) { + f.onMsg(msg.(connectivity.State)) +} + +// TestConnectivityStateSubscriber confirms updates sent by the balancer in +// rapid succession are not missed by the subscriber. +func (s) TestConnectivityStateSubscriber(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + sendStates := []connectivity.State{ + connectivity.Connecting, + connectivity.Ready, + connectivity.Idle, + connectivity.Connecting, + connectivity.Idle, + connectivity.Connecting, + connectivity.Ready, + } + wantStates := append(sendStates, connectivity.Shutdown) + + const testBalName = "any" + bf := stub.BalancerFuncs{ + UpdateClientConnState: func(bd *stub.BalancerData, _ balancer.ClientConnState) error { + // Send the expected states in rapid succession. + for _, s := range sendStates { + t.Logf("Sending state update %s", s) + bd.ClientConn.UpdateState(balancer.State{ConnectivityState: s}) + } + return nil + }, + } + stub.Register(testBalName, bf) + + // Create the ClientConn. + const testResName = "any" + rb := manual.NewBuilderWithScheme(testResName) + cc, err := grpc.Dial(testResName+":///", + grpc.WithResolvers(rb), + grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, testBalName)), + grpc.WithTransportCredentials(insecure.NewCredentials()), + ) + if err != nil { + t.Fatalf("Unexpected error from grpc.Dial: %v", err) + } + + // Subscribe to state updates. Use a buffer size of 1 to allow the + // Shutdown state to go into the channel when Close()ing. + connCh := make(chan connectivity.State, 1) + s := &funcConnectivityStateSubscriber{ + onMsg: func(s connectivity.State) { + select { + case connCh <- s: + case <-ctx.Done(): + } + if s == connectivity.Shutdown { + close(connCh) + } + }, + } + + internal.SubscribeToConnectivityStateChanges.(func(cc *grpc.ClientConn, s grpcsync.Subscriber) func())(cc, s) + + // Send an update from the resolver that will trigger the LB policy's UpdateClientConnState. + go rb.UpdateState(resolver.State{}) + + // Verify the resulting states. + for i, want := range wantStates { + if i == len(sendStates) { + // Trigger Shutdown to be sent by the channel. Use a goroutine to + // ensure the operation does not block. + cc.Close() + } + select { + case got := <-connCh: + if got != want { + t.Errorf("Update %v was %s; want %s", i, got, want) + } else { + t.Logf("Update %v was %s as expected", i, got) + } + case <-ctx.Done(): + t.Fatalf("Timed out waiting for state update %v: %s", i, want) + } + } +} diff --git a/test/clientconn_test.go b/test/clientconn_test.go new file mode 100644 index 000000000000..4432701fb3d1 --- /dev/null +++ b/test/clientconn_test.go @@ -0,0 +1,85 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/channelz" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/status" +) + +// TestClientConnClose_WithPendingRPC tests the scenario where the channel has +// not yet received any update from the name resolver and hence RPCs are +// blocking. The test verifies that closing the ClientConn unblocks the RPC with +// the expected error code. +func (s) TestClientConnClose_WithPendingRPC(t *testing.T) { + r := manual.NewBuilderWithScheme("whatever") + cc, err := grpc.Dial(r.Scheme()+":///test.server", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) + } + client := testgrpc.NewTestServiceClient(cc) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + doneErrCh := make(chan error, 1) + go func() { + // This RPC would block until the ClientConn is closed, because the + // resolver has not provided its first update yet. + _, err := client.EmptyCall(ctx, &testpb.Empty{}) + if status.Code(err) != codes.Canceled || !strings.Contains(err.Error(), "client connection is closing") { + doneErrCh <- fmt.Errorf("EmptyCall() = %v, want %s", err, codes.Canceled) + } + doneErrCh <- nil + }() + + // Make sure that there is one pending RPC on the ClientConn before attempting + // to close it. If we don't do this, cc.Close() can happen before the above + // goroutine gets to make the RPC. + for { + if err := ctx.Err(); err != nil { + t.Fatal(err) + } + tcs, _ := channelz.GetTopChannels(0, 0) + if len(tcs) != 1 { + t.Fatalf("there should only be one top channel, not %d", len(tcs)) + } + started := tcs[0].ChannelData.CallsStarted + completed := tcs[0].ChannelData.CallsSucceeded + tcs[0].ChannelData.CallsFailed + if (started - completed) == 1 { + break + } + time.Sleep(defaultTestShortTimeout) + } + cc.Close() + if err := <-doneErrCh; err != nil { + t.Fatal(err) + } +} diff --git a/test/clienttester.go b/test/clienttester.go new file mode 100644 index 000000000000..7e223091164d --- /dev/null +++ b/test/clienttester.go @@ -0,0 +1,109 @@ +/* + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package test + +import ( + "bytes" + "io" + "net" + "testing" + + "golang.org/x/net/http2" +) + +var ( + clientPreface = []byte(http2.ClientPreface) +) + +func newClientTester(t *testing.T, conn net.Conn) *clientTester { + ct := &clientTester{ + t: t, + conn: conn, + } + ct.fr = http2.NewFramer(conn, conn) + ct.greet() + return ct +} + +type clientTester struct { + t *testing.T + conn net.Conn + fr *http2.Framer +} + +// greet() performs the necessary steps for http2 connection establishment on +// the server side. +func (ct *clientTester) greet() { + ct.wantClientPreface() + ct.wantSettingsFrame() + ct.writeSettingsFrame() + ct.writeSettingsAck() + + for { + f, err := ct.fr.ReadFrame() + if err != nil { + ct.t.Errorf("error reading frame from client side: %v", err) + } + switch f := f.(type) { + case *http2.SettingsFrame: + if f.IsAck() { // HTTP/2 handshake completed. + return + } + default: + ct.t.Errorf("during greet, unexpected frame type %T", f) + } + } +} + +func (ct *clientTester) wantClientPreface() { + preface := make([]byte, len(clientPreface)) + if _, err := io.ReadFull(ct.conn, preface); err != nil { + ct.t.Errorf("Error at server-side while reading preface from client. Err: %v", err) + } + if !bytes.Equal(preface, clientPreface) { + ct.t.Errorf("received bogus greeting from client %q", preface) + } +} + +func (ct *clientTester) wantSettingsFrame() { + frame, err := ct.fr.ReadFrame() + if err != nil { + ct.t.Errorf("error reading initial settings frame from client: %v", err) + } + _, ok := frame.(*http2.SettingsFrame) + if !ok { + ct.t.Errorf("initial frame sent from client is not a settings frame, type %T", frame) + } +} + +func (ct *clientTester) writeSettingsFrame() { + if err := ct.fr.WriteSettings(); err != nil { + ct.t.Fatalf("Error writing initial SETTINGS frame from client to server: %v", err) + } +} + +func (ct *clientTester) writeSettingsAck() { + if err := ct.fr.WriteSettingsAck(); err != nil { + ct.t.Fatalf("Error writing ACK of client's SETTINGS: %v", err) + } +} + +func (ct *clientTester) writeGoAway(maxStreamID uint32, code http2.ErrCode, debugData []byte) { + if err := ct.fr.WriteGoAway(maxStreamID, code, debugData); err != nil { + ct.t.Fatalf("Error writing GOAWAY: %v", err) + } +} diff --git a/test/codec_perf/perf.pb.go b/test/codec_perf/perf.pb.go index b98764ade4ec..28fb33d9ff1b 100644 --- a/test/codec_perf/perf.pb.go +++ b/test/codec_perf/perf.pb.go @@ -1,80 +1,163 @@ +// Copyright 2017 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Messages used for performance tests that may not reference grpc directly for +// reasons of import cycles. + // Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.31.0 +// protoc v4.22.0 // source: test/codec_perf/perf.proto package codec_perf import ( - fmt "fmt" - proto "github.com/golang/protobuf/proto" - math "math" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" ) -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) // Buffer is a message that contains a body of bytes that is used to exercise // encoding and decoding overheads. type Buffer struct { - Body []byte `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Body []byte `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"` } -func (m *Buffer) Reset() { *m = Buffer{} } -func (m *Buffer) String() string { return proto.CompactTextString(m) } -func (*Buffer) ProtoMessage() {} -func (*Buffer) Descriptor() ([]byte, []int) { - return fileDescriptor_a913550de912e506, []int{0} +func (x *Buffer) Reset() { + *x = Buffer{} + if protoimpl.UnsafeEnabled { + mi := &file_test_codec_perf_perf_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *Buffer) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Buffer.Unmarshal(m, b) +func (x *Buffer) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *Buffer) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Buffer.Marshal(b, m, deterministic) + +func (*Buffer) ProtoMessage() {} + +func (x *Buffer) ProtoReflect() protoreflect.Message { + mi := &file_test_codec_perf_perf_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -func (m *Buffer) XXX_Merge(src proto.Message) { - xxx_messageInfo_Buffer.Merge(m, src) + +// Deprecated: Use Buffer.ProtoReflect.Descriptor instead. +func (*Buffer) Descriptor() ([]byte, []int) { + return file_test_codec_perf_perf_proto_rawDescGZIP(), []int{0} } -func (m *Buffer) XXX_Size() int { - return xxx_messageInfo_Buffer.Size(m) + +func (x *Buffer) GetBody() []byte { + if x != nil { + return x.Body + } + return nil } -func (m *Buffer) XXX_DiscardUnknown() { - xxx_messageInfo_Buffer.DiscardUnknown(m) + +var File_test_codec_perf_perf_proto protoreflect.FileDescriptor + +var file_test_codec_perf_perf_proto_rawDesc = []byte{ + 0x0a, 0x1a, 0x74, 0x65, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x64, 0x65, 0x63, 0x5f, 0x70, 0x65, 0x72, + 0x66, 0x2f, 0x70, 0x65, 0x72, 0x66, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0a, 0x63, 0x6f, + 0x64, 0x65, 0x63, 0x2e, 0x70, 0x65, 0x72, 0x66, 0x22, 0x1c, 0x0a, 0x06, 0x42, 0x75, 0x66, 0x66, + 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, + 0x52, 0x04, 0x62, 0x6f, 0x64, 0x79, 0x42, 0x28, 0x5a, 0x26, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, + 0x2e, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x67, 0x72, 0x70, 0x63, + 0x2f, 0x74, 0x65, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x64, 0x65, 0x63, 0x5f, 0x70, 0x65, 0x72, 0x66, + 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } -var xxx_messageInfo_Buffer proto.InternalMessageInfo +var ( + file_test_codec_perf_perf_proto_rawDescOnce sync.Once + file_test_codec_perf_perf_proto_rawDescData = file_test_codec_perf_perf_proto_rawDesc +) -func (m *Buffer) GetBody() []byte { - if m != nil { - return m.Body - } - return nil +func file_test_codec_perf_perf_proto_rawDescGZIP() []byte { + file_test_codec_perf_perf_proto_rawDescOnce.Do(func() { + file_test_codec_perf_perf_proto_rawDescData = protoimpl.X.CompressGZIP(file_test_codec_perf_perf_proto_rawDescData) + }) + return file_test_codec_perf_perf_proto_rawDescData } -func init() { - proto.RegisterType((*Buffer)(nil), "codec.perf.Buffer") +var file_test_codec_perf_perf_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_test_codec_perf_perf_proto_goTypes = []interface{}{ + (*Buffer)(nil), // 0: codec.perf.Buffer +} +var file_test_codec_perf_perf_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name } -func init() { proto.RegisterFile("test/codec_perf/perf.proto", fileDescriptor_a913550de912e506) } - -var fileDescriptor_a913550de912e506 = []byte{ - // 118 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x2a, 0x49, 0x2d, 0x2e, - 0xd1, 0x4f, 0xce, 0x4f, 0x49, 0x4d, 0x8e, 0x2f, 0x48, 0x2d, 0x4a, 0xd3, 0x07, 0x11, 0x7a, 0x05, - 0x45, 0xf9, 0x25, 0xf9, 0x42, 0x5c, 0x60, 0x61, 0x3d, 0x90, 0x88, 0x92, 0x0c, 0x17, 0x9b, 0x53, - 0x69, 0x5a, 0x5a, 0x6a, 0x91, 0x90, 0x10, 0x17, 0x4b, 0x52, 0x7e, 0x4a, 0xa5, 0x04, 0xa3, 0x02, - 0xa3, 0x06, 0x4f, 0x10, 0x98, 0xed, 0xa4, 0x11, 0xa5, 0x96, 0x9e, 0x9f, 0x9f, 0x9e, 0x93, 0xaa, - 0x97, 0x9e, 0x9f, 0x93, 0x98, 0x97, 0xae, 0x97, 0x5f, 0x94, 0xae, 0x9f, 0x5e, 0x54, 0x90, 0xac, - 0x8f, 0x66, 0x7c, 0x12, 0x1b, 0xd8, 0x68, 0x63, 0x40, 0x00, 0x00, 0x00, 0xff, 0xff, 0x0c, 0xdb, - 0x49, 0x8b, 0x78, 0x00, 0x00, 0x00, +func init() { file_test_codec_perf_perf_proto_init() } +func file_test_codec_perf_perf_proto_init() { + if File_test_codec_perf_perf_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_test_codec_perf_perf_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Buffer); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_test_codec_perf_perf_proto_rawDesc, + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_test_codec_perf_perf_proto_goTypes, + DependencyIndexes: file_test_codec_perf_perf_proto_depIdxs, + MessageInfos: file_test_codec_perf_perf_proto_msgTypes, + }.Build() + File_test_codec_perf_perf_proto = out.File + file_test_codec_perf_perf_proto_rawDesc = nil + file_test_codec_perf_perf_proto_goTypes = nil + file_test_codec_perf_perf_proto_depIdxs = nil } diff --git a/test/compressor_test.go b/test/compressor_test.go new file mode 100644 index 000000000000..5a4aec3a1c98 --- /dev/null +++ b/test/compressor_test.go @@ -0,0 +1,707 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "bytes" + "compress/gzip" + "context" + "io" + "reflect" + "strings" + "sync/atomic" + "testing" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/encoding" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +func (s) TestCompressServerHasNoSupport(t *testing.T) { + for _, e := range listTestEnv() { + testCompressServerHasNoSupport(t, e) + } +} + +func testCompressServerHasNoSupport(t *testing.T, e env) { + te := newTest(t, e) + te.serverCompression = false + te.clientCompression = false + te.clientNopCompression = true + te.startServer(&testServer{security: e.security}) + defer te.tearDown() + tc := testgrpc.NewTestServiceClient(te.clientConn()) + + const argSize = 271828 + const respSize = 314159 + payload, err := newPayload(testpb.PayloadType_COMPRESSABLE, argSize) + if err != nil { + t.Fatal(err) + } + req := &testpb.SimpleRequest{ + ResponseType: testpb.PayloadType_COMPRESSABLE, + ResponseSize: respSize, + Payload: payload, + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.UnaryCall(ctx, req); err == nil || status.Code(err) != codes.Unimplemented { + t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code %s", err, codes.Unimplemented) + } + // Streaming RPC + stream, err := tc.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("%v.FullDuplexCall(_) = _, %v, want ", tc, err) + } + if _, err := stream.Recv(); err == nil || status.Code(err) != codes.Unimplemented { + t.Fatalf("%v.Recv() = %v, want error code %s", stream, err, codes.Unimplemented) + } +} + +func (s) TestCompressOK(t *testing.T) { + for _, e := range listTestEnv() { + testCompressOK(t, e) + } +} + +func testCompressOK(t *testing.T, e env) { + te := newTest(t, e) + te.serverCompression = true + te.clientCompression = true + te.startServer(&testServer{security: e.security}) + defer te.tearDown() + tc := testgrpc.NewTestServiceClient(te.clientConn()) + + // Unary call + const argSize = 271828 + const respSize = 314159 + payload, err := newPayload(testpb.PayloadType_COMPRESSABLE, argSize) + if err != nil { + t.Fatal(err) + } + req := &testpb.SimpleRequest{ + ResponseType: testpb.PayloadType_COMPRESSABLE, + ResponseSize: respSize, + Payload: payload, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctx = metadata.NewOutgoingContext(ctx, metadata.Pairs("something", "something")) + if _, err := tc.UnaryCall(ctx, req); err != nil { + t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, ", err) + } + // Streaming RPC + stream, err := tc.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("%v.FullDuplexCall(_) = _, %v, want ", tc, err) + } + respParam := []*testpb.ResponseParameters{ + { + Size: 31415, + }, + } + payload, err = newPayload(testpb.PayloadType_COMPRESSABLE, int32(31415)) + if err != nil { + t.Fatal(err) + } + sreq := &testpb.StreamingOutputCallRequest{ + ResponseType: testpb.PayloadType_COMPRESSABLE, + ResponseParameters: respParam, + Payload: payload, + } + if err := stream.Send(sreq); err != nil { + t.Fatalf("%v.Send(%v) = %v, want ", stream, sreq, err) + } + stream.CloseSend() + if _, err := stream.Recv(); err != nil { + t.Fatalf("%v.Recv() = %v, want ", stream, err) + } + if _, err := stream.Recv(); err != io.EOF { + t.Fatalf("%v.Recv() = %v, want io.EOF", stream, err) + } +} + +func (s) TestIdentityEncoding(t *testing.T) { + for _, e := range listTestEnv() { + testIdentityEncoding(t, e) + } +} + +func testIdentityEncoding(t *testing.T, e env) { + te := newTest(t, e) + te.startServer(&testServer{security: e.security}) + defer te.tearDown() + tc := testgrpc.NewTestServiceClient(te.clientConn()) + + // Unary call + payload, err := newPayload(testpb.PayloadType_COMPRESSABLE, 5) + if err != nil { + t.Fatal(err) + } + req := &testpb.SimpleRequest{ + ResponseType: testpb.PayloadType_COMPRESSABLE, + ResponseSize: 10, + Payload: payload, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctx = metadata.NewOutgoingContext(ctx, metadata.Pairs("something", "something")) + if _, err := tc.UnaryCall(ctx, req); err != nil { + t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, ", err) + } + // Streaming RPC + stream, err := tc.FullDuplexCall(ctx, grpc.UseCompressor("identity")) + if err != nil { + t.Fatalf("%v.FullDuplexCall(_) = _, %v, want ", tc, err) + } + payload, err = newPayload(testpb.PayloadType_COMPRESSABLE, int32(31415)) + if err != nil { + t.Fatal(err) + } + sreq := &testpb.StreamingOutputCallRequest{ + ResponseType: testpb.PayloadType_COMPRESSABLE, + ResponseParameters: []*testpb.ResponseParameters{{Size: 10}}, + Payload: payload, + } + if err := stream.Send(sreq); err != nil { + t.Fatalf("%v.Send(%v) = %v, want ", stream, sreq, err) + } + stream.CloseSend() + if _, err := stream.Recv(); err != nil { + t.Fatalf("%v.Recv() = %v, want ", stream, err) + } + if _, err := stream.Recv(); err != io.EOF { + t.Fatalf("%v.Recv() = %v, want io.EOF", stream, err) + } +} + +// renameCompressor is a grpc.Compressor wrapper that allows customizing the +// Type() of another compressor. +type renameCompressor struct { + grpc.Compressor + name string +} + +func (r *renameCompressor) Type() string { return r.name } + +// renameDecompressor is a grpc.Decompressor wrapper that allows customizing the +// Type() of another Decompressor. +type renameDecompressor struct { + grpc.Decompressor + name string +} + +func (r *renameDecompressor) Type() string { return r.name } + +func (s) TestClientForwardsGrpcAcceptEncodingHeader(t *testing.T) { + wantGrpcAcceptEncodingCh := make(chan []string, 1) + defer close(wantGrpcAcceptEncodingCh) + + compressor := renameCompressor{Compressor: grpc.NewGZIPCompressor(), name: "testgzip"} + decompressor := renameDecompressor{Decompressor: grpc.NewGZIPDecompressor(), name: "testgzip"} + + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return nil, status.Errorf(codes.Internal, "no metadata in context") + } + if got, want := md["grpc-accept-encoding"], <-wantGrpcAcceptEncodingCh; !reflect.DeepEqual(got, want) { + return nil, status.Errorf(codes.Internal, "got grpc-accept-encoding=%q; want [%q]", got, want) + } + return &testpb.Empty{}, nil + }, + } + if err := ss.Start([]grpc.ServerOption{grpc.RPCDecompressor(&decompressor)}); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + wantGrpcAcceptEncodingCh <- []string{"gzip"} + if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("ss.Client.EmptyCall(_, _) = _, %v; want _, nil", err) + } + + wantGrpcAcceptEncodingCh <- []string{"gzip"} + if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}, grpc.UseCompressor("gzip")); err != nil { + t.Fatalf("ss.Client.EmptyCall(_, _) = _, %v; want _, nil", err) + } + + // Use compressor directly which is not registered via + // encoding.RegisterCompressor. + if err := ss.StartClient(grpc.WithCompressor(&compressor)); err != nil { + t.Fatalf("Error starting client: %v", err) + } + wantGrpcAcceptEncodingCh <- []string{"gzip,testgzip"} + if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("ss.Client.EmptyCall(_, _) = _, %v; want _, nil", err) + } +} + +// wrapCompressor is a wrapper of encoding.Compressor which maintains count of +// Compressor method invokes. +type wrapCompressor struct { + encoding.Compressor + compressInvokes int32 +} + +func (wc *wrapCompressor) Compress(w io.Writer) (io.WriteCloser, error) { + atomic.AddInt32(&wc.compressInvokes, 1) + return wc.Compressor.Compress(w) +} + +func setupGzipWrapCompressor(t *testing.T) *wrapCompressor { + oldC := encoding.GetCompressor("gzip") + c := &wrapCompressor{Compressor: oldC} + encoding.RegisterCompressor(c) + t.Cleanup(func() { + encoding.RegisterCompressor(oldC) + }) + return c +} + +func (s) TestSetSendCompressorSuccess(t *testing.T) { + for _, tt := range []struct { + name string + desc string + dialOpts []grpc.DialOption + resCompressor string + wantCompressInvokes int32 + }{ + { + name: "identity_request_and_gzip_response", + desc: "request is uncompressed and response is gzip compressed", + resCompressor: "gzip", + wantCompressInvokes: 1, + }, + { + name: "gzip_request_and_identity_response", + desc: "request is gzip compressed and response is uncompressed with identity", + resCompressor: "identity", + dialOpts: []grpc.DialOption{ + // Use WithCompressor instead of UseCompressor to avoid counting + // the client's compressor usage. + grpc.WithCompressor(grpc.NewGZIPCompressor()), + }, + wantCompressInvokes: 0, + }, + } { + t.Run(tt.name, func(t *testing.T) { + t.Run("unary", func(t *testing.T) { + testUnarySetSendCompressorSuccess(t, tt.resCompressor, tt.wantCompressInvokes, tt.dialOpts) + }) + + t.Run("stream", func(t *testing.T) { + testStreamSetSendCompressorSuccess(t, tt.resCompressor, tt.wantCompressInvokes, tt.dialOpts) + }) + }) + } +} + +func testUnarySetSendCompressorSuccess(t *testing.T, resCompressor string, wantCompressInvokes int32, dialOpts []grpc.DialOption) { + wc := setupGzipWrapCompressor(t) + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { + if err := grpc.SetSendCompressor(ctx, resCompressor); err != nil { + return nil, err + } + return &testpb.Empty{}, nil + }, + } + if err := ss.Start(nil, dialOpts...); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("Unexpected unary call error, got: %v, want: nil", err) + } + + compressInvokes := atomic.LoadInt32(&wc.compressInvokes) + if compressInvokes != wantCompressInvokes { + t.Fatalf("Unexpected compress invokes, got:%d, want: %d", compressInvokes, wantCompressInvokes) + } +} + +func testStreamSetSendCompressorSuccess(t *testing.T, resCompressor string, wantCompressInvokes int32, dialOpts []grpc.DialOption) { + wc := setupGzipWrapCompressor(t) + ss := &stubserver.StubServer{ + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + if _, err := stream.Recv(); err != nil { + return err + } + + if err := grpc.SetSendCompressor(stream.Context(), resCompressor); err != nil { + return err + } + + return stream.Send(&testpb.StreamingOutputCallResponse{}) + }, + } + if err := ss.Start(nil, dialOpts...); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + s, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("Unexpected full duplex call error, got: %v, want: nil", err) + } + + if err := s.Send(&testpb.StreamingOutputCallRequest{}); err != nil { + t.Fatalf("Unexpected full duplex call send error, got: %v, want: nil", err) + } + + if _, err := s.Recv(); err != nil { + t.Fatalf("Unexpected full duplex recv error, got: %v, want: nil", err) + } + + compressInvokes := atomic.LoadInt32(&wc.compressInvokes) + if compressInvokes != wantCompressInvokes { + t.Fatalf("Unexpected compress invokes, got:%d, want: %d", compressInvokes, wantCompressInvokes) + } +} + +func (s) TestUnregisteredSetSendCompressorFailure(t *testing.T) { + resCompressor := "snappy2" + wantErr := status.Error(codes.Unknown, "unable to set send compressor: compressor not registered \"snappy2\"") + + t.Run("unary", func(t *testing.T) { + testUnarySetSendCompressorFailure(t, resCompressor, wantErr) + }) + + t.Run("stream", func(t *testing.T) { + testStreamSetSendCompressorFailure(t, resCompressor, wantErr) + }) +} + +func (s) TestUnadvertisedSetSendCompressorFailure(t *testing.T) { + // Disable client compressor advertisement. + defer func(b bool) { envconfig.AdvertiseCompressors = b }(envconfig.AdvertiseCompressors) + envconfig.AdvertiseCompressors = false + + resCompressor := "gzip" + wantErr := status.Error(codes.Unknown, "unable to set send compressor: client does not support compressor \"gzip\"") + + t.Run("unary", func(t *testing.T) { + testUnarySetSendCompressorFailure(t, resCompressor, wantErr) + }) + + t.Run("stream", func(t *testing.T) { + testStreamSetSendCompressorFailure(t, resCompressor, wantErr) + }) +} + +func testUnarySetSendCompressorFailure(t *testing.T, resCompressor string, wantErr error) { + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { + if err := grpc.SetSendCompressor(ctx, resCompressor); err != nil { + return nil, err + } + return &testpb.Empty{}, nil + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); !equalError(err, wantErr) { + t.Fatalf("Unexpected unary call error, got: %v, want: %v", err, wantErr) + } +} + +func testStreamSetSendCompressorFailure(t *testing.T, resCompressor string, wantErr error) { + ss := &stubserver.StubServer{ + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + if _, err := stream.Recv(); err != nil { + return err + } + + if err := grpc.SetSendCompressor(stream.Context(), resCompressor); err != nil { + return err + } + + return stream.Send(&testpb.StreamingOutputCallResponse{}) + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v, want: nil", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + s, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("Unexpected full duplex call error, got: %v, want: nil", err) + } + + if err := s.Send(&testpb.StreamingOutputCallRequest{}); err != nil { + t.Fatalf("Unexpected full duplex call send error, got: %v, want: nil", err) + } + + if _, err := s.Recv(); !equalError(err, wantErr) { + t.Fatalf("Unexpected full duplex recv error, got: %v, want: nil", err) + } +} + +func (s) TestUnarySetSendCompressorAfterHeaderSendFailure(t *testing.T) { + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { + // Send headers early and then set send compressor. + grpc.SendHeader(ctx, metadata.MD{}) + err := grpc.SetSendCompressor(ctx, "gzip") + if err == nil { + t.Error("Wanted set send compressor error") + return &testpb.Empty{}, nil + } + return nil, err + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + wantErr := status.Error(codes.Unknown, "transport: set send compressor called after headers sent or stream done") + if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); !equalError(err, wantErr) { + t.Fatalf("Unexpected unary call error, got: %v, want: %v", err, wantErr) + } +} + +func (s) TestStreamSetSendCompressorAfterHeaderSendFailure(t *testing.T) { + ss := &stubserver.StubServer{ + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + // Send headers early and then set send compressor. + grpc.SendHeader(stream.Context(), metadata.MD{}) + err := grpc.SetSendCompressor(stream.Context(), "gzip") + if err == nil { + t.Error("Wanted set send compressor error") + } + return err + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + wantErr := status.Error(codes.Unknown, "transport: set send compressor called after headers sent or stream done") + s, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("Unexpected full duplex call error, got: %v, want: nil", err) + } + + if _, err := s.Recv(); !equalError(err, wantErr) { + t.Fatalf("Unexpected full duplex recv error, got: %v, want: %v", err, wantErr) + } +} + +func (s) TestClientSupportedCompressors(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + for _, tt := range []struct { + desc string + ctx context.Context + want []string + }{ + { + desc: "No additional grpc-accept-encoding header", + ctx: ctx, + want: []string{"gzip"}, + }, + { + desc: "With additional grpc-accept-encoding header", + ctx: metadata.AppendToOutgoingContext(ctx, + "grpc-accept-encoding", "test-compressor-1", + "grpc-accept-encoding", "test-compressor-2", + ), + want: []string{"gzip", "test-compressor-1", "test-compressor-2"}, + }, + { + desc: "With additional empty grpc-accept-encoding header", + ctx: metadata.AppendToOutgoingContext(ctx, + "grpc-accept-encoding", "", + ), + want: []string{"gzip"}, + }, + } { + t.Run(tt.desc, func(t *testing.T) { + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { + got, err := grpc.ClientSupportedCompressors(ctx) + if err != nil { + return nil, err + } + + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("unexpected client compressors got: %v, want: %v", got, tt.want) + } + + return &testpb.Empty{}, nil + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v, want: nil", err) + } + defer ss.Stop() + + _, err := ss.Client.EmptyCall(tt.ctx, &testpb.Empty{}) + if err != nil { + t.Fatalf("Unexpected unary call error, got: %v, want: nil", err) + } + }) + } +} + +func (s) TestCompressorRegister(t *testing.T) { + for _, e := range listTestEnv() { + testCompressorRegister(t, e) + } +} + +func testCompressorRegister(t *testing.T, e env) { + te := newTest(t, e) + te.clientCompression = false + te.serverCompression = false + te.clientUseCompression = true + + te.startServer(&testServer{security: e.security}) + defer te.tearDown() + tc := testgrpc.NewTestServiceClient(te.clientConn()) + + // Unary call + const argSize = 271828 + const respSize = 314159 + payload, err := newPayload(testpb.PayloadType_COMPRESSABLE, argSize) + if err != nil { + t.Fatal(err) + } + req := &testpb.SimpleRequest{ + ResponseType: testpb.PayloadType_COMPRESSABLE, + ResponseSize: respSize, + Payload: payload, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctx = metadata.NewOutgoingContext(ctx, metadata.Pairs("something", "something")) + if _, err := tc.UnaryCall(ctx, req); err != nil { + t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, ", err) + } + // Streaming RPC + stream, err := tc.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("%v.FullDuplexCall(_) = _, %v, want ", tc, err) + } + respParam := []*testpb.ResponseParameters{ + { + Size: 31415, + }, + } + payload, err = newPayload(testpb.PayloadType_COMPRESSABLE, int32(31415)) + if err != nil { + t.Fatal(err) + } + sreq := &testpb.StreamingOutputCallRequest{ + ResponseType: testpb.PayloadType_COMPRESSABLE, + ResponseParameters: respParam, + Payload: payload, + } + if err := stream.Send(sreq); err != nil { + t.Fatalf("%v.Send(%v) = %v, want ", stream, sreq, err) + } + if _, err := stream.Recv(); err != nil { + t.Fatalf("%v.Recv() = %v, want ", stream, err) + } +} + +type badGzipCompressor struct{} + +func (badGzipCompressor) Do(w io.Writer, p []byte) error { + buf := &bytes.Buffer{} + gzw := gzip.NewWriter(buf) + if _, err := gzw.Write(p); err != nil { + return err + } + err := gzw.Close() + bs := buf.Bytes() + if len(bs) >= 6 { + bs[len(bs)-6] ^= 1 // modify checksum at end by 1 byte + } + w.Write(bs) + return err +} + +func (badGzipCompressor) Type() string { + return "gzip" +} + +func (s) TestGzipBadChecksum(t *testing.T) { + ss := &stubserver.StubServer{ + UnaryCallF: func(ctx context.Context, _ *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{}, nil + }, + } + if err := ss.Start(nil, grpc.WithCompressor(badGzipCompressor{})); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + p, err := newPayload(testpb.PayloadType_COMPRESSABLE, int32(1024)) + if err != nil { + t.Fatalf("Unexpected error from newPayload: %v", err) + } + if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{Payload: p}); err == nil || + status.Code(err) != codes.Internal || + !strings.Contains(status.Convert(err).Message(), gzip.ErrChecksum.Error()) { + t.Errorf("ss.Client.UnaryCall(_) = _, %v\n\twant: _, status(codes.Internal, contains %q)", err, gzip.ErrChecksum) + } +} diff --git a/test/config_selector_test.go b/test/config_selector_test.go new file mode 100644 index 000000000000..69782ef9ce0b --- /dev/null +++ b/test/config_selector_test.go @@ -0,0 +1,216 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "google.golang.org/grpc/codes" + iresolver "google.golang.org/grpc/internal/resolver" + "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + testpb "google.golang.org/grpc/interop/grpc_testing" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/status" +) + +type funcConfigSelector struct { + f func(iresolver.RPCInfo) (*iresolver.RPCConfig, error) +} + +func (f funcConfigSelector) SelectConfig(i iresolver.RPCInfo) (*iresolver.RPCConfig, error) { + return f.f(i) +} + +func (s) TestConfigSelector(t *testing.T) { + gotContextChan := testutils.NewChannelWithSize(1) + + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { + gotContextChan.SendContext(ctx, ctx) + return &testpb.Empty{}, nil + }, + } + ss.R = manual.NewBuilderWithScheme("confSel") + + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + const normalTimeout = 10 * time.Second + ctxDeadline := time.Now().Add(normalTimeout) + ctx, cancel := context.WithTimeout(context.Background(), normalTimeout) + defer cancel() + + const longTimeout = 30 * time.Second + longCtxDeadline := time.Now().Add(longTimeout) + longdeadlineCtx, cancel := context.WithTimeout(context.Background(), longTimeout) + defer cancel() + shorterTimeout := 3 * time.Second + + testMD := metadata.MD{"footest": []string{"bazbar"}} + mdOut := metadata.MD{"handler": []string{"value"}} + + var onCommittedCalled bool + + testCases := []struct { + name string + md metadata.MD // MD sent with RPC + config *iresolver.RPCConfig // config returned by config selector + csErr error // error returned by config selector + + wantMD metadata.MD + wantDeadline time.Time + wantTimeout time.Duration + wantErr error + }{{ + name: "basic", + md: testMD, + config: &iresolver.RPCConfig{}, + wantMD: testMD, + wantDeadline: ctxDeadline, + }, { + name: "alter MD", + md: testMD, + config: &iresolver.RPCConfig{ + Context: metadata.NewOutgoingContext(ctx, mdOut), + }, + wantMD: mdOut, + wantDeadline: ctxDeadline, + }, { + name: "erroring SelectConfig", + csErr: status.Errorf(codes.Unavailable, "cannot send RPC"), + wantErr: status.Errorf(codes.Unavailable, "cannot send RPC"), + }, { + name: "alter timeout; remove MD", + md: testMD, + config: &iresolver.RPCConfig{ + Context: longdeadlineCtx, // no metadata + }, + wantMD: nil, + wantDeadline: longCtxDeadline, + }, { + name: "nil config", + md: metadata.MD{}, + config: nil, + wantMD: nil, + wantDeadline: ctxDeadline, + }, { + name: "alter timeout via method config; remove MD", + md: testMD, + config: &iresolver.RPCConfig{ + MethodConfig: serviceconfig.MethodConfig{ + Timeout: &shorterTimeout, + }, + }, + wantMD: nil, + wantTimeout: shorterTimeout, + }, { + name: "onCommitted callback", + md: testMD, + config: &iresolver.RPCConfig{ + OnCommitted: func() { + onCommittedCalled = true + }, + }, + wantMD: testMD, + wantDeadline: ctxDeadline, + }} + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var gotInfo *iresolver.RPCInfo + state := iresolver.SetConfigSelector(resolver.State{ + Addresses: []resolver.Address{{Addr: ss.Address}}, + ServiceConfig: parseServiceConfig(t, ss.R, "{}"), + }, funcConfigSelector{ + f: func(i iresolver.RPCInfo) (*iresolver.RPCConfig, error) { + gotInfo = &i + cfg := tc.config + if cfg != nil && cfg.Context == nil { + cfg.Context = i.Context + } + return cfg, tc.csErr + }, + }) + ss.R.UpdateState(state) // Blocks until config selector is applied + + onCommittedCalled = false + ctx := metadata.NewOutgoingContext(ctx, tc.md) + startTime := time.Now() + if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); fmt.Sprint(err) != fmt.Sprint(tc.wantErr) { + t.Fatalf("client.EmptyCall(_, _) = _, %v; want _, %v", err, tc.wantErr) + } else if err != nil { + return // remaining checks are invalid + } + + if gotInfo == nil { + t.Fatalf("no config selector data") + } + + if want := "/grpc.testing.TestService/EmptyCall"; gotInfo.Method != want { + t.Errorf("gotInfo.Method = %q; want %q", gotInfo.Method, want) + } + + gotContextI, ok := gotContextChan.ReceiveOrFail() + if !ok { + t.Fatalf("no context received") + } + gotContext := gotContextI.(context.Context) + + gotMD, _ := metadata.FromOutgoingContext(gotInfo.Context) + if diff := cmp.Diff(tc.md, gotMD); diff != "" { + t.Errorf("gotInfo.Context contains MD %v; want %v\nDiffs: %v", gotMD, tc.md, diff) + } + + gotMD, _ = metadata.FromIncomingContext(gotContext) + // Remove entries from gotMD not in tc.wantMD (e.g. authority header). + for k := range gotMD { + if _, ok := tc.wantMD[k]; !ok { + delete(gotMD, k) + } + } + if diff := cmp.Diff(tc.wantMD, gotMD, cmpopts.EquateEmpty()); diff != "" { + t.Errorf("received md = %v; want %v\nDiffs: %v", gotMD, tc.wantMD, diff) + } + + wantDeadline := tc.wantDeadline + if wantDeadline == (time.Time{}) { + wantDeadline = startTime.Add(tc.wantTimeout) + } + deadlineGot, _ := gotContext.Deadline() + if diff := deadlineGot.Sub(wantDeadline); diff > time.Second || diff < -time.Second { + t.Errorf("received deadline = %v; want ~%v", deadlineGot, wantDeadline) + } + + if tc.config != nil && tc.config.OnCommitted != nil && !onCommittedCalled { + t.Errorf("OnCommitted callback not called") + } + }) + } +} diff --git a/test/context_canceled_test.go b/test/context_canceled_test.go index 781f63f0c04e..510de99c4f28 100644 --- a/test/context_canceled_test.go +++ b/test/context_canceled_test.go @@ -26,14 +26,17 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/encoding/gzip" + "google.golang.org/grpc/internal/stubserver" "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" - testpb "google.golang.org/grpc/test/grpc_testing" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" ) func (s) TestContextCanceled(t *testing.T) { - ss := &stubServer{ - fullDuplexCall: func(stream testpb.TestService_FullDuplexCallServer) error { + ss := &stubserver.StubServer{ + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { stream.SetTrailer(metadata.New(map[string]string{"a": "b"})) return status.Error(codes.PermissionDenied, "perm denied") }, @@ -51,7 +54,7 @@ func (s) TestContextCanceled(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), delay) defer cancel() - str, err := ss.client.FullDuplexCall(ctx) + str, err := ss.Client.FullDuplexCall(ctx) if err != nil { continue } @@ -121,8 +124,8 @@ func (s) TestContextCanceled(t *testing.T) { // first one, but `case ctx.Done()` wins the second one, the compression info // will be inconsistent, and it causes internal error. func (s) TestCancelWhileRecvingWithCompression(t *testing.T) { - ss := &stubServer{ - fullDuplexCall: func(stream testpb.TestService_FullDuplexCallServer) error { + ss := &stubserver.StubServer{ + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { for { if err := stream.Send(&testpb.StreamingOutputCallResponse{ Payload: nil, @@ -138,8 +141,8 @@ func (s) TestCancelWhileRecvingWithCompression(t *testing.T) { defer ss.Stop() for i := 0; i < 10; i++ { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - s, err := ss.client.FullDuplexCall(ctx, grpc.UseCompressor(gzip.Name)) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + s, err := ss.Client.FullDuplexCall(ctx, grpc.UseCompressor(gzip.Name)) if err != nil { t.Fatalf("failed to start bidi streaming RPC: %v", err) } diff --git a/test/control_plane_status_test.go b/test/control_plane_status_test.go new file mode 100644 index 000000000000..087dd30dd670 --- /dev/null +++ b/test/control_plane_status_test.go @@ -0,0 +1,234 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "context" + "strings" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/base" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/internal/balancer/stub" + iresolver "google.golang.org/grpc/internal/resolver" + "google.golang.org/grpc/internal/stubserver" + testpb "google.golang.org/grpc/interop/grpc_testing" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/status" +) + +func (s) TestConfigSelectorStatusCodes(t *testing.T) { + testCases := []struct { + name string + csErr error + want error + }{{ + name: "legal status code", + csErr: status.Errorf(codes.Unavailable, "this error is fine"), + want: status.Errorf(codes.Unavailable, "this error is fine"), + }, { + name: "illegal status code", + csErr: status.Errorf(codes.NotFound, "this error is bad"), + want: status.Errorf(codes.Internal, "this error is bad"), + }} + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + } + ss.R = manual.NewBuilderWithScheme("confSel") + + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + state := iresolver.SetConfigSelector(resolver.State{ + Addresses: []resolver.Address{{Addr: ss.Address}}, + ServiceConfig: parseServiceConfig(t, ss.R, "{}"), + }, funcConfigSelector{ + f: func(i iresolver.RPCInfo) (*iresolver.RPCConfig, error) { + return nil, tc.csErr + }, + }) + ss.R.UpdateState(state) // Blocks until config selector is applied + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != status.Code(tc.want) || !strings.Contains(err.Error(), status.Convert(tc.want).Message()) { + t.Fatalf("client.EmptyCall(_, _) = _, %v; want _, %v", err, tc.want) + } + }) + } +} + +func (s) TestPickerStatusCodes(t *testing.T) { + testCases := []struct { + name string + pickerErr error + want error + }{{ + name: "legal status code", + pickerErr: status.Errorf(codes.Unavailable, "this error is fine"), + want: status.Errorf(codes.Unavailable, "this error is fine"), + }, { + name: "illegal status code", + pickerErr: status.Errorf(codes.NotFound, "this error is bad"), + want: status.Errorf(codes.Internal, "this error is bad"), + }} + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + } + + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + // Create a stub balancer that creates a picker that always returns + // an error. + sbf := stub.BalancerFuncs{ + UpdateClientConnState: func(d *stub.BalancerData, _ balancer.ClientConnState) error { + d.ClientConn.UpdateState(balancer.State{ + ConnectivityState: connectivity.TransientFailure, + Picker: base.NewErrPicker(tc.pickerErr), + }) + return nil + }, + } + stub.Register("testPickerStatusCodesBalancer", sbf) + + ss.NewServiceConfig(`{"loadBalancingConfig": [{"testPickerStatusCodesBalancer":{}}] }`) + + // Make calls until pickerErr is received. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + var lastErr error + for ctx.Err() == nil { + if _, lastErr = ss.Client.EmptyCall(ctx, &testpb.Empty{}); status.Code(lastErr) == status.Code(tc.want) && strings.Contains(lastErr.Error(), status.Convert(tc.want).Message()) { + // Success! + return + } + time.Sleep(time.Millisecond) + } + + t.Fatalf("client.EmptyCall(_, _) = _, %v; want _, %v", lastErr, tc.want) + }) + } +} + +func (s) TestCallCredsFromDialOptionsStatusCodes(t *testing.T) { + testCases := []struct { + name string + credsErr error + want error + }{{ + name: "legal status code", + credsErr: status.Errorf(codes.Unavailable, "this error is fine"), + want: status.Errorf(codes.Unavailable, "this error is fine"), + }, { + name: "illegal status code", + credsErr: status.Errorf(codes.NotFound, "this error is bad"), + want: status.Errorf(codes.Internal, "this error is bad"), + }} + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + } + + errChan := make(chan error, 1) + creds := &testPerRPCCredentials{errChan: errChan} + + if err := ss.Start(nil, grpc.WithPerRPCCredentials(creds)); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + errChan <- tc.credsErr + + if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != status.Code(tc.want) || !strings.Contains(err.Error(), status.Convert(tc.want).Message()) { + t.Fatalf("client.EmptyCall(_, _) = _, %v; want _, %v", err, tc.want) + } + }) + } +} + +func (s) TestCallCredsFromCallOptionsStatusCodes(t *testing.T) { + testCases := []struct { + name string + credsErr error + want error + }{{ + name: "legal status code", + credsErr: status.Errorf(codes.Unavailable, "this error is fine"), + want: status.Errorf(codes.Unavailable, "this error is fine"), + }, { + name: "illegal status code", + credsErr: status.Errorf(codes.NotFound, "this error is bad"), + want: status.Errorf(codes.Internal, "this error is bad"), + }} + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + } + + errChan := make(chan error, 1) + creds := &testPerRPCCredentials{errChan: errChan} + + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + errChan <- tc.credsErr + + if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}, grpc.PerRPCCredentials(creds)); status.Code(err) != status.Code(tc.want) || !strings.Contains(err.Error(), status.Convert(tc.want).Message()) { + t.Fatalf("client.EmptyCall(_, _) = _, %v; want _, %v", err, tc.want) + } + }) + } +} diff --git a/test/creds_test.go b/test/creds_test.go index 46bdd30dc85e..ef73fe936c80 100644 --- a/test/creds_test.go +++ b/test/creds_test.go @@ -31,13 +31,17 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/connectivity" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/testutils" "google.golang.org/grpc/metadata" "google.golang.org/grpc/resolver" "google.golang.org/grpc/resolver/manual" "google.golang.org/grpc/status" "google.golang.org/grpc/tap" - testpb "google.golang.org/grpc/test/grpc_testing" "google.golang.org/grpc/testdata" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" ) const ( @@ -52,7 +56,7 @@ type testCredsBundle struct { func (c *testCredsBundle) TransportCredentials() credentials.TransportCredentials { if c.mode == bundlePerRPCOnly { - return nil + return insecure.NewCredentials() } creds, err := credentials.NewClientTLSFromFile(testdata.Path("x509/server_ca_cert.pem"), "x.test.example.com") @@ -67,7 +71,7 @@ func (c *testCredsBundle) PerRPCCredentials() credentials.PerRPCCredentials { if c.mode == bundleTLSOnly { return nil } - return testPerRPCCredentials{} + return testPerRPCCredentials{authdata: authdata} } func (c *testCredsBundle) NewWithMode(mode string) (credentials.Bundle, error) { @@ -91,8 +95,10 @@ func (s) TestCredsBundleBoth(t *testing.T) { defer te.tearDown() cc := te.clientConn() - tc := testpb.NewTestServiceClient(cc) - if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); err != nil { + tc := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil { t.Fatalf("Test failed. Reason: %v", err) } } @@ -113,8 +119,10 @@ func (s) TestCredsBundleTransportCredentials(t *testing.T) { defer te.tearDown() cc := te.clientConn() - tc := testpb.NewTestServiceClient(cc) - if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); err != nil { + tc := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil { t.Fatalf("Test failed. Reason: %v", err) } } @@ -129,8 +137,10 @@ func (s) TestCredsBundlePerRPCCredentials(t *testing.T) { defer te.tearDown() cc := te.clientConn() - tc := testpb.NewTestServiceClient(cc) - if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); err != nil { + tc := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil { t.Fatalf("Test failed. Reason: %v", err) } } @@ -163,9 +173,11 @@ func (s) TestNonFailFastRPCSucceedOnTimeoutCreds(t *testing.T) { defer te.tearDown() cc := te.clientConn(grpc.WithTransportCredentials(&clientTimeoutCreds{})) - tc := testpb.NewTestServiceClient(cc) + tc := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() // This unary call should succeed, because ClientHandshake will succeed for the second time. - if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { te.t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want ", err) } } @@ -187,9 +199,9 @@ func (s) TestGRPCMethodAccessibleToCredsViaContextRequestInfo(t *testing.T) { defer te.tearDown() cc := te.clientConn(grpc.WithPerRPCCredentials(&methodTestCreds{})) - tc := testpb.NewTestServiceClient(cc) + tc := testgrpc.NewTestServiceClient(cc) - ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); status.Convert(err).Message() != wantMethod { t.Fatalf("ss.client.EmptyCall(_, _) = _, %v; want _, _.Message()=%q", err, wantMethod) @@ -222,7 +234,7 @@ func (s) TestFailFastRPCErrorOnBadCertificates(t *testing.T) { defer te.tearDown() opts := []grpc.DialOption{grpc.WithTransportCredentials(clientAlwaysFailCred{})} - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() cc, err := grpc.DialContext(ctx, te.srvAddr, opts...) if err != nil { @@ -230,13 +242,13 @@ func (s) TestFailFastRPCErrorOnBadCertificates(t *testing.T) { } defer cc.Close() - tc := testpb.NewTestServiceClient(cc) + tc := testgrpc.NewTestServiceClient(cc) for i := 0; i < 1000; i++ { // This loop runs for at most 1 second. The first several RPCs will fail // with Unavailable because the connection hasn't started. When the // first connection failed with creds error, the next RPC should also // fail with the expected error. - if _, err = tc.EmptyCall(context.Background(), &testpb.Empty{}); strings.Contains(err.Error(), clientAlwaysFailCredErrorMsg) { + if _, err = tc.EmptyCall(ctx, &testpb.Empty{}); strings.Contains(err.Error(), clientAlwaysFailCredErrorMsg) { return } time.Sleep(time.Millisecond) @@ -250,21 +262,20 @@ func (s) TestWaitForReadyRPCErrorOnBadCertificates(t *testing.T) { defer te.tearDown() opts := []grpc.DialOption{grpc.WithTransportCredentials(clientAlwaysFailCred{})} - dctx, dcancel := context.WithTimeout(context.Background(), 10*time.Second) - defer dcancel() - cc, err := grpc.DialContext(dctx, te.srvAddr, opts...) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + cc, err := grpc.DialContext(ctx, te.srvAddr, opts...) if err != nil { t.Fatalf("Dial(_) = %v, want %v", err, nil) } defer cc.Close() - tc := testpb.NewTestServiceClient(cc) - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + tc := testgrpc.NewTestServiceClient(cc) + ctx, cancel = context.WithTimeout(context.Background(), defaultTestShortTimeout) defer cancel() - if _, err = tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); strings.Contains(err.Error(), clientAlwaysFailCredErrorMsg) { - return + if _, err = tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); !strings.Contains(err.Error(), clientAlwaysFailCredErrorMsg) { + t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want err.Error() contains %q", err, clientAlwaysFailCredErrorMsg) } - te.t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want err.Error() contains %q", err, clientAlwaysFailCredErrorMsg) } var ( @@ -275,10 +286,17 @@ var ( } ) -type testPerRPCCredentials struct{} +type testPerRPCCredentials struct { + authdata map[string]string + errChan chan error +} func (cr testPerRPCCredentials) GetRequestMetadata(ctx context.Context, uri ...string) (map[string]string, error) { - return authdata, nil + var err error + if cr.errChan != nil { + err = <-cr.errChan + } + return cr.authdata, err } func (cr testPerRPCCredentials) RequireTransportSecurity() bool { @@ -311,13 +329,15 @@ func (s) TestPerRPCCredentialsViaDialOptions(t *testing.T) { func testPerRPCCredentialsViaDialOptions(t *testing.T, e env) { te := newTest(t, e) te.tapHandle = authHandle - te.perRPCCreds = testPerRPCCredentials{} + te.perRPCCreds = testPerRPCCredentials{authdata: authdata} te.startServer(&testServer{security: e.security}) defer te.tearDown() cc := te.clientConn() - tc := testpb.NewTestServiceClient(cc) - if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); err != nil { + tc := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil { t.Fatalf("Test failed. Reason: %v", err) } } @@ -335,8 +355,10 @@ func testPerRPCCredentialsViaCallOptions(t *testing.T, e env) { defer te.tearDown() cc := te.clientConn() - tc := testpb.NewTestServiceClient(cc) - if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}, grpc.PerRPCCredentials(testPerRPCCredentials{})); err != nil { + tc := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.PerRPCCredentials(testPerRPCCredentials{authdata: authdata})); err != nil { t.Fatalf("Test failed. Reason: %v", err) } } @@ -349,7 +371,7 @@ func (s) TestPerRPCCredentialsViaDialOptionsAndCallOptions(t *testing.T) { func testPerRPCCredentialsViaDialOptionsAndCallOptions(t *testing.T, e env) { te := newTest(t, e) - te.perRPCCreds = testPerRPCCredentials{} + te.perRPCCreds = testPerRPCCredentials{authdata: authdata} // When credentials are provided via both dial options and call options, // we apply both sets. te.tapHandle = func(ctx context.Context, _ *tap.Info) (context.Context, error) { @@ -375,8 +397,10 @@ func testPerRPCCredentialsViaDialOptionsAndCallOptions(t *testing.T, e env) { defer te.tearDown() cc := te.clientConn() - tc := testpb.NewTestServiceClient(cc) - if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}, grpc.PerRPCCredentials(testPerRPCCredentials{})); err != nil { + tc := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.PerRPCCredentials(testPerRPCCredentials{authdata: authdata})); err != nil { t.Fatalf("Test failed. Reason: %v", err) } } @@ -420,17 +444,9 @@ func (s) TestCredsHandshakeAuthority(t *testing.T) { defer cc.Close() r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: lis.Addr().String()}}}) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() - for { - s := cc.GetState() - if s == connectivity.Ready { - break - } - if !cc.WaitForStateChange(ctx, s) { - t.Fatalf("ClientConn is not ready after 100 ms") - } - } + testutils.AwaitState(ctx, t, cc, connectivity.Ready) if cred.got != testAuthority { t.Fatalf("client creds got authority: %q, want: %q", cred.got, testAuthority) @@ -460,17 +476,9 @@ func (s) TestCredsHandshakeServerNameAuthority(t *testing.T) { defer cc.Close() r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: lis.Addr().String(), ServerName: testServerName}}}) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() - for { - s := cc.GetState() - if s == connectivity.Ready { - break - } - if !cc.WaitForStateChange(ctx, s) { - t.Fatalf("ClientConn is not ready after 100 ms") - } - } + testutils.AwaitState(ctx, t, cc, connectivity.Ready) if cred.got != testServerName { t.Fatalf("client creds got authority: %q, want: %q", cred.got, testAuthority) diff --git a/test/end2end_test.go b/test/end2end_test.go index f26997ea808b..9d1b4a78b0ce 100644 --- a/test/end2end_test.go +++ b/test/end2end_test.go @@ -21,9 +21,9 @@ package test import ( "bufio" "bytes" - "compress/gzip" "context" "crypto/tls" + "encoding/json" "errors" "flag" "fmt" @@ -42,22 +42,23 @@ import ( "time" "github.com/golang/protobuf/proto" - anypb "github.com/golang/protobuf/ptypes/any" "golang.org/x/net/http2" "golang.org/x/net/http2/hpack" - spb "google.golang.org/genproto/googleapis/rpc/status" "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/roundrobin" "google.golang.org/grpc/codes" "google.golang.org/grpc/connectivity" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/encoding" - _ "google.golang.org/grpc/encoding/gzip" "google.golang.org/grpc/health" - healthgrpc "google.golang.org/grpc/health/grpc_health_v1" - healthpb "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/binarylog" "google.golang.org/grpc/internal/channelz" "google.golang.org/grpc/internal/grpcsync" "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/stubserver" "google.golang.org/grpc/internal/testutils" "google.golang.org/grpc/internal/transport" "google.golang.org/grpc/metadata" @@ -68,14 +69,24 @@ import ( "google.golang.org/grpc/stats" "google.golang.org/grpc/status" "google.golang.org/grpc/tap" - testpb "google.golang.org/grpc/test/grpc_testing" + "google.golang.org/grpc/test/bufconn" "google.golang.org/grpc/testdata" + + anypb "github.com/golang/protobuf/ptypes/any" + spb "google.golang.org/genproto/googleapis/rpc/status" + healthgrpc "google.golang.org/grpc/health/grpc_health_v1" + healthpb "google.golang.org/grpc/health/grpc_health_v1" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + + _ "google.golang.org/grpc/encoding/gzip" ) const defaultHealthService = "grpc.health.v1.Health" func init() { channelz.TurnOn() + balancer.Register(triggerRPCBlockPickerBalancerBuilder{}) } type s struct { @@ -126,6 +137,8 @@ var ( var raceMode bool // set by race.go in race mode type testServer struct { + testgrpc.UnimplementedTestServiceServer + security string // indicate the authentication protocol used by this server. earlyFail bool // whether to error out the execution of a service handler prematurely. setAndSendHeader bool // whether to call setHeader and sendHeader. @@ -134,8 +147,6 @@ type testServer struct { unaryCallSleepTime time.Duration } -var _ testpb.UnstableTestServiceService = (*testServer)(nil) - func (s *testServer) EmptyCall(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { if md, ok := metadata.FromIncomingContext(ctx); ok { // For testing purpose, returns an error if user-agent is failAppUA. @@ -159,8 +170,6 @@ func newPayload(t testpb.PayloadType, size int32) (*testpb.Payload, error) { body := make([]byte, size) switch t { case testpb.PayloadType_COMPRESSABLE: - case testpb.PayloadType_UNCOMPRESSABLE: - return nil, fmt.Errorf("PayloadType UNCOMPRESSABLE is not supported") default: return nil, fmt.Errorf("unsupported payload type: %d", t) } @@ -241,7 +250,7 @@ func (s *testServer) UnaryCall(ctx context.Context, in *testpb.SimpleRequest) (* }, nil } -func (s *testServer) StreamingOutputCall(args *testpb.StreamingOutputCallRequest, stream testpb.TestService_StreamingOutputCallServer) error { +func (s *testServer) StreamingOutputCall(args *testpb.StreamingOutputCallRequest, stream testgrpc.TestService_StreamingOutputCallServer) error { if md, ok := metadata.FromIncomingContext(stream.Context()); ok { if _, exists := md[":authority"]; !exists { return status.Errorf(codes.DataLoss, "expected an :authority metadata: %v", md) @@ -272,7 +281,7 @@ func (s *testServer) StreamingOutputCall(args *testpb.StreamingOutputCallRequest return nil } -func (s *testServer) StreamingInputCall(stream testpb.TestService_StreamingInputCallServer) error { +func (s *testServer) StreamingInputCall(stream testgrpc.TestService_StreamingInputCallServer) error { var sum int for { in, err := stream.Recv() @@ -292,7 +301,7 @@ func (s *testServer) StreamingInputCall(stream testpb.TestService_StreamingInput } } -func (s *testServer) FullDuplexCall(stream testpb.TestService_FullDuplexCallServer) error { +func (s *testServer) FullDuplexCall(stream testgrpc.TestService_FullDuplexCallServer) error { md, ok := metadata.FromIncomingContext(stream.Context()) if ok { if s.setAndSendHeader { @@ -356,7 +365,7 @@ func (s *testServer) FullDuplexCall(stream testpb.TestService_FullDuplexCallServ } } -func (s *testServer) HalfDuplexCall(stream testpb.TestService_HalfDuplexCallServer) error { +func (s *testServer) HalfDuplexCall(stream testgrpc.TestService_HalfDuplexCallServer) error { var msgBuf []*testpb.StreamingOutputCallRequest for { in, err := stream.Recv() @@ -466,7 +475,7 @@ type test struct { // expose the server's health using the default health service // implementation. This should only be used when a non-default health service // implementation is required. - healthServer healthpb.HealthServer + healthServer healthgrpc.HealthServer maxStream uint32 tapHandle tap.ServerInHandle maxServerMsgSize *int @@ -505,19 +514,15 @@ type test struct { customDialOptions []grpc.DialOption resolverScheme string - // All test dialing is blocking by default. Set this to true if dial - // should be non-blocking. - nonBlockingDial bool - // These are are set once startServer is called. The common case is to have // only one testServer. srv stopper - hSrv healthpb.HealthServer + hSrv healthgrpc.HealthServer srvAddr string // These are are set once startServers is called. srvs []stopper - hSrvs []healthpb.HealthServer + hSrvs []healthgrpc.HealthServer srvAddrs []string cc *grpc.ClientConn // nil until requested via clientConn @@ -562,11 +567,11 @@ func newTest(t *testing.T, e env) *test { e: e, maxStream: math.MaxUint32, } - te.ctx, te.cancel = context.WithCancel(context.Background()) + te.ctx, te.cancel = context.WithTimeout(context.Background(), defaultTestTimeout) return te } -func (te *test) listenAndServe(ts interface{}, listen func(network, address string) (net.Listener, error)) net.Listener { +func (te *test) listenAndServe(ts testgrpc.TestServiceServer, listen func(network, address string) (net.Listener, error)) net.Listener { te.t.Helper() te.t.Logf("Running test in %s environment...", te.e.name) sopts := []grpc.ServerOption{grpc.MaxConcurrentStreams(te.maxStream)} @@ -626,7 +631,7 @@ func (te *test) listenAndServe(ts interface{}, listen func(network, address stri sopts = append(sopts, te.customServerOptions...) s := grpc.NewServer(sopts...) if ts != nil { - testpb.RegisterTestServiceService(s, testpb.NewTestServiceService(ts)) + testgrpc.RegisterTestServiceServer(s, ts) } // Create a new default health server if enableHealthServer is set, or use @@ -684,27 +689,30 @@ type wrapHS struct { } func (w wrapHS) GracefulStop() { - w.s.Shutdown(context.Background()) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + w.s.Shutdown(ctx) } func (w wrapHS) Stop() { w.s.Close() + w.s.Handler.(*grpc.Server).Stop() } -func (te *test) startServerWithConnControl(ts interface{}) *listenerWrapper { +func (te *test) startServerWithConnControl(ts testgrpc.TestServiceServer) *listenerWrapper { l := te.listenAndServe(ts, listenWithConnControl) return l.(*listenerWrapper) } // startServer starts a gRPC server exposing the provided TestService // implementation. Callers should defer a call to te.tearDown to clean up -func (te *test) startServer(ts interface{}) { +func (te *test) startServer(ts testgrpc.TestServiceServer) { te.t.Helper() te.listenAndServe(ts, net.Listen) } // startServers starts 'num' gRPC servers exposing the provided TestService. -func (te *test) startServers(ts interface{}, num int) { +func (te *test) startServers(ts testgrpc.TestServiceServer, num int) { for i := 0; i < num; i++ { te.startServer(ts) te.srvs = append(te.srvs, te.srv.(*grpc.Server)) @@ -801,7 +809,7 @@ func (te *test) configDial(opts ...grpc.DialOption) ([]grpc.DialOption, string) case "empty": // Don't add any transport creds option. default: - opts = append(opts, grpc.WithInsecure()) + opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) } // TODO(bar) switch balancer case "pick_first". var scheme string @@ -811,7 +819,7 @@ func (te *test) configDial(opts ...grpc.DialOption) ([]grpc.DialOption, string) scheme = te.resolverScheme + ":///" } if te.e.balancer != "" { - opts = append(opts, grpc.WithBalancerName(te.e.balancer)) + opts = append(opts, grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, te.e.balancer))) } if te.clientInitialWindowSize > 0 { opts = append(opts, grpc.WithInitialWindowSize(te.clientInitialWindowSize)) @@ -825,10 +833,6 @@ func (te *test) configDial(opts ...grpc.DialOption) ([]grpc.DialOption, string) if te.customCodec != nil { opts = append(opts, grpc.WithDefaultCallOptions(grpc.ForceCodec(te.customCodec))) } - if !te.nonBlockingDial && te.srvAddr != "" { - // Only do a blocking dial if server is up. - opts = append(opts, grpc.WithBlock()) - } if te.srvAddr == "" { te.srvAddr = "client.side.only.test" } @@ -892,6 +896,32 @@ type lazyConn struct { beLazy int32 } +// possible conn closed errors. +const possibleConnResetMsg = "connection reset by peer" +const possibleEOFMsg = "error reading from server: EOF" + +// isConnClosedErr checks the error msg for possible conn closed messages. There +// is a raceyness in the timing of when TCP packets are sent from client to +// server, and when we tell the server to stop, so we need to check for both of +// these possible error messages: +// 1. If the call to ss.S.Stop() causes the server's sockets to close while +// there's still in-fight data from the client on the TCP connection, then +// the kernel can send an RST back to the client (also see +// https://stackoverflow.com/questions/33053507/econnreset-in-send-linux-c). +// Note that while this condition is expected to be rare due to the +// test httpServer start synchronization, in theory it should be possible, +// e.g. if the client sends a BDP ping at the right time. +// 2. If, for example, the call to ss.S.Stop() happens after the RPC headers +// have been received at the server, then the TCP connection can shutdown +// gracefully when the server's socket closes. +// 3. If there is an actual io.EOF received because the client stopped the stream. +func isConnClosedErr(err error) bool { + errContainsConnResetMsg := strings.Contains(err.Error(), possibleConnResetMsg) + errContainsEOFMsg := strings.Contains(err.Error(), possibleEOFMsg) + + return errContainsConnResetMsg || errContainsEOFMsg || err == io.EOF +} + func (l *lazyConn) Write(b []byte) (int, error) { if atomic.LoadInt32(&(l.beLazy)) == 1 { time.Sleep(time.Second) @@ -916,12 +946,14 @@ func (s) TestContextDeadlineNotIgnored(t *testing.T) { defer te.tearDown() cc := te.clientConn() - tc := testpb.NewTestServiceClient(cc) - if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); err != nil { + tc := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil { t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, ", err) } + cancel() atomic.StoreInt32(&(lc.beLazy), 1) - ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + ctx, cancel = context.WithTimeout(context.Background(), defaultTestShortTimeout) defer cancel() t1 := time.Now() if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.DeadlineExceeded { @@ -941,36 +973,24 @@ func (s) TestTimeoutOnDeadServer(t *testing.T) { func testTimeoutOnDeadServer(t *testing.T, e env) { te := newTest(t, e) te.userAgent = testAppUA - te.declareLogNoise( - "transport: http2Client.notifyError got notified that the client transport was broken EOF", - "grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing", - "grpc: addrConn.resetTransport failed to create client transport: connection error", - ) te.startServer(&testServer{security: e.security}) defer te.tearDown() cc := te.clientConn() - tc := testpb.NewTestServiceClient(cc) - if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + tc := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, ", err) } + // Wait for the client to report READY, stop the server, then wait for the + // client to notice the connection is gone. + testutils.AwaitState(ctx, t, cc, connectivity.Ready) te.srv.Stop() - - // Wait for the client to notice the connection is gone. - ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) - state := cc.GetState() - for ; state == connectivity.Ready && cc.WaitForStateChange(ctx, state); state = cc.GetState() { - } - cancel() - if state == connectivity.Ready { - t.Fatalf("Timed out waiting for non-ready state") - } - ctx, cancel = context.WithTimeout(context.Background(), time.Millisecond) - _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)) - cancel() - if e.balancer != "" && status.Code(err) != codes.DeadlineExceeded { - // If e.balancer == nil, the ac will stop reconnecting because the dialer returns non-temp error, - // the error will be an internal error. + testutils.AwaitNotState(ctx, t, cc, connectivity.Ready) + ctx, cancel = context.WithTimeout(ctx, defaultTestShortTimeout) + defer cancel() + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); status.Code(err) != codes.DeadlineExceeded { t.Fatalf("TestService/EmptyCall(%v, _) = _, %v, want _, error code: %s", ctx, err, codes.DeadlineExceeded) } awaitNewConnLogOutput() @@ -996,492 +1016,164 @@ func testServerGracefulStopIdempotent(t *testing.T, e env) { } } -func (s) TestServerGoAway(t *testing.T) { - for _, e := range listTestEnv() { - if e.name == "handler-tls" { - continue - } - testServerGoAway(t, e) - } -} - -func testServerGoAway(t *testing.T, e env) { - te := newTest(t, e) - te.userAgent = testAppUA - te.startServer(&testServer{security: e.security}) - defer te.tearDown() - - cc := te.clientConn() - tc := testpb.NewTestServiceClient(cc) - // Finish an RPC to make sure the connection is good. - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { - t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, ", err) - } - ch := make(chan struct{}) - go func() { - te.srv.GracefulStop() - close(ch) - }() - // Loop until the server side GoAway signal is propagated to the client. - for { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) - if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil && status.Code(err) != codes.DeadlineExceeded { - cancel() - break - } - cancel() - } - // A new RPC should fail. - ctx, cancel = context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.Unavailable && status.Code(err) != codes.Internal { - t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, %s or %s", err, codes.Unavailable, codes.Internal) +func (s) TestDetailedConnectionCloseErrorPropagatesToRpcError(t *testing.T) { + rpcStartedOnServer := make(chan struct{}) + rpcDoneOnClient := make(chan struct{}) + ss := &stubserver.StubServer{ + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + close(rpcStartedOnServer) + <-rpcDoneOnClient + return status.Error(codes.Internal, "arbitrary status") + }, } - <-ch - awaitNewConnLogOutput() -} - -func (s) TestServerGoAwayPendingRPC(t *testing.T) { - for _, e := range listTestEnv() { - if e.name == "handler-tls" { - continue - } - testServerGoAwayPendingRPC(t, e) + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) } -} + defer ss.Stop() -func testServerGoAwayPendingRPC(t *testing.T, e env) { - te := newTest(t, e) - te.userAgent = testAppUA - te.declareLogNoise( - "transport: http2Client.notifyError got notified that the client transport was broken EOF", - "grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing", - "grpc: addrConn.resetTransport failed to create client transport: connection error", - ) - te.startServer(&testServer{security: e.security}) - defer te.tearDown() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() - cc := te.clientConn() - tc := testpb.NewTestServiceClient(cc) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - stream, err := tc.FullDuplexCall(ctx, grpc.WaitForReady(true)) + // Start an RPC. Then, while the RPC is still being accepted or handled at the server, abruptly + // stop the server, killing the connection. The RPC error message should include details about the specific + // connection error that was encountered. + stream, err := ss.Client.FullDuplexCall(ctx) if err != nil { - t.Fatalf("%v.FullDuplexCall(_) = _, %v, want ", tc, err) - } - // Finish an RPC to make sure the connection is good. - if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { - t.Fatalf("%v.EmptyCall(_, _, _) = _, %v, want _, ", tc, err) - } - ch := make(chan struct{}) - go func() { - te.srv.GracefulStop() - close(ch) - }() - // Loop until the server side GoAway signal is propagated to the client. - start := time.Now() - errored := false - for time.Since(start) < time.Second { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) - _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)) - cancel() - if err != nil { - errored = true - break - } - } - if !errored { - t.Fatalf("GoAway never received by client") - } - respParam := []*testpb.ResponseParameters{{Size: 1}} - payload, err := newPayload(testpb.PayloadType_COMPRESSABLE, int32(100)) - if err != nil { - t.Fatal(err) - } - req := &testpb.StreamingOutputCallRequest{ - ResponseType: testpb.PayloadType_COMPRESSABLE, - ResponseParameters: respParam, - Payload: payload, - } - // The existing RPC should be still good to proceed. - if err := stream.Send(req); err != nil { - t.Fatalf("%v.Send(_) = %v, want ", stream, err) + t.Fatalf("%v.FullDuplexCall = _, %v, want _, ", ss.Client, err) } - if _, err := stream.Recv(); err != nil { - t.Fatalf("%v.Recv() = _, %v, want _, ", stream, err) + // Block until the RPC has been started on the server. This ensures that the ClientConn will find a healthy + // connection for the RPC to go out on initially, and that the TCP connection will shut down strictly after + // the RPC has been started on it. + <-rpcStartedOnServer + ss.S.Stop() + // The precise behavior of this test is subject to raceyness around the timing + // of when TCP packets are sent from client to server, and when we tell the + // server to stop, so we need to account for both possible error messages. + if _, err := stream.Recv(); err == io.EOF || !isConnClosedErr(err) { + t.Fatalf("%v.Recv() = _, %v, want _, rpc error containing substring: %q OR %q", stream, err, possibleConnResetMsg, possibleEOFMsg) } - // The RPC will run until canceled. - cancel() - <-ch - awaitNewConnLogOutput() + close(rpcDoneOnClient) } -func (s) TestServerMultipleGoAwayPendingRPC(t *testing.T) { +func (s) TestFailFast(t *testing.T) { for _, e := range listTestEnv() { - if e.name == "handler-tls" { - continue - } - testServerMultipleGoAwayPendingRPC(t, e) + testFailFast(t, e) } } -func testServerMultipleGoAwayPendingRPC(t *testing.T, e env) { +func testFailFast(t *testing.T, e env) { te := newTest(t, e) te.userAgent = testAppUA - te.declareLogNoise( - "transport: http2Client.notifyError got notified that the client transport was broken EOF", - "grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing", - "grpc: addrConn.resetTransport failed to create client transport: connection error", - ) te.startServer(&testServer{security: e.security}) defer te.tearDown() cc := te.clientConn() - tc := testpb.NewTestServiceClient(cc) - ctx, cancel := context.WithCancel(context.Background()) - stream, err := tc.FullDuplexCall(ctx, grpc.WaitForReady(true)) - if err != nil { - t.Fatalf("%v.FullDuplexCall(_) = _, %v, want ", tc, err) - } - // Finish an RPC to make sure the connection is good. - if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { - t.Fatalf("%v.EmptyCall(_, _, _) = _, %v, want _, ", tc, err) + tc := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, ", err) } - ch1 := make(chan struct{}) - go func() { - te.srv.GracefulStop() - close(ch1) - }() - ch2 := make(chan struct{}) - go func() { - te.srv.GracefulStop() - close(ch2) - }() - // Loop until the server side GoAway signal is propagated to the client. + // Stop the server and tear down all the existing connections. + te.srv.Stop() + // Loop until the server teardown is propagated to the client. for { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) - if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { - cancel() + if err := ctx.Err(); err != nil { + t.Fatalf("EmptyCall did not return UNAVAILABLE before timeout") + } + _, err := tc.EmptyCall(ctx, &testpb.Empty{}) + if status.Code(err) == codes.Unavailable { break } - cancel() - } - select { - case <-ch1: - t.Fatal("GracefulStop() terminated early") - case <-ch2: - t.Fatal("GracefulStop() terminated early") - default: - } - respParam := []*testpb.ResponseParameters{ - { - Size: 1, - }, - } - payload, err := newPayload(testpb.PayloadType_COMPRESSABLE, int32(100)) - if err != nil { - t.Fatal(err) - } - req := &testpb.StreamingOutputCallRequest{ - ResponseType: testpb.PayloadType_COMPRESSABLE, - ResponseParameters: respParam, - Payload: payload, - } - // The existing RPC should be still good to proceed. - if err := stream.Send(req); err != nil { - t.Fatalf("%v.Send(%v) = %v, want ", stream, req, err) + t.Logf("%v.EmptyCall(_, _) = _, %v", tc, err) + time.Sleep(10 * time.Millisecond) } - if _, err := stream.Recv(); err != nil { - t.Fatalf("%v.Recv() = _, %v, want _, ", stream, err) + // The client keeps reconnecting and ongoing fail-fast RPCs should fail with code.Unavailable. + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.Unavailable { + t.Fatalf("TestService/EmptyCall(_, _, _) = _, %v, want _, error code: %s", err, codes.Unavailable) } - if err := stream.CloseSend(); err != nil { - t.Fatalf("%v.CloseSend() = %v, want ", stream, err) + if _, err := tc.StreamingInputCall(ctx); status.Code(err) != codes.Unavailable { + t.Fatalf("TestService/StreamingInputCall(_) = _, %v, want _, error code: %s", err, codes.Unavailable) } - <-ch1 - <-ch2 - cancel() - awaitNewConnLogOutput() -} -func (s) TestConcurrentClientConnCloseAndServerGoAway(t *testing.T) { - for _, e := range listTestEnv() { - if e.name == "handler-tls" { - continue - } - testConcurrentClientConnCloseAndServerGoAway(t, e) - } + awaitNewConnLogOutput() } -func testConcurrentClientConnCloseAndServerGoAway(t *testing.T, e env) { +func testServiceConfigSetup(t *testing.T, e env) *test { te := newTest(t, e) te.userAgent = testAppUA te.declareLogNoise( - "transport: http2Client.notifyError got notified that the client transport was broken EOF", - "grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing", - "grpc: addrConn.resetTransport failed to create client transport: connection error", + "Failed to dial : context canceled; please retry.", ) - te.startServer(&testServer{security: e.security}) - defer te.tearDown() + return te +} - cc := te.clientConn() - tc := testpb.NewTestServiceClient(cc) - if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { - t.Fatalf("%v.EmptyCall(_, _, _) = _, %v, want _, ", tc, err) - } - ch := make(chan struct{}) - // Close ClientConn and Server concurrently. - go func() { - te.srv.GracefulStop() - close(ch) - }() - go func() { - cc.Close() - }() - <-ch +func newBool(b bool) (a *bool) { + return &b } -func (s) TestConcurrentServerStopAndGoAway(t *testing.T) { - for _, e := range listTestEnv() { - if e.name == "handler-tls" { - continue - } - testConcurrentServerStopAndGoAway(t, e) - } +func newInt(b int) (a *int) { + return &b } -func testConcurrentServerStopAndGoAway(t *testing.T, e env) { - te := newTest(t, e) - te.userAgent = testAppUA - te.declareLogNoise( - "transport: http2Client.notifyError got notified that the client transport was broken EOF", - "grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing", - "grpc: addrConn.resetTransport failed to create client transport: connection error", - ) - te.startServer(&testServer{security: e.security}) +func newDuration(b time.Duration) (a *time.Duration) { + a = new(time.Duration) + *a = b + return +} + +func (s) TestGetMethodConfig(t *testing.T) { + te := testServiceConfigSetup(t, tcpClearRREnv) defer te.tearDown() + r := manual.NewBuilderWithScheme("whatever") - cc := te.clientConn() - tc := testpb.NewTestServiceClient(cc) - stream, err := tc.FullDuplexCall(context.Background(), grpc.WaitForReady(true)) - if err != nil { - t.Fatalf("%v.FullDuplexCall(_) = _, %v, want ", tc, err) - } - // Finish an RPC to make sure the connection is good. - if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { - t.Fatalf("%v.EmptyCall(_, _, _) = _, %v, want _, ", tc, err) - } - ch := make(chan struct{}) - go func() { - te.srv.GracefulStop() - close(ch) - }() - // Loop until the server side GoAway signal is propagated to the client. + te.resolverScheme = r.Scheme() + cc := te.clientConn(grpc.WithResolvers(r)) + addrs := []resolver.Address{{Addr: te.srvAddr}} + r.UpdateState(resolver.State{ + Addresses: addrs, + ServiceConfig: parseServiceConfig(t, r, `{ + "methodConfig": [ + { + "name": [ + { + "service": "grpc.testing.TestService", + "method": "EmptyCall" + } + ], + "waitForReady": true, + "timeout": ".001s" + }, + { + "name": [ + { + "service": "grpc.testing.TestService" + } + ], + "waitForReady": false + } + ] +}`)}) + + tc := testgrpc.NewTestServiceClient(cc) + + // Make sure service config has been processed by grpc. for { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) - if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { - cancel() + if cc.GetMethodConfig("/grpc.testing.TestService/EmptyCall").WaitForReady != nil { break } - cancel() + time.Sleep(time.Millisecond) } - // Stop the server and close all the connections. - te.srv.Stop() - respParam := []*testpb.ResponseParameters{ - { - Size: 1, - }, - } - payload, err := newPayload(testpb.PayloadType_COMPRESSABLE, int32(100)) - if err != nil { - t.Fatal(err) - } - req := &testpb.StreamingOutputCallRequest{ - ResponseType: testpb.PayloadType_COMPRESSABLE, - ResponseParameters: respParam, - Payload: payload, - } - sendStart := time.Now() - for { - if err := stream.Send(req); err == io.EOF { - // stream.Send should eventually send io.EOF - break - } else if err != nil { - // Send should never return a transport-level error. - t.Fatalf("stream.Send(%v) = %v; want ", req, err) - } - if time.Since(sendStart) > 2*time.Second { - t.Fatalf("stream.Send(_) did not return io.EOF after 2s") - } - time.Sleep(time.Millisecond) - } - if _, err := stream.Recv(); err == nil || err == io.EOF { - t.Fatalf("%v.Recv() = _, %v, want _, ", stream, err) - } - <-ch - awaitNewConnLogOutput() -} - -func (s) TestClientConnCloseAfterGoAwayWithActiveStream(t *testing.T) { - for _, e := range listTestEnv() { - if e.name == "handler-tls" { - continue - } - testClientConnCloseAfterGoAwayWithActiveStream(t, e) - } -} - -func testClientConnCloseAfterGoAwayWithActiveStream(t *testing.T, e env) { - te := newTest(t, e) - te.startServer(&testServer{security: e.security}) - defer te.tearDown() - cc := te.clientConn() - tc := testpb.NewTestServiceClient(cc) - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - if _, err := tc.FullDuplexCall(ctx); err != nil { - t.Fatalf("%v.FullDuplexCall(_) = _, %v, want _, ", tc, err) - } - done := make(chan struct{}) - go func() { - te.srv.GracefulStop() - close(done) - }() - time.Sleep(50 * time.Millisecond) - cc.Close() - timeout := time.NewTimer(time.Second) - select { - case <-done: - case <-timeout.C: - t.Fatalf("Test timed-out.") - } -} - -func (s) TestFailFast(t *testing.T) { - for _, e := range listTestEnv() { - testFailFast(t, e) - } -} - -func testFailFast(t *testing.T, e env) { - te := newTest(t, e) - te.userAgent = testAppUA - te.declareLogNoise( - "transport: http2Client.notifyError got notified that the client transport was broken EOF", - "grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing", - "grpc: addrConn.resetTransport failed to create client transport: connection error", - ) - te.startServer(&testServer{security: e.security}) - defer te.tearDown() - - cc := te.clientConn() - tc := testpb.NewTestServiceClient(cc) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil { - t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, ", err) - } - // Stop the server and tear down all the existing connections. - te.srv.Stop() - // Loop until the server teardown is propagated to the client. - for { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - _, err := tc.EmptyCall(ctx, &testpb.Empty{}) - cancel() - if status.Code(err) == codes.Unavailable { - break - } - t.Logf("%v.EmptyCall(_, _) = _, %v", tc, err) - time.Sleep(10 * time.Millisecond) - } - // The client keeps reconnecting and ongoing fail-fast RPCs should fail with code.Unavailable. - if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); status.Code(err) != codes.Unavailable { - t.Fatalf("TestService/EmptyCall(_, _, _) = _, %v, want _, error code: %s", err, codes.Unavailable) - } - if _, err := tc.StreamingInputCall(context.Background()); status.Code(err) != codes.Unavailable { - t.Fatalf("TestService/StreamingInputCall(_) = _, %v, want _, error code: %s", err, codes.Unavailable) - } - - awaitNewConnLogOutput() -} - -func testServiceConfigSetup(t *testing.T, e env) *test { - te := newTest(t, e) - te.userAgent = testAppUA - te.declareLogNoise( - "transport: http2Client.notifyError got notified that the client transport was broken EOF", - "grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing", - "grpc: addrConn.resetTransport failed to create client transport: connection error", - "Failed to dial : context canceled; please retry.", - ) - return te -} - -func newBool(b bool) (a *bool) { - return &b -} - -func newInt(b int) (a *int) { - return &b -} - -func newDuration(b time.Duration) (a *time.Duration) { - a = new(time.Duration) - *a = b - return -} - -func (s) TestGetMethodConfig(t *testing.T) { - te := testServiceConfigSetup(t, tcpClearRREnv) - defer te.tearDown() - r := manual.NewBuilderWithScheme("whatever") - - te.resolverScheme = r.Scheme() - cc := te.clientConn(grpc.WithResolvers(r)) - addrs := []resolver.Address{{Addr: te.srvAddr}} - r.UpdateState(resolver.State{ - Addresses: addrs, - ServiceConfig: parseCfg(r, `{ - "methodConfig": [ - { - "name": [ - { - "service": "grpc.testing.TestService", - "method": "EmptyCall" - } - ], - "waitForReady": true, - "timeout": ".001s" - }, - { - "name": [ - { - "service": "grpc.testing.TestService" - } - ], - "waitForReady": false - } - ] -}`)}) - - tc := testpb.NewTestServiceClient(cc) - - // Make sure service config has been processed by grpc. - for { - if cc.GetMethodConfig("/grpc.testing.TestService/EmptyCall").WaitForReady != nil { - break - } - time.Sleep(time.Millisecond) - } - - // The following RPCs are expected to become non-fail-fast ones with 1ms deadline. - var err error - if _, err = tc.EmptyCall(context.Background(), &testpb.Empty{}); status.Code(err) != codes.DeadlineExceeded { - t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, %s", err, codes.DeadlineExceeded) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + // The following RPCs are expected to become non-fail-fast ones with 1ms deadline. + var err error + if _, err = tc.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.DeadlineExceeded { + t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, %s", err, codes.DeadlineExceeded) } - r.UpdateState(resolver.State{Addresses: addrs, ServiceConfig: parseCfg(r, `{ + r.UpdateState(resolver.State{Addresses: addrs, ServiceConfig: parseServiceConfig(t, r, `{ "methodConfig": [ { "name": [ @@ -1512,7 +1204,7 @@ func (s) TestGetMethodConfig(t *testing.T) { time.Sleep(time.Millisecond) } // The following RPCs are expected to become fail-fast. - if _, err = tc.EmptyCall(context.Background(), &testpb.Empty{}); status.Code(err) != codes.Unavailable { + if _, err = tc.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.Unavailable { t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, %s", err, codes.Unavailable) } } @@ -1528,7 +1220,7 @@ func (s) TestServiceConfigWaitForReady(t *testing.T) { addrs := []resolver.Address{{Addr: te.srvAddr}} r.UpdateState(resolver.State{ Addresses: addrs, - ServiceConfig: parseCfg(r, `{ + ServiceConfig: parseServiceConfig(t, r, `{ "methodConfig": [ { "name": [ @@ -1547,7 +1239,7 @@ func (s) TestServiceConfigWaitForReady(t *testing.T) { ] }`)}) - tc := testpb.NewTestServiceClient(cc) + tc := testgrpc.NewTestServiceClient(cc) // Make sure service config has been processed by grpc. for { @@ -1556,13 +1248,14 @@ func (s) TestServiceConfigWaitForReady(t *testing.T) { } time.Sleep(time.Millisecond) } - + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() // The following RPCs are expected to become non-fail-fast ones with 1ms deadline. var err error - if _, err = tc.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true)); status.Code(err) != codes.DeadlineExceeded { + if _, err = tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); status.Code(err) != codes.DeadlineExceeded { t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, %s", err, codes.DeadlineExceeded) } - if _, err := tc.FullDuplexCall(context.Background(), grpc.WaitForReady(true)); status.Code(err) != codes.DeadlineExceeded { + if _, err := tc.FullDuplexCall(ctx, grpc.WaitForReady(true)); status.Code(err) != codes.DeadlineExceeded { t.Fatalf("TestService/FullDuplexCall(_) = _, %v, want %s", err, codes.DeadlineExceeded) } @@ -1570,7 +1263,7 @@ func (s) TestServiceConfigWaitForReady(t *testing.T) { // Case2:Client API set failfast to be false, and service config set wait_for_ready to be true, and the rpc will wait until deadline exceeds. r.UpdateState(resolver.State{ Addresses: addrs, - ServiceConfig: parseCfg(r, `{ + ServiceConfig: parseServiceConfig(t, r, `{ "methodConfig": [ { "name": [ @@ -1597,10 +1290,10 @@ func (s) TestServiceConfigWaitForReady(t *testing.T) { time.Sleep(time.Millisecond) } // The following RPCs are expected to become non-fail-fast ones with 1ms deadline. - if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); status.Code(err) != codes.DeadlineExceeded { + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.DeadlineExceeded { t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, %s", err, codes.DeadlineExceeded) } - if _, err := tc.FullDuplexCall(context.Background()); status.Code(err) != codes.DeadlineExceeded { + if _, err := tc.FullDuplexCall(ctx); status.Code(err) != codes.DeadlineExceeded { t.Fatalf("TestService/FullDuplexCall(_) = _, %v, want %s", err, codes.DeadlineExceeded) } } @@ -1616,7 +1309,7 @@ func (s) TestServiceConfigTimeout(t *testing.T) { addrs := []resolver.Address{{Addr: te.srvAddr}} r.UpdateState(resolver.State{ Addresses: addrs, - ServiceConfig: parseCfg(r, `{ + ServiceConfig: parseServiceConfig(t, r, `{ "methodConfig": [ { "name": [ @@ -1635,7 +1328,7 @@ func (s) TestServiceConfigTimeout(t *testing.T) { ] }`)}) - tc := testpb.NewTestServiceClient(cc) + tc := testgrpc.NewTestServiceClient(cc) // Make sure service config has been processed by grpc. for { @@ -1647,13 +1340,13 @@ func (s) TestServiceConfigTimeout(t *testing.T) { // The following RPCs are expected to become non-fail-fast ones with 1ns deadline. var err error - ctx, cancel := context.WithTimeout(context.Background(), time.Nanosecond) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) if _, err = tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); status.Code(err) != codes.DeadlineExceeded { t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, %s", err, codes.DeadlineExceeded) } cancel() - ctx, cancel = context.WithTimeout(context.Background(), time.Nanosecond) + ctx, cancel = context.WithTimeout(context.Background(), defaultTestShortTimeout) if _, err = tc.FullDuplexCall(ctx, grpc.WaitForReady(true)); status.Code(err) != codes.DeadlineExceeded { t.Fatalf("TestService/FullDuplexCall(_) = _, %v, want %s", err, codes.DeadlineExceeded) } @@ -1663,7 +1356,7 @@ func (s) TestServiceConfigTimeout(t *testing.T) { // Case2: Client API sets timeout to be 1hr and ServiceConfig sets timeout to be 1ns. Timeout should be 1ns (min of 1ns and 1hr) and the rpc will wait until deadline exceeds. r.UpdateState(resolver.State{ Addresses: addrs, - ServiceConfig: parseCfg(r, `{ + ServiceConfig: parseServiceConfig(t, r, `{ "methodConfig": [ { "name": [ @@ -1690,17 +1383,15 @@ func (s) TestServiceConfigTimeout(t *testing.T) { time.Sleep(time.Millisecond) } - ctx, cancel = context.WithTimeout(context.Background(), time.Hour) + ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() if _, err = tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); status.Code(err) != codes.DeadlineExceeded { t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, %s", err, codes.DeadlineExceeded) } - cancel() - ctx, cancel = context.WithTimeout(context.Background(), time.Hour) if _, err = tc.FullDuplexCall(ctx, grpc.WaitForReady(true)); status.Code(err) != codes.DeadlineExceeded { t.Fatalf("TestService/FullDuplexCall(_) = _, %v, want %s", err, codes.DeadlineExceeded) } - cancel() } func (s) TestServiceConfigMaxMsgSize(t *testing.T) { @@ -1730,12 +1421,11 @@ func (s) TestServiceConfigMaxMsgSize(t *testing.T) { defer te1.tearDown() te1.resolverScheme = r.Scheme() - te1.nonBlockingDial = true te1.startServer(&testServer{security: e.security}) cc1 := te1.clientConn(grpc.WithResolvers(r)) addrs := []resolver.Address{{Addr: te1.srvAddr}} - sc := parseCfg(r, `{ + sc := parseServiceConfig(t, r, `{ "methodConfig": [ { "name": [ @@ -1754,7 +1444,7 @@ func (s) TestServiceConfigMaxMsgSize(t *testing.T) { ] }`) r.UpdateState(resolver.State{Addresses: addrs, ServiceConfig: sc}) - tc := testpb.NewTestServiceClient(cc1) + tc := testgrpc.NewTestServiceClient(cc1) req := &testpb.SimpleRequest{ ResponseType: testpb.PayloadType_COMPRESSABLE, @@ -1768,16 +1458,17 @@ func (s) TestServiceConfigMaxMsgSize(t *testing.T) { } time.Sleep(time.Millisecond) } - + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() // Test for unary RPC recv. - if _, err = tc.UnaryCall(context.Background(), req, grpc.WaitForReady(true)); err == nil || status.Code(err) != codes.ResourceExhausted { + if _, err = tc.UnaryCall(ctx, req, grpc.WaitForReady(true)); err == nil || status.Code(err) != codes.ResourceExhausted { t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted) } // Test for unary RPC send. req.Payload = extraLargePayload req.ResponseSize = int32(smallSize) - if _, err := tc.UnaryCall(context.Background(), req); err == nil || status.Code(err) != codes.ResourceExhausted { + if _, err := tc.UnaryCall(ctx, req); err == nil || status.Code(err) != codes.ResourceExhausted { t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted) } @@ -1817,7 +1508,6 @@ func (s) TestServiceConfigMaxMsgSize(t *testing.T) { // Case2: Client API set maxReqSize to 1024 (send), maxRespSize to 1024 (recv). Sc sets maxReqSize to 2048 (send), maxRespSize to 2048 (recv). te2 := testServiceConfigSetup(t, e) te2.resolverScheme = r.Scheme() - te2.nonBlockingDial = true te2.maxClientReceiveMsgSize = newInt(1024) te2.maxClientSendMsgSize = newInt(1024) @@ -1825,7 +1515,7 @@ func (s) TestServiceConfigMaxMsgSize(t *testing.T) { defer te2.tearDown() cc2 := te2.clientConn(grpc.WithResolvers(r)) r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: te2.srvAddr}}, ServiceConfig: sc}) - tc = testpb.NewTestServiceClient(cc2) + tc = testgrpc.NewTestServiceClient(cc2) for { if cc2.GetMethodConfig("/grpc.testing.TestService/FullDuplexCall").MaxReqSize != nil { @@ -1838,14 +1528,14 @@ func (s) TestServiceConfigMaxMsgSize(t *testing.T) { req.Payload = smallPayload req.ResponseSize = int32(largeSize) - if _, err = tc.UnaryCall(context.Background(), req, grpc.WaitForReady(true)); err == nil || status.Code(err) != codes.ResourceExhausted { + if _, err = tc.UnaryCall(ctx, req, grpc.WaitForReady(true)); err == nil || status.Code(err) != codes.ResourceExhausted { t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted) } // Test for unary RPC send. req.Payload = largePayload req.ResponseSize = int32(smallSize) - if _, err := tc.UnaryCall(context.Background(), req); err == nil || status.Code(err) != codes.ResourceExhausted { + if _, err := tc.UnaryCall(ctx, req); err == nil || status.Code(err) != codes.ResourceExhausted { t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted) } @@ -1877,7 +1567,6 @@ func (s) TestServiceConfigMaxMsgSize(t *testing.T) { // Case3: Client API set maxReqSize to 4096 (send), maxRespSize to 4096 (recv). Sc sets maxReqSize to 2048 (send), maxRespSize to 2048 (recv). te3 := testServiceConfigSetup(t, e) te3.resolverScheme = r.Scheme() - te3.nonBlockingDial = true te3.maxClientReceiveMsgSize = newInt(4096) te3.maxClientSendMsgSize = newInt(4096) @@ -1886,7 +1575,7 @@ func (s) TestServiceConfigMaxMsgSize(t *testing.T) { cc3 := te3.clientConn(grpc.WithResolvers(r)) r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: te3.srvAddr}}, ServiceConfig: sc}) - tc = testpb.NewTestServiceClient(cc3) + tc = testgrpc.NewTestServiceClient(cc3) for { if cc3.GetMethodConfig("/grpc.testing.TestService/FullDuplexCall").MaxReqSize != nil { @@ -1899,24 +1588,24 @@ func (s) TestServiceConfigMaxMsgSize(t *testing.T) { req.Payload = smallPayload req.ResponseSize = int32(largeSize) - if _, err = tc.UnaryCall(context.Background(), req, grpc.WaitForReady(true)); err != nil { + if _, err = tc.UnaryCall(ctx, req, grpc.WaitForReady(true)); err != nil { t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want ", err) } req.ResponseSize = int32(extraLargeSize) - if _, err := tc.UnaryCall(context.Background(), req); err == nil || status.Code(err) != codes.ResourceExhausted { + if _, err := tc.UnaryCall(ctx, req); err == nil || status.Code(err) != codes.ResourceExhausted { t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted) } // Test for unary RPC send. req.Payload = largePayload req.ResponseSize = int32(smallSize) - if _, err := tc.UnaryCall(context.Background(), req); err != nil { + if _, err := tc.UnaryCall(ctx, req); err != nil { t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want ", err) } req.Payload = extraLargePayload - if _, err = tc.UnaryCall(context.Background(), req); err == nil || status.Code(err) != codes.ResourceExhausted { + if _, err = tc.UnaryCall(ctx, req); err == nil || status.Code(err) != codes.ResourceExhausted { t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted) } @@ -1970,13 +1659,12 @@ func (s) TestStreamingRPCWithTimeoutInServiceConfigRecv(t *testing.T) { r := manual.NewBuilderWithScheme("whatever") te.resolverScheme = r.Scheme() - te.nonBlockingDial = true cc := te.clientConn(grpc.WithResolvers(r)) - tc := testpb.NewTestServiceClient(cc) + tc := testgrpc.NewTestServiceClient(cc) r.UpdateState(resolver.State{ Addresses: []resolver.Address{{Addr: te.srvAddr}}, - ServiceConfig: parseCfg(r, `{ + ServiceConfig: parseServiceConfig(t, r, `{ "methodConfig": [ { "name": [ @@ -1998,7 +1686,7 @@ func (s) TestStreamingRPCWithTimeoutInServiceConfigRecv(t *testing.T) { time.Sleep(time.Millisecond) } - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() stream, err := tc.FullDuplexCall(ctx, grpc.WaitForReady(true)) if err != nil { @@ -2042,15 +1730,12 @@ func testPreloaderClientSend(t *testing.T, e env) { te := newTest(t, e) te.userAgent = testAppUA te.declareLogNoise( - "transport: http2Client.notifyError got notified that the client transport was broken EOF", - "grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing", - "grpc: addrConn.resetTransport failed to create client transport: connection error", "Failed to dial : context canceled; please retry.", ) te.startServer(&testServer{security: e.security}) defer te.tearDown() - tc := testpb.NewTestServiceClient(te.clientConn()) + tc := testgrpc.NewTestServiceClient(te.clientConn()) // Test for streaming RPC recv. // Set context for send with proper RPC Information @@ -2106,6 +1791,61 @@ func testPreloaderClientSend(t *testing.T, e env) { } } +func (s) TestPreloaderSenderSend(t *testing.T) { + ss := &stubserver.StubServer{ + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + for i := 0; i < 10; i++ { + preparedMsg := &grpc.PreparedMsg{} + err := preparedMsg.Encode(stream, &testpb.StreamingOutputCallResponse{ + Payload: &testpb.Payload{ + Body: []byte{'0' + uint8(i)}, + }, + }) + if err != nil { + return err + } + stream.SendMsg(preparedMsg) + } + return nil + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + stream, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("ss.Client.EmptyCall(_, _) = _, %v; want _, nil", err) + } + + var ngot int + var buf bytes.Buffer + for { + reply, err := stream.Recv() + if err == io.EOF { + break + } + if err != nil { + t.Fatal(err) + } + ngot++ + if buf.Len() > 0 { + buf.WriteByte(',') + } + buf.Write(reply.GetPayload().GetBody()) + } + if want := 10; ngot != want { + t.Errorf("Got %d replies, want %d", ngot, want) + } + if got, want := buf.String(), "0,1,2,3,4,5,6,7,8,9"; got != want { + t.Errorf("Got replies %q; want %q", got, want) + } +} + func (s) TestMaxMsgSizeClientDefault(t *testing.T) { for _, e := range listTestEnv() { testMaxMsgSizeClientDefault(t, e) @@ -2116,15 +1856,12 @@ func testMaxMsgSizeClientDefault(t *testing.T, e env) { te := newTest(t, e) te.userAgent = testAppUA te.declareLogNoise( - "transport: http2Client.notifyError got notified that the client transport was broken EOF", - "grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing", - "grpc: addrConn.resetTransport failed to create client transport: connection error", "Failed to dial : context canceled; please retry.", ) te.startServer(&testServer{security: e.security}) defer te.tearDown() - tc := testpb.NewTestServiceClient(te.clientConn()) + tc := testgrpc.NewTestServiceClient(te.clientConn()) const smallSize = 1 const largeSize = 4 * 1024 * 1024 @@ -2137,8 +1874,11 @@ func testMaxMsgSizeClientDefault(t *testing.T, e env) { ResponseSize: int32(largeSize), Payload: smallPayload, } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() // Test for unary RPC recv. - if _, err := tc.UnaryCall(context.Background(), req); err == nil || status.Code(err) != codes.ResourceExhausted { + if _, err := tc.UnaryCall(ctx, req); err == nil || status.Code(err) != codes.ResourceExhausted { t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted) } @@ -2180,15 +1920,12 @@ func testMaxMsgSizeClientAPI(t *testing.T, e env) { te.maxClientReceiveMsgSize = newInt(1024) te.maxClientSendMsgSize = newInt(1024) te.declareLogNoise( - "transport: http2Client.notifyError got notified that the client transport was broken EOF", - "grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing", - "grpc: addrConn.resetTransport failed to create client transport: connection error", "Failed to dial : context canceled; please retry.", ) te.startServer(&testServer{security: e.security}) defer te.tearDown() - tc := testpb.NewTestServiceClient(te.clientConn()) + tc := testgrpc.NewTestServiceClient(te.clientConn()) const smallSize = 1 const largeSize = 1024 @@ -2206,15 +1943,18 @@ func testMaxMsgSizeClientAPI(t *testing.T, e env) { ResponseSize: int32(largeSize), Payload: smallPayload, } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() // Test for unary RPC recv. - if _, err := tc.UnaryCall(context.Background(), req); err == nil || status.Code(err) != codes.ResourceExhausted { + if _, err := tc.UnaryCall(ctx, req); err == nil || status.Code(err) != codes.ResourceExhausted { t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted) } // Test for unary RPC send. req.Payload = largePayload req.ResponseSize = int32(smallSize) - if _, err := tc.UnaryCall(context.Background(), req); err == nil || status.Code(err) != codes.ResourceExhausted { + if _, err := tc.UnaryCall(ctx, req); err == nil || status.Code(err) != codes.ResourceExhausted { t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted) } @@ -2265,15 +2005,12 @@ func testMaxMsgSizeServerAPI(t *testing.T, e env) { te.maxServerReceiveMsgSize = newInt(1024) te.maxServerSendMsgSize = newInt(1024) te.declareLogNoise( - "transport: http2Client.notifyError got notified that the client transport was broken EOF", - "grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing", - "grpc: addrConn.resetTransport failed to create client transport: connection error", "Failed to dial : context canceled; please retry.", ) te.startServer(&testServer{security: e.security}) defer te.tearDown() - tc := testpb.NewTestServiceClient(te.clientConn()) + tc := testgrpc.NewTestServiceClient(te.clientConn()) const smallSize = 1 const largeSize = 1024 @@ -2291,15 +2028,18 @@ func testMaxMsgSizeServerAPI(t *testing.T, e env) { ResponseSize: int32(largeSize), Payload: smallPayload, } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() // Test for unary RPC send. - if _, err := tc.UnaryCall(context.Background(), req); err == nil || status.Code(err) != codes.ResourceExhausted { + if _, err := tc.UnaryCall(ctx, req); err == nil || status.Code(err) != codes.ResourceExhausted { t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted) } // Test for unary RPC recv. req.Payload = largePayload req.ResponseSize = int32(smallSize) - if _, err := tc.UnaryCall(context.Background(), req); err == nil || status.Code(err) != codes.ResourceExhausted { + if _, err := tc.UnaryCall(ctx, req); err == nil || status.Code(err) != codes.ResourceExhausted { t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted) } @@ -2356,10 +2096,13 @@ type myTap struct { func (t *myTap) handle(ctx context.Context, info *tap.Info) (context.Context, error) { if info != nil { - if info.FullMethodName == "/grpc.testing.TestService/EmptyCall" { + switch info.FullMethodName { + case "/grpc.testing.TestService/EmptyCall": t.cnt++ - } else if info.FullMethodName == "/grpc.testing.TestService/UnaryCall" { + case "/grpc.testing.TestService/UnaryCall": return nil, fmt.Errorf("tap error") + case "/grpc.testing.TestService/FullDuplexCall": + return nil, status.Errorf(codes.FailedPrecondition, "test custom error") } } return ctx, nil @@ -2370,17 +2113,14 @@ func testTap(t *testing.T, e env) { te.userAgent = testAppUA ttap := &myTap{} te.tapHandle = ttap.handle - te.declareLogNoise( - "transport: http2Client.notifyError got notified that the client transport was broken EOF", - "grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing", - "grpc: addrConn.resetTransport failed to create client transport: connection error", - ) te.startServer(&testServer{security: e.security}) defer te.tearDown() cc := te.clientConn() - tc := testpb.NewTestServiceClient(cc) - if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); err != nil { + tc := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil { t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, ", err) } if ttap.cnt != 1 { @@ -2397,386 +2137,87 @@ func testTap(t *testing.T, e env) { ResponseSize: 45, Payload: payload, } - if _, err := tc.UnaryCall(context.Background(), req); status.Code(err) != codes.Unavailable { - t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, %s", err, codes.Unavailable) + if _, err := tc.UnaryCall(ctx, req); status.Code(err) != codes.PermissionDenied { + t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, %s", err, codes.PermissionDenied) } -} - -// healthCheck is a helper function to make a unary health check RPC and return -// the response. -func healthCheck(d time.Duration, cc *grpc.ClientConn, service string) (*healthpb.HealthCheckResponse, error) { - ctx, cancel := context.WithTimeout(context.Background(), d) - defer cancel() - hc := healthgrpc.NewHealthClient(cc) - return hc.Check(ctx, &healthpb.HealthCheckRequest{Service: service}) -} - -// verifyHealthCheckStatus is a helper function to verify that the current -// health status of the service matches the one passed in 'wantStatus'. -func verifyHealthCheckStatus(t *testing.T, d time.Duration, cc *grpc.ClientConn, service string, wantStatus healthpb.HealthCheckResponse_ServingStatus) { - t.Helper() - resp, err := healthCheck(d, cc, service) + str, err := tc.FullDuplexCall(ctx) if err != nil { - t.Fatalf("Health/Check(_, _) = _, %v, want _, ", err) + t.Fatalf("Unexpected error creating stream: %v", err) } - if resp.Status != wantStatus { - t.Fatalf("Got the serving status %v, want %v", resp.Status, wantStatus) + if _, err := str.Recv(); status.Code(err) != codes.FailedPrecondition { + t.Fatalf("FullDuplexCall Recv() = _, %v, want _, %s", err, codes.FailedPrecondition) } } -// verifyHealthCheckErrCode is a helper function to verify that a unary health -// check RPC returns an error with a code set to 'wantCode'. -func verifyHealthCheckErrCode(t *testing.T, d time.Duration, cc *grpc.ClientConn, service string, wantCode codes.Code) { - t.Helper() - if _, err := healthCheck(d, cc, service); status.Code(err) != wantCode { - t.Fatalf("Health/Check() got errCode %v, want %v", status.Code(err), wantCode) +func (s) TestEmptyUnaryWithUserAgent(t *testing.T) { + for _, e := range listTestEnv() { + testEmptyUnaryWithUserAgent(t, e) } } -// newHealthCheckStream is a helper function to start a health check streaming -// RPC, and returns the stream. -func newHealthCheckStream(t *testing.T, cc *grpc.ClientConn, service string) (healthgrpc.Health_WatchClient, context.CancelFunc) { - t.Helper() - ctx, cancel := context.WithCancel(context.Background()) - hc := healthgrpc.NewHealthClient(cc) - stream, err := hc.Watch(ctx, &healthpb.HealthCheckRequest{Service: service}) - if err != nil { - t.Fatalf("hc.Watch(_, %v) failed: %v", service, err) - } - return stream, cancel -} +func testEmptyUnaryWithUserAgent(t *testing.T, e env) { + te := newTest(t, e) + te.userAgent = testAppUA + te.startServer(&testServer{security: e.security}) + defer te.tearDown() -// healthWatchChecker is a helper function to verify that the next health -// status returned on the given stream matches the one passed in 'wantStatus'. -func healthWatchChecker(t *testing.T, stream healthgrpc.Health_WatchClient, wantStatus healthpb.HealthCheckResponse_ServingStatus) { - t.Helper() - response, err := stream.Recv() - if err != nil { - t.Fatalf("stream.Recv() failed: %v", err) + cc := te.clientConn() + tc := testgrpc.NewTestServiceClient(cc) + var header metadata.MD + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + reply, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.Header(&header)) + if err != nil || !proto.Equal(&testpb.Empty{}, reply) { + t.Fatalf("TestService/EmptyCall(_, _) = %v, %v, want %v, ", reply, err, &testpb.Empty{}) } - if response.Status != wantStatus { - t.Fatalf("got servingStatus %v, want %v", response.Status, wantStatus) + if v, ok := header["ua"]; !ok || !strings.HasPrefix(v[0], testAppUA) { + t.Fatalf("header[\"ua\"] = %q, %t, want string with prefix %q, true", v, ok, testAppUA) } + + te.srv.Stop() } -// TestHealthCheckSuccess invokes the unary Check() RPC on the health server in -// a successful case. -func (s) TestHealthCheckSuccess(t *testing.T) { +func (s) TestFailedEmptyUnary(t *testing.T) { for _, e := range listTestEnv() { - testHealthCheckSuccess(t, e) + if e.name == "handler-tls" { + // This test covers status details, but + // Grpc-Status-Details-Bin is not support in handler_server. + continue + } + testFailedEmptyUnary(t, e) } } -func testHealthCheckSuccess(t *testing.T, e env) { +func testFailedEmptyUnary(t *testing.T, e env) { te := newTest(t, e) - te.enableHealthServer = true + te.userAgent = failAppUA te.startServer(&testServer{security: e.security}) - te.setHealthServingStatus(defaultHealthService, healthpb.HealthCheckResponse_SERVING) defer te.tearDown() + tc := testgrpc.NewTestServiceClient(te.clientConn()) - verifyHealthCheckErrCode(t, 1*time.Second, te.clientConn(), defaultHealthService, codes.OK) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctx = metadata.NewOutgoingContext(ctx, testMetadata) + wantErr := detailedError + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); !testutils.StatusErrEqual(err, wantErr) { + t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, %v", err, wantErr) + } } -// TestHealthCheckFailure invokes the unary Check() RPC on the health server -// with an expired context and expects the RPC to fail. -func (s) TestHealthCheckFailure(t *testing.T) { +func (s) TestLargeUnary(t *testing.T) { for _, e := range listTestEnv() { - testHealthCheckFailure(t, e) + testLargeUnary(t, e) } } -func testHealthCheckFailure(t *testing.T, e env) { +func testLargeUnary(t *testing.T, e env) { te := newTest(t, e) - te.declareLogNoise( - "Failed to dial ", - "grpc: the client connection is closing; please retry", - ) - te.enableHealthServer = true te.startServer(&testServer{security: e.security}) - te.setHealthServingStatus(defaultHealthService, healthpb.HealthCheckResponse_SERVING) defer te.tearDown() + tc := testgrpc.NewTestServiceClient(te.clientConn()) - verifyHealthCheckErrCode(t, 0*time.Second, te.clientConn(), defaultHealthService, codes.DeadlineExceeded) - awaitNewConnLogOutput() -} - -// TestHealthCheckOff makes a unary Check() RPC on the health server where the -// health status of the defaultHealthService is not set, and therefore expects -// an error code 'codes.NotFound'. -func (s) TestHealthCheckOff(t *testing.T) { - for _, e := range listTestEnv() { - // TODO(bradfitz): Temporarily skip this env due to #619. - if e.name == "handler-tls" { - continue - } - testHealthCheckOff(t, e) - } -} - -func testHealthCheckOff(t *testing.T, e env) { - te := newTest(t, e) - te.enableHealthServer = true - te.startServer(&testServer{security: e.security}) - defer te.tearDown() - - verifyHealthCheckErrCode(t, 1*time.Second, te.clientConn(), defaultHealthService, codes.NotFound) -} - -// TestHealthWatchMultipleClients makes a streaming Watch() RPC on the health -// server with multiple clients and expects the same status on both streams. -func (s) TestHealthWatchMultipleClients(t *testing.T) { - for _, e := range listTestEnv() { - testHealthWatchMultipleClients(t, e) - } -} - -func testHealthWatchMultipleClients(t *testing.T, e env) { - te := newTest(t, e) - te.enableHealthServer = true - te.startServer(&testServer{security: e.security}) - defer te.tearDown() - - cc := te.clientConn() - stream1, cf1 := newHealthCheckStream(t, cc, defaultHealthService) - defer cf1() - healthWatchChecker(t, stream1, healthpb.HealthCheckResponse_SERVICE_UNKNOWN) - - stream2, cf2 := newHealthCheckStream(t, cc, defaultHealthService) - defer cf2() - healthWatchChecker(t, stream2, healthpb.HealthCheckResponse_SERVICE_UNKNOWN) - - te.setHealthServingStatus(defaultHealthService, healthpb.HealthCheckResponse_NOT_SERVING) - healthWatchChecker(t, stream1, healthpb.HealthCheckResponse_NOT_SERVING) - healthWatchChecker(t, stream2, healthpb.HealthCheckResponse_NOT_SERVING) -} - -// TestHealthWatchSameStatusmakes a streaming Watch() RPC on the health server -// and makes sure that the health status of the server is as expected after -// multiple calls to SetServingStatus with the same status. -func (s) TestHealthWatchSameStatus(t *testing.T) { - for _, e := range listTestEnv() { - testHealthWatchSameStatus(t, e) - } -} - -func testHealthWatchSameStatus(t *testing.T, e env) { - te := newTest(t, e) - te.enableHealthServer = true - te.startServer(&testServer{security: e.security}) - defer te.tearDown() - - stream, cf := newHealthCheckStream(t, te.clientConn(), defaultHealthService) - defer cf() - - healthWatchChecker(t, stream, healthpb.HealthCheckResponse_SERVICE_UNKNOWN) - te.setHealthServingStatus(defaultHealthService, healthpb.HealthCheckResponse_SERVING) - healthWatchChecker(t, stream, healthpb.HealthCheckResponse_SERVING) - te.setHealthServingStatus(defaultHealthService, healthpb.HealthCheckResponse_SERVING) - te.setHealthServingStatus(defaultHealthService, healthpb.HealthCheckResponse_NOT_SERVING) - healthWatchChecker(t, stream, healthpb.HealthCheckResponse_NOT_SERVING) -} - -// TestHealthWatchServiceStatusSetBeforeStartingServer starts a health server -// on which the health status for the defaultService is set before the gRPC -// server is started, and expects the correct health status to be returned. -func (s) TestHealthWatchServiceStatusSetBeforeStartingServer(t *testing.T) { - for _, e := range listTestEnv() { - testHealthWatchSetServiceStatusBeforeStartingServer(t, e) - } -} - -func testHealthWatchSetServiceStatusBeforeStartingServer(t *testing.T, e env) { - hs := health.NewServer() - te := newTest(t, e) - te.healthServer = hs - hs.SetServingStatus(defaultHealthService, healthpb.HealthCheckResponse_SERVING) - te.startServer(&testServer{security: e.security}) - defer te.tearDown() - - stream, cf := newHealthCheckStream(t, te.clientConn(), defaultHealthService) - defer cf() - healthWatchChecker(t, stream, healthpb.HealthCheckResponse_SERVING) -} - -// TestHealthWatchDefaultStatusChange verifies the simple case where the -// service starts off with a SERVICE_UNKNOWN status (because SetServingStatus -// hasn't been called yet) and then moves to SERVING after SetServingStatus is -// called. -func (s) TestHealthWatchDefaultStatusChange(t *testing.T) { - for _, e := range listTestEnv() { - testHealthWatchDefaultStatusChange(t, e) - } -} - -func testHealthWatchDefaultStatusChange(t *testing.T, e env) { - te := newTest(t, e) - te.enableHealthServer = true - te.startServer(&testServer{security: e.security}) - defer te.tearDown() - - stream, cf := newHealthCheckStream(t, te.clientConn(), defaultHealthService) - defer cf() - healthWatchChecker(t, stream, healthpb.HealthCheckResponse_SERVICE_UNKNOWN) - te.setHealthServingStatus(defaultHealthService, healthpb.HealthCheckResponse_SERVING) - healthWatchChecker(t, stream, healthpb.HealthCheckResponse_SERVING) -} - -// TestHealthWatchSetServiceStatusBeforeClientCallsWatch verifies the case -// where the health status is set to SERVING before the client calls Watch(). -func (s) TestHealthWatchSetServiceStatusBeforeClientCallsWatch(t *testing.T) { - for _, e := range listTestEnv() { - testHealthWatchSetServiceStatusBeforeClientCallsWatch(t, e) - } -} - -func testHealthWatchSetServiceStatusBeforeClientCallsWatch(t *testing.T, e env) { - te := newTest(t, e) - te.enableHealthServer = true - te.startServer(&testServer{security: e.security}) - te.setHealthServingStatus(defaultHealthService, healthpb.HealthCheckResponse_SERVING) - defer te.tearDown() - - stream, cf := newHealthCheckStream(t, te.clientConn(), defaultHealthService) - defer cf() - healthWatchChecker(t, stream, healthpb.HealthCheckResponse_SERVING) -} - -// TestHealthWatchOverallServerHealthChange verifies setting the overall status -// of the server by using the empty service name. -func (s) TestHealthWatchOverallServerHealthChange(t *testing.T) { - for _, e := range listTestEnv() { - testHealthWatchOverallServerHealthChange(t, e) - } -} - -func testHealthWatchOverallServerHealthChange(t *testing.T, e env) { - te := newTest(t, e) - te.enableHealthServer = true - te.startServer(&testServer{security: e.security}) - defer te.tearDown() - - stream, cf := newHealthCheckStream(t, te.clientConn(), "") - defer cf() - healthWatchChecker(t, stream, healthpb.HealthCheckResponse_SERVING) - te.setHealthServingStatus("", healthpb.HealthCheckResponse_NOT_SERVING) - healthWatchChecker(t, stream, healthpb.HealthCheckResponse_NOT_SERVING) -} - -// TestUnknownHandler verifies that an expected error is returned (by setting -// the unknownHandler on the server) for a service which is not exposed to the -// client. -func (s) TestUnknownHandler(t *testing.T) { - // An example unknownHandler that returns a different code and a different - // method, making sure that we do not expose what methods are implemented to - // a client that is not authenticated. - unknownHandler := func(srv interface{}, stream grpc.ServerStream) error { - return status.Error(codes.Unauthenticated, "user unauthenticated") - } - for _, e := range listTestEnv() { - // TODO(bradfitz): Temporarily skip this env due to #619. - if e.name == "handler-tls" { - continue - } - testUnknownHandler(t, e, unknownHandler) - } -} - -func testUnknownHandler(t *testing.T, e env, unknownHandler grpc.StreamHandler) { - te := newTest(t, e) - te.unknownHandler = unknownHandler - te.startServer(&testServer{security: e.security}) - defer te.tearDown() - verifyHealthCheckErrCode(t, 1*time.Second, te.clientConn(), "", codes.Unauthenticated) -} - -// TestHealthCheckServingStatus makes a streaming Watch() RPC on the health -// server and verifies a bunch of health status transitions. -func (s) TestHealthCheckServingStatus(t *testing.T) { - for _, e := range listTestEnv() { - testHealthCheckServingStatus(t, e) - } -} - -func testHealthCheckServingStatus(t *testing.T, e env) { - te := newTest(t, e) - te.enableHealthServer = true - te.startServer(&testServer{security: e.security}) - defer te.tearDown() - - cc := te.clientConn() - verifyHealthCheckStatus(t, 1*time.Second, cc, "", healthpb.HealthCheckResponse_SERVING) - verifyHealthCheckErrCode(t, 1*time.Second, cc, defaultHealthService, codes.NotFound) - te.setHealthServingStatus(defaultHealthService, healthpb.HealthCheckResponse_SERVING) - verifyHealthCheckStatus(t, 1*time.Second, cc, defaultHealthService, healthpb.HealthCheckResponse_SERVING) - te.setHealthServingStatus(defaultHealthService, healthpb.HealthCheckResponse_NOT_SERVING) - verifyHealthCheckStatus(t, 1*time.Second, cc, defaultHealthService, healthpb.HealthCheckResponse_NOT_SERVING) -} - -func (s) TestEmptyUnaryWithUserAgent(t *testing.T) { - for _, e := range listTestEnv() { - testEmptyUnaryWithUserAgent(t, e) - } -} - -func testEmptyUnaryWithUserAgent(t *testing.T, e env) { - te := newTest(t, e) - te.userAgent = testAppUA - te.startServer(&testServer{security: e.security}) - defer te.tearDown() - - cc := te.clientConn() - tc := testpb.NewTestServiceClient(cc) - var header metadata.MD - reply, err := tc.EmptyCall(context.Background(), &testpb.Empty{}, grpc.Header(&header)) - if err != nil || !proto.Equal(&testpb.Empty{}, reply) { - t.Fatalf("TestService/EmptyCall(_, _) = %v, %v, want %v, ", reply, err, &testpb.Empty{}) - } - if v, ok := header["ua"]; !ok || !strings.HasPrefix(v[0], testAppUA) { - t.Fatalf("header[\"ua\"] = %q, %t, want string with prefix %q, true", v, ok, testAppUA) - } - - te.srv.Stop() -} - -func (s) TestFailedEmptyUnary(t *testing.T) { - for _, e := range listTestEnv() { - if e.name == "handler-tls" { - // This test covers status details, but - // Grpc-Status-Details-Bin is not support in handler_server. - continue - } - testFailedEmptyUnary(t, e) - } -} - -func testFailedEmptyUnary(t *testing.T, e env) { - te := newTest(t, e) - te.userAgent = failAppUA - te.startServer(&testServer{security: e.security}) - defer te.tearDown() - tc := testpb.NewTestServiceClient(te.clientConn()) - - ctx := metadata.NewOutgoingContext(context.Background(), testMetadata) - wantErr := detailedError - if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); !testutils.StatusErrEqual(err, wantErr) { - t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, %v", err, wantErr) - } -} - -func (s) TestLargeUnary(t *testing.T) { - for _, e := range listTestEnv() { - testLargeUnary(t, e) - } -} - -func testLargeUnary(t *testing.T, e env) { - te := newTest(t, e) - te.startServer(&testServer{security: e.security}) - defer te.tearDown() - tc := testpb.NewTestServiceClient(te.clientConn()) - - const argSize = 271828 - const respSize = 314159 + const argSize = 271828 + const respSize = 314159 payload, err := newPayload(testpb.PayloadType_COMPRESSABLE, argSize) if err != nil { @@ -2788,7 +2229,10 @@ func testLargeUnary(t *testing.T, e env) { ResponseSize: respSize, Payload: payload, } - reply, err := tc.UnaryCall(context.Background(), req) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + reply, err := tc.UnaryCall(ctx, req) if err != nil { t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, ", err) } @@ -2812,7 +2256,7 @@ func testExceedMsgLimit(t *testing.T, e env) { te.maxServerMsgSize, te.maxClientMsgSize = newInt(maxMsgSize), newInt(maxMsgSize) te.startServer(&testServer{security: e.security}) defer te.tearDown() - tc := testpb.NewTestServiceClient(te.clientConn()) + tc := testgrpc.NewTestServiceClient(te.clientConn()) largeSize := int32(maxMsgSize + 1) const smallSize = 1 @@ -2832,13 +2276,16 @@ func testExceedMsgLimit(t *testing.T, e env) { ResponseSize: smallSize, Payload: largePayload, } - if _, err := tc.UnaryCall(context.Background(), req); err == nil || status.Code(err) != codes.ResourceExhausted { + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.UnaryCall(ctx, req); err == nil || status.Code(err) != codes.ResourceExhausted { t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted) } // Make sure the client cannot receive a unary RPC of largeSize. req.ResponseSize = largeSize req.Payload = smallPayload - if _, err := tc.UnaryCall(context.Background(), req); err == nil || status.Code(err) != codes.ResourceExhausted { + if _, err := tc.UnaryCall(ctx, req); err == nil || status.Code(err) != codes.ResourceExhausted { t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted) } @@ -2891,9 +2338,11 @@ func testPeerClientSide(t *testing.T, e env) { te.userAgent = testAppUA te.startServer(&testServer{security: e.security}) defer te.tearDown() - tc := testpb.NewTestServiceClient(te.clientConn()) + tc := testgrpc.NewTestServiceClient(te.clientConn()) peer := new(peer.Peer) - if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}, grpc.Peer(peer), grpc.WaitForReady(true)); err != nil { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer), grpc.WaitForReady(true)); err != nil { t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, ", err) } pa := peer.Addr.String() @@ -2931,9 +2380,9 @@ func testPeerNegative(t *testing.T, e env) { defer te.tearDown() cc := te.clientConn() - tc := testpb.NewTestServiceClient(cc) + tc := testgrpc.NewTestServiceClient(cc) peer := new(peer.Peer) - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) cancel() tc.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer)) } @@ -2950,10 +2399,12 @@ func testPeerFailedRPC(t *testing.T, e env) { te.startServer(&testServer{security: e.security}) defer te.tearDown() - tc := testpb.NewTestServiceClient(te.clientConn()) + tc := testgrpc.NewTestServiceClient(te.clientConn()) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() // first make a successful request to the server - if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); err != nil { + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil { t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, ", err) } @@ -2969,7 +2420,7 @@ func testPeerFailedRPC(t *testing.T, e env) { } peer := new(peer.Peer) - if _, err := tc.UnaryCall(context.Background(), req, grpc.Peer(peer)); err == nil || status.Code(err) != codes.ResourceExhausted { + if _, err := tc.UnaryCall(ctx, req, grpc.Peer(peer)); err == nil || status.Code(err) != codes.ResourceExhausted { t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted) } else { pa := peer.Addr.String() @@ -3003,7 +2454,7 @@ func testMetadataUnaryRPC(t *testing.T, e env) { te := newTest(t, e) te.startServer(&testServer{security: e.security}) defer te.tearDown() - tc := testpb.NewTestServiceClient(te.clientConn()) + tc := testgrpc.NewTestServiceClient(te.clientConn()) const argSize = 2718 const respSize = 314 @@ -3019,7 +2470,9 @@ func testMetadataUnaryRPC(t *testing.T, e env) { Payload: payload, } var header, trailer metadata.MD - ctx := metadata.NewOutgoingContext(context.Background(), testMetadata) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctx = metadata.NewOutgoingContext(ctx, testMetadata) if _, err := tc.UnaryCall(ctx, req, grpc.Header(&header), grpc.Trailer(&trailer)); err != nil { t.Fatalf("TestService.UnaryCall(%v, _, _, _) = _, %v; want _, ", ctx, err) } @@ -3029,6 +2482,7 @@ func testMetadataUnaryRPC(t *testing.T, e env) { delete(header, "date") // the Date header is also optional delete(header, "user-agent") delete(header, "content-type") + delete(header, "grpc-accept-encoding") } if !reflect.DeepEqual(header, testMetadata) { t.Fatalf("Received header metadata %v, want %v", header, testMetadata) @@ -3048,9 +2502,11 @@ func testMetadataOrderUnaryRPC(t *testing.T, e env) { te := newTest(t, e) te.startServer(&testServer{security: e.security}) defer te.tearDown() - tc := testpb.NewTestServiceClient(te.clientConn()) + tc := testgrpc.NewTestServiceClient(te.clientConn()) - ctx := metadata.NewOutgoingContext(context.Background(), testMetadata) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctx = metadata.NewOutgoingContext(ctx, testMetadata) ctx = metadata.AppendToOutgoingContext(ctx, "key1", "value2") ctx = metadata.AppendToOutgoingContext(ctx, "key1", "value3") @@ -3068,6 +2524,7 @@ func testMetadataOrderUnaryRPC(t *testing.T, e env) { delete(header, "date") // the Date header is also optional delete(header, "user-agent") delete(header, "content-type") + delete(header, "grpc-accept-encoding") } if !reflect.DeepEqual(header, newMetadata) { @@ -3085,7 +2542,7 @@ func testMultipleSetTrailerUnaryRPC(t *testing.T, e env) { te := newTest(t, e) te.startServer(&testServer{security: e.security, multipleSetTrailer: true}) defer te.tearDown() - tc := testpb.NewTestServiceClient(te.clientConn()) + tc := testgrpc.NewTestServiceClient(te.clientConn()) const ( argSize = 1 @@ -3102,7 +2559,9 @@ func testMultipleSetTrailerUnaryRPC(t *testing.T, e env) { Payload: payload, } var trailer metadata.MD - ctx := metadata.NewOutgoingContext(context.Background(), testMetadata) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctx = metadata.NewOutgoingContext(ctx, testMetadata) if _, err := tc.UnaryCall(ctx, req, grpc.Trailer(&trailer), grpc.WaitForReady(true)); err != nil { t.Fatalf("TestService.UnaryCall(%v, _, _, _) = _, %v; want _, ", ctx, err) } @@ -3122,9 +2581,11 @@ func testMultipleSetTrailerStreamingRPC(t *testing.T, e env) { te := newTest(t, e) te.startServer(&testServer{security: e.security, multipleSetTrailer: true}) defer te.tearDown() - tc := testpb.NewTestServiceClient(te.clientConn()) + tc := testgrpc.NewTestServiceClient(te.clientConn()) - ctx := metadata.NewOutgoingContext(context.Background(), testMetadata) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctx = metadata.NewOutgoingContext(ctx, testMetadata) stream, err := tc.FullDuplexCall(ctx, grpc.WaitForReady(true)) if err != nil { t.Fatalf("%v.FullDuplexCall(_) = _, %v, want ", tc, err) @@ -3157,7 +2618,7 @@ func testSetAndSendHeaderUnaryRPC(t *testing.T, e env) { te := newTest(t, e) te.startServer(&testServer{security: e.security, setAndSendHeader: true}) defer te.tearDown() - tc := testpb.NewTestServiceClient(te.clientConn()) + tc := testgrpc.NewTestServiceClient(te.clientConn()) const ( argSize = 1 @@ -3174,12 +2635,16 @@ func testSetAndSendHeaderUnaryRPC(t *testing.T, e env) { Payload: payload, } var header metadata.MD - ctx := metadata.NewOutgoingContext(context.Background(), testMetadata) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctx = metadata.NewOutgoingContext(ctx, testMetadata) if _, err := tc.UnaryCall(ctx, req, grpc.Header(&header), grpc.WaitForReady(true)); err != nil { t.Fatalf("TestService.UnaryCall(%v, _, _, _) = _, %v; want _, ", ctx, err) } delete(header, "user-agent") delete(header, "content-type") + delete(header, "grpc-accept-encoding") + expectedHeader := metadata.Join(testMetadata, testMetadata2) if !reflect.DeepEqual(header, expectedHeader) { t.Fatalf("Received header metadata %v, want %v", header, expectedHeader) @@ -3200,7 +2665,7 @@ func testMultipleSetHeaderUnaryRPC(t *testing.T, e env) { te := newTest(t, e) te.startServer(&testServer{security: e.security, setHeaderOnly: true}) defer te.tearDown() - tc := testpb.NewTestServiceClient(te.clientConn()) + tc := testgrpc.NewTestServiceClient(te.clientConn()) const ( argSize = 1 @@ -3218,12 +2683,15 @@ func testMultipleSetHeaderUnaryRPC(t *testing.T, e env) { } var header metadata.MD - ctx := metadata.NewOutgoingContext(context.Background(), testMetadata) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctx = metadata.NewOutgoingContext(ctx, testMetadata) if _, err := tc.UnaryCall(ctx, req, grpc.Header(&header), grpc.WaitForReady(true)); err != nil { t.Fatalf("TestService.UnaryCall(%v, _, _, _) = _, %v; want _, ", ctx, err) } delete(header, "user-agent") delete(header, "content-type") + delete(header, "grpc-accept-encoding") expectedHeader := metadata.Join(testMetadata, testMetadata2) if !reflect.DeepEqual(header, expectedHeader) { t.Fatalf("Received header metadata %v, want %v", header, expectedHeader) @@ -3244,7 +2712,7 @@ func testMultipleSetHeaderUnaryRPCError(t *testing.T, e env) { te := newTest(t, e) te.startServer(&testServer{security: e.security, setHeaderOnly: true}) defer te.tearDown() - tc := testpb.NewTestServiceClient(te.clientConn()) + tc := testgrpc.NewTestServiceClient(te.clientConn()) const ( argSize = 1 @@ -3261,12 +2729,15 @@ func testMultipleSetHeaderUnaryRPCError(t *testing.T, e env) { Payload: payload, } var header metadata.MD - ctx := metadata.NewOutgoingContext(context.Background(), testMetadata) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctx = metadata.NewOutgoingContext(ctx, testMetadata) if _, err := tc.UnaryCall(ctx, req, grpc.Header(&header), grpc.WaitForReady(true)); err == nil { t.Fatalf("TestService.UnaryCall(%v, _, _, _) = _, %v; want _, ", ctx, err) } delete(header, "user-agent") delete(header, "content-type") + delete(header, "grpc-accept-encoding") expectedHeader := metadata.Join(testMetadata, testMetadata2) if !reflect.DeepEqual(header, expectedHeader) { t.Fatalf("Received header metadata %v, want %v", header, expectedHeader) @@ -3287,9 +2758,11 @@ func testSetAndSendHeaderStreamingRPC(t *testing.T, e env) { te := newTest(t, e) te.startServer(&testServer{security: e.security, setAndSendHeader: true}) defer te.tearDown() - tc := testpb.NewTestServiceClient(te.clientConn()) + tc := testgrpc.NewTestServiceClient(te.clientConn()) - ctx := metadata.NewOutgoingContext(context.Background(), testMetadata) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctx = metadata.NewOutgoingContext(ctx, testMetadata) stream, err := tc.FullDuplexCall(ctx) if err != nil { t.Fatalf("%v.FullDuplexCall(_) = _, %v, want ", tc, err) @@ -3307,6 +2780,7 @@ func testSetAndSendHeaderStreamingRPC(t *testing.T, e env) { } delete(header, "user-agent") delete(header, "content-type") + delete(header, "grpc-accept-encoding") expectedHeader := metadata.Join(testMetadata, testMetadata2) if !reflect.DeepEqual(header, expectedHeader) { t.Fatalf("Received header metadata %v, want %v", header, expectedHeader) @@ -3327,13 +2801,15 @@ func testMultipleSetHeaderStreamingRPC(t *testing.T, e env) { te := newTest(t, e) te.startServer(&testServer{security: e.security, setHeaderOnly: true}) defer te.tearDown() - tc := testpb.NewTestServiceClient(te.clientConn()) + tc := testgrpc.NewTestServiceClient(te.clientConn()) const ( argSize = 1 respSize = 1 ) - ctx := metadata.NewOutgoingContext(context.Background(), testMetadata) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctx = metadata.NewOutgoingContext(ctx, testMetadata) stream, err := tc.FullDuplexCall(ctx) if err != nil { t.Fatalf("%v.FullDuplexCall(_) = _, %v, want ", tc, err) @@ -3370,6 +2846,7 @@ func testMultipleSetHeaderStreamingRPC(t *testing.T, e env) { } delete(header, "user-agent") delete(header, "content-type") + delete(header, "grpc-accept-encoding") expectedHeader := metadata.Join(testMetadata, testMetadata2) if !reflect.DeepEqual(header, expectedHeader) { t.Fatalf("Received header metadata %v, want %v", header, expectedHeader) @@ -3391,13 +2868,13 @@ func testMultipleSetHeaderStreamingRPCError(t *testing.T, e env) { te := newTest(t, e) te.startServer(&testServer{security: e.security, setHeaderOnly: true}) defer te.tearDown() - tc := testpb.NewTestServiceClient(te.clientConn()) + tc := testgrpc.NewTestServiceClient(te.clientConn()) const ( argSize = 1 respSize = -1 ) - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() ctx = metadata.NewOutgoingContext(ctx, testMetadata) stream, err := tc.FullDuplexCall(ctx) @@ -3430,6 +2907,7 @@ func testMultipleSetHeaderStreamingRPCError(t *testing.T, e env) { } delete(header, "user-agent") delete(header, "content-type") + delete(header, "grpc-accept-encoding") expectedHeader := metadata.Join(testMetadata, testMetadata2) if !reflect.DeepEqual(header, expectedHeader) { t.Fatalf("Received header metadata %v, want %v", header, expectedHeader) @@ -3456,7 +2934,7 @@ func testMalformedHTTP2Metadata(t *testing.T, e env) { te := newTest(t, e) te.startServer(&testServer{security: e.security}) defer te.tearDown() - tc := testpb.NewTestServiceClient(te.clientConn()) + tc := testgrpc.NewTestServiceClient(te.clientConn()) payload, err := newPayload(testpb.PayloadType_COMPRESSABLE, 2718) if err != nil { @@ -3468,78 +2946,95 @@ func testMalformedHTTP2Metadata(t *testing.T, e env) { ResponseSize: 314, Payload: payload, } - ctx := metadata.NewOutgoingContext(context.Background(), malformedHTTP2Metadata) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctx = metadata.NewOutgoingContext(ctx, malformedHTTP2Metadata) if _, err := tc.UnaryCall(ctx, req); status.Code(err) != codes.Internal { t.Fatalf("TestService.UnaryCall(%v, _) = _, %v; want _, %s", ctx, err, codes.Internal) } } +// Tests that the client transparently retries correctly when receiving a +// RST_STREAM with code REFUSED_STREAM. func (s) TestTransparentRetry(t *testing.T) { - for _, e := range listTestEnv() { - if e.name == "handler-tls" { - // Fails with RST_STREAM / FLOW_CONTROL_ERROR - continue - } - testTransparentRetry(t, e) - } -} - -// This test makes sure RPCs are retried times when they receive a RST_STREAM -// with the REFUSED_STREAM error code, which the InTapHandle provokes. -func testTransparentRetry(t *testing.T, e env) { - te := newTest(t, e) - attempts := 0 - successAttempt := 2 - te.tapHandle = func(ctx context.Context, _ *tap.Info) (context.Context, error) { - attempts++ - if attempts < successAttempt { - return nil, errors.New("not now") - } - return ctx, nil - } - te.startServer(&testServer{security: e.security}) - defer te.tearDown() - - cc := te.clientConn() - tsc := testpb.NewTestServiceClient(cc) testCases := []struct { - successAttempt int - failFast bool - errCode codes.Code + failFast bool + errCode codes.Code }{{ - successAttempt: 1, + // success attempt: 1, (stream ID 1) }, { - successAttempt: 2, + // success attempt: 2, (stream IDs 3, 5) }, { - successAttempt: 3, - errCode: codes.Unavailable, + // no success attempt (stream IDs 7, 9) + errCode: codes.Unavailable, }, { - successAttempt: 1, - failFast: true, + // success attempt: 1 (stream ID 11), + failFast: true, }, { - successAttempt: 2, - failFast: true, + // success attempt: 2 (stream IDs 13, 15), + failFast: true, }, { - successAttempt: 3, - failFast: true, - errCode: codes.Unavailable, + // no success attempt (stream IDs 17, 19) + failFast: true, + errCode: codes.Unavailable, }} - for _, tc := range testCases { - attempts = 0 - successAttempt = tc.successAttempt - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - _, err := tsc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(!tc.failFast)) - cancel() - if status.Code(err) != tc.errCode { - t.Errorf("%+v: tsc.EmptyCall(_, _) = _, %v, want _, Code=%v", tc, err, tc.errCode) + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Failed to listen. Err: %v", err) + } + defer lis.Close() + server := &httpServer{ + responses: []httpServerResponse{{ + trailers: [][]string{{ + ":status", "200", + "content-type", "application/grpc", + "grpc-status", "0", + }}, + }}, + refuseStream: func(i uint32) bool { + switch i { + case 1, 5, 11, 15: // these stream IDs succeed + return false + } + return true // these are refused + }, + } + server.start(t, lis) + cc, err := grpc.Dial(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("failed to dial due to err: %v", err) + } + defer cc.Close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + client := testgrpc.NewTestServiceClient(cc) + + for i, tc := range testCases { + stream, err := client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("error creating stream due to err: %v", err) + } + code := func(err error) codes.Code { + if err == io.EOF { + return codes.OK + } + return status.Code(err) } + if _, err := stream.Recv(); code(err) != tc.errCode { + t.Fatalf("%v: stream.Recv() = _, %v, want error code: %v", i, err, tc.errCode) + } + } } func (s) TestCancel(t *testing.T) { for _, e := range listTestEnv() { - testCancel(t, e) + t.Run(e.name, func(t *testing.T) { + testCancel(t, e) + }) } } @@ -3550,7 +3045,7 @@ func testCancel(t *testing.T, e env) { defer te.tearDown() cc := te.clientConn() - tc := testpb.NewTestServiceClient(cc) + tc := testgrpc.NewTestServiceClient(cc) const argSize = 2718 const respSize = 314 @@ -3565,7 +3060,7 @@ func testCancel(t *testing.T, e env) { ResponseSize: respSize, Payload: payload, } - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) time.AfterFunc(1*time.Millisecond, cancel) if r, err := tc.UnaryCall(ctx, req); status.Code(err) != codes.Canceled { t.Fatalf("TestService/UnaryCall(_, _) = %v, %v; want _, error code: %s", r, err, codes.Canceled) @@ -3587,12 +3082,12 @@ func testCancelNoIO(t *testing.T, e env) { defer te.tearDown() cc := te.clientConn() - tc := testpb.NewTestServiceClient(cc) + tc := testgrpc.NewTestServiceClient(cc) // Start one blocked RPC for which we'll never send streaming // input. This will consume the 1 maximum concurrent streams, // causing future RPCs to hang. - ctx, cancelFirst := context.WithCancel(context.Background()) + ctx, cancelFirst := context.WithTimeout(context.Background(), defaultTestTimeout) _, err := tc.StreamingInputCall(ctx) if err != nil { t.Fatalf("%v.StreamingInputCall(_) = _, %v, want _, ", tc, err) @@ -3605,7 +3100,7 @@ func testCancelNoIO(t *testing.T, e env) { // succeeding. // TODO(bradfitz): add internal test hook for this (Issue 534) for { - ctx, cancelSecond := context.WithTimeout(context.Background(), 50*time.Millisecond) + ctx, cancelSecond := context.WithTimeout(context.Background(), defaultTestShortTimeout) _, err := tc.StreamingInputCall(ctx) cancelSecond() if err == nil { @@ -3627,7 +3122,7 @@ func testCancelNoIO(t *testing.T, e env) { }() // This should be blocked until the 1st is canceled, then succeed. - ctx, cancelThird := context.WithTimeout(context.Background(), 500*time.Millisecond) + ctx, cancelThird := context.WithTimeout(context.Background(), defaultTestShortTimeout) if _, err := tc.StreamingInputCall(ctx); err != nil { t.Errorf("%v.StreamingInputCall(_) = _, %v, want _, ", tc, err) } @@ -3653,7 +3148,7 @@ func testNoService(t *testing.T, e env) { defer te.tearDown() cc := te.clientConn() - tc := testpb.NewTestServiceClient(cc) + tc := testgrpc.NewTestServiceClient(cc) stream, err := tc.FullDuplexCall(te.ctx, grpc.WaitForReady(true)) if err != nil { @@ -3674,7 +3169,7 @@ func testPingPong(t *testing.T, e env) { te := newTest(t, e) te.startServer(&testServer{security: e.security}) defer te.tearDown() - tc := testpb.NewTestServiceClient(te.clientConn()) + tc := testgrpc.NewTestServiceClient(te.clientConn()) stream, err := tc.FullDuplexCall(te.ctx) if err != nil { @@ -3733,7 +3228,7 @@ func testMetadataStreamingRPC(t *testing.T, e env) { te := newTest(t, e) te.startServer(&testServer{security: e.security}) defer te.tearDown() - tc := testpb.NewTestServiceClient(te.clientConn()) + tc := testgrpc.NewTestServiceClient(te.clientConn()) ctx := metadata.NewOutgoingContext(te.ctx, testMetadata) stream, err := tc.FullDuplexCall(ctx) @@ -3748,6 +3243,7 @@ func testMetadataStreamingRPC(t *testing.T, e env) { delete(headerMD, "trailer") // ignore if present delete(headerMD, "user-agent") delete(headerMD, "content-type") + delete(headerMD, "grpc-accept-encoding") if err != nil || !reflect.DeepEqual(testMetadata, headerMD) { t.Errorf("#1 %v.Header() = %v, %v, want %v, ", stream, headerMD, err, testMetadata) } @@ -3756,6 +3252,7 @@ func testMetadataStreamingRPC(t *testing.T, e env) { delete(headerMD, "trailer") // ignore if present delete(headerMD, "user-agent") delete(headerMD, "content-type") + delete(headerMD, "grpc-accept-encoding") if err != nil || !reflect.DeepEqual(testMetadata, headerMD) { t.Errorf("#2 %v.Header() = %v, %v, want %v, ", stream, headerMD, err, testMetadata) } @@ -3810,7 +3307,7 @@ func testServerStreaming(t *testing.T, e env) { te := newTest(t, e) te.startServer(&testServer{security: e.security}) defer te.tearDown() - tc := testpb.NewTestServiceClient(te.clientConn()) + tc := testgrpc.NewTestServiceClient(te.clientConn()) respParam := make([]*testpb.ResponseParameters, len(respSizes)) for i, s := range respSizes { @@ -3822,7 +3319,10 @@ func testServerStreaming(t *testing.T, e env) { ResponseType: testpb.PayloadType_COMPRESSABLE, ResponseParameters: respParam, } - stream, err := tc.StreamingOutputCall(context.Background(), req) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + stream, err := tc.StreamingOutputCall(ctx, req) if err != nil { t.Fatalf("%v.StreamingOutputCall(_) = _, %v, want ", tc, err) } @@ -3865,7 +3365,7 @@ func testFailedServerStreaming(t *testing.T, e env) { te.userAgent = failAppUA te.startServer(&testServer{security: e.security}) defer te.tearDown() - tc := testpb.NewTestServiceClient(te.clientConn()) + tc := testgrpc.NewTestServiceClient(te.clientConn()) respParam := make([]*testpb.ResponseParameters, len(respSizes)) for i, s := range respSizes { @@ -3892,15 +3392,17 @@ func equalError(x, y error) bool { return x == y || (x != nil && y != nil && x.Error() == y.Error()) } -// concurrentSendServer is a TestServiceService whose +// concurrentSendServer is a TestServiceServer whose // StreamingOutputCall makes ten serial Send calls, sending payloads // "0".."9", inclusive. TestServerStreamingConcurrent verifies they // were received in the correct order, and that there were no races. // -// All other TestServiceService methods return unimplemented if called. -type concurrentSendServer struct{} +// All other TestServiceServer methods crash if called. +type concurrentSendServer struct { + testgrpc.TestServiceServer +} -func (s concurrentSendServer) StreamingOutputCall(args *testpb.StreamingOutputCallRequest, stream testpb.TestService_StreamingOutputCallServer) error { +func (s concurrentSendServer) StreamingOutputCall(args *testpb.StreamingOutputCallRequest, stream testgrpc.TestService_StreamingOutputCallServer) error { for i := 0; i < 10; i++ { stream.Send(&testpb.StreamingOutputCallResponse{ Payload: &testpb.Payload{ @@ -3924,11 +3426,13 @@ func testServerStreamingConcurrent(t *testing.T, e env) { defer te.tearDown() cc := te.clientConn() - tc := testpb.NewTestServiceClient(cc) + tc := testgrpc.NewTestServiceClient(cc) doStreamingCall := func() { req := &testpb.StreamingOutputCallRequest{} - stream, err := tc.StreamingOutputCall(context.Background(), req) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + stream, err := tc.StreamingOutputCall(ctx, req) if err != nil { t.Errorf("%v.StreamingOutputCall(_) = _, %v, want ", tc, err) return @@ -4003,9 +3507,9 @@ func testClientStreaming(t *testing.T, e env, sizes []int) { te := newTest(t, e) te.startServer(&testServer{security: e.security}) defer te.tearDown() - tc := testpb.NewTestServiceClient(te.clientConn()) + tc := testgrpc.NewTestServiceClient(te.clientConn()) - ctx, cancel := context.WithTimeout(te.ctx, time.Second*30) + ctx, cancel := context.WithTimeout(te.ctx, defaultTestTimeout) defer cancel() stream, err := tc.StreamingInputCall(ctx) if err != nil { @@ -4049,7 +3553,7 @@ func testClientStreamingError(t *testing.T, e env) { te := newTest(t, e) te.startServer(&testServer{security: e.security, earlyFail: true}) defer te.tearDown() - tc := testpb.NewTestServiceClient(te.clientConn()) + tc := testgrpc.NewTestServiceClient(te.clientConn()) stream, err := tc.StreamingInputCall(te.ctx) if err != nil { @@ -4096,7 +3600,7 @@ func testExceedMaxStreamsLimit(t *testing.T, e env) { defer te.tearDown() cc := te.clientConn() - tc := testpb.NewTestServiceClient(cc) + tc := testgrpc.NewTestServiceClient(cc) _, err := tc.StreamingInputCall(te.ctx) if err != nil { @@ -4104,7 +3608,7 @@ func testExceedMaxStreamsLimit(t *testing.T, e env) { } // Loop until receiving the new max stream setting from the server. for { - ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) defer cancel() _, err := tc.StreamingInputCall(ctx) if err == nil { @@ -4136,15 +3640,15 @@ func testStreamsQuotaRecovery(t *testing.T, e env) { defer te.tearDown() cc := te.clientConn() - tc := testpb.NewTestServiceClient(cc) - ctx, cancel := context.WithCancel(context.Background()) + tc := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() if _, err := tc.StreamingInputCall(ctx); err != nil { t.Fatalf("tc.StreamingInputCall(_) = _, %v, want _, ", err) } // Loop until the new max stream setting is effective. for { - ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) _, err := tc.StreamingInputCall(ctx) cancel() if err == nil { @@ -4173,7 +3677,7 @@ func testStreamsQuotaRecovery(t *testing.T, e env) { Payload: payload, } // No rpc should go through due to the max streams limit. - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) defer cancel() if _, err := tc.UnaryCall(ctx, req, grpc.WaitForReady(true)); status.Code(err) != codes.DeadlineExceeded { t.Errorf("tc.UnaryCall(_, _) = _, %v, want _, %s", err, codes.DeadlineExceeded) @@ -4184,192 +3688,38 @@ func testStreamsQuotaRecovery(t *testing.T, e env) { cancel() // A new stream should be allowed after canceling the first one. - ctx, cancel = context.WithTimeout(context.Background(), 5*time.Second) + ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() if _, err := tc.StreamingInputCall(ctx); err != nil { t.Fatalf("tc.StreamingInputCall(_) = _, %v, want _, %v", err, nil) } } -func (s) TestCompressServerHasNoSupport(t *testing.T) { +func (s) TestUnaryClientInterceptor(t *testing.T) { for _, e := range listTestEnv() { - testCompressServerHasNoSupport(t, e) + testUnaryClientInterceptor(t, e) + } +} + +func failOkayRPC(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + err := invoker(ctx, method, req, reply, cc, opts...) + if err == nil { + return status.Error(codes.NotFound, "") } + return err } -func testCompressServerHasNoSupport(t *testing.T, e env) { +func testUnaryClientInterceptor(t *testing.T, e env) { te := newTest(t, e) - te.serverCompression = false - te.clientCompression = false - te.clientNopCompression = true + te.userAgent = testAppUA + te.unaryClientInt = failOkayRPC te.startServer(&testServer{security: e.security}) defer te.tearDown() - tc := testpb.NewTestServiceClient(te.clientConn()) - const argSize = 271828 - const respSize = 314159 - payload, err := newPayload(testpb.PayloadType_COMPRESSABLE, argSize) - if err != nil { - t.Fatal(err) - } - req := &testpb.SimpleRequest{ - ResponseType: testpb.PayloadType_COMPRESSABLE, - ResponseSize: respSize, - Payload: payload, - } - if _, err := tc.UnaryCall(context.Background(), req); err == nil || status.Code(err) != codes.Unimplemented { - t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code %s", err, codes.Unimplemented) - } - // Streaming RPC - stream, err := tc.FullDuplexCall(context.Background()) - if err != nil { - t.Fatalf("%v.FullDuplexCall(_) = _, %v, want ", tc, err) - } - if _, err := stream.Recv(); err == nil || status.Code(err) != codes.Unimplemented { - t.Fatalf("%v.Recv() = %v, want error code %s", stream, err, codes.Unimplemented) - } -} - -func (s) TestCompressOK(t *testing.T) { - for _, e := range listTestEnv() { - testCompressOK(t, e) - } -} - -func testCompressOK(t *testing.T, e env) { - te := newTest(t, e) - te.serverCompression = true - te.clientCompression = true - te.startServer(&testServer{security: e.security}) - defer te.tearDown() - tc := testpb.NewTestServiceClient(te.clientConn()) - - // Unary call - const argSize = 271828 - const respSize = 314159 - payload, err := newPayload(testpb.PayloadType_COMPRESSABLE, argSize) - if err != nil { - t.Fatal(err) - } - req := &testpb.SimpleRequest{ - ResponseType: testpb.PayloadType_COMPRESSABLE, - ResponseSize: respSize, - Payload: payload, - } - ctx := metadata.NewOutgoingContext(context.Background(), metadata.Pairs("something", "something")) - if _, err := tc.UnaryCall(ctx, req); err != nil { - t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, ", err) - } - // Streaming RPC - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - stream, err := tc.FullDuplexCall(ctx) - if err != nil { - t.Fatalf("%v.FullDuplexCall(_) = _, %v, want ", tc, err) - } - respParam := []*testpb.ResponseParameters{ - { - Size: 31415, - }, - } - payload, err = newPayload(testpb.PayloadType_COMPRESSABLE, int32(31415)) - if err != nil { - t.Fatal(err) - } - sreq := &testpb.StreamingOutputCallRequest{ - ResponseType: testpb.PayloadType_COMPRESSABLE, - ResponseParameters: respParam, - Payload: payload, - } - if err := stream.Send(sreq); err != nil { - t.Fatalf("%v.Send(%v) = %v, want ", stream, sreq, err) - } - stream.CloseSend() - if _, err := stream.Recv(); err != nil { - t.Fatalf("%v.Recv() = %v, want ", stream, err) - } - if _, err := stream.Recv(); err != io.EOF { - t.Fatalf("%v.Recv() = %v, want io.EOF", stream, err) - } -} - -func (s) TestIdentityEncoding(t *testing.T) { - for _, e := range listTestEnv() { - testIdentityEncoding(t, e) - } -} - -func testIdentityEncoding(t *testing.T, e env) { - te := newTest(t, e) - te.startServer(&testServer{security: e.security}) - defer te.tearDown() - tc := testpb.NewTestServiceClient(te.clientConn()) - - // Unary call - payload, err := newPayload(testpb.PayloadType_COMPRESSABLE, 5) - if err != nil { - t.Fatal(err) - } - req := &testpb.SimpleRequest{ - ResponseType: testpb.PayloadType_COMPRESSABLE, - ResponseSize: 10, - Payload: payload, - } - ctx := metadata.NewOutgoingContext(context.Background(), metadata.Pairs("something", "something")) - if _, err := tc.UnaryCall(ctx, req); err != nil { - t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, ", err) - } - // Streaming RPC - ctx, cancel := context.WithCancel(context.Background()) + tc := testgrpc.NewTestServiceClient(te.clientConn()) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() - stream, err := tc.FullDuplexCall(ctx, grpc.UseCompressor("identity")) - if err != nil { - t.Fatalf("%v.FullDuplexCall(_) = _, %v, want ", tc, err) - } - payload, err = newPayload(testpb.PayloadType_COMPRESSABLE, int32(31415)) - if err != nil { - t.Fatal(err) - } - sreq := &testpb.StreamingOutputCallRequest{ - ResponseType: testpb.PayloadType_COMPRESSABLE, - ResponseParameters: []*testpb.ResponseParameters{{Size: 10}}, - Payload: payload, - } - if err := stream.Send(sreq); err != nil { - t.Fatalf("%v.Send(%v) = %v, want ", stream, sreq, err) - } - stream.CloseSend() - if _, err := stream.Recv(); err != nil { - t.Fatalf("%v.Recv() = %v, want ", stream, err) - } - if _, err := stream.Recv(); err != io.EOF { - t.Fatalf("%v.Recv() = %v, want io.EOF", stream, err) - } -} - -func (s) TestUnaryClientInterceptor(t *testing.T) { - for _, e := range listTestEnv() { - testUnaryClientInterceptor(t, e) - } -} - -func failOkayRPC(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { - err := invoker(ctx, method, req, reply, cc, opts...) - if err == nil { - return status.Error(codes.NotFound, "") - } - return err -} - -func testUnaryClientInterceptor(t *testing.T, e env) { - te := newTest(t, e) - te.userAgent = testAppUA - te.unaryClientInt = failOkayRPC - te.startServer(&testServer{security: e.security}) - defer te.tearDown() - - tc := testpb.NewTestServiceClient(te.clientConn()) - if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); status.Code(err) != codes.NotFound { + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.NotFound { t.Fatalf("%v.EmptyCall(_, _) = _, %v, want _, error code %s", tc, err, codes.NotFound) } } @@ -4394,7 +3744,7 @@ func testStreamClientInterceptor(t *testing.T, e env) { te.startServer(&testServer{security: e.security}) defer te.tearDown() - tc := testpb.NewTestServiceClient(te.clientConn()) + tc := testgrpc.NewTestServiceClient(te.clientConn()) respParam := []*testpb.ResponseParameters{ { Size: int32(1), @@ -4409,7 +3759,9 @@ func testStreamClientInterceptor(t *testing.T, e env) { ResponseParameters: respParam, Payload: payload, } - if _, err := tc.StreamingOutputCall(context.Background(), req); status.Code(err) != codes.NotFound { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.StreamingOutputCall(ctx, req); status.Code(err) != codes.NotFound { t.Fatalf("%v.StreamingOutputCall(_) = _, %v, want _, error code %s", tc, err, codes.NotFound) } } @@ -4420,7 +3772,7 @@ func (s) TestUnaryServerInterceptor(t *testing.T) { } } -func errInjector(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { +func errInjector(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { return nil, status.Error(codes.PermissionDenied, "") } @@ -4430,8 +3782,10 @@ func testUnaryServerInterceptor(t *testing.T, e env) { te.startServer(&testServer{security: e.security}) defer te.tearDown() - tc := testpb.NewTestServiceClient(te.clientConn()) - if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); status.Code(err) != codes.PermissionDenied { + tc := testgrpc.NewTestServiceClient(te.clientConn()) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.PermissionDenied { t.Fatalf("%v.EmptyCall(_, _) = _, %v, want _, error code %s", tc, err, codes.PermissionDenied) } } @@ -4446,7 +3800,7 @@ func (s) TestStreamServerInterceptor(t *testing.T) { } } -func fullDuplexOnly(srv interface{}, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { +func fullDuplexOnly(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { if info.FullMethod == "/grpc.testing.TestService/FullDuplexCall" { return handler(srv, ss) } @@ -4460,7 +3814,7 @@ func testStreamServerInterceptor(t *testing.T, e env) { te.startServer(&testServer{security: e.security}) defer te.tearDown() - tc := testpb.NewTestServiceClient(te.clientConn()) + tc := testgrpc.NewTestServiceClient(te.clientConn()) respParam := []*testpb.ResponseParameters{ { Size: int32(1), @@ -4475,14 +3829,16 @@ func testStreamServerInterceptor(t *testing.T, e env) { ResponseParameters: respParam, Payload: payload, } - s1, err := tc.StreamingOutputCall(context.Background(), req) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + s1, err := tc.StreamingOutputCall(ctx, req) if err != nil { t.Fatalf("%v.StreamingOutputCall(_) = _, %v, want _, ", tc, err) } if _, err := s1.Recv(); status.Code(err) != codes.PermissionDenied { t.Fatalf("%v.StreamingInputCall(_) = _, %v, want _, error code %s", tc, err, codes.PermissionDenied) } - s2, err := tc.FullDuplexCall(context.Background()) + s2, err := tc.FullDuplexCall(ctx) if err != nil { t.Fatalf("%v.FullDuplexCall(_) = _, %v, want ", tc, err) } @@ -4494,25 +3850,26 @@ func testStreamServerInterceptor(t *testing.T, e env) { } } -// funcServer implements methods of TestServiceService using funcs, +// funcServer implements methods of TestServiceServer using funcs, // similar to an http.HandlerFunc. -// Any unimplemented method will return unimplemented. Tests implement the method(s) +// Any unimplemented method will crash. Tests implement the method(s) // they need. type funcServer struct { + testgrpc.TestServiceServer unaryCall func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) - streamingInputCall func(stream testpb.TestService_StreamingInputCallServer) error - fullDuplexCall func(stream testpb.TestService_FullDuplexCallServer) error + streamingInputCall func(stream testgrpc.TestService_StreamingInputCallServer) error + fullDuplexCall func(stream testgrpc.TestService_FullDuplexCallServer) error } func (s *funcServer) UnaryCall(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { return s.unaryCall(ctx, in) } -func (s *funcServer) StreamingInputCall(stream testpb.TestService_StreamingInputCallServer) error { +func (s *funcServer) StreamingInputCall(stream testgrpc.TestService_StreamingInputCallServer) error { return s.streamingInputCall(stream) } -func (s *funcServer) FullDuplexCall(stream testpb.TestService_FullDuplexCallServer) error { +func (s *funcServer) FullDuplexCall(stream testgrpc.TestService_FullDuplexCallServer) error { return s.fullDuplexCall(stream) } @@ -4608,7 +3965,7 @@ func (s) TestClientRequestBodyErrorCancelStreamingInput(t *testing.T) { func testClientRequestBodyErrorCancelStreamingInput(t *testing.T, e env) { te := newTest(t, e) recvErr := make(chan error, 1) - ts := &funcServer{streamingInputCall: func(stream testpb.TestService_StreamingInputCallServer) error { + ts := &funcServer{streamingInputCall: func(stream testgrpc.TestService_StreamingInputCallServer) error { _, err := stream.Recv() recvErr <- err return nil @@ -4650,7 +4007,7 @@ func testClientInitialHeaderEndStream(t *testing.T, e env) { // checking. handlerDone := make(chan struct{}) te := newTest(t, e) - ts := &funcServer{streamingInputCall: func(stream testpb.TestService_StreamingInputCallServer) error { + ts := &funcServer{streamingInputCall: func(stream testgrpc.TestService_StreamingInputCallServer) error { defer close(handlerDone) // Block on serverTester receiving RST_STREAM. This ensures server has closed // stream before stream.Recv(). @@ -4694,7 +4051,7 @@ func testClientSendDataAfterCloseSend(t *testing.T, e env) { // checking. handlerDone := make(chan struct{}) te := newTest(t, e) - ts := &funcServer{streamingInputCall: func(stream testpb.TestService_StreamingInputCallServer) error { + ts := &funcServer{streamingInputCall: func(stream testgrpc.TestService_StreamingInputCallServer) error { defer close(handlerDone) // Block on serverTester receiving RST_STREAM. This ensures server has closed // stream before stream.Recv(). @@ -4713,8 +4070,8 @@ func testClientSendDataAfterCloseSend(t *testing.T, e env) { } if err := stream.SendMsg(nil); err == nil { t.Error("expected error sending message on stream after stream closed due to illegal data") - } else if status.Code(err) != codes.Internal { - t.Errorf("expected internal error, instead received '%v'", err) + } else if status.Code(err) != codes.Canceled { + t.Errorf("expected cancel error, instead received '%v'", err) } return nil }} @@ -4746,7 +4103,7 @@ func (s) TestClientResourceExhaustedCancelFullDuplex(t *testing.T) { func testClientResourceExhaustedCancelFullDuplex(t *testing.T, e env) { te := newTest(t, e) recvErr := make(chan error, 1) - ts := &funcServer{fullDuplexCall: func(stream testpb.TestService_FullDuplexCallServer) error { + ts := &funcServer{fullDuplexCall: func(stream testgrpc.TestService_FullDuplexCallServer) error { defer close(recvErr) _, err := stream.Recv() if err != nil { @@ -4784,8 +4141,11 @@ func testClientResourceExhaustedCancelFullDuplex(t *testing.T, e env) { // client side when server send a large message. te.maxClientReceiveMsgSize = newInt(10) cc := te.clientConn() - tc := testpb.NewTestServiceClient(cc) - stream, err := tc.FullDuplexCall(context.Background()) + tc := testgrpc.NewTestServiceClient(cc) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + stream, err := tc.FullDuplexCall(ctx) if err != nil { t.Fatalf("%v.FullDuplexCall(_) = _, %v, want ", tc, err) } @@ -4835,9 +4195,9 @@ func (s) TestFailfastRPCFailOnFatalHandshakeError(t *testing.T) { } defer cc.Close() - tc := testpb.NewTestServiceClient(cc) + tc := testgrpc.NewTestServiceClient(cc) // This unary call should fail, but not timeout. - ctx, cancel := context.WithTimeout(context.Background(), time.Second) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(false)); status.Code(err) != codes.Unavailable { t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want ", err) @@ -4853,11 +4213,9 @@ func (s) TestFlowControlLogicalRace(t *testing.T) { itemSize = 1 << 10 recvCount = 2 maxFailures = 3 - - requestTimeout = time.Second * 5 ) - requestCount := 10000 + requestCount := 3000 if raceMode { requestCount = 1000 } @@ -4869,68 +4227,57 @@ func (s) TestFlowControlLogicalRace(t *testing.T) { defer lis.Close() s := grpc.NewServer() - testpb.RegisterTestServiceService(s, testpb.NewTestServiceService(&flowControlLogicalRaceServer{ + testgrpc.RegisterTestServiceServer(s, &flowControlLogicalRaceServer{ itemCount: itemCount, itemSize: itemSize, - })) + }) defer s.Stop() go s.Serve(lis) - ctx := context.Background() - - cc, err := grpc.Dial(lis.Addr().String(), grpc.WithInsecure(), grpc.WithBlock()) + cc, err := grpc.Dial(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { t.Fatalf("grpc.Dial(%q) = %v", lis.Addr().String(), err) } defer cc.Close() - cl := testpb.NewTestServiceClient(cc) + cl := testgrpc.NewTestServiceClient(cc) failures := 0 for i := 0; i < requestCount; i++ { - ctx, cancel := context.WithTimeout(ctx, requestTimeout) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) output, err := cl.StreamingOutputCall(ctx, &testpb.StreamingOutputCallRequest{}) if err != nil { t.Fatalf("StreamingOutputCall; err = %q", err) } - j := 0 - loop: - for ; j < recvCount; j++ { - _, err := output.Recv() - if err != nil { - if err == io.EOF { - break loop - } - switch status.Code(err) { - case codes.DeadlineExceeded: - break loop - default: - t.Fatalf("Recv; err = %q", err) + for j := 0; j < recvCount; j++ { + if _, err := output.Recv(); err != nil { + if err == io.EOF || status.Code(err) == codes.DeadlineExceeded { + t.Errorf("got %d responses to request %d", j, i) + failures++ + break } + t.Fatalf("Recv; err = %q", err) } } cancel() - <-ctx.Done() - - if j < recvCount { - t.Errorf("got %d responses to request %d", j, i) - failures++ - if failures >= maxFailures { - // Continue past the first failure to see if the connection is - // entirely broken, or if only a single RPC was affected - break - } + + if failures >= maxFailures { + // Continue past the first failure to see if the connection is + // entirely broken, or if only a single RPC was affected + t.Fatalf("Too many failures received; aborting") } } } type flowControlLogicalRaceServer struct { + testgrpc.TestServiceServer + itemSize int itemCount int } -func (s *flowControlLogicalRaceServer) StreamingOutputCall(req *testpb.StreamingOutputCallRequest, srv testpb.TestService_StreamingOutputCallServer) error { +func (s *flowControlLogicalRaceServer) StreamingOutputCall(req *testpb.StreamingOutputCallRequest, srv testgrpc.TestService_StreamingOutputCallServer) error { for i := 0; i < s.itemCount; i++ { err := srv.Send(&testpb.StreamingOutputCallResponse{ Payload: &testpb.Payload{ @@ -5069,142 +4416,110 @@ func (fw *filterWriter) Write(p []byte) (n int, err error) { return fw.dst.Write(p) } -// stubServer is a server that is easy to customize within individual test -// cases. -type stubServer struct { - // Customizable implementations of server handlers. - emptyCall func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) - unaryCall func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) - fullDuplexCall func(stream testpb.TestService_FullDuplexCallServer) error - - // A client connected to this service the test may use. Created in Start(). - client testpb.TestServiceClient - cc *grpc.ClientConn - s *grpc.Server - - // Parameters for Listen and Dial. Defaults will be used if these are empty - // before Start. - network string - address string - target string - - cleanups []func() // Lambdas executed in Stop(); populated by Start(). - - r *manual.Resolver -} - -func (ss *stubServer) EmptyCall(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { - return ss.emptyCall(ctx, in) -} - -func (ss *stubServer) UnaryCall(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { - return ss.unaryCall(ctx, in) -} - -func (ss *stubServer) FullDuplexCall(stream testpb.TestService_FullDuplexCallServer) error { - return ss.fullDuplexCall(stream) -} +func (s) TestGRPCMethod(t *testing.T) { + var method string + var ok bool -// Start starts the server and creates a client connected to it. -func (ss *stubServer) Start(sopts []grpc.ServerOption, dopts ...grpc.DialOption) error { - if ss.network == "" { - ss.network = "tcp" - } - if ss.address == "" { - ss.address = "localhost:0" - } - if ss.target == "" { - ss.r = manual.NewBuilderWithScheme("whatever") + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { + method, ok = grpc.Method(ctx) + return &testpb.Empty{}, nil + }, } - - lis, err := net.Listen(ss.network, ss.address) - if err != nil { - return fmt.Errorf("net.Listen(%q, %q) = %v", ss.network, ss.address, err) + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) } - ss.address = lis.Addr().String() - ss.cleanups = append(ss.cleanups, func() { lis.Close() }) + defer ss.Stop() - s := grpc.NewServer(sopts...) - testpb.RegisterTestServiceService(s, testpb.NewTestServiceService(ss)) - go s.Serve(lis) - ss.cleanups = append(ss.cleanups, s.Stop) - ss.s = s + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() - opts := append([]grpc.DialOption{grpc.WithInsecure()}, dopts...) - if ss.r != nil { - ss.target = ss.r.Scheme() + ":///" + ss.address - opts = append(opts, grpc.WithResolvers(ss.r)) + if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("ss.Client.EmptyCall(_, _) = _, %v; want _, nil", err) } - cc, err := grpc.Dial(ss.target, opts...) - if err != nil { - return fmt.Errorf("grpc.Dial(%q) = %v", ss.target, err) - } - ss.cc = cc - if ss.r != nil { - ss.r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: ss.address}}}) - } - if err := ss.waitForReady(cc); err != nil { - return err + if want := "/grpc.testing.TestService/EmptyCall"; !ok || method != want { + t.Fatalf("grpc.Method(_) = %q, %v; want %q, true", method, ok, want) } +} - ss.cleanups = append(ss.cleanups, func() { cc.Close() }) - - ss.client = testpb.NewTestServiceClient(cc) - return nil +// renameProtoCodec is an encoding.Codec wrapper that allows customizing the +// Name() of another codec. +type renameProtoCodec struct { + encoding.Codec + name string } -func (ss *stubServer) newServiceConfig(sc string) { - if ss.r != nil { - ss.r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: ss.address}}, ServiceConfig: parseCfg(ss.r, sc)}) +func (r *renameProtoCodec) Name() string { return r.name } + +// TestForceCodecName confirms that the ForceCodec call option sets the subtype +// in the content-type header according to the Name() of the codec provided. +func (s) TestForceCodecName(t *testing.T) { + wantContentTypeCh := make(chan []string, 1) + defer close(wantContentTypeCh) + + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return nil, status.Errorf(codes.Internal, "no metadata in context") + } + if got, want := md["content-type"], <-wantContentTypeCh; !reflect.DeepEqual(got, want) { + return nil, status.Errorf(codes.Internal, "got content-type=%q; want [%q]", got, want) + } + return &testpb.Empty{}, nil + }, } -} + if err := ss.Start([]grpc.ServerOption{grpc.ForceServerCodec(encoding.GetCodec("proto"))}); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() -func (ss *stubServer) waitForReady(cc *grpc.ClientConn) error { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() - for { - s := cc.GetState() - if s == connectivity.Ready { - return nil - } - if !cc.WaitForStateChange(ctx, s) { - // ctx got timeout or canceled. - return ctx.Err() - } + + codec := &renameProtoCodec{Codec: encoding.GetCodec("proto"), name: "some-test-name"} + wantContentTypeCh <- []string{"application/grpc+some-test-name"} + if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}, grpc.ForceCodec(codec)); err != nil { + t.Fatalf("ss.Client.EmptyCall(_, _) = _, %v; want _, nil", err) } -} -func (ss *stubServer) Stop() { - for i := len(ss.cleanups) - 1; i >= 0; i-- { - ss.cleanups[i]() + // Confirm the name is converted to lowercase before transmitting. + codec.name = "aNoTHeRNaME" + wantContentTypeCh <- []string{"application/grpc+anothername"} + if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}, grpc.ForceCodec(codec)); err != nil { + t.Fatalf("ss.Client.EmptyCall(_, _) = _, %v; want _, nil", err) } } -func (s) TestGRPCMethod(t *testing.T) { - var method string - var ok bool - - ss := &stubServer{ - emptyCall: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { - method, ok = grpc.Method(ctx) +func (s) TestForceServerCodec(t *testing.T) { + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil }, } - if err := ss.Start(nil); err != nil { + codec := &countingProtoCodec{} + if err := ss.Start([]grpc.ServerOption{grpc.ForceServerCodec(codec)}); err != nil { t.Fatalf("Error starting endpoint server: %v", err) } defer ss.Stop() - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() - if _, err := ss.client.EmptyCall(ctx, &testpb.Empty{}); err != nil { - t.Fatalf("ss.client.EmptyCall(_, _) = _, %v; want _, nil", err) + if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("ss.Client.EmptyCall(_, _) = _, %v; want _, nil", err) } - if want := "/grpc.testing.TestService/EmptyCall"; !ok || method != want { - t.Fatalf("grpc.Method(_) = %q, %v; want %q, true", method, ok, want) + unmarshalCount := atomic.LoadInt32(&codec.unmarshalCount) + const wantUnmarshalCount = 1 + if unmarshalCount != wantUnmarshalCount { + t.Fatalf("protoCodec.unmarshalCount = %d; want %d", unmarshalCount, wantUnmarshalCount) + } + marshalCount := atomic.LoadInt32(&codec.marshalCount) + const wantMarshalCount = 1 + if marshalCount != wantMarshalCount { + t.Fatalf("protoCodec.marshalCount = %d; want %d", marshalCount, wantMarshalCount) } } @@ -5212,8 +4527,8 @@ func (s) TestUnaryProxyDoesNotForwardMetadata(t *testing.T) { const mdkey = "somedata" // endpoint ensures mdkey is NOT in metadata and returns an error if it is. - endpoint := &stubServer{ - emptyCall: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { + endpoint := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { if md, ok := metadata.FromIncomingContext(ctx); !ok || md[mdkey] != nil { return nil, status.Errorf(codes.Internal, "endpoint: md=%v; want !contains(%q)", md, mdkey) } @@ -5227,12 +4542,12 @@ func (s) TestUnaryProxyDoesNotForwardMetadata(t *testing.T) { // proxy ensures mdkey IS in metadata, then forwards the RPC to endpoint // without explicitly copying the metadata. - proxy := &stubServer{ - emptyCall: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { + proxy := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { if md, ok := metadata.FromIncomingContext(ctx); !ok || md[mdkey] == nil { return nil, status.Errorf(codes.Internal, "proxy: md=%v; want contains(%q)", md, mdkey) } - return endpoint.client.EmptyCall(ctx, in) + return endpoint.Client.EmptyCall(ctx, in) }, } if err := proxy.Start(nil); err != nil { @@ -5240,18 +4555,18 @@ func (s) TestUnaryProxyDoesNotForwardMetadata(t *testing.T) { } defer proxy.Stop() - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() md := metadata.Pairs(mdkey, "val") ctx = metadata.NewOutgoingContext(ctx, md) // Sanity check that endpoint properly errors when it sees mdkey. - _, err := endpoint.client.EmptyCall(ctx, &testpb.Empty{}) + _, err := endpoint.Client.EmptyCall(ctx, &testpb.Empty{}) if s, ok := status.FromError(err); !ok || s.Code() != codes.Internal { - t.Fatalf("endpoint.client.EmptyCall(_, _) = _, %v; want _, ", err) + t.Fatalf("endpoint.Client.EmptyCall(_, _) = _, %v; want _, ", err) } - if _, err := proxy.client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + if _, err := proxy.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil { t.Fatal(err.Error()) } } @@ -5262,7 +4577,7 @@ func (s) TestStreamingProxyDoesNotForwardMetadata(t *testing.T) { // doFDC performs a FullDuplexCall with client and returns the error from the // first stream.Recv call, or nil if that error is io.EOF. Calls t.Fatal if // the stream cannot be established. - doFDC := func(ctx context.Context, client testpb.TestServiceClient) error { + doFDC := func(ctx context.Context, client testgrpc.TestServiceClient) error { stream, err := client.FullDuplexCall(ctx) if err != nil { t.Fatalf("Unwanted error: %v", err) @@ -5274,8 +4589,8 @@ func (s) TestStreamingProxyDoesNotForwardMetadata(t *testing.T) { } // endpoint ensures mdkey is NOT in metadata and returns an error if it is. - endpoint := &stubServer{ - fullDuplexCall: func(stream testpb.TestService_FullDuplexCallServer) error { + endpoint := &stubserver.StubServer{ + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { ctx := stream.Context() if md, ok := metadata.FromIncomingContext(ctx); !ok || md[mdkey] != nil { return status.Errorf(codes.Internal, "endpoint: md=%v; want !contains(%q)", md, mdkey) @@ -5290,13 +4605,13 @@ func (s) TestStreamingProxyDoesNotForwardMetadata(t *testing.T) { // proxy ensures mdkey IS in metadata, then forwards the RPC to endpoint // without explicitly copying the metadata. - proxy := &stubServer{ - fullDuplexCall: func(stream testpb.TestService_FullDuplexCallServer) error { + proxy := &stubserver.StubServer{ + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { ctx := stream.Context() if md, ok := metadata.FromIncomingContext(ctx); !ok || md[mdkey] == nil { return status.Errorf(codes.Internal, "endpoint: md=%v; want !contains(%q)", md, mdkey) } - return doFDC(ctx, endpoint.client) + return doFDC(ctx, endpoint.Client) }, } if err := proxy.Start(nil); err != nil { @@ -5304,19 +4619,19 @@ func (s) TestStreamingProxyDoesNotForwardMetadata(t *testing.T) { } defer proxy.Stop() - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() md := metadata.Pairs(mdkey, "val") ctx = metadata.NewOutgoingContext(ctx, md) // Sanity check that endpoint properly errors when it sees mdkey in ctx. - err := doFDC(ctx, endpoint.client) + err := doFDC(ctx, endpoint.Client) if s, ok := status.FromError(err); !ok || s.Code() != codes.Internal { t.Fatalf("stream.Recv() = _, %v; want _, ", err) } - if err := doFDC(ctx, proxy.client); err != nil { - t.Fatalf("doFDC(_, proxy.client) = %v; want nil", err) + if err := doFDC(ctx, proxy.Client); err != nil { + t.Fatalf("doFDC(_, proxy.Client) = %v; want nil", err) } } @@ -5327,8 +4642,8 @@ func (s) TestStatsTagsAndTrace(t *testing.T) { // endpoint ensures Tags() and Trace() in context match those that were added // by the client and returns an error if not. - endpoint := &stubServer{ - emptyCall: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { + endpoint := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { md, _ := metadata.FromIncomingContext(ctx) if tg := stats.Tags(ctx); !reflect.DeepEqual(tg, tags) { return nil, status.Errorf(codes.Internal, "stats.Tags(%v)=%v; want %v", ctx, tg, tags) @@ -5350,7 +4665,7 @@ func (s) TestStatsTagsAndTrace(t *testing.T) { } defer endpoint.Stop() - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() testCases := []struct { @@ -5365,12 +4680,12 @@ func (s) TestStatsTagsAndTrace(t *testing.T) { } for _, tc := range testCases { - _, err := endpoint.client.EmptyCall(tc.ctx, &testpb.Empty{}) + _, err := endpoint.Client.EmptyCall(tc.ctx, &testpb.Empty{}) if tc.want == codes.OK && err != nil { - t.Fatalf("endpoint.client.EmptyCall(%v, _) = _, %v; want _, nil", tc.ctx, err) + t.Fatalf("endpoint.Client.EmptyCall(%v, _) = _, %v; want _, nil", tc.ctx, err) } if s, ok := status.FromError(err); !ok || s.Code() != tc.want { - t.Fatalf("endpoint.client.EmptyCall(%v, _) = _, %v; want _, ", tc.ctx, err, tc.want) + t.Fatalf("endpoint.Client.EmptyCall(%v, _) = _, %v; want _, ", tc.ctx, err, tc.want) } } } @@ -5387,8 +4702,8 @@ func (s) TestTapTimeout(t *testing.T) { }), } - ss := &stubServer{ - emptyCall: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { <-ctx.Done() return nil, status.Errorf(codes.Canceled, ctx.Err().Error()) }, @@ -5401,19 +4716,19 @@ func (s) TestTapTimeout(t *testing.T) { // This was known to be flaky; test several times. for i := 0; i < 10; i++ { // Set our own deadline in case the server hangs. - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - res, err := ss.client.EmptyCall(ctx, &testpb.Empty{}) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + res, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}) cancel() if s, ok := status.FromError(err); !ok || s.Code() != codes.Canceled { - t.Fatalf("ss.client.EmptyCall(context.Background(), _) = %v, %v; want nil, ", res, err) + t.Fatalf("ss.Client.EmptyCall(ctx, _) = %v, %v; want nil, ", res, err) } } } func (s) TestClientWriteFailsAfterServerClosesStream(t *testing.T) { - ss := &stubServer{ - fullDuplexCall: func(stream testpb.TestService_FullDuplexCallServer) error { + ss := &stubserver.StubServer{ + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { return status.Errorf(codes.Internal, "") }, } @@ -5422,9 +4737,9 @@ func (s) TestClientWriteFailsAfterServerClosesStream(t *testing.T) { t.Fatalf("Error starting endpoint server: %v", err) } defer ss.Stop() - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() - stream, err := ss.client.FullDuplexCall(ctx) + stream, err := ss.Client.FullDuplexCall(ctx) if err != nil { t.Fatalf("Error while creating stream: %v", err) } @@ -5488,8 +4803,10 @@ func testConfigurableWindowSize(t *testing.T, e env, wc windowSizeConfig) { defer te.tearDown() cc := te.clientConn() - tc := testpb.NewTestServiceClient(cc) - stream, err := tc.FullDuplexCall(context.Background()) + tc := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + stream, err := tc.FullDuplexCall(ctx) if err != nil { t.Fatalf("%v.FullDuplexCall(_) = _, %v, want ", tc, err) } @@ -5538,18 +4855,10 @@ func testWaitForReadyConnection(t *testing.T, e env) { defer te.tearDown() cc := te.clientConn() // Non-blocking dial. - tc := testpb.NewTestServiceClient(cc) - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - state := cc.GetState() - // Wait for connection to be Ready. - for ; state != connectivity.Ready && cc.WaitForStateChange(ctx, state); state = cc.GetState() { - } - if state != connectivity.Ready { - t.Fatalf("Want connection state to be Ready, got %v", state) - } - ctx, cancel = context.WithTimeout(context.Background(), time.Second) + tc := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() + testutils.AwaitState(ctx, t, cc, connectivity.Ready) // Make a fail-fast RPC. if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil { t.Fatalf("TestService/EmptyCall(_,_) = _, %v, want _, nil", err) @@ -5560,14 +4869,14 @@ type errCodec struct { noError bool } -func (c *errCodec) Marshal(v interface{}) ([]byte, error) { +func (c *errCodec) Marshal(v any) ([]byte, error) { if c.noError { return []byte{}, nil } return nil, fmt.Errorf("3987^12 + 4365^12 = 4472^12") } -func (c *errCodec) Unmarshal(data []byte, v interface{}) error { +func (c *errCodec) Unmarshal(data []byte, v any) error { return nil } @@ -5575,6 +4884,33 @@ func (c *errCodec) Name() string { return "Fermat's near-miss." } +type countingProtoCodec struct { + marshalCount int32 + unmarshalCount int32 +} + +func (p *countingProtoCodec) Marshal(v any) ([]byte, error) { + atomic.AddInt32(&p.marshalCount, 1) + vv, ok := v.(proto.Message) + if !ok { + return nil, fmt.Errorf("failed to marshal, message is %T, want proto.Message", v) + } + return proto.Marshal(vv) +} + +func (p *countingProtoCodec) Unmarshal(data []byte, v any) error { + atomic.AddInt32(&p.unmarshalCount, 1) + vv, ok := v.(proto.Message) + if !ok { + return fmt.Errorf("failed to unmarshal, message is %T, want proto.Message", v) + } + return proto.Unmarshal(data, vv) +} + +func (*countingProtoCodec) Name() string { + return "proto" +} + func (s) TestEncodeDoesntPanic(t *testing.T) { for _, e := range listTestEnv() { testEncodeDoesntPanic(t, e) @@ -5588,12 +4924,14 @@ func testEncodeDoesntPanic(t *testing.T, e env) { te.startServer(&testServer{security: e.security}) defer te.tearDown() te.customCodec = nil - tc := testpb.NewTestServiceClient(te.clientConn()) + tc := testgrpc.NewTestServiceClient(te.clientConn()) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() // Failure case, should not panic. - tc.EmptyCall(context.Background(), &testpb.Empty{}) + tc.EmptyCall(ctx, &testpb.Empty{}) erc.noError = true // Passing case. - if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); err != nil { + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil { t.Fatalf("EmptyCall(_, _) = _, %v, want _, ", err) } } @@ -5621,7 +4959,7 @@ func testSvrWriteStatusEarlyWrite(t *testing.T, e env) { } te.startServer(&testServer{security: e.security}) defer te.tearDown() - tc := testpb.NewTestServiceClient(te.clientConn()) + tc := testgrpc.NewTestServiceClient(te.clientConn()) respParam := []*testpb.ResponseParameters{ { Size: int32(smallSize), @@ -5659,442 +4997,43 @@ func testSvrWriteStatusEarlyWrite(t *testing.T, e env) { } } -// The following functions with function name ending with TD indicates that they -// should be deleted after old service config API is deprecated and deleted. -func testServiceConfigSetupTD(t *testing.T, e env) (*test, chan grpc.ServiceConfig) { - te := newTest(t, e) - // We write before read. - ch := make(chan grpc.ServiceConfig, 1) - te.sc = ch - te.userAgent = testAppUA - te.declareLogNoise( - "transport: http2Client.notifyError got notified that the client transport was broken EOF", - "grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing", - "grpc: addrConn.resetTransport failed to create client transport: connection error", - "Failed to dial : context canceled; please retry.", - ) - return te, ch -} +// TestMalformedStreamMethod starts a test server and sends an RPC with a +// malformed method name. The server should respond with an UNIMPLEMENTED status +// code in this case. +func (s) TestMalformedStreamMethod(t *testing.T) { + const testMethod = "a-method-name-without-any-slashes" + te := newTest(t, tcpClearRREnv) + te.startServer(nil) + defer te.tearDown() -func (s) TestServiceConfigGetMethodConfigTD(t *testing.T) { - for _, e := range listTestEnv() { - testGetMethodConfigTD(t, e) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + err := te.clientConn().Invoke(ctx, testMethod, nil, nil) + if gotCode := status.Code(err); gotCode != codes.Unimplemented { + t.Fatalf("Invoke with method %q, got code %s, want %s", testMethod, gotCode, codes.Unimplemented) } } -func testGetMethodConfigTD(t *testing.T, e env) { - te, ch := testServiceConfigSetupTD(t, e) - defer te.tearDown() - - mc1 := grpc.MethodConfig{ - WaitForReady: newBool(true), - Timeout: newDuration(time.Millisecond), - } - mc2 := grpc.MethodConfig{WaitForReady: newBool(false)} - m := make(map[string]grpc.MethodConfig) - m["/grpc.testing.TestService/EmptyCall"] = mc1 - m["/grpc.testing.TestService/"] = mc2 - sc := grpc.ServiceConfig{ - Methods: m, +func (s) TestMethodFromServerStream(t *testing.T) { + const testMethod = "/package.service/method" + e := tcpClearRREnv + te := newTest(t, e) + var method string + var ok bool + te.unknownHandler = func(srv any, stream grpc.ServerStream) error { + method, ok = grpc.MethodFromServerStream(stream) + return nil } - ch <- sc - cc := te.clientConn() - tc := testpb.NewTestServiceClient(cc) - // The following RPCs are expected to become non-fail-fast ones with 1ms deadline. - if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); status.Code(err) != codes.DeadlineExceeded { - t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, %s", err, codes.DeadlineExceeded) + te.startServer(nil) + defer te.tearDown() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + _ = te.clientConn().Invoke(ctx, testMethod, nil, nil) + if !ok || method != testMethod { + t.Fatalf("Invoke with method %q, got %q, %v, want %q, true", testMethod, method, ok, testMethod) } - - m = make(map[string]grpc.MethodConfig) - m["/grpc.testing.TestService/UnaryCall"] = mc1 - m["/grpc.testing.TestService/"] = mc2 - sc = grpc.ServiceConfig{ - Methods: m, - } - ch <- sc - // Wait for the new service config to propagate. - for { - if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); status.Code(err) == codes.DeadlineExceeded { - continue - } - break - } - // The following RPCs are expected to become fail-fast. - if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); status.Code(err) != codes.Unavailable { - t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, %s", err, codes.Unavailable) - } -} - -func (s) TestServiceConfigWaitForReadyTD(t *testing.T) { - for _, e := range listTestEnv() { - testServiceConfigWaitForReadyTD(t, e) - } -} - -func testServiceConfigWaitForReadyTD(t *testing.T, e env) { - te, ch := testServiceConfigSetupTD(t, e) - defer te.tearDown() - - // Case1: Client API set failfast to be false, and service config set wait_for_ready to be false, Client API should win, and the rpc will wait until deadline exceeds. - mc := grpc.MethodConfig{ - WaitForReady: newBool(false), - Timeout: newDuration(time.Millisecond), - } - m := make(map[string]grpc.MethodConfig) - m["/grpc.testing.TestService/EmptyCall"] = mc - m["/grpc.testing.TestService/FullDuplexCall"] = mc - sc := grpc.ServiceConfig{ - Methods: m, - } - ch <- sc - - cc := te.clientConn() - tc := testpb.NewTestServiceClient(cc) - // The following RPCs are expected to become non-fail-fast ones with 1ms deadline. - if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}, grpc.WaitForReady(true)); status.Code(err) != codes.DeadlineExceeded { - t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, %s", err, codes.DeadlineExceeded) - } - if _, err := tc.FullDuplexCall(context.Background(), grpc.WaitForReady(true)); status.Code(err) != codes.DeadlineExceeded { - t.Fatalf("TestService/FullDuplexCall(_) = _, %v, want %s", err, codes.DeadlineExceeded) - } - - // Generate a service config update. - // Case2: Client API does not set failfast, and service config set wait_for_ready to be true, and the rpc will wait until deadline exceeds. - mc.WaitForReady = newBool(true) - m = make(map[string]grpc.MethodConfig) - m["/grpc.testing.TestService/EmptyCall"] = mc - m["/grpc.testing.TestService/FullDuplexCall"] = mc - sc = grpc.ServiceConfig{ - Methods: m, - } - ch <- sc - - // Wait for the new service config to take effect. - mc = cc.GetMethodConfig("/grpc.testing.TestService/EmptyCall") - for { - if !*mc.WaitForReady { - time.Sleep(100 * time.Millisecond) - mc = cc.GetMethodConfig("/grpc.testing.TestService/EmptyCall") - continue - } - break - } - // The following RPCs are expected to become non-fail-fast ones with 1ms deadline. - if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); status.Code(err) != codes.DeadlineExceeded { - t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, %s", err, codes.DeadlineExceeded) - } - if _, err := tc.FullDuplexCall(context.Background()); status.Code(err) != codes.DeadlineExceeded { - t.Fatalf("TestService/FullDuplexCall(_) = _, %v, want %s", err, codes.DeadlineExceeded) - } -} - -func (s) TestServiceConfigTimeoutTD(t *testing.T) { - for _, e := range listTestEnv() { - testServiceConfigTimeoutTD(t, e) - } -} - -func testServiceConfigTimeoutTD(t *testing.T, e env) { - te, ch := testServiceConfigSetupTD(t, e) - defer te.tearDown() - - // Case1: Client API sets timeout to be 1ns and ServiceConfig sets timeout to be 1hr. Timeout should be 1ns (min of 1ns and 1hr) and the rpc will wait until deadline exceeds. - mc := grpc.MethodConfig{ - Timeout: newDuration(time.Hour), - } - m := make(map[string]grpc.MethodConfig) - m["/grpc.testing.TestService/EmptyCall"] = mc - m["/grpc.testing.TestService/FullDuplexCall"] = mc - sc := grpc.ServiceConfig{ - Methods: m, - } - ch <- sc - - cc := te.clientConn() - tc := testpb.NewTestServiceClient(cc) - // The following RPCs are expected to become non-fail-fast ones with 1ns deadline. - ctx, cancel := context.WithTimeout(context.Background(), time.Nanosecond) - if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); status.Code(err) != codes.DeadlineExceeded { - t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, %s", err, codes.DeadlineExceeded) - } - cancel() - ctx, cancel = context.WithTimeout(context.Background(), time.Nanosecond) - if _, err := tc.FullDuplexCall(ctx, grpc.WaitForReady(true)); status.Code(err) != codes.DeadlineExceeded { - t.Fatalf("TestService/FullDuplexCall(_) = _, %v, want %s", err, codes.DeadlineExceeded) - } - cancel() - - // Generate a service config update. - // Case2: Client API sets timeout to be 1hr and ServiceConfig sets timeout to be 1ns. Timeout should be 1ns (min of 1ns and 1hr) and the rpc will wait until deadline exceeds. - mc.Timeout = newDuration(time.Nanosecond) - m = make(map[string]grpc.MethodConfig) - m["/grpc.testing.TestService/EmptyCall"] = mc - m["/grpc.testing.TestService/FullDuplexCall"] = mc - sc = grpc.ServiceConfig{ - Methods: m, - } - ch <- sc - - // Wait for the new service config to take effect. - mc = cc.GetMethodConfig("/grpc.testing.TestService/FullDuplexCall") - for { - if *mc.Timeout != time.Nanosecond { - time.Sleep(100 * time.Millisecond) - mc = cc.GetMethodConfig("/grpc.testing.TestService/FullDuplexCall") - continue - } - break - } - - ctx, cancel = context.WithTimeout(context.Background(), time.Hour) - if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); status.Code(err) != codes.DeadlineExceeded { - t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, %s", err, codes.DeadlineExceeded) - } - cancel() - - ctx, cancel = context.WithTimeout(context.Background(), time.Hour) - if _, err := tc.FullDuplexCall(ctx, grpc.WaitForReady(true)); status.Code(err) != codes.DeadlineExceeded { - t.Fatalf("TestService/FullDuplexCall(_) = _, %v, want %s", err, codes.DeadlineExceeded) - } - cancel() -} - -func (s) TestServiceConfigMaxMsgSizeTD(t *testing.T) { - for _, e := range listTestEnv() { - testServiceConfigMaxMsgSizeTD(t, e) - } -} - -func testServiceConfigMaxMsgSizeTD(t *testing.T, e env) { - // Setting up values and objects shared across all test cases. - const smallSize = 1 - const largeSize = 1024 - const extraLargeSize = 2048 - - smallPayload, err := newPayload(testpb.PayloadType_COMPRESSABLE, smallSize) - if err != nil { - t.Fatal(err) - } - largePayload, err := newPayload(testpb.PayloadType_COMPRESSABLE, largeSize) - if err != nil { - t.Fatal(err) - } - extraLargePayload, err := newPayload(testpb.PayloadType_COMPRESSABLE, extraLargeSize) - if err != nil { - t.Fatal(err) - } - - mc := grpc.MethodConfig{ - MaxReqSize: newInt(extraLargeSize), - MaxRespSize: newInt(extraLargeSize), - } - - m := make(map[string]grpc.MethodConfig) - m["/grpc.testing.TestService/UnaryCall"] = mc - m["/grpc.testing.TestService/FullDuplexCall"] = mc - sc := grpc.ServiceConfig{ - Methods: m, - } - // Case1: sc set maxReqSize to 2048 (send), maxRespSize to 2048 (recv). - te1, ch1 := testServiceConfigSetupTD(t, e) - te1.startServer(&testServer{security: e.security}) - defer te1.tearDown() - - ch1 <- sc - tc := testpb.NewTestServiceClient(te1.clientConn()) - - req := &testpb.SimpleRequest{ - ResponseType: testpb.PayloadType_COMPRESSABLE, - ResponseSize: int32(extraLargeSize), - Payload: smallPayload, - } - // Test for unary RPC recv. - if _, err := tc.UnaryCall(context.Background(), req); err == nil || status.Code(err) != codes.ResourceExhausted { - t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted) - } - - // Test for unary RPC send. - req.Payload = extraLargePayload - req.ResponseSize = int32(smallSize) - if _, err := tc.UnaryCall(context.Background(), req); err == nil || status.Code(err) != codes.ResourceExhausted { - t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted) - } - - // Test for streaming RPC recv. - respParam := []*testpb.ResponseParameters{ - { - Size: int32(extraLargeSize), - }, - } - sreq := &testpb.StreamingOutputCallRequest{ - ResponseType: testpb.PayloadType_COMPRESSABLE, - ResponseParameters: respParam, - Payload: smallPayload, - } - stream, err := tc.FullDuplexCall(te1.ctx) - if err != nil { - t.Fatalf("%v.FullDuplexCall(_) = _, %v, want ", tc, err) - } - if err := stream.Send(sreq); err != nil { - t.Fatalf("%v.Send(%v) = %v, want ", stream, sreq, err) - } - if _, err := stream.Recv(); err == nil || status.Code(err) != codes.ResourceExhausted { - t.Fatalf("%v.Recv() = _, %v, want _, error code: %s", stream, err, codes.ResourceExhausted) - } - - // Test for streaming RPC send. - respParam[0].Size = int32(smallSize) - sreq.Payload = extraLargePayload - stream, err = tc.FullDuplexCall(te1.ctx) - if err != nil { - t.Fatalf("%v.FullDuplexCall(_) = _, %v, want ", tc, err) - } - if err := stream.Send(sreq); err == nil || status.Code(err) != codes.ResourceExhausted { - t.Fatalf("%v.Send(%v) = %v, want _, error code: %s", stream, sreq, err, codes.ResourceExhausted) - } - - // Case2: Client API set maxReqSize to 1024 (send), maxRespSize to 1024 (recv). Sc sets maxReqSize to 2048 (send), maxRespSize to 2048 (recv). - te2, ch2 := testServiceConfigSetupTD(t, e) - te2.maxClientReceiveMsgSize = newInt(1024) - te2.maxClientSendMsgSize = newInt(1024) - te2.startServer(&testServer{security: e.security}) - defer te2.tearDown() - ch2 <- sc - tc = testpb.NewTestServiceClient(te2.clientConn()) - - // Test for unary RPC recv. - req.Payload = smallPayload - req.ResponseSize = int32(largeSize) - - if _, err := tc.UnaryCall(context.Background(), req); err == nil || status.Code(err) != codes.ResourceExhausted { - t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted) - } - - // Test for unary RPC send. - req.Payload = largePayload - req.ResponseSize = int32(smallSize) - if _, err := tc.UnaryCall(context.Background(), req); err == nil || status.Code(err) != codes.ResourceExhausted { - t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted) - } - - // Test for streaming RPC recv. - stream, err = tc.FullDuplexCall(te2.ctx) - respParam[0].Size = int32(largeSize) - sreq.Payload = smallPayload - if err != nil { - t.Fatalf("%v.FullDuplexCall(_) = _, %v, want ", tc, err) - } - if err := stream.Send(sreq); err != nil { - t.Fatalf("%v.Send(%v) = %v, want ", stream, sreq, err) - } - if _, err := stream.Recv(); err == nil || status.Code(err) != codes.ResourceExhausted { - t.Fatalf("%v.Recv() = _, %v, want _, error code: %s", stream, err, codes.ResourceExhausted) - } - - // Test for streaming RPC send. - respParam[0].Size = int32(smallSize) - sreq.Payload = largePayload - stream, err = tc.FullDuplexCall(te2.ctx) - if err != nil { - t.Fatalf("%v.FullDuplexCall(_) = _, %v, want ", tc, err) - } - if err := stream.Send(sreq); err == nil || status.Code(err) != codes.ResourceExhausted { - t.Fatalf("%v.Send(%v) = %v, want _, error code: %s", stream, sreq, err, codes.ResourceExhausted) - } - - // Case3: Client API set maxReqSize to 4096 (send), maxRespSize to 4096 (recv). Sc sets maxReqSize to 2048 (send), maxRespSize to 2048 (recv). - te3, ch3 := testServiceConfigSetupTD(t, e) - te3.maxClientReceiveMsgSize = newInt(4096) - te3.maxClientSendMsgSize = newInt(4096) - te3.startServer(&testServer{security: e.security}) - defer te3.tearDown() - ch3 <- sc - tc = testpb.NewTestServiceClient(te3.clientConn()) - - // Test for unary RPC recv. - req.Payload = smallPayload - req.ResponseSize = int32(largeSize) - - if _, err := tc.UnaryCall(context.Background(), req); err != nil { - t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want ", err) - } - - req.ResponseSize = int32(extraLargeSize) - if _, err := tc.UnaryCall(context.Background(), req); err == nil || status.Code(err) != codes.ResourceExhausted { - t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted) - } - - // Test for unary RPC send. - req.Payload = largePayload - req.ResponseSize = int32(smallSize) - if _, err := tc.UnaryCall(context.Background(), req); err != nil { - t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want ", err) - } - - req.Payload = extraLargePayload - if _, err := tc.UnaryCall(context.Background(), req); err == nil || status.Code(err) != codes.ResourceExhausted { - t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted) - } - - // Test for streaming RPC recv. - stream, err = tc.FullDuplexCall(te3.ctx) - if err != nil { - t.Fatalf("%v.FullDuplexCall(_) = _, %v, want ", tc, err) - } - respParam[0].Size = int32(largeSize) - sreq.Payload = smallPayload - - if err := stream.Send(sreq); err != nil { - t.Fatalf("%v.Send(%v) = %v, want ", stream, sreq, err) - } - if _, err := stream.Recv(); err != nil { - t.Fatalf("%v.Recv() = _, %v, want ", stream, err) - } - - respParam[0].Size = int32(extraLargeSize) - - if err := stream.Send(sreq); err != nil { - t.Fatalf("%v.Send(%v) = %v, want ", stream, sreq, err) - } - if _, err := stream.Recv(); err == nil || status.Code(err) != codes.ResourceExhausted { - t.Fatalf("%v.Recv() = _, %v, want _, error code: %s", stream, err, codes.ResourceExhausted) - } - - // Test for streaming RPC send. - respParam[0].Size = int32(smallSize) - sreq.Payload = largePayload - stream, err = tc.FullDuplexCall(te3.ctx) - if err != nil { - t.Fatalf("%v.FullDuplexCall(_) = _, %v, want ", tc, err) - } - if err := stream.Send(sreq); err != nil { - t.Fatalf("%v.Send(%v) = %v, want ", stream, sreq, err) - } - sreq.Payload = extraLargePayload - if err := stream.Send(sreq); err == nil || status.Code(err) != codes.ResourceExhausted { - t.Fatalf("%v.Send(%v) = %v, want _, error code: %s", stream, sreq, err, codes.ResourceExhausted) - } -} - -func (s) TestMethodFromServerStream(t *testing.T) { - const testMethod = "/package.service/method" - e := tcpClearRREnv - te := newTest(t, e) - var method string - var ok bool - te.unknownHandler = func(srv interface{}, stream grpc.ServerStream) error { - method, ok = grpc.MethodFromServerStream(stream) - return nil - } - - te.startServer(nil) - defer te.tearDown() - _ = te.clientConn().Invoke(context.Background(), testMethod, nil, nil) - if !ok || method != testMethod { - t.Fatalf("Invoke with method %q, got %q, %v, want %q, true", testMethod, method, ok, testMethod) - } -} +} func (s) TestInterceptorCanAccessCallOptions(t *testing.T) { e := tcpClearRREnv @@ -6139,7 +5078,7 @@ func (s) TestInterceptorCanAccessCallOptions(t *testing.T) { } } - te.unaryClientInt = func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + te.unaryClientInt = func(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { populateOpts(opts) return nil } @@ -6152,12 +5091,14 @@ func (s) TestInterceptorCanAccessCallOptions(t *testing.T) { grpc.WaitForReady(true), grpc.MaxCallRecvMsgSize(1010), } - tc := testpb.NewTestServiceClient(te.clientConn(grpc.WithDefaultCallOptions(defaults...))) + tc := testgrpc.NewTestServiceClient(te.clientConn(grpc.WithDefaultCallOptions(defaults...))) var headers metadata.MD var trailers metadata.MD var pr peer.Peer - tc.UnaryCall(context.Background(), &testpb.SimpleRequest{}, + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + tc.UnaryCall(ctx, &testpb.SimpleRequest{}, grpc.MaxCallRecvMsgSize(100), grpc.MaxCallSendMsgSize(200), grpc.PerRPCCredentials(testPerRPCCredentials{}), @@ -6180,7 +5121,7 @@ func (s) TestInterceptorCanAccessCallOptions(t *testing.T) { observedOpts = observedOptions{} // reset - tc.StreamingInputCall(context.Background(), + tc.StreamingInputCall(ctx, grpc.WaitForReady(false), grpc.MaxCallSendMsgSize(2020), grpc.UseCompressor("comp-type"), @@ -6198,77 +5139,16 @@ func (s) TestInterceptorCanAccessCallOptions(t *testing.T) { } } -func (s) TestCompressorRegister(t *testing.T) { - for _, e := range listTestEnv() { - testCompressorRegister(t, e) - } -} - -func testCompressorRegister(t *testing.T, e env) { - te := newTest(t, e) - te.clientCompression = false - te.serverCompression = false - te.clientUseCompression = true - - te.startServer(&testServer{security: e.security}) - defer te.tearDown() - tc := testpb.NewTestServiceClient(te.clientConn()) - - // Unary call - const argSize = 271828 - const respSize = 314159 - payload, err := newPayload(testpb.PayloadType_COMPRESSABLE, argSize) - if err != nil { - t.Fatal(err) - } - req := &testpb.SimpleRequest{ - ResponseType: testpb.PayloadType_COMPRESSABLE, - ResponseSize: respSize, - Payload: payload, - } - ctx := metadata.NewOutgoingContext(context.Background(), metadata.Pairs("something", "something")) - if _, err := tc.UnaryCall(ctx, req); err != nil { - t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, ", err) - } - // Streaming RPC - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - stream, err := tc.FullDuplexCall(ctx) - if err != nil { - t.Fatalf("%v.FullDuplexCall(_) = _, %v, want ", tc, err) - } - respParam := []*testpb.ResponseParameters{ - { - Size: 31415, - }, - } - payload, err = newPayload(testpb.PayloadType_COMPRESSABLE, int32(31415)) - if err != nil { - t.Fatal(err) - } - sreq := &testpb.StreamingOutputCallRequest{ - ResponseType: testpb.PayloadType_COMPRESSABLE, - ResponseParameters: respParam, - Payload: payload, - } - if err := stream.Send(sreq); err != nil { - t.Fatalf("%v.Send(%v) = %v, want ", stream, sreq, err) - } - if _, err := stream.Recv(); err != nil { - t.Fatalf("%v.Recv() = %v, want ", stream, err) - } -} - func (s) TestServeExitsWhenListenerClosed(t *testing.T) { - ss := &stubServer{ - emptyCall: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + ss := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil }, } s := grpc.NewServer() defer s.Stop() - testpb.RegisterTestServiceService(s, testpb.NewTestServiceService(ss)) + testgrpc.RegisterTestServiceServer(s, ss) lis, err := net.Listen("tcp", "localhost:0") if err != nil { @@ -6281,13 +5161,13 @@ func (s) TestServeExitsWhenListenerClosed(t *testing.T) { close(done) }() - cc, err := grpc.Dial(lis.Addr().String(), grpc.WithInsecure(), grpc.WithBlock()) + cc, err := grpc.Dial(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { t.Fatalf("Failed to dial server: %v", err) } defer cc.Close() - c := testpb.NewTestServiceClient(cc) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + c := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() if _, err := c.EmptyCall(ctx, &testpb.Empty{}); err != nil { t.Fatalf("Failed to send test RPC to server: %v", err) @@ -6313,8 +5193,8 @@ func (s) TestStatusInvalidUTF8Message(t *testing.T) { wantMsg = "���" ) - ss := &stubServer{ - emptyCall: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { return nil, status.Errorf(codes.Internal, origMsg) }, } @@ -6323,11 +5203,11 @@ func (s) TestStatusInvalidUTF8Message(t *testing.T) { } defer ss.Stop() - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() - if _, err := ss.client.EmptyCall(ctx, &testpb.Empty{}); status.Convert(err).Message() != wantMsg { - t.Fatalf("ss.client.EmptyCall(_, _) = _, %v (msg %q); want _, err with msg %q", err, status.Convert(err).Message(), wantMsg) + if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); status.Convert(err).Message() != wantMsg { + t.Fatalf("ss.Client.EmptyCall(_, _) = _, %v (msg %q); want _, err with msg %q", err, status.Convert(err).Message(), wantMsg) } } @@ -6335,15 +5215,15 @@ func (s) TestStatusInvalidUTF8Message(t *testing.T) { // will fail to marshal the status because of the invalid utf8 message. Details // will be dropped when sending. func (s) TestStatusInvalidUTF8Details(t *testing.T) { - grpctest.TLogger.ExpectError("transport: failed to marshal rpc status") + grpctest.TLogger.ExpectError("Failed to marshal rpc status") var ( origMsg = string([]byte{0xff, 0xfe, 0xfd}) wantMsg = "���" ) - ss := &stubServer{ - emptyCall: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { st := status.New(codes.Internal, origMsg) st, err := st.WithDetails(&testpb.Empty{}) if err != nil { @@ -6357,13 +5237,13 @@ func (s) TestStatusInvalidUTF8Details(t *testing.T) { } defer ss.Stop() - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() - _, err := ss.client.EmptyCall(ctx, &testpb.Empty{}) + _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}) st := status.Convert(err) if st.Message() != wantMsg { - t.Fatalf("ss.client.EmptyCall(_, _) = _, %v (msg %q); want _, err with msg %q", err, st.Message(), wantMsg) + t.Fatalf("ss.Client.EmptyCall(_, _) = _, %v (msg %q); want _, err with msg %q", err, st.Message(), wantMsg) } if len(st.Details()) != 0 { // Details should be dropped on the server side. @@ -6371,49 +5251,6 @@ func (s) TestStatusInvalidUTF8Details(t *testing.T) { } } -func (s) TestClientDoesntDeadlockWhileWritingErrornousLargeMessages(t *testing.T) { - for _, e := range listTestEnv() { - if e.httpHandler { - continue - } - testClientDoesntDeadlockWhileWritingErrornousLargeMessages(t, e) - } -} - -func testClientDoesntDeadlockWhileWritingErrornousLargeMessages(t *testing.T, e env) { - te := newTest(t, e) - te.userAgent = testAppUA - smallSize := 1024 - te.maxServerReceiveMsgSize = &smallSize - te.startServer(&testServer{security: e.security}) - defer te.tearDown() - tc := testpb.NewTestServiceClient(te.clientConn()) - payload, err := newPayload(testpb.PayloadType_COMPRESSABLE, 1048576) - if err != nil { - t.Fatal(err) - } - req := &testpb.SimpleRequest{ - ResponseType: testpb.PayloadType_COMPRESSABLE, - Payload: payload, - } - var wg sync.WaitGroup - for i := 0; i < 10; i++ { - wg.Add(1) - go func() { - defer wg.Done() - for j := 0; j < 100; j++ { - ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*10)) - defer cancel() - if _, err := tc.UnaryCall(ctx, req); status.Code(err) != codes.ResourceExhausted { - t.Errorf("TestService/UnaryCall(_,_) = _. %v, want code: %s", err, codes.ResourceExhausted) - return - } - } - }() - } - wg.Wait() -} - func (s) TestRPCTimeout(t *testing.T) { for _, e := range listTestEnv() { testRPCTimeout(t, e) @@ -6426,7 +5263,7 @@ func testRPCTimeout(t *testing.T, e env) { defer te.tearDown() cc := te.clientConn() - tc := testpb.NewTestServiceClient(cc) + tc := testgrpc.NewTestServiceClient(cc) const argSize = 2718 const respSize = 314 @@ -6462,8 +5299,8 @@ func (s) TestDisabledIOBuffers(t *testing.T) { Payload: payload, } - ss := &stubServer{ - fullDuplexCall: func(stream testpb.TestService_FullDuplexCallServer) error { + ss := &stubserver.StubServer{ + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { for { in, err := stream.Recv() if err == io.EOF { @@ -6487,28 +5324,26 @@ func (s) TestDisabledIOBuffers(t *testing.T) { } s := grpc.NewServer(grpc.WriteBufferSize(0), grpc.ReadBufferSize(0)) - testpb.RegisterTestServiceService(s, testpb.NewTestServiceService(ss)) + testgrpc.RegisterTestServiceServer(s, ss) lis, err := net.Listen("tcp", "localhost:0") if err != nil { t.Fatalf("Failed to create listener: %v", err) } - done := make(chan struct{}) go func() { s.Serve(lis) - close(done) }() defer s.Stop() - dctx, dcancel := context.WithTimeout(context.Background(), 5*time.Second) + dctx, dcancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer dcancel() - cc, err := grpc.DialContext(dctx, lis.Addr().String(), grpc.WithInsecure(), grpc.WithBlock(), grpc.WithWriteBufferSize(0), grpc.WithReadBufferSize(0)) + cc, err := grpc.DialContext(dctx, lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithWriteBufferSize(0), grpc.WithReadBufferSize(0)) if err != nil { t.Fatalf("Failed to dial server") } defer cc.Close() - c := testpb.NewTestServiceClient(cc) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + c := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() stream, err := c.FullDuplexCall(ctx, grpc.WaitForReady(true)) if err != nil { @@ -6549,8 +5384,8 @@ func testServerMaxHeaderListSizeClientUserViolation(t *testing.T, e env) { defer te.tearDown() cc := te.clientConn() - tc := testpb.NewTestServiceClient(cc) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + tc := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() metadata.AppendToOutgoingContext(ctx, "oversize", string(make([]byte, 216))) var err error @@ -6581,8 +5416,8 @@ func testClientMaxHeaderListSizeServerUserViolation(t *testing.T, e env) { defer te.tearDown() cc := te.clientConn() - tc := testpb.NewTestServiceClient(cc) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + tc := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() var err error if err = verifyResultWithDelay(func() (bool, error) { @@ -6612,8 +5447,8 @@ func testServerMaxHeaderListSizeClientIntentionalViolation(t *testing.T, e env) defer te.tearDown() cc, dw := te.clientConnWithConnControl() - tc := &testServiceClientWrapper{TestServiceClient: testpb.NewTestServiceClient(cc)} - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + tc := &testServiceClientWrapper{TestServiceClient: testgrpc.NewTestServiceClient(cc)} + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() stream, err := tc.FullDuplexCall(ctx) if err != nil { @@ -6653,8 +5488,8 @@ func testClientMaxHeaderListSizeServerIntentionalViolation(t *testing.T, e env) lw := te.startServerWithConnControl(&testServer{security: e.security, setHeaderOnly: true}) defer te.tearDown() cc, _ := te.clientConnWithConnControl() - tc := &testServiceClientWrapper{TestServiceClient: testpb.NewTestServiceClient(cc)} - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + tc := &testServiceClientWrapper{TestServiceClient: testgrpc.NewTestServiceClient(cc)} + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() stream, err := tc.FullDuplexCall(ctx) if err != nil { @@ -6700,16 +5535,16 @@ func (s) TestNetPipeConn(t *testing.T) { ts := &funcServer{unaryCall: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { return &testpb.SimpleResponse{}, nil }} - testpb.RegisterTestServiceService(s, testpb.NewTestServiceService(ts)) + testgrpc.RegisterTestServiceServer(s, ts) go s.Serve(pl) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() - cc, err := grpc.DialContext(ctx, "", grpc.WithInsecure(), grpc.WithDialer(pl.Dialer())) + cc, err := grpc.DialContext(ctx, "", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDialer(pl.Dialer())) if err != nil { t.Fatalf("Error creating client: %v", err) } defer cc.Close() - client := testpb.NewTestServiceClient(cc) + client := testgrpc.NewTestServiceClient(cc) if _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil { t.Fatalf("UnaryCall(_) = _, %v; want _, nil", err) } @@ -6725,120 +5560,34 @@ func testLargeTimeout(t *testing.T, e env) { te := newTest(t, e) te.declareLogNoise("Server.processUnaryRPC failed to write status") - ts := &funcServer{} - te.startServer(ts) - defer te.tearDown() - tc := testpb.NewTestServiceClient(te.clientConn()) - - timeouts := []time.Duration{ - time.Duration(math.MaxInt64), // will be (correctly) converted to - // 2562048 hours, which overflows upon converting back to an int64 - 2562047 * time.Hour, // the largest timeout that does not overflow - } - - for i, maxTimeout := range timeouts { - ts.unaryCall = func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { - deadline, ok := ctx.Deadline() - timeout := time.Until(deadline) - minTimeout := maxTimeout - 5*time.Second - if !ok || timeout < minTimeout || timeout > maxTimeout { - t.Errorf("ctx.Deadline() = (now+%v), %v; want [%v, %v], true", timeout, ok, minTimeout, maxTimeout) - return nil, status.Error(codes.OutOfRange, "deadline error") - } - return &testpb.SimpleResponse{}, nil - } - - ctx, cancel := context.WithTimeout(context.Background(), maxTimeout) - defer cancel() - - if _, err := tc.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil { - t.Errorf("case %v: UnaryCall(_) = _, %v; want _, nil", i, err) - } - } -} - -// Proxies typically send GO_AWAY followed by connection closure a minute or so later. This -// test ensures that the connection is re-created after GO_AWAY and not affected by the -// subsequent (old) connection closure. -func (s) TestGoAwayThenClose(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) - defer cancel() - - lis1, err := net.Listen("tcp", "localhost:0") - if err != nil { - t.Fatalf("Error while listening. Err: %v", err) - } - s1 := grpc.NewServer() - defer s1.Stop() - ts := &funcServer{ - unaryCall: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { - return &testpb.SimpleResponse{}, nil - }, - fullDuplexCall: func(stream testpb.TestService_FullDuplexCallServer) error { - // Wait forever. - _, err := stream.Recv() - if err == nil { - t.Error("expected to never receive any message") - } - return err - }, - } - testpb.RegisterTestServiceService(s1, testpb.NewTestServiceService(ts)) - go s1.Serve(lis1) - - conn2Established := grpcsync.NewEvent() - lis2, err := listenWithNotifyingListener("tcp", "localhost:0", conn2Established) - if err != nil { - t.Fatalf("Error while listening. Err: %v", err) - } - s2 := grpc.NewServer() - defer s2.Stop() - testpb.RegisterTestServiceService(s2, testpb.NewTestServiceService(ts)) - go s2.Serve(lis2) - - r := manual.NewBuilderWithScheme("whatever") - r.InitialState(resolver.State{Addresses: []resolver.Address{ - {Addr: lis1.Addr().String()}, - }}) - cc, err := grpc.DialContext(ctx, r.Scheme()+":///", grpc.WithResolvers(r), grpc.WithInsecure()) - if err != nil { - t.Fatalf("Error creating client: %v", err) - } - defer cc.Close() - - client := testpb.NewTestServiceClient(cc) - - // Should go on connection 1. We use a long-lived RPC because it will cause GracefulStop to send GO_AWAY, but the - // connection doesn't get closed until the server stops and the client receives. - stream, err := client.FullDuplexCall(ctx) - if err != nil { - t.Fatalf("FullDuplexCall(_) = _, %v; want _, nil", err) - } - - r.UpdateState(resolver.State{Addresses: []resolver.Address{ - {Addr: lis1.Addr().String()}, - {Addr: lis2.Addr().String()}, - }}) - - // Send GO_AWAY to connection 1. - go s1.GracefulStop() + ts := &funcServer{} + te.startServer(ts) + defer te.tearDown() + tc := testgrpc.NewTestServiceClient(te.clientConn()) - // Wait for connection 2 to be established. - <-conn2Established.Done() + timeouts := []time.Duration{ + time.Duration(math.MaxInt64), // will be (correctly) converted to + // 2562048 hours, which overflows upon converting back to an int64 + 2562047 * time.Hour, // the largest timeout that does not overflow + } - // Close connection 1. - s1.Stop() + for i, maxTimeout := range timeouts { + ts.unaryCall = func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + deadline, ok := ctx.Deadline() + timeout := time.Until(deadline) + minTimeout := maxTimeout - 5*time.Second + if !ok || timeout < minTimeout || timeout > maxTimeout { + t.Errorf("ctx.Deadline() = (now+%v), %v; want [%v, %v], true", timeout, ok, minTimeout, maxTimeout) + return nil, status.Error(codes.OutOfRange, "deadline error") + } + return &testpb.SimpleResponse{}, nil + } - // Wait for client to close. - _, err = stream.Recv() - if err == nil { - t.Fatal("expected the stream to die, but got a successful Recv") - } + ctx, cancel := context.WithTimeout(context.Background(), maxTimeout) + defer cancel() - // Do a bunch of RPCs, make sure it stays stable. These should go to connection 2. - for i := 0; i < 10; i++ { - if _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil { - t.Fatalf("UnaryCall(_) = _, %v; want _, nil", err) + if _, err := tc.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil { + t.Errorf("case %v: UnaryCall(_) = _, %v; want _, nil", i, err) } } } @@ -6868,24 +5617,23 @@ func (s) TestRPCWaitsForResolver(t *testing.T) { r := manual.NewBuilderWithScheme("whatever") te.resolverScheme = r.Scheme() - te.nonBlockingDial = true cc := te.clientConn(grpc.WithResolvers(r)) - tc := testpb.NewTestServiceClient(cc) + tc := testgrpc.NewTestServiceClient(cc) - ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) defer cancel() // With no resolved addresses yet, this will timeout. if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.DeadlineExceeded { t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, %s", err, codes.DeadlineExceeded) } - ctx, cancel = context.WithTimeout(context.Background(), 10*time.Second) + ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() go func() { time.Sleep(time.Second) r.UpdateState(resolver.State{ Addresses: []resolver.Address{{Addr: te.srvAddr}}, - ServiceConfig: parseCfg(r, `{ + ServiceConfig: parseServiceConfig(t, r, `{ "methodConfig": [ { "name": [ @@ -6902,7 +5650,11 @@ func (s) TestRPCWaitsForResolver(t *testing.T) { // We wait a second before providing a service config and resolving // addresses. So this will wait for that and then honor the // maxRequestMessageBytes it contains. - if _, err := tc.UnaryCall(ctx, &testpb.SimpleRequest{ResponseType: testpb.PayloadType_UNCOMPRESSABLE}); status.Code(err) != codes.ResourceExhausted { + payload, err := newPayload(testpb.PayloadType_COMPRESSABLE, 1) + if err != nil { + t.Fatal(err) + } + if _, err := tc.UnaryCall(ctx, &testpb.SimpleRequest{Payload: payload}); status.Code(err) != codes.ResourceExhausted { t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, nil", err) } if got := ctx.Err(); got != nil { @@ -6913,378 +5665,877 @@ func (s) TestRPCWaitsForResolver(t *testing.T) { } } -func (s) TestHTTPHeaderFrameErrorHandlingHTTPMode(t *testing.T) { - // Non-gRPC content-type fallback path. - for httpCode := range transport.HTTPStatusConvTab { - doHTTPHeaderTest(t, transport.HTTPStatusConvTab[int(httpCode)], []string{ - ":status", fmt.Sprintf("%d", httpCode), - "content-type", "text/html", // non-gRPC content type to switch to HTTP mode. - "grpc-status", "1", // Make up a gRPC status error - "grpc-status-details-bin", "???", // Make up a gRPC field parsing error - }) +type httpServerResponse struct { + headers [][]string + payload []byte + trailers [][]string +} + +type httpServer struct { + // If waitForEndStream is set, wait for the client to send a frame with end + // stream in it before sending a response/refused stream. + waitForEndStream bool + refuseStream func(uint32) bool + responses []httpServerResponse +} + +func (s *httpServer) writeHeader(framer *http2.Framer, sid uint32, headerFields []string, endStream bool) error { + if len(headerFields)%2 == 1 { + panic("odd number of kv args") } - // Missing content-type fallback path. - for httpCode := range transport.HTTPStatusConvTab { - doHTTPHeaderTest(t, transport.HTTPStatusConvTab[int(httpCode)], []string{ - ":status", fmt.Sprintf("%d", httpCode), - // Omitting content type to switch to HTTP mode. - "grpc-status", "1", // Make up a gRPC status error - "grpc-status-details-bin", "???", // Make up a gRPC field parsing error - }) + var buf bytes.Buffer + henc := hpack.NewEncoder(&buf) + for len(headerFields) > 0 { + k, v := headerFields[0], headerFields[1] + headerFields = headerFields[2:] + henc.WriteField(hpack.HeaderField{Name: k, Value: v}) } - // Malformed HTTP status when fallback. - doHTTPHeaderTest(t, codes.Internal, []string{ - ":status", "abc", - // Omitting content type to switch to HTTP mode. - "grpc-status", "1", // Make up a gRPC status error - "grpc-status-details-bin", "???", // Make up a gRPC field parsing error + return framer.WriteHeaders(http2.HeadersFrameParam{ + StreamID: sid, + BlockFragment: buf.Bytes(), + EndStream: endStream, + EndHeaders: true, }) } -// Testing erroneous ResponseHeader or Trailers-only (delivered in the first HEADERS frame). -func (s) TestHTTPHeaderFrameErrorHandlingInitialHeader(t *testing.T) { - for _, test := range []struct { - header []string - errCode codes.Code - }{ - { - // missing gRPC status. - header: []string{ - ":status", "403", - "content-type", "application/grpc", - }, - errCode: codes.Unknown, - }, - { - // malformed grpc-status. - header: []string{ - ":status", "502", - "content-type", "application/grpc", - "grpc-status", "abc", - }, - errCode: codes.Internal, - }, - { - // Malformed grpc-tags-bin field. - header: []string{ - ":status", "502", - "content-type", "application/grpc", - "grpc-status", "0", - "grpc-tags-bin", "???", - }, - errCode: codes.Internal, +func (s *httpServer) writePayload(framer *http2.Framer, sid uint32, payload []byte) error { + return framer.WriteData(sid, false, payload) +} + +func (s *httpServer) start(t *testing.T, lis net.Listener) { + // Launch an HTTP server to send back header. + go func() { + conn, err := lis.Accept() + if err != nil { + t.Errorf("Error accepting connection: %v", err) + return + } + defer conn.Close() + // Read preface sent by client. + if _, err = io.ReadFull(conn, make([]byte, len(http2.ClientPreface))); err != nil { + t.Errorf("Error at server-side while reading preface from client. Err: %v", err) + return + } + reader := bufio.NewReader(conn) + writer := bufio.NewWriter(conn) + framer := http2.NewFramer(writer, reader) + if err = framer.WriteSettingsAck(); err != nil { + t.Errorf("Error at server-side while sending Settings ack. Err: %v", err) + return + } + writer.Flush() // necessary since client is expecting preface before declaring connection fully setup. + var sid uint32 + // Loop until framer returns possible conn closed errors. + for requestNum := 0; ; requestNum = (requestNum + 1) % len(s.responses) { + // Read frames until a header is received. + for { + frame, err := framer.ReadFrame() + if err != nil { + if !isConnClosedErr(err) { + t.Errorf("Error at server-side while reading frame. got: %q, want: rpc error containing substring %q OR %q", err, possibleConnResetMsg, possibleEOFMsg) + } + return + } + sid = 0 + switch fr := frame.(type) { + case *http2.HeadersFrame: + // Respond after this if we are not waiting for an end + // stream or if this frame ends it. + if !s.waitForEndStream || fr.StreamEnded() { + sid = fr.Header().StreamID + } + + case *http2.DataFrame: + // Respond after this if we were waiting for an end stream + // and this frame ends it. (If we were not waiting for an + // end stream, this stream was already responded to when + // the headers were received.) + if s.waitForEndStream && fr.StreamEnded() { + sid = fr.Header().StreamID + } + } + if sid != 0 { + if s.refuseStream == nil || !s.refuseStream(sid) { + break + } + framer.WriteRSTStream(sid, http2.ErrCodeRefusedStream) + writer.Flush() + } + } + + response := s.responses[requestNum] + for _, header := range response.headers { + if err = s.writeHeader(framer, sid, header, false); err != nil { + t.Errorf("Error at server-side while writing headers. Err: %v", err) + return + } + writer.Flush() + } + if response.payload != nil { + if err = s.writePayload(framer, sid, response.payload); err != nil { + t.Errorf("Error at server-side while writing payload. Err: %v", err) + return + } + writer.Flush() + } + for i, trailer := range response.trailers { + if err = s.writeHeader(framer, sid, trailer, i == len(response.trailers)-1); err != nil { + t.Errorf("Error at server-side while writing trailers. Err: %v", err) + return + } + writer.Flush() + } + } + }() +} + +func (s) TestClientCancellationPropagatesUnary(t *testing.T) { + wg := &sync.WaitGroup{} + called, done := make(chan struct{}), make(chan struct{}) + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + close(called) + <-ctx.Done() + err := ctx.Err() + if err != context.Canceled { + t.Errorf("ctx.Err() = %v; want context.Canceled", err) + } + close(done) + return nil, err }, - { - // gRPC status error. - header: []string{ - ":status", "502", - "content-type", "application/grpc", - "grpc-status", "3", - }, - errCode: codes.InvalidArgument, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + + wg.Add(1) + go func() { + if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.Canceled { + t.Errorf("ss.Client.EmptyCall() = _, %v; want _, Code()=codes.Canceled", err) + } + wg.Done() + }() + + select { + case <-called: + case <-time.After(5 * time.Second): + t.Fatalf("failed to perform EmptyCall after 10s") + } + cancel() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatalf("server failed to close done chan due to cancellation propagation") + } + wg.Wait() +} + +// When an RPC is canceled, it's possible that the last Recv() returns before +// all call options' after are executed. +func (s) TestCanceledRPCCallOptionRace(t *testing.T) { + ss := &stubserver.StubServer{ + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + err := stream.Send(&testpb.StreamingOutputCallResponse{}) + if err != nil { + return err + } + <-stream.Context().Done() + return nil }, - } { - doHTTPHeaderTest(t, test.errCode, test.header) + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + const count = 1000 + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + var wg sync.WaitGroup + wg.Add(count) + for i := 0; i < count; i++ { + go func() { + defer wg.Done() + var p peer.Peer + ctx, cancel := context.WithCancel(ctx) + defer cancel() + stream, err := ss.Client.FullDuplexCall(ctx, grpc.Peer(&p)) + if err != nil { + t.Errorf("_.FullDuplexCall(_) = _, %v", err) + return + } + if err := stream.Send(&testpb.StreamingOutputCallRequest{}); err != nil { + t.Errorf("_ has error %v while sending", err) + return + } + if _, err := stream.Recv(); err != nil { + t.Errorf("%v.Recv() = %v", stream, err) + return + } + cancel() + if _, err := stream.Recv(); status.Code(err) != codes.Canceled { + t.Errorf("%v compleled with error %v, want %s", stream, err, codes.Canceled) + return + } + // If recv returns before call options are executed, peer.Addr is not set, + // fail the test. + if p.Addr == nil { + t.Errorf("peer.Addr is nil, want non-nil") + return + } + }() + } + wg.Wait() +} + +func (s) TestClientSettingsFloodCloseConn(t *testing.T) { + // Tests that the server properly closes its transport if the client floods + // settings frames and then closes the connection. + + // Minimize buffer sizes to stimulate failure condition more quickly. + s := grpc.NewServer(grpc.WriteBufferSize(20)) + l := bufconn.Listen(20) + go s.Serve(l) + + // Dial our server and handshake. + conn, err := l.Dial() + if err != nil { + t.Fatalf("Error dialing bufconn: %v", err) + } + + n, err := conn.Write([]byte(http2.ClientPreface)) + if err != nil || n != len(http2.ClientPreface) { + t.Fatalf("Error writing client preface: %v, %v", n, err) + } + + fr := http2.NewFramer(conn, conn) + f, err := fr.ReadFrame() + if err != nil { + t.Fatalf("Error reading initial settings frame: %v", err) + } + if _, ok := f.(*http2.SettingsFrame); ok { + if err := fr.WriteSettingsAck(); err != nil { + t.Fatalf("Error writing settings ack: %v", err) + } + } else { + t.Fatalf("Error reading initial settings frame: type=%T", f) + } + + // Confirm settings can be written, and that an ack is read. + if err = fr.WriteSettings(); err != nil { + t.Fatalf("Error writing settings frame: %v", err) + } + if f, err = fr.ReadFrame(); err != nil { + t.Fatalf("Error reading frame: %v", err) + } + if sf, ok := f.(*http2.SettingsFrame); !ok || !sf.IsAck() { + t.Fatalf("Unexpected frame: %v", f) + } + + // Flood settings frames until a timeout occurs, indiciating the server has + // stopped reading from the connection, then close the conn. + for { + conn.SetWriteDeadline(time.Now().Add(50 * time.Millisecond)) + if err := fr.WriteSettings(); err != nil { + if to, ok := err.(interface{ Timeout() bool }); !ok || !to.Timeout() { + t.Fatalf("Received unexpected write error: %v", err) + } + break + } + } + conn.Close() + + // If the server does not handle this situation correctly, it will never + // close the transport. This is because its loopyWriter.run() will have + // exited, and thus not handle the goAway the draining process initiates. + // Also, we would see a goroutine leak in this case, as the reader would be + // blocked on the controlBuf's throttle() method indefinitely. + + timer := time.AfterFunc(5*time.Second, func() { + t.Errorf("Timeout waiting for GracefulStop to return") + s.Stop() + }) + s.GracefulStop() + timer.Stop() +} + +func unaryInterceptorVerifyConn(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + conn := transport.GetConnection(ctx) + if conn == nil { + return nil, status.Error(codes.NotFound, "connection was not in context") + } + return nil, status.Error(codes.OK, "") +} + +// TestUnaryServerInterceptorGetsConnection tests whether the accepted conn on +// the server gets to any unary interceptors on the server side. +func (s) TestUnaryServerInterceptorGetsConnection(t *testing.T) { + ss := &stubserver.StubServer{} + if err := ss.Start([]grpc.ServerOption{grpc.UnaryInterceptor(unaryInterceptorVerifyConn)}); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.OK { + t.Fatalf("ss.Client.EmptyCall(_, _) = _, %v, want _, error code %s", err, codes.OK) + } +} + +func streamingInterceptorVerifyConn(srv any, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + conn := transport.GetConnection(ss.Context()) + if conn == nil { + return status.Error(codes.NotFound, "connection was not in context") + } + return status.Error(codes.OK, "") +} + +// TestStreamingServerInterceptorGetsConnection tests whether the accepted conn on +// the server gets to any streaming interceptors on the server side. +func (s) TestStreamingServerInterceptorGetsConnection(t *testing.T) { + ss := &stubserver.StubServer{} + if err := ss.Start([]grpc.ServerOption{grpc.StreamInterceptor(streamingInterceptorVerifyConn)}); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + s, err := ss.Client.StreamingOutputCall(ctx, &testpb.StreamingOutputCallRequest{}) + if err != nil { + t.Fatalf("ss.Client.StreamingOutputCall(_) = _, %v, want _, ", err) + } + if _, err := s.Recv(); err != io.EOF { + t.Fatalf("ss.Client.StreamingInputCall(_) = _, %v, want _, %v", err, io.EOF) } } -// Testing non-Trailers-only Trailers (delievered in second HEADERS frame) -func (s) TestHTTPHeaderFrameErrorHandlingNormalTrailer(t *testing.T) { - for _, test := range []struct { - responseHeader []string - trailer []string - errCode codes.Code +// unaryInterceptorVerifyAuthority verifies there is an unambiguous :authority +// once the request gets to an interceptor. An unambiguous :authority is defined +// as at most a single :authority header, and no host header according to A41. +func unaryInterceptorVerifyAuthority(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return nil, status.Error(codes.NotFound, "metadata was not in context") + } + authority := md.Get(":authority") + if len(authority) > 1 { // Should be an unambiguous authority by the time it gets to interceptor. + return nil, status.Error(codes.NotFound, ":authority value had more than one value") + } + // Host header shouldn't be present by the time it gets to the interceptor + // level (should either be renamed to :authority or explicitly deleted). + host := md.Get("host") + if len(host) != 0 { + return nil, status.Error(codes.NotFound, "host header should not be present in metadata") + } + // Pass back the authority for verification on client - NotFound so + // grpc-message will be available to read for verification. + if len(authority) == 0 { + // Represent no :authority header present with an empty string. + return nil, status.Error(codes.NotFound, "") + } + return nil, status.Error(codes.NotFound, authority[0]) +} + +// TestAuthorityHeader tests that the eventual :authority that reaches the grpc +// layer is unambiguous due to logic added in A41. +func (s) TestAuthorityHeader(t *testing.T) { + tests := []struct { + name string + headers []string + wantAuthority string }{ + // "If :authority is missing, Host must be renamed to :authority." - A41 { - responseHeader: []string{ - ":status", "200", + name: "Missing :authority", + // Codepath triggered by incoming headers with no authority but with + // a host. + headers: []string{ + ":method", "POST", + ":path", "/grpc.testing.TestService/UnaryCall", "content-type", "application/grpc", + "te", "trailers", + "host", "localhost", }, - trailer: []string{ - // trailer missing grpc-status - ":status", "502", - }, - errCode: codes.Unknown, + wantAuthority: "localhost", }, { - responseHeader: []string{ - ":status", "404", + name: "Missing :authority and host", + // Codepath triggered by incoming headers with no :authority and no + // host. + headers: []string{ + ":method", "POST", + ":path", "/grpc.testing.TestService/UnaryCall", "content-type", "application/grpc", + "te", "trailers", }, - trailer: []string{ - // malformed grpc-status-details-bin field - "grpc-status", "0", - "grpc-status-details-bin", "????", + wantAuthority: "", + }, + // "If :authority is present, Host must be discarded." - A41 + { + name: ":authority and host present", + // Codepath triggered by incoming headers with both an authority + // header and a host header. + headers: []string{ + ":method", "POST", + ":path", "/grpc.testing.TestService/UnaryCall", + ":authority", "localhost", + "content-type", "application/grpc", + "host", "localhost2", }, - errCode: codes.Internal, + wantAuthority: "localhost", }, - } { - doHTTPHeaderTest(t, test.errCode, test.responseHeader, test.trailer) } -} + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + te := newTest(t, tcpClearRREnv) + ts := &funcServer{unaryCall: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{}, nil + }} + te.unaryServerInt = unaryInterceptorVerifyAuthority + te.startServer(ts) + defer te.tearDown() + success := testutils.NewChannel() + te.withServerTester(func(st *serverTester) { + st.writeHeaders(http2.HeadersFrameParam{ + StreamID: 1, + BlockFragment: st.encodeHeader(test.headers...), + EndStream: false, + EndHeaders: true, + }) + st.writeData(1, true, []byte{0, 0, 0, 0, 0}) + + for { + frame := st.wantAnyFrame() + f, ok := frame.(*http2.MetaHeadersFrame) + if !ok { + continue + } + for _, header := range f.Fields { + if header.Name == "grpc-message" { + success.Send(header.Value) + return + } + } + } + }) -func (s) TestHTTPHeaderFrameErrorHandlingMoreThanTwoHeaders(t *testing.T) { - header := []string{ - ":status", "200", - "content-type", "application/grpc", + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + gotAuthority, err := success.Receive(ctx) + if err != nil { + t.Fatalf("Error receiving from channel: %v", err) + } + if gotAuthority != test.wantAuthority { + t.Fatalf("gotAuthority: %v, wantAuthority %v", gotAuthority, test.wantAuthority) + } + }) } - doHTTPHeaderTest(t, codes.Internal, header, header, header) } -type httpServer struct { - headerFields [][]string +// wrapCloseListener tracks Accepts/Closes and maintains a counter of the +// number of open connections. +type wrapCloseListener struct { + net.Listener + connsOpen int32 } -func (s *httpServer) writeHeader(framer *http2.Framer, sid uint32, headerFields []string, endStream bool) error { - if len(headerFields)%2 == 1 { - panic("odd number of kv args") - } +// wrapCloseListener is returned by wrapCloseListener.Accept and decrements its +// connsOpen when Close is called. +type wrapCloseConn struct { + net.Conn + lis *wrapCloseListener + closeOnce sync.Once +} - var buf bytes.Buffer - henc := hpack.NewEncoder(&buf) - for len(headerFields) > 0 { - k, v := headerFields[0], headerFields[1] - headerFields = headerFields[2:] - henc.WriteField(hpack.HeaderField{Name: k, Value: v}) +func (w *wrapCloseListener) Accept() (net.Conn, error) { + conn, err := w.Listener.Accept() + if err != nil { + return nil, err } + atomic.AddInt32(&w.connsOpen, 1) + return &wrapCloseConn{Conn: conn, lis: w}, nil +} - return framer.WriteHeaders(http2.HeadersFrameParam{ - StreamID: sid, - BlockFragment: buf.Bytes(), - EndStream: endStream, - EndHeaders: true, - }) +func (w *wrapCloseConn) Close() error { + defer w.closeOnce.Do(func() { atomic.AddInt32(&w.lis.connsOpen, -1) }) + return w.Conn.Close() } -func (s *httpServer) start(t *testing.T, lis net.Listener) { - // Launch an HTTP server to send back header. - go func() { - conn, err := lis.Accept() +// TestServerClosesConn ensures conn.Close is always closed even if the client +// doesn't complete the HTTP/2 handshake. +func (s) TestServerClosesConn(t *testing.T) { + lis := bufconn.Listen(20) + wrapLis := &wrapCloseListener{Listener: lis} + + s := grpc.NewServer() + go s.Serve(wrapLis) + defer s.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + for i := 0; i < 10; i++ { + conn, err := lis.DialContext(ctx) if err != nil { - t.Errorf("Error accepting connection: %v", err) - return + t.Fatalf("Dial = _, %v; want _, nil", err) } - defer conn.Close() - // Read preface sent by client. - if _, err = io.ReadFull(conn, make([]byte, len(http2.ClientPreface))); err != nil { - t.Errorf("Error at server-side while reading preface from client. Err: %v", err) - return - } - reader := bufio.NewReader(conn) - writer := bufio.NewWriter(conn) - framer := http2.NewFramer(writer, reader) - if err = framer.WriteSettingsAck(); err != nil { - t.Errorf("Error at server-side while sending Settings ack. Err: %v", err) + conn.Close() + } + for ctx.Err() == nil { + if atomic.LoadInt32(&wrapLis.connsOpen) == 0 { return } - writer.Flush() // necessary since client is expecting preface before declaring connection fully setup. - - var sid uint32 - // Read frames until a header is received. - for { - frame, err := framer.ReadFrame() - if err != nil { - t.Errorf("Error at server-side while reading frame. Err: %v", err) - return - } - if hframe, ok := frame.(*http2.HeadersFrame); ok { - sid = hframe.Header().StreamID - break - } - } - for i, headers := range s.headerFields { - if err = s.writeHeader(framer, sid, headers, i == len(s.headerFields)-1); err != nil { - t.Errorf("Error at server-side while writing headers. Err: %v", err) - return - } - writer.Flush() - } - }() + time.Sleep(50 * time.Millisecond) + } + t.Fatalf("timed out waiting for conns to be closed by server; still open: %v", atomic.LoadInt32(&wrapLis.connsOpen)) } -func doHTTPHeaderTest(t *testing.T, errCode codes.Code, headerFields ...[]string) { - t.Helper() - lis, err := net.Listen("tcp", "localhost:0") - if err != nil { - t.Fatalf("Failed to listen. Err: %v", err) +// TestNilStatsHandler ensures we do not panic as a result of a nil stats +// handler. +func (s) TestNilStatsHandler(t *testing.T) { + grpctest.TLogger.ExpectErrorN("ignoring nil parameter", 2) + ss := &stubserver.StubServer{ + UnaryCallF: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{}, nil + }, } - defer lis.Close() - server := &httpServer{ - headerFields: headerFields, + if err := ss.Start([]grpc.ServerOption{grpc.StatsHandler(nil)}, grpc.WithStatsHandler(nil)); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) } - server.start(t, lis) - cc, err := grpc.Dial(lis.Addr().String(), grpc.WithInsecure()) - if err != nil { - t.Fatalf("failed to dial due to err: %v", err) + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil { + t.Fatalf("Unexpected error from UnaryCall: %v", err) + } +} + +// TestUnexpectedEOF tests a scenario where a client invokes two unary RPC +// calls. The first call receives a payload which exceeds max grpc receive +// message length, and the second gets a large response. This second RPC should +// not fail with unexpected.EOF. +func (s) TestUnexpectedEOF(t *testing.T) { + ss := &stubserver.StubServer{ + UnaryCallF: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{ + Payload: &testpb.Payload{ + Body: bytes.Repeat([]byte("a"), int(in.ResponseSize)), + }, + }, nil + }, } - defer cc.Close() - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + if err := ss.Start([]grpc.ServerOption{}); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() - client := testpb.NewTestServiceClient(cc) - stream, err := client.FullDuplexCall(ctx) - if err != nil { - t.Fatalf("error creating stream due to err: %v", err) + for i := 0; i < 10; i++ { + // exceeds grpc.DefaultMaxRecvMessageSize, this should error with + // RESOURCE_EXHAUSTED error. + _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{ResponseSize: 4194304}) + if code := status.Code(err); code != codes.ResourceExhausted { + t.Fatalf("UnaryCall RPC returned error: %v, want status code %v", err, codes.ResourceExhausted) + } + // Larger response that doesn't exceed DefaultMaxRecvMessageSize, this + // should work normally. + if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{ResponseSize: 275075}); err != nil { + t.Fatalf("UnaryCall RPC failed: %v", err) + } + } +} + +// TestRecvWhileReturningStatus performs a Recv in a service handler while the +// handler returns its status. A race condition could result in the server +// sending the first headers frame without the HTTP :status header. This can +// happen when the failed Recv (due to the handler returning) and the handler's +// status both attempt to write the status, which would be the first headers +// frame sent, simultaneously. +func (s) TestRecvWhileReturningStatus(t *testing.T) { + ss := &stubserver.StubServer{ + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + // The client never sends, so this Recv blocks until the server + // returns and causes stream operations to return errors. + go stream.Recv() + return nil + }, } - if _, err := stream.Recv(); err == nil || status.Code(err) != errCode { - t.Fatalf("stream.Recv() = _, %v, want error code: %v", err, errCode) + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + for i := 0; i < 100; i++ { + stream, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("Error while creating stream: %v", err) + } + if _, err := stream.Recv(); err != io.EOF { + t.Fatalf("stream.Recv() = %v, want io.EOF", err) + } } } -func parseCfg(r *manual.Resolver, s string) *serviceconfig.ParseResult { - g := r.CC.ParseServiceConfig(s) - if g.Err != nil { - panic(fmt.Sprintf("Error parsing config %q: %v", s, g.Err)) +type mockBinaryLogger struct { + mml *mockMethodLogger +} + +func newMockBinaryLogger() *mockBinaryLogger { + return &mockBinaryLogger{ + mml: &mockMethodLogger{}, } - return g } -func (s) TestClientCancellationPropagatesUnary(t *testing.T) { - wg := &sync.WaitGroup{} - called, done := make(chan struct{}), make(chan struct{}) - ss := &stubServer{ - emptyCall: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { - close(called) - <-ctx.Done() - err := ctx.Err() - if err != context.Canceled { - t.Errorf("ctx.Err() = %v; want context.Canceled", err) +func (mbl *mockBinaryLogger) GetMethodLogger(string) binarylog.MethodLogger { + return mbl.mml +} + +type mockMethodLogger struct { + events uint64 +} + +func (mml *mockMethodLogger) Log(context.Context, binarylog.LogEntryConfig) { + atomic.AddUint64(&mml.events, 1) +} + +// TestGlobalBinaryLoggingOptions tests the binary logging options for client +// and server side. The test configures a binary logger to be plumbed into every +// created ClientConn and server. It then makes a unary RPC call, and a +// streaming RPC call. A certain amount of logging calls should happen as a +// result of the stream operations on each of these calls. +func (s) TestGlobalBinaryLoggingOptions(t *testing.T) { + csbl := newMockBinaryLogger() + ssbl := newMockBinaryLogger() + + internal.AddGlobalDialOptions.(func(opt ...grpc.DialOption))(internal.WithBinaryLogger.(func(bl binarylog.Logger) grpc.DialOption)(csbl)) + internal.AddGlobalServerOptions.(func(opt ...grpc.ServerOption))(internal.BinaryLogger.(func(bl binarylog.Logger) grpc.ServerOption)(ssbl)) + defer func() { + internal.ClearGlobalDialOptions() + internal.ClearGlobalServerOptions() + }() + ss := &stubserver.StubServer{ + UnaryCallF: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{}, nil + }, + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + _, err := stream.Recv() + if err == io.EOF { + return nil } - close(done) - return nil, err + return status.Errorf(codes.Unknown, "expected client to call CloseSend") }, } + + // No client or server options specified, because should pick up configured + // global options. if err := ss.Start(nil); err != nil { t.Fatalf("Error starting endpoint server: %v", err) } defer ss.Stop() - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + // Make a Unary RPC. This should cause Log calls on the MethodLogger. + if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil { + t.Fatalf("Unexpected error from UnaryCall: %v", err) + } + if csbl.mml.events != 5 { + t.Fatalf("want 5 client side binary logging events, got %v", csbl.mml.events) + } + if ssbl.mml.events != 5 { + t.Fatalf("want 5 server side binary logging events, got %v", ssbl.mml.events) + } - wg.Add(1) - go func() { - if _, err := ss.client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.Canceled { - t.Errorf("ss.client.EmptyCall() = _, %v; want _, Code()=codes.Canceled", err) - } - wg.Done() - }() + // Make a streaming RPC. This should cause Log calls on the MethodLogger. + stream, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("ss.Client.FullDuplexCall failed: %f", err) + } - select { - case <-called: - case <-time.After(5 * time.Second): - t.Fatalf("failed to perform EmptyCall after 10s") + stream.CloseSend() + if _, err = stream.Recv(); err != io.EOF { + t.Fatalf("unexpected error: %v, expected an EOF error", err) } - cancel() - select { - case <-done: - case <-time.After(5 * time.Second): - t.Fatalf("server failed to close done chan due to cancellation propagation") + + if csbl.mml.events != 8 { + t.Fatalf("want 8 client side binary logging events, got %v", csbl.mml.events) } - wg.Wait() + if ssbl.mml.events != 8 { + t.Fatalf("want 8 server side binary logging events, got %v", ssbl.mml.events) + } +} + +type statsHandlerRecordEvents struct { + mu sync.Mutex + s []stats.RPCStats +} + +func (*statsHandlerRecordEvents) TagRPC(ctx context.Context, _ *stats.RPCTagInfo) context.Context { + return ctx +} +func (h *statsHandlerRecordEvents) HandleRPC(_ context.Context, s stats.RPCStats) { + h.mu.Lock() + defer h.mu.Unlock() + h.s = append(h.s, s) } +func (*statsHandlerRecordEvents) TagConn(ctx context.Context, _ *stats.ConnTagInfo) context.Context { + return ctx +} +func (*statsHandlerRecordEvents) HandleConn(context.Context, stats.ConnStats) {} -type badGzipCompressor struct{} +type triggerRPCBlockPicker struct { + pickDone func() +} -func (badGzipCompressor) Do(w io.Writer, p []byte) error { - buf := &bytes.Buffer{} - gzw := gzip.NewWriter(buf) - if _, err := gzw.Write(p); err != nil { - return err +func (bp *triggerRPCBlockPicker) Pick(pi balancer.PickInfo) (balancer.PickResult, error) { + bp.pickDone() + return balancer.PickResult{}, balancer.ErrNoSubConnAvailable +} + +const name = "triggerRPCBlockBalancer" + +type triggerRPCBlockPickerBalancerBuilder struct{} + +func (triggerRPCBlockPickerBalancerBuilder) Build(cc balancer.ClientConn, bOpts balancer.BuildOptions) balancer.Balancer { + b := &triggerRPCBlockBalancer{ + blockingPickerDone: grpcsync.NewEvent(), + ClientConn: cc, } - err := gzw.Close() - bs := buf.Bytes() - if len(bs) >= 6 { - bs[len(bs)-6] ^= 1 // modify checksum at end by 1 byte + // round_robin child to complete balancer tree with a usable leaf policy and + // have RPCs actually work. + builder := balancer.Get(roundrobin.Name) + rr := builder.Build(b, bOpts) + if rr == nil { + panic("round robin builder returned nil") } - w.Write(bs) + b.Balancer = rr + return b +} + +func (triggerRPCBlockPickerBalancerBuilder) ParseConfig(json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + return &bpbConfig{}, nil +} + +func (triggerRPCBlockPickerBalancerBuilder) Name() string { + return name +} + +type bpbConfig struct { + serviceconfig.LoadBalancingConfig +} + +// triggerRPCBlockBalancer uses a child RR balancer, but blocks all UpdateState +// calls until the first Pick call. That first Pick returns +// ErrNoSubConnAvailable to make the RPC block and trigger the appropriate stats +// handler callout. After the first Pick call, it will forward at least one +// READY picker update from the child, causing RPCs to proceed as normal using a +// round robin balancer's picker if it updates with a READY picker. +type triggerRPCBlockBalancer struct { + stateMu sync.Mutex + childState balancer.State + + blockingPickerDone *grpcsync.Event + // embed a ClientConn to wrap only UpdateState() operation + balancer.ClientConn + // embed a Balancer to wrap only UpdateClientConnState() operation + balancer.Balancer +} + +func (bpb *triggerRPCBlockBalancer) UpdateClientConnState(s balancer.ClientConnState) error { + err := bpb.Balancer.UpdateClientConnState(s) + bpb.ClientConn.UpdateState(balancer.State{ + ConnectivityState: connectivity.Connecting, + Picker: &triggerRPCBlockPicker{ + pickDone: func() { + bpb.stateMu.Lock() + defer bpb.stateMu.Unlock() + bpb.blockingPickerDone.Fire() + if bpb.childState.ConnectivityState == connectivity.Ready { + bpb.ClientConn.UpdateState(bpb.childState) + } + }, + }, + }) return err } -func (badGzipCompressor) Type() string { - return "gzip" +func (bpb *triggerRPCBlockBalancer) UpdateState(state balancer.State) { + bpb.stateMu.Lock() + defer bpb.stateMu.Unlock() + bpb.childState = state + if bpb.blockingPickerDone.HasFired() { // guard first one to get a picker sending ErrNoSubConnAvailable first + if state.ConnectivityState == connectivity.Ready { + bpb.ClientConn.UpdateState(state) // after the first rr picker update, only forward once READY for deterministic picker counts + } + } } -func (s) TestGzipBadChecksum(t *testing.T) { - ss := &stubServer{ - unaryCall: func(ctx context.Context, _ *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { +// TestRPCBlockingOnPickerStatsCall tests the emission of a stats handler call +// that represents the RPC had to block waiting for a new picker due to +// ErrNoSubConnAvailable being returned from the first picker call. +func (s) TestRPCBlockingOnPickerStatsCall(t *testing.T) { + sh := &statsHandlerRecordEvents{} + ss := &stubserver.StubServer{ + UnaryCallF: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { return &testpb.SimpleResponse{}, nil }, } - if err := ss.Start(nil, grpc.WithCompressor(badGzipCompressor{})); err != nil { + + if err := ss.StartServer(); err != nil { t.Fatalf("Error starting endpoint server: %v", err) } defer ss.Stop() - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() + lbCfgJSON := `{ + "loadBalancingConfig": [ + { + "triggerRPCBlockBalancer": {} + } + ] + }` + + sc := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(lbCfgJSON) + mr := manual.NewBuilderWithScheme("pickerupdatedbalancer") + defer mr.Close() + mr.InitialState(resolver.State{ + Addresses: []resolver.Address{ + {Addr: ss.Address}, + }, + ServiceConfig: sc, + }) - p, err := newPayload(testpb.PayloadType_COMPRESSABLE, int32(1024)) + cc, err := grpc.Dial(mr.Scheme()+":///", grpc.WithResolvers(mr), grpc.WithStatsHandler(sh), grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { - t.Fatalf("Unexpected error from newPayload: %v", err) + t.Fatalf("grpc.Dial() failed: %v", err) } - if _, err := ss.client.UnaryCall(ctx, &testpb.SimpleRequest{Payload: p}); err == nil || - status.Code(err) != codes.Internal || - !strings.Contains(status.Convert(err).Message(), gzip.ErrChecksum.Error()) { - t.Errorf("ss.client.UnaryCall(_) = _, %v\n\twant: _, status(codes.Internal, contains %q)", err, gzip.ErrChecksum) + defer cc.Close() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testServiceClient := testgrpc.NewTestServiceClient(cc) + if _, err := testServiceClient.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil { + t.Fatalf("Unexpected error from UnaryCall: %v", err) } -} -// When an RPC is canceled, it's possible that the last Recv() returns before -// all call options' after are executed. -func (s) TestCanceledRPCCallOptionRace(t *testing.T) { - ss := &stubServer{ - fullDuplexCall: func(stream testpb.TestService_FullDuplexCallServer) error { - err := stream.Send(&testpb.StreamingOutputCallResponse{}) - if err != nil { - return err - } - <-stream.Context().Done() - return nil - }, - } - if err := ss.Start(nil); err != nil { - t.Fatalf("Error starting endpoint server: %v", err) + var pickerUpdatedCount uint + for _, stat := range sh.s { + if _, ok := stat.(*stats.PickerUpdated); ok { + pickerUpdatedCount++ + } } - defer ss.Stop() - - const count = 1000 - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - var wg sync.WaitGroup - wg.Add(count) - for i := 0; i < count; i++ { - go func() { - defer wg.Done() - var p peer.Peer - ctx, cancel := context.WithCancel(ctx) - defer cancel() - stream, err := ss.client.FullDuplexCall(ctx, grpc.Peer(&p)) - if err != nil { - t.Errorf("_.FullDuplexCall(_) = _, %v", err) - return - } - if err := stream.Send(&testpb.StreamingOutputCallRequest{}); err != nil { - t.Errorf("_ has error %v while sending", err) - return - } - if _, err := stream.Recv(); err != nil { - t.Errorf("%v.Recv() = %v", stream, err) - return - } - cancel() - if _, err := stream.Recv(); status.Code(err) != codes.Canceled { - t.Errorf("%v compleled with error %v, want %s", stream, err, codes.Canceled) - return - } - // If recv returns before call options are executed, peer.Addr is not set, - // fail the test. - if p.Addr == nil { - t.Errorf("peer.Addr is nil, want non-nil") - return - } - }() + if pickerUpdatedCount != 1 { + t.Fatalf("sh.pickerUpdated count: %v, want: %v", pickerUpdatedCount, 2) } - wg.Wait() } diff --git a/test/goaway_test.go b/test/goaway_test.go index 93d964803c42..2a8ff0bfcc04 100644 --- a/test/goaway_test.go +++ b/test/goaway_test.go @@ -20,13 +20,29 @@ package test import ( "context" + "io" "net" + "strings" "testing" "time" + "golang.org/x/net/http2" "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" "google.golang.org/grpc/keepalive" - testpb "google.golang.org/grpc/test/grpc_testing" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/status" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" ) // TestGracefulClientOnGoAway attempts to ensure that when the server sends a @@ -40,15 +56,15 @@ func (s) TestGracefulClientOnGoAway(t *testing.T) { const maxConnAge = 100 * time.Millisecond const testTime = maxConnAge * 10 - ss := &stubServer{ - emptyCall: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + ss := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil }, } s := grpc.NewServer(grpc.KeepaliveParams(keepalive.ServerParameters{MaxConnectionAge: maxConnAge})) defer s.Stop() - testpb.RegisterTestServiceService(s, testpb.NewTestServiceService(ss)) + testgrpc.RegisterTestServiceServer(s, ss) lis, err := net.Listen("tcp", "localhost:0") if err != nil { @@ -56,19 +72,692 @@ func (s) TestGracefulClientOnGoAway(t *testing.T) { } go s.Serve(lis) - cc, err := grpc.Dial(lis.Addr().String(), grpc.WithInsecure()) + cc, err := grpc.Dial(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { t.Fatalf("Failed to dial server: %v", err) } defer cc.Close() - c := testpb.NewTestServiceClient(cc) + c := testgrpc.NewTestServiceClient(cc) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() endTime := time.Now().Add(testTime) for time.Now().Before(endTime) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second) if _, err := c.EmptyCall(ctx, &testpb.Empty{}); err != nil { t.Fatalf("EmptyCall(_, _) = _, %v; want _, ", err) } + } +} + +func (s) TestDetailedGoAwayErrorOnGracefulClosePropagatesToRPCError(t *testing.T) { + rpcDoneOnClient := make(chan struct{}) + ss := &stubserver.StubServer{ + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + <-rpcDoneOnClient + return status.Error(codes.Internal, "arbitrary status") + }, + } + sopts := []grpc.ServerOption{ + grpc.KeepaliveParams(keepalive.ServerParameters{ + MaxConnectionAge: time.Millisecond * 100, + MaxConnectionAgeGrace: time.Nanosecond, // ~instantaneously, but non-zero to avoid default + }), + } + if err := ss.Start(sopts); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + stream, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("%v.FullDuplexCall = _, %v, want _, ", ss.Client, err) + } + const expectedErrorMessageSubstring = "received prior goaway: code: NO_ERROR" + _, err = stream.Recv() + close(rpcDoneOnClient) + if err == nil || !strings.Contains(err.Error(), expectedErrorMessageSubstring) { + t.Fatalf("%v.Recv() = _, %v, want _, rpc error containing substring: %q", stream, err, expectedErrorMessageSubstring) + } +} + +func (s) TestDetailedGoAwayErrorOnAbruptClosePropagatesToRPCError(t *testing.T) { + grpctest.TLogger.ExpectError("Client received GoAway with error code ENHANCE_YOUR_CALM and debug data equal to ASCII \"too_many_pings\"") + // set the min keepalive time very low so that this test can take + // a reasonable amount of time + prev := internal.KeepaliveMinPingTime + internal.KeepaliveMinPingTime = time.Millisecond + defer func() { internal.KeepaliveMinPingTime = prev }() + + rpcDoneOnClient := make(chan struct{}) + ss := &stubserver.StubServer{ + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + <-rpcDoneOnClient + return status.Error(codes.Internal, "arbitrary status") + }, + } + sopts := []grpc.ServerOption{ + grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{ + MinTime: time.Second * 1000, /* arbitrary, large value */ + }), + } + dopts := []grpc.DialOption{ + grpc.WithKeepaliveParams(keepalive.ClientParameters{ + Time: time.Millisecond, /* should trigger "too many pings" error quickly */ + Timeout: time.Second * 1000, /* arbitrary, large value */ + PermitWithoutStream: false, + }), + } + if err := ss.Start(sopts, dopts...); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + stream, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("%v.FullDuplexCall = _, %v, want _, ", ss.Client, err) + } + const expectedErrorMessageSubstring = `received prior goaway: code: ENHANCE_YOUR_CALM, debug data: "too_many_pings"` + _, err = stream.Recv() + close(rpcDoneOnClient) + if err == nil || !strings.Contains(err.Error(), expectedErrorMessageSubstring) { + t.Fatalf("%v.Recv() = _, %v, want _, rpc error containing substring: |%v|", stream, err, expectedErrorMessageSubstring) + } +} + +func (s) TestClientConnCloseAfterGoAwayWithActiveStream(t *testing.T) { + for _, e := range listTestEnv() { + if e.name == "handler-tls" { + continue + } + testClientConnCloseAfterGoAwayWithActiveStream(t, e) + } +} + +func testClientConnCloseAfterGoAwayWithActiveStream(t *testing.T, e env) { + te := newTest(t, e) + te.startServer(&testServer{security: e.security}) + defer te.tearDown() + cc := te.clientConn() + tc := testgrpc.NewTestServiceClient(cc) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.FullDuplexCall(ctx); err != nil { + t.Fatalf("%v.FullDuplexCall(_) = _, %v, want _, ", tc, err) + } + done := make(chan struct{}) + go func() { + te.srv.GracefulStop() + close(done) + }() + time.Sleep(50 * time.Millisecond) + cc.Close() + timeout := time.NewTimer(time.Second) + select { + case <-done: + case <-timeout.C: + t.Fatalf("Test timed-out.") + } +} + +func (s) TestServerGoAway(t *testing.T) { + for _, e := range listTestEnv() { + if e.name == "handler-tls" { + continue + } + testServerGoAway(t, e) + } +} + +func testServerGoAway(t *testing.T, e env) { + te := newTest(t, e) + te.userAgent = testAppUA + te.startServer(&testServer{security: e.security}) + defer te.tearDown() + + cc := te.clientConn() + tc := testgrpc.NewTestServiceClient(cc) + // Finish an RPC to make sure the connection is good. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, ", err) + } + ch := make(chan struct{}) + go func() { + te.srv.GracefulStop() + close(ch) + }() + // Loop until the server side GoAway signal is propagated to the client. + for { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil && status.Code(err) != codes.DeadlineExceeded { + cancel() + break + } + cancel() + } + // A new RPC should fail. + ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.Unavailable && status.Code(err) != codes.Internal { + t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, %s or %s", err, codes.Unavailable, codes.Internal) + } + <-ch + awaitNewConnLogOutput() +} + +func (s) TestServerGoAwayPendingRPC(t *testing.T) { + for _, e := range listTestEnv() { + if e.name == "handler-tls" { + continue + } + testServerGoAwayPendingRPC(t, e) + } +} + +func testServerGoAwayPendingRPC(t *testing.T, e env) { + te := newTest(t, e) + te.userAgent = testAppUA + te.declareLogNoise( + "transport: http2Client.notifyError got notified that the client transport was broken EOF", + "grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing", + "grpc: addrConn.resetTransport failed to create client transport: connection error", + ) + te.startServer(&testServer{security: e.security}) + defer te.tearDown() + + cc := te.clientConn() + tc := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + stream, err := tc.FullDuplexCall(ctx, grpc.WaitForReady(true)) + if err != nil { + t.Fatalf("%v.FullDuplexCall(_) = _, %v, want ", tc, err) + } + // Finish an RPC to make sure the connection is good. + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("%v.EmptyCall(_, _, _) = _, %v, want _, ", tc, err) + } + ch := make(chan struct{}) + go func() { + te.srv.GracefulStop() + close(ch) + }() + // Loop until the server side GoAway signal is propagated to the client. + start := time.Now() + errored := false + for time.Since(start) < time.Second { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)) + cancel() + if err != nil { + errored = true + break + } + } + if !errored { + t.Fatalf("GoAway never received by client") + } + respParam := []*testpb.ResponseParameters{{Size: 1}} + payload, err := newPayload(testpb.PayloadType_COMPRESSABLE, int32(100)) + if err != nil { + t.Fatal(err) + } + req := &testpb.StreamingOutputCallRequest{ + ResponseType: testpb.PayloadType_COMPRESSABLE, + ResponseParameters: respParam, + Payload: payload, + } + // The existing RPC should be still good to proceed. + if err := stream.Send(req); err != nil { + t.Fatalf("%v.Send(_) = %v, want ", stream, err) + } + if _, err := stream.Recv(); err != nil { + t.Fatalf("%v.Recv() = _, %v, want _, ", stream, err) + } + // The RPC will run until canceled. + cancel() + <-ch + awaitNewConnLogOutput() +} + +func (s) TestServerMultipleGoAwayPendingRPC(t *testing.T) { + for _, e := range listTestEnv() { + if e.name == "handler-tls" { + continue + } + testServerMultipleGoAwayPendingRPC(t, e) + } +} + +func testServerMultipleGoAwayPendingRPC(t *testing.T, e env) { + te := newTest(t, e) + te.userAgent = testAppUA + te.declareLogNoise( + "transport: http2Client.notifyError got notified that the client transport was broken EOF", + "grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing", + "grpc: addrConn.resetTransport failed to create client transport: connection error", + ) + te.startServer(&testServer{security: e.security}) + defer te.tearDown() + + cc := te.clientConn() + tc := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + stream, err := tc.FullDuplexCall(ctx, grpc.WaitForReady(true)) + if err != nil { + t.Fatalf("%v.FullDuplexCall(_) = _, %v, want ", tc, err) + } + // Finish an RPC to make sure the connection is good. + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("%v.EmptyCall(_, _, _) = _, %v, want _, ", tc, err) + } + ch1 := make(chan struct{}) + go func() { + te.srv.GracefulStop() + close(ch1) + }() + ch2 := make(chan struct{}) + go func() { + te.srv.GracefulStop() + close(ch2) + }() + // Loop until the server side GoAway signal is propagated to the client. + + for { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + cancel() + break + } + cancel() + } + select { + case <-ch1: + t.Fatal("GracefulStop() terminated early") + case <-ch2: + t.Fatal("GracefulStop() terminated early") + default: + } + respParam := []*testpb.ResponseParameters{ + { + Size: 1, + }, + } + payload, err := newPayload(testpb.PayloadType_COMPRESSABLE, int32(100)) + if err != nil { + t.Fatal(err) + } + req := &testpb.StreamingOutputCallRequest{ + ResponseType: testpb.PayloadType_COMPRESSABLE, + ResponseParameters: respParam, + Payload: payload, + } + // The existing RPC should be still good to proceed. + if err := stream.Send(req); err != nil { + t.Fatalf("%v.Send(%v) = %v, want ", stream, req, err) + } + if _, err := stream.Recv(); err != nil { + t.Fatalf("%v.Recv() = _, %v, want _, ", stream, err) + } + if err := stream.CloseSend(); err != nil { + t.Fatalf("%v.CloseSend() = %v, want ", stream, err) + } + + <-ch1 + <-ch2 + cancel() + awaitNewConnLogOutput() +} + +func (s) TestConcurrentClientConnCloseAndServerGoAway(t *testing.T) { + for _, e := range listTestEnv() { + if e.name == "handler-tls" { + continue + } + testConcurrentClientConnCloseAndServerGoAway(t, e) + } +} + +func testConcurrentClientConnCloseAndServerGoAway(t *testing.T, e env) { + te := newTest(t, e) + te.userAgent = testAppUA + te.declareLogNoise( + "transport: http2Client.notifyError got notified that the client transport was broken EOF", + "grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing", + "grpc: addrConn.resetTransport failed to create client transport: connection error", + ) + te.startServer(&testServer{security: e.security}) + defer te.tearDown() + + cc := te.clientConn() + tc := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("%v.EmptyCall(_, _, _) = _, %v, want _, ", tc, err) + } + ch := make(chan struct{}) + // Close ClientConn and Server concurrently. + go func() { + te.srv.GracefulStop() + close(ch) + }() + go func() { + cc.Close() + }() + <-ch +} + +func (s) TestConcurrentServerStopAndGoAway(t *testing.T) { + for _, e := range listTestEnv() { + if e.name == "handler-tls" { + continue + } + testConcurrentServerStopAndGoAway(t, e) + } +} + +func testConcurrentServerStopAndGoAway(t *testing.T, e env) { + te := newTest(t, e) + te.userAgent = testAppUA + te.declareLogNoise( + "transport: http2Client.notifyError got notified that the client transport was broken EOF", + "grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing", + "grpc: addrConn.resetTransport failed to create client transport: connection error", + ) + te.startServer(&testServer{security: e.security}) + defer te.tearDown() + + cc := te.clientConn() + tc := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + stream, err := tc.FullDuplexCall(ctx, grpc.WaitForReady(true)) + if err != nil { + t.Fatalf("%v.FullDuplexCall(_) = _, %v, want ", tc, err) + } + + // Finish an RPC to make sure the connection is good. + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("%v.EmptyCall(_, _, _) = _, %v, want _, ", tc, err) + } + + ch := make(chan struct{}) + go func() { + te.srv.GracefulStop() + close(ch) + }() + // Loop until the server side GoAway signal is propagated to the client. + for { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + cancel() + break + } cancel() } + // Stop the server and close all the connections. + te.srv.Stop() + respParam := []*testpb.ResponseParameters{ + { + Size: 1, + }, + } + payload, err := newPayload(testpb.PayloadType_COMPRESSABLE, int32(100)) + if err != nil { + t.Fatal(err) + } + req := &testpb.StreamingOutputCallRequest{ + ResponseType: testpb.PayloadType_COMPRESSABLE, + ResponseParameters: respParam, + Payload: payload, + } + sendStart := time.Now() + for { + if err := stream.Send(req); err == io.EOF { + // stream.Send should eventually send io.EOF + break + } else if err != nil { + // Send should never return a transport-level error. + t.Fatalf("stream.Send(%v) = %v; want ", req, err) + } + if time.Since(sendStart) > 2*time.Second { + t.Fatalf("stream.Send(_) did not return io.EOF after 2s") + } + time.Sleep(time.Millisecond) + } + if _, err := stream.Recv(); err == nil || err == io.EOF { + t.Fatalf("%v.Recv() = _, %v, want _, ", stream, err) + } + <-ch + awaitNewConnLogOutput() +} + +// Proxies typically send GO_AWAY followed by connection closure a minute or so later. This +// test ensures that the connection is re-created after GO_AWAY and not affected by the +// subsequent (old) connection closure. +func (s) TestGoAwayThenClose(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + lis1, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Error while listening. Err: %v", err) + } + s1 := grpc.NewServer() + defer s1.Stop() + ts := &funcServer{ + unaryCall: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{}, nil + }, + fullDuplexCall: func(stream testgrpc.TestService_FullDuplexCallServer) error { + if err := stream.Send(&testpb.StreamingOutputCallResponse{}); err != nil { + t.Errorf("unexpected error from send: %v", err) + return err + } + // Wait forever. + _, err := stream.Recv() + if err == nil { + t.Error("expected to never receive any message") + } + return err + }, + } + testgrpc.RegisterTestServiceServer(s1, ts) + go s1.Serve(lis1) + + conn2Established := grpcsync.NewEvent() + lis2, err := listenWithNotifyingListener("tcp", "localhost:0", conn2Established) + if err != nil { + t.Fatalf("Error while listening. Err: %v", err) + } + s2 := grpc.NewServer() + defer s2.Stop() + testgrpc.RegisterTestServiceServer(s2, ts) + + r := manual.NewBuilderWithScheme("whatever") + r.InitialState(resolver.State{Addresses: []resolver.Address{ + {Addr: lis1.Addr().String()}, + {Addr: lis2.Addr().String()}, + }}) + cc, err := grpc.DialContext(ctx, r.Scheme()+":///", grpc.WithResolvers(r), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Error creating client: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + + t.Log("Waiting for the ClientConn to enter READY state.") + testutils.AwaitState(ctx, t, cc, connectivity.Ready) + + // We make a streaming RPC and do an one-message-round-trip to make sure + // it's created on connection 1. + // + // We use a long-lived RPC because it will cause GracefulStop to send + // GO_AWAY, but the connection won't get closed until the server stops and + // the client receives the error. + t.Log("Creating first streaming RPC to server 1.") + stream, err := client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("FullDuplexCall(_) = _, %v; want _, nil", err) + } + if _, err = stream.Recv(); err != nil { + t.Fatalf("unexpected error from first recv: %v", err) + } + + go s2.Serve(lis2) + + t.Log("Gracefully stopping server 1.") + go s1.GracefulStop() + + t.Log("Waiting for the ClientConn to enter IDLE state.") + testutils.AwaitState(ctx, t, cc, connectivity.Idle) + + t.Log("Performing another RPC to create a connection to server 2.") + if _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil { + t.Fatalf("UnaryCall(_) = _, %v; want _, nil", err) + } + + t.Log("Waiting for a connection to server 2.") + select { + case <-conn2Established.Done(): + case <-ctx.Done(): + t.Fatalf("timed out waiting for connection 2 to be established") + } + + // Close the listener for server2 to prevent it from allowing new connections. + lis2.Close() + + t.Log("Hard closing connection 1.") + s1.Stop() + + t.Log("Waiting for the first stream to error.") + if _, err = stream.Recv(); err == nil { + t.Fatal("expected the stream to die, but got a successful Recv") + } + + t.Log("Ensuring connection 2 is stable.") + for i := 0; i < 10; i++ { + if _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil { + t.Fatalf("UnaryCall(_) = _, %v; want _, nil", err) + } + } +} + +// TestGoAwayStreamIDSmallerThanCreatedStreams tests the scenario where a server +// sends a goaway with a stream id that is smaller than some created streams on +// the client, while the client is simultaneously creating new streams. This +// should not induce a deadlock. +func (s) TestGoAwayStreamIDSmallerThanCreatedStreams(t *testing.T) { + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("error listening: %v", err) + } + + ctCh := testutils.NewChannel() + go func() { + conn, err := lis.Accept() + if err != nil { + t.Errorf("error in lis.Accept(): %v", err) + } + ct := newClientTester(t, conn) + ctCh.Send(ct) + }() + + cc, err := grpc.Dial(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("error dialing: %v", err) + } + defer cc.Close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + val, err := ctCh.Receive(ctx) + if err != nil { + t.Fatalf("timeout waiting for client transport (should be given after http2 creation)") + } + ct := val.(*clientTester) + + tc := testgrpc.NewTestServiceClient(cc) + someStreamsCreated := grpcsync.NewEvent() + goAwayWritten := grpcsync.NewEvent() + go func() { + for i := 0; i < 20; i++ { + if i == 10 { + <-goAwayWritten.Done() + } + tc.FullDuplexCall(ctx) + if i == 4 { + someStreamsCreated.Fire() + } + } + }() + + <-someStreamsCreated.Done() + ct.writeGoAway(1, http2.ErrCodeNo, []byte{}) + goAwayWritten.Fire() +} + +// TestTwoGoAwayPingFrames tests the scenario where you get two go away ping +// frames from the client during graceful shutdown. This should not crash the +// server. +func (s) TestTwoGoAwayPingFrames(t *testing.T) { + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Failed to listen: %v", err) + } + defer lis.Close() + s := grpc.NewServer() + defer s.Stop() + go s.Serve(lis) + + conn, err := net.DialTimeout("tcp", lis.Addr().String(), defaultTestTimeout) + if err != nil { + t.Fatalf("Failed to dial: %v", err) + } + + st := newServerTesterFromConn(t, conn) + st.greet() + pingReceivedClientSide := testutils.NewChannel() + go func() { + for { + f, err := st.readFrame() + if err != nil { + return + } + switch f.(type) { + case *http2.GoAwayFrame: + case *http2.PingFrame: + pingReceivedClientSide.Send(nil) + default: + t.Errorf("server tester received unexpected frame type %T", f) + } + } + }() + gsDone := testutils.NewChannel() + go func() { + s.GracefulStop() + gsDone.Send(nil) + }() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := pingReceivedClientSide.Receive(ctx); err != nil { + t.Fatalf("Error waiting for ping frame client side from graceful shutdown: %v", err) + } + // Write two goaway pings here. + st.writePing(true, [8]byte{1, 6, 1, 8, 0, 3, 3, 9}) + st.writePing(true, [8]byte{1, 6, 1, 8, 0, 3, 3, 9}) + // Close the conn to finish up the Graceful Shutdown process. + conn.Close() + if _, err := gsDone.Receive(ctx); err != nil { + t.Fatalf("Error waiting for graceful shutdown of the server: %v", err) + } } diff --git a/test/gracefulstop_test.go b/test/gracefulstop_test.go index f7659524d1d8..f0697e7e328b 100644 --- a/test/gracefulstop_test.go +++ b/test/gracefulstop_test.go @@ -24,12 +24,16 @@ import ( "net" "sync" "testing" - "time" + "golang.org/x/net/http2" "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/stubserver" "google.golang.org/grpc/status" - testpb "google.golang.org/grpc/test/grpc_testing" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" ) type delayListener struct { @@ -107,8 +111,8 @@ func (s) TestGracefulStop(t *testing.T) { } d := func(ctx context.Context, _ string) (net.Conn, error) { return dlis.Dial(ctx) } - ss := &stubServer{ - fullDuplexCall: func(stream testpb.TestService_FullDuplexCallServer) error { + ss := &stubserver.StubServer{ + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { _, err := stream.Recv() if err != nil { return err @@ -117,7 +121,7 @@ func (s) TestGracefulStop(t *testing.T) { }, } s := grpc.NewServer() - testpb.RegisterTestServiceService(s, testpb.NewTestServiceService(ss)) + testgrpc.RegisterTestServiceServer(s, ss) // 1. Start Server wg := sync.WaitGroup{} @@ -143,22 +147,72 @@ func (s) TestGracefulStop(t *testing.T) { // Now dial. The listener's Accept method will return a valid connection, // even though GracefulStop has closed the listener. - ctx, dialCancel := context.WithTimeout(context.Background(), 5*time.Second) + ctx, dialCancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer dialCancel() - cc, err := grpc.DialContext(ctx, "", grpc.WithInsecure(), grpc.WithContextDialer(d)) + cc, err := grpc.DialContext(ctx, "", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithContextDialer(d)) if err != nil { t.Fatalf("grpc.DialContext(_, %q, _) = %v", lis.Addr().String(), err) } - client := testpb.NewTestServiceClient(cc) + client := testgrpc.NewTestServiceClient(cc) defer cc.Close() // 4. Send an RPC on the new connection. // The server would send a GOAWAY first, but we are delaying the server's // writes for now until the client writes more than the preface. - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) if _, err = client.FullDuplexCall(ctx); err == nil || status.Code(err) != codes.Unavailable { t.Fatalf("FullDuplexCall= _, %v; want _, ", err) } cancel() wg.Wait() } + +func (s) TestGracefulStopClosesConnAfterLastStream(t *testing.T) { + // This test ensures that a server closes the connections to its clients + // when the final stream has completed after a GOAWAY. + + handlerCalled := make(chan struct{}) + gracefulStopCalled := make(chan struct{}) + + ts := &funcServer{streamingInputCall: func(stream testgrpc.TestService_StreamingInputCallServer) error { + close(handlerCalled) // Initiate call to GracefulStop. + <-gracefulStopCalled // Wait for GOAWAYs to be received by the client. + return nil + }} + + te := newTest(t, tcpClearEnv) + te.startServer(ts) + defer te.tearDown() + + te.withServerTester(func(st *serverTester) { + st.writeHeadersGRPC(1, "/grpc.testing.TestService/StreamingInputCall", false) + + <-handlerCalled // Wait for the server to invoke its handler. + + // Gracefully stop the server. + gracefulStopDone := make(chan struct{}) + go func() { + te.srv.GracefulStop() + close(gracefulStopDone) + }() + st.wantGoAway(http2.ErrCodeNo) // Server sends a GOAWAY due to GracefulStop. + pf := st.wantPing() // Server sends a ping to verify client receipt. + st.writePing(true, pf.Data) // Send ping ack to confirm. + st.wantGoAway(http2.ErrCodeNo) // Wait for subsequent GOAWAY to indicate no new stream processing. + + close(gracefulStopCalled) // Unblock server handler. + + fr := st.wantAnyFrame() // Wait for trailer. + hdr, ok := fr.(*http2.MetaHeadersFrame) + if !ok { + t.Fatalf("Received unexpected frame of type (%T) from server: %v; want HEADERS", fr, fr) + } + if !hdr.StreamEnded() { + t.Fatalf("Received unexpected HEADERS frame from server: %v; want END_STREAM set", fr) + } + + st.wantRSTStream(http2.ErrCodeNo) // Server should send RST_STREAM because client did not half-close. + + <-gracefulStopDone // Wait for GracefulStop to return. + }) +} diff --git a/test/grpc_testing/test.pb.go b/test/grpc_testing/test.pb.go deleted file mode 100644 index 70e3b89228bc..000000000000 --- a/test/grpc_testing/test.pb.go +++ /dev/null @@ -1,568 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. -// source: test/grpc_testing/test.proto - -package grpc_testing - -import ( - fmt "fmt" - proto "github.com/golang/protobuf/proto" - math "math" -) - -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package - -// The type of payload that should be returned. -type PayloadType int32 - -const ( - // Compressable text format. - PayloadType_COMPRESSABLE PayloadType = 0 - // Uncompressable binary format. - PayloadType_UNCOMPRESSABLE PayloadType = 1 - // Randomly chosen from all other formats defined in this enum. - PayloadType_RANDOM PayloadType = 2 -) - -var PayloadType_name = map[int32]string{ - 0: "COMPRESSABLE", - 1: "UNCOMPRESSABLE", - 2: "RANDOM", -} - -var PayloadType_value = map[string]int32{ - "COMPRESSABLE": 0, - "UNCOMPRESSABLE": 1, - "RANDOM": 2, -} - -func (x PayloadType) String() string { - return proto.EnumName(PayloadType_name, int32(x)) -} - -func (PayloadType) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_139516ae706ad4b7, []int{0} -} - -type Empty struct { - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *Empty) Reset() { *m = Empty{} } -func (m *Empty) String() string { return proto.CompactTextString(m) } -func (*Empty) ProtoMessage() {} -func (*Empty) Descriptor() ([]byte, []int) { - return fileDescriptor_139516ae706ad4b7, []int{0} -} - -func (m *Empty) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Empty.Unmarshal(m, b) -} -func (m *Empty) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Empty.Marshal(b, m, deterministic) -} -func (m *Empty) XXX_Merge(src proto.Message) { - xxx_messageInfo_Empty.Merge(m, src) -} -func (m *Empty) XXX_Size() int { - return xxx_messageInfo_Empty.Size(m) -} -func (m *Empty) XXX_DiscardUnknown() { - xxx_messageInfo_Empty.DiscardUnknown(m) -} - -var xxx_messageInfo_Empty proto.InternalMessageInfo - -// A block of data, to simply increase gRPC message size. -type Payload struct { - // The type of data in body. - Type PayloadType `protobuf:"varint,1,opt,name=type,proto3,enum=grpc.testing.PayloadType" json:"type,omitempty"` - // Primary contents of payload. - Body []byte `protobuf:"bytes,2,opt,name=body,proto3" json:"body,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *Payload) Reset() { *m = Payload{} } -func (m *Payload) String() string { return proto.CompactTextString(m) } -func (*Payload) ProtoMessage() {} -func (*Payload) Descriptor() ([]byte, []int) { - return fileDescriptor_139516ae706ad4b7, []int{1} -} - -func (m *Payload) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Payload.Unmarshal(m, b) -} -func (m *Payload) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Payload.Marshal(b, m, deterministic) -} -func (m *Payload) XXX_Merge(src proto.Message) { - xxx_messageInfo_Payload.Merge(m, src) -} -func (m *Payload) XXX_Size() int { - return xxx_messageInfo_Payload.Size(m) -} -func (m *Payload) XXX_DiscardUnknown() { - xxx_messageInfo_Payload.DiscardUnknown(m) -} - -var xxx_messageInfo_Payload proto.InternalMessageInfo - -func (m *Payload) GetType() PayloadType { - if m != nil { - return m.Type - } - return PayloadType_COMPRESSABLE -} - -func (m *Payload) GetBody() []byte { - if m != nil { - return m.Body - } - return nil -} - -// Unary request. -type SimpleRequest struct { - // Desired payload type in the response from the server. - // If response_type is RANDOM, server randomly chooses one from other formats. - ResponseType PayloadType `protobuf:"varint,1,opt,name=response_type,json=responseType,proto3,enum=grpc.testing.PayloadType" json:"response_type,omitempty"` - // Desired payload size in the response from the server. - // If response_type is COMPRESSABLE, this denotes the size before compression. - ResponseSize int32 `protobuf:"varint,2,opt,name=response_size,json=responseSize,proto3" json:"response_size,omitempty"` - // Optional input payload sent along with the request. - Payload *Payload `protobuf:"bytes,3,opt,name=payload,proto3" json:"payload,omitempty"` - // Whether SimpleResponse should include username. - FillUsername bool `protobuf:"varint,4,opt,name=fill_username,json=fillUsername,proto3" json:"fill_username,omitempty"` - // Whether SimpleResponse should include OAuth scope. - FillOauthScope bool `protobuf:"varint,5,opt,name=fill_oauth_scope,json=fillOauthScope,proto3" json:"fill_oauth_scope,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *SimpleRequest) Reset() { *m = SimpleRequest{} } -func (m *SimpleRequest) String() string { return proto.CompactTextString(m) } -func (*SimpleRequest) ProtoMessage() {} -func (*SimpleRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_139516ae706ad4b7, []int{2} -} - -func (m *SimpleRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_SimpleRequest.Unmarshal(m, b) -} -func (m *SimpleRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_SimpleRequest.Marshal(b, m, deterministic) -} -func (m *SimpleRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_SimpleRequest.Merge(m, src) -} -func (m *SimpleRequest) XXX_Size() int { - return xxx_messageInfo_SimpleRequest.Size(m) -} -func (m *SimpleRequest) XXX_DiscardUnknown() { - xxx_messageInfo_SimpleRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_SimpleRequest proto.InternalMessageInfo - -func (m *SimpleRequest) GetResponseType() PayloadType { - if m != nil { - return m.ResponseType - } - return PayloadType_COMPRESSABLE -} - -func (m *SimpleRequest) GetResponseSize() int32 { - if m != nil { - return m.ResponseSize - } - return 0 -} - -func (m *SimpleRequest) GetPayload() *Payload { - if m != nil { - return m.Payload - } - return nil -} - -func (m *SimpleRequest) GetFillUsername() bool { - if m != nil { - return m.FillUsername - } - return false -} - -func (m *SimpleRequest) GetFillOauthScope() bool { - if m != nil { - return m.FillOauthScope - } - return false -} - -// Unary response, as configured by the request. -type SimpleResponse struct { - // Payload to increase message size. - Payload *Payload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"` - // The user the request came from, for verifying authentication was - // successful when the client expected it. - Username string `protobuf:"bytes,2,opt,name=username,proto3" json:"username,omitempty"` - // OAuth scope. - OauthScope string `protobuf:"bytes,3,opt,name=oauth_scope,json=oauthScope,proto3" json:"oauth_scope,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *SimpleResponse) Reset() { *m = SimpleResponse{} } -func (m *SimpleResponse) String() string { return proto.CompactTextString(m) } -func (*SimpleResponse) ProtoMessage() {} -func (*SimpleResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_139516ae706ad4b7, []int{3} -} - -func (m *SimpleResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_SimpleResponse.Unmarshal(m, b) -} -func (m *SimpleResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_SimpleResponse.Marshal(b, m, deterministic) -} -func (m *SimpleResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_SimpleResponse.Merge(m, src) -} -func (m *SimpleResponse) XXX_Size() int { - return xxx_messageInfo_SimpleResponse.Size(m) -} -func (m *SimpleResponse) XXX_DiscardUnknown() { - xxx_messageInfo_SimpleResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_SimpleResponse proto.InternalMessageInfo - -func (m *SimpleResponse) GetPayload() *Payload { - if m != nil { - return m.Payload - } - return nil -} - -func (m *SimpleResponse) GetUsername() string { - if m != nil { - return m.Username - } - return "" -} - -func (m *SimpleResponse) GetOauthScope() string { - if m != nil { - return m.OauthScope - } - return "" -} - -// Client-streaming request. -type StreamingInputCallRequest struct { - // Optional input payload sent along with the request. - Payload *Payload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *StreamingInputCallRequest) Reset() { *m = StreamingInputCallRequest{} } -func (m *StreamingInputCallRequest) String() string { return proto.CompactTextString(m) } -func (*StreamingInputCallRequest) ProtoMessage() {} -func (*StreamingInputCallRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_139516ae706ad4b7, []int{4} -} - -func (m *StreamingInputCallRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_StreamingInputCallRequest.Unmarshal(m, b) -} -func (m *StreamingInputCallRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_StreamingInputCallRequest.Marshal(b, m, deterministic) -} -func (m *StreamingInputCallRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_StreamingInputCallRequest.Merge(m, src) -} -func (m *StreamingInputCallRequest) XXX_Size() int { - return xxx_messageInfo_StreamingInputCallRequest.Size(m) -} -func (m *StreamingInputCallRequest) XXX_DiscardUnknown() { - xxx_messageInfo_StreamingInputCallRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_StreamingInputCallRequest proto.InternalMessageInfo - -func (m *StreamingInputCallRequest) GetPayload() *Payload { - if m != nil { - return m.Payload - } - return nil -} - -// Client-streaming response. -type StreamingInputCallResponse struct { - // Aggregated size of payloads received from the client. - AggregatedPayloadSize int32 `protobuf:"varint,1,opt,name=aggregated_payload_size,json=aggregatedPayloadSize,proto3" json:"aggregated_payload_size,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *StreamingInputCallResponse) Reset() { *m = StreamingInputCallResponse{} } -func (m *StreamingInputCallResponse) String() string { return proto.CompactTextString(m) } -func (*StreamingInputCallResponse) ProtoMessage() {} -func (*StreamingInputCallResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_139516ae706ad4b7, []int{5} -} - -func (m *StreamingInputCallResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_StreamingInputCallResponse.Unmarshal(m, b) -} -func (m *StreamingInputCallResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_StreamingInputCallResponse.Marshal(b, m, deterministic) -} -func (m *StreamingInputCallResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_StreamingInputCallResponse.Merge(m, src) -} -func (m *StreamingInputCallResponse) XXX_Size() int { - return xxx_messageInfo_StreamingInputCallResponse.Size(m) -} -func (m *StreamingInputCallResponse) XXX_DiscardUnknown() { - xxx_messageInfo_StreamingInputCallResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_StreamingInputCallResponse proto.InternalMessageInfo - -func (m *StreamingInputCallResponse) GetAggregatedPayloadSize() int32 { - if m != nil { - return m.AggregatedPayloadSize - } - return 0 -} - -// Configuration for a particular response. -type ResponseParameters struct { - // Desired payload sizes in responses from the server. - // If response_type is COMPRESSABLE, this denotes the size before compression. - Size int32 `protobuf:"varint,1,opt,name=size,proto3" json:"size,omitempty"` - // Desired interval between consecutive responses in the response stream in - // microseconds. - IntervalUs int32 `protobuf:"varint,2,opt,name=interval_us,json=intervalUs,proto3" json:"interval_us,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *ResponseParameters) Reset() { *m = ResponseParameters{} } -func (m *ResponseParameters) String() string { return proto.CompactTextString(m) } -func (*ResponseParameters) ProtoMessage() {} -func (*ResponseParameters) Descriptor() ([]byte, []int) { - return fileDescriptor_139516ae706ad4b7, []int{6} -} - -func (m *ResponseParameters) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_ResponseParameters.Unmarshal(m, b) -} -func (m *ResponseParameters) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_ResponseParameters.Marshal(b, m, deterministic) -} -func (m *ResponseParameters) XXX_Merge(src proto.Message) { - xxx_messageInfo_ResponseParameters.Merge(m, src) -} -func (m *ResponseParameters) XXX_Size() int { - return xxx_messageInfo_ResponseParameters.Size(m) -} -func (m *ResponseParameters) XXX_DiscardUnknown() { - xxx_messageInfo_ResponseParameters.DiscardUnknown(m) -} - -var xxx_messageInfo_ResponseParameters proto.InternalMessageInfo - -func (m *ResponseParameters) GetSize() int32 { - if m != nil { - return m.Size - } - return 0 -} - -func (m *ResponseParameters) GetIntervalUs() int32 { - if m != nil { - return m.IntervalUs - } - return 0 -} - -// Server-streaming request. -type StreamingOutputCallRequest struct { - // Desired payload type in the response from the server. - // If response_type is RANDOM, the payload from each response in the stream - // might be of different types. This is to simulate a mixed type of payload - // stream. - ResponseType PayloadType `protobuf:"varint,1,opt,name=response_type,json=responseType,proto3,enum=grpc.testing.PayloadType" json:"response_type,omitempty"` - // Configuration for each expected response message. - ResponseParameters []*ResponseParameters `protobuf:"bytes,2,rep,name=response_parameters,json=responseParameters,proto3" json:"response_parameters,omitempty"` - // Optional input payload sent along with the request. - Payload *Payload `protobuf:"bytes,3,opt,name=payload,proto3" json:"payload,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *StreamingOutputCallRequest) Reset() { *m = StreamingOutputCallRequest{} } -func (m *StreamingOutputCallRequest) String() string { return proto.CompactTextString(m) } -func (*StreamingOutputCallRequest) ProtoMessage() {} -func (*StreamingOutputCallRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_139516ae706ad4b7, []int{7} -} - -func (m *StreamingOutputCallRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_StreamingOutputCallRequest.Unmarshal(m, b) -} -func (m *StreamingOutputCallRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_StreamingOutputCallRequest.Marshal(b, m, deterministic) -} -func (m *StreamingOutputCallRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_StreamingOutputCallRequest.Merge(m, src) -} -func (m *StreamingOutputCallRequest) XXX_Size() int { - return xxx_messageInfo_StreamingOutputCallRequest.Size(m) -} -func (m *StreamingOutputCallRequest) XXX_DiscardUnknown() { - xxx_messageInfo_StreamingOutputCallRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_StreamingOutputCallRequest proto.InternalMessageInfo - -func (m *StreamingOutputCallRequest) GetResponseType() PayloadType { - if m != nil { - return m.ResponseType - } - return PayloadType_COMPRESSABLE -} - -func (m *StreamingOutputCallRequest) GetResponseParameters() []*ResponseParameters { - if m != nil { - return m.ResponseParameters - } - return nil -} - -func (m *StreamingOutputCallRequest) GetPayload() *Payload { - if m != nil { - return m.Payload - } - return nil -} - -// Server-streaming response, as configured by the request and parameters. -type StreamingOutputCallResponse struct { - // Payload to increase response size. - Payload *Payload `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *StreamingOutputCallResponse) Reset() { *m = StreamingOutputCallResponse{} } -func (m *StreamingOutputCallResponse) String() string { return proto.CompactTextString(m) } -func (*StreamingOutputCallResponse) ProtoMessage() {} -func (*StreamingOutputCallResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_139516ae706ad4b7, []int{8} -} - -func (m *StreamingOutputCallResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_StreamingOutputCallResponse.Unmarshal(m, b) -} -func (m *StreamingOutputCallResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_StreamingOutputCallResponse.Marshal(b, m, deterministic) -} -func (m *StreamingOutputCallResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_StreamingOutputCallResponse.Merge(m, src) -} -func (m *StreamingOutputCallResponse) XXX_Size() int { - return xxx_messageInfo_StreamingOutputCallResponse.Size(m) -} -func (m *StreamingOutputCallResponse) XXX_DiscardUnknown() { - xxx_messageInfo_StreamingOutputCallResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_StreamingOutputCallResponse proto.InternalMessageInfo - -func (m *StreamingOutputCallResponse) GetPayload() *Payload { - if m != nil { - return m.Payload - } - return nil -} - -func init() { - proto.RegisterEnum("grpc.testing.PayloadType", PayloadType_name, PayloadType_value) - proto.RegisterType((*Empty)(nil), "grpc.testing.Empty") - proto.RegisterType((*Payload)(nil), "grpc.testing.Payload") - proto.RegisterType((*SimpleRequest)(nil), "grpc.testing.SimpleRequest") - proto.RegisterType((*SimpleResponse)(nil), "grpc.testing.SimpleResponse") - proto.RegisterType((*StreamingInputCallRequest)(nil), "grpc.testing.StreamingInputCallRequest") - proto.RegisterType((*StreamingInputCallResponse)(nil), "grpc.testing.StreamingInputCallResponse") - proto.RegisterType((*ResponseParameters)(nil), "grpc.testing.ResponseParameters") - proto.RegisterType((*StreamingOutputCallRequest)(nil), "grpc.testing.StreamingOutputCallRequest") - proto.RegisterType((*StreamingOutputCallResponse)(nil), "grpc.testing.StreamingOutputCallResponse") -} - -func init() { proto.RegisterFile("test/grpc_testing/test.proto", fileDescriptor_139516ae706ad4b7) } - -var fileDescriptor_139516ae706ad4b7 = []byte{ - // 615 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x55, 0xdd, 0x6e, 0xd3, 0x4c, - 0x10, 0xfd, 0xb6, 0x4d, 0xff, 0x26, 0x69, 0x14, 0x6d, 0x55, 0x7d, 0x69, 0x5a, 0x89, 0xc8, 0x5c, - 0x60, 0x2a, 0x91, 0xa0, 0x20, 0xb8, 0x04, 0xf5, 0x57, 0x54, 0x6a, 0x9b, 0x62, 0x37, 0x37, 0xdc, - 0x44, 0xdb, 0x76, 0x6a, 0x2c, 0x6d, 0xbc, 0xcb, 0x7a, 0x5d, 0x91, 0x5e, 0xf0, 0x62, 0xbc, 0x0c, - 0x0f, 0xc1, 0x03, 0xa0, 0xdd, 0xd8, 0xa9, 0xd3, 0xb8, 0x22, 0x05, 0xc1, 0x55, 0x36, 0x33, 0x67, - 0xce, 0x9c, 0xe3, 0x19, 0x7b, 0x61, 0x4b, 0x63, 0xac, 0xdb, 0x81, 0x92, 0x97, 0x7d, 0x73, 0x0a, - 0xa3, 0xa0, 0x6d, 0x7e, 0x5b, 0x52, 0x09, 0x2d, 0x68, 0xc5, 0x24, 0x5a, 0x69, 0xc2, 0x59, 0x82, - 0x85, 0x83, 0x81, 0xd4, 0x43, 0xe7, 0x18, 0x96, 0xce, 0xd8, 0x90, 0x0b, 0x76, 0x45, 0x5f, 0x40, - 0x49, 0x0f, 0x25, 0xd6, 0x49, 0x93, 0xb8, 0xd5, 0xce, 0x46, 0x2b, 0x5f, 0xd0, 0x4a, 0x41, 0xe7, - 0x43, 0x89, 0x9e, 0x85, 0x51, 0x0a, 0xa5, 0x0b, 0x71, 0x35, 0xac, 0xcf, 0x35, 0x89, 0x5b, 0xf1, - 0xec, 0xd9, 0xf9, 0x41, 0x60, 0xd5, 0x0f, 0x07, 0x92, 0xa3, 0x87, 0x9f, 0x13, 0x8c, 0x35, 0x7d, - 0x0b, 0xab, 0x0a, 0x63, 0x29, 0xa2, 0x18, 0xfb, 0xb3, 0xb1, 0x57, 0x32, 0xbc, 0xf9, 0x47, 0x9f, - 0xe6, 0xea, 0xe3, 0xf0, 0x16, 0x6d, 0xbb, 0x85, 0x3b, 0x90, 0x1f, 0xde, 0x22, 0x6d, 0xc3, 0x92, - 0x1c, 0x31, 0xd4, 0xe7, 0x9b, 0xc4, 0x2d, 0x77, 0xd6, 0x0b, 0xe9, 0xbd, 0x0c, 0x65, 0x58, 0xaf, - 0x43, 0xce, 0xfb, 0x49, 0x8c, 0x2a, 0x62, 0x03, 0xac, 0x97, 0x9a, 0xc4, 0x5d, 0xf6, 0x2a, 0x26, - 0xd8, 0x4b, 0x63, 0xd4, 0x85, 0x9a, 0x05, 0x09, 0x96, 0xe8, 0x4f, 0xfd, 0xf8, 0x52, 0x48, 0xac, - 0x2f, 0x58, 0x5c, 0xd5, 0xc4, 0xbb, 0x26, 0xec, 0x9b, 0xa8, 0xf3, 0x15, 0xaa, 0x99, 0xeb, 0x91, - 0xaa, 0xbc, 0x22, 0x32, 0x93, 0xa2, 0x06, 0x2c, 0x8f, 0xc5, 0x18, 0x8b, 0x2b, 0xde, 0xf8, 0x3f, - 0x7d, 0x02, 0xe5, 0xbc, 0x86, 0x79, 0x9b, 0x06, 0x71, 0xd7, 0xff, 0x18, 0x36, 0x7c, 0xad, 0x90, - 0x0d, 0xc2, 0x28, 0x38, 0x8a, 0x64, 0xa2, 0xf7, 0x18, 0xe7, 0xd9, 0x04, 0x1e, 0x2b, 0xc5, 0x39, - 0x87, 0x46, 0x11, 0x5b, 0xea, 0xec, 0x0d, 0xfc, 0xcf, 0x82, 0x40, 0x61, 0xc0, 0x34, 0x5e, 0xf5, - 0xd3, 0x9a, 0xd1, 0x68, 0x88, 0x1d, 0xcd, 0xfa, 0x5d, 0x3a, 0xa5, 0x36, 0x33, 0x72, 0x8e, 0x80, - 0x66, 0x1c, 0x67, 0x4c, 0xb1, 0x01, 0x6a, 0x54, 0xb1, 0x59, 0xa2, 0x5c, 0xa9, 0x3d, 0x1b, 0xbb, - 0x61, 0xa4, 0x51, 0xdd, 0x30, 0x33, 0xa0, 0x74, 0xe0, 0x90, 0x85, 0x7a, 0xb1, 0xf3, 0x9d, 0xe4, - 0x14, 0x76, 0x13, 0x7d, 0xcf, 0xf0, 0x9f, 0xae, 0xdc, 0x07, 0x58, 0x1b, 0xd7, 0xcb, 0xb1, 0xd4, - 0xfa, 0x5c, 0x73, 0xde, 0x2d, 0x77, 0x9a, 0x93, 0x2c, 0xd3, 0x96, 0x3c, 0xaa, 0xa6, 0x6d, 0x3e, - 0x76, 0x41, 0x9d, 0x53, 0xd8, 0x2c, 0x74, 0xf8, 0x9b, 0xeb, 0xb5, 0xfd, 0x0e, 0xca, 0x39, 0xc3, - 0xb4, 0x06, 0x95, 0xbd, 0xee, 0xc9, 0x99, 0x77, 0xe0, 0xfb, 0x3b, 0xbb, 0xc7, 0x07, 0xb5, 0xff, - 0x28, 0x85, 0x6a, 0xef, 0x74, 0x22, 0x46, 0x28, 0xc0, 0xa2, 0xb7, 0x73, 0xba, 0xdf, 0x3d, 0xa9, - 0xcd, 0x75, 0xbe, 0x95, 0xa0, 0x7c, 0x8e, 0xb1, 0xf6, 0x51, 0xdd, 0x84, 0x97, 0x48, 0x5f, 0xc3, - 0x8a, 0xfd, 0x80, 0x18, 0x59, 0x74, 0x6d, 0xb2, 0xbb, 0x4d, 0x34, 0x8a, 0x82, 0xf4, 0x10, 0x56, - 0x7a, 0x11, 0x53, 0xa3, 0xb2, 0xcd, 0x49, 0xc4, 0xc4, 0x87, 0xa3, 0xb1, 0x55, 0x9c, 0x4c, 0x1f, - 0x00, 0x87, 0xb5, 0x82, 0xe7, 0x43, 0xdd, 0x7b, 0x45, 0x0f, 0x2e, 0x49, 0xe3, 0xf9, 0x0c, 0xc8, - 0x51, 0xaf, 0x97, 0x84, 0x86, 0x40, 0xa7, 0xdf, 0x08, 0xfa, 0xec, 0x01, 0x8a, 0xfb, 0x6f, 0x60, - 0xc3, 0xfd, 0x35, 0x70, 0xd4, 0xca, 0x35, 0xad, 0xaa, 0x87, 0x09, 0xe7, 0xfb, 0x89, 0xe4, 0xf8, - 0xe5, 0xaf, 0x79, 0x72, 0x89, 0x75, 0x55, 0x7d, 0xcf, 0xf8, 0xf5, 0x3f, 0x68, 0xb5, 0xbb, 0xfd, - 0xd1, 0x0d, 0x84, 0x08, 0x38, 0xb6, 0x02, 0xc1, 0x59, 0x14, 0xb4, 0x84, 0x0a, 0xec, 0x4d, 0xd5, - 0x9e, 0xba, 0xb3, 0x2e, 0x16, 0xed, 0x7d, 0xf5, 0xea, 0x67, 0x00, 0x00, 0x00, 0xff, 0xff, 0x7e, - 0x50, 0x51, 0x5b, 0xcf, 0x06, 0x00, 0x00, -} diff --git a/test/grpc_testing/test.proto b/test/grpc_testing/test.proto deleted file mode 100644 index 0c6650401d59..000000000000 --- a/test/grpc_testing/test.proto +++ /dev/null @@ -1,156 +0,0 @@ -// Copyright 2017 gRPC authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// An integration test service that covers all the method signature permutations -// of unary/streaming requests/responses. -syntax = "proto3"; - -option go_package = "google.golang.org/grpc/test/grpc_testing"; - -package grpc.testing; - -message Empty {} - -// The type of payload that should be returned. -enum PayloadType { - // Compressable text format. - COMPRESSABLE = 0; - - // Uncompressable binary format. - UNCOMPRESSABLE = 1; - - // Randomly chosen from all other formats defined in this enum. - RANDOM = 2; -} - -// A block of data, to simply increase gRPC message size. -message Payload { - // The type of data in body. - PayloadType type = 1; - // Primary contents of payload. - bytes body = 2; -} - -// Unary request. -message SimpleRequest { - // Desired payload type in the response from the server. - // If response_type is RANDOM, server randomly chooses one from other formats. - PayloadType response_type = 1; - - // Desired payload size in the response from the server. - // If response_type is COMPRESSABLE, this denotes the size before compression. - int32 response_size = 2; - - // Optional input payload sent along with the request. - Payload payload = 3; - - // Whether SimpleResponse should include username. - bool fill_username = 4; - - // Whether SimpleResponse should include OAuth scope. - bool fill_oauth_scope = 5; -} - -// Unary response, as configured by the request. -message SimpleResponse { - // Payload to increase message size. - Payload payload = 1; - - // The user the request came from, for verifying authentication was - // successful when the client expected it. - string username = 2; - - // OAuth scope. - string oauth_scope = 3; -} - -// Client-streaming request. -message StreamingInputCallRequest { - // Optional input payload sent along with the request. - Payload payload = 1; - - // Not expecting any payload from the response. -} - -// Client-streaming response. -message StreamingInputCallResponse { - // Aggregated size of payloads received from the client. - int32 aggregated_payload_size = 1; -} - -// Configuration for a particular response. -message ResponseParameters { - // Desired payload sizes in responses from the server. - // If response_type is COMPRESSABLE, this denotes the size before compression. - int32 size = 1; - - // Desired interval between consecutive responses in the response stream in - // microseconds. - int32 interval_us = 2; -} - -// Server-streaming request. -message StreamingOutputCallRequest { - // Desired payload type in the response from the server. - // If response_type is RANDOM, the payload from each response in the stream - // might be of different types. This is to simulate a mixed type of payload - // stream. - PayloadType response_type = 1; - - // Configuration for each expected response message. - repeated ResponseParameters response_parameters = 2; - - // Optional input payload sent along with the request. - Payload payload = 3; -} - -// Server-streaming response, as configured by the request and parameters. -message StreamingOutputCallResponse { - // Payload to increase response size. - Payload payload = 1; -} - -// A simple service to test the various types of RPCs and experiment with -// performance with various types of payload. -service TestService { - // One empty request followed by one empty response. - rpc EmptyCall(Empty) returns (Empty); - - // One request followed by one response. - // The server returns the client payload as-is. - rpc UnaryCall(SimpleRequest) returns (SimpleResponse); - - // One request followed by a sequence of responses (streamed download). - // The server returns the payload with client desired type and sizes. - rpc StreamingOutputCall(StreamingOutputCallRequest) - returns (stream StreamingOutputCallResponse); - - // A sequence of requests followed by one response (streamed upload). - // The server returns the aggregated size of client payload as the result. - rpc StreamingInputCall(stream StreamingInputCallRequest) - returns (StreamingInputCallResponse); - - // A sequence of requests with each request served by the server immediately. - // As one request could lead to multiple responses, this interface - // demonstrates the idea of full duplexing. - rpc FullDuplexCall(stream StreamingOutputCallRequest) - returns (stream StreamingOutputCallResponse); - - // A sequence of requests followed by a sequence of responses. - // The server buffers all the client requests and then serves them in order. A - // stream of responses are returned to the client when the server starts with - // first request. - rpc HalfDuplexCall(stream StreamingOutputCallRequest) - returns (stream StreamingOutputCallResponse); -} diff --git a/test/grpc_testing/test_grpc.pb.go b/test/grpc_testing/test_grpc.pb.go deleted file mode 100644 index 2ab12ac94210..000000000000 --- a/test/grpc_testing/test_grpc.pb.go +++ /dev/null @@ -1,511 +0,0 @@ -// Code generated by protoc-gen-go-grpc. DO NOT EDIT. - -package grpc_testing - -import ( - context "context" - grpc "google.golang.org/grpc" - codes "google.golang.org/grpc/codes" - status "google.golang.org/grpc/status" -) - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the grpc package it is being compiled against. -const _ = grpc.SupportPackageIsVersion7 - -// TestServiceClient is the client API for TestService service. -// -// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. -type TestServiceClient interface { - // One empty request followed by one empty response. - EmptyCall(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error) - // One request followed by one response. - // The server returns the client payload as-is. - UnaryCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (*SimpleResponse, error) - // One request followed by a sequence of responses (streamed download). - // The server returns the payload with client desired type and sizes. - StreamingOutputCall(ctx context.Context, in *StreamingOutputCallRequest, opts ...grpc.CallOption) (TestService_StreamingOutputCallClient, error) - // A sequence of requests followed by one response (streamed upload). - // The server returns the aggregated size of client payload as the result. - StreamingInputCall(ctx context.Context, opts ...grpc.CallOption) (TestService_StreamingInputCallClient, error) - // A sequence of requests with each request served by the server immediately. - // As one request could lead to multiple responses, this interface - // demonstrates the idea of full duplexing. - FullDuplexCall(ctx context.Context, opts ...grpc.CallOption) (TestService_FullDuplexCallClient, error) - // A sequence of requests followed by a sequence of responses. - // The server buffers all the client requests and then serves them in order. A - // stream of responses are returned to the client when the server starts with - // first request. - HalfDuplexCall(ctx context.Context, opts ...grpc.CallOption) (TestService_HalfDuplexCallClient, error) -} - -type testServiceClient struct { - cc grpc.ClientConnInterface -} - -func NewTestServiceClient(cc grpc.ClientConnInterface) TestServiceClient { - return &testServiceClient{cc} -} - -var testServiceEmptyCallStreamDesc = &grpc.StreamDesc{ - StreamName: "EmptyCall", -} - -func (c *testServiceClient) EmptyCall(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*Empty, error) { - out := new(Empty) - err := c.cc.Invoke(ctx, "/grpc.testing.TestService/EmptyCall", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -var testServiceUnaryCallStreamDesc = &grpc.StreamDesc{ - StreamName: "UnaryCall", -} - -func (c *testServiceClient) UnaryCall(ctx context.Context, in *SimpleRequest, opts ...grpc.CallOption) (*SimpleResponse, error) { - out := new(SimpleResponse) - err := c.cc.Invoke(ctx, "/grpc.testing.TestService/UnaryCall", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -var testServiceStreamingOutputCallStreamDesc = &grpc.StreamDesc{ - StreamName: "StreamingOutputCall", - ServerStreams: true, -} - -func (c *testServiceClient) StreamingOutputCall(ctx context.Context, in *StreamingOutputCallRequest, opts ...grpc.CallOption) (TestService_StreamingOutputCallClient, error) { - stream, err := c.cc.NewStream(ctx, testServiceStreamingOutputCallStreamDesc, "/grpc.testing.TestService/StreamingOutputCall", opts...) - if err != nil { - return nil, err - } - x := &testServiceStreamingOutputCallClient{stream} - if err := x.ClientStream.SendMsg(in); err != nil { - return nil, err - } - if err := x.ClientStream.CloseSend(); err != nil { - return nil, err - } - return x, nil -} - -type TestService_StreamingOutputCallClient interface { - Recv() (*StreamingOutputCallResponse, error) - grpc.ClientStream -} - -type testServiceStreamingOutputCallClient struct { - grpc.ClientStream -} - -func (x *testServiceStreamingOutputCallClient) Recv() (*StreamingOutputCallResponse, error) { - m := new(StreamingOutputCallResponse) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -var testServiceStreamingInputCallStreamDesc = &grpc.StreamDesc{ - StreamName: "StreamingInputCall", - ClientStreams: true, -} - -func (c *testServiceClient) StreamingInputCall(ctx context.Context, opts ...grpc.CallOption) (TestService_StreamingInputCallClient, error) { - stream, err := c.cc.NewStream(ctx, testServiceStreamingInputCallStreamDesc, "/grpc.testing.TestService/StreamingInputCall", opts...) - if err != nil { - return nil, err - } - x := &testServiceStreamingInputCallClient{stream} - return x, nil -} - -type TestService_StreamingInputCallClient interface { - Send(*StreamingInputCallRequest) error - CloseAndRecv() (*StreamingInputCallResponse, error) - grpc.ClientStream -} - -type testServiceStreamingInputCallClient struct { - grpc.ClientStream -} - -func (x *testServiceStreamingInputCallClient) Send(m *StreamingInputCallRequest) error { - return x.ClientStream.SendMsg(m) -} - -func (x *testServiceStreamingInputCallClient) CloseAndRecv() (*StreamingInputCallResponse, error) { - if err := x.ClientStream.CloseSend(); err != nil { - return nil, err - } - m := new(StreamingInputCallResponse) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -var testServiceFullDuplexCallStreamDesc = &grpc.StreamDesc{ - StreamName: "FullDuplexCall", - ServerStreams: true, - ClientStreams: true, -} - -func (c *testServiceClient) FullDuplexCall(ctx context.Context, opts ...grpc.CallOption) (TestService_FullDuplexCallClient, error) { - stream, err := c.cc.NewStream(ctx, testServiceFullDuplexCallStreamDesc, "/grpc.testing.TestService/FullDuplexCall", opts...) - if err != nil { - return nil, err - } - x := &testServiceFullDuplexCallClient{stream} - return x, nil -} - -type TestService_FullDuplexCallClient interface { - Send(*StreamingOutputCallRequest) error - Recv() (*StreamingOutputCallResponse, error) - grpc.ClientStream -} - -type testServiceFullDuplexCallClient struct { - grpc.ClientStream -} - -func (x *testServiceFullDuplexCallClient) Send(m *StreamingOutputCallRequest) error { - return x.ClientStream.SendMsg(m) -} - -func (x *testServiceFullDuplexCallClient) Recv() (*StreamingOutputCallResponse, error) { - m := new(StreamingOutputCallResponse) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -var testServiceHalfDuplexCallStreamDesc = &grpc.StreamDesc{ - StreamName: "HalfDuplexCall", - ServerStreams: true, - ClientStreams: true, -} - -func (c *testServiceClient) HalfDuplexCall(ctx context.Context, opts ...grpc.CallOption) (TestService_HalfDuplexCallClient, error) { - stream, err := c.cc.NewStream(ctx, testServiceHalfDuplexCallStreamDesc, "/grpc.testing.TestService/HalfDuplexCall", opts...) - if err != nil { - return nil, err - } - x := &testServiceHalfDuplexCallClient{stream} - return x, nil -} - -type TestService_HalfDuplexCallClient interface { - Send(*StreamingOutputCallRequest) error - Recv() (*StreamingOutputCallResponse, error) - grpc.ClientStream -} - -type testServiceHalfDuplexCallClient struct { - grpc.ClientStream -} - -func (x *testServiceHalfDuplexCallClient) Send(m *StreamingOutputCallRequest) error { - return x.ClientStream.SendMsg(m) -} - -func (x *testServiceHalfDuplexCallClient) Recv() (*StreamingOutputCallResponse, error) { - m := new(StreamingOutputCallResponse) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -// TestServiceService is the service API for TestService service. -// Fields should be assigned to their respective handler implementations only before -// RegisterTestServiceService is called. Any unassigned fields will result in the -// handler for that method returning an Unimplemented error. -type TestServiceService struct { - // One empty request followed by one empty response. - EmptyCall func(context.Context, *Empty) (*Empty, error) - // One request followed by one response. - // The server returns the client payload as-is. - UnaryCall func(context.Context, *SimpleRequest) (*SimpleResponse, error) - // One request followed by a sequence of responses (streamed download). - // The server returns the payload with client desired type and sizes. - StreamingOutputCall func(*StreamingOutputCallRequest, TestService_StreamingOutputCallServer) error - // A sequence of requests followed by one response (streamed upload). - // The server returns the aggregated size of client payload as the result. - StreamingInputCall func(TestService_StreamingInputCallServer) error - // A sequence of requests with each request served by the server immediately. - // As one request could lead to multiple responses, this interface - // demonstrates the idea of full duplexing. - FullDuplexCall func(TestService_FullDuplexCallServer) error - // A sequence of requests followed by a sequence of responses. - // The server buffers all the client requests and then serves them in order. A - // stream of responses are returned to the client when the server starts with - // first request. - HalfDuplexCall func(TestService_HalfDuplexCallServer) error -} - -func (s *TestServiceService) emptyCall(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - if s.EmptyCall == nil { - return nil, status.Errorf(codes.Unimplemented, "method EmptyCall not implemented") - } - in := new(Empty) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return s.EmptyCall(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: s, - FullMethod: "/grpc.testing.TestService/EmptyCall", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return s.EmptyCall(ctx, req.(*Empty)) - } - return interceptor(ctx, in, info, handler) -} -func (s *TestServiceService) unaryCall(_ interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - if s.UnaryCall == nil { - return nil, status.Errorf(codes.Unimplemented, "method UnaryCall not implemented") - } - in := new(SimpleRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return s.UnaryCall(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: s, - FullMethod: "/grpc.testing.TestService/UnaryCall", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return s.UnaryCall(ctx, req.(*SimpleRequest)) - } - return interceptor(ctx, in, info, handler) -} -func (s *TestServiceService) streamingOutputCall(_ interface{}, stream grpc.ServerStream) error { - if s.StreamingOutputCall == nil { - return status.Errorf(codes.Unimplemented, "method StreamingOutputCall not implemented") - } - m := new(StreamingOutputCallRequest) - if err := stream.RecvMsg(m); err != nil { - return err - } - return s.StreamingOutputCall(m, &testServiceStreamingOutputCallServer{stream}) -} -func (s *TestServiceService) streamingInputCall(_ interface{}, stream grpc.ServerStream) error { - if s.StreamingInputCall == nil { - return status.Errorf(codes.Unimplemented, "method StreamingInputCall not implemented") - } - return s.StreamingInputCall(&testServiceStreamingInputCallServer{stream}) -} -func (s *TestServiceService) fullDuplexCall(_ interface{}, stream grpc.ServerStream) error { - if s.FullDuplexCall == nil { - return status.Errorf(codes.Unimplemented, "method FullDuplexCall not implemented") - } - return s.FullDuplexCall(&testServiceFullDuplexCallServer{stream}) -} -func (s *TestServiceService) halfDuplexCall(_ interface{}, stream grpc.ServerStream) error { - if s.HalfDuplexCall == nil { - return status.Errorf(codes.Unimplemented, "method HalfDuplexCall not implemented") - } - return s.HalfDuplexCall(&testServiceHalfDuplexCallServer{stream}) -} - -type TestService_StreamingOutputCallServer interface { - Send(*StreamingOutputCallResponse) error - grpc.ServerStream -} - -type testServiceStreamingOutputCallServer struct { - grpc.ServerStream -} - -func (x *testServiceStreamingOutputCallServer) Send(m *StreamingOutputCallResponse) error { - return x.ServerStream.SendMsg(m) -} - -type TestService_StreamingInputCallServer interface { - SendAndClose(*StreamingInputCallResponse) error - Recv() (*StreamingInputCallRequest, error) - grpc.ServerStream -} - -type testServiceStreamingInputCallServer struct { - grpc.ServerStream -} - -func (x *testServiceStreamingInputCallServer) SendAndClose(m *StreamingInputCallResponse) error { - return x.ServerStream.SendMsg(m) -} - -func (x *testServiceStreamingInputCallServer) Recv() (*StreamingInputCallRequest, error) { - m := new(StreamingInputCallRequest) - if err := x.ServerStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -type TestService_FullDuplexCallServer interface { - Send(*StreamingOutputCallResponse) error - Recv() (*StreamingOutputCallRequest, error) - grpc.ServerStream -} - -type testServiceFullDuplexCallServer struct { - grpc.ServerStream -} - -func (x *testServiceFullDuplexCallServer) Send(m *StreamingOutputCallResponse) error { - return x.ServerStream.SendMsg(m) -} - -func (x *testServiceFullDuplexCallServer) Recv() (*StreamingOutputCallRequest, error) { - m := new(StreamingOutputCallRequest) - if err := x.ServerStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -type TestService_HalfDuplexCallServer interface { - Send(*StreamingOutputCallResponse) error - Recv() (*StreamingOutputCallRequest, error) - grpc.ServerStream -} - -type testServiceHalfDuplexCallServer struct { - grpc.ServerStream -} - -func (x *testServiceHalfDuplexCallServer) Send(m *StreamingOutputCallResponse) error { - return x.ServerStream.SendMsg(m) -} - -func (x *testServiceHalfDuplexCallServer) Recv() (*StreamingOutputCallRequest, error) { - m := new(StreamingOutputCallRequest) - if err := x.ServerStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -// RegisterTestServiceService registers a service implementation with a gRPC server. -func RegisterTestServiceService(s grpc.ServiceRegistrar, srv *TestServiceService) { - sd := grpc.ServiceDesc{ - ServiceName: "grpc.testing.TestService", - Methods: []grpc.MethodDesc{ - { - MethodName: "EmptyCall", - Handler: srv.emptyCall, - }, - { - MethodName: "UnaryCall", - Handler: srv.unaryCall, - }, - }, - Streams: []grpc.StreamDesc{ - { - StreamName: "StreamingOutputCall", - Handler: srv.streamingOutputCall, - ServerStreams: true, - }, - { - StreamName: "StreamingInputCall", - Handler: srv.streamingInputCall, - ClientStreams: true, - }, - { - StreamName: "FullDuplexCall", - Handler: srv.fullDuplexCall, - ServerStreams: true, - ClientStreams: true, - }, - { - StreamName: "HalfDuplexCall", - Handler: srv.halfDuplexCall, - ServerStreams: true, - ClientStreams: true, - }, - }, - Metadata: "test/grpc_testing/test.proto", - } - - s.RegisterService(&sd, nil) -} - -// NewTestServiceService creates a new TestServiceService containing the -// implemented methods of the TestService service in s. Any unimplemented -// methods will result in the gRPC server returning an UNIMPLEMENTED status to the client. -// This includes situations where the method handler is misspelled or has the wrong -// signature. For this reason, this function should be used with great care and -// is not recommended to be used by most users. -func NewTestServiceService(s interface{}) *TestServiceService { - ns := &TestServiceService{} - if h, ok := s.(interface { - EmptyCall(context.Context, *Empty) (*Empty, error) - }); ok { - ns.EmptyCall = h.EmptyCall - } - if h, ok := s.(interface { - UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error) - }); ok { - ns.UnaryCall = h.UnaryCall - } - if h, ok := s.(interface { - StreamingOutputCall(*StreamingOutputCallRequest, TestService_StreamingOutputCallServer) error - }); ok { - ns.StreamingOutputCall = h.StreamingOutputCall - } - if h, ok := s.(interface { - StreamingInputCall(TestService_StreamingInputCallServer) error - }); ok { - ns.StreamingInputCall = h.StreamingInputCall - } - if h, ok := s.(interface { - FullDuplexCall(TestService_FullDuplexCallServer) error - }); ok { - ns.FullDuplexCall = h.FullDuplexCall - } - if h, ok := s.(interface { - HalfDuplexCall(TestService_HalfDuplexCallServer) error - }); ok { - ns.HalfDuplexCall = h.HalfDuplexCall - } - return ns -} - -// UnstableTestServiceService is the service API for TestService service. -// New methods may be added to this interface if they are added to the service -// definition, which is not a backward-compatible change. For this reason, -// use of this type is not recommended. -type UnstableTestServiceService interface { - // One empty request followed by one empty response. - EmptyCall(context.Context, *Empty) (*Empty, error) - // One request followed by one response. - // The server returns the client payload as-is. - UnaryCall(context.Context, *SimpleRequest) (*SimpleResponse, error) - // One request followed by a sequence of responses (streamed download). - // The server returns the payload with client desired type and sizes. - StreamingOutputCall(*StreamingOutputCallRequest, TestService_StreamingOutputCallServer) error - // A sequence of requests followed by one response (streamed upload). - // The server returns the aggregated size of client payload as the result. - StreamingInputCall(TestService_StreamingInputCallServer) error - // A sequence of requests with each request served by the server immediately. - // As one request could lead to multiple responses, this interface - // demonstrates the idea of full duplexing. - FullDuplexCall(TestService_FullDuplexCallServer) error - // A sequence of requests followed by a sequence of responses. - // The server buffers all the client requests and then serves them in order. A - // stream of responses are returned to the client when the server starts with - // first request. - HalfDuplexCall(TestService_HalfDuplexCallServer) error -} diff --git a/test/healthcheck_test.go b/test/healthcheck_test.go index 0137494efb8f..2b5b5a82d93c 100644 --- a/test/healthcheck_test.go +++ b/test/healthcheck_test.go @@ -30,16 +30,20 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/connectivity" - _ "google.golang.org/grpc/health" - healthgrpc "google.golang.org/grpc/health/grpc_health_v1" - healthpb "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/health" "google.golang.org/grpc/internal" "google.golang.org/grpc/internal/channelz" "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/testutils" "google.golang.org/grpc/resolver" "google.golang.org/grpc/resolver/manual" "google.golang.org/grpc/status" - testpb "google.golang.org/grpc/test/grpc_testing" + + healthgrpc "google.golang.org/grpc/health/grpc_health_v1" + healthpb "google.golang.org/grpc/health/grpc_health_v1" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" ) var testHealthCheckFunc = internal.HealthCheckFunc @@ -48,7 +52,7 @@ func newTestHealthServer() *testHealthServer { return newTestHealthServerWithWatchFunc(defaultWatchFunc) } -func newTestHealthServerWithWatchFunc(f func(s *testHealthServer, in *healthpb.HealthCheckRequest, stream healthgrpc.Health_WatchServer) error) *testHealthServer { +func newTestHealthServerWithWatchFunc(f healthWatchFunc) *testHealthServer { return &testHealthServer{ watchFunc: f, update: make(chan struct{}, 1), @@ -82,9 +86,11 @@ func defaultWatchFunc(s *testHealthServer, in *healthpb.HealthCheckRequest, stre return nil } +type healthWatchFunc func(s *testHealthServer, in *healthpb.HealthCheckRequest, stream healthgrpc.Health_WatchServer) error + type testHealthServer struct { - healthpb.UnimplementedHealthServer - watchFunc func(s *testHealthServer, in *healthpb.HealthCheckRequest, stream healthgrpc.Health_WatchServer) error + healthgrpc.UnimplementedHealthServer + watchFunc healthWatchFunc mu sync.Mutex status map[string]healthpb.HealthCheckResponse_ServingStatus update chan struct{} @@ -116,7 +122,7 @@ func (s *testHealthServer) SetServingStatus(service string, status healthpb.Heal func setupHealthCheckWrapper() (hcEnterChan chan struct{}, hcExitChan chan struct{}, wrapper internal.HealthChecker) { hcEnterChan = make(chan struct{}) hcExitChan = make(chan struct{}) - wrapper = func(ctx context.Context, newStream func(string) (interface{}, error), update func(connectivity.State, error), service string) error { + wrapper = func(ctx context.Context, newStream func(string) (any, error), update func(connectivity.State, error), service string) error { close(hcEnterChan) defer close(hcExitChan) return testHealthCheckFunc(ctx, newStream, update, service) @@ -124,25 +130,26 @@ func setupHealthCheckWrapper() (hcEnterChan chan struct{}, hcExitChan chan struc return } -type svrConfig struct { - specialWatchFunc func(s *testHealthServer, in *healthpb.HealthCheckRequest, stream healthgrpc.Health_WatchServer) error -} +func setupServer(t *testing.T, watchFunc healthWatchFunc) (*grpc.Server, net.Listener, *testHealthServer) { + t.Helper() -func setupServer(sc *svrConfig) (s *grpc.Server, lis net.Listener, ts *testHealthServer, deferFunc func(), err error) { - s = grpc.NewServer() - lis, err = net.Listen("tcp", "localhost:0") + lis, err := net.Listen("tcp", "localhost:0") if err != nil { - return nil, nil, nil, func() {}, fmt.Errorf("failed to listen due to err %v", err) + t.Fatalf("net.Listen() failed: %v", err) } - if sc.specialWatchFunc != nil { - ts = newTestHealthServerWithWatchFunc(sc.specialWatchFunc) + + var ts *testHealthServer + if watchFunc != nil { + ts = newTestHealthServerWithWatchFunc(watchFunc) } else { ts = newTestHealthServer() } + s := grpc.NewServer() healthgrpc.RegisterHealthServer(s, ts) - testpb.RegisterTestServiceService(s, testpb.NewTestServiceService(&testServer{})) + testgrpc.RegisterTestServiceServer(s, &testServer{}) go s.Serve(lis) - return s, lis, ts, s.Stop, nil + t.Cleanup(func() { s.Stop() }) + return s, lis, ts } type clientConfig struct { @@ -151,28 +158,34 @@ type clientConfig struct { extraDialOption []grpc.DialOption } -func setupClient(c *clientConfig) (cc *grpc.ClientConn, r *manual.Resolver, deferFunc func(), err error) { - r = manual.NewBuilderWithScheme("whatever") - var opts []grpc.DialOption - opts = append(opts, grpc.WithInsecure(), grpc.WithResolvers(r), grpc.WithBalancerName(c.balancerName)) - if c.testHealthCheckFuncWrapper != nil { - opts = append(opts, internal.WithHealthCheckFunc.(func(internal.HealthChecker) grpc.DialOption)(c.testHealthCheckFuncWrapper)) +func setupClient(t *testing.T, c *clientConfig) (*grpc.ClientConn, *manual.Resolver) { + t.Helper() + + r := manual.NewBuilderWithScheme("whatever") + opts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(r), + } + if c != nil { + if c.balancerName != "" { + opts = append(opts, grpc.WithDefaultServiceConfig(fmt.Sprintf(`{"loadBalancingConfig": [{"%s":{}}]}`, c.balancerName))) + } + if c.testHealthCheckFuncWrapper != nil { + opts = append(opts, internal.WithHealthCheckFunc.(func(internal.HealthChecker) grpc.DialOption)(c.testHealthCheckFuncWrapper)) + } + opts = append(opts, c.extraDialOption...) } - opts = append(opts, c.extraDialOption...) - cc, err = grpc.Dial(r.Scheme()+":///test.server", opts...) - if err != nil { - return nil, nil, nil, fmt.Errorf("dial failed due to err: %v", err) + cc, err := grpc.Dial(r.Scheme()+":///test.server", opts...) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) } - return cc, r, func() { cc.Close() }, nil + t.Cleanup(func() { cc.Close() }) + return cc, r } func (s) TestHealthCheckWatchStateChange(t *testing.T) { - _, lis, ts, deferFunc, err := setupServer(&svrConfig{}) - defer deferFunc() - if err != nil { - t.Fatal(err) - } + _, lis, ts := setupServer(t, nil) // The table below shows the expected series of addrConn connectivity transitions when server // updates its health status. As there's only one addrConn corresponds with the ClientConn in this @@ -188,59 +201,45 @@ func (s) TestHealthCheckWatchStateChange(t *testing.T) { //+------------------------------+-------------------------------------------+ ts.SetServingStatus("foo", healthpb.HealthCheckResponse_NOT_SERVING) - cc, r, deferFunc, err := setupClient(&clientConfig{balancerName: "round_robin"}) - if err != nil { - t.Fatal(err) - } - defer deferFunc() - + cc, r := setupClient(t, nil) r.UpdateState(resolver.State{ Addresses: []resolver.Address{{Addr: lis.Addr().String()}}, - ServiceConfig: parseCfg(r, `{ + ServiceConfig: parseServiceConfig(t, r, `{ "healthCheckConfig": { "serviceName": "foo" - } + }, + "loadBalancingConfig": [{"round_robin":{}}] }`)}) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() - if ok := cc.WaitForStateChange(ctx, connectivity.Idle); !ok { - t.Fatal("ClientConn is still in IDLE state when the context times out.") - } - if ok := cc.WaitForStateChange(ctx, connectivity.Connecting); !ok { - t.Fatal("ClientConn is still in CONNECTING state when the context times out.") - } + testutils.AwaitNotState(ctx, t, cc, connectivity.Idle) + testutils.AwaitNotState(ctx, t, cc, connectivity.Connecting) + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) if s := cc.GetState(); s != connectivity.TransientFailure { t.Fatalf("ClientConn is in %v state, want TRANSIENT FAILURE", s) } ts.SetServingStatus("foo", healthpb.HealthCheckResponse_SERVING) - if ok := cc.WaitForStateChange(ctx, connectivity.TransientFailure); !ok { - t.Fatal("ClientConn is still in TRANSIENT FAILURE state when the context times out.") - } + testutils.AwaitNotState(ctx, t, cc, connectivity.TransientFailure) if s := cc.GetState(); s != connectivity.Ready { t.Fatalf("ClientConn is in %v state, want READY", s) } ts.SetServingStatus("foo", healthpb.HealthCheckResponse_SERVICE_UNKNOWN) - if ok := cc.WaitForStateChange(ctx, connectivity.Ready); !ok { - t.Fatal("ClientConn is still in READY state when the context times out.") - } + testutils.AwaitNotState(ctx, t, cc, connectivity.Ready) if s := cc.GetState(); s != connectivity.TransientFailure { t.Fatalf("ClientConn is in %v state, want TRANSIENT FAILURE", s) } ts.SetServingStatus("foo", healthpb.HealthCheckResponse_SERVING) - if ok := cc.WaitForStateChange(ctx, connectivity.TransientFailure); !ok { - t.Fatal("ClientConn is still in TRANSIENT FAILURE state when the context times out.") - } + testutils.AwaitNotState(ctx, t, cc, connectivity.TransientFailure) if s := cc.GetState(); s != connectivity.Ready { t.Fatalf("ClientConn is in %v state, want READY", s) } ts.SetServingStatus("foo", healthpb.HealthCheckResponse_UNKNOWN) - if ok := cc.WaitForStateChange(ctx, connectivity.Ready); !ok { - t.Fatal("ClientConn is still in READY state when the context times out.") - } + testutils.AwaitNotState(ctx, t, cc, connectivity.Ready) if s := cc.GetState(); s != connectivity.TransientFailure { t.Fatalf("ClientConn is in %v state, want TRANSIENT FAILURE", s) } @@ -257,28 +256,20 @@ func (s) TestHealthCheckHealthServerNotRegistered(t *testing.T) { go s.Serve(lis) defer s.Stop() - cc, r, deferFunc, err := setupClient(&clientConfig{balancerName: "round_robin"}) - if err != nil { - t.Fatal(err) - } - defer deferFunc() - + cc, r := setupClient(t, nil) r.UpdateState(resolver.State{ Addresses: []resolver.Address{{Addr: lis.Addr().String()}}, - ServiceConfig: parseCfg(r, `{ + ServiceConfig: parseServiceConfig(t, r, `{ "healthCheckConfig": { "serviceName": "foo" - } + }, + "loadBalancingConfig": [{"round_robin":{}}] }`)}) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - if ok := cc.WaitForStateChange(ctx, connectivity.Idle); !ok { - t.Fatal("ClientConn is still in IDLE state when the context times out.") - } - if ok := cc.WaitForStateChange(ctx, connectivity.Connecting); !ok { - t.Fatal("ClientConn is still in CONNECTING state when the context times out.") - } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testutils.AwaitNotState(ctx, t, cc, connectivity.Idle) + testutils.AwaitNotState(ctx, t, cc, connectivity.Connecting) if s := cc.GetState(); s != connectivity.Ready { t.Fatalf("ClientConn is in %v state, want READY", s) } @@ -287,37 +278,26 @@ func (s) TestHealthCheckHealthServerNotRegistered(t *testing.T) { // In the case of a goaway received, the health check stream should be terminated and health check // function should exit. func (s) TestHealthCheckWithGoAway(t *testing.T) { - hcEnterChan, hcExitChan, testHealthCheckFuncWrapper := setupHealthCheckWrapper() - - s, lis, ts, deferFunc, err := setupServer(&svrConfig{}) - defer deferFunc() - if err != nil { - t.Fatal(err) - } - + s, lis, ts := setupServer(t, nil) ts.SetServingStatus("foo", healthpb.HealthCheckResponse_SERVING) - cc, r, deferFunc, err := setupClient(&clientConfig{ - balancerName: "round_robin", - testHealthCheckFuncWrapper: testHealthCheckFuncWrapper, - }) - if err != nil { - t.Fatal(err) - } - defer deferFunc() - - tc := testpb.NewTestServiceClient(cc) + hcEnterChan, hcExitChan, testHealthCheckFuncWrapper := setupHealthCheckWrapper() + cc, r := setupClient(t, &clientConfig{testHealthCheckFuncWrapper: testHealthCheckFuncWrapper}) + tc := testgrpc.NewTestServiceClient(cc) r.UpdateState(resolver.State{ Addresses: []resolver.Address{{Addr: lis.Addr().String()}}, - ServiceConfig: parseCfg(r, `{ + ServiceConfig: parseServiceConfig(t, r, `{ "healthCheckConfig": { "serviceName": "foo" - } + }, + "loadBalancingConfig": [{"round_robin":{}}] }`)}) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() // make some rpcs to make sure connection is working. if err := verifyResultWithDelay(func() (bool, error) { - if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); err != nil { + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil { return false, fmt.Errorf("TestService/EmptyCall(_, _) = _, %v, want _, ", err) } return true, nil @@ -326,8 +306,6 @@ func (s) TestHealthCheckWithGoAway(t *testing.T) { } // the stream rpc will persist through goaway event. - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() stream, err := tc.FullDuplexCall(ctx, grpc.WaitForReady(true)) if err != nil { t.Fatalf("%v.FullDuplexCall(_) = _, %v, want ", tc, err) @@ -378,38 +356,26 @@ func (s) TestHealthCheckWithGoAway(t *testing.T) { } func (s) TestHealthCheckWithConnClose(t *testing.T) { - hcEnterChan, hcExitChan, testHealthCheckFuncWrapper := setupHealthCheckWrapper() - - s, lis, ts, deferFunc, err := setupServer(&svrConfig{}) - defer deferFunc() - if err != nil { - t.Fatal(err) - } - + s, lis, ts := setupServer(t, nil) ts.SetServingStatus("foo", healthpb.HealthCheckResponse_SERVING) - cc, r, deferFunc, err := setupClient(&clientConfig{ - balancerName: "round_robin", - testHealthCheckFuncWrapper: testHealthCheckFuncWrapper, - }) - if err != nil { - t.Fatal(err) - } - defer deferFunc() - - tc := testpb.NewTestServiceClient(cc) - + hcEnterChan, hcExitChan, testHealthCheckFuncWrapper := setupHealthCheckWrapper() + cc, r := setupClient(t, &clientConfig{testHealthCheckFuncWrapper: testHealthCheckFuncWrapper}) + tc := testgrpc.NewTestServiceClient(cc) r.UpdateState(resolver.State{ Addresses: []resolver.Address{{Addr: lis.Addr().String()}}, - ServiceConfig: parseCfg(r, `{ + ServiceConfig: parseServiceConfig(t, r, `{ "healthCheckConfig": { "serviceName": "foo" - } + }, + "loadBalancingConfig": [{"round_robin":{}}] }`)}) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() // make some rpcs to make sure connection is working. if err := verifyResultWithDelay(func() (bool, error) { - if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); err != nil { + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil { return false, fmt.Errorf("TestService/EmptyCall(_, _) = _, %v, want _, ", err) } return true, nil @@ -440,39 +406,28 @@ func (s) TestHealthCheckWithConnClose(t *testing.T) { // addrConn drain happens when addrConn gets torn down due to its address being no longer in the // address list returned by the resolver. func (s) TestHealthCheckWithAddrConnDrain(t *testing.T) { - hcEnterChan, hcExitChan, testHealthCheckFuncWrapper := setupHealthCheckWrapper() - - _, lis, ts, deferFunc, err := setupServer(&svrConfig{}) - defer deferFunc() - if err != nil { - t.Fatal(err) - } - + _, lis, ts := setupServer(t, nil) ts.SetServingStatus("foo", healthpb.HealthCheckResponse_SERVING) - cc, r, deferFunc, err := setupClient(&clientConfig{ - balancerName: "round_robin", - testHealthCheckFuncWrapper: testHealthCheckFuncWrapper, - }) - if err != nil { - t.Fatal(err) - } - defer deferFunc() - - tc := testpb.NewTestServiceClient(cc) - sc := parseCfg(r, `{ + hcEnterChan, hcExitChan, testHealthCheckFuncWrapper := setupHealthCheckWrapper() + cc, r := setupClient(t, &clientConfig{testHealthCheckFuncWrapper: testHealthCheckFuncWrapper}) + tc := testgrpc.NewTestServiceClient(cc) + sc := parseServiceConfig(t, r, `{ "healthCheckConfig": { "serviceName": "foo" - } + }, + "loadBalancingConfig": [{"round_robin":{}}] }`) r.UpdateState(resolver.State{ Addresses: []resolver.Address{{Addr: lis.Addr().String()}}, ServiceConfig: sc, }) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() // make some rpcs to make sure connection is working. if err := verifyResultWithDelay(func() (bool, error) { - if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); err != nil { + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil { return false, fmt.Errorf("TestService/EmptyCall(_, _) = _, %v, want _, ", err) } return true, nil @@ -481,8 +436,6 @@ func (s) TestHealthCheckWithAddrConnDrain(t *testing.T) { } // the stream rpc will persist through goaway event. - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() stream, err := tc.FullDuplexCall(ctx, grpc.WaitForReady(true)) if err != nil { t.Fatalf("%v.FullDuplexCall(_) = _, %v, want ", tc, err) @@ -533,37 +486,26 @@ func (s) TestHealthCheckWithAddrConnDrain(t *testing.T) { // ClientConn close will lead to its addrConns being torn down. func (s) TestHealthCheckWithClientConnClose(t *testing.T) { - hcEnterChan, hcExitChan, testHealthCheckFuncWrapper := setupHealthCheckWrapper() - - _, lis, ts, deferFunc, err := setupServer(&svrConfig{}) - defer deferFunc() - if err != nil { - t.Fatal(err) - } - + _, lis, ts := setupServer(t, nil) ts.SetServingStatus("foo", healthpb.HealthCheckResponse_SERVING) - cc, r, deferFunc, err := setupClient(&clientConfig{ - balancerName: "round_robin", - testHealthCheckFuncWrapper: testHealthCheckFuncWrapper, - }) - if err != nil { - t.Fatal(err) - } - defer deferFunc() - - tc := testpb.NewTestServiceClient(cc) + hcEnterChan, hcExitChan, testHealthCheckFuncWrapper := setupHealthCheckWrapper() + cc, r := setupClient(t, &clientConfig{testHealthCheckFuncWrapper: testHealthCheckFuncWrapper}) + tc := testgrpc.NewTestServiceClient(cc) r.UpdateState(resolver.State{ Addresses: []resolver.Address{{Addr: lis.Addr().String()}}, - ServiceConfig: parseCfg(r, `{ + ServiceConfig: parseServiceConfig(t, r, `{ "healthCheckConfig": { "serviceName": "foo" - } + }, + "loadBalancingConfig": [{"round_robin":{}}] }`)}) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() // make some rpcs to make sure connection is working. if err := verifyResultWithDelay(func() (bool, error) { - if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); err != nil { + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil { return false, fmt.Errorf("TestService/EmptyCall(_, _) = _, %v, want _, ", err) } return true, nil @@ -596,47 +538,34 @@ func (s) TestHealthCheckWithClientConnClose(t *testing.T) { // closes the skipReset channel(since it has not been closed inside health check func) to unblock // onGoAway/onClose goroutine. func (s) TestHealthCheckWithoutSetConnectivityStateCalledAddrConnShutDown(t *testing.T) { - hcEnterChan, hcExitChan, testHealthCheckFuncWrapper := setupHealthCheckWrapper() - - _, lis, ts, deferFunc, err := setupServer(&svrConfig{ - specialWatchFunc: func(s *testHealthServer, in *healthpb.HealthCheckRequest, stream healthgrpc.Health_WatchServer) error { - if in.Service != "delay" { - return status.Error(codes.FailedPrecondition, - "this special Watch function only handles request with service name to be \"delay\"") - } - // Do nothing to mock a delay of health check response from server side. - // This case is to help with the test that covers the condition that setConnectivityState is not - // called inside HealthCheckFunc before the func returns. - select { - case <-stream.Context().Done(): - case <-time.After(5 * time.Second): - } - return nil - }, - }) - defer deferFunc() - if err != nil { - t.Fatal(err) + watchFunc := func(s *testHealthServer, in *healthpb.HealthCheckRequest, stream healthgrpc.Health_WatchServer) error { + if in.Service != "delay" { + return status.Error(codes.FailedPrecondition, + "this special Watch function only handles request with service name to be \"delay\"") + } + // Do nothing to mock a delay of health check response from server side. + // This case is to help with the test that covers the condition that setConnectivityState is not + // called inside HealthCheckFunc before the func returns. + select { + case <-stream.Context().Done(): + case <-time.After(5 * time.Second): + } + return nil } - + _, lis, ts := setupServer(t, watchFunc) ts.SetServingStatus("delay", healthpb.HealthCheckResponse_SERVING) - _, r, deferFunc, err := setupClient(&clientConfig{ - balancerName: "round_robin", - testHealthCheckFuncWrapper: testHealthCheckFuncWrapper, - }) - if err != nil { - t.Fatal(err) - } - defer deferFunc() + hcEnterChan, hcExitChan, testHealthCheckFuncWrapper := setupHealthCheckWrapper() + _, r := setupClient(t, &clientConfig{testHealthCheckFuncWrapper: testHealthCheckFuncWrapper}) // The serviceName "delay" is specially handled at server side, where response will not be sent // back to client immediately upon receiving the request (client should receive no response until // test ends). - sc := parseCfg(r, `{ + sc := parseServiceConfig(t, r, `{ "healthCheckConfig": { "serviceName": "delay" - } + }, + "loadBalancingConfig": [{"round_robin":{}}] }`) r.UpdateState(resolver.State{ Addresses: []resolver.Address{{Addr: lis.Addr().String()}}, @@ -672,49 +601,36 @@ func (s) TestHealthCheckWithoutSetConnectivityStateCalledAddrConnShutDown(t *tes // closes the allowedToReset channel(since it has not been closed inside health check func) to unblock // onGoAway/onClose goroutine. func (s) TestHealthCheckWithoutSetConnectivityStateCalled(t *testing.T) { - hcEnterChan, hcExitChan, testHealthCheckFuncWrapper := setupHealthCheckWrapper() - - s, lis, ts, deferFunc, err := setupServer(&svrConfig{ - specialWatchFunc: func(s *testHealthServer, in *healthpb.HealthCheckRequest, stream healthgrpc.Health_WatchServer) error { - if in.Service != "delay" { - return status.Error(codes.FailedPrecondition, - "this special Watch function only handles request with service name to be \"delay\"") - } - // Do nothing to mock a delay of health check response from server side. - // This case is to help with the test that covers the condition that setConnectivityState is not - // called inside HealthCheckFunc before the func returns. - select { - case <-stream.Context().Done(): - case <-time.After(5 * time.Second): - } - return nil - }, - }) - defer deferFunc() - if err != nil { - t.Fatal(err) + watchFunc := func(s *testHealthServer, in *healthpb.HealthCheckRequest, stream healthgrpc.Health_WatchServer) error { + if in.Service != "delay" { + return status.Error(codes.FailedPrecondition, + "this special Watch function only handles request with service name to be \"delay\"") + } + // Do nothing to mock a delay of health check response from server side. + // This case is to help with the test that covers the condition that setConnectivityState is not + // called inside HealthCheckFunc before the func returns. + select { + case <-stream.Context().Done(): + case <-time.After(5 * time.Second): + } + return nil } - + s, lis, ts := setupServer(t, watchFunc) ts.SetServingStatus("delay", healthpb.HealthCheckResponse_SERVING) - _, r, deferFunc, err := setupClient(&clientConfig{ - balancerName: "round_robin", - testHealthCheckFuncWrapper: testHealthCheckFuncWrapper, - }) - if err != nil { - t.Fatal(err) - } - defer deferFunc() + hcEnterChan, hcExitChan, testHealthCheckFuncWrapper := setupHealthCheckWrapper() + _, r := setupClient(t, &clientConfig{testHealthCheckFuncWrapper: testHealthCheckFuncWrapper}) // The serviceName "delay" is specially handled at server side, where response will not be sent // back to client immediately upon receiving the request (client should receive no response until // test ends). r.UpdateState(resolver.State{ Addresses: []resolver.Address{{Addr: lis.Addr().String()}}, - ServiceConfig: parseCfg(r, `{ + ServiceConfig: parseServiceConfig(t, r, `{ "healthCheckConfig": { "serviceName": "delay" - } + }, + "loadBalancingConfig": [{"round_robin":{}}] }`)}) select { @@ -744,30 +660,25 @@ func (s) TestHealthCheckWithoutSetConnectivityStateCalled(t *testing.T) { func testHealthCheckDisableWithDialOption(t *testing.T, addr string) { hcEnterChan, _, testHealthCheckFuncWrapper := setupHealthCheckWrapper() - - cc, r, deferFunc, err := setupClient(&clientConfig{ - balancerName: "round_robin", + cc, r := setupClient(t, &clientConfig{ testHealthCheckFuncWrapper: testHealthCheckFuncWrapper, extraDialOption: []grpc.DialOption{grpc.WithDisableHealthCheck()}, }) - if err != nil { - t.Fatal(err) - } - defer deferFunc() - - tc := testpb.NewTestServiceClient(cc) - + tc := testgrpc.NewTestServiceClient(cc) r.UpdateState(resolver.State{ Addresses: []resolver.Address{{Addr: addr}}, - ServiceConfig: parseCfg(r, `{ + ServiceConfig: parseServiceConfig(t, r, `{ "healthCheckConfig": { "serviceName": "foo" - } + }, + "loadBalancingConfig": [{"round_robin":{}}] }`)}) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() // send some rpcs to make sure transport has been created and is ready for use. if err := verifyResultWithDelay(func() (bool, error) { - if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); err != nil { + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil { return false, fmt.Errorf("TestService/EmptyCall(_, _) = _, %v, want _, ", err) } return true, nil @@ -784,29 +695,24 @@ func testHealthCheckDisableWithDialOption(t *testing.T, addr string) { func testHealthCheckDisableWithBalancer(t *testing.T, addr string) { hcEnterChan, _, testHealthCheckFuncWrapper := setupHealthCheckWrapper() - - cc, r, deferFunc, err := setupClient(&clientConfig{ - balancerName: "pick_first", + cc, r := setupClient(t, &clientConfig{ testHealthCheckFuncWrapper: testHealthCheckFuncWrapper, }) - if err != nil { - t.Fatal(err) - } - defer deferFunc() - - tc := testpb.NewTestServiceClient(cc) - + tc := testgrpc.NewTestServiceClient(cc) r.UpdateState(resolver.State{ Addresses: []resolver.Address{{Addr: addr}}, - ServiceConfig: parseCfg(r, `{ + ServiceConfig: parseServiceConfig(t, r, `{ "healthCheckConfig": { "serviceName": "foo" - } + }, + "loadBalancingConfig": [{"pick_first":{}}] }`)}) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() // send some rpcs to make sure transport has been created and is ready for use. if err := verifyResultWithDelay(func() (bool, error) { - if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); err != nil { + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil { return false, fmt.Errorf("TestService/EmptyCall(_, _) = _, %v, want _, ", err) } return true, nil @@ -823,23 +729,15 @@ func testHealthCheckDisableWithBalancer(t *testing.T, addr string) { func testHealthCheckDisableWithServiceConfig(t *testing.T, addr string) { hcEnterChan, _, testHealthCheckFuncWrapper := setupHealthCheckWrapper() - - cc, r, deferFunc, err := setupClient(&clientConfig{ - balancerName: "round_robin", - testHealthCheckFuncWrapper: testHealthCheckFuncWrapper, - }) - if err != nil { - t.Fatal(err) - } - defer deferFunc() - - tc := testpb.NewTestServiceClient(cc) - + cc, r := setupClient(t, &clientConfig{testHealthCheckFuncWrapper: testHealthCheckFuncWrapper}) + tc := testgrpc.NewTestServiceClient(cc) r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: addr}}}) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() // send some rpcs to make sure transport has been created and is ready for use. if err := verifyResultWithDelay(func() (bool, error) { - if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); err != nil { + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); err != nil { return false, fmt.Errorf("TestService/EmptyCall(_, _) = _, %v, want _, ", err) } return true, nil @@ -855,11 +753,7 @@ func testHealthCheckDisableWithServiceConfig(t *testing.T, addr string) { } func (s) TestHealthCheckDisable(t *testing.T) { - _, lis, ts, deferFunc, err := setupServer(&svrConfig{}) - defer deferFunc() - if err != nil { - t.Fatal(err) - } + _, lis, ts := setupServer(t, nil) ts.SetServingStatus("foo", healthpb.HealthCheckResponse_SERVING) // test client side disabling configuration. @@ -869,32 +763,23 @@ func (s) TestHealthCheckDisable(t *testing.T) { } func (s) TestHealthCheckChannelzCountingCallSuccess(t *testing.T) { - _, lis, _, deferFunc, err := setupServer(&svrConfig{ - specialWatchFunc: func(s *testHealthServer, in *healthpb.HealthCheckRequest, stream healthgrpc.Health_WatchServer) error { - if in.Service != "channelzSuccess" { - return status.Error(codes.FailedPrecondition, - "this special Watch function only handles request with service name to be \"channelzSuccess\"") - } - return status.Error(codes.OK, "fake success") - }, - }) - defer deferFunc() - if err != nil { - t.Fatal(err) - } - - _, r, deferFunc, err := setupClient(&clientConfig{balancerName: "round_robin"}) - if err != nil { - t.Fatal(err) + watchFunc := func(s *testHealthServer, in *healthpb.HealthCheckRequest, stream healthgrpc.Health_WatchServer) error { + if in.Service != "channelzSuccess" { + return status.Error(codes.FailedPrecondition, + "this special Watch function only handles request with service name to be \"channelzSuccess\"") + } + return status.Error(codes.OK, "fake success") } - defer deferFunc() + _, lis, _ := setupServer(t, watchFunc) + _, r := setupClient(t, nil) r.UpdateState(resolver.State{ Addresses: []resolver.Address{{Addr: lis.Addr().String()}}, - ServiceConfig: parseCfg(r, `{ + ServiceConfig: parseServiceConfig(t, r, `{ "healthCheckConfig": { "serviceName": "channelzSuccess" - } + }, + "loadBalancingConfig": [{"round_robin":{}}] }`)}) if err := verifyResultWithDelay(func() (bool, error) { @@ -925,32 +810,23 @@ func (s) TestHealthCheckChannelzCountingCallSuccess(t *testing.T) { } func (s) TestHealthCheckChannelzCountingCallFailure(t *testing.T) { - _, lis, _, deferFunc, err := setupServer(&svrConfig{ - specialWatchFunc: func(s *testHealthServer, in *healthpb.HealthCheckRequest, stream healthgrpc.Health_WatchServer) error { - if in.Service != "channelzFailure" { - return status.Error(codes.FailedPrecondition, - "this special Watch function only handles request with service name to be \"channelzFailure\"") - } - return status.Error(codes.Internal, "fake failure") - }, - }) - if err != nil { - t.Fatal(err) - } - defer deferFunc() - - _, r, deferFunc, err := setupClient(&clientConfig{balancerName: "round_robin"}) - if err != nil { - t.Fatal(err) + watchFunc := func(s *testHealthServer, in *healthpb.HealthCheckRequest, stream healthgrpc.Health_WatchServer) error { + if in.Service != "channelzFailure" { + return status.Error(codes.FailedPrecondition, + "this special Watch function only handles request with service name to be \"channelzFailure\"") + } + return status.Error(codes.Internal, "fake failure") } - defer deferFunc() + _, lis, _ := setupServer(t, watchFunc) + _, r := setupClient(t, nil) r.UpdateState(resolver.State{ Addresses: []resolver.Address{{Addr: lis.Addr().String()}}, - ServiceConfig: parseCfg(r, `{ + ServiceConfig: parseServiceConfig(t, r, `{ "healthCheckConfig": { "serviceName": "channelzFailure" - } + }, + "loadBalancingConfig": [{"round_robin":{}}] }`)}) if err := verifyResultWithDelay(func() (bool, error) { @@ -979,3 +855,313 @@ func (s) TestHealthCheckChannelzCountingCallFailure(t *testing.T) { t.Fatal(err) } } + +// healthCheck is a helper function to make a unary health check RPC and return +// the response. +func healthCheck(d time.Duration, cc *grpc.ClientConn, service string) (*healthpb.HealthCheckResponse, error) { + ctx, cancel := context.WithTimeout(context.Background(), d) + defer cancel() + hc := healthgrpc.NewHealthClient(cc) + return hc.Check(ctx, &healthpb.HealthCheckRequest{Service: service}) +} + +// verifyHealthCheckStatus is a helper function to verify that the current +// health status of the service matches the one passed in 'wantStatus'. +func verifyHealthCheckStatus(t *testing.T, d time.Duration, cc *grpc.ClientConn, service string, wantStatus healthpb.HealthCheckResponse_ServingStatus) { + t.Helper() + resp, err := healthCheck(d, cc, service) + if err != nil { + t.Fatalf("Health/Check(_, _) = _, %v, want _, ", err) + } + if resp.Status != wantStatus { + t.Fatalf("Got the serving status %v, want %v", resp.Status, wantStatus) + } +} + +// verifyHealthCheckErrCode is a helper function to verify that a unary health +// check RPC returns an error with a code set to 'wantCode'. +func verifyHealthCheckErrCode(t *testing.T, d time.Duration, cc *grpc.ClientConn, service string, wantCode codes.Code) { + t.Helper() + if _, err := healthCheck(d, cc, service); status.Code(err) != wantCode { + t.Fatalf("Health/Check() got errCode %v, want %v", status.Code(err), wantCode) + } +} + +// newHealthCheckStream is a helper function to start a health check streaming +// RPC, and returns the stream. +func newHealthCheckStream(t *testing.T, cc *grpc.ClientConn, service string) (healthgrpc.Health_WatchClient, context.CancelFunc) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + hc := healthgrpc.NewHealthClient(cc) + stream, err := hc.Watch(ctx, &healthpb.HealthCheckRequest{Service: service}) + if err != nil { + t.Fatalf("hc.Watch(_, %v) failed: %v", service, err) + } + return stream, cancel +} + +// healthWatchChecker is a helper function to verify that the next health +// status returned on the given stream matches the one passed in 'wantStatus'. +func healthWatchChecker(t *testing.T, stream healthgrpc.Health_WatchClient, wantStatus healthpb.HealthCheckResponse_ServingStatus) { + t.Helper() + response, err := stream.Recv() + if err != nil { + t.Fatalf("stream.Recv() failed: %v", err) + } + if response.Status != wantStatus { + t.Fatalf("got servingStatus %v, want %v", response.Status, wantStatus) + } +} + +// TestHealthCheckSuccess invokes the unary Check() RPC on the health server in +// a successful case. +func (s) TestHealthCheckSuccess(t *testing.T) { + for _, e := range listTestEnv() { + testHealthCheckSuccess(t, e) + } +} + +func testHealthCheckSuccess(t *testing.T, e env) { + te := newTest(t, e) + te.enableHealthServer = true + te.startServer(&testServer{security: e.security}) + te.setHealthServingStatus(defaultHealthService, healthpb.HealthCheckResponse_SERVING) + defer te.tearDown() + + verifyHealthCheckErrCode(t, 1*time.Second, te.clientConn(), defaultHealthService, codes.OK) +} + +// TestHealthCheckFailure invokes the unary Check() RPC on the health server +// with an expired context and expects the RPC to fail. +func (s) TestHealthCheckFailure(t *testing.T) { + for _, e := range listTestEnv() { + testHealthCheckFailure(t, e) + } +} + +func testHealthCheckFailure(t *testing.T, e env) { + te := newTest(t, e) + te.declareLogNoise( + "Failed to dial ", + "grpc: the client connection is closing; please retry", + ) + te.enableHealthServer = true + te.startServer(&testServer{security: e.security}) + te.setHealthServingStatus(defaultHealthService, healthpb.HealthCheckResponse_SERVING) + defer te.tearDown() + + verifyHealthCheckErrCode(t, 0*time.Second, te.clientConn(), defaultHealthService, codes.DeadlineExceeded) + awaitNewConnLogOutput() +} + +// TestHealthCheckOff makes a unary Check() RPC on the health server where the +// health status of the defaultHealthService is not set, and therefore expects +// an error code 'codes.NotFound'. +func (s) TestHealthCheckOff(t *testing.T) { + for _, e := range listTestEnv() { + // TODO(bradfitz): Temporarily skip this env due to #619. + if e.name == "handler-tls" { + continue + } + testHealthCheckOff(t, e) + } +} + +func testHealthCheckOff(t *testing.T, e env) { + te := newTest(t, e) + te.enableHealthServer = true + te.startServer(&testServer{security: e.security}) + defer te.tearDown() + + verifyHealthCheckErrCode(t, 1*time.Second, te.clientConn(), defaultHealthService, codes.NotFound) +} + +// TestHealthWatchMultipleClients makes a streaming Watch() RPC on the health +// server with multiple clients and expects the same status on both streams. +func (s) TestHealthWatchMultipleClients(t *testing.T) { + for _, e := range listTestEnv() { + testHealthWatchMultipleClients(t, e) + } +} + +func testHealthWatchMultipleClients(t *testing.T, e env) { + te := newTest(t, e) + te.enableHealthServer = true + te.startServer(&testServer{security: e.security}) + defer te.tearDown() + + cc := te.clientConn() + stream1, cf1 := newHealthCheckStream(t, cc, defaultHealthService) + defer cf1() + healthWatchChecker(t, stream1, healthpb.HealthCheckResponse_SERVICE_UNKNOWN) + + stream2, cf2 := newHealthCheckStream(t, cc, defaultHealthService) + defer cf2() + healthWatchChecker(t, stream2, healthpb.HealthCheckResponse_SERVICE_UNKNOWN) + + te.setHealthServingStatus(defaultHealthService, healthpb.HealthCheckResponse_NOT_SERVING) + healthWatchChecker(t, stream1, healthpb.HealthCheckResponse_NOT_SERVING) + healthWatchChecker(t, stream2, healthpb.HealthCheckResponse_NOT_SERVING) +} + +// TestHealthWatchSameStatusmakes a streaming Watch() RPC on the health server +// and makes sure that the health status of the server is as expected after +// multiple calls to SetServingStatus with the same status. +func (s) TestHealthWatchSameStatus(t *testing.T) { + for _, e := range listTestEnv() { + testHealthWatchSameStatus(t, e) + } +} + +func testHealthWatchSameStatus(t *testing.T, e env) { + te := newTest(t, e) + te.enableHealthServer = true + te.startServer(&testServer{security: e.security}) + defer te.tearDown() + + stream, cf := newHealthCheckStream(t, te.clientConn(), defaultHealthService) + defer cf() + + healthWatchChecker(t, stream, healthpb.HealthCheckResponse_SERVICE_UNKNOWN) + te.setHealthServingStatus(defaultHealthService, healthpb.HealthCheckResponse_SERVING) + healthWatchChecker(t, stream, healthpb.HealthCheckResponse_SERVING) + te.setHealthServingStatus(defaultHealthService, healthpb.HealthCheckResponse_SERVING) + te.setHealthServingStatus(defaultHealthService, healthpb.HealthCheckResponse_NOT_SERVING) + healthWatchChecker(t, stream, healthpb.HealthCheckResponse_NOT_SERVING) +} + +// TestHealthWatchServiceStatusSetBeforeStartingServer starts a health server +// on which the health status for the defaultService is set before the gRPC +// server is started, and expects the correct health status to be returned. +func (s) TestHealthWatchServiceStatusSetBeforeStartingServer(t *testing.T) { + for _, e := range listTestEnv() { + testHealthWatchSetServiceStatusBeforeStartingServer(t, e) + } +} + +func testHealthWatchSetServiceStatusBeforeStartingServer(t *testing.T, e env) { + hs := health.NewServer() + te := newTest(t, e) + te.healthServer = hs + hs.SetServingStatus(defaultHealthService, healthpb.HealthCheckResponse_SERVING) + te.startServer(&testServer{security: e.security}) + defer te.tearDown() + + stream, cf := newHealthCheckStream(t, te.clientConn(), defaultHealthService) + defer cf() + healthWatchChecker(t, stream, healthpb.HealthCheckResponse_SERVING) +} + +// TestHealthWatchDefaultStatusChange verifies the simple case where the +// service starts off with a SERVICE_UNKNOWN status (because SetServingStatus +// hasn't been called yet) and then moves to SERVING after SetServingStatus is +// called. +func (s) TestHealthWatchDefaultStatusChange(t *testing.T) { + for _, e := range listTestEnv() { + testHealthWatchDefaultStatusChange(t, e) + } +} + +func testHealthWatchDefaultStatusChange(t *testing.T, e env) { + te := newTest(t, e) + te.enableHealthServer = true + te.startServer(&testServer{security: e.security}) + defer te.tearDown() + + stream, cf := newHealthCheckStream(t, te.clientConn(), defaultHealthService) + defer cf() + healthWatchChecker(t, stream, healthpb.HealthCheckResponse_SERVICE_UNKNOWN) + te.setHealthServingStatus(defaultHealthService, healthpb.HealthCheckResponse_SERVING) + healthWatchChecker(t, stream, healthpb.HealthCheckResponse_SERVING) +} + +// TestHealthWatchSetServiceStatusBeforeClientCallsWatch verifies the case +// where the health status is set to SERVING before the client calls Watch(). +func (s) TestHealthWatchSetServiceStatusBeforeClientCallsWatch(t *testing.T) { + for _, e := range listTestEnv() { + testHealthWatchSetServiceStatusBeforeClientCallsWatch(t, e) + } +} + +func testHealthWatchSetServiceStatusBeforeClientCallsWatch(t *testing.T, e env) { + te := newTest(t, e) + te.enableHealthServer = true + te.startServer(&testServer{security: e.security}) + te.setHealthServingStatus(defaultHealthService, healthpb.HealthCheckResponse_SERVING) + defer te.tearDown() + + stream, cf := newHealthCheckStream(t, te.clientConn(), defaultHealthService) + defer cf() + healthWatchChecker(t, stream, healthpb.HealthCheckResponse_SERVING) +} + +// TestHealthWatchOverallServerHealthChange verifies setting the overall status +// of the server by using the empty service name. +func (s) TestHealthWatchOverallServerHealthChange(t *testing.T) { + for _, e := range listTestEnv() { + testHealthWatchOverallServerHealthChange(t, e) + } +} + +func testHealthWatchOverallServerHealthChange(t *testing.T, e env) { + te := newTest(t, e) + te.enableHealthServer = true + te.startServer(&testServer{security: e.security}) + defer te.tearDown() + + stream, cf := newHealthCheckStream(t, te.clientConn(), "") + defer cf() + healthWatchChecker(t, stream, healthpb.HealthCheckResponse_SERVING) + te.setHealthServingStatus("", healthpb.HealthCheckResponse_NOT_SERVING) + healthWatchChecker(t, stream, healthpb.HealthCheckResponse_NOT_SERVING) +} + +// TestUnknownHandler verifies that an expected error is returned (by setting +// the unknownHandler on the server) for a service which is not exposed to the +// client. +func (s) TestUnknownHandler(t *testing.T) { + // An example unknownHandler that returns a different code and a different + // method, making sure that we do not expose what methods are implemented to + // a client that is not authenticated. + unknownHandler := func(srv any, stream grpc.ServerStream) error { + return status.Error(codes.Unauthenticated, "user unauthenticated") + } + for _, e := range listTestEnv() { + // TODO(bradfitz): Temporarily skip this env due to #619. + if e.name == "handler-tls" { + continue + } + testUnknownHandler(t, e, unknownHandler) + } +} + +func testUnknownHandler(t *testing.T, e env, unknownHandler grpc.StreamHandler) { + te := newTest(t, e) + te.unknownHandler = unknownHandler + te.startServer(&testServer{security: e.security}) + defer te.tearDown() + verifyHealthCheckErrCode(t, 1*time.Second, te.clientConn(), "", codes.Unauthenticated) +} + +// TestHealthCheckServingStatus makes a streaming Watch() RPC on the health +// server and verifies a bunch of health status transitions. +func (s) TestHealthCheckServingStatus(t *testing.T) { + for _, e := range listTestEnv() { + testHealthCheckServingStatus(t, e) + } +} + +func testHealthCheckServingStatus(t *testing.T, e env) { + te := newTest(t, e) + te.enableHealthServer = true + te.startServer(&testServer{security: e.security}) + defer te.tearDown() + + cc := te.clientConn() + verifyHealthCheckStatus(t, 1*time.Second, cc, "", healthpb.HealthCheckResponse_SERVING) + verifyHealthCheckErrCode(t, 1*time.Second, cc, defaultHealthService, codes.NotFound) + te.setHealthServingStatus(defaultHealthService, healthpb.HealthCheckResponse_SERVING) + verifyHealthCheckStatus(t, 1*time.Second, cc, defaultHealthService, healthpb.HealthCheckResponse_SERVING) + te.setHealthServingStatus(defaultHealthService, healthpb.HealthCheckResponse_NOT_SERVING) + verifyHealthCheckStatus(t, 1*time.Second, cc, defaultHealthService, healthpb.HealthCheckResponse_NOT_SERVING) +} diff --git a/test/http_header_end2end_test.go b/test/http_header_end2end_test.go new file mode 100644 index 000000000000..77867133f95c --- /dev/null +++ b/test/http_header_end2end_test.go @@ -0,0 +1,261 @@ +/* +* +* Copyright 2022 gRPC authors. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* + */ +package test + +import ( + "context" + "fmt" + "net" + "testing" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/transport" + "google.golang.org/grpc/status" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" +) + +func (s) TestHTTPHeaderFrameErrorHandlingHTTPMode(t *testing.T) { + type test struct { + name string + header []string + errCode codes.Code + } + + var tests []test + + // Non-gRPC content-type fallback path. + for httpCode := range transport.HTTPStatusConvTab { + tests = append(tests, test{ + name: fmt.Sprintf("Non-gRPC content-type fallback path with httpCode: %v", httpCode), + header: []string{ + ":status", fmt.Sprintf("%d", httpCode), + "content-type", "text/html", // non-gRPC content type to switch to HTTP mode. + "grpc-status", "1", // Make up a gRPC status error + "grpc-status-details-bin", "???", // Make up a gRPC field parsing error + }, + errCode: transport.HTTPStatusConvTab[int(httpCode)], + }) + } + + // Missing content-type fallback path. + for httpCode := range transport.HTTPStatusConvTab { + tests = append(tests, test{ + name: fmt.Sprintf("Missing content-type fallback path with httpCode: %v", httpCode), + header: []string{ + ":status", fmt.Sprintf("%d", httpCode), + // Omitting content type to switch to HTTP mode. + "grpc-status", "1", // Make up a gRPC status error + "grpc-status-details-bin", "???", // Make up a gRPC field parsing error + }, + errCode: transport.HTTPStatusConvTab[int(httpCode)], + }) + } + + // Malformed HTTP status when fallback. + tests = append(tests, test{ + name: "Malformed HTTP status when fallback", + header: []string{ + ":status", "abc", + // Omitting content type to switch to HTTP mode. + "grpc-status", "1", // Make up a gRPC status error + "grpc-status-details-bin", "???", // Make up a gRPC field parsing error + }, + errCode: codes.Internal, + }) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + serverAddr, cleanup, err := startServer(t, test.header) + if err != nil { + t.Fatal(err) + } + defer cleanup() + if err := doHTTPHeaderTest(serverAddr, test.errCode); err != nil { + t.Error(err) + } + }) + } +} + +// Testing erroneous ResponseHeader or Trailers-only (delivered in the first HEADERS frame). +func (s) TestHTTPHeaderFrameErrorHandlingInitialHeader(t *testing.T) { + for _, test := range []struct { + name string + header []string + errCode codes.Code + }{ + { + name: "missing gRPC status", + header: []string{ + ":status", "403", + "content-type", "application/grpc", + }, + errCode: codes.PermissionDenied, + }, + { + name: "malformed grpc-status", + header: []string{ + ":status", "502", + "content-type", "application/grpc", + "grpc-status", "abc", + }, + errCode: codes.Internal, + }, + { + name: "Malformed grpc-tags-bin field", + header: []string{ + ":status", "502", + "content-type", "application/grpc", + "grpc-status", "0", + "grpc-tags-bin", "???", + }, + errCode: codes.Unavailable, + }, + { + name: "gRPC status error", + header: []string{ + ":status", "502", + "content-type", "application/grpc", + "grpc-status", "3", + }, + errCode: codes.Unavailable, + }, + } { + t.Run(test.name, func(t *testing.T) { + serverAddr, cleanup, err := startServer(t, test.header) + if err != nil { + t.Fatal(err) + } + defer cleanup() + if err := doHTTPHeaderTest(serverAddr, test.errCode); err != nil { + t.Error(err) + } + }) + } +} + +// Testing non-Trailers-only Trailers (delivered in second HEADERS frame) +func (s) TestHTTPHeaderFrameErrorHandlingNormalTrailer(t *testing.T) { + tests := []struct { + name string + responseHeader []string + trailer []string + errCode codes.Code + }{ + { + name: "trailer missing grpc-status", + responseHeader: []string{ + ":status", "200", + "content-type", "application/grpc", + }, + trailer: []string{ + // trailer missing grpc-status + ":status", "502", + }, + errCode: codes.Unavailable, + }, + { + name: "malformed grpc-status-details-bin field with status 404", + responseHeader: []string{ + ":status", "404", + "content-type", "application/grpc", + }, + trailer: []string{ + // malformed grpc-status-details-bin field + "grpc-status", "0", + "grpc-status-details-bin", "????", + }, + errCode: codes.Unimplemented, + }, + { + name: "malformed grpc-status-details-bin field with status 200", + responseHeader: []string{ + ":status", "200", + "content-type", "application/grpc", + }, + trailer: []string{ + // malformed grpc-status-details-bin field + "grpc-status", "0", + "grpc-status-details-bin", "????", + }, + errCode: codes.Internal, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + serverAddr, cleanup, err := startServer(t, test.responseHeader, test.trailer) + if err != nil { + t.Fatal(err) + } + defer cleanup() + if err := doHTTPHeaderTest(serverAddr, test.errCode); err != nil { + t.Error(err) + } + }) + + } +} + +func (s) TestHTTPHeaderFrameErrorHandlingMoreThanTwoHeaders(t *testing.T) { + header := []string{ + ":status", "200", + "content-type", "application/grpc", + } + serverAddr, cleanup, err := startServer(t, header, header, header) + if err != nil { + t.Fatal(err) + } + defer cleanup() + if err := doHTTPHeaderTest(serverAddr, codes.Internal); err != nil { + t.Fatal(err) + } +} + +func startServer(t *testing.T, headerFields ...[]string) (serverAddr string, cleanup func(), err error) { + t.Helper() + + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + return "", nil, fmt.Errorf("listening on %q: %v", "localhost:0", err) + } + server := &httpServer{responses: []httpServerResponse{{trailers: headerFields}}} + server.start(t, lis) + return lis.Addr().String(), func() { lis.Close() }, nil +} + +func doHTTPHeaderTest(lisAddr string, errCode codes.Code) error { + cc, err := grpc.Dial(lisAddr, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + return fmt.Errorf("dial(%q): %v", lisAddr, err) + } + defer cc.Close() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + client := testgrpc.NewTestServiceClient(cc) + stream, err := client.FullDuplexCall(ctx) + if err != nil { + return fmt.Errorf("creating FullDuplex stream: %v", err) + } + if _, err := stream.Recv(); err == nil || status.Code(err) != errCode { + return fmt.Errorf("stream.Recv() = %v, want error code: %v", err, errCode) + } + return nil +} diff --git a/test/insecure_creds_test.go b/test/insecure_creds_test.go new file mode 100644 index 000000000000..0647c81232ae --- /dev/null +++ b/test/insecure_creds_test.go @@ -0,0 +1,207 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "context" + "net" + "strings" + "testing" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +// testLegacyPerRPCCredentials is a PerRPCCredentials that has yet incorporated security level. +type testLegacyPerRPCCredentials struct{} + +func (cr testLegacyPerRPCCredentials) GetRequestMetadata(ctx context.Context, uri ...string) (map[string]string, error) { + return nil, nil +} + +func (cr testLegacyPerRPCCredentials) RequireTransportSecurity() bool { + return true +} + +func getSecurityLevel(ai credentials.AuthInfo) credentials.SecurityLevel { + if c, ok := ai.(interface { + GetCommonAuthInfo() credentials.CommonAuthInfo + }); ok { + return c.GetCommonAuthInfo().SecurityLevel + } + return credentials.InvalidSecurityLevel +} + +// TestInsecureCreds tests the use of insecure creds on the server and client +// side, and verifies that expect security level and auth info are returned. +// Also verifies that this credential can interop with existing `WithInsecure` +// DialOption. +func (s) TestInsecureCreds(t *testing.T) { + tests := []struct { + desc string + clientInsecureCreds bool + serverInsecureCreds bool + }{ + { + desc: "client and server insecure creds", + clientInsecureCreds: true, + serverInsecureCreds: true, + }, + { + desc: "client only insecure creds", + clientInsecureCreds: true, + }, + { + desc: "server only insecure creds", + serverInsecureCreds: true, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { + if !test.serverInsecureCreds { + return &testpb.Empty{}, nil + } + + pr, ok := peer.FromContext(ctx) + if !ok { + return nil, status.Error(codes.DataLoss, "Failed to get peer from ctx") + } + // Check security level. + secLevel := getSecurityLevel(pr.AuthInfo) + if secLevel == credentials.InvalidSecurityLevel { + return nil, status.Errorf(codes.Unauthenticated, "peer.AuthInfo does not implement GetCommonAuthInfo()") + } + if secLevel != credentials.NoSecurity { + return nil, status.Errorf(codes.Unauthenticated, "Wrong security level: got %q, want %q", secLevel, credentials.NoSecurity) + } + return &testpb.Empty{}, nil + }, + } + + sOpts := []grpc.ServerOption{} + if test.serverInsecureCreds { + sOpts = append(sOpts, grpc.Creds(insecure.NewCredentials())) + } + s := grpc.NewServer(sOpts...) + defer s.Stop() + + testgrpc.RegisterTestServiceServer(s, ss) + + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("net.Listen(tcp, localhost:0) failed: %v", err) + } + + go s.Serve(lis) + + addr := lis.Addr().String() + opts := []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())} + if test.clientInsecureCreds { + opts = []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())} + } + cc, err := grpc.Dial(addr, opts...) + if err != nil { + t.Fatalf("grpc.Dial(%q) failed: %v", addr, err) + } + defer cc.Close() + + c := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err = c.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall(_, _) = _, %v; want _, ", err) + } + }) + } +} + +func (s) TestInsecureCreds_WithPerRPCCredentials_AsCallOption(t *testing.T) { + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + } + + s := grpc.NewServer(grpc.Creds(insecure.NewCredentials())) + defer s.Stop() + testgrpc.RegisterTestServiceServer(s, ss) + + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("net.Listen(tcp, localhost:0) failed: %v", err) + } + go s.Serve(lis) + + addr := lis.Addr().String() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + dopts := []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())} + copts := []grpc.CallOption{grpc.PerRPCCredentials(testLegacyPerRPCCredentials{})} + cc, err := grpc.Dial(addr, dopts...) + if err != nil { + t.Fatalf("grpc.Dial(%q) failed: %v", addr, err) + } + defer cc.Close() + + const wantErr = "transport: cannot send secure credentials on an insecure connection" + c := testgrpc.NewTestServiceClient(cc) + if _, err = c.EmptyCall(ctx, &testpb.Empty{}, copts...); err == nil || !strings.Contains(err.Error(), wantErr) { + t.Fatalf("insecure credentials with per-RPC credentials requiring transport security returned error: %v; want %s", err, wantErr) + } +} + +func (s) TestInsecureCreds_WithPerRPCCredentials_AsDialOption(t *testing.T) { + ss := &stubserver.StubServer{ + EmptyCallF: func(_ context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + } + + s := grpc.NewServer(grpc.Creds(insecure.NewCredentials())) + defer s.Stop() + testgrpc.RegisterTestServiceServer(s, ss) + + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("net.Listen(tcp, localhost:0) failed: %v", err) + } + go s.Serve(lis) + + addr := lis.Addr().String() + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithPerRPCCredentials(testLegacyPerRPCCredentials{}), + } + const wantErr = "the credentials require transport level security" + if _, err := grpc.Dial(addr, dopts...); err == nil || !strings.Contains(err.Error(), wantErr) { + t.Fatalf("grpc.Dial(%q) returned err %v, want: %v", addr, err, wantErr) + } +} diff --git a/test/interceptor_test.go b/test/interceptor_test.go new file mode 100644 index 000000000000..f6db1d282a4c --- /dev/null +++ b/test/interceptor_test.go @@ -0,0 +1,281 @@ +/* + * + * Copyright 2022 gRPC authors. + + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "context" + "fmt" + "testing" + + "google.golang.org/grpc" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +type parentCtxkey struct{} +type firstInterceptorCtxkey struct{} +type secondInterceptorCtxkey struct{} +type baseInterceptorCtxKey struct{} + +const ( + parentCtxVal = "parent" + firstInterceptorCtxVal = "firstInterceptor" + secondInterceptorCtxVal = "secondInterceptor" + baseInterceptorCtxVal = "baseInterceptor" +) + +// TestUnaryClientInterceptor_ContextValuePropagation verifies that a unary +// interceptor receives context values specified in the context passed to the +// RPC call. +func (s) TestUnaryClientInterceptor_ContextValuePropagation(t *testing.T) { + errCh := testutils.NewChannel() + unaryInt := func(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + if got, ok := ctx.Value(parentCtxkey{}).(string); !ok || got != parentCtxVal { + errCh.Send(fmt.Errorf("unaryInt got %q in context.Val, want %q", got, parentCtxVal)) + } + errCh.Send(nil) + return invoker(ctx, method, req, reply, cc, opts...) + } + + // Start a stub server and use the above unary interceptor while creating a + // ClientConn to it. + ss := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil }, + } + if err := ss.Start(nil, grpc.WithUnaryInterceptor(unaryInt)); err != nil { + t.Fatalf("Failed to start stub server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := ss.Client.EmptyCall(context.WithValue(ctx, parentCtxkey{}, parentCtxVal), &testpb.Empty{}); err != nil { + t.Fatalf("ss.Client.EmptyCall() failed: %v", err) + } + val, err := errCh.Receive(ctx) + if err != nil { + t.Fatalf("timeout when waiting for unary interceptor to be invoked: %v", err) + } + if val != nil { + t.Fatalf("unary interceptor failed: %v", val) + } +} + +// TestChainUnaryClientInterceptor_ContextValuePropagation verifies that a chain +// of unary interceptors receive context values specified in the original call +// as well as the ones specified by prior interceptors in the chain. +func (s) TestChainUnaryClientInterceptor_ContextValuePropagation(t *testing.T) { + errCh := testutils.NewChannel() + firstInt := func(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + if got, ok := ctx.Value(parentCtxkey{}).(string); !ok || got != parentCtxVal { + errCh.SendContext(ctx, fmt.Errorf("first interceptor got %q in context.Val, want %q", got, parentCtxVal)) + } + if ctx.Value(firstInterceptorCtxkey{}) != nil { + errCh.SendContext(ctx, fmt.Errorf("first interceptor should not have %T in context", firstInterceptorCtxkey{})) + } + if ctx.Value(secondInterceptorCtxkey{}) != nil { + errCh.SendContext(ctx, fmt.Errorf("first interceptor should not have %T in context", secondInterceptorCtxkey{})) + } + firstCtx := context.WithValue(ctx, firstInterceptorCtxkey{}, firstInterceptorCtxVal) + return invoker(firstCtx, method, req, reply, cc, opts...) + } + + secondInt := func(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + if got, ok := ctx.Value(parentCtxkey{}).(string); !ok || got != parentCtxVal { + errCh.SendContext(ctx, fmt.Errorf("second interceptor got %q in context.Val, want %q", got, parentCtxVal)) + } + if got, ok := ctx.Value(firstInterceptorCtxkey{}).(string); !ok || got != firstInterceptorCtxVal { + errCh.SendContext(ctx, fmt.Errorf("second interceptor got %q in context.Val, want %q", got, firstInterceptorCtxVal)) + } + if ctx.Value(secondInterceptorCtxkey{}) != nil { + errCh.SendContext(ctx, fmt.Errorf("second interceptor should not have %T in context", secondInterceptorCtxkey{})) + } + secondCtx := context.WithValue(ctx, secondInterceptorCtxkey{}, secondInterceptorCtxVal) + return invoker(secondCtx, method, req, reply, cc, opts...) + } + + lastInt := func(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + if got, ok := ctx.Value(parentCtxkey{}).(string); !ok || got != parentCtxVal { + errCh.SendContext(ctx, fmt.Errorf("last interceptor got %q in context.Val, want %q", got, parentCtxVal)) + } + if got, ok := ctx.Value(firstInterceptorCtxkey{}).(string); !ok || got != firstInterceptorCtxVal { + errCh.SendContext(ctx, fmt.Errorf("last interceptor got %q in context.Val, want %q", got, firstInterceptorCtxVal)) + } + if got, ok := ctx.Value(secondInterceptorCtxkey{}).(string); !ok || got != secondInterceptorCtxVal { + errCh.SendContext(ctx, fmt.Errorf("last interceptor got %q in context.Val, want %q", got, secondInterceptorCtxVal)) + } + errCh.SendContext(ctx, nil) + return invoker(ctx, method, req, reply, cc, opts...) + } + + // Start a stub server and use the above chain of interceptors while creating + // a ClientConn to it. + ss := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil }, + } + if err := ss.Start(nil, grpc.WithChainUnaryInterceptor(firstInt, secondInt, lastInt)); err != nil { + t.Fatalf("Failed to start stub server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := ss.Client.EmptyCall(context.WithValue(ctx, parentCtxkey{}, parentCtxVal), &testpb.Empty{}); err != nil { + t.Fatalf("ss.Client.EmptyCall() failed: %v", err) + } + val, err := errCh.Receive(ctx) + if err != nil { + t.Fatalf("timeout when waiting for unary interceptor to be invoked: %v", err) + } + if val != nil { + t.Fatalf("unary interceptor failed: %v", val) + } +} + +// TestChainOnBaseUnaryClientInterceptor_ContextValuePropagation verifies that +// unary interceptors specified as a base interceptor or as a chain interceptor +// receive context values specified in the original call as well as the ones +// specified by interceptors in the chain. +func (s) TestChainOnBaseUnaryClientInterceptor_ContextValuePropagation(t *testing.T) { + errCh := testutils.NewChannel() + baseInt := func(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + if got, ok := ctx.Value(parentCtxkey{}).(string); !ok || got != parentCtxVal { + errCh.SendContext(ctx, fmt.Errorf("base interceptor got %q in context.Val, want %q", got, parentCtxVal)) + } + if ctx.Value(baseInterceptorCtxKey{}) != nil { + errCh.SendContext(ctx, fmt.Errorf("baseinterceptor should not have %T in context", baseInterceptorCtxKey{})) + } + baseCtx := context.WithValue(ctx, baseInterceptorCtxKey{}, baseInterceptorCtxVal) + return invoker(baseCtx, method, req, reply, cc, opts...) + } + + chainInt := func(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + if got, ok := ctx.Value(parentCtxkey{}).(string); !ok || got != parentCtxVal { + errCh.SendContext(ctx, fmt.Errorf("chain interceptor got %q in context.Val, want %q", got, parentCtxVal)) + } + if got, ok := ctx.Value(baseInterceptorCtxKey{}).(string); !ok || got != baseInterceptorCtxVal { + errCh.SendContext(ctx, fmt.Errorf("chain interceptor got %q in context.Val, want %q", got, baseInterceptorCtxVal)) + } + errCh.SendContext(ctx, nil) + return invoker(ctx, method, req, reply, cc, opts...) + } + + // Start a stub server and use the above chain of interceptors while creating + // a ClientConn to it. + ss := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil }, + } + if err := ss.Start(nil, grpc.WithUnaryInterceptor(baseInt), grpc.WithChainUnaryInterceptor(chainInt)); err != nil { + t.Fatalf("Failed to start stub server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := ss.Client.EmptyCall(context.WithValue(ctx, parentCtxkey{}, parentCtxVal), &testpb.Empty{}); err != nil { + t.Fatalf("ss.Client.EmptyCall() failed: %v", err) + } + val, err := errCh.Receive(ctx) + if err != nil { + t.Fatalf("timeout when waiting for unary interceptor to be invoked: %v", err) + } + if val != nil { + t.Fatalf("unary interceptor failed: %v", val) + } +} + +// TestChainStreamClientInterceptor_ContextValuePropagation verifies that a +// chain of stream interceptors receive context values specified in the original +// call as well as the ones specified by the prior interceptors in the chain. +func (s) TestChainStreamClientInterceptor_ContextValuePropagation(t *testing.T) { + errCh := testutils.NewChannel() + firstInt := func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) { + if got, ok := ctx.Value(parentCtxkey{}).(string); !ok || got != parentCtxVal { + errCh.SendContext(ctx, fmt.Errorf("first interceptor got %q in context.Val, want %q", got, parentCtxVal)) + } + if ctx.Value(firstInterceptorCtxkey{}) != nil { + errCh.SendContext(ctx, fmt.Errorf("first interceptor should not have %T in context", firstInterceptorCtxkey{})) + } + if ctx.Value(secondInterceptorCtxkey{}) != nil { + errCh.SendContext(ctx, fmt.Errorf("first interceptor should not have %T in context", secondInterceptorCtxkey{})) + } + firstCtx := context.WithValue(ctx, firstInterceptorCtxkey{}, firstInterceptorCtxVal) + return streamer(firstCtx, desc, cc, method, opts...) + } + + secondInt := func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) { + if got, ok := ctx.Value(parentCtxkey{}).(string); !ok || got != parentCtxVal { + errCh.SendContext(ctx, fmt.Errorf("second interceptor got %q in context.Val, want %q", got, parentCtxVal)) + } + if got, ok := ctx.Value(firstInterceptorCtxkey{}).(string); !ok || got != firstInterceptorCtxVal { + errCh.SendContext(ctx, fmt.Errorf("second interceptor got %q in context.Val, want %q", got, firstInterceptorCtxVal)) + } + if ctx.Value(secondInterceptorCtxkey{}) != nil { + errCh.SendContext(ctx, fmt.Errorf("second interceptor should not have %T in context", secondInterceptorCtxkey{})) + } + secondCtx := context.WithValue(ctx, secondInterceptorCtxkey{}, secondInterceptorCtxVal) + return streamer(secondCtx, desc, cc, method, opts...) + } + + lastInt := func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) { + if got, ok := ctx.Value(parentCtxkey{}).(string); !ok || got != parentCtxVal { + errCh.SendContext(ctx, fmt.Errorf("last interceptor got %q in context.Val, want %q", got, parentCtxVal)) + } + if got, ok := ctx.Value(firstInterceptorCtxkey{}).(string); !ok || got != firstInterceptorCtxVal { + errCh.SendContext(ctx, fmt.Errorf("last interceptor got %q in context.Val, want %q", got, firstInterceptorCtxVal)) + } + if got, ok := ctx.Value(secondInterceptorCtxkey{}).(string); !ok || got != secondInterceptorCtxVal { + errCh.SendContext(ctx, fmt.Errorf("last interceptor got %q in context.Val, want %q", got, secondInterceptorCtxVal)) + } + errCh.SendContext(ctx, nil) + return streamer(ctx, desc, cc, method, opts...) + } + + // Start a stub server and use the above chain of interceptors while creating + // a ClientConn to it. + ss := &stubserver.StubServer{ + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + if _, err := stream.Recv(); err != nil { + return err + } + return stream.Send(&testpb.StreamingOutputCallResponse{}) + }, + } + if err := ss.Start(nil, grpc.WithChainStreamInterceptor(firstInt, secondInt, lastInt)); err != nil { + t.Fatalf("Failed to start stub server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := ss.Client.FullDuplexCall(context.WithValue(ctx, parentCtxkey{}, parentCtxVal)); err != nil { + t.Fatalf("ss.Client.FullDuplexCall() failed: %v", err) + } + val, err := errCh.Receive(ctx) + if err != nil { + t.Fatalf("timeout when waiting for stream interceptor to be invoked: %v", err) + } + if val != nil { + t.Fatalf("stream interceptor failed: %v", val) + } +} diff --git a/test/invoke_test.go b/test/invoke_test.go new file mode 100644 index 000000000000..4ee65251e450 --- /dev/null +++ b/test/invoke_test.go @@ -0,0 +1,152 @@ +/* + * + * Copyright 2022 gRPC authors. + + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "context" + "strings" + "testing" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal/stubserver" + testpb "google.golang.org/grpc/interop/grpc_testing" + "google.golang.org/grpc/status" +) + +// TestInvoke verifies a straightforward invocation of ClientConn.Invoke(). +func (s) TestInvoke(t *testing.T) { + ss := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Failed to start stub server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := ss.CC.Invoke(ctx, "/grpc.testing.TestService/EmptyCall", &testpb.Empty{}, &testpb.Empty{}); err != nil { + t.Fatalf("grpc.Invoke(\"/grpc.testing.TestService/EmptyCall\") failed: %v", err) + } +} + +// TestInvokeLargeErr verifies an invocation of ClientConn.Invoke() where the +// server returns a really large error message. +func (s) TestInvokeLargeErr(t *testing.T) { + largeErrorStr := strings.Repeat("A", 1024*1024) + ss := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, status.Error(codes.Internal, largeErrorStr) + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Failed to start stub server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + err := ss.CC.Invoke(ctx, "/grpc.testing.TestService/EmptyCall", &testpb.Empty{}, &testpb.Empty{}) + if err == nil { + t.Fatal("grpc.Invoke(\"/grpc.testing.TestService/EmptyCall\") succeeded when expected to fail") + } + st, ok := status.FromError(err) + if !ok { + t.Fatal("grpc.Invoke(\"/grpc.testing.TestService/EmptyCall\") received non-status error") + } + if status.Code(err) != codes.Internal || st.Message() != largeErrorStr { + t.Fatalf("grpc.Invoke(\"/grpc.testing.TestService/EmptyCall\") failed with error: %v, want an error of code %d and desc size %d", err, codes.Internal, len(largeErrorStr)) + } +} + +// TestInvokeErrorSpecialChars tests an invocation of ClientConn.Invoke() and +// verifies that error messages don't get mangled. +func (s) TestInvokeErrorSpecialChars(t *testing.T) { + const weirdError = "format verbs: %v%s" + ss := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, status.Error(codes.Internal, weirdError) + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Failed to start stub server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + err := ss.CC.Invoke(ctx, "/grpc.testing.TestService/EmptyCall", &testpb.Empty{}, &testpb.Empty{}) + if err == nil { + t.Fatal("grpc.Invoke(\"/grpc.testing.TestService/EmptyCall\") succeeded when expected to fail") + } + st, ok := status.FromError(err) + if !ok { + t.Fatal("grpc.Invoke(\"/grpc.testing.TestService/EmptyCall\") received non-status error") + } + if status.Code(err) != codes.Internal || st.Message() != weirdError { + t.Fatalf("grpc.Invoke(\"/grpc.testing.TestService/EmptyCall\") failed with error: %v, want %v", err, weirdError) + } +} + +// TestInvokeCancel tests an invocation of ClientConn.Invoke() with a cancelled +// context and verifies that the request is not actually sent to the server. +func (s) TestInvokeCancel(t *testing.T) { + cancelled := 0 + ss := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + cancelled++ + return &testpb.Empty{}, nil + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Failed to start stub server: %v", err) + } + defer ss.Stop() + + for i := 0; i < 100; i++ { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + cancel() + ss.CC.Invoke(ctx, "/grpc.testing.TestService/EmptyCall", &testpb.Empty{}, &testpb.Empty{}) + } + if cancelled != 0 { + t.Fatalf("server received %d of 100 cancelled requests", cancelled) + } +} + +// TestInvokeCancelClosedNonFail tests an invocation of ClientConn.Invoke() with +// a cancelled non-failfast RPC on a closed ClientConn and verifies that the +// call terminates with an error. +func (s) TestInvokeCancelClosedNonFailFast(t *testing.T) { + ss := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Failed to start stub server: %v", err) + } + defer ss.Stop() + + ss.CC.Close() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + cancel() + if err := ss.CC.Invoke(ctx, "/grpc.testing.TestService/EmptyCall", &testpb.Empty{}, &testpb.Empty{}, grpc.WaitForReady(true)); err == nil { + t.Fatal("ClientConn.Invoke() on closed connection succeeded when expected to fail") + } +} diff --git a/test/kokoro/psm-security.cfg b/test/kokoro/psm-security.cfg new file mode 100644 index 000000000000..040efe9d707e --- /dev/null +++ b/test/kokoro/psm-security.cfg @@ -0,0 +1,13 @@ +# Config file for internal CI + +# Location of the continuous shell script in repository. +build_file: "grpc-go/test/kokoro/psm-security.sh" +timeout_mins: 240 + +action { + define_artifacts { + regex: "artifacts/**/*sponge_log.xml" + regex: "artifacts/**/*.log" + strip_prefix: "artifacts" + } +} diff --git a/test/kokoro/psm-security.sh b/test/kokoro/psm-security.sh new file mode 100755 index 000000000000..46e3709d2e82 --- /dev/null +++ b/test/kokoro/psm-security.sh @@ -0,0 +1,166 @@ +#!/usr/bin/env bash +# Copyright 2021 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +# Constants +readonly GITHUB_REPOSITORY_NAME="grpc-go" +readonly TEST_DRIVER_INSTALL_SCRIPT_URL="https://raw.githubusercontent.com/${TEST_DRIVER_REPO_OWNER:-grpc}/grpc/${TEST_DRIVER_BRANCH:-master}/tools/internal_ci/linux/grpc_xds_k8s_install_test_driver.sh" +## xDS test server/client Docker images +readonly SERVER_IMAGE_NAME="gcr.io/grpc-testing/xds-interop/go-server" +readonly CLIENT_IMAGE_NAME="gcr.io/grpc-testing/xds-interop/go-client" +readonly FORCE_IMAGE_BUILD="${FORCE_IMAGE_BUILD:-0}" + +####################################### +# Builds test app Docker images and pushes them to GCR +# Globals: +# SERVER_IMAGE_NAME: Test server Docker image name +# CLIENT_IMAGE_NAME: Test client Docker image name +# GIT_COMMIT: SHA-1 of git commit being built +# TESTING_VERSION: version branch under test, f.e. v1.42.x, master +# Arguments: +# None +# Outputs: +# Writes the output of `gcloud builds submit` to stdout, stderr +####################################### +build_test_app_docker_images() { + echo "Building Go xDS interop test app Docker images" + docker build -f "${SRC_DIR}/interop/xds/client/Dockerfile" -t "${CLIENT_IMAGE_NAME}:${GIT_COMMIT}" "${SRC_DIR}" + docker build -f "${SRC_DIR}/interop/xds/server/Dockerfile" -t "${SERVER_IMAGE_NAME}:${GIT_COMMIT}" "${SRC_DIR}" + gcloud -q auth configure-docker + docker push "${CLIENT_IMAGE_NAME}:${GIT_COMMIT}" + docker push "${SERVER_IMAGE_NAME}:${GIT_COMMIT}" + if is_version_branch "${TESTING_VERSION}"; then + tag_and_push_docker_image "${CLIENT_IMAGE_NAME}" "${GIT_COMMIT}" "${TESTING_VERSION}" + tag_and_push_docker_image "${SERVER_IMAGE_NAME}" "${GIT_COMMIT}" "${TESTING_VERSION}" + fi +} + +####################################### +# Builds test app and its docker images unless they already exist +# Globals: +# SERVER_IMAGE_NAME: Test server Docker image name +# CLIENT_IMAGE_NAME: Test client Docker image name +# GIT_COMMIT: SHA-1 of git commit being built +# FORCE_IMAGE_BUILD +# Arguments: +# None +# Outputs: +# Writes the output to stdout, stderr +####################################### +build_docker_images_if_needed() { + # Check if images already exist + server_tags="$(gcloud_gcr_list_image_tags "${SERVER_IMAGE_NAME}" "${GIT_COMMIT}")" + printf "Server image: %s:%s\n" "${SERVER_IMAGE_NAME}" "${GIT_COMMIT}" + echo "${server_tags:-Server image not found}" + + client_tags="$(gcloud_gcr_list_image_tags "${CLIENT_IMAGE_NAME}" "${GIT_COMMIT}")" + printf "Client image: %s:%s\n" "${CLIENT_IMAGE_NAME}" "${GIT_COMMIT}" + echo "${client_tags:-Client image not found}" + + # Build if any of the images are missing, or FORCE_IMAGE_BUILD=1 + if [[ "${FORCE_IMAGE_BUILD}" == "1" || -z "${server_tags}" || -z "${client_tags}" ]]; then + build_test_app_docker_images + else + echo "Skipping Go test app build" + fi +} + +####################################### +# Executes the test case +# Globals: +# TEST_DRIVER_FLAGFILE: Relative path to test driver flagfile +# KUBE_CONTEXT: The name of kubectl context with GKE cluster access +# TEST_XML_OUTPUT_DIR: Output directory for the test xUnit XML report +# SERVER_IMAGE_NAME: Test server Docker image name +# CLIENT_IMAGE_NAME: Test client Docker image name +# GIT_COMMIT: SHA-1 of git commit being built +# TESTING_VERSION: version branch under test: used by the framework to determine the supported PSM +# features. +# Arguments: +# Test case name +# Outputs: +# Writes the output of test execution to stdout, stderr +# Test xUnit report to ${TEST_XML_OUTPUT_DIR}/${test_name}/sponge_log.xml +####################################### +run_test() { + # Test driver usage: + # https://github.com/grpc/grpc/tree/master/tools/run_tests/xds_k8s_test_driver#basic-usage + local test_name="${1:?Usage: run_test test_name}" + set -x + local out_dir="${TEST_XML_OUTPUT_DIR}/${test_name}" + mkdir -pv "${out_dir}" + python -m "tests.${test_name}" \ + --flagfile="${TEST_DRIVER_FLAGFILE}" \ + --kube_context="${KUBE_CONTEXT}" \ + --server_image="${SERVER_IMAGE_NAME}:${GIT_COMMIT}" \ + --client_image="${CLIENT_IMAGE_NAME}:${GIT_COMMIT}" \ + --testing_version="${TESTING_VERSION}" \ + --nocheck_local_certs \ + --force_cleanup \ + --collect_app_logs \ + --log_dir="${out_dir}" \ + --xml_output_file="${out_dir}/sponge_log.xml" \ + |& tee "${out_dir}/sponge_log.log" +} + +####################################### +# Main function: provision software necessary to execute tests, and run them +# Globals: +# KOKORO_ARTIFACTS_DIR +# GITHUB_REPOSITORY_NAME +# SRC_DIR: Populated with absolute path to the source repo +# TEST_DRIVER_REPO_DIR: Populated with the path to the repo containing +# the test driver +# TEST_DRIVER_FULL_DIR: Populated with the path to the test driver source code +# TEST_DRIVER_FLAGFILE: Populated with relative path to test driver flagfile +# TEST_XML_OUTPUT_DIR: Populated with the path to test xUnit XML report +# GIT_ORIGIN_URL: Populated with the origin URL of git repo used for the build +# GIT_COMMIT: Populated with the SHA-1 of git commit being built +# GIT_COMMIT_SHORT: Populated with the short SHA-1 of git commit being built +# KUBE_CONTEXT: Populated with name of kubectl context with GKE cluster access +# Arguments: +# None +# Outputs: +# Writes the output of test execution to stdout, stderr +####################################### +main() { + local script_dir + script_dir="$(dirname "$0")" + + # Source the test driver from the master branch. + echo "Sourcing test driver install script from: ${TEST_DRIVER_INSTALL_SCRIPT_URL}" + source /dev/stdin <<< "$(curl -s "${TEST_DRIVER_INSTALL_SCRIPT_URL}")" + + activate_gke_cluster GKE_CLUSTER_PSM_SECURITY + + set -x + if [[ -n "${KOKORO_ARTIFACTS_DIR}" ]]; then + kokoro_setup_test_driver "${GITHUB_REPOSITORY_NAME}" + else + local_setup_test_driver "${script_dir}" + fi + build_docker_images_if_needed + # Run tests + cd "${TEST_DRIVER_FULL_DIR}" + local failed_tests=0 + test_suites=("baseline_test" "security_test" "authz_test") + for test in "${test_suites[@]}"; do + run_test $test || (( ++failed_tests )) + done + echo "Failed test suites: ${failed_tests}" +} + +main "$@" diff --git a/test/kokoro/xds.cfg b/test/kokoro/xds.cfg index 09ab0dacc463..a1e4ed0bb5e6 100644 --- a/test/kokoro/xds.cfg +++ b/test/kokoro/xds.cfg @@ -2,7 +2,7 @@ # Location of the continuous shell script in repository. build_file: "grpc-go/test/kokoro/xds.sh" -timeout_mins: 90 +timeout_mins: 360 action { define_artifacts { regex: "**/*sponge_log.*" diff --git a/test/kokoro/xds.sh b/test/kokoro/xds.sh index 23c9d0119425..75865c340e9e 100755 --- a/test/kokoro/xds.sh +++ b/test/kokoro/xds.sh @@ -7,16 +7,17 @@ cd github export GOPATH="${HOME}/gopath" pushd grpc-go/interop/xds/client -branch=$(git branch --all --no-color --contains "${KOKORO_GITHUB_COMMIT}" \ - | grep -v HEAD | head -1) -shopt -s extglob -branch="${branch//[[:space:]]}" -branch="${branch##remotes/origin/}" -shopt -u extglob -go build +# Install a version of Go supported by gRPC for the new features, e.g. +# errors.Is() +gofilename=go1.21.0.linux-amd64.tar.gz +curl --retry 3 -O -L "https://go.dev/dl/${gofilename}" +sudo tar -C /usr/local -xf "${gofilename}" +sudo ln -s /usr/local/go/bin/go /usr/bin/go +# Retry go build on errors (e.g. go get connection errors), for at most 3 times +for i in 1 2 3; do go build && break || sleep 5; done popd -git clone -b "${branch}" --single-branch --depth=1 https://github.com/grpc/grpc.git +git clone -b master --single-branch --depth=1 https://github.com/grpc/grpc.git grpc/tools/run_tests/helper_scripts/prep_xds.sh @@ -27,12 +28,14 @@ grpc/tools/run_tests/helper_scripts/prep_xds.sh # they are added into "all". GRPC_GO_LOG_VERBOSITY_LEVEL=99 GRPC_GO_LOG_SEVERITY_LEVEL=info \ python3 grpc/tools/run_tests/run_xds_tests.py \ - --test_case="all,path_matching,header_matching" \ + --test_case="ping_pong,circuit_breaking" \ --project_id=grpc-testing \ - --source_image=projects/grpc-testing/global/images/xds-test-server-2 \ + --project_num=830293263384 \ + --source_image=projects/grpc-testing/global/images/xds-test-server-5 \ --path_to_server_binary=/java_server/grpc-java/interop-testing/build/install/grpc-interop-testing/bin/xds-test-server \ --gcp_suffix=$(date '+%s') \ --verbose \ + ${XDS_V3_OPT-} \ --client_cmd="grpc-go/interop/xds/client/client \ --server=xds:///{server_uri} \ --stats_port={stats_port} \ @@ -40,4 +43,3 @@ GRPC_GO_LOG_VERBOSITY_LEVEL=99 GRPC_GO_LOG_SEVERITY_LEVEL=info \ {fail_on_failed_rpc} \ {rpcs_to_send} \ {metadata_to_send}" - diff --git a/test/kokoro/xds_k8s_lb.cfg b/test/kokoro/xds_k8s_lb.cfg new file mode 100644 index 000000000000..5b989a6fe073 --- /dev/null +++ b/test/kokoro/xds_k8s_lb.cfg @@ -0,0 +1,13 @@ +# Config file for internal CI + +# Location of the continuous shell script in repository. +build_file: "grpc-go/test/kokoro/xds_k8s_lb.sh" +timeout_mins: 180 + +action { + define_artifacts { + regex: "artifacts/**/*sponge_log.xml" + regex: "artifacts/**/*.log" + strip_prefix: "artifacts" + } +} diff --git a/test/kokoro/xds_k8s_lb.sh b/test/kokoro/xds_k8s_lb.sh new file mode 100755 index 000000000000..9a543ed4da85 --- /dev/null +++ b/test/kokoro/xds_k8s_lb.sh @@ -0,0 +1,180 @@ +#!/usr/bin/env bash +# Copyright 2022 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +# Constants +readonly GITHUB_REPOSITORY_NAME="grpc-go" +readonly TEST_DRIVER_INSTALL_SCRIPT_URL="https://raw.githubusercontent.com/${TEST_DRIVER_REPO_OWNER:-grpc}/grpc/${TEST_DRIVER_BRANCH:-master}/tools/internal_ci/linux/grpc_xds_k8s_install_test_driver.sh" +## xDS test server/client Docker images +readonly SERVER_IMAGE_NAME="gcr.io/grpc-testing/xds-interop/go-server" +readonly CLIENT_IMAGE_NAME="gcr.io/grpc-testing/xds-interop/go-client" +readonly FORCE_IMAGE_BUILD="${FORCE_IMAGE_BUILD:-0}" + +####################################### +# Builds test app Docker images and pushes them to GCR +# Globals: +# SERVER_IMAGE_NAME: Test server Docker image name +# CLIENT_IMAGE_NAME: Test client Docker image name +# GIT_COMMIT: SHA-1 of git commit being built +# TESTING_VERSION: version branch under test, f.e. v1.42.x, master +# Arguments: +# None +# Outputs: +# Writes the output of `gcloud builds submit` to stdout, stderr +####################################### +build_test_app_docker_images() { + echo "Building Go xDS interop test app Docker images" + docker build -f "${SRC_DIR}/interop/xds/client/Dockerfile" -t "${CLIENT_IMAGE_NAME}:${GIT_COMMIT}" "${SRC_DIR}" + docker build -f "${SRC_DIR}/interop/xds/server/Dockerfile" -t "${SERVER_IMAGE_NAME}:${GIT_COMMIT}" "${SRC_DIR}" + gcloud -q auth configure-docker + docker push "${CLIENT_IMAGE_NAME}:${GIT_COMMIT}" + docker push "${SERVER_IMAGE_NAME}:${GIT_COMMIT}" + if is_version_branch "${TESTING_VERSION}"; then + tag_and_push_docker_image "${CLIENT_IMAGE_NAME}" "${GIT_COMMIT}" "${TESTING_VERSION}" + tag_and_push_docker_image "${SERVER_IMAGE_NAME}" "${GIT_COMMIT}" "${TESTING_VERSION}" + fi +} + +####################################### +# Builds test app and its docker images unless they already exist +# Globals: +# SERVER_IMAGE_NAME: Test server Docker image name +# CLIENT_IMAGE_NAME: Test client Docker image name +# GIT_COMMIT: SHA-1 of git commit being built +# FORCE_IMAGE_BUILD +# Arguments: +# None +# Outputs: +# Writes the output to stdout, stderr +####################################### +build_docker_images_if_needed() { + # Check if images already exist + server_tags="$(gcloud_gcr_list_image_tags "${SERVER_IMAGE_NAME}" "${GIT_COMMIT}")" + printf "Server image: %s:%s\n" "${SERVER_IMAGE_NAME}" "${GIT_COMMIT}" + echo "${server_tags:-Server image not found}" + + client_tags="$(gcloud_gcr_list_image_tags "${CLIENT_IMAGE_NAME}" "${GIT_COMMIT}")" + printf "Client image: %s:%s\n" "${CLIENT_IMAGE_NAME}" "${GIT_COMMIT}" + echo "${client_tags:-Client image not found}" + + # Build if any of the images are missing, or FORCE_IMAGE_BUILD=1 + if [[ "${FORCE_IMAGE_BUILD}" == "1" || -z "${server_tags}" || -z "${client_tags}" ]]; then + build_test_app_docker_images + else + echo "Skipping Go test app build" + fi +} + +####################################### +# Executes the test case +# Globals: +# TEST_DRIVER_FLAGFILE: Relative path to test driver flagfile +# KUBE_CONTEXT: The name of kubectl context with GKE cluster access +# SECONDARY_KUBE_CONTEXT: The name of kubectl context with secondary GKE cluster access, if any +# TEST_XML_OUTPUT_DIR: Output directory for the test xUnit XML report +# SERVER_IMAGE_NAME: Test server Docker image name +# CLIENT_IMAGE_NAME: Test client Docker image name +# GIT_COMMIT: SHA-1 of git commit being built +# TESTING_VERSION: version branch under test: used by the framework to determine the supported PSM +# features. +# Arguments: +# Test case name +# Outputs: +# Writes the output of test execution to stdout, stderr +# Test xUnit report to ${TEST_XML_OUTPUT_DIR}/${test_name}/sponge_log.xml +####################################### +run_test() { + # Test driver usage: + # https://github.com/grpc/grpc/tree/master/tools/run_tests/xds_k8s_test_driver#basic-usage + local test_name="${1:?Usage: run_test test_name}" + local out_dir="${TEST_XML_OUTPUT_DIR}/${test_name}" + mkdir -pv "${out_dir}" + set -x + python -m "tests.${test_name}" \ + --flagfile="${TEST_DRIVER_FLAGFILE}" \ + --kube_context="${KUBE_CONTEXT}" \ + --secondary_kube_context="${SECONDARY_KUBE_CONTEXT}" \ + --server_image="${SERVER_IMAGE_NAME}:${GIT_COMMIT}" \ + --client_image="${CLIENT_IMAGE_NAME}:${GIT_COMMIT}" \ + --testing_version="${TESTING_VERSION}" \ + --force_cleanup \ + --collect_app_logs \ + --log_dir="${out_dir}" \ + --xml_output_file="${out_dir}/sponge_log.xml" \ + |& tee "${out_dir}/sponge_log.log" +} + +####################################### +# Main function: provision software necessary to execute tests, and run them +# Globals: +# KOKORO_ARTIFACTS_DIR +# GITHUB_REPOSITORY_NAME +# SRC_DIR: Populated with absolute path to the source repo +# TEST_DRIVER_REPO_DIR: Populated with the path to the repo containing +# the test driver +# TEST_DRIVER_FULL_DIR: Populated with the path to the test driver source code +# TEST_DRIVER_FLAGFILE: Populated with relative path to test driver flagfile +# TEST_XML_OUTPUT_DIR: Populated with the path to test xUnit XML report +# GIT_ORIGIN_URL: Populated with the origin URL of git repo used for the build +# GIT_COMMIT: Populated with the SHA-1 of git commit being built +# GIT_COMMIT_SHORT: Populated with the short SHA-1 of git commit being built +# KUBE_CONTEXT: Populated with name of kubectl context with GKE cluster access +# Arguments: +# None +# Outputs: +# Writes the output of test execution to stdout, stderr +####################################### +main() { + local script_dir + script_dir="$(dirname "$0")" + + # Source the test driver from the master branch. + echo "Sourcing test driver install script from: ${TEST_DRIVER_INSTALL_SCRIPT_URL}" + source /dev/stdin <<< "$(curl -s "${TEST_DRIVER_INSTALL_SCRIPT_URL}")" + + activate_gke_cluster GKE_CLUSTER_PSM_LB + activate_secondary_gke_cluster GKE_CLUSTER_PSM_LB + + set -x + if [[ -n "${KOKORO_ARTIFACTS_DIR}" ]]; then + kokoro_setup_test_driver "${GITHUB_REPOSITORY_NAME}" + else + local_setup_test_driver "${script_dir}" + fi + build_docker_images_if_needed + # Run tests + cd "${TEST_DRIVER_FULL_DIR}" + local failed_tests=0 + test_suites=( + "affinity_test" + "api_listener_test" + "change_backend_service_test" + "custom_lb_test" + "failover_test" + "outlier_detection_test" + "remove_neg_test" + "round_robin_test" + ) + if [[ "${TESTING_VERSION}" =~ "master" ]]; then + test_suites+=('bootstrap_generator_test') + fi + for test in "${test_suites[@]}"; do + run_test $test || (( ++failed_tests )) + done + echo "Failed test suites: ${failed_tests}" +} + +main "$@" diff --git a/test/kokoro/xds_url_map.cfg b/test/kokoro/xds_url_map.cfg new file mode 100644 index 000000000000..49ebc48e93c6 --- /dev/null +++ b/test/kokoro/xds_url_map.cfg @@ -0,0 +1,13 @@ +# Config file for internal CI + +# Location of the continuous shell script in repository. +build_file: "grpc-go/test/kokoro/xds_url_map.sh" +timeout_mins: 60 + +action { + define_artifacts { + regex: "artifacts/**/*sponge_log.xml" + regex: "artifacts/**/*.log" + strip_prefix: "artifacts" + } +} diff --git a/test/kokoro/xds_url_map.sh b/test/kokoro/xds_url_map.sh new file mode 100755 index 000000000000..a571ea1f00ff --- /dev/null +++ b/test/kokoro/xds_url_map.sh @@ -0,0 +1,147 @@ +#!/usr/bin/env bash +# Copyright 2021 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +# Constants +readonly GITHUB_REPOSITORY_NAME="grpc-go" +readonly TEST_DRIVER_INSTALL_SCRIPT_URL="https://raw.githubusercontent.com/${TEST_DRIVER_REPO_OWNER:-grpc}/grpc/${TEST_DRIVER_BRANCH:-master}/tools/internal_ci/linux/grpc_xds_k8s_install_test_driver.sh" +## xDS test client Docker images +readonly CLIENT_IMAGE_NAME="gcr.io/grpc-testing/xds-interop/go-client" +readonly FORCE_IMAGE_BUILD="${FORCE_IMAGE_BUILD:-0}" + +####################################### +# Builds test app Docker images and pushes them to GCR +# Globals: +# CLIENT_IMAGE_NAME: Test client Docker image name +# GIT_COMMIT: SHA-1 of git commit being built +# Arguments: +# None +# Outputs: +# Writes the output of `gcloud builds submit` to stdout, stderr +####################################### +build_test_app_docker_images() { + echo "Building Go xDS interop test app Docker images" + docker build -f "${SRC_DIR}/interop/xds/client/Dockerfile" -t "${CLIENT_IMAGE_NAME}:${GIT_COMMIT}" "${SRC_DIR}" + gcloud -q auth configure-docker + docker push "${CLIENT_IMAGE_NAME}:${GIT_COMMIT}" + if is_version_branch "${TESTING_VERSION}"; then + tag_and_push_docker_image "${CLIENT_IMAGE_NAME}" "${GIT_COMMIT}" "${TESTING_VERSION}" + fi +} + +####################################### +# Builds test app and its docker images unless they already exist +# Globals: +# CLIENT_IMAGE_NAME: Test client Docker image name +# GIT_COMMIT: SHA-1 of git commit being built +# FORCE_IMAGE_BUILD +# Arguments: +# None +# Outputs: +# Writes the output to stdout, stderr +####################################### +build_docker_images_if_needed() { + # Check if images already exist + client_tags="$(gcloud_gcr_list_image_tags "${CLIENT_IMAGE_NAME}" "${GIT_COMMIT}")" + printf "Client image: %s:%s\n" "${CLIENT_IMAGE_NAME}" "${GIT_COMMIT}" + echo "${client_tags:-Client image not found}" + + # Build if any of the images are missing, or FORCE_IMAGE_BUILD=1 + if [[ "${FORCE_IMAGE_BUILD}" == "1" || -z "${client_tags}" ]]; then + build_test_app_docker_images + else + echo "Skipping Go test app build" + fi +} + +####################################### +# Executes the test case +# Globals: +# TEST_DRIVER_FLAGFILE: Relative path to test driver flagfile +# KUBE_CONTEXT: The name of kubectl context with GKE cluster access +# TEST_XML_OUTPUT_DIR: Output directory for the test xUnit XML report +# CLIENT_IMAGE_NAME: Test client Docker image name +# GIT_COMMIT: SHA-1 of git commit being built +# TESTING_VERSION: version branch under test: used by the framework to determine the supported PSM +# features. +# Arguments: +# Test case name +# Outputs: +# Writes the output of test execution to stdout, stderr +# Test xUnit report to ${TEST_XML_OUTPUT_DIR}/${test_name}/sponge_log.xml +####################################### +run_test() { + # Test driver usage: + # https://github.com/grpc/grpc/tree/master/tools/run_tests/xds_k8s_test_driver#basic-usage + local test_name="${1:?Usage: run_test test_name}" + local out_dir="${TEST_XML_OUTPUT_DIR}/${test_name}" + mkdir -pv "${out_dir}" + set -x + python -m "tests.${test_name}" \ + --flagfile="${TEST_DRIVER_FLAGFILE}" \ + --flagfile="config/url-map.cfg" \ + --kube_context="${KUBE_CONTEXT}" \ + --client_image="${CLIENT_IMAGE_NAME}:${GIT_COMMIT}" \ + --testing_version="${TESTING_VERSION}" \ + --collect_app_logs \ + --log_dir="${out_dir}" \ + --xml_output_file="${out_dir}/sponge_log.xml" \ + |& tee "${out_dir}/sponge_log.log" +} + +####################################### +# Main function: provision software necessary to execute tests, and run them +# Globals: +# KOKORO_ARTIFACTS_DIR +# GITHUB_REPOSITORY_NAME +# SRC_DIR: Populated with absolute path to the source repo +# TEST_DRIVER_REPO_DIR: Populated with the path to the repo containing +# the test driver +# TEST_DRIVER_FULL_DIR: Populated with the path to the test driver source code +# TEST_DRIVER_FLAGFILE: Populated with relative path to test driver flagfile +# TEST_XML_OUTPUT_DIR: Populated with the path to test xUnit XML report +# GIT_ORIGIN_URL: Populated with the origin URL of git repo used for the build +# GIT_COMMIT: Populated with the SHA-1 of git commit being built +# GIT_COMMIT_SHORT: Populated with the short SHA-1 of git commit being built +# KUBE_CONTEXT: Populated with name of kubectl context with GKE cluster access +# Arguments: +# None +# Outputs: +# Writes the output of test execution to stdout, stderr +####################################### +main() { + local script_dir + script_dir="$(dirname "$0")" + + # Source the test driver from the master branch. + echo "Sourcing test driver install script from: ${TEST_DRIVER_INSTALL_SCRIPT_URL}" + source /dev/stdin <<< "$(curl -s "${TEST_DRIVER_INSTALL_SCRIPT_URL}")" + + activate_gke_cluster GKE_CLUSTER_PSM_BASIC + + set -x + if [[ -n "${KOKORO_ARTIFACTS_DIR}" ]]; then + kokoro_setup_test_driver "${GITHUB_REPOSITORY_NAME}" + else + local_setup_test_driver "${script_dir}" + fi + build_docker_images_if_needed + # Run tests + cd "${TEST_DRIVER_FULL_DIR}" + run_test url_map || echo "Failed url_map test" +} + +main "$@" diff --git a/test/kokoro/xds_v3.cfg b/test/kokoro/xds_v3.cfg new file mode 100644 index 000000000000..1991efd325d3 --- /dev/null +++ b/test/kokoro/xds_v3.cfg @@ -0,0 +1,11 @@ +# Config file for internal CI + +# Location of the continuous shell script in repository. +build_file: "grpc-go/test/kokoro/xds_v3.sh" +timeout_mins: 360 +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + } +} diff --git a/test/kokoro/xds_v3.sh b/test/kokoro/xds_v3.sh new file mode 100755 index 000000000000..73eb50d248a2 --- /dev/null +++ b/test/kokoro/xds_v3.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +XDS_V3_OPT="--xds_v3_support" `dirname $0`/xds.sh diff --git a/test/local_creds_test.go b/test/local_creds_test.go index 12af20b2b34c..b5186971490a 100644 --- a/test/local_creds_test.go +++ b/test/local_creds_test.go @@ -29,23 +29,33 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/credentials/local" + "google.golang.org/grpc/internal/stubserver" "google.golang.org/grpc/peer" "google.golang.org/grpc/status" - testpb "google.golang.org/grpc/test/grpc_testing" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" ) func testLocalCredsE2ESucceed(network, address string) error { - ss := &stubServer{ - emptyCall: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { pr, ok := peer.FromContext(ctx) if !ok { return nil, status.Error(codes.DataLoss, "Failed to get peer from ctx") } + type internalInfo interface { + GetCommonAuthInfo() credentials.CommonAuthInfo + } + var secLevel credentials.SecurityLevel + if info, ok := (pr.AuthInfo).(internalInfo); ok { + secLevel = info.GetCommonAuthInfo().SecurityLevel + } else { + return nil, status.Errorf(codes.Unauthenticated, "peer.AuthInfo does not implement GetCommonAuthInfo()") + } // Check security level - info := pr.AuthInfo.(local.Info) - secLevel := info.CommonAuthInfo.SecurityLevel switch network { case "unix": if secLevel != credentials.PrivacyAndIntegrity { @@ -64,7 +74,7 @@ func testLocalCredsE2ESucceed(network, address string) error { s := grpc.NewServer(sopts...) defer s.Stop() - testpb.RegisterTestServiceService(s, testpb.NewTestServiceService(ss)) + testgrpc.RegisterTestServiceServer(s, ss) lis, err := net.Listen(network, address) if err != nil { @@ -92,8 +102,8 @@ func testLocalCredsE2ESucceed(network, address string) error { } defer cc.Close() - c := testpb.NewTestServiceClient(cc) - ctx, cancel := context.WithTimeout(context.Background(), time.Second) + c := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() if _, err = c.EmptyCall(ctx, &testpb.Empty{}); err != nil { @@ -152,8 +162,8 @@ func spoofDialer(addr net.Addr) func(target string, t time.Duration) (net.Conn, } func testLocalCredsE2EFail(dopts []grpc.DialOption) error { - ss := &stubServer{ - emptyCall: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + ss := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil }, } @@ -162,7 +172,7 @@ func testLocalCredsE2EFail(dopts []grpc.DialOption) error { s := grpc.NewServer(sopts...) defer s.Stop() - testpb.RegisterTestServiceService(s, testpb.NewTestServiceService(ss)) + testgrpc.RegisterTestServiceServer(s, ss) lis, err := net.Listen("tcp", "localhost:0") if err != nil { @@ -187,8 +197,8 @@ func testLocalCredsE2EFail(dopts []grpc.DialOption) error { } defer cc.Close() - c := testpb.NewTestServiceClient(cc) - ctx, cancel := context.WithTimeout(context.Background(), time.Second) + c := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() _, err = c.EmptyCall(ctx, &testpb.Empty{}) @@ -210,7 +220,7 @@ func (s) TestLocalCredsClientFail(t *testing.T) { func (s) TestLocalCredsServerFail(t *testing.T) { // Use insecure at client-side which should lead to server-side failure. - opts := []grpc.DialOption{grpc.WithInsecure()} + opts := []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())} if err := testLocalCredsE2EFail(opts); status.Code(err) != codes.Unavailable { t.Fatalf("testLocalCredsE2EFail() = %v; want %v", err, codes.Unavailable) } diff --git a/test/metadata_test.go b/test/metadata_test.go new file mode 100644 index 000000000000..8153ef5be0bf --- /dev/null +++ b/test/metadata_test.go @@ -0,0 +1,155 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "context" + "fmt" + "io" + "reflect" + "strings" + "testing" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +func (s) TestInvalidMetadata(t *testing.T) { + grpctest.TLogger.ExpectErrorN("stream: failed to validate md when setting trailer", 5) + + tests := []struct { + name string + md metadata.MD + appendMD []string + want error + recv error + }{ + { + name: "invalid key", + md: map[string][]string{string(rune(0x19)): {"testVal"}}, + want: status.Error(codes.Internal, "header key \"\\x19\" contains illegal characters not in [0-9a-z-_.]"), + recv: status.Error(codes.Internal, "invalid header field"), + }, + { + name: "invalid value", + md: map[string][]string{"test": {string(rune(0x19))}}, + want: status.Error(codes.Internal, "header key \"test\" contains value with non-printable ASCII characters"), + recv: status.Error(codes.Internal, "invalid header field"), + }, + { + name: "invalid appended value", + md: map[string][]string{"test": {"test"}}, + appendMD: []string{"/", "value"}, + want: status.Error(codes.Internal, "header key \"/\" contains illegal characters not in [0-9a-z-_.]"), + recv: status.Error(codes.Internal, "invalid header field"), + }, + { + name: "empty appended key", + md: map[string][]string{"test": {"test"}}, + appendMD: []string{"", "value"}, + want: status.Error(codes.Internal, "there is an empty key in the header"), + recv: status.Error(codes.Internal, "invalid header field"), + }, + { + name: "empty key", + md: map[string][]string{"": {"test"}}, + want: status.Error(codes.Internal, "there is an empty key in the header"), + recv: status.Error(codes.Internal, "invalid header field"), + }, + { + name: "-bin key with arbitrary value", + md: map[string][]string{"test-bin": {string(rune(0x19))}}, + want: nil, + recv: io.EOF, + }, + { + name: "valid key and value", + md: map[string][]string{"test": {"value"}}, + want: nil, + recv: io.EOF, + }, + } + + testNum := 0 + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + _, err := stream.Recv() + if err != nil { + return err + } + test := tests[testNum] + testNum++ + // merge original md and added md. + md := metadata.Join(test.md, metadata.Pairs(test.appendMD...)) + + if err := stream.SetHeader(md); !reflect.DeepEqual(test.want, err) { + return fmt.Errorf("call stream.SendHeader(md) validate metadata which is %v got err :%v, want err :%v", md, err, test.want) + } + if err := stream.SendHeader(md); !reflect.DeepEqual(test.want, err) { + return fmt.Errorf("call stream.SendHeader(md) validate metadata which is %v got err :%v, want err :%v", md, err, test.want) + } + stream.SetTrailer(md) + return nil + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting ss endpoint server: %v", err) + } + defer ss.Stop() + + for _, test := range tests { + t.Run("unary "+test.name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + ctx = metadata.NewOutgoingContext(ctx, test.md) + ctx = metadata.AppendToOutgoingContext(ctx, test.appendMD...) + if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); !reflect.DeepEqual(test.want, err) { + t.Errorf("call ss.Client.EmptyCall() validate metadata which is %v got err :%v, want err :%v", test.md, err, test.want) + } + }) + } + + // call the stream server's api to drive the server-side unit testing + for _, test := range tests { + t.Run("streaming "+test.name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + stream, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Errorf("call ss.Client.FullDuplexCall got err :%v", err) + return + } + if err := stream.Send(&testpb.StreamingOutputCallRequest{}); err != nil { + t.Errorf("call ss.Client stream Send(nil) will success but got err :%v", err) + } + if _, err := stream.Recv(); status.Code(err) != status.Code(test.recv) || !strings.Contains(err.Error(), test.recv.Error()) { + t.Errorf("stream.Recv() = _, get err :%v, want err :%v", err, test.recv) + } + }) + } +} diff --git a/test/parse_config.go b/test/parse_config.go new file mode 100644 index 000000000000..f375a3aa8a18 --- /dev/null +++ b/test/parse_config.go @@ -0,0 +1,38 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "testing" + + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/serviceconfig" +) + +// parseServiceConfig is a test helper which uses the manual resolver to parse +// the given service config. It calls t.Fatal() if service config parsing fails. +func parseServiceConfig(t *testing.T, r *manual.Resolver, sc string) *serviceconfig.ParseResult { + t.Helper() + + scpr := r.CC.ParseServiceConfig(sc) + if scpr.Err != nil { + t.Fatalf("Failed to parse service config %q: %v", sc, scpr.Err) + } + return scpr +} diff --git a/test/pickfirst_test.go b/test/pickfirst_test.go new file mode 100644 index 000000000000..fc9e1c48f352 --- /dev/null +++ b/test/pickfirst_test.go @@ -0,0 +1,1000 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/backoff" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/channelz" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/grpcrand" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/pickfirst" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/serviceconfig" + "google.golang.org/grpc/status" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +const pickFirstServiceConfig = `{"loadBalancingConfig": [{"pick_first":{}}]}` + +// setupPickFirst performs steps required for pick_first tests. It starts a +// bunch of backends exporting the TestService, creates a ClientConn to them +// with service config specifying the use of the pick_first LB policy. +func setupPickFirst(t *testing.T, backendCount int, opts ...grpc.DialOption) (*grpc.ClientConn, *manual.Resolver, []*stubserver.StubServer) { + t.Helper() + + r := manual.NewBuilderWithScheme("whatever") + + backends := make([]*stubserver.StubServer, backendCount) + addrs := make([]resolver.Address, backendCount) + for i := 0; i < backendCount; i++ { + backend := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + } + if err := backend.StartServer(); err != nil { + t.Fatalf("Failed to start backend: %v", err) + } + t.Logf("Started TestService backend at: %q", backend.Address) + t.Cleanup(func() { backend.Stop() }) + + backends[i] = backend + addrs[i] = resolver.Address{Addr: backend.Address} + } + + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(r), + grpc.WithDefaultServiceConfig(pickFirstServiceConfig), + } + dopts = append(dopts, opts...) + cc, err := grpc.Dial(r.Scheme()+":///test.server", dopts...) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) + } + t.Cleanup(func() { cc.Close() }) + + // At this point, the resolver has not returned any addresses to the channel. + // This RPC must block until the context expires. + sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(sCtx, &testpb.Empty{}); status.Code(err) != codes.DeadlineExceeded { + t.Fatalf("EmptyCall() = %s, want %s", status.Code(err), codes.DeadlineExceeded) + } + return cc, r, backends +} + +// stubBackendsToResolverAddrs converts from a set of stub server backends to +// resolver addresses. Useful when pushing addresses to the manual resolver. +func stubBackendsToResolverAddrs(backends []*stubserver.StubServer) []resolver.Address { + addrs := make([]resolver.Address, len(backends)) + for i, backend := range backends { + addrs[i] = resolver.Address{Addr: backend.Address} + } + return addrs +} + +// TestPickFirst_OneBackend tests the most basic scenario for pick_first. It +// brings up a single backend and verifies that all RPCs get routed to it. +func (s) TestPickFirst_OneBackend(t *testing.T) { + cc, r, backends := setupPickFirst(t, 1) + + addrs := stubBackendsToResolverAddrs(backends) + r.UpdateState(resolver.State{Addresses: addrs}) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } +} + +// TestPickFirst_MultipleBackends tests the scenario with multiple backends and +// verifies that all RPCs get routed to the first one. +func (s) TestPickFirst_MultipleBackends(t *testing.T) { + cc, r, backends := setupPickFirst(t, 2) + + addrs := stubBackendsToResolverAddrs(backends) + r.UpdateState(resolver.State{Addresses: addrs}) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } +} + +// TestPickFirst_OneServerDown tests the scenario where we have multiple +// backends and pick_first is working as expected. Verifies that RPCs get routed +// to the next backend in the list when the first one goes down. +func (s) TestPickFirst_OneServerDown(t *testing.T) { + cc, r, backends := setupPickFirst(t, 2) + + addrs := stubBackendsToResolverAddrs(backends) + r.UpdateState(resolver.State{Addresses: addrs}) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + // Stop the backend which is currently being used. RPCs should get routed to + // the next backend in the list. + backends[0].Stop() + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { + t.Fatal(err) + } +} + +// TestPickFirst_AllServersDown tests the scenario where we have multiple +// backends and pick_first is working as expected. When all backends go down, +// the test verifies that RPCs fail with appropriate status code. +func (s) TestPickFirst_AllServersDown(t *testing.T) { + cc, r, backends := setupPickFirst(t, 2) + + addrs := stubBackendsToResolverAddrs(backends) + r.UpdateState(resolver.State{Addresses: addrs}) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + for _, b := range backends { + b.Stop() + } + + client := testgrpc.NewTestServiceClient(cc) + for { + if ctx.Err() != nil { + t.Fatalf("channel failed to move to Unavailable after all backends were stopped: %v", ctx.Err()) + } + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) == codes.Unavailable { + return + } + time.Sleep(defaultTestShortTimeout) + } +} + +// TestPickFirst_AddressesRemoved tests the scenario where we have multiple +// backends and pick_first is working as expected. It then verifies that when +// addresses are removed by the name resolver, RPCs get routed appropriately. +func (s) TestPickFirst_AddressesRemoved(t *testing.T) { + cc, r, backends := setupPickFirst(t, 3) + + addrs := stubBackendsToResolverAddrs(backends) + r.UpdateState(resolver.State{Addresses: addrs}) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + // Remove the first backend from the list of addresses originally pushed. + // RPCs should get routed to the first backend in the new list. + r.UpdateState(resolver.State{Addresses: []resolver.Address{addrs[1], addrs[2]}}) + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { + t.Fatal(err) + } + + // Append the backend that we just removed to the end of the list. + // Nothing should change. + r.UpdateState(resolver.State{Addresses: []resolver.Address{addrs[1], addrs[2], addrs[0]}}) + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { + t.Fatal(err) + } + + // Remove the first backend from the existing list of addresses. + // RPCs should get routed to the first backend in the new list. + r.UpdateState(resolver.State{Addresses: []resolver.Address{addrs[2], addrs[0]}}) + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[2]); err != nil { + t.Fatal(err) + } + + // Remove the first backend from the existing list of addresses. + // RPCs should get routed to the first backend in the new list. + r.UpdateState(resolver.State{Addresses: []resolver.Address{addrs[0]}}) + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } +} + +// TestPickFirst_NewAddressWhileBlocking tests the case where pick_first is +// configured on a channel, things are working as expected and then a resolver +// updates removes all addresses. An RPC attempted at this point in time will be +// blocked because there are no valid backends. This test verifies that when new +// backends are added, the RPC is able to complete. +func (s) TestPickFirst_NewAddressWhileBlocking(t *testing.T) { + cc, r, backends := setupPickFirst(t, 2) + addrs := stubBackendsToResolverAddrs(backends) + r.UpdateState(resolver.State{Addresses: addrs}) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + // Send a resolver update with no addresses. This should push the channel into + // TransientFailure. + r.UpdateState(resolver.State{}) + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + + doneCh := make(chan struct{}) + client := testgrpc.NewTestServiceClient(cc) + go func() { + // The channel is currently in TransientFailure and this RPC will block + // until the channel becomes Ready, which will only happen when we push a + // resolver update with a valid backend address. + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Errorf("EmptyCall() = %v, want ", err) + } + close(doneCh) + }() + + // Make sure that there is one pending RPC on the ClientConn before attempting + // to push new addresses through the name resolver. If we don't do this, the + // resolver update can happen before the above goroutine gets to make the RPC. + for { + if err := ctx.Err(); err != nil { + t.Fatal(err) + } + tcs, _ := channelz.GetTopChannels(0, 0) + if len(tcs) != 1 { + t.Fatalf("there should only be one top channel, not %d", len(tcs)) + } + started := tcs[0].ChannelData.CallsStarted + completed := tcs[0].ChannelData.CallsSucceeded + tcs[0].ChannelData.CallsFailed + if (started - completed) == 1 { + break + } + time.Sleep(defaultTestShortTimeout) + } + + // Send a resolver update with a valid backend to push the channel to Ready + // and unblock the above RPC. + r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: backends[0].Address}}}) + + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for blocked RPC to complete") + case <-doneCh: + } +} + +// TestPickFirst_StickyTransientFailure tests the case where pick_first is +// configured on a channel, and the backend is configured to close incoming +// connections as soon as they are accepted. The test verifies that the channel +// enters TransientFailure and stays there. The test also verifies that the +// pick_first LB policy is constantly trying to reconnect to the backend. +func (s) TestPickFirst_StickyTransientFailure(t *testing.T) { + // Spin up a local server which closes the connection as soon as it receives + // one. It also sends a signal on a channel whenver it received a connection. + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("Failed to create listener: %v", err) + } + t.Cleanup(func() { lis.Close() }) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + connCh := make(chan struct{}, 1) + go func() { + for { + conn, err := lis.Accept() + if err != nil { + return + } + select { + case connCh <- struct{}{}: + conn.Close() + case <-ctx.Done(): + return + } + } + }() + + // Dial the above server with a ConnectParams that does a constant backoff + // of defaultTestShortTimeout duration. + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithDefaultServiceConfig(pickFirstServiceConfig), + grpc.WithConnectParams(grpc.ConnectParams{ + Backoff: backoff.Config{ + BaseDelay: defaultTestShortTimeout, + Multiplier: float64(0), + Jitter: float64(0), + MaxDelay: defaultTestShortTimeout, + }, + }), + } + cc, err := grpc.Dial(lis.Addr().String(), dopts...) + if err != nil { + t.Fatalf("Failed to dial server at %q: %v", lis.Addr(), err) + } + t.Cleanup(func() { cc.Close() }) + + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + + // Spawn a goroutine to ensure that the channel stays in TransientFailure. + // The call to cc.WaitForStateChange will return false when the main + // goroutine exits and the context is cancelled. + go func() { + if cc.WaitForStateChange(ctx, connectivity.TransientFailure) { + if state := cc.GetState(); state != connectivity.Shutdown { + t.Errorf("Unexpected state change from TransientFailure to %s", cc.GetState()) + } + } + }() + + // Ensures that the pick_first LB policy is constantly trying to reconnect. + for i := 0; i < 10; i++ { + select { + case <-connCh: + case <-time.After(2 * defaultTestShortTimeout): + t.Error("Timeout when waiting for pick_first to reconnect") + } + } +} + +// Tests the PF LB policy with shuffling enabled. +func (s) TestPickFirst_ShuffleAddressList(t *testing.T) { + defer func(old bool) { envconfig.PickFirstLBConfig = old }(envconfig.PickFirstLBConfig) + envconfig.PickFirstLBConfig = true + const serviceConfig = `{"loadBalancingConfig": [{"pick_first":{ "shuffleAddressList": true }}]}` + + // Install a shuffler that always reverses two entries. + origShuf := grpcrand.Shuffle + defer func() { grpcrand.Shuffle = origShuf }() + grpcrand.Shuffle = func(n int, f func(int, int)) { + if n != 2 { + t.Errorf("Shuffle called with n=%v; want 2", n) + return + } + f(0, 1) // reverse the two addresses + } + + // Set up our backends. + cc, r, backends := setupPickFirst(t, 2) + addrs := stubBackendsToResolverAddrs(backends) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Push an update with both addresses and shuffling disabled. We should + // connect to backend 0. + r.UpdateState(resolver.State{Addresses: []resolver.Address{addrs[0], addrs[1]}}) + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + // Send a config with shuffling enabled. This will reverse the addresses, + // but the channel should still be connected to backend 0. + shufState := resolver.State{ + ServiceConfig: parseServiceConfig(t, r, serviceConfig), + Addresses: []resolver.Address{addrs[0], addrs[1]}, + } + r.UpdateState(shufState) + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + // Send a resolver update with no addresses. This should push the channel + // into TransientFailure. + r.UpdateState(resolver.State{}) + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + + // Send the same config as last time with shuffling enabled. Since we are + // not connected to backend 0, we should connect to backend 1. + r.UpdateState(shufState) + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { + t.Fatal(err) + } +} + +// Tests the PF LB policy with the environment variable support of address list +// shuffling disabled. +func (s) TestPickFirst_ShuffleAddressListDisabled(t *testing.T) { + defer func(old bool) { envconfig.PickFirstLBConfig = old }(envconfig.PickFirstLBConfig) + envconfig.PickFirstLBConfig = false + const serviceConfig = `{"loadBalancingConfig": [{"pick_first":{ "shuffleAddressList": true }}]}` + + // Install a shuffler that always reverses two entries. + origShuf := grpcrand.Shuffle + defer func() { grpcrand.Shuffle = origShuf }() + grpcrand.Shuffle = func(n int, f func(int, int)) { + if n != 2 { + t.Errorf("Shuffle called with n=%v; want 2", n) + return + } + f(0, 1) // reverse the two addresses + } + + // Set up our backends. + cc, r, backends := setupPickFirst(t, 2) + addrs := stubBackendsToResolverAddrs(backends) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Send a config with shuffling enabled. This will reverse the addresses, + // so we should connect to backend 1 if shuffling is supported. However + // with it disabled at the start of the test, we will connect to backend 0 + // instead. + shufState := resolver.State{ + ServiceConfig: parseServiceConfig(t, r, serviceConfig), + Addresses: []resolver.Address{addrs[0], addrs[1]}, + } + r.UpdateState(shufState) + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } +} + +// Test config parsing with the env var turned on and off for various scenarios. +func (s) TestPickFirst_ParseConfig_Success(t *testing.T) { + // Install a shuffler that always reverses two entries. + origShuf := grpcrand.Shuffle + defer func() { grpcrand.Shuffle = origShuf }() + grpcrand.Shuffle = func(n int, f func(int, int)) { + if n != 2 { + t.Errorf("Shuffle called with n=%v; want 2", n) + return + } + f(0, 1) // reverse the two addresses + } + + tests := []struct { + name string + envVar bool + serviceConfig string + wantFirstAddr bool + }{ + { + name: "env var disabled with empty pickfirst config", + envVar: false, + serviceConfig: `{"loadBalancingConfig": [{"pick_first":{}}]}`, + wantFirstAddr: true, + }, + { + name: "env var disabled with non-empty good pickfirst config", + envVar: false, + serviceConfig: `{"loadBalancingConfig": [{"pick_first":{ "shuffleAddressList": true }}]}`, + wantFirstAddr: true, + }, + { + name: "env var disabled with non-empty bad pickfirst config", + envVar: false, + serviceConfig: `{"loadBalancingConfig": [{"pick_first":{ "shuffleAddressList": 666 }}]}`, + wantFirstAddr: true, + }, + { + name: "env var enabled with empty pickfirst config", + envVar: true, + serviceConfig: `{"loadBalancingConfig": [{"pick_first":{}}]}`, + wantFirstAddr: true, + }, + { + name: "env var enabled with empty good pickfirst config", + envVar: true, + serviceConfig: `{"loadBalancingConfig": [{"pick_first":{ "shuffleAddressList": true }}]}`, + wantFirstAddr: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + // Set the env var as specified by the test table. + origPickFirstLBConfig := envconfig.PickFirstLBConfig + envconfig.PickFirstLBConfig = test.envVar + defer func() { envconfig.PickFirstLBConfig = origPickFirstLBConfig }() + + // Set up our backends. + cc, r, backends := setupPickFirst(t, 2) + addrs := stubBackendsToResolverAddrs(backends) + + r.UpdateState(resolver.State{ + ServiceConfig: parseServiceConfig(t, r, test.serviceConfig), + Addresses: addrs, + }) + + // Some tests expect address shuffling to happen, and indicate that + // by setting wantFirstAddr to false (since our shuffling function + // defined at the top of this test, simply reverses the list of + // addresses provided to it). + wantAddr := addrs[0] + if !test.wantFirstAddr { + wantAddr = addrs[1] + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := pickfirst.CheckRPCsToBackend(ctx, cc, wantAddr); err != nil { + t.Fatal(err) + } + }) + } +} + +// Test config parsing for a bad service config. +func (s) TestPickFirst_ParseConfig_Failure(t *testing.T) { + origPickFirstLBConfig := envconfig.PickFirstLBConfig + envconfig.PickFirstLBConfig = true + defer func() { envconfig.PickFirstLBConfig = origPickFirstLBConfig }() + + // Service config should fail with the below config. Name resolvers are + // expected to perform this parsing before they push the parsed service + // config to the channel. + const sc = `{"loadBalancingConfig": [{"pick_first":{ "shuffleAddressList": 666 }}]}` + scpr := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(sc) + if scpr.Err == nil { + t.Fatalf("ParseConfig() succeeded and returned %+v, when expected to fail", scpr) + } +} + +// setupPickFirstWithListenerWrapper is very similar to setupPickFirst, but uses +// a wrapped listener that the test can use to track accepted connections. +func setupPickFirstWithListenerWrapper(t *testing.T, backendCount int, opts ...grpc.DialOption) (*grpc.ClientConn, *manual.Resolver, []*stubserver.StubServer, []*testutils.ListenerWrapper) { + t.Helper() + + backends := make([]*stubserver.StubServer, backendCount) + addrs := make([]resolver.Address, backendCount) + listeners := make([]*testutils.ListenerWrapper, backendCount) + for i := 0; i < backendCount; i++ { + lis := testutils.NewListenerWrapper(t, nil) + backend := &stubserver.StubServer{ + Listener: lis, + EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + } + if err := backend.StartServer(); err != nil { + t.Fatalf("Failed to start backend: %v", err) + } + t.Logf("Started TestService backend at: %q", backend.Address) + t.Cleanup(func() { backend.Stop() }) + + backends[i] = backend + addrs[i] = resolver.Address{Addr: backend.Address} + listeners[i] = lis + } + + r := manual.NewBuilderWithScheme("whatever") + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(r), + grpc.WithDefaultServiceConfig(pickFirstServiceConfig), + } + dopts = append(dopts, opts...) + cc, err := grpc.Dial(r.Scheme()+":///test.server", dopts...) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) + } + t.Cleanup(func() { cc.Close() }) + + // At this point, the resolver has not returned any addresses to the channel. + // This RPC must block until the context expires. + sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(sCtx, &testpb.Empty{}); status.Code(err) != codes.DeadlineExceeded { + t.Fatalf("EmptyCall() = %s, want %s", status.Code(err), codes.DeadlineExceeded) + } + return cc, r, backends, listeners +} + +// TestPickFirst_AddressUpdateWithAttributes tests the case where an address +// update received by the pick_first LB policy differs in attributes. Addresses +// which differ in attributes are considered different from the perspective of +// subconn creation and connection establishment and the test verifies that new +// connections are created when attributes change. +func (s) TestPickFirst_AddressUpdateWithAttributes(t *testing.T) { + cc, r, backends, listeners := setupPickFirstWithListenerWrapper(t, 2) + + // Add a set of attributes to the addresses before pushing them to the + // pick_first LB policy through the manual resolver. + addrs := stubBackendsToResolverAddrs(backends) + for i := range addrs { + addrs[i].Attributes = addrs[i].Attributes.WithValue("test-attribute-1", fmt.Sprintf("%d", i)) + } + r.UpdateState(resolver.State{Addresses: addrs}) + + // Ensure that RPCs succeed to the first backend in the list. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + // Grab the wrapped connection from the listener wrapper. This will be used + // to verify the connection is closed. + val, err := listeners[0].NewConnCh.Receive(ctx) + if err != nil { + t.Fatalf("Failed to receive new connection from wrapped listener: %v", err) + } + conn := val.(*testutils.ConnWrapper) + + // Add another set of attributes to the addresses, and push them to the + // pick_first LB policy through the manual resolver. Leave the order of the + // addresses unchanged. + for i := range addrs { + addrs[i].Attributes = addrs[i].Attributes.WithValue("test-attribute-2", fmt.Sprintf("%d", i)) + } + r.UpdateState(resolver.State{Addresses: addrs}) + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + // A change in the address attributes results in the new address being + // considered different to the current address. This will result in the old + // connection being closed and a new connection to the same backend (since + // address order is not modified). + if _, err := conn.CloseCh.Receive(ctx); err != nil { + t.Fatalf("Timeout when expecting existing connection to be closed: %v", err) + } + val, err = listeners[0].NewConnCh.Receive(ctx) + if err != nil { + t.Fatalf("Failed to receive new connection from wrapped listener: %v", err) + } + conn = val.(*testutils.ConnWrapper) + + // Add another set of attributes to the addresses, and push them to the + // pick_first LB policy through the manual resolver. Reverse of the order + // of addresses. + for i := range addrs { + addrs[i].Attributes = addrs[i].Attributes.WithValue("test-attribute-3", fmt.Sprintf("%d", i)) + } + addrs[0], addrs[1] = addrs[1], addrs[0] + r.UpdateState(resolver.State{Addresses: addrs}) + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + // Ensure that the old connection is closed and a new connection is + // established to the first address in the new list. + if _, err := conn.CloseCh.Receive(ctx); err != nil { + t.Fatalf("Timeout when expecting existing connection to be closed: %v", err) + } + _, err = listeners[1].NewConnCh.Receive(ctx) + if err != nil { + t.Fatalf("Failed to receive new connection from wrapped listener: %v", err) + } +} + +// TestPickFirst_AddressUpdateWithBalancerAttributes tests the case where an +// address update received by the pick_first LB policy differs in balancer +// attributes, which are meant only for consumption by LB policies. In this +// case, the test verifies that new connections are not created when the address +// update only changes the balancer attributes. +func (s) TestPickFirst_AddressUpdateWithBalancerAttributes(t *testing.T) { + cc, r, backends, listeners := setupPickFirstWithListenerWrapper(t, 2) + + // Add a set of balancer attributes to the addresses before pushing them to + // the pick_first LB policy through the manual resolver. + addrs := stubBackendsToResolverAddrs(backends) + for i := range addrs { + addrs[i].BalancerAttributes = addrs[i].BalancerAttributes.WithValue("test-attribute-1", fmt.Sprintf("%d", i)) + } + r.UpdateState(resolver.State{Addresses: addrs}) + + // Ensure that RPCs succeed to the expected backend. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + // Grab the wrapped connection from the listener wrapper. This will be used + // to verify the connection is not closed. + val, err := listeners[0].NewConnCh.Receive(ctx) + if err != nil { + t.Fatalf("Failed to receive new connection from wrapped listener: %v", err) + } + conn := val.(*testutils.ConnWrapper) + + // Add a set of balancer attributes to the addresses before pushing them to + // the pick_first LB policy through the manual resolver. Leave the order of + // the addresses unchanged. + for i := range addrs { + addrs[i].BalancerAttributes = addrs[i].BalancerAttributes.WithValue("test-attribute-2", fmt.Sprintf("%d", i)) + } + r.UpdateState(resolver.State{Addresses: addrs}) + + // Ensure that no new connection is established, and ensure that the old + // connection is not closed. + for i := range listeners { + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := listeners[i].NewConnCh.Receive(sCtx); err != context.DeadlineExceeded { + t.Fatalf("Unexpected error when expecting no new connection: %v", err) + } + } + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := conn.CloseCh.Receive(sCtx); err != context.DeadlineExceeded { + t.Fatalf("Unexpected error when expecting existing connection to stay active: %v", err) + } + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + // Add a set of balancer attributes to the addresses before pushing them to + // the pick_first LB policy through the manual resolver. Reverse of the + // order of addresses. + for i := range addrs { + addrs[i].BalancerAttributes = addrs[i].BalancerAttributes.WithValue("test-attribute-3", fmt.Sprintf("%d", i)) + } + addrs[0], addrs[1] = addrs[1], addrs[0] + r.UpdateState(resolver.State{Addresses: addrs}) + + // Ensure that no new connection is established, and ensure that the old + // connection is not closed. + for i := range listeners { + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := listeners[i].NewConnCh.Receive(sCtx); err != context.DeadlineExceeded { + t.Fatalf("Unexpected error when expecting no new connection: %v", err) + } + } + sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := conn.CloseCh.Receive(sCtx); err != context.DeadlineExceeded { + t.Fatalf("Unexpected error when expecting existing connection to stay active: %v", err) + } + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[1]); err != nil { + t.Fatal(err) + } +} + +// Tests the case where the pick_first LB policy receives an error from the name +// resolver without previously receiving a good update. Verifies that the +// channel moves to TRANSIENT_FAILURE and that error received from the name +// resolver is propagated to the caller of an RPC. +func (s) TestPickFirst_ResolverError_NoPreviousUpdate(t *testing.T) { + cc, r, _ := setupPickFirst(t, 0) + + nrErr := errors.New("error from name resolver") + r.ReportError(nrErr) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + + client := testgrpc.NewTestServiceClient(cc) + _, err := client.EmptyCall(ctx, &testpb.Empty{}) + if err == nil { + t.Fatalf("EmptyCall() succeeded when expected to fail with error: %v", nrErr) + } + if !strings.Contains(err.Error(), nrErr.Error()) { + t.Fatalf("EmptyCall() failed with error: %v, want error: %v", err, nrErr) + } +} + +// Tests the case where the pick_first LB policy receives an error from the name +// resolver after receiving a good update (and the channel is currently READY). +// The test verifies that the channel continues to use the previously received +// good update. +func (s) TestPickFirst_ResolverError_WithPreviousUpdate_Ready(t *testing.T) { + cc, r, backends := setupPickFirst(t, 1) + + addrs := stubBackendsToResolverAddrs(backends) + r.UpdateState(resolver.State{Addresses: addrs}) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + nrErr := errors.New("error from name resolver") + r.ReportError(nrErr) + + // Ensure that RPCs continue to succeed for the next second. + client := testgrpc.NewTestServiceClient(cc) + for end := time.Now().Add(time.Second); time.Now().Before(end); <-time.After(defaultTestShortTimeout) { + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + } +} + +// Tests the case where the pick_first LB policy receives an error from the name +// resolver after receiving a good update (and the channel is currently in +// CONNECTING state). The test verifies that the channel continues to use the +// previously received good update, and that RPCs don't fail with the error +// received from the name resolver. +func (s) TestPickFirst_ResolverError_WithPreviousUpdate_Connecting(t *testing.T) { + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("net.Listen() failed: %v", err) + } + + // Listen on a local port and act like a server that blocks until the + // channel reaches CONNECTING and closes the connection without sending a + // server preface. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + waitForConnecting := make(chan struct{}) + go func() { + conn, err := lis.Accept() + if err != nil { + t.Errorf("Unexpected error when accepting a connection: %v", err) + } + defer conn.Close() + + select { + case <-waitForConnecting: + case <-ctx.Done(): + t.Error("Timeout when waiting for channel to move to CONNECTING state") + } + }() + + r := manual.NewBuilderWithScheme("whatever") + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(r), + grpc.WithDefaultServiceConfig(pickFirstServiceConfig), + } + cc, err := grpc.Dial(r.Scheme()+":///test.server", dopts...) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) + } + t.Cleanup(func() { cc.Close() }) + + addrs := []resolver.Address{{Addr: lis.Addr().String()}} + r.UpdateState(resolver.State{Addresses: addrs}) + testutils.AwaitState(ctx, t, cc, connectivity.Connecting) + + nrErr := errors.New("error from name resolver") + r.ReportError(nrErr) + + // RPCs should fail with deadline exceed error as long as they are in + // CONNECTING and not the error returned by the name resolver. + client := testgrpc.NewTestServiceClient(cc) + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := client.EmptyCall(sCtx, &testpb.Empty{}); !strings.Contains(err.Error(), context.DeadlineExceeded.Error()) { + t.Fatalf("EmptyCall() failed with error: %v, want error: %v", err, context.DeadlineExceeded) + } + + // Closing this channel leads to closing of the connection by our listener. + // gRPC should see this as a connection error. + close(waitForConnecting) + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + checkForConnectionError(ctx, t, cc) +} + +// Tests the case where the pick_first LB policy receives an error from the name +// resolver after receiving a good update. The previous good update though has +// seen the channel move to TRANSIENT_FAILURE. The test verifies that the +// channel fails RPCs with the new error from the resolver. +func (s) TestPickFirst_ResolverError_WithPreviousUpdate_TransientFailure(t *testing.T) { + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("net.Listen() failed: %v", err) + } + + // Listen on a local port and act like a server that closes the connection + // without sending a server preface. + go func() { + conn, err := lis.Accept() + if err != nil { + t.Errorf("Unexpected error when accepting a connection: %v", err) + } + conn.Close() + }() + + r := manual.NewBuilderWithScheme("whatever") + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(r), + grpc.WithDefaultServiceConfig(pickFirstServiceConfig), + } + cc, err := grpc.Dial(r.Scheme()+":///test.server", dopts...) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) + } + t.Cleanup(func() { cc.Close() }) + + addrs := []resolver.Address{{Addr: lis.Addr().String()}} + r.UpdateState(resolver.State{Addresses: addrs}) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + checkForConnectionError(ctx, t, cc) + + // An error from the name resolver should result in RPCs failing with that + // error instead of the old error that caused the channel to move to + // TRANSIENT_FAILURE in the first place. + nrErr := errors.New("error from name resolver") + r.ReportError(nrErr) + client := testgrpc.NewTestServiceClient(cc) + for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) { + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); strings.Contains(err.Error(), nrErr.Error()) { + break + } + } + if ctx.Err() != nil { + t.Fatal("Timeout when waiting for RPCs to fail with error returned by the name resolver") + } +} + +func checkForConnectionError(ctx context.Context, t *testing.T, cc *grpc.ClientConn) { + t.Helper() + + // RPCs may fail on the client side in two ways, once the fake server closes + // the accepted connection: + // - writing the client preface succeeds, but not reading the server preface + // - writing the client preface fails + // In either case, we should see it fail with UNAVAILABLE. + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.Unavailable { + t.Fatalf("EmptyCall() failed with error: %v, want code %v", err, codes.Unavailable) + } +} + +// Tests the case where the pick_first LB policy receives an update from the +// name resolver with no addresses after receiving a good update. The test +// verifies that the channel fails RPCs with an error indicating the fact that +// the name resolver returned no addresses. +func (s) TestPickFirst_ResolverError_ZeroAddresses_WithPreviousUpdate(t *testing.T) { + cc, r, backends := setupPickFirst(t, 1) + + addrs := stubBackendsToResolverAddrs(backends) + r.UpdateState(resolver.State{Addresses: addrs}) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := pickfirst.CheckRPCsToBackend(ctx, cc, addrs[0]); err != nil { + t.Fatal(err) + } + + r.UpdateState(resolver.State{}) + wantErr := "produced zero addresses" + client := testgrpc.NewTestServiceClient(cc) + for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) { + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); strings.Contains(err.Error(), wantErr) { + break + } + } + if ctx.Err() != nil { + t.Fatal("Timeout when waiting for RPCs to fail with error returned by the name resolver") + } +} diff --git a/test/race.go b/test/race_test.go similarity index 97% rename from test/race.go rename to test/race_test.go index acfa0dfae37c..d99f0a410ac6 100644 --- a/test/race.go +++ b/test/race_test.go @@ -1,3 +1,4 @@ +//go:build race // +build race /* diff --git a/test/recv_buffer_pool_test.go b/test/recv_buffer_pool_test.go new file mode 100644 index 000000000000..9e7b4aaaea6b --- /dev/null +++ b/test/recv_buffer_pool_test.go @@ -0,0 +1,89 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "bytes" + "context" + "io" + "testing" + + "google.golang.org/grpc" + "google.golang.org/grpc/internal/stubserver" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +func (s) TestRecvBufferPool(t *testing.T) { + ss := &stubserver.StubServer{ + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + for i := 0; i < 10; i++ { + preparedMsg := &grpc.PreparedMsg{} + err := preparedMsg.Encode(stream, &testpb.StreamingOutputCallResponse{ + Payload: &testpb.Payload{ + Body: []byte{'0' + uint8(i)}, + }, + }) + if err != nil { + return err + } + stream.SendMsg(preparedMsg) + } + return nil + }, + } + if err := ss.Start( + []grpc.ServerOption{grpc.RecvBufferPool(grpc.NewSharedBufferPool())}, + grpc.WithRecvBufferPool(grpc.NewSharedBufferPool()), + ); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + stream, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("ss.Client.FullDuplexCall failed: %f", err) + } + + var ngot int + var buf bytes.Buffer + for { + reply, err := stream.Recv() + if err == io.EOF { + break + } + if err != nil { + t.Fatal(err) + } + ngot++ + if buf.Len() > 0 { + buf.WriteByte(',') + } + buf.Write(reply.GetPayload().GetBody()) + } + if want := 10; ngot != want { + t.Errorf("Got %d replies, want %d", ngot, want) + } + if got, want := buf.String(), "0,1,2,3,4,5,6,7,8,9"; got != want { + t.Errorf("Got replies %q; want %q", got, want) + } +} diff --git a/test/resolver_update_test.go b/test/resolver_update_test.go new file mode 100644 index 000000000000..e9ae0df6ebd8 --- /dev/null +++ b/test/resolver_update_test.go @@ -0,0 +1,259 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/balancer/stub" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/serviceconfig" + "google.golang.org/grpc/status" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +// TestResolverUpdateDuringBuild_ServiceConfigParseError makes the +// resolver.Builder call into the ClientConn, during the Build call, with a +// service config parsing error. +// +// We use two separate mutexes in the code which make sure there is no data race +// in this code path, and also that there is no deadlock. +func (s) TestResolverUpdateDuringBuild_ServiceConfigParseError(t *testing.T) { + // Setting InitialState on the manual resolver makes it call into the + // ClientConn during the Build call. + r := manual.NewBuilderWithScheme("whatever") + r.InitialState(resolver.State{ServiceConfig: &serviceconfig.ParseResult{Err: errors.New("resolver build err")}}) + + cc, err := grpc.Dial(r.Scheme()+":///test.server", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("Dial(_, _) = _, %v; want _, nil", err) + } + defer cc.Close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + client := testgrpc.NewTestServiceClient(cc) + const wantMsg = "error parsing service config" + const wantCode = codes.Unavailable + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != wantCode || !strings.Contains(status.Convert(err).Message(), wantMsg) { + t.Fatalf("EmptyCall RPC failed: %v; want code: %v, want message: %q", err, wantCode, wantMsg) + } +} + +type fakeConfig struct { + serviceconfig.Config +} + +// TestResolverUpdateDuringBuild_ServiceConfigInvalidTypeError makes the +// resolver.Builder call into the ClientConn, during the Build call, with an +// invalid service config type. +// +// We use two separate mutexes in the code which make sure there is no data race +// in this code path, and also that there is no deadlock. +func (s) TestResolverUpdateDuringBuild_ServiceConfigInvalidTypeError(t *testing.T) { + // Setting InitialState on the manual resolver makes it call into the + // ClientConn during the Build call. + r := manual.NewBuilderWithScheme("whatever") + r.InitialState(resolver.State{ServiceConfig: &serviceconfig.ParseResult{Config: fakeConfig{}}}) + + cc, err := grpc.Dial(r.Scheme()+":///test.server", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("Dial(_, _) = _, %v; want _, nil", err) + } + defer cc.Close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + client := testgrpc.NewTestServiceClient(cc) + const wantMsg = "illegal service config type" + const wantCode = codes.Unavailable + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != wantCode || !strings.Contains(status.Convert(err).Message(), wantMsg) { + t.Fatalf("EmptyCall RPC failed: %v; want code: %v, want message: %q", err, wantCode, wantMsg) + } +} + +// TestResolverUpdate_InvalidServiceConfigAsFirstUpdate makes the resolver send +// an update with an invalid service config as its first update. This should +// make the ClientConn apply the failing LB policy, and should result in RPC +// errors indicating the failing service config. +func (s) TestResolverUpdate_InvalidServiceConfigAsFirstUpdate(t *testing.T) { + r := manual.NewBuilderWithScheme("whatever") + + cc, err := grpc.Dial(r.Scheme()+":///test.server", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("Dial(_, _) = _, %v; want _, nil", err) + } + defer cc.Close() + + scpr := r.CC.ParseServiceConfig("bad json service config") + r.UpdateState(resolver.State{ServiceConfig: scpr}) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + client := testgrpc.NewTestServiceClient(cc) + const wantMsg = "error parsing service config" + const wantCode = codes.Unavailable + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != wantCode || !strings.Contains(status.Convert(err).Message(), wantMsg) { + t.Fatalf("EmptyCall RPC failed: %v; want code: %v, want message: %q", err, wantCode, wantMsg) + } +} + +func verifyClientConnStateUpdate(got, want balancer.ClientConnState) error { + if got, want := got.ResolverState.Addresses, want.ResolverState.Addresses; !cmp.Equal(got, want) { + return fmt.Errorf("update got unexpected addresses: %v, want %v", got, want) + } + if got, want := got.ResolverState.ServiceConfig.Config, want.ResolverState.ServiceConfig.Config; !internal.EqualServiceConfigForTesting(got, want) { + return fmt.Errorf("received unexpected service config: \ngot: %v \nwant: %v", got, want) + } + if got, want := got.BalancerConfig, want.BalancerConfig; !cmp.Equal(got, want) { + return fmt.Errorf("received unexpected balancer config: \ngot: %v \nwant: %v", cmp.Diff(nil, got), cmp.Diff(nil, want)) + } + return nil +} + +// TestResolverUpdate_InvalidServiceConfigAfterGoodUpdate tests the scenario +// where the resolver sends an update with an invalid service config after +// having sent a good update. This should result in the ClientConn discarding +// the new invalid service config, and continuing to use the old good config. +func (s) TestResolverUpdate_InvalidServiceConfigAfterGoodUpdate(t *testing.T) { + type wrappingBalancerConfig struct { + serviceconfig.LoadBalancingConfig + Config string `json:"config,omitempty"` + } + + // Register a stub balancer which uses a "pick_first" balancer underneath and + // signals on a channel when it receives ClientConn updates. + ccUpdateCh := testutils.NewChannel() + stub.Register(t.Name(), stub.BalancerFuncs{ + Init: func(bd *stub.BalancerData) { + pf := balancer.Get(grpc.PickFirstBalancerName) + bd.Data = pf.Build(bd.ClientConn, bd.BuildOptions) + }, + ParseConfig: func(lbCfg json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + cfg := &wrappingBalancerConfig{} + if err := json.Unmarshal(lbCfg, cfg); err != nil { + return nil, err + } + return cfg, nil + }, + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + if _, ok := ccs.BalancerConfig.(*wrappingBalancerConfig); !ok { + return fmt.Errorf("received balancer config of unsupported type %T", ccs.BalancerConfig) + } + bal := bd.Data.(balancer.Balancer) + ccUpdateCh.Send(ccs) + ccs.BalancerConfig = nil + return bal.UpdateClientConnState(ccs) + }, + }) + + // Start a backend exposing the test service. + backend := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil }, + } + if err := backend.StartServer(); err != nil { + t.Fatalf("Failed to start backend: %v", err) + } + t.Logf("Started TestService backend at: %q", backend.Address) + defer backend.Stop() + + r := manual.NewBuilderWithScheme("whatever") + + cc, err := grpc.Dial(r.Scheme()+":///test.server", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("Dial(_, _) = _, %v; want _, nil", err) + } + defer cc.Close() + + // Push a resolver update and verify that our balancer receives the update. + addrs := []resolver.Address{{Addr: backend.Address}} + const lbCfg = "wrapping balancer LB policy config" + goodSC := r.CC.ParseServiceConfig(fmt.Sprintf(` +{ + "loadBalancingConfig": [ + { + "%v": { + "config": "%s" + } + } + ] +}`, t.Name(), lbCfg)) + r.UpdateState(resolver.State{Addresses: addrs, ServiceConfig: goodSC}) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + wantCCS := balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: addrs, + ServiceConfig: goodSC, + }, + BalancerConfig: &wrappingBalancerConfig{Config: lbCfg}, + } + ccs, err := ccUpdateCh.Receive(ctx) + if err != nil { + t.Fatalf("Timeout when waiting for ClientConnState update from grpc") + } + gotCCS := ccs.(balancer.ClientConnState) + if err := verifyClientConnStateUpdate(gotCCS, wantCCS); err != nil { + t.Fatal(err) + } + + // Ensure RPCs are successful. + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall RPC failed: %v", err) + } + + // Push a bad resolver update and ensure that the update is propagated to our + // stub balancer. But since the pushed update contains an invalid service + // config, our balancer should continue to see the old loadBalancingConfig. + badSC := r.CC.ParseServiceConfig("bad json service config") + wantCCS.ResolverState.ServiceConfig = badSC + r.UpdateState(resolver.State{Addresses: addrs, ServiceConfig: badSC}) + ccs, err = ccUpdateCh.Receive(ctx) + if err != nil { + t.Fatalf("Timeout when waiting for ClientConnState update from grpc") + } + gotCCS = ccs.(balancer.ClientConnState) + if err := verifyClientConnStateUpdate(gotCCS, wantCCS); err != nil { + t.Fatal(err) + } + + // RPCs should continue to be successful since the ClientConn is using the old + // good service config. + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall RPC failed: %v", err) + } +} diff --git a/test/retry_test.go b/test/retry_test.go index f0ab380e96bc..49becb359097 100644 --- a/test/retry_test.go +++ b/test/retry_test.go @@ -22,32 +22,33 @@ import ( "context" "fmt" "io" - "os" + "net" + "reflect" "strconv" "strings" + "sync" "testing" "time" "github.com/golang/protobuf/proto" "google.golang.org/grpc" "google.golang.org/grpc/codes" - "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/stubserver" "google.golang.org/grpc/metadata" + "google.golang.org/grpc/stats" "google.golang.org/grpc/status" - testpb "google.golang.org/grpc/test/grpc_testing" -) -func enableRetry() func() { - old := envconfig.Retry - envconfig.Retry = true - return func() { envconfig.Retry = old } -} + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) func (s) TestRetryUnary(t *testing.T) { - defer enableRetry()() i := -1 - ss := &stubServer{ - emptyCall: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + ss := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (r *testpb.Empty, err error) { + defer func() { t.Logf("server call %v returning err %v", i, err) }() i++ switch i { case 0, 2, 5: @@ -58,11 +59,8 @@ func (s) TestRetryUnary(t *testing.T) { return nil, status.New(codes.AlreadyExists, "retryable error").Err() }, } - if err := ss.Start([]grpc.ServerOption{}); err != nil { - t.Fatalf("Error starting endpoint server: %v", err) - } - defer ss.Stop() - ss.newServiceConfig(`{ + if err := ss.Start([]grpc.ServerOption{}, + grpc.WithDefaultServiceConfig(`{ "methodConfig": [{ "name": [{"service": "grpc.testing.TestService"}], "waitForReady": true, @@ -73,18 +71,10 @@ func (s) TestRetryUnary(t *testing.T) { "BackoffMultiplier": 1.0, "RetryableStatusCodes": [ "ALREADY_EXISTS" ] } - }]}`) - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - for { - if ctx.Err() != nil { - t.Fatalf("Timed out waiting for service config update") - } - if ss.cc.GetMethodConfig("/grpc.testing.TestService/EmptyCall").WaitForReady != nil { - break - } - time.Sleep(time.Millisecond) + }]}`)); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) } - cancel() + defer ss.Stop() testCases := []struct { code codes.Code @@ -98,71 +88,10 @@ func (s) TestRetryUnary(t *testing.T) { {codes.Internal, 11}, {codes.AlreadyExists, 15}, } - for _, tc := range testCases { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - _, err := ss.client.EmptyCall(ctx, &testpb.Empty{}) - cancel() - if status.Code(err) != tc.code { - t.Fatalf("EmptyCall(_, _) = _, %v; want _, ", err, tc.code) - } - if i != tc.count { - t.Fatalf("i = %v; want %v", i, tc.count) - } - } -} - -func (s) TestRetryDisabledByDefault(t *testing.T) { - if strings.EqualFold(os.Getenv("GRPC_GO_RETRY"), "on") { - return - } - i := -1 - ss := &stubServer{ - emptyCall: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { - i++ - switch i { - case 0: - return nil, status.New(codes.AlreadyExists, "retryable error").Err() - } - return &testpb.Empty{}, nil - }, - } - if err := ss.Start([]grpc.ServerOption{}); err != nil { - t.Fatalf("Error starting endpoint server: %v", err) - } - defer ss.Stop() - ss.newServiceConfig(`{ - "methodConfig": [{ - "name": [{"service": "grpc.testing.TestService"}], - "waitForReady": true, - "retryPolicy": { - "MaxAttempts": 4, - "InitialBackoff": ".01s", - "MaxBackoff": ".01s", - "BackoffMultiplier": 1.0, - "RetryableStatusCodes": [ "ALREADY_EXISTS" ] - } - }]}`) - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - for { - if ctx.Err() != nil { - t.Fatalf("Timed out waiting for service config update") - } - if ss.cc.GetMethodConfig("/grpc.testing.TestService/EmptyCall").WaitForReady != nil { - break - } - time.Sleep(time.Millisecond) - } - cancel() - - testCases := []struct { - code codes.Code - count int - }{ - {codes.AlreadyExists, 0}, - } - for _, tc := range testCases { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - _, err := ss.client.EmptyCall(ctx, &testpb.Empty{}) + for num, tc := range testCases { + t.Log("Case", num) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}) cancel() if status.Code(err) != tc.code { t.Fatalf("EmptyCall(_, _) = _, %v; want _, ", err, tc.code) @@ -174,10 +103,9 @@ func (s) TestRetryDisabledByDefault(t *testing.T) { } func (s) TestRetryThrottling(t *testing.T) { - defer enableRetry()() i := -1 - ss := &stubServer{ - emptyCall: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + ss := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { i++ switch i { case 0, 3, 6, 10, 11, 12, 13, 14, 16, 18: @@ -186,11 +114,8 @@ func (s) TestRetryThrottling(t *testing.T) { return nil, status.New(codes.Unavailable, "retryable error").Err() }, } - if err := ss.Start([]grpc.ServerOption{}); err != nil { - t.Fatalf("Error starting endpoint server: %v", err) - } - defer ss.Stop() - ss.newServiceConfig(`{ + if err := ss.Start([]grpc.ServerOption{}, + grpc.WithDefaultServiceConfig(`{ "methodConfig": [{ "name": [{"service": "grpc.testing.TestService"}], "waitForReady": true, @@ -206,18 +131,10 @@ func (s) TestRetryThrottling(t *testing.T) { "maxTokens": 10, "tokenRatio": 0.5 } - }`) - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - for { - if ctx.Err() != nil { - t.Fatalf("Timed out waiting for service config update") - } - if ss.cc.GetMethodConfig("/grpc.testing.TestService/EmptyCall").WaitForReady != nil { - break - } - time.Sleep(time.Millisecond) + }`)); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) } - cancel() + defer ss.Stop() testCases := []struct { code codes.Code @@ -237,8 +154,8 @@ func (s) TestRetryThrottling(t *testing.T) { {codes.Unavailable, 17}, // tokens = 4.5 } for _, tc := range testCases { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - _, err := ss.client.EmptyCall(ctx, &testpb.Empty{}) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}) cancel() if status.Code(err) != tc.code { t.Errorf("EmptyCall(_, _) = _, %v; want _, ", err, tc.code) @@ -250,7 +167,6 @@ func (s) TestRetryThrottling(t *testing.T) { } func (s) TestRetryStreaming(t *testing.T) { - defer enableRetry()() req := func(b byte) *testpb.StreamingOutputCallRequest { return &testpb.StreamingOutputCallRequest{Payload: &testpb.Payload{Body: []byte{b}}} } @@ -260,12 +176,12 @@ func (s) TestRetryStreaming(t *testing.T) { largePayload, _ := newPayload(testpb.PayloadType_COMPRESSABLE, 500) - type serverOp func(stream testpb.TestService_FullDuplexCallServer) error - type clientOp func(stream testpb.TestService_FullDuplexCallClient) error + type serverOp func(stream testgrpc.TestService_FullDuplexCallServer) error + type clientOp func(stream testgrpc.TestService_FullDuplexCallClient) error // Server Operations sAttempts := func(n int) serverOp { - return func(stream testpb.TestService_FullDuplexCallServer) error { + return func(stream testgrpc.TestService_FullDuplexCallServer) error { const key = "grpc-previous-rpc-attempts" md, ok := metadata.FromIncomingContext(stream.Context()) if !ok { @@ -278,7 +194,7 @@ func (s) TestRetryStreaming(t *testing.T) { } } sReq := func(b byte) serverOp { - return func(stream testpb.TestService_FullDuplexCallServer) error { + return func(stream testgrpc.TestService_FullDuplexCallServer) error { want := req(b) if got, err := stream.Recv(); err != nil || !proto.Equal(got, want) { return status.Errorf(codes.Internal, "server: Recv() = %v, %v; want %v, ", got, err, want) @@ -287,7 +203,7 @@ func (s) TestRetryStreaming(t *testing.T) { } } sReqPayload := func(p *testpb.Payload) serverOp { - return func(stream testpb.TestService_FullDuplexCallServer) error { + return func(stream testgrpc.TestService_FullDuplexCallServer) error { want := &testpb.StreamingOutputCallRequest{Payload: p} if got, err := stream.Recv(); err != nil || !proto.Equal(got, want) { return status.Errorf(codes.Internal, "server: Recv() = %v, %v; want %v, ", got, err, want) @@ -295,8 +211,13 @@ func (s) TestRetryStreaming(t *testing.T) { return nil } } + sHdr := func() serverOp { + return func(stream testgrpc.TestService_FullDuplexCallServer) error { + return stream.SendHeader(metadata.Pairs("test_header", "test_value")) + } + } sRes := func(b byte) serverOp { - return func(stream testpb.TestService_FullDuplexCallServer) error { + return func(stream testgrpc.TestService_FullDuplexCallServer) error { msg := res(b) if err := stream.Send(msg); err != nil { return status.Errorf(codes.Internal, "server: Send(%v) = %v; want ", msg, err) @@ -305,12 +226,12 @@ func (s) TestRetryStreaming(t *testing.T) { } } sErr := func(c codes.Code) serverOp { - return func(stream testpb.TestService_FullDuplexCallServer) error { - return status.New(c, "").Err() + return func(stream testgrpc.TestService_FullDuplexCallServer) error { + return status.New(c, "this is a test error").Err() } } sCloseSend := func() serverOp { - return func(stream testpb.TestService_FullDuplexCallServer) error { + return func(stream testgrpc.TestService_FullDuplexCallServer) error { if msg, err := stream.Recv(); msg != nil || err != io.EOF { return status.Errorf(codes.Internal, "server: Recv() = %v, %v; want , io.EOF", msg, err) } @@ -318,7 +239,7 @@ func (s) TestRetryStreaming(t *testing.T) { } } sPushback := func(s string) serverOp { - return func(stream testpb.TestService_FullDuplexCallServer) error { + return func(stream testgrpc.TestService_FullDuplexCallServer) error { stream.SetTrailer(metadata.MD{"grpc-retry-pushback-ms": []string{s}}) return nil } @@ -326,7 +247,7 @@ func (s) TestRetryStreaming(t *testing.T) { // Client Operations cReq := func(b byte) clientOp { - return func(stream testpb.TestService_FullDuplexCallClient) error { + return func(stream testgrpc.TestService_FullDuplexCallClient) error { msg := req(b) if err := stream.Send(msg); err != nil { return fmt.Errorf("client: Send(%v) = %v; want ", msg, err) @@ -335,7 +256,7 @@ func (s) TestRetryStreaming(t *testing.T) { } } cReqPayload := func(p *testpb.Payload) clientOp { - return func(stream testpb.TestService_FullDuplexCallClient) error { + return func(stream testgrpc.TestService_FullDuplexCallClient) error { msg := &testpb.StreamingOutputCallRequest{Payload: p} if err := stream.Send(msg); err != nil { return fmt.Errorf("client: Send(%v) = %v; want ", msg, err) @@ -344,7 +265,7 @@ func (s) TestRetryStreaming(t *testing.T) { } } cRes := func(b byte) clientOp { - return func(stream testpb.TestService_FullDuplexCallClient) error { + return func(stream testgrpc.TestService_FullDuplexCallClient) error { want := res(b) if got, err := stream.Recv(); err != nil || !proto.Equal(got, want) { return fmt.Errorf("client: Recv() = %v, %v; want %v, ", got, err, want) @@ -353,8 +274,8 @@ func (s) TestRetryStreaming(t *testing.T) { } } cErr := func(c codes.Code) clientOp { - return func(stream testpb.TestService_FullDuplexCallClient) error { - want := status.New(c, "").Err() + return func(stream testgrpc.TestService_FullDuplexCallClient) error { + want := status.New(c, "this is a test error").Err() if c == codes.OK { want = io.EOF } @@ -368,7 +289,7 @@ func (s) TestRetryStreaming(t *testing.T) { } } cCloseSend := func() clientOp { - return func(stream testpb.TestService_FullDuplexCallClient) error { + return func(stream testgrpc.TestService_FullDuplexCallClient) error { if err := stream.CloseSend(); err != nil { return fmt.Errorf("client: CloseSend() = %v; want ", err) } @@ -377,13 +298,13 @@ func (s) TestRetryStreaming(t *testing.T) { } var curTime time.Time cGetTime := func() clientOp { - return func(_ testpb.TestService_FullDuplexCallClient) error { + return func(_ testgrpc.TestService_FullDuplexCallClient) error { curTime = time.Now() return nil } } cCheckElapsed := func(d time.Duration) clientOp { - return func(_ testpb.TestService_FullDuplexCallClient) error { + return func(_ testgrpc.TestService_FullDuplexCallClient) error { if elapsed := time.Since(curTime); elapsed < d { return fmt.Errorf("elapsed time: %v; want >= %v", elapsed, d) } @@ -391,13 +312,18 @@ func (s) TestRetryStreaming(t *testing.T) { } } cHdr := func() clientOp { - return func(stream testpb.TestService_FullDuplexCallClient) error { + return func(stream testgrpc.TestService_FullDuplexCallClient) error { _, err := stream.Header() + if err == io.EOF { + // The stream ended successfully; convert to nil to avoid + // erroring the test case. + err = nil + } return err } } cCtx := func() clientOp { - return func(stream testpb.TestService_FullDuplexCallClient) error { + return func(stream testgrpc.TestService_FullDuplexCallClient) error { stream.Context() return nil } @@ -446,9 +372,13 @@ func (s) TestRetryStreaming(t *testing.T) { sReq(1), sRes(3), sErr(codes.Unavailable), }, clientOps: []clientOp{cReq(1), cRes(3), cErr(codes.Unavailable)}, + }, { + desc: "Retry via ClientStream.Header()", + serverOps: []serverOp{sReq(1), sErr(codes.Unavailable), sReq(1), sAttempts(1)}, + clientOps: []clientOp{cReq(1), cHdr() /* this should cause a retry */, cErr(codes.OK)}, }, { desc: "No retry after header", - serverOps: []serverOp{sReq(1), sErr(codes.Unavailable)}, + serverOps: []serverOp{sReq(1), sHdr(), sErr(codes.Unavailable)}, clientOps: []clientOp{cReq(1), cHdr(), cErr(codes.Unavailable)}, }, { desc: "No retry after context", @@ -485,8 +415,8 @@ func (s) TestRetryStreaming(t *testing.T) { var serverOpIter int var serverOps []serverOp - ss := &stubServer{ - fullDuplexCall: func(stream testpb.TestService_FullDuplexCallServer) error { + ss := &stubserver.StubServer{ + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { for serverOpIter < len(serverOps) { op := serverOps[serverOpIter] serverOpIter++ @@ -497,11 +427,8 @@ func (s) TestRetryStreaming(t *testing.T) { return nil }, } - if err := ss.Start([]grpc.ServerOption{}, grpc.WithDefaultCallOptions(grpc.MaxRetryRPCBufferSize(200))); err != nil { - t.Fatalf("Error starting endpoint server: %v", err) - } - defer ss.Stop() - ss.newServiceConfig(`{ + if err := ss.Start([]grpc.ServerOption{}, grpc.WithDefaultCallOptions(grpc.MaxRetryRPCBufferSize(200)), + grpc.WithDefaultServiceConfig(`{ "methodConfig": [{ "name": [{"service": "grpc.testing.TestService"}], "waitForReady": true, @@ -512,27 +439,28 @@ func (s) TestRetryStreaming(t *testing.T) { "BackoffMultiplier": 1.0, "RetryableStatusCodes": [ "UNAVAILABLE" ] } - }]}`) - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + }]}`)); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() for { if ctx.Err() != nil { t.Fatalf("Timed out waiting for service config update") } - if ss.cc.GetMethodConfig("/grpc.testing.TestService/FullDuplexCall").WaitForReady != nil { + if ss.CC.GetMethodConfig("/grpc.testing.TestService/FullDuplexCall").WaitForReady != nil { break } time.Sleep(time.Millisecond) } - cancel() for _, tc := range testCases { func() { serverOpIter = 0 serverOps = tc.serverOps - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - stream, err := ss.client.FullDuplexCall(ctx) + stream, err := ss.Client.FullDuplexCall(ctx) if err != nil { t.Fatalf("%v: Error while creating stream: %v", tc.desc, err) } @@ -548,3 +476,252 @@ func (s) TestRetryStreaming(t *testing.T) { }() } } + +type retryStatsHandler struct { + mu sync.Mutex + s []stats.RPCStats +} + +func (*retryStatsHandler) TagRPC(ctx context.Context, _ *stats.RPCTagInfo) context.Context { + return ctx +} +func (h *retryStatsHandler) HandleRPC(_ context.Context, s stats.RPCStats) { + // these calls come in nondeterministically - so can just ignore + if _, ok := s.(*stats.PickerUpdated); ok { + return + } + h.mu.Lock() + h.s = append(h.s, s) + h.mu.Unlock() +} +func (*retryStatsHandler) TagConn(ctx context.Context, _ *stats.ConnTagInfo) context.Context { + return ctx +} +func (*retryStatsHandler) HandleConn(context.Context, stats.ConnStats) {} + +func (s) TestRetryStats(t *testing.T) { + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("Failed to listen. Err: %v", err) + } + defer lis.Close() + server := &httpServer{ + waitForEndStream: true, + responses: []httpServerResponse{{ + trailers: [][]string{{ + ":status", "200", + "content-type", "application/grpc", + "grpc-status", "14", // UNAVAILABLE + "grpc-message", "unavailable retry", + "grpc-retry-pushback-ms", "10", + }}, + }, { + headers: [][]string{{ + ":status", "200", + "content-type", "application/grpc", + }}, + payload: []byte{0, 0, 0, 0, 0}, // header for 0-byte response message. + trailers: [][]string{{ + "grpc-status", "0", // OK + }}, + }}, + refuseStream: func(i uint32) bool { + return i == 1 + }, + } + server.start(t, lis) + handler := &retryStatsHandler{} + cc, err := grpc.Dial(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithStatsHandler(handler), + grpc.WithDefaultServiceConfig((`{ + "methodConfig": [{ + "name": [{"service": "grpc.testing.TestService"}], + "retryPolicy": { + "MaxAttempts": 4, + "InitialBackoff": ".01s", + "MaxBackoff": ".01s", + "BackoffMultiplier": 1.0, + "RetryableStatusCodes": [ "UNAVAILABLE" ] + } + }]}`))) + if err != nil { + t.Fatalf("failed to dial due to err: %v", err) + } + defer cc.Close() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + client := testgrpc.NewTestServiceClient(cc) + + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("unexpected EmptyCall error: %v", err) + } + handler.mu.Lock() + want := []stats.RPCStats{ + &stats.Begin{}, + &stats.OutHeader{FullMethod: "/grpc.testing.TestService/EmptyCall"}, + &stats.OutPayload{WireLength: 5}, + &stats.End{}, + + &stats.Begin{IsTransparentRetryAttempt: true}, + &stats.OutHeader{FullMethod: "/grpc.testing.TestService/EmptyCall"}, + &stats.OutPayload{WireLength: 5}, + &stats.InTrailer{Trailer: metadata.Pairs("content-type", "application/grpc", "grpc-retry-pushback-ms", "10")}, + &stats.End{}, + + &stats.Begin{}, + &stats.OutHeader{FullMethod: "/grpc.testing.TestService/EmptyCall"}, + &stats.OutPayload{WireLength: 5}, + &stats.InHeader{}, + &stats.InPayload{WireLength: 5}, + &stats.InTrailer{}, + &stats.End{}, + } + + toString := func(ss []stats.RPCStats) (ret []string) { + for _, s := range ss { + ret = append(ret, fmt.Sprintf("%T - %v", s, s)) + } + return ret + } + t.Logf("Handler received frames:\n%v\n---\nwant:\n%v\n", + strings.Join(toString(handler.s), "\n"), + strings.Join(toString(want), "\n")) + + if len(handler.s) != len(want) { + t.Fatalf("received unexpected number of RPCStats: got %v; want %v", len(handler.s), len(want)) + } + + // There is a race between receiving the payload (triggered by the + // application / gRPC library) and receiving the trailer (triggered at the + // transport layer). Adjust the received stats accordingly if necessary. + const tIdx, pIdx = 13, 14 + _, okT := handler.s[tIdx].(*stats.InTrailer) + _, okP := handler.s[pIdx].(*stats.InPayload) + if okT && okP { + handler.s[pIdx], handler.s[tIdx] = handler.s[tIdx], handler.s[pIdx] + } + + for i := range handler.s { + w, s := want[i], handler.s[i] + + // Validate the event type + if reflect.TypeOf(w) != reflect.TypeOf(s) { + t.Fatalf("at position %v: got %T; want %T", i, s, w) + } + wv, sv := reflect.ValueOf(w).Elem(), reflect.ValueOf(s).Elem() + + // Validate that Client is always true + if sv.FieldByName("Client").Interface().(bool) != true { + t.Fatalf("at position %v: got Client=false; want true", i) + } + + // Validate any set fields in want + for i := 0; i < wv.NumField(); i++ { + if !wv.Field(i).IsZero() { + if got, want := sv.Field(i).Interface(), wv.Field(i).Interface(); !reflect.DeepEqual(got, want) { + name := reflect.TypeOf(w).Elem().Field(i).Name + t.Fatalf("at position %v, field %v: got %v; want %v", i, name, got, want) + } + } + } + + // Since the above only tests non-zero-value fields, test + // IsTransparentRetryAttempt=false explicitly when needed. + if wb, ok := w.(*stats.Begin); ok && !wb.IsTransparentRetryAttempt { + if s.(*stats.Begin).IsTransparentRetryAttempt { + t.Fatalf("at position %v: got IsTransparentRetryAttempt=true; want false", i) + } + } + } + + // Validate timings between last Begin and preceding End. + end := handler.s[8].(*stats.End) + begin := handler.s[9].(*stats.Begin) + diff := begin.BeginTime.Sub(end.EndTime) + if diff < 10*time.Millisecond || diff > 50*time.Millisecond { + t.Fatalf("pushback time before final attempt = %v; want ~10ms", diff) + } +} + +func (s) TestRetryTransparentWhenCommitted(t *testing.T) { + // With MaxConcurrentStreams=1: + // + // 1. Create stream 1 that is retriable. + // 2. Stream 1 is created and fails with a retriable code. + // 3. Create dummy stream 2, blocking indefinitely. + // 4. Stream 1 retries (and blocks until stream 2 finishes) + // 5. Stream 1 is canceled manually. + // + // If there is no bug, the stream is done and errors with CANCELED. With a bug: + // + // 6. Stream 1 has a nil stream (attempt.s). Operations like CloseSend will panic. + + first := grpcsync.NewEvent() + ss := &stubserver.StubServer{ + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + // signal? + if !first.HasFired() { + first.Fire() + t.Log("returned first error") + return status.Error(codes.AlreadyExists, "first attempt fails and is retriable") + } + t.Log("blocking") + <-stream.Context().Done() + return stream.Context().Err() + }, + } + + if err := ss.Start([]grpc.ServerOption{grpc.MaxConcurrentStreams(1)}, + grpc.WithDefaultServiceConfig(`{ + "methodConfig": [{ + "name": [{"service": "grpc.testing.TestService"}], + "waitForReady": true, + "retryPolicy": { + "MaxAttempts": 2, + "InitialBackoff": ".1s", + "MaxBackoff": ".1s", + "BackoffMultiplier": 1.0, + "RetryableStatusCodes": [ "ALREADY_EXISTS" ] + } + }]}`)); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx1, cancel1 := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel1() + ctx2, cancel2 := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel2() + + stream1, err := ss.Client.FullDuplexCall(ctx1) + if err != nil { + t.Fatalf("Error creating stream 1: %v", err) + } + + // Create dummy stream to block indefinitely. + _, err = ss.Client.FullDuplexCall(ctx2) + if err != nil { + t.Errorf("Error creating stream 2: %v", err) + } + + stream1Closed := grpcsync.NewEvent() + go func() { + _, err := stream1.Recv() + // Will trigger a retry when it sees the ALREADY_EXISTS error + if status.Code(err) != codes.Canceled { + t.Errorf("Expected stream1 to be canceled; got error: %v", err) + } + stream1Closed.Fire() + }() + + // Wait longer than the retry backoff timer. + time.Sleep(200 * time.Millisecond) + cancel1() + + // Operations on the stream should not panic. + <-stream1Closed.Done() + stream1.CloseSend() + stream1.Recv() + stream1.Send(&testpb.StreamingOutputCallRequest{}) +} diff --git a/test/roundrobin_test.go b/test/roundrobin_test.go new file mode 100644 index 000000000000..b4b17895b053 --- /dev/null +++ b/test/roundrobin_test.go @@ -0,0 +1,317 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "context" + "strings" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/channelz" + imetadata "google.golang.org/grpc/internal/metadata" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + rrutil "google.golang.org/grpc/internal/testutils/roundrobin" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/status" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +const rrServiceConfig = `{"loadBalancingConfig": [{"round_robin":{}}]}` + +func testRoundRobinBasic(ctx context.Context, t *testing.T, opts ...grpc.DialOption) (*grpc.ClientConn, *manual.Resolver, []*stubserver.StubServer) { + t.Helper() + + r := manual.NewBuilderWithScheme("whatever") + + const backendCount = 5 + backends := make([]*stubserver.StubServer, backendCount) + addrs := make([]resolver.Address, backendCount) + for i := 0; i < backendCount; i++ { + backend := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil }, + } + if err := backend.StartServer(); err != nil { + t.Fatalf("Failed to start backend: %v", err) + } + t.Logf("Started TestService backend at: %q", backend.Address) + t.Cleanup(func() { backend.Stop() }) + + backends[i] = backend + addrs[i] = resolver.Address{Addr: backend.Address} + } + + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(r), + grpc.WithDefaultServiceConfig(rrServiceConfig), + } + dopts = append(dopts, opts...) + cc, err := grpc.Dial(r.Scheme()+":///test.server", dopts...) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) + } + t.Cleanup(func() { cc.Close() }) + client := testgrpc.NewTestServiceClient(cc) + + // At this point, the resolver has not returned any addresses to the channel. + // This RPC must block until the context expires. + sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + if _, err := client.EmptyCall(sCtx, &testpb.Empty{}); status.Code(err) != codes.DeadlineExceeded { + t.Fatalf("EmptyCall() = %s, want %s", status.Code(err), codes.DeadlineExceeded) + } + + r.UpdateState(resolver.State{Addresses: addrs}) + if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs); err != nil { + t.Fatal(err) + } + return cc, r, backends +} + +// TestRoundRobin_Basic tests the most basic scenario for round_robin. It brings +// up a bunch of backends and verifies that RPCs are getting round robin-ed +// across these backends. +func (s) TestRoundRobin_Basic(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testRoundRobinBasic(ctx, t) +} + +// TestRoundRobin_AddressesRemoved tests the scenario where a bunch of backends +// are brought up, and round_robin is configured as the LB policy and RPCs are +// being correctly round robin-ed across these backends. We then send a resolver +// update with no addresses and verify that the channel enters TransientFailure +// and RPCs fail with an expected error message. +func (s) TestRoundRobin_AddressesRemoved(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + cc, r, _ := testRoundRobinBasic(ctx, t) + + // Send a resolver update with no addresses. This should push the channel into + // TransientFailure. + r.UpdateState(resolver.State{Addresses: []resolver.Address{}}) + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + + const msgWant = "produced zero addresses" + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); !strings.Contains(status.Convert(err).Message(), msgWant) { + t.Fatalf("EmptyCall() = %v, want Contains(Message(), %q)", err, msgWant) + } +} + +// TestRoundRobin_NewAddressWhileBlocking tests the case where round_robin is +// configured on a channel, things are working as expected and then a resolver +// updates removes all addresses. An RPC attempted at this point in time will be +// blocked because there are no valid backends. This test verifies that when new +// backends are added, the RPC is able to complete. +func (s) TestRoundRobin_NewAddressWhileBlocking(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + cc, r, backends := testRoundRobinBasic(ctx, t) + + // Send a resolver update with no addresses. This should push the channel into + // TransientFailure. + r.UpdateState(resolver.State{Addresses: []resolver.Address{}}) + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + + client := testgrpc.NewTestServiceClient(cc) + doneCh := make(chan struct{}) + go func() { + // The channel is currently in TransientFailure and this RPC will block + // until the channel becomes Ready, which will only happen when we push a + // resolver update with a valid backend address. + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Errorf("EmptyCall() = %v, want ", err) + } + close(doneCh) + }() + + // Make sure that there is one pending RPC on the ClientConn before attempting + // to push new addresses through the name resolver. If we don't do this, the + // resolver update can happen before the above goroutine gets to make the RPC. + for { + if err := ctx.Err(); err != nil { + t.Fatal(err) + } + tcs, _ := channelz.GetTopChannels(0, 0) + if len(tcs) != 1 { + t.Fatalf("there should only be one top channel, not %d", len(tcs)) + } + started := tcs[0].ChannelData.CallsStarted + completed := tcs[0].ChannelData.CallsSucceeded + tcs[0].ChannelData.CallsFailed + if (started - completed) == 1 { + break + } + time.Sleep(defaultTestShortTimeout) + } + + // Send a resolver update with a valid backend to push the channel to Ready + // and unblock the above RPC. + r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: backends[0].Address}}}) + + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for blocked RPC to complete") + case <-doneCh: + } +} + +// TestRoundRobin_OneServerDown tests the scenario where a channel is configured +// to round robin across a set of backends, and things are working correctly. +// One backend goes down. The test verifies that going forward, RPCs are round +// robin-ed across the remaining set of backends. +func (s) TestRoundRobin_OneServerDown(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + cc, _, backends := testRoundRobinBasic(ctx, t) + + // Stop one backend. RPCs should round robin across the remaining backends. + backends[len(backends)-1].Stop() + + addrs := make([]resolver.Address, len(backends)-1) + for i := 0; i < len(backends)-1; i++ { + addrs[i] = resolver.Address{Addr: backends[i].Address} + } + client := testgrpc.NewTestServiceClient(cc) + if err := rrutil.CheckRoundRobinRPCs(ctx, client, addrs); err != nil { + t.Fatalf("RPCs are not being round robined across remaining servers: %v", err) + } +} + +// TestRoundRobin_AllServersDown tests the scenario where a channel is +// configured to round robin across a set of backends, and things are working +// correctly. Then, all backends go down. The test verifies that the channel +// moves to TransientFailure and failfast RPCs fail with Unavailable. +func (s) TestRoundRobin_AllServersDown(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + cc, _, backends := testRoundRobinBasic(ctx, t) + + // Stop all backends. + for _, b := range backends { + b.Stop() + } + + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + + // Failfast RPCs should fail with Unavailable. + client := testgrpc.NewTestServiceClient(cc) + + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.Unavailable { + t.Fatalf("EmptyCall got err: %v; want Unavailable", err) + } +} + +// TestRoundRobin_UpdateAddressAttributes tests the scenario where the addresses +// returned by the resolver contain attributes. The test verifies that the +// attributes contained in the addresses show up as RPC metadata in the backend. +func (s) TestRoundRobin_UpdateAddressAttributes(t *testing.T) { + const ( + testMDKey = "test-md" + testMDValue = "test-md-value" + ) + r := manual.NewBuilderWithScheme("whatever") + + // Spin up a StubServer to serve as a backend. The implementation verifies + // that the expected metadata is received. + testMDChan := make(chan []string, 1) + backend := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, _ *testpb.Empty) (*testpb.Empty, error) { + md, ok := metadata.FromIncomingContext(ctx) + if ok { + select { + case testMDChan <- md[testMDKey]: + case <-ctx.Done(): + return nil, ctx.Err() + } + } + return &testpb.Empty{}, nil + }, + } + if err := backend.StartServer(); err != nil { + t.Fatalf("Failed to start backend: %v", err) + } + t.Logf("Started TestService backend at: %q", backend.Address) + t.Cleanup(func() { backend.Stop() }) + + // Dial the backend with round_robin as the LB policy. + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(r), + grpc.WithDefaultServiceConfig(rrServiceConfig), + } + cc, err := grpc.Dial(r.Scheme()+":///test.server", dopts...) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) + } + t.Cleanup(func() { cc.Close() }) + + // Send a resolver update with no address attributes. + addr := resolver.Address{Addr: backend.Address} + r.UpdateState(resolver.State{Addresses: []resolver.Address{addr}}) + + // Make an RPC and ensure it does not contain the metadata we are looking for. + client := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() = %v, want ", err) + } + select { + case <-ctx.Done(): + t.Fatalf("Timeout when waiting for metadata received in RPC") + case md := <-testMDChan: + if len(md) != 0 { + t.Fatalf("received metadata %v, want nil", md) + } + } + + // Send a resolver update with address attributes. + addrWithAttributes := imetadata.Set(addr, metadata.Pairs(testMDKey, testMDValue)) + r.UpdateState(resolver.State{Addresses: []resolver.Address{addrWithAttributes}}) + + // Make an RPC and ensure it contains the metadata we are looking for. The + // resolver update isn't processed synchronously, so we wait some time before + // failing if some RPCs do not contain it. +Done: + for { + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() = %v, want ", err) + } + select { + case <-ctx.Done(): + t.Fatalf("Timeout when waiting for metadata received in RPC") + case md := <-testMDChan: + if len(md) == 1 && md[0] == testMDValue { + break Done + } + } + time.Sleep(defaultTestShortTimeout) + } +} diff --git a/test/server_test.go b/test/server_test.go index c6a5fe74bd55..eb0872d0c113 100644 --- a/test/server_test.go +++ b/test/server_test.go @@ -25,19 +25,57 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal/stubserver" "google.golang.org/grpc/status" - testpb "google.golang.org/grpc/test/grpc_testing" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" ) type ctxKey string +// TestServerReturningContextError verifies that if a context error is returned +// by the service handler, the status will have the correct status code, not +// Unknown. +func (s) TestServerReturningContextError(t *testing.T) { + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { + return nil, context.DeadlineExceeded + }, + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + return context.DeadlineExceeded + }, + } + if err := ss.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}) + if s, ok := status.FromError(err); !ok || s.Code() != codes.DeadlineExceeded { + t.Fatalf("ss.Client.EmptyCall() got error %v; want ", err) + } + + stream, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("unexpected error starting the stream: %v", err) + } + _, err = stream.Recv() + if s, ok := status.FromError(err); !ok || s.Code() != codes.DeadlineExceeded { + t.Fatalf("ss.Client.FullDuplexCall().Recv() got error %v; want ", err) + } + +} + func (s) TestChainUnaryServerInterceptor(t *testing.T) { var ( firstIntKey = ctxKey("firstIntKey") secondIntKey = ctxKey("secondIntKey") ) - firstInt := func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { + firstInt := func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { if ctx.Value(firstIntKey) != nil { return nil, status.Errorf(codes.Internal, "first interceptor should not have %v in context", firstIntKey) } @@ -63,7 +101,7 @@ func (s) TestChainUnaryServerInterceptor(t *testing.T) { }, nil } - secondInt := func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { + secondInt := func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { if ctx.Value(firstIntKey) == nil { return nil, status.Errorf(codes.Internal, "second interceptor should have %v in context", firstIntKey) } @@ -89,7 +127,7 @@ func (s) TestChainUnaryServerInterceptor(t *testing.T) { }, nil } - lastInt := func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { + lastInt := func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { if ctx.Value(firstIntKey) == nil { return nil, status.Errorf(codes.Internal, "last interceptor should have %v in context", firstIntKey) } @@ -118,8 +156,8 @@ func (s) TestChainUnaryServerInterceptor(t *testing.T) { grpc.ChainUnaryInterceptor(firstInt, secondInt, lastInt), } - ss := &stubServer{ - unaryCall: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + ss := &stubserver.StubServer{ + UnaryCallF: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { payload, err := newPayload(testpb.PayloadType_COMPRESSABLE, 0) if err != nil { return nil, status.Errorf(codes.Aborted, "failed to make payload: %v", err) @@ -135,9 +173,11 @@ func (s) TestChainUnaryServerInterceptor(t *testing.T) { } defer ss.Stop() - resp, err := ss.client.UnaryCall(context.Background(), &testpb.SimpleRequest{}) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + resp, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{}) if s, ok := status.FromError(err); !ok || s.Code() != codes.OK { - t.Fatalf("ss.client.UnaryCall(context.Background(), _) = %v, %v; want nil, ", resp, err) + t.Fatalf("ss.Client.UnaryCall(ctx, _) = %v, %v; want nil, ", resp, err) } respBytes := resp.Payload.GetBody() @@ -149,7 +189,7 @@ func (s) TestChainUnaryServerInterceptor(t *testing.T) { func (s) TestChainOnBaseUnaryServerInterceptor(t *testing.T) { baseIntKey := ctxKey("baseIntKey") - baseInt := func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { + baseInt := func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { if ctx.Value(baseIntKey) != nil { return nil, status.Errorf(codes.Internal, "base interceptor should not have %v in context", baseIntKey) } @@ -158,7 +198,7 @@ func (s) TestChainOnBaseUnaryServerInterceptor(t *testing.T) { return handler(baseCtx, req) } - chainInt := func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { + chainInt := func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { if ctx.Value(baseIntKey) == nil { return nil, status.Errorf(codes.Internal, "chain interceptor should have %v in context", baseIntKey) } @@ -171,8 +211,8 @@ func (s) TestChainOnBaseUnaryServerInterceptor(t *testing.T) { grpc.ChainUnaryInterceptor(chainInt), } - ss := &stubServer{ - emptyCall: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil }, } @@ -181,16 +221,18 @@ func (s) TestChainOnBaseUnaryServerInterceptor(t *testing.T) { } defer ss.Stop() - resp, err := ss.client.EmptyCall(context.Background(), &testpb.Empty{}) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + resp, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}) if s, ok := status.FromError(err); !ok || s.Code() != codes.OK { - t.Fatalf("ss.client.EmptyCall(context.Background(), _) = %v, %v; want nil, ", resp, err) + t.Fatalf("ss.Client.EmptyCall(ctx, _) = %v, %v; want nil, ", resp, err) } } func (s) TestChainStreamServerInterceptor(t *testing.T) { callCounts := make([]int, 4) - firstInt := func(srv interface{}, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + firstInt := func(srv any, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { if callCounts[0] != 0 { return status.Errorf(codes.Internal, "callCounts[0] should be 0, but got=%d", callCounts[0]) } @@ -207,7 +249,7 @@ func (s) TestChainStreamServerInterceptor(t *testing.T) { return handler(srv, stream) } - secondInt := func(srv interface{}, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + secondInt := func(srv any, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { if callCounts[0] != 1 { return status.Errorf(codes.Internal, "callCounts[0] should be 1, but got=%d", callCounts[0]) } @@ -224,7 +266,7 @@ func (s) TestChainStreamServerInterceptor(t *testing.T) { return handler(srv, stream) } - lastInt := func(srv interface{}, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + lastInt := func(srv any, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { if callCounts[0] != 1 { return status.Errorf(codes.Internal, "callCounts[0] should be 1, but got=%d", callCounts[0]) } @@ -245,8 +287,8 @@ func (s) TestChainStreamServerInterceptor(t *testing.T) { grpc.ChainStreamInterceptor(firstInt, secondInt, lastInt), } - ss := &stubServer{ - fullDuplexCall: func(stream testpb.TestService_FullDuplexCallServer) error { + ss := &stubserver.StubServer{ + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { if callCounts[0] != 1 { return status.Errorf(codes.Internal, "callCounts[0] should be 1, but got=%d", callCounts[0]) } @@ -268,7 +310,9 @@ func (s) TestChainStreamServerInterceptor(t *testing.T) { } defer ss.Stop() - stream, err := ss.client.FullDuplexCall(context.Background()) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + stream, err := ss.Client.FullDuplexCall(ctx) if err != nil { t.Fatalf("failed to FullDuplexCall: %v", err) } diff --git a/test/servertester.go b/test/servertester.go index 9758e8eb6cf8..3701a0e094d9 100644 --- a/test/servertester.go +++ b/test/servertester.go @@ -138,19 +138,46 @@ func (st *serverTester) writeSettingsAck() { } } +func (st *serverTester) wantGoAway(errCode http2.ErrCode) *http2.GoAwayFrame { + f, err := st.readFrame() + if err != nil { + st.t.Fatalf("Error while expecting an RST frame: %v", err) + } + gaf, ok := f.(*http2.GoAwayFrame) + if !ok { + st.t.Fatalf("got a %T; want *http2.GoAwayFrame", f) + } + if gaf.ErrCode != errCode { + st.t.Fatalf("expected GOAWAY error code '%v', got '%v'", errCode.String(), gaf.ErrCode.String()) + } + return gaf +} + +func (st *serverTester) wantPing() *http2.PingFrame { + f, err := st.readFrame() + if err != nil { + st.t.Fatalf("Error while expecting an RST frame: %v", err) + } + pf, ok := f.(*http2.PingFrame) + if !ok { + st.t.Fatalf("got a %T; want *http2.GoAwayFrame", f) + } + return pf +} + func (st *serverTester) wantRSTStream(errCode http2.ErrCode) *http2.RSTStreamFrame { f, err := st.readFrame() if err != nil { st.t.Fatalf("Error while expecting an RST frame: %v", err) } - sf, ok := f.(*http2.RSTStreamFrame) + rf, ok := f.(*http2.RSTStreamFrame) if !ok { st.t.Fatalf("got a %T; want *http2.RSTStreamFrame", f) } - if sf.ErrCode != errCode { - st.t.Fatalf("expected RST error code '%v', got '%v'", errCode.String(), sf.ErrCode.String()) + if rf.ErrCode != errCode { + st.t.Fatalf("expected RST error code '%v', got '%v'", errCode.String(), rf.ErrCode.String()) } - return sf + return rf } func (st *serverTester) wantSettings() *http2.SettingsFrame { @@ -273,3 +300,9 @@ func (st *serverTester) writeRSTStream(streamID uint32, code http2.ErrCode) { st.t.Fatalf("Error writing RST_STREAM: %v", err) } } + +func (st *serverTester) writePing(ack bool, data [8]byte) { + if err := st.fr.WritePing(ack, data); err != nil { + st.t.Fatalf("Error writing PING: %v", err) + } +} diff --git a/test/service_config_deprecated_test.go b/test/service_config_deprecated_test.go new file mode 100644 index 000000000000..a1fd44d853d9 --- /dev/null +++ b/test/service_config_deprecated_test.go @@ -0,0 +1,463 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "context" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +// The following functions with function name ending with TD indicates that they +// should be deleted after old service config API is deprecated and deleted. +func testServiceConfigSetupTD(t *testing.T, e env) (*test, chan grpc.ServiceConfig) { + te := newTest(t, e) + // We write before read. + ch := make(chan grpc.ServiceConfig, 1) + te.sc = ch + te.userAgent = testAppUA + te.declareLogNoise( + "transport: http2Client.notifyError got notified that the client transport was broken EOF", + "grpc: addrConn.transportMonitor exits due to: grpc: the connection is closing", + "grpc: addrConn.resetTransport failed to create client transport: connection error", + "Failed to dial : context canceled; please retry.", + ) + return te, ch +} + +func (s) TestServiceConfigGetMethodConfigTD(t *testing.T) { + for _, e := range listTestEnv() { + testGetMethodConfigTD(t, e) + } +} + +func testGetMethodConfigTD(t *testing.T, e env) { + te, ch := testServiceConfigSetupTD(t, e) + defer te.tearDown() + + mc1 := grpc.MethodConfig{ + WaitForReady: newBool(true), + Timeout: newDuration(time.Millisecond), + } + mc2 := grpc.MethodConfig{WaitForReady: newBool(false)} + m := make(map[string]grpc.MethodConfig) + m["/grpc.testing.TestService/EmptyCall"] = mc1 + m["/grpc.testing.TestService/"] = mc2 + sc := grpc.ServiceConfig{ + Methods: m, + } + ch <- sc + + cc := te.clientConn() + tc := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + // The following RPCs are expected to become non-fail-fast ones with 1ms deadline. + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.DeadlineExceeded { + t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, %s", err, codes.DeadlineExceeded) + } + + m = make(map[string]grpc.MethodConfig) + m["/grpc.testing.TestService/UnaryCall"] = mc1 + m["/grpc.testing.TestService/"] = mc2 + sc = grpc.ServiceConfig{ + Methods: m, + } + ch <- sc + // Wait for the new service config to propagate. + for { + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.DeadlineExceeded { + break + } + } + // The following RPCs are expected to become fail-fast. + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.Unavailable { + t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, %s", err, codes.Unavailable) + } +} + +func (s) TestServiceConfigWaitForReadyTD(t *testing.T) { + for _, e := range listTestEnv() { + testServiceConfigWaitForReadyTD(t, e) + } +} + +func testServiceConfigWaitForReadyTD(t *testing.T, e env) { + te, ch := testServiceConfigSetupTD(t, e) + defer te.tearDown() + + // Case1: Client API set failfast to be false, and service config set wait_for_ready to be false, Client API should win, and the rpc will wait until deadline exceeds. + mc := grpc.MethodConfig{ + WaitForReady: newBool(false), + Timeout: newDuration(time.Millisecond), + } + m := make(map[string]grpc.MethodConfig) + m["/grpc.testing.TestService/EmptyCall"] = mc + m["/grpc.testing.TestService/FullDuplexCall"] = mc + sc := grpc.ServiceConfig{ + Methods: m, + } + ch <- sc + + cc := te.clientConn() + tc := testgrpc.NewTestServiceClient(cc) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + // The following RPCs are expected to become non-fail-fast ones with 1ms deadline. + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); status.Code(err) != codes.DeadlineExceeded { + t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, %s", err, codes.DeadlineExceeded) + } + if _, err := tc.FullDuplexCall(ctx, grpc.WaitForReady(true)); status.Code(err) != codes.DeadlineExceeded { + t.Fatalf("TestService/FullDuplexCall(_) = _, %v, want %s", err, codes.DeadlineExceeded) + } + + // Generate a service config update. + // Case2: Client API does not set failfast, and service config set wait_for_ready to be true, and the rpc will wait until deadline exceeds. + mc.WaitForReady = newBool(true) + m = make(map[string]grpc.MethodConfig) + m["/grpc.testing.TestService/EmptyCall"] = mc + m["/grpc.testing.TestService/FullDuplexCall"] = mc + sc = grpc.ServiceConfig{ + Methods: m, + } + ch <- sc + + // Wait for the new service config to take effect. + ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) { + mc = cc.GetMethodConfig("/grpc.testing.TestService/FullDuplexCall") + if *mc.WaitForReady { + break + } + } + if ctx.Err() != nil { + t.Fatalf("Timeout when waiting for service config to take effect") + } + + // The following RPCs are expected to become non-fail-fast ones with 1ms deadline. + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.DeadlineExceeded { + t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, %s", err, codes.DeadlineExceeded) + } + if _, err := tc.FullDuplexCall(ctx); status.Code(err) != codes.DeadlineExceeded { + t.Fatalf("TestService/FullDuplexCall(_) = _, %v, want %s", err, codes.DeadlineExceeded) + } +} + +func (s) TestServiceConfigTimeoutTD(t *testing.T) { + for _, e := range listTestEnv() { + testServiceConfigTimeoutTD(t, e) + } +} + +func testServiceConfigTimeoutTD(t *testing.T, e env) { + te, ch := testServiceConfigSetupTD(t, e) + defer te.tearDown() + + // Case1: Client API sets timeout to be 1ns and ServiceConfig sets timeout + // to be 1hr. Timeout should be 1ns (min of 1ns and 1hr) and the rpc will + // wait until deadline exceeds. + mc := grpc.MethodConfig{ + Timeout: newDuration(time.Hour), + } + m := make(map[string]grpc.MethodConfig) + m["/grpc.testing.TestService/EmptyCall"] = mc + m["/grpc.testing.TestService/FullDuplexCall"] = mc + sc := grpc.ServiceConfig{ + Methods: m, + } + ch <- sc + + cc := te.clientConn() + tc := testgrpc.NewTestServiceClient(cc) + // The following RPCs are expected to become non-fail-fast ones with 1ns deadline. + ctx, cancel := context.WithTimeout(context.Background(), time.Nanosecond) + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); status.Code(err) != codes.DeadlineExceeded { + t.Fatalf("TestService/EmptyCall(_, _) = _, %v, want _, %s", err, codes.DeadlineExceeded) + } + cancel() + ctx, cancel = context.WithTimeout(context.Background(), time.Nanosecond) + if _, err := tc.FullDuplexCall(ctx, grpc.WaitForReady(true)); status.Code(err) != codes.DeadlineExceeded { + t.Fatalf("TestService/FullDuplexCall(_) = _, %v, want %s", err, codes.DeadlineExceeded) + } + cancel() + + // Generate a service config update. + // Case2: Client API sets timeout to be the default and ServiceConfig sets + // timeout to be 1ns. Timeout should be 1ns (min of 1ns and the default) + // and the rpc will wait until deadline exceeds. + mc.Timeout = newDuration(time.Nanosecond) + m = make(map[string]grpc.MethodConfig) + m["/grpc.testing.TestService/EmptyCall"] = mc + m["/grpc.testing.TestService/FullDuplexCall"] = mc + sc = grpc.ServiceConfig{ + Methods: m, + } + ch <- sc + + // Wait for the new service config to take effect. + ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + for ; ctx.Err() == nil; <-time.After(time.Millisecond) { + mc = cc.GetMethodConfig("/grpc.testing.TestService/FullDuplexCall") + if *mc.Timeout == time.Nanosecond { + break + } + } + if ctx.Err() != nil { + t.Fatalf("Timeout when waiting for service config to take effect") + } + + ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := tc.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); status.Code(err) != codes.DeadlineExceeded || ctx.Err() != nil { + t.Fatalf("TestService/EmptyCall(_, _) = _, %v and ctx.Err() = %v; want _, %s and ctx.Err() = nil", err, ctx.Err(), codes.DeadlineExceeded) + } + + if _, err := tc.FullDuplexCall(ctx, grpc.WaitForReady(true)); status.Code(err) != codes.DeadlineExceeded || ctx.Err() != nil { + t.Fatalf("TestService/FullDuplexCall(_) = _, %v and ctx.Err() = %v; want _, %s and ctx.Err() = nil", err, ctx.Err(), codes.DeadlineExceeded) + } +} + +func (s) TestServiceConfigMaxMsgSizeTD(t *testing.T) { + for _, e := range listTestEnv() { + testServiceConfigMaxMsgSizeTD(t, e) + } +} + +func testServiceConfigMaxMsgSizeTD(t *testing.T, e env) { + // Setting up values and objects shared across all test cases. + const smallSize = 1 + const largeSize = 1024 + const extraLargeSize = 2048 + + smallPayload, err := newPayload(testpb.PayloadType_COMPRESSABLE, smallSize) + if err != nil { + t.Fatal(err) + } + largePayload, err := newPayload(testpb.PayloadType_COMPRESSABLE, largeSize) + if err != nil { + t.Fatal(err) + } + extraLargePayload, err := newPayload(testpb.PayloadType_COMPRESSABLE, extraLargeSize) + if err != nil { + t.Fatal(err) + } + + mc := grpc.MethodConfig{ + MaxReqSize: newInt(extraLargeSize), + MaxRespSize: newInt(extraLargeSize), + } + + m := make(map[string]grpc.MethodConfig) + m["/grpc.testing.TestService/UnaryCall"] = mc + m["/grpc.testing.TestService/FullDuplexCall"] = mc + sc := grpc.ServiceConfig{ + Methods: m, + } + // Case1: sc set maxReqSize to 2048 (send), maxRespSize to 2048 (recv). + te1, ch1 := testServiceConfigSetupTD(t, e) + te1.startServer(&testServer{security: e.security}) + defer te1.tearDown() + + ch1 <- sc + tc := testgrpc.NewTestServiceClient(te1.clientConn()) + + req := &testpb.SimpleRequest{ + ResponseType: testpb.PayloadType_COMPRESSABLE, + ResponseSize: int32(extraLargeSize), + Payload: smallPayload, + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + // Test for unary RPC recv. + if _, err := tc.UnaryCall(ctx, req); err == nil || status.Code(err) != codes.ResourceExhausted { + t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted) + } + + // Test for unary RPC send. + req.Payload = extraLargePayload + req.ResponseSize = int32(smallSize) + if _, err := tc.UnaryCall(ctx, req); err == nil || status.Code(err) != codes.ResourceExhausted { + t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted) + } + + // Test for streaming RPC recv. + respParam := []*testpb.ResponseParameters{ + { + Size: int32(extraLargeSize), + }, + } + sreq := &testpb.StreamingOutputCallRequest{ + ResponseType: testpb.PayloadType_COMPRESSABLE, + ResponseParameters: respParam, + Payload: smallPayload, + } + stream, err := tc.FullDuplexCall(te1.ctx) + if err != nil { + t.Fatalf("%v.FullDuplexCall(_) = _, %v, want ", tc, err) + } + if err := stream.Send(sreq); err != nil { + t.Fatalf("%v.Send(%v) = %v, want ", stream, sreq, err) + } + if _, err := stream.Recv(); err == nil || status.Code(err) != codes.ResourceExhausted { + t.Fatalf("%v.Recv() = _, %v, want _, error code: %s", stream, err, codes.ResourceExhausted) + } + + // Test for streaming RPC send. + respParam[0].Size = int32(smallSize) + sreq.Payload = extraLargePayload + stream, err = tc.FullDuplexCall(te1.ctx) + if err != nil { + t.Fatalf("%v.FullDuplexCall(_) = _, %v, want ", tc, err) + } + if err := stream.Send(sreq); err == nil || status.Code(err) != codes.ResourceExhausted { + t.Fatalf("%v.Send(%v) = %v, want _, error code: %s", stream, sreq, err, codes.ResourceExhausted) + } + + // Case2: Client API set maxReqSize to 1024 (send), maxRespSize to 1024 (recv). Sc sets maxReqSize to 2048 (send), maxRespSize to 2048 (recv). + te2, ch2 := testServiceConfigSetupTD(t, e) + te2.maxClientReceiveMsgSize = newInt(1024) + te2.maxClientSendMsgSize = newInt(1024) + te2.startServer(&testServer{security: e.security}) + defer te2.tearDown() + ch2 <- sc + tc = testgrpc.NewTestServiceClient(te2.clientConn()) + + // Test for unary RPC recv. + req.Payload = smallPayload + req.ResponseSize = int32(largeSize) + + if _, err := tc.UnaryCall(ctx, req); err == nil || status.Code(err) != codes.ResourceExhausted { + t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted) + } + + // Test for unary RPC send. + req.Payload = largePayload + req.ResponseSize = int32(smallSize) + if _, err := tc.UnaryCall(ctx, req); err == nil || status.Code(err) != codes.ResourceExhausted { + t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted) + } + + // Test for streaming RPC recv. + stream, err = tc.FullDuplexCall(te2.ctx) + respParam[0].Size = int32(largeSize) + sreq.Payload = smallPayload + if err != nil { + t.Fatalf("%v.FullDuplexCall(_) = _, %v, want ", tc, err) + } + if err := stream.Send(sreq); err != nil { + t.Fatalf("%v.Send(%v) = %v, want ", stream, sreq, err) + } + if _, err := stream.Recv(); err == nil || status.Code(err) != codes.ResourceExhausted { + t.Fatalf("%v.Recv() = _, %v, want _, error code: %s", stream, err, codes.ResourceExhausted) + } + + // Test for streaming RPC send. + respParam[0].Size = int32(smallSize) + sreq.Payload = largePayload + stream, err = tc.FullDuplexCall(te2.ctx) + if err != nil { + t.Fatalf("%v.FullDuplexCall(_) = _, %v, want ", tc, err) + } + if err := stream.Send(sreq); err == nil || status.Code(err) != codes.ResourceExhausted { + t.Fatalf("%v.Send(%v) = %v, want _, error code: %s", stream, sreq, err, codes.ResourceExhausted) + } + + // Case3: Client API set maxReqSize to 4096 (send), maxRespSize to 4096 (recv). Sc sets maxReqSize to 2048 (send), maxRespSize to 2048 (recv). + te3, ch3 := testServiceConfigSetupTD(t, e) + te3.maxClientReceiveMsgSize = newInt(4096) + te3.maxClientSendMsgSize = newInt(4096) + te3.startServer(&testServer{security: e.security}) + defer te3.tearDown() + ch3 <- sc + tc = testgrpc.NewTestServiceClient(te3.clientConn()) + + // Test for unary RPC recv. + req.Payload = smallPayload + req.ResponseSize = int32(largeSize) + + if _, err := tc.UnaryCall(ctx, req); err != nil { + t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want ", err) + } + + req.ResponseSize = int32(extraLargeSize) + if _, err := tc.UnaryCall(ctx, req); err == nil || status.Code(err) != codes.ResourceExhausted { + t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted) + } + + // Test for unary RPC send. + req.Payload = largePayload + req.ResponseSize = int32(smallSize) + if _, err := tc.UnaryCall(ctx, req); err != nil { + t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want ", err) + } + + req.Payload = extraLargePayload + if _, err := tc.UnaryCall(ctx, req); err == nil || status.Code(err) != codes.ResourceExhausted { + t.Fatalf("TestService/UnaryCall(_, _) = _, %v, want _, error code: %s", err, codes.ResourceExhausted) + } + + // Test for streaming RPC recv. + stream, err = tc.FullDuplexCall(te3.ctx) + if err != nil { + t.Fatalf("%v.FullDuplexCall(_) = _, %v, want ", tc, err) + } + respParam[0].Size = int32(largeSize) + sreq.Payload = smallPayload + + if err := stream.Send(sreq); err != nil { + t.Fatalf("%v.Send(%v) = %v, want ", stream, sreq, err) + } + if _, err := stream.Recv(); err != nil { + t.Fatalf("%v.Recv() = _, %v, want ", stream, err) + } + + respParam[0].Size = int32(extraLargeSize) + + if err := stream.Send(sreq); err != nil { + t.Fatalf("%v.Send(%v) = %v, want ", stream, sreq, err) + } + if _, err := stream.Recv(); err == nil || status.Code(err) != codes.ResourceExhausted { + t.Fatalf("%v.Recv() = _, %v, want _, error code: %s", stream, err, codes.ResourceExhausted) + } + + // Test for streaming RPC send. + respParam[0].Size = int32(smallSize) + sreq.Payload = largePayload + stream, err = tc.FullDuplexCall(te3.ctx) + if err != nil { + t.Fatalf("%v.FullDuplexCall(_) = _, %v, want ", tc, err) + } + if err := stream.Send(sreq); err != nil { + t.Fatalf("%v.Send(%v) = %v, want ", stream, sreq, err) + } + sreq.Payload = extraLargePayload + if err := stream.Send(sreq); err == nil || status.Code(err) != codes.ResourceExhausted { + t.Fatalf("%v.Send(%v) = %v, want _, error code: %s", stream, sreq, err, codes.ResourceExhausted) + } +} diff --git a/test/stream_cleanup_test.go b/test/stream_cleanup_test.go index cb31b4eb2876..0f705bab2507 100644 --- a/test/stream_cleanup_test.go +++ b/test/stream_cleanup_test.go @@ -26,8 +26,11 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal/stubserver" "google.golang.org/grpc/status" - testpb "google.golang.org/grpc/test/grpc_testing" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" ) func (s) TestStreamCleanup(t *testing.T) { @@ -35,25 +38,27 @@ func (s) TestStreamCleanup(t *testing.T) { const bodySize = 2 * initialWindowSize // Something that is not going to fit in a single window const callRecvMsgSize uint = 1 // The maximum message size the client can receive - ss := &stubServer{ - unaryCall: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + ss := &stubserver.StubServer{ + UnaryCallF: func(ctx context.Context, in *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { return &testpb.SimpleResponse{Payload: &testpb.Payload{ Body: make([]byte, bodySize), }}, nil }, - emptyCall: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil }, } - if err := ss.Start([]grpc.ServerOption{grpc.MaxConcurrentStreams(1)}, grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(int(callRecvMsgSize))), grpc.WithInitialWindowSize(int32(initialWindowSize))); err != nil { + if err := ss.Start(nil, grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(int(callRecvMsgSize))), grpc.WithInitialWindowSize(int32(initialWindowSize))); err != nil { t.Fatalf("Error starting endpoint server: %v", err) } defer ss.Stop() - if _, err := ss.client.UnaryCall(context.Background(), &testpb.SimpleRequest{}); status.Code(err) != codes.ResourceExhausted { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := ss.Client.UnaryCall(ctx, &testpb.SimpleRequest{}); status.Code(err) != codes.ResourceExhausted { t.Fatalf("should fail with ResourceExhausted, message's body size: %v, maximum message size the client can receive: %v", bodySize, callRecvMsgSize) } - if _, err := ss.client.EmptyCall(context.Background(), &testpb.Empty{}); err != nil { + if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil { t.Fatalf("should succeed, err: %v", err) } } @@ -64,8 +69,8 @@ func (s) TestStreamCleanupAfterSendStatus(t *testing.T) { serverReturnedStatus := make(chan struct{}) - ss := &stubServer{ - fullDuplexCall: func(stream testpb.TestService_FullDuplexCallServer) error { + ss := &stubserver.StubServer{ + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { defer func() { close(serverReturnedStatus) }() @@ -76,7 +81,7 @@ func (s) TestStreamCleanupAfterSendStatus(t *testing.T) { }) }, } - if err := ss.Start([]grpc.ServerOption{grpc.MaxConcurrentStreams(1)}, grpc.WithInitialWindowSize(int32(initialWindowSize))); err != nil { + if err := ss.Start(nil, grpc.WithInitialWindowSize(int32(initialWindowSize))); err != nil { t.Fatalf("Error starting endpoint server: %v", err) } defer ss.Stop() @@ -86,9 +91,9 @@ func (s) TestStreamCleanupAfterSendStatus(t *testing.T) { // 1. Make a long living stream RPC. So server's activeStream list is not // empty. - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() - stream, err := ss.client.FullDuplexCall(ctx) + stream, err := ss.Client.FullDuplexCall(ctx) if err != nil { t.Fatalf("FullDuplexCall= _, %v; want _, ", err) } @@ -113,7 +118,7 @@ func (s) TestStreamCleanupAfterSendStatus(t *testing.T) { gracefulStopDone := make(chan struct{}) go func() { defer close(gracefulStopDone) - ss.s.GracefulStop() + ss.S.GracefulStop() }() // 4. Make sure the stream is not broken. @@ -129,6 +134,6 @@ func (s) TestStreamCleanupAfterSendStatus(t *testing.T) { case <-gracefulStopDone: timer.Stop() case <-timer.C: - t.Fatalf("s.GracefulStop() didn't finish without 1 second after the last RPC") + t.Fatalf("s.GracefulStop() didn't finish within 1 second after the last RPC") } } diff --git a/test/subconn_test.go b/test/subconn_test.go new file mode 100644 index 000000000000..cd2ac5a5432d --- /dev/null +++ b/test/subconn_test.go @@ -0,0 +1,126 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "context" + "errors" + "fmt" + "testing" + + "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/base" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/internal/balancer/stub" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + testpb "google.golang.org/grpc/interop/grpc_testing" + "google.golang.org/grpc/resolver" +) + +type tsccPicker struct { + sc balancer.SubConn +} + +func (p *tsccPicker) Pick(info balancer.PickInfo) (balancer.PickResult, error) { + return balancer.PickResult{SubConn: p.sc}, nil +} + +// TestSubConnEmpty tests that removing all addresses from a SubConn and then +// re-adding them does not cause a panic and properly reconnects. +func (s) TestSubConnEmpty(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // sc is the one SubConn used throughout the test. Created on demand and + // re-used on every update. + var sc balancer.SubConn + + // Simple custom balancer that sets the address list to empty if the + // resolver produces no addresses. Pickfirst, by default, will remove the + // SubConn in this case instead. + bal := stub.BalancerFuncs{ + UpdateClientConnState: func(d *stub.BalancerData, ccs balancer.ClientConnState) error { + if sc == nil { + var err error + sc, err = d.ClientConn.NewSubConn(ccs.ResolverState.Addresses, balancer.NewSubConnOptions{ + StateListener: func(state balancer.SubConnState) { + switch state.ConnectivityState { + case connectivity.Ready: + d.ClientConn.UpdateState(balancer.State{ + ConnectivityState: connectivity.Ready, + Picker: &tsccPicker{sc: sc}, + }) + case connectivity.TransientFailure: + d.ClientConn.UpdateState(balancer.State{ + ConnectivityState: connectivity.TransientFailure, + Picker: base.NewErrPicker(fmt.Errorf("error connecting: %v", state.ConnectionError)), + }) + } + }, + }) + if err != nil { + t.Errorf("error creating initial subconn: %v", err) + } + } else { + d.ClientConn.UpdateAddresses(sc, ccs.ResolverState.Addresses) + } + sc.Connect() + + if len(ccs.ResolverState.Addresses) == 0 { + d.ClientConn.UpdateState(balancer.State{ + ConnectivityState: connectivity.TransientFailure, + Picker: base.NewErrPicker(errors.New("no addresses")), + }) + } else { + d.ClientConn.UpdateState(balancer.State{ + ConnectivityState: connectivity.Connecting, + Picker: &tsccPicker{sc: sc}, + }) + } + return nil + }, + } + stub.Register("tscc", bal) + + // Start the stub server with our stub balancer. + ss := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + } + if err := ss.Start(nil, grpc.WithDefaultServiceConfig(`{"loadBalancingConfig": [{"tscc":{}}]}`)); err != nil { + t.Fatalf("Error starting server: %v", err) + } + defer ss.Stop() + if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall failed: %v", err) + } + + t.Log("Removing addresses from resolver and SubConn") + ss.R.UpdateState(resolver.State{Addresses: []resolver.Address{}}) + testutils.AwaitState(ctx, t, ss.CC, connectivity.TransientFailure) + + t.Log("Re-adding addresses to resolver and SubConn") + ss.R.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: ss.Address}}}) + if _, err := ss.Client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall failed: %v", err) + } +} diff --git a/security/advancedtls/sni_appengine.go b/test/timeouts.go similarity index 69% rename from security/advancedtls/sni_appengine.go rename to test/timeouts.go index fffbb0107ddd..1c0c2123938a 100644 --- a/security/advancedtls/sni_appengine.go +++ b/test/timeouts.go @@ -1,5 +1,3 @@ -// +build appengine - /* * * Copyright 2020 gRPC authors. @@ -18,13 +16,14 @@ * */ -package advancedtls +package test -import ( - "crypto/tls" -) +import "time" -// buildGetCertificates is a no-op for appengine builds. -func buildGetCertificates(clientHello *tls.ClientHelloInfo, o *ServerOptions) (*tls.Certificate, error) { - return nil, nil -} +const ( + // Default timeout for tests in this package. + defaultTestTimeout = 10 * time.Second + // Default short timeout, to be used when waiting for events which are not + // expected to happen. + defaultTestShortTimeout = 100 * time.Millisecond +) diff --git a/test/tools/go.mod b/test/tools/go.mod index e683f01362c9..33856e80e68a 100644 --- a/test/tools/go.mod +++ b/test/tools/go.mod @@ -1,12 +1,19 @@ module google.golang.org/grpc/test/tools -go 1.11 +go 1.19 require ( - github.com/BurntSushi/toml v0.3.1 // indirect github.com/client9/misspell v0.3.4 - github.com/golang/protobuf v1.3.3 - golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 - golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135 - honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc + github.com/golang/protobuf v1.5.3 + golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 + golang.org/x/tools v0.11.0 + honnef.co/go/tools v0.4.3 +) + +require ( + github.com/BurntSushi/toml v1.3.2 // indirect + golang.org/x/exp/typeparams v0.0.0-20230713183714-613f0c0eb8a1 // indirect + golang.org/x/mod v0.12.0 // indirect + golang.org/x/sys v0.10.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect ) diff --git a/test/tools/go.sum b/test/tools/go.sum index 8dd58fbbf069..f8b2d644b624 100644 --- a/test/tools/go.sum +++ b/test/tools/go.sum @@ -1,18 +1,39 @@ -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= +github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 h1:XQyxROzUlZH+WIQwySDgnISgOivlhjIEwaQaJEJrrN0= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/exp/typeparams v0.0.0-20230713183714-613f0c0eb8a1 h1:VXDua8UTGWl3e7L5kCk5Vyt0LA3QpsyRu6XXL7K3v1w= +golang.org/x/exp/typeparams v0.0.0-20230713183714-613f0c0eb8a1/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135 h1:5Beo0mZN8dRzgrMMkDp0jc8YXQKx9DiJ2k1dkvGsn5A= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc h1:/hemPrYIhOhy8zYrNj+069zDB68us2sMGsfkFJO0iZs= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.11.0 h1:EMCa6U9S2LtZXLAMoWiR/R8dAQFRqbAitmbJ2UKhoi8= +golang.org/x/tools v0.11.0/go.mod h1:anzJrxPjNtfgiYQYirP2CPGzGLxrH2u2QBhn6Bf3qY8= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +honnef.co/go/tools v0.4.3 h1:o/n5/K5gXqk8Gozvs2cnL0F2S1/g1vcGCAx2vETjITw= +honnef.co/go/tools v0.4.3/go.mod h1:36ZgoUOrqOk1GxwHhyryEkq8FQWkUO2xGuSMhUCcdvA= diff --git a/test/tools/tools.go b/test/tools/tools.go index 511dc2534462..646a144ccca1 100644 --- a/test/tools/tools.go +++ b/test/tools/tools.go @@ -1,3 +1,4 @@ +//go:build tools // +build tools /* @@ -18,10 +19,9 @@ * */ -// This package exists to cause `go mod` and `go get` to believe these tools -// are dependencies, even though they are not runtime dependencies of any grpc -// package. This means they will appear in our `go.mod` file, but will not be -// a part of the build. +// This file is not intended to be compiled. Because some of these imports are +// not actual go packages, we use a build constraint at the top of this file to +// prevent tools from inspecting the imports. package tools diff --git a/test/tools/tools_vet.go b/test/tools/tools_vet.go new file mode 100644 index 000000000000..06ab2fd10be2 --- /dev/null +++ b/test/tools/tools_vet.go @@ -0,0 +1,21 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package tools is used to pin specific versions of external tools in this +// module's go.mod that gRPC uses for internal testing. +package tools diff --git a/test/transport_test.go b/test/transport_test.go new file mode 100644 index 000000000000..d58bdf8acd77 --- /dev/null +++ b/test/transport_test.go @@ -0,0 +1,155 @@ +/* +* +* Copyright 2023 gRPC authors. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* + */ +package test + +import ( + "context" + "io" + "net" + "sync" + "testing" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/transport" + "google.golang.org/grpc/status" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +// connWrapperWithCloseCh wraps a net.Conn and fires an event when closed. +type connWrapperWithCloseCh struct { + net.Conn + close *grpcsync.Event +} + +// Close closes the connection and sends a value on the close channel. +func (cw *connWrapperWithCloseCh) Close() error { + cw.close.Fire() + return cw.Conn.Close() +} + +// These custom creds are used for storing the connections made by the client. +// The closeCh in conn can be used to detect when conn is closed. +type transportRestartCheckCreds struct { + mu sync.Mutex + connections []*connWrapperWithCloseCh +} + +func (c *transportRestartCheckCreds) ServerHandshake(rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) { + return rawConn, nil, nil +} +func (c *transportRestartCheckCreds) ClientHandshake(ctx context.Context, authority string, rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) { + c.mu.Lock() + defer c.mu.Unlock() + conn := &connWrapperWithCloseCh{Conn: rawConn, close: grpcsync.NewEvent()} + c.connections = append(c.connections, conn) + return conn, nil, nil +} +func (c *transportRestartCheckCreds) Info() credentials.ProtocolInfo { + return credentials.ProtocolInfo{} +} +func (c *transportRestartCheckCreds) Clone() credentials.TransportCredentials { + return c +} +func (c *transportRestartCheckCreds) OverrideServerName(s string) error { + return nil +} + +// Tests that the client transport drains and restarts when next stream ID exceeds +// MaxStreamID. This test also verifies that subsequent RPCs use a new client +// transport and the old transport is closed. +func (s) TestClientTransportRestartsAfterStreamIDExhausted(t *testing.T) { + // Set the transport's MaxStreamID to 4 to cause connection to drain after 2 RPCs. + originalMaxStreamID := transport.MaxStreamID + transport.MaxStreamID = 4 + defer func() { + transport.MaxStreamID = originalMaxStreamID + }() + + ss := &stubserver.StubServer{ + FullDuplexCallF: func(stream testgrpc.TestService_FullDuplexCallServer) error { + if _, err := stream.Recv(); err != nil { + return status.Errorf(codes.Internal, "unexpected error receiving: %v", err) + } + if err := stream.Send(&testpb.StreamingOutputCallResponse{}); err != nil { + return status.Errorf(codes.Internal, "unexpected error sending: %v", err) + } + if recv, err := stream.Recv(); err != io.EOF { + return status.Errorf(codes.Internal, "Recv = %v, %v; want _, io.EOF", recv, err) + } + return nil + }, + } + + creds := &transportRestartCheckCreds{} + if err := ss.Start(nil, grpc.WithTransportCredentials(creds)); err != nil { + t.Fatalf("Starting stubServer: %v", err) + } + defer ss.Stop() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + var streams []testgrpc.TestService_FullDuplexCallClient + + const numStreams = 3 + // expected number of conns when each stream is created i.e., 3rd stream is created + // on a new connection. + expectedNumConns := [numStreams]int{1, 1, 2} + + // Set up 3 streams. + for i := 0; i < numStreams; i++ { + s, err := ss.Client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("Creating FullDuplex stream: %v", err) + } + streams = append(streams, s) + // Verify expected num of conns after each stream is created. + if len(creds.connections) != expectedNumConns[i] { + t.Fatalf("Got number of connections created: %v, want: %v", len(creds.connections), expectedNumConns[i]) + } + } + + // Verify all streams still work. + for i, stream := range streams { + if err := stream.Send(&testpb.StreamingOutputCallRequest{}); err != nil { + t.Fatalf("Sending on stream %d: %v", i, err) + } + if _, err := stream.Recv(); err != nil { + t.Fatalf("Receiving on stream %d: %v", i, err) + } + } + + for i, stream := range streams { + if err := stream.CloseSend(); err != nil { + t.Fatalf("CloseSend() on stream %d: %v", i, err) + } + } + + // Verifying first connection was closed. + select { + case <-creds.connections[0].close.Done(): + case <-ctx.Done(): + t.Fatal("Timeout expired when waiting for first client transport to close") + } +} diff --git a/test/xds/xds_client_ack_nack_test.go b/test/xds/xds_client_ack_nack_test.go new file mode 100644 index 000000000000..87ff0077cd70 --- /dev/null +++ b/test/xds/xds_client_ack_nack_test.go @@ -0,0 +1,190 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds_test + +import ( + "context" + "fmt" + "testing" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +// We are interested in LDS, RDS, CDS and EDS resources as part of the regular +// xDS flow on the client. +const wantResources = 4 + +// seenAllACKs returns true if the provided ackVersions map contains valid acks +// for all the resources that we are interested in. If `wantNonEmpty` is true, +// only non-empty ack versions are considered valid. +func seenAllACKs(acksVersions map[string]string, wantNonEmpty bool) bool { + if len(acksVersions) != wantResources { + return false + } + for _, ack := range acksVersions { + if wantNonEmpty && ack == "" { + return false + } + } + return true +} + +// TestClientResourceVersionAfterStreamRestart tests the scenario where the +// xdsClient's ADS stream to the management server gets broken. This test +// verifies that the version number on the initial request on the new stream +// indicates the most recent version seen by the client on the previous stream. +func (s) TestClientResourceVersionAfterStreamRestart(t *testing.T) { + // Create a restartable listener which can close existing connections. + l, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("testutils.LocalTCPListener() failed: %v", err) + } + lis := testutils.NewRestartableListener(l) + + // We depend on the fact that the management server assigns monotonically + // increasing stream IDs starting at 1. + const ( + idBeforeRestart = 1 + idAfterRestart = 2 + ) + + // Events of importance in the test, in the order in which they are expected + // to happen. + acksReceivedBeforeRestart := grpcsync.NewEvent() + streamRestarted := grpcsync.NewEvent() + acksReceivedAfterRestart := grpcsync.NewEvent() + + // Map from stream id to a map of resource type to resource version. + ackVersionsMap := make(map[int64]map[string]string) + managementServer, nodeID, _, resolver, cleanup1 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{ + Listener: lis, + OnStreamRequest: func(id int64, req *v3discoverypb.DiscoveryRequest) error { + // Return early under the following circumstances: + // - Received all the requests we wanted to see. This is to avoid + // any stray requests leading to test flakes. + // - Request contains no resource names. Such requests are usually + // seen when the xdsclient is shutting down and is no longer + // interested in the resources that it had subscribed to earlier. + if acksReceivedAfterRestart.HasFired() || len(req.GetResourceNames()) == 0 { + return nil + } + // Create a stream specific map to store ack versions if this is the + // first time we are seeing this stream id. + if ackVersionsMap[id] == nil { + ackVersionsMap[id] = make(map[string]string) + } + ackVersionsMap[id][req.GetTypeUrl()] = req.GetVersionInfo() + // Prior to stream restart, we are interested only in non-empty + // resource versions. The xdsclient first sends out requests with an + // empty version string. After receipt of requested resource, it + // sends out another request for the same resource, but this time + // with a non-empty version string, to serve as an ACK. + if seenAllACKs(ackVersionsMap[idBeforeRestart], true) { + acksReceivedBeforeRestart.Fire() + } + // After stream restart, we expect the xdsclient to send out + // requests with version string set to the previously ACKed + // versions. If it sends out requests with empty version string, it + // is a bug and we want this test to catch it. + if seenAllACKs(ackVersionsMap[idAfterRestart], false) { + acksReceivedAfterRestart.Fire() + } + return nil + }, + OnStreamClosed: func(int64, *v3corepb.Node) { + streamRestarted.Fire() + }, + }) + defer cleanup1() + + server := stubserver.StartTestService(t, nil) + defer server.Stop() + + const serviceName = "my-service-client-side-xds" + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: "localhost", + Port: testutils.ParsePort(t, server.Address), + SecLevel: e2e.SecurityLevelNone, + }) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create a ClientConn and make a successful RPC. + cc, err := grpc.Dial(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("rpc EmptyCall() failed: %v", err) + } + + // A successful RPC means that the xdsclient received all requested + // resources. The ACKs from the xdsclient may get a little delayed. So, we + // need to wait for all ACKs to be received on the management server before + // restarting the stream. + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for all resources to be ACKed prior to stream restart") + case <-acksReceivedBeforeRestart.Done(): + } + + // Stop the listener on the management server. This will cause the client to + // backoff and recreate the stream. + lis.Stop() + + // Wait for the stream to be closed on the server. + <-streamRestarted.Done() + + // Restart the listener on the management server to be able to accept + // reconnect attempts from the client. + lis.Restart() + + // Wait for all the previously sent resources to be re-requested. + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for all resources to be ACKed post stream restart") + case <-acksReceivedAfterRestart.Done(): + } + + if diff := cmp.Diff(ackVersionsMap[idBeforeRestart], ackVersionsMap[idAfterRestart]); diff != "" { + t.Fatalf("unexpected diff in ack versions before and after stream restart (-want, +got):\n%s", diff) + } + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("rpc EmptyCall() failed: %v", err) + } +} diff --git a/test/xds/xds_client_affinity_test.go b/test/xds/xds_client_affinity_test.go new file mode 100644 index 000000000000..7fff019fa526 --- /dev/null +++ b/test/xds/xds_client_affinity_test.go @@ -0,0 +1,135 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds_test + +import ( + "context" + "fmt" + "testing" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +// hashRouteConfig returns a RouteConfig resource with hash policy set to +// header "session_id". +func hashRouteConfig(routeName, ldsTarget, clusterName string) *v3routepb.RouteConfiguration { + return &v3routepb.RouteConfiguration{ + Name: routeName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{ldsTarget}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}}, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName}, + HashPolicy: []*v3routepb.RouteAction_HashPolicy{{ + PolicySpecifier: &v3routepb.RouteAction_HashPolicy_Header_{ + Header: &v3routepb.RouteAction_HashPolicy_Header{ + HeaderName: "session_id", + }, + }, + Terminal: true, + }}, + }}, + }}, + }}, + } +} + +// ringhashCluster returns a Cluster resource that picks ringhash as the lb +// policy. +func ringhashCluster(clusterName, edsServiceName string) *v3clusterpb.Cluster { + return &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: edsServiceName, + }, + LbPolicy: v3clusterpb.Cluster_RING_HASH, + } +} + +// TestClientSideAffinitySanityCheck tests that the affinity config can be +// propagated to pick the ring_hash policy. It doesn't test the affinity +// behavior in ring_hash policy. +func (s) TestClientSideAffinitySanityCheck(t *testing.T) { + defer func() func() { + old := envconfig.XDSRingHash + envconfig.XDSRingHash = true + return func() { envconfig.XDSRingHash = old } + }()() + + managementServer, nodeID, _, resolver, cleanup1 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup1() + + server := stubserver.StartTestService(t, nil) + defer server.Stop() + + const serviceName = "my-service-client-side-xds" + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: "localhost", + Port: testutils.ParsePort(t, server.Address), + SecLevel: e2e.SecurityLevelNone, + }) + // Replace RDS and CDS resources with ringhash config, but keep the resource + // names. + resources.Routes = []*v3routepb.RouteConfiguration{hashRouteConfig( + resources.Routes[0].Name, + resources.Listeners[0].Name, + resources.Clusters[0].Name, + )} + resources.Clusters = []*v3clusterpb.Cluster{ringhashCluster( + resources.Clusters[0].Name, + resources.Clusters[0].EdsClusterConfig.ServiceName, + )} + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create a ClientConn and make a successful RPC. + cc, err := grpc.Dial(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("rpc EmptyCall() failed: %v", err) + } +} diff --git a/test/xds/xds_client_custom_lb_test.go b/test/xds/xds_client_custom_lb_test.go new file mode 100644 index 000000000000..b72686106c6d --- /dev/null +++ b/test/xds/xds_client_custom_lb_test.go @@ -0,0 +1,279 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds_test + +import ( + "context" + "fmt" + "testing" + "time" + + "google.golang.org/grpc" + _ "google.golang.org/grpc/balancer/leastrequest" // To register least_request + _ "google.golang.org/grpc/balancer/weightedroundrobin" // To register weighted_round_robin + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/roundrobin" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/resolver" + + v3xdsxdstypepb "github.com/cncf/xds/go/xds/type/v3" + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3clientsideweightedroundrobinpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3" + v3leastrequestpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/least_request/v3" + v3roundrobinpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/round_robin/v3" + v3wrrlocalitypb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/wrr_locality/v3" + "github.com/golang/protobuf/proto" + structpb "github.com/golang/protobuf/ptypes/struct" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +// wrrLocality is a helper that takes a proto message and returns a +// WrrLocalityProto with the proto message marshaled into a proto.Any as a +// child. +func wrrLocality(m proto.Message) *v3wrrlocalitypb.WrrLocality { + return &v3wrrlocalitypb.WrrLocality{ + EndpointPickingPolicy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: testutils.MarshalAny(m), + }, + }, + }, + }, + } +} + +// clusterWithLBConfiguration returns a cluster resource with the proto message +// passed Marshaled to an any and specified through the load_balancing_policy +// field. +func clusterWithLBConfiguration(clusterName, edsServiceName string, secLevel e2e.SecurityLevel, m proto.Message) *v3clusterpb.Cluster { + cluster := e2e.DefaultCluster(clusterName, edsServiceName, secLevel) + cluster.LoadBalancingPolicy = &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: testutils.MarshalAny(m), + }, + }, + }, + } + return cluster +} + +// TestWRRLocality tests RPC distribution across a scenario with 5 backends, +// with 2 backends in a locality with weight 1, and 3 backends in a second +// locality with weight 2. Through xDS, the test configures a +// wrr_locality_balancer with either a round robin or custom (specifying pick +// first) child load balancing policy, and asserts the correct distribution +// based on the locality weights and the endpoint picking policy specified. +func (s) TestWrrLocality(t *testing.T) { + oldCustomLBSupport := envconfig.XDSCustomLBPolicy + envconfig.XDSCustomLBPolicy = true + defer func() { + envconfig.XDSCustomLBPolicy = oldCustomLBSupport + }() + oldLeastRequestLBSupport := envconfig.LeastRequestLB + envconfig.LeastRequestLB = true + defer func() { + envconfig.LeastRequestLB = oldLeastRequestLBSupport + }() + + backend1 := stubserver.StartTestService(t, nil) + port1 := testutils.ParsePort(t, backend1.Address) + defer backend1.Stop() + backend2 := stubserver.StartTestService(t, nil) + port2 := testutils.ParsePort(t, backend2.Address) + defer backend2.Stop() + backend3 := stubserver.StartTestService(t, nil) + port3 := testutils.ParsePort(t, backend3.Address) + defer backend3.Stop() + backend4 := stubserver.StartTestService(t, nil) + port4 := testutils.ParsePort(t, backend4.Address) + defer backend4.Stop() + backend5 := stubserver.StartTestService(t, nil) + port5 := testutils.ParsePort(t, backend5.Address) + defer backend5.Stop() + const serviceName = "my-service-client-side-xds" + + tests := []struct { + name string + // Configuration will be specified through load_balancing_policy field. + wrrLocalityConfiguration *v3wrrlocalitypb.WrrLocality + addressDistributionWant []struct { + addr string + count int + } + }{ + { + name: "rr_child", + wrrLocalityConfiguration: wrrLocality(&v3roundrobinpb.RoundRobin{}), + // Each addresses expected probability is locality weight of + // locality / total locality weights multiplied by 1 / number of + // endpoints in each locality (due to round robin across endpoints + // in a locality). Thus, address 1 and address 2 have 1/3 * 1/2 + // probability, and addresses 3 4 5 have 2/3 * 1/3 probability of + // being routed to. + addressDistributionWant: []struct { + addr string + count int + }{ + {addr: backend1.Address, count: 6}, + {addr: backend2.Address, count: 6}, + {addr: backend3.Address, count: 8}, + {addr: backend4.Address, count: 8}, + {addr: backend5.Address, count: 8}, + }, + }, + // This configures custom lb as the child of wrr_locality, which points + // to our pick_first implementation. Thus, the expected distribution of + // addresses is locality weight of locality / total locality weights as + // the probability of picking the first backend within the locality + // (e.g. Address 1 for locality 1, and Address 3 for locality 2). + { + name: "custom_lb_child_pick_first", + wrrLocalityConfiguration: wrrLocality(&v3xdsxdstypepb.TypedStruct{ + TypeUrl: "type.googleapis.com/pick_first", + Value: &structpb.Struct{}, + }), + addressDistributionWant: []struct { + addr string + count int + }{ + {addr: backend1.Address, count: 1}, + {addr: backend3.Address, count: 2}, + }, + }, + // Sanity check for weighted round robin. Don't need to test super + // specific behaviors, as that is covered in unit tests. Set up weighted + // round robin as the endpoint picking policy with per RPC load reports + // enabled. Due the server not sending trailers with load reports, the + // weighted round robin policy should essentially function as round + // robin, and thus should have the same distribution as round robin + // above. + { + name: "custom_lb_child_wrr/", + wrrLocalityConfiguration: wrrLocality(&v3clientsideweightedroundrobinpb.ClientSideWeightedRoundRobin{ + EnableOobLoadReport: &wrapperspb.BoolValue{ + Value: false, + }, + // BlackoutPeriod long enough to cause load report weights to + // trigger in the scope of test case, but no load reports + // configured anyway. + BlackoutPeriod: durationpb.New(10 * time.Second), + WeightExpirationPeriod: durationpb.New(10 * time.Second), + WeightUpdatePeriod: durationpb.New(time.Second), + ErrorUtilizationPenalty: &wrapperspb.FloatValue{Value: 1}, + }), + addressDistributionWant: []struct { + addr string + count int + }{ + {addr: backend1.Address, count: 6}, + {addr: backend2.Address, count: 6}, + {addr: backend3.Address, count: 8}, + {addr: backend4.Address, count: 8}, + {addr: backend5.Address, count: 8}, + }, + }, + { + name: "custom_lb_least_request", + wrrLocalityConfiguration: wrrLocality(&v3leastrequestpb.LeastRequest{ + ChoiceCount: wrapperspb.UInt32(2), + }), + // The test performs a Unary RPC, and blocks until the RPC returns, + // and then makes the next Unary RPC. Thus, over iterations, no RPC + // counts are present. This causes least request's randomness of + // indexes to sample to converge onto a round robin distribution per + // locality. Thus, expect the same distribution as round robin + // above. + addressDistributionWant: []struct { + addr string + count int + }{ + {addr: backend1.Address, count: 6}, + {addr: backend2.Address, count: 6}, + {addr: backend3.Address, count: 8}, + {addr: backend4.Address, count: 8}, + {addr: backend5.Address, count: 8}, + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + managementServer, nodeID, _, r, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup() + routeConfigName := "route-" + serviceName + clusterName := "cluster-" + serviceName + endpointsName := "endpoints-" + serviceName + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeConfigName, serviceName, clusterName)}, + Clusters: []*v3clusterpb.Cluster{clusterWithLBConfiguration(clusterName, endpointsName, e2e.SecurityLevelNone, test.wrrLocalityConfiguration)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.EndpointResourceWithOptions(e2e.EndpointOptions{ + ClusterName: endpointsName, + Host: "localhost", + Localities: []e2e.LocalityOptions{ + { + Backends: []e2e.BackendOptions{{Port: port1}, {Port: port2}}, + Weight: 1, + }, + { + Backends: []e2e.BackendOptions{{Port: port3}, {Port: port4}, {Port: port5}}, + Weight: 2, + }, + }, + })}, + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + cc, err := grpc.Dial(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("Failed to dial local test server: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + var addrDistWant []resolver.Address + for _, addrAndCount := range test.addressDistributionWant { + for i := 0; i < addrAndCount.count; i++ { + addrDistWant = append(addrDistWant, resolver.Address{Addr: addrAndCount.addr}) + } + } + if err := roundrobin.CheckWeightedRoundRobinRPCs(ctx, client, addrDistWant); err != nil { + t.Fatalf("Error in expected round robin: %v", err) + } + }) + } +} diff --git a/test/xds/xds_client_federation_test.go b/test/xds/xds_client_federation_test.go new file mode 100644 index 000000000000..1aebcd226104 --- /dev/null +++ b/test/xds/xds_client_federation_test.go @@ -0,0 +1,276 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds_test + +import ( + "context" + "fmt" + "strings" + "testing" + + "github.com/google/uuid" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/bootstrap" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/status" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +// TestClientSideFederation tests that federation is supported. +// +// In this test, some xDS responses contain resource names in another authority +// (in the new resource name style): +// - LDS: old style, no authority (default authority) +// - RDS: new style, in a different authority +// - CDS: old style, no authority (default authority) +// - EDS: new style, in a different authority +func (s) TestClientSideFederation(t *testing.T) { + oldXDSFederation := envconfig.XDSFederation + envconfig.XDSFederation = true + defer func() { envconfig.XDSFederation = oldXDSFederation }() + + // Start a management server as the default authority. + serverDefaultAuth, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) + if err != nil { + t.Fatalf("Failed to spin up the xDS management server: %v", err) + } + t.Cleanup(serverDefaultAuth.Stop) + + // Start another management server as the other authority. + const nonDefaultAuth = "non-default-auth" + serverAnotherAuth, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) + if err != nil { + t.Fatalf("Failed to spin up the xDS management server: %v", err) + } + t.Cleanup(serverAnotherAuth.Stop) + + // Create a bootstrap file in a temporary directory. + nodeID := uuid.New().String() + bootstrapContents, err := bootstrap.Contents(bootstrap.Options{ + NodeID: nodeID, + ServerURI: serverDefaultAuth.Address, + ServerListenerResourceNameTemplate: e2e.ServerListenerResourceNameTemplate, + // Specify the address of the non-default authority. + Authorities: map[string]string{nonDefaultAuth: serverAnotherAuth.Address}, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap file: %v", err) + } + + resolverBuilder := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error)) + resolver, err := resolverBuilder(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS resolver for testing: %v", err) + } + server := stubserver.StartTestService(t, nil) + defer server.Stop() + + const serviceName = "my-service-client-side-xds" + // LDS is old style name. + ldsName := serviceName + // RDS is new style, with the non default authority. + rdsName := fmt.Sprintf("xdstp://%s/envoy.config.route.v3.RouteConfiguration/%s", nonDefaultAuth, "route-"+serviceName) + // CDS is old style name. + cdsName := "cluster-" + serviceName + // EDS is new style, with the non default authority. + edsName := fmt.Sprintf("xdstp://%s/envoy.config.route.v3.ClusterLoadAssignment/%s", nonDefaultAuth, "endpoints-"+serviceName) + + // Split resources, put LDS/CDS in the default authority, and put RDS/EDS in + // the other authority. + resourcesDefault := e2e.UpdateOptions{ + NodeID: nodeID, + // This has only LDS and CDS. + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(ldsName, rdsName)}, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(cdsName, edsName, e2e.SecurityLevelNone)}, + SkipValidation: true, + } + resourcesAnother := e2e.UpdateOptions{ + NodeID: nodeID, + // This has only RDS and EDS. + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(rdsName, ldsName, cdsName)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(edsName, "localhost", []uint32{testutils.ParsePort(t, server.Address)})}, + SkipValidation: true, + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + // This has only LDS and CDS. + if err := serverDefaultAuth.Update(ctx, resourcesDefault); err != nil { + t.Fatal(err) + } + // This has only RDS and EDS. + if err := serverAnotherAuth.Update(ctx, resourcesAnother); err != nil { + t.Fatal(err) + } + + // Create a ClientConn and make a successful RPC. + cc, err := grpc.Dial(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("rpc EmptyCall() failed: %v", err) + } +} + +// TestFederation_UnknownAuthorityInDialTarget tests the case where a ClientConn +// is created with a dial target containing an authority which is not specified +// in the bootstrap configuration. The test verifies that RPCs on the ClientConn +// fail with an appropriate error. +func (s) TestFederation_UnknownAuthorityInDialTarget(t *testing.T) { + oldXDSFederation := envconfig.XDSFederation + envconfig.XDSFederation = true + defer func() { envconfig.XDSFederation = oldXDSFederation }() + + // Setting up the management server is not *really* required for this test + // case. All we need is a bootstrap configuration which does not contain the + // authority mentioned in the dial target. But setting up the management + // server and actually making an RPC ensures that the xDS client is + // configured properly, and when we dial with an unknown authority in the + // next step, we can be sure that the error we receive is legitimate. + managementServer, nodeID, _, resolver, cleanup1 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup1() + + server := stubserver.StartTestService(t, nil) + defer server.Stop() + + const serviceName = "my-service-client-side-xds" + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: "localhost", + Port: testutils.ParsePort(t, server.Address), + SecLevel: e2e.SecurityLevelNone, + }) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create a ClientConn and make a successful RPC. + target := fmt.Sprintf("xds:///%s", serviceName) + cc, err := grpc.Dial(target, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver)) + if err != nil { + t.Fatalf("Dialing target %q: %v", target, err) + } + defer cc.Close() + t.Log("Created ClientConn to test service") + + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() RPC: %v", err) + } + t.Log("Successfully performed an EmptyCall RPC") + + target = fmt.Sprintf("xds://unknown-authority/%s", serviceName) + t.Logf("Dialing target %q with unknown authority which is expected to fail", target) + const wantErr = `authority "unknown-authority" is not found in the bootstrap file` + _, err = grpc.Dial(target, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver)) + if err == nil || !strings.Contains(err.Error(), wantErr) { + t.Fatalf("grpc.Dial(%q) returned %v, want: %s", target, err, wantErr) + } +} + +// TestFederation_UnknownAuthorityInReceivedResponse tests the case where the +// LDS resource associated with the dial target contains an RDS resource name +// with an authority which is not specified in the bootstrap configuration. The +// test verifies that RPCs fail with an appropriate error. +func (s) TestFederation_UnknownAuthorityInReceivedResponse(t *testing.T) { + oldXDSFederation := envconfig.XDSFederation + envconfig.XDSFederation = true + defer func() { envconfig.XDSFederation = oldXDSFederation }() + + mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) + if err != nil { + t.Fatalf("Failed to spin up the xDS management server: %v", err) + } + defer mgmtServer.Stop() + + nodeID := uuid.New().String() + bootstrapContents, err := bootstrap.Contents(bootstrap.Options{ + NodeID: nodeID, + ServerURI: mgmtServer.Address, + ServerListenerResourceNameTemplate: e2e.ServerListenerResourceNameTemplate, + }) + if err != nil { + t.Fatal(err) + } + + resolverBuilder := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error)) + resolver, err := resolverBuilder(bootstrapContents) + if err != nil { + t.Fatalf("Creating xDS resolver for testing: %v", err) + } + + // LDS is old style name. + // RDS is new style, with an unknown authority. + const serviceName = "my-service-client-side-xds" + const unknownAuthority = "unknown-authority" + ldsName := serviceName + rdsName := fmt.Sprintf("xdstp://%s/envoy.config.route.v3.RouteConfiguration/%s", unknownAuthority, "route-"+serviceName) + + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(ldsName, rdsName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(rdsName, ldsName, "cluster-"+serviceName)}, + SkipValidation: true, // This update has only LDS and RDS resources. + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + target := fmt.Sprintf("xds:///%s", serviceName) + cc, err := grpc.Dial(target, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver)) + if err != nil { + t.Fatalf("Dialing target %q: %v", target, err) + } + defer cc.Close() + t.Log("Created ClientConn to test service") + + client := testgrpc.NewTestServiceClient(cc) + _, err = client.EmptyCall(ctx, &testpb.Empty{}) + if err == nil { + t.Fatal("EmptyCall RPC succeeded for target with unknown authority when expected to fail") + } + if got, want := status.Code(err), codes.Unavailable; got != want { + t.Fatalf("EmptyCall RPC returned status code: %v, want %v", got, want) + } + if wantErr := `failed to find authority "unknown-authority"`; !strings.Contains(err.Error(), wantErr) { + t.Fatalf("EmptyCall RPC returned error: %v, want %v", err, wantErr) + } +} diff --git a/test/xds/xds_client_ignore_resource_deletion_test.go b/test/xds/xds_client_ignore_resource_deletion_test.go new file mode 100644 index 000000000000..5978ab4d7fe9 --- /dev/null +++ b/test/xds/xds_client_ignore_resource_deletion_test.go @@ -0,0 +1,484 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds_test + +import ( + "context" + "fmt" + "net" + "sync" + "testing" + "time" + + "github.com/google/uuid" + "google.golang.org/grpc" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/bootstrap" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/xds" + + clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +const ( + serviceName = "my-service-xds" + rdsName = "route-" + serviceName + cdsName1 = "cluster1-" + serviceName + cdsName2 = "cluster2-" + serviceName + edsName1 = "eds1-" + serviceName + edsName2 = "eds2-" + serviceName +) + +var ( + // This route configuration resource contains two routes: + // - a route for the EmptyCall rpc, to be sent to cluster1 + // - a route for the UnaryCall rpc, to be sent to cluster2 + defaultRouteConfigWithTwoRoutes = &routepb.RouteConfiguration{ + Name: rdsName, + VirtualHosts: []*routepb.VirtualHost{{ + Domains: []string{serviceName}, + Routes: []*routepb.Route{ + { + Match: &routepb.RouteMatch{PathSpecifier: &routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"}}, + Action: &routepb.Route_Route{Route: &routepb.RouteAction{ + ClusterSpecifier: &routepb.RouteAction_Cluster{Cluster: cdsName1}, + }}, + }, + { + Match: &routepb.RouteMatch{PathSpecifier: &routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/UnaryCall"}}, + Action: &routepb.Route_Route{Route: &routepb.RouteAction{ + ClusterSpecifier: &routepb.RouteAction_Cluster{Cluster: cdsName2}, + }}, + }, + }, + }}, + } +) + +// This test runs subtest each for a Listener resource and a Cluster resource deletion +// in the response from the server for the following cases: +// - testResourceDeletionIgnored: When ignore_resource_deletion is set, the +// xDSClient should not delete the resource. +// - testResourceDeletionNotIgnored: When ignore_resource_deletion is unset, +// the xDSClient should delete the resource. +// +// Resource deletion is only applicable to Listener and Cluster resources. +func (s) TestIgnoreResourceDeletionOnClient(t *testing.T) { + server1 := stubserver.StartTestService(t, nil) + t.Cleanup(server1.Stop) + + server2 := stubserver.StartTestService(t, nil) + t.Cleanup(server2.Stop) + + initialResourceOnServer := func(nodeID string) e2e.UpdateOptions { + return e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*listenerpb.Listener{e2e.DefaultClientListener(serviceName, rdsName)}, + Routes: []*routepb.RouteConfiguration{defaultRouteConfigWithTwoRoutes}, + Clusters: []*clusterpb.Cluster{ + e2e.DefaultCluster(cdsName1, edsName1, e2e.SecurityLevelNone), + e2e.DefaultCluster(cdsName2, edsName2, e2e.SecurityLevelNone), + }, + Endpoints: []*endpointpb.ClusterLoadAssignment{ + e2e.DefaultEndpoint(edsName1, "localhost", []uint32{testutils.ParsePort(t, server1.Address)}), + e2e.DefaultEndpoint(edsName2, "localhost", []uint32{testutils.ParsePort(t, server2.Address)}), + }, + SkipValidation: true, + } + } + + tests := []struct { + name string + updateResource func(r *e2e.UpdateOptions) + }{ + { + name: "listener", + updateResource: func(r *e2e.UpdateOptions) { + r.Listeners = nil + }, + }, + { + name: "cluster", + updateResource: func(r *e2e.UpdateOptions) { + r.Clusters = nil + }, + }, + } + for _, test := range tests { + t.Run(fmt.Sprintf("%s resource deletion ignored", test.name), func(t *testing.T) { + testResourceDeletionIgnored(t, initialResourceOnServer, test.updateResource) + }) + t.Run(fmt.Sprintf("%s resource deletion not ignored", test.name), func(t *testing.T) { + testResourceDeletionNotIgnored(t, initialResourceOnServer, test.updateResource) + }) + } +} + +// This subtest tests the scenario where the bootstrap config has "ignore_resource_deletion" +// set in "server_features" field. This subtest verifies that the resource was +// not deleted by the xDSClient when a resource is missing the xDS response and +// RPCs continue to succeed. +func testResourceDeletionIgnored(t *testing.T, initialResource func(string) e2e.UpdateOptions, updateResource func(r *e2e.UpdateOptions)) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + t.Cleanup(cancel) + mgmtServer := startManagementServer(t) + nodeID := uuid.New().String() + bs := generateBootstrapContents(t, mgmtServer.Address, true, nodeID) + xdsR := xdsResolverBuilder(t, bs) + resources := initialResource(nodeID) + + // Update the management server with initial resources setup. + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + cc, err := grpc.Dial(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsR)) + if err != nil { + t.Fatalf("Failed to dial local test server: %v.", err) + } + t.Cleanup(func() { cc.Close() }) + + if err := verifyRPCtoAllEndpoints(cc); err != nil { + t.Fatal(err) + } + + // Mutate resource and update on the server. + updateResource(&resources) + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Make an RPC every 50ms for the next 500ms. This is to ensure that the + // updated resource is received from the management server and is processed by + // gRPC. Since resource deletions are ignored by the xDS client, we expect RPCs + // to all endpoints to keep succeeding. + timer := time.NewTimer(500 * time.Millisecond) + ticker := time.NewTicker(50 * time.Millisecond) + t.Cleanup(ticker.Stop) + for { + if err := verifyRPCtoAllEndpoints(cc); err != nil { + t.Fatal(err) + } + select { + case <-ctx.Done(): + return + case <-timer.C: + return + case <-ticker.C: + } + } +} + +// This subtest tests the scenario where the bootstrap config has "ignore_resource_deletion" +// not set in "server_features" field. This subtest verifies that the resource was +// deleted by the xDSClient when a resource is missing the xDS response and subsequent +// RPCs fail. +func testResourceDeletionNotIgnored(t *testing.T, initialResource func(string) e2e.UpdateOptions, updateResource func(r *e2e.UpdateOptions)) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout*1000) + t.Cleanup(cancel) + mgmtServer := startManagementServer(t) + nodeID := uuid.New().String() + bs := generateBootstrapContents(t, mgmtServer.Address, false, nodeID) + xdsR := xdsResolverBuilder(t, bs) + resources := initialResource(nodeID) + + // Update the management server with initial resources setup. + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + cc, err := grpc.Dial(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsR)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + t.Cleanup(func() { cc.Close() }) + + if err := verifyRPCtoAllEndpoints(cc); err != nil { + t.Fatal(err) + } + + // Mutate resource and update on the server. + updateResource(&resources) + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Spin up go routines to verify RPCs fail after the update. + client := testgrpc.NewTestServiceClient(cc) + wg := sync.WaitGroup{} + wg.Add(2) + go func() { + defer wg.Done() + for ; ctx.Err() == nil; <-time.After(10 * time.Millisecond) { + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + return + } + } + }() + go func() { + defer wg.Done() + for ; ctx.Err() == nil; <-time.After(10 * time.Millisecond) { + if _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil { + return + } + } + }() + + wg.Wait() + if ctx.Err() != nil { + t.Fatal("Context expired before RPCs failed.") + } +} + +// This helper creates a management server for the test. +func startManagementServer(t *testing.T) *e2e.ManagementServer { + t.Helper() + mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) + if err != nil { + t.Fatalf("Failed to start management server: %v", err) + } + t.Cleanup(mgmtServer.Stop) + return mgmtServer +} + +// This helper generates a custom bootstrap config for the test. +func generateBootstrapContents(t *testing.T, serverURI string, ignoreResourceDeletion bool, nodeID string) []byte { + t.Helper() + bootstrapContents, err := bootstrap.Contents(bootstrap.Options{ + NodeID: nodeID, + ServerURI: serverURI, + ServerListenerResourceNameTemplate: e2e.ServerListenerResourceNameTemplate, + IgnoreResourceDeletion: ignoreResourceDeletion, + }) + if err != nil { + t.Fatal(err) + } + return bootstrapContents +} + +// This helper creates an XDS resolver Builder from the bootstrap config passed +// as parameter. +func xdsResolverBuilder(t *testing.T, bs []byte) resolver.Builder { + t.Helper() + resolverBuilder := internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error)) + xdsR, err := resolverBuilder(bs) + if err != nil { + t.Fatalf("Creating xDS resolver for testing failed for config %q: %v", string(bs), err) + } + return xdsR +} + +// This helper creates an xDS-enabled gRPC server using the listener and the +// bootstrap config passed. It then registers the test service on the newly +// created gRPC server and starts serving. +func setupGRPCServerWithModeChangeChannelAndServe(t *testing.T, bootstrapContents []byte, lis net.Listener) chan connectivity.ServingMode { + t.Helper() + updateCh := make(chan connectivity.ServingMode, 1) + + // Create a server option to get notified about serving mode changes. + modeChangeOpt := xds.ServingModeCallback(func(addr net.Addr, args xds.ServingModeChangeArgs) { + t.Logf("Serving mode for listener %q changed to %q, err: %v", addr.String(), args.Mode, args.Err) + updateCh <- args.Mode + }) + server, err := xds.NewGRPCServer(grpc.Creds(insecure.NewCredentials()), modeChangeOpt, xds.BootstrapContentsForTesting(bootstrapContents)) + if err != nil { + t.Fatalf("Failed to create an xDS enabled gRPC server: %v", err) + } + t.Cleanup(server.Stop) + testgrpc.RegisterTestServiceServer(server, &testService{}) + + // Serve. + go func() { + if err := server.Serve(lis); err != nil { + t.Errorf("Serve() failed: %v", err) + } + }() + + return updateCh +} + +// This helper creates a new TCP listener. This helper also uses this listener to +// create a resource update with a listener resource. This helper returns the +// resource update and the TCP listener. +func resourceWithListenerForGRPCServer(t *testing.T, nodeID string) (e2e.UpdateOptions, net.Listener) { + t.Helper() + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("testutils.LocalTCPListener() failed: %v", err) + } + t.Cleanup(func() { lis.Close() }) + host, port, err := hostPortFromListener(lis) + if err != nil { + t.Fatalf("Failed to retrieve host and port of listener at %q: %v", lis.Addr(), err) + } + listener := e2e.DefaultServerListener(host, port, e2e.SecurityLevelNone) + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*listenerpb.Listener{listener}, + } + return resources, lis +} + +// This test creates a gRPC server which provides server-side xDS functionality +// by talking to a custom management server. This tests the scenario where bootstrap +// config with "server_features" includes "ignore_resource_deletion". In which +// case, when the listener resource is deleted on the management server, the gRPC +// server should continue to serve RPCs. +func (s) TestListenerResourceDeletionOnServerIgnored(t *testing.T) { + mgmtServer := startManagementServer(t) + nodeID := uuid.New().String() + bs := generateBootstrapContents(t, mgmtServer.Address, true, nodeID) + xdsR := xdsResolverBuilder(t, bs) + resources, lis := resourceWithListenerForGRPCServer(t, nodeID) + modeChangeCh := setupGRPCServerWithModeChangeChannelAndServe(t, bs, lis) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Wait for the server to update to ServingModeServing mode. + select { + case <-ctx.Done(): + t.Fatal("Test timed out waiting for a server to change to ServingModeServing.") + case mode := <-modeChangeCh: + if mode != connectivity.ServingModeServing { + t.Fatalf("Server switched to mode %v, want %v", mode, connectivity.ServingModeServing) + } + } + + // Create a ClientConn and make a successful RPCs. + cc, err := grpc.Dial(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsR)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + if err := verifyRPCtoAllEndpoints(cc); err != nil { + t.Fatal(err) + } + + // Update without a listener resource. + if err := mgmtServer.Update(ctx, e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*listenerpb.Listener{}, + }); err != nil { + t.Fatal(err) + } + + // Perform RPCs every 100 ms for 1s and verify that the serving mode does not + // change on gRPC server. + timer := time.NewTimer(500 * time.Millisecond) + ticker := time.NewTicker(50 * time.Millisecond) + t.Cleanup(ticker.Stop) + for { + if err := verifyRPCtoAllEndpoints(cc); err != nil { + t.Fatal(err) + } + select { + case <-timer.C: + return + case mode := <-modeChangeCh: + t.Fatalf("Server switched to mode: %v when no switch was expected", mode) + case <-ticker.C: + } + } +} + +// This test creates a gRPC server which provides server-side xDS functionality +// by talking to a custom management server. This tests the scenario where bootstrap +// config with "server_features" does not include "ignore_resource_deletion". In +// which case, when the listener resource is deleted on the management server, the +// gRPC server should stop serving RPCs and switch mode to ServingModeNotServing. +func (s) TestListenerResourceDeletionOnServerNotIgnored(t *testing.T) { + mgmtServer := startManagementServer(t) + nodeID := uuid.New().String() + bs := generateBootstrapContents(t, mgmtServer.Address, false, nodeID) + xdsR := xdsResolverBuilder(t, bs) + resources, lis := resourceWithListenerForGRPCServer(t, nodeID) + updateCh := setupGRPCServerWithModeChangeChannelAndServe(t, bs, lis) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Wait for the listener to move to "serving" mode. + select { + case <-ctx.Done(): + t.Fatal("Test timed out waiting for a mode change update.") + case mode := <-updateCh: + if mode != connectivity.ServingModeServing { + t.Fatalf("Listener received new mode %v, want %v", mode, connectivity.ServingModeServing) + } + } + + // Create a ClientConn and make a successful RPCs. + cc, err := grpc.Dial(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(xdsR)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + if err := verifyRPCtoAllEndpoints(cc); err != nil { + t.Fatal(err) + } + + if err := mgmtServer.Update(ctx, e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*listenerpb.Listener{}, // empty listener resource + }); err != nil { + t.Fatal(err) + } + + select { + case <-ctx.Done(): + t.Fatalf("timed out waiting for a mode change update: %v", err) + case mode := <-updateCh: + if mode != connectivity.ServingModeNotServing { + t.Fatalf("listener received new mode %v, want %v", mode, connectivity.ServingModeNotServing) + } + } +} + +// This helper makes both UnaryCall and EmptyCall RPCs using the ClientConn that +// is passed to this function. This helper panics for any failed RPCs. +func verifyRPCtoAllEndpoints(cc grpc.ClientConnInterface) error { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + return fmt.Errorf("rpc EmptyCall() failed: %v", err) + } + if _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}); err != nil { + return fmt.Errorf("rpc UnaryCall() failed: %v", err) + } + return nil +} diff --git a/test/xds/xds_client_integration_test.go b/test/xds/xds_client_integration_test.go new file mode 100644 index 000000000000..ba96e6b25d9a --- /dev/null +++ b/test/xds/xds_client_integration_test.go @@ -0,0 +1,83 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds_test + +import ( + "context" + "fmt" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +const ( + defaultTestTimeout = 10 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond // For events expected to *not* happen. +) + +func (s) TestClientSideXDS(t *testing.T) { + managementServer, nodeID, _, resolver, cleanup1 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup1() + + server := stubserver.StartTestService(t, nil) + defer server.Stop() + + const serviceName = "my-service-client-side-xds" + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: "localhost", + Port: testutils.ParsePort(t, server.Address), + SecLevel: e2e.SecurityLevelNone, + }) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create a ClientConn and make a successful RPC. + cc, err := grpc.Dial(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("rpc EmptyCall() failed: %v", err) + } +} diff --git a/test/xds/xds_client_outlier_detection_test.go b/test/xds/xds_client_outlier_detection_test.go new file mode 100644 index 000000000000..d91b35a883aa --- /dev/null +++ b/test/xds/xds_client_outlier_detection_test.go @@ -0,0 +1,325 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds_test + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/resolver" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +// TestOutlierDetection_NoopConfig tests the scenario where the Outlier +// Detection feature is enabled on the gRPC client, but it receives no Outlier +// Detection configuration from the management server. This should result in a +// no-op Outlier Detection configuration being used to configure the Outlier +// Detection balancer. This test verifies that an RPC is able to proceed +// normally with this configuration. +func (s) TestOutlierDetection_NoopConfig(t *testing.T) { + managementServer, nodeID, _, resolver, cleanup1 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup1() + + server := &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return &testpb.Empty{}, nil }, + } + server.StartServer() + t.Logf("Started test service backend at %q", server.Address) + defer server.Stop() + + const serviceName = "my-service-client-side-xds" + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: "localhost", + Port: testutils.ParsePort(t, server.Address), + SecLevel: e2e.SecurityLevelNone, + }) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create a ClientConn and make a successful RPC. + cc, err := grpc.Dial(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("rpc EmptyCall() failed: %v", err) + } +} + +// clientResourcesMultipleBackendsAndOD returns xDS resources which correspond +// to multiple upstreams, corresponding different backends listening on +// different localhost:port combinations. The resources also configure an +// Outlier Detection Balancer configured through the passed in Outlier Detection +// proto. +func clientResourcesMultipleBackendsAndOD(params e2e.ResourceParams, ports []uint32, od *v3clusterpb.OutlierDetection) e2e.UpdateOptions { + routeConfigName := "route-" + params.DialTarget + clusterName := "cluster-" + params.DialTarget + endpointsName := "endpoints-" + params.DialTarget + return e2e.UpdateOptions{ + NodeID: params.NodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(params.DialTarget, routeConfigName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(routeConfigName, params.DialTarget, clusterName)}, + Clusters: []*v3clusterpb.Cluster{clusterWithOutlierDetection(clusterName, endpointsName, params.SecLevel, od)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(endpointsName, params.Host, ports)}, + } +} + +func clusterWithOutlierDetection(clusterName, edsServiceName string, secLevel e2e.SecurityLevel, od *v3clusterpb.OutlierDetection) *v3clusterpb.Cluster { + cluster := e2e.DefaultCluster(clusterName, edsServiceName, secLevel) + cluster.OutlierDetection = od + return cluster +} + +// checkRoundRobinRPCs verifies that EmptyCall RPCs on the given ClientConn, +// connected to a server exposing the test.grpc_testing.TestService, are +// roundrobined across the given backend addresses. +// +// Returns a non-nil error if context deadline expires before RPCs start to get +// roundrobined across the given backends. +func checkRoundRobinRPCs(ctx context.Context, client testgrpc.TestServiceClient, addrs []resolver.Address) error { + wantAddrCount := make(map[string]int) + for _, addr := range addrs { + wantAddrCount[addr.Addr]++ + } + for ; ctx.Err() == nil; <-time.After(time.Millisecond) { + // Perform 3 iterations. + var iterations [][]string + for i := 0; i < 3; i++ { + iteration := make([]string, len(addrs)) + for c := 0; c < len(addrs); c++ { + var peer peer.Peer + client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&peer)) + if peer.Addr != nil { + iteration[c] = peer.Addr.String() + } + } + iterations = append(iterations, iteration) + } + // Ensure the the first iteration contains all addresses in addrs. + gotAddrCount := make(map[string]int) + for _, addr := range iterations[0] { + gotAddrCount[addr]++ + } + if diff := cmp.Diff(gotAddrCount, wantAddrCount); diff != "" { + continue + } + // Ensure all three iterations contain the same addresses. + if !cmp.Equal(iterations[0], iterations[1]) || !cmp.Equal(iterations[0], iterations[2]) { + continue + } + return nil + } + return fmt.Errorf("timeout when waiting for roundrobin distribution of RPCs across addresses: %v", addrs) +} + +// TestOutlierDetectionWithOutlier tests the Outlier Detection Balancer e2e. It +// spins up three backends, one which consistently errors, and configures the +// ClientConn using xDS to connect to all three of those backends. The Outlier +// Detection Balancer should eject the connection to the backend which +// constantly errors, causing RPC's to not be routed to that upstream, and only +// be Round Robined across the two healthy upstreams. Other than the intervals +// the unhealthy upstream is ejected, RPC's should regularly round robin across +// all three upstreams. +func (s) TestOutlierDetectionWithOutlier(t *testing.T) { + managementServer, nodeID, _, r, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup() + + // Working backend 1. + backend1 := stubserver.StartTestService(t, nil) + port1 := testutils.ParsePort(t, backend1.Address) + defer backend1.Stop() + + // Working backend 2. + backend2 := stubserver.StartTestService(t, nil) + port2 := testutils.ParsePort(t, backend2.Address) + defer backend2.Stop() + + // Backend 3 that will always return an error and eventually ejected. + backend3 := stubserver.StartTestService(t, &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return nil, errors.New("some error") }, + }) + port3 := testutils.ParsePort(t, backend3.Address) + defer backend3.Stop() + + const serviceName = "my-service-client-side-xds" + resources := clientResourcesMultipleBackendsAndOD(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: "localhost", + SecLevel: e2e.SecurityLevelNone, + }, []uint32{port1, port2, port3}, &v3clusterpb.OutlierDetection{ + Interval: &durationpb.Duration{Nanos: 50000000}, // .5 seconds + BaseEjectionTime: &durationpb.Duration{Seconds: 30}, + MaxEjectionTime: &durationpb.Duration{Seconds: 300}, + MaxEjectionPercent: &wrapperspb.UInt32Value{Value: 1}, + FailurePercentageThreshold: &wrapperspb.UInt32Value{Value: 50}, + EnforcingFailurePercentage: &wrapperspb.UInt32Value{Value: 100}, + FailurePercentageRequestVolume: &wrapperspb.UInt32Value{Value: 8}, + FailurePercentageMinimumHosts: &wrapperspb.UInt32Value{Value: 3}, + }) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + cc, err := grpc.Dial(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + + fullAddresses := []resolver.Address{ + {Addr: backend1.Address}, + {Addr: backend2.Address}, + {Addr: backend3.Address}, + } + // At first, due to no statistics on each of the backends, the 3 + // upstreams should all be round robined across. + if err = checkRoundRobinRPCs(ctx, client, fullAddresses); err != nil { + t.Fatalf("error in expected round robin: %v", err) + } + + // The addresses which don't return errors. + okAddresses := []resolver.Address{ + {Addr: backend1.Address}, + {Addr: backend2.Address}, + } + // After calling the three upstreams, one of them constantly error + // and should eventually be ejected for a period of time. This + // period of time should cause the RPC's to be round robined only + // across the two that are healthy. + if err = checkRoundRobinRPCs(ctx, client, okAddresses); err != nil { + t.Fatalf("error in expected round robin: %v", err) + } +} + +// TestOutlierDetectionXDSDefaultOn tests that Outlier Detection is by default +// configured on in the xDS Flow. If the Outlier Detection proto message is +// present with SuccessRateEjection unset, then Outlier Detection should be +// turned on. The test setups and xDS system with xDS resources with Outlier +// Detection present in the CDS update, but with SuccessRateEjection unset, and +// asserts that Outlier Detection is turned on and ejects upstreams. +func (s) TestOutlierDetectionXDSDefaultOn(t *testing.T) { + managementServer, nodeID, _, r, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup() + + // Working backend 1. + backend1 := stubserver.StartTestService(t, nil) + port1 := testutils.ParsePort(t, backend1.Address) + defer backend1.Stop() + + // Working backend 2. + backend2 := stubserver.StartTestService(t, nil) + port2 := testutils.ParsePort(t, backend2.Address) + defer backend2.Stop() + + // Backend 3 that will always return an error and eventually ejected. + backend3 := stubserver.StartTestService(t, &stubserver.StubServer{ + EmptyCallF: func(context.Context, *testpb.Empty) (*testpb.Empty, error) { return nil, errors.New("some error") }, + }) + port3 := testutils.ParsePort(t, backend3.Address) + defer backend3.Stop() + + // Configure CDS resources with Outlier Detection set but + // EnforcingSuccessRate unset. This should cause Outlier Detection to be + // configured with SuccessRateEjection present in configuration, which will + // eventually be populated with its default values along with the knobs set + // as SuccessRate fields in the proto, and thus Outlier Detection should be + // on and actively eject upstreams. + const serviceName = "my-service-client-side-xds" + resources := clientResourcesMultipleBackendsAndOD(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: "localhost", + SecLevel: e2e.SecurityLevelNone, + }, []uint32{port1, port2, port3}, &v3clusterpb.OutlierDetection{ + // Need to set knobs to trigger ejection within the test time frame. + Interval: &durationpb.Duration{Nanos: 50000000}, + // EnforcingSuccessRateSet to nil, causes success rate algorithm to be + // turned on. + SuccessRateMinimumHosts: &wrapperspb.UInt32Value{Value: 1}, + SuccessRateRequestVolume: &wrapperspb.UInt32Value{Value: 8}, + SuccessRateStdevFactor: &wrapperspb.UInt32Value{Value: 1}, + }) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + cc, err := grpc.Dial(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + + fullAddresses := []resolver.Address{ + {Addr: backend1.Address}, + {Addr: backend2.Address}, + {Addr: backend3.Address}, + } + // At first, due to no statistics on each of the backends, the 3 + // upstreams should all be round robined across. + if err = checkRoundRobinRPCs(ctx, client, fullAddresses); err != nil { + t.Fatalf("error in expected round robin: %v", err) + } + + // The addresses which don't return errors. + okAddresses := []resolver.Address{ + {Addr: backend1.Address}, + {Addr: backend2.Address}, + } + // After calling the three upstreams, one of them constantly error + // and should eventually be ejected for a period of time. This + // period of time should cause the RPC's to be round robined only + // across the two that are healthy. + if err = checkRoundRobinRPCs(ctx, client, okAddresses); err != nil { + t.Fatalf("error in expected round robin: %v", err) + } +} diff --git a/test/xds/xds_client_retry_test.go b/test/xds/xds_client_retry_test.go new file mode 100644 index 000000000000..d7cb7b4bfb3c --- /dev/null +++ b/test/xds/xds_client_retry_test.go @@ -0,0 +1,181 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds_test + +import ( + "context" + "fmt" + "testing" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/wrapperspb" + + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +func (s) TestClientSideRetry(t *testing.T) { + ctr := 0 + errs := []codes.Code{codes.ResourceExhausted} + + managementServer, nodeID, _, resolver, cleanup1 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup1() + + server := stubserver.StartTestService(t, &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { + defer func() { ctr++ }() + if ctr < len(errs) { + return nil, status.Errorf(errs[ctr], "this should be retried") + } + return &testpb.Empty{}, nil + }, + }) + defer server.Stop() + + const serviceName = "my-service-client-side-xds" + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: "localhost", + Port: testutils.ParsePort(t, server.Address), + SecLevel: e2e.SecurityLevelNone, + }) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create a ClientConn and make a successful RPC. + cc, err := grpc.Dial(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + defer cancel() + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); status.Code(err) != codes.ResourceExhausted { + t.Fatalf("rpc EmptyCall() = _, %v; want _, ResourceExhausted", err) + } + + testCases := []struct { + name string + vhPolicy *v3routepb.RetryPolicy + routePolicy *v3routepb.RetryPolicy + errs []codes.Code // the errors returned by the server for each RPC + tryAgainErr codes.Code // the error that would be returned if we are still using the old retry policies. + errWant codes.Code + }{{ + name: "virtualHost only, fail", + vhPolicy: &v3routepb.RetryPolicy{ + RetryOn: "resource-exhausted,unavailable", + NumRetries: &wrapperspb.UInt32Value{Value: 1}, + }, + errs: []codes.Code{codes.ResourceExhausted, codes.Unavailable}, + routePolicy: nil, + tryAgainErr: codes.ResourceExhausted, + errWant: codes.Unavailable, + }, { + name: "virtualHost only", + vhPolicy: &v3routepb.RetryPolicy{ + RetryOn: "resource-exhausted, unavailable", + NumRetries: &wrapperspb.UInt32Value{Value: 2}, + }, + errs: []codes.Code{codes.ResourceExhausted, codes.Unavailable}, + routePolicy: nil, + tryAgainErr: codes.Unavailable, + errWant: codes.OK, + }, { + name: "virtualHost+route, fail", + vhPolicy: &v3routepb.RetryPolicy{ + RetryOn: "resource-exhausted,unavailable", + NumRetries: &wrapperspb.UInt32Value{Value: 2}, + }, + routePolicy: &v3routepb.RetryPolicy{ + RetryOn: "resource-exhausted", + NumRetries: &wrapperspb.UInt32Value{Value: 2}, + }, + errs: []codes.Code{codes.ResourceExhausted, codes.Unavailable}, + tryAgainErr: codes.OK, + errWant: codes.Unavailable, + }, { + name: "virtualHost+route", + vhPolicy: &v3routepb.RetryPolicy{ + RetryOn: "resource-exhausted", + NumRetries: &wrapperspb.UInt32Value{Value: 2}, + }, + routePolicy: &v3routepb.RetryPolicy{ + RetryOn: "unavailable", + NumRetries: &wrapperspb.UInt32Value{Value: 2}, + }, + errs: []codes.Code{codes.Unavailable}, + tryAgainErr: codes.Unavailable, + errWant: codes.OK, + }, { + name: "virtualHost+route, not enough attempts", + vhPolicy: &v3routepb.RetryPolicy{ + RetryOn: "unavailable", + NumRetries: &wrapperspb.UInt32Value{Value: 2}, + }, + routePolicy: &v3routepb.RetryPolicy{ + RetryOn: "unavailable", + NumRetries: &wrapperspb.UInt32Value{Value: 1}, + }, + errs: []codes.Code{codes.Unavailable, codes.Unavailable}, + tryAgainErr: codes.OK, + errWant: codes.Unavailable, + }} + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + errs = tc.errs + + // Confirm tryAgainErr is correct before updating resources. + ctr = 0 + _, err := client.EmptyCall(ctx, &testpb.Empty{}) + if code := status.Code(err); code != tc.tryAgainErr { + t.Fatalf("with old retry policy: EmptyCall() = _, %v; want _, %v", err, tc.tryAgainErr) + } + + resources.Routes[0].VirtualHosts[0].RetryPolicy = tc.vhPolicy + resources.Routes[0].VirtualHosts[0].Routes[0].GetRoute().RetryPolicy = tc.routePolicy + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + for { + ctr = 0 + _, err := client.EmptyCall(ctx, &testpb.Empty{}) + if code := status.Code(err); code == tc.tryAgainErr { + continue + } else if code != tc.errWant { + t.Fatalf("rpc EmptyCall() = _, %v; want _, %v", err, tc.errWant) + } + break + } + }) + } +} diff --git a/test/xds/xds_rls_clusterspecifier_plugin_test.go b/test/xds/xds_rls_clusterspecifier_plugin_test.go new file mode 100644 index 000000000000..bca198081a7c --- /dev/null +++ b/test/xds/xds_rls_clusterspecifier_plugin_test.go @@ -0,0 +1,178 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds_test + +import ( + "context" + "fmt" + "testing" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/rls" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/protobuf/types/known/durationpb" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + rlspb "google.golang.org/grpc/internal/proto/grpc_lookup_v1" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + + _ "google.golang.org/grpc/balancer/rls" // Register the RLS Load Balancing policy. +) + +// defaultClientResourcesWithRLSCSP returns a set of resources (LDS, RDS, CDS, EDS) for a +// client to connect to a server with a RLS Load Balancer as a child of Cluster Manager. +func defaultClientResourcesWithRLSCSP(lb e2e.LoadBalancingPolicy, params e2e.ResourceParams, rlsProto *rlspb.RouteLookupConfig) e2e.UpdateOptions { + routeConfigName := "route-" + params.DialTarget + clusterName := "cluster-" + params.DialTarget + endpointsName := "endpoints-" + params.DialTarget + return e2e.UpdateOptions{ + NodeID: params.NodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(params.DialTarget, routeConfigName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.RouteConfigResourceWithOptions(e2e.RouteConfigOptions{ + RouteConfigName: routeConfigName, + ListenerName: params.DialTarget, + ClusterSpecifierType: e2e.RouteConfigClusterSpecifierTypeClusterSpecifierPlugin, + ClusterSpecifierPluginName: "rls-csp", + ClusterSpecifierPluginConfig: testutils.MarshalAny(&rlspb.RouteLookupClusterSpecifier{ + RouteLookupConfig: rlsProto, + }), + })}, + Clusters: []*v3clusterpb.Cluster{e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: clusterName, + ServiceName: endpointsName, + Policy: lb, + SecurityLevel: params.SecLevel, + })}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(endpointsName, params.Host, []uint32{params.Port})}, + } +} + +// TestRLSinxDS tests an xDS configured system with an RLS Balancer present. +// +// This test sets up the RLS Balancer using the RLS Cluster Specifier Plugin, +// spins up a test service and has a fake RLS Server correctly respond with a +// target corresponding to this test service. This test asserts an RPC proceeds +// as normal with the RLS Balancer as part of system. +func (s) TestRLSinxDS(t *testing.T) { + tests := []struct { + name string + lbPolicy e2e.LoadBalancingPolicy + }{ + { + name: "roundrobin", + lbPolicy: e2e.LoadBalancingPolicyRoundRobin, + }, + { + name: "ringhash", + lbPolicy: e2e.LoadBalancingPolicyRingHash, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + testRLSinxDS(t, test.lbPolicy) + }) + } +} + +func testRLSinxDS(t *testing.T, lbPolicy e2e.LoadBalancingPolicy) { + oldRLS := envconfig.XDSRLS + envconfig.XDSRLS = true + internal.RegisterRLSClusterSpecifierPluginForTesting() + defer func() { + envconfig.XDSRLS = oldRLS + internal.UnregisterRLSClusterSpecifierPluginForTesting() + }() + + // Set up all components and configuration necessary - management server, + // xDS resolver, fake RLS Server, and xDS configuration which specifies an + // RLS Balancer that communicates to this set up fake RLS Server. + managementServer, nodeID, _, resolver, cleanup1 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup1() + + server := stubserver.StartTestService(t, nil) + defer server.Stop() + + lis := testutils.NewListenerWrapper(t, nil) + rlsServer, rlsRequestCh := rls.SetupFakeRLSServer(t, lis) + rlsProto := &rlspb.RouteLookupConfig{ + GrpcKeybuilders: []*rlspb.GrpcKeyBuilder{{Names: []*rlspb.GrpcKeyBuilder_Name{{Service: "grpc.testing.TestService"}}}}, + LookupService: rlsServer.Address, + LookupServiceTimeout: durationpb.New(defaultTestTimeout), + CacheSizeBytes: 1024, + } + + const serviceName = "my-service-client-side-xds" + resources := defaultClientResourcesWithRLSCSP(lbPolicy, e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: "localhost", + Port: testutils.ParsePort(t, server.Address), + SecLevel: e2e.SecurityLevelNone, + }, rlsProto) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Configure the fake RLS Server to set the RLS Balancers child CDS + // Cluster's name as the target for the RPC to use. + rlsServer.SetResponseCallback(func(_ context.Context, req *rlspb.RouteLookupRequest) *rls.RouteLookupResponse { + return &rls.RouteLookupResponse{Resp: &rlspb.RouteLookupResponse{Targets: []string{"cluster-" + serviceName}}} + }) + + // Create a ClientConn and make a successful RPC. + cc, err := grpc.Dial(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + // Successfully sending the RPC will require the RLS Load Balancer to + // communicate with the fake RLS Server for information about the target. + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("rpc EmptyCall() failed: %v", err) + } + + // These RLS Verifications makes sure the RLS Load Balancer is actually part + // of the xDS Configured system that correctly sends out RPC. + + // Verify connection is established to RLS Server. + if _, err = lis.NewConnCh.Receive(ctx); err != nil { + t.Fatal("Timeout when waiting for RLS LB policy to create control channel") + } + + // Verify an rls request is sent out to fake RLS Server. + select { + case <-ctx.Done(): + t.Fatalf("Timeout when waiting for an RLS request to be sent out") + case <-rlsRequestCh: + } +} diff --git a/test/xds/xds_security_config_nack_test.go b/test/xds/xds_security_config_nack_test.go new file mode 100644 index 000000000000..1dc3250935bf --- /dev/null +++ b/test/xds/xds_security_config_nack_test.go @@ -0,0 +1,367 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds_test + +import ( + "context" + "fmt" + "testing" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + xdscreds "google.golang.org/grpc/credentials/xds" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +func (s) TestUnmarshalListener_WithUpdateValidatorFunc(t *testing.T) { + const ( + serviceName = "my-service-client-side-xds" + missingIdentityProviderInstance = "missing-identity-provider-instance" + missingRootProviderInstance = "missing-root-provider-instance" + ) + + tests := []struct { + name string + securityConfig *v3corepb.TransportSocket + wantErr bool + }{ + { + name: "both identity and root providers are not present in bootstrap", + securityConfig: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: missingIdentityProviderInstance, + }, + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{ + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: missingRootProviderInstance, + }, + }, + }, + }, + }), + }, + }, + wantErr: true, + }, + { + name: "only identity provider is not present in bootstrap", + securityConfig: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: missingIdentityProviderInstance, + }, + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{ + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: e2e.ServerSideCertProviderInstance, + }, + }, + }, + }, + }), + }, + }, + wantErr: true, + }, + { + name: "only root provider is not present in bootstrap", + securityConfig: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: e2e.ServerSideCertProviderInstance, + }, + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{ + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: missingRootProviderInstance, + }, + }, + }, + }, + }), + }, + }, + wantErr: true, + }, + { + name: "both identity and root providers are present in bootstrap", + securityConfig: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: e2e.ServerSideCertProviderInstance, + }, + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{ + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: e2e.ServerSideCertProviderInstance, + }, + }, + }, + }, + }), + }, + }, + wantErr: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + managementServer, nodeID, bootstrapContents, resolver, cleanup1 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup1() + + lis, cleanup2 := setupGRPCServer(t, bootstrapContents) + defer cleanup2() + + // Grab the host and port of the server and create client side xDS + // resources corresponding to it. + host, port, err := hostPortFromListener(lis) + if err != nil { + t.Fatalf("failed to retrieve host and port of server: %v", err) + } + + // Create xDS resources to be consumed on the client side. This + // includes the listener, route configuration, cluster (with + // security configuration) and endpoint resources. + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: host, + Port: port, + SecLevel: e2e.SecurityLevelMTLS, + }) + + // Create an inbound xDS listener resource for the server side. + inboundLis := e2e.DefaultServerListener(host, port, e2e.SecurityLevelMTLS) + for _, fc := range inboundLis.GetFilterChains() { + fc.TransportSocket = test.securityConfig + } + resources.Listeners = append(resources.Listeners, inboundLis) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create client-side xDS credentials with an insecure fallback. + creds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()}) + if err != nil { + t.Fatal(err) + } + + // Create a ClientConn with the xds scheme and make an RPC. + cc, err := grpc.DialContext(ctx, fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(creds), grpc.WithResolvers(resolver)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + // Make a context with a shorter timeout from the top level test + // context for cases where we expect failures. + timeout := defaultTestTimeout + if test.wantErr { + timeout = defaultTestShortTimeout + } + ctx, cancel = context.WithTimeout(ctx, timeout) + defer cancel() + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); (err != nil) != test.wantErr { + t.Fatalf("EmptyCall() returned err: %v, wantErr %v", err, test.wantErr) + } + }) + } +} + +func (s) TestUnmarshalCluster_WithUpdateValidatorFunc(t *testing.T) { + const ( + serviceName = "my-service-client-side-xds" + missingIdentityProviderInstance = "missing-identity-provider-instance" + missingRootProviderInstance = "missing-root-provider-instance" + ) + + tests := []struct { + name string + securityConfig *v3corepb.TransportSocket + wantErr bool + }{ + { + name: "both identity and root providers are not present in bootstrap", + securityConfig: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: missingIdentityProviderInstance, + }, + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{ + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: missingRootProviderInstance, + }, + }, + }, + }, + }), + }, + }, + wantErr: true, + }, + { + name: "only identity provider is not present in bootstrap", + securityConfig: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: missingIdentityProviderInstance, + }, + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{ + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: e2e.ClientSideCertProviderInstance, + }, + }, + }, + }, + }), + }, + }, + wantErr: true, + }, + { + name: "only root provider is not present in bootstrap", + securityConfig: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: e2e.ClientSideCertProviderInstance, + }, + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{ + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: missingRootProviderInstance, + }, + }, + }, + }, + }), + }, + }, + wantErr: true, + }, + { + name: "both identity and root providers are present in bootstrap", + securityConfig: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: e2e.ClientSideCertProviderInstance, + }, + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{ + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: e2e.ClientSideCertProviderInstance, + }, + }, + }, + }, + }), + }, + }, + wantErr: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + // SetupManagementServer() sets up a bootstrap file with certificate + // provider instance names: `e2e.ServerSideCertProviderInstance` and + // `e2e.ClientSideCertProviderInstance`. + managementServer, nodeID, _, resolver, cleanup1 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup1() + + server := stubserver.StartTestService(t, nil) + defer server.Stop() + + // This creates a `Cluster` resource with a security config which + // refers to `e2e.ClientSideCertProviderInstance` for both root and + // identity certs. + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: "localhost", + Port: testutils.ParsePort(t, server.Address), + SecLevel: e2e.SecurityLevelMTLS, + }) + resources.Clusters[0].TransportSocket = test.securityConfig + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + cc, err := grpc.Dial(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + // Make a context with a shorter timeout from the top level test + // context for cases where we expect failures. + timeout := defaultTestTimeout + if test.wantErr { + timeout = defaultTestShortTimeout + } + ctx2, cancel2 := context.WithTimeout(ctx, timeout) + defer cancel2() + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx2, &testpb.Empty{}, grpc.WaitForReady(true)); (err != nil) != test.wantErr { + t.Fatalf("EmptyCall() returned err: %v, wantErr %v", err, test.wantErr) + } + }) + } +} diff --git a/test/xds/xds_server_integration_test.go b/test/xds/xds_server_integration_test.go new file mode 100644 index 000000000000..1c9d05137cfa --- /dev/null +++ b/test/xds/xds_server_integration_test.go @@ -0,0 +1,373 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds_test + +import ( + "context" + "fmt" + "net" + "strconv" + "testing" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + xdscreds "google.golang.org/grpc/credentials/xds" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/status" + "google.golang.org/grpc/xds" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +type testService struct { + testgrpc.TestServiceServer +} + +func (*testService) EmptyCall(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil +} + +func (*testService) UnaryCall(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) { + return &testpb.SimpleResponse{}, nil +} + +// setupGRPCServer performs the following: +// - spin up an xDS-enabled gRPC server, configure it with xdsCredentials and +// register the test service on it +// - create a local TCP listener and start serving on it +// +// Returns the following: +// - local listener on which the xDS-enabled gRPC server is serving on +// - cleanup function to be invoked by the tests when done +func setupGRPCServer(t *testing.T, bootstrapContents []byte) (net.Listener, func()) { + t.Helper() + + // Configure xDS credentials to be used on the server-side. + creds, err := xdscreds.NewServerCredentials(xdscreds.ServerOptions{ + FallbackCreds: insecure.NewCredentials(), + }) + if err != nil { + t.Fatal(err) + } + + // Create a server option to get notified about serving mode changes. We don't + // do anything other than throwing a log entry here. But this is required, + // since the server code emits a log entry at the default level (which is + // ERROR) if no callback is registered for serving mode changes. Our + // testLogger fails the test if there is any log entry at ERROR level. It does + // provide an ExpectError() method, but that takes a string and it would be + // painful to construct the exact error message expected here. Instead this + // works just fine. + modeChangeOpt := xds.ServingModeCallback(func(addr net.Addr, args xds.ServingModeChangeArgs) { + t.Logf("Serving mode for listener %q changed to %q, err: %v", addr.String(), args.Mode, args.Err) + }) + + // Initialize an xDS-enabled gRPC server and register the stubServer on it. + server, err := xds.NewGRPCServer(grpc.Creds(creds), modeChangeOpt, xds.BootstrapContentsForTesting(bootstrapContents)) + if err != nil { + t.Fatalf("Failed to create an xDS enabled gRPC server: %v", err) + } + testgrpc.RegisterTestServiceServer(server, &testService{}) + + // Create a local listener and pass it to Serve(). + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("testutils.LocalTCPListener() failed: %v", err) + } + + go func() { + if err := server.Serve(lis); err != nil { + t.Errorf("Serve() failed: %v", err) + } + }() + + return lis, func() { + server.Stop() + } +} + +func hostPortFromListener(lis net.Listener) (string, uint32, error) { + host, p, err := net.SplitHostPort(lis.Addr().String()) + if err != nil { + return "", 0, fmt.Errorf("net.SplitHostPort(%s) failed: %v", lis.Addr().String(), err) + } + port, err := strconv.ParseInt(p, 10, 32) + if err != nil { + return "", 0, fmt.Errorf("strconv.ParseInt(%s, 10, 32) failed: %v", p, err) + } + return host, uint32(port), nil +} + +// TestServerSideXDS_Fallback is an e2e test which verifies xDS credentials +// fallback functionality. +// +// The following sequence of events happen as part of this test: +// - An xDS-enabled gRPC server is created and xDS credentials are configured. +// - xDS is enabled on the client by the use of the xds:/// scheme, and xDS +// credentials are configured. +// - Control plane is configured to not send any security configuration to both +// the client and the server. This results in both of them using the +// configured fallback credentials (which is insecure creds in this case). +func (s) TestServerSideXDS_Fallback(t *testing.T) { + managementServer, nodeID, bootstrapContents, resolver, cleanup1 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup1() + + lis, cleanup2 := setupGRPCServer(t, bootstrapContents) + defer cleanup2() + + // Grab the host and port of the server and create client side xDS resources + // corresponding to it. This contains default resources with no security + // configuration in the Cluster resources. + host, port, err := hostPortFromListener(lis) + if err != nil { + t.Fatalf("failed to retrieve host and port of server: %v", err) + } + const serviceName = "my-service-fallback" + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: host, + Port: port, + SecLevel: e2e.SecurityLevelNone, + }) + + // Create an inbound xDS listener resource for the server side that does not + // contain any security configuration. This should force the server-side + // xdsCredentials to use fallback. + inboundLis := e2e.DefaultServerListener(host, port, e2e.SecurityLevelNone) + resources.Listeners = append(resources.Listeners, inboundLis) + + // Setup the management server with client and server-side resources. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create client-side xDS credentials with an insecure fallback. + creds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{ + FallbackCreds: insecure.NewCredentials(), + }) + if err != nil { + t.Fatal(err) + } + + // Create a ClientConn with the xds scheme and make a successful RPC. + cc, err := grpc.DialContext(ctx, fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(creds), grpc.WithResolvers(resolver)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Errorf("rpc EmptyCall() failed: %v", err) + } +} + +// TestServerSideXDS_FileWatcherCerts is an e2e test which verifies xDS +// credentials with file watcher certificate provider. +// +// The following sequence of events happen as part of this test: +// - An xDS-enabled gRPC server is created and xDS credentials are configured. +// - xDS is enabled on the client by the use of the xds:/// scheme, and xDS +// credentials are configured. +// - Control plane is configured to send security configuration to both the +// client and the server, pointing to the file watcher certificate provider. +// We verify both TLS and mTLS scenarios. +func (s) TestServerSideXDS_FileWatcherCerts(t *testing.T) { + tests := []struct { + name string + secLevel e2e.SecurityLevel + }{ + { + name: "tls", + secLevel: e2e.SecurityLevelTLS, + }, + { + name: "mtls", + secLevel: e2e.SecurityLevelMTLS, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + managementServer, nodeID, bootstrapContents, resolver, cleanup1 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup1() + + lis, cleanup2 := setupGRPCServer(t, bootstrapContents) + defer cleanup2() + + // Grab the host and port of the server and create client side xDS + // resources corresponding to it. + host, port, err := hostPortFromListener(lis) + if err != nil { + t.Fatalf("failed to retrieve host and port of server: %v", err) + } + + // Create xDS resources to be consumed on the client side. This + // includes the listener, route configuration, cluster (with + // security configuration) and endpoint resources. + serviceName := "my-service-file-watcher-certs-" + test.name + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: host, + Port: port, + SecLevel: test.secLevel, + }) + + // Create an inbound xDS listener resource for the server side that + // contains security configuration pointing to the file watcher + // plugin. + inboundLis := e2e.DefaultServerListener(host, port, test.secLevel) + resources.Listeners = append(resources.Listeners, inboundLis) + + // Setup the management server with client and server resources. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create client-side xDS credentials with an insecure fallback. + creds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{ + FallbackCreds: insecure.NewCredentials(), + }) + if err != nil { + t.Fatal(err) + } + + // Create a ClientConn with the xds scheme and make an RPC. + cc, err := grpc.DialContext(ctx, fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(creds), grpc.WithResolvers(resolver)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("rpc EmptyCall() failed: %v", err) + } + }) + } +} + +// TestServerSideXDS_SecurityConfigChange is an e2e test where xDS is enabled on +// the server-side and xdsCredentials are configured for security. The control +// plane initially does not any security configuration. This forces the +// xdsCredentials to use fallback creds, which is this case is insecure creds. +// We verify that a client connecting with TLS creds is not able to successfully +// make an RPC. The control plane then sends a listener resource with security +// configuration pointing to the use of the file_watcher plugin and we verify +// that the same client is now able to successfully make an RPC. +func (s) TestServerSideXDS_SecurityConfigChange(t *testing.T) { + managementServer, nodeID, bootstrapContents, resolver, cleanup1 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup1() + + lis, cleanup2 := setupGRPCServer(t, bootstrapContents) + defer cleanup2() + + // Grab the host and port of the server and create client side xDS resources + // corresponding to it. This contains default resources with no security + // configuration in the Cluster resource. This should force the xDS + // credentials on the client to use its fallback. + host, port, err := hostPortFromListener(lis) + if err != nil { + t.Fatalf("failed to retrieve host and port of server: %v", err) + } + const serviceName = "my-service-security-config-change" + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: host, + Port: port, + SecLevel: e2e.SecurityLevelNone, + }) + + // Create an inbound xDS listener resource for the server side that does not + // contain any security configuration. This should force the xDS credentials + // on server to use its fallback. + inboundLis := e2e.DefaultServerListener(host, port, e2e.SecurityLevelNone) + resources.Listeners = append(resources.Listeners, inboundLis) + + // Setup the management server with client and server-side resources. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create client-side xDS credentials with an insecure fallback. + xdsCreds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{ + FallbackCreds: insecure.NewCredentials(), + }) + if err != nil { + t.Fatal(err) + } + + // Create a ClientConn with the xds scheme and make a successful RPC. + xdsCC, err := grpc.DialContext(ctx, fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(xdsCreds), grpc.WithResolvers(resolver)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer xdsCC.Close() + + client := testgrpc.NewTestServiceClient(xdsCC) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("rpc EmptyCall() failed: %v", err) + } + + // Create a ClientConn with TLS creds. This should fail since the server is + // using fallback credentials which in this case in insecure creds. + tlsCreds := e2e.CreateClientTLSCredentials(t) + tlsCC, err := grpc.DialContext(ctx, lis.Addr().String(), grpc.WithTransportCredentials(tlsCreds)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer tlsCC.Close() + + // We don't set 'waitForReady` here since we want this call to failfast. + client = testgrpc.NewTestServiceClient(tlsCC) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.Unavailable { + t.Fatal("rpc EmptyCall() succeeded when expected to fail") + } + + // Switch server and client side resources with ones that contain required + // security configuration for mTLS with a file watcher certificate provider. + resources = e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: host, + Port: port, + SecLevel: e2e.SecurityLevelMTLS, + }) + inboundLis = e2e.DefaultServerListener(host, port, e2e.SecurityLevelMTLS) + resources.Listeners = append(resources.Listeners, inboundLis) + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Make another RPC with `waitForReady` set and expect this to succeed. + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("rpc EmptyCall() failed: %v", err) + } +} diff --git a/test/xds/xds_server_rbac_test.go b/test/xds/xds_server_rbac_test.go new file mode 100644 index 000000000000..fa684104281b --- /dev/null +++ b/test/xds/xds_server_rbac_test.go @@ -0,0 +1,993 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds_test + +import ( + "context" + "encoding/json" + "fmt" + "net" + "strconv" + "strings" + "testing" + + v3xdsxdstypepb "github.com/cncf/xds/go/xds/type/v3" + v3routerpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3" + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc" + "google.golang.org/grpc/authz/audit" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/structpb" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3rbacpb "github.com/envoyproxy/go-control-plane/envoy/config/rbac/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + rpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/rbac/v3" + v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3" + wrapperspb "github.com/golang/protobuf/ptypes/wrappers" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +// TestServerSideXDS_RouteConfiguration is an e2e test which verifies routing +// functionality. The xDS enabled server will be set up with route configuration +// where the route configuration has routes with the correct routing actions +// (NonForwardingAction), and the RPC's matching those routes should proceed as +// normal. +func (s) TestServerSideXDS_RouteConfiguration(t *testing.T) { + oldRBAC := envconfig.XDSRBAC + envconfig.XDSRBAC = true + defer func() { + envconfig.XDSRBAC = oldRBAC + }() + managementServer, nodeID, bootstrapContents, resolver, cleanup1 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup1() + + lis, cleanup2 := setupGRPCServer(t, bootstrapContents) + defer cleanup2() + + host, port, err := hostPortFromListener(lis) + if err != nil { + t.Fatalf("failed to retrieve host and port of server: %v", err) + } + const serviceName = "my-service-fallback" + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: host, + Port: port, + SecLevel: e2e.SecurityLevelNone, + }) + + // Create an inbound xDS listener resource with route configuration which + // selectively will allow RPC's through or not. This will test routing in + // xds(Unary|Stream)Interceptors. + vhs := []*v3routepb.VirtualHost{ + // Virtual host that will never be matched to test Virtual Host selection. + { + Domains: []string{"this will not match*"}, + Routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}, + }, + Action: &v3routepb.Route_NonForwardingAction{}, + }, + }, + }, + // This Virtual Host will actually get matched to. + { + Domains: []string{"*"}, + Routes: []*v3routepb.Route{ + // A routing rule that can be selectively triggered based on properties about incoming RPC. + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"}, + // "Fully-qualified RPC method name with leading slash. Same as :path header". + }, + // Correct Action, so RPC's that match this route should proceed to interceptor processing. + Action: &v3routepb.Route_NonForwardingAction{}, + }, + // This routing rule is matched the same way as the one above, + // except has an incorrect action for the server side. However, + // since routing chooses the first route which matches an + // incoming RPC, this should never get invoked (iteration + // through this route slice is deterministic). + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"}, + // "Fully-qualified RPC method name with leading slash. Same as :path header". + }, + // Incorrect Action, so RPC's that match this route should get denied. + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: ""}}, + }, + }, + // Another routing rule that can be selectively triggered based on incoming RPC. + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/UnaryCall"}, + }, + // Wrong action (!Non_Forwarding_Action) so RPC's that match this route should get denied. + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: ""}}, + }, + }, + // Another routing rule that can be selectively triggered based on incoming RPC. + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/StreamingInputCall"}, + }, + // Wrong action (!Non_Forwarding_Action) so RPC's that match this route should get denied. + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: ""}}, + }, + }, + // Not matching route, this is be able to get invoked logically (i.e. doesn't have to match the Route configurations above). + }}, + } + inboundLis := &v3listenerpb.Listener{ + Name: fmt.Sprintf(e2e.ServerListenerResourceNameTemplate, net.JoinHostPort(host, strconv.Itoa(int(port)))), + Address: &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: host, + PortSpecifier: &v3corepb.SocketAddress_PortValue{ + PortValue: port, + }, + }, + }, + }, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "v4-wildcard", + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{ + { + AddressPrefix: "0.0.0.0", + PrefixLen: &wrapperspb.UInt32Value{ + Value: uint32(0), + }, + }, + }, + SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK, + SourcePrefixRanges: []*v3corepb.CidrRange{ + { + AddressPrefix: "0.0.0.0", + PrefixLen: &wrapperspb.UInt32Value{ + Value: uint32(0), + }, + }, + }, + }, + Filters: []*v3listenerpb.Filter{ + { + Name: "filter-1", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + HttpFilters: []*v3httppb.HttpFilter{e2e.HTTPFilter("router", &v3routerpb.Router{})}, + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: &v3routepb.RouteConfiguration{ + Name: "routeName", + VirtualHosts: vhs, + }, + }, + }), + }, + }, + }, + }, + { + Name: "v6-wildcard", + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{ + { + AddressPrefix: "::", + PrefixLen: &wrapperspb.UInt32Value{ + Value: uint32(0), + }, + }, + }, + SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK, + SourcePrefixRanges: []*v3corepb.CidrRange{ + { + AddressPrefix: "::", + PrefixLen: &wrapperspb.UInt32Value{ + Value: uint32(0), + }, + }, + }, + }, + Filters: []*v3listenerpb.Filter{ + { + Name: "filter-1", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + HttpFilters: []*v3httppb.HttpFilter{e2e.HTTPFilter("router", &v3routerpb.Router{})}, + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: &v3routepb.RouteConfiguration{ + Name: "routeName", + VirtualHosts: vhs, + }, + }, + }), + }, + }, + }, + }, + }, + } + resources.Listeners = append(resources.Listeners, inboundLis) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + // Setup the management server with client and server-side resources. + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + cc, err := grpc.DialContext(ctx, fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + + // This Empty Call should match to a route with a correct action + // (NonForwardingAction). Thus, this RPC should proceed as normal. There is + // a routing rule that this RPC would match to that has an incorrect action, + // but the server should only use the first route matched to with the + // correct action. + if _, err = client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("rpc EmptyCall() failed: %v", err) + } + + // This Unary Call should match to a route with an incorrect action. Thus, + // this RPC should not go through as per A36, and this call should receive + // an error with codes.Unavailable. + if _, err = client.UnaryCall(ctx, &testpb.SimpleRequest{}); status.Code(err) != codes.Unavailable { + t.Fatalf("client.UnaryCall() = _, %v, want _, error code %s", err, codes.Unavailable) + } + + // This Streaming Call should match to a route with an incorrect action. + // Thus, this RPC should not go through as per A36, and this call should + // receive an error with codes.Unavailable. + stream, err := client.StreamingInputCall(ctx) + if err != nil { + t.Fatalf("StreamingInputCall(_) = _, %v, want ", err) + } + if _, err = stream.CloseAndRecv(); status.Code(err) != codes.Unavailable || !strings.Contains(err.Error(), "the incoming RPC matched to a route that was not of action type non forwarding") { + t.Fatalf("streaming RPC should have been denied") + } + + // This Full Duplex should not match to a route, and thus should return an + // error and not proceed. + dStream, err := client.FullDuplexCall(ctx) + if err != nil { + t.Fatalf("FullDuplexCall(_) = _, %v, want ", err) + } + if _, err = dStream.Recv(); status.Code(err) != codes.Unavailable || !strings.Contains(err.Error(), "the incoming RPC did not match a configured Route") { + t.Fatalf("streaming RPC should have been denied") + } +} + +// serverListenerWithRBACHTTPFilters returns an xds Listener resource with HTTP Filters defined in the HCM, and a route +// configuration that always matches to a route and a VH. +func serverListenerWithRBACHTTPFilters(host string, port uint32, rbacCfg *rpb.RBAC) *v3listenerpb.Listener { + // Rather than declare typed config inline, take a HCM proto and append the + // RBAC Filters to it. + hcm := &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: &v3routepb.RouteConfiguration{ + Name: "routeName", + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{"*"}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}, + }, + Action: &v3routepb.Route_NonForwardingAction{}, + }}, + // This tests override parsing + building when RBAC Filter + // passed both normal and override config. + TypedPerFilterConfig: map[string]*anypb.Any{ + "rbac": testutils.MarshalAny(&rpb.RBACPerRoute{Rbac: rbacCfg}), + }, + }}}, + }, + } + hcm.HttpFilters = nil + hcm.HttpFilters = append(hcm.HttpFilters, e2e.HTTPFilter("rbac", rbacCfg)) + hcm.HttpFilters = append(hcm.HttpFilters, e2e.RouterHTTPFilter) + + return &v3listenerpb.Listener{ + Name: fmt.Sprintf(e2e.ServerListenerResourceNameTemplate, net.JoinHostPort(host, strconv.Itoa(int(port)))), + Address: &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: host, + PortSpecifier: &v3corepb.SocketAddress_PortValue{ + PortValue: port, + }, + }, + }, + }, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "v4-wildcard", + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{ + { + AddressPrefix: "0.0.0.0", + PrefixLen: &wrapperspb.UInt32Value{ + Value: uint32(0), + }, + }, + }, + SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK, + SourcePrefixRanges: []*v3corepb.CidrRange{ + { + AddressPrefix: "0.0.0.0", + PrefixLen: &wrapperspb.UInt32Value{ + Value: uint32(0), + }, + }, + }, + }, + Filters: []*v3listenerpb.Filter{ + { + Name: "filter-1", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(hcm), + }, + }, + }, + }, + { + Name: "v6-wildcard", + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{ + { + AddressPrefix: "::", + PrefixLen: &wrapperspb.UInt32Value{ + Value: uint32(0), + }, + }, + }, + SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK, + SourcePrefixRanges: []*v3corepb.CidrRange{ + { + AddressPrefix: "::", + PrefixLen: &wrapperspb.UInt32Value{ + Value: uint32(0), + }, + }, + }, + }, + Filters: []*v3listenerpb.Filter{ + { + Name: "filter-1", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(hcm), + }, + }, + }, + }, + }, + } +} + +// TestRBACHTTPFilter tests the xds configured RBAC HTTP Filter. It sets up the +// full end to end flow, and makes sure certain RPC's are successful and proceed +// as normal and certain RPC's are denied by the RBAC HTTP Filter which gets +// called by hooked xds interceptors. +func (s) TestRBACHTTPFilter(t *testing.T) { + oldRBAC := envconfig.XDSRBAC + envconfig.XDSRBAC = true + defer func() { + envconfig.XDSRBAC = oldRBAC + }() + internal.RegisterRBACHTTPFilterForTesting() + defer internal.UnregisterRBACHTTPFilterForTesting() + tests := []struct { + name string + rbacCfg *rpb.RBAC + wantStatusEmptyCall codes.Code + wantStatusUnaryCall codes.Code + wantAuthzOutcomes map[bool]int + eventContent *audit.Event + }{ + // This test tests an RBAC HTTP Filter which is configured to allow any RPC. + // Any RPC passing through this RBAC HTTP Filter should proceed as normal. + { + name: "allow-anything", + rbacCfg: &rpb.RBAC{ + Rules: &v3rbacpb.RBAC{ + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "anyone": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + AuditLoggingOptions: &v3rbacpb.RBAC_AuditLoggingOptions{ + AuditCondition: v3rbacpb.RBAC_AuditLoggingOptions_ON_ALLOW, + LoggerConfigs: []*v3rbacpb.RBAC_AuditLoggingOptions_AuditLoggerConfig{ + { + AuditLogger: &v3corepb.TypedExtensionConfig{ + Name: "stat_logger", + TypedConfig: createXDSTypedStruct(t, map[string]any{}, "stat_logger"), + }, + IsOptional: false, + }, + }, + }, + }, + }, + wantStatusEmptyCall: codes.OK, + wantStatusUnaryCall: codes.OK, + wantAuthzOutcomes: map[bool]int{true: 2, false: 0}, + // TODO(gtcooke94) add policy name (RBAC filter name) once + // https://github.com/grpc/grpc-go/pull/6327 is merged. + eventContent: &audit.Event{ + FullMethodName: "/grpc.testing.TestService/UnaryCall", + MatchedRule: "anyone", + Authorized: true, + }, + }, + // This test tests an RBAC HTTP Filter which is configured to allow only + // RPC's with certain paths ("UnaryCall"). Only unary calls passing + // through this RBAC HTTP Filter should proceed as normal, and any + // others should be denied. + { + name: "allow-certain-path", + rbacCfg: &rpb.RBAC{ + Rules: &v3rbacpb.RBAC{ + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "certain-path": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_UrlPath{UrlPath: &v3matcherpb.PathMatcher{Rule: &v3matcherpb.PathMatcher_Path{Path: &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "/grpc.testing.TestService/UnaryCall"}}}}}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + wantStatusEmptyCall: codes.PermissionDenied, + wantStatusUnaryCall: codes.OK, + }, + // This test that a RBAC Config with nil rules means that every RPC is + // allowed. This maps to the line "If absent, no enforcing RBAC policy + // will be applied" from the RBAC Proto documentation for the Rules + // field. + { + name: "absent-rules", + rbacCfg: &rpb.RBAC{ + Rules: nil, + }, + wantStatusEmptyCall: codes.OK, + wantStatusUnaryCall: codes.OK, + }, + // The two tests below test that configuring the xDS RBAC HTTP Filter + // with :authority and host header matchers end up being logically + // equivalent. This represents functionality from this line in A41 - + // "As documented for HeaderMatcher, Envoy aliases :authority and Host + // in its header map implementation, so they should be treated + // equivalent for the RBAC matchers; there must be no behavior change + // depending on which of the two header names is used in the RBAC + // policy." + + // This test tests an xDS RBAC Filter with an :authority header matcher. + { + name: "match-on-authority", + rbacCfg: &rpb.RBAC{ + Rules: &v3rbacpb.RBAC{ + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "match-on-authority": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Header{Header: &v3routepb.HeaderMatcher{Name: ":authority", HeaderMatchSpecifier: &v3routepb.HeaderMatcher_PrefixMatch{PrefixMatch: "my-service-fallback"}}}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + wantStatusEmptyCall: codes.OK, + wantStatusUnaryCall: codes.OK, + }, + // This test tests that configuring an xDS RBAC Filter with a host + // header matcher has the same behavior as if it was configured with + // :authority. Since host and authority are aliased, this should still + // continue to match on incoming RPC's :authority, just as the test + // above. + { + name: "match-on-host", + rbacCfg: &rpb.RBAC{ + Rules: &v3rbacpb.RBAC{ + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "match-on-authority": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Header{Header: &v3routepb.HeaderMatcher{Name: "host", HeaderMatchSpecifier: &v3routepb.HeaderMatcher_PrefixMatch{PrefixMatch: "my-service-fallback"}}}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + wantStatusEmptyCall: codes.OK, + wantStatusUnaryCall: codes.OK, + }, + // This test tests that the RBAC HTTP Filter hard codes the :method + // header to POST. Since the RBAC Configuration says to deny every RPC + // with a method :POST, every RPC tried should be denied. + { + name: "deny-post", + rbacCfg: &rpb.RBAC{ + Rules: &v3rbacpb.RBAC{ + Action: v3rbacpb.RBAC_DENY, + Policies: map[string]*v3rbacpb.Policy{ + "post-method": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Header{Header: &v3routepb.HeaderMatcher{Name: ":method", HeaderMatchSpecifier: &v3routepb.HeaderMatcher_ExactMatch{ExactMatch: "POST"}}}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + wantStatusEmptyCall: codes.PermissionDenied, + wantStatusUnaryCall: codes.PermissionDenied, + }, + // This test tests that RBAC ignores the TE: trailers header (which is + // hardcoded in http2_client.go for every RPC). Since the RBAC + // Configuration says to only ALLOW RPC's with a TE: Trailers, every RPC + // tried should be denied. + { + name: "allow-only-te", + rbacCfg: &rpb.RBAC{ + Rules: &v3rbacpb.RBAC{ + Action: v3rbacpb.RBAC_ALLOW, + Policies: map[string]*v3rbacpb.Policy{ + "post-method": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Header{Header: &v3routepb.HeaderMatcher{Name: "TE", HeaderMatchSpecifier: &v3routepb.HeaderMatcher_ExactMatch{ExactMatch: "trailers"}}}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + wantStatusEmptyCall: codes.PermissionDenied, + wantStatusUnaryCall: codes.PermissionDenied, + }, + // This test tests that an RBAC Config with Action.LOG configured allows + // every RPC through. This maps to the line "At this time, if the + // RBAC.action is Action.LOG then the policy will be completely ignored, + // as if RBAC was not configurated." from A41 + { + name: "action-log", + rbacCfg: &rpb.RBAC{ + Rules: &v3rbacpb.RBAC{ + Action: v3rbacpb.RBAC_LOG, + Policies: map[string]*v3rbacpb.Policy{ + "anyone": { + Permissions: []*v3rbacpb.Permission{ + {Rule: &v3rbacpb.Permission_Any{Any: true}}, + }, + Principals: []*v3rbacpb.Principal{ + {Identifier: &v3rbacpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }, + wantStatusEmptyCall: codes.OK, + wantStatusUnaryCall: codes.OK, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + func() { + lb := &loggerBuilder{ + authzDecisionStat: map[bool]int{true: 0, false: 0}, + lastEvent: &audit.Event{}, + } + audit.RegisterLoggerBuilder(lb) + + managementServer, nodeID, bootstrapContents, resolver, cleanup1 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup1() + + lis, cleanup2 := setupGRPCServer(t, bootstrapContents) + defer cleanup2() + + host, port, err := hostPortFromListener(lis) + if err != nil { + t.Fatalf("failed to retrieve host and port of server: %v", err) + } + const serviceName = "my-service-fallback" + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: host, + Port: port, + SecLevel: e2e.SecurityLevelNone, + }) + inboundLis := serverListenerWithRBACHTTPFilters(host, port, test.rbacCfg) + resources.Listeners = append(resources.Listeners, inboundLis) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + // Setup the management server with client and server-side resources. + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + cc, err := grpc.DialContext(ctx, fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); status.Code(err) != test.wantStatusEmptyCall { + t.Fatalf("EmptyCall() returned err with status: %v, wantStatusEmptyCall: %v", status.Code(err), test.wantStatusEmptyCall) + } + + if _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}); status.Code(err) != test.wantStatusUnaryCall { + t.Fatalf("UnaryCall() returned err with status: %v, wantStatusUnaryCall: %v", err, test.wantStatusUnaryCall) + } + + // Toggle the RBAC Env variable off, this should disable RBAC and allow any RPC"s through (will not go through + // routing or processed by HTTP Filters and thus will never get denied by RBAC). + envconfig.XDSRBAC = false + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.OK { + t.Fatalf("EmptyCall() returned err with status: %v, once RBAC is disabled all RPC's should proceed as normal", status.Code(err)) + } + if _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}); status.Code(err) != codes.OK { + t.Fatalf("UnaryCall() returned err with status: %v, once RBAC is disabled all RPC's should proceed as normal", status.Code(err)) + } + // Toggle RBAC back on for next iterations. + envconfig.XDSRBAC = true + + if test.wantAuthzOutcomes != nil { + if diff := cmp.Diff(lb.authzDecisionStat, test.wantAuthzOutcomes); diff != "" { + t.Fatalf("authorization decision do not match\ndiff (-got +want):\n%s", diff) + } + } + if test.eventContent != nil { + if diff := cmp.Diff(lb.lastEvent, test.eventContent); diff != "" { + t.Fatalf("unexpected event\ndiff (-got +want):\n%s", diff) + } + } + }() + }) + } +} + +// serverListenerWithBadRouteConfiguration returns an xds Listener resource with +// a Route Configuration that will never successfully match in order to test +// RBAC Environment variable being toggled on and off. +func serverListenerWithBadRouteConfiguration(host string, port uint32) *v3listenerpb.Listener { + return &v3listenerpb.Listener{ + Name: fmt.Sprintf(e2e.ServerListenerResourceNameTemplate, net.JoinHostPort(host, strconv.Itoa(int(port)))), + Address: &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: host, + PortSpecifier: &v3corepb.SocketAddress_PortValue{ + PortValue: port, + }, + }, + }, + }, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "v4-wildcard", + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{ + { + AddressPrefix: "0.0.0.0", + PrefixLen: &wrapperspb.UInt32Value{ + Value: uint32(0), + }, + }, + }, + SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK, + SourcePrefixRanges: []*v3corepb.CidrRange{ + { + AddressPrefix: "0.0.0.0", + PrefixLen: &wrapperspb.UInt32Value{ + Value: uint32(0), + }, + }, + }, + }, + Filters: []*v3listenerpb.Filter{ + { + Name: "filter-1", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: &v3routepb.RouteConfiguration{ + Name: "routeName", + VirtualHosts: []*v3routepb.VirtualHost{{ + // Incoming RPC's will try and match to Virtual Hosts based on their :authority header. + // Thus, incoming RPC's will never match to a Virtual Host (server side requires matching + // to a VH/Route of type Non Forwarding Action to proceed normally), and all incoming RPC's + // with this route configuration will be denied. + Domains: []string{"will-never-match"}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}, + }, + Action: &v3routepb.Route_NonForwardingAction{}, + }}}}}, + }, + HttpFilters: []*v3httppb.HttpFilter{e2e.RouterHTTPFilter}, + }), + }, + }, + }, + }, + { + Name: "v6-wildcard", + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{ + { + AddressPrefix: "::", + PrefixLen: &wrapperspb.UInt32Value{ + Value: uint32(0), + }, + }, + }, + SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK, + SourcePrefixRanges: []*v3corepb.CidrRange{ + { + AddressPrefix: "::", + PrefixLen: &wrapperspb.UInt32Value{ + Value: uint32(0), + }, + }, + }, + }, + Filters: []*v3listenerpb.Filter{ + { + Name: "filter-1", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: &v3routepb.RouteConfiguration{ + Name: "routeName", + VirtualHosts: []*v3routepb.VirtualHost{{ + // Incoming RPC's will try and match to Virtual Hosts based on their :authority header. + // Thus, incoming RPC's will never match to a Virtual Host (server side requires matching + // to a VH/Route of type Non Forwarding Action to proceed normally), and all incoming RPC's + // with this route configuration will be denied. + Domains: []string{"will-never-match"}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}, + }, + Action: &v3routepb.Route_NonForwardingAction{}, + }}}}}, + }, + HttpFilters: []*v3httppb.HttpFilter{e2e.RouterHTTPFilter}, + }), + }, + }, + }, + }, + }, + } +} + +func (s) TestRBACToggledOn_WithBadRouteConfiguration(t *testing.T) { + // Turn RBAC support on. + oldRBAC := envconfig.XDSRBAC + envconfig.XDSRBAC = true + defer func() { + envconfig.XDSRBAC = oldRBAC + }() + + managementServer, nodeID, bootstrapContents, resolver, cleanup1 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup1() + + lis, cleanup2 := setupGRPCServer(t, bootstrapContents) + defer cleanup2() + + host, port, err := hostPortFromListener(lis) + if err != nil { + t.Fatalf("failed to retrieve host and port of server: %v", err) + } + const serviceName = "my-service-fallback" + + // The inbound listener needs a route table that will never match on a VH, + // and thus shouldn't allow incoming RPC's to proceed. + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: host, + Port: port, + SecLevel: e2e.SecurityLevelNone, + }) + // Since RBAC support is turned ON, all the RPC's should get denied with + // status code Unavailable due to not matching to a route of type Non + // Forwarding Action (Route Table not configured properly). + inboundLis := serverListenerWithBadRouteConfiguration(host, port) + resources.Listeners = append(resources.Listeners, inboundLis) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + // Setup the management server with client and server-side resources. + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + cc, err := grpc.DialContext(ctx, fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.Unavailable { + t.Fatalf("EmptyCall() returned err with status: %v, if RBAC is disabled all RPC's should proceed as normal", status.Code(err)) + } + if _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}); status.Code(err) != codes.Unavailable { + t.Fatalf("UnaryCall() returned err with status: %v, if RBAC is disabled all RPC's should proceed as normal", status.Code(err)) + } +} + +func (s) TestRBACToggledOff_WithBadRouteConfiguration(t *testing.T) { + // Turn RBAC support off. + oldRBAC := envconfig.XDSRBAC + envconfig.XDSRBAC = false + defer func() { + envconfig.XDSRBAC = oldRBAC + }() + + managementServer, nodeID, bootstrapContents, resolver, cleanup1 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup1() + + lis, cleanup2 := setupGRPCServer(t, bootstrapContents) + defer cleanup2() + + host, port, err := hostPortFromListener(lis) + if err != nil { + t.Fatalf("failed to retrieve host and port of server: %v", err) + } + const serviceName = "my-service-fallback" + + // The inbound listener needs a route table that will never match on a VH, + // and thus shouldn't allow incoming RPC's to proceed. + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: host, + Port: port, + SecLevel: e2e.SecurityLevelNone, + }) + // This bad route configuration shouldn't affect incoming RPC's from + // proceeding as normal, as the configuration shouldn't be parsed due to the + // RBAC Environment variable not being set to true. + inboundLis := serverListenerWithBadRouteConfiguration(host, port) + resources.Listeners = append(resources.Listeners, inboundLis) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + // Setup the management server with client and server-side resources. + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + cc, err := grpc.DialContext(ctx, fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.OK { + t.Fatalf("EmptyCall() returned err with status: %v, if RBAC is disabled all RPC's should proceed as normal", status.Code(err)) + } + if _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}); status.Code(err) != codes.OK { + t.Fatalf("UnaryCall() returned err with status: %v, if RBAC is disabled all RPC's should proceed as normal", status.Code(err)) + } +} + +type statAuditLogger struct { + authzDecisionStat map[bool]int // Map to hold counts of authorization decisions + lastEvent *audit.Event // Field to store last received event +} + +func (s *statAuditLogger) Log(event *audit.Event) { + s.authzDecisionStat[event.Authorized]++ + *s.lastEvent = *event +} + +type loggerBuilder struct { + authzDecisionStat map[bool]int + lastEvent *audit.Event +} + +func (loggerBuilder) Name() string { + return "stat_logger" +} + +func (lb *loggerBuilder) Build(audit.LoggerConfig) audit.Logger { + return &statAuditLogger{ + authzDecisionStat: lb.authzDecisionStat, + lastEvent: lb.lastEvent, + } +} + +func (*loggerBuilder) ParseLoggerConfig(config json.RawMessage) (audit.LoggerConfig, error) { + return nil, nil +} + +// This is used when converting a custom config from raw JSON to a TypedStruct. +// The TypeURL of the TypeStruct will be "grpc.authz.audit_logging/". +const typeURLPrefix = "grpc.authz.audit_logging/" + +// Builds custom configs for audit logger RBAC protos. +func createXDSTypedStruct(t *testing.T, in map[string]any, name string) *anypb.Any { + t.Helper() + pb, err := structpb.NewStruct(in) + if err != nil { + t.Fatalf("createXDSTypedStruct failed during structpb.NewStruct: %v", err) + } + typedStruct := &v3xdsxdstypepb.TypedStruct{ + TypeUrl: typeURLPrefix + name, + Value: pb, + } + customConfig, err := anypb.New(typedStruct) + if err != nil { + t.Fatalf("createXDSTypedStruct failed during anypb.New: %v", err) + } + return customConfig +} diff --git a/test/xds/xds_server_serving_mode_test.go b/test/xds/xds_server_serving_mode_test.go new file mode 100644 index 000000000000..b459952b8e0b --- /dev/null +++ b/test/xds/xds_server_serving_mode_test.go @@ -0,0 +1,401 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds_test + +import ( + "context" + "fmt" + "net" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + xdscreds "google.golang.org/grpc/credentials/xds" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/xds" + + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +// TestServerSideXDS_RedundantUpdateSuppression tests the scenario where the +// control plane sends the same resource update. It verifies that the mode +// change callback is not invoked and client connections to the server are not +// recycled. +func (s) TestServerSideXDS_RedundantUpdateSuppression(t *testing.T) { + managementServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup() + + creds, err := xdscreds.NewServerCredentials(xdscreds.ServerOptions{FallbackCreds: insecure.NewCredentials()}) + if err != nil { + t.Fatal(err) + } + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("testutils.LocalTCPListener() failed: %v", err) + } + updateCh := make(chan connectivity.ServingMode, 1) + + // Create a server option to get notified about serving mode changes. + modeChangeOpt := xds.ServingModeCallback(func(addr net.Addr, args xds.ServingModeChangeArgs) { + t.Logf("serving mode for listener %q changed to %q, err: %v", addr.String(), args.Mode, args.Err) + updateCh <- args.Mode + }) + + // Initialize an xDS-enabled gRPC server and register the stubServer on it. + server, err := xds.NewGRPCServer(grpc.Creds(creds), modeChangeOpt, xds.BootstrapContentsForTesting(bootstrapContents)) + if err != nil { + t.Fatalf("Failed to create an xDS enabled gRPC server: %v", err) + } + defer server.Stop() + testgrpc.RegisterTestServiceServer(server, &testService{}) + + // Setup the management server to respond with the listener resources. + host, port, err := hostPortFromListener(lis) + if err != nil { + t.Fatalf("failed to retrieve host and port of server: %v", err) + } + listener := e2e.DefaultServerListener(host, port, e2e.SecurityLevelNone) + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listener}, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + go func() { + if err := server.Serve(lis); err != nil { + t.Errorf("Serve() failed: %v", err) + } + }() + + // Wait for the listener to move to "serving" mode. + select { + case <-ctx.Done(): + t.Fatalf("timed out waiting for a mode change update: %v", err) + case mode := <-updateCh: + if mode != connectivity.ServingModeServing { + t.Fatalf("listener received new mode %v, want %v", mode, connectivity.ServingModeServing) + } + } + + // Create a ClientConn and make a successful RPCs. + cc, err := grpc.Dial(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + waitForSuccessfulRPC(ctx, t, cc) + + // Start a goroutine to make sure that we do not see any connectivity state + // changes on the client connection. If redundant updates are not + // suppressed, server will recycle client connections. + errCh := make(chan error, 1) + go func() { + prev := connectivity.Ready // We know we are READY since we just did an RPC. + for { + curr := cc.GetState() + if !(curr == connectivity.Ready || curr == connectivity.Idle) { + errCh <- fmt.Errorf("unexpected connectivity state change {%s --> %s} on the client connection", prev, curr) + return + } + if !cc.WaitForStateChange(ctx, curr) { + // Break out of the for loop when the context has been cancelled. + break + } + prev = curr + } + errCh <- nil + }() + + // Update the management server with the same listener resource. This will + // update the resource version though, and should result in a the management + // server sending the same resource to the xDS-enabled gRPC server. + if err := managementServer.Update(ctx, e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listener}, + }); err != nil { + t.Fatal(err) + } + + // Since redundant resource updates are suppressed, we should not see the + // mode change callback being invoked. + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + select { + case <-sCtx.Done(): + case mode := <-updateCh: + t.Fatalf("unexpected mode change callback with new mode %v", mode) + } + + // Make sure RPCs continue to succeed. + waitForSuccessfulRPC(ctx, t, cc) + + // Cancel the context to ensure that the WaitForStateChange call exits early + // and returns false. + cancel() + if err := <-errCh; err != nil { + t.Fatal(err) + } +} + +// TestServerSideXDS_ServingModeChanges tests the serving mode functionality in +// xDS enabled gRPC servers. It verifies that appropriate mode changes happen in +// the server, and also verifies behavior of clientConns under these modes. +func (s) TestServerSideXDS_ServingModeChanges(t *testing.T) { + managementServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup() + + // Configure xDS credentials to be used on the server-side. + creds, err := xdscreds.NewServerCredentials(xdscreds.ServerOptions{ + FallbackCreds: insecure.NewCredentials(), + }) + if err != nil { + t.Fatal(err) + } + + // Create two local listeners and pass it to Serve(). + lis1, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("testutils.LocalTCPListener() failed: %v", err) + } + lis2, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("testutils.LocalTCPListener() failed: %v", err) + } + + // Create a couple of channels on which mode updates will be pushed. + updateCh1 := make(chan connectivity.ServingMode, 1) + updateCh2 := make(chan connectivity.ServingMode, 1) + + // Create a server option to get notified about serving mode changes, and + // push the updated mode on the channels created above. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + modeChangeOpt := xds.ServingModeCallback(func(addr net.Addr, args xds.ServingModeChangeArgs) { + t.Logf("serving mode for listener %q changed to %q, err: %v", addr.String(), args.Mode, args.Err) + switch addr.String() { + case lis1.Addr().String(): + updateCh1 <- args.Mode + case lis2.Addr().String(): + updateCh2 <- args.Mode + default: + t.Errorf("serving mode callback invoked for unknown listener address: %q", addr.String()) + } + }) + + // Initialize an xDS-enabled gRPC server and register the stubServer on it. + server, err := xds.NewGRPCServer(grpc.Creds(creds), modeChangeOpt, xds.BootstrapContentsForTesting(bootstrapContents)) + if err != nil { + t.Fatalf("Failed to create an xDS enabled gRPC server: %v", err) + } + defer server.Stop() + testgrpc.RegisterTestServiceServer(server, &testService{}) + + // Setup the management server to respond with server-side Listener + // resources for both listeners. + host1, port1, err := hostPortFromListener(lis1) + if err != nil { + t.Fatalf("failed to retrieve host and port of server: %v", err) + } + listener1 := e2e.DefaultServerListener(host1, port1, e2e.SecurityLevelNone) + host2, port2, err := hostPortFromListener(lis2) + if err != nil { + t.Fatalf("failed to retrieve host and port of server: %v", err) + } + listener2 := e2e.DefaultServerListener(host2, port2, e2e.SecurityLevelNone) + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listener1, listener2}, + } + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + go func() { + if err := server.Serve(lis1); err != nil { + t.Errorf("Serve() failed: %v", err) + } + }() + go func() { + if err := server.Serve(lis2); err != nil { + t.Errorf("Serve() failed: %v", err) + } + }() + + // Wait for both listeners to move to "serving" mode. + select { + case <-ctx.Done(): + t.Fatalf("timed out waiting for a mode change update: %v", err) + case mode := <-updateCh1: + if mode != connectivity.ServingModeServing { + t.Fatalf("listener received new mode %v, want %v", mode, connectivity.ServingModeServing) + } + } + select { + case <-ctx.Done(): + t.Fatalf("timed out waiting for a mode change update: %v", err) + case mode := <-updateCh2: + if mode != connectivity.ServingModeServing { + t.Fatalf("listener received new mode %v, want %v", mode, connectivity.ServingModeServing) + } + } + + // Create a ClientConn to the first listener and make a successful RPCs. + cc1, err := grpc.Dial(lis1.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc1.Close() + waitForSuccessfulRPC(ctx, t, cc1) + + // Create a ClientConn to the second listener and make a successful RPCs. + cc2, err := grpc.Dial(lis2.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc2.Close() + waitForSuccessfulRPC(ctx, t, cc2) + + // Update the management server to remove the second listener resource. This + // should push only the second listener into "not-serving" mode. + if err := managementServer.Update(ctx, e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listener1}, + }); err != nil { + t.Fatal(err) + } + + // Wait for lis2 to move to "not-serving" mode. + select { + case <-ctx.Done(): + t.Fatalf("timed out waiting for a mode change update: %v", err) + case mode := <-updateCh2: + if mode != connectivity.ServingModeNotServing { + t.Fatalf("listener received new mode %v, want %v", mode, connectivity.ServingModeNotServing) + } + } + + // Make sure RPCs succeed on cc1 and fail on cc2. + waitForSuccessfulRPC(ctx, t, cc1) + waitForFailedRPC(ctx, t, cc2) + + // Update the management server to remove the first listener resource as + // well. This should push the first listener into "not-serving" mode. Second + // listener is already in "not-serving" mode. + if err := managementServer.Update(ctx, e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{}, + }); err != nil { + t.Fatal(err) + } + + // Wait for lis1 to move to "not-serving" mode. lis2 was already removed + // from the xdsclient's resource cache. So, lis2's callback will not be + // invoked this time around. + select { + case <-ctx.Done(): + t.Fatalf("timed out waiting for a mode change update: %v", err) + case mode := <-updateCh1: + if mode != connectivity.ServingModeNotServing { + t.Fatalf("listener received new mode %v, want %v", mode, connectivity.ServingModeNotServing) + } + } + + // Make sure RPCs fail on both. + waitForFailedRPC(ctx, t, cc1) + waitForFailedRPC(ctx, t, cc2) + + // Make sure new connection attempts to "not-serving" servers fail. We use a + // short timeout since we expect this to fail. + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := grpc.DialContext(sCtx, lis1.Addr().String(), grpc.WithBlock(), grpc.WithTransportCredentials(insecure.NewCredentials())); err == nil { + t.Fatal("successfully created clientConn to a server in \"not-serving\" state") + } + + // Update the management server with both listener resources. + if err := managementServer.Update(ctx, e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listener1, listener2}, + }); err != nil { + t.Fatal(err) + } + + // Wait for both listeners to move to "serving" mode. + select { + case <-ctx.Done(): + t.Fatalf("timed out waiting for a mode change update: %v", err) + case mode := <-updateCh1: + if mode != connectivity.ServingModeServing { + t.Fatalf("listener received new mode %v, want %v", mode, connectivity.ServingModeServing) + } + } + select { + case <-ctx.Done(): + t.Fatalf("timed out waiting for a mode change update: %v", err) + case mode := <-updateCh2: + if mode != connectivity.ServingModeServing { + t.Fatalf("listener received new mode %v, want %v", mode, connectivity.ServingModeServing) + } + } + + // The clientConns created earlier should be able to make RPCs now. + waitForSuccessfulRPC(ctx, t, cc1) + waitForSuccessfulRPC(ctx, t, cc2) +} + +func waitForSuccessfulRPC(ctx context.Context, t *testing.T, cc *grpc.ClientConn) { + t.Helper() + + c := testgrpc.NewTestServiceClient(cc) + if _, err := c.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("rpc EmptyCall() failed: %v", err) + } +} + +func waitForFailedRPC(ctx context.Context, t *testing.T, cc *grpc.ClientConn) { + t.Helper() + + // Attempt one RPC before waiting for the ticker to expire. + c := testgrpc.NewTestServiceClient(cc) + if _, err := c.EmptyCall(ctx, &testpb.Empty{}); err != nil { + return + } + + ticker := time.NewTimer(1 * time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + t.Fatalf("failure when waiting for RPCs to fail: %v", ctx.Err()) + case <-ticker.C: + if _, err := c.EmptyCall(ctx, &testpb.Empty{}); err != nil { + return + } + } + } +} diff --git a/testdata/x509/README.md b/testdata/x509/README.md index e64a385e5f97..661caf4ac858 100644 --- a/testdata/x509/README.md +++ b/testdata/x509/README.md @@ -3,104 +3,4 @@ gRPC-Go tests. How were these test certs/keys generated ? ------------------------------------------ -0. Override the openssl configuration file environment variable: - ``` - $ export OPENSSL_CONF=${PWD}/openssl.cnf - ``` - -1. Generate a self-signed CA certificate along with its private key: - ``` - $ openssl req -x509 \ - -newkey rsa:4096 \ - -nodes \ - -days 3650 \ - -keyout ca_key.pem \ - -out ca_cert.pem \ - -subj /C=US/ST=CA/L=SVL/O=gRPC/CN=test-ca/ \ - -config ./openssl.cnf \ - -extensions test_ca - ``` - - To view the CA cert: - ``` - $ openssl x509 -text -noout -in ca_cert.pem - ``` - -2.a Generate a private key for the server: - ``` - $ openssl genrsa -out server_key.pem 4096 - ``` - -2.b Generate a private key for the client: - ``` - $ openssl genrsa -out client_key.pem 4096 - ``` - -3.a Generate a CSR for the server: - ``` - $ openssl req -new \ - -key server_key.pem \ - -days 3650 \ - -out server_csr.pem \ - -subj /C=US/ST=CA/L=SVL/O=gRPC/CN=test-server/ \ - -config ./openssl.cnf \ - -reqexts test_server - ``` - - To view the CSR: - ``` - $ openssl req -text -noout -in server_csr.pem - ``` - -3.b Generate a CSR for the client: - ``` - $ openssl req -new \ - -key client_key.pem \ - -days 3650 \ - -out client_csr.pem \ - -subj /C=US/ST=CA/L=SVL/O=gRPC/CN=test-client/ \ - -config ./openssl.cnf \ - -reqexts test_client - ``` - - To view the CSR: - ``` - $ openssl req -text -noout -in client_csr.pem - ``` - -4.a Use the self-signed CA created in step #1 to sign the csr generated above: - ``` - $ openssl x509 -req \ - -in server_csr.pem \ - -CAkey ca_key.pem \ - -CA ca_cert.pem \ - -days 3650 \ - -set_serial 1000 \ - -out server_cert.pem \ - -extfile ./openssl.cnf \ - -extensions test_server - ``` - -4.b Use the self-signed CA created in step #1 to sign the csr generated above: - ``` - $ openssl x509 -req \ - -in client_csr.pem \ - -CAkey ca_key.pem \ - -CA ca_cert.pem \ - -days 3650 \ - -set_serial 1000 \ - -out client_cert.pem \ - -extfile ./openssl.cnf \ - -extensions test_client - ``` - -5.a Verify the `server_cert.pem` is trusted by `ca_cert.pem`: - ``` - $ openssl verify -verbose -CAfile ca_cert.pem server_cert.pem - ``` - -5.b Verify the `client_cert.pem` is trusted by `ca_cert.pem`: - ``` - $ openssl verify -verbose -CAfile ca_cert.pem client_cert.pem - ``` - +Run `./create.sh` diff --git a/testdata/x509/client1_cert.pem b/testdata/x509/client1_cert.pem index 714136918f30..6f82cc3be84f 100644 --- a/testdata/x509/client1_cert.pem +++ b/testdata/x509/client1_cert.pem @@ -1,32 +1,32 @@ -----BEGIN CERTIFICATE----- -MIIFcTCCA1mgAwIBAgICA+gwDQYJKoZIhvcNAQEFBQAwUDELMAkGA1UEBhMCVVMx +MIIFcTCCA1mgAwIBAgICA+gwDQYJKoZIhvcNAQELBQAwUDELMAkGA1UEBhMCVVMx CzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTALBgNVBAoMBGdSUEMxFzAVBgNV -BAMMDnRlc3QtY2xpZW50X2NhMB4XDTIwMDgwNDAyMDAwMFoXDTMwMDgwMjAyMDAw -MFowTjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTAL +BAMMDnRlc3QtY2xpZW50X2NhMB4XDTIxMTIyMzE4NDI1MVoXDTMxMTIyMTE4NDI1 +MVowTjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTAL BgNVBAoMBGdSUEMxFTATBgNVBAMMDHRlc3QtY2xpZW50MTCCAiIwDQYJKoZIhvcN -AQEBBQADggIPADCCAgoCggIBAK3fSafgFyHediP0fonPcc/pH010l2jqryUNsEfr -PhxR//ccr7sBvbcInwvj3NJ9XqF4V4ws9h/QbPMLXg1FBcC/LpYjo6VZoNjuJLt2 -DTG2gGcTEL+4G2w/4ztrrmunLxa53P3URIgMgMYhCTIXK2enVbpy637X8WhPYOrq -w+NXnDaTwT8uLGfMVEAKNvXzf8Ras8OHjgTZJEpkgXVjREhUhOPszrBsyYKnI/f2 -QSDnvgSJbrkBLFRqluT/ciqccryBWy0qJOStVhha1I2tId+dvJsTgQa/NLBASbsU -LkIIUV375K0raINYeg/kA6MK6YDwcCtrVbQa8fu7drxxBiY3tSoDLVn1FYz7iTJ3 -PvtpwsGAqTEsSW3k7l2MTz3iuqcAgL8tI1CpyacwNPfy7j67mH3akY95sh2nmTVj -rsW2uuFSC/cc00bH+IMVZnztE7+fpgZvU63BVnf9d9TMgDe8kwMwbq7dFi9irr12 -8Szpbdnt028dgsrjpbOgPpMYJehRK2Q0I7+99cLeJa1V5ySeFhf+uhNpW9RDi/qp -TJGAG+rE3qAbVVoD9GrOispNZW7Hby4/q8pkNoafXmilqIf6mOri/88AYOMXbH4X -i8mJgIeN2AjJmEGVPBPM25ZjN+ZurWqfdSasXuiIJmJzw9ExcIEjzAjoMl9CNVVy -c77lAgMBAAGjVzBVMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFEdlWQy5/06l3GTu -rqJTuMgy4JuKMA4GA1UdDwEB/wQEAwIF4DAWBgNVHSUBAf8EDDAKBggrBgEFBQcD -AjANBgkqhkiG9w0BAQUFAAOCAgEAAmzrEprlWBxCQDzFZy5pZVIa1FniD+23qXlV -n0Fhhr0eF2udYR2tfzf0VM9WcBHHoRzX5fwNkGmIWiXAISgamMl4sHHZn6Ig0i9h -k9/fI4bYtrCiOqjYRG6VA8OZSD98bD+NtQEPQneO5F5buL0by4FUugu6Ls0Ovpk0 -yhb2pgKFhbFbMC6ev1AK9IpJZgz2q9/rjkJedGjnu35ze+94tw5Fe1FIIkg24ZQk -C4e4DzSpRz5s51LS+dS5hDGuvglWn7SrwGuGujz8iMQdAJa3WSP5WmjbuUFaD8pH -6afrjAhMZoWgxNubLkypkFUW/3W5JwTLnj5wPhPpBtHX6NQ30/FgN89j7+0Zp064 -i4Ur1ykhHgbdUb/EB28sXs+/CkmfmFx44M68yhoJ5euUzRF5gGmxSgRn3+RVsw2E -ju0YQBVvH8JjAt0XCi9SY+vCpe+EG0uV4HxEO6DDdSslsMkuiAeM7pvZTM3FbZyt -BXpWs/L71OF37ouUbt1TD+C1fsCUovjGi4AE0KXeO1rv4u2mTGfxtOOUFKt2dFDa -E1sjyJm1+WjDgIqNjbubM6zpvNtix0xaOXqg7MAt4OJnKAeRQELgJhe6Rt2ROKGq -Hoy8uIjcA26/lwclj2h7fwiKznlxqfDxVsiwmCTdJJb76w69UQvyIRY3tlJr3c3+ -O4VSONQ= +AQEBBQADggIPADCCAgoCggIBALUoje/J3uPOJ0dapY2s7mGLVPhYRaHyRnJE2/TY +zFOB0IisAF3R7BIDufQrHhk3fh0JazCw95TDD9rxsKEVs6Z50lmDkrg/bjlsniE/ +n+M1JacaLQW7xfh2L+Ei4jvMr101nAsimd6IxFU9m3+2SFbhPBG/GWWJ2ZKqQblz +DVMpNg9FYNmMe45vLevOhdPQBE4cVoAPhI9Je+P4Koslebhor0koUeQVeYdBbCq3 +3dQJPAHjBST6mD9mJI4yVrE3Xso3LO85WROUPhRYQyXhrgU15W6g9qTpMTfkriUe +FYLCtAPU9LBodyvjYLuwoEoyRVsA6Zh/vABteD8Afl552fV9KwN2fRVbTDAxQCp7 +P8gE3/rD1RKv7KBNJ/LrwMu7g4VO+tzYDxWee+eXPQ6M/zRWAb3E0v3UNHsF1ZBl +rlFhEiRShHrXDEKMQwCTSrRjwYajUpZ/Hq2USDgkLepKmTmCaoBfWHPyZwblqSTn +A4DNOh5N23eJyrLnJOPYjzZqEPfX5hDTjFRdVTQxtmYlJ1muwtlNyuwZDImhjO6G +54pPj/bV6gy1+YpIQBemPoXtqqmcRiEVWSV5zAizwRaWf85tqpxb1Tjuj2OpD9le +oO4JX0HLjhyQBoKspNohu2I4+s7ex/w92bf76cTpYTbMJqIp37YZmfPVztHVaMl4 +W0xRAgMBAAGjVzBVMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFMRdhhib+RS6IJpQ +zFsaKH1BNbyZMA4GA1UdDwEB/wQEAwIF4DAWBgNVHSUBAf8EDDAKBggrBgEFBQcD +AjANBgkqhkiG9w0BAQsFAAOCAgEAHyQwLSo/UdSoZCKcdeb0DCkeABRMUPIysroj +gQJCe3cAKOT+TxL6Qnp6jzM2/vt2T/lzlRp7RAB1jH3J4wy5re4cbCn1sbII3Rog +Nm4PKcw6hfUET44t1Gk9DsCjgvxEIirFBWVpxfn+YDI916iH1fkNURaMP+yxpQBL +3K4bmxanBiyBUHC8cyChLMD2NwXjOAA4pZFk0ohpmK0YUk4ra3Z3Q30DCH6NZ1ZP +aOMDHrCXU6MLlmPk8yiOnotgjqiYEgi3Bzxd/OHpR41Xo8k6g3UrN2GEQFs17ibQ +CQasxodOar5Vezu6ZKCYk5TaY4lugT34w+qxi8tVF54WY2jtWY5PUmU6ZT2Dw5cn +CQzlPUdEebOc1hltTvsD049/2lZmGlMXk0dykxy51jYAYznf2rb3cnC1vu1Wgi3w +J28xXBYD8AvME9jaJ6g3L+KR+AFCSLqpUsTxvu9zKf6pLrVtOCl+9G69uOK/wono +yMGNeel8rkzwzzr1LNrhmcKHqipkq83vqxIUT/mbpBUKO1ZXVG/TWKS6bpBTc4Pn +hBCIvGOSyoKuEiXnFr6fqLhLskUNcCNl7iOfA9h/MhS5ZufJXhhXu3Wbo/KC/mNh +y+fr1S9AyA+EJaYtJRKAOeewGvXYb881UNXWGCQU1aVNJnujRKFyhd07sEjxsad9 +Bn/aYes= -----END CERTIFICATE----- diff --git a/testdata/x509/client1_key.pem b/testdata/x509/client1_key.pem index b7a3930254fd..6cd652c55435 100644 --- a/testdata/x509/client1_key.pem +++ b/testdata/x509/client1_key.pem @@ -1,51 +1,51 @@ -----BEGIN RSA PRIVATE KEY----- -MIIJKQIBAAKCAgEArd9Jp+AXId52I/R+ic9xz+kfTXSXaOqvJQ2wR+s+HFH/9xyv -uwG9twifC+Pc0n1eoXhXjCz2H9Bs8wteDUUFwL8uliOjpVmg2O4ku3YNMbaAZxMQ -v7gbbD/jO2uua6cvFrnc/dREiAyAxiEJMhcrZ6dVunLrftfxaE9g6urD41ecNpPB -Py4sZ8xUQAo29fN/xFqzw4eOBNkkSmSBdWNESFSE4+zOsGzJgqcj9/ZBIOe+BIlu -uQEsVGqW5P9yKpxyvIFbLSok5K1WGFrUja0h3528mxOBBr80sEBJuxQuQghRXfvk -rStog1h6D+QDowrpgPBwK2tVtBrx+7t2vHEGJje1KgMtWfUVjPuJMnc++2nCwYCp -MSxJbeTuXYxPPeK6pwCAvy0jUKnJpzA09/LuPruYfdqRj3myHaeZNWOuxba64VIL -9xzTRsf4gxVmfO0Tv5+mBm9TrcFWd/131MyAN7yTAzBurt0WL2KuvXbxLOlt2e3T -bx2CyuOls6A+kxgl6FErZDQjv731wt4lrVXnJJ4WF/66E2lb1EOL+qlMkYAb6sTe -oBtVWgP0as6Kyk1lbsdvLj+rymQ2hp9eaKWoh/qY6uL/zwBg4xdsfheLyYmAh43Y -CMmYQZU8E8zblmM35m6tap91Jqxe6IgmYnPD0TFwgSPMCOgyX0I1VXJzvuUCAwEA -AQKCAgEAidNL4aUC8TgU0h+HBtrHzxVuWMmpE+OkfmzBZeEV1QEzM8Erk8OnjSVq -XdR8QOZcUwa/7z/cwg9Hrck+/qnOC6IA3cbWe8X2eL8dovPLNbMDSbGVP0RDiKWE -DKApHPDjpNIkWZkf0fCHS4b4cRpor7u3exqJjnzCwfraSp1aNiZGkATD1L9XN9iC -mFkAhCpHB3EWulIDw9gUqlvNOy46/FLzHHGkzbkOa2DuZCpyKhFJUPNYL5K8fxYX -EuNirmBhmwe3LLARmqvEaX3mq3+oMEgrL4pgZua+b1AmogM3P+S0CxoXhSW5rRQ/ -fcUzFNUbj7gIUoK85w3M780ELBAz3F0j9cy1/DcidV0T8SAzKVrpiJvvK59XYzzn -3J4JFmAsZ0PYgkPhZyPY6hNysRFapPwJyNC+I1NVRpSNHifMsYNEX5dV4M6Qtmv4 -7QmtvUubpJ+vo75W0DNzQ8Ar4BaBVZ6YzKTW58/Ob9Y1o6knUJv/lElE9RLyJBrn -PgtFMPDjf2FzYaA45+zVtQBDk3rljLatS6WZxWg+qh+4RPQjS6sKzNB7U728oiZj -1PRMbeUGKAZDb6FWTZ5nlvai3Z1VDwmLdBBSACnUWLOhXqmnkWY0q9d3kSGnMih4 -Au1A2sCFhhoowoyEkbbmlvORDSo6jfqdYKxP2rUQV1DJBPepo6kCggEBANQzH//s -CTcB9fMSIpiEUmcFBBineA+uMXDbmNRzeJMoFwSxt7TXQpVqfMOXTaa4aDuCSCNX -VLIf10aP4Myx9A0i+t4A9IAn9Ps+wCu5Yb43XmiEc2PC/AZYuviYfP/rIptTS0o8 -z8zAc1cLdDYBww76DcKdagAQABZQaqPARlGEHAvqmr5fjR0oWfcGeEvzqdv7WbGf -9nyuAWl1ldMmILysW0GRDudFhp5rit6A3uCq7LB5Qb14dGrek5k+y8lnzjp30r0O -9QxUuxZVuvh4ujiDnQI5tVWbhD/jgIUF91Nm/Vw0bZMdcp0iA9r8EGmFaHNi0by7 -rMw/6Pqcxd75qP8CggEBANHC5PZLyZEmt6C/c1ohTIZtf2g5ghXSHARfwMvRTVJ5 -4HksZp/FQSe3080l4yACXDpfxJ6pm5WNFNhy4Xi09wkEIWk+bSOqBgk+DvItgqkY -em3q1EUUdhzIB3OXqWRcpgmc78hLiD33GkCTM9BR6W2Q/5TY7o5ULOjkiDKiVL+r -+juFlXQtUTOak0Mwu1RRDQE6z96N5Ffg2rHxjNu1HxQK7OsSfc/lrwOyqnXaB7kR -7CThI4xpSmtyMq4prxehM1YhKk2rJmT4hW+M636uyxZCBg1Aoqqnoxv0sQTHH6k/ -RU1+ZU38RYLzid2qNBom86RS1fWX60H3CH4EX3AVFBsCggEAFMOv9O4W9MAHXjK/ -GeeQ3K3b+cGheP9VrTJ/4QIvoU7B+d6eGF8cD9zsuoL6wT64TGJyRqsMCaYd/bSk -jcM4G3T50XGMe2HtkgxQ57ZrPx7R6S5U0EVLPh++pAbf7HcI2uQqsOgEeYe3gaQI -SiSf/r4vTIT00269Y3GZDc8J0n439F6Pp+NXvqutKgQDD4OXcoRFAaGikA7C6pvr -/k5z06KWB3N3XuApzSS+4QkBRkDTim1DJpQ76B1BmjRP4rR6tLP29jMZfYxpBkV7 -V0cRCeivG4GkIe1m4o2TjPDJg+rHDhe/RS8TgRbMA8i4nmrEjs3zsiE3RoFWffeL -UUdi5wKCAQB86+rb26rBbSNy8lHKXYZrkI6ODaGxSR4yZKw3NgEsmzTaNV0wzZLO -CqZyyJuJFp7CjQJV04C7AfhmJ5SsBGoSzojvWqQ41ysdGf5gsEXeWpufFnkwYs0s -utvlNW9GO/8OPo525LTQ4naZ+pCjAgVYoT/073SzAuJ0GJYcQZzjQZKXHCkztUFk -0CvfmggWYOaz0si1LB/PTjQwQUC4IBfQIemS3cJbq9gdBayK3zw2NbxDAmnfV11g -u/P+0QhbtD8Ujk/ZTZJiE7e0BWLCYWrFaLCd995ob8mt/n3l8IikjO/DBQFj/leP -c2apwpGg+Y2kUUjnKICNGofONOB5qbP9AoIBAQCSKpGUVqnsb0PSqjrhr8B1VFvS -4MZfe0ds6/GrB02D7owHPhPaSJsXhBXVri/ECSx2WripMujbZ3tZH4IPub848PYv -9668O1RxKRkyoknyUn5TO58dhYbp3VO7P7EqfVfqEezyQ8bDfVGxrIbMA+kXJosi -T052e3yNin6Q1r+R3cWCg0dHBGDCCkpKdD861LkYjfyipw+u8c4O+CefTHvd8XV8 -EXGn+NBBAPG42bBsJMa+P/1k9qJbflbUfQy/lPGxMspVD8xwWWeEOJEFTgGmoLWE -cNtabvDCEiQ6+DjBBE2Cl656MjX9uv0Dn830so/PLr6FWK0JSy9sGIRTrPC/ +MIIJKgIBAAKCAgEAtSiN78ne484nR1qljazuYYtU+FhFofJGckTb9NjMU4HQiKwA +XdHsEgO59CseGTd+HQlrMLD3lMMP2vGwoRWzpnnSWYOSuD9uOWyeIT+f4zUlpxot +BbvF+HYv4SLiO8yvXTWcCyKZ3ojEVT2bf7ZIVuE8Eb8ZZYnZkqpBuXMNUyk2D0Vg +2Yx7jm8t686F09AEThxWgA+Ej0l74/gqiyV5uGivSShR5BV5h0FsKrfd1Ak8AeMF +JPqYP2YkjjJWsTdeyjcs7zlZE5Q+FFhDJeGuBTXlbqD2pOkxN+SuJR4VgsK0A9T0 +sGh3K+Ngu7CgSjJFWwDpmH+8AG14PwB+XnnZ9X0rA3Z9FVtMMDFAKns/yATf+sPV +Eq/soE0n8uvAy7uDhU763NgPFZ5755c9Doz/NFYBvcTS/dQ0ewXVkGWuUWESJFKE +etcMQoxDAJNKtGPBhqNSln8erZRIOCQt6kqZOYJqgF9Yc/JnBuWpJOcDgM06Hk3b +d4nKsuck49iPNmoQ99fmENOMVF1VNDG2ZiUnWa7C2U3K7BkMiaGM7obnik+P9tXq +DLX5ikhAF6Y+he2qqZxGIRVZJXnMCLPBFpZ/zm2qnFvVOO6PY6kP2V6g7glfQcuO +HJAGgqyk2iG7Yjj6zt7H/D3Zt/vpxOlhNswmoinfthmZ89XO0dVoyXhbTFECAwEA +AQKCAgEAjtzrijWVy+sQuMm4k1DUMSKzIKJkT4GDoqvBFoc+I4DVVmLmaxaYZ+B+ +bhruwo4rq3R5Ds4QgUWPJGfDllVJ9rhNdYA4XYrQPwL0dV36ljCcf/o5lTLuvbFe +stpStTwG86fKZlGkLIWI53wNPBshUzqOp6QfwB6E8Y/JAxnDYVi3pDVfWlDaQ4pU +GYklqtN6AauBX75dGK6nwDE+Q7uLES2lRjlA03FIBK1IQyv7CTM7GnXQ4cep9x1z +KJx0F4+F9kyq6AE+yRz4FA1C7wXZuYw2YhcYSxcHVH/IAceGyTcIxZjUWqYXjQnk +iD+TONAKN+kxTq01MtUhpfWasqC/i+6QU1eqf5YWpd6GsRKyrGgO02NND/SM6Z3V ++S9og4QAjdUyc8dkN+udd1K1CeYNFbmhrYpF2aS9k/PjDP3L137hDW6Cy+thIjZP +u9OB6ba2yUrbQDlmkCbh0vX+77HKAbT5bj8h9r7MqzNsPsgkaKS8gZ79T/Whr/ft +Xiu+eo/u1jtjwUjNMKGxQ9XiU2UU7QccthHHLcYaiv4eySHXA75h+Sho9cD1Vvs/ +ms1/nbCSuU9TSK0UK/V8YjeDA0eVGtDCX3weIW2ECQ80SoT7uf+fhjaLkvOadb7f +1O9DvYVYZvblxUm8ajOh+/n9VyB/I9R9Q8GdGiauXy16uXLZMdECggEBAPEx+4aR +XfuXmnEoPk3GpvyrRlrMy02aklrATKcxeAYo2uiIcuQv65E3AmE1GHpoxmAKjLty +fuUfGdT7f4uGeF6p+IEkW4jQm56UFbCdun9kduEaN9FRylTBqUKWIY2rtRS6nHZ8 +bAkL/6Uv3g9NWx95rV7HnAfC2n6AIvc8LRfQVVqSvjPbsEPvJAT2353D0Rb7vC2M +1hKeBrSNBiy57EKnrMDOhNpBvSBU0Zc+YsBRNAimKyBz7dt35H+THkFaEk9vGtG0 +QkDvngPzSX99Ojwk2mo9jGrh7LHErWih5C73IfvYUh3kyEwbZ5y25i9Z0F37boIG +jHSVvcPp+9x9PNUCggEBAMBHLyhBUAQVZFXtWysr0BjO34XffgkSt1XQa8cVxif7 +glWauUZtjfC7PT/qgY0mx2dI2bDcKiQQCBlVavP1RLRwj3rZv23eit7z13UgHSa6 +3dnsgpO2Zux6qoV48lO4xbuFqZtW+MP+9jthKwr95r8lmZ4cmGQwXXcqNsR7skFt +30Uhcyn+MTfyLwcqt8g9i98rrJmbPAuIME/Sz9DLIi6UxQLI6MeEn92AzECNDp18 +CypOL+sDrLw/7HNHNoSblgm628BHpBgT2qaOYnawRr0gni7MHXOAbDopKYDAtLuU +ZMFjlILdfiSDouhvKtMlZG9arTB0TasdAQJGPz53H40CggEBAJ4JDvJsOzVHb2Vn +ZfNWD0INA0spVqhheDXIPDFsg2UdzdmA1i7XizUZ4xBIVuKV1i1FnFKRwb1ktGtN +4pNMJ4B3RCFx7hvl+6FbDB8uKe2gqRfzMtGPEtCYF8xOTGvkLwEHCM/F1I/U8cuN +YqWKHQOxmTw58+1N6hXq5X4zSqSI1/RBpCiccJEClwo9q+VWUaEKjpEV74pBSslw +gbQ6mihOby3h40CSxFXz3WSI9vFmA38LScS40Qf1NZ21iqRtXQP5G4x93M9pcZLL +DMRhDBAuYYItE91QbONJqAmf0cBII1c9tQhrSCY96pTPbmFmKtX5kb3Whp85Ih7F +KEafNIUCggEBALMnoIDZmjyz0fFeX3wyLotu9kY+n6jEj56dvE6bsy694grxR4Cf +w4lybPeJAX0LjPBnqK5p9bn0VheEx0rYVVPrLUVCbmNo3+wtN6wiaAcWRnAvNtt7 +MRtWkFwc/W2U1GiNeiMLPm8guT1KpFhxiva/igsQic2QYwYNh0o8FzNvtIEtUajm +9+Uw+zCqVON2tUUT5JabVa9JDfrSamAZZZgRdh/KI1sD8BDrWWUsCVojoiOhBnTr +z5730ND4oYudjIc0XF0kY3krxqc6M/Ry+vZt1fW0qhxcpHrsr4cQB1ZgRiELL+1f +g5FyNfBs5HIofRRkYMqtE1FEjRQZcAQ76mECggEAaOUtM9BZuV9gEwmG4hmFfeXq +vJOMvlsDkRRbLuDQ1B8Vw3v7lt1+K+KfBt96MoQe08MyXM7sIMB+hn+zakNaM2W6 +UzTnAPQQAo+wELqj6U3DrV7zw7I1hZTA9G7qxMAQBEmk3u2q4/zWDAcyAx3D9JVj +L3G14pYf0drFLChnknVTPRaF0Q5upLYzCPLMa9w0FLKy6fkfdWdpzyjvW7+JEeFY +koA98hrottqJB2CcqehQDSCUHKKbd5U15y1NV1BQloaPJLwpPAVTkBszQSHanltN +l9POJBJlfQ1eWL88wHdKiLbtOg6PTfAmfghIRxakjHvxBgFO1/xG6Lxm7QwUDQ== -----END RSA PRIVATE KEY----- diff --git a/testdata/x509/client2_cert.pem b/testdata/x509/client2_cert.pem index 6777622f891b..2cffdeafb590 100644 --- a/testdata/x509/client2_cert.pem +++ b/testdata/x509/client2_cert.pem @@ -1,32 +1,32 @@ -----BEGIN CERTIFICATE----- -MIIFcTCCA1mgAwIBAgICA+gwDQYJKoZIhvcNAQEFBQAwUDELMAkGA1UEBhMCVVMx +MIIFcTCCA1mgAwIBAgICA+gwDQYJKoZIhvcNAQELBQAwUDELMAkGA1UEBhMCVVMx CzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTALBgNVBAoMBGdSUEMxFzAVBgNV -BAMMDnRlc3QtY2xpZW50X2NhMB4XDTIwMDgwNDAyMDAwMloXDTMwMDgwMjAyMDAw +BAMMDnRlc3QtY2xpZW50X2NhMB4XDTIxMTIyMzE4NDI1MloXDTMxMTIyMTE4NDI1 MlowTjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTAL BgNVBAoMBGdSUEMxFTATBgNVBAMMDHRlc3QtY2xpZW50MjCCAiIwDQYJKoZIhvcN -AQEBBQADggIPADCCAgoCggIBAOBsqW26eD9t1JvEsQH2PjcstaknRoeNi+yqxQ4w -DIWrngdeL9/achgzCx4lCbcdTv0QatGg3manPEXem5qsmR9dKj+EKPXnV2qI6PSI -fv1AwcLma0Ph2F/zMASNP5wkwgv6MaIyYx3n+F4iBGQToUaj1l9XS5E30w7k2VxN -KR7zDOGSKifavuGP2nVT8NKgXUjsh3X9F72ZIPZwvaPYkbmikOshDr8TchSgof6+ -9ng+sSmYt/Vm0yWspjJfk2qJldeIXgGRVCwOl0qBsziEk0HJSpAjjy/u9GcTGz3A -qRQ7wPmoKU5MwnNQKZGE73JRra6zk64YiqWdkg7x2WuE1Dp661bvP9iwC4EgUqXL -ZEQkISsDpT8RkWqp2G5crvyrk/cf8I8TbsPi9Q6Eg3dRkqCN8H1mkZT1assOp+G3 -2F7jOvagZfLik3xoSbvpD+u2vMRe30uPKZBNhEZv2PU2YaSEXu+a5qT328uFK254 -rLFi1DZU0eXlj9Y///nMo5kUoq3z4WcL1rnDRSJk2JZJ6Ln5SXN4lbNuvn7dFjKA -VoQa4texrCSf8jtRKzexhBi28n6LAorJT57E/mo0ZvfL6aJb7cUjbhQZZmC5Kqoa -lMaiEnoPxhMqG4m+n6bfYGLqfZlsDiTVzcgEd+RxGTlhaUIg65ZMGK5982PvV3vr -AeyRAgMBAAGjVzBVMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFHoqiLSZ1N8EEtPo -vmr1u4X0tKyKMA4GA1UdDwEB/wQEAwIF4DAWBgNVHSUBAf8EDDAKBggrBgEFBQcD -AjANBgkqhkiG9w0BAQUFAAOCAgEAZ+NQzQk5L/55+58WYQk81SyFWXLjj3RVO3fB -jUgIaxd87IrVeLKrnfoa9mMaS2Qf3SfEMhovRy6Jb2jfxbG0wLQnhx1bqNtaNLAr -2pGG/Yu+4ZzN4iIloP9dn98tYFkHOLOLfIwNEh4Yg6IB9eg+qcbDg5JlqGkGRSsu -IOS9XD2CY9zQ3KLGlVCWZ1EfW8u+du1GIUMx0DEwEYZ/zYnyTa2bCdBD7aetmbKZ -yjSQ0Ole1W3z1Q3uF8CQjZ2dr/wQ3nmxj5Km9PN7/Q9iHn7RyeypWxt5utzSG5Bf -egL0ER8kmYeKHZeagdRbKWPRyUjEligndLzh8Vi75hGFBDAx/pB0aVf81HEStKKw -WCuL0PKpKIoIqNE8aJ6jTo0OEL+Z+6uam0vSnuVqHkeigbNsmefyR82TmiJYDahM -3CBp6Q5gfw4WKIY/0JuJnN4Ym+zIgv2kKRVHGK3SHhiaCUGt2BydN9MxSjl1/B+v -U7kYVj73MJZHSl96w1mnXXFOevxb7SOP23QmTKfqmU0NakfRMcHcjnG6M5mlnIDg -DjpSJd1TLoCS1SfIyc+Fibd4grsRucnuo0iHuFqV8TZ4hi0qKKE8UG8En5KNiQDl -exFgZo6FGQa1mJxQiSfhL3VoyeZ/b3QRG+mNVDmgHsZSTjfMppfmyBJRT4HIlsHS -dWeIeN4= +AQEBBQADggIPADCCAgoCggIBALb7KLFguOBiEHR8FBI/0AFs2X3s7fN5ZCOkTf4p +s9LwAcBWm5/zUqzvZCSui+4sr3qN1b1D+Xbc4xH9+WcxfbeoA2w4d2FKKJ0qaShD +Mu4XTQfj9B7g5GZ+FUeP9rScqgJ+WFeOM6QoAgRrFAS0AMP21TpDue4AVKmD1trX +z3f1DaRBtcUa4zlk2J0GBQDPyPB4worxhZ0IW+OLz2RIl8AWJBDFKMgscxEx239t +GTOY9H6hPI7Py2koknj7LBNc84lf2PVFw5OytQYglmtkFQqntyVxtETAOL4pFOjj +Zw6MAnQBGLS6nhiXG1LkZuvWJn1T5ewhci4qNVpv/8LtrPc2Fv7jb86I5XJvdOGv +hYC6IwS9Psg9oCYaIzyandoSj8SEwXaQuD98ROBUs1raasLedg3d4xNeZCRRmnzE +me+IpHm/wS4hTMxQJHYVewNB68fl6FoyRAqoXNy5yi8uMJKbjqb9E7niQCQRO9vQ +eX0TrB23uoUXbdTz95uMiw8yy/xz11/h59TxN9cOiqDf74tymgH0jTwO8eg/bzKU +zTXxTANcfGDvS1vR+yaZDxNbZ/3A4+NzNF0M/Z7AHgEzUcx4yu2shrXXr3dhNpzb +crk9yoCnEAiP1i54euqaOOrp6O34cRCCBE18j+oEEIIfYdMRXorCPxHRzdHKlJ5K +OfRZAgMBAAGjVzBVMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFCIxr8jA3srbonwd +2AoxV5teDkuzMA4GA1UdDwEB/wQEAwIF4DAWBgNVHSUBAf8EDDAKBggrBgEFBQcD +AjANBgkqhkiG9w0BAQsFAAOCAgEAT4hUMxXeg6cqFyyg8TeStBI3fWtmVKahlsba +Nh1KlZe5ZVTwWKh6ULn2zvSqy0t28wpER8Ky2a/yBxsssvPKGQBUgUUmUOSy0Zzk +ICU+pDJrVtZ02vOPlrx23SpnE3EFs3yXMGO5B0RGScHG62YjHyBDPJH3Gun4CB2W +PpthtYIX2FN5g15T3r6UZy62w7SjUEu5z5Ke7IOiAcnXNOXtozv4J9v3bt2Kg7WB +YS80r6b6cyOhO57jobdRBcdsWWogBXBn+ciaL8z3gLNWPs6YooA/6/95Vq0uV8t3 +xfq0XH1dbcdZOnalSwNEyOgLKxQ/yggOd9ridk9e5cGBBIfw1v1N1qDkOWjcdEoR +qjAjR4pgUa+d92/HNLYG8SqVGqACjUUQM6tigw6tHUbeqpk1iI7vT8Cl6Er6bEYE +tMTWEcWAI7GsqJXl2SJPMsObjBg34aZJnU+xxedMDF+OYZXzYeYk2De7uhXUi3iu +46alockzYqOdN6vE99Y7757C1X3N62PnEUhZN0Ri9D7i1yjQ9t0CCucam6hcqcEH +fcDIsXTQz0l97iztkPhcLd3kzAg2pXopwuHkhd3Ih5So5/m1V0rjHVVtrbSkN8/u +JlHy0/tNsJ1OaJKRqd665M9IhaRrc9KP0lzHoA1ZpUsRKo/Be8gNUaw9EP1CMDny +kKizg2s= -----END CERTIFICATE----- diff --git a/testdata/x509/client2_key.pem b/testdata/x509/client2_key.pem index dc4cca65a312..a9563fbf1427 100644 --- a/testdata/x509/client2_key.pem +++ b/testdata/x509/client2_key.pem @@ -1,51 +1,51 @@ -----BEGIN RSA PRIVATE KEY----- -MIIJKAIBAAKCAgEA4Gypbbp4P23Um8SxAfY+Nyy1qSdGh42L7KrFDjAMhaueB14v -39pyGDMLHiUJtx1O/RBq0aDeZqc8Rd6bmqyZH10qP4Qo9edXaojo9Ih+/UDBwuZr -Q+HYX/MwBI0/nCTCC/oxojJjHef4XiIEZBOhRqPWX1dLkTfTDuTZXE0pHvMM4ZIq -J9q+4Y/adVPw0qBdSOyHdf0XvZkg9nC9o9iRuaKQ6yEOvxNyFKCh/r72eD6xKZi3 -9WbTJaymMl+TaomV14heAZFULA6XSoGzOISTQclKkCOPL+70ZxMbPcCpFDvA+agp -TkzCc1ApkYTvclGtrrOTrhiKpZ2SDvHZa4TUOnrrVu8/2LALgSBSpctkRCQhKwOl -PxGRaqnYblyu/KuT9x/wjxNuw+L1DoSDd1GSoI3wfWaRlPVqyw6n4bfYXuM69qBl -8uKTfGhJu+kP67a8xF7fS48pkE2ERm/Y9TZhpIRe75rmpPfby4UrbnissWLUNlTR -5eWP1j//+cyjmRSirfPhZwvWucNFImTYlknouflJc3iVs26+ft0WMoBWhBri17Gs -JJ/yO1ErN7GEGLbyfosCislPnsT+ajRm98vpolvtxSNuFBlmYLkqqhqUxqISeg/G -Eyobib6fpt9gYup9mWwOJNXNyAR35HEZOWFpQiDrlkwYrn3zY+9Xe+sB7JECAwEA -AQKCAgA4kiuDRWXaV00olsQnwnKcZeDE6umUcdG7rrBNiz8c0s3a/ZsDyoTIJNXA -m4V/axvmHqVOgkaNicpfsmV279sJVOq5aA8LLW2TpT9TpLSeEhzFjF+tlNh+F0cb -Xp+SNJHVgxPP1vO1LiwlTl3c/DXDILmA/vhFetTxBC7mXWzoKEwu8DFAKpvDMAfZ -W3dxIItjPnxG+a1qVZdBh9nF22mgaaIuIv8cm0I+gN9U374xQVxXJ+/3JBxFeufJ -+t2mFVh4JB/ONVwKXwMz/M24iXK1OpBZFR2a75kcAmzzfAUi3I0gYYtH+YFqn+Ja -lC/nmT82sn2ffQA2DyoqKjysJad5PWHByyepPGA6mkrAwaxn8YFsd0Yu14LaWCfO -5jKQzMvDhuAavAkaeT8EJnQdOeztXHYGV7S8rDQOgXM58W8e9+SchceJzkl1MYKf -99xXveelRaTaGOWBK1E6xPQP7iKJTeh1/Xjk0ylEnWPG5VvjcbNFwleDAnhyDTwB -OqcW2L3IV208MmDEmLuSBAFjHg8u5+/hLnsv+qozAX4yWhITZL67uBufVjKbhTi9 -viFUJ8/yGP9kIrJosQ4iDZgZv1juQLEhAw/W1eIV0gCxy/ZFfxAJXgKThZJWgSAI -FTNf3mKZOiUpuG5+Pe5fFtDa1/vmvQaE5y2lzh8ztLtFboaboQKCAQEA+a4nh2bD -WR6UC/3xQ22/Uwvntw91P18L+HyzNtgKCKKKVpwjWdaK/o9jdnRajK4s/hYKcIND -szaSjnD1vXWezw61aXZgOBai+xGdMWJFbTIRFfFcJqvFwN4cOmURX2NzLn7JPCp2 -y8HUdP0u55n0Ax9/qSkh4Eysxcy9+RMAcJ7LIsqSSlsSY9tQ78QS8ymJeePdf8xl -Ha3rlaGLpoLt/8gfYLjMfpyfUnuWrwRK79aBBKbkG7sdi5Cahnw1ZN1vxdOjpKcu -5/NhJ5OZxU9OSm91uzSkQFfsLe2t1JLnjuvcPASlhMIskhpGof8qCrjct1e7sYeo -UpyVknF7InNGMwKCAQEA5hrbpi7Nny2H4Uu7Z6aIUf3dIuPLl6f9VDkIqRon1HXx -4+1gQWhEwclB18FzFVDv4h4YAGv4upGHYo8DNl8GYrcIpZQ2dxz3QfTH5jl+tKmF -FfHIRKuBJVgXw+nVrE8HzF1M1UTCwCb8SnDg1dV8U5OfJy01LOEnp1sNW64T/pDy -unCnY2+k/ncqGmeWUKL4mbKN8GmfzIGMhwi8yiM3Cdbmk0kETDK/NIwgl+YLX6dt -lHe2g5OVoDgVatC8ViVmoQVmuuPASP1K4TPUAtRi0A0BYqPB2O/vFZ1f+yD1sJM7 -kILtz91DPB5v+7txwjD5S558TC1l8L9JCH12R7BWKwKCAQEAz4Z5RImdhM1tsCn6 -BlmJ1LToe7dVdL7DbF35d3RJorO22BYfK+Su0rbLrQE44gVDUE1xj+MKukJ5vfsV -xculm+RV1LqXbwchoB0b0pgjrIcYvGxIc7wCOjRisgafUfGPIu4uxNtmsiUBOdvW -yJmlv5LGwQt3JL+WOzHaFNQ+YV0a6mgE/9iCiI0Z0K/gMEwuACntSPPSd8C/Nzd2 -o4ff2eG0cugm0HXN1vjyXbXrsz1PL1an8oSsIfym83D50ERdSsiGE60Bx7j637JG -9UDdifDqohc3DmQF4obTHQSdgqV4AEq8aIQcF7PPUYaMoyzUB2/cicp/lWqgx3+b -IR8/EQKCAQBtZA9P7agrKEYUwSASooTkFb/vOkQrkN1KEOMhISIWSwv3w32jGqK1 -TaxTmc/QLm4cHRpj+PCCIXUvUbXBP2OVwlYGAXPzJH4XiPsPY/3sfTqbuBnxK2d2 -DW8e4CeIhvm6GhDQwqOjHeWKrib1AUzdnqxmv4MsFs33Lb4n+5Xdy6LZJ30sNINH -xfbqHpzDMPbmepAn3s7tNhlMiMbXge5Eazmqg2fbobRsksFb9S0rCDl7/31xB9R2 -GrNz2E/w1E759ctkxalACcpzTWRZBAcFyWkDL76UF1yd9fcPOBgVHamPhe7whsvT -5NRv5CisnQOnA20r+dkgno9lzd9RLW+JAoIBADJ0vUL2nJZkM6reh4+bDAoRDP3s -U6JNPAmkMvWsiMckm+WKUtUo84VDBSIKX897z5sZ1AfkWS8P9MqyiDbPiJCuuIkq -h9OJIHVEQ8NfmD/sl/3TE+ig0OzIbZUL3sssL1Iadkkn9hNnYIY1nt5QsKsWJ1m7 -u2+6DHTkj0TAM6SGt41TvRQyLS/fGomqmAkqYNuN3jdEGF5cFJoeyhOh/EoMP3RC -LabPAhwUZzIH+JO93Ws5nuKOTPnryDQOM4Ug09aPLaJW5GRmfKVie1iDV6sp7KBI -7OqHcuieCyxXHrFRESmxkMj87DaQ5mTo/q8qoZ1nOZ58vohAjbPvIaQ+vL8= +MIIJKQIBAAKCAgEAtvsosWC44GIQdHwUEj/QAWzZfezt83lkI6RN/imz0vABwFab +n/NSrO9kJK6L7iyveo3VvUP5dtzjEf35ZzF9t6gDbDh3YUoonSppKEMy7hdNB+P0 +HuDkZn4VR4/2tJyqAn5YV44zpCgCBGsUBLQAw/bVOkO57gBUqYPW2tfPd/UNpEG1 +xRrjOWTYnQYFAM/I8HjCivGFnQhb44vPZEiXwBYkEMUoyCxzETHbf20ZM5j0fqE8 +js/LaSiSePssE1zziV/Y9UXDk7K1BiCWa2QVCqe3JXG0RMA4vikU6ONnDowCdAEY +tLqeGJcbUuRm69YmfVPl7CFyLio1Wm//wu2s9zYW/uNvzojlcm904a+FgLojBL0+ +yD2gJhojPJqd2hKPxITBdpC4P3xE4FSzWtpqwt52Dd3jE15kJFGafMSZ74ikeb/B +LiFMzFAkdhV7A0Hrx+XoWjJECqhc3LnKLy4wkpuOpv0TueJAJBE729B5fROsHbe6 +hRdt1PP3m4yLDzLL/HPXX+Hn1PE31w6KoN/vi3KaAfSNPA7x6D9vMpTNNfFMA1x8 +YO9LW9H7JpkPE1tn/cDj43M0XQz9nsAeATNRzHjK7ayGtdevd2E2nNtyuT3KgKcQ +CI/WLnh66po46uno7fhxEIIETXyP6gQQgh9h0xFeisI/EdHN0cqUnko59FkCAwEA +AQKCAgEAtP73H42nEfyufiqFyA9q9x3ufMsyDFYVIdRSeYhSoeJaOSDyS2NqcjlR ++57UN0HoSfemZtKoHlUcHx3z54li65m72P55x7iNN/lNj0/5Pt25ioaHYUvfYSpy +bhkPVVRqLpE/XUwB9OzGIgyw/n33C+BKxplbfvrAw/TvQAWc6PFzDvkYjeGsxYbl +ZV0g8c6W2pb5CGsjWVN9YTVYbcAIqy67egMr9eVR5L5GemM2PH2dyuw+dJ1CfcBu +MlFxJa4aD9bJSsQ5Uw3AVlFBuPSEg8emN9mjESZ6ek80qbDWreL8QjcbcxntbDF8 +C6B11e48oFeu5MWopdWGdPC4Mt7a6Pjjy/ESGHcKqiDPP0VdcEgKpmowXI2CtXfz +k9bbIAoveMgFThX4eb/d5DzYXID9MkSd3GdZXMa2Z2xqX3/S4dujWKda0VlN61vd +3sX0Xd6Wno91vobjFx3tqhqumKpZ/1MjNDqzB6v0lRzxaiFPT5/h6hTIzGKslzvJ +H9bTUyoocXo6Xuskw5FHcM3VFriJljfFOi09eqVvldSvaBosYc6MIRuw8zuGuSon +q9ZBIYDgdnfuXhMh2cohEUOHoi692FgGsC651rl/bgHncx3q7IS995vz8NzmZF2V +dpN9q4v6nwBuePQs23s2MAEF5REyeOR/eA7gWbtGASnZMfmyk6kCggEBAOMGYgAZ +JMr3dY0bZsp2hdS2quHau2mUIEvV4FMvu/FLLiFq0JMHNe3vlmZrTLR/JW9TfK00 +ymPXEc/v01raJDeG2Y7I0086v65tDmdHyEE30MLEYNww2XsPqZIoacJNEvr/Jmki +O/nSaUszxJ4ygOGA6u7oJi1YJ+l2Oe8kQ6UMxoDHSDO0Y2Fjhg+TKCF0+a0Y9ddP +Q8k0B3MOXUcUuGk5ZtnXXJbDBq2Fvpf/pGsCp9twESyu/nbGSGKAClH3hfxXNU/C +jBUbX9Jyxgw5ZqWqRIt7O5NBliav3MClKKKVBYWQiju0SjV4GC2s7kF3lfohFq6R +ltGgn0pxaXsLqbMCggEBAM5Vubx62pQ9O36FPCp4yLCzMeGuuTA0Wq8LoC1OtBvq +OtHjKZdmx+Pe4W224iyceK1hEYNd18Byv1w/FSJPR5U9W+jk/GYTQ1WlMosAeGYG +fNvuLCJUDxO1cDimRIuCiQeAeYiAgCmyfmsdaUEiwMYsI22ITlSeKavIULiZDvc6 +JUQfDgfsmmD9ZtxVhwyGuLgqnHEOxXv4Cti511/iYbwM1NMB1tvmuDpjZAwpQMpl +/Fq4N/cNCe91/sAF/a3VXMZlxXey1kGmXLlCPFdzGGMGelI0v8cbB+dJ11NnZDC7 +EZPknj4jiEHVkN7/jl+WVk4zhfr8l85xh5Q+nP8/C8MCggEAPNUkA3S5WC2w8Qur +oorZ16LO7VAoMeVANjHsNz4uNTz48nllxFAFUmmFupH77s23ITqUyPDBXrlti3Nv +BgQ3+i0HNOx5Oty6KioM1v30Gg2zwczPS5FHZWNQA9sSY781W85s43UJ7ypDjqQj +hmRwBnz99uB8AmCB6VwFsB/ehGaE9lLv9PLcQmdhr+C1uylWEd0DWxthRZPMfzcV +JYvW0lNQTQUZSUifDHYvGRmmXApNIk7IO1n006zUDpjSqx4RaAmSPnoaATnhlkms +6e+joraaQWnXD+FeM6WiGHjpB4+4+A5ADDmGPQeeKvcQrLg3ltuw8TwP1sIcjN0Q +76izYwKCAQEAlYaQPCN3pTeelrhs+oZfQZYKjvb0oxc9pF6zbEH9ycD7cUDC0kIc +l2jcSore6t9VoKeYbm+iO4esX2gjo6J6SI+XvHW85ygMgtNdhlgH6D/JWgQGnbX2 +2xyAP71WLReiv/n9mMsulYkRjgRZU2eg9bvkzKqbwTyBDEj1HmFk9AqCGRS8MUfo +NGNOmFuuq4gx8tyGVHQU7xq4mYhLqOPAWeuei29oyiEv3rhKN3npxwMTVpbrj7A2 +Q/9pZrSwurnFKs1zxaOnGxo5VdPHMMRqptB58nrhg6N2HcloLrvdYmcefOOPPY64 +XqUrAD+IaILk9nTmIhXM2UFytB6P3XVNywKCAQBsOZZ7Bk3LEZHpqqlqOy7U9jjI +39tir93AEIf46i2Rn0YgynuTpsh458E4LEH88ZXJCDdfOtFPTEqa0wnm1DHhpai2 +qyiNeXWFpmhbsEgLR4RfiASVyl1W4febZT+JpcVkYtkMwro6u6Owy8L34SO+rPCb +X2IyPqQ1+lj/9ZvXU9AGaFgZNQ9bui3sK3ifvNYLGbPTBM939hNdOoI5mW44eEHQ +ZDBKjiMNnkWlNnFJk2DEyGVIQak7pVgSygX+RkMAP/OjDPNO3DyS81WXZKuxOlda +rWV0liu6hYAAz4Bq881oXzTviG24BgUjNJVq0qYtIzrsbW7fDYirkn5ap/7k -----END RSA PRIVATE KEY----- diff --git a/testdata/x509/client_ca_cert.pem b/testdata/x509/client_ca_cert.pem index ad6fea8ecfeb..026a4e478412 100644 --- a/testdata/x509/client_ca_cert.pem +++ b/testdata/x509/client_ca_cert.pem @@ -1,34 +1,35 @@ -----BEGIN CERTIFICATE----- -MIIF6jCCA9KgAwIBAgIJAKa2/29Hc+P6MA0GCSqGSIb3DQEBCwUAMFAxCzAJBgNV -BAYTAlVTMQswCQYDVQQIDAJDQTEMMAoGA1UEBwwDU1ZMMQ0wCwYDVQQKDARnUlBD -MRcwFQYDVQQDDA50ZXN0LWNsaWVudF9jYTAeFw0yMDA4MDQwMTU5NTdaFw0zMDA4 -MDIwMTU5NTdaMFAxCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEMMAoGA1UEBwwD -U1ZMMQ0wCwYDVQQKDARnUlBDMRcwFQYDVQQDDA50ZXN0LWNsaWVudF9jYTCCAiIw -DQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAL9LRGKPAszSvRzSKwgP6niaPy0w -wbSILjrp60WVHB9jjOSIvgCaTev9Tz/+zCaxCqM/hIrBNXI+ITZuzNUBx3+rz4Ns -VdYVhEsilc5gjl/dqsvD/FJdRKHKDrSzvKznwEs7KpGX1AdYoWBYZ8jNaQcDdopU -VhZdE/196akrTRejZQhnjNaaCXKCjrubfeFGpZ4hTsDHLjzuTYkiZ7m5q0Kdiri0 -9gKNdp6b5edyLuuMimEviEsZbYritZbwP1kwGiOMSQi2tzBGUcIANugqxMhSUrgy -JQ45Eew8mLnNqEOgk3nuWf4m0LPzTlJ/R70TmLIVyJrZ51GcLYmTZ/czsfkhXaPT -sTuBRgqFhJNb2ukjq8XPJH7O0wOhbUKT7MCRXSlFttUCIZ8aOmufv5mYLuaGx0sd -8uJEEMZHKDeMZOZNsyTZNaged77Onf+AoUkSH25aTdjU+bpUn/0CO2aJDqwp04Rq -7qOrtGQ76miNnw4Fe/eHJuUoqp8VH4dUmFO3vZ24N+kSzF5LDwEbgyybQN/cot0i -rjm8iqcimwS+BISEm7UvIeK0AEzXmxNC1mXEwvY0lkIci6TpH2Fy7OGaCu5MTru0 -XrOORWqxMLo65bTQ0ciUSxw8DartL4xobOW2UY+EUO6Da8yhVRbO59cC8dBbA9J4 -fH60efPhziFt4aKvAgMBAAGjgcYwgcMwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4E -FgQUaGCg8RtquvpSIbS9Va1yqdqyXuYwgYAGA1UdIwR5MHeAFGhgoPEbarr6UiG0 -vVWtcqnasl7moVSkUjBQMQswCQYDVQQGEwJVUzELMAkGA1UECAwCQ0ExDDAKBgNV -BAcMA1NWTDENMAsGA1UECgwEZ1JQQzEXMBUGA1UEAwwOdGVzdC1jbGllbnRfY2GC -CQCmtv9vR3Pj+jAOBgNVHQ8BAf8EBAMCAgQwDQYJKoZIhvcNAQELBQADggIBAEmD -XMO4s+WP86pM35pFMbrz3qsjiB6Cw3tqSoUaw0eIi88uWk3ismREV/24wY4j+10+ -5NACcPLykZLh0XpQjHMIZu4FEZSsQP0ExnHluaS/XaFf8hIy/qLFcm5x6wZ08AeU -M+daf9BmCSrjuW7u2bMxIrRLcnLMQG1kX3t3aEQLl/GA62g6Ll3MlHBGDILdvdNA -jIscctNhnrCPLBc+ykifa5NIBhz1PWU1RTr9JyNJwLaO2To9LJcpZKda2LJJ6xYQ -/lzPBg0aJgw9rOOgdenhb4ijQ5nMWZqCDZZFiKej3e6pj+M9E4a6OlelHiRPZT7j -q0bSoDDNTCviGlap/LDCBTvzyU/c8hgJ2XSUMfOL5RTXQTmqF7eQEMepmNl+J9HT -FYv80eOtk3O6rnIVHJ25zjLcLTD8iDzH3eX61bhMphI65jr4ltC6fGetXn9xINX4 -lpuxpMg5sRIYLl6lUdBcp1pMdsjEWUdiPcAxhjYqthb9MeSgmAG0cEJ+EbgGbiJA -m2DpQ8HkQjd5gc2mCs1X5HKiFWr3ERTeQwzBwUZmNaupfgbDWpKi8xrz91r3tLVN -eFjyd2z+0VtM82KP8D34ZVqssjp3jS8N9H1h3NoPqZPtFN3DjXfFV7BsfrcGR9CN -mwNfZlxB487I+gXYIwAG2Tp1UYNQ1JDDfkF39Uu5 +MIIGAjCCA+qgAwIBAgIUZzkKhtgm6Y3RaksChHMIJFKV+U4wDQYJKoZIhvcNAQEL +BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTAL +BgNVBAoMBGdSUEMxFzAVBgNVBAMMDnRlc3QtY2xpZW50X2NhMB4XDTIxMTIyMzE4 +NDI1MFoXDTMxMTIyMTE4NDI1MFowUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNB +MQwwCgYDVQQHDANTVkwxDTALBgNVBAoMBGdSUEMxFzAVBgNVBAMMDnRlc3QtY2xp +ZW50X2NhMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA1Rk7zsuwXn8r +KMHk+gmvaftFmlY+NHs1mKJPzyCGFnablnJtHU4hDpSvvNitoQ0OcurOo9V9ALlA +U2uw/1q6Yhg1Am4cXwSWHG0/GwCQAdPTVb7W1MiAd/IB5bx9xrwfjrpGLjVLS3/y +nOKP+kl1bf6WAcLEPClvH+kSG8xMwvg58ot7ipWQcWBTSuZLaz89d2yfxpvtwrvS +YDemY6f8Tkxil+kDjb2Jo/zdRDz8eIEOs1PcdztrdWWeQaYJVX6aEOHCfdVNOHw3 +jNQKyVREUgXjr/pkwo9fTnZjQdBUhZIo7NuPPG25t5qZK3dUDuLcVRQ5Vt0/45pZ +/HkZDCkxmSynZWz2gPClOHVPOG8Eqi0Mbd3XxQSsd1Go667oFotLvTuynbYhdh4s +xAJWXbFV26HgDXI5wXueXrs1n0stUlbD6KahfeoYBu+idX7gB4RftqhqlbIazu3y +hj22k8cMQEPkLhzmUwRt64juLA0+FRG0Hfr8vdZD+f91Qbv86Qw3c1/lckQIOlyI +MerljNbCbHJm9KOZGf1zizwvMVtVzuVtr6RY+Loov4gzhJ5kNSk/YDMQC42c2Yhz +Lr5y9EGe/cL8QXdKfjKNeJjCbzxTTFiVBq5XRKUgjz6ga+F7KGO7ayMBrexZ7+ap +z7ydlUYS+xp43hqdisAGmUMJdDVlHCMCAwEAAaOB0zCB0DAPBgNVHRMBAf8EBTAD +AQH/MB0GA1UdDgQWBBTq92tDG5TfVvTqbu1bA593K6aAwjCBjQYDVR0jBIGFMIGC +gBTq92tDG5TfVvTqbu1bA593K6aAwqFUpFIwUDELMAkGA1UEBhMCVVMxCzAJBgNV +BAgMAkNBMQwwCgYDVQQHDANTVkwxDTALBgNVBAoMBGdSUEMxFzAVBgNVBAMMDnRl +c3QtY2xpZW50X2NhghRnOQqG2CbpjdFqSwKEcwgkUpX5TjAOBgNVHQ8BAf8EBAMC +AgQwDQYJKoZIhvcNAQELBQADggIBAMHOXRUUq5vf9G2NvnAR1lb0fTKx4/6B9rhU +Nli9uIoWGQyMu8icEMistUp4AdHWdhutKX9NS0Fe3e5ef6qIYCng0gVBE3fTHJd4 +V8MhGtyaK0K/gpTrJdClwK/litRIEjCFwNYEK8vtuqNjR82d8IuFjnbinb+IGCH0 +sLRGvvZch+dwM5N9BVRq20M2FZhyI+fWZmt1ZiBwnfy3xM+enD2I+/LOUFoxAmGS +m2vnS+ULhq7fLaK6vgyUIGqRDQMxYEql9QGzRIspV9vVhRuOCmowlJbgCv++eOUG +FvjlAPlQRGJ+ShpXO5n2pEkdjIJOrLf4kyviLDHffIl5I80fRWzv7GJ1HP+Bb9qO +LZGaiO3SelPhvJGTSV5uSZpgkFsBbgdbbGI60W2QQIHEwG0HdjnNk17+TmVEUoCj +rWK/Kzw5py1Egtibju4CiJ8uIKeew+2pfdnnyHoCVwCfdACc4dwRpet6fQvkRcru +5PR5MzZqUI2+bjg/hJrHj7SVpxpjcr3OZdh05T+heCVuPp+9mHBmcxbeA8rkMZAq +vILLwgwEriSbKy9Y1GLs2oaPNaWEpN9Q6kZPUwtwlzjHG3OOtldeXPpMVpg6Sb0y +3NnRfvfV/g2gm68S21j6qhGM2aeQCdCu5insqnR8GS5/stmuyCNnlst24JBneE0i +louEQ0EV -----END CERTIFICATE----- diff --git a/testdata/x509/client_ca_key.pem b/testdata/x509/client_ca_key.pem index 3fcfdb7074e1..750e20bb0f2b 100644 --- a/testdata/x509/client_ca_key.pem +++ b/testdata/x509/client_ca_key.pem @@ -1,52 +1,52 @@ -----BEGIN PRIVATE KEY----- -MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQC/S0RijwLM0r0c -0isID+p4mj8tMMG0iC466etFlRwfY4zkiL4Amk3r/U8//swmsQqjP4SKwTVyPiE2 -bszVAcd/q8+DbFXWFYRLIpXOYI5f3arLw/xSXUShyg60s7ys58BLOyqRl9QHWKFg -WGfIzWkHA3aKVFYWXRP9fempK00Xo2UIZ4zWmglygo67m33hRqWeIU7Axy487k2J -Ime5uatCnYq4tPYCjXaem+Xnci7rjIphL4hLGW2K4rWW8D9ZMBojjEkItrcwRlHC -ADboKsTIUlK4MiUOORHsPJi5zahDoJN57ln+JtCz805Sf0e9E5iyFcia2edRnC2J -k2f3M7H5IV2j07E7gUYKhYSTW9rpI6vFzyR+ztMDoW1Ck+zAkV0pRbbVAiGfGjpr -n7+ZmC7mhsdLHfLiRBDGRyg3jGTmTbMk2TWoHne+zp3/gKFJEh9uWk3Y1Pm6VJ/9 -AjtmiQ6sKdOEau6jq7RkO+pojZ8OBXv3hyblKKqfFR+HVJhTt72duDfpEsxeSw8B -G4Msm0Df3KLdIq45vIqnIpsEvgSEhJu1LyHitABM15sTQtZlxML2NJZCHIuk6R9h -cuzhmgruTE67tF6zjkVqsTC6OuW00NHIlEscPA2q7S+MaGzltlGPhFDug2vMoVUW -zufXAvHQWwPSeHx+tHnz4c4hbeGirwIDAQABAoICAQC9otcLQazL8kpprOvd1TFj -F75zhTcySiJSYxzKYTR85YqB8BEztcRzoy2SSnyGCtJ53Xj+uOTL+U2hkZvbuiTU -qzVPmvFJBxGcDpAmBFCANtafpA2adT2Zih6kAt6TJjfaHLBpnvMhyTpJsbpJNWDe -BA/auBqTlvg/PziJbRTCz0dUWpsjD5c3/reSwmW7EvcSWQCiWZK78p3IyeO8GZTu -uBESZMrQ4v5p5DC5Ddf3yN5R0/YwROf0XCUamdajCu2Ouf6Y9dGKuNtKED5eUC++ -SuYYFhXoEKl04OmioH8jc6dfo+tw6XfSPOwzGly60xd3y+KPqF8J52K5VPkm9geC -NEttAEKEpwLX4cAsxzQ09WaL0fq+XSpwWZYuAJI4F8zPadckbzittkAFGnwH6t5N -ydaYoAcGxz9x97qbu2iS9SiN1cWQ+OSMF+o3o02WcLNcIBOVIKivV1FuLgQEPfXw -bi9egAOUI5TUvoVO8mG3Drk5+Ii6PPxEaCKfp6x0xXA+t8JrmOCsEoYRiPhCc65B -gHZC1+mgngYUs6PYmkPgTgBfYAe2wYpn7uaCEo06tNfe0kPqLzr2uMEKZNY1IfoM -5RMxic9qKac3Qp2Lf/XG/90L/wO+kVpv/HSWh8JAZXezYD9f+EhrDuYae8KlsKXE -Z+XGmMdgIarHLGnXoAqpeQKCAQEA3002LSywsvGM1KZz5tQoIyaor2kD8tW8vyS8 -7TlozM1TI58ALtDyV/LCrvS5jJEIbsdlrrOeBhQOS3RPjSQQdEQKfSly1TF9mhE2 -vDLznxFOQNpdkkzGwfLxI/5mMbeHN6960XAcfVD5QTDdpKPY+74uQU3HzQCx6Net -+UK3aT6CeIvgWn0xNnR5Fk2EnQHKUduqm0sRj5c2S8qUO6HxD/VPNRCT7G+faex9 -tP2VIHxwF4iH1WOmwQWxTLpy9wR7UYYxpFBvQN6gglHuMeY6tublTnvfhpMdZ5NU -Zd1Trzrh4w4sXRWStHkphJK9aQzHEclZq5ktvdJtFd/GGZfsrQKCAQEA2040LYoT -JcassmJs9UzgJeVkJIB61wmBc/qvqwKqy28niLubSYNaETO6cQzPrnlZjk6LKa+M -HumrlA3DjobbkmA1YI/OAhIHjGMEtaxsOTUz0rMR5RDOvRc3hXo2qKsXfDQGUtr+ -1DrCmnI/iVm8+F8HtV9tEHzrGEaCmeMLHQWCxveNoGDnZRCZds52ApoFxiLnVq3N -+ocQEsWwdOg+8ZdyfF7RqzW2e22WoCkTJYApGutDfu1eXHXlOeBrBNPiHMzK3pbA -n6+oqcxB23NRttNeUkge3UezjfQfuGqR7CLi0yF2L236MGBOGuXo4bGaUMgEz277 -ZBT7YfWhZpn8SwKCAQADqM5Ee0ECDbdTHM81bzChMtb82Om5pwsKzt1Rvekbwhmk -scxc+AugqVfLajNIPHA48IeYD1V9oAKD9gn/tCGY5iyN1IoPOFpolfOhrewUJUJ1 -CZ8S8LMpJoQRJPAjzHAo13VZzU6KNzN+gACB3DWIGpvDcjTeBS7lM/Oj7BX5YY7d -zt0EXpzZ2ZrKZMbRk9/u63ymQtqs0buQDmfTelnq+wgrRHRIIaQpJjkBKE6zU5a6 -rAAd3R40d5VqPnv31Fj5Awv5N2A7XeqfeBxBMRaxPKNxX9JP8EVBF0cAzFm8u2hM -QkUz2VCoKHwnsgfsmssAXZ5ck4wOWk5zV1F1xemZAoIBAAR+esVAIhpREvLo33C7 -bZB5Pe8djubfM/7rcTQg7t0SXw4HQixke7EEjVqJt6vMotAuvd1R0p5DjZeQHKTM -EK3UOOPMrp0OP4dZ9BvA98rIU1KLBt/Z01K+qg2bLomQT//klQiXokc5GQnPM4we -AahZUjAeT37aAHtT3pNGutCSb1aidg2GTtecWni7zGFLRLkFuBXno+PxZpvr3yzW -IYwT3W29B7Dpfd7TpRWNIe5PzQfXMF/mf1uHsvXXqnnD2ctbSwD6t+HN2Lf6DpNv -ron/lNw8zB0evgg3q3q8/FaJdHp9Ig3gxBK/tnoIohgV6qKjJq4ViSNI5sngHbmb -iDcCggEBAJ+Wg1Y3UnPShQjAAUyOeqfdLnb0h6ocz5Flog4I49023ro102xav/Rr -O6NzaH8nBHt4OKYWPgwa1ANZ1ujXfnqU531OlB7p8vllDcECSR9qnSE0vMO8hvbU -flREfjy2inQ9kVwCqLbYHh2XEYZ7sEwQ7p0dz1v9G1ytBslwyeC3h2aMIg4utT/k -73y0T5Nq0e6Mas5w0ZBemzKNHoKw7N05g2rrELL4hRfkGMrEIsSaANPDRM+4cI1k -a3CAv0mex+5XeBskUCtvU+xrCH6isDovDhCT/CSAjuEatezby6tLk8PeaH0uEaxr -MhPlrQvyfY9eITe9uSQtiTQRg+Z4U5E= +MIIJRAIBADANBgkqhkiG9w0BAQEFAASCCS4wggkqAgEAAoICAQDVGTvOy7Befyso +weT6Ca9p+0WaVj40ezWYok/PIIYWdpuWcm0dTiEOlK+82K2hDQ5y6s6j1X0AuUBT +a7D/WrpiGDUCbhxfBJYcbT8bAJAB09NVvtbUyIB38gHlvH3GvB+OukYuNUtLf/Kc +4o/6SXVt/pYBwsQ8KW8f6RIbzEzC+Dnyi3uKlZBxYFNK5ktrPz13bJ/Gm+3Cu9Jg +N6Zjp/xOTGKX6QONvYmj/N1EPPx4gQ6zU9x3O2t1ZZ5BpglVfpoQ4cJ91U04fDeM +1ArJVERSBeOv+mTCj19OdmNB0FSFkijs2488bbm3mpkrd1QO4txVFDlW3T/jmln8 +eRkMKTGZLKdlbPaA8KU4dU84bwSqLQxt3dfFBKx3UajrrugWi0u9O7KdtiF2HizE +AlZdsVXboeANcjnBe55euzWfSy1SVsPopqF96hgG76J1fuAHhF+2qGqVshrO7fKG +PbaTxwxAQ+QuHOZTBG3riO4sDT4VEbQd+vy91kP5/3VBu/zpDDdzX+VyRAg6XIgx +6uWM1sJscmb0o5kZ/XOLPC8xW1XO5W2vpFj4uii/iDOEnmQ1KT9gMxALjZzZiHMu +vnL0QZ79wvxBd0p+Mo14mMJvPFNMWJUGrldEpSCPPqBr4XsoY7trIwGt7Fnv5qnP +vJ2VRhL7GnjeGp2KwAaZQwl0NWUcIwIDAQABAoICAQCgH7bmG/4p84qdtJx3GaH6 +k/noD9fsHYzXZVds/zZiWLtuoArHk3aZezZWQ8asFqB9z1x4lSm5ynnAdVJpfmZA +4Ymrisu8xjh5ocliY9jR1radXqoU95g5CNtOIoWsOJ3J5MRpYlhyofDO3Btt6ZbY +kQ1sw0orHsNGih62Tpx7gIQicZbiOqJv3v6XcFbJfpqUS0X/uhk9U16wOADKL2cR ++qm3Fjs6XWq4k4A8D0tyzR8btu8ZlMeZTkNNdxLacCgaeVlorke5IvWm14pHYA96 +Rryg9hiSbaMi1SieQonQWFRyLkUCFj0P7pYbqC28hdEkCO9RCy0/vDLT2LbugWGn +JBdPIQqRYggGnEdRocvflx6f2Xdid9I4zrI2XWnorbypqVIdmhVivCCWK8PNqKE2 +YcRg8TRQHvyOXoR55Sodrxp6KycUc65nduGe5jsyjA9hlQ0Jfxhr4gv1LuytnVCx +Z2q2PFF/cznrSLlU8uBT9Lb2gGQXRyI/rxp6g6zwnTKLvMXsQqBrt2hzlE2vkdiz +I8EcLp99IT4CwSJAyGFdR/2ZmXg2Hy20GiGc5RisMIsXvj2gVt26XHrbb+LnYHMq +0+5d5QoMnTMZC+JczoDiw64vQlzJGcM1VWFDOMn9g7UALgofCQv9/nZrWLjw5hIB +FCli4JhtwjNUP5Gz2sqx4QKCAQEA/zY1Op8i0j2xaxeaysZg6midIKyvS0a+E+CJ +qfNE0qmwCEmG/T67+IvKIwXqfBGBrBntg3De4rTDxhTVL4S4Ue89WYzB/sf9J52e +6HEpBCujRJcdb8ouxSfDkpkMYXsVsVTIjckbQl731cm4qk7L8DS1GuE0oZs9I3kx +iQSzJ1+GzRotnzO6a11NU5n5N7NM+97x9z/BHFmd3fUOlUkYdpr67PVBNKRaj46k +Ifs0Og7cZNh2JCzhVOOYrf/x9DybjCJnPHLVuNMqYOHTNI/LgpFytM26+Lnmu1X3 +mcohVacygr55oZaCC0dz6CijEOpX9SL9sUZJb/tJ01Sxv/pgMQKCAQEA1cG6Oa7h +1Hl1f6Qqg9qiWNTHur6dBOw4lt/ej8vexB6y9c85WoMfXUBFpiQta9Nt+XCrC/UU +wY0EqdQir+Ydwg92ddX+1eKb7NmNLi/moUF+s0V+9uPvgGcz6xVlSMQKTgsYxZnZ +CE8ZSBTSD3dYyIadGQHaFoP4PsABzGfzYjWnQpvk4SZf055Qs0Gt6vIBlbs7R/O/ +wPajzFYf/o0mAaPAAdPpuK6C3Q1J4Gp5LKkHtzY79XFl336uXQV3AwxU29sAkmVS +/COFl772Ev55P5nV8NsEQaChoyuNHO11YQtZEyh7zTwx+R4SfnTivffVQNfusnIa +gKuj2Eoq24XgkwKCAQAeHRJY0XA1aIwnu8hLBu9mmWN4+IdSlY1WIRd9UzQau2UH +BU4FUcKySCRYz5jkfNhVK1YIPWg/Td8P32NsUPfCyzzs9Rvq6UQoyYN3n+qcEF4a +eM5DY5LzNobwJFj+o5xiqUNk34b05OnPcxb0GYoc1MtN2abxLrUfG2zJ4yEUk0P/ +rYgWke78Pi0ioTdz6Bc8XQkmCILLypNDHmhTGyXk0NKs5R+Fi6MX71fUnqSB+UDu +MVB3YkhQUO6yEVJGZGRiO6j8y/wF6/zDI8JdIF5+EJV9Wg0mziC4mCM4JU6bobfn +D3ygoXbEx/CYQztCgrRQO4m9wjJmITuL0SGMKonxAoIBAQCCelJ2S23GCK3UUB0z +hw16M8gHEbs++gJA9j4ggE1mYWbT7L4RpeBLR6Q8GfEv1EtY65E9J0iYLMAf+kGC +JXEct9uTaiC35i9PkCxBeTPKUvRH8a/ifJgBRP3IDbNZi3DO2q8wTwzPqZjBCxR+ +JFepb6INVbgN7lhl1UZDw2ApHp8OZaJ8XLQ5tHWGNh03QKn+/97buMnfu62YWSoG +c5ozfgUCGJyeAsgWrrndpqB4xmTTTOOkmqeYmPdOCLvwvGJAIZpjwj25cuVlD0ed +qH/SdtDEyKv8c1S3CSqF8dyodAjXTOrlCE1oxxZ64lZVpyYhAq3NdyD+Uccdi4hF +n57JAoIBAQDFuv2cmOl34qQz1vd+R0axxNSQEwYC9wug9WG8PEARRmk6vCIMy/AZ +XnHXZ4aV9ds9q0J4hGrx0C1vjGHSpBFR+kulI+KcIITtHLPTAgE7e1UXGATVz0+B +ES3qvzJ1eXhl7hrrFfYUdmPok7pJUhf37qqhKajcRfVtHccaB6J7v/sbAMCIXP1B +ij7EESZgM+NLwOQ/iAM2Bpuphn+gxdV2oqgorx3kLymzffhmn0oq6qfn818DH5ps +sPgi2bndSxG9jNtpCIPPC9ltMNwWxuB+3f+wd2pKIjBulJ9tb+72s/Vb4v7EOmJ/ +c/xqN5lRsGXGduw76PipTrLpy3/LkZDL -----END PRIVATE KEY----- diff --git a/testdata/x509/client_with_spiffe_cert.pem b/testdata/x509/client_with_spiffe_cert.pem new file mode 100644 index 000000000000..b982fcbe554f --- /dev/null +++ b/testdata/x509/client_with_spiffe_cert.pem @@ -0,0 +1,33 @@ +-----BEGIN CERTIFICATE----- +MIIFxzCCA6+gAwIBAgICA+gwDQYJKoZIhvcNAQELBQAwUDELMAkGA1UEBhMCVVMx +CzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTALBgNVBAoMBGdSUEMxFzAVBgNV +BAMMDnRlc3QtY2xpZW50X2NhMB4XDTIzMDUyMjA1MDA1NVoXDTMzMDUxOTA1MDA1 +NVowTjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTAL +BgNVBAoMBGdSUEMxFTATBgNVBAMMDHRlc3QtY2xpZW50MTCCAiIwDQYJKoZIhvcN +AQEBBQADggIPADCCAgoCggIBANXyLXGYzQFwLGwjzkeuo/y41voDH1Y9J+ee4qJU +OFuMKKXx5ai7n7dik4//J12OqJbbr416cFkKmcojwwbAdncXMV58EF82Bt8QRov0 +Vtoio/wxlyRlxDlVYwr56W+0EVP9Q+kzA/dTnMgOQYIeSix96CUQRy8XDu1YX3rk +fiUkND9xxuQw8OXi3LXguv/lilLVC/lXiXwa0RWEgMZZU2S1/lAElAG3aZuuWULG +K+PpKPuqkcptbUPCvNN1eUs9/D82aoFuqRCmpTC+7bUO+SJSggpUHcgTbXT9i6OO +9eR0ijcaQjtb0Y6ro+Cv60YOnlGC8It3KoY2SxioyqdceRUohqs4T4hjBEckzz11 +AC0Pj0Gp4NJPcOY68EjhD5rvncn76RRr3z2XZpd+2Nz+Fldxk/aaejfdgqs9lo1g +C+aP+nk9oqSpFAc+rpHsblLZehUur/FHhenn1pYWqkSJsAG0sFW4sDHATRIfva3c +HNHB5kBzruGymywBGO0xOw7+s5XzPiNnbXT5FBY1rKG7RwlqdtDh6LWJRHmEblWV +tPHNiY+rrStv0rN7Hk/YKcSXd5JiTjk3GXjO1YJJVEraEWHlxzdGy+xu/m0iJLed +pxZwuxxdZ/Q2+Ht+X9pO2DsW8BQFbddCwbooxKygwSlmHCN1gRSWqWMZY5nzsxxY +tic9AgMBAAGjgawwgakwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUyiXne0d3g9zN +gjhCdl2d9ykxIfgwDgYDVR0PAQH/BAQDAgXgMBYGA1UdJQEB/wQMMAoGCCsGAQUF +BwMCMDEGA1UdEQQqMCiGJnNwaWZmZTovL2Zvby5iYXIuY29tL2NsaWVudC93b3Jr +bG9hZC8xMB8GA1UdIwQYMBaAFOr3a0MblN9W9Opu7VsDn3crpoDCMA0GCSqGSIb3 +DQEBCwUAA4ICAQB3pt3mLXDDcReko9eTFahkNyU2zGP7CSi1RcgfP1aJDLBTjePb +JUhoY14tSpOGSliXWscXbNveW+Yk+tB411r8SJuXIkaYZM2BJQDWFzL7aLfAQSx5 +rf8tHVyKO89uBoQtgEnxZp9BFhBp/b2y5DLrZWjM6W9s21C9S9UIFjD8UwrKicNH +HGxIC6AZ6yc0x2Nsq/KW1IZ6HDueZRB4tud75wwkPVpS5fb+XqIJEBP7lgYrJjGp +aLLxV2vn1kX2/qbH31hhWVpNyPkpFsT+IbkPFLDyQoZKHbewD6M56+KBRTTENETQ +hFLgJB0HiICJ2I6cqw1UbDJMJFkcnThsuI8Wg9dxZ+OffYeZ5bnFCVIg0WUi9oMK +JDXZAqYDwBaQHyNszaYzZ5VE2Gd/K8PEDevW4RblI+vAOamIM5w1DjQHWf7n1byt +nGwnxt4IQ5vwlrdX3FDcEkhacHdcniX/FTpYrfOistPh+QpBAvA92DG1CbAf2nKY +yXLx+Ho7tUEBGioU4XvRHccwumfatf5z+JO/EvIi2yWd1tanl5J3o/sSs9ixJfx4 +aSuM+zAwf8EM+YGqYMCZ896+T6/r7NAg+YIDYu1K5b5QqYyPanqNqUf9VTR4oQ4v ++jdb5PkujXbjENvkAhNbUyUbQJ+IU0KHm3/sdhRPN5tuc9C+BTSQvlmKkw== +-----END CERTIFICATE----- diff --git a/testdata/x509/client_with_spiffe_key.pem b/testdata/x509/client_with_spiffe_key.pem new file mode 100644 index 000000000000..6adcdc3122c3 --- /dev/null +++ b/testdata/x509/client_with_spiffe_key.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDV8i1xmM0BcCxs +I85HrqP8uNb6Ax9WPSfnnuKiVDhbjCil8eWou5+3YpOP/yddjqiW26+NenBZCpnK +I8MGwHZ3FzFefBBfNgbfEEaL9FbaIqP8MZckZcQ5VWMK+elvtBFT/UPpMwP3U5zI +DkGCHkosfeglEEcvFw7tWF965H4lJDQ/ccbkMPDl4ty14Lr/5YpS1Qv5V4l8GtEV +hIDGWVNktf5QBJQBt2mbrllCxivj6Sj7qpHKbW1DwrzTdXlLPfw/NmqBbqkQpqUw +vu21DvkiUoIKVB3IE210/YujjvXkdIo3GkI7W9GOq6Pgr+tGDp5RgvCLdyqGNksY +qMqnXHkVKIarOE+IYwRHJM89dQAtD49BqeDST3DmOvBI4Q+a753J++kUa989l2aX +ftjc/hZXcZP2mno33YKrPZaNYAvmj/p5PaKkqRQHPq6R7G5S2XoVLq/xR4Xp59aW +FqpEibABtLBVuLAxwE0SH72t3BzRweZAc67hspssARjtMTsO/rOV8z4jZ210+RQW +Nayhu0cJanbQ4ei1iUR5hG5VlbTxzYmPq60rb9Kzex5P2CnEl3eSYk45Nxl4ztWC +SVRK2hFh5cc3Rsvsbv5tIiS3nacWcLscXWf0Nvh7fl/aTtg7FvAUBW3XQsG6KMSs +oMEpZhwjdYEUlqljGWOZ87McWLYnPQIDAQABAoICAAY5tM7QZm67R9+hrxfw4f6x +ljfSLXBB+U5JFkko8DbhvjEN9+PQCda5PJf9EbUsOIWjQNl6DZjZsR3rqnog0ZGn +kB0yuPs8RDjrbVIXOwu/5EurWb2KZIpSjL4+BWflsndiMD6x6FSjDzXXDFrv7LKc +u0uQzLF3F00avDSEP5NvGUIbWnE7Z1cZIdj9ABQAJuVAI8gOnwaIdTsODv02jjGp +BgxoBbKDFsSb7yb9QzuvhizEitd8FajaGsqAaZYh6JwiRjkb8jl0z+u6MoqJNACm +q/gG+JLg1deIpS6OM2OBbKAr2G+HvXJMVklsdQkl1b+DcuJsBkW/gLHn/3WdQDyq +t9sB8XrUx3S5dy6oroj9tcrwwlpUPbx3/37BX7OEn/NlIWZojI62hGexoFaggu3O +Jg0JJYH8THlQqs9G/oXmRTQKngse2FLEIPePie9vIIvokiQtG4T6miOK+6QmxYZq +H+KPT8AQG8j7AEexo4is1mEayapmTxftIYANOLFaT82BhsUOZksa698Sz7k1Cf/o +MSFn6CocGLflMEzdBqEq0wbBkdeuKUKlXG3ztXlqElN1xFdbzkaP/Tzl1ooq3kLR +0cLBCJNrHxTo1tuW4qTn+s4GLHpM4PE4YZgMehVWtyRFBb7mrSXsESw03POvulBx +65vA86DtCPm/jVuC5lQBAoIBAQD8IWDHYtQnvn/za6etc0fc7eTyq1jmoS/gh33y +eHaY6IccUN2LXCxgTJYwmfy57De58Im5AcOnkgGvw2Hw2i6+A5i4tKkyCa9awW4A +M20QOnyQpF+9uiIqGzI72skpfH20SvgTstTFtgGr3UBOqTfcApL+1X4guvGnY+Cx +uHUAPzbis9G3CNOWb4iiLhUcBnTDZyB3MPM4S1U8E5JLFo86+va6gbqdBP4ac+KH +08XDk/z6ohp9p796o6IiBQyZEsVaYLCrzjSOXeFfE5Fyj2z53oGlws+/PdhXKo02 +3++zRESiLVuGbCmAN17nKwDbZu9kFfGNP2WdwhJt9Yey91I9AoIBAQDZOsXWNkyP +zoDcSrvJznMPFcwQCbMNyU7A+axXpAsxDqn16AQj5/t1PWqufjRSdC7gVUWFcQ2K +ldUHkNyGtqHVCcNpqsMZJT51NlgTOl1A3GLnmm+tAiMCcr7aySeNnlj08fW278Ek +fnxpgUqGtXjTFpArULSFdZulXNPAP85ZDBburJtdhMfiT3GyQ1iRZcXkzsUVzNU1 +nGGk0jtCodlzQKiz3/aHO63G0GAjtdPuXpzGm7nBJSgLD0GabkCdkHDFULOaraYy +t1zsCsg7tQWa4KGRDNkcJKzoz3zf1sI4g87UJceGoXdB+mfluyKtnFhqjFalFW8Y +14Yb8YYdYHkBAoIBAC1pZaED7+poqWsSjNT02pC0WHRM4GpJxfHO9aRihhnsZ8l1 +1zFunJ+Lq9F9KsPiA/d9l5C2/KKF7b/WlSFoatrWkv9RqtfUXr0d8c4fdRljL2Rt +9sCZceXbmCSnt2u9fHaouh3yK9ige5SU+Swx1lnOLOOxWFJU2Ymot6PK8Wfl+uDC +OpeZA2MpG5b6bdrqXsWDIZnWOzh8eRGlBMh5e7rH0QCutQnrCEmDbd3BCvG7Cemq +oNLZD+fq6Rzvg+FePCWXHLsVHOo3how1XhEgPCSVKwzMFdcAMKMiiuTDWM0VEreT +K9T+TktFrdY9LJ5X3+5K9YLXVFohxmf/vT1CxpECggEBAIfegeVU+xgrYl/nAoPb +9A1oZcVWO78QvYhn4YrDmRhrApVDNGu86oPPEU3otBMqhjNcQmqPZpfa1W6xBa3g +x2H3hFkwLG0q5WDsx7PnGnK6JcaUyurcXkdmu8ceb/XdJ+i0+ioc1aJc1rYq3xFY +qiTlhPECvpaHE/4fDHa/sfHyZNmN7nNU3KzJYeTMyLXQgTF2vsC+6FBq6ovrzpMD +pn224I35NDorcqrapHdRgCgk10xGFK4g7mXUegT8lr+2m0JfEqdZm403MRCWQd1O +gR35CDUwYw9+RQQs2v8qVTqB/riklKK5lV0YISoInU0XcBncg0koGd/g1gneTDNN +pwECggEBAM4sDCCPplzbyd0yXLGo9P3RYIsNFnKnIm0YGRPrevBaiux3Qhr7Whpi +eV04BJ7Q58Z2WFzPFMhdXU45y4c6jIbmikdplEW1TASgXxOTvTfhg8P8ljdLPx+R +3CvQi4BPkJ3ZtYrHLKXKF/9aseyHLlSzuNUAJ6H0YxVi0tmzCFG82SWcFOzhR2Ec +cWDptGTRt9YY+Eo5rhPYbX/s8fCcW2u9QGnRnX35F8vJOp8Q7eCONIaN6faV4Yos +1wk6WXjZfDgEdjxmrnqXrgxdup82uD4Q1agmkxAjPl/9frLtHMW87Y0OixJb/Sve +eSCMKThlBQ57WubHTi2TbFBVKph/rP0= +-----END PRIVATE KEY----- diff --git a/testdata/x509/client_with_spiffe_openssl.cnf b/testdata/x509/client_with_spiffe_openssl.cnf new file mode 100644 index 000000000000..cf96f271d4a5 --- /dev/null +++ b/testdata/x509/client_with_spiffe_openssl.cnf @@ -0,0 +1,17 @@ +[req] +distinguished_name = req_distinguished_name +attributes = req_attributes + +[req_distinguished_name] + +[req_attributes] + +[test_client] +basicConstraints = critical,CA:FALSE +subjectKeyIdentifier = hash +keyUsage = critical,nonRepudiation,digitalSignature,keyEncipherment +extendedKeyUsage = critical,clientAuth +subjectAltName = @alt_names + +[alt_names] +URI = spiffe://foo.bar.com/client/workload/1 \ No newline at end of file diff --git a/testdata/x509/create.sh b/testdata/x509/create.sh index ecfcfd19f544..378bd10cf24f 100755 --- a/testdata/x509/create.sh +++ b/testdata/x509/create.sh @@ -9,7 +9,8 @@ openssl req -x509 \ -out server_ca_cert.pem \ -subj /C=US/ST=CA/L=SVL/O=gRPC/CN=test-server_ca/ \ -config ./openssl.cnf \ - -extensions test_ca + -extensions test_ca \ + -sha256 # Create the client CA certs. openssl req -x509 \ @@ -20,7 +21,8 @@ openssl req -x509 \ -out client_ca_cert.pem \ -subj /C=US/ST=CA/L=SVL/O=gRPC/CN=test-client_ca/ \ -config ./openssl.cnf \ - -extensions test_ca + -extensions test_ca \ + -sha256 # Generate two server certs. openssl genrsa -out server1_key.pem 4096 @@ -39,7 +41,8 @@ openssl x509 -req \ -set_serial 1000 \ -out server1_cert.pem \ -extfile ./openssl.cnf \ - -extensions test_server + -extensions test_server \ + -sha256 openssl verify -verbose -CAfile server_ca_cert.pem server1_cert.pem openssl genrsa -out server2_key.pem 4096 @@ -58,7 +61,8 @@ openssl x509 -req \ -set_serial 1000 \ -out server2_cert.pem \ -extfile ./openssl.cnf \ - -extensions test_server + -extensions test_server \ + -sha256 openssl verify -verbose -CAfile server_ca_cert.pem server2_cert.pem # Generate two client certs. @@ -78,7 +82,8 @@ openssl x509 -req \ -set_serial 1000 \ -out client1_cert.pem \ -extfile ./openssl.cnf \ - -extensions test_client + -extensions test_client \ + -sha256 openssl verify -verbose -CAfile client_ca_cert.pem client1_cert.pem openssl genrsa -out client2_key.pem 4096 @@ -97,8 +102,50 @@ openssl x509 -req \ -set_serial 1000 \ -out client2_cert.pem \ -extfile ./openssl.cnf \ - -extensions test_client + -extensions test_client \ + -sha256 openssl verify -verbose -CAfile client_ca_cert.pem client2_cert.pem +# Generate a cert with SPIFFE ID. +openssl req -x509 \ + -newkey rsa:4096 \ + -keyout spiffe_key.pem \ + -out spiffe_cert.pem \ + -nodes \ + -days 3650 \ + -subj /C=US/ST=CA/L=SVL/O=gRPC/CN=test-client1/ \ + -addext "subjectAltName = URI:spiffe://foo.bar.com/client/workload/1" \ + -sha256 + +# Generate a cert with SPIFFE ID and another SAN URI field(which doesn't meet SPIFFE specs). +openssl req -x509 \ + -newkey rsa:4096 \ + -keyout multiple_uri_key.pem \ + -out multiple_uri_cert.pem \ + -nodes \ + -days 3650 \ + -subj /C=US/ST=CA/L=SVL/O=gRPC/CN=test-client1/ \ + -addext "subjectAltName = URI:spiffe://foo.bar.com/client/workload/1, URI:https://bar.baz.com/client" \ + -sha256 + +# Generate a cert with SPIFFE ID using client_with_spiffe_openssl.cnf +openssl req -new \ + -key client_with_spiffe_key.pem \ + -out client_with_spiffe_csr.pem \ + -subj /C=US/ST=CA/L=SVL/O=gRPC/CN=test-client1/ \ + -config ./client_with_spiffe_openssl.cnf \ + -reqexts test_client +openssl x509 -req \ + -in client_with_spiffe_csr.pem \ + -CAkey client_ca_key.pem \ + -CA client_ca_cert.pem \ + -days 3650 \ + -set_serial 1000 \ + -out client_with_spiffe_cert.pem \ + -extfile ./client_with_spiffe_openssl.cnf \ + -extensions test_client \ + -sha256 +openssl verify -verbose -CAfile client_with_spiffe_cert.pem + # Cleanup the CSRs. rm *_csr.pem diff --git a/testdata/x509/multiple_uri_cert.pem b/testdata/x509/multiple_uri_cert.pem new file mode 100644 index 000000000000..210b844448cf --- /dev/null +++ b/testdata/x509/multiple_uri_cert.pem @@ -0,0 +1,34 @@ +-----BEGIN CERTIFICATE----- +MIIFzjCCA7agAwIBAgIUA0Tqj0ezdOI2R+W4smJis+1NRucwDQYJKoZIhvcNAQEL +BQAwTjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTAL +BgNVBAoMBGdSUEMxFTATBgNVBAMMDHRlc3QtY2xpZW50MTAeFw0yMTEyMjMxODQy +NTRaFw0zMTEyMjExODQyNTRaME4xCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEM +MAoGA1UEBwwDU1ZMMQ0wCwYDVQQKDARnUlBDMRUwEwYDVQQDDAx0ZXN0LWNsaWVu +dDEwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCnfQMbGQzzWl17NuJQ +annAeiYFX9mcyKi0ywS1BOsNpDn8SRgeW7Ymj8EqMYIUv0VK5QDpsJKUQ4ZDBE/f +drplyXhbR0/aMpAATxP4AjnzH0pw49aIe/n3KTdBZj/KF52qO7WG2rcJ5GSVr8Z6 +H8FeP02GRt2iOAbLqV5/k52gBEEzjSJ1Be1DiRAMOQL/Wahyo1XTfvB0UeG7nIFN +OFiTdQmgS0s6OUYWns8J2jcSO8XoSKCxuNz+ZzkisX7xicRFjkweLKNLuYmS4U1x +wd19JMizaCRoK8E6NnOGVrxP/r9am5ft3QgC6AIkzZvNcylcHkkhZo5++X5qC18u +mKjPgxCOzAop/pGeiItfkjnshccLPgoPBI1W/gO6puxDaTV3HFGZdy5rtV/6MxjE +byUf69aKaBEY9d+mIw0OB8TfyqzNSoU578rTbuEqwiG2f1IDe0KMB2xkikDXGXPz +YYRVdm4bmgVmY3fjpoiM06+aSN1IbYEnmvuB98z1nf8VyRdR39jVk80udfHHdQWu +xTBMEHBtAjT6HC0tKIkeGKB5kmozPIVL+6EbI0JyuRVB6wyCrmnpv0Mw1NzSF5NM +JiYD/5ScSKhnFogYEstq8Lgj5atyrqi703bNGDVGmbTqChBG133r30WNT9JJ4rzS +KznDJhoHOgQqagkzGgHsKeAmiwIDAQABo4GjMIGgMB0GA1UdDgQWBBRHg8AR1psw +kslOTh5a2nxGAmo33zAfBgNVHSMEGDAWgBRHg8AR1pswkslOTh5a2nxGAmo33zAP +BgNVHRMBAf8EBTADAQH/ME0GA1UdEQRGMESGJnNwaWZmZTovL2Zvby5iYXIuY29t +L2NsaWVudC93b3JrbG9hZC8xhhpodHRwczovL2Jhci5iYXouY29tL2NsaWVudDAN +BgkqhkiG9w0BAQsFAAOCAgEAOkL6WsETiUWJT2lhMXEHGpLwu1Q4nETr4O51+V7t +AJJd7oGS/QRL0K6YNDgNQW6GOUZptvTEOSAO2irNohP+0+ITZAClF46ggB0pRAeD +COWjnG9h1aonMtVlnswh2xVYfg4jd+qfQZ07jN9tATn5ZBpFpcaxvcyAYc/eq6x/ +DKf7HBBWq9XWyRxZJuPD9qhyGPDzI/E2yr2ahLJFSGMRbTDivDUbw0yHbzmYnY2g +uPrVAAD4DuKsJxyZrA2/Hs7ZspBMTyUjWj7KSw64AcDvFDQgPBXDfG4CMSRH3Eh5 +J2F48ej7T6J1+PbJ81ISifGjUZH50haskBG5TKQqRX65p5LIVrDThsEM+YpfEyOB +mD2ylbxNs/X3b9fk07iS2HirfKZ0cKSINZPU+hEroasqxCcAY0E28Kzw0SdAGCGf +iZNRT0mNVgTPg7Bnrb7JhCBrm0aid0/nYFX+fqeKuS2lcdAcx6U5EgH0KnHg+9/N +NbSv+RtRiGWv5RqWF/Pk4bdHPvlzp/qiFfX9dQIOBtrFph9XUt/bEf6hZgaMKvT1 +QbQuM+rmf2ghjbqpCRP9iZUYBzOOvDZ8IeugguDvyBgrGaUSpreMzMC52B0fp2jB +Ib89u6yiKNNZzBGGE0d9y2qsju7q3IoV+eUwqbCUvGvcal+gdAfhO7Pvr3dD40z+ +g58= +-----END CERTIFICATE----- diff --git a/testdata/x509/multiple_uri_key.pem b/testdata/x509/multiple_uri_key.pem new file mode 100644 index 000000000000..621c1b2c9a5a --- /dev/null +++ b/testdata/x509/multiple_uri_key.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJRAIBADANBgkqhkiG9w0BAQEFAASCCS4wggkqAgEAAoICAQCnfQMbGQzzWl17 +NuJQannAeiYFX9mcyKi0ywS1BOsNpDn8SRgeW7Ymj8EqMYIUv0VK5QDpsJKUQ4ZD +BE/fdrplyXhbR0/aMpAATxP4AjnzH0pw49aIe/n3KTdBZj/KF52qO7WG2rcJ5GSV +r8Z6H8FeP02GRt2iOAbLqV5/k52gBEEzjSJ1Be1DiRAMOQL/Wahyo1XTfvB0UeG7 +nIFNOFiTdQmgS0s6OUYWns8J2jcSO8XoSKCxuNz+ZzkisX7xicRFjkweLKNLuYmS +4U1xwd19JMizaCRoK8E6NnOGVrxP/r9am5ft3QgC6AIkzZvNcylcHkkhZo5++X5q +C18umKjPgxCOzAop/pGeiItfkjnshccLPgoPBI1W/gO6puxDaTV3HFGZdy5rtV/6 +MxjEbyUf69aKaBEY9d+mIw0OB8TfyqzNSoU578rTbuEqwiG2f1IDe0KMB2xkikDX +GXPzYYRVdm4bmgVmY3fjpoiM06+aSN1IbYEnmvuB98z1nf8VyRdR39jVk80udfHH +dQWuxTBMEHBtAjT6HC0tKIkeGKB5kmozPIVL+6EbI0JyuRVB6wyCrmnpv0Mw1NzS +F5NMJiYD/5ScSKhnFogYEstq8Lgj5atyrqi703bNGDVGmbTqChBG133r30WNT9JJ +4rzSKznDJhoHOgQqagkzGgHsKeAmiwIDAQABAoICAHkAXOUP1QZe65hfz2LPecRv +utY5KCsX4KI05eKtee9yDR5R5GXSVidHxgLon5TDlpkEFwO9uDf7DJ2QGPBVg1aU +FirDu1HlI5nFh6SuXxVhLtOeFtil0LIaibvq1fz30MUyu/OAQaqY4X4u7lI+bOHd +E/IFcouGtIogg4/hoof/aueGeDVZIc+fzwM1kQ/Pw12G2TOhyrAOk+mJqPST15I4 +hMrUerXGuPcQpnz0tMKsgk9NYSLkbmwxQNrqps5zfGPP6PgHwbWshlKiCOQ9bfnC +QGk0vNCxg7i9q/qK4SNd5Prd3AZRoD8RRLM4A+6K23+ctbK2uA3Ny+Fq88njKlkN +jYxHPlkZ+b6nGzYxwZ4pbVYR/rpmrKdnrns1t4l/9GCOwMhDZe1jnRZaimQOoQQs +8hHMwxDisqsOjzd5ozQU3dVgxmG/n0jGjjtBIVp8usGe//AqRmZ7SVRNrglgG6FI +vqYxwCvum+DEJ4X5ONDyyKddmccGkCpj1lX6xPBtEkp7VupKx4KHW6ufteQYSkdh +U80RrCPaIKoFm1y5Jes9vOtICtvRk4PVNfLXBBycn0WR6aUQEkFHDvSHFlCnROPD +UdABH1r3bSMruz5vQrdIA/6XGHXzjHmq1WN0pKynwberFwazAxDlD/1G1ZbPD4Dg ++Z9cpyZ7Tlxj+T4hSfExAoIBAQDT8GU76CpiHL1VkIA9Khk3j6Nglq8ALQjRUH2S +ReqcAhvpUiSr6G9lt9PWGm5iWv4qpRoln+HyBDXpWOxaTR/PNi07J0NeTGxk2+in +IYNMDg9IILDGlbv5tKen4yKD9Yb7VHD73DxhDbSs2U+o49eTdfXmlQfErd15alZy +m19Xl2r2jPtYF8diPEp2m2dUamArnA8Vd3jdFAxTsfP8eNeAcCn6R3jEfIFyMm02 +X2H4iGO4Ec1ykVdiFxiemElLjm3Z7vVek88Md0KgjuAxhv5AuogRYtgMIMlhNhco +hfNgXzVtvxuDFJ3J2rlA20T0htSz6xsy9ZVUYRWkDIf0tPRJAoIBAQDKTufkXmcC +wXhkjnUMmYXqnmul6CPeYbXyfX1CZhtAlZSXtzAJ/2e7eRlqb7iFJuYfTdpkbN6i +pMrSb5wfcPtl8RRC+MERMCbe/LB4DND70QSLy/u8mWIsjbkdIT6SCi8rlNzfaL7U +cOb4uzmuXOhZHvdw8q9YTjXlT/EjKoMa2a+RggVLnICTm5sG6tsfkgiH5+DSsFCL +kKQt9Gtc66Q8HfyJW4ljK2JAOjYR7w3+bDXsodrxUQrl5maMALeBdkenc9uzwsmE +2an/R0NQToJJku+gRKVbJxN9jyMEVMygmygxfmW5qUyFZ8W2dd1Rkcr9jzNdVPlT +a6KgEGHPP5wzAoIBAQCqma3DpUTIqT227JRtp7+Yu+TVUTYZPCcG5nXOEr2hSwlQ +rTCbuIRDKtA4XhpQzdIeXbxIYQStnboP1eabYc2jLIcIQLi35WizX1lNf2qDBCZE +9xuVHt6rSEJUoD8eXbuEABraghOQREoVgO/gkVbsel2weHJCXXoTzAc+RddfWKFf +SWjhJnL2nnWKN9nbV62GLR7vNrZxrzuk+2/c4SEHYEJKFtIdx+MjG3hR9kGUn6U1 +fA8Wk+v1J4ZH02ncig/fB703nl9iN3XIbHoHJBTx4bS52gjy6klwGOxXUEvyXXFS +oCzzPNsuqwPIMzi0ZPw+v5erU4ga3fNflD60Oh0RAoIBAQCXV84MPj7rhdZNy3Bu +246eBKNdOrtSimA1poEFIiNy/jNqB+WNJR7x1VcZE7jDC2WNt40QIY2vuH3uTQZL +UxcOnPneW/76n74EhJ5zQIs6RpQTDKcm4Mvbrq3zx8HqOGovPS66hr5zaH6xRkaR +VPmQaiULvtFDy0ZwZIxtFUl81aqMvOq/NLXPNtITq6/+/x0YpnO+yZ2Hus3Hfxiu +K63yNzCLhQnTQUo/6Aw5AE/ErCju++oxKsJvWBwQ0hx1YgmakIakBK0CkF6nFSWb +NxAqgBx5FcFp3mKrRGAaxmFKKKg51me9K5SOHCKBK81ETz++zdjMElxudo/zFC5H +fzuXAoIBAQDTFfoLQ2XC0PI5x77zpse3zOvIEvsh3e3tLUhOHLoLTUqX0Nvp/7m7 +ohTHrPU4lf4ERInL4Kz7y2iI2yjRiKD+ARYHDYVx3QgttNeUFnsODnseJTnBcgFB +EWlThVuxbvEzJAZXNVtKTHmCLKFHqc9epyfdI59uD5lXQjvNp4uNEPiEE1AX0d8d +0OuFfQ6bSK1rZNv6IkRPTl+LCWuyHvZNWO1WZ4IUCg4XLr1IXtQaRIMRDxyxbHwc +6vwk0JtRxCt6wvIb/YWUK6qjjbfcfH70iDComZRTyXLGrr8w+vtmqvD43MUGIEmq +XHot6Ki4FhjC4ks5oT9m2q6TerUyIg8Z +-----END PRIVATE KEY----- diff --git a/testdata/x509/server1_cert.pem b/testdata/x509/server1_cert.pem index 3e48a52fd108..ed6bc02c4ce9 100644 --- a/testdata/x509/server1_cert.pem +++ b/testdata/x509/server1_cert.pem @@ -1,32 +1,32 @@ -----BEGIN CERTIFICATE----- -MIIFeDCCA2CgAwIBAgICA+gwDQYJKoZIhvcNAQEFBQAwUDELMAkGA1UEBhMCVVMx +MIIFeDCCA2CgAwIBAgICA+gwDQYJKoZIhvcNAQELBQAwUDELMAkGA1UEBhMCVVMx CzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTALBgNVBAoMBGdSUEMxFzAVBgNV -BAMMDnRlc3Qtc2VydmVyX2NhMB4XDTIwMDgwNDAxNTk1OFoXDTMwMDgwMjAxNTk1 -OFowTjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTAL +BAMMDnRlc3Qtc2VydmVyX2NhMB4XDTIxMTIyMzE4NDI1MFoXDTMxMTIyMTE4NDI1 +MFowTjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTAL BgNVBAoMBGdSUEMxFTATBgNVBAMMDHRlc3Qtc2VydmVyMTCCAiIwDQYJKoZIhvcN -AQEBBQADggIPADCCAgoCggIBAKonkszKvSg1IUvpfW3PAeDPLgLrXboOWJCXv3RD -5q6vf29+IBCaljSJmU6T7SplokUML5ZkY6adjX6awG+LH3tOMg9zvXpHuSPRpFUk -2oLFtaWuzJ+NC5HIM0wWDvdZ6KQsiPFbNxk2Rhkk+QKsiiptZy2yf/AbDY0sVieZ -BJZJ+os+BdFIk7+XUgDutPdSAutTANhrGycYa4iYAfDGQApz3sndSSsM2KVc0w5F -gW6w2UBC4ggc1ZaWdbVtkYo+0dCsrl1J7WUNsz8v8mjGsvm9eFuJjKFBiDhCF+xg -4Xzu1Wz7zV97994la/xMImQR4QDdky9IgKcJMVUGua6U0GE5lmt2wnd3aAI228Vm -6SnK7kKvnD8vRUyM9ByeRoMlrAuYb0AjnVBr/MTFbOaii6w2v3RjU0j6YFzp8+67 -ihOW9nkb1ayqSXD3T4QUD0p75Ne7/zz1r2amIh9pmSJlugLexVDpb86vXg9RnXjb -Zn2HTEkXsL5eHUIlQzuhK+gdmj+MLGf/Yzp3fdaJsA0cJfMjj5Ubb2gR4VwzrHy9 -AD2Kjjzs06pTtpULChwpr9IBTLEsZfw/4uW4II4pfe6Rwn4bGHFifjx0+3svlsSo -jdHcXEMHvdRPhWGUZ0rne+IK6Qxgb3OMZu7a04vV0RqvgovxM6hre3e0UzBJG45Y -qlQjAgMBAAGjXjBcMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFFL5HUzehgKNfgdz -4nuw5fru5OTPMA4GA1UdDwEB/wQEAwIDqDAdBgNVHREEFjAUghIqLnRlc3QuZXhh -bXBsZS5jb20wDQYJKoZIhvcNAQEFBQADggIBAHMPYTF4StfSx9869EoitlEi7Oz2 -YTOForDbsY9i0VnIamhIi9CpjekAGLo8SVojeAk7UV3ayiu0hEMAHJWbicgWTwWM -JvZWWfrIk/2WYyBWWTa711DuW26cvtbSebFzXsovNeTqMICiTeYbvOAK826UdH/o -OqNiHL+UO5xR1Xmqa2hKmLSl5J1n+zgm94l6SROzc9c5YDzn03U+8dlhoyXCwlTv -JRprOD+lupccxcKj5Tfh9/G6PjKsgxW+DZ+rvQV5f/l7c4m/bBrgS8tru4t2Xip0 -NhQW4qHnL0wXdTjaOG/1liLppjcp7SsP+vKF4shUvp+P8NQuAswBp/QtqUse5EYl -EUARWrjEpV4OHSKThkMackMg5E32keiOvQE6iICxtU+m2V+C3xXM3G2cGlDDx5Ob -tan0c9fZXoygrN2mc94GPogfwFGxwivajvvJIs/bsB3RkcIuLbi2UB76Wwoq+ZvH -15xxNZI1rpaDhjEuqwbSGPMPVpFtF5VERgYQ9LaDgj7yorwSQ1YLY8R1y0vSiAR2 -2YeOaBH1ZLPF9v9os1iK4TIC8XQfPv7ll2WdDwfbe2ux5GVbDBD4bPhP9s3F4a+f -oPhikWsUY4eN5CfS76x6xL0L60TL1AlWLlwuubTxpvNhv3GSyxjfunjcGiXDml20 -6S80qO4hepxzzjol +AQEBBQADggIPADCCAgoCggIBAKtk472NGPcQhDL9U6wsYWGOachAw5XX/a7lUBh/ +yowV+qD/SRCbspeBfiNdMNoXh/LPgyePWhAhskT2XaSJZ5cYD6VpI9Q55lFnFzR7 +Q1bw7BLaD2q83BJkrUSGyDnxH+LdQc2+Gq7rj1PIpIDBaJDtdd8U9bcpP6rH+S9Q +yGQw5OniPCCsUrnx8ym/3lAhKdn2OWXLq+F1avim8AN5dQj0fvAI2kMQdswKgY2M +bs5E932WPtjwLbe80A7RtHPIrqvsdVIoaZav7g3liKekisBuJGLMtTX3hBct5an2 +eKu3Q901bEQXeMWrToekc+DnUsmQ5TwkXatWiE+/sMWg80KNyWt8rulI4ATF8go8 +7Jl8duyb/jvULXjTRdDae6w34gNZjq9jZH2qSVIiLV3Jy0GadyRVJDVyE8Lz8EiI +XkbhgbjL8fpNG8cjN+58sK3TNDuP480A/Pi/9I1BoPYTSPCD6H1KPQJwF2GZVmgj +epF328/RGjl0bfaY458RRYZafydblBpIhDsLBBRmDMkFh4SghAgOwoQBjsEZpCmF +efzzPmJybfloBdmBiqfrEXP8t3J4jBzP5+qhYZRxHik0ignOWwyDtQQSUa2JTJoE +/ET8bkO88XLL7hkAlF+eLVV8ao4oXRh5yjf1c4PvJ/Zfr80mYJYOvOlA8Me9/+A7 +jZr3AgMBAAGjXjBcMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFJmANOTd7MASs5/K +mnBYcpvzmgLcMA4GA1UdDwEB/wQEAwIDqDAdBgNVHREEFjAUghIqLnRlc3QuZXhh +bXBsZS5jb20wDQYJKoZIhvcNAQELBQADggIBAFH3XxzcRkP5053jgcyV1/L3hNY3 +gAKxdDjYSw9uk2saJXz4gjfvc8xyIsWv35QU2yzzvN3xaGfDP+qVh1fNmC/7DdUF +LJzYTJb/wm4atVM6oYzdBhu/b3NqtMPb06NFKqyEX4SSN2QjeUVUF1QgAkjmsCiE +79NOuoCO0aWcxgdKd18Wl9MLtG/PtlCMLRcPlx4FX6OYLcOeqFEtcOQXKYDWej0r +9m8JQ2DAGRSa3AOUYskN12GacEmchC86cnlMsL2AycnX1YzOBawgp4KKSur40iPg +S1+8LRjZA5Dz88+a+DXb619ckABO7v8b0AVlbkVVmaXNnhBEU4vqdfQa4BygaGGl +BG8hYYMoNBHpNDr8bBWwwl/WVpGUIMHpOnIJnG5gAGrNiAxH1/6DYFC+cXEOH5J2 +NpkJ5O3Jm9a1xtwBs3tSp8GEORqVrpIjiK+bUWQacss9nsyyO6Oo0S33javSR8AN +nJrGHBE01/QytEpJ53d3N0btZrByhiFZkh8BG4NdhXZsAaQjVy6EEHxfJBsfl8Z6 +UGX4T/TTkASNDWA4B+/nRD/BxrcSegDb8fE34GY9M0IWgQtmMIdR49bOxygzYMFK +lrh+dwGqZ9/xqJ3ro7sYphhJ+Gk5YL5lkZygF2/F9GJY3zOcKrFUalfJgaqKOaTO +6zW3ZjSyDhQoFNR1 -----END CERTIFICATE----- diff --git a/testdata/x509/server1_key.pem b/testdata/x509/server1_key.pem index e71ad0ac9753..5e5331ab5d98 100644 --- a/testdata/x509/server1_key.pem +++ b/testdata/x509/server1_key.pem @@ -1,51 +1,51 @@ -----BEGIN RSA PRIVATE KEY----- -MIIJKQIBAAKCAgEAqieSzMq9KDUhS+l9bc8B4M8uAutdug5YkJe/dEPmrq9/b34g -EJqWNImZTpPtKmWiRQwvlmRjpp2NfprAb4sfe04yD3O9eke5I9GkVSTagsW1pa7M -n40LkcgzTBYO91nopCyI8Vs3GTZGGST5AqyKKm1nLbJ/8BsNjSxWJ5kElkn6iz4F -0UiTv5dSAO6091IC61MA2GsbJxhriJgB8MZACnPeyd1JKwzYpVzTDkWBbrDZQELi -CBzVlpZ1tW2Rij7R0KyuXUntZQ2zPy/yaMay+b14W4mMoUGIOEIX7GDhfO7VbPvN -X3v33iVr/EwiZBHhAN2TL0iApwkxVQa5rpTQYTmWa3bCd3doAjbbxWbpKcruQq+c -Py9FTIz0HJ5GgyWsC5hvQCOdUGv8xMVs5qKLrDa/dGNTSPpgXOnz7ruKE5b2eRvV -rKpJcPdPhBQPSnvk17v/PPWvZqYiH2mZImW6At7FUOlvzq9eD1GdeNtmfYdMSRew -vl4dQiVDO6Er6B2aP4wsZ/9jOnd91omwDRwl8yOPlRtvaBHhXDOsfL0APYqOPOzT -qlO2lQsKHCmv0gFMsSxl/D/i5bggjil97pHCfhsYcWJ+PHT7ey+WxKiN0dxcQwe9 -1E+FYZRnSud74grpDGBvc4xm7trTi9XRGq+Ci/EzqGt7d7RTMEkbjliqVCMCAwEA -AQKCAgEAjU6UEVMFSBDnd/2OVtUlQCeOlIoWql8jmeEL9Gg3eTbx5AugYWmf+D2V -fbZHrX/+BM2b74+rWkFZspyd14R4PpSv6jk6UASkcmS1zqfud8/tjIzgDli6FPVn -9HYVM8IM+9qoV5hi56M1D8iuq1PS4m081Kx6p1IwLN93JSdksdL6KQz3E9jsKp5m -UbPrwcDv/7JM723zfMJA+40Rf32EzalwicAl9YSTnrC57g428VAY+88Pm6EmmAqX -8nXt+hs1b9EYdQziA5wfEgiljfIFzHVXMN3IVlrv35iz+XBzkqddw0ZSRkvTiz8U -sNAhd22JqIhapVfWz+FIgM43Ag9ABUMNWoQlaT0+2KlhkL+cZ6J1nfpMTBEIatz0 -A/l4TGcvdDhREODrS5jrxwJNx/LMRENtFFnRzAPzX4RdkFvi8SOioAWRBvs1TZFo -ZLq2bzDOzDjs+EPQVx0SmjZEiBRhI6nC8Way00IdQi3T546r6qTKfPmXgjl5/fVO -J4adGVbEUnI/7+fqL2N82WVr+Le585EFP/6IL5FO++sAIGDqAOzEQhyRaLhmnz+D -GboeS/Tac9XdymFbrEvEMB4EFS3nsZHTeahfiqVd/SuXFDTHZ6kiqXweuhfsP1uW -7tGlnqtn+3zmLO6XRENPVvmjn7DhU255yjiKFdUqkajcoOYyWPECggEBANuYk+sr -UTScvJoh/VRHuqd9NkVVIoqfoTN61x6V1OuNNcmjMWsOIsH+n4SifLlUW6xCKaSK -8x8RJYfE9bnObv/NqM4DMhuaNd52bPKFi8IBbHSZpuRE/UEyJhMDpoto04H1GXx4 -1S49tndiNxQOv1/VojB4BH7kapY0yp30drK1CrocGN+YOUddxI9lOQpgt2AyoXVk -ehdyamK4uzQmkMyyGQljrV5EQbmyPCqZ1l/d0MJ9DixOBxnPDR9Ov9qrG4Dy6S/k -cH8PythqHTGTdlXgsBJaWEl2PyQupo3OhfiCV+79B9uxPfKvk5CIMVbnYxKgu+ly -RKSTSX+GHVgNwicCggEBAMZcwQIAA+I39sTRg/Vn/MxmUBAu3h2+oJcuZ3FQh4v5 -SL80BWEsooK9Oe4MzxyWkU+8FieFu5G6iXaSx8f3Wv6j90IzA3g6Xr9M5xBm5qUN -IqzF+hUZuKAEMY1NcPlFTa2NlrkT8JdfQvJ+D5QrcBIMFmg9cKG5x9yD7MfHTJkf -ztMDFOwP3n7ahKRBowfe7/unAEFf6hYFtYjV+bqMDmBFVmk2CIVtjFgO9BNBQ/LB -zGcnwo2VigWBIjRDF5BgV0v+2g0PZGaxJ362RigZjzJojx3gYj6kaZYX8yb6ttGo -RPGt1A9woz6m0G0fLLMlce1dpbBAna14UVY7AEVt56UCggEAVvii/Oz3CINbHyB/ -GLYf8t3gdK03NPfr/FuWf4KQBYqz1txPYjsDARo7S2ifRTdn51186LIvgApmdtNH -DwP3alClnpIdclktJKJ6m8LQi1HNBpEkTBwWwY9/DODRQT2PJ1VPdsDUja/baIT5 -k3QTz3zo85FVFnyYyky2QsDjkfup9/PQ1h2P8fftNW29naKYff0PfVMCF+80u0y2 -t/zeNHQE/nb/3unhrg4tTiIHiYhsedrVli6BGXOrms6xpYVHK1cJi/JJq8kxaWz9 -ivkAURrgISSu+sleUJI5XMiCvt3AveJxDk2wX0Gyi/eksuqJjoMiaV7cWOIMpfkT -/h/U2QKCAQAFirvduXBiVpvvXccpCRG4CDe+bADKpfPIpYRAVzaiQ4GzzdlEoMGd -k3nV28fBjbdbme6ohgT6ilKi3HD2dkO1j5Et6Uz0g/T3tUdTXvycqeRJHXLiOgi9 -d8CGqR456KTF74nBe/whzoiJS9pVkm0cI/hQSz8lVZJu58SqxDewo4HcxV5FRiA6 -PRKtoCPU6Xac+kp4iRx6JwiuXQQQIS+ZovZKFDdiuu/L2gcZrp4eXym9zA+UcxQb -GUOCYEl9QCPQPLuM19w/Pj3TPXZyUlx81Q0Cka1NALzuc5bYhPKsot3iPrAJCmWV -L4XtNozCKI6pSg+CABwnp4/mL9nPFsX9AoIBAQDHiDhG9jtBdgtAEog6oL2Z98qR -u5+nONtLQ61I5R22eZYOgWfxnz08fTtpaHaVWNLNzF0ApyxjxD+zkFHcMJDUuHkR -O0yxUbCaof7u8EFtq8P9ux4xjtCnZW+9da0Y07zBrcXTsHYnAOiqNbtvVYd6RPiW -AaE61hgvj1c9/BQh2lUcroQx+yJI8uAAQrfYtXzm90rb6qk6rWy4li2ybMjB+LmP -cIQIXIUzdwE5uhBnwIre74cIZRXFJBqFY01+mT8ShPUWJkpOe0Fojrkl633TUuNf -9thZ++Fjvs4s7alFH5Hc7Ulk4v/O1+owdjqERd8zlu7+568C9s50CGwFnH0d +MIIJKQIBAAKCAgEAq2TjvY0Y9xCEMv1TrCxhYY5pyEDDldf9ruVQGH/KjBX6oP9J +EJuyl4F+I10w2heH8s+DJ49aECGyRPZdpIlnlxgPpWkj1DnmUWcXNHtDVvDsEtoP +arzcEmStRIbIOfEf4t1Bzb4aruuPU8ikgMFokO113xT1tyk/qsf5L1DIZDDk6eI8 +IKxSufHzKb/eUCEp2fY5Zcur4XVq+KbwA3l1CPR+8AjaQxB2zAqBjYxuzkT3fZY+ +2PAtt7zQDtG0c8iuq+x1Uihplq/uDeWIp6SKwG4kYsy1NfeEFy3lqfZ4q7dD3TVs +RBd4xatOh6Rz4OdSyZDlPCRdq1aIT7+wxaDzQo3Ja3yu6UjgBMXyCjzsmXx27Jv+ +O9QteNNF0Np7rDfiA1mOr2NkfapJUiItXcnLQZp3JFUkNXITwvPwSIheRuGBuMvx ++k0bxyM37nywrdM0O4/jzQD8+L/0jUGg9hNI8IPofUo9AnAXYZlWaCN6kXfbz9Ea +OXRt9pjjnxFFhlp/J1uUGkiEOwsEFGYMyQWHhKCECA7ChAGOwRmkKYV5/PM+YnJt ++WgF2YGKp+sRc/y3cniMHM/n6qFhlHEeKTSKCc5bDIO1BBJRrYlMmgT8RPxuQ7zx +csvuGQCUX54tVXxqjihdGHnKN/Vzg+8n9l+vzSZglg686UDwx73/4DuNmvcCAwEA +AQKCAgEAknXlUv42vjGD9pqZnMBT+uyaooANYoevBXx5ZGYXbHv/rwJXqnSSOXtz +kb651zRSfQAswGp0eOKClwG8ZbTxK6FpBV2CO4G6ugcRQkyu76Vy5m0mzXxTxvf3 +RF60zSaqq8+MwsbXwHAVC3CielBMDcSNfDNKAdmiyUqXOoKaq1tI0j/8R6NaEgGa +XCvUSr78J4CL7dwMpd4TqiXlZeKtSxi7PF0kPjjce2Hi8VV2/pbaspvoWrNrLd6Q +IIm83VA5SzsFyk40ZIs0LvXdP/yQgP3d4/uwQkyfuLsEzaeL2JkDyg0z1kAEeU35 +DlpOl3q1OP+zlCAzVw3b7+ILqeXu2KOLPBCoRTo2wWNKutxcExmA0oIfMSgyb/P9 +36bNsdmT6b/6C9ZeJpsXZWedx7bmlEYg1patfk1CgD6WMynsl+fQ34F658v/JWzY +b8JpqAG+2ov4j4EyRnwK267/6u/Yw4CvQw+8giKIDMv6W81b0GWpCBT7PkTKgUaW +1Hq46z6xZ/NttiTU6qMsgUskFheU7IuO1KHl1kpLBERio9S87ZxkjLGfvxl4+sS7 +7OdgIVsM1d50RYy4pipNplQXw998kitPvNFcSOK5vnxqW5sz4SRzVAUl5sAqJ5GC +0MPqq/I+H84BBQwwviQ9WsBuA1+YW6NxfY/mldAzAjg3/yUMwokCggEBANTyvgnw +3SjBOKzg56QXRs1L2eqQzqtDzyOn2BXlMvT3bJFdqOHROhqqD0oSSbYZuCoJAJPh +/W2bqengGSbsGwKKLHDN0yPWR12QVq3y9gbK/L6Qktjp2zDhHqNgr+4SSSl0gMMF +bz6Nzn+0SO6C3m+M6hAgfsuizIhSCxBSSLccFSIT0ZiYRzq7rN8FlirpSIhd9cQ5 +B3q41lebUHpfciPr7K+psmCXO9NqtSvXtcMA/n7GyIGPBVDp8kMUwzuH9OVOkZkK +Z1a42uuYKs/zgnbXV7kCZ6iBQkt7A2Scv/IIwdeaKxfTv87e8UqMkYPuf4wgqLq1 +USoMoHdz6JmQnw0CggEBAM4LfX4PIIGT+OQG5kn54ZcsqerPljPj19EBe8U3jyf3 +OzxDkSX0f2fmBVDjqj8QRO/PcXQbGUXWhJN1DvvuJITE93Y5Pj8DJX3CfLEnjE+b ++sfuSxNawH9NwvwNt42NWDleMAgfMot3J+MwlXb31BWixCMj41Vl4VrkHEzX/M6R +aAWcTqeY59KumtOvZO/4U98VvNyOipLHWcWZuwJLzeUjvyZ+hJQ6kBGJJYNezYm/ +qhHZ5k4bz8HAUSgih+Yb3DNmREk3YqIM4Skq843YdF8SEnFw/4b7PKMfmBXy3X72 +f51wdCiSkIm9o+/QLaduExQd7AhmFO8avZa5bRNNwRMCggEAKY/DBX+sOoMTw7IV +o9IjMHhobL6ch5Kxf/0HUKauPl94IhsMlh5W39NnLobJOjBk4FdndHV8GAN0sz/Y +yN72GpXLPKz/U5RD04ATWtn7qLG/iJYBAzMJY83cQ/jf/XA2NVAWvXl3D9dvgT83 +qM2ECnOPT1x4QthgYQ7aN/JHXO2vNjp2AvldlZoBkHmvqGpljLACAq06x3oB45Fd +sLSmO1qVlGdjeDSsKYQ/HfJ4+DlecnHrulWmrPcsIGmR/TF427Rs+FiueJ+VorvN +R074nKdE6MgOYTXxMXgt3lo1oFCTPLhLRtg+LGsY3vr2f7Bx1nCdXet7juBuBUJr +GGXAlQKCAQEAxNVHMfijdgX022kX8A2Ni4x4Wj+q3rFHR3viUDnOQUC2TtDBRX/3 +gjrEU0zaI1qYcHs8h80nbIcMqY1HHjaWnltHh6IRq8KGu0fjNJ1yNc7tWLd08u1c +PYD8xysXcVtYr50hx3B+KatP6IJOFpOUAIM4WdV74+XqzZhizKn88R0JQWrb3NF+ +jM6OS7EffPs+rDuo6w4kpSlZwiIk+4GNFNv8THrKjowPeyEIPCKBuZjmkB0YHQAG +jbH6FZw/NPzidBu7GjKVv/cL1fcZKiVgrj2mbsai5MD3YWHaOQWEwTgcGzwFS4kQ +GPWYOY0nP+4wvaQECtXyI6To/qbu42UBDwKCAQAd+/H4mepB8cgchZobFco6yTt3 +qgS1D9sOFgWYZLkxIuQTQH0C1cLHXIiqf0NrJAOBbaJ+vVuCrqv2MUASaxKoDK7h +1W413yKIELWbTk7yEttw0M0T2PXmI5dNdKuw5I5hw+MmjLOFmyRzvX870ihDnM4F +MISxV7K45t1EHjMsz66fMc8BIkophwK3/7FSok5XhYdLQdQS9Rshv0PXQmffkVUM +UTlrwgH/3WGRhkFbmcdBMlawHQGvjiyZ+Gz+wF1uhzEYweT6wUfaHZePxX0hXqom +WVS6ojlUji6NNJqFN8DB4q8V5/EShj4fpdjenDap5IxFgDSxgSCShR2FGTCW -----END RSA PRIVATE KEY----- diff --git a/testdata/x509/server2_cert.pem b/testdata/x509/server2_cert.pem index dc20b468e5d9..753065b49ad5 100644 --- a/testdata/x509/server2_cert.pem +++ b/testdata/x509/server2_cert.pem @@ -1,32 +1,32 @@ -----BEGIN CERTIFICATE----- -MIIFeDCCA2CgAwIBAgICA+gwDQYJKoZIhvcNAQEFBQAwUDELMAkGA1UEBhMCVVMx +MIIFeDCCA2CgAwIBAgICA+gwDQYJKoZIhvcNAQELBQAwUDELMAkGA1UEBhMCVVMx CzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTALBgNVBAoMBGdSUEMxFzAVBgNV -BAMMDnRlc3Qtc2VydmVyX2NhMB4XDTIwMDgwNDAxNTk1OVoXDTMwMDgwMjAxNTk1 -OVowTjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTAL +BAMMDnRlc3Qtc2VydmVyX2NhMB4XDTIxMTIyMzE4NDI1MVoXDTMxMTIyMTE4NDI1 +MVowTjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTAL BgNVBAoMBGdSUEMxFTATBgNVBAMMDHRlc3Qtc2VydmVyMjCCAiIwDQYJKoZIhvcN -AQEBBQADggIPADCCAgoCggIBANluCTNFJz8gsMgn2ixQuk4YphdLfbsgOlk2lRFx -mYBpfD2hfZpnr6c67WNIWBuvMy57z+FWcmmA2iVabEs4OGPaQj5R6cngai01QNPO -d0gPpcAW/4KuVAYOYiYWSrVOTj8aTZm4buG/VMZMUKUMS0JNXSuYLZrgD23Rsr5K -j6q2fqRFtcC89QW9opafa4oTmkp6Kz/WrphF4EsK1fbelZ8xQ4+TOkIJegZMS+vA -r3itgA3ha1xqzUU9+A4xTg8HybRzJMAbtzO0DJMzmfDXXwIzAsdsYerDgaoYlBtP -5Fnod19g8k8NIJduF8dPRfnyn8fFVisT4fWet59/1jcXUbdsgdPLuuY59sxT/C8o -HLfn26w4Wda0Sc2XN5qhXwezkPX51mOw2siP81jFeHRQE+J0IOfxjfpdbI1+xdIF -vsu42NdmYa7a7ejhilZxDYRZSaJLLYE/ZDiGfTBZVoVKRNbM0EZ7VRCN9pN6i5jd -WsHCjdq1u9rzplA0D3KrycUvlpZc7xFaJxTiVFGiJugJmTJoUpQHnF6chZsGukhA -pypSB/f+r4tPa81N5X9f9vG0WBXiKGaoWVJXmNOQHaqAYz7maO/JCetjtUn6IH7V -Ti0qK4yeVh/5GZzC7xFfTmO4oWbz6Cb9FKPSsVjvo/n2Zo0e7CSVKc9oFFBwgjg+ -p6bvAgMBAAGjXjBcMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFNpFUJbVy5fb+Uvm -jKzuUDbuWctCMA4GA1UdDwEB/wQEAwIDqDAdBgNVHREEFjAUghIqLnRlc3QuZXhh -bXBsZS5jb20wDQYJKoZIhvcNAQEFBQADggIBAIaUCF04BpWeQkeUsslTSN44Q95U -oNlRD19fNXWF8eae7Wl53dFkRhn2nyqx0uoHvFZ5oRhF4v8kzM1cyW4RyLk9WTnh -Lmg/jfr84bSdWvN8nW5T2jNvq0ltSY414MFu4fHf8/GMbpIKtafFkisFXmhKm8Uc -zVilTn9Wn087Lkg3FHYVU2v0oWfupM5Qvq6tvZxT2v+7nmES6Cip8Z9U7km04yxV -hDy6YFdz2UDUYlZaQCsLPmaiIxR/EclSsL6KnMW3UjMyxX8Eft1WPwvTzlQKQFDs -uEfbq+Cl+cogMaGq1VvAA9cvCUSa1hTathWayKH2q1mPH8sqtbFyged7XXh8mkkf -8qeYTqfeL74I405Gl3u3/EjVnhSLpOqQOgn2E5HnV0bZaJmGHdU0DIvOyKauinyg -U4hnL8WBv5en9owQvE+DrivbcG9brqEY3wot0XNzB7pxXjrWdw/PMc/HNPbBsT8s -Zg0gwxwvpffGemc1L8tiM8aHOp8eR1oVr4szuNDAbAfdEgpwBctXs5JJg81zsmGe -2jJfHFAeqwhUZgCoF/FjJ+IHxOFZx9IVwrlawPadIFgVh2I0rFUcME0B1/Vk46Gg -BOiuP9keVX+qhKtqjnfabN9l5iX+zpniHIarke2o6W7nYIgdOtdbmH4YNZxjyidj -9w/3d/4ItCavbKAn +AQEBBQADggIPADCCAgoCggIBAMqkUFp6xBzIksawPZpDQCZS+ZE/Pjfab4q7CUd/ +pN0Ss0U9MRBBnYj767qvwrOGQYdpkK0NWh+BtUOQHaqsnD+ykr1x2i27uMvYBnkn +91t8EmfW8u5cj2lZGM8SRXaiCc2tv3ZFDKeizIp5BcxLsTubRZNMTOYxUCpFoABo +d6DGYXaU6LFNbIhXQ/vTlbDa85a5EjS5SWeBzEEFgBHHQDdjh+cqS4bzAgI+klYT +uYM3PMbUuOg49hVQjW1pXSGO8Yvha0fzZP4upUKi6h1+/xBUZdDiWuynEJZuYPAC +xqgnmgSkexNLJwrg/6JPr04TfbKagffKDHKhloI62CZJ9/VRwo7OF8dsl1Y+cr3T +XjDtsxmypDIi31szDYhE5V1P+FAGP3sewQBRh597x1RA8tGE3ijIV6iqSlSo/Lpx +HaVM5n3PWY5vUNFWdbawMDuCUvi9GYn3JVnt4YUxjsMxLC39FibzUmwD7JXqx86t +ArPRLU6knrcod72dKRnUGi446Z4dElqMngyu2Z08RB5iYzDwT+xugwki0gezQ2uB +59/968+MI3eVe/QWMwtIAoZuHKvPsAHOb9bAcBvLQi94MWLbBUkAga3XRF6CNAGq +LPNdhznCNYdXMnMdBQHAzu7V5+yIB/ZFQ519Fm4RinCG6clIi326A6GNYenAJ/Jr +Xa5tAgMBAAGjXjBcMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFBfefRYaa/BmcX6g +eSQSMrh0zktEMA4GA1UdDwEB/wQEAwIDqDAdBgNVHREEFjAUghIqLnRlc3QuZXhh +bXBsZS5jb20wDQYJKoZIhvcNAQELBQADggIBALpTW6UICT8SyCY7VNUja51t3+XF +QoD8xKu2qS69G/oDbxe5SF3ldymvkQPqVwnW9e34mjxD1Au4IF8zQWv2EMm96wDD +0Js0/yAMjw6/60f0hF9lpQEqe24W+wgbRV4Fzt3/rybLte+4L9chvq1plHqQ3sfa +s99D7bfPSLX8n82ppNmm0U81kXmtNAw3P+vStBqRgrZNkbkkgsoGZsTgzKuD+H81 +WUzIqmIAfTmkDN47SXYuneULlFNWwtHgTWv4jq2/ptYo75MQq+ExwTDGM168x17u +yaC634INTjfNd04exiktBJXWmAS8K4aYgvHPgcIlzyidR7taI0X1O4mR4qomh5W2 +fVmGkpQZCmkW80whgycY3ui2fdWYOs3XGdfz53fJdN2vWebpUjw7+1owlmqFJhEe +Ct0wqLLeE8rdOfKueh3/xi7CxoeYM2fVjN4gHPojcQ3Mcs7wiJDm0WFaITi6+KDS +LmGhSHKaiXiGKsbLykN0DygDQYa/c4t6NfzoRGWKMhdAcRXicZaxnwPD65psAijv +ZDgONwXeHKgfk7DnE3rs+D9xuh2ciw7lkcbImYmOCMoV88qG0t1uIwlM3xh8S7Oa +DH6q4vj3pF63QS57uRtwCBCOa3xcYKTDJbdRyUKgAejVoz8bqTI8lBjnRTtxlQFi +4ugkg86X1370IY84 -----END CERTIFICATE----- diff --git a/testdata/x509/server2_key.pem b/testdata/x509/server2_key.pem index b0f6ddf70dce..82c947d70fdb 100644 --- a/testdata/x509/server2_key.pem +++ b/testdata/x509/server2_key.pem @@ -1,51 +1,51 @@ -----BEGIN RSA PRIVATE KEY----- -MIIJKQIBAAKCAgEA2W4JM0UnPyCwyCfaLFC6ThimF0t9uyA6WTaVEXGZgGl8PaF9 -mmevpzrtY0hYG68zLnvP4VZyaYDaJVpsSzg4Y9pCPlHpyeBqLTVA0853SA+lwBb/ -gq5UBg5iJhZKtU5OPxpNmbhu4b9UxkxQpQxLQk1dK5gtmuAPbdGyvkqPqrZ+pEW1 -wLz1Bb2ilp9rihOaSnorP9aumEXgSwrV9t6VnzFDj5M6Qgl6BkxL68CveK2ADeFr -XGrNRT34DjFODwfJtHMkwBu3M7QMkzOZ8NdfAjMCx2xh6sOBqhiUG0/kWeh3X2Dy -Tw0gl24Xx09F+fKfx8VWKxPh9Z63n3/WNxdRt2yB08u65jn2zFP8Lygct+fbrDhZ -1rRJzZc3mqFfB7OQ9fnWY7DayI/zWMV4dFAT4nQg5/GN+l1sjX7F0gW+y7jY12Zh -rtrt6OGKVnENhFlJokstgT9kOIZ9MFlWhUpE1szQRntVEI32k3qLmN1awcKN2rW7 -2vOmUDQPcqvJxS+WllzvEVonFOJUUaIm6AmZMmhSlAecXpyFmwa6SECnKlIH9/6v -i09rzU3lf1/28bRYFeIoZqhZUleY05AdqoBjPuZo78kJ62O1SfogftVOLSorjJ5W -H/kZnMLvEV9OY7ihZvPoJv0Uo9KxWO+j+fZmjR7sJJUpz2gUUHCCOD6npu8CAwEA -AQKCAgB1i31B0HLlN+EadCEIsCPoMH8qPM+eKFAjBtUT9xwLRfu6veFPZhqaB8tq -TyQC43aB/MFnivqTeut0IixFhgFGSiph0prXXpFIG3AOkaH+vSbYcBZ2KZSXKZN6 -D7cXyVuX1bp6DjEzreJAyeUXNUxCbdyewsh04Ai3UBSXt2tv2PUiDeWyavTzw49w -aoMSxII3HVDgVElTXQNizlrZ+X9d7p4dsnReWw0y9nBc5XB3hyShXGpULhEHC/dc -hN80VPuAqHcHvHQQaZgaxFzGzUg5wiYQddGBv2wL7vmywkArMvfGAn08q1YhR41n -XL3x4G7s6wwogbk4tjOC8PN4GQ09YxbxJVLSyVIHX/v8tYe8H8acsw4LonkawZVm -HOgwMpz/hcm7P+ClYjAVUWZjCJt02svDV9U1BPEdBtOXrMDwlBfVuFxtM4GDkKmZ -GjCLnthpvBXfw6stDKuwE9g+TYVcRMsPhksjE9ZasTTVtFU/qZXhc2bDuJkWaUAd -yAtxBOQYF9mBN4g35NSE7k8FE3HxNDJx+zstodweq6qhinXchuKAeViap94tneeG -hoSt9PgMnOnx7V0wIK7DaGCH3ssxbjRQ2wRLdTNYAzhV+tkeDex2zf2xtOvqtWIC -l5gUSTUnaEYX5wVbCPAJIOAI1TtMe501PfXyZa8wb6p9eSHMAQKCAQEA9DJPKkjI -p+FLBn5iFgGE6DgiFD+K6OyGtr0Mle8D+XFDtlZClF1sjZHacb3OQgDnlSYXZaZR -iN5jKuVJsrhCgm9g0QYDwtp+m7JMMX9A19qZbbK71w9Qi80wuRZze0nktr6RKiyS -+x8VXkeSHPUSw7VbzbE/CCm551Z/ORoU5fXnDstPKk/M8K2NSYywwzwaEkEuu0NQ -/syGxaAW8mThruDAZ4gtJns6IyTmM+8KgkSnbwK5mlOMPhJ+6bHDyeV3OJe2lSVW -ZRA9kzDFAKlotpwRaSwBdu6chCdDhQGn/WlofJHCt2t5Fh9mK89AXQsXfjAh0O1N -7zrU/yeNIXJd7wKCAQEA4/CD665RVUwNffb7fa0vnt6Rkj47FdM/BmWpLnb1IC7L -87Fe9uryaNtghLD6T87vF3MtH2rEfQ2qwR9VRC4MyB5kNvozBVtJbKLy2oRD0/Lp -GSLhjAiKrzu8Dmwv/5iQhrSRr3mqn/eoIx5ydgot/+OzxgH5Q4CGYvzZUcIMVpi+ -eq4/39vLPQoa5tvT+n0G81sCCVR+sBtBbgVq8WaiqW6UunqP+B4+bPG6jbYMjdcD -w+ylakjJdAofl5SqcUcUy0UzI1pEjKnlLYyCyuVMlkhVZoaQiX9TTOTZ0jAXnbps -sDS0fwW1/8J5cSXxIA3q1WVtshst2LwwaCgYlVhHAQKCAQEA0x+v7BnzSZnx+JJK -EUaM9wyZAjKR0aG1Msat2+9C22W+qiVX+Nfw41EHsLDuY4hOsFe3gM3TzmafDFYi -ap79+bF73hu6IrwvHEOBtoWTtUusvPf7iQsXk1b62fr8KsqPMCQAc5sIFI8iNVnh -jKGh8Iya63Jj0ZXpwYW6Bs9y5AK/Gr5SGn3V7PvPnJhDtvf+fmvWkFa57yE7IB+x -1y27JSvxjVFh39RIRlw/nwT7a/cZX1PWzgOPy5bIHRnw8VwvwEECvV4DnOr2oYxX -tqPBAahbMTe3qHDR5zvfF16ANArvKEwJMfV8QdExz4ym1Aqj7BiHFBAnAj82Kcez -MAimBwKCAQADw2LKL1SUbe8DF2LLjmJs4wvQOErNb3Fo76C9baVaZKtlWJZSyUo7 -RPPw/OMFEkuMPZCPJjocPm+FRLkpqQD5BNduuO7CteEedApCZVChXS9QBO1oXHO9 -tOTD8DFSrPgl4TFOjlmszm/uNIB7Rmu//8hmCn5NCQAu/jGwUd3WSCtM5zeSwJQ4 -a8RJ73MufYXx2pzL/qMg0TJhWKGNXr5swbCe64sY85bgQZVs5YaLiPM89tk8SftZ -eRlQbVnrCNtlB71yZfkfwWZRPDKkmuiKyqLuUGZufrWnXVfjSnv5VKyatCQOvM9m -a5WJsrCqcNBhuYz4Fc7J90FtVswhGxYBAoIBAQCzNj4K/OrC5fX2sidbcaEU/9S+ -r8JZeCaxAAFepoFKE0LyspNrsW0CZ3Ana3B7SqfH3nAFLoLxExCgMm1WkvwLp22X -23Gav6cRG4XJjZjyLKW+rcowuhI2Hb6FE2UvshcDzlHpkISpjeY62Qx5gcoLeLlj -eQpqg59wL5ZweCOcgV/K2nrOILlmQR/GQ68XxvBLoj3J46fc+/iI8G1roGI2H6n6 -tRqmOxRFdmchkPfLPYq5Z71LTWD7m1E27k8apttT8P2mfQhZZ3YYERyiRTTYdO0i -0ZIi5+OqzTuZuefgurtHDnJe4rFT3/jZzKmI3IfbuRITxmxSgPd7cpuM22uo +MIIJKQIBAAKCAgEAyqRQWnrEHMiSxrA9mkNAJlL5kT8+N9pvirsJR3+k3RKzRT0x +EEGdiPvruq/Cs4ZBh2mQrQ1aH4G1Q5AdqqycP7KSvXHaLbu4y9gGeSf3W3wSZ9by +7lyPaVkYzxJFdqIJza2/dkUMp6LMinkFzEuxO5tFk0xM5jFQKkWgAGh3oMZhdpTo +sU1siFdD+9OVsNrzlrkSNLlJZ4HMQQWAEcdAN2OH5ypLhvMCAj6SVhO5gzc8xtS4 +6Dj2FVCNbWldIY7xi+FrR/Nk/i6lQqLqHX7/EFRl0OJa7KcQlm5g8ALGqCeaBKR7 +E0snCuD/ok+vThN9spqB98oMcqGWgjrYJkn39VHCjs4Xx2yXVj5yvdNeMO2zGbKk +MiLfWzMNiETlXU/4UAY/ex7BAFGHn3vHVEDy0YTeKMhXqKpKVKj8unEdpUzmfc9Z +jm9Q0VZ1trAwO4JS+L0ZifclWe3hhTGOwzEsLf0WJvNSbAPslerHzq0Cs9EtTqSe +tyh3vZ0pGdQaLjjpnh0SWoyeDK7ZnTxEHmJjMPBP7G6DCSLSB7NDa4Hn3/3rz4wj +d5V79BYzC0gChm4cq8+wAc5v1sBwG8tCL3gxYtsFSQCBrddEXoI0Aaos812HOcI1 +h1cycx0FAcDO7tXn7IgH9kVDnX0WbhGKcIbpyUiLfboDoY1h6cAn8mtdrm0CAwEA +AQKCAgBa3LaS+304Es+Ne7UDmKgJByeUczEoxi9Bm4AbqSZ5Yksz/q4jReinZZ5b +hTfeW5LCbxlKHzSL8BMhClvjDaa6AQ4/F+/mlcfUzzaH2N3XDZkLKpyfOK2tZR/0 +qZKwERQoP4IcO/XirOLeLEnnQwFjYsodtBa/GNmDOtj1leIeGxXUoAx+g+Lod4iq +QENcm7ChoraBIZvCZ7b4aMj2L8uhimWDx7k593itHPVs10dViM0dsoB+0Bu3jvj7 +WEVEKN4yBI+gIYjlWHENohMryqf/4HgO45A1kOulKDUbKYN+HtO2xTHSgt4syJqX +YveOILs5/IHOY7CVLdNY7Z3B/WTKtO4UCzGtFWsD0Ai6rtfnSj2aAmq5uHWDwPHl +3fHdTK6knHdlaPWbeQiBIk4bT6L/JjH38dqBU17/RathjoCDQngNmid7AgSnv5o0 +5ugTCTzzFTUz7FnA9uYcEWIq7xDB5gVFTcvWcARYd2BsLM/9gF1Hh7r4A8gsHj2i +0+7Saw6mvAsPXp0JH+8idBk8khV8v6Uy1arF5aYF8yNus/hVr5fUBnnK6MZ+F2Nr +VOa49n5BhbWm/IsVYdgnnk4uwUx4yNuwMZ9/nSsDEZ6IiX9KZjap6zKNwhoodLV2 +T9WYKC7JMOEBr1CzoQL9JzvyYulZUp0F5SBMbB9kZj6j1gQRGQKCAQEA5QbA1hqt +0iy5KjH26Rok2pwi3z49o/sFPyg4TBOs5Zx7T/iBQM3ZlyOUdGT4uYFX83Xljbo1 +As/UIM1wzCSUbyyHGs7RTuoAWJ35d94UwdfOmw5j5ETAtbA0EVxnLCqbp2ArqC60 +UT/M9Yk9HK8bj6wwb0kZ/wsfwn78Jts8GhtCozPg/c2KWq3ce2bcHMuGhlaWq5Jb +XlHrZBIoL1tYsT8LWe4wc8Agm/w0ZC4rM6B0A8rxdsrPAc3WmlvNvnA8DA7OAJZ5 +j6zThsQ6FdSic8CI6p5vw4pyFZXtdERIjX2jkWVLVdb8adQKhqcglBvYZt5e73tk +a1OfgG0tX3M8TwKCAQEA4oIOLgVJWc9lfWFaLiRIjKlVtnOqfR8Vy9e7i2dBylgO +ZAoVM3ROYLsbDRsMJqMN/IQIuSOMYP+lIeeLoJlcqor49+z6em45Kbt5ZiK1VRoG +78Zi/UZ6cb/vcB17zmeBuUHYEmHLbB8PWbL+qFEipVDAf0q1jD9VnRw6ntt5K0oM +AKPH7jiCxfo3GU62nOZgcT1rwnA0FHBl9vvcr+237a/+NoNkRqQxTX4y5kA5l0nP +9EEWYvOlCKbkYiPKHOZMGZ4MWb0FlFp46KPxiM/x8XxB6NFUacEJUKE27NYgj69+ +5C62A34YtLKptD2+CeAKqxYOYIt3Tj4qJBrJ4m6OgwKCAQBxoXwjvnDnipEEQm4D +EZmfbUBQCw2CQpVD1Ky58jkiYxU7hEx83qVKu7h4V3CgeXAttx0ByJVso7jX3ZZN +cwjCcBFIV7y5rpglX5vawTEDTBOSEv20z/fdLWNoCbSW0T0ROkHu291TQphqaoEL +rkW6bvBJBrgDNn23flGU5clYGpZhaugChOxUOVbfUxV6o/BGzsdKsP7sOTDVIb0W +YfgLWQBEykz34SdMvUExQ0bkAoQNLa/IBK/YcUw8obfe+MiSIvZKjF4bzt/USZ+Y +HTvMuoY0Ag/psNMRqqV5vjdRHDj/doZ+PIBX8YCXdmxPj9E6mLH5l/sm1QKaMZEF +fqM5AoIBAQCPe8lVt72Wab2lpgTFU/CtQhtsv2qRZh6diSRhk2BmuE8tagGyHYwE +1KG3NJoG46VZf54zAWTMkUTe7FlTu7KqyewayYCGC8qkOAEYBQaPSTR5sVdFj97C +rc4UXGjwADt5yk8AnfiJnkdQEAYnQ3ZJ+JRoTkAg/oHSS26K8QaZuIdP5HAi5KNa +nD1JB8bAL2OKeFkJy6ACDo1Y3oUW4ORxadoEWEkuQpaEu1us5aRVxMk5tf1jY2n4 +yBfGX1uJ4Qz18VtrgUTGjGUpIalAfFGMIqVxwSDS+RhYfjdX4fCwdIBSNZDRN5CY +7tB3v+DhSo4XgJpM6CwEYXa6dknK6TPXAoIBAQCo/+MThNIrKwIsbKxyz3Xzg4Ut +6JUQxd3YU1kNcNde8BSuNp+05RcUIc7wpRTrrJwAc9uNHVNjKz73WcpjMh3dHYJY +VF0nbUzM2m+KzLdTJtRTYMaFGJjiHbttetNGJoKomNrMea/vEjZh5WadkBagKBGp +u85H3Ff1vYdMyCTBiU6eyabxc08/ZEaFJaQALjVC4e/mQdluCHyfmqeY1awKLmCK +vf5ZvBC6ISOMMibqcRT5ocjAvO3j+3d9Ce3ExX5U1fu/xYb3YoWQtdX+qIAT2KYq +QG1vDy0VieGJlUiDPooWin/X830pxyYJ79w7XN67JIZdlrtIEVDJe4/xghXt -----END RSA PRIVATE KEY----- diff --git a/testdata/x509/server_ca_cert.pem b/testdata/x509/server_ca_cert.pem index eee033e8cb05..2760c56b4335 100644 --- a/testdata/x509/server_ca_cert.pem +++ b/testdata/x509/server_ca_cert.pem @@ -1,34 +1,35 @@ -----BEGIN CERTIFICATE----- -MIIF6jCCA9KgAwIBAgIJAKnJpgBC9CHNMA0GCSqGSIb3DQEBCwUAMFAxCzAJBgNV -BAYTAlVTMQswCQYDVQQIDAJDQTEMMAoGA1UEBwwDU1ZMMQ0wCwYDVQQKDARnUlBD -MRcwFQYDVQQDDA50ZXN0LXNlcnZlcl9jYTAeFw0yMDA4MDQwMTU5NTdaFw0zMDA4 -MDIwMTU5NTdaMFAxCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEMMAoGA1UEBwwD -U1ZMMQ0wCwYDVQQKDARnUlBDMRcwFQYDVQQDDA50ZXN0LXNlcnZlcl9jYTCCAiIw -DQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAMZFKSUi+PlQ6z/aTz1Jp9lqrFAY -38cEIzpxS9ktQiWvLoYICImXRFhCH/h+WjmiyV8zYHcbft63BTUwgXJFuE0cxsJY -mqOUYL2wTD5PzgoN0B9KVgKyyi0SQ6WH9+D2ZvYAolHb1l6pYuxxk1bQL2OA80Cc -K659UioynIQtJ52NRqGRDI2EYsC9XRuhfddnDu/RwBaiv3ix84R3VAqcgRyOeGwH -cX2e+aX0m6ULnsiyPXG9y9wQi956CGGZimInV63S+sU3Mc6PuUt8rwFlmSXCZ/07 -D8No5ljNUo6Vt2BpAMQzSz+SU4PUFE7Vxbq4ypI+2ZbkI80YjDwF52/pMauqZFIP -Kjw0b2yyWD/F4hLmR7Rx9d8EFWRLZm2VYSVMiQTwANpb+uL7+kH8UE3QF7tryH8K -G65mMh18XiERgSAWgs5Z8j/B1W5bl17PVx2Ii1dYp0IquyAVjCIKRrFituvoXXZj -FHHpb/aUDpW0SYrT5dmDhAAGFkYfMTFd4EOj6bWepZtRRjPeIHR9B2yx8U0tFSMf -tuHCj95l2izJDUfKhVIkigpbRrElI2QqXAPIyIOqcdzlgtI6DIanCd/CwsfdyaEs -7AnW2mFWarbkxpw92RdGxYy6WXbdM+2EdY+cWKys06upINcnG2zvkCflAE39fg9F -BVCJC71oO3laXnf7AgMBAAGjgcYwgcMwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4E -FgQUBuToaw2a+AV/vfbooJn3yzwA3lMwgYAGA1UdIwR5MHeAFAbk6GsNmvgFf732 -6KCZ98s8AN5ToVSkUjBQMQswCQYDVQQGEwJVUzELMAkGA1UECAwCQ0ExDDAKBgNV -BAcMA1NWTDENMAsGA1UECgwEZ1JQQzEXMBUGA1UEAwwOdGVzdC1zZXJ2ZXJfY2GC -CQCpyaYAQvQhzTAOBgNVHQ8BAf8EBAMCAgQwDQYJKoZIhvcNAQELBQADggIBALUz -P2SiZAXZDwCH8kzHbLqsqacSM81bUSuG153t3fhwZU8hzXgQqifFububLkrLaRCj -VvtIS3XsbHmKYD1TBOOCZy5zE2KdpWYW47LmogBqUllKCSD099UHFB2YUepK9Zci -oxYJMhNWIhkoJ/NJMp70A8PZtxUvZafeUQl6xueo1yPbfQubg0lG9Pp2xkmTypSv -WJkpRyX8GSJYFoFFYdNcvICVw7E/Zg+PGXe8gjpAGWW8KxxaohPsdLid6f3KauJM -UCi/WQECzIpNzxQDSqnGeoqbZp+2y6mhgECQ3mG/K75n0fX0aV88DNwTd1o0xOpv -lHJo8VD9mvwnapbm/Bc7NWIzCjL8fo0IviRkmAuoz525eBy6NsUCf1f432auvNbg -OUaGGrY6Kse9sF8Tsc8XMoT9AfGQaR8Ay7oJHjaCZccvuxpB2n//L1UAjMRPYd2y -XAiSN2xz7WauUh4+v48lKbWa+dwn1G0pa6ZGB7IGBUbgva8Fi3iqVh3UZoz+0PFM -qVLG2SzhfMTMHg0kF+rI4eOcEKc1j3A83DmTTPZDz3APn53weJLJhKzrgQiI1JRW -boAJ4VFQF6zjxeecCIIiekH6saYKnol2yL6ksm0jyHoFejkrHWrzoRAwIhTf9avj -G7QS5fiSQk4PXCX42J5aS/zISy85RT120bkBjV/P +MIIGAjCCA+qgAwIBAgIUVaYPCm+rhznxJTRWV7wJKkmRuW0wDQYJKoZIhvcNAQEL +BQAwUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTAL +BgNVBAoMBGdSUEMxFzAVBgNVBAMMDnRlc3Qtc2VydmVyX2NhMB4XDTIxMTIyMzE4 +NDI0OVoXDTMxMTIyMTE4NDI0OVowUDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNB +MQwwCgYDVQQHDANTVkwxDTALBgNVBAoMBGdSUEMxFzAVBgNVBAMMDnRlc3Qtc2Vy +dmVyX2NhMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAxo6Cn80nk3i5 +PgmYnMBicmJEykEz5YbJEuyN+Mjv1wivqc23P75qvu7u0FPePptHZK+Q3PCnv7BZ +jc+MDQzZhUWN8jwenMGOxpVrX0zjK7Q0u92YbrHgxE9fkRA5fZcXGzZlrhsJQJUA +G+0QGCSzjWZvSab2JrVn/gYEzikcl81Q6zAJkTI9vACZC0vnTc6XsVC8QCpT71fb +qQwE4Bvr1tyuA6biB4H40RiLGWuG+8BoVn1pgSL/9GzRnsEnSN2KCfaqzk9VMDnP +TQRx0yJY+Zl5FB/ufeEJH8hh1OS3dAJhR7IYLktlm8S68dSI/oTs811BWIw1dOqa +KbpElXS5Tr9usGOehxy7q6dlazj2+nDzIhQ/20koX0dqyN1O8Pzi4OWcR5YQEBDO +8Bp9v6JNowwbMkZGSg/C1GMNwN4rEhLlAgpv8/4CoZwlQM0oROWiiZwczpVniDiq +6dYtTUhuJwC0cgJLSswDXpnAlp30hPB7EV5MIdr+9ybuRAx59Nl+ZB8g6utuWNaA +lNTrAsouwWBalHmY/f4/ltEnHkwgKCReYFHpDNuDVtnxhtEfGzd5IxQWNl2etWCR +Nnf7Z3DQHLTduIQD2R0qp73tqFK8T1DR/HZkbZnZPvBqmUIXvCnHJKKB3ntrkpqR +bQHMq6Tkv3NL+N3XpZwEz0AOeiRE+gsCAwEAAaOB0zCB0DAPBgNVHRMBAf8EBTAD +AQH/MB0GA1UdDgQWBBQl88evytJrL7t5BGRbJUeMOW4jaTCBjQYDVR0jBIGFMIGC +gBQl88evytJrL7t5BGRbJUeMOW4jaaFUpFIwUDELMAkGA1UEBhMCVVMxCzAJBgNV +BAgMAkNBMQwwCgYDVQQHDANTVkwxDTALBgNVBAoMBGdSUEMxFzAVBgNVBAMMDnRl +c3Qtc2VydmVyX2NhghRVpg8Kb6uHOfElNFZXvAkqSZG5bTAOBgNVHQ8BAf8EBAMC +AgQwDQYJKoZIhvcNAQELBQADggIBAMGqi2F7ccNQ0FSiALPUjO0VvQrUqdWLrc9Z +67rr7wBu4bEzchM+HQP9GwbnSnH9yT0pnYj2H6idAfqTww1kKuR4CYMkGsNJ9PYW +AgYdrC67HKT2xhy9YmrUItIe/pM6rRO6oNA8Np3IAEmC0gpVMqmPqHeLvwhxcf4f +izsi148gTGOxBIWVNupImFrOaztKV6SbVwA+wdHNJvXz4MEEYlMlgHFfkrAEXHfO +6QmHXru8C0BIQaMOiVZDN8YCwmsrcGFYjHFRS/OnYblrRxuVDdhpMmNiQRJLhZHi +jf6WOpJS7o50FmC8bG1CE0CqMNF/qz3Hap36Rm2w/xSems2dIqMr6FsH34KkyXzm +pCHN162g720orV1uExpgfRSfv+IaklN1sM98WkTqAkz9p6OPPEo4VHeVGUk/mFuv +aVnByrk7qmpTLBGDk7dFI0GjsNwOz619omgYZGliRU+7rDXP4fN6EPlF5sQO7MJX +REOSZvVcHPpIAIqTFRR4SBnwYGsEPQbTKTH7jJROg0TGmiKeN4N1syb4KNos2Wfp +ZZB+f2qmn6LXS6d2kI692UomRfGVNoBEsAhWBW7FzpU/WnT+aF97VpvWUEqzg/AS +61tKM/t/ap6kNTLaPGSWTk9Ade/KuMmg+nSrL6S1Xa0T1rl2Qjd5h7U6JLWHL94v +GPxPuBsN -----END CERTIFICATE----- diff --git a/testdata/x509/server_ca_key.pem b/testdata/x509/server_ca_key.pem index 114e2a37a11e..1f69f3435663 100644 --- a/testdata/x509/server_ca_key.pem +++ b/testdata/x509/server_ca_key.pem @@ -1,52 +1,52 @@ -----BEGIN PRIVATE KEY----- -MIIJQQIBADANBgkqhkiG9w0BAQEFAASCCSswggknAgEAAoICAQDGRSklIvj5UOs/ -2k89SafZaqxQGN/HBCM6cUvZLUIlry6GCAiJl0RYQh/4flo5oslfM2B3G37etwU1 -MIFyRbhNHMbCWJqjlGC9sEw+T84KDdAfSlYCssotEkOlh/fg9mb2AKJR29ZeqWLs -cZNW0C9jgPNAnCuufVIqMpyELSedjUahkQyNhGLAvV0boX3XZw7v0cAWor94sfOE -d1QKnIEcjnhsB3F9nvml9JulC57Isj1xvcvcEIveeghhmYpiJ1et0vrFNzHOj7lL -fK8BZZklwmf9Ow/DaOZYzVKOlbdgaQDEM0s/klOD1BRO1cW6uMqSPtmW5CPNGIw8 -Bedv6TGrqmRSDyo8NG9sslg/xeIS5ke0cfXfBBVkS2ZtlWElTIkE8ADaW/ri+/pB -/FBN0Be7a8h/ChuuZjIdfF4hEYEgFoLOWfI/wdVuW5dez1cdiItXWKdCKrsgFYwi -CkaxYrbr6F12YxRx6W/2lA6VtEmK0+XZg4QABhZGHzExXeBDo+m1nqWbUUYz3iB0 -fQdssfFNLRUjH7bhwo/eZdosyQ1HyoVSJIoKW0axJSNkKlwDyMiDqnHc5YLSOgyG -pwnfwsLH3cmhLOwJ1tphVmq25MacPdkXRsWMull23TPthHWPnFisrNOrqSDXJxts -75An5QBN/X4PRQVQiQu9aDt5Wl53+wIDAQABAoICADoDco6TNRZ+PtdoIVdlfd93 -/wNQw+mPpF8tV2wsefZc09gT8auQv0az0nb7QZsrrpBUkB1Jxk2Ub8mob7fn/o1R -pjanhlfmyoe2VhjFcRwv/n2pWpFfjxixB2of5r/EWUwR02zwTkFUfsWAVgRI1hTf -Xk3BZGah9LC0LmfeboEDHW+Y6XtfCSYsQlobXp7wYMZ7MSFubWf7aa2Q3N5d/MlG -RqYVZ3fCVHnioMgiJkvDG4d0aXnyvXpTarBkJMGjkVwjJ40dIU23cBhOW0alW7JY -t+S4q1waDYxeR5HA7O8gykCeYZ4wSo+ANpD6q+h+uYchLLmh93fDfwTxFU8BhK6a -Dp8ikyZe7hjEba5a7ZvfOXedOZoLqGuUF4P5wI0Hfdslqwq34QSqMiHJuQGa+dM+ -tqnxTw8TjylYysMJxkqipA91uhO9AWxUc37jkWOY255kXcQdKwx5TdQN25XDDjK3 -BNiGtWIEuRMoflO2tL8AmaATOYbVuC3rSm9vtK0jre09MwLxihuzd8fgGBrtEx5S -UMaBAGDG1F0lcdxQY/h1byL5g1y//N472Ir0PLGczMPBigy+ZEy2GNtwUniwWOWH -z8CE8BbCr4PMxaqR/qU4hmEw6E3mB8w0WMMGQRn9+jKwxSZaIsE518Wa7oVEx02d -LZOu9b4xNslw8HjwaSKBAoIBAQDvas21s1EhtgKZaLXNyVsWaX6Mg1h0puylvqlg -G7t7F7XRV4gPb21e65y29V42vG/r2KB/AJh8eHTFYrOPSPPT1ZZxfxD7yuJGliYc -LwMU9QWkks5bFEP8nHogBv5nA47Ve+ctgrkwhZneWS896EI0Ulzw90oeOYgzJAmP -u0IVx6k0SlYKw5b31xWdwRehAIiz0UFufn88QtM3Fhj530It/+mqvrT/MR93XIIm -0tFLOIGz0Tp4yLleB1h//9xFdLUgDAGXgyC2ivlq5H31rGwkZr0Ixiwm8VOq1yvF -/ZofDN37RIrIbC2O0shFbU/L4KC99Uu5gDk/bu7INwLrmK3JAoIBAQDUAMNz0Ewg -cR1hlJ1mDD0IuKHjgjwoOslJ+r/P9tMfXkNudx7IRrnsNlJzB0RYYFaZiyXi4dXn -nN1C1ePIXo/kfw18Bvl+GIUrHV9EZrMJ+OfdWyXOzv6kfkT4B+axUJpqCTA3aalr -+mI+EpSjw5IHzgEL9cZBlms2YSu6cDxKYXm8sjQ7w0OKSQFsdv4rIfT+xwVyVHMW -1vn2tYdxnnidzuGUFt1Fhx8SnNHSu6K3rvjoc80jjg3TuOeKtil5AwVhtxqpX5HV -XAdQwFSZigSkjypnvIlJ9YLVl+64U24UQXBZc3qZdImZqKn38Dfalaiz52CWZPtt -N6HFzJTAjcmjAoIBAEgWe4wLSxGAcTXp3lvxFfkgmJcMcVTmcfjR+MPUddXxZLB8 -z53+KgtbnBtGtDB8+qIj3ud+sWtBBb/tIS2yhKAy/pJ79QwroYgpa54u1Zm40RMl -lPa9ml70ap08Hdu8qYREQ25jnwkqIRNe/SeByHVim1N+0hVZs1XasvpRIuvV62+w -NkoVbF6Bp6ORYWD7/S1Pg4kWk478fAZpI+oQvCeHl77unyb7joLtGs8/yP8CK6OO -CzIVFiNmyNH5o0RSiLr2goAxXmc4XzM9S2Pun70yJhb/PIoZPd0B3s9FteNFh41B -rRv93pXTh7PH3y//Gcc4la1sG1CrQUCNt9ZiaWkCggEABHHXpy/wyKVWdltFSYRs -KyijzD9Iv5cr7S8ioluMZZX2V/SLYquI7ljdNagrWKb8ac+vBaiycV6qjOIrGmJR -Jfs77yO+S1R8RkEhZC+7BTSAt/VXP5S7Zft3urN/tKv58MsshZzjfm4LbT26fAx3 -nU5GW1fVxj4/FS7IWepMeUq94KTjz3Tyj42kR//eqEzX9Bd8F7+JgisTpoZ7xngK -E1TpCc/I59JDZoJ/K6nfaXZzpXv4CwzJYWz4/cF/8ReNH1VVa8OjLRP220yM+YMZ -QdH2k6IyRqitC4lZ6edl4WrVzipLobf9woj0t0wD/8MvfEYXkk+frdSCwcDeRYMz -fQKCAQB/kbirzZfwH60I6FIlCHohRDJ3csBU3n94UqSOqi/zl6ts+Vgrdq/36UuK -lww51o1FmtnpI1jaVw8Sug7N4xYUkgCmyFvLU3SUOw68xzPxi7k9NwI+M1jH4ZMK -JVJXHaxx2bY35rf+y1NKOge24uw//C1aEmKq4Dolql6ZiJlVGUna9lp+VmcDa+XW -OzGfJWMZeSh2kI8cJrTCrar21zRfF2c6IsoKdDBAmZV1qSgzymzUYtYQ2P1s+qRS -Cs891gpYRQMchfec7FefWdFYXgEfLRp+nz4WoLaIwK+oftPHl96V1z9rS1Zs2HXD -okA9YtMucwgrhGFv9T0QtBuq4aEC +MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQDGjoKfzSeTeLk+ +CZicwGJyYkTKQTPlhskS7I34yO/XCK+pzbc/vmq+7u7QU94+m0dkr5Dc8Ke/sFmN +z4wNDNmFRY3yPB6cwY7GlWtfTOMrtDS73ZhuseDET1+REDl9lxcbNmWuGwlAlQAb +7RAYJLONZm9JpvYmtWf+BgTOKRyXzVDrMAmRMj28AJkLS+dNzpexULxAKlPvV9up +DATgG+vW3K4DpuIHgfjRGIsZa4b7wGhWfWmBIv/0bNGewSdI3YoJ9qrOT1UwOc9N +BHHTIlj5mXkUH+594QkfyGHU5Ld0AmFHshguS2WbxLrx1Ij+hOzzXUFYjDV06pop +ukSVdLlOv26wY56HHLurp2VrOPb6cPMiFD/bSShfR2rI3U7w/OLg5ZxHlhAQEM7w +Gn2/ok2jDBsyRkZKD8LUYw3A3isSEuUCCm/z/gKhnCVAzShE5aKJnBzOlWeIOKrp +1i1NSG4nALRyAktKzANemcCWnfSE8HsRXkwh2v73Ju5EDHn02X5kHyDq625Y1oCU +1OsCyi7BYFqUeZj9/j+W0SceTCAoJF5gUekM24NW2fGG0R8bN3kjFBY2XZ61YJE2 +d/tncNActN24hAPZHSqnve2oUrxPUNH8dmRtmdk+8GqZQhe8KcckooHee2uSmpFt +AcyrpOS/c0v43delnATPQA56JET6CwIDAQABAoICAQC80PSi5kMGWD1AI3v/RGvZ +/l0QQOULFhvMZSu1M8/wGxCBV2E1uuxj2W88qSSlQKCpvNLzZ979yMPAuWejWV7Y +/4W2nzk1NFODwL+0hrdY7itfo6C7U2g9BoYIuvcQ2Udd12LmKEuqIIdUByHQ88XT +Z1/ZGG7n7IaR6ENVkX7hVJvoq2vNqYtPZvoi5fF16koSkoYSNq5O4qu+m/Fe9O5X +CtBoJKC5Jv3oSYCtkbVxXk1aQjS8Wv4v//NvFps3DYWhZ/KR8ps+GxtpUBq1/unB +ohKj8qGnDwLQOIvgGgfiyAieV1vrWkOr1283XTdRYjK6Uyo6/Eoxfo9PsxRZVACK +lpkRGn2p1GTbx442INrOwhJbJtKAcSlM7V8m2dAhgMTXXJzPtl0eCNWGy0s1obpK +1p9qTUz23s3WA425TAXQwIFpq0CdrXPgaXmZYw5LlfuwQhUmw5SQBOg8dX30Y5rb +vrrBBj1+2kAEYyeC3aQ3HwP6qAevWtmlbyCLMS5pYcPTRIEJlVX7OMCZSmvYnOku +0Vghgr2g+FZjNgQMo0HYEq6bb1jmpnN6tY4y3H7Zq76xavkFBWzaz8aHEB1koBoW +e0z+AaojAXEMaAKzsA3hF74HZA4kZf9kFi4er4ZLlnZFrU7UCN99mQfsV0LU+75E +rpzoRVYE8CZN9egiphZFYQKCAQEA8AUt1mi3tDfbyuDnut9KVkraCDAQ5WX+31H9 +BTPdIoXaeYx29zCKSEaBAMvf43s2pDgVPVyuc7KVFp8l3LZZ+znNWxU3eVznE2Ua +1QUTR8ZxueUIyOZxuSHU8xqWvxZ7SLZYTqTVmYGpORjvuQIgS/+6GrAbCNcZLNIo +I7U4/Nx4zJ3bkF1xaKJcaVXQRwG0cHQEZZdgkaE+J/ak3WTRC/11tEr4agGo+UDF +6XBQryp3HxPMnZyAv44zOB9gqD/FvnxH6S5ybc93eISQXZoJQuuad3GuEn28Gz5x +Bsr6zzSkhsTp2cEr2AVSH+b/SCseZ/2JEE0mjPxgjXRVLQrnGQKCAQEA08akFJjH +nq0aEcL8HWbUE2PuuXcq/vUZHFNv1gG+e6vZ7gU7bq+5SS3+APuerPQrhE+3mKsn +WKUfkr2Rgn2Yxieo/u+SKMbDaNNT5h08KgA/joewRYDeInACw9QHsKPKArWmWn6d +6fHYl5d3rxUPGUaM0fsdOvayX+xeN4PDn+puqzInhh3TFbeyvkpu1b/xZZ17u7Xz +yvu1PRHNt5eZB6QZ/GF3d1r63CTC7m3GF/wikoV9Ygy0Me8gw8WxqnF42GFRyUIa +wHXfb3w4Y4rZ17b7x5H18FF4tHMgXN0pNbGk+PoIFB3Erzfqi95xVyZh/MROwFiL +qy2KeRoniUbCwwKCAQBDvxJ7DD+dzI5rKyP9KP1Qcfwsh3SdazaPThL+nu7xyZoq +6KzDhJ3jXJMY6HKfQK3hmDrWgQx0d5mBMxZ6v7WSJXSDGu/3f3Nxk/4I1k/k2GxN +LgpWukSrHpN+sqiN8wiFM4KlX/0yQNjE1vcC30jCasHauo5G5n+imQbfXU1igdBO +4NeSXe2evQUcbi5FfIOzoeuDyUBmmn5yxTkvjD89BSNt6iNHuIQ7Jj82bo83geLx +kKMWcZAdgUOPubuMgcOMyoN5m7SMrhxolfIxmUK38sw8noeljHvFrNA2PKCiT5eI +upfO8KkxZf8SJh8z/YetjnBbe4tADBQsmQNZnVQxAoIBABNKmxPNPxHzTtajXngH +L/Z8OfjnJCGJjjoIV7209vcpFncaPum8VDKYX/US9sdmjrhE0sKzhKgMkq25WxH6 +Avq6Dij7BeN1B8P6zD/AFgT1dNS1A5exP4r/jSDtpa2vne1VQswnkJcJEuPsRljK +oE97H8TZDTab1m/qhkKkXCOrJV2u+e67tMjbrQqsmSAblg/dorHcx1KMT1w6zPSW +eLg7eKqG7m0O+p8nMiKqGUuCClwykNNnuNp7oA51adPO9mUvqFWfEfTKSApN1I0s +zt9ZqeHqJ+82XLqDakVLWD+t6QtNK4M5mvsjKtiG8OgxdOejslDPQBnd0ilp+oQE +0CUCggEATL5jg3Kcbq9HcaFEdgqzVXXcqNI4hOVWjP3Vp6W390PUZkmls1Igms3l +zc9fhibHv5gRq/Y3xQGUuP0dKuPuTykjXUTRozeal5ErkiLU6PZeVlz7xUHbLrF9 +J/IARiZtnfhcDh8qo33QZ4m4HUB5b54/4Gdzr7go8o0UVI50gXY65kwz5hV79/7P +S38hxS+X+k2FkWuCNrP6ECkC/L8jNM1h2gR4Ez6FziiNxYbOhQgsaTQRKjca/v/E +tzX9oZjMqiWtWKcxgSCZs/1DscXjjCsi64f/oOBShF1x4ZeiMoZTMhQBLMlbVx/n +JxRWkMCYLEbAZXWJXEKHyqaiIAOpkw== -----END PRIVATE KEY----- diff --git a/testdata/x509/spiffe_cert.pem b/testdata/x509/spiffe_cert.pem new file mode 100644 index 000000000000..bbe7d66a4bfb --- /dev/null +++ b/testdata/x509/spiffe_cert.pem @@ -0,0 +1,33 @@ +-----BEGIN CERTIFICATE----- +MIIFsjCCA5qgAwIBAgIURygVMMzdr+Q7rsUaz189JozyHMwwDQYJKoZIhvcNAQEL +BQAwTjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMQwwCgYDVQQHDANTVkwxDTAL +BgNVBAoMBGdSUEMxFTATBgNVBAMMDHRlc3QtY2xpZW50MTAeFw0yMTEyMjMxODQy +NTJaFw0zMTEyMjExODQyNTJaME4xCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEM +MAoGA1UEBwwDU1ZMMQ0wCwYDVQQKDARnUlBDMRUwEwYDVQQDDAx0ZXN0LWNsaWVu +dDEwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDJ4AqpGetyVSqGUuBJ +LVFla+7bEfca7UYzfVSSZLZ/X+JDmWIVN8UIPuFib5jhMEc3XaUnFXUmM7zEtz/Z +G5hapwLwOb2C3ZxOP6PQjYCJxbkLie+b43UQrFu1xxd3vMhVJgcj/AIxEpmszuqO +a6kUrkYifjJADQ+64kZgl66bsTdXMCzpxyFl9xUfff59L8OX+HUfAcoZz3emjg3Z +JPYURQEmjdZTOau1EjFilwHgd989Jt7NKgx30NXoHmw7nusVBIY94fL2VKN3f1XV +m0dHu5NI279Q6zr0ZBU7k5T3IeHnzsUesQS4NGlklDWoVTKk73Uv9Pna8yQsSW75 +7PEbHOGp9Knu4bnoGPOlsG81yIPipO6hTgGFK24pF97M9kpGbWqYX4+2vLlrCAfc +msHqaUPmQlYeRVTT6vw7ctYo2kyUYGtnODXk76LqewRBVvkzx75QUhfjAyb740Yc +DmIenc56Tq6gebJHjhEmVSehR6xIpXP7SVeurTyhPsEQnpJHtgs4dcwWOZp7BvPN +zHXmJqfr7vsshie3vS5kQ0u1e1yqAqXgyDjqKXOkx+dpgUTehSJHhPNHvTc5LXRs +vvXKYz6FrwR/DZ8t7BNEvPeLjFgxpH7QVJFLCvCbXs5K6yYbsnLfxFIBPRnrbJkI +sK+sQwnRdnsiUdPsTkG5B2lQfQIDAQABo4GHMIGEMB0GA1UdDgQWBBQ2lBp0PiRH +HvQ5IRURm8aHsj4RETAfBgNVHSMEGDAWgBQ2lBp0PiRHHvQ5IRURm8aHsj4RETAP +BgNVHRMBAf8EBTADAQH/MDEGA1UdEQQqMCiGJnNwaWZmZTovL2Zvby5iYXIuY29t +L2NsaWVudC93b3JrbG9hZC8xMA0GCSqGSIb3DQEBCwUAA4ICAQA1mSkgRclAl+E/ +aS9zJ7t8+Y4n3T24nOKKveSIjxXm/zjhWqVsLYBI6kglWtih2+PELvU8JdPqNZK3 +4Kl0Q6FWpVSGDdWN1i6NyORt2ocggL3ke3iXxRk3UpUKJmqwz81VhA2KUHnMlyE0 +IufFfZNwNWWHBv13uJfRbjeQpKPhU+yf4DeXrsWcvrZlGvAET+mcplafUzCp7Iv+ +PcISJtUerbxbVtuHVeZCLlgDXWkLAWJN8rf0dIG4x060LJ+j6j9uRVhb9sZn1HJV ++j4XdIYm1VKilluhOtNwP2d3Ox/JuTBxf7hFHXZPfMagQE5k5PzmxRaCAEMJ1l2D +vUbZw+shJfSNoWcBo2qadnUaWT3BmmJRBDh7ZReib/RQ1Rd4ygOyzP3E0vkV4/gq +yjLdApXh5PZP8KLQZ+1JN/sdWt7VfIt9wYOpkIqujdll51ESHzwQeAK9WVCB4UvV +z6zdhItB9CRbXPreWC+wCB1xDovIzFKOVsLs5+Gqs1m7VinG2LxbDqaKyo/FB0Hx +x0acBNzezLWoDwXYQrN0T0S4pnqhKD1CYPpdArBkNezUYAjS725FkApuK+mnBX3U +0msBffEaUEOkcyar1EW2m/33vpetD/k3eQQkmvQf4Hbiu9AF+9cNDm/hMuXEw5EX +GA91fn0891b5eEW8BJHXX0jri0aN8g== +-----END CERTIFICATE----- diff --git a/testdata/x509/spiffe_key.pem b/testdata/x509/spiffe_key.pem new file mode 100644 index 000000000000..77a33e9d0af0 --- /dev/null +++ b/testdata/x509/spiffe_key.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQDJ4AqpGetyVSqG +UuBJLVFla+7bEfca7UYzfVSSZLZ/X+JDmWIVN8UIPuFib5jhMEc3XaUnFXUmM7zE +tz/ZG5hapwLwOb2C3ZxOP6PQjYCJxbkLie+b43UQrFu1xxd3vMhVJgcj/AIxEpms +zuqOa6kUrkYifjJADQ+64kZgl66bsTdXMCzpxyFl9xUfff59L8OX+HUfAcoZz3em +jg3ZJPYURQEmjdZTOau1EjFilwHgd989Jt7NKgx30NXoHmw7nusVBIY94fL2VKN3 +f1XVm0dHu5NI279Q6zr0ZBU7k5T3IeHnzsUesQS4NGlklDWoVTKk73Uv9Pna8yQs +SW757PEbHOGp9Knu4bnoGPOlsG81yIPipO6hTgGFK24pF97M9kpGbWqYX4+2vLlr +CAfcmsHqaUPmQlYeRVTT6vw7ctYo2kyUYGtnODXk76LqewRBVvkzx75QUhfjAyb7 +40YcDmIenc56Tq6gebJHjhEmVSehR6xIpXP7SVeurTyhPsEQnpJHtgs4dcwWOZp7 +BvPNzHXmJqfr7vsshie3vS5kQ0u1e1yqAqXgyDjqKXOkx+dpgUTehSJHhPNHvTc5 +LXRsvvXKYz6FrwR/DZ8t7BNEvPeLjFgxpH7QVJFLCvCbXs5K6yYbsnLfxFIBPRnr +bJkIsK+sQwnRdnsiUdPsTkG5B2lQfQIDAQABAoICAQCXbolwqfHVFQ/OLRLzsZvy +UZGeIY7UUxKrAyPSoNvJFpr7DG7n7arOcaTOG1p56aYyYPvHIrB7FKpQggnSCYIy +1j89BoMjTKu4gsKWad72+ivB/RmRPYGOHUy6QftXpXQ9c0Y99weJ2iMO3zRR2269 +BbG0pCd7ppCbJquWP5IKVlhl/cxjHS3vd/YPZorlS1QUhpsMxGHfFKLzfHHk5nX1 +ZIHlctZIHeWw8VG8W/xbbnA2RhcxnY42vqAG+/NCkgZUAM3WU8zWfU0WEZ3Imy9Q +HuPv7m9H+vyBYSYQR7eh3nfAVHnHeRBKQX6hpQ/PEwHneXzVmZVnaaZD1l35+oQs +9mCmIC5PkwGe+vJI+Rxt5UElgzRDMVF6YuHfmobQQn5mT0Jsbdn6tfaFXa+e8+Ja +4WSlv/rVNSvCxNSwV0fOGIIrV1CIElf9ei7Go/Jewz94Eh2PdoOL9gNPqBeFFLmt +mS7HgST/Dkn8yAvyYgY5IDDWDiauECMV2F37QRk9stX+dUFhDQU3P/CcMH0SRyjx +vXRRvWY/5rWuinJcIr1Kb0OHiDztrM/4wAYc1BQjsL6CPKT3aAmCs/fxf0JflK/o +pvzsK5+AyBrvEBh9SXPcQYYEj6ZYCGZ/rPVlmgOjT/d4+xZf0FBUi3g+np7V15Km +ao4LgfcSQF5xUVTiyGo1AQKCAQEA423AcschurSrHwnmL1E5UIaYoFyKok2W3GZ6 +nu8Gp49GlF1v+CQ4w0zMNgigTTHxPyShK4/lC1n8W7liuo2TD98Km4qsnkD04R1z +Jf2PNBoVfMFEwxTA0t3LYfqJc3xLxWTR7wbvIRwtrcM/tFbU0RQAZ0/eE5Vine6t +fU8HBn/kSnUeCXMjFqheRccluYQgwES5ayJjnvf4Yrwe+In89+C6JE01LetqtxiL +U9X6iO9VpF9JVI7CPTBd7cZI3jfO+N2dXGlnt/Dp95S3RymoRSyMA1ILCCFZOeTo +Kh26/6hfgc2ox6ttN+s/1J9xPaXFmvbEAT27n+bDFVIfvF9QwQKCAQEA4zx4xLsW +AdljDLSDcLziWE1ikvO25hWiH3Q7ZyFi7vqby4d7FdgazNERrFZaCNQeSLfMakLd +zO1URfWsG7+6XvY45VtgVlqw2+uAHlE/B3FtrSeaIDr68AvWfPo02vckRMgTCBch +MEvul58A658mlybOkTHoRyaeDDD+83sIHAnyFzublxfsfbdgqzFW1RxfiUwmAM7w +9rI+PFPQnBgBkfyjcOfaVx/I1uvm3Nnl4ZUAmEdz6YlJdN6EhM8SjCP2LsLvhDUz +kjZ5WJ50ybRs3DWhDH/d06DmFlGwgu894TKBHqrq2fgDDukpZkH7cHldcjugzlHJ +c2CAO9MxI4UyvQKCAQAogEgQaKP6EuiSe3nRnV5el8mgbTqHEtg14c4edaSyvFIu +Y8Fn6FNvfEK1sK2TcbxrqUNGdbatYdYOI6KQZFv3LJo//t8kw56YZF04O8J/3dFL +yUNMlmqMYtEwXqSRu2Xm/kBgl9SICfOciTPUEs6NeUllHJUI2caZJ4Mf2K4Am0/1 +bovt1OI/y7YWKRPvyLboZpS6noItMi26r5O4YSJ6pjuf8VvyFIWJm8ZcJLQcJLsU +rZ9qfo3axb1EddZONJQYP6chaOf+mtmfrI1DEAkWYIuCn961EPNJ2xj5PxgpJTv0 +6sIO5NlrZuqUG9zXxKi/Iwjey7aZEEhXiKt8KWFBAoIBABtcRaJScHTqit2VwpnJ +dGtzbeIJzETp5+pnoVtqjrH9pNKdznkz2w48QieBAjg76iWRU+Cbin9JODNwQDfb +HwKeHP2owfHD27WvJm8AE1m/E5icwxcMYviSRFIqAkE3LrvFZ107A7j/+4twDrlQ +IWJjvs2Gt9QRV0hagegpMTHHFMotWC+aJtSARvh16WGhl/M9IvpH8IWTsqCq6txQ +m6fLRpaqpASHhDQ0lUiUR/Sgb0DmoZNF/3096bDgCfirv9GjkRlXGo2JV5UPBzre +KZleL7UElF4N6oZXcaxiSA4ceaWKqNpz3VJnSp/QZAkH4/OEMHmHKX1l6irJ5AnF +2PUCggEAQzXltQN+24eNh9nD9BzJR7CYOJQ4CzmHbkxJJZXGJtSCv0mYcEsSRmpH +5mW9hr8w6Y5WL4XfMduUX9od8exJtXl5d7Fl8oUNh+CwDr9smEjEDchh+6d11ZCi +Ervr1XOmNyOtpnQ1+N2nbbBEMLVns9yX6oNnl9mBdwpvwko7Q9ahyTvWnE2oG5At +8VeX/34k6BWdqPCJfnISMVbt6D+J0kaqaVw6BplTNSRs93bmqKwpcrRHMTida+bO +l9t8Cy3TguZdRWTJZ0kFmze0fV6dXYookZoUIeisTZBl701tOsvysZjxtMQJfTLJ +Io+0lEzXxTbCbBP/iyizjo62XTgpdQ== +-----END PRIVATE KEY----- diff --git a/trace.go b/trace.go index 07a2d26b3e77..9ded79321ba7 100644 --- a/trace.go +++ b/trace.go @@ -97,8 +97,8 @@ func truncate(x string, l int) string { // payload represents an RPC request or response payload. type payload struct { - sent bool // whether this is an outgoing payload - msg interface{} // e.g. a proto.Message + sent bool // whether this is an outgoing payload + msg any // e.g. a proto.Message // TODO(dsymonds): add stringifying info to codec, and limit how much we hold here? } @@ -111,7 +111,7 @@ func (p payload) String() string { type fmtStringer struct { format string - a []interface{} + a []any } func (f *fmtStringer) String() string { diff --git a/version.go b/version.go index 674c12f9a7e1..914ce665f53e 100644 --- a/version.go +++ b/version.go @@ -19,4 +19,4 @@ package grpc // Version is the current grpc version. -const Version = "1.32.0-dev" +const Version = "1.58.0" diff --git a/vet.sh b/vet.sh index c52c1bdbf4c9..bbc9e2e3c8e3 100755 --- a/vet.sh +++ b/vet.sh @@ -28,34 +28,23 @@ cleanup() { } trap cleanup EXIT -PATH="${GOPATH}/bin:${GOROOT}/bin:${PATH}" +PATH="${HOME}/go/bin:${GOROOT}/bin:${PATH}" +go version if [[ "$1" = "-install" ]]; then - # Check for module support - if go help mod >& /dev/null; then - # Install the pinned versions as defined in module tools. - pushd ./test/tools - go install \ - golang.org/x/lint/golint \ - golang.org/x/tools/cmd/goimports \ - honnef.co/go/tools/cmd/staticcheck \ - github.com/client9/misspell/cmd/misspell - popd - else - # Ye olde `go get` incantation. - # Note: this gets the latest version of all tools (vs. the pinned versions - # with Go modules). - go get -u \ - golang.org/x/lint/golint \ - golang.org/x/tools/cmd/goimports \ - honnef.co/go/tools/cmd/staticcheck \ - github.com/client9/misspell/cmd/misspell - fi + # Install the pinned versions as defined in module tools. + pushd ./test/tools + go install \ + golang.org/x/lint/golint \ + golang.org/x/tools/cmd/goimports \ + honnef.co/go/tools/cmd/staticcheck \ + github.com/client9/misspell/cmd/misspell + popd if [[ -z "${VET_SKIP_PROTO}" ]]; then - if [[ "${TRAVIS}" = "true" ]]; then - PROTOBUF_VERSION=3.3.0 + if [[ "${GITHUB_ACTIONS}" = "true" ]]; then + PROTOBUF_VERSION=22.0 # a.k.a v4.22.0 in pb.go files. PROTOC_FILENAME=protoc-${PROTOBUF_VERSION}-linux-x86_64.zip - pushd /home/travis + pushd /home/runner/go wget https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/${PROTOC_FILENAME} unzip ${PROTOC_FILENAME} bin/protoc --version @@ -69,8 +58,20 @@ elif [[ "$#" -ne 0 ]]; then die "Unknown argument(s): $*" fi +# - Check that generated proto files are up to date. +if [[ -z "${VET_SKIP_PROTO}" ]]; then + make proto && git status --porcelain 2>&1 | fail_on_output || \ + (git status; git --no-pager diff; exit 1) +fi + +if [[ -n "${VET_ONLY_PROTO}" ]]; then + exit 0 +fi + # - Ensure all source files contain a copyright message. -not git grep -L "\(Copyright [0-9]\{4,\} gRPC authors\)\|DO NOT EDIT" -- '*.go' +# (Done in two parts because Darwin "git grep" has broken support for compound +# exclusion matches.) +(grep -L "DO NOT EDIT" $(git grep -L "\(Copyright [0-9]\{4,\} gRPC authors\)" -- '*.go') || true) | fail_on_output # - Make sure all tests in grpc and grpc/test use leakcheck via Teardown. not grep 'func Test[^(]' *_test.go @@ -83,8 +84,11 @@ not git grep -l 'x/net/context' -- "*.go" # thread safety. git grep -l '"math/rand"' -- "*.go" 2>&1 | not grep -v '^examples\|^stress\|grpcrand\|^benchmark\|wrr_test' +# - Do not use "interface{}"; use "any" instead. +git grep -l 'interface{}' -- "*.go" 2>&1 | not grep -v '\.pb\.go\|protoc-gen-go-grpc' + # - Do not call grpclog directly. Use grpclog.Component instead. -git grep -l 'grpclog.I\|grpclog.W\|grpclog.E\|grpclog.F\|grpclog.V' -- "*.go" | not grep -v '^grpclog/component.go\|^internal/grpctest/tlogger_test.go' +git grep -l -e 'grpclog.I' --or -e 'grpclog.W' --or -e 'grpclog.E' --or -e 'grpclog.F' --or -e 'grpclog.V' -- "*.go" | not grep -v '^grpclog/component.go\|^internal/grpctest/tlogger_test.go' # - Ensure all ptypes proto packages are renamed when importing. not git grep "\(import \|^\s*\)\"github.com/golang/protobuf/ptypes/" -- "*.go" @@ -92,34 +96,32 @@ not git grep "\(import \|^\s*\)\"github.com/golang/protobuf/ptypes/" -- "*.go" # - Ensure all xds proto imports are renamed to *pb or *grpc. git grep '"github.com/envoyproxy/go-control-plane/envoy' -- '*.go' ':(exclude)*.pb.go' | not grep -v 'pb "\|grpc "' -# - gofmt, goimports, golint (with exceptions for generated code), go vet. -gofmt -s -d -l . 2>&1 | fail_on_output -goimports -l . 2>&1 | not grep -vE "\.pb\.go" -golint ./... 2>&1 | not grep -vE "\.pb\.go:" -go vet -all ./... - misspell -error . -# - Check that generated proto files are up to date. -if [[ -z "${VET_SKIP_PROTO}" ]]; then - PATH="/home/travis/bin:${PATH}" make proto && \ - git status --porcelain 2>&1 | fail_on_output || \ - (git status; git --no-pager diff; exit 1) -fi - -# - Check that our modules are tidy. -if go help mod >& /dev/null; then - find . -name 'go.mod' | xargs -IXXX bash -c 'cd $(dirname XXX); go mod tidy' +# - gofmt, goimports, golint (with exceptions for generated code), go vet, +# go mod tidy. +# Perform these checks on each module inside gRPC. +for MOD_FILE in $(find . -name 'go.mod'); do + MOD_DIR=$(dirname ${MOD_FILE}) + pushd ${MOD_DIR} + go vet -all ./... | fail_on_output + gofmt -s -d -l . 2>&1 | fail_on_output + goimports -l . 2>&1 | not grep -vE "\.pb\.go" + golint ./... 2>&1 | not grep -vE "/grpc_testing_not_regenerate/.*\.pb\.go:" + + go mod tidy -compat=1.19 git status --porcelain 2>&1 | fail_on_output || \ (git status; git --no-pager diff; exit 1) -fi + popd +done # - Collection of static analysis checks # # TODO(dfawley): don't use deprecated functions in examples or first-party # plugins. +# TODO(dfawley): enable ST1019 (duplicate imports) but allow for protobufs. SC_OUT="$(mktemp)" -staticcheck -go 1.9 -checks 'inherit,-ST1015' ./... > "${SC_OUT}" || true +staticcheck -go 1.19 -checks 'inherit,-ST1015,-ST1019,-SA1019' ./... > "${SC_OUT}" || true # Error if anything other than deprecation warnings are printed. not grep -v "is deprecated:.*SA1019" "${SC_OUT}" # Only ignore the following deprecated types/fields/functions. @@ -129,8 +131,11 @@ not grep -Fv '.CredsBundle .NewAddress .NewServiceConfig .Type is deprecated: use Attributes +BuildVersion is deprecated balancer.ErrTransientFailure balancer.Picker +extDesc.Filename is deprecated +github.com/golang/protobuf/jsonpb is deprecated grpc.CallCustomCodec grpc.Code grpc.Compressor @@ -143,7 +148,6 @@ grpc.NewGZIPDecompressor grpc.RPCCompressor grpc.RPCDecompressor grpc.ServiceConfig -grpc.WithBalancerName grpc.WithCompressor grpc.WithDecompressor grpc.WithDialer @@ -152,8 +156,24 @@ grpc.WithServiceConfig grpc.WithTimeout http.CloseNotifier info.SecurityVersion -resolver.Backend -resolver.GRPCLB' "${SC_OUT}" +proto is deprecated +proto.InternalMessageInfo is deprecated +proto.EnumName is deprecated +proto.ErrInternalBadWireType is deprecated +proto.FileDescriptor is deprecated +proto.Marshaler is deprecated +proto.MessageType is deprecated +proto.RegisterEnum is deprecated +proto.RegisterFile is deprecated +proto.RegisterType is deprecated +proto.RegisterExtension is deprecated +proto.RegisteredExtension is deprecated +proto.RegisteredExtensions is deprecated +proto.RegisterMapType is deprecated +proto.Unmarshaler is deprecated +Target is deprecated: Use the Target field in the BuildOptions instead. +xxx_messageInfo_ +' "${SC_OUT}" # - special golint on package comments. lint_package_comment_per_package() { diff --git a/xds/bootstrap/bootstrap.go b/xds/bootstrap/bootstrap.go new file mode 100644 index 000000000000..fcb99bdfd967 --- /dev/null +++ b/xds/bootstrap/bootstrap.go @@ -0,0 +1,64 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package bootstrap provides the functionality to register possible options +// for aspects of the xDS client through the bootstrap file. +// +// # Experimental +// +// Notice: This package is EXPERIMENTAL and may be changed or removed +// in a later release. +package bootstrap + +import ( + "encoding/json" + + "google.golang.org/grpc/credentials" +) + +// registry is a map from credential type name to Credential builder. +var registry = make(map[string]Credentials) + +// Credentials interface encapsulates a credentials.Bundle builder +// that can be used for communicating with the xDS Management server. +type Credentials interface { + // Build returns a credential bundle associated with this credential. + Build(config json.RawMessage) (credentials.Bundle, error) + // Name returns the credential name associated with this credential. + Name() string +} + +// RegisterCredentials registers Credentials used for connecting to the xds +// management server. +// +// NOTE: this function must only be called during initialization time (i.e. in +// an init() function), and is not thread-safe. If multiple credentials are +// registered with the same name, the one registered last will take effect. +func RegisterCredentials(c Credentials) { + registry[c.Name()] = c +} + +// GetCredentials returns the credentials associated with a given name. +// If no credentials are registered with the name, nil will be returned. +func GetCredentials(name string) Credentials { + if c, ok := registry[name]; ok { + return c + } + + return nil +} diff --git a/xds/bootstrap/bootstrap_test.go b/xds/bootstrap/bootstrap_test.go new file mode 100644 index 000000000000..80ae31ccd2e3 --- /dev/null +++ b/xds/bootstrap/bootstrap_test.go @@ -0,0 +1,63 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package bootstrap + +import ( + "encoding/json" + "testing" + + "google.golang.org/grpc/credentials" +) + +const testCredsBuilderName = "test_creds" + +var builder = &testCredsBuilder{} + +func init() { + RegisterCredentials(builder) +} + +type testCredsBuilder struct { + config json.RawMessage +} + +func (t *testCredsBuilder) Build(config json.RawMessage) (credentials.Bundle, error) { + t.config = config + return nil, nil +} + +func (t *testCredsBuilder) Name() string { + return testCredsBuilderName +} + +func TestRegisterNew(t *testing.T) { + c := GetCredentials(testCredsBuilderName) + if c == nil { + t.Fatalf("GetCredentials(%q) credential = nil", testCredsBuilderName) + } + + const sampleConfig = "sample_config" + rawMessage := json.RawMessage(sampleConfig) + if _, err := c.Build(rawMessage); err != nil { + t.Errorf("Build(%v) error = %v, want nil", rawMessage, err) + } + + if got, want := string(builder.config), sampleConfig; got != want { + t.Errorf("Build config = %v, want %v", got, want) + } +} diff --git a/xds/csds/csds.go b/xds/csds/csds.go new file mode 100644 index 000000000000..8d03124811a4 --- /dev/null +++ b/xds/csds/csds.go @@ -0,0 +1,181 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package csds implements features to dump the status (xDS responses) the +// xds_client is using. +// +// Notice: This package is EXPERIMENTAL and may be changed or removed in a later +// release. +package csds + +import ( + "context" + "fmt" + "io" + "sync" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/grpclog" + internalgrpclog "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/status" + "google.golang.org/grpc/xds/internal/xdsclient" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" + "google.golang.org/protobuf/types/known/timestamppb" + + v3adminpb "github.com/envoyproxy/go-control-plane/envoy/admin/v3" + v3statusgrpc "github.com/envoyproxy/go-control-plane/envoy/service/status/v3" + v3statuspb "github.com/envoyproxy/go-control-plane/envoy/service/status/v3" +) + +var logger = grpclog.Component("xds") + +const prefix = "[csds-server %p] " + +func prefixLogger(s *ClientStatusDiscoveryServer) *internalgrpclog.PrefixLogger { + return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, s)) +} + +// ClientStatusDiscoveryServer provides an implementation of the Client Status +// Discovery Service (CSDS) for exposing the xDS config of a given client. See +// https://github.com/envoyproxy/envoy/blob/main/api/envoy/service/status/v3/csds.proto. +// +// For more details about the gRPC implementation of CSDS, refer to gRPC A40 at: +// https://github.com/grpc/proposal/blob/master/A40-csds-support.md. +type ClientStatusDiscoveryServer struct { + logger *internalgrpclog.PrefixLogger + + mu sync.Mutex + xdsClient xdsclient.XDSClient + xdsClientClose func() +} + +// NewClientStatusDiscoveryServer returns an implementation of the CSDS server +// that can be registered on a gRPC server. +func NewClientStatusDiscoveryServer() (*ClientStatusDiscoveryServer, error) { + c, close, err := xdsclient.New() + if err != nil { + logger.Warningf("Failed to create xDS client: %v", err) + } + s := &ClientStatusDiscoveryServer{xdsClient: c, xdsClientClose: close} + s.logger = prefixLogger(s) + s.logger.Infof("Created CSDS server, with xdsClient %p", c) + return s, nil +} + +// StreamClientStatus implementations interface ClientStatusDiscoveryServiceServer. +func (s *ClientStatusDiscoveryServer) StreamClientStatus(stream v3statusgrpc.ClientStatusDiscoveryService_StreamClientStatusServer) error { + for { + req, err := stream.Recv() + if err == io.EOF { + return nil + } + if err != nil { + return err + } + resp, err := s.buildClientStatusRespForReq(req) + if err != nil { + return err + } + if err := stream.Send(resp); err != nil { + return err + } + } +} + +// FetchClientStatus implementations interface ClientStatusDiscoveryServiceServer. +func (s *ClientStatusDiscoveryServer) FetchClientStatus(_ context.Context, req *v3statuspb.ClientStatusRequest) (*v3statuspb.ClientStatusResponse, error) { + return s.buildClientStatusRespForReq(req) +} + +// buildClientStatusRespForReq fetches the status from the client, and returns +// the response to be sent back to xdsclient. +// +// If it returns an error, the error is a status error. +func (s *ClientStatusDiscoveryServer) buildClientStatusRespForReq(req *v3statuspb.ClientStatusRequest) (*v3statuspb.ClientStatusResponse, error) { + s.mu.Lock() + defer s.mu.Unlock() + + if s.xdsClient == nil { + return &v3statuspb.ClientStatusResponse{}, nil + } + // Field NodeMatchers is unsupported, by design + // https://github.com/grpc/proposal/blob/master/A40-csds-support.md#detail-node-matching. + if len(req.NodeMatchers) != 0 { + return nil, status.Errorf(codes.InvalidArgument, "node_matchers are not supported, request contains node_matchers: %v", req.NodeMatchers) + } + + dump := s.xdsClient.DumpResources() + ret := &v3statuspb.ClientStatusResponse{ + Config: []*v3statuspb.ClientConfig{ + { + Node: s.xdsClient.BootstrapConfig().NodeProto, + GenericXdsConfigs: dumpToGenericXdsConfig(dump), + }, + }, + } + return ret, nil +} + +// Close cleans up the resources. +func (s *ClientStatusDiscoveryServer) Close() { + if s.xdsClientClose != nil { + s.xdsClientClose() + } +} + +func dumpToGenericXdsConfig(dump map[string]map[string]xdsresource.UpdateWithMD) []*v3statuspb.ClientConfig_GenericXdsConfig { + var ret []*v3statuspb.ClientConfig_GenericXdsConfig + for typeURL, updates := range dump { + for name, update := range updates { + config := &v3statuspb.ClientConfig_GenericXdsConfig{ + TypeUrl: typeURL, + Name: name, + VersionInfo: update.MD.Version, + XdsConfig: update.Raw, + LastUpdated: timestamppb.New(update.MD.Timestamp), + ClientStatus: serviceStatusToProto(update.MD.Status), + } + if errState := update.MD.ErrState; errState != nil { + config.ErrorState = &v3adminpb.UpdateFailureState{ + LastUpdateAttempt: timestamppb.New(errState.Timestamp), + Details: errState.Err.Error(), + VersionInfo: errState.Version, + } + } + ret = append(ret, config) + } + } + return ret +} + +func serviceStatusToProto(serviceStatus xdsresource.ServiceStatus) v3adminpb.ClientResourceStatus { + switch serviceStatus { + case xdsresource.ServiceStatusUnknown: + return v3adminpb.ClientResourceStatus_UNKNOWN + case xdsresource.ServiceStatusRequested: + return v3adminpb.ClientResourceStatus_REQUESTED + case xdsresource.ServiceStatusNotExist: + return v3adminpb.ClientResourceStatus_DOES_NOT_EXIST + case xdsresource.ServiceStatusACKed: + return v3adminpb.ClientResourceStatus_ACKED + case xdsresource.ServiceStatusNACKed: + return v3adminpb.ClientResourceStatus_NACKED + default: + return v3adminpb.ClientResourceStatus_UNKNOWN + } +} diff --git a/xds/csds/csds_e2e_test.go b/xds/csds/csds_e2e_test.go new file mode 100644 index 000000000000..481e93929fa2 --- /dev/null +++ b/xds/csds/csds_e2e_test.go @@ -0,0 +1,461 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package csds_test + +import ( + "context" + "fmt" + "io" + "sort" + "strings" + "testing" + "time" + + "github.com/golang/protobuf/proto" + "github.com/google/go-cmp/cmp" + "github.com/google/uuid" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/bootstrap" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/xds/csds" + "google.golang.org/grpc/xds/internal/xdsclient" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" + "google.golang.org/protobuf/testing/protocmp" + "google.golang.org/protobuf/types/known/anypb" + + v3adminpb "github.com/envoyproxy/go-control-plane/envoy/admin/v3" + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3statuspb "github.com/envoyproxy/go-control-plane/envoy/service/status/v3" + v3statuspbgrpc "github.com/envoyproxy/go-control-plane/envoy/service/status/v3" + + _ "google.golang.org/grpc/xds/internal/httpfilter/router" // Register the router filter +) + +const defaultTestTimeout = 5 * time.Second + +var cmpOpts = cmp.Options{ + cmp.Transformer("sort", func(in []*v3statuspb.ClientConfig_GenericXdsConfig) []*v3statuspb.ClientConfig_GenericXdsConfig { + out := append([]*v3statuspb.ClientConfig_GenericXdsConfig(nil), in...) + sort.Slice(out, func(i, j int) bool { + a, b := out[i], out[j] + if a == nil { + return true + } + if b == nil { + return false + } + if strings.Compare(a.TypeUrl, b.TypeUrl) == 0 { + return strings.Compare(a.Name, b.Name) < 0 + } + return strings.Compare(a.TypeUrl, b.TypeUrl) < 0 + }) + return out + }), + protocmp.Transform(), + protocmp.IgnoreFields((*v3statuspb.ClientConfig_GenericXdsConfig)(nil), "last_updated"), + protocmp.IgnoreFields((*v3adminpb.UpdateFailureState)(nil), "last_update_attempt", "details"), +} + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// The following watcher implementations are no-ops since we don't really care +// about the callback received by these watchers in the test. We only care +// whether CSDS reports the expected state. + +type unimplementedListenerWatcher struct{} + +func (unimplementedListenerWatcher) OnUpdate(*xdsresource.ListenerResourceData) {} +func (unimplementedListenerWatcher) OnError(error) {} +func (unimplementedListenerWatcher) OnResourceDoesNotExist() {} + +type unimplementedRouteConfigWatcher struct{} + +func (unimplementedRouteConfigWatcher) OnUpdate(*xdsresource.RouteConfigResourceData) {} +func (unimplementedRouteConfigWatcher) OnError(error) {} +func (unimplementedRouteConfigWatcher) OnResourceDoesNotExist() {} + +type unimplementedClusterWatcher struct{} + +func (unimplementedClusterWatcher) OnUpdate(*xdsresource.ClusterResourceData) {} +func (unimplementedClusterWatcher) OnError(error) {} +func (unimplementedClusterWatcher) OnResourceDoesNotExist() {} + +type unimplementedEndpointsWatcher struct{} + +func (unimplementedEndpointsWatcher) OnUpdate(*xdsresource.EndpointsResourceData) {} +func (unimplementedEndpointsWatcher) OnError(error) {} +func (unimplementedEndpointsWatcher) OnResourceDoesNotExist() {} + +func (s) TestCSDS(t *testing.T) { + // Spin up a xDS management server on a local port. + nodeID := uuid.New().String() + mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) + if err != nil { + t.Fatal(err) + } + defer mgmtServer.Stop() + + // Create a bootstrap file in a temporary directory. + bootstrapCleanup, err := bootstrap.CreateFile(bootstrap.Options{ + NodeID: nodeID, + ServerURI: mgmtServer.Address, + }) + if err != nil { + t.Fatal(err) + } + defer bootstrapCleanup() + + // Create an xDS client. This will end up using the same singleton as used + // by the CSDS service. + xdsC, close, err := xdsclient.New() + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Initialize an gRPC server and register CSDS on it. + server := grpc.NewServer() + csdss, err := csds.NewClientStatusDiscoveryServer() + if err != nil { + t.Fatal(err) + } + v3statuspbgrpc.RegisterClientStatusDiscoveryServiceServer(server, csdss) + defer func() { + server.Stop() + csdss.Close() + }() + + // Create a local listener and pass it to Serve(). + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("testutils.LocalTCPListener() failed: %v", err) + } + go func() { + if err := server.Serve(lis); err != nil { + t.Errorf("Serve() failed: %v", err) + } + }() + + // Create a client to the CSDS server. + conn, err := grpc.Dial(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to dial CSDS server %q: %v", lis.Addr().String(), err) + } + c := v3statuspbgrpc.NewClientStatusDiscoveryServiceClient(conn) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + stream, err := c.StreamClientStatus(ctx, grpc.WaitForReady(true)) + if err != nil { + t.Fatalf("Failed to create a stream for CSDS: %v", err) + } + defer conn.Close() + + // Verify that the xDS client reports an empty config. + if err := checkClientStatusResponse(stream, nil); err != nil { + t.Fatal(err) + } + + // Initialize the xDS resources to be used in this test. + ldsTargets := []string{"lds.target.good:0000", "lds.target.good:1111"} + rdsTargets := []string{"route-config-0", "route-config-1"} + cdsTargets := []string{"cluster-0", "cluster-1"} + edsTargets := []string{"endpoints-0", "endpoints-1"} + listeners := make([]*v3listenerpb.Listener, len(ldsTargets)) + listenerAnys := make([]*anypb.Any, len(ldsTargets)) + for i := range ldsTargets { + listeners[i] = e2e.DefaultClientListener(ldsTargets[i], rdsTargets[i]) + listenerAnys[i] = testutils.MarshalAny(listeners[i]) + } + routes := make([]*v3routepb.RouteConfiguration, len(rdsTargets)) + routeAnys := make([]*anypb.Any, len(rdsTargets)) + for i := range rdsTargets { + routes[i] = e2e.DefaultRouteConfig(rdsTargets[i], ldsTargets[i], cdsTargets[i]) + routeAnys[i] = testutils.MarshalAny(routes[i]) + } + clusters := make([]*v3clusterpb.Cluster, len(cdsTargets)) + clusterAnys := make([]*anypb.Any, len(cdsTargets)) + for i := range cdsTargets { + clusters[i] = e2e.DefaultCluster(cdsTargets[i], edsTargets[i], e2e.SecurityLevelNone) + clusterAnys[i] = testutils.MarshalAny(clusters[i]) + } + endpoints := make([]*v3endpointpb.ClusterLoadAssignment, len(edsTargets)) + endpointAnys := make([]*anypb.Any, len(edsTargets)) + ips := []string{"0.0.0.0", "1.1.1.1"} + ports := []uint32{123, 456} + for i := range edsTargets { + endpoints[i] = e2e.DefaultEndpoint(edsTargets[i], ips[i], ports[i:i+1]) + endpointAnys[i] = testutils.MarshalAny(endpoints[i]) + } + + // Register watches on the xDS client for two resources of each type. + for _, target := range ldsTargets { + xdsresource.WatchListener(xdsC, target, unimplementedListenerWatcher{}) + } + for _, target := range rdsTargets { + xdsresource.WatchRouteConfig(xdsC, target, unimplementedRouteConfigWatcher{}) + } + for _, target := range cdsTargets { + xdsresource.WatchCluster(xdsC, target, unimplementedClusterWatcher{}) + } + for _, target := range edsTargets { + xdsresource.WatchEndpoints(xdsC, target, unimplementedEndpointsWatcher{}) + } + + // Verify that the xDS client reports the resources as being in "Requested" + // state. + want := []*v3statuspb.ClientConfig_GenericXdsConfig{} + for i := range ldsTargets { + want = append(want, makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[i], "", v3adminpb.ClientResourceStatus_REQUESTED, nil)) + } + for i := range rdsTargets { + want = append(want, makeGenericXdsConfig("type.googleapis.com/envoy.config.route.v3.RouteConfiguration", rdsTargets[i], "", v3adminpb.ClientResourceStatus_REQUESTED, nil)) + } + for i := range cdsTargets { + want = append(want, makeGenericXdsConfig("type.googleapis.com/envoy.config.cluster.v3.Cluster", cdsTargets[i], "", v3adminpb.ClientResourceStatus_REQUESTED, nil)) + } + for i := range edsTargets { + want = append(want, makeGenericXdsConfig("type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", edsTargets[i], "", v3adminpb.ClientResourceStatus_REQUESTED, nil)) + } + for { + if err := ctx.Err(); err != nil { + t.Fatalf("Timeout when waiting for resources in \"Requested\" state: %v", err) + } + if err := checkClientStatusResponse(stream, want); err == nil { + break + } + time.Sleep(time.Millisecond * 100) + } + + // Configure the management server with two resources of each type, + // corresponding to the watches registered above. + if err := mgmtServer.Update(ctx, e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: listeners, + Routes: routes, + Clusters: clusters, + Endpoints: endpoints, + }); err != nil { + t.Fatal(err) + } + + // Verify that the xDS client reports the resources as being in "ACKed" + // state, and in version "1". + want = nil + for i := range ldsTargets { + want = append(want, makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[i], "1", v3adminpb.ClientResourceStatus_ACKED, listenerAnys[i])) + } + for i := range rdsTargets { + want = append(want, makeGenericXdsConfig("type.googleapis.com/envoy.config.route.v3.RouteConfiguration", rdsTargets[i], "1", v3adminpb.ClientResourceStatus_ACKED, routeAnys[i])) + } + for i := range cdsTargets { + want = append(want, makeGenericXdsConfig("type.googleapis.com/envoy.config.cluster.v3.Cluster", cdsTargets[i], "1", v3adminpb.ClientResourceStatus_ACKED, clusterAnys[i])) + } + for i := range edsTargets { + want = append(want, makeGenericXdsConfig("type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", edsTargets[i], "1", v3adminpb.ClientResourceStatus_ACKED, endpointAnys[i])) + } + for { + if err := ctx.Err(); err != nil { + t.Fatalf("Timeout when waiting for resources in \"ACKed\" state: %v", err) + } + err := checkClientStatusResponse(stream, want) + if err == nil { + break + } + time.Sleep(time.Millisecond * 100) + } + + // Update the first resource of each type in the management server to a + // value which is expected to be NACK'ed by the xDS client. + const nackResourceIdx = 0 + listeners[nackResourceIdx].ApiListener = &v3listenerpb.ApiListener{} + routes[nackResourceIdx].VirtualHosts = []*v3routepb.VirtualHost{{Routes: []*v3routepb.Route{{}}}} + clusters[nackResourceIdx].ClusterDiscoveryType = &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_STATIC} + endpoints[nackResourceIdx].Endpoints = []*v3endpointpb.LocalityLbEndpoints{{}} + if err := mgmtServer.Update(ctx, e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: listeners, + Routes: routes, + Clusters: clusters, + Endpoints: endpoints, + SkipValidation: true, + }); err != nil { + t.Fatal(err) + } + + // Verify that the xDS client reports the first resource of each type as + // being in "NACKed" state, and the second resource of each type to be in + // "ACKed" state. The version for the ACKed resource would be "2", while + // that for the NACKed resource would be "1". In the NACKed resource, the + // version which is NACKed is stored in the ErrorState field. + want = nil + for i := range ldsTargets { + config := makeGenericXdsConfig("type.googleapis.com/envoy.config.listener.v3.Listener", ldsTargets[i], "2", v3adminpb.ClientResourceStatus_ACKED, listenerAnys[i]) + if i == nackResourceIdx { + config.VersionInfo = "1" + config.ClientStatus = v3adminpb.ClientResourceStatus_NACKED + config.ErrorState = &v3adminpb.UpdateFailureState{VersionInfo: "2"} + } + want = append(want, config) + } + for i := range rdsTargets { + config := makeGenericXdsConfig("type.googleapis.com/envoy.config.route.v3.RouteConfiguration", rdsTargets[i], "2", v3adminpb.ClientResourceStatus_ACKED, routeAnys[i]) + if i == nackResourceIdx { + config.VersionInfo = "1" + config.ClientStatus = v3adminpb.ClientResourceStatus_NACKED + config.ErrorState = &v3adminpb.UpdateFailureState{VersionInfo: "2"} + } + want = append(want, config) + } + for i := range cdsTargets { + config := makeGenericXdsConfig("type.googleapis.com/envoy.config.cluster.v3.Cluster", cdsTargets[i], "2", v3adminpb.ClientResourceStatus_ACKED, clusterAnys[i]) + if i == nackResourceIdx { + config.VersionInfo = "1" + config.ClientStatus = v3adminpb.ClientResourceStatus_NACKED + config.ErrorState = &v3adminpb.UpdateFailureState{VersionInfo: "2"} + } + want = append(want, config) + } + for i := range edsTargets { + config := makeGenericXdsConfig("type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", edsTargets[i], "2", v3adminpb.ClientResourceStatus_ACKED, endpointAnys[i]) + if i == nackResourceIdx { + config.VersionInfo = "1" + config.ClientStatus = v3adminpb.ClientResourceStatus_NACKED + config.ErrorState = &v3adminpb.UpdateFailureState{VersionInfo: "2"} + } + want = append(want, config) + } + for { + if err := ctx.Err(); err != nil { + t.Fatalf("Timeout when waiting for resources in \"NACKed\" state: %v", err) + } + err := checkClientStatusResponse(stream, want) + if err == nil { + break + } + time.Sleep(time.Millisecond * 100) + } +} + +func makeGenericXdsConfig(typeURL, name, version string, status v3adminpb.ClientResourceStatus, config *anypb.Any) *v3statuspb.ClientConfig_GenericXdsConfig { + return &v3statuspb.ClientConfig_GenericXdsConfig{ + TypeUrl: typeURL, + Name: name, + VersionInfo: version, + ClientStatus: status, + XdsConfig: config, + } +} + +func checkClientStatusResponse(stream v3statuspbgrpc.ClientStatusDiscoveryService_StreamClientStatusClient, want []*v3statuspb.ClientConfig_GenericXdsConfig) error { + if err := stream.Send(&v3statuspb.ClientStatusRequest{Node: nil}); err != nil { + if err != io.EOF { + return fmt.Errorf("failed to send ClientStatusRequest: %v", err) + } + // If the stream has closed, we call Recv() until it returns a non-nil + // error to get the actual error on the stream. + for { + if _, err := stream.Recv(); err != nil { + return fmt.Errorf("failed to recv ClientStatusResponse: %v", err) + } + } + } + resp, err := stream.Recv() + if err != nil { + return fmt.Errorf("failed to recv ClientStatusResponse: %v", err) + } + + if n := len(resp.Config); n != 1 { + return fmt.Errorf("got %d configs, want 1: %v", n, proto.MarshalTextString(resp)) + } + + if diff := cmp.Diff(resp.Config[0].GenericXdsConfigs, want, cmpOpts); diff != "" { + return fmt.Errorf(diff) + } + return nil +} + +func (s) TestCSDSNoXDSClient(t *testing.T) { + // Create a bootstrap file in a temporary directory. Since we pass empty + // options, it would end up creating a bootstrap file with an empty + // serverURI which will fail xDS client creation. + bootstrapCleanup, err := bootstrap.CreateFile(bootstrap.Options{}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { bootstrapCleanup() }) + + // Initialize an gRPC server and register CSDS on it. + server := grpc.NewServer() + csdss, err := csds.NewClientStatusDiscoveryServer() + if err != nil { + t.Fatal(err) + } + defer csdss.Close() + v3statuspbgrpc.RegisterClientStatusDiscoveryServiceServer(server, csdss) + + // Create a local listener and pass it to Serve(). + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("testutils.LocalTCPListener() failed: %v", err) + } + go func() { + if err := server.Serve(lis); err != nil { + t.Errorf("Serve() failed: %v", err) + } + }() + defer server.Stop() + + // Create a client to the CSDS server. + conn, err := grpc.Dial(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("Failed to dial CSDS server %q: %v", lis.Addr().String(), err) + } + defer conn.Close() + c := v3statuspbgrpc.NewClientStatusDiscoveryServiceClient(conn) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + stream, err := c.StreamClientStatus(ctx, grpc.WaitForReady(true)) + if err != nil { + t.Fatalf("Failed to create a stream for CSDS: %v", err) + } + + if err := stream.Send(&v3statuspb.ClientStatusRequest{Node: nil}); err != nil { + t.Fatalf("Failed to send ClientStatusRequest: %v", err) + } + r, err := stream.Recv() + if err != nil { + // io.EOF is not ok. + t.Fatalf("Failed to recv ClientStatusResponse: %v", err) + } + if n := len(r.Config); n != 0 { + t.Fatalf("got %d configs, want 0: %v", n, proto.MarshalTextString(r)) + } +} diff --git a/xds/googledirectpath/googlec2p.go b/xds/googledirectpath/googlec2p.go new file mode 100644 index 000000000000..f8f749835c24 --- /dev/null +++ b/xds/googledirectpath/googlec2p.go @@ -0,0 +1,216 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package googledirectpath implements a resolver that configures xds to make +// cloud to prod directpath connection. +// +// It's a combo of DNS and xDS resolvers. It delegates to DNS if +// - not on GCE, or +// - xDS bootstrap env var is set (so this client needs to do normal xDS, not +// direct path, and clients with this scheme is not part of the xDS mesh). +package googledirectpath + +import ( + "fmt" + "net/url" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/googlecloud" + internalgrpclog "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/grpcrand" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/xds/internal/xdsclient" + "google.golang.org/grpc/xds/internal/xdsclient/bootstrap" + "google.golang.org/protobuf/types/known/structpb" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + + _ "google.golang.org/grpc/xds" // To register xds resolvers and balancers. +) + +const ( + c2pScheme = "google-c2p" + c2pExperimentalScheme = "google-c2p-experimental" + c2pAuthority = "traffic-director-c2p.xds.googleapis.com" + + tdURL = "dns:///directpath-pa.googleapis.com" + httpReqTimeout = 10 * time.Second + zoneURL = "http://metadata.google.internal/computeMetadata/v1/instance/zone" + ipv6URL = "http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/ipv6s" + + gRPCUserAgentName = "gRPC Go" + clientFeatureNoOverprovisioning = "envoy.lb.does_not_support_overprovisioning" + ipv6CapableMetadataName = "TRAFFICDIRECTOR_DIRECTPATH_C2P_IPV6_CAPABLE" + + logPrefix = "[google-c2p-resolver]" + + dnsName, xdsName = "dns", "xds" +) + +// For overriding in unittests. +var ( + onGCE = googlecloud.OnGCE + + newClientWithConfig = func(config *bootstrap.Config) (xdsclient.XDSClient, func(), error) { + return xdsclient.NewWithConfig(config) + } + + logger = internalgrpclog.NewPrefixLogger(grpclog.Component("directpath"), logPrefix) +) + +func init() { + resolver.Register(c2pResolverBuilder{ + scheme: c2pScheme, + }) + // TODO(apolcyn): remove this experimental scheme before the 1.52 release + resolver.Register(c2pResolverBuilder{ + scheme: c2pExperimentalScheme, + }) +} + +type c2pResolverBuilder struct { + scheme string +} + +func (c2pResolverBuilder) Build(t resolver.Target, cc resolver.ClientConn, opts resolver.BuildOptions) (resolver.Resolver, error) { + if t.URL.Host != "" { + return nil, fmt.Errorf("google-c2p URI scheme does not support authorities") + } + + if !runDirectPath() { + // If not xDS, fallback to DNS. + t.URL.Scheme = dnsName + return resolver.Get(dnsName).Build(t, cc, opts) + } + + // Note that the following calls to getZone() and getIPv6Capable() does I/O, + // and has 10 seconds timeout each. + // + // This should be fine in most of the cases. In certain error cases, this + // could block Dial() for up to 10 seconds (each blocking call has its own + // goroutine). + zoneCh, ipv6CapableCh := make(chan string), make(chan bool) + go func() { zoneCh <- getZone(httpReqTimeout) }() + go func() { ipv6CapableCh <- getIPv6Capable(httpReqTimeout) }() + + balancerName := envconfig.C2PResolverTestOnlyTrafficDirectorURI + if balancerName == "" { + balancerName = tdURL + } + serverConfig, err := bootstrap.ServerConfigFromJSON([]byte(fmt.Sprintf(` + { + "server_uri": "%s", + "channel_creds": [{"type": "google_default"}], + "server_features": ["xds_v3", "ignore_resource_deletion"] + }`, balancerName))) + if err != nil { + return nil, fmt.Errorf("failed to build bootstrap configuration: %v", err) + } + config := &bootstrap.Config{ + XDSServer: serverConfig, + ClientDefaultListenerResourceNameTemplate: "%s", + Authorities: map[string]*bootstrap.Authority{ + c2pAuthority: { + XDSServer: serverConfig, + }, + }, + NodeProto: newNode(<-zoneCh, <-ipv6CapableCh), + } + + // Create singleton xds client with this config. The xds client will be + // used by the xds resolver later. + _, close, err := newClientWithConfig(config) + if err != nil { + return nil, fmt.Errorf("failed to start xDS client: %v", err) + } + + // Create and return an xDS resolver. + t.URL.Scheme = xdsName + if envconfig.XDSFederation { + t = resolver.Target{ + URL: url.URL{ + Scheme: xdsName, + Host: c2pAuthority, + Path: t.URL.Path, + }, + } + } + xdsR, err := resolver.Get(xdsName).Build(t, cc, opts) + if err != nil { + close() + return nil, err + } + return &c2pResolver{ + Resolver: xdsR, + clientCloseFunc: close, + }, nil +} + +func (b c2pResolverBuilder) Scheme() string { + return b.scheme +} + +type c2pResolver struct { + resolver.Resolver + clientCloseFunc func() +} + +func (r *c2pResolver) Close() { + r.Resolver.Close() + r.clientCloseFunc() +} + +var ipv6EnabledMetadata = &structpb.Struct{ + Fields: map[string]*structpb.Value{ + ipv6CapableMetadataName: structpb.NewBoolValue(true), + }, +} + +var id = fmt.Sprintf("C2P-%d", grpcrand.Int()) + +// newNode makes a copy of defaultNode, and populate it's Metadata and +// Locality fields. +func newNode(zone string, ipv6Capable bool) *v3corepb.Node { + ret := &v3corepb.Node{ + // Not all required fields are set in defaultNote. Metadata will be set + // if ipv6 is enabled. Locality will be set to the value from metadata. + Id: id, + UserAgentName: gRPCUserAgentName, + UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version}, + ClientFeatures: []string{clientFeatureNoOverprovisioning}, + } + ret.Locality = &v3corepb.Locality{Zone: zone} + if ipv6Capable { + ret.Metadata = ipv6EnabledMetadata + } + return ret +} + +// runDirectPath returns whether this resolver should use direct path. +// +// direct path is enabled if this client is running on GCE, and the normal xDS +// is not used (bootstrap env vars are not set) or federation is enabled. +func runDirectPath() bool { + if !onGCE() { + return false + } + return envconfig.XDSFederation || envconfig.XDSBootstrapFileName == "" && envconfig.XDSBootstrapFileContent == "" +} diff --git a/xds/googledirectpath/googlec2p_test.go b/xds/googledirectpath/googlec2p_test.go new file mode 100644 index 000000000000..945202fe71a9 --- /dev/null +++ b/xds/googledirectpath/googlec2p_test.go @@ -0,0 +1,258 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package googledirectpath + +import ( + "fmt" + "strconv" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/xds/internal/xdsclient" + "google.golang.org/grpc/xds/internal/xdsclient/bootstrap" + "google.golang.org/protobuf/testing/protocmp" + "google.golang.org/protobuf/types/known/structpb" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" +) + +type emptyResolver struct { + resolver.Resolver + scheme string +} + +func (er *emptyResolver) Build(_ resolver.Target, _ resolver.ClientConn, _ resolver.BuildOptions) (resolver.Resolver, error) { + return er, nil +} + +func (er *emptyResolver) Scheme() string { + return er.scheme +} + +func (er *emptyResolver) Close() {} + +var ( + testDNSResolver = &emptyResolver{scheme: "dns"} + testXDSResolver = &emptyResolver{scheme: "xds"} +) + +func replaceResolvers() func() { + oldDNS := resolver.Get("dns") + resolver.Register(testDNSResolver) + oldXDS := resolver.Get("xds") + resolver.Register(testXDSResolver) + return func() { + resolver.Register(oldDNS) + resolver.Register(oldXDS) + } +} + +// Test that when bootstrap env is set, fallback to DNS. +func TestBuildWithBootstrapEnvSet(t *testing.T) { + defer replaceResolvers()() + builder := resolver.Get(c2pScheme) + + for i, envP := range []*string{&envconfig.XDSBootstrapFileName, &envconfig.XDSBootstrapFileContent} { + t.Run(strconv.Itoa(i), func(t *testing.T) { + // Set bootstrap config env var. + oldEnv := *envP + *envP = "does not matter" + defer func() { *envP = oldEnv }() + + // Build should return DNS, not xDS. + r, err := builder.Build(resolver.Target{}, nil, resolver.BuildOptions{}) + if err != nil { + t.Fatalf("failed to build resolver: %v", err) + } + if r != testDNSResolver { + t.Fatalf("want dns resolver, got %#v", r) + } + }) + } +} + +// Test that when not on GCE, fallback to DNS. +func TestBuildNotOnGCE(t *testing.T) { + defer replaceResolvers()() + builder := resolver.Get(c2pScheme) + + oldOnGCE := onGCE + onGCE = func() bool { return false } + defer func() { onGCE = oldOnGCE }() + + // Build should return DNS, not xDS. + r, err := builder.Build(resolver.Target{}, nil, resolver.BuildOptions{}) + if err != nil { + t.Fatalf("failed to build resolver: %v", err) + } + if r != testDNSResolver { + t.Fatalf("want dns resolver, got %#v", r) + } +} + +type testXDSClient struct { + xdsclient.XDSClient + closed chan struct{} +} + +func (c *testXDSClient) Close() { + c.closed <- struct{}{} +} + +// Test that when xDS is built, the client is built with the correct config. +func TestBuildXDS(t *testing.T) { + defer replaceResolvers()() + builder := resolver.Get(c2pScheme) + + oldOnGCE := onGCE + onGCE = func() bool { return true } + defer func() { onGCE = oldOnGCE }() + + const testZone = "test-zone" + oldGetZone := getZone + getZone = func(time.Duration) string { return testZone } + defer func() { getZone = oldGetZone }() + + for _, tt := range []struct { + name string + ipv6 bool + tdURI string // traffic director URI will be overridden if this is set. + }{ + {name: "ipv6 true", ipv6: true}, + {name: "ipv6 false", ipv6: false}, + {name: "override TD URI", ipv6: true, tdURI: "test-uri"}, + } { + t.Run(tt.name, func(t *testing.T) { + oldGetIPv6Capability := getIPv6Capable + getIPv6Capable = func(time.Duration) bool { return tt.ipv6 } + defer func() { getIPv6Capable = oldGetIPv6Capability }() + + if tt.tdURI != "" { + oldURI := envconfig.C2PResolverTestOnlyTrafficDirectorURI + envconfig.C2PResolverTestOnlyTrafficDirectorURI = tt.tdURI + defer func() { + envconfig.C2PResolverTestOnlyTrafficDirectorURI = oldURI + }() + } + + tXDSClient := &testXDSClient{closed: make(chan struct{}, 1)} + + configCh := make(chan *bootstrap.Config, 1) + oldNewClient := newClientWithConfig + newClientWithConfig = func(config *bootstrap.Config) (xdsclient.XDSClient, func(), error) { + configCh <- config + return tXDSClient, func() { tXDSClient.Close() }, nil + } + defer func() { newClientWithConfig = oldNewClient }() + + // Build should return DNS, not xDS. + r, err := builder.Build(resolver.Target{}, nil, resolver.BuildOptions{}) + if err != nil { + t.Fatalf("failed to build resolver: %v", err) + } + rr := r.(*c2pResolver) + if rrr := rr.Resolver; rrr != testXDSResolver { + t.Fatalf("want xds resolver, got %#v, ", rrr) + } + + wantNode := &v3corepb.Node{ + Id: id, + Metadata: nil, + Locality: &v3corepb.Locality{Zone: testZone}, + UserAgentName: gRPCUserAgentName, + UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version}, + ClientFeatures: []string{clientFeatureNoOverprovisioning}, + } + if tt.ipv6 { + wantNode.Metadata = &structpb.Struct{ + Fields: map[string]*structpb.Value{ + ipv6CapableMetadataName: { + Kind: &structpb.Value_BoolValue{BoolValue: true}, + }, + }, + } + } + wantServerConfig, err := bootstrap.ServerConfigFromJSON([]byte(fmt.Sprintf(`{ + "server_uri": "%s", + "channel_creds": [{"type": "google_default"}], + "server_features": ["xds_v3", "ignore_resource_deletion"] + }`, tdURL))) + if err != nil { + t.Fatalf("Failed to build server bootstrap config: %v", err) + } + wantConfig := &bootstrap.Config{ + XDSServer: wantServerConfig, + ClientDefaultListenerResourceNameTemplate: "%s", + Authorities: map[string]*bootstrap.Authority{ + "traffic-director-c2p.xds.googleapis.com": { + XDSServer: wantServerConfig, + }, + }, + NodeProto: wantNode, + } + if tt.tdURI != "" { + wantConfig.XDSServer.ServerURI = tt.tdURI + } + cmpOpts := cmp.Options{ + cmpopts.IgnoreFields(bootstrap.ServerConfig{}, "Creds"), + cmp.AllowUnexported(bootstrap.ServerConfig{}), + protocmp.Transform(), + } + select { + case gotConfig := <-configCh: + if diff := cmp.Diff(wantConfig, gotConfig, cmpOpts); diff != "" { + t.Fatalf("Unexpected diff in bootstrap config (-want +got):\n%s", diff) + } + case <-time.After(time.Second): + t.Fatalf("timeout waiting for client config") + } + + r.Close() + select { + case <-tXDSClient.closed: + case <-time.After(time.Second): + t.Fatalf("timeout waiting for client close") + } + }) + } +} + +// TestDialFailsWhenTargetContainsAuthority attempts to Dial a target URI of +// google-c2p scheme with a non-empty authority and verifies that it fails with +// an expected error. +func TestBuildFailsWhenCalledWithAuthority(t *testing.T) { + uri := "google-c2p://an-authority/resource" + cc, err := grpc.Dial(uri, grpc.WithTransportCredentials(insecure.NewCredentials())) + defer func() { + if cc != nil { + cc.Close() + } + }() + wantErr := "google-c2p URI scheme does not support authorities" + if err == nil || !strings.Contains(err.Error(), wantErr) { + t.Fatalf("grpc.Dial(%s) returned error: %v, want: %v", uri, err, wantErr) + } +} diff --git a/xds/googledirectpath/utils.go b/xds/googledirectpath/utils.go new file mode 100644 index 000000000000..de33cf48d0e5 --- /dev/null +++ b/xds/googledirectpath/utils.go @@ -0,0 +1,96 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package googledirectpath + +import ( + "bytes" + "fmt" + "io" + "net/http" + "net/url" + "sync" + "time" +) + +func getFromMetadata(timeout time.Duration, urlStr string) ([]byte, error) { + parsedURL, err := url.Parse(urlStr) + if err != nil { + return nil, err + } + client := &http.Client{Timeout: timeout} + req := &http.Request{ + Method: http.MethodGet, + URL: parsedURL, + Header: http.Header{"Metadata-Flavor": {"Google"}}, + } + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed communicating with metadata server: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("metadata server returned resp with non-OK: %v", resp) + } + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed reading from metadata server: %v", err) + } + return body, nil +} + +var ( + zone string + zoneOnce sync.Once +) + +// Defined as var to be overridden in tests. +var getZone = func(timeout time.Duration) string { + zoneOnce.Do(func() { + qualifiedZone, err := getFromMetadata(timeout, zoneURL) + if err != nil { + logger.Warningf("could not discover instance zone: %v", err) + return + } + i := bytes.LastIndexByte(qualifiedZone, '/') + if i == -1 { + logger.Warningf("could not parse zone from metadata server: %s", qualifiedZone) + return + } + zone = string(qualifiedZone[i+1:]) + }) + return zone +} + +var ( + ipv6Capable bool + ipv6CapableOnce sync.Once +) + +// Defined as var to be overridden in tests. +var getIPv6Capable = func(timeout time.Duration) bool { + ipv6CapableOnce.Do(func() { + _, err := getFromMetadata(timeout, ipv6URL) + if err != nil { + logger.Warningf("could not discover ipv6 capability: %v", err) + return + } + ipv6Capable = true + }) + return ipv6Capable +} diff --git a/xds/internal/balancer/balancer.go b/xds/internal/balancer/balancer.go index d0b1bb786c7d..ff27af026db5 100644 --- a/xds/internal/balancer/balancer.go +++ b/xds/internal/balancer/balancer.go @@ -20,8 +20,12 @@ package balancer import ( - _ "google.golang.org/grpc/xds/internal/balancer/cdsbalancer" // Register the CDS balancer - _ "google.golang.org/grpc/xds/internal/balancer/edsbalancer" // Register the EDS balancer - _ "google.golang.org/grpc/xds/internal/balancer/weightedtarget" // Register the weighted_target balancer - _ "google.golang.org/grpc/xds/internal/balancer/xdsrouting" // Register the xds_routing balancer + _ "google.golang.org/grpc/balancer/leastrequest" // Register the least_request_experimental balancer + _ "google.golang.org/grpc/balancer/weightedtarget" // Register the weighted_target balancer + _ "google.golang.org/grpc/xds/internal/balancer/cdsbalancer" // Register the CDS balancer + _ "google.golang.org/grpc/xds/internal/balancer/clusterimpl" // Register the xds_cluster_impl balancer + _ "google.golang.org/grpc/xds/internal/balancer/clustermanager" // Register the xds_cluster_manager balancer + _ "google.golang.org/grpc/xds/internal/balancer/clusterresolver" // Register the xds_cluster_resolver balancer + _ "google.golang.org/grpc/xds/internal/balancer/outlierdetection" // Register the outlier_detection balancer + _ "google.golang.org/grpc/xds/internal/balancer/priority" // Register the priority balancer ) diff --git a/xds/internal/balancer/balancergroup/balancergroup_test.go b/xds/internal/balancer/balancergroup/balancergroup_test.go deleted file mode 100644 index 347404989d01..000000000000 --- a/xds/internal/balancer/balancergroup/balancergroup_test.go +++ /dev/null @@ -1,917 +0,0 @@ -/* - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// All tests in this file are combination of balancer group and -// weighted_balancerstate_aggregator, aka weighted_target tests. The difference -// is weighted_target tests cannot add sub-balancers to balancer group directly, -// they instead uses balancer config to control sub-balancers. Even though not -// very suited, the tests still cover all the functionality. -// -// TODO: the tests should be moved to weighted_target, and balancer group's -// tests should use a mock balancerstate_aggregator. - -package balancergroup - -import ( - "fmt" - "testing" - "time" - - orcapb "github.com/cncf/udpa/go/udpa/data/orca/v1" - "github.com/google/go-cmp/cmp" - "google.golang.org/grpc" - "google.golang.org/grpc/balancer" - "google.golang.org/grpc/balancer/roundrobin" - "google.golang.org/grpc/connectivity" - "google.golang.org/grpc/resolver" - "google.golang.org/grpc/xds/internal" - "google.golang.org/grpc/xds/internal/balancer/lrs" - "google.golang.org/grpc/xds/internal/balancer/weightedtarget/weightedaggregator" - "google.golang.org/grpc/xds/internal/testutils" -) - -var ( - rrBuilder = balancer.Get(roundrobin.Name) - pfBuilder = balancer.Get(grpc.PickFirstBalancerName) - testBalancerIDs = []internal.LocalityID{{Region: "b1"}, {Region: "b2"}, {Region: "b3"}} - testBackendAddrs []resolver.Address -) - -const testBackendAddrsCount = 12 - -func init() { - for i := 0; i < testBackendAddrsCount; i++ { - testBackendAddrs = append(testBackendAddrs, resolver.Address{Addr: fmt.Sprintf("%d.%d.%d.%d:%d", i, i, i, i, i)}) - } - - // Disable caching for all tests. It will be re-enabled in caching specific - // tests. - DefaultSubBalancerCloseTimeout = time.Millisecond -} - -func subConnFromPicker(p balancer.Picker) func() balancer.SubConn { - return func() balancer.SubConn { - scst, _ := p.Pick(balancer.PickInfo{}) - return scst.SubConn - } -} - -func newTestBalancerGroup(t *testing.T, loadStore lrs.Store) (*testutils.TestClientConn, *weightedaggregator.Aggregator, *BalancerGroup) { - cc := testutils.NewTestClientConn(t) - gator := weightedaggregator.New(cc, nil, testutils.NewTestWRR) - gator.Start() - bg := New(cc, gator, loadStore, nil) - bg.Start() - return cc, gator, bg -} - -// 1 balancer, 1 backend -> 2 backends -> 1 backend. -func (s) TestBalancerGroup_OneRR_AddRemoveBackend(t *testing.T) { - cc, gator, bg := newTestBalancerGroup(t, nil) - - // Add one balancer to group. - gator.Add(testBalancerIDs[0], 1) - bg.Add(testBalancerIDs[0], rrBuilder) - // Send one resolved address. - bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:1]}}) - - // Send subconn state change. - sc1 := <-cc.NewSubConnCh - bg.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - bg.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - - // Test pick with one backend. - p1 := <-cc.NewPickerCh - for i := 0; i < 5; i++ { - gotSCSt, _ := p1.Pick(balancer.PickInfo{}) - if !cmp.Equal(gotSCSt.SubConn, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) { - t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc1) - } - } - - // Send two addresses. - bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:2]}}) - // Expect one new subconn, send state update. - sc2 := <-cc.NewSubConnCh - bg.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - bg.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - - // Test roundrobin pick. - p2 := <-cc.NewPickerCh - want := []balancer.SubConn{sc1, sc2} - if err := testutils.IsRoundRobin(want, subConnFromPicker(p2)); err != nil { - t.Fatalf("want %v, got %v", want, err) - } - - // Remove the first address. - bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[1:2]}}) - scToRemove := <-cc.RemoveSubConnCh - if !cmp.Equal(scToRemove, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) { - t.Fatalf("RemoveSubConn, want %v, got %v", sc1, scToRemove) - } - bg.UpdateSubConnState(scToRemove, balancer.SubConnState{ConnectivityState: connectivity.Shutdown}) - - // Test pick with only the second subconn. - p3 := <-cc.NewPickerCh - for i := 0; i < 5; i++ { - gotSC, _ := p3.Pick(balancer.PickInfo{}) - if !cmp.Equal(gotSC.SubConn, sc2, cmp.AllowUnexported(testutils.TestSubConn{})) { - t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSC, sc2) - } - } -} - -// 2 balancers, each with 1 backend. -func (s) TestBalancerGroup_TwoRR_OneBackend(t *testing.T) { - cc, gator, bg := newTestBalancerGroup(t, nil) - - // Add two balancers to group and send one resolved address to both - // balancers. - gator.Add(testBalancerIDs[0], 1) - bg.Add(testBalancerIDs[0], rrBuilder) - bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:1]}}) - sc1 := <-cc.NewSubConnCh - - gator.Add(testBalancerIDs[1], 1) - bg.Add(testBalancerIDs[1], rrBuilder) - bg.UpdateClientConnState(testBalancerIDs[1], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:1]}}) - sc2 := <-cc.NewSubConnCh - - // Send state changes for both subconns. - bg.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - bg.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - bg.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - bg.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - - // Test roundrobin on the last picker. - p1 := <-cc.NewPickerCh - want := []balancer.SubConn{sc1, sc2} - if err := testutils.IsRoundRobin(want, subConnFromPicker(p1)); err != nil { - t.Fatalf("want %v, got %v", want, err) - } -} - -// 2 balancers, each with more than 1 backends. -func (s) TestBalancerGroup_TwoRR_MoreBackends(t *testing.T) { - cc, gator, bg := newTestBalancerGroup(t, nil) - - // Add two balancers to group and send one resolved address to both - // balancers. - gator.Add(testBalancerIDs[0], 1) - bg.Add(testBalancerIDs[0], rrBuilder) - bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:2]}}) - sc1 := <-cc.NewSubConnCh - sc2 := <-cc.NewSubConnCh - - gator.Add(testBalancerIDs[1], 1) - bg.Add(testBalancerIDs[1], rrBuilder) - bg.UpdateClientConnState(testBalancerIDs[1], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[2:4]}}) - sc3 := <-cc.NewSubConnCh - sc4 := <-cc.NewSubConnCh - - // Send state changes for both subconns. - bg.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - bg.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - bg.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - bg.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - bg.UpdateSubConnState(sc3, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - bg.UpdateSubConnState(sc3, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - bg.UpdateSubConnState(sc4, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - bg.UpdateSubConnState(sc4, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - - // Test roundrobin on the last picker. - p1 := <-cc.NewPickerCh - want := []balancer.SubConn{sc1, sc2, sc3, sc4} - if err := testutils.IsRoundRobin(want, subConnFromPicker(p1)); err != nil { - t.Fatalf("want %v, got %v", want, err) - } - - // Turn sc2's connection down, should be RR between balancers. - bg.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) - p2 := <-cc.NewPickerCh - // Expect two sc1's in the result, because balancer1 will be picked twice, - // but there's only one sc in it. - want = []balancer.SubConn{sc1, sc1, sc3, sc4} - if err := testutils.IsRoundRobin(want, subConnFromPicker(p2)); err != nil { - t.Fatalf("want %v, got %v", want, err) - } - - // Remove sc3's addresses. - bg.UpdateClientConnState(testBalancerIDs[1], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[3:4]}}) - scToRemove := <-cc.RemoveSubConnCh - if !cmp.Equal(scToRemove, sc3, cmp.AllowUnexported(testutils.TestSubConn{})) { - t.Fatalf("RemoveSubConn, want %v, got %v", sc3, scToRemove) - } - bg.UpdateSubConnState(scToRemove, balancer.SubConnState{ConnectivityState: connectivity.Shutdown}) - p3 := <-cc.NewPickerCh - want = []balancer.SubConn{sc1, sc4} - if err := testutils.IsRoundRobin(want, subConnFromPicker(p3)); err != nil { - t.Fatalf("want %v, got %v", want, err) - } - - // Turn sc1's connection down. - bg.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) - p4 := <-cc.NewPickerCh - want = []balancer.SubConn{sc4} - if err := testutils.IsRoundRobin(want, subConnFromPicker(p4)); err != nil { - t.Fatalf("want %v, got %v", want, err) - } - - // Turn last connection to connecting. - bg.UpdateSubConnState(sc4, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - p5 := <-cc.NewPickerCh - for i := 0; i < 5; i++ { - if _, err := p5.Pick(balancer.PickInfo{}); err != balancer.ErrNoSubConnAvailable { - t.Fatalf("want pick error %v, got %v", balancer.ErrNoSubConnAvailable, err) - } - } - - // Turn all connections down. - bg.UpdateSubConnState(sc4, balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) - p6 := <-cc.NewPickerCh - for i := 0; i < 5; i++ { - if _, err := p6.Pick(balancer.PickInfo{}); err != balancer.ErrTransientFailure { - t.Fatalf("want pick error %v, got %v", balancer.ErrTransientFailure, err) - } - } -} - -// 2 balancers with different weights. -func (s) TestBalancerGroup_TwoRR_DifferentWeight_MoreBackends(t *testing.T) { - cc, gator, bg := newTestBalancerGroup(t, nil) - - // Add two balancers to group and send two resolved addresses to both - // balancers. - gator.Add(testBalancerIDs[0], 2) - bg.Add(testBalancerIDs[0], rrBuilder) - bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:2]}}) - sc1 := <-cc.NewSubConnCh - sc2 := <-cc.NewSubConnCh - - gator.Add(testBalancerIDs[1], 1) - bg.Add(testBalancerIDs[1], rrBuilder) - bg.UpdateClientConnState(testBalancerIDs[1], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[2:4]}}) - sc3 := <-cc.NewSubConnCh - sc4 := <-cc.NewSubConnCh - - // Send state changes for both subconns. - bg.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - bg.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - bg.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - bg.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - bg.UpdateSubConnState(sc3, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - bg.UpdateSubConnState(sc3, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - bg.UpdateSubConnState(sc4, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - bg.UpdateSubConnState(sc4, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - - // Test roundrobin on the last picker. - p1 := <-cc.NewPickerCh - want := []balancer.SubConn{sc1, sc1, sc2, sc2, sc3, sc4} - if err := testutils.IsRoundRobin(want, subConnFromPicker(p1)); err != nil { - t.Fatalf("want %v, got %v", want, err) - } -} - -// totally 3 balancers, add/remove balancer. -func (s) TestBalancerGroup_ThreeRR_RemoveBalancer(t *testing.T) { - cc, gator, bg := newTestBalancerGroup(t, nil) - - // Add three balancers to group and send one resolved address to both - // balancers. - gator.Add(testBalancerIDs[0], 1) - bg.Add(testBalancerIDs[0], rrBuilder) - bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:1]}}) - sc1 := <-cc.NewSubConnCh - - gator.Add(testBalancerIDs[1], 1) - bg.Add(testBalancerIDs[1], rrBuilder) - bg.UpdateClientConnState(testBalancerIDs[1], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[1:2]}}) - sc2 := <-cc.NewSubConnCh - - gator.Add(testBalancerIDs[2], 1) - bg.Add(testBalancerIDs[2], rrBuilder) - bg.UpdateClientConnState(testBalancerIDs[2], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[1:2]}}) - sc3 := <-cc.NewSubConnCh - - // Send state changes for both subconns. - bg.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - bg.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - bg.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - bg.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - bg.UpdateSubConnState(sc3, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - bg.UpdateSubConnState(sc3, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - - p1 := <-cc.NewPickerCh - want := []balancer.SubConn{sc1, sc2, sc3} - if err := testutils.IsRoundRobin(want, subConnFromPicker(p1)); err != nil { - t.Fatalf("want %v, got %v", want, err) - } - - // Remove the second balancer, while the others two are ready. - gator.Remove(testBalancerIDs[1]) - bg.Remove(testBalancerIDs[1]) - gator.BuildAndUpdate() - scToRemove := <-cc.RemoveSubConnCh - if !cmp.Equal(scToRemove, sc2, cmp.AllowUnexported(testutils.TestSubConn{})) { - t.Fatalf("RemoveSubConn, want %v, got %v", sc2, scToRemove) - } - p2 := <-cc.NewPickerCh - want = []balancer.SubConn{sc1, sc3} - if err := testutils.IsRoundRobin(want, subConnFromPicker(p2)); err != nil { - t.Fatalf("want %v, got %v", want, err) - } - - // move balancer 3 into transient failure. - bg.UpdateSubConnState(sc3, balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) - // Remove the first balancer, while the third is transient failure. - gator.Remove(testBalancerIDs[0]) - bg.Remove(testBalancerIDs[0]) - gator.BuildAndUpdate() - scToRemove = <-cc.RemoveSubConnCh - if !cmp.Equal(scToRemove, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) { - t.Fatalf("RemoveSubConn, want %v, got %v", sc1, scToRemove) - } - p3 := <-cc.NewPickerCh - for i := 0; i < 5; i++ { - if _, err := p3.Pick(balancer.PickInfo{}); err != balancer.ErrTransientFailure { - t.Fatalf("want pick error %v, got %v", balancer.ErrTransientFailure, err) - } - } -} - -// 2 balancers, change balancer weight. -func (s) TestBalancerGroup_TwoRR_ChangeWeight_MoreBackends(t *testing.T) { - cc, gator, bg := newTestBalancerGroup(t, nil) - - // Add two balancers to group and send two resolved addresses to both - // balancers. - gator.Add(testBalancerIDs[0], 2) - bg.Add(testBalancerIDs[0], rrBuilder) - bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:2]}}) - sc1 := <-cc.NewSubConnCh - sc2 := <-cc.NewSubConnCh - - gator.Add(testBalancerIDs[1], 1) - bg.Add(testBalancerIDs[1], rrBuilder) - bg.UpdateClientConnState(testBalancerIDs[1], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[2:4]}}) - sc3 := <-cc.NewSubConnCh - sc4 := <-cc.NewSubConnCh - - // Send state changes for both subconns. - bg.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - bg.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - bg.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - bg.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - bg.UpdateSubConnState(sc3, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - bg.UpdateSubConnState(sc3, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - bg.UpdateSubConnState(sc4, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - bg.UpdateSubConnState(sc4, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - - // Test roundrobin on the last picker. - p1 := <-cc.NewPickerCh - want := []balancer.SubConn{sc1, sc1, sc2, sc2, sc3, sc4} - if err := testutils.IsRoundRobin(want, subConnFromPicker(p1)); err != nil { - t.Fatalf("want %v, got %v", want, err) - } - - gator.UpdateWeight(testBalancerIDs[0], 3) - gator.BuildAndUpdate() - - // Test roundrobin with new weight. - p2 := <-cc.NewPickerCh - want = []balancer.SubConn{sc1, sc1, sc1, sc2, sc2, sc2, sc3, sc4} - if err := testutils.IsRoundRobin(want, subConnFromPicker(p2)); err != nil { - t.Fatalf("want %v, got %v", want, err) - } -} - -func (s) TestBalancerGroup_LoadReport(t *testing.T) { - testLoadStore := testutils.NewTestLoadStore() - cc, gator, bg := newTestBalancerGroup(t, testLoadStore) - - backendToBalancerID := make(map[balancer.SubConn]internal.LocalityID) - - // Add two balancers to group and send two resolved addresses to both - // balancers. - gator.Add(testBalancerIDs[0], 2) - bg.Add(testBalancerIDs[0], rrBuilder) - bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:2]}}) - sc1 := <-cc.NewSubConnCh - sc2 := <-cc.NewSubConnCh - backendToBalancerID[sc1] = testBalancerIDs[0] - backendToBalancerID[sc2] = testBalancerIDs[0] - - gator.Add(testBalancerIDs[1], 1) - bg.Add(testBalancerIDs[1], rrBuilder) - bg.UpdateClientConnState(testBalancerIDs[1], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[2:4]}}) - sc3 := <-cc.NewSubConnCh - sc4 := <-cc.NewSubConnCh - backendToBalancerID[sc3] = testBalancerIDs[1] - backendToBalancerID[sc4] = testBalancerIDs[1] - - // Send state changes for both subconns. - bg.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - bg.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - bg.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - bg.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - bg.UpdateSubConnState(sc3, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - bg.UpdateSubConnState(sc3, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - bg.UpdateSubConnState(sc4, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - bg.UpdateSubConnState(sc4, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - - // Test roundrobin on the last picker. - p1 := <-cc.NewPickerCh - var ( - wantStart []internal.LocalityID - wantEnd []internal.LocalityID - wantCost []testutils.TestServerLoad - ) - for i := 0; i < 10; i++ { - scst, _ := p1.Pick(balancer.PickInfo{}) - locality := backendToBalancerID[scst.SubConn] - wantStart = append(wantStart, locality) - if scst.Done != nil && scst.SubConn != sc1 { - scst.Done(balancer.DoneInfo{ - ServerLoad: &orcapb.OrcaLoadReport{ - CpuUtilization: 10, - MemUtilization: 5, - RequestCost: map[string]float64{"pic": 3.14}, - Utilization: map[string]float64{"piu": 3.14}, - }, - }) - wantEnd = append(wantEnd, locality) - wantCost = append(wantCost, - testutils.TestServerLoad{Name: serverLoadCPUName, D: 10}, - testutils.TestServerLoad{Name: serverLoadMemoryName, D: 5}, - testutils.TestServerLoad{Name: "pic", D: 3.14}, - testutils.TestServerLoad{Name: "piu", D: 3.14}) - } - } - - if !cmp.Equal(testLoadStore.CallsStarted, wantStart) { - t.Fatalf("want started: %v, got: %v", testLoadStore.CallsStarted, wantStart) - } - if !cmp.Equal(testLoadStore.CallsEnded, wantEnd) { - t.Fatalf("want ended: %v, got: %v", testLoadStore.CallsEnded, wantEnd) - } - if !cmp.Equal(testLoadStore.CallsCost, wantCost, cmp.AllowUnexported(testutils.TestServerLoad{})) { - t.Fatalf("want cost: %v, got: %v", testLoadStore.CallsCost, wantCost) - } -} - -// Create a new balancer group, add balancer and backends, but not start. -// - b1, weight 2, backends [0,1] -// - b2, weight 1, backends [2,3] -// Start the balancer group and check behavior. -// -// Close the balancer group, call add/remove/change weight/change address. -// - b2, weight 3, backends [0,3] -// - b3, weight 1, backends [1,2] -// Start the balancer group again and check for behavior. -func (s) TestBalancerGroup_start_close(t *testing.T) { - cc := testutils.NewTestClientConn(t) - gator := weightedaggregator.New(cc, nil, testutils.NewTestWRR) - gator.Start() - bg := New(cc, gator, nil, nil) - - // Add two balancers to group and send two resolved addresses to both - // balancers. - gator.Add(testBalancerIDs[0], 2) - bg.Add(testBalancerIDs[0], rrBuilder) - bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:2]}}) - gator.Add(testBalancerIDs[1], 1) - bg.Add(testBalancerIDs[1], rrBuilder) - bg.UpdateClientConnState(testBalancerIDs[1], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[2:4]}}) - - bg.Start() - - m1 := make(map[resolver.Address]balancer.SubConn) - for i := 0; i < 4; i++ { - addrs := <-cc.NewSubConnAddrsCh - sc := <-cc.NewSubConnCh - m1[addrs[0]] = sc - bg.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - bg.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - } - - // Test roundrobin on the last picker. - p1 := <-cc.NewPickerCh - want := []balancer.SubConn{ - m1[testBackendAddrs[0]], m1[testBackendAddrs[0]], - m1[testBackendAddrs[1]], m1[testBackendAddrs[1]], - m1[testBackendAddrs[2]], m1[testBackendAddrs[3]], - } - if err := testutils.IsRoundRobin(want, subConnFromPicker(p1)); err != nil { - t.Fatalf("want %v, got %v", want, err) - } - - gator.Stop() - bg.Close() - for i := 0; i < 4; i++ { - bg.UpdateSubConnState(<-cc.RemoveSubConnCh, balancer.SubConnState{ConnectivityState: connectivity.Shutdown}) - } - - // Add b3, weight 1, backends [1,2]. - gator.Add(testBalancerIDs[2], 1) - bg.Add(testBalancerIDs[2], rrBuilder) - bg.UpdateClientConnState(testBalancerIDs[2], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[1:3]}}) - - // Remove b1. - gator.Remove(testBalancerIDs[0]) - bg.Remove(testBalancerIDs[0]) - - // Update b2 to weight 3, backends [0,3]. - gator.UpdateWeight(testBalancerIDs[1], 3) - bg.UpdateClientConnState(testBalancerIDs[1], balancer.ClientConnState{ResolverState: resolver.State{Addresses: append([]resolver.Address(nil), testBackendAddrs[0], testBackendAddrs[3])}}) - - gator.Start() - bg.Start() - - m2 := make(map[resolver.Address]balancer.SubConn) - for i := 0; i < 4; i++ { - addrs := <-cc.NewSubConnAddrsCh - sc := <-cc.NewSubConnCh - m2[addrs[0]] = sc - bg.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - bg.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - } - - // Test roundrobin on the last picker. - p2 := <-cc.NewPickerCh - want = []balancer.SubConn{ - m2[testBackendAddrs[0]], m2[testBackendAddrs[0]], m2[testBackendAddrs[0]], - m2[testBackendAddrs[3]], m2[testBackendAddrs[3]], m2[testBackendAddrs[3]], - m2[testBackendAddrs[1]], m2[testBackendAddrs[2]], - } - if err := testutils.IsRoundRobin(want, subConnFromPicker(p2)); err != nil { - t.Fatalf("want %v, got %v", want, err) - } -} - -// Test that balancer group start() doesn't deadlock if the balancer calls back -// into balancer group inline when it gets an update. -// -// The potential deadlock can happen if we -// - hold a lock and send updates to balancer (e.g. update resolved addresses) -// - the balancer calls back (NewSubConn or update picker) in line -// The callback will try to hold hte same lock again, which will cause a -// deadlock. -// -// This test starts the balancer group with a test balancer, will updates picker -// whenever it gets an address update. It's expected that start() doesn't block -// because of deadlock. -func (s) TestBalancerGroup_start_close_deadlock(t *testing.T) { - cc := testutils.NewTestClientConn(t) - gator := weightedaggregator.New(cc, nil, testutils.NewTestWRR) - gator.Start() - bg := New(cc, gator, nil, nil) - - gator.Add(testBalancerIDs[0], 2) - bg.Add(testBalancerIDs[0], &testutils.TestConstBalancerBuilder{}) - bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:2]}}) - gator.Add(testBalancerIDs[1], 1) - bg.Add(testBalancerIDs[1], &testutils.TestConstBalancerBuilder{}) - bg.UpdateClientConnState(testBalancerIDs[1], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[2:4]}}) - - bg.Start() -} - -// Test that at init time, with two sub-balancers, if one sub-balancer reports -// transient_failure, the picks won't fail with transient_failure, and should -// instead wait for the other sub-balancer. -func (s) TestBalancerGroup_InitOneSubBalancerTransientFailure(t *testing.T) { - cc, gator, bg := newTestBalancerGroup(t, nil) - - // Add two balancers to group and send one resolved address to both - // balancers. - gator.Add(testBalancerIDs[0], 1) - bg.Add(testBalancerIDs[0], rrBuilder) - bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:1]}}) - sc1 := <-cc.NewSubConnCh - - gator.Add(testBalancerIDs[1], 1) - bg.Add(testBalancerIDs[1], rrBuilder) - bg.UpdateClientConnState(testBalancerIDs[1], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:1]}}) - <-cc.NewSubConnCh - - // Set one subconn to TransientFailure, this will trigger one sub-balancer - // to report transient failure. - bg.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) - - p1 := <-cc.NewPickerCh - for i := 0; i < 5; i++ { - r, err := p1.Pick(balancer.PickInfo{}) - if err != balancer.ErrNoSubConnAvailable { - t.Fatalf("want pick to fail with %v, got result %v, err %v", balancer.ErrNoSubConnAvailable, r, err) - } - } -} - -// Test that with two sub-balancers, both in transient_failure, if one turns -// connecting, the overall state stays in transient_failure, and all picks -// return transient failure error. -func (s) TestBalancerGroup_SubBalancerTurnsConnectingFromTransientFailure(t *testing.T) { - cc, gator, bg := newTestBalancerGroup(t, nil) - - // Add two balancers to group and send one resolved address to both - // balancers. - gator.Add(testBalancerIDs[0], 1) - bg.Add(testBalancerIDs[0], pfBuilder) - bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:1]}}) - sc1 := <-cc.NewSubConnCh - - gator.Add(testBalancerIDs[1], 1) - bg.Add(testBalancerIDs[1], pfBuilder) - bg.UpdateClientConnState(testBalancerIDs[1], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:1]}}) - sc2 := <-cc.NewSubConnCh - - // Set both subconn to TransientFailure, this will put both sub-balancers in - // transient failure. - bg.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) - bg.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) - - p1 := <-cc.NewPickerCh - for i := 0; i < 5; i++ { - r, err := p1.Pick(balancer.PickInfo{}) - if err != balancer.ErrTransientFailure { - t.Fatalf("want pick to fail with %v, got result %v, err %v", balancer.ErrTransientFailure, r, err) - } - } - - // Set one subconn to Connecting, it shouldn't change the overall state. - bg.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - - p2 := <-cc.NewPickerCh - for i := 0; i < 5; i++ { - r, err := p2.Pick(balancer.PickInfo{}) - if err != balancer.ErrTransientFailure { - t.Fatalf("want pick to fail with %v, got result %v, err %v", balancer.ErrTransientFailure, r, err) - } - } -} - -func replaceDefaultSubBalancerCloseTimeout(n time.Duration) func() { - old := DefaultSubBalancerCloseTimeout - DefaultSubBalancerCloseTimeout = n - return func() { DefaultSubBalancerCloseTimeout = old } -} - -// initBalancerGroupForCachingTest creates a balancer group, and initialize it -// to be ready for caching tests. -// -// Two rr balancers are added to bg, each with 2 ready subConns. A sub-balancer -// is removed later, so the balancer group returned has one sub-balancer in its -// own map, and one sub-balancer in cache. -func initBalancerGroupForCachingTest(t *testing.T) (*weightedaggregator.Aggregator, *BalancerGroup, *testutils.TestClientConn, map[resolver.Address]balancer.SubConn) { - cc := testutils.NewTestClientConn(t) - gator := weightedaggregator.New(cc, nil, testutils.NewTestWRR) - gator.Start() - bg := New(cc, gator, nil, nil) - - // Add two balancers to group and send two resolved addresses to both - // balancers. - gator.Add(testBalancerIDs[0], 2) - bg.Add(testBalancerIDs[0], rrBuilder) - bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:2]}}) - gator.Add(testBalancerIDs[1], 1) - bg.Add(testBalancerIDs[1], rrBuilder) - bg.UpdateClientConnState(testBalancerIDs[1], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[2:4]}}) - - bg.Start() - - m1 := make(map[resolver.Address]balancer.SubConn) - for i := 0; i < 4; i++ { - addrs := <-cc.NewSubConnAddrsCh - sc := <-cc.NewSubConnCh - m1[addrs[0]] = sc - bg.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - bg.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - } - - // Test roundrobin on the last picker. - p1 := <-cc.NewPickerCh - want := []balancer.SubConn{ - m1[testBackendAddrs[0]], m1[testBackendAddrs[0]], - m1[testBackendAddrs[1]], m1[testBackendAddrs[1]], - m1[testBackendAddrs[2]], m1[testBackendAddrs[3]], - } - if err := testutils.IsRoundRobin(want, subConnFromPicker(p1)); err != nil { - t.Fatalf("want %v, got %v", want, err) - } - - gator.Remove(testBalancerIDs[1]) - bg.Remove(testBalancerIDs[1]) - gator.BuildAndUpdate() - // Don't wait for SubConns to be removed after close, because they are only - // removed after close timeout. - for i := 0; i < 10; i++ { - select { - case <-cc.RemoveSubConnCh: - t.Fatalf("Got request to remove subconn, want no remove subconn (because subconns were still in cache)") - default: - } - time.Sleep(time.Millisecond) - } - // Test roundrobin on the with only sub-balancer0. - p2 := <-cc.NewPickerCh - want = []balancer.SubConn{ - m1[testBackendAddrs[0]], m1[testBackendAddrs[1]], - } - if err := testutils.IsRoundRobin(want, subConnFromPicker(p2)); err != nil { - t.Fatalf("want %v, got %v", want, err) - } - - return gator, bg, cc, m1 -} - -// Test that if a sub-balancer is removed, and re-added within close timeout, -// the subConns won't be re-created. -func (s) TestBalancerGroup_locality_caching(t *testing.T) { - defer replaceDefaultSubBalancerCloseTimeout(10 * time.Second)() - gator, bg, cc, addrToSC := initBalancerGroupForCachingTest(t) - - // Turn down subconn for addr2, shouldn't get picker update because - // sub-balancer1 was removed. - bg.UpdateSubConnState(addrToSC[testBackendAddrs[2]], balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) - for i := 0; i < 10; i++ { - select { - case <-cc.NewPickerCh: - t.Fatalf("Got new picker, want no new picker (because the sub-balancer was removed)") - default: - } - time.Sleep(time.Millisecond) - } - - // Sleep, but sleep less then close timeout. - time.Sleep(time.Millisecond * 100) - - // Re-add sub-balancer-1, because subconns were in cache, no new subconns - // should be created. But a new picker will still be generated, with subconn - // states update to date. - gator.Add(testBalancerIDs[1], 1) - bg.Add(testBalancerIDs[1], rrBuilder) - - p3 := <-cc.NewPickerCh - want := []balancer.SubConn{ - addrToSC[testBackendAddrs[0]], addrToSC[testBackendAddrs[0]], - addrToSC[testBackendAddrs[1]], addrToSC[testBackendAddrs[1]], - // addr2 is down, b2 only has addr3 in READY state. - addrToSC[testBackendAddrs[3]], addrToSC[testBackendAddrs[3]], - } - if err := testutils.IsRoundRobin(want, subConnFromPicker(p3)); err != nil { - t.Fatalf("want %v, got %v", want, err) - } - - for i := 0; i < 10; i++ { - select { - case <-cc.NewSubConnAddrsCh: - t.Fatalf("Got new subconn, want no new subconn (because subconns were still in cache)") - default: - } - time.Sleep(time.Millisecond * 10) - } -} - -// Sub-balancers are put in cache when they are removed. If balancer group is -// closed within close timeout, all subconns should still be rmeoved -// immediately. -func (s) TestBalancerGroup_locality_caching_close_group(t *testing.T) { - defer replaceDefaultSubBalancerCloseTimeout(10 * time.Second)() - _, bg, cc, addrToSC := initBalancerGroupForCachingTest(t) - - bg.Close() - // The balancer group is closed. The subconns should be removed immediately. - removeTimeout := time.After(time.Millisecond * 500) - scToRemove := map[balancer.SubConn]int{ - addrToSC[testBackendAddrs[0]]: 1, - addrToSC[testBackendAddrs[1]]: 1, - addrToSC[testBackendAddrs[2]]: 1, - addrToSC[testBackendAddrs[3]]: 1, - } - for i := 0; i < len(scToRemove); i++ { - select { - case sc := <-cc.RemoveSubConnCh: - c := scToRemove[sc] - if c == 0 { - t.Fatalf("Got removeSubConn for %v when there's %d remove expected", sc, c) - } - scToRemove[sc] = c - 1 - case <-removeTimeout: - t.Fatalf("timeout waiting for subConns (from balancer in cache) to be removed") - } - } -} - -// Sub-balancers in cache will be closed if not re-added within timeout, and -// subConns will be removed. -func (s) TestBalancerGroup_locality_caching_not_readd_within_timeout(t *testing.T) { - defer replaceDefaultSubBalancerCloseTimeout(time.Second)() - _, _, cc, addrToSC := initBalancerGroupForCachingTest(t) - - // The sub-balancer is not re-added withtin timeout. The subconns should be - // removed. - removeTimeout := time.After(DefaultSubBalancerCloseTimeout) - scToRemove := map[balancer.SubConn]int{ - addrToSC[testBackendAddrs[2]]: 1, - addrToSC[testBackendAddrs[3]]: 1, - } - for i := 0; i < len(scToRemove); i++ { - select { - case sc := <-cc.RemoveSubConnCh: - c := scToRemove[sc] - if c == 0 { - t.Fatalf("Got removeSubConn for %v when there's %d remove expected", sc, c) - } - scToRemove[sc] = c - 1 - case <-removeTimeout: - t.Fatalf("timeout waiting for subConns (from balancer in cache) to be removed") - } - } -} - -// Wrap the rr builder, so it behaves the same, but has a different pointer. -type noopBalancerBuilderWrapper struct { - balancer.Builder -} - -// After removing a sub-balancer, re-add with same ID, but different balancer -// builder. Old subconns should be removed, and new subconns should be created. -func (s) TestBalancerGroup_locality_caching_readd_with_different_builder(t *testing.T) { - defer replaceDefaultSubBalancerCloseTimeout(10 * time.Second)() - gator, bg, cc, addrToSC := initBalancerGroupForCachingTest(t) - - // Re-add sub-balancer-1, but with a different balancer builder. The - // sub-balancer was still in cache, but cann't be reused. This should cause - // old sub-balancer's subconns to be removed immediately, and new subconns - // to be created. - gator.Add(testBalancerIDs[1], 1) - bg.Add(testBalancerIDs[1], &noopBalancerBuilderWrapper{rrBuilder}) - - // The cached sub-balancer should be closed, and the subconns should be - // removed immediately. - removeTimeout := time.After(time.Millisecond * 500) - scToRemove := map[balancer.SubConn]int{ - addrToSC[testBackendAddrs[2]]: 1, - addrToSC[testBackendAddrs[3]]: 1, - } - for i := 0; i < len(scToRemove); i++ { - select { - case sc := <-cc.RemoveSubConnCh: - c := scToRemove[sc] - if c == 0 { - t.Fatalf("Got removeSubConn for %v when there's %d remove expected", sc, c) - } - scToRemove[sc] = c - 1 - case <-removeTimeout: - t.Fatalf("timeout waiting for subConns (from balancer in cache) to be removed") - } - } - - bg.UpdateClientConnState(testBalancerIDs[1], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[4:6]}}) - - newSCTimeout := time.After(time.Millisecond * 500) - scToAdd := map[resolver.Address]int{ - testBackendAddrs[4]: 1, - testBackendAddrs[5]: 1, - } - for i := 0; i < len(scToAdd); i++ { - select { - case addr := <-cc.NewSubConnAddrsCh: - c := scToAdd[addr[0]] - if c == 0 { - t.Fatalf("Got newSubConn for %v when there's %d new expected", addr, c) - } - scToAdd[addr[0]] = c - 1 - sc := <-cc.NewSubConnCh - addrToSC[addr[0]] = sc - bg.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - bg.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - case <-newSCTimeout: - t.Fatalf("timeout waiting for subConns (from new sub-balancer) to be newed") - } - } - - // Test roundrobin on the new picker. - p3 := <-cc.NewPickerCh - want := []balancer.SubConn{ - addrToSC[testBackendAddrs[0]], addrToSC[testBackendAddrs[0]], - addrToSC[testBackendAddrs[1]], addrToSC[testBackendAddrs[1]], - addrToSC[testBackendAddrs[4]], addrToSC[testBackendAddrs[5]], - } - if err := testutils.IsRoundRobin(want, subConnFromPicker(p3)); err != nil { - t.Fatalf("want %v, got %v", want, err) - } -} diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer.go b/xds/internal/balancer/cdsbalancer/cdsbalancer.go index c6ab764763aa..85a081d09df5 100644 --- a/xds/internal/balancer/cdsbalancer/cdsbalancer.go +++ b/xds/internal/balancer/cdsbalancer/cdsbalancer.go @@ -21,69 +21,104 @@ import ( "encoding/json" "errors" "fmt" - "sync" - "google.golang.org/grpc/attributes" "google.golang.org/grpc/balancer" "google.golang.org/grpc/balancer/base" "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/tls/certprovider" + "google.golang.org/grpc/internal/balancer/nop" "google.golang.org/grpc/internal/buffer" + xdsinternal "google.golang.org/grpc/internal/credentials/xds" + "google.golang.org/grpc/internal/envconfig" "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/pretty" "google.golang.org/grpc/resolver" "google.golang.org/grpc/serviceconfig" - "google.golang.org/grpc/xds/internal/balancer/edsbalancer" - - xdsinternal "google.golang.org/grpc/xds/internal" - xdsclient "google.golang.org/grpc/xds/internal/client" + "google.golang.org/grpc/xds/internal/balancer/clusterresolver" + "google.golang.org/grpc/xds/internal/xdsclient" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" ) const ( cdsName = "cds_experimental" - edsName = "eds_experimental" ) var ( - errBalancerClosed = errors.New("cdsBalancer is closed") + errBalancerClosed = errors.New("cds_experimental LB policy is closed") - // newEDSBalancer is a helper function to build a new edsBalancer and will be - // overridden in unittests. - newEDSBalancer = func(cc balancer.ClientConn, opts balancer.BuildOptions) (balancer.Balancer, error) { - builder := balancer.Get(edsName) + // newChildBalancer is a helper function to build a new cluster_resolver + // balancer and will be overridden in unittests. + newChildBalancer = func(cc balancer.ClientConn, opts balancer.BuildOptions) (balancer.Balancer, error) { + builder := balancer.Get(clusterresolver.Name) if builder == nil { - return nil, fmt.Errorf("xds: no balancer builder with name %v", edsName) + return nil, fmt.Errorf("xds: no balancer builder with name %v", clusterresolver.Name) } - // We directly pass the parent clientConn to the - // underlying edsBalancer because the cdsBalancer does - // not deal with subConns. + // We directly pass the parent clientConn to the underlying + // cluster_resolver balancer because the cdsBalancer does not deal with + // subConns. return builder.Build(cc, opts), nil } + buildProvider = buildProviderFunc ) func init() { - balancer.Register(cdsBB{}) + balancer.Register(bb{}) } -// cdsBB (short for cdsBalancerBuilder) implements the balancer.Builder -// interface to help build a cdsBalancer. +// bb implements the balancer.Builder interface to help build a cdsBalancer. // It also implements the balancer.ConfigParser interface to help parse the // JSON service config, to be passed to the cdsBalancer. -type cdsBB struct{} +type bb struct{} // Build creates a new CDS balancer with the ClientConn. -func (cdsBB) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { +func (bb) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { + builder := balancer.Get(clusterresolver.Name) + if builder == nil { + // Shouldn't happen, registered through imported Cluster Resolver, + // defensive programming. + logger.Errorf("%q LB policy is needed but not registered", clusterresolver.Name) + return nop.NewBalancer(cc, fmt.Errorf("%q LB policy is needed but not registered", clusterresolver.Name)) + } + crParser, ok := builder.(balancer.ConfigParser) + if !ok { + // Shouldn't happen, imported Cluster Resolver builder has this method. + logger.Errorf("%q LB policy does not implement a config parser", clusterresolver.Name) + return nop.NewBalancer(cc, fmt.Errorf("%q LB policy does not implement a config parser", clusterresolver.Name)) + } b := &cdsBalancer{ - cc: cc, bOpts: opts, updateCh: buffer.NewUnbounded(), + closed: grpcsync.NewEvent(), + done: grpcsync.NewEvent(), + crParser: crParser, + xdsHI: xdsinternal.NewHandshakeInfo(nil, nil), } b.logger = prefixLogger((b)) b.logger.Infof("Created") + var creds credentials.TransportCredentials + switch { + case opts.DialCreds != nil: + creds = opts.DialCreds + case opts.CredsBundle != nil: + creds = opts.CredsBundle.TransportCredentials() + } + if xc, ok := creds.(interface{ UsesXDS() bool }); ok && xc.UsesXDS() { + b.xdsCredsInUse = true + } + b.logger.Infof("xDS credentials in use: %v", b.xdsCredsInUse) + b.clusterHandler = newClusterHandler(b) + b.ccw = &ccWrapper{ + ClientConn: cc, + xdsHI: b.xdsHI, + } go b.run() return b } // Name returns the name of balancers built by this builder. -func (cdsBB) Name() string { +func (bb) Name() string { return cdsName } @@ -96,7 +131,7 @@ type lbConfig struct { // ParseConfig parses the JSON load balancer config provided into an // internal form or returns an error if the config is invalid. -func (cdsBB) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { +func (bb) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { var cfg lbConfig if err := json.Unmarshal(c, &cfg); err != nil { return nil, fmt.Errorf("xds: unable to unmarshal lbconfig: %s, error: %v", string(c), err) @@ -104,162 +139,305 @@ func (cdsBB) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig, return &cfg, nil } -// xdsClientInterface contains methods from xdsClient.Client which are used by -// the cdsBalancer. This will be faked out in unittests. -type xdsClientInterface interface { - WatchCluster(string, func(xdsclient.ClusterUpdate, error)) func() - Close() -} - // ccUpdate wraps a clientConn update received from gRPC (pushed from the // xdsResolver). A valid clusterName causes the cdsBalancer to register a CDS // watcher with the xdsClient, while a non-nil error causes it to cancel the -// existing watch and propagate the error to the underlying edsBalancer. +// existing watch and propagate the error to the underlying cluster_resolver +// balancer. type ccUpdate struct { - client xdsClientInterface clusterName string err error } -// scUpdate wraps a subConn update received from gRPC. This is directly passed -// on to the edsBalancer. -type scUpdate struct { - subConn balancer.SubConn - state balancer.SubConnState +type exitIdle struct{} + +// cdsBalancer implements a CDS based LB policy. It instantiates a +// cluster_resolver balancer to further resolve the serviceName received from +// CDS, into localities and endpoints. Implements the balancer.Balancer +// interface which is exposed to gRPC and implements the balancer.ClientConn +// interface which is exposed to the cluster_resolver balancer. +type cdsBalancer struct { + ccw *ccWrapper // ClientConn interface passed to child LB. + bOpts balancer.BuildOptions // BuildOptions passed to child LB. + updateCh *buffer.Unbounded // Channel for gRPC and xdsClient updates. + xdsClient xdsclient.XDSClient // xDS client to watch Cluster resource. + clusterHandler *clusterHandler // To watch the clusters. + childLB balancer.Balancer + logger *grpclog.PrefixLogger + closed *grpcsync.Event + done *grpcsync.Event + crParser balancer.ConfigParser + + // The certificate providers are cached here to that they can be closed when + // a new provider is to be created. + cachedRoot certprovider.Provider + cachedIdentity certprovider.Provider + xdsHI *xdsinternal.HandshakeInfo + xdsCredsInUse bool } -// watchUpdate wraps the information received from a registered CDS watcher. A -// non-nil error is propagated to the underlying edsBalancer. A valid update -// results in creating a new edsBalancer (if one doesn't already exist) and -// pushing the update to it. -type watchUpdate struct { - cds xdsclient.ClusterUpdate - err error +// handleClientConnUpdate handles a ClientConnUpdate received from gRPC. Good +// updates lead to registration of a CDS watch. Updates with error lead to +// cancellation of existing watch and propagation of the same error to the +// cluster_resolver balancer. +func (b *cdsBalancer) handleClientConnUpdate(update *ccUpdate) { + // We first handle errors, if any, and then proceed with handling the + // update, only if the status quo has changed. + if err := update.err; err != nil { + b.handleErrorFromUpdate(err, true) + return + } + b.clusterHandler.updateRootCluster(update.clusterName) } -// closeUpdate is an empty struct used to notify the run() goroutine that a -// Close has been called on the balancer. -type closeUpdate struct{} +// handleSecurityConfig processes the security configuration received from the +// management server, creates appropriate certificate provider plugins, and +// updates the HandhakeInfo which is added as an address attribute in +// NewSubConn() calls. +func (b *cdsBalancer) handleSecurityConfig(config *xdsresource.SecurityConfig) error { + // If xdsCredentials are not in use, i.e, the user did not want to get + // security configuration from an xDS server, we should not be acting on the + // received security config here. Doing so poses a security threat. + if !b.xdsCredsInUse { + return nil + } -// cdsBalancer implements a CDS based LB policy. It instantiates an EDS based -// LB policy to further resolve the serviceName received from CDS, into -// localities and endpoints. Implements the balancer.Balancer interface which -// is exposed to gRPC and implements the balancer.ClientConn interface which is -// exposed to the edsBalancer. -type cdsBalancer struct { - cc balancer.ClientConn - bOpts balancer.BuildOptions - updateCh *buffer.Unbounded - client xdsClientInterface - cancelWatch func() - edsLB balancer.Balancer - clusterToWatch string - - logger *grpclog.PrefixLogger - - // The only thing protected by this mutex is the closed boolean. This is - // checked by all methods before acting on updates. - mu sync.Mutex - closed bool + // Security config being nil is a valid case where the management server has + // not sent any security configuration. The xdsCredentials implementation + // handles this by delegating to its fallback credentials. + if config == nil { + // We need to explicitly set the fields to nil here since this might be + // a case of switching from a good security configuration to an empty + // one where fallback credentials are to be used. + b.xdsHI.SetRootCertProvider(nil) + b.xdsHI.SetIdentityCertProvider(nil) + b.xdsHI.SetSANMatchers(nil) + return nil + } + + bc := b.xdsClient.BootstrapConfig() + if bc == nil || bc.CertProviderConfigs == nil { + // Bootstrap did not find any certificate provider configs, but the user + // has specified xdsCredentials and the management server has sent down + // security configuration. + return errors.New("xds: certificate_providers config missing in bootstrap file") + } + cpc := bc.CertProviderConfigs + + // A root provider is required whether we are using TLS or mTLS. + rootProvider, err := buildProvider(cpc, config.RootInstanceName, config.RootCertName, false, true) + if err != nil { + return err + } + + // The identity provider is only present when using mTLS. + var identityProvider certprovider.Provider + if name, cert := config.IdentityInstanceName, config.IdentityCertName; name != "" { + var err error + identityProvider, err = buildProvider(cpc, name, cert, true, false) + if err != nil { + return err + } + } + + // Close the old providers and cache the new ones. + if b.cachedRoot != nil { + b.cachedRoot.Close() + } + if b.cachedIdentity != nil { + b.cachedIdentity.Close() + } + b.cachedRoot = rootProvider + b.cachedIdentity = identityProvider + + // We set all fields here, even if some of them are nil, since they + // could have been non-nil earlier. + b.xdsHI.SetRootCertProvider(rootProvider) + b.xdsHI.SetIdentityCertProvider(identityProvider) + b.xdsHI.SetSANMatchers(config.SubjectAltNameMatchers) + return nil } -// run is a long-running goroutine which handles all updates from gRPC. All -// methods which are invoked directly by gRPC or xdsClient simply push an -// update onto a channel which is read and acted upon right here. -// -// 1. Good clientConn updates lead to registration of a CDS watch. Updates with -// error lead to cancellation of existing watch and propagation of the same -// error to the edsBalancer. -// 2. SubConn updates are passthrough and are simply handed over to the -// underlying edsBalancer. -// 3. Watch API updates lead to clientConn updates being invoked on the -// underlying edsBalancer. -// 4. Close results in cancellation of the CDS watch and closing of the -// underlying edsBalancer and is the only way to exit this goroutine. -func (b *cdsBalancer) run() { - for { - u := <-b.updateCh.Get() - b.updateCh.Load() - switch update := u.(type) { - case *ccUpdate: - // We first handle errors, if any, and then proceed with handling - // the update, only if the status quo has changed. - if err := update.err; err != nil { - b.handleErrorFromUpdate(err, true) - } - if b.client == update.client && b.clusterToWatch == update.clusterName { - break - } - if update.client != nil { - // Since the cdsBalancer doesn't own the xdsClient object, we - // don't have to bother about closing the old client here, but - // we still need to cancel the watch on the old client. - if b.cancelWatch != nil { - b.cancelWatch() - } - b.client = update.client +func buildProviderFunc(configs map[string]*certprovider.BuildableConfig, instanceName, certName string, wantIdentity, wantRoot bool) (certprovider.Provider, error) { + cfg, ok := configs[instanceName] + if !ok { + return nil, fmt.Errorf("certificate provider instance %q not found in bootstrap file", instanceName) + } + provider, err := cfg.Build(certprovider.BuildOptions{ + CertName: certName, + WantIdentity: wantIdentity, + WantRoot: wantRoot, + }) + if err != nil { + // This error is not expected since the bootstrap process parses the + // config and makes sure that it is acceptable to the plugin. Still, it + // is possible that the plugin parses the config successfully, but its + // Build() method errors out. + return nil, fmt.Errorf("xds: failed to get security plugin instance (%+v): %v", cfg, err) + } + return provider, nil +} + +// handleWatchUpdate handles a watch update from the xDS Client. Good updates +// lead to clientConn updates being invoked on the underlying cluster_resolver balancer. +func (b *cdsBalancer) handleWatchUpdate(update clusterHandlerUpdate) { + if err := update.err; err != nil { + b.logger.Warningf("Watch error from xds-client %p: %v", b.xdsClient, err) + b.handleErrorFromUpdate(err, false) + return + } + + b.logger.Infof("Received Cluster resource contains content: %s, security config: %s", pretty.ToJSON(update.updates), pretty.ToJSON(update.securityCfg)) + + // Process the security config from the received update before building the + // child policy or forwarding the update to it. We do this because the child + // policy may try to create a new subConn inline. Processing the security + // configuration here and setting up the handshakeInfo will make sure that + // such attempts are handled properly. + if err := b.handleSecurityConfig(update.securityCfg); err != nil { + // If the security config is invalid, for example, if the provider + // instance is not found in the bootstrap config, we need to put the + // channel in transient failure. + b.logger.Warningf("Received Cluster resource contains invalid security config: %v", err) + b.handleErrorFromUpdate(err, false) + return + } + + // The first good update from the watch API leads to the instantiation of an + // cluster_resolver balancer. Further updates/errors are propagated to the existing + // cluster_resolver balancer. + if b.childLB == nil { + childLB, err := newChildBalancer(b.ccw, b.bOpts) + if err != nil { + b.logger.Errorf("Failed to create child policy of type %s: %v", clusterresolver.Name, err) + return + } + b.childLB = childLB + b.logger.Infof("Created child policy %p of type %s", b.childLB, clusterresolver.Name) + } + + dms := make([]clusterresolver.DiscoveryMechanism, len(update.updates)) + for i, cu := range update.updates { + switch cu.ClusterType { + case xdsresource.ClusterTypeEDS: + dms[i] = clusterresolver.DiscoveryMechanism{ + Type: clusterresolver.DiscoveryMechanismTypeEDS, + Cluster: cu.ClusterName, + EDSServiceName: cu.EDSServiceName, + MaxConcurrentRequests: cu.MaxRequests, } - if update.clusterName != "" { - cancelWatch := b.client.WatchCluster(update.clusterName, b.handleClusterUpdate) - b.logger.Infof("Watch started on resource name %v with xds-client %p", update.clusterName, b.client) - b.cancelWatch = func() { - cancelWatch() - b.logger.Infof("Watch cancelled on resource name %v with xds-client %p", update.clusterName, b.client) + if cu.LRSServerConfig == xdsresource.ClusterLRSServerSelf { + bootstrapConfig := b.xdsClient.BootstrapConfig() + parsedName := xdsresource.ParseName(cu.ClusterName) + if parsedName.Scheme == xdsresource.FederationScheme { + // Is a federation resource name, find the corresponding + // authority server config. + if cfg, ok := bootstrapConfig.Authorities[parsedName.Authority]; ok { + dms[i].LoadReportingServer = cfg.XDSServer + } + } else { + // Not a federation resource name, use the default + // authority. + dms[i].LoadReportingServer = bootstrapConfig.XDSServer } - b.clusterToWatch = update.clusterName } - case *scUpdate: - if b.edsLB == nil { - b.logger.Errorf("xds: received scUpdate {%+v} with no edsBalancer", update) - break + case xdsresource.ClusterTypeLogicalDNS: + dms[i] = clusterresolver.DiscoveryMechanism{ + Type: clusterresolver.DiscoveryMechanismTypeLogicalDNS, + Cluster: cu.ClusterName, + DNSHostname: cu.DNSHostName, } - b.edsLB.UpdateSubConnState(update.subConn, update.state) - case *watchUpdate: - if err := update.err; err != nil { - b.logger.Warningf("Watch error from xds-client %p: %v", b.client, err) - b.handleErrorFromUpdate(err, false) - break + default: + b.logger.Infof("Unexpected cluster type %v when handling update from cluster handler", cu.ClusterType) + } + if envconfig.XDSOutlierDetection { + odJSON := cu.OutlierDetection + // "In the cds LB policy, if the outlier_detection field is not set in + // the Cluster resource, a "no-op" outlier_detection config will be + // generated in the corresponding DiscoveryMechanism config, with all + // fields unset." - A50 + if odJSON == nil { + // This will pick up top level defaults in Cluster Resolver + // ParseConfig, but sre and fpe will be nil still so still a + // "no-op" config. + odJSON = json.RawMessage(`{}`) } + dms[i].OutlierDetection = odJSON + } + } - b.logger.Infof("Watch update from xds-client %p, content: %+v", b.client, update.cds) - // The first good update from the watch API leads to the - // instantiation of an edsBalancer. Further updates/errors are - // propagated to the existing edsBalancer. - if b.edsLB == nil { - var err error - b.edsLB, err = newEDSBalancer(b.cc, b.bOpts) - if b.edsLB == nil { - b.logger.Errorf("Failed to create child policy of type %s, %v", edsName, err) - break - } - b.logger.Infof("Created child policy %p of type %s", b.edsLB, edsName) - } - lbCfg := &edsbalancer.EDSConfig{EDSServiceName: update.cds.ServiceName} - if update.cds.EnableLRS { - // An empty string here indicates that the edsBalancer - // should use the same xDS server for load reporting as - // it does for EDS requests/responses. - lbCfg.LrsLoadReportingServerName = new(string) + // Prepare Cluster Resolver config, marshal into JSON, and then Parse it to + // get configuration to send downward to Cluster Resolver. + lbCfg := &clusterresolver.LBConfig{ + DiscoveryMechanisms: dms, + XDSLBPolicy: update.lbPolicy, + } + crLBCfgJSON, err := json.Marshal(lbCfg) + if err != nil { + // Shouldn't happen, since we just prepared struct. + b.logger.Errorf("cds_balancer: error marshalling prepared config: %v", lbCfg) + return + } + + var sc serviceconfig.LoadBalancingConfig + if sc, err = b.crParser.ParseConfig(crLBCfgJSON); err != nil { + b.logger.Errorf("cds_balancer: cluster_resolver config generated %v is invalid: %v", string(crLBCfgJSON), err) + return + } + + ccState := balancer.ClientConnState{ + ResolverState: xdsclient.SetClient(resolver.State{}, b.xdsClient), + BalancerConfig: sc, + } + if err := b.childLB.UpdateClientConnState(ccState); err != nil { + b.logger.Errorf("Encountered error when sending config {%+v} to child policy: %v", ccState, err) + } +} +// run is a long-running goroutine which handles all updates from gRPC. All +// methods which are invoked directly by gRPC or xdsClient simply push an +// update onto a channel which is read and acted upon right here. +func (b *cdsBalancer) run() { + for { + select { + case u, ok := <-b.updateCh.Get(): + if !ok { + return } - ccState := balancer.ClientConnState{ - ResolverState: resolver.State{Attributes: attributes.New(xdsinternal.XDSClientID, b.client)}, - BalancerConfig: lbCfg, + b.updateCh.Load() + switch update := u.(type) { + case *ccUpdate: + b.handleClientConnUpdate(update) + case exitIdle: + if b.childLB == nil { + b.logger.Errorf("Received ExitIdle with no child policy") + break + } + // This implementation assumes the child balancer supports + // ExitIdle (but still checks for the interface's existence to + // avoid a panic if not). If the child does not, no subconns + // will be connected. + if ei, ok := b.childLB.(balancer.ExitIdler); ok { + ei.ExitIdle() + } } - if err := b.edsLB.UpdateClientConnState(ccState); err != nil { - b.logger.Errorf("xds: edsBalancer.UpdateClientConnState(%+v) returned error: %v", ccState, err) + case u := <-b.clusterHandler.updateChannel: + b.handleWatchUpdate(u) + case <-b.closed.Done(): + b.clusterHandler.close() + if b.childLB != nil { + b.childLB.Close() + b.childLB = nil } - case *closeUpdate: - if b.cancelWatch != nil { - b.cancelWatch() - b.cancelWatch = nil + if b.cachedRoot != nil { + b.cachedRoot.Close() } - if b.edsLB != nil { - b.edsLB.Close() - b.edsLB = nil + if b.cachedIdentity != nil { + b.cachedIdentity.Close() } - // This is the *ONLY* point of return from this function. + b.updateCh.Close() b.logger.Infof("Shutdown") + b.done.Fire() return } } @@ -278,110 +456,120 @@ func (b *cdsBalancer) run() { // - If it's from xds client, it means CDS resource were removed. The CDS // watcher should keep watching. // -// In both cases, the error will be forwarded to EDS balancer. And if error is -// resource-not-found, the child EDS balancer will stop watching EDS. +// In both cases, the error will be forwarded to the child balancer. And if +// error is resource-not-found, the child balancer will stop watching EDS. func (b *cdsBalancer) handleErrorFromUpdate(err error, fromParent bool) { - // TODO: connection errors will be sent to the eds balancers directly, and - // also forwarded by the parent balancers/resolvers. So the eds balancer may - // see the same error multiple times. We way want to only forward the error - // to eds if it's not a connection error. - // // This is not necessary today, because xds client never sends connection // errors. - - if fromParent && xdsclient.ErrType(err) == xdsclient.ErrorTypeResourceNotFound { - if b.cancelWatch != nil { - b.cancelWatch() - } + if fromParent && xdsresource.ErrType(err) == xdsresource.ErrorTypeResourceNotFound { + b.clusterHandler.close() } - if b.edsLB != nil { - b.edsLB.ResolverError(err) + if b.childLB != nil { + if xdsresource.ErrType(err) != xdsresource.ErrorTypeConnection { + // Connection errors will be sent to the child balancers directly. + // There's no need to forward them. + b.childLB.ResolverError(err) + } } else { - // If eds balancer was never created, fail the RPCs with + // If child balancer was never created, fail the RPCs with // errors. - b.cc.UpdateState(balancer.State{ + b.ccw.UpdateState(balancer.State{ ConnectivityState: connectivity.TransientFailure, Picker: base.NewErrPicker(err), }) } } -// handleClusterUpdate is the CDS watch API callback. It simply pushes the -// received information on to the update channel for run() to pick it up. -func (b *cdsBalancer) handleClusterUpdate(cu xdsclient.ClusterUpdate, err error) { - if b.isClosed() { - b.logger.Warningf("xds: received cluster update {%+v} after cdsBalancer was closed", cu) - return - } - b.updateCh.Put(&watchUpdate{cds: cu, err: err}) -} - // UpdateClientConnState receives the serviceConfig (which contains the // clusterName to watch for in CDS) and the xdsClient object from the // xdsResolver. func (b *cdsBalancer) UpdateClientConnState(state balancer.ClientConnState) error { - if b.isClosed() { - b.logger.Warningf("xds: received ClientConnState {%+v} after cdsBalancer was closed", state) + if b.closed.HasFired() { + b.logger.Errorf("Received balancer config after close") return errBalancerClosed } - b.logger.Infof("Receive update from resolver, balancer config: %+v", state.BalancerConfig) + if b.xdsClient == nil { + c := xdsclient.FromResolverState(state.ResolverState) + if c == nil { + return balancer.ErrBadResolverState + } + b.xdsClient = c + } + b.logger.Infof("Received balancer config update: %s", pretty.ToJSON(state.BalancerConfig)) + // The errors checked here should ideally never happen because the // ServiceConfig in this case is prepared by the xdsResolver and is not // something that is received on the wire. lbCfg, ok := state.BalancerConfig.(*lbConfig) if !ok { - b.logger.Warningf("xds: unexpected LoadBalancingConfig type: %T", state.BalancerConfig) + b.logger.Warningf("Received unexpected balancer config type: %T", state.BalancerConfig) return balancer.ErrBadResolverState } if lbCfg.ClusterName == "" { - b.logger.Warningf("xds: no clusterName found in LoadBalancingConfig: %+v", lbCfg) - return balancer.ErrBadResolverState - } - client := state.ResolverState.Attributes.Value(xdsinternal.XDSClientID) - if client == nil { - b.logger.Warningf("xds: no xdsClient found in resolver state attributes") - return balancer.ErrBadResolverState - } - newClient, ok := client.(xdsClientInterface) - if !ok { - b.logger.Warningf("xds: unexpected xdsClient type: %T", client) + b.logger.Warningf("Received balancer config with no cluster name") return balancer.ErrBadResolverState } - b.updateCh.Put(&ccUpdate{client: newClient, clusterName: lbCfg.ClusterName}) + b.updateCh.Put(&ccUpdate{clusterName: lbCfg.ClusterName}) return nil } // ResolverError handles errors reported by the xdsResolver. func (b *cdsBalancer) ResolverError(err error) { - if b.isClosed() { - b.logger.Warningf("xds: received resolver error {%v} after cdsBalancer was closed", err) + if b.closed.HasFired() { + b.logger.Warningf("Received resolver error after close: %v", err) return } - b.updateCh.Put(&ccUpdate{err: err}) } // UpdateSubConnState handles subConn updates from gRPC. func (b *cdsBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { - if b.isClosed() { - b.logger.Warningf("xds: received subConn update {%v, %v} after cdsBalancer was closed", sc, state) - return - } - b.updateCh.Put(&scUpdate{subConn: sc, state: state}) + b.logger.Errorf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, state) } -// Close closes the cdsBalancer and the underlying edsBalancer. +// Close cancels the CDS watch, closes the child policy and closes the +// cdsBalancer. func (b *cdsBalancer) Close() { - b.mu.Lock() - b.closed = true - b.mu.Unlock() - b.updateCh.Put(&closeUpdate{}) + b.closed.Fire() + <-b.done.Done() +} + +func (b *cdsBalancer) ExitIdle() { + b.updateCh.Put(exitIdle{}) +} + +// ccWrapper wraps the balancer.ClientConn passed to the CDS balancer at +// creation and intercepts the NewSubConn() and UpdateAddresses() call from the +// child policy to add security configuration required by xDS credentials. +// +// Other methods of the balancer.ClientConn interface are not overridden and +// hence get the original implementation. +type ccWrapper struct { + balancer.ClientConn + + // The certificate providers in this HandshakeInfo are updated based on the + // received security configuration in the Cluster resource. + xdsHI *xdsinternal.HandshakeInfo +} + +// NewSubConn intercepts NewSubConn() calls from the child policy and adds an +// address attribute which provides all information required by the xdsCreds +// handshaker to perform the TLS handshake. +func (ccw *ccWrapper) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) { + newAddrs := make([]resolver.Address, len(addrs)) + for i, addr := range addrs { + newAddrs[i] = xdsinternal.SetHandshakeInfo(addr, ccw.xdsHI) + } + // No need to override opts.StateListener; just forward all calls to the + // child that created the SubConn. + return ccw.ClientConn.NewSubConn(newAddrs, opts) } -func (b *cdsBalancer) isClosed() bool { - b.mu.Lock() - closed := b.closed - b.mu.Unlock() - return closed +func (ccw *ccWrapper) UpdateAddresses(sc balancer.SubConn, addrs []resolver.Address) { + newAddrs := make([]resolver.Address, len(addrs)) + for i, addr := range addrs { + newAddrs[i] = xdsinternal.SetHandshakeInfo(addr, ccw.xdsHI) + } + ccw.ClientConn.UpdateAddresses(sc, newAddrs) } diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go b/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go new file mode 100644 index 000000000000..11a7d33ed523 --- /dev/null +++ b/xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go @@ -0,0 +1,720 @@ +/* + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cdsbalancer + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "fmt" + "os" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc" + "google.golang.org/grpc/attributes" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/credentials/tls/certprovider" + "google.golang.org/grpc/credentials/xds" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/balancer/stub" + xdscredsinternal "google.golang.org/grpc/internal/credentials/xds" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + xdsbootstrap "google.golang.org/grpc/internal/testutils/xds/bootstrap" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/serviceconfig" + "google.golang.org/grpc/testdata" + "google.golang.org/grpc/xds/internal/xdsclient" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + + _ "google.golang.org/grpc/credentials/tls/certprovider/pemfile" // Register the file watcher certificate provider plugin. +) + +// testCCWrapper wraps a balancer.ClientConn and intercepts NewSubConn and +// returns the xDS handshake info back to the test for inspection. +type testCCWrapper struct { + balancer.ClientConn + handshakeInfoCh chan *xdscredsinternal.HandshakeInfo +} + +// NewSubConn forwards the call to the underlying balancer.ClientConn, but +// before that, it validates the following: +// - there is only one address in the addrs slice +// - the single address contains xDS handshake information, which is then +// pushed onto the handshakeInfoCh channel +func (tcc *testCCWrapper) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) { + if len(addrs) != 1 { + return nil, fmt.Errorf("NewSubConn got %d addresses, want 1", len(addrs)) + } + getHI := internal.GetXDSHandshakeInfoForTesting.(func(attr *attributes.Attributes) *xdscredsinternal.HandshakeInfo) + hi := getHI(addrs[0].Attributes) + if hi == nil { + return nil, fmt.Errorf("NewSubConn got address without xDS handshake info") + } + sc, err := tcc.ClientConn.NewSubConn(addrs, opts) + select { + case tcc.handshakeInfoCh <- hi: + default: + } + return sc, err +} + +// Registers a wrapped cds LB policy for the duration of this test that retains +// all the functionality of the original cds LB policy, but overrides the +// NewSubConn method passed to the policy and makes the xDS handshake +// information passed to NewSubConn available to the test. +// +// Accepts as argument a channel onto which xDS handshake information passed to +// NewSubConn is written to. +func registerWrappedCDSPolicyWithNewSubConnOverride(t *testing.T, ch chan *xdscredsinternal.HandshakeInfo) { + cdsBuilder := balancer.Get(cdsName) + internal.BalancerUnregister(cdsBuilder.Name()) + var ccWrapper *testCCWrapper + stub.Register(cdsBuilder.Name(), stub.BalancerFuncs{ + Init: func(bd *stub.BalancerData) { + ccWrapper = &testCCWrapper{ + ClientConn: bd.ClientConn, + handshakeInfoCh: ch, + } + bd.Data = cdsBuilder.Build(ccWrapper, bd.BuildOptions) + }, + ParseConfig: func(lbCfg json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + return cdsBuilder.(balancer.ConfigParser).ParseConfig(lbCfg) + }, + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + bal := bd.Data.(balancer.Balancer) + return bal.UpdateClientConnState(ccs) + }, + Close: func(bd *stub.BalancerData) { + bal := bd.Data.(balancer.Balancer) + bal.Close() + }, + }) + t.Cleanup(func() { balancer.Register(cdsBuilder) }) +} + +// Common setup for security tests: +// - creates an xDS client with the specified bootstrap configuration +// - creates a manual resolver that specifies cds as the top-level LB policy +// - creates a channel that uses the passed in client creds and the manual +// resolver +// - creates a test server that uses the passed in server creds +// +// Returns the following: +// - a client channel to make RPCs +// - address of the test backend server +func setupForSecurityTests(t *testing.T, bootstrapContents []byte, clientCreds, serverCreds credentials.TransportCredentials) (*grpc.ClientConn, string) { + t.Helper() + + xdsClient, xdsClose, err := xdsclient.NewWithBootstrapContentsForTesting(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + t.Cleanup(xdsClose) + + // Create a manual resolver that configures the CDS LB policy as the + // top-level LB policy on the channel. + r := manual.NewBuilderWithScheme("whatever") + jsonSC := fmt.Sprintf(`{ + "loadBalancingConfig":[{ + "cds_experimental":{ + "cluster": "%s" + } + }] + }`, clusterName) + scpr := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(jsonSC) + state := xdsclient.SetClient(resolver.State{ServiceConfig: scpr}, xdsClient) + r.InitialState(state) + + // Create a ClientConn with the specified transport credentials. + cc, err := grpc.Dial(r.Scheme()+":///test.service", grpc.WithTransportCredentials(clientCreds), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("Failed to dial local test server: %v", err) + } + t.Cleanup(func() { cc.Close() }) + + // Start a test service backend with the specified transport credentials. + sOpts := []grpc.ServerOption{} + if serverCreds != nil { + sOpts = append(sOpts, grpc.Creds(serverCreds)) + } + server := stubserver.StartTestService(t, nil, sOpts...) + t.Cleanup(server.Stop) + + return cc, server.Address +} + +// Creates transport credentials to be used on the client side that rely on xDS +// to provide the security configuration. It falls back to insecure creds if no +// security information is received from the management server. +func xdsClientCredsWithInsecureFallback(t *testing.T) credentials.TransportCredentials { + t.Helper() + + xdsCreds, err := xds.NewClientCredentials(xds.ClientOptions{FallbackCreds: insecure.NewCredentials()}) + if err != nil { + t.Fatalf("Failed to create xDS credentials: %v", err) + } + return xdsCreds +} + +// Creates transport credentials to be used on the server side from certificate +// files in testdata/x509. +// +// The certificate returned by this function has a CommonName of "test-server1". +func tlsServerCreds(t *testing.T) credentials.TransportCredentials { + t.Helper() + + cert, err := tls.LoadX509KeyPair(testdata.Path("x509/server1_cert.pem"), testdata.Path("x509/server1_key.pem")) + if err != nil { + t.Fatalf("Failed to load server cert and key: %v", err) + + } + pemData, err := os.ReadFile(testdata.Path("x509/client_ca_cert.pem")) + if err != nil { + t.Fatalf("Failed to read client CA cert: %v", err) + } + roots := x509.NewCertPool() + roots.AppendCertsFromPEM(pemData) + cfg := &tls.Config{ + Certificates: []tls.Certificate{cert}, + ClientCAs: roots, + } + return credentials.NewTLS(cfg) +} + +// Checks the AuthInfo available in the peer if it matches the expected security +// level of the connection. +func verifySecurityInformationFromPeer(t *testing.T, pr *peer.Peer, wantSecLevel e2e.SecurityLevel) { + // This is not a true helper in the Go sense, because it does not perform + // setup or cleanup tasks. Marking it a helper is to ensure that when the + // test fails, the line information of the caller is outputted instead of + // from here. + // + // And this function directly calls t.Fatalf() instead of returning an error + // and letting the caller decide what to do with it. This is also OK since + // all callers will simply end up calling t.Fatalf() with the returned + // error, and can't add any contextual information of value to the error + // message. + t.Helper() + + switch wantSecLevel { + case e2e.SecurityLevelNone: + if pr.AuthInfo.AuthType() != "insecure" { + t.Fatalf("AuthType() is %s, want insecure", pr.AuthInfo.AuthType()) + } + case e2e.SecurityLevelMTLS: + ai, ok := pr.AuthInfo.(credentials.TLSInfo) + if !ok { + t.Fatalf("AuthInfo type is %T, want %T", pr.AuthInfo, credentials.TLSInfo{}) + } + if len(ai.State.PeerCertificates) != 1 { + t.Fatalf("Number of peer certificates is %d, want 1", len(ai.State.PeerCertificates)) + } + cert := ai.State.PeerCertificates[0] + const wantCommonName = "test-server1" + if cn := cert.Subject.CommonName; cn != wantCommonName { + t.Fatalf("Common name in peer certificate is %s, want %s", cn, wantCommonName) + } + } +} + +// Tests the case where xDS credentials are not in use, but the cds LB policy +// receives a Cluster update with security configuration. Verifies that the +// security configuration is not parsed by the cds LB policy by looking at the +// xDS handshake info passed to NewSubConn. +func (s) TestSecurityConfigWithoutXDSCreds(t *testing.T) { + // Register a wrapped cds LB policy for the duration of this test that writes + // the xDS handshake info passed to NewSubConn onto the given channel. + handshakeInfoCh := make(chan *xdscredsinternal.HandshakeInfo, 1) + registerWrappedCDSPolicyWithNewSubConnOverride(t, handshakeInfoCh) + + // Spin up an xDS management server. + mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + t.Cleanup(cleanup) + + // Create a grpc channel with insecure creds talking to a test server with + // insecure credentials. + cc, serverAddress := setupForSecurityTests(t, bootstrapContents, insecure.NewCredentials(), nil) + + // Configure cluster and endpoints resources in the management server. The + // cluster resource is configured to return security configuration. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, serviceName, e2e.SecurityLevelMTLS)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, "localhost", []uint32{testutils.ParsePort(t, serverAddress)})}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Verify that a successful RPC can be made. + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + + // Ensure that the xDS handshake info passed to NewSubConn is empty. + var gotHI *xdscredsinternal.HandshakeInfo + select { + case gotHI = <-handshakeInfoCh: + case <-ctx.Done(): + t.Fatal("Timeout when waiting to read handshake info passed to NewSubConn") + } + wantHI := xdscredsinternal.NewHandshakeInfo(nil, nil) + if !cmp.Equal(gotHI, wantHI) { + t.Fatalf("NewSubConn got handshake info %+v, want %+v", gotHI, wantHI) + } +} + +// Tests the case where xDS credentials are in use, but the cds LB policy +// receives a Cluster update without security configuration. Verifies that the +// xDS handshake info passed to NewSubConn specified the use of fallback +// credentials. +func (s) TestNoSecurityConfigWithXDSCreds(t *testing.T) { + // Register a wrapped cds LB policy for the duration of this test that writes + // the xDS handshake info passed to NewSubConn onto the given channel. + handshakeInfoCh := make(chan *xdscredsinternal.HandshakeInfo, 1) + registerWrappedCDSPolicyWithNewSubConnOverride(t, handshakeInfoCh) + + // Spin up an xDS management server. + mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + t.Cleanup(cleanup) + + // Create a grpc channel with xDS creds talking to a test server with + // insecure credentials. + cc, serverAddress := setupForSecurityTests(t, bootstrapContents, xdsClientCredsWithInsecureFallback(t), nil) + + // Configure cluster and endpoints resources in the management server. The + // cluster resource is not configured to return any security configuration. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, serviceName, e2e.SecurityLevelNone)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, "localhost", []uint32{testutils.ParsePort(t, serverAddress)})}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Verify that a successful RPC can be made. + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + + // Ensure that the xDS handshake info passed to NewSubConn is empty. + var gotHI *xdscredsinternal.HandshakeInfo + select { + case gotHI = <-handshakeInfoCh: + case <-ctx.Done(): + t.Fatal("Timeout when waiting to read handshake info passed to NewSubConn") + } + wantHI := xdscredsinternal.NewHandshakeInfo(nil, nil) + if !cmp.Equal(gotHI, wantHI) { + t.Fatalf("NewSubConn got handshake info %+v, want %+v", gotHI, wantHI) + } + if !gotHI.UseFallbackCreds() { + t.Fatal("NewSubConn got hanshake info that does not specify the use of fallback creds") + } +} + +// Tests the case where the security config returned by the management server +// cannot be resolved based on the contents of the bootstrap config. Verifies +// that the cds LB policy puts the channel in TRANSIENT_FAILURE. +func (s) TestSecurityConfigNotFoundInBootstrap(t *testing.T) { + // Spin up an xDS management server. + mgmtServer, nodeID, _, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + t.Cleanup(cleanup) + + // Ignore the bootstrap configuration returned by the above call to + // e2e.SetupManagementServer and create a new one that does not have + // ceritificate providers configuration. + bootstrapContents, err := xdsbootstrap.Contents(xdsbootstrap.Options{ + NodeID: nodeID, + ServerURI: mgmtServer.Address, + ServerListenerResourceNameTemplate: e2e.ServerListenerResourceNameTemplate, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + + // Create a grpc channel with xDS creds. + cc, _ := setupForSecurityTests(t, bootstrapContents, xdsClientCredsWithInsecureFallback(t), nil) + + // Configure a cluster resource that contains security configuration, in the + // management server. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, serviceName, e2e.SecurityLevelMTLS)}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) +} + +// A ceritificate provider builder that returns a nil Provider from the starter +// func passed to certprovider.NewBuildableConfig(). +type errCertProviderBuilder struct{} + +const errCertProviderName = "err-cert-provider" + +func (e errCertProviderBuilder) ParseConfig(any) (*certprovider.BuildableConfig, error) { + // Returning a nil Provider simulates the case where an error is encountered + // at the time of building the Provider. + bc := certprovider.NewBuildableConfig(errCertProviderName, nil, func(certprovider.BuildOptions) certprovider.Provider { return nil }) + return bc, nil +} + +func (e errCertProviderBuilder) Name() string { + return errCertProviderName +} + +func init() { + certprovider.Register(errCertProviderBuilder{}) +} + +// Tests the case where the certprovider.Store returns an error when the cds LB +// policy attempts to build a certificate provider. Verifies that the cds LB +// policy puts the channel in TRANSIENT_FAILURE. +func (s) TestCertproviderStoreError(t *testing.T) { + mgmtServer, nodeID, _, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + t.Cleanup(cleanup) + + // Ignore the bootstrap configuration returned by the above call to + // e2e.SetupManagementServer and create a new one that includes ceritificate + // providers configuration for errCertProviderBuilder. + providerCfg := json.RawMessage(fmt.Sprintf(`{ + "plugin_name": "%s", + "config": {} + }`, errCertProviderName)) + bootstrapContents, err := xdsbootstrap.Contents(xdsbootstrap.Options{ + NodeID: nodeID, + ServerURI: mgmtServer.Address, + CertificateProviders: map[string]json.RawMessage{e2e.ClientSideCertProviderInstance: providerCfg}, + ServerListenerResourceNameTemplate: e2e.ServerListenerResourceNameTemplate, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + + // Create a grpc channel with xDS creds. + cc, _ := setupForSecurityTests(t, bootstrapContents, xdsClientCredsWithInsecureFallback(t), nil) + + // Configure a cluster resource that contains security configuration, in the + // management server. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, serviceName, e2e.SecurityLevelMTLS)}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) +} + +// Tests the case where the cds LB policy receives security configuration as +// part of the Cluster resource that can be successfully resolved using the +// bootstrap file contents. Verifies that the connection between the client and +// the server is secure. +func (s) TestGoodSecurityConfig(t *testing.T) { + // Spin up an xDS management server. + mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + t.Cleanup(cleanup) + + // Create a grpc channel with xDS creds talking to a test server with TLS + // credentials. + cc, serverAddress := setupForSecurityTests(t, bootstrapContents, xdsClientCredsWithInsecureFallback(t), tlsServerCreds(t)) + + // Configure cluster and endpoints resources in the management server. The + // cluster resource is configured to return security configuration. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, serviceName, e2e.SecurityLevelMTLS)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, "localhost", []uint32{testutils.ParsePort(t, serverAddress)})}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Verify that a successful RPC can be made over a secure connection. + client := testgrpc.NewTestServiceClient(cc) + peer := &peer.Peer{} + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + verifySecurityInformationFromPeer(t, peer, e2e.SecurityLevelMTLS) +} + +// Tests the case where the cds LB policy receives security configuration as +// part of the Cluster resource that contains a certificate provider instance +// that is missing in the bootstrap file. Verifies that the channel moves to +// TRANSIENT_FAILURE. Subsequently, the cds LB policy receives a cluster +// resource that contains a certificate provider that is present in the +// bootstrap file. Verifies that the connection between the client and the +// server is secure. +func (s) TestSecurityConfigUpdate_BadToGood(t *testing.T) { + // Spin up an xDS management server. + mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + t.Cleanup(cleanup) + + // Create a grpc channel with xDS creds talking to a test server with TLS + // credentials. + cc, serverAddress := setupForSecurityTests(t, bootstrapContents, xdsClientCredsWithInsecureFallback(t), tlsServerCreds(t)) + + // Configure cluster and endpoints resources in the management server. The + // cluster resource contains security configuration with a certificate + // provider instance that is missing in the bootstrap configuration. + cluster := e2e.DefaultCluster(clusterName, serviceName, e2e.SecurityLevelNone) + cluster.TransportSocket = &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{ + ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: "unknown-certificate-provider-instance", + }, + }, + }, + }), + }, + } + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{cluster}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, "localhost", []uint32{testutils.ParsePort(t, serverAddress)})}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + + // Update the management server with a Cluster resource that contains a + // certificate provider instance that is present in the bootstrap + // configuration. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, serviceName, e2e.SecurityLevelMTLS)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, "localhost", []uint32{testutils.ParsePort(t, serverAddress)})}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Verify that a successful RPC can be made over a secure connection. + client := testgrpc.NewTestServiceClient(cc) + peer := &peer.Peer{} + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + verifySecurityInformationFromPeer(t, peer, e2e.SecurityLevelMTLS) +} + +// Tests the case where the cds LB policy receives security configuration as +// part of the Cluster resource that can be successfully resolved using the +// bootstrap file contents. Verifies that the connection between the client and +// the server is secure. Subsequently, the cds LB policy receives a cluster +// resource without security configuration. Verifies that this results in the +// use of fallback credentials, which in this case is insecure creds. +func (s) TestSecurityConfigUpdate_GoodToFallback(t *testing.T) { + // Spin up an xDS management server. + mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + t.Cleanup(cleanup) + + // Create a grpc channel with xDS creds talking to a test server with TLS + // credentials. + cc, serverAddress := setupForSecurityTests(t, bootstrapContents, xdsClientCredsWithInsecureFallback(t), tlsServerCreds(t)) + + // Configure cluster and endpoints resources in the management server. The + // cluster resource is configured to return security configuration. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, serviceName, e2e.SecurityLevelMTLS)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, "localhost", []uint32{testutils.ParsePort(t, serverAddress)})}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Verify that a successful RPC can be made over a secure connection. + client := testgrpc.NewTestServiceClient(cc) + peer := &peer.Peer{} + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + verifySecurityInformationFromPeer(t, peer, e2e.SecurityLevelMTLS) + + // Start a test service backend that does not expect a secure connection. + insecureServer := stubserver.StartTestService(t, nil) + t.Cleanup(insecureServer.Stop) + + // Update the resources in the management server to contain no security + // configuration. This should result in the use of fallback credentials, + // which is insecure in our case. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, serviceName, e2e.SecurityLevelNone)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, "localhost", []uint32{testutils.ParsePort(t, insecureServer.Address)})}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Wait for the connection to move to the new backend that expects + // connections without security. + for ctx.Err() == nil { + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil { + t.Logf("EmptyCall() failed: %v", err) + } + if peer.Addr.String() == insecureServer.Address { + break + } + } + if ctx.Err() != nil { + t.Fatal("Timed out when waiting for connection to switch to second backend") + } + verifySecurityInformationFromPeer(t, peer, e2e.SecurityLevelNone) +} + +// Tests the case where the cds LB policy receives security configuration as +// part of the Cluster resource that can be successfully resolved using the +// bootstrap file contents. Verifies that the connection between the client and +// the server is secure. Subsequently, the cds LB policy receives a cluster +// resource that is NACKed by the xDS client. Test verifies that the cds LB +// policy continues to use the previous good configuration, but the error from +// the xDS client is propagated to the child policy. +func (s) TestSecurityConfigUpdate_GoodToBad(t *testing.T) { + // Register a wrapped clusterresolver LB policy (child policy of the cds LB + // policy) for the duration of this test that makes the resolver error + // pushed to it available to the test. + _, resolverErrCh, _, _ := registerWrappedClusterResolverPolicy(t) + + // Spin up an xDS management server. + mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + t.Cleanup(cleanup) + + // Create a grpc channel with xDS creds talking to a test server with TLS + // credentials. + cc, serverAddress := setupForSecurityTests(t, bootstrapContents, xdsClientCredsWithInsecureFallback(t), tlsServerCreds(t)) + + // Configure cluster and endpoints resources in the management server. The + // cluster resource is configured to return security configuration. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, serviceName, e2e.SecurityLevelMTLS)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, "localhost", []uint32{testutils.ParsePort(t, serverAddress)})}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Verify that a successful RPC can be made over a secure connection. + client := testgrpc.NewTestServiceClient(cc) + peer := &peer.Peer{} + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + verifySecurityInformationFromPeer(t, peer, e2e.SecurityLevelMTLS) + + // Configure cluster and endpoints resources in the management server. The + // cluster resource contains security configuration with a certificate + // provider instance that is missing in the bootstrap configuration. + cluster := e2e.DefaultCluster(clusterName, serviceName, e2e.SecurityLevelNone) + cluster.TransportSocket = &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{ + ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: "unknown-certificate-provider-instance", + }, + }, + }, + }), + }, + } + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{cluster}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, "localhost", []uint32{testutils.ParsePort(t, serverAddress)})}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + const wantNACKErr = "instance name \"unknown-certificate-provider-instance\" missing in bootstrap configuration" + select { + case err := <-resolverErrCh: + if !strings.Contains(err.Error(), wantNACKErr) { + t.Fatalf("Child policy got resolver error: %v, want err: %v", err, wantNACKErr) + } + case <-ctx.Done(): + t.Fatal("Timeout when waiting for resolver error to be pushed to the child policy") + } + + // Verify that a successful RPC can be made over a secure connection. + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + verifySecurityInformationFromPeer(t, peer, e2e.SecurityLevelMTLS) +} diff --git a/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go b/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go index 358e6c3895ef..1d02c4810912 100644 --- a/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go +++ b/xds/internal/balancer/cdsbalancer/cdsbalancer_test.go @@ -17,32 +17,53 @@ package cdsbalancer import ( + "context" "encoding/json" "errors" "fmt" + "strings" "testing" "time" "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" - "google.golang.org/grpc/attributes" + "google.golang.org/grpc" "google.golang.org/grpc/balancer" + "google.golang.org/grpc/codes" "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/balancer/stub" "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" "google.golang.org/grpc/serviceconfig" - xdsinternal "google.golang.org/grpc/xds/internal" - "google.golang.org/grpc/xds/internal/balancer/edsbalancer" - xdsclient "google.golang.org/grpc/xds/internal/client" - "google.golang.org/grpc/xds/internal/testutils" - "google.golang.org/grpc/xds/internal/testutils/fakeclient" + "google.golang.org/grpc/status" + "google.golang.org/grpc/xds/internal/balancer/clusterresolver" + "google.golang.org/grpc/xds/internal/xdsclient" + "google.golang.org/grpc/xds/internal/xdsclient/bootstrap" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource/version" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/wrapperspb" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + + _ "google.golang.org/grpc/xds/internal/balancer/ringhash" // Register the ring_hash LB policy ) const ( - clusterName = "cluster1" - serviceName = "service1" - defaultTestTimeout = 2 * time.Second + clusterName = "cluster1" + serviceName = "service1" + defaultTestTimeout = 5 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond // For events expected to *not* happen. ) type s struct { @@ -53,526 +74,940 @@ func Test(t *testing.T) { grpctest.RunSubTests(t, s{}) } -type testClientConn struct { - balancer.ClientConn - - newPickerCh *testutils.Channel // The last picker updated. -} - -func newTestClientConn() *testClientConn { - return &testClientConn{ - newPickerCh: testutils.NewChannelWithSize(1), +func waitForResourceNames(ctx context.Context, resourceNamesCh chan []string, wantNames []string) error { + for ctx.Err() == nil { + select { + case <-ctx.Done(): + case gotNames := <-resourceNamesCh: + if cmp.Equal(gotNames, wantNames) { + return nil + } + } } + if ctx.Err() != nil { + return fmt.Errorf("Timeout when waiting for appropriate Cluster resources to be requested") + } + return nil } -func (tcc *testClientConn) UpdateState(bs balancer.State) { - tcc.newPickerCh.Replace(bs) -} +// Registers a wrapped cluster_resolver LB policy (child policy of the cds LB +// policy) for the duration of this test that retains all the functionality of +// the former, but makes certain events available for inspection by the test. +// +// Returns the following: +// - a channel to read received load balancing configuration +// - a channel to read received resolver error +// - a channel that is closed when ExitIdle() is called +// - a channel that is closed when the balancer is closed +func registerWrappedClusterResolverPolicy(t *testing.T) (chan serviceconfig.LoadBalancingConfig, chan error, chan struct{}, chan struct{}) { + clusterresolverBuilder := balancer.Get(clusterresolver.Name) + internal.BalancerUnregister(clusterresolverBuilder.Name()) -// cdsWatchInfo wraps the update and the error sent in a CDS watch callback. -type cdsWatchInfo struct { - update xdsclient.ClusterUpdate - err error -} + lbCfgCh := make(chan serviceconfig.LoadBalancingConfig, 1) + resolverErrCh := make(chan error, 1) + exitIdleCh := make(chan struct{}) + closeCh := make(chan struct{}) -// invokeWatchCb invokes the CDS watch callback registered by the cdsBalancer -// and waits for appropriate state to be pushed to the provided edsBalancer. -func invokeWatchCbAndWait(xdsC *fakeclient.Client, cdsW cdsWatchInfo, wantCCS balancer.ClientConnState, edsB *testEDSBalancer) error { - xdsC.InvokeWatchClusterCallback(cdsW.update, cdsW.err) - if cdsW.err != nil { - return edsB.waitForResolverError(cdsW.err) - } - return edsB.waitForClientConnUpdate(wantCCS) -} + stub.Register(clusterresolver.Name, stub.BalancerFuncs{ + Init: func(bd *stub.BalancerData) { + bd.Data = clusterresolverBuilder.Build(bd.ClientConn, bd.BuildOptions) + }, + ParseConfig: func(lbCfg json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + return clusterresolverBuilder.(balancer.ConfigParser).ParseConfig(lbCfg) + }, + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + select { + case lbCfgCh <- ccs.BalancerConfig: + default: + } + bal := bd.Data.(balancer.Balancer) + return bal.UpdateClientConnState(ccs) + }, + ResolverError: func(bd *stub.BalancerData, err error) { + select { + case resolverErrCh <- err: + default: + } + bal := bd.Data.(balancer.Balancer) + bal.ResolverError(err) + }, + ExitIdle: func(bd *stub.BalancerData) { + bal := bd.Data.(balancer.Balancer) + bal.(balancer.ExitIdler).ExitIdle() + close(exitIdleCh) + }, + Close: func(bd *stub.BalancerData) { + bal := bd.Data.(balancer.Balancer) + bal.Close() + close(closeCh) + }, + }) + t.Cleanup(func() { balancer.Register(clusterresolverBuilder) }) -// testEDSBalancer is a fake edsBalancer used to verify different actions from -// the cdsBalancer. It contains a bunch of channels to signal different events -// to the test. -type testEDSBalancer struct { - // ccsCh is a channel used to signal the receipt of a ClientConn update. - ccsCh chan balancer.ClientConnState - // scStateCh is a channel used to signal the receipt of a SubConn update. - scStateCh chan subConnWithState - // resolverErrCh is a channel used to signal a resolver error. - resolverErrCh chan error - // closeCh is a channel used to signal the closing of this balancer. - closeCh chan struct{} + return lbCfgCh, resolverErrCh, exitIdleCh, closeCh } -type subConnWithState struct { - sc balancer.SubConn - state balancer.SubConnState -} +// Registers a wrapped cds LB policy for the duration of this test that retains +// all the functionality of the original cds LB policy, but makes the newly +// built policy available to the test to directly invoke any balancer methods. +// +// Returns a channel on which the newly built cds LB policy is written to. +func registerWrappedCDSPolicy(t *testing.T) chan balancer.Balancer { + cdsBuilder := balancer.Get(cdsName) + internal.BalancerUnregister(cdsBuilder.Name()) + cdsBalancerCh := make(chan balancer.Balancer, 1) + stub.Register(cdsBuilder.Name(), stub.BalancerFuncs{ + Init: func(bd *stub.BalancerData) { + bal := cdsBuilder.Build(bd.ClientConn, bd.BuildOptions) + bd.Data = bal + cdsBalancerCh <- bal + }, + ParseConfig: func(lbCfg json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + return cdsBuilder.(balancer.ConfigParser).ParseConfig(lbCfg) + }, + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + bal := bd.Data.(balancer.Balancer) + return bal.UpdateClientConnState(ccs) + }, + Close: func(bd *stub.BalancerData) { + bal := bd.Data.(balancer.Balancer) + bal.Close() + }, + }) + t.Cleanup(func() { balancer.Register(cdsBuilder) }) -func newTestEDSBalancer() *testEDSBalancer { - return &testEDSBalancer{ - ccsCh: make(chan balancer.ClientConnState, 1), - scStateCh: make(chan subConnWithState, 1), - resolverErrCh: make(chan error, 1), - closeCh: make(chan struct{}, 1), - } + return cdsBalancerCh } -func (tb *testEDSBalancer) UpdateClientConnState(ccs balancer.ClientConnState) error { - tb.ccsCh <- ccs - return nil -} +// Performs the following setup required for tests: +// - Spins up an xDS management server +// - Creates an xDS client talking to this management server +// - Creates a manual resolver that configures the cds LB policy as the +// top-level policy, and pushes an initial configuration to it +// - Creates a gRPC channel with the above manual resolver +// +// Returns the following: +// - the xDS management server +// - the nodeID expected by the management server +// - the grpc channel to the test backend service +// - the manual resolver configured on the channel +// - the xDS cient used the grpc channel +// - a channel on which requested cluster resource names are sent +// - a channel used to signal that previously requested cluster resources are +// no longer requested +func setupWithManagementServer(t *testing.T) (*e2e.ManagementServer, string, *grpc.ClientConn, *manual.Resolver, xdsclient.XDSClient, chan []string, chan struct{}) { + t.Helper() -func (tb *testEDSBalancer) ResolverError(err error) { - tb.resolverErrCh <- err -} + cdsResourceRequestedCh := make(chan []string, 1) + cdsResourceCanceledCh := make(chan struct{}, 1) + mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{ + OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + if req.GetTypeUrl() == version.V3ClusterURL { + switch len(req.GetResourceNames()) { + case 0: + select { + case cdsResourceCanceledCh <- struct{}{}: + default: + } + default: + select { + case cdsResourceRequestedCh <- req.GetResourceNames(): + default: + } + } + } + return nil + }, + }) + t.Cleanup(cleanup) -func (tb *testEDSBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { - tb.scStateCh <- subConnWithState{sc: sc, state: state} -} + xdsC, xdsClose, err := xdsclient.NewWithBootstrapContentsForTesting(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + t.Cleanup(xdsClose) -func (tb *testEDSBalancer) Close() { - tb.closeCh <- struct{}{} -} + r := manual.NewBuilderWithScheme("whatever") + jsonSC := fmt.Sprintf(`{ + "loadBalancingConfig":[{ + "cds_experimental":{ + "cluster": "%s" + } + }] + }`, clusterName) + scpr := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(jsonSC) + r.InitialState(xdsclient.SetClient(resolver.State{ServiceConfig: scpr}, xdsC)) -// waitForClientConnUpdate verifies if the testEDSBalancer receives the -// provided ClientConnState within a reasonable amount of time. -func (tb *testEDSBalancer) waitForClientConnUpdate(wantCCS balancer.ClientConnState) error { - timer := time.NewTimer(defaultTestTimeout) - select { - case <-timer.C: - return errors.New("Timeout when expecting ClientConn update on EDS balancer") - case gotCCS := <-tb.ccsCh: - timer.Stop() - if !cmp.Equal(gotCCS, wantCCS, cmpopts.IgnoreUnexported(attributes.Attributes{})) { - return fmt.Errorf("received ClientConnState: %+v, want %+v", gotCCS, wantCCS) - } - return nil + cc, err := grpc.Dial(r.Scheme()+":///test.service", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("Failed to dial local test server: %v", err) } + t.Cleanup(func() { cc.Close() }) + + return mgmtServer, nodeID, cc, r, xdsC, cdsResourceRequestedCh, cdsResourceCanceledCh } -// waitForSubConnUpdate verifies if the testEDSBalancer receives the provided -// SubConn update within a reasonable amount of time. -func (tb *testEDSBalancer) waitForSubConnUpdate(wantSCS subConnWithState) error { - timer := time.NewTimer(defaultTestTimeout) +// Helper function to compare the load balancing configuration received on the +// channel with the expected one. Both configs are marshalled to JSON and then +// compared. +// +// Returns an error if marshalling to JSON fails, or if the load balancing +// configurations don't match, or if the context deadline expires before reading +// a child policy configuration off of the lbCfgCh. +func compareLoadBalancingConfig(ctx context.Context, lbCfgCh chan serviceconfig.LoadBalancingConfig, wantChildCfg serviceconfig.LoadBalancingConfig) error { + wantJSON, err := json.Marshal(wantChildCfg) + if err != nil { + return fmt.Errorf("failed to marshal expected child config to JSON: %v", err) + } select { - case <-timer.C: - return errors.New("Timeout when expecting SubConn update on EDS balancer") - case gotSCS := <-tb.scStateCh: - timer.Stop() - if !cmp.Equal(gotSCS, wantSCS, cmp.AllowUnexported(subConnWithState{})) { - return fmt.Errorf("received SubConnState: %+v, want %+v", gotSCS, wantSCS) + case lbCfg := <-lbCfgCh: + gotJSON, err := json.Marshal(lbCfg) + if err != nil { + return fmt.Errorf("failed to marshal received LB config into JSON: %v", err) + } + if diff := cmp.Diff(wantJSON, gotJSON); diff != "" { + return fmt.Errorf("child policy received unexpected diff in config (-want +got):\n%s", diff) } - return nil + case <-ctx.Done(): + return fmt.Errorf("timeout when waiting for child policy to receive its configuration") } + return nil } -// waitForResolverError verifies if the testEDSBalancer receives the -// provided resolver error within a reasonable amount of time. -func (tb *testEDSBalancer) waitForResolverError(wantErr error) error { - timer := time.NewTimer(defaultTestTimeout) - select { - case <-timer.C: - return errors.New("Timeout when expecting a resolver error") - case gotErr := <-tb.resolverErrCh: - timer.Stop() - if gotErr != wantErr { - return fmt.Errorf("received resolver error: %v, want %v", gotErr, wantErr) - } - return nil +// Tests the functionality that handles LB policy configuration. Verifies that +// the appropriate xDS resource is requested corresponding to the provided LB +// policy configuration. Also verifies that when the LB policy receives the same +// configuration again, it does not send out a new request, and when the +// configuration changes, it stops requesting the old cluster resource and +// starts requesting the new one. +func (s) TestConfigurationUpdate_Success(t *testing.T) { + _, _, _, r, xdsClient, cdsResourceRequestedCh, _ := setupWithManagementServer(t) + + // Verify that the specified cluster resource is requested. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + wantNames := []string{clusterName} + if err := waitForResourceNames(ctx, cdsResourceRequestedCh, wantNames); err != nil { + t.Fatal(err) } -} -// waitForClose verifies that the edsBalancer is closed with a reasonable -// amount of time. -func (tb *testEDSBalancer) waitForClose() error { - timer := time.NewTimer(defaultTestTimeout) + // Push the same configuration again. + jsonSC := fmt.Sprintf(`{ + "loadBalancingConfig":[{ + "cds_experimental":{ + "cluster": "%s" + } + }] + }`, clusterName) + scpr := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(jsonSC) + r.UpdateState(xdsclient.SetClient(resolver.State{ServiceConfig: scpr}, xdsClient)) + + // Verify that a new CDS request is not sent. + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() select { - case <-timer.C: - return errors.New("Timeout when expecting a close") - case <-tb.closeCh: - timer.Stop() - return nil + case <-sCtx.Done(): + case gotNames := <-cdsResourceRequestedCh: + t.Fatalf("CDS resources %v requested when none expected", gotNames) } -} -// cdsCCS is a helper function to construct a good update passed from the -// xdsResolver to the cdsBalancer. -func cdsCCS(cluster string, xdsClient interface{}) balancer.ClientConnState { - const cdsLBConfig = `{ - "loadBalancingConfig":[ - { - "cds_experimental":{ - "Cluster": "%s" - } - } - ] - }` - jsonSC := fmt.Sprintf(cdsLBConfig, cluster) - return balancer.ClientConnState{ - ResolverState: resolver.State{ - ServiceConfig: internal.ParseServiceConfigForTesting.(func(string) *serviceconfig.ParseResult)(jsonSC), - Attributes: attributes.New(xdsinternal.XDSClientID, xdsClient), - }, - BalancerConfig: &lbConfig{ClusterName: clusterName}, + // Push an updated configuration with a different cluster name. + newClusterName := clusterName + "-new" + jsonSC = fmt.Sprintf(`{ + "loadBalancingConfig":[{ + "cds_experimental":{ + "cluster": "%s" + } + }] + }`, newClusterName) + scpr = internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(jsonSC) + r.UpdateState(xdsclient.SetClient(resolver.State{ServiceConfig: scpr}, xdsClient)) + + // Verify that the new cluster name is requested and the old one is no + // longer requested. + wantNames = []string{newClusterName} + if err := waitForResourceNames(ctx, cdsResourceRequestedCh, wantNames); err != nil { + t.Fatal(err) } } -// edsCCS is a helper function to construct a good update passed from the -// cdsBalancer to the edsBalancer. -func edsCCS(service string, enableLRS bool, xdsClient interface{}) balancer.ClientConnState { - lbCfg := &edsbalancer.EDSConfig{EDSServiceName: service} - if enableLRS { - lbCfg.LrsLoadReportingServerName = new(string) - } - return balancer.ClientConnState{ - ResolverState: resolver.State{Attributes: attributes.New(xdsinternal.XDSClientID, xdsClient)}, - BalancerConfig: lbCfg, +// Tests the case where a configuration with an empty cluster name is pushed to +// the CDS LB policy. Verifies that ErrBadResolverState is returned. +func (s) TestConfigurationUpdate_EmptyCluster(t *testing.T) { + // Setup a management server and an xDS client to talk to it. + _, _, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + t.Cleanup(cleanup) + xdsClient, xdsClose, err := xdsclient.NewWithBootstrapContentsForTesting(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) } -} + t.Cleanup(xdsClose) -// setup creates a cdsBalancer and an edsBalancer (and overrides the -// newEDSBalancer function to return it), and also returns a cleanup function. -func setup() (*cdsBalancer, *testEDSBalancer, *testClientConn, func()) { - builder := cdsBB{} - tcc := newTestClientConn() - cdsB := builder.Build(tcc, balancer.BuildOptions{}) + // Create a manual resolver that configures the CDS LB policy as the + // top-level LB policy on the channel, and pushes a configuration with an + // empty cluster name. Also, register a callback with the manual resolver to + // receive the error returned by the balancer when a configuration with an + // empty cluster name is pushed. + r := manual.NewBuilderWithScheme("whatever") + updateStateErrCh := make(chan error, 1) + r.UpdateStateCallback = func(err error) { updateStateErrCh <- err } + jsonSC := `{ + "loadBalancingConfig":[{ + "cds_experimental":{ + "cluster": "" + } + }] + }` + scpr := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(jsonSC) + r.InitialState(xdsclient.SetClient(resolver.State{ServiceConfig: scpr}, xdsClient)) - edsB := newTestEDSBalancer() - oldEDSBalancerBuilder := newEDSBalancer - newEDSBalancer = func(cc balancer.ClientConn, opts balancer.BuildOptions) (balancer.Balancer, error) { - return edsB, nil + // Create a ClientConn with the above manual resolver. + cc, err := grpc.Dial(r.Scheme()+":///test.service", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("Failed to dial: %v", err) } + t.Cleanup(func() { cc.Close() }) - return cdsB.(*cdsBalancer), edsB, tcc, func() { - newEDSBalancer = oldEDSBalancerBuilder + select { + case <-time.After(defaultTestTimeout): + t.Fatalf("Timed out waiting for error from the LB policy") + case err := <-updateStateErrCh: + if err != balancer.ErrBadResolverState { + t.Fatalf("For a configuration update with an empty cluster name, got error %v from the LB policy, want %v", err, balancer.ErrBadResolverState) + } } } -// setupWithWatch does everything that setup does, and also pushes a ClientConn -// update to the cdsBalancer and waits for a CDS watch call to be registered. -func setupWithWatch(t *testing.T) (*fakeclient.Client, *cdsBalancer, *testEDSBalancer, *testClientConn, func()) { - t.Helper() +// Tests the case where a configuration with a missing xDS client is pushed to +// the CDS LB policy. Verifies that ErrBadResolverState is returned. +func (s) TestConfigurationUpdate_MissingXdsClient(t *testing.T) { + // Create a manual resolver that configures the CDS LB policy as the + // top-level LB policy on the channel, and pushes a configuration that is + // missing the xDS client. Also, register a callback with the manual + // resolver to receive the error returned by the balancer. + r := manual.NewBuilderWithScheme("whatever") + updateStateErrCh := make(chan error, 1) + r.UpdateStateCallback = func(err error) { updateStateErrCh <- err } + jsonSC := `{ + "loadBalancingConfig":[{ + "cds_experimental":{ + "cluster": "foo" + } + }] + }` + scpr := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(jsonSC) + r.InitialState(resolver.State{ServiceConfig: scpr}) - xdsC := fakeclient.NewClient() - cdsB, edsB, tcc, cancel := setup() - if err := cdsB.UpdateClientConnState(cdsCCS(clusterName, xdsC)); err != nil { - t.Fatalf("cdsBalancer.UpdateClientConnState failed with error: %v", err) - } - gotCluster, err := xdsC.WaitForWatchCluster() + // Create a ClientConn with the above manual resolver. + cc, err := grpc.Dial(r.Scheme()+":///test.service", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) if err != nil { - t.Fatalf("xdsClient.WatchCDS failed with error: %v", err) + t.Fatalf("Failed to dial: %v", err) } - if gotCluster != clusterName { - t.Fatalf("xdsClient.WatchCDS called for cluster: %v, want: %v", gotCluster, clusterName) + t.Cleanup(func() { cc.Close() }) + + select { + case <-time.After(defaultTestTimeout): + t.Fatalf("Timed out waiting for error from the LB policy") + case err := <-updateStateErrCh: + if err != balancer.ErrBadResolverState { + t.Fatalf("For a configuration update missing the xDS client, got error %v from the LB policy, want %v", err, balancer.ErrBadResolverState) + } } - return xdsC, cdsB, edsB, tcc, cancel } -// TestUpdateClientConnState invokes the UpdateClientConnState method on the -// cdsBalancer with different inputs and verifies that the CDS watch API on the -// provided xdsClient is invoked appropriately. -func (s) TestUpdateClientConnState(t *testing.T) { - xdsC := fakeclient.NewClient() - +// Tests success scenarios where the cds LB policy receives a cluster resource +// from the management server. Verifies that the load balancing configuration +// pushed to the child is as expected. +func (s) TestClusterUpdate_Success(t *testing.T) { tests := []struct { - name string - ccs balancer.ClientConnState - wantErr error - wantCluster string + name string + clusterResource *v3clusterpb.Cluster + wantChildCfg serviceconfig.LoadBalancingConfig }{ { - name: "bad-lbCfg-type", - ccs: balancer.ClientConnState{BalancerConfig: nil}, - wantErr: balancer.ErrBadResolverState, - }, - { - name: "empty-cluster-in-lbCfg", - ccs: balancer.ClientConnState{BalancerConfig: &lbConfig{ClusterName: ""}}, - wantErr: balancer.ErrBadResolverState, + name: "happy-case-with-circuit-breakers", + clusterResource: func() *v3clusterpb.Cluster { + c := e2e.DefaultCluster(clusterName, serviceName, e2e.SecurityLevelNone) + c.CircuitBreakers = &v3clusterpb.CircuitBreakers{ + Thresholds: []*v3clusterpb.CircuitBreakers_Thresholds{ + { + Priority: v3corepb.RoutingPriority_DEFAULT, + MaxRequests: wrapperspb.UInt32(512), + }, + { + Priority: v3corepb.RoutingPriority_HIGH, + MaxRequests: nil, + }, + }, + } + return c + }(), + wantChildCfg: &clusterresolver.LBConfig{ + DiscoveryMechanisms: []clusterresolver.DiscoveryMechanism{{ + Cluster: clusterName, + Type: clusterresolver.DiscoveryMechanismTypeEDS, + EDSServiceName: serviceName, + MaxConcurrentRequests: newUint32(512), + OutlierDetection: json.RawMessage(`{}`), + }}, + XDSLBPolicy: json.RawMessage(`[{"xds_wrr_locality_experimental": {"childPolicy": [{"round_robin": {}}]}}]`), + }, }, { - name: "no-xdsClient-in-attributes", - ccs: balancer.ClientConnState{ - ResolverState: resolver.State{ - Attributes: attributes.New("key", "value"), - }, - BalancerConfig: &lbConfig{ClusterName: clusterName}, + name: "happy-case-with-ring-hash-lb-policy", + clusterResource: func() *v3clusterpb.Cluster { + c := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: clusterName, + ServiceName: serviceName, + SecurityLevel: e2e.SecurityLevelNone, + Policy: e2e.LoadBalancingPolicyRingHash, + }) + c.LbConfig = &v3clusterpb.Cluster_RingHashLbConfig_{ + RingHashLbConfig: &v3clusterpb.Cluster_RingHashLbConfig{ + MinimumRingSize: &wrapperspb.UInt64Value{Value: 100}, + MaximumRingSize: &wrapperspb.UInt64Value{Value: 1000}, + }, + } + return c + }(), + wantChildCfg: &clusterresolver.LBConfig{ + DiscoveryMechanisms: []clusterresolver.DiscoveryMechanism{{ + Cluster: clusterName, + Type: clusterresolver.DiscoveryMechanismTypeEDS, + EDSServiceName: serviceName, + OutlierDetection: json.RawMessage(`{}`), + }}, + XDSLBPolicy: json.RawMessage(`[{"ring_hash_experimental": {"minRingSize":100, "maxRingSize":1000}}]`), }, - wantErr: balancer.ErrBadResolverState, }, { - name: "bad-xdsClient-in-attributes", - ccs: balancer.ClientConnState{ - ResolverState: resolver.State{ - Attributes: attributes.New(xdsinternal.XDSClientID, "value"), - }, - BalancerConfig: &lbConfig{ClusterName: clusterName}, + name: "happy-case-outlier-detection-xds-defaults", // OD proto set but no proto fields set + clusterResource: func() *v3clusterpb.Cluster { + c := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: clusterName, + ServiceName: serviceName, + SecurityLevel: e2e.SecurityLevelNone, + Policy: e2e.LoadBalancingPolicyRingHash, + }) + c.OutlierDetection = &v3clusterpb.OutlierDetection{} + return c + }(), + wantChildCfg: &clusterresolver.LBConfig{ + DiscoveryMechanisms: []clusterresolver.DiscoveryMechanism{{ + Cluster: clusterName, + Type: clusterresolver.DiscoveryMechanismTypeEDS, + EDSServiceName: serviceName, + OutlierDetection: json.RawMessage(`{"successRateEjection":{}}`), + }}, + XDSLBPolicy: json.RawMessage(`[{"ring_hash_experimental": {"minRingSize":1024, "maxRingSize":8388608}}]`), }, - wantErr: balancer.ErrBadResolverState, }, { - name: "happy-good-case", - ccs: cdsCCS(clusterName, xdsC), - wantCluster: clusterName, + name: "happy-case-outlier-detection-all-fields-set", + clusterResource: func() *v3clusterpb.Cluster { + c := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: clusterName, + ServiceName: serviceName, + SecurityLevel: e2e.SecurityLevelNone, + Policy: e2e.LoadBalancingPolicyRingHash, + }) + c.OutlierDetection = &v3clusterpb.OutlierDetection{ + Interval: durationpb.New(10 * time.Second), + BaseEjectionTime: durationpb.New(30 * time.Second), + MaxEjectionTime: durationpb.New(300 * time.Second), + MaxEjectionPercent: wrapperspb.UInt32(10), + SuccessRateStdevFactor: wrapperspb.UInt32(1900), + EnforcingSuccessRate: wrapperspb.UInt32(100), + SuccessRateMinimumHosts: wrapperspb.UInt32(5), + SuccessRateRequestVolume: wrapperspb.UInt32(100), + FailurePercentageThreshold: wrapperspb.UInt32(85), + EnforcingFailurePercentage: wrapperspb.UInt32(5), + FailurePercentageMinimumHosts: wrapperspb.UInt32(5), + FailurePercentageRequestVolume: wrapperspb.UInt32(50), + } + return c + }(), + wantChildCfg: &clusterresolver.LBConfig{ + DiscoveryMechanisms: []clusterresolver.DiscoveryMechanism{{ + Cluster: clusterName, + Type: clusterresolver.DiscoveryMechanismTypeEDS, + EDSServiceName: serviceName, + OutlierDetection: json.RawMessage(`{ + "interval": "10s", + "baseEjectionTime": "30s", + "maxEjectionTime": "300s", + "maxEjectionPercent": 10, + "successRateEjection": { + "stdevFactor": 1900, + "enforcementPercentage": 100, + "minimumHosts": 5, + "requestVolume": 100 + }, + "failurePercentageEjection": { + "threshold": 85, + "enforcementPercentage": 5, + "minimumHosts": 5, + "requestVolume": 50 + } + }`), + }}, + XDSLBPolicy: json.RawMessage(`[{"ring_hash_experimental": {"minRingSize":1024, "maxRingSize":8388608}}]`), + }, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - cdsB, _, _, cancel := setup() - defer func() { - cancel() - cdsB.Close() - }() - - if err := cdsB.UpdateClientConnState(test.ccs); err != test.wantErr { - t.Fatalf("cdsBalancer.UpdateClientConnState failed with error: %v", err) - } - if test.wantErr != nil { - // When we wanted an error and got it, we should return early. - return - } - gotCluster, err := xdsC.WaitForWatchCluster() - if err != nil { - t.Fatalf("xdsClient.WatchCDS failed with error: %v", err) + lbCfgCh, _, _, _ := registerWrappedClusterResolverPolicy(t) + mgmtServer, nodeID, _, _, _, _, _ := setupWithManagementServer(t) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{test.clusterResource}, + SkipValidation: true, + }); err != nil { + t.Fatal(err) } - if gotCluster != test.wantCluster { - t.Fatalf("xdsClient.WatchCDS called for cluster: %v, want: %v", gotCluster, test.wantCluster) + + if err := compareLoadBalancingConfig(ctx, lbCfgCh, test.wantChildCfg); err != nil { + t.Fatal(err) } }) } } -// TestUpdateClientConnStateAfterClose invokes the UpdateClientConnState method -// on the cdsBalancer after close and verifies that it returns an error. -func (s) TestUpdateClientConnStateAfterClose(t *testing.T) { - cdsB, _, _, cancel := setup() +// Tests a single success scenario where the cds LB policy receives a cluster +// resource from the management server with LRS enabled. Verifies that the load +// balancing configuration pushed to the child is as expected. +func (s) TestClusterUpdate_SuccessWithLRS(t *testing.T) { + lbCfgCh, _, _, _ := registerWrappedClusterResolverPolicy(t) + mgmtServer, nodeID, _, _, _, _, _ := setupWithManagementServer(t) + + clusterResource := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: clusterName, + ServiceName: serviceName, + EnableLRS: true, + }) + wantChildCfg := &clusterresolver.LBConfig{ + DiscoveryMechanisms: []clusterresolver.DiscoveryMechanism{{ + Cluster: clusterName, + Type: clusterresolver.DiscoveryMechanismTypeEDS, + EDSServiceName: serviceName, + LoadReportingServer: &bootstrap.ServerConfig{ + ServerURI: mgmtServer.Address, + Creds: bootstrap.ChannelCreds{Type: "insecure"}, + }, + OutlierDetection: json.RawMessage(`{}`), + }}, + XDSLBPolicy: json.RawMessage(`[{"xds_wrr_locality_experimental": {"childPolicy": [{"round_robin": {}}]}}]`), + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() - cdsB.Close() + if err := mgmtServer.Update(ctx, e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{clusterResource}, + SkipValidation: true, + }); err != nil { + t.Fatal(err) + } - if err := cdsB.UpdateClientConnState(cdsCCS(clusterName, fakeclient.NewClient())); err != errBalancerClosed { - t.Fatalf("UpdateClientConnState() after close returned %v, want %v", err, errBalancerClosed) + if err := compareLoadBalancingConfig(ctx, lbCfgCh, wantChildCfg); err != nil { + t.Fatal(err) } } -// TestUpdateClientConnStateWithSameState verifies that a ClientConnState -// update with the same cluster and xdsClient does not cause the cdsBalancer to -// create a new watch. -func (s) TestUpdateClientConnStateWithSameState(t *testing.T) { - xdsC, cdsB, _, _, cancel := setupWithWatch(t) - defer func() { - cancel() - cdsB.Close() - }() +// Tests scenarios for a bad cluster update received from the management server. +// +// - when a bad cluster resource update is received without any previous good +// update from the management server, the cds LB policy is expected to put +// the channel in TRANSIENT_FAILURE. +// - when a bad cluster resource update is received after a previous good +// update from the management server, the cds LB policy is expected to +// continue using the previous good update. +// - when the cluster resource is removed after a previous good +// update from the management server, the cds LB policy is expected to put +// the channel in TRANSIENT_FAILURE. +func (s) TestClusterUpdate_Failure(t *testing.T) { + _, resolverErrCh, _, _ := registerWrappedClusterResolverPolicy(t) + mgmtServer, nodeID, cc, _, _, cdsResourceRequestedCh, cdsResourceCanceledCh := setupWithManagementServer(t) - if err := cdsB.UpdateClientConnState(cdsCCS(clusterName, xdsC)); err != nil { - t.Fatalf("cdsBalancer.UpdateClientConnState failed with error: %v", err) - } - if _, err := xdsC.WaitForWatchCluster(); err != testutils.ErrRecvTimeout { - t.Fatalf("waiting for WatchCluster() should have timed out, but returned error: %v", err) + // Verify that the specified cluster resource is requested. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + wantNames := []string{clusterName} + if err := waitForResourceNames(ctx, cdsResourceRequestedCh, wantNames); err != nil { + t.Fatal(err) } -} -// TestHandleClusterUpdate invokes the registered CDS watch callback with -// different updates and verifies that the expect ClientConnState is propagated -// to the edsBalancer. -func (s) TestHandleClusterUpdate(t *testing.T) { - xdsC, cdsB, edsB, _, cancel := setupWithWatch(t) - defer func() { - cancel() - cdsB.Close() - }() + // Configure the management server to return a cluster resource that + // contains a config_source_specifier for the `lrs_server` field which is not + // set to `self`, and hence is expected to be NACKed by the client. + cluster := e2e.DefaultCluster(clusterName, serviceName, e2e.SecurityLevelNone) + cluster.LrsServer = &v3corepb.ConfigSource{ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{}} + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{cluster}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } - tests := []struct { - name string - cdsUpdate xdsclient.ClusterUpdate - updateErr error - wantCCS balancer.ClientConnState - }{ - { - name: "happy-case-with-lrs", - cdsUpdate: xdsclient.ClusterUpdate{ServiceName: serviceName, EnableLRS: true}, - wantCCS: edsCCS(serviceName, true, xdsC), - }, - { - name: "happy-case-without-lrs", - cdsUpdate: xdsclient.ClusterUpdate{ServiceName: serviceName}, - wantCCS: edsCCS(serviceName, false, xdsC), - }, + // Verify that the watch for the cluster resource is not cancelled. + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + select { + case <-sCtx.Done(): + case <-cdsResourceCanceledCh: + t.Fatal("Watch for cluster resource is cancelled when not expected to") } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - if err := invokeWatchCbAndWait(xdsC, cdsWatchInfo{test.cdsUpdate, test.updateErr}, test.wantCCS, edsB); err != nil { - t.Fatal(err) - } - }) + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + + // Ensure that the NACK error is propagated to the RPC caller. + const wantClusterNACKErr = "unsupported config_source_specifier" + client := testgrpc.NewTestServiceClient(cc) + _, err := client.EmptyCall(ctx, &testpb.Empty{}) + if code := status.Code(err); code != codes.Unavailable { + t.Fatalf("EmptyCall() failed with code: %v, want %v", code, codes.Unavailable) + } + if err != nil && !strings.Contains(err.Error(), wantClusterNACKErr) { + t.Fatalf("EmptyCall() failed with err: %v, want err containing: %v", err, wantClusterNACKErr) } -} -// TestHandleClusterUpdateError covers the cases that an error is returned from -// the watcher. -// -// Includes error with and without a child eds balancer, and whether error is a -// resource-not-found error. -func (s) TestHandleClusterUpdateError(t *testing.T) { - xdsC, cdsB, edsB, tcc, cancel := setupWithWatch(t) - defer func() { - cancel() - cdsB.Close() - }() - - // An error before eds balancer is built. Should result in an error picker. - // And this is not a resource not found error, watch shouldn't be canceled. - err1 := errors.New("cdsBalancer resolver error 1") - xdsC.InvokeWatchClusterCallback(xdsclient.ClusterUpdate{}, err1) - if err := xdsC.WaitForCancelClusterWatch(); err == nil { - t.Fatal("watch was canceled, want not canceled (timeout error)") - } - if err := edsB.waitForResolverError(err1); err == nil { - t.Fatal("eds balancer shouldn't get error (shouldn't be built yet)") - } - state, err := tcc.newPickerCh.Receive() - if err != nil { - t.Fatalf("failed to get picker, expect an error picker") + // Start a test service backend. + server := stubserver.StartTestService(t, nil) + t.Cleanup(server.Stop) + + // Configure cluster and endpoints resources in the management server. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, serviceName, e2e.SecurityLevelNone)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, "localhost", []uint32{testutils.ParsePort(t, server.Address)})}, + SkipValidation: true, } - picker := state.(balancer.State).Picker - if _, perr := picker.Pick(balancer.PickInfo{}); perr == nil { - t.Fatalf("want picker to always fail, got nil") + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) } - cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName} - wantCCS := edsCCS(serviceName, false, xdsC) - if err := invokeWatchCbAndWait(xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil { + // Verify that a successful RPC can be made. + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + + // Send the bad cluster resource again. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{cluster}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, "localhost", []uint32{testutils.ParsePort(t, server.Address)})}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { t.Fatal(err) } - // An error after eds balancer is build, eds should receive the error. This - // is not a resource not found error, watch shouldn't be canceled - err2 := errors.New("cdsBalancer resolver error 2") - xdsC.InvokeWatchClusterCallback(xdsclient.ClusterUpdate{}, err2) - if err := xdsC.WaitForCancelClusterWatch(); err == nil { - t.Fatal("watch was canceled, want not canceled (timeout error)") + // Verify that the watch for the cluster resource is not cancelled. + sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + select { + case <-sCtx.Done(): + case <-cdsResourceCanceledCh: + t.Fatal("Watch for cluster resource is cancelled when not expected to") } - if err := edsB.waitForResolverError(err2); err != nil { - t.Fatalf("eds balancer should get error, waitForError failed: %v", err) + + // Verify that a successful RPC can be made, using the previously received + // good configuration. + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) } - // A resource not found error. Watch should not be canceled because this - // means CDS resource is removed, and eds should receive the error. - resourceErr := xdsclient.NewErrorf(xdsclient.ErrorTypeResourceNotFound, "cdsBalancer resource not found error") - xdsC.InvokeWatchClusterCallback(xdsclient.ClusterUpdate{}, resourceErr) - if err := xdsC.WaitForCancelClusterWatch(); err == nil { - t.Fatalf("want watch to be not canceled, watchForCancel should timeout") + // Verify that the resolver error is pushed to the child policy. + select { + case err := <-resolverErrCh: + if !strings.Contains(err.Error(), wantClusterNACKErr) { + t.Fatalf("Error pushed to child policy is %v, want %v", err, wantClusterNACKErr) + } + case <-ctx.Done(): + t.Fatal("Timeout when waiting for resolver error to be pushed to the child policy") + } + + // Remove the cluster resource from the management server, triggering a + // resource-not-found error. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + SkipValidation: true, } - if err := edsB.waitForResolverError(resourceErr); err != nil { - t.Fatalf("eds balancer should get resource-not-found error, waitForError failed: %v", err) + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Verify that the watch for the cluster resource is not cancelled. + sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + select { + case <-sCtx.Done(): + case <-cdsResourceCanceledCh: + t.Fatal("Watch for cluster resource is cancelled when not expected to") + } + + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + + // Ensure RPC fails with Unavailable. The actual error message depends on + // the picker returned from the priority LB policy, and therefore not + // checking for it here. + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.Unavailable { + t.Fatalf("EmptyCall() failed with code: %v, want %v", status.Code(err), codes.Unavailable) } } -// TestResolverError verifies that resolvers errors (with type -// resource-not-found or others) are handled correctly. +// Tests the following scenarios for resolver errors: +// - when a resolver error is received without any previous good update from the +// management server, the cds LB policy is expected to put the channel in +// TRANSIENT_FAILURE. +// - when a resolver error is received (one that is not a resource-not-found +// error), with a previous good update from the management server, the cds LB +// policy is expected to push the error down the child policy, but is expected +// to continue to use the previously received good configuration. +// - when a resolver error is received (one that is a resource-not-found +// error, which is usually the case when the LDS resource is removed), +// with a previous good update from the management server, the cds LB policy +// is expected to push the error down the child policy and put the channel in +// TRANSIENT_FAILURE. It is also expected to cancel the CDS watch. func (s) TestResolverError(t *testing.T) { - xdsC, cdsB, edsB, tcc, cancel := setupWithWatch(t) - defer func() { - cancel() - cdsB.Close() - }() - - // An error before eds balancer is built. Should result in an error picker. - // Not a resource not found error, watch shouldn't be canceled. - err1 := errors.New("cdsBalancer resolver error 1") - cdsB.ResolverError(err1) - if err := xdsC.WaitForCancelClusterWatch(); err == nil { - t.Fatal("watch was canceled, want not canceled (timeout error)") - } - if err := edsB.waitForResolverError(err1); err == nil { - t.Fatal("eds balancer shouldn't get error (shouldn't be built yet)") - } - state, err := tcc.newPickerCh.Receive() - if err != nil { - t.Fatalf("failed to get picker, expect an error picker") + _, resolverErrCh, _, _ := registerWrappedClusterResolverPolicy(t) + mgmtServer, nodeID, cc, r, _, cdsResourceRequestedCh, cdsResourceCanceledCh := setupWithManagementServer(t) + + // Verify that the specified cluster resource is requested. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + wantNames := []string{clusterName} + if err := waitForResourceNames(ctx, cdsResourceRequestedCh, wantNames); err != nil { + t.Fatal(err) } - picker := state.(balancer.State).Picker - if _, perr := picker.Pick(balancer.PickInfo{}); perr == nil { - t.Fatalf("want picker to always fail, got nil") + + // Push a resolver error that is not a resource-not-found error. + resolverErr := errors.New("resolver-error-not-a-resource-not-found-error") + r.ReportError(resolverErr) + + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + + // Drain the resolver error channel. + select { + case <-resolverErrCh: + default: } - cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName} - wantCCS := edsCCS(serviceName, false, xdsC) - if err := invokeWatchCbAndWait(xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil { + // Ensure that the resolver error is propagated to the RPC caller. + client := testgrpc.NewTestServiceClient(cc) + _, err := client.EmptyCall(ctx, &testpb.Empty{}) + if code := status.Code(err); code != codes.Unavailable { + t.Fatalf("EmptyCall() failed with code: %v, want %v", code, codes.Unavailable) + } + if err != nil && !strings.Contains(err.Error(), resolverErr.Error()) { + t.Fatalf("EmptyCall() failed with err: %v, want %v", err, resolverErr) + } + + // Also verify that the watch for the cluster resource is not cancelled. + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + select { + case <-sCtx.Done(): + case <-cdsResourceCanceledCh: + t.Fatal("Watch for cluster resource is cancelled when not expected to") + } + + // Start a test service backend. + server := stubserver.StartTestService(t, nil) + t.Cleanup(server.Stop) + + // Configure good cluster and endpoints resources in the management server. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, serviceName, e2e.SecurityLevelNone)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, "localhost", []uint32{testutils.ParsePort(t, server.Address)})}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { t.Fatal(err) } - // Not a resource not found error, watch shouldn't be canceled, and eds - // should receive the error. - err2 := errors.New("cdsBalancer resolver error 2") - cdsB.ResolverError(err2) - if err := xdsC.WaitForCancelClusterWatch(); err == nil { - t.Fatal("watch was canceled, want not canceled (timeout error)") + // Verify that a successful RPC can be made. + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) } - if err := edsB.waitForResolverError(err2); err != nil { - t.Fatalf("eds balancer should get error, waitForError failed: %v", err) + + // Again push a resolver error that is not a resource-not-found error. + r.ReportError(resolverErr) + + // And again verify that the watch for the cluster resource is not + // cancelled. + sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + select { + case <-sCtx.Done(): + case <-cdsResourceCanceledCh: + t.Fatal("Watch for cluster resource is cancelled when not expected to") } - // A resource not found error. Watch should be canceled, and eds should - // receive the error. - resourceErr := xdsclient.NewErrorf(xdsclient.ErrorTypeResourceNotFound, "cdsBalancer resource not found error") - cdsB.ResolverError(resourceErr) - if err := xdsC.WaitForCancelClusterWatch(); err != nil { - t.Fatalf("want watch to be canceled, watchForCancel failed: %v", err) + // Verify that a successful RPC can be made, using the previously received + // good configuration. + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) } - if err := edsB.waitForResolverError(resourceErr); err != nil { - t.Fatalf("eds balancer should get resource-not-found error, waitForError failed: %v", err) + + // Verify that the resolver error is pushed to the child policy. + select { + case err := <-resolverErrCh: + if err != resolverErr { + t.Fatalf("Error pushed to child policy is %v, want %v", err, resolverErr) + } + case <-ctx.Done(): + t.Fatal("Timeout when waiting for resolver error to be pushed to the child policy") } -} -// TestUpdateSubConnState pushes a SubConn update to the cdsBalancer and -// verifies that the update is propagated to the edsBalancer. -func (s) TestUpdateSubConnState(t *testing.T) { - xdsC, cdsB, edsB, _, cancel := setupWithWatch(t) - defer func() { - cancel() - cdsB.Close() - }() - - cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName} - wantCCS := edsCCS(serviceName, false, xdsC) - if err := invokeWatchCbAndWait(xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil { - t.Fatal(err) + // Push a resource-not-found-error this time around. + resolverErr = xdsresource.NewErrorf(xdsresource.ErrorTypeResourceNotFound, "xds resource not found error") + r.ReportError(resolverErr) + + // Wait for the CDS resource to be not requested anymore. + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for CDS resource to be not requested") + case <-cdsResourceCanceledCh: } - var sc balancer.SubConn - state := balancer.SubConnState{ConnectivityState: connectivity.Ready} - cdsB.UpdateSubConnState(sc, state) - if err := edsB.waitForSubConnUpdate(subConnWithState{sc: sc, state: state}); err != nil { - t.Fatal(err) + // Verify that the resolver error is pushed to the child policy. + select { + case err := <-resolverErrCh: + if err != resolverErr { + t.Fatalf("Error pushed to child policy is %v, want %v", err, resolverErr) + } + case <-ctx.Done(): + t.Fatal("Timeout when waiting for resolver error to be pushed to the child policy") + } + + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + + // Ensure RPC fails with Unavailable. The actual error message depends on + // the picker returned from the priority LB policy, and therefore not + // checking for it here. + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.Unavailable { + t.Fatalf("EmptyCall() failed with code: %v, want %v", status.Code(err), codes.Unavailable) } } -// TestClose calls Close() on the cdsBalancer, and verifies that the underlying -// edsBalancer is also closed. +// Tests that closing the cds LB policy results in the cluster resource watch +// being cancelled and the child policy being closed. func (s) TestClose(t *testing.T) { - xdsC, cdsB, edsB, _, cancel := setupWithWatch(t) - defer cancel() + cdsBalancerCh := registerWrappedCDSPolicy(t) + _, _, _, childPolicyCloseCh := registerWrappedClusterResolverPolicy(t) + mgmtServer, nodeID, cc, _, _, _, cdsResourceCanceledCh := setupWithManagementServer(t) - cdsUpdate := xdsclient.ClusterUpdate{ServiceName: serviceName} - wantCCS := edsCCS(serviceName, false, xdsC) - if err := invokeWatchCbAndWait(xdsC, cdsWatchInfo{cdsUpdate, nil}, wantCCS, edsB); err != nil { + // Start a test service backend. + server := stubserver.StartTestService(t, nil) + t.Cleanup(server.Stop) + + // Configure cluster and endpoints resources in the management server. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, serviceName, e2e.SecurityLevelNone)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, "localhost", []uint32{testutils.ParsePort(t, server.Address)})}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { t.Fatal(err) } - cdsB.Close() - if err := xdsC.WaitForCancelClusterWatch(); err != nil { - t.Fatal(err) + // Verify that a successful RPC can be made. + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + + // Retrieve the cds LB policy and close it. + var cdsBal balancer.Balancer + select { + case cdsBal = <-cdsBalancerCh: + case <-ctx.Done(): + t.Fatal("Timeout when waiting for cds LB policy to be created") + } + cdsBal.Close() + + // Wait for the CDS resource to be not requested anymore. + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for CDS resource to be not requested") + case <-cdsResourceCanceledCh: + } + // Wait for the child policy to be closed. + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for the child policy to be closed") + case <-childPolicyCloseCh: } - if err := edsB.waitForClose(); err != nil { +} + +// Tests that calling ExitIdle on the cds LB policy results in the call being +// propagated to the child policy. +func (s) TestExitIdle(t *testing.T) { + cdsBalancerCh := registerWrappedCDSPolicy(t) + _, _, exitIdleCh, _ := registerWrappedClusterResolverPolicy(t) + mgmtServer, nodeID, cc, _, _, _, _ := setupWithManagementServer(t) + + // Start a test service backend. + server := stubserver.StartTestService(t, nil) + t.Cleanup(server.Stop) + + // Configure cluster and endpoints resources in the management server. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, serviceName, e2e.SecurityLevelNone)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(serviceName, "localhost", []uint32{testutils.ParsePort(t, server.Address)})}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { t.Fatal(err) } + + // Verify that a successful RPC can be made. + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + + // Retrieve the cds LB policy and call ExitIdle() on it. + var cdsBal balancer.Balancer + select { + case cdsBal = <-cdsBalancerCh: + case <-ctx.Done(): + t.Fatal("Timeout when waiting for cds LB policy to be created") + } + cdsBal.(balancer.ExitIdler).ExitIdle() + + // Wait for ExitIdle to be called on the child policy. + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for the child policy to be closed") + case <-exitIdleCh: + } } -// TestParseConfig exercises the config parsing functionality in the cds -// balancer builder. +// TestParseConfig verifies the ParseConfig() method in the CDS balancer. func (s) TestParseConfig(t *testing.T) { - bb := cdsBB{} - if gotName := bb.Name(); gotName != cdsName { - t.Fatalf("cdsBB.Name() = %v, want %v", gotName, cdsName) + bb := balancer.Get(cdsName) + if bb == nil { + t.Fatalf("balancer.Get(%q) returned nil", cdsName) + } + parser, ok := bb.(balancer.ConfigParser) + if !ok { + t.Fatalf("balancer %q does not implement the ConfigParser interface", cdsName) } tests := []struct { @@ -582,33 +1017,43 @@ func (s) TestParseConfig(t *testing.T) { wantErr bool }{ { - name: "good-lb-config", + name: "good-config", input: json.RawMessage(`{"Cluster": "cluster1"}`), - wantCfg: &lbConfig{ClusterName: clusterName}, + wantCfg: &lbConfig{ClusterName: "cluster1"}, }, { - name: "unknown-fields-in-lb-config", + name: "unknown-fields-in-config", input: json.RawMessage(`{"Unknown": "foobar"}`), wantCfg: &lbConfig{ClusterName: ""}, }, { - name: "empty-lb-config", + name: "empty-config", input: json.RawMessage(""), wantErr: true, }, + { + name: "bad-config", + input: json.RawMessage(`{"Cluster": 5}`), + wantErr: true, + }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - gotCfg, gotErr := bb.ParseConfig(test.input) + gotCfg, gotErr := parser.ParseConfig(test.input) if (gotErr != nil) != test.wantErr { - t.Fatalf("bb.ParseConfig(%v) = %v, wantErr %v", string(test.input), gotErr, test.wantErr) + t.Fatalf("ParseConfig(%v) = %v, wantErr %v", string(test.input), gotErr, test.wantErr) } - if !test.wantErr { - if !cmp.Equal(gotCfg, test.wantCfg) { - t.Fatalf("bb.ParseConfig(%v) = %v, want %v", string(test.input), gotCfg, test.wantCfg) - } + if test.wantErr { + return + } + if !cmp.Equal(gotCfg, test.wantCfg) { + t.Fatalf("ParseConfig(%v) = %v, want %v", string(test.input), gotCfg, test.wantCfg) } }) } } + +func newUint32(i uint32) *uint32 { + return &i +} diff --git a/xds/internal/balancer/cdsbalancer/cluster_handler.go b/xds/internal/balancer/cdsbalancer/cluster_handler.go new file mode 100644 index 000000000000..aa2d9674a790 --- /dev/null +++ b/xds/internal/balancer/cdsbalancer/cluster_handler.go @@ -0,0 +1,368 @@ +/* + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cdsbalancer + +import ( + "encoding/json" + "errors" + "sync" + + "google.golang.org/grpc/xds/internal/xdsclient" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" +) + +const maxDepth = 16 + +var ( + errNotReceivedUpdate = errors.New("tried to construct a cluster update on a cluster that has not received an update") + errExceedsMaxDepth = errors.New("aggregate cluster graph exceeds max depth") +) + +// clusterHandlerUpdate wraps the information received from the registered CDS +// watcher. A non-nil error is propagated to the underlying cluster_resolver +// balancer. A valid update results in creating a new cluster_resolver balancer +// (if one doesn't already exist) and pushing the update to it. +type clusterHandlerUpdate struct { + // securityCfg is the Security Config from the top (root) cluster. + securityCfg *xdsresource.SecurityConfig + + // lbPolicy is the the child of the cluster_impl policy, for all priorities. + lbPolicy json.RawMessage + + // updates is a list of ClusterUpdates from all the leaf clusters. + updates []xdsresource.ClusterUpdate + err error +} + +// clusterHandler will be given a name representing a cluster. It will then +// update the CDS policy constantly with a list of Clusters to pass down to +// XdsClusterResolverLoadBalancingPolicyConfig in a stream like fashion. +type clusterHandler struct { + parent *cdsBalancer + + // A mutex to protect entire tree of clusters. + clusterMutex sync.Mutex + rootClusterName string + + createdClusters map[string]*clusterNode + + // A way to ping CDS Balancer about any updates or errors to a Node in the + // tree. This will either get called from this handler constructing an + // update or from a child with an error. Capacity of one as the only update + // CDS Balancer cares about is the most recent update. + updateChannel chan clusterHandlerUpdate +} + +func newClusterHandler(parent *cdsBalancer) *clusterHandler { + return &clusterHandler{ + parent: parent, + updateChannel: make(chan clusterHandlerUpdate, 1), + createdClusters: make(map[string]*clusterNode), + } +} + +func (ch *clusterHandler) updateRootCluster(rootClusterName string) { + ch.clusterMutex.Lock() + defer ch.clusterMutex.Unlock() + if ch.createdClusters[ch.rootClusterName] == nil { + // Construct a root node on first update. + createClusterNode(rootClusterName, ch.parent.xdsClient, ch, 0) + ch.rootClusterName = rootClusterName + return + } + // Check if root cluster was changed. If it was, delete old one and start + // new one, if not do nothing. + if rootClusterName != ch.rootClusterName { + ch.createdClusters[ch.rootClusterName].delete() + createClusterNode(rootClusterName, ch.parent.xdsClient, ch, 0) + ch.rootClusterName = rootClusterName + } +} + +// This function tries to construct a cluster update to send to CDS. +func (ch *clusterHandler) constructClusterUpdate() { + if ch.createdClusters[ch.rootClusterName] == nil { + // If root is nil, this handler is closed, ignore the update. + return + } + clusterUpdate, err := ch.createdClusters[ch.rootClusterName].constructClusterUpdate(make(map[string]bool)) + if err != nil { + // If there was an error received no op, as this can mean one of the + // children hasn't received an update yet, or the graph continued to + // stay in an error state. If the graph continues to stay in an error + // state, no new error needs to be written to the update buffer as that + // would be redundant information. + return + } + if clusterUpdate == nil { + // This means that there was an aggregated cluster with no EDS or DNS as + // leaf nodes. No update to be written. + return + } + // For a ClusterUpdate, the only update CDS cares about is the most + // recent one, so opportunistically drain the update channel before + // sending the new update. + select { + case <-ch.updateChannel: + default: + } + + ch.updateChannel <- clusterHandlerUpdate{ + securityCfg: ch.createdClusters[ch.rootClusterName].clusterUpdate.SecurityCfg, + lbPolicy: ch.createdClusters[ch.rootClusterName].clusterUpdate.LBPolicy, + updates: clusterUpdate, + } +} + +// close() is meant to be called by CDS when the CDS balancer is closed, and it +// cancels the watches for every cluster in the cluster tree. +func (ch *clusterHandler) close() { + ch.clusterMutex.Lock() + defer ch.clusterMutex.Unlock() + if ch.createdClusters[ch.rootClusterName] == nil { + return + } + ch.createdClusters[ch.rootClusterName].delete() + ch.rootClusterName = "" +} + +// This logically represents a cluster. This handles all the logic for starting +// and stopping a cluster watch, handling any updates, and constructing a list +// recursively for the ClusterHandler. +type clusterNode struct { + // A way to cancel the watch for the cluster. + cancelFunc func() + + // A list of children, as the Node can be an aggregate Cluster. + children []string + + // A ClusterUpdate in order to build a list of cluster updates for CDS to + // send down to child XdsClusterResolverLoadBalancingPolicy. + clusterUpdate xdsresource.ClusterUpdate + + // This boolean determines whether this Node has received an update or not. + // This isn't the best practice, but this will protect a list of Cluster + // Updates from being constructed if a cluster in the tree has not received + // an update yet. + receivedUpdate bool + + clusterHandler *clusterHandler + + depth int32 + refCount int32 + + // maxDepthErr is set if this cluster node is an aggregate cluster and has a + // child that causes the graph to exceed the maximum depth allowed. This is + // used to show a cluster graph as being in an error state when it constructs + // a cluster update. + maxDepthErr error +} + +// CreateClusterNode creates a cluster node from a given clusterName. This will +// also start the watch for that cluster. +func createClusterNode(clusterName string, xdsClient xdsclient.XDSClient, topLevelHandler *clusterHandler, depth int32) { + // If the cluster has already been created, simply return, which ignores + // duplicates. + if topLevelHandler.createdClusters[clusterName] != nil { + topLevelHandler.createdClusters[clusterName].refCount++ + return + } + c := &clusterNode{ + clusterHandler: topLevelHandler, + depth: depth, + refCount: 1, + } + // Communicate with the xds client here. + topLevelHandler.parent.logger.Infof("CDS watch started on %v", clusterName) + cancel := xdsClient.WatchCluster(clusterName, c.handleResp) + c.cancelFunc = func() { + topLevelHandler.parent.logger.Infof("CDS watch canceled on %v", clusterName) + cancel() + } + topLevelHandler.createdClusters[clusterName] = c +} + +// This function cancels the cluster watch on the cluster and all of it's +// children. +func (c *clusterNode) delete() { + c.refCount-- + if c.refCount == 0 { + c.cancelFunc() + delete(c.clusterHandler.createdClusters, c.clusterUpdate.ClusterName) + for _, child := range c.children { + if c.clusterHandler.createdClusters[child] != nil { + c.clusterHandler.createdClusters[child].delete() + } + } + } +} + +// Construct cluster update (potentially a list of ClusterUpdates) for a node. +func (c *clusterNode) constructClusterUpdate(clustersSeen map[string]bool) ([]xdsresource.ClusterUpdate, error) { + // If the cluster has not yet received an update, the cluster update is not + // yet ready. + if !c.receivedUpdate { + return nil, errNotReceivedUpdate + } + if c.maxDepthErr != nil { + return nil, c.maxDepthErr + } + // Ignore duplicates. It's ok to ignore duplicates because the second + // occurrence of a cluster will never be used. I.e. in [C, D, C], the second + // C will never be used (the only way to fall back to lower priority D is if + // C is down, which means second C will never be chosen). Thus, [C, D, C] is + // logically equivalent to [C, D]. + if clustersSeen[c.clusterUpdate.ClusterName] { + return []xdsresource.ClusterUpdate{}, nil + } + clustersSeen[c.clusterUpdate.ClusterName] = true + + // Base case - LogicalDNS or EDS. Both of these cluster types will be tied + // to a single ClusterUpdate. + if c.clusterUpdate.ClusterType != xdsresource.ClusterTypeAggregate { + return []xdsresource.ClusterUpdate{c.clusterUpdate}, nil + } + + // If an aggregate construct a list by recursively calling down to all of + // it's children. + var childrenUpdates []xdsresource.ClusterUpdate + for _, child := range c.children { + childUpdateList, err := c.clusterHandler.createdClusters[child].constructClusterUpdate(clustersSeen) + if err != nil { + return nil, err + } + childrenUpdates = append(childrenUpdates, childUpdateList...) + } + return childrenUpdates, nil +} + +// handleResp handles a xds response for a particular cluster. This function +// also handles any logic with regards to any child state that may have changed. +// At the end of the handleResp(), the clusterUpdate will be pinged in certain +// situations to try and construct an update to send back to CDS. +func (c *clusterNode) handleResp(clusterUpdate xdsresource.ClusterUpdate, err error) { + c.clusterHandler.clusterMutex.Lock() + defer c.clusterHandler.clusterMutex.Unlock() + if err != nil { // Write this error for run() to pick up in CDS LB policy. + // For a ClusterUpdate, the only update CDS cares about is the most + // recent one, so opportunistically drain the update channel before + // sending the new update. + select { + case <-c.clusterHandler.updateChannel: + default: + } + c.clusterHandler.updateChannel <- clusterHandlerUpdate{err: err} + c.receivedUpdate = false + c.maxDepthErr = nil + return + } + + c.receivedUpdate = true + c.clusterUpdate = clusterUpdate + + // If the cluster was a leaf node, if the cluster update received had change + // in the cluster update then the overall cluster update would change and + // there is a possibility for the overall update to build so ping cluster + // handler to return. Also, if there was any children from previously, + // delete the children, as the cluster type is no longer an aggregate + // cluster. + if clusterUpdate.ClusterType != xdsresource.ClusterTypeAggregate { + for _, child := range c.children { + c.clusterHandler.createdClusters[child].delete() + } + c.children = nil + c.maxDepthErr = nil + // This is an update in the one leaf node, should try to send an update + // to the parent CDS balancer. + // + // Note that this update might be a duplicate from the previous one. + // Because the update contains not only the cluster name to watch, but + // also the extra fields (e.g. security config). There's no good way to + // compare all the fields. + c.clusterHandler.constructClusterUpdate() + return + } + + // Aggregate cluster handling. + if len(clusterUpdate.PrioritizedClusterNames) >= 1 { + if c.depth == maxDepth-1 { + // For a ClusterUpdate, the only update CDS cares about is the most + // recent one, so opportunistically drain the update channel before + // sending the new update. + select { + case <-c.clusterHandler.updateChannel: + default: + } + c.clusterHandler.updateChannel <- clusterHandlerUpdate{err: errExceedsMaxDepth} + c.children = []string{} + c.maxDepthErr = errExceedsMaxDepth + return + } + } + + newChildren := make(map[string]bool) + for _, childName := range clusterUpdate.PrioritizedClusterNames { + newChildren[childName] = true + } + + // These booleans help determine whether this callback will ping the overall + // clusterHandler to try and construct an update to send back to CDS. This + // will be determined by whether there would be a change in the overall + // clusterUpdate for the whole tree (ex. change in clusterUpdate for current + // cluster or a deleted child) and also if there's even a possibility for + // the update to build (ex. if a child is created and a watch is started, + // that child hasn't received an update yet due to the mutex lock on this + // callback). + var createdChild bool + + // This map will represent the current children of the cluster. It will be + // first added to in order to represent the new children. It will then have + // any children deleted that are no longer present. + mapCurrentChildren := make(map[string]bool) + for _, child := range c.children { + mapCurrentChildren[child] = true + } + + // Add and construct any new child nodes. + for child := range newChildren { + if _, inChildrenAlready := mapCurrentChildren[child]; !inChildrenAlready { + createClusterNode(child, c.clusterHandler.parent.xdsClient, c.clusterHandler, c.depth+1) + } + } + + // Delete any child nodes no longer in the aggregate cluster's children. + for child := range mapCurrentChildren { + if _, stillAChild := newChildren[child]; !stillAChild { + c.clusterHandler.createdClusters[child].delete() + delete(mapCurrentChildren, child) + } + } + + c.children = clusterUpdate.PrioritizedClusterNames + + c.maxDepthErr = nil + // If the cluster is an aggregate cluster, if this callback created any new + // child cluster nodes, then there's no possibility for a full cluster + // update to successfully build, as those created children will not have + // received an update yet. Even if this update did not delete a child, there + // is still a possibility for the cluster update to build, as the aggregate + // cluster can ignore duplicated children and thus the update can fill out + // the full cluster update tree. + if !createdChild { + c.clusterHandler.constructClusterUpdate() + } +} diff --git a/xds/internal/balancer/cdsbalancer/cluster_handler_test.go b/xds/internal/balancer/cdsbalancer/cluster_handler_test.go new file mode 100644 index 000000000000..ee989ec3ef73 --- /dev/null +++ b/xds/internal/balancer/cdsbalancer/cluster_handler_test.go @@ -0,0 +1,1095 @@ +/* + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cdsbalancer + +import ( + "context" + "errors" + "fmt" + "testing" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/xds/internal/testutils/fakeclient" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" +) + +const ( + edsService = "EDS Service" + logicalDNSService = "Logical DNS Service" + edsService2 = "EDS Service 2" + logicalDNSService2 = "Logical DNS Service 2" + aggregateClusterService = "Aggregate Cluster Service" +) + +// setupTests creates a clusterHandler with a fake xds client for control over +// xds client. +func setupTests() (*clusterHandler, *fakeclient.Client) { + xdsC := fakeclient.NewClient() + ch := newClusterHandler(&cdsBalancer{xdsClient: xdsC}) + return ch, xdsC +} + +// Simplest case: the cluster handler receives a cluster name, handler starts a +// watch for that cluster, xds client returns that it is a Leaf Node (EDS or +// LogicalDNS), not a tree, so expectation that update is written to buffer +// which will be read by CDS LB. +func (s) TestSuccessCaseLeafNode(t *testing.T) { + tests := []struct { + name string + clusterName string + clusterUpdate xdsresource.ClusterUpdate + }{ + { + name: "test-update-root-cluster-EDS-success", + clusterName: edsService, + clusterUpdate: xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeEDS, + ClusterName: edsService, + }, + }, + { + name: "test-update-root-cluster-Logical-DNS-success", + clusterName: logicalDNSService, + clusterUpdate: xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeLogicalDNS, + ClusterName: logicalDNSService, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ch, fakeClient := setupTests() + // When you first update the root cluster, it should hit the code + // path which will start a cluster node for that root. Updating the + // root cluster logically represents a ping from a ClientConn. + ch.updateRootCluster(test.clusterName) + // Starting a cluster node involves communicating with the + // xdsClient, telling it to watch a cluster. + ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer ctxCancel() + gotCluster, err := fakeClient.WaitForWatchCluster(ctx) + if err != nil { + t.Fatalf("xdsClient.WatchCDS failed with error: %v", err) + } + if gotCluster != test.clusterName { + t.Fatalf("xdsClient.WatchCDS called for cluster: %v, want: %v", gotCluster, test.clusterName) + } + // Invoke callback with xds client with a certain clusterUpdate. Due + // to this cluster update filling out the whole cluster tree, as the + // cluster is of a root type (EDS or Logical DNS) and not an + // aggregate cluster, this should trigger the ClusterHandler to + // write to the update buffer to update the CDS policy. + fakeClient.InvokeWatchClusterCallback(test.clusterUpdate, nil) + select { + case chu := <-ch.updateChannel: + if diff := cmp.Diff(chu.updates, []xdsresource.ClusterUpdate{test.clusterUpdate}); diff != "" { + t.Fatalf("got unexpected cluster update, diff (-got, +want): %v", diff) + } + case <-ctx.Done(): + t.Fatal("Timed out waiting for update from update channel.") + } + // Close the clusterHandler. This is meant to be called when the CDS + // Balancer is closed, and the call should cancel the watch for this + // cluster. + ch.close() + clusterNameDeleted, err := fakeClient.WaitForCancelClusterWatch(ctx) + if err != nil { + t.Fatalf("xdsClient.CancelCDS failed with error: %v", err) + } + if clusterNameDeleted != test.clusterName { + t.Fatalf("xdsClient.CancelCDS called for cluster %v, want: %v", clusterNameDeleted, logicalDNSService) + } + }) + } +} + +// The cluster handler receives a cluster name, handler starts a watch for that +// cluster, xds client returns that it is a Leaf Node (EDS or LogicalDNS), not a +// tree, so expectation that first update is written to buffer which will be +// read by CDS LB. Then, send a new cluster update that is different, with the +// expectation that it is also written to the update buffer to send back to CDS. +func (s) TestSuccessCaseLeafNodeThenNewUpdate(t *testing.T) { + tests := []struct { + name string + clusterName string + clusterUpdate xdsresource.ClusterUpdate + newClusterUpdate xdsresource.ClusterUpdate + }{ + {name: "test-update-root-cluster-then-new-update-EDS-success", + clusterName: edsService, + clusterUpdate: xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeEDS, + ClusterName: edsService, + }, + newClusterUpdate: xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeEDS, + ClusterName: edsService2, + }, + }, + { + name: "test-update-root-cluster-then-new-update-Logical-DNS-success", + clusterName: logicalDNSService, + clusterUpdate: xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeLogicalDNS, + ClusterName: logicalDNSService, + }, + newClusterUpdate: xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeLogicalDNS, + ClusterName: logicalDNSService2, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ch, fakeClient := setupTests() + ch.updateRootCluster(test.clusterName) + ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer ctxCancel() + _, err := fakeClient.WaitForWatchCluster(ctx) + if err != nil { + t.Fatalf("xdsClient.WatchCDS failed with error: %v", err) + } + fakeClient.InvokeWatchClusterCallback(test.clusterUpdate, nil) + select { + case <-ch.updateChannel: + case <-ctx.Done(): + t.Fatal("Timed out waiting for update from updateChannel.") + } + + // Check that sending the same cluster update also induces an update + // to be written to update buffer. + fakeClient.InvokeWatchClusterCallback(test.clusterUpdate, nil) + shouldNotHappenCtx, shouldNotHappenCtxCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer shouldNotHappenCtxCancel() + select { + case <-ch.updateChannel: + case <-shouldNotHappenCtx.Done(): + t.Fatal("Timed out waiting for update from updateChannel.") + } + + // Above represents same thing as the simple + // TestSuccessCaseLeafNode, extra behavior + validation (clusterNode + // which is a leaf receives a changed clusterUpdate, which should + // ping clusterHandler, which should then write to the update + // buffer). + fakeClient.InvokeWatchClusterCallback(test.newClusterUpdate, nil) + select { + case chu := <-ch.updateChannel: + if diff := cmp.Diff(chu.updates, []xdsresource.ClusterUpdate{test.newClusterUpdate}); diff != "" { + t.Fatalf("got unexpected cluster update, diff (-got, +want): %v", diff) + } + case <-ctx.Done(): + t.Fatal("Timed out waiting for update from updateChannel.") + } + }) + } +} + +// TestUpdateRootClusterAggregateSuccess tests the case where an aggregate +// cluster is a root pointing to two child clusters one of type EDS and the +// other of type LogicalDNS. This test will then send cluster updates for both +// the children, and at the end there should be a successful clusterUpdate +// written to the update buffer to send back to CDS. +func (s) TestUpdateRootClusterAggregateSuccess(t *testing.T) { + ch, fakeClient := setupTests() + ch.updateRootCluster(aggregateClusterService) + + ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer ctxCancel() + gotCluster, err := fakeClient.WaitForWatchCluster(ctx) + if err != nil { + t.Fatalf("xdsClient.WatchCDS failed with error: %v", err) + } + if gotCluster != aggregateClusterService { + t.Fatalf("xdsClient.WatchCDS called for cluster: %v, want: %v", gotCluster, aggregateClusterService) + } + + // The xdsClient telling the clusterNode that the cluster type is an + // aggregate cluster which will cause a lot of downstream behavior. For a + // cluster type that isn't an aggregate, the behavior is simple. The + // clusterNode will simply get a successful update, which will then ping the + // clusterHandler which will successfully build an update to send to the CDS + // policy. In the aggregate cluster case, the handleResp callback must also + // start watches for the aggregate cluster's children. The ping to the + // clusterHandler at the end of handleResp should be a no-op, as neither the + // EDS or LogicalDNS child clusters have received an update yet. + fakeClient.InvokeWatchClusterCallback(xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeAggregate, + ClusterName: aggregateClusterService, + PrioritizedClusterNames: []string{edsService, logicalDNSService}, + }, nil) + + // xds client should be called to start a watch for one of the child + // clusters of the aggregate. The order of the children in the update + // written to the buffer to send to CDS matters, however there is no + // guarantee on the order it will start the watches of the children. + gotCluster, err = fakeClient.WaitForWatchCluster(ctx) + if err != nil { + t.Fatalf("xdsClient.WatchCDS failed with error: %v", err) + } + if gotCluster != edsService { + if gotCluster != logicalDNSService { + t.Fatalf("xdsClient.WatchCDS called for cluster: %v, want: %v", gotCluster, edsService) + } + } + + // xds client should then be called to start a watch for the second child + // cluster. + gotCluster, err = fakeClient.WaitForWatchCluster(ctx) + if err != nil { + t.Fatalf("xdsClient.WatchCDS failed with error: %v", err) + } + if gotCluster != edsService { + if gotCluster != logicalDNSService { + t.Fatalf("xdsClient.WatchCDS called for cluster: %v, want: %v", gotCluster, logicalDNSService) + } + } + + // The handleResp() call on the root aggregate cluster should not ping the + // cluster handler to try and construct an update, as the handleResp() + // callback knows that when a child is created, it cannot possibly build a + // successful update yet. Thus, there should be nothing in the update + // channel. + + shouldNotHappenCtx, shouldNotHappenCtxCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer shouldNotHappenCtxCancel() + + select { + case <-ch.updateChannel: + t.Fatal("Cluster Handler wrote an update to updateChannel when it shouldn't have, as each node in the full cluster tree has not yet received an update") + case <-shouldNotHappenCtx.Done(): + } + + // Send callback for the EDS child cluster. + fakeClient.InvokeWatchClusterCallback(xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeEDS, + ClusterName: edsService, + }, nil) + + // EDS child cluster will ping the Cluster Handler, to try an update, which + // still won't successfully build as the LogicalDNS child of the root + // aggregate cluster has not yet received and handled an update. + select { + case <-ch.updateChannel: + t.Fatal("Cluster Handler wrote an update to updateChannel when it shouldn't have, as each node in the full cluster tree has not yet received an update") + case <-shouldNotHappenCtx.Done(): + } + + // Invoke callback for Logical DNS child cluster. + + fakeClient.InvokeWatchClusterCallback(xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeLogicalDNS, + ClusterName: logicalDNSService, + }, nil) + + // Will Ping Cluster Handler, which will finally successfully build an + // update as all nodes in the tree of clusters have received an update. + // Since this cluster is an aggregate cluster comprised of two children, the + // returned update should be length 2, as the xds cluster resolver LB policy + // only cares about the full list of LogicalDNS and EDS clusters + // representing the base nodes of the tree of clusters. This list should be + // ordered as per the cluster update. + select { + case chu := <-ch.updateChannel: + if diff := cmp.Diff(chu.updates, []xdsresource.ClusterUpdate{{ + ClusterType: xdsresource.ClusterTypeEDS, + ClusterName: edsService, + }, { + ClusterType: xdsresource.ClusterTypeLogicalDNS, + ClusterName: logicalDNSService, + }}); diff != "" { + t.Fatalf("got unexpected cluster update, diff (-got, +want): %v", diff) + } + case <-ctx.Done(): + t.Fatal("Timed out waiting for the cluster update to be written to the update buffer.") + } +} + +// TestUpdateRootClusterAggregateThenChangeChild tests the scenario where you +// have an aggregate cluster with an EDS child and a LogicalDNS child, then you +// change one of the children and send an update for the changed child. This +// should write a new update to the update buffer to send back to CDS. +func (s) TestUpdateRootClusterAggregateThenChangeChild(t *testing.T) { + // This initial code is the same as the test for the aggregate success case, + // except without validations. This will get this test to the point where it + // can change one of the children. + ch, fakeClient := setupTests() + ch.updateRootCluster(aggregateClusterService) + + ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer ctxCancel() + _, err := fakeClient.WaitForWatchCluster(ctx) + if err != nil { + t.Fatalf("xdsClient.WatchCDS failed with error: %v", err) + } + + fakeClient.InvokeWatchClusterCallback(xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeAggregate, + ClusterName: aggregateClusterService, + PrioritizedClusterNames: []string{edsService, logicalDNSService}, + }, nil) + fakeClient.WaitForWatchCluster(ctx) + fakeClient.WaitForWatchCluster(ctx) + fakeClient.InvokeWatchClusterCallback(xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeEDS, + ClusterName: edsService, + }, nil) + fakeClient.InvokeWatchClusterCallback(xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeLogicalDNS, + ClusterName: logicalDNSService, + }, nil) + + select { + case <-ch.updateChannel: + case <-ctx.Done(): + t.Fatal("Timed out waiting for the cluster update to be written to the update buffer.") + } + + fakeClient.InvokeWatchClusterCallback(xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeAggregate, + ClusterName: aggregateClusterService, + PrioritizedClusterNames: []string{edsService, logicalDNSService2}, + }, nil) + + // The cluster update let's the aggregate cluster know that it's children + // are now edsService and logicalDNSService2, which implies that the + // aggregateCluster lost it's old logicalDNSService child. Thus, the + // logicalDNSService child should be deleted. + clusterNameDeleted, err := fakeClient.WaitForCancelClusterWatch(ctx) + if err != nil { + t.Fatalf("xdsClient.CancelCDS failed with error: %v", err) + } + if clusterNameDeleted != logicalDNSService { + t.Fatalf("xdsClient.CancelCDS called for cluster %v, want: %v", clusterNameDeleted, logicalDNSService) + } + + // The handleResp() callback should then start a watch for + // logicalDNSService2. + clusterNameCreated, err := fakeClient.WaitForWatchCluster(ctx) + if err != nil { + t.Fatalf("xdsClient.WatchCDS failed with error: %v", err) + } + if clusterNameCreated != logicalDNSService2 { + t.Fatalf("xdsClient.WatchCDS called for cluster %v, want: %v", clusterNameCreated, logicalDNSService2) + } + + // handleResp() should try and send an update here, but it will fail as + // logicalDNSService2 has not yet received an update. + shouldNotHappenCtx, shouldNotHappenCtxCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer shouldNotHappenCtxCancel() + select { + case <-ch.updateChannel: + t.Fatal("Cluster Handler wrote an update to updateChannel when it shouldn't have, as each node in the full cluster tree has not yet received an update") + case <-shouldNotHappenCtx.Done(): + } + + // Invoke a callback for the new logicalDNSService2 - this will fill out the + // tree with successful updates. + fakeClient.InvokeWatchClusterCallback(xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeLogicalDNS, + ClusterName: logicalDNSService2, + }, nil) + + // Behavior: This update make every node in the tree of cluster have + // received an update. Thus, at the end of this callback, when you ping the + // clusterHandler to try and construct an update, the update should now + // successfully be written to update buffer to send back to CDS. This new + // update should contain the new child of LogicalDNS2. + + select { + case chu := <-ch.updateChannel: + if diff := cmp.Diff(chu.updates, []xdsresource.ClusterUpdate{{ + ClusterType: xdsresource.ClusterTypeEDS, + ClusterName: edsService, + }, { + ClusterType: xdsresource.ClusterTypeLogicalDNS, + ClusterName: logicalDNSService2, + }}); diff != "" { + t.Fatalf("got unexpected cluster update, diff (-got, +want): %v", diff) + } + case <-ctx.Done(): + t.Fatal("Timed out waiting for the cluster update to be written to the update buffer.") + } +} + +// TestUpdateRootClusterAggregateThenChangeRootToEDS tests the situation where +// you have a fully updated aggregate cluster (where AggregateCluster success +// test gets you) as the root cluster, then you update that root cluster to a +// cluster of type EDS. +func (s) TestUpdateRootClusterAggregateThenChangeRootToEDS(t *testing.T) { + // This initial code is the same as the test for the aggregate success case, + // except without validations. This will get this test to the point where it + // can update the root cluster to one of type EDS. + ch, fakeClient := setupTests() + ch.updateRootCluster(aggregateClusterService) + + ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer ctxCancel() + _, err := fakeClient.WaitForWatchCluster(ctx) + if err != nil { + t.Fatalf("xdsClient.WatchCDS failed with error: %v", err) + } + + fakeClient.InvokeWatchClusterCallback(xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeAggregate, + ClusterName: aggregateClusterService, + PrioritizedClusterNames: []string{edsService, logicalDNSService}, + }, nil) + fakeClient.WaitForWatchCluster(ctx) + fakeClient.WaitForWatchCluster(ctx) + fakeClient.InvokeWatchClusterCallback(xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeEDS, + ClusterName: edsService, + }, nil) + fakeClient.InvokeWatchClusterCallback(xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeLogicalDNS, + ClusterName: logicalDNSService, + }, nil) + + select { + case <-ch.updateChannel: + case <-ctx.Done(): + t.Fatal("Timed out waiting for the cluster update to be written to the update buffer.") + } + + // Changes the root aggregate cluster to a EDS cluster. This should delete + // the root aggregate cluster and all of it's children by successfully + // canceling the watches for them. + ch.updateRootCluster(edsService2) + + // Reads from the cancel channel, should first be type Aggregate, then EDS + // then Logical DNS. + clusterNameDeleted, err := fakeClient.WaitForCancelClusterWatch(ctx) + if err != nil { + t.Fatalf("xdsClient.CancelCDS failed with error: %v", err) + } + if clusterNameDeleted != aggregateClusterService { + t.Fatalf("xdsClient.CancelCDS called for cluster %v, want: %v", clusterNameDeleted, logicalDNSService) + } + + clusterNameDeleted, err = fakeClient.WaitForCancelClusterWatch(ctx) + if err != nil { + t.Fatalf("xdsClient.CancelCDS failed with error: %v", err) + } + if clusterNameDeleted != edsService { + t.Fatalf("xdsClient.CancelCDS called for cluster %v, want: %v", clusterNameDeleted, logicalDNSService) + } + + clusterNameDeleted, err = fakeClient.WaitForCancelClusterWatch(ctx) + if err != nil { + t.Fatalf("xdsClient.CancelCDS failed with error: %v", err) + } + if clusterNameDeleted != logicalDNSService { + t.Fatalf("xdsClient.CancelCDS called for cluster %v, want: %v", clusterNameDeleted, logicalDNSService) + } + + // After deletion, it should start a watch for the EDS Cluster. The behavior + // for this EDS Cluster receiving an update from xds client and then + // successfully writing an update to send back to CDS is already tested in + // the updateEDS success case. + gotCluster, err := fakeClient.WaitForWatchCluster(ctx) + if err != nil { + t.Fatalf("xdsClient.WatchCDS failed with error: %v", err) + } + if gotCluster != edsService2 { + t.Fatalf("xdsClient.WatchCDS called for cluster: %v, want: %v", gotCluster, edsService2) + } +} + +// TestHandleRespInvokedWithError tests that when handleResp is invoked with an +// error, that the error is successfully written to the update buffer. +func (s) TestHandleRespInvokedWithError(t *testing.T) { + ch, fakeClient := setupTests() + ch.updateRootCluster(edsService) + ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer ctxCancel() + _, err := fakeClient.WaitForWatchCluster(ctx) + if err != nil { + t.Fatalf("xdsClient.WatchCDS failed with error: %v", err) + } + fakeClient.InvokeWatchClusterCallback(xdsresource.ClusterUpdate{}, errors.New("some error")) + select { + case chu := <-ch.updateChannel: + if chu.err.Error() != "some error" { + t.Fatalf("Did not receive the expected error, instead received: %v", chu.err.Error()) + } + case <-ctx.Done(): + t.Fatal("Timed out waiting for update from update channel.") + } +} + +// TestSwitchClusterNodeBetweenLeafAndAggregated tests having an existing +// cluster node switch between a leaf and an aggregated cluster. When the +// cluster switches from a leaf to an aggregated cluster, it should add +// children, and when it switches back to a leaf, it should delete those new +// children and also successfully write a cluster update to the update buffer. +func (s) TestSwitchClusterNodeBetweenLeafAndAggregated(t *testing.T) { + // Getting the test to the point where there's a root cluster which is a eds + // leaf. + ch, fakeClient := setupTests() + ch.updateRootCluster(edsService2) + ctx, ctxCancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer ctxCancel() + _, err := fakeClient.WaitForWatchCluster(ctx) + if err != nil { + t.Fatalf("xdsClient.WatchCDS failed with error: %v", err) + } + fakeClient.InvokeWatchClusterCallback(xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeEDS, + ClusterName: edsService2, + }, nil) + select { + case <-ch.updateChannel: + case <-ctx.Done(): + t.Fatal("Timed out waiting for update from update channel.") + } + // Switch the cluster to an aggregate cluster, this should cause two new + // child watches to be created. + fakeClient.InvokeWatchClusterCallback(xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeAggregate, + ClusterName: edsService2, + PrioritizedClusterNames: []string{edsService, logicalDNSService}, + }, nil) + + // xds client should be called to start a watch for one of the child + // clusters of the aggregate. The order of the children in the update + // written to the buffer to send to CDS matters, however there is no + // guarantee on the order it will start the watches of the children. + gotCluster, err := fakeClient.WaitForWatchCluster(ctx) + if err != nil { + t.Fatalf("xdsClient.WatchCDS failed with error: %v", err) + } + if gotCluster != edsService { + if gotCluster != logicalDNSService { + t.Fatalf("xdsClient.WatchCDS called for cluster: %v, want: %v", gotCluster, edsService) + } + } + + // xds client should then be called to start a watch for the second child + // cluster. + gotCluster, err = fakeClient.WaitForWatchCluster(ctx) + if err != nil { + t.Fatalf("xdsClient.WatchCDS failed with error: %v", err) + } + if gotCluster != edsService { + if gotCluster != logicalDNSService { + t.Fatalf("xdsClient.WatchCDS called for cluster: %v, want: %v", gotCluster, logicalDNSService) + } + } + + // After starting a watch for the second child cluster, there should be no + // more watches started on the xds client. + shouldNotHappenCtx, shouldNotHappenCtxCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer shouldNotHappenCtxCancel() + gotCluster, err = fakeClient.WaitForWatchCluster(shouldNotHappenCtx) + if err == nil { + t.Fatalf("xdsClient.WatchCDS called for cluster: %v, no more watches should be started.", gotCluster) + } + + // The handleResp() call on the root aggregate cluster should not ping the + // cluster handler to try and construct an update, as the handleResp() + // callback knows that when a child is created, it cannot possibly build a + // successful update yet. Thus, there should be nothing in the update + // channel. + + shouldNotHappenCtx, shouldNotHappenCtxCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer shouldNotHappenCtxCancel() + + select { + case <-ch.updateChannel: + t.Fatal("Cluster Handler wrote an update to updateChannel when it shouldn't have, as each node in the full cluster tree has not yet received an update") + case <-shouldNotHappenCtx.Done(): + } + + // Switch the cluster back to an EDS Cluster. This should cause the two + // children to be deleted. + fakeClient.InvokeWatchClusterCallback(xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeEDS, + ClusterName: edsService2, + }, nil) + + // Should delete the two children (no guarantee of ordering deleted, which + // is ok), then successfully write an update to the update buffer as the + // full cluster tree has received updates. + clusterNameDeleted, err := fakeClient.WaitForCancelClusterWatch(ctx) + if err != nil { + t.Fatalf("xdsClient.CancelCDS failed with error: %v", err) + } + // No guarantee of ordering, so one of the children should be deleted first. + if clusterNameDeleted != edsService { + if clusterNameDeleted != logicalDNSService { + t.Fatalf("xdsClient.CancelCDS called for cluster %v, want either: %v or: %v", clusterNameDeleted, edsService, logicalDNSService) + } + } + // Then the other child should be deleted. + clusterNameDeleted, err = fakeClient.WaitForCancelClusterWatch(ctx) + if err != nil { + t.Fatalf("xdsClient.CancelCDS failed with error: %v", err) + } + if clusterNameDeleted != edsService { + if clusterNameDeleted != logicalDNSService { + t.Fatalf("xdsClient.CancelCDS called for cluster %v, want either: %v or: %v", clusterNameDeleted, edsService, logicalDNSService) + } + } + + // After cancelling a watch for the second child cluster, there should be no + // more watches cancelled on the xds client. + shouldNotHappenCtx, shouldNotHappenCtxCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer shouldNotHappenCtxCancel() + gotCluster, err = fakeClient.WaitForCancelClusterWatch(shouldNotHappenCtx) + if err == nil { + t.Fatalf("xdsClient.WatchCDS called for cluster: %v, no more watches should be cancelled.", gotCluster) + } + + // Then an update should successfully be written to the update buffer. + select { + case chu := <-ch.updateChannel: + if diff := cmp.Diff(chu.updates, []xdsresource.ClusterUpdate{{ + ClusterType: xdsresource.ClusterTypeEDS, + ClusterName: edsService2, + }}); diff != "" { + t.Fatalf("got unexpected cluster update, diff (-got, +want): %v", diff) + } + case <-ctx.Done(): + t.Fatal("Timed out waiting for update from update channel.") + } +} + +// TestExceedsMaxStackDepth tests the scenario where an aggregate cluster +// exceeds the maximum depth, which is 16. This should cause an error to be +// written to the update buffer. +func (s) TestExceedsMaxStackDepth(t *testing.T) { + ch, fakeClient := setupTests() + ch.updateRootCluster("cluster0") + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + _, err := fakeClient.WaitForWatchCluster(ctx) + if err != nil { + t.Fatalf("xdsClient.WatchCDS failed with error: %v", err) + } + + for i := 0; i <= 15; i++ { + fakeClient.InvokeWatchClusterCallback(xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeAggregate, + ClusterName: "cluster" + fmt.Sprint(i), + PrioritizedClusterNames: []string{"cluster" + fmt.Sprint(i+1)}, + }, nil) + if i == 15 { + // The 16th iteration will try and create a cluster which exceeds + // max stack depth and will thus error, so no CDS Watch will be + // started for the child. + continue + } + _, err = fakeClient.WaitForWatchCluster(ctx) + if err != nil { + t.Fatalf("xdsClient.WatchCDS failed with error: %v", err) + } + } + select { + case chu := <-ch.updateChannel: + if chu.err.Error() != "aggregate cluster graph exceeds max depth" { + t.Fatalf("Did not receive the expected error, instead received: %v", chu.err.Error()) + } + case <-ctx.Done(): + t.Fatal("Timed out waiting for an error to be written to update channel.") + } +} + +// TestDiamondDependency tests a diamond shaped aggregate cluster (A->[B,C]; +// B->D; C->D). Due to both B and C pointing to D as it's child, it should be +// ignored for C. Once all 4 clusters have received a CDS update, an update +// should be then written to the update buffer, specifying a single Cluster D. +func (s) TestDiamondDependency(t *testing.T) { + ch, fakeClient := setupTests() + ch.updateRootCluster("clusterA") + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + _, err := fakeClient.WaitForWatchCluster(ctx) + if err != nil { + t.Fatalf("xdsClient.WatchCDS failed with error: %v", err) + } + fakeClient.InvokeWatchClusterCallback(xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeAggregate, + ClusterName: "clusterA", + PrioritizedClusterNames: []string{"clusterB", "clusterC"}, + }, nil) + // Two watches should be started for both child clusters. + _, err = fakeClient.WaitForWatchCluster(ctx) + if err != nil { + t.Fatalf("xdsClient.WatchCDS failed with error: %v", err) + } + _, err = fakeClient.WaitForWatchCluster(ctx) + if err != nil { + t.Fatalf("xdsClient.WatchCDS failed with error: %v", err) + } + // B -> D. + fakeClient.InvokeWatchClusterCallback(xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeAggregate, + ClusterName: "clusterB", + PrioritizedClusterNames: []string{"clusterD"}, + }, nil) + _, err = fakeClient.WaitForWatchCluster(ctx) + if err != nil { + t.Fatalf("xdsClient.WatchCDS failed with error: %v", err) + } + + // This shouldn't cause an update to be written to the update buffer, + // as cluster C has not received a cluster update yet. + fakeClient.InvokeWatchClusterCallback(xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeEDS, + ClusterName: "clusterD", + }, nil) + + sCtx, cancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer cancel() + select { + case <-ch.updateChannel: + t.Fatal("an update should not have been written to the update buffer") + case <-sCtx.Done(): + } + + // This update for C should cause an update to be written to the update + // buffer. When you search this aggregated cluster graph, each node has + // received an update. This update should only contain one clusterD, as + // clusterC does not add a clusterD child update due to the clusterD update + // already having been added as a child of clusterB. + fakeClient.InvokeWatchClusterCallback(xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeAggregate, + ClusterName: "clusterC", + PrioritizedClusterNames: []string{"clusterD"}, + }, nil) + + select { + case chu := <-ch.updateChannel: + if diff := cmp.Diff(chu.updates, []xdsresource.ClusterUpdate{{ + ClusterType: xdsresource.ClusterTypeEDS, + ClusterName: "clusterD", + }}); diff != "" { + t.Fatalf("got unexpected cluster update, diff (-got, +want): %v", diff) + } + case <-ctx.Done(): + t.Fatal("Timed out waiting for the cluster update to be written to the update buffer.") + } +} + +// TestIgnoreDups tests the cluster (A->[B, C]; B->[C, D]). Only one watch +// should be started for cluster C. The update written to the update buffer +// should only contain one instance of cluster C correctly as a higher priority +// than D. +func (s) TestIgnoreDups(t *testing.T) { + ch, fakeClient := setupTests() + ch.updateRootCluster("clusterA") + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + _, err := fakeClient.WaitForWatchCluster(ctx) + if err != nil { + t.Fatalf("xdsClient.WatchCDS failed with error: %v", err) + } + fakeClient.InvokeWatchClusterCallback(xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeAggregate, + ClusterName: "clusterA", + PrioritizedClusterNames: []string{"clusterB", "clusterC"}, + }, nil) + // Two watches should be started, one for each child cluster. + _, err = fakeClient.WaitForWatchCluster(ctx) + if err != nil { + t.Fatalf("xdsClient.WatchCDS failed with error: %v", err) + } + _, err = fakeClient.WaitForWatchCluster(ctx) + if err != nil { + t.Fatalf("xdsClient.WatchCDS failed with error: %v", err) + } + // The child cluster C should not have a watch started for it, as it is + // already part of the aggregate cluster graph as the child of the root + // cluster clusterA and has already had a watch started for it. + fakeClient.InvokeWatchClusterCallback(xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeAggregate, + ClusterName: "clusterB", + PrioritizedClusterNames: []string{"clusterC", "clusterD"}, + }, nil) + // Only one watch should be started, which should be for clusterD. + name, err := fakeClient.WaitForWatchCluster(ctx) + if err != nil { + t.Fatalf("xdsClient.WatchCDS failed with error: %v", err) + } + if name != "clusterD" { + t.Fatalf("xdsClient.WatchCDS called for cluster: %v, want: clusterD", name) + } + + sCtx, cancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer cancel() + if _, err = fakeClient.WaitForWatchCluster(sCtx); err == nil { + t.Fatalf("only one watch should have been started for the children of clusterB") + } + + // This update should not cause an update to be written to the update + // buffer, as each cluster in the tree has not yet received a cluster + // update. With cluster B ignoring cluster C, the system should function as + // if cluster C was not a child of cluster B (meaning all 4 clusters should + // be required to get an update). + fakeClient.InvokeWatchClusterCallback(xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeEDS, + ClusterName: "clusterC", + }, nil) + sCtx, cancel = context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer cancel() + select { + case <-ch.updateChannel: + t.Fatal("an update should not have been written to the update buffer") + case <-sCtx.Done(): + } + + // This update causes all 4 clusters in the aggregated cluster graph to have + // received an update, so an update should be written to the update buffer + // with only a single occurrence of cluster C as a higher priority than + // cluster D. + fakeClient.InvokeWatchClusterCallback(xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeEDS, + ClusterName: "clusterD", + }, nil) + select { + case chu := <-ch.updateChannel: + if diff := cmp.Diff(chu.updates, []xdsresource.ClusterUpdate{{ + ClusterType: xdsresource.ClusterTypeEDS, + ClusterName: "clusterC", + }, { + ClusterType: xdsresource.ClusterTypeEDS, + ClusterName: "clusterD", + }}); diff != "" { + t.Fatalf("got unexpected cluster update, diff (-got, +want): %v", diff) + } + case <-ctx.Done(): + t.Fatal("Timed out waiting for the cluster update to be written to the update buffer.") + } + + // Delete A's ref to C by updating A with only child B. Since B still has a + // reference to C, C's watch should not be canceled, and also an update + // should correctly be built. + fakeClient.InvokeWatchClusterCallback(xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeAggregate, + ClusterName: "clusterA", + PrioritizedClusterNames: []string{"clusterB"}, + }, nil) + + select { + case chu := <-ch.updateChannel: + if diff := cmp.Diff(chu.updates, []xdsresource.ClusterUpdate{{ + ClusterType: xdsresource.ClusterTypeEDS, + ClusterName: "clusterC", + }, { + ClusterType: xdsresource.ClusterTypeEDS, + ClusterName: "clusterD", + }}); diff != "" { + t.Fatalf("got unexpected cluster update, diff (-got, +want): %v", diff) + } + case <-ctx.Done(): + t.Fatal("Timed out waiting for the cluster update to be written to the update buffer.") + } +} + +// TestErrorStateWholeTree tests the scenario where the aggregate cluster graph +// exceeds max depth. An error should be written to the update channel. +// Afterward, if a valid response comes in for another cluster, no update should +// be written to the update channel, as the aggregate cluster graph is still in +// the same error state. +func (s) TestErrorStateWholeTree(t *testing.T) { + ch, fakeClient := setupTests() + ch.updateRootCluster("cluster0") + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + _, err := fakeClient.WaitForWatchCluster(ctx) + if err != nil { + t.Fatalf("xdsClient.WatchCDS failed with error: %v", err) + } + + for i := 0; i <= 15; i++ { + fakeClient.InvokeWatchClusterCallback(xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeAggregate, + ClusterName: "cluster" + fmt.Sprint(i), + PrioritizedClusterNames: []string{"cluster" + fmt.Sprint(i+1)}, + }, nil) + if i == 15 { + // The 16th iteration will try and create a cluster which exceeds + // max stack depth and will thus error, so no CDS Watch will be + // started for the child. + continue + } + _, err = fakeClient.WaitForWatchCluster(ctx) + if err != nil { + t.Fatalf("xdsClient.WatchCDS failed with error: %v", err) + } + } + select { + case chu := <-ch.updateChannel: + if chu.err.Error() != "aggregate cluster graph exceeds max depth" { + t.Fatalf("Did not receive the expected error, instead received: %v", chu.err.Error()) + } + case <-ctx.Done(): + t.Fatal("Timed out waiting for an error to be written to update channel.") + } + + // Invoke a cluster callback for a node in the graph that rests within the + // allowed depth. This will cause the system to try and construct a cluster + // update, and it shouldn't write an update as the aggregate cluster graph + // is still in an error state. Since the graph continues to stay in an error + // state, no new error needs to be written to the update buffer as that + // would be redundant information. + fakeClient.InvokeWatchClusterCallback(xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeAggregate, + ClusterName: "cluster3", + PrioritizedClusterNames: []string{"cluster4"}, + }, nil) + + sCtx, cancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer cancel() + select { + case <-ch.updateChannel: + t.Fatal("an update should not have been written to the update buffer") + case <-sCtx.Done(): + } + + // Invoke the same cluster update for cluster 15, specifying it has a child + // cluster16. This should cause an error to be written to the update buffer, + // as it still exceeds the max depth. + fakeClient.InvokeWatchClusterCallback(xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeAggregate, + ClusterName: "cluster15", + PrioritizedClusterNames: []string{"cluster16"}, + }, nil) + select { + case chu := <-ch.updateChannel: + if chu.err.Error() != "aggregate cluster graph exceeds max depth" { + t.Fatalf("Did not receive the expected error, instead received: %v", chu.err.Error()) + } + case <-ctx.Done(): + t.Fatal("Timed out waiting for an error to be written to update channel.") + } + + // When you remove the child of cluster15 that causes the graph to be in the + // error state of exceeding max depth, the update should successfully + // construct and be written to the update buffer. + fakeClient.InvokeWatchClusterCallback(xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeEDS, + ClusterName: "cluster15", + }, nil) + + select { + case chu := <-ch.updateChannel: + if diff := cmp.Diff(chu.updates, []xdsresource.ClusterUpdate{{ + ClusterType: xdsresource.ClusterTypeEDS, + ClusterName: "cluster15", + }}); diff != "" { + t.Fatalf("got unexpected cluster update, diff (-got, +want): %v", diff) + } + case <-ctx.Done(): + t.Fatal("Timed out waiting for the cluster update to be written to the update buffer.") + } +} + +// TestNodeChildOfItself tests the scenario where the aggregate cluster graph +// has a node that has child node of itself. The case for this is A -> A, and +// since there is no base cluster (EDS or Logical DNS), no update should be +// written if it tries to build a cluster update. +func (s) TestNodeChildOfItself(t *testing.T) { + ch, fakeClient := setupTests() + ch.updateRootCluster("clusterA") + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + _, err := fakeClient.WaitForWatchCluster(ctx) + if err != nil { + t.Fatalf("xdsClient.WatchCDS failed with error: %v", err) + } + // Invoke the callback informing the cluster handler that clusterA has a + // child that it is itself. Due to this child cluster being a duplicate, no + // watch should be started. Since there are no leaf nodes (i.e. EDS or + // Logical DNS), no update should be written to the update buffer. + fakeClient.InvokeWatchClusterCallback(xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeAggregate, + ClusterName: "clusterA", + PrioritizedClusterNames: []string{"clusterA"}, + }, nil) + sCtx, cancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer cancel() + if _, err := fakeClient.WaitForWatchCluster(sCtx); err == nil { + t.Fatal("Watch should not have been started for clusterA") + } + sCtx, cancel = context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer cancel() + select { + case <-ch.updateChannel: + t.Fatal("update should not have been written to update buffer") + case <-sCtx.Done(): + } + + // Invoke the callback again informing the cluster handler that clusterA has + // a child that it is itself. Due to this child cluster being a duplicate, + // no watch should be started. Since there are no leaf nodes (i.e. EDS or + // Logical DNS), no update should be written to the update buffer. + fakeClient.InvokeWatchClusterCallback(xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeAggregate, + ClusterName: "clusterA", + PrioritizedClusterNames: []string{"clusterA"}, + }, nil) + + sCtx, cancel = context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer cancel() + if _, err := fakeClient.WaitForWatchCluster(sCtx); err == nil { + t.Fatal("Watch should not have been started for clusterA") + } + sCtx, cancel = context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer cancel() + select { + case <-ch.updateChannel: + t.Fatal("update should not have been written to update buffer, as clusterB has not received an update yet") + case <-sCtx.Done(): + } + + // Inform the cluster handler that clusterA now has clusterB as a child. + // This should not cancel the watch for A, as it is still the root cluster + // and still has a ref count, not write an update to update buffer as + // cluster B has not received an update yet, and start a new watch for + // cluster B as it is not a duplicate. + fakeClient.InvokeWatchClusterCallback(xdsresource.ClusterUpdate{ + ClusterType: xdsresource.ClusterTypeAggregate, + ClusterName: "clusterA", + PrioritizedClusterNames: []string{"clusterB"}, + }, nil) + + sCtx, cancel = context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer cancel() + if _, err := fakeClient.WaitForCancelClusterWatch(sCtx); err == nil { + t.Fatal("clusterA should not have been canceled, as it is still the root cluster") + } + + sCtx, cancel = context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer cancel() + select { + case <-ch.updateChannel: + t.Fatal("update should not have been written to update buffer, as clusterB has not received an update yet") + case <-sCtx.Done(): + } + + gotCluster, err := fakeClient.WaitForWatchCluster(ctx) + if err != nil { + t.Fatalf("xdsClient.WatchCDS failed with error: %v", err) + } + if gotCluster != "clusterB" { + t.Fatalf("xdsClient.WatchCDS called for cluster: %v, want: %v", gotCluster, "clusterB") + } +} diff --git a/xds/internal/balancer/clusterimpl/balancer_test.go b/xds/internal/balancer/clusterimpl/balancer_test.go new file mode 100644 index 000000000000..2f0099d0b6c5 --- /dev/null +++ b/xds/internal/balancer/clusterimpl/balancer_test.go @@ -0,0 +1,789 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package clusterimpl + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/base" + "google.golang.org/grpc/balancer/roundrobin" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/balancer/stub" + "google.golang.org/grpc/internal/grpctest" + internalserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/resolver" + xdsinternal "google.golang.org/grpc/xds/internal" + "google.golang.org/grpc/xds/internal/testutils/fakeclient" + "google.golang.org/grpc/xds/internal/xdsclient" + "google.golang.org/grpc/xds/internal/xdsclient/bootstrap" + "google.golang.org/grpc/xds/internal/xdsclient/load" +) + +const ( + defaultTestTimeout = 5 * time.Second + defaultShortTestTimeout = 100 * time.Microsecond + + testClusterName = "test-cluster" + testServiceName = "test-eds-service" +) + +var ( + testBackendAddrs = []resolver.Address{ + {Addr: "1.1.1.1:1"}, + } + testLRSServerConfig = &bootstrap.ServerConfig{ + ServerURI: "trafficdirector.googleapis.com:443", + Creds: bootstrap.ChannelCreds{ + Type: "google_default", + }, + } + + cmpOpts = cmp.Options{ + cmpopts.EquateEmpty(), + cmpopts.IgnoreFields(load.Data{}, "ReportInterval"), + } +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func init() { + NewRandomWRR = testutils.NewTestWRR +} + +// TestDropByCategory verifies that the balancer correctly drops the picks, and +// that the drops are reported. +func (s) TestDropByCategory(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + defer xdsclient.ClearCounterForTesting(testClusterName, testServiceName) + xdsC := fakeclient.NewClient() + + builder := balancer.Get(Name) + cc := testutils.NewTestClientConn(t) + b := builder.Build(cc, balancer.BuildOptions{}) + defer b.Close() + + const ( + dropReason = "test-dropping-category" + dropNumerator = 1 + dropDenominator = 2 + ) + if err := b.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: xdsclient.SetClient(resolver.State{Addresses: testBackendAddrs}, xdsC), + BalancerConfig: &LBConfig{ + Cluster: testClusterName, + EDSServiceName: testServiceName, + LoadReportingServer: testLRSServerConfig, + DropCategories: []DropConfig{{ + Category: dropReason, + RequestsPerMillion: million * dropNumerator / dropDenominator, + }}, + ChildPolicy: &internalserviceconfig.BalancerConfig{ + Name: roundrobin.Name, + }, + }, + }); err != nil { + t.Fatalf("unexpected error from UpdateClientConnState: %v", err) + } + + got, err := xdsC.WaitForReportLoad(ctx) + if err != nil { + t.Fatalf("xdsClient.ReportLoad failed with error: %v", err) + } + if got.Server != testLRSServerConfig { + t.Fatalf("xdsClient.ReportLoad called with {%q}: want {%q}", got.Server, testLRSServerConfig) + } + + sc1 := <-cc.NewSubConnCh + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + // This should get the connecting picker. + if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil { + t.Fatal(err.Error()) + } + + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + // Test pick with one backend. + + const rpcCount = 20 + if err := cc.WaitForPicker(ctx, func(p balancer.Picker) error { + for i := 0; i < rpcCount; i++ { + gotSCSt, err := p.Pick(balancer.PickInfo{}) + // Even RPCs are dropped. + if i%2 == 0 { + if err == nil || !strings.Contains(err.Error(), "dropped") { + return fmt.Errorf("pick.Pick, got %v, %v, want error RPC dropped", gotSCSt, err) + } + continue + } + if err != nil || gotSCSt.SubConn != sc1 { + return fmt.Errorf("picker.Pick, got %v, %v, want SubConn=%v", gotSCSt, err, sc1) + } + if gotSCSt.Done != nil { + gotSCSt.Done(balancer.DoneInfo{}) + } + } + return nil + }); err != nil { + t.Fatal(err.Error()) + } + + // Dump load data from the store and compare with expected counts. + loadStore := xdsC.LoadStore() + if loadStore == nil { + t.Fatal("loadStore is nil in xdsClient") + } + const dropCount = rpcCount * dropNumerator / dropDenominator + wantStatsData0 := []*load.Data{{ + Cluster: testClusterName, + Service: testServiceName, + TotalDrops: dropCount, + Drops: map[string]uint64{dropReason: dropCount}, + LocalityStats: map[string]load.LocalityData{ + assertString(xdsinternal.LocalityID{}.ToString): {RequestStats: load.RequestData{Succeeded: rpcCount - dropCount}}, + }, + }} + + gotStatsData0 := loadStore.Stats([]string{testClusterName}) + if diff := cmp.Diff(gotStatsData0, wantStatsData0, cmpOpts); diff != "" { + t.Fatalf("got unexpected reports, diff (-got, +want): %v", diff) + } + + // Send an update with new drop configs. + const ( + dropReason2 = "test-dropping-category-2" + dropNumerator2 = 1 + dropDenominator2 = 4 + ) + if err := b.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: xdsclient.SetClient(resolver.State{Addresses: testBackendAddrs}, xdsC), + BalancerConfig: &LBConfig{ + Cluster: testClusterName, + EDSServiceName: testServiceName, + LoadReportingServer: testLRSServerConfig, + DropCategories: []DropConfig{{ + Category: dropReason2, + RequestsPerMillion: million * dropNumerator2 / dropDenominator2, + }}, + ChildPolicy: &internalserviceconfig.BalancerConfig{ + Name: roundrobin.Name, + }, + }, + }); err != nil { + t.Fatalf("unexpected error from UpdateClientConnState: %v", err) + } + + if err := cc.WaitForPicker(ctx, func(p balancer.Picker) error { + for i := 0; i < rpcCount; i++ { + gotSCSt, err := p.Pick(balancer.PickInfo{}) + // Even RPCs are dropped. + if i%4 == 0 { + if err == nil || !strings.Contains(err.Error(), "dropped") { + return fmt.Errorf("pick.Pick, got %v, %v, want error RPC dropped", gotSCSt, err) + } + continue + } + if err != nil || gotSCSt.SubConn != sc1 { + return fmt.Errorf("picker.Pick, got %v, %v, want SubConn=%v", gotSCSt, err, sc1) + } + if gotSCSt.Done != nil { + gotSCSt.Done(balancer.DoneInfo{}) + } + } + return nil + }); err != nil { + t.Fatal(err.Error()) + } + + const dropCount2 = rpcCount * dropNumerator2 / dropDenominator2 + wantStatsData1 := []*load.Data{{ + Cluster: testClusterName, + Service: testServiceName, + TotalDrops: dropCount2, + Drops: map[string]uint64{dropReason2: dropCount2}, + LocalityStats: map[string]load.LocalityData{ + assertString(xdsinternal.LocalityID{}.ToString): {RequestStats: load.RequestData{Succeeded: rpcCount - dropCount2}}, + }, + }} + + gotStatsData1 := loadStore.Stats([]string{testClusterName}) + if diff := cmp.Diff(gotStatsData1, wantStatsData1, cmpOpts); diff != "" { + t.Fatalf("got unexpected reports, diff (-got, +want): %v", diff) + } +} + +// TestDropCircuitBreaking verifies that the balancer correctly drops the picks +// due to circuit breaking, and that the drops are reported. +func (s) TestDropCircuitBreaking(t *testing.T) { + defer xdsclient.ClearCounterForTesting(testClusterName, testServiceName) + xdsC := fakeclient.NewClient() + + builder := balancer.Get(Name) + cc := testutils.NewTestClientConn(t) + b := builder.Build(cc, balancer.BuildOptions{}) + defer b.Close() + + var maxRequest uint32 = 50 + if err := b.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: xdsclient.SetClient(resolver.State{Addresses: testBackendAddrs}, xdsC), + BalancerConfig: &LBConfig{ + Cluster: testClusterName, + EDSServiceName: testServiceName, + LoadReportingServer: testLRSServerConfig, + MaxConcurrentRequests: &maxRequest, + ChildPolicy: &internalserviceconfig.BalancerConfig{ + Name: roundrobin.Name, + }, + }, + }); err != nil { + t.Fatalf("unexpected error from UpdateClientConnState: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + got, err := xdsC.WaitForReportLoad(ctx) + if err != nil { + t.Fatalf("xdsClient.ReportLoad failed with error: %v", err) + } + if got.Server != testLRSServerConfig { + t.Fatalf("xdsClient.ReportLoad called with {%q}: want {%q}", got.Server, testLRSServerConfig) + } + + sc1 := <-cc.NewSubConnCh + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + // This should get the connecting picker. + if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil { + t.Fatal(err.Error()) + } + + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + // Test pick with one backend. + const rpcCount = 100 + if err := cc.WaitForPicker(ctx, func(p balancer.Picker) error { + dones := []func(){} + for i := 0; i < rpcCount; i++ { + gotSCSt, err := p.Pick(balancer.PickInfo{}) + if i < 50 && err != nil { + return fmt.Errorf("The first 50%% picks should be non-drops, got error %v", err) + } else if i > 50 && err == nil { + return fmt.Errorf("The second 50%% picks should be drops, got error ") + } + dones = append(dones, func() { + if gotSCSt.Done != nil { + gotSCSt.Done(balancer.DoneInfo{}) + } + }) + } + for _, done := range dones { + done() + } + + dones = []func(){} + // Pick without drops. + for i := 0; i < 50; i++ { + gotSCSt, err := p.Pick(balancer.PickInfo{}) + if err != nil { + t.Errorf("The third 50%% picks should be non-drops, got error %v", err) + } + dones = append(dones, func() { + if gotSCSt.Done != nil { + gotSCSt.Done(balancer.DoneInfo{}) + } + }) + } + for _, done := range dones { + done() + } + + return nil + }); err != nil { + t.Fatal(err.Error()) + } + + // Dump load data from the store and compare with expected counts. + loadStore := xdsC.LoadStore() + if loadStore == nil { + t.Fatal("loadStore is nil in xdsClient") + } + + wantStatsData0 := []*load.Data{{ + Cluster: testClusterName, + Service: testServiceName, + TotalDrops: uint64(maxRequest), + LocalityStats: map[string]load.LocalityData{ + assertString(xdsinternal.LocalityID{}.ToString): {RequestStats: load.RequestData{Succeeded: uint64(rpcCount - maxRequest + 50)}}, + }, + }} + + gotStatsData0 := loadStore.Stats([]string{testClusterName}) + if diff := cmp.Diff(gotStatsData0, wantStatsData0, cmpOpts); diff != "" { + t.Fatalf("got unexpected drop reports, diff (-got, +want): %v", diff) + } +} + +// TestPickerUpdateAfterClose covers the case where a child policy sends a +// picker update after the cluster_impl policy is closed. Because picker updates +// are handled in the run() goroutine, which exits before Close() returns, we +// expect the above picker update to be dropped. +func (s) TestPickerUpdateAfterClose(t *testing.T) { + defer xdsclient.ClearCounterForTesting(testClusterName, testServiceName) + xdsC := fakeclient.NewClient() + + builder := balancer.Get(Name) + cc := testutils.NewTestClientConn(t) + b := builder.Build(cc, balancer.BuildOptions{}) + + // Create a stub balancer which waits for the cluster_impl policy to be + // closed before sending a picker update (upon receipt of a subConn state + // change). + closeCh := make(chan struct{}) + const childPolicyName = "stubBalancer-TestPickerUpdateAfterClose" + stub.Register(childPolicyName, stub.BalancerFuncs{ + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + // Create a subConn which will be used later on to test the race + // between StateListener() and Close(). + sc, err := bd.ClientConn.NewSubConn(ccs.ResolverState.Addresses, balancer.NewSubConnOptions{ + StateListener: func(balancer.SubConnState) { + go func() { + // Wait for Close() to be called on the parent policy before + // sending the picker update. + <-closeCh + bd.ClientConn.UpdateState(balancer.State{ + Picker: base.NewErrPicker(errors.New("dummy error picker")), + }) + }() + }, + }) + if err != nil { + return err + } + sc.Connect() + return nil + }, + }) + + var maxRequest uint32 = 50 + if err := b.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: xdsclient.SetClient(resolver.State{Addresses: testBackendAddrs}, xdsC), + BalancerConfig: &LBConfig{ + Cluster: testClusterName, + EDSServiceName: testServiceName, + MaxConcurrentRequests: &maxRequest, + ChildPolicy: &internalserviceconfig.BalancerConfig{ + Name: childPolicyName, + }, + }, + }); err != nil { + b.Close() + t.Fatalf("unexpected error from UpdateClientConnState: %v", err) + } + + // Send a subConn state change to trigger a picker update. The stub balancer + // that we use as the child policy will not send a picker update until the + // parent policy is closed. + sc1 := <-cc.NewSubConnCh + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + b.Close() + close(closeCh) + + select { + case <-cc.NewPickerCh: + t.Fatalf("unexpected picker update after balancer is closed") + case <-time.After(defaultShortTestTimeout): + } +} + +// TestClusterNameInAddressAttributes covers the case that cluster name is +// attached to the subconn address attributes. +func (s) TestClusterNameInAddressAttributes(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + defer xdsclient.ClearCounterForTesting(testClusterName, testServiceName) + xdsC := fakeclient.NewClient() + + builder := balancer.Get(Name) + cc := testutils.NewTestClientConn(t) + b := builder.Build(cc, balancer.BuildOptions{}) + defer b.Close() + + if err := b.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: xdsclient.SetClient(resolver.State{Addresses: testBackendAddrs}, xdsC), + BalancerConfig: &LBConfig{ + Cluster: testClusterName, + EDSServiceName: testServiceName, + ChildPolicy: &internalserviceconfig.BalancerConfig{ + Name: roundrobin.Name, + }, + }, + }); err != nil { + t.Fatalf("unexpected error from UpdateClientConnState: %v", err) + } + + sc1 := <-cc.NewSubConnCh + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + // This should get the connecting picker. + if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil { + t.Fatal(err.Error()) + } + + addrs1 := <-cc.NewSubConnAddrsCh + if got, want := addrs1[0].Addr, testBackendAddrs[0].Addr; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + cn, ok := internal.GetXDSHandshakeClusterName(addrs1[0].Attributes) + if !ok || cn != testClusterName { + t.Fatalf("sc is created with addr with cluster name %v, %v, want cluster name %v", cn, ok, testClusterName) + } + + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + // Test pick with one backend. + if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil { + t.Fatal(err.Error()) + } + + const testClusterName2 = "test-cluster-2" + var addr2 = resolver.Address{Addr: "2.2.2.2"} + if err := b.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: xdsclient.SetClient(resolver.State{Addresses: []resolver.Address{addr2}}, xdsC), + BalancerConfig: &LBConfig{ + Cluster: testClusterName2, + EDSServiceName: testServiceName, + ChildPolicy: &internalserviceconfig.BalancerConfig{ + Name: roundrobin.Name, + }, + }, + }); err != nil { + t.Fatalf("unexpected error from UpdateClientConnState: %v", err) + } + + addrs2 := <-cc.NewSubConnAddrsCh + if got, want := addrs2[0].Addr, addr2.Addr; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + // New addresses should have the new cluster name. + cn2, ok := internal.GetXDSHandshakeClusterName(addrs2[0].Attributes) + if !ok || cn2 != testClusterName2 { + t.Fatalf("sc is created with addr with cluster name %v, %v, want cluster name %v", cn2, ok, testClusterName2) + } +} + +// TestReResolution verifies that when a SubConn turns transient failure, +// re-resolution is triggered. +func (s) TestReResolution(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + defer xdsclient.ClearCounterForTesting(testClusterName, testServiceName) + xdsC := fakeclient.NewClient() + + builder := balancer.Get(Name) + cc := testutils.NewTestClientConn(t) + b := builder.Build(cc, balancer.BuildOptions{}) + defer b.Close() + + if err := b.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: xdsclient.SetClient(resolver.State{Addresses: testBackendAddrs}, xdsC), + BalancerConfig: &LBConfig{ + Cluster: testClusterName, + EDSServiceName: testServiceName, + ChildPolicy: &internalserviceconfig.BalancerConfig{ + Name: roundrobin.Name, + }, + }, + }); err != nil { + t.Fatalf("unexpected error from UpdateClientConnState: %v", err) + } + + sc1 := <-cc.NewSubConnCh + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + // This should get the connecting picker. + if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil { + t.Fatal(err.Error()) + } + + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) + // This should get the transient failure picker. + if err := cc.WaitForErrPicker(ctx); err != nil { + t.Fatal(err.Error()) + } + + // The transient failure should trigger a re-resolution. + select { + case <-cc.ResolveNowCh: + case <-time.After(defaultTestTimeout): + t.Fatalf("timeout waiting for ResolveNow()") + } + + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + // Test pick with one backend. + if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil { + t.Fatal(err.Error()) + } + + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) + // This should get the transient failure picker. + if err := cc.WaitForErrPicker(ctx); err != nil { + t.Fatal(err.Error()) + } + + // The transient failure should trigger a re-resolution. + select { + case <-cc.ResolveNowCh: + case <-time.After(defaultTestTimeout): + t.Fatalf("timeout waiting for ResolveNow()") + } +} + +func (s) TestLoadReporting(t *testing.T) { + var testLocality = xdsinternal.LocalityID{ + Region: "test-region", + Zone: "test-zone", + SubZone: "test-sub-zone", + } + + xdsC := fakeclient.NewClient() + + builder := balancer.Get(Name) + cc := testutils.NewTestClientConn(t) + b := builder.Build(cc, balancer.BuildOptions{}) + defer b.Close() + + addrs := make([]resolver.Address, len(testBackendAddrs)) + for i, a := range testBackendAddrs { + addrs[i] = xdsinternal.SetLocalityID(a, testLocality) + } + if err := b.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: xdsclient.SetClient(resolver.State{Addresses: addrs}, xdsC), + BalancerConfig: &LBConfig{ + Cluster: testClusterName, + EDSServiceName: testServiceName, + LoadReportingServer: testLRSServerConfig, + // Locality: testLocality, + ChildPolicy: &internalserviceconfig.BalancerConfig{ + Name: roundrobin.Name, + }, + }, + }); err != nil { + t.Fatalf("unexpected error from UpdateClientConnState: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + got, err := xdsC.WaitForReportLoad(ctx) + if err != nil { + t.Fatalf("xdsClient.ReportLoad failed with error: %v", err) + } + if got.Server != testLRSServerConfig { + t.Fatalf("xdsClient.ReportLoad called with {%q}: want {%q}", got.Server, testLRSServerConfig) + } + + sc1 := <-cc.NewSubConnCh + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + // This should get the connecting picker. + if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil { + t.Fatal(err.Error()) + } + + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + // Test pick with one backend. + const successCount = 5 + const errorCount = 5 + if err := cc.WaitForPicker(ctx, func(p balancer.Picker) error { + for i := 0; i < successCount; i++ { + gotSCSt, err := p.Pick(balancer.PickInfo{}) + if gotSCSt.SubConn != sc1 { + return fmt.Errorf("picker.Pick, got %v, %v, want SubConn=%v", gotSCSt, err, sc1) + } + gotSCSt.Done(balancer.DoneInfo{}) + } + for i := 0; i < errorCount; i++ { + gotSCSt, err := p.Pick(balancer.PickInfo{}) + if gotSCSt.SubConn != sc1 { + return fmt.Errorf("picker.Pick, got %v, %v, want SubConn=%v", gotSCSt, err, sc1) + } + gotSCSt.Done(balancer.DoneInfo{Err: fmt.Errorf("error")}) + } + return nil + }); err != nil { + t.Fatal(err.Error()) + } + + // Dump load data from the store and compare with expected counts. + loadStore := xdsC.LoadStore() + if loadStore == nil { + t.Fatal("loadStore is nil in xdsClient") + } + sds := loadStore.Stats([]string{testClusterName}) + if len(sds) == 0 { + t.Fatalf("loads for cluster %v not found in store", testClusterName) + } + sd := sds[0] + if sd.Cluster != testClusterName || sd.Service != testServiceName { + t.Fatalf("got unexpected load for %q, %q, want %q, %q", sd.Cluster, sd.Service, testClusterName, testServiceName) + } + testLocalityJSON, _ := testLocality.ToString() + localityData, ok := sd.LocalityStats[testLocalityJSON] + if !ok { + t.Fatalf("loads for %v not found in store", testLocality) + } + reqStats := localityData.RequestStats + if reqStats.Succeeded != successCount { + t.Errorf("got succeeded %v, want %v", reqStats.Succeeded, successCount) + } + if reqStats.Errored != errorCount { + t.Errorf("got errord %v, want %v", reqStats.Errored, errorCount) + } + if reqStats.InProgress != 0 { + t.Errorf("got inProgress %v, want %v", reqStats.InProgress, 0) + } + + b.Close() + if err := xdsC.WaitForCancelReportLoad(ctx); err != nil { + t.Fatalf("unexpected error waiting form load report to be canceled: %v", err) + } +} + +// TestUpdateLRSServer covers the cases +// - the init config specifies "" as the LRS server +// - config modifies LRS server to a different string +// - config sets LRS server to nil to stop load reporting +func (s) TestUpdateLRSServer(t *testing.T) { + var testLocality = xdsinternal.LocalityID{ + Region: "test-region", + Zone: "test-zone", + SubZone: "test-sub-zone", + } + + xdsC := fakeclient.NewClient() + + builder := balancer.Get(Name) + cc := testutils.NewTestClientConn(t) + b := builder.Build(cc, balancer.BuildOptions{}) + defer b.Close() + + addrs := make([]resolver.Address, len(testBackendAddrs)) + for i, a := range testBackendAddrs { + addrs[i] = xdsinternal.SetLocalityID(a, testLocality) + } + if err := b.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: xdsclient.SetClient(resolver.State{Addresses: addrs}, xdsC), + BalancerConfig: &LBConfig{ + Cluster: testClusterName, + EDSServiceName: testServiceName, + LoadReportingServer: testLRSServerConfig, + ChildPolicy: &internalserviceconfig.BalancerConfig{ + Name: roundrobin.Name, + }, + }, + }); err != nil { + t.Fatalf("unexpected error from UpdateClientConnState: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + got, err := xdsC.WaitForReportLoad(ctx) + if err != nil { + t.Fatalf("xdsClient.ReportLoad failed with error: %v", err) + } + if got.Server != testLRSServerConfig { + t.Fatalf("xdsClient.ReportLoad called with {%q}: want {%q}", got.Server, testLRSServerConfig) + } + + testLRSServerConfig2 := &bootstrap.ServerConfig{ + ServerURI: "trafficdirector-another.googleapis.com:443", + Creds: bootstrap.ChannelCreds{ + Type: "google_default", + }, + } + // Update LRS server to a different name. + if err := b.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: xdsclient.SetClient(resolver.State{Addresses: addrs}, xdsC), + BalancerConfig: &LBConfig{ + Cluster: testClusterName, + EDSServiceName: testServiceName, + LoadReportingServer: testLRSServerConfig2, + ChildPolicy: &internalserviceconfig.BalancerConfig{ + Name: roundrobin.Name, + }, + }, + }); err != nil { + t.Fatalf("unexpected error from UpdateClientConnState: %v", err) + } + if err := xdsC.WaitForCancelReportLoad(ctx); err != nil { + t.Fatalf("unexpected error waiting form load report to be canceled: %v", err) + } + got2, err2 := xdsC.WaitForReportLoad(ctx) + if err2 != nil { + t.Fatalf("xdsClient.ReportLoad failed with error: %v", err2) + } + if got2.Server != testLRSServerConfig2 { + t.Fatalf("xdsClient.ReportLoad called with {%q}: want {%q}", got2.Server, testLRSServerConfig2) + } + + // Update LRS server to nil, to disable LRS. + if err := b.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: xdsclient.SetClient(resolver.State{Addresses: addrs}, xdsC), + BalancerConfig: &LBConfig{ + Cluster: testClusterName, + EDSServiceName: testServiceName, + ChildPolicy: &internalserviceconfig.BalancerConfig{ + Name: roundrobin.Name, + }, + }, + }); err != nil { + t.Fatalf("unexpected error from UpdateClientConnState: %v", err) + } + if err := xdsC.WaitForCancelReportLoad(ctx); err != nil { + t.Fatalf("unexpected error waiting form load report to be canceled: %v", err) + } + + shortCtx, shortCancel := context.WithTimeout(context.Background(), defaultShortTestTimeout) + defer shortCancel() + if s, err := xdsC.WaitForReportLoad(shortCtx); err != context.DeadlineExceeded { + t.Fatalf("unexpected load report to server: %q", s) + } +} + +func assertString(f func() (string, error)) string { + s, err := f() + if err != nil { + panic(err.Error()) + } + return s +} diff --git a/xds/internal/balancer/clusterimpl/clusterimpl.go b/xds/internal/balancer/clusterimpl/clusterimpl.go new file mode 100644 index 000000000000..407d2deff7d6 --- /dev/null +++ b/xds/internal/balancer/clusterimpl/clusterimpl.go @@ -0,0 +1,492 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package clusterimpl implements the xds_cluster_impl balancing policy. It +// handles the cluster features (e.g. circuit_breaking, RPC dropping). +// +// Note that it doesn't handle name resolution, which is done by policy +// xds_cluster_resolver. +package clusterimpl + +import ( + "encoding/json" + "fmt" + "sync" + "sync/atomic" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/balancer/gracefulswitch" + "google.golang.org/grpc/internal/buffer" + "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" + xdsinternal "google.golang.org/grpc/xds/internal" + "google.golang.org/grpc/xds/internal/balancer/loadstore" + "google.golang.org/grpc/xds/internal/xdsclient" + "google.golang.org/grpc/xds/internal/xdsclient/bootstrap" + "google.golang.org/grpc/xds/internal/xdsclient/load" +) + +const ( + // Name is the name of the cluster_impl balancer. + Name = "xds_cluster_impl_experimental" + defaultRequestCountMax = 1024 +) + +func init() { + balancer.Register(bb{}) +} + +type bb struct{} + +func (bb) Build(cc balancer.ClientConn, bOpts balancer.BuildOptions) balancer.Balancer { + b := &clusterImplBalancer{ + ClientConn: cc, + bOpts: bOpts, + closed: grpcsync.NewEvent(), + done: grpcsync.NewEvent(), + loadWrapper: loadstore.NewWrapper(), + pickerUpdateCh: buffer.NewUnbounded(), + requestCountMax: defaultRequestCountMax, + } + b.logger = prefixLogger(b) + b.child = gracefulswitch.NewBalancer(b, bOpts) + go b.run() + b.logger.Infof("Created") + return b +} + +func (bb) Name() string { + return Name +} + +func (bb) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + return parseConfig(c) +} + +type clusterImplBalancer struct { + balancer.ClientConn + + // mu guarantees mutual exclusion between Close() and handling of picker + // update to the parent ClientConn in run(). It's to make sure that the + // run() goroutine doesn't send picker update to parent after the balancer + // is closed. + // + // It's only used by the run() goroutine, but not the other exported + // functions. Because the exported functions are guaranteed to be + // synchronized with Close(). + mu sync.Mutex + closed *grpcsync.Event + done *grpcsync.Event + + bOpts balancer.BuildOptions + logger *grpclog.PrefixLogger + xdsClient xdsclient.XDSClient + + config *LBConfig + child *gracefulswitch.Balancer + cancelLoadReport func() + edsServiceName string + lrsServer *bootstrap.ServerConfig + loadWrapper *loadstore.Wrapper + + clusterNameMu sync.Mutex + clusterName string + + // childState/drops/requestCounter keeps the state used by the most recently + // generated picker. All fields can only be accessed in run(). And run() is + // the only goroutine that sends picker to the parent ClientConn. All + // requests to update picker need to be sent to pickerUpdateCh. + childState balancer.State + dropCategories []DropConfig // The categories for drops. + drops []*dropper + requestCounterCluster string // The cluster name for the request counter. + requestCounterService string // The service name for the request counter. + requestCounter *xdsclient.ClusterRequestsCounter + requestCountMax uint32 + pickerUpdateCh *buffer.Unbounded +} + +// updateLoadStore checks the config for load store, and decides whether it +// needs to restart the load reporting stream. +func (b *clusterImplBalancer) updateLoadStore(newConfig *LBConfig) error { + var updateLoadClusterAndService bool + + // ClusterName is different, restart. ClusterName is from ClusterName and + // EDSServiceName. + clusterName := b.getClusterName() + if clusterName != newConfig.Cluster { + updateLoadClusterAndService = true + b.setClusterName(newConfig.Cluster) + clusterName = newConfig.Cluster + } + if b.edsServiceName != newConfig.EDSServiceName { + updateLoadClusterAndService = true + b.edsServiceName = newConfig.EDSServiceName + } + if updateLoadClusterAndService { + // This updates the clusterName and serviceName that will be reported + // for the loads. The update here is too early, the perfect timing is + // when the picker is updated with the new connection. But from this + // balancer's point of view, it's impossible to tell. + // + // On the other hand, this will almost never happen. Each LRS policy + // shouldn't get updated config. The parent should do a graceful switch + // when the clusterName or serviceName is changed. + b.loadWrapper.UpdateClusterAndService(clusterName, b.edsServiceName) + } + + var ( + stopOldLoadReport bool + startNewLoadReport bool + ) + + // Check if it's necessary to restart load report. + if b.lrsServer == nil { + if newConfig.LoadReportingServer != nil { + // Old is nil, new is not nil, start new LRS. + b.lrsServer = newConfig.LoadReportingServer + startNewLoadReport = true + } + // Old is nil, new is nil, do nothing. + } else if newConfig.LoadReportingServer == nil { + // Old is not nil, new is nil, stop old, don't start new. + b.lrsServer = newConfig.LoadReportingServer + stopOldLoadReport = true + } else { + // Old is not nil, new is not nil, compare string values, if + // different, stop old and start new. + if !b.lrsServer.Equal(newConfig.LoadReportingServer) { + b.lrsServer = newConfig.LoadReportingServer + stopOldLoadReport = true + startNewLoadReport = true + } + } + + if stopOldLoadReport { + if b.cancelLoadReport != nil { + b.cancelLoadReport() + b.cancelLoadReport = nil + if !startNewLoadReport { + // If a new LRS stream will be started later, no need to update + // it to nil here. + b.loadWrapper.UpdateLoadStore(nil) + } + } + } + if startNewLoadReport { + var loadStore *load.Store + if b.xdsClient != nil { + loadStore, b.cancelLoadReport = b.xdsClient.ReportLoad(b.lrsServer) + } + b.loadWrapper.UpdateLoadStore(loadStore) + } + + return nil +} + +func (b *clusterImplBalancer) UpdateClientConnState(s balancer.ClientConnState) error { + if b.closed.HasFired() { + b.logger.Warningf("xds: received ClientConnState {%+v} after clusterImplBalancer was closed", s) + return nil + } + + b.logger.Infof("Received update from resolver, balancer config: %+v", pretty.ToJSON(s.BalancerConfig)) + newConfig, ok := s.BalancerConfig.(*LBConfig) + if !ok { + return fmt.Errorf("unexpected balancer config with type: %T", s.BalancerConfig) + } + + // Need to check for potential errors at the beginning of this function, so + // that on errors, we reject the whole config, instead of applying part of + // it. + bb := balancer.Get(newConfig.ChildPolicy.Name) + if bb == nil { + return fmt.Errorf("balancer %q not registered", newConfig.ChildPolicy.Name) + } + + if b.xdsClient == nil { + c := xdsclient.FromResolverState(s.ResolverState) + if c == nil { + return balancer.ErrBadResolverState + } + b.xdsClient = c + } + + // Update load reporting config. This needs to be done before updating the + // child policy because we need the loadStore from the updated client to be + // passed to the ccWrapper, so that the next picker from the child policy + // will pick up the new loadStore. + if err := b.updateLoadStore(newConfig); err != nil { + return err + } + + if b.config == nil || b.config.ChildPolicy.Name != newConfig.ChildPolicy.Name { + if err := b.child.SwitchTo(bb); err != nil { + return fmt.Errorf("error switching to child of type %q: %v", newConfig.ChildPolicy.Name, err) + } + } + b.config = newConfig + + // Notify run() of this new config, in case drop and request counter need + // update (which means a new picker needs to be generated). + b.pickerUpdateCh.Put(newConfig) + + // Addresses and sub-balancer config are sent to sub-balancer. + return b.child.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: s.ResolverState, + BalancerConfig: b.config.ChildPolicy.Config, + }) +} + +func (b *clusterImplBalancer) ResolverError(err error) { + if b.closed.HasFired() { + b.logger.Warningf("xds: received resolver error {%+v} after clusterImplBalancer was closed", err) + return + } + b.child.ResolverError(err) +} + +func (b *clusterImplBalancer) updateSubConnState(sc balancer.SubConn, s balancer.SubConnState, cb func(balancer.SubConnState)) { + if b.closed.HasFired() { + b.logger.Warningf("xds: received subconn state change {%+v, %+v} after clusterImplBalancer was closed", sc, s) + return + } + + // Trigger re-resolution when a SubConn turns transient failure. This is + // necessary for the LogicalDNS in cluster_resolver policy to re-resolve. + // + // Note that this happens not only for the addresses from DNS, but also for + // EDS (cluster_impl doesn't know if it's DNS or EDS, only the parent + // knows). The parent priority policy is configured to ignore re-resolution + // signal from the EDS children. + if s.ConnectivityState == connectivity.TransientFailure { + b.ClientConn.ResolveNow(resolver.ResolveNowOptions{}) + } + + if cb != nil { + cb(s) + } +} + +func (b *clusterImplBalancer) UpdateSubConnState(sc balancer.SubConn, s balancer.SubConnState) { + b.logger.Errorf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, s) +} + +func (b *clusterImplBalancer) Close() { + b.mu.Lock() + b.closed.Fire() + b.mu.Unlock() + + b.child.Close() + b.childState = balancer.State{} + b.pickerUpdateCh.Close() + <-b.done.Done() + b.logger.Infof("Shutdown") +} + +func (b *clusterImplBalancer) ExitIdle() { + b.child.ExitIdle() +} + +// Override methods to accept updates from the child LB. + +func (b *clusterImplBalancer) UpdateState(state balancer.State) { + // Instead of updating parent ClientConn inline, send state to run(). + b.pickerUpdateCh.Put(state) +} + +func (b *clusterImplBalancer) setClusterName(n string) { + b.clusterNameMu.Lock() + defer b.clusterNameMu.Unlock() + b.clusterName = n +} + +func (b *clusterImplBalancer) getClusterName() string { + b.clusterNameMu.Lock() + defer b.clusterNameMu.Unlock() + return b.clusterName +} + +// scWrapper is a wrapper of SubConn with locality ID. The locality ID can be +// retrieved from the addresses when creating SubConn. +// +// All SubConns passed to the child policies are wrapped in this, so that the +// picker can get the localityID from the picked SubConn, and do load reporting. +// +// After wrapping, all SubConns to and from the parent ClientConn (e.g. for +// SubConn state update, update/remove SubConn) must be the original SubConns. +// All SubConns to and from the child policy (NewSubConn, forwarding SubConn +// state update) must be the wrapper. The balancer keeps a map from the original +// SubConn to the wrapper for this purpose. +type scWrapper struct { + balancer.SubConn + // locality needs to be atomic because it can be updated while being read by + // the picker. + locality atomic.Value // type xdsinternal.LocalityID +} + +func (scw *scWrapper) updateLocalityID(lID xdsinternal.LocalityID) { + scw.locality.Store(lID) +} + +func (scw *scWrapper) localityID() xdsinternal.LocalityID { + lID, _ := scw.locality.Load().(xdsinternal.LocalityID) + return lID +} + +func (b *clusterImplBalancer) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) { + clusterName := b.getClusterName() + newAddrs := make([]resolver.Address, len(addrs)) + var lID xdsinternal.LocalityID + for i, addr := range addrs { + newAddrs[i] = internal.SetXDSHandshakeClusterName(addr, clusterName) + lID = xdsinternal.GetLocalityID(newAddrs[i]) + } + var sc balancer.SubConn + oldListener := opts.StateListener + opts.StateListener = func(state balancer.SubConnState) { b.updateSubConnState(sc, state, oldListener) } + sc, err := b.ClientConn.NewSubConn(newAddrs, opts) + if err != nil { + return nil, err + } + // Wrap this SubConn in a wrapper, and add it to the map. + ret := &scWrapper{SubConn: sc} + ret.updateLocalityID(lID) + return ret, nil +} + +func (b *clusterImplBalancer) RemoveSubConn(sc balancer.SubConn) { + b.logger.Errorf("RemoveSubConn(%v) called unexpectedly", sc) +} + +func (b *clusterImplBalancer) UpdateAddresses(sc balancer.SubConn, addrs []resolver.Address) { + clusterName := b.getClusterName() + newAddrs := make([]resolver.Address, len(addrs)) + var lID xdsinternal.LocalityID + for i, addr := range addrs { + newAddrs[i] = internal.SetXDSHandshakeClusterName(addr, clusterName) + lID = xdsinternal.GetLocalityID(newAddrs[i]) + } + if scw, ok := sc.(*scWrapper); ok { + scw.updateLocalityID(lID) + // Need to get the original SubConn from the wrapper before calling + // parent ClientConn. + sc = scw.SubConn + } + b.ClientConn.UpdateAddresses(sc, newAddrs) +} + +type dropConfigs struct { + drops []*dropper + requestCounter *xdsclient.ClusterRequestsCounter + requestCountMax uint32 +} + +// handleDropAndRequestCount compares drop and request counter in newConfig with +// the one currently used by picker. It returns a new dropConfigs if a new +// picker needs to be generated, otherwise it returns nil. +func (b *clusterImplBalancer) handleDropAndRequestCount(newConfig *LBConfig) *dropConfigs { + // Compare new drop config. And update picker if it's changed. + var updatePicker bool + if !equalDropCategories(b.dropCategories, newConfig.DropCategories) { + b.dropCategories = newConfig.DropCategories + b.drops = make([]*dropper, 0, len(newConfig.DropCategories)) + for _, c := range newConfig.DropCategories { + b.drops = append(b.drops, newDropper(c)) + } + updatePicker = true + } + + // Compare cluster name. And update picker if it's changed, because circuit + // breaking's stream counter will be different. + if b.requestCounterCluster != newConfig.Cluster || b.requestCounterService != newConfig.EDSServiceName { + b.requestCounterCluster = newConfig.Cluster + b.requestCounterService = newConfig.EDSServiceName + b.requestCounter = xdsclient.GetClusterRequestsCounter(newConfig.Cluster, newConfig.EDSServiceName) + updatePicker = true + } + // Compare upper bound of stream count. And update picker if it's changed. + // This is also for circuit breaking. + var newRequestCountMax uint32 = 1024 + if newConfig.MaxConcurrentRequests != nil { + newRequestCountMax = *newConfig.MaxConcurrentRequests + } + if b.requestCountMax != newRequestCountMax { + b.requestCountMax = newRequestCountMax + updatePicker = true + } + + if !updatePicker { + return nil + } + return &dropConfigs{ + drops: b.drops, + requestCounter: b.requestCounter, + requestCountMax: b.requestCountMax, + } +} + +func (b *clusterImplBalancer) run() { + defer b.done.Fire() + for { + select { + case update, ok := <-b.pickerUpdateCh.Get(): + if !ok { + return + } + b.pickerUpdateCh.Load() + b.mu.Lock() + if b.closed.HasFired() { + b.mu.Unlock() + return + } + switch u := update.(type) { + case balancer.State: + b.childState = u + b.ClientConn.UpdateState(balancer.State{ + ConnectivityState: b.childState.ConnectivityState, + Picker: newPicker(b.childState, &dropConfigs{ + drops: b.drops, + requestCounter: b.requestCounter, + requestCountMax: b.requestCountMax, + }, b.loadWrapper), + }) + case *LBConfig: + dc := b.handleDropAndRequestCount(u) + if dc != nil && b.childState.Picker != nil { + b.ClientConn.UpdateState(balancer.State{ + ConnectivityState: b.childState.ConnectivityState, + Picker: newPicker(b.childState, dc, b.loadWrapper), + }) + } + } + b.mu.Unlock() + case <-b.closed.Done(): + if b.cancelLoadReport != nil { + b.cancelLoadReport() + b.cancelLoadReport = nil + } + return + } + } +} diff --git a/xds/internal/balancer/clusterimpl/config.go b/xds/internal/balancer/clusterimpl/config.go new file mode 100644 index 000000000000..cfddc6fb2a1b --- /dev/null +++ b/xds/internal/balancer/clusterimpl/config.go @@ -0,0 +1,67 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package clusterimpl + +import ( + "encoding/json" + + internalserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/serviceconfig" + "google.golang.org/grpc/xds/internal/xdsclient/bootstrap" +) + +// DropConfig contains the category, and drop ratio. +type DropConfig struct { + Category string + RequestsPerMillion uint32 +} + +// LBConfig is the balancer config for cluster_impl balancer. +type LBConfig struct { + serviceconfig.LoadBalancingConfig `json:"-"` + + Cluster string `json:"cluster,omitempty"` + EDSServiceName string `json:"edsServiceName,omitempty"` + // LoadReportingServer is the LRS server to send load reports to. If not + // present, load reporting will be disabled. + LoadReportingServer *bootstrap.ServerConfig `json:"lrsLoadReportingServer,omitempty"` + MaxConcurrentRequests *uint32 `json:"maxConcurrentRequests,omitempty"` + DropCategories []DropConfig `json:"dropCategories,omitempty"` + ChildPolicy *internalserviceconfig.BalancerConfig `json:"childPolicy,omitempty"` +} + +func parseConfig(c json.RawMessage) (*LBConfig, error) { + var cfg LBConfig + if err := json.Unmarshal(c, &cfg); err != nil { + return nil, err + } + return &cfg, nil +} + +func equalDropCategories(a, b []DropConfig) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} diff --git a/xds/internal/balancer/clusterimpl/config_test.go b/xds/internal/balancer/clusterimpl/config_test.go new file mode 100644 index 000000000000..b001b8fdf0a4 --- /dev/null +++ b/xds/internal/balancer/clusterimpl/config_test.go @@ -0,0 +1,145 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package clusterimpl + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "google.golang.org/grpc/balancer" + _ "google.golang.org/grpc/balancer/roundrobin" + _ "google.golang.org/grpc/balancer/weightedtarget" + internalserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/xds/internal/xdsclient/bootstrap" +) + +const ( + testJSONConfig = `{ + "cluster": "test_cluster", + "edsServiceName": "test-eds", + "lrsLoadReportingServer": { + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ { "type": "google_default" } ] + }, + "maxConcurrentRequests": 123, + "dropCategories": [ + { + "category": "drop-1", + "requestsPerMillion": 314 + }, + { + "category": "drop-2", + "requestsPerMillion": 159 + } + ], + "childPolicy": [ + { + "weighted_target_experimental": { + "targets": { + "wt-child-1": { + "weight": 75, + "childPolicy":[{"round_robin":{}}] + }, + "wt-child-2": { + "weight": 25, + "childPolicy":[{"round_robin":{}}] + } + } + } + } + ] +}` + + wtName = "weighted_target_experimental" +) + +var ( + wtConfigParser = balancer.Get(wtName).(balancer.ConfigParser) + wtConfigJSON = `{ + "targets": { + "wt-child-1": { + "weight": 75, + "childPolicy":[{"round_robin":{}}] + }, + "wt-child-2": { + "weight": 25, + "childPolicy":[{"round_robin":{}}] + } + } +}` + + wtConfig, _ = wtConfigParser.ParseConfig([]byte(wtConfigJSON)) +) + +func TestParseConfig(t *testing.T) { + tests := []struct { + name string + js string + want *LBConfig + wantErr bool + }{ + { + name: "empty json", + js: "", + want: nil, + wantErr: true, + }, + { + name: "bad json", + js: "{", + want: nil, + wantErr: true, + }, + { + name: "OK", + js: testJSONConfig, + want: &LBConfig{ + Cluster: "test_cluster", + EDSServiceName: "test-eds", + LoadReportingServer: testLRSServerConfig, + MaxConcurrentRequests: newUint32(123), + DropCategories: []DropConfig{ + {Category: "drop-1", RequestsPerMillion: 314}, + {Category: "drop-2", RequestsPerMillion: 159}, + }, + ChildPolicy: &internalserviceconfig.BalancerConfig{ + Name: wtName, + Config: wtConfig, + }, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parseConfig([]byte(tt.js)) + if (err != nil) != tt.wantErr { + t.Fatalf("parseConfig() error = %v, wantErr %v", err, tt.wantErr) + } + if !cmp.Equal(got, tt.want, cmpopts.IgnoreFields(bootstrap.ServerConfig{}, "Creds")) { + t.Errorf("parseConfig() got unexpected result, diff: %v", cmp.Diff(got, tt.want)) + } + }) + } +} + +func newUint32(i uint32) *uint32 { + return &i +} diff --git a/xds/internal/balancer/lrs/logging.go b/xds/internal/balancer/clusterimpl/logging.go similarity index 85% rename from xds/internal/balancer/lrs/logging.go rename to xds/internal/balancer/clusterimpl/logging.go index 602dac099597..3bbd1b0d7837 100644 --- a/xds/internal/balancer/lrs/logging.go +++ b/xds/internal/balancer/clusterimpl/logging.go @@ -16,7 +16,7 @@ * */ -package lrs +package clusterimpl import ( "fmt" @@ -25,10 +25,10 @@ import ( internalgrpclog "google.golang.org/grpc/internal/grpclog" ) -const prefix = "[lrs-lb %p] " +const prefix = "[xds-cluster-impl-lb %p] " var logger = grpclog.Component("xds") -func prefixLogger(p *lrsBalancer) *internalgrpclog.PrefixLogger { +func prefixLogger(p *clusterImplBalancer) *internalgrpclog.PrefixLogger { return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p)) } diff --git a/xds/internal/balancer/clusterimpl/picker.go b/xds/internal/balancer/clusterimpl/picker.go new file mode 100644 index 000000000000..3f354424f28e --- /dev/null +++ b/xds/internal/balancer/clusterimpl/picker.go @@ -0,0 +1,190 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package clusterimpl + +import ( + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/internal/wrr" + "google.golang.org/grpc/status" + "google.golang.org/grpc/xds/internal/xdsclient" + "google.golang.org/grpc/xds/internal/xdsclient/load" + + v3orcapb "github.com/cncf/xds/go/xds/data/orca/v3" +) + +// NewRandomWRR is used when calculating drops. It's exported so that tests can +// override it. +var NewRandomWRR = wrr.NewRandom + +const million = 1000000 + +type dropper struct { + category string + w wrr.WRR +} + +// greatest common divisor (GCD) via Euclidean algorithm +func gcd(a, b uint32) uint32 { + for b != 0 { + t := b + b = a % b + a = t + } + return a +} + +func newDropper(c DropConfig) *dropper { + w := NewRandomWRR() + gcdv := gcd(c.RequestsPerMillion, million) + // Return true for RequestPerMillion, false for the rest. + w.Add(true, int64(c.RequestsPerMillion/gcdv)) + w.Add(false, int64((million-c.RequestsPerMillion)/gcdv)) + + return &dropper{ + category: c.Category, + w: w, + } +} + +func (d *dropper) drop() (ret bool) { + return d.w.Next().(bool) +} + +const ( + serverLoadCPUName = "cpu_utilization" + serverLoadMemoryName = "mem_utilization" +) + +// loadReporter wraps the methods from the loadStore that are used here. +type loadReporter interface { + CallStarted(locality string) + CallFinished(locality string, err error) + CallServerLoad(locality, name string, val float64) + CallDropped(locality string) +} + +// Picker implements RPC drop, circuit breaking drop and load reporting. +type picker struct { + drops []*dropper + s balancer.State + loadStore loadReporter + counter *xdsclient.ClusterRequestsCounter + countMax uint32 +} + +func newPicker(s balancer.State, config *dropConfigs, loadStore load.PerClusterReporter) *picker { + return &picker{ + drops: config.drops, + s: s, + loadStore: loadStore, + counter: config.requestCounter, + countMax: config.requestCountMax, + } +} + +func (d *picker) Pick(info balancer.PickInfo) (balancer.PickResult, error) { + // Don't drop unless the inner picker is READY. Similar to + // https://github.com/grpc/grpc-go/issues/2622. + if d.s.ConnectivityState == connectivity.Ready { + // Check if this RPC should be dropped by category. + for _, dp := range d.drops { + if dp.drop() { + if d.loadStore != nil { + d.loadStore.CallDropped(dp.category) + } + return balancer.PickResult{}, status.Errorf(codes.Unavailable, "RPC is dropped") + } + } + } + + // Check if this RPC should be dropped by circuit breaking. + if d.counter != nil { + if err := d.counter.StartRequest(d.countMax); err != nil { + // Drops by circuit breaking are reported with empty category. They + // will be reported only in total drops, but not in per category. + if d.loadStore != nil { + d.loadStore.CallDropped("") + } + return balancer.PickResult{}, status.Errorf(codes.Unavailable, err.Error()) + } + } + + var lIDStr string + pr, err := d.s.Picker.Pick(info) + if scw, ok := pr.SubConn.(*scWrapper); ok { + // This OK check also covers the case err!=nil, because SubConn will be + // nil. + pr.SubConn = scw.SubConn + var e error + // If locality ID isn't found in the wrapper, an empty locality ID will + // be used. + lIDStr, e = scw.localityID().ToString() + if e != nil { + logger.Infof("failed to marshal LocalityID: %#v, loads won't be reported", scw.localityID()) + } + } + + if err != nil { + if d.counter != nil { + // Release one request count if this pick fails. + d.counter.EndRequest() + } + return pr, err + } + + if d.loadStore != nil { + d.loadStore.CallStarted(lIDStr) + oldDone := pr.Done + pr.Done = func(info balancer.DoneInfo) { + if oldDone != nil { + oldDone(info) + } + d.loadStore.CallFinished(lIDStr, info.Err) + + load, ok := info.ServerLoad.(*v3orcapb.OrcaLoadReport) + if !ok || load == nil { + return + } + d.loadStore.CallServerLoad(lIDStr, serverLoadCPUName, load.CpuUtilization) + d.loadStore.CallServerLoad(lIDStr, serverLoadMemoryName, load.MemUtilization) + for n, c := range load.RequestCost { + d.loadStore.CallServerLoad(lIDStr, n, c) + } + for n, c := range load.Utilization { + d.loadStore.CallServerLoad(lIDStr, n, c) + } + } + } + + if d.counter != nil { + // Update Done() so that when the RPC finishes, the request count will + // be released. + oldDone := pr.Done + pr.Done = func(doneInfo balancer.DoneInfo) { + d.counter.EndRequest() + if oldDone != nil { + oldDone(doneInfo) + } + } + } + + return pr, err +} diff --git a/xds/internal/balancer/clusterimpl/tests/balancer_test.go b/xds/internal/balancer/clusterimpl/tests/balancer_test.go new file mode 100644 index 000000000000..1cb8492949a0 --- /dev/null +++ b/xds/internal/balancer/clusterimpl/tests/balancer_test.go @@ -0,0 +1,156 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package clusterimpl_test + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/status" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + + _ "google.golang.org/grpc/xds" +) + +const ( + defaultTestTimeout = 5 * time.Second + defaultTestShortTimeout = 100 * time.Millisecond +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// TestConfigUpdateWithSameLoadReportingServerConfig tests the scenario where +// the clusterimpl LB policy receives a config update with no change in the load +// reporting server configuration. The test verifies that the existing load +// repoting stream is not terminated and that a new load reporting stream is not +// created. +func (s) TestConfigUpdateWithSameLoadReportingServerConfig(t *testing.T) { + // Create an xDS management server that serves ADS and LRS requests. + opts := e2e.ManagementServerOptions{SupportLoadReportingService: true} + mgmtServer, nodeID, _, resolver, mgmtServerCleanup := e2e.SetupManagementServer(t, opts) + defer mgmtServerCleanup() + + // Start a server backend exposing the test service. + server := stubserver.StartTestService(t, nil) + defer server.Stop() + + // Configure the xDS management server with default resources. Override the + // default cluster to include an LRS server config pointing to self. + const serviceName = "my-test-xds-service" + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: "localhost", + Port: testutils.ParsePort(t, server.Address), + SecLevel: e2e.SecurityLevelNone, + }) + resources.Clusters[0].LrsServer = &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Self{ + Self: &v3corepb.SelfConfigSource{}, + }, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create a ClientConn and make a successful RPC. + cc, err := grpc.Dial(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(resolver)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("rpc EmptyCall() failed: %v", err) + } + + // Ensure that an LRS stream is created. + if _, err := mgmtServer.LRSServer.LRSStreamOpenChan.Receive(ctx); err != nil { + t.Fatalf("Failure when waiting for an LRS stream to be opened: %v", err) + } + + // Configure a new resource on the management server with drop config that + // drops all RPCs, but with no change in the load reporting server config. + resources.Endpoints = []*v3endpointpb.ClusterLoadAssignment{ + e2e.EndpointResourceWithOptions(e2e.EndpointOptions{ + ClusterName: "endpoints-" + serviceName, + Host: "localhost", + Localities: []e2e.LocalityOptions{ + { + Backends: []e2e.BackendOptions{{Port: testutils.ParsePort(t, server.Address)}}, + Weight: 1, + }, + }, + DropPercents: map[string]int{"test-drop-everything": 100}, + }), + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Repeatedly send RPCs until we sees that they are getting dropped, or the + // test context deadline expires. The former indicates that new config with + // drops has been applied. + for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) { + _, err := client.EmptyCall(ctx, &testpb.Empty{}) + if err != nil && status.Code(err) == codes.Unavailable && strings.Contains(err.Error(), "RPC is dropped") { + break + } + } + if ctx.Err() != nil { + t.Fatalf("Timeout when waiting for RPCs to be dropped after config update") + } + + // Ensure that the old LRS stream is not closed. + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := mgmtServer.LRSServer.LRSStreamCloseChan.Receive(sCtx); err == nil { + t.Fatal("LRS stream closed when expected not to") + } + + // Also ensure that a new LRS stream is not created. + sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := mgmtServer.LRSServer.LRSStreamOpenChan.Receive(sCtx); err == nil { + t.Fatal("New LRS stream created when expected not to") + } +} diff --git a/xds/internal/balancer/xdsrouting/balancerstateaggregator.go b/xds/internal/balancer/clustermanager/balancerstateaggregator.go similarity index 61% rename from xds/internal/balancer/xdsrouting/balancerstateaggregator.go rename to xds/internal/balancer/clustermanager/balancerstateaggregator.go index 9f6250e5c804..4b971a3e241b 100644 --- a/xds/internal/balancer/xdsrouting/balancerstateaggregator.go +++ b/xds/internal/balancer/clustermanager/balancerstateaggregator.go @@ -16,7 +16,7 @@ * */ -package xdsrouting +package clustermanager import ( "fmt" @@ -26,7 +26,6 @@ import ( "google.golang.org/grpc/balancer/base" "google.golang.org/grpc/connectivity" "google.golang.org/grpc/internal/grpclog" - "google.golang.org/grpc/xds/internal" ) type subBalancerState struct { @@ -48,8 +47,6 @@ type balancerStateAggregator struct { logger *grpclog.PrefixLogger mu sync.Mutex - // routes, one for each matcher. - routes []route // If started is false, no updates should be sent to the parent cc. A closed // sub-balancer could still send pickers to this aggregator. This makes sure // that no updates will be forwarded to parent when the whole balancer group @@ -59,42 +56,47 @@ type balancerStateAggregator struct { // started. // // If an ID is not in map, it's either removed or never added. - idToPickerState map[internal.LocalityID]*subBalancerState + idToPickerState map[string]*subBalancerState + // Set when UpdateState call propagation is paused. + pauseUpdateState bool + // Set when UpdateState call propagation is paused and an UpdateState call + // is suppressed. + needUpdateStateOnResume bool } func newBalancerStateAggregator(cc balancer.ClientConn, logger *grpclog.PrefixLogger) *balancerStateAggregator { return &balancerStateAggregator{ cc: cc, logger: logger, - idToPickerState: make(map[internal.LocalityID]*subBalancerState), + idToPickerState: make(map[string]*subBalancerState), } } // Start starts the aggregator. It can be called after Close to restart the // aggretator. -func (rbsa *balancerStateAggregator) start() { - rbsa.mu.Lock() - defer rbsa.mu.Unlock() - rbsa.started = true +func (bsa *balancerStateAggregator) start() { + bsa.mu.Lock() + defer bsa.mu.Unlock() + bsa.started = true } // Close closes the aggregator. When the aggregator is closed, it won't call // parent ClientConn to update balancer state. -func (rbsa *balancerStateAggregator) close() { - rbsa.mu.Lock() - defer rbsa.mu.Unlock() - rbsa.started = false - rbsa.clearStates() +func (bsa *balancerStateAggregator) close() { + bsa.mu.Lock() + defer bsa.mu.Unlock() + bsa.started = false + bsa.clearStates() } // add adds a sub-balancer state with weight. It adds a place holder, and waits // for the real sub-balancer to update state. // -// This is called when there's a new action. -func (rbsa *balancerStateAggregator) add(id internal.LocalityID) { - rbsa.mu.Lock() - defer rbsa.mu.Unlock() - rbsa.idToPickerState[id] = &subBalancerState{ +// This is called when there's a new child. +func (bsa *balancerStateAggregator) add(id string) { + bsa.mu.Lock() + defer bsa.mu.Unlock() + bsa.idToPickerState[id] = &subBalancerState{ // Start everything in CONNECTING, so if one of the sub-balancers // reports TransientFailure, the RPCs will still wait for the other // sub-balancers. @@ -109,35 +111,47 @@ func (rbsa *balancerStateAggregator) add(id internal.LocalityID) { // remove removes the sub-balancer state. Future updates from this sub-balancer, // if any, will be ignored. // -// This is called when an action is removed. -func (rbsa *balancerStateAggregator) remove(id internal.LocalityID) { - rbsa.mu.Lock() - defer rbsa.mu.Unlock() - if _, ok := rbsa.idToPickerState[id]; !ok { +// This is called when a child is removed. +func (bsa *balancerStateAggregator) remove(id string) { + bsa.mu.Lock() + defer bsa.mu.Unlock() + if _, ok := bsa.idToPickerState[id]; !ok { return } // Remove id and picker from picker map. This also results in future updates // for this ID to be ignored. - delete(rbsa.idToPickerState, id) + delete(bsa.idToPickerState, id) } -// updateRoutes updates the routes. Note that it doesn't trigger an update to -// the parent ClientConn. The caller should decide when it's necessary, and call -// buildAndUpdate. -func (rbsa *balancerStateAggregator) updateRoutes(newRoutes []route) { - rbsa.mu.Lock() - defer rbsa.mu.Unlock() - rbsa.routes = newRoutes +// pauseStateUpdates causes UpdateState calls to not propagate to the parent +// ClientConn. The last state will be remembered and propagated when +// ResumeStateUpdates is called. +func (bsa *balancerStateAggregator) pauseStateUpdates() { + bsa.mu.Lock() + defer bsa.mu.Unlock() + bsa.pauseUpdateState = true + bsa.needUpdateStateOnResume = false +} + +// resumeStateUpdates will resume propagating UpdateState calls to the parent, +// and call UpdateState on the parent if any UpdateState call was suppressed. +func (bsa *balancerStateAggregator) resumeStateUpdates() { + bsa.mu.Lock() + defer bsa.mu.Unlock() + bsa.pauseUpdateState = false + if bsa.needUpdateStateOnResume { + bsa.cc.UpdateState(bsa.build()) + } } // UpdateState is called to report a balancer state change from sub-balancer. // It's usually called by the balancer group. // // It calls parent ClientConn's UpdateState with the new aggregated state. -func (rbsa *balancerStateAggregator) UpdateState(id internal.LocalityID, state balancer.State) { - rbsa.mu.Lock() - defer rbsa.mu.Unlock() - pickerSt, ok := rbsa.idToPickerState[id] +func (bsa *balancerStateAggregator) UpdateState(id string, state balancer.State) { + bsa.mu.Lock() + defer bsa.mu.Unlock() + pickerSt, ok := bsa.idToPickerState[id] if !ok { // All state starts with an entry in pickStateMap. If ID is not in map, // it's either removed, or never existed. @@ -152,18 +166,24 @@ func (rbsa *balancerStateAggregator) UpdateState(id internal.LocalityID, state b } pickerSt.state = state - if !rbsa.started { + if !bsa.started { + return + } + if bsa.pauseUpdateState { + // If updates are paused, do not call UpdateState, but remember that we + // need to call it when they are resumed. + bsa.needUpdateStateOnResume = true return } - rbsa.cc.UpdateState(rbsa.build()) + bsa.cc.UpdateState(bsa.build()) } // clearState Reset everything to init state (Connecting) but keep the entry in // map (to keep the weight). // -// Caller must hold rbsa.mu. -func (rbsa *balancerStateAggregator) clearStates() { - for _, pState := range rbsa.idToPickerState { +// Caller must hold bsa.mu. +func (bsa *balancerStateAggregator) clearStates() { + for _, pState := range bsa.idToPickerState { pState.state = balancer.State{ ConnectivityState: connectivity.Connecting, Picker: base.NewErrPicker(balancer.ErrNoSubConnAvailable), @@ -174,19 +194,25 @@ func (rbsa *balancerStateAggregator) clearStates() { // buildAndUpdate combines the sub-state from each sub-balancer into one state, // and update it to parent ClientConn. -func (rbsa *balancerStateAggregator) buildAndUpdate() { - rbsa.mu.Lock() - defer rbsa.mu.Unlock() - if !rbsa.started { +func (bsa *balancerStateAggregator) buildAndUpdate() { + bsa.mu.Lock() + defer bsa.mu.Unlock() + if !bsa.started { + return + } + if bsa.pauseUpdateState { + // If updates are paused, do not call UpdateState, but remember that we + // need to call it when they are resumed. + bsa.needUpdateStateOnResume = true return } - rbsa.cc.UpdateState(rbsa.build()) + bsa.cc.UpdateState(bsa.build()) } -// build combines sub-states into one. The picker will do routing pick. +// build combines sub-states into one. The picker will do a child pick. // -// Caller must hold rbsa.mu. -func (rbsa *balancerStateAggregator) build() balancer.State { +// Caller must hold bsa.mu. +func (bsa *balancerStateAggregator) build() balancer.State { // TODO: the majority of this function (and UpdateState) is exactly the same // as weighted_target's state aggregator. Try to make a general utility // function/struct to handle the logic. @@ -195,13 +221,18 @@ func (rbsa *balancerStateAggregator) build() balancer.State { // handling the special connecting after ready, as in UpdateState(). Then a // function to calculate the aggregated connectivity state as in this // function. - var readyN, connectingN int - for _, ps := range rbsa.idToPickerState { + // + // TODO: use balancer.ConnectivityStateEvaluator to calculate the aggregated + // state. + var readyN, connectingN, idleN int + for _, ps := range bsa.idToPickerState { switch ps.stateToAggregate { case connectivity.Ready: readyN++ case connectivity.Connecting: connectingN++ + case connectivity.Idle: + idleN++ } } var aggregatedState connectivity.State @@ -210,18 +241,20 @@ func (rbsa *balancerStateAggregator) build() balancer.State { aggregatedState = connectivity.Ready case connectingN > 0: aggregatedState = connectivity.Connecting + case idleN > 0: + aggregatedState = connectivity.Idle default: aggregatedState = connectivity.TransientFailure } // The picker's return error might not be consistent with the - // aggregatedState. Because for routing, we want to always build picker with - // all sub-pickers (not even ready sub-pickers), so even if the overall - // state is Ready, pick for certain RPCs can behave like Connecting or - // TransientFailure. - rbsa.logger.Infof("Child pickers with routes: %s, actions: %+v", rbsa.routes, rbsa.idToPickerState) + // aggregatedState. Because for this LB policy, we want to always build + // picker with all sub-pickers (not only ready sub-pickers), so even if the + // overall state is Ready, pick for certain RPCs can behave like Connecting + // or TransientFailure. + bsa.logger.Infof("Child pickers: %+v", bsa.idToPickerState) return balancer.State{ ConnectivityState: aggregatedState, - Picker: newPickerGroup(rbsa.routes, rbsa.idToPickerState), + Picker: newPickerGroup(bsa.idToPickerState), } } diff --git a/xds/internal/balancer/clustermanager/clustermanager.go b/xds/internal/balancer/clustermanager/clustermanager.go new file mode 100644 index 000000000000..db8332b90eac --- /dev/null +++ b/xds/internal/balancer/clustermanager/clustermanager.go @@ -0,0 +1,163 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package clustermanager implements the cluster manager LB policy for xds. +package clustermanager + +import ( + "encoding/json" + "fmt" + "time" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal/balancergroup" + internalgrpclog "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/hierarchy" + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" +) + +const balancerName = "xds_cluster_manager_experimental" + +func init() { + balancer.Register(bb{}) +} + +type bb struct{} + +func (bb) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { + b := &bal{} + b.logger = prefixLogger(b) + b.stateAggregator = newBalancerStateAggregator(cc, b.logger) + b.stateAggregator.start() + b.bg = balancergroup.New(balancergroup.Options{ + CC: cc, + BuildOpts: opts, + StateAggregator: b.stateAggregator, + Logger: b.logger, + SubBalancerCloseTimeout: time.Duration(0), // Disable caching of removed child policies + }) + b.bg.Start() + b.logger.Infof("Created") + return b +} + +func (bb) Name() string { + return balancerName +} + +func (bb) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + return parseConfig(c) +} + +type bal struct { + logger *internalgrpclog.PrefixLogger + + // TODO: make this package not dependent on xds specific code. Same as for + // weighted target balancer. + bg *balancergroup.BalancerGroup + stateAggregator *balancerStateAggregator + + children map[string]childConfig +} + +func (b *bal) updateChildren(s balancer.ClientConnState, newConfig *lbConfig) { + update := false + addressesSplit := hierarchy.Group(s.ResolverState.Addresses) + + // Remove sub-pickers and sub-balancers that are not in the new cluster list. + for name := range b.children { + if _, ok := newConfig.Children[name]; !ok { + b.stateAggregator.remove(name) + b.bg.Remove(name) + update = true + } + } + + // For sub-balancers in the new cluster list, + // - add to balancer group if it's new, + // - forward the address/balancer config update. + for name, newT := range newConfig.Children { + if _, ok := b.children[name]; !ok { + // If this is a new sub-balancer, add it to the picker map. + b.stateAggregator.add(name) + // Then add to the balancer group. + b.bg.Add(name, balancer.Get(newT.ChildPolicy.Name)) + } else { + // Already present, check for type change and if so send down a new builder. + if newT.ChildPolicy.Name != b.children[name].ChildPolicy.Name { + b.bg.UpdateBuilder(name, balancer.Get(newT.ChildPolicy.Name)) + } + } + // TODO: handle error? How to aggregate errors and return? + _ = b.bg.UpdateClientConnState(name, balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: addressesSplit[name], + ServiceConfig: s.ResolverState.ServiceConfig, + Attributes: s.ResolverState.Attributes, + }, + BalancerConfig: newT.ChildPolicy.Config, + }) + } + + b.children = newConfig.Children + if update { + b.stateAggregator.buildAndUpdate() + } +} + +func (b *bal) UpdateClientConnState(s balancer.ClientConnState) error { + newConfig, ok := s.BalancerConfig.(*lbConfig) + if !ok { + return fmt.Errorf("unexpected balancer config with type: %T", s.BalancerConfig) + } + b.logger.Infof("update with config %+v, resolver state %+v", pretty.ToJSON(s.BalancerConfig), s.ResolverState) + + b.stateAggregator.pauseStateUpdates() + defer b.stateAggregator.resumeStateUpdates() + b.updateChildren(s, newConfig) + return nil +} + +func (b *bal) ResolverError(err error) { + b.bg.ResolverError(err) +} + +func (b *bal) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { + b.logger.Errorf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, state) +} + +func (b *bal) Close() { + b.stateAggregator.close() + b.bg.Close() + b.logger.Infof("Shutdown") +} + +func (b *bal) ExitIdle() { + b.bg.ExitIdle() +} + +const prefix = "[xds-cluster-manager-lb %p] " + +var logger = grpclog.Component("xds") + +func prefixLogger(p *bal) *internalgrpclog.PrefixLogger { + return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p)) +} diff --git a/xds/internal/balancer/clustermanager/clustermanager_test.go b/xds/internal/balancer/clustermanager/clustermanager_test.go new file mode 100644 index 000000000000..39e32d60993d --- /dev/null +++ b/xds/internal/balancer/clustermanager/clustermanager_test.go @@ -0,0 +1,763 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package clustermanager + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/balancer/stub" + "google.golang.org/grpc/internal/channelz" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/hierarchy" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/status" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +const ( + defaultTestTimeout = 5 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond + testBackendAddrsCount = 12 +) + +var testBackendAddrStrs []string + +func init() { + for i := 0; i < testBackendAddrsCount; i++ { + testBackendAddrStrs = append(testBackendAddrStrs, fmt.Sprintf("%d.%d.%d.%d:%d", i, i, i, i, i)) + } +} + +func testPick(t *testing.T, p balancer.Picker, info balancer.PickInfo, wantSC balancer.SubConn, wantErr error) { + t.Helper() + for i := 0; i < 5; i++ { + gotSCSt, err := p.Pick(info) + if fmt.Sprint(err) != fmt.Sprint(wantErr) { + t.Fatalf("picker.Pick(%+v), got error %v, want %v", info, err, wantErr) + } + if gotSCSt.SubConn != wantSC { + t.Fatalf("picker.Pick(%+v), got %v, want SubConn=%v", info, gotSCSt, wantSC) + } + } +} + +func TestClusterPicks(t *testing.T) { + cc := testutils.NewTestClientConn(t) + builder := balancer.Get(balancerName) + parser := builder.(balancer.ConfigParser) + bal := builder.Build(cc, balancer.BuildOptions{}) + + configJSON1 := `{ +"children": { + "cds:cluster_1":{ "childPolicy": [{"round_robin":""}] }, + "cds:cluster_2":{ "childPolicy": [{"round_robin":""}] } +} +}` + config1, err := parser.ParseConfig([]byte(configJSON1)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + + // Send the config, and an address with hierarchy path ["cluster_1"]. + wantAddrs := []resolver.Address{ + {Addr: testBackendAddrStrs[0], BalancerAttributes: nil}, + {Addr: testBackendAddrStrs[1], BalancerAttributes: nil}, + } + if err := bal.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Addresses: []resolver.Address{ + hierarchy.Set(wantAddrs[0], []string{"cds:cluster_1"}), + hierarchy.Set(wantAddrs[1], []string{"cds:cluster_2"}), + }}, + BalancerConfig: config1, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + m1 := make(map[resolver.Address]balancer.SubConn) + // Verify that a subconn is created with the address, and the hierarchy path + // in the address is cleared. + for range wantAddrs { + addrs := <-cc.NewSubConnAddrsCh + if len(hierarchy.Get(addrs[0])) != 0 { + t.Fatalf("NewSubConn with address %+v, attrs %+v, want address with hierarchy cleared", addrs[0], addrs[0].BalancerAttributes) + } + sc := <-cc.NewSubConnCh + // Clear the attributes before adding to map. + addrs[0].BalancerAttributes = nil + m1[addrs[0]] = sc + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + } + + p1 := <-cc.NewPickerCh + for _, tt := range []struct { + pickInfo balancer.PickInfo + wantSC balancer.SubConn + wantErr error + }{ + { + pickInfo: balancer.PickInfo{ + Ctx: SetPickedCluster(context.Background(), "cds:cluster_1"), + }, + wantSC: m1[wantAddrs[0]], + }, + { + pickInfo: balancer.PickInfo{ + Ctx: SetPickedCluster(context.Background(), "cds:cluster_2"), + }, + wantSC: m1[wantAddrs[1]], + }, + { + pickInfo: balancer.PickInfo{ + Ctx: SetPickedCluster(context.Background(), "notacluster"), + }, + wantErr: status.Errorf(codes.Unavailable, `unknown cluster selected for RPC: "notacluster"`), + }, + } { + testPick(t, p1, tt.pickInfo, tt.wantSC, tt.wantErr) + } +} + +// TestConfigUpdateAddCluster covers the cases the balancer receives config +// update with extra clusters. +func TestConfigUpdateAddCluster(t *testing.T) { + cc := testutils.NewTestClientConn(t) + builder := balancer.Get(balancerName) + parser := builder.(balancer.ConfigParser) + bal := builder.Build(cc, balancer.BuildOptions{}) + + configJSON1 := `{ +"children": { + "cds:cluster_1":{ "childPolicy": [{"round_robin":""}] }, + "cds:cluster_2":{ "childPolicy": [{"round_robin":""}] } +} +}` + config1, err := parser.ParseConfig([]byte(configJSON1)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + + // Send the config, and an address with hierarchy path ["cluster_1"]. + wantAddrs := []resolver.Address{ + {Addr: testBackendAddrStrs[0], BalancerAttributes: nil}, + {Addr: testBackendAddrStrs[1], BalancerAttributes: nil}, + } + if err := bal.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Addresses: []resolver.Address{ + hierarchy.Set(wantAddrs[0], []string{"cds:cluster_1"}), + hierarchy.Set(wantAddrs[1], []string{"cds:cluster_2"}), + }}, + BalancerConfig: config1, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + m1 := make(map[resolver.Address]balancer.SubConn) + // Verify that a subconn is created with the address, and the hierarchy path + // in the address is cleared. + for range wantAddrs { + addrs := <-cc.NewSubConnAddrsCh + if len(hierarchy.Get(addrs[0])) != 0 { + t.Fatalf("NewSubConn with address %+v, attrs %+v, want address with hierarchy cleared", addrs[0], addrs[0].BalancerAttributes) + } + sc := <-cc.NewSubConnCh + // Clear the attributes before adding to map. + addrs[0].BalancerAttributes = nil + m1[addrs[0]] = sc + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + } + + p1 := <-cc.NewPickerCh + for _, tt := range []struct { + pickInfo balancer.PickInfo + wantSC balancer.SubConn + wantErr error + }{ + { + pickInfo: balancer.PickInfo{ + Ctx: SetPickedCluster(context.Background(), "cds:cluster_1"), + }, + wantSC: m1[wantAddrs[0]], + }, + { + pickInfo: balancer.PickInfo{ + Ctx: SetPickedCluster(context.Background(), "cds:cluster_2"), + }, + wantSC: m1[wantAddrs[1]], + }, + { + pickInfo: balancer.PickInfo{ + Ctx: SetPickedCluster(context.Background(), "cds:notacluster"), + }, + wantErr: status.Errorf(codes.Unavailable, `unknown cluster selected for RPC: "cds:notacluster"`), + }, + } { + testPick(t, p1, tt.pickInfo, tt.wantSC, tt.wantErr) + } + + // A config update with different routes, and different actions. Expect a + // new subconn and a picker update. + configJSON2 := `{ +"children": { + "cds:cluster_1":{ "childPolicy": [{"round_robin":""}] }, + "cds:cluster_2":{ "childPolicy": [{"round_robin":""}] }, + "cds:cluster_3":{ "childPolicy": [{"round_robin":""}] } +} +}` + config2, err := parser.ParseConfig([]byte(configJSON2)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + wantAddrs = append(wantAddrs, resolver.Address{Addr: testBackendAddrStrs[2], BalancerAttributes: nil}) + if err := bal.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Addresses: []resolver.Address{ + hierarchy.Set(wantAddrs[0], []string{"cds:cluster_1"}), + hierarchy.Set(wantAddrs[1], []string{"cds:cluster_2"}), + hierarchy.Set(wantAddrs[2], []string{"cds:cluster_3"}), + }}, + BalancerConfig: config2, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // Expect exactly one new subconn. + addrs := <-cc.NewSubConnAddrsCh + if len(hierarchy.Get(addrs[0])) != 0 { + t.Fatalf("NewSubConn with address %+v, attrs %+v, want address with hierarchy cleared", addrs[0], addrs[0].BalancerAttributes) + } + sc := <-cc.NewSubConnCh + // Clear the attributes before adding to map. + addrs[0].BalancerAttributes = nil + m1[addrs[0]] = sc + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // Should have no more newSubConn. + select { + case <-time.After(time.Millisecond * 500): + case <-cc.NewSubConnCh: + addrs := <-cc.NewSubConnAddrsCh + t.Fatalf("unexpected NewSubConn with address %v", addrs) + } + + p2 := <-cc.NewPickerCh + for _, tt := range []struct { + pickInfo balancer.PickInfo + wantSC balancer.SubConn + wantErr error + }{ + { + pickInfo: balancer.PickInfo{ + Ctx: SetPickedCluster(context.Background(), "cds:cluster_1"), + }, + wantSC: m1[wantAddrs[0]], + }, + { + pickInfo: balancer.PickInfo{ + Ctx: SetPickedCluster(context.Background(), "cds:cluster_2"), + }, + wantSC: m1[wantAddrs[1]], + }, + { + pickInfo: balancer.PickInfo{ + Ctx: SetPickedCluster(context.Background(), "cds:cluster_3"), + }, + wantSC: m1[wantAddrs[2]], + }, + { + pickInfo: balancer.PickInfo{ + Ctx: SetPickedCluster(context.Background(), "cds:notacluster"), + }, + wantErr: status.Errorf(codes.Unavailable, `unknown cluster selected for RPC: "cds:notacluster"`), + }, + } { + testPick(t, p2, tt.pickInfo, tt.wantSC, tt.wantErr) + } +} + +// TestRoutingConfigUpdateDeleteAll covers the cases the balancer receives +// config update with no clusters. Pick should fail with details in error. +func TestRoutingConfigUpdateDeleteAll(t *testing.T) { + cc := testutils.NewTestClientConn(t) + builder := balancer.Get(balancerName) + parser := builder.(balancer.ConfigParser) + bal := builder.Build(cc, balancer.BuildOptions{}) + + configJSON1 := `{ +"children": { + "cds:cluster_1":{ "childPolicy": [{"round_robin":""}] }, + "cds:cluster_2":{ "childPolicy": [{"round_robin":""}] } +} +}` + config1, err := parser.ParseConfig([]byte(configJSON1)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + + // Send the config, and an address with hierarchy path ["cluster_1"]. + wantAddrs := []resolver.Address{ + {Addr: testBackendAddrStrs[0], BalancerAttributes: nil}, + {Addr: testBackendAddrStrs[1], BalancerAttributes: nil}, + } + if err := bal.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Addresses: []resolver.Address{ + hierarchy.Set(wantAddrs[0], []string{"cds:cluster_1"}), + hierarchy.Set(wantAddrs[1], []string{"cds:cluster_2"}), + }}, + BalancerConfig: config1, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + m1 := make(map[resolver.Address]balancer.SubConn) + // Verify that a subconn is created with the address, and the hierarchy path + // in the address is cleared. + for range wantAddrs { + addrs := <-cc.NewSubConnAddrsCh + if len(hierarchy.Get(addrs[0])) != 0 { + t.Fatalf("NewSubConn with address %+v, attrs %+v, want address with hierarchy cleared", addrs[0], addrs[0].BalancerAttributes) + } + sc := <-cc.NewSubConnCh + // Clear the attributes before adding to map. + addrs[0].BalancerAttributes = nil + m1[addrs[0]] = sc + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + } + + p1 := <-cc.NewPickerCh + for _, tt := range []struct { + pickInfo balancer.PickInfo + wantSC balancer.SubConn + wantErr error + }{ + { + pickInfo: balancer.PickInfo{ + Ctx: SetPickedCluster(context.Background(), "cds:cluster_1"), + }, + wantSC: m1[wantAddrs[0]], + }, + { + pickInfo: balancer.PickInfo{ + Ctx: SetPickedCluster(context.Background(), "cds:cluster_2"), + }, + wantSC: m1[wantAddrs[1]], + }, + { + pickInfo: balancer.PickInfo{ + Ctx: SetPickedCluster(context.Background(), "cds:notacluster"), + }, + wantErr: status.Errorf(codes.Unavailable, `unknown cluster selected for RPC: "cds:notacluster"`), + }, + } { + testPick(t, p1, tt.pickInfo, tt.wantSC, tt.wantErr) + } + + // A config update with no clusters. + configJSON2 := `{}` + config2, err := parser.ParseConfig([]byte(configJSON2)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + if err := bal.UpdateClientConnState(balancer.ClientConnState{ + BalancerConfig: config2, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // Expect two removed subconns. + for range wantAddrs { + select { + case <-time.After(time.Millisecond * 500): + t.Fatalf("timeout waiting for remove subconn") + case <-cc.ShutdownSubConnCh: + } + } + + p2 := <-cc.NewPickerCh + for i := 0; i < 5; i++ { + gotSCSt, err := p2.Pick(balancer.PickInfo{Ctx: SetPickedCluster(context.Background(), "cds:notacluster")}) + if fmt.Sprint(err) != status.Errorf(codes.Unavailable, `unknown cluster selected for RPC: "cds:notacluster"`).Error() { + t.Fatalf("picker.Pick, got %v, %v, want error %v", gotSCSt, err, `unknown cluster selected for RPC: "cds:notacluster"`) + } + } + + // Resend the previous config with clusters + if err := bal.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Addresses: []resolver.Address{ + hierarchy.Set(wantAddrs[0], []string{"cds:cluster_1"}), + hierarchy.Set(wantAddrs[1], []string{"cds:cluster_2"}), + }}, + BalancerConfig: config1, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + m2 := make(map[resolver.Address]balancer.SubConn) + // Verify that a subconn is created with the address, and the hierarchy path + // in the address is cleared. + for range wantAddrs { + addrs := <-cc.NewSubConnAddrsCh + if len(hierarchy.Get(addrs[0])) != 0 { + t.Fatalf("NewSubConn with address %+v, attrs %+v, want address with hierarchy cleared", addrs[0], addrs[0].BalancerAttributes) + } + sc := <-cc.NewSubConnCh + // Clear the attributes before adding to map. + addrs[0].BalancerAttributes = nil + m2[addrs[0]] = sc + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + } + + p3 := <-cc.NewPickerCh + for _, tt := range []struct { + pickInfo balancer.PickInfo + wantSC balancer.SubConn + wantErr error + }{ + { + pickInfo: balancer.PickInfo{ + Ctx: SetPickedCluster(context.Background(), "cds:cluster_1"), + }, + wantSC: m2[wantAddrs[0]], + }, + { + pickInfo: balancer.PickInfo{ + Ctx: SetPickedCluster(context.Background(), "cds:cluster_2"), + }, + wantSC: m2[wantAddrs[1]], + }, + { + pickInfo: balancer.PickInfo{ + Ctx: SetPickedCluster(context.Background(), "cds:notacluster"), + }, + wantErr: status.Errorf(codes.Unavailable, `unknown cluster selected for RPC: "cds:notacluster"`), + }, + } { + testPick(t, p3, tt.pickInfo, tt.wantSC, tt.wantErr) + } +} + +func TestClusterManagerForwardsBalancerBuildOptions(t *testing.T) { + const ( + userAgent = "ua" + defaultTestTimeout = 1 * time.Second + ) + + // Setup the stub balancer such that we can read the build options passed to + // it in the UpdateClientConnState method. + ccsCh := testutils.NewChannel() + bOpts := balancer.BuildOptions{ + DialCreds: insecure.NewCredentials(), + ChannelzParentID: channelz.NewIdentifierForTesting(channelz.RefChannel, 1234, nil), + CustomUserAgent: userAgent, + } + stub.Register(t.Name(), stub.BalancerFuncs{ + UpdateClientConnState: func(bd *stub.BalancerData, _ balancer.ClientConnState) error { + if !cmp.Equal(bd.BuildOptions, bOpts) { + err := fmt.Errorf("buildOptions in child balancer: %v, want %v", bd, bOpts) + ccsCh.Send(err) + return err + } + ccsCh.Send(nil) + return nil + }, + }) + + cc := testutils.NewTestClientConn(t) + builder := balancer.Get(balancerName) + parser := builder.(balancer.ConfigParser) + bal := builder.Build(cc, bOpts) + + configJSON1 := fmt.Sprintf(`{ +"children": { + "cds:cluster_1":{ "childPolicy": [{"%s":""}] } +} +}`, t.Name()) + config1, err := parser.ParseConfig([]byte(configJSON1)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + + if err := bal.UpdateClientConnState(balancer.ClientConnState{BalancerConfig: config1}); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + v, err := ccsCh.Receive(ctx) + if err != nil { + t.Fatalf("timed out waiting for UpdateClientConnState result: %v", err) + } + if v != nil { + t.Fatal(v) + } +} + +const initIdleBalancerName = "test-init-Idle-balancer" + +var errTestInitIdle = fmt.Errorf("init Idle balancer error 0") + +func init() { + stub.Register(initIdleBalancerName, stub.BalancerFuncs{ + UpdateClientConnState: func(bd *stub.BalancerData, opts balancer.ClientConnState) error { + sc, err := bd.ClientConn.NewSubConn(opts.ResolverState.Addresses, balancer.NewSubConnOptions{ + StateListener: func(state balancer.SubConnState) { + err := fmt.Errorf("wrong picker error") + if state.ConnectivityState == connectivity.Idle { + err = errTestInitIdle + } + bd.ClientConn.UpdateState(balancer.State{ + ConnectivityState: state.ConnectivityState, + Picker: &testutils.TestConstPicker{Err: err}, + }) + }, + }) + if err != nil { + return err + } + sc.Connect() + return nil + }, + }) +} + +// TestInitialIdle covers the case that if the child reports Idle, the overall +// state will be Idle. +func TestInitialIdle(t *testing.T) { + cc := testutils.NewTestClientConn(t) + builder := balancer.Get(balancerName) + parser := builder.(balancer.ConfigParser) + bal := builder.Build(cc, balancer.BuildOptions{}) + + configJSON1 := `{ +"children": { + "cds:cluster_1":{ "childPolicy": [{"test-init-Idle-balancer":""}] } +} +}` + config1, err := parser.ParseConfig([]byte(configJSON1)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + + // Send the config, and an address with hierarchy path ["cluster_1"]. + wantAddrs := []resolver.Address{ + {Addr: testBackendAddrStrs[0], BalancerAttributes: nil}, + } + if err := bal.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Addresses: []resolver.Address{ + hierarchy.Set(wantAddrs[0], []string{"cds:cluster_1"}), + }}, + BalancerConfig: config1, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // Verify that a subconn is created with the address, and the hierarchy path + // in the address is cleared. + for range wantAddrs { + sc := <-cc.NewSubConnCh + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Idle}) + } + + if state1 := <-cc.NewStateCh; state1 != connectivity.Idle { + t.Fatalf("Received aggregated state: %v, want Idle", state1) + } +} + +// TestClusterGracefulSwitch tests the graceful switch functionality for a child +// of the cluster manager. At first, the child is configured as a round robin +// load balancer, and thus should behave accordingly. The test then gracefully +// switches this child to a pick first load balancer. Once that balancer updates +// it's state and completes the graceful switch process the new picker should +// reflect this change. +func TestClusterGracefulSwitch(t *testing.T) { + cc := testutils.NewTestClientConn(t) + builder := balancer.Get(balancerName) + parser := builder.(balancer.ConfigParser) + bal := builder.Build(cc, balancer.BuildOptions{}) + + configJSON1 := `{ +"children": { + "csp:cluster":{ "childPolicy": [{"round_robin":""}] } +} +}` + config1, err := parser.ParseConfig([]byte(configJSON1)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + wantAddrs := []resolver.Address{ + {Addr: testBackendAddrStrs[0], BalancerAttributes: nil}, + {Addr: testBackendAddrStrs[1], BalancerAttributes: nil}, + } + if err := bal.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Addresses: []resolver.Address{ + hierarchy.Set(wantAddrs[0], []string{"csp:cluster"}), + }}, + BalancerConfig: config1, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + sc1 := <-cc.NewSubConnCh + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + p1 := <-cc.NewPickerCh + pi := balancer.PickInfo{ + Ctx: SetPickedCluster(context.Background(), "csp:cluster"), + } + testPick(t, p1, pi, sc1, nil) + + childPolicyName := t.Name() + stub.Register(childPolicyName, stub.BalancerFuncs{ + Init: func(bd *stub.BalancerData) { + bd.Data = balancer.Get(grpc.PickFirstBalancerName).Build(bd.ClientConn, bd.BuildOptions) + }, + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + bal := bd.Data.(balancer.Balancer) + return bal.UpdateClientConnState(ccs) + }, + }) + // Same cluster, different balancer type. + configJSON2 := fmt.Sprintf(`{ +"children": { + "csp:cluster":{ "childPolicy": [{"%s":""}] } +} +}`, childPolicyName) + config2, err := parser.ParseConfig([]byte(configJSON2)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + if err := bal.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Addresses: []resolver.Address{ + hierarchy.Set(wantAddrs[1], []string{"csp:cluster"}), + }}, + BalancerConfig: config2, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + sc2 := <-cc.NewSubConnCh + // Update the pick first balancers SubConn as CONNECTING. This will cause + // the pick first balancer to UpdateState() with CONNECTING, which shouldn't send + // a Picker update back, as the Graceful Switch process is not complete. + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer cancel() + select { + case <-cc.NewPickerCh: + t.Fatalf("No new picker should have been sent due to the Graceful Switch process not completing") + case <-ctx.Done(): + } + + // Update the pick first balancers SubConn as READY. This will cause + // the pick first balancer to UpdateState() with READY, which should send a + // Picker update back, as the Graceful Switch process is complete. This + // Picker should always pick the pick first's created SubConn. + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + p2 := <-cc.NewPickerCh + testPick(t, p2, pi, sc2, nil) + // The Graceful Switch process completing for the child should cause the + // SubConns for the balancer being gracefully switched from to get deleted. + ctx, cancel = context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + select { + case <-ctx.Done(): + t.Fatalf("error waiting for sc.Shutdown()") + case rsc := <-cc.ShutdownSubConnCh: + // The SubConn removed should have been the created SubConn + // from the child before switching. + if rsc != sc1 { + t.Fatalf("Shutdown() got: %v, want %v", rsc, sc1) + } + } +} + +// tcc wraps a testutils.TestClientConn but stores all state transitions in a +// slice. +type tcc struct { + *testutils.TestClientConn + states []balancer.State +} + +func (t *tcc) UpdateState(bs balancer.State) { + t.states = append(t.states, bs) + t.TestClientConn.UpdateState(bs) +} + +func (s) TestUpdateStatePauses(t *testing.T) { + cc := &tcc{TestClientConn: testutils.NewTestClientConn(t)} + + balFuncs := stub.BalancerFuncs{ + UpdateClientConnState: func(bd *stub.BalancerData, s balancer.ClientConnState) error { + bd.ClientConn.UpdateState(balancer.State{ConnectivityState: connectivity.TransientFailure, Picker: nil}) + bd.ClientConn.UpdateState(balancer.State{ConnectivityState: connectivity.Ready, Picker: nil}) + return nil + }, + } + stub.Register("update_state_balancer", balFuncs) + + builder := balancer.Get(balancerName) + parser := builder.(balancer.ConfigParser) + bal := builder.Build(cc, balancer.BuildOptions{}) + + configJSON1 := `{ +"children": { + "cds:cluster_1":{ "childPolicy": [{"update_state_balancer":""}] } +} +}` + config1, err := parser.ParseConfig([]byte(configJSON1)) + if err != nil { + t.Fatalf("failed to parse balancer config: %v", err) + } + + // Send the config, and an address with hierarchy path ["cluster_1"]. + wantAddrs := []resolver.Address{ + {Addr: testBackendAddrStrs[0], BalancerAttributes: nil}, + } + if err := bal.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Addresses: []resolver.Address{ + hierarchy.Set(wantAddrs[0], []string{"cds:cluster_1"}), + }}, + BalancerConfig: config1, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // Verify that the only state update is the second one called by the child. + if len(cc.states) != 1 || cc.states[0].ConnectivityState != connectivity.Ready { + t.Fatalf("cc.states = %v; want [connectivity.Ready]", cc.states) + } +} diff --git a/xds/internal/balancer/weightedtarget/weightedtarget_config.go b/xds/internal/balancer/clustermanager/config.go similarity index 66% rename from xds/internal/balancer/weightedtarget/weightedtarget_config.go rename to xds/internal/balancer/clustermanager/config.go index 747ce918bc68..3a5625ff19e1 100644 --- a/xds/internal/balancer/weightedtarget/weightedtarget_config.go +++ b/xds/internal/balancer/clustermanager/config.go @@ -16,7 +16,7 @@ * */ -package weightedtarget +package clustermanager import ( "encoding/json" @@ -25,32 +25,22 @@ import ( "google.golang.org/grpc/serviceconfig" ) -type target struct { - // Weight is the weight of the child policy. - Weight uint32 +type childConfig struct { // ChildPolicy is the child policy and it's config. ChildPolicy *internalserviceconfig.BalancerConfig } -// lbConfig is the balancer config for weighted_target. The proto representation -// is: -// -// message WeightedTargetConfig { -// message Target { -// uint32 weight = 1; -// repeated LoadBalancingConfig child_policy = 2; -// } -// map targets = 1; -// } +// lbConfig is the balancer config for xds routing policy. type lbConfig struct { serviceconfig.LoadBalancingConfig - Targets map[string]target + Children map[string]childConfig } func parseConfig(c json.RawMessage) (*lbConfig, error) { - var cfg lbConfig - if err := json.Unmarshal(c, &cfg); err != nil { + cfg := &lbConfig{} + if err := json.Unmarshal(c, cfg); err != nil { return nil, err } - return &cfg, nil + + return cfg, nil } diff --git a/xds/internal/balancer/clustermanager/config_test.go b/xds/internal/balancer/clustermanager/config_test.go new file mode 100644 index 000000000000..23c25755ee30 --- /dev/null +++ b/xds/internal/balancer/clustermanager/config_test.go @@ -0,0 +1,144 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package clustermanager + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/balancer" + _ "google.golang.org/grpc/balancer/weightedtarget" + internalserviceconfig "google.golang.org/grpc/internal/serviceconfig" + _ "google.golang.org/grpc/xds/internal/balancer/cdsbalancer" +) + +const ( + testJSONConfig = `{ + "children":{ + "cds:cluster_1":{ + "childPolicy":[{ + "cds_experimental":{"cluster":"cluster_1"} + }] + }, + "weighted:cluster_1_cluster_2_1":{ + "childPolicy":[{ + "weighted_target_experimental":{ + "targets": { + "cluster_1" : { + "weight":75, + "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}] + }, + "cluster_2" : { + "weight":25, + "childPolicy":[{"cds_experimental":{"cluster":"cluster_2"}}] + } + } + } + }] + }, + "weighted:cluster_1_cluster_3_1":{ + "childPolicy":[{ + "weighted_target_experimental":{ + "targets": { + "cluster_1": { + "weight":99, + "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}] + }, + "cluster_3": { + "weight":1, + "childPolicy":[{"cds_experimental":{"cluster":"cluster_3"}}] + } + } + } + }] + } + } +} +` + + cdsName = "cds_experimental" + wtName = "weighted_target_experimental" +) + +var ( + cdsConfigParser = balancer.Get(cdsName).(balancer.ConfigParser) + cdsConfigJSON1 = `{"cluster":"cluster_1"}` + cdsConfig1, _ = cdsConfigParser.ParseConfig([]byte(cdsConfigJSON1)) + + wtConfigParser = balancer.Get(wtName).(balancer.ConfigParser) + wtConfigJSON1 = `{ + "targets": { + "cluster_1" : { "weight":75, "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}] }, + "cluster_2" : { "weight":25, "childPolicy":[{"cds_experimental":{"cluster":"cluster_2"}}] } + } }` + wtConfig1, _ = wtConfigParser.ParseConfig([]byte(wtConfigJSON1)) + wtConfigJSON2 = `{ + "targets": { + "cluster_1": { "weight":99, "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}] }, + "cluster_3": { "weight":1, "childPolicy":[{"cds_experimental":{"cluster":"cluster_3"}}] } + } }` + wtConfig2, _ = wtConfigParser.ParseConfig([]byte(wtConfigJSON2)) +) + +func Test_parseConfig(t *testing.T) { + tests := []struct { + name string + js string + want *lbConfig + wantErr bool + }{ + { + name: "empty json", + js: "", + want: nil, + wantErr: true, + }, + { + name: "OK", + js: testJSONConfig, + want: &lbConfig{ + Children: map[string]childConfig{ + "cds:cluster_1": {ChildPolicy: &internalserviceconfig.BalancerConfig{ + Name: cdsName, Config: cdsConfig1}, + }, + "weighted:cluster_1_cluster_2_1": {ChildPolicy: &internalserviceconfig.BalancerConfig{ + Name: wtName, Config: wtConfig1}, + }, + "weighted:cluster_1_cluster_3_1": {ChildPolicy: &internalserviceconfig.BalancerConfig{ + Name: wtName, Config: wtConfig2}, + }, + }, + }, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parseConfig([]byte(tt.js)) + if (err != nil) != tt.wantErr { + t.Errorf("parseConfig() error = %v, wantErr %v", err, tt.wantErr) + return + } + if d := cmp.Diff(got, tt.want, cmp.AllowUnexported(lbConfig{})); d != "" { + t.Errorf("parseConfig() got unexpected result, diff: %v", d) + } + }) + } +} diff --git a/xds/internal/balancer/clustermanager/picker.go b/xds/internal/balancer/clustermanager/picker.go new file mode 100644 index 000000000000..015cd2b7af49 --- /dev/null +++ b/xds/internal/balancer/clustermanager/picker.go @@ -0,0 +1,70 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package clustermanager + +import ( + "context" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// pickerGroup contains a list of pickers. If the picker isn't ready, the pick +// will be queued. +type pickerGroup struct { + pickers map[string]balancer.Picker +} + +func newPickerGroup(idToPickerState map[string]*subBalancerState) *pickerGroup { + pickers := make(map[string]balancer.Picker) + for id, st := range idToPickerState { + pickers[id] = st.state.Picker + } + return &pickerGroup{ + pickers: pickers, + } +} + +func (pg *pickerGroup) Pick(info balancer.PickInfo) (balancer.PickResult, error) { + cluster := getPickedCluster(info.Ctx) + if p := pg.pickers[cluster]; p != nil { + return p.Pick(info) + } + return balancer.PickResult{}, status.Errorf(codes.Unavailable, "unknown cluster selected for RPC: %q", cluster) +} + +type clusterKey struct{} + +func getPickedCluster(ctx context.Context) string { + cluster, _ := ctx.Value(clusterKey{}).(string) + return cluster +} + +// GetPickedClusterForTesting returns the cluster in the context; to be used +// for testing only. +func GetPickedClusterForTesting(ctx context.Context) string { + return getPickedCluster(ctx) +} + +// SetPickedCluster adds the selected cluster to the context for the +// xds_cluster_manager LB policy to pick. +func SetPickedCluster(ctx context.Context, cluster string) context.Context { + return context.WithValue(ctx, clusterKey{}, cluster) +} diff --git a/xds/internal/balancer/clusterresolver/clusterresolver.go b/xds/internal/balancer/clusterresolver/clusterresolver.go new file mode 100644 index 000000000000..6a60bc308a96 --- /dev/null +++ b/xds/internal/balancer/clusterresolver/clusterresolver.go @@ -0,0 +1,402 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package clusterresolver contains the implementation of the +// cluster_resolver_experimental LB policy which resolves endpoint addresses +// using a list of one or more discovery mechanisms. +package clusterresolver + +import ( + "encoding/json" + "errors" + "fmt" + + "google.golang.org/grpc/attributes" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/base" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/internal/balancer/nop" + "google.golang.org/grpc/internal/buffer" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" + "google.golang.org/grpc/xds/internal/balancer/outlierdetection" + "google.golang.org/grpc/xds/internal/balancer/priority" + "google.golang.org/grpc/xds/internal/xdsclient" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" +) + +// Name is the name of the cluster_resolver balancer. +const Name = "cluster_resolver_experimental" + +var ( + errBalancerClosed = errors.New("cdsBalancer is closed") + newChildBalancer = func(bb balancer.Builder, cc balancer.ClientConn, o balancer.BuildOptions) balancer.Balancer { + return bb.Build(cc, o) + } +) + +func init() { + balancer.Register(bb{}) +} + +type bb struct{} + +// Build helps implement the balancer.Builder interface. +func (bb) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { + priorityBuilder := balancer.Get(priority.Name) + if priorityBuilder == nil { + logger.Errorf("%q LB policy is needed but not registered", priority.Name) + return nop.NewBalancer(cc, fmt.Errorf("%q LB policy is needed but not registered", priority.Name)) + } + priorityConfigParser, ok := priorityBuilder.(balancer.ConfigParser) + if !ok { + logger.Errorf("%q LB policy does not implement a config parser", priority.Name) + return nop.NewBalancer(cc, fmt.Errorf("%q LB policy does not implement a config parser", priority.Name)) + } + + b := &clusterResolverBalancer{ + bOpts: opts, + updateCh: buffer.NewUnbounded(), + closed: grpcsync.NewEvent(), + done: grpcsync.NewEvent(), + + priorityBuilder: priorityBuilder, + priorityConfigParser: priorityConfigParser, + } + b.logger = prefixLogger(b) + b.logger.Infof("Created") + + b.resourceWatcher = newResourceResolver(b, b.logger) + b.cc = &ccWrapper{ + ClientConn: cc, + b: b, + resourceWatcher: b.resourceWatcher, + } + + go b.run() + return b +} + +func (bb) Name() string { + return Name +} + +func (bb) ParseConfig(j json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + odBuilder := balancer.Get(outlierdetection.Name) + if odBuilder == nil { + // Shouldn't happen, registered through imported Outlier Detection, + // defensive programming. + return nil, fmt.Errorf("%q LB policy is needed but not registered", outlierdetection.Name) + } + odParser, ok := odBuilder.(balancer.ConfigParser) + if !ok { + // Shouldn't happen, imported Outlier Detection builder has this method. + return nil, fmt.Errorf("%q LB policy does not implement a config parser", outlierdetection.Name) + } + + var cfg *LBConfig + if err := json.Unmarshal(j, &cfg); err != nil { + return nil, fmt.Errorf("unable to unmarshal balancer config %s into cluster-resolver config, error: %v", string(j), err) + } + + if envconfig.XDSOutlierDetection { + for i, dm := range cfg.DiscoveryMechanisms { + lbCfg, err := odParser.ParseConfig(dm.OutlierDetection) + if err != nil { + return nil, fmt.Errorf("error parsing Outlier Detection config %v: %v", dm.OutlierDetection, err) + } + odCfg, ok := lbCfg.(*outlierdetection.LBConfig) + if !ok { + // Shouldn't happen, Parser built at build time with Outlier Detection + // builder pulled from gRPC LB Registry. + return nil, fmt.Errorf("odParser returned config with unexpected type %T: %v", lbCfg, lbCfg) + } + cfg.DiscoveryMechanisms[i].outlierDetection = *odCfg + } + } + if err := json.Unmarshal(cfg.XDSLBPolicy, &cfg.xdsLBPolicy); err != nil { + // This will never occur, valid configuration is emitted from the xDS + // Client. Validity is already checked in the xDS Client, however, this + // double validation is present because Unmarshalling and Validating are + // coupled into one json.Unmarshal operation). We will switch this in + // the future to two separate operations. + return nil, fmt.Errorf("error unmarshaling xDS LB Policy: %v", err) + } + return cfg, nil +} + +// ccUpdate wraps a clientConn update received from gRPC. +type ccUpdate struct { + state balancer.ClientConnState + err error +} + +type exitIdle struct{} + +// clusterResolverBalancer resolves endpoint addresses using a list of one or +// more discovery mechanisms. +type clusterResolverBalancer struct { + cc balancer.ClientConn + bOpts balancer.BuildOptions + updateCh *buffer.Unbounded // Channel for updates from gRPC. + resourceWatcher *resourceResolver + logger *grpclog.PrefixLogger + closed *grpcsync.Event + done *grpcsync.Event + + priorityBuilder balancer.Builder + priorityConfigParser balancer.ConfigParser + + config *LBConfig + configRaw *serviceconfig.ParseResult + xdsClient xdsclient.XDSClient // xDS client to watch EDS resource. + attrsWithClient *attributes.Attributes // Attributes with xdsClient attached to be passed to the child policies. + + child balancer.Balancer + priorities []priorityConfig + watchUpdateReceived bool +} + +// handleClientConnUpdate handles a ClientConnUpdate received from gRPC. +// +// A good update results in creation of endpoint resolvers for the configured +// discovery mechanisms. An update with an error results in cancellation of any +// existing endpoint resolution and propagation of the same to the child policy. +func (b *clusterResolverBalancer) handleClientConnUpdate(update *ccUpdate) { + if err := update.err; err != nil { + b.handleErrorFromUpdate(err, true) + return + } + + b.logger.Infof("Received new balancer config: %v", pretty.ToJSON(update.state.BalancerConfig)) + cfg, _ := update.state.BalancerConfig.(*LBConfig) + if cfg == nil { + b.logger.Warningf("Ignoring unsupported balancer configuration of type: %T", update.state.BalancerConfig) + return + } + + b.config = cfg + b.configRaw = update.state.ResolverState.ServiceConfig + b.resourceWatcher.updateMechanisms(cfg.DiscoveryMechanisms) + + // The child policy is created only after all configured discovery + // mechanisms have been successfully returned endpoints. If that is not the + // case, we return early. + if !b.watchUpdateReceived { + return + } + b.updateChildConfig() +} + +// handleResourceUpdate handles a resource update or error from the resource +// resolver by propagating the same to the child LB policy. +func (b *clusterResolverBalancer) handleResourceUpdate(update *resourceUpdate) { + if err := update.err; err != nil { + b.handleErrorFromUpdate(err, false) + return + } + + b.watchUpdateReceived = true + b.priorities = update.priorities + + // An update from the resource resolver contains resolved endpoint addresses + // for all configured discovery mechanisms ordered by priority. This is used + // to generate configuration for the priority LB policy. + b.updateChildConfig() +} + +// updateChildConfig builds child policy configuration using endpoint addresses +// returned by the resource resolver and child policy configuration provided by +// parent LB policy. +// +// A child policy is created if one doesn't already exist. The newly built +// configuration is then pushed to the child policy. +func (b *clusterResolverBalancer) updateChildConfig() { + if b.child == nil { + b.child = newChildBalancer(b.priorityBuilder, b.cc, b.bOpts) + } + + childCfgBytes, addrs, err := buildPriorityConfigJSON(b.priorities, &b.config.xdsLBPolicy) + if err != nil { + b.logger.Warningf("Failed to build child policy config: %v", err) + return + } + childCfg, err := b.priorityConfigParser.ParseConfig(childCfgBytes) + if err != nil { + b.logger.Warningf("Failed to parse child policy config. This should never happen because the config was generated: %v", err) + return + } + b.logger.Infof("Built child policy config: %v", pretty.ToJSON(childCfg)) + + endpoints := make([]resolver.Endpoint, len(addrs)) + for i, a := range addrs { + endpoints[i].Attributes = a.BalancerAttributes + endpoints[i].Addresses = []resolver.Address{a} + endpoints[i].Addresses[0].BalancerAttributes = nil + } + if err := b.child.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Endpoints: endpoints, + Addresses: addrs, + ServiceConfig: b.configRaw, + Attributes: b.attrsWithClient, + }, + BalancerConfig: childCfg, + }); err != nil { + b.logger.Warningf("Failed to push config to child policy: %v", err) + } +} + +// handleErrorFromUpdate handles errors from the parent LB policy and endpoint +// resolvers. fromParent is true if error is from the parent LB policy. In both +// cases, the error is propagated to the child policy, if one exists. +func (b *clusterResolverBalancer) handleErrorFromUpdate(err error, fromParent bool) { + b.logger.Warningf("Received error: %v", err) + + // A resource-not-found error from the parent LB policy means that the LDS + // or CDS resource was removed. This should result in endpoint resolvers + // being stopped here. + // + // A resource-not-found error from the EDS endpoint resolver means that the + // EDS resource was removed. No action needs to be taken for this, and we + // should continue watching the same EDS resource. + if fromParent && xdsresource.ErrType(err) == xdsresource.ErrorTypeResourceNotFound { + b.resourceWatcher.stop(false) + } + + if b.child != nil { + b.child.ResolverError(err) + return + } + b.cc.UpdateState(balancer.State{ + ConnectivityState: connectivity.TransientFailure, + Picker: base.NewErrPicker(err), + }) +} + +// run is a long-running goroutine that handles updates from gRPC and endpoint +// resolvers. The methods handling the individual updates simply push them onto +// a channel which is read and acted upon from here. +func (b *clusterResolverBalancer) run() { + for { + select { + case u, ok := <-b.updateCh.Get(): + if !ok { + return + } + b.updateCh.Load() + switch update := u.(type) { + case *ccUpdate: + b.handleClientConnUpdate(update) + case exitIdle: + if b.child == nil { + b.logger.Errorf("xds: received ExitIdle with no child balancer") + break + } + // This implementation assumes the child balancer supports + // ExitIdle (but still checks for the interface's existence to + // avoid a panic if not). If the child does not, no subconns + // will be connected. + if ei, ok := b.child.(balancer.ExitIdler); ok { + ei.ExitIdle() + } + } + case u := <-b.resourceWatcher.updateChannel: + b.handleResourceUpdate(u) + + // Close results in stopping the endpoint resolvers and closing the + // underlying child policy and is the only way to exit this goroutine. + case <-b.closed.Done(): + b.resourceWatcher.stop(true) + + if b.child != nil { + b.child.Close() + b.child = nil + } + b.updateCh.Close() + // This is the *ONLY* point of return from this function. + b.logger.Infof("Shutdown") + b.done.Fire() + return + } + } +} + +// Following are methods to implement the balancer interface. + +func (b *clusterResolverBalancer) UpdateClientConnState(state balancer.ClientConnState) error { + if b.closed.HasFired() { + b.logger.Warningf("Received update from gRPC {%+v} after close", state) + return errBalancerClosed + } + + if b.xdsClient == nil { + c := xdsclient.FromResolverState(state.ResolverState) + if c == nil { + return balancer.ErrBadResolverState + } + b.xdsClient = c + b.attrsWithClient = state.ResolverState.Attributes + } + + b.updateCh.Put(&ccUpdate{state: state}) + return nil +} + +// ResolverError handles errors reported by the xdsResolver. +func (b *clusterResolverBalancer) ResolverError(err error) { + if b.closed.HasFired() { + b.logger.Warningf("Received resolver error {%v} after close", err) + return + } + b.updateCh.Put(&ccUpdate{err: err}) +} + +// UpdateSubConnState handles subConn updates from gRPC. +func (b *clusterResolverBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { + b.logger.Errorf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, state) +} + +// Close closes the cdsBalancer and the underlying child balancer. +func (b *clusterResolverBalancer) Close() { + b.closed.Fire() + <-b.done.Done() +} + +func (b *clusterResolverBalancer) ExitIdle() { + b.updateCh.Put(exitIdle{}) +} + +// ccWrapper overrides ResolveNow(), so that re-resolution from the child +// policies will trigger the DNS resolver in cluster_resolver balancer. It +// also intercepts NewSubConn calls in case children don't set the +// StateListener, to allow redirection to happen via this cluster_resolver +// balancer. +type ccWrapper struct { + balancer.ClientConn + b *clusterResolverBalancer + resourceWatcher *resourceResolver +} + +func (c *ccWrapper) ResolveNow(resolver.ResolveNowOptions) { + c.resourceWatcher.resolveNow() +} diff --git a/xds/internal/balancer/edsbalancer/util.go b/xds/internal/balancer/clusterresolver/clusterresolver_test.go similarity index 56% rename from xds/internal/balancer/edsbalancer/util.go rename to xds/internal/balancer/clusterresolver/clusterresolver_test.go index 132950426466..bdf6e60b35c6 100644 --- a/xds/internal/balancer/edsbalancer/util.go +++ b/xds/internal/balancer/clusterresolver/clusterresolver_test.go @@ -1,4 +1,5 @@ /* + * * Copyright 2019 gRPC authors. * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -12,33 +13,31 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. + * */ -package edsbalancer +package clusterresolver import ( - "google.golang.org/grpc/internal/wrr" - xdsclient "google.golang.org/grpc/xds/internal/client" -) - -var newRandomWRR = wrr.NewRandom + "testing" + "time" -type dropper struct { - c xdsclient.OverloadDropConfig - w wrr.WRR -} + "google.golang.org/grpc/internal/grpctest" +) -func newDropper(c xdsclient.OverloadDropConfig) *dropper { - w := newRandomWRR() - w.Add(true, int64(c.Numerator)) - w.Add(false, int64(c.Denominator-c.Numerator)) +const ( + defaultTestTimeout = 5 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond + testEDSService = "test-eds-service-name" + testClusterName = "test-cluster-name" + testClusterName2 = "google_cfe_some-name" + testBalancerNameFooBar = "foo.bar" +) - return &dropper{ - c: c, - w: w, - } +type s struct { + grpctest.Tester } -func (d *dropper) drop() (ret bool) { - return d.w.Next().(bool) +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) } diff --git a/xds/internal/balancer/clusterresolver/config.go b/xds/internal/balancer/clusterresolver/config.go new file mode 100644 index 000000000000..c67608819185 --- /dev/null +++ b/xds/internal/balancer/clusterresolver/config.go @@ -0,0 +1,158 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package clusterresolver + +import ( + "bytes" + "encoding/json" + "fmt" + + internalserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/serviceconfig" + "google.golang.org/grpc/xds/internal/balancer/outlierdetection" + "google.golang.org/grpc/xds/internal/xdsclient/bootstrap" +) + +// DiscoveryMechanismType is the type of discovery mechanism. +type DiscoveryMechanismType int + +const ( + // DiscoveryMechanismTypeEDS is eds. + DiscoveryMechanismTypeEDS DiscoveryMechanismType = iota // `json:"EDS"` + // DiscoveryMechanismTypeLogicalDNS is DNS. + DiscoveryMechanismTypeLogicalDNS // `json:"LOGICAL_DNS"` +) + +// MarshalJSON marshals a DiscoveryMechanismType to a quoted json string. +// +// This is necessary to handle enum (as strings) from JSON. +// +// Note that this needs to be defined on the type not pointer, otherwise the +// variables of this type will marshal to int not string. +func (t DiscoveryMechanismType) MarshalJSON() ([]byte, error) { + buffer := bytes.NewBufferString(`"`) + switch t { + case DiscoveryMechanismTypeEDS: + buffer.WriteString("EDS") + case DiscoveryMechanismTypeLogicalDNS: + buffer.WriteString("LOGICAL_DNS") + } + buffer.WriteString(`"`) + return buffer.Bytes(), nil +} + +// UnmarshalJSON unmarshals a quoted json string to the DiscoveryMechanismType. +func (t *DiscoveryMechanismType) UnmarshalJSON(b []byte) error { + var s string + err := json.Unmarshal(b, &s) + if err != nil { + return err + } + switch s { + case "EDS": + *t = DiscoveryMechanismTypeEDS + case "LOGICAL_DNS": + *t = DiscoveryMechanismTypeLogicalDNS + default: + return fmt.Errorf("unable to unmarshal string %q to type DiscoveryMechanismType", s) + } + return nil +} + +// DiscoveryMechanism is the discovery mechanism, can be either EDS or DNS. +// +// For DNS, the ClientConn target will be used for name resolution. +// +// For EDS, if EDSServiceName is not empty, it will be used for watching. If +// EDSServiceName is empty, Cluster will be used. +type DiscoveryMechanism struct { + // Cluster is the cluster name. + Cluster string `json:"cluster,omitempty"` + // LoadReportingServer is the LRS server to send load reports to. If not + // present, load reporting will be disabled. + LoadReportingServer *bootstrap.ServerConfig `json:"lrsLoadReportingServer,omitempty"` + // MaxConcurrentRequests is the maximum number of outstanding requests can + // be made to the upstream cluster. Default is 1024. + MaxConcurrentRequests *uint32 `json:"maxConcurrentRequests,omitempty"` + // Type is the discovery mechanism type. + Type DiscoveryMechanismType `json:"type,omitempty"` + // EDSServiceName is the EDS service name, as returned in CDS. May be unset + // if not specified in CDS. For type EDS only. + // + // This is used for EDS watch if set. If unset, Cluster is used for EDS + // watch. + EDSServiceName string `json:"edsServiceName,omitempty"` + // DNSHostname is the DNS name to resolve in "host:port" form. For type + // LOGICAL_DNS only. + DNSHostname string `json:"dnsHostname,omitempty"` + // OutlierDetection is the Outlier Detection LB configuration for this + // priority. + OutlierDetection json.RawMessage `json:"outlierDetection,omitempty"` + outlierDetection outlierdetection.LBConfig +} + +// Equal returns whether the DiscoveryMechanism is the same with the parameter. +func (dm DiscoveryMechanism) Equal(b DiscoveryMechanism) bool { + od := &dm.outlierDetection + switch { + case dm.Cluster != b.Cluster: + return false + case !equalUint32P(dm.MaxConcurrentRequests, b.MaxConcurrentRequests): + return false + case dm.Type != b.Type: + return false + case dm.EDSServiceName != b.EDSServiceName: + return false + case dm.DNSHostname != b.DNSHostname: + return false + case !od.EqualIgnoringChildPolicy(&b.outlierDetection): + return false + } + + if dm.LoadReportingServer == nil && b.LoadReportingServer == nil { + return true + } + if (dm.LoadReportingServer != nil) != (b.LoadReportingServer != nil) { + return false + } + return dm.LoadReportingServer.String() == b.LoadReportingServer.String() +} + +func equalUint32P(a, b *uint32) bool { + if a == nil && b == nil { + return true + } + if a == nil || b == nil { + return false + } + return *a == *b +} + +// LBConfig is the config for cluster resolver balancer. +type LBConfig struct { + serviceconfig.LoadBalancingConfig `json:"-"` + // DiscoveryMechanisms is an ordered list of discovery mechanisms. + // + // Must have at least one element. Results from each discovery mechanism are + // concatenated together in successive priorities. + DiscoveryMechanisms []DiscoveryMechanism `json:"discoveryMechanisms,omitempty"` + + // XDSLBPolicy specifies the policy for locality picking and endpoint picking. + XDSLBPolicy json.RawMessage `json:"xdsLbPolicy,omitempty"` + xdsLBPolicy internalserviceconfig.BalancerConfig +} diff --git a/xds/internal/balancer/clusterresolver/config_test.go b/xds/internal/balancer/clusterresolver/config_test.go new file mode 100644 index 000000000000..608c17ef78c8 --- /dev/null +++ b/xds/internal/balancer/clusterresolver/config_test.go @@ -0,0 +1,365 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package clusterresolver + +import ( + "encoding/json" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "google.golang.org/grpc/balancer" + iserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/xds/internal/balancer/outlierdetection" + "google.golang.org/grpc/xds/internal/balancer/ringhash" + "google.golang.org/grpc/xds/internal/xdsclient/bootstrap" +) + +func TestDiscoveryMechanismTypeMarshalJSON(t *testing.T) { + tests := []struct { + name string + typ DiscoveryMechanismType + want string + }{ + { + name: "eds", + typ: DiscoveryMechanismTypeEDS, + want: `"EDS"`, + }, + { + name: "dns", + typ: DiscoveryMechanismTypeLogicalDNS, + want: `"LOGICAL_DNS"`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got, err := json.Marshal(tt.typ); err != nil || string(got) != tt.want { + t.Fatalf("DiscoveryMechanismTypeEDS.MarshalJSON() = (%v, %v), want (%s, nil)", string(got), err, tt.want) + } + }) + } +} +func TestDiscoveryMechanismTypeUnmarshalJSON(t *testing.T) { + tests := []struct { + name string + js string + want DiscoveryMechanismType + wantErr bool + }{ + { + name: "eds", + js: `"EDS"`, + want: DiscoveryMechanismTypeEDS, + }, + { + name: "dns", + js: `"LOGICAL_DNS"`, + want: DiscoveryMechanismTypeLogicalDNS, + }, + { + name: "error", + js: `"1234"`, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var got DiscoveryMechanismType + err := json.Unmarshal([]byte(tt.js), &got) + if (err != nil) != tt.wantErr { + t.Fatalf("DiscoveryMechanismTypeEDS.UnmarshalJSON() error = %v, wantErr %v", err, tt.wantErr) + } + if diff := cmp.Diff(got, tt.want); diff != "" { + t.Fatalf("DiscoveryMechanismTypeEDS.UnmarshalJSON() got unexpected output, diff (-got +want): %v", diff) + } + }) + } +} + +const ( + testJSONConfig1 = `{ + "discoveryMechanisms": [{ + "cluster": "test-cluster-name", + "lrsLoadReportingServer": { + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ { "type": "google_default" } ] + }, + "maxConcurrentRequests": 314, + "type": "EDS", + "edsServiceName": "test-eds-service-name", + "outlierDetection": {} + }], + "xdsLbPolicy":[{"ROUND_ROBIN":{}}] +}` + testJSONConfig2 = `{ + "discoveryMechanisms": [{ + "cluster": "test-cluster-name", + "lrsLoadReportingServer": { + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ { "type": "google_default" } ] + }, + "maxConcurrentRequests": 314, + "type": "EDS", + "edsServiceName": "test-eds-service-name", + "outlierDetection": {} + },{ + "type": "LOGICAL_DNS", + "outlierDetection": {} + }], + "xdsLbPolicy":[{"ROUND_ROBIN":{}}] +}` + testJSONConfig3 = `{ + "discoveryMechanisms": [{ + "cluster": "test-cluster-name", + "lrsLoadReportingServer": { + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ { "type": "google_default" } ] + }, + "maxConcurrentRequests": 314, + "type": "EDS", + "edsServiceName": "test-eds-service-name", + "outlierDetection": {} + }], + "xdsLbPolicy":[{"ROUND_ROBIN":{}}] +}` + testJSONConfig4 = `{ + "discoveryMechanisms": [{ + "cluster": "test-cluster-name", + "lrsLoadReportingServer": { + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ { "type": "google_default" } ] + }, + "maxConcurrentRequests": 314, + "type": "EDS", + "edsServiceName": "test-eds-service-name", + "outlierDetection": {} + }], + "xdsLbPolicy":[{"ring_hash_experimental":{}}] +}` + testJSONConfig5 = `{ + "discoveryMechanisms": [{ + "cluster": "test-cluster-name", + "lrsLoadReportingServer": { + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ { "type": "google_default" } ] + }, + "maxConcurrentRequests": 314, + "type": "EDS", + "edsServiceName": "test-eds-service-name", + "outlierDetection": {} + }], + "xdsLbPolicy":[{"ROUND_ROBIN":{}}] +}` +) + +var testLRSServerConfig = &bootstrap.ServerConfig{ + ServerURI: "trafficdirector.googleapis.com:443", + Creds: bootstrap.ChannelCreds{ + Type: "google_default", + }, +} + +func TestParseConfig(t *testing.T) { + tests := []struct { + name string + js string + want *LBConfig + wantErr bool + }{ + { + name: "empty json", + js: "", + want: nil, + wantErr: true, + }, + { + name: "OK with one discovery mechanism", + js: testJSONConfig1, + want: &LBConfig{ + DiscoveryMechanisms: []DiscoveryMechanism{ + { + Cluster: testClusterName, + LoadReportingServer: testLRSServerConfig, + MaxConcurrentRequests: newUint32(testMaxRequests), + Type: DiscoveryMechanismTypeEDS, + EDSServiceName: testEDSService, + outlierDetection: outlierdetection.LBConfig{ + Interval: iserviceconfig.Duration(10 * time.Second), // default interval + BaseEjectionTime: iserviceconfig.Duration(30 * time.Second), + MaxEjectionTime: iserviceconfig.Duration(300 * time.Second), + MaxEjectionPercent: 10, + // sre and fpe are both nil + }, + }, + }, + xdsLBPolicy: iserviceconfig.BalancerConfig{ // do we want to make this not pointer + Name: "ROUND_ROBIN", + Config: nil, + }, + }, + wantErr: false, + }, + { + name: "OK with multiple discovery mechanisms", + js: testJSONConfig2, + want: &LBConfig{ + DiscoveryMechanisms: []DiscoveryMechanism{ + { + Cluster: testClusterName, + LoadReportingServer: testLRSServerConfig, + MaxConcurrentRequests: newUint32(testMaxRequests), + Type: DiscoveryMechanismTypeEDS, + EDSServiceName: testEDSService, + outlierDetection: outlierdetection.LBConfig{ + Interval: iserviceconfig.Duration(10 * time.Second), // default interval + BaseEjectionTime: iserviceconfig.Duration(30 * time.Second), + MaxEjectionTime: iserviceconfig.Duration(300 * time.Second), + MaxEjectionPercent: 10, + // sre and fpe are both nil + }, + }, + { + Type: DiscoveryMechanismTypeLogicalDNS, + outlierDetection: outlierdetection.LBConfig{ + Interval: iserviceconfig.Duration(10 * time.Second), // default interval + BaseEjectionTime: iserviceconfig.Duration(30 * time.Second), + MaxEjectionTime: iserviceconfig.Duration(300 * time.Second), + MaxEjectionPercent: 10, + // sre and fpe are both nil + }, + }, + }, + xdsLBPolicy: iserviceconfig.BalancerConfig{ + Name: "ROUND_ROBIN", + Config: nil, + }, + }, + wantErr: false, + }, + { + name: "OK with picking policy round_robin", + js: testJSONConfig3, + want: &LBConfig{ + DiscoveryMechanisms: []DiscoveryMechanism{ + { + Cluster: testClusterName, + LoadReportingServer: testLRSServerConfig, + MaxConcurrentRequests: newUint32(testMaxRequests), + Type: DiscoveryMechanismTypeEDS, + EDSServiceName: testEDSService, + outlierDetection: outlierdetection.LBConfig{ + Interval: iserviceconfig.Duration(10 * time.Second), // default interval + BaseEjectionTime: iserviceconfig.Duration(30 * time.Second), + MaxEjectionTime: iserviceconfig.Duration(300 * time.Second), + MaxEjectionPercent: 10, + // sre and fpe are both nil + }, + }, + }, + xdsLBPolicy: iserviceconfig.BalancerConfig{ + Name: "ROUND_ROBIN", + Config: nil, + }, + }, + wantErr: false, + }, + { + name: "OK with picking policy ring_hash", + js: testJSONConfig4, + want: &LBConfig{ + DiscoveryMechanisms: []DiscoveryMechanism{ + { + Cluster: testClusterName, + LoadReportingServer: testLRSServerConfig, + MaxConcurrentRequests: newUint32(testMaxRequests), + Type: DiscoveryMechanismTypeEDS, + EDSServiceName: testEDSService, + outlierDetection: outlierdetection.LBConfig{ + Interval: iserviceconfig.Duration(10 * time.Second), // default interval + BaseEjectionTime: iserviceconfig.Duration(30 * time.Second), + MaxEjectionTime: iserviceconfig.Duration(300 * time.Second), + MaxEjectionPercent: 10, + // sre and fpe are both nil + }, + }, + }, + xdsLBPolicy: iserviceconfig.BalancerConfig{ + Name: ringhash.Name, + Config: &ringhash.LBConfig{MinRingSize: 1024, MaxRingSize: 4096}, // Ringhash LB config with default min and max. + }, + }, + wantErr: false, + }, + { + name: "noop-outlier-detection", + js: testJSONConfig5, + want: &LBConfig{ + DiscoveryMechanisms: []DiscoveryMechanism{ + { + Cluster: testClusterName, + LoadReportingServer: testLRSServerConfig, + MaxConcurrentRequests: newUint32(testMaxRequests), + Type: DiscoveryMechanismTypeEDS, + EDSServiceName: testEDSService, + outlierDetection: outlierdetection.LBConfig{ + Interval: iserviceconfig.Duration(10 * time.Second), // default interval + BaseEjectionTime: iserviceconfig.Duration(30 * time.Second), + MaxEjectionTime: iserviceconfig.Duration(300 * time.Second), + MaxEjectionPercent: 10, + // sre and fpe are both nil + }, + }, + }, + xdsLBPolicy: iserviceconfig.BalancerConfig{ + Name: "ROUND_ROBIN", + Config: nil, + }, + }, + wantErr: false, + }, + } + for _, tt := range tests { + b := balancer.Get(Name) + if b == nil { + t.Fatalf("LB policy %q not registered", Name) + } + cfgParser, ok := b.(balancer.ConfigParser) + if !ok { + t.Fatalf("LB policy %q does not support config parsing", Name) + } + t.Run(tt.name, func(t *testing.T) { + got, err := cfgParser.ParseConfig([]byte(tt.js)) + if (err != nil) != tt.wantErr { + t.Fatalf("parseConfig() error = %v, wantErr %v", err, tt.wantErr) + } + if tt.wantErr { + return + } + if diff := cmp.Diff(got, tt.want, cmp.AllowUnexported(LBConfig{}), cmpopts.IgnoreFields(LBConfig{}, "XDSLBPolicy")); diff != "" { + t.Errorf("parseConfig() got unexpected output, diff (-got +want): %v", diff) + } + }) + } +} + +func newUint32(i uint32) *uint32 { + return &i +} diff --git a/xds/internal/balancer/clusterresolver/configbuilder.go b/xds/internal/balancer/clusterresolver/configbuilder.go new file mode 100644 index 000000000000..d1fb717d878b --- /dev/null +++ b/xds/internal/balancer/clusterresolver/configbuilder.go @@ -0,0 +1,310 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package clusterresolver + +import ( + "encoding/json" + "fmt" + "sort" + + "google.golang.org/grpc/balancer/weightedroundrobin" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/hierarchy" + internalserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/xds/internal" + "google.golang.org/grpc/xds/internal/balancer/clusterimpl" + "google.golang.org/grpc/xds/internal/balancer/outlierdetection" + "google.golang.org/grpc/xds/internal/balancer/priority" + "google.golang.org/grpc/xds/internal/balancer/wrrlocality" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" +) + +const million = 1000000 + +// priorityConfig is config for one priority. For example, if there an EDS and a +// DNS, the priority list will be [priorityConfig{EDS}, priorityConfig{DNS}]. +// +// Each priorityConfig corresponds to one discovery mechanism from the LBConfig +// generated by the CDS balancer. The CDS balancer resolves the cluster name to +// an ordered list of discovery mechanisms (if the top cluster is an aggregated +// cluster), one for each underlying cluster. +type priorityConfig struct { + mechanism DiscoveryMechanism + // edsResp is set only if type is EDS. + edsResp xdsresource.EndpointsUpdate + // addresses is set only if type is DNS. + addresses []string + // Each discovery mechanism has a name generator so that the child policies + // can reuse names between updates (EDS updates for example). + childNameGen *nameGenerator +} + +// buildPriorityConfigJSON builds balancer config for the passed in +// priorities. +// +// The built tree of balancers (see test for the output struct). +// +// ┌────────┐ +// │priority│ +// └┬──────┬┘ +// │ │ +// ┌──────────▼─┐ ┌─▼──────────┐ +// │cluster_impl│ │cluster_impl│ +// └──────┬─────┘ └─────┬──────┘ +// │ │ +// ┌──────▼─────┐ ┌─────▼──────┐ +// │xDSLBPolicy │ │xDSLBPolicy │ (Locality and Endpoint picking layer) +// └────────────┘ └────────────┘ +func buildPriorityConfigJSON(priorities []priorityConfig, xdsLBPolicy *internalserviceconfig.BalancerConfig) ([]byte, []resolver.Address, error) { + pc, addrs, err := buildPriorityConfig(priorities, xdsLBPolicy) + if err != nil { + return nil, nil, fmt.Errorf("failed to build priority config: %v", err) + } + ret, err := json.Marshal(pc) + if err != nil { + return nil, nil, fmt.Errorf("failed to marshal built priority config struct into json: %v", err) + } + return ret, addrs, nil +} + +func buildPriorityConfig(priorities []priorityConfig, xdsLBPolicy *internalserviceconfig.BalancerConfig) (*priority.LBConfig, []resolver.Address, error) { + var ( + retConfig = &priority.LBConfig{Children: make(map[string]*priority.Child)} + retAddrs []resolver.Address + ) + for _, p := range priorities { + switch p.mechanism.Type { + case DiscoveryMechanismTypeEDS: + names, configs, addrs, err := buildClusterImplConfigForEDS(p.childNameGen, p.edsResp, p.mechanism, xdsLBPolicy) + if err != nil { + return nil, nil, err + } + retConfig.Priorities = append(retConfig.Priorities, names...) + retAddrs = append(retAddrs, addrs...) + var odCfgs map[string]*outlierdetection.LBConfig + if envconfig.XDSOutlierDetection { + odCfgs = convertClusterImplMapToOutlierDetection(configs, p.mechanism.outlierDetection) + for n, c := range odCfgs { + retConfig.Children[n] = &priority.Child{ + Config: &internalserviceconfig.BalancerConfig{Name: outlierdetection.Name, Config: c}, + // Ignore all re-resolution from EDS children. + IgnoreReresolutionRequests: true, + } + } + continue + } + for n, c := range configs { + retConfig.Children[n] = &priority.Child{ + Config: &internalserviceconfig.BalancerConfig{Name: clusterimpl.Name, Config: c}, + // Ignore all re-resolution from EDS children. + IgnoreReresolutionRequests: true, + } + + } + case DiscoveryMechanismTypeLogicalDNS: + name, config, addrs := buildClusterImplConfigForDNS(p.childNameGen, p.addresses, p.mechanism) + retConfig.Priorities = append(retConfig.Priorities, name) + retAddrs = append(retAddrs, addrs...) + var odCfg *outlierdetection.LBConfig + if envconfig.XDSOutlierDetection { + odCfg = makeClusterImplOutlierDetectionChild(config, p.mechanism.outlierDetection) + retConfig.Children[name] = &priority.Child{ + Config: &internalserviceconfig.BalancerConfig{Name: outlierdetection.Name, Config: odCfg}, + // Not ignore re-resolution from DNS children, they will trigger + // DNS to re-resolve. + IgnoreReresolutionRequests: false, + } + continue + } + retConfig.Children[name] = &priority.Child{ + Config: &internalserviceconfig.BalancerConfig{Name: clusterimpl.Name, Config: config}, + // Not ignore re-resolution from DNS children, they will trigger + // DNS to re-resolve. + IgnoreReresolutionRequests: false, + } + } + } + return retConfig, retAddrs, nil +} + +func convertClusterImplMapToOutlierDetection(ciCfgs map[string]*clusterimpl.LBConfig, odCfg outlierdetection.LBConfig) map[string]*outlierdetection.LBConfig { + odCfgs := make(map[string]*outlierdetection.LBConfig, len(ciCfgs)) + for n, c := range ciCfgs { + odCfgs[n] = makeClusterImplOutlierDetectionChild(c, odCfg) + } + return odCfgs +} + +func makeClusterImplOutlierDetectionChild(ciCfg *clusterimpl.LBConfig, odCfg outlierdetection.LBConfig) *outlierdetection.LBConfig { + odCfgRet := odCfg + odCfgRet.ChildPolicy = &internalserviceconfig.BalancerConfig{Name: clusterimpl.Name, Config: ciCfg} + return &odCfgRet +} + +func buildClusterImplConfigForDNS(g *nameGenerator, addrStrs []string, mechanism DiscoveryMechanism) (string, *clusterimpl.LBConfig, []resolver.Address) { + // Endpoint picking policy for DNS is hardcoded to pick_first. + const childPolicy = "pick_first" + retAddrs := make([]resolver.Address, 0, len(addrStrs)) + pName := fmt.Sprintf("priority-%v", g.prefix) + for _, addrStr := range addrStrs { + retAddrs = append(retAddrs, hierarchy.Set(resolver.Address{Addr: addrStr}, []string{pName})) + } + return pName, &clusterimpl.LBConfig{ + Cluster: mechanism.Cluster, + ChildPolicy: &internalserviceconfig.BalancerConfig{Name: childPolicy}, + }, retAddrs +} + +// buildClusterImplConfigForEDS returns a list of cluster_impl configs, one for +// each priority, sorted by priority, and the addresses for each priority (with +// hierarchy attributes set). +// +// For example, if there are two priorities, the returned values will be +// - ["p0", "p1"] +// - map{"p0":p0_config, "p1":p1_config} +// - [p0_address_0, p0_address_1, p1_address_0, p1_address_1] +// - p0 addresses' hierarchy attributes are set to p0 +func buildClusterImplConfigForEDS(g *nameGenerator, edsResp xdsresource.EndpointsUpdate, mechanism DiscoveryMechanism, xdsLBPolicy *internalserviceconfig.BalancerConfig) ([]string, map[string]*clusterimpl.LBConfig, []resolver.Address, error) { + drops := make([]clusterimpl.DropConfig, 0, len(edsResp.Drops)) + for _, d := range edsResp.Drops { + drops = append(drops, clusterimpl.DropConfig{ + Category: d.Category, + RequestsPerMillion: d.Numerator * million / d.Denominator, + }) + } + + // Localities of length 0 is triggered by an NACK or resource-not-found + // error before update, or a empty localities list in a update. In either + // case want to create a priority, and send down empty address list, causing + // TF for that priority. "If any discovery mechanism instance experiences an + // error retrieving data, and it has not previously reported any results, it + // should report a result that is a single priority with no endpoints." - + // A37 + priorities := [][]xdsresource.Locality{{}} + if len(edsResp.Localities) != 0 { + priorities = groupLocalitiesByPriority(edsResp.Localities) + } + retNames := g.generate(priorities) + retConfigs := make(map[string]*clusterimpl.LBConfig, len(retNames)) + var retAddrs []resolver.Address + for i, pName := range retNames { + priorityLocalities := priorities[i] + cfg, addrs, err := priorityLocalitiesToClusterImpl(priorityLocalities, pName, mechanism, drops, xdsLBPolicy) + if err != nil { + return nil, nil, nil, err + } + retConfigs[pName] = cfg + retAddrs = append(retAddrs, addrs...) + } + return retNames, retConfigs, retAddrs, nil +} + +// groupLocalitiesByPriority returns the localities grouped by priority. +// +// The returned list is sorted from higher priority to lower. Each item in the +// list is a group of localities. +// +// For example, for L0-p0, L1-p0, L2-p1, results will be +// - [[L0, L1], [L2]] +func groupLocalitiesByPriority(localities []xdsresource.Locality) [][]xdsresource.Locality { + var priorityIntSlice []int + priorities := make(map[int][]xdsresource.Locality) + for _, locality := range localities { + priority := int(locality.Priority) + priorities[priority] = append(priorities[priority], locality) + priorityIntSlice = append(priorityIntSlice, priority) + } + // Sort the priorities based on the int value, deduplicate, and then turn + // the sorted list into a string list. This will be child names, in priority + // order. + sort.Ints(priorityIntSlice) + priorityIntSliceDeduped := dedupSortedIntSlice(priorityIntSlice) + ret := make([][]xdsresource.Locality, 0, len(priorityIntSliceDeduped)) + for _, p := range priorityIntSliceDeduped { + ret = append(ret, priorities[p]) + } + return ret +} + +func dedupSortedIntSlice(a []int) []int { + if len(a) == 0 { + return a + } + i, j := 0, 1 + for ; j < len(a); j++ { + if a[i] == a[j] { + continue + } + i++ + if i != j { + a[i] = a[j] + } + } + return a[:i+1] +} + +// priorityLocalitiesToClusterImpl takes a list of localities (with the same +// priority), and generates a cluster impl policy config, and a list of +// addresses with their path hierarchy set to [priority-name, locality-name], so +// priority and the xDS LB Policy know which child policy each address is for. +func priorityLocalitiesToClusterImpl(localities []xdsresource.Locality, priorityName string, mechanism DiscoveryMechanism, drops []clusterimpl.DropConfig, xdsLBPolicy *internalserviceconfig.BalancerConfig) (*clusterimpl.LBConfig, []resolver.Address, error) { + var addrs []resolver.Address + for _, locality := range localities { + var lw uint32 = 1 + if locality.Weight != 0 { + lw = locality.Weight + } + localityStr, err := locality.ID.ToString() + if err != nil { + localityStr = fmt.Sprintf("%+v", locality.ID) + } + for _, endpoint := range locality.Endpoints { + // Filter out all "unhealthy" endpoints (unknown and healthy are + // both considered to be healthy: + // https://www.envoyproxy.io/docs/envoy/latest/api-v2/api/v2/core/health_check.proto#envoy-api-enum-core-healthstatus). + if endpoint.HealthStatus != xdsresource.EndpointHealthStatusHealthy && endpoint.HealthStatus != xdsresource.EndpointHealthStatusUnknown { + continue + } + addr := resolver.Address{Addr: endpoint.Address} + addr = hierarchy.Set(addr, []string{priorityName, localityStr}) + addr = internal.SetLocalityID(addr, locality.ID) + // "To provide the xds_wrr_locality load balancer information about + // locality weights received from EDS, the cluster resolver will + // populate a new locality weight attribute for each address The + // attribute will have the weight (as an integer) of the locality + // the address is part of." - A52 + addr = wrrlocality.SetAddrInfo(addr, wrrlocality.AddrInfo{LocalityWeight: lw}) + var ew uint32 = 1 + if endpoint.Weight != 0 { + ew = endpoint.Weight + } + addr = weightedroundrobin.SetAddrInfo(addr, weightedroundrobin.AddrInfo{Weight: lw * ew}) + addrs = append(addrs, addr) + } + } + return &clusterimpl.LBConfig{ + Cluster: mechanism.Cluster, + EDSServiceName: mechanism.EDSServiceName, + LoadReportingServer: mechanism.LoadReportingServer, + MaxConcurrentRequests: mechanism.MaxConcurrentRequests, + DropCategories: drops, + ChildPolicy: xdsLBPolicy, + }, addrs, nil +} diff --git a/xds/internal/balancer/clusterresolver/configbuilder_childname.go b/xds/internal/balancer/clusterresolver/configbuilder_childname.go new file mode 100644 index 000000000000..119f4c474752 --- /dev/null +++ b/xds/internal/balancer/clusterresolver/configbuilder_childname.go @@ -0,0 +1,88 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package clusterresolver + +import ( + "fmt" + + "google.golang.org/grpc/xds/internal" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" +) + +// nameGenerator generates a child name for a list of priorities (each priority +// is a list of localities). +// +// The purpose of this generator is to reuse names between updates. So the +// struct keeps state between generate() calls, and a later generate() might +// return names returned by the previous call. +type nameGenerator struct { + existingNames map[internal.LocalityID]string + prefix uint64 + nextID uint64 +} + +func newNameGenerator(prefix uint64) *nameGenerator { + return &nameGenerator{prefix: prefix} +} + +// generate returns a list of names for the given list of priorities. +// +// Each priority is a list of localities. The name for the priority is picked as +// - for each locality in this priority, if it exists in the existing names, +// this priority will reuse the name +// - if no reusable name is found for this priority, a new name is generated +// +// For example: +// - update 1: [[L1], [L2], [L3]] --> ["0", "1", "2"] +// - update 2: [[L1], [L2], [L3]] --> ["0", "1", "2"] +// - update 3: [[L1, L2], [L3]] --> ["0", "2"] (Two priorities were merged) +// - update 4: [[L1], [L4]] --> ["0", "3",] (A priority was split, and a new priority was added) +func (ng *nameGenerator) generate(priorities [][]xdsresource.Locality) []string { + var ret []string + usedNames := make(map[string]bool) + newNames := make(map[internal.LocalityID]string) + for _, priority := range priorities { + var nameFound string + for _, locality := range priority { + if name, ok := ng.existingNames[locality.ID]; ok { + if !usedNames[name] { + nameFound = name + // Found a name to use. No need to process the remaining + // localities. + break + } + } + } + + if nameFound == "" { + // No appropriate used name is found. Make a new name. + nameFound = fmt.Sprintf("priority-%d-%d", ng.prefix, ng.nextID) + ng.nextID++ + } + + ret = append(ret, nameFound) + // All localities in this priority share the same name. Add them all to + // the new map. + for _, l := range priority { + newNames[l.ID] = nameFound + } + usedNames[nameFound] = true + } + ng.existingNames = newNames + return ret +} diff --git a/xds/internal/balancer/clusterresolver/configbuilder_childname_test.go b/xds/internal/balancer/clusterresolver/configbuilder_childname_test.go new file mode 100644 index 000000000000..6a3dbba83a4b --- /dev/null +++ b/xds/internal/balancer/clusterresolver/configbuilder_childname_test.go @@ -0,0 +1,111 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package clusterresolver + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/xds/internal" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" +) + +func Test_nameGenerator_generate(t *testing.T) { + tests := []struct { + name string + prefix uint64 + input1 [][]xdsresource.Locality + input2 [][]xdsresource.Locality + want []string + }{ + { + name: "init, two new priorities", + prefix: 3, + input1: nil, + input2: [][]xdsresource.Locality{ + {{ID: internal.LocalityID{Zone: "L0"}}}, + {{ID: internal.LocalityID{Zone: "L1"}}}, + }, + want: []string{"priority-3-0", "priority-3-1"}, + }, + { + name: "one new priority", + prefix: 1, + input1: [][]xdsresource.Locality{ + {{ID: internal.LocalityID{Zone: "L0"}}}, + }, + input2: [][]xdsresource.Locality{ + {{ID: internal.LocalityID{Zone: "L0"}}}, + {{ID: internal.LocalityID{Zone: "L1"}}}, + }, + want: []string{"priority-1-0", "priority-1-1"}, + }, + { + name: "merge two priorities", + prefix: 4, + input1: [][]xdsresource.Locality{ + {{ID: internal.LocalityID{Zone: "L0"}}}, + {{ID: internal.LocalityID{Zone: "L1"}}}, + {{ID: internal.LocalityID{Zone: "L2"}}}, + }, + input2: [][]xdsresource.Locality{ + {{ID: internal.LocalityID{Zone: "L0"}}, {ID: internal.LocalityID{Zone: "L1"}}}, + {{ID: internal.LocalityID{Zone: "L2"}}}, + }, + want: []string{"priority-4-0", "priority-4-2"}, + }, + { + name: "swap two priorities", + input1: [][]xdsresource.Locality{ + {{ID: internal.LocalityID{Zone: "L0"}}}, + {{ID: internal.LocalityID{Zone: "L1"}}}, + {{ID: internal.LocalityID{Zone: "L2"}}}, + }, + input2: [][]xdsresource.Locality{ + {{ID: internal.LocalityID{Zone: "L1"}}}, + {{ID: internal.LocalityID{Zone: "L0"}}}, + {{ID: internal.LocalityID{Zone: "L2"}}}, + }, + want: []string{"priority-0-1", "priority-0-0", "priority-0-2"}, + }, + { + name: "split priority", + input1: [][]xdsresource.Locality{ + {{ID: internal.LocalityID{Zone: "L0"}}, {ID: internal.LocalityID{Zone: "L1"}}}, + {{ID: internal.LocalityID{Zone: "L2"}}}, + }, + input2: [][]xdsresource.Locality{ + {{ID: internal.LocalityID{Zone: "L0"}}}, + {{ID: internal.LocalityID{Zone: "L1"}}}, // This gets a newly generated name, sice "0-0" was already picked. + {{ID: internal.LocalityID{Zone: "L2"}}}, + }, + want: []string{"priority-0-0", "priority-0-2", "priority-0-1"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ng := newNameGenerator(tt.prefix) + got1 := ng.generate(tt.input1) + t.Logf("%v", got1) + got := ng.generate(tt.input2) + if diff := cmp.Diff(got, tt.want); diff != "" { + t.Errorf("generate() = got: %v, want: %v, diff (-got +want): %s", got, tt.want, diff) + } + }) + } +} diff --git a/xds/internal/balancer/clusterresolver/configbuilder_test.go b/xds/internal/balancer/clusterresolver/configbuilder_test.go new file mode 100644 index 000000000000..b30686b18561 --- /dev/null +++ b/xds/internal/balancer/clusterresolver/configbuilder_test.go @@ -0,0 +1,706 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package clusterresolver + +import ( + "bytes" + "encoding/json" + "fmt" + "sort" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/attributes" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/roundrobin" + "google.golang.org/grpc/balancer/weightedroundrobin" + "google.golang.org/grpc/internal/hierarchy" + iserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/xds/internal" + "google.golang.org/grpc/xds/internal/balancer/clusterimpl" + "google.golang.org/grpc/xds/internal/balancer/outlierdetection" + "google.golang.org/grpc/xds/internal/balancer/priority" + "google.golang.org/grpc/xds/internal/balancer/ringhash" + "google.golang.org/grpc/xds/internal/balancer/wrrlocality" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" +) + +const ( + testLRSServer = "test-lrs-server" + testMaxRequests = 314 + testEDSServiceName = "service-name-from-parent" + testDropCategory = "test-drops" + testDropOverMillion = 1 + + localityCount = 5 + addressPerLocality = 2 +) + +var ( + testLocalityIDs []internal.LocalityID + testAddressStrs [][]string + testEndpoints [][]xdsresource.Endpoint + + testLocalitiesP0, testLocalitiesP1 []xdsresource.Locality + + addrCmpOpts = cmp.Options{ + cmp.AllowUnexported(attributes.Attributes{}), + cmp.Transformer("SortAddrs", func(in []resolver.Address) []resolver.Address { + out := append([]resolver.Address(nil), in...) // Copy input to avoid mutating it + sort.Slice(out, func(i, j int) bool { + return out[i].Addr < out[j].Addr + }) + return out + }), + } + + noopODCfg = outlierdetection.LBConfig{ + Interval: iserviceconfig.Duration(10 * time.Second), // default interval + BaseEjectionTime: iserviceconfig.Duration(30 * time.Second), + MaxEjectionTime: iserviceconfig.Duration(300 * time.Second), + MaxEjectionPercent: 10, + } +) + +func init() { + for i := 0; i < localityCount; i++ { + testLocalityIDs = append(testLocalityIDs, internal.LocalityID{Zone: fmt.Sprintf("test-zone-%d", i)}) + var ( + addrs []string + ends []xdsresource.Endpoint + ) + for j := 0; j < addressPerLocality; j++ { + addr := fmt.Sprintf("addr-%d-%d", i, j) + addrs = append(addrs, addr) + ends = append(ends, xdsresource.Endpoint{ + Address: addr, + HealthStatus: xdsresource.EndpointHealthStatusHealthy, + }) + } + testAddressStrs = append(testAddressStrs, addrs) + testEndpoints = append(testEndpoints, ends) + } + + testLocalitiesP0 = []xdsresource.Locality{ + { + Endpoints: testEndpoints[0], + ID: testLocalityIDs[0], + Weight: 20, + Priority: 0, + }, + { + Endpoints: testEndpoints[1], + ID: testLocalityIDs[1], + Weight: 80, + Priority: 0, + }, + } + testLocalitiesP1 = []xdsresource.Locality{ + { + Endpoints: testEndpoints[2], + ID: testLocalityIDs[2], + Weight: 20, + Priority: 1, + }, + { + Endpoints: testEndpoints[3], + ID: testLocalityIDs[3], + Weight: 80, + Priority: 1, + }, + } +} + +// TestBuildPriorityConfigJSON is a sanity check that the built balancer config +// can be parsed. The behavior test is covered by TestBuildPriorityConfig. +func TestBuildPriorityConfigJSON(t *testing.T) { + gotConfig, _, err := buildPriorityConfigJSON([]priorityConfig{ + { + mechanism: DiscoveryMechanism{ + Cluster: testClusterName, + LoadReportingServer: testLRSServerConfig, + MaxConcurrentRequests: newUint32(testMaxRequests), + Type: DiscoveryMechanismTypeEDS, + EDSServiceName: testEDSServiceName, + }, + edsResp: xdsresource.EndpointsUpdate{ + Drops: []xdsresource.OverloadDropConfig{ + { + Category: testDropCategory, + Numerator: testDropOverMillion, + Denominator: million, + }, + }, + Localities: []xdsresource.Locality{ + testLocalitiesP0[0], + testLocalitiesP0[1], + testLocalitiesP1[0], + testLocalitiesP1[1], + }, + }, + childNameGen: newNameGenerator(0), + }, + { + mechanism: DiscoveryMechanism{ + Type: DiscoveryMechanismTypeLogicalDNS, + }, + addresses: testAddressStrs[4], + childNameGen: newNameGenerator(1), + }, + }, nil) + if err != nil { + t.Fatalf("buildPriorityConfigJSON(...) failed: %v", err) + } + + var prettyGot bytes.Buffer + if err := json.Indent(&prettyGot, gotConfig, ">>> ", " "); err != nil { + t.Fatalf("json.Indent() failed: %v", err) + } + // Print the indented json if this test fails. + t.Log(prettyGot.String()) + + priorityB := balancer.Get(priority.Name) + if _, err = priorityB.(balancer.ConfigParser).ParseConfig(gotConfig); err != nil { + t.Fatalf("ParseConfig(%+v) failed: %v", gotConfig, err) + } +} + +// TestBuildPriorityConfig tests the priority config generation. Each top level +// balancer per priority should be an Outlier Detection balancer, with a Cluster +// Impl Balancer as a child. +func TestBuildPriorityConfig(t *testing.T) { + gotConfig, _, _ := buildPriorityConfig([]priorityConfig{ + { + // EDS - OD config should be the top level for both of the EDS + // priorities balancer This EDS priority will have multiple sub + // priorities. The Outlier Detection configuration specified in the + // Discovery Mechanism should be the top level for each sub + // priorities balancer. + mechanism: DiscoveryMechanism{ + Cluster: testClusterName, + Type: DiscoveryMechanismTypeEDS, + EDSServiceName: testEDSServiceName, + outlierDetection: noopODCfg, + }, + edsResp: xdsresource.EndpointsUpdate{ + Localities: []xdsresource.Locality{ + testLocalitiesP0[0], + testLocalitiesP0[1], + testLocalitiesP1[0], + testLocalitiesP1[1], + }, + }, + childNameGen: newNameGenerator(0), + }, + { + // This OD config should wrap the Logical DNS priorities balancer. + mechanism: DiscoveryMechanism{ + Cluster: testClusterName2, + Type: DiscoveryMechanismTypeLogicalDNS, + outlierDetection: noopODCfg, + }, + addresses: testAddressStrs[4], + childNameGen: newNameGenerator(1), + }, + }, nil) + + wantConfig := &priority.LBConfig{ + Children: map[string]*priority.Child{ + "priority-0-0": { + Config: &iserviceconfig.BalancerConfig{ + Name: outlierdetection.Name, + Config: &outlierdetection.LBConfig{ + Interval: iserviceconfig.Duration(10 * time.Second), // default interval + BaseEjectionTime: iserviceconfig.Duration(30 * time.Second), + MaxEjectionTime: iserviceconfig.Duration(300 * time.Second), + MaxEjectionPercent: 10, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: clusterimpl.Name, + Config: &clusterimpl.LBConfig{ + Cluster: testClusterName, + EDSServiceName: testEDSServiceName, + DropCategories: []clusterimpl.DropConfig{}, + }, + }, + }, + }, + IgnoreReresolutionRequests: true, + }, + "priority-0-1": { + Config: &iserviceconfig.BalancerConfig{ + Name: outlierdetection.Name, + Config: &outlierdetection.LBConfig{ + Interval: iserviceconfig.Duration(10 * time.Second), // default interval + BaseEjectionTime: iserviceconfig.Duration(30 * time.Second), + MaxEjectionTime: iserviceconfig.Duration(300 * time.Second), + MaxEjectionPercent: 10, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: clusterimpl.Name, + Config: &clusterimpl.LBConfig{ + Cluster: testClusterName, + EDSServiceName: testEDSServiceName, + DropCategories: []clusterimpl.DropConfig{}, + }, + }, + }, + }, + IgnoreReresolutionRequests: true, + }, + "priority-1": { + Config: &iserviceconfig.BalancerConfig{ + Name: outlierdetection.Name, + Config: &outlierdetection.LBConfig{ + Interval: iserviceconfig.Duration(10 * time.Second), // default interval + BaseEjectionTime: iserviceconfig.Duration(30 * time.Second), + MaxEjectionTime: iserviceconfig.Duration(300 * time.Second), + MaxEjectionPercent: 10, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: clusterimpl.Name, + Config: &clusterimpl.LBConfig{ + Cluster: testClusterName2, + ChildPolicy: &iserviceconfig.BalancerConfig{Name: "pick_first"}, + }, + }, + }, + }, + IgnoreReresolutionRequests: false, + }, + }, + Priorities: []string{"priority-0-0", "priority-0-1", "priority-1"}, + } + if diff := cmp.Diff(gotConfig, wantConfig); diff != "" { + t.Errorf("buildPriorityConfig() diff (-got +want) %v", diff) + } +} + +func TestBuildClusterImplConfigForDNS(t *testing.T) { + gotName, gotConfig, gotAddrs := buildClusterImplConfigForDNS(newNameGenerator(3), testAddressStrs[0], DiscoveryMechanism{Cluster: testClusterName2, Type: DiscoveryMechanismTypeLogicalDNS}) + wantName := "priority-3" + wantConfig := &clusterimpl.LBConfig{ + Cluster: testClusterName2, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: "pick_first", + }, + } + wantAddrs := []resolver.Address{ + hierarchy.Set(resolver.Address{Addr: testAddressStrs[0][0]}, []string{"priority-3"}), + hierarchy.Set(resolver.Address{Addr: testAddressStrs[0][1]}, []string{"priority-3"}), + } + + if diff := cmp.Diff(gotName, wantName); diff != "" { + t.Errorf("buildClusterImplConfigForDNS() diff (-got +want) %v", diff) + } + if diff := cmp.Diff(gotConfig, wantConfig); diff != "" { + t.Errorf("buildClusterImplConfigForDNS() diff (-got +want) %v", diff) + } + if diff := cmp.Diff(gotAddrs, wantAddrs, addrCmpOpts); diff != "" { + t.Errorf("buildClusterImplConfigForDNS() diff (-got +want) %v", diff) + } +} + +func TestBuildClusterImplConfigForEDS(t *testing.T) { + gotNames, gotConfigs, gotAddrs, _ := buildClusterImplConfigForEDS( + newNameGenerator(2), + xdsresource.EndpointsUpdate{ + Drops: []xdsresource.OverloadDropConfig{ + { + Category: testDropCategory, + Numerator: testDropOverMillion, + Denominator: million, + }, + }, + Localities: []xdsresource.Locality{ + { + Endpoints: testEndpoints[3], + ID: testLocalityIDs[3], + Weight: 80, + Priority: 1, + }, { + Endpoints: testEndpoints[1], + ID: testLocalityIDs[1], + Weight: 80, + Priority: 0, + }, { + Endpoints: testEndpoints[2], + ID: testLocalityIDs[2], + Weight: 20, + Priority: 1, + }, { + Endpoints: testEndpoints[0], + ID: testLocalityIDs[0], + Weight: 20, + Priority: 0, + }, + }, + }, + DiscoveryMechanism{ + Cluster: testClusterName, + MaxConcurrentRequests: newUint32(testMaxRequests), + LoadReportingServer: testLRSServerConfig, + Type: DiscoveryMechanismTypeEDS, + EDSServiceName: testEDSServiceName, + }, + nil, + ) + + wantNames := []string{ + fmt.Sprintf("priority-%v-%v", 2, 0), + fmt.Sprintf("priority-%v-%v", 2, 1), + } + wantConfigs := map[string]*clusterimpl.LBConfig{ + "priority-2-0": { + Cluster: testClusterName, + EDSServiceName: testEDSServiceName, + LoadReportingServer: testLRSServerConfig, + MaxConcurrentRequests: newUint32(testMaxRequests), + DropCategories: []clusterimpl.DropConfig{ + { + Category: testDropCategory, + RequestsPerMillion: testDropOverMillion, + }, + }, + }, + "priority-2-1": { + Cluster: testClusterName, + EDSServiceName: testEDSServiceName, + LoadReportingServer: testLRSServerConfig, + MaxConcurrentRequests: newUint32(testMaxRequests), + DropCategories: []clusterimpl.DropConfig{ + { + Category: testDropCategory, + RequestsPerMillion: testDropOverMillion, + }, + }, + }, + } + wantAddrs := []resolver.Address{ + testAddrWithAttrs(testAddressStrs[0][0], 20, 1, "priority-2-0", &testLocalityIDs[0]), + testAddrWithAttrs(testAddressStrs[0][1], 20, 1, "priority-2-0", &testLocalityIDs[0]), + testAddrWithAttrs(testAddressStrs[1][0], 80, 1, "priority-2-0", &testLocalityIDs[1]), + testAddrWithAttrs(testAddressStrs[1][1], 80, 1, "priority-2-0", &testLocalityIDs[1]), + testAddrWithAttrs(testAddressStrs[2][0], 20, 1, "priority-2-1", &testLocalityIDs[2]), + testAddrWithAttrs(testAddressStrs[2][1], 20, 1, "priority-2-1", &testLocalityIDs[2]), + testAddrWithAttrs(testAddressStrs[3][0], 80, 1, "priority-2-1", &testLocalityIDs[3]), + testAddrWithAttrs(testAddressStrs[3][1], 80, 1, "priority-2-1", &testLocalityIDs[3]), + } + + if diff := cmp.Diff(gotNames, wantNames); diff != "" { + t.Errorf("buildClusterImplConfigForEDS() diff (-got +want) %v", diff) + } + if diff := cmp.Diff(gotConfigs, wantConfigs); diff != "" { + t.Errorf("buildClusterImplConfigForEDS() diff (-got +want) %v", diff) + } + if diff := cmp.Diff(gotAddrs, wantAddrs, addrCmpOpts); diff != "" { + t.Errorf("buildClusterImplConfigForEDS() diff (-got +want) %v", diff) + } + +} + +func TestGroupLocalitiesByPriority(t *testing.T) { + tests := []struct { + name string + localities []xdsresource.Locality + wantLocalities [][]xdsresource.Locality + }{ + { + name: "1 locality 1 priority", + localities: []xdsresource.Locality{testLocalitiesP0[0]}, + wantLocalities: [][]xdsresource.Locality{ + {testLocalitiesP0[0]}, + }, + }, + { + name: "2 locality 1 priority", + localities: []xdsresource.Locality{testLocalitiesP0[0], testLocalitiesP0[1]}, + wantLocalities: [][]xdsresource.Locality{ + {testLocalitiesP0[0], testLocalitiesP0[1]}, + }, + }, + { + name: "1 locality in each", + localities: []xdsresource.Locality{testLocalitiesP0[0], testLocalitiesP1[0]}, + wantLocalities: [][]xdsresource.Locality{ + {testLocalitiesP0[0]}, + {testLocalitiesP1[0]}, + }, + }, + { + name: "2 localities in each sorted", + localities: []xdsresource.Locality{ + testLocalitiesP0[0], testLocalitiesP0[1], + testLocalitiesP1[0], testLocalitiesP1[1]}, + wantLocalities: [][]xdsresource.Locality{ + {testLocalitiesP0[0], testLocalitiesP0[1]}, + {testLocalitiesP1[0], testLocalitiesP1[1]}, + }, + }, + { + // The localities are given in order [p1, p0, p1, p0], but the + // returned priority list must be sorted [p0, p1], because the list + // order is the priority order. + name: "2 localities in each needs to sort", + localities: []xdsresource.Locality{ + testLocalitiesP1[1], testLocalitiesP0[1], + testLocalitiesP1[0], testLocalitiesP0[0]}, + wantLocalities: [][]xdsresource.Locality{ + {testLocalitiesP0[1], testLocalitiesP0[0]}, + {testLocalitiesP1[1], testLocalitiesP1[0]}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotLocalities := groupLocalitiesByPriority(tt.localities) + if diff := cmp.Diff(gotLocalities, tt.wantLocalities); diff != "" { + t.Errorf("groupLocalitiesByPriority() diff(-got +want) %v", diff) + } + }) + } +} + +func TestDedupSortedIntSlice(t *testing.T) { + tests := []struct { + name string + a []int + want []int + }{ + { + name: "empty", + a: []int{}, + want: []int{}, + }, + { + name: "no dup", + a: []int{0, 1, 2, 3}, + want: []int{0, 1, 2, 3}, + }, + { + name: "with dup", + a: []int{0, 0, 1, 1, 1, 2, 3}, + want: []int{0, 1, 2, 3}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := dedupSortedIntSlice(tt.a); !cmp.Equal(got, tt.want) { + t.Errorf("dedupSortedIntSlice() = %v, want %v, diff %v", got, tt.want, cmp.Diff(got, tt.want)) + } + }) + } +} + +func TestPriorityLocalitiesToClusterImpl(t *testing.T) { + tests := []struct { + name string + localities []xdsresource.Locality + priorityName string + mechanism DiscoveryMechanism + childPolicy *iserviceconfig.BalancerConfig + wantConfig *clusterimpl.LBConfig + wantAddrs []resolver.Address + wantErr bool + }{{ + name: "round robin as child, no LRS", + localities: []xdsresource.Locality{ + { + Endpoints: []xdsresource.Endpoint{ + {Address: "addr-1-1", HealthStatus: xdsresource.EndpointHealthStatusHealthy, Weight: 90}, + {Address: "addr-1-2", HealthStatus: xdsresource.EndpointHealthStatusHealthy, Weight: 10}, + }, + ID: internal.LocalityID{Zone: "test-zone-1"}, + Weight: 20, + }, + { + Endpoints: []xdsresource.Endpoint{ + {Address: "addr-2-1", HealthStatus: xdsresource.EndpointHealthStatusHealthy, Weight: 90}, + {Address: "addr-2-2", HealthStatus: xdsresource.EndpointHealthStatusHealthy, Weight: 10}, + }, + ID: internal.LocalityID{Zone: "test-zone-2"}, + Weight: 80, + }, + }, + priorityName: "test-priority", + childPolicy: &iserviceconfig.BalancerConfig{Name: roundrobin.Name}, + mechanism: DiscoveryMechanism{ + Cluster: testClusterName, + Type: DiscoveryMechanismTypeEDS, + EDSServiceName: testEDSService, + }, + // lrsServer is nil, so LRS policy will not be used. + wantConfig: &clusterimpl.LBConfig{ + Cluster: testClusterName, + EDSServiceName: testEDSService, + ChildPolicy: &iserviceconfig.BalancerConfig{Name: roundrobin.Name}, + }, + wantAddrs: []resolver.Address{ + testAddrWithAttrs("addr-1-1", 20, 90, "test-priority", &internal.LocalityID{Zone: "test-zone-1"}), + testAddrWithAttrs("addr-1-2", 20, 10, "test-priority", &internal.LocalityID{Zone: "test-zone-1"}), + testAddrWithAttrs("addr-2-1", 80, 90, "test-priority", &internal.LocalityID{Zone: "test-zone-2"}), + testAddrWithAttrs("addr-2-2", 80, 10, "test-priority", &internal.LocalityID{Zone: "test-zone-2"}), + }, + }, + { + name: "ring_hash as child", + localities: []xdsresource.Locality{ + { + Endpoints: []xdsresource.Endpoint{ + {Address: "addr-1-1", HealthStatus: xdsresource.EndpointHealthStatusHealthy, Weight: 90}, + {Address: "addr-1-2", HealthStatus: xdsresource.EndpointHealthStatusHealthy, Weight: 10}, + }, + ID: internal.LocalityID{Zone: "test-zone-1"}, + Weight: 20, + }, + { + Endpoints: []xdsresource.Endpoint{ + {Address: "addr-2-1", HealthStatus: xdsresource.EndpointHealthStatusHealthy, Weight: 90}, + {Address: "addr-2-2", HealthStatus: xdsresource.EndpointHealthStatusHealthy, Weight: 10}, + }, + ID: internal.LocalityID{Zone: "test-zone-2"}, + Weight: 80, + }, + }, + priorityName: "test-priority", + childPolicy: &iserviceconfig.BalancerConfig{Name: ringhash.Name, Config: &ringhash.LBConfig{MinRingSize: 1, MaxRingSize: 2}}, + // lrsServer is nil, so LRS policy will not be used. + wantConfig: &clusterimpl.LBConfig{ + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: ringhash.Name, + Config: &ringhash.LBConfig{MinRingSize: 1, MaxRingSize: 2}, + }, + }, + wantAddrs: []resolver.Address{ + testAddrWithAttrs("addr-1-1", 20, 90, "test-priority", &internal.LocalityID{Zone: "test-zone-1"}), + testAddrWithAttrs("addr-1-2", 20, 10, "test-priority", &internal.LocalityID{Zone: "test-zone-1"}), + testAddrWithAttrs("addr-2-1", 80, 90, "test-priority", &internal.LocalityID{Zone: "test-zone-2"}), + testAddrWithAttrs("addr-2-2", 80, 10, "test-priority", &internal.LocalityID{Zone: "test-zone-2"}), + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, got1, err := priorityLocalitiesToClusterImpl(tt.localities, tt.priorityName, tt.mechanism, nil, tt.childPolicy) + if (err != nil) != tt.wantErr { + t.Fatalf("priorityLocalitiesToClusterImpl() error = %v, wantErr %v", err, tt.wantErr) + } + if diff := cmp.Diff(got, tt.wantConfig); diff != "" { + t.Errorf("localitiesToWeightedTarget() diff (-got +want) %v", diff) + } + if diff := cmp.Diff(got1, tt.wantAddrs, cmp.AllowUnexported(attributes.Attributes{})); diff != "" { + t.Errorf("localitiesToWeightedTarget() diff (-got +want) %v", diff) + } + }) + } +} + +func assertString(f func() (string, error)) string { + s, err := f() + if err != nil { + panic(err.Error()) + } + return s +} + +func testAddrWithAttrs(addrStr string, localityWeight, endpointWeight uint32, priority string, lID *internal.LocalityID) resolver.Address { + addr := resolver.Address{Addr: addrStr} + path := []string{priority} + if lID != nil { + path = append(path, assertString(lID.ToString)) + addr = internal.SetLocalityID(addr, *lID) + } + addr = hierarchy.Set(addr, path) + addr = wrrlocality.SetAddrInfo(addr, wrrlocality.AddrInfo{LocalityWeight: localityWeight}) + addr = weightedroundrobin.SetAddrInfo(addr, weightedroundrobin.AddrInfo{Weight: localityWeight * endpointWeight}) + return addr +} + +func TestConvertClusterImplMapToOutlierDetection(t *testing.T) { + tests := []struct { + name string + ciCfgsMap map[string]*clusterimpl.LBConfig + odCfg outlierdetection.LBConfig + wantODCfgs map[string]*outlierdetection.LBConfig + }{ + { + name: "single-entry-noop", + ciCfgsMap: map[string]*clusterimpl.LBConfig{ + "child1": { + Cluster: "cluster1", + }, + }, + odCfg: outlierdetection.LBConfig{ + Interval: 1<<63 - 1, + }, + wantODCfgs: map[string]*outlierdetection.LBConfig{ + "child1": { + Interval: 1<<63 - 1, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: clusterimpl.Name, + Config: &clusterimpl.LBConfig{ + Cluster: "cluster1", + }, + }, + }, + }, + }, + { + name: "multiple-entries-noop", + ciCfgsMap: map[string]*clusterimpl.LBConfig{ + "child1": { + Cluster: "cluster1", + }, + "child2": { + Cluster: "cluster2", + }, + }, + odCfg: outlierdetection.LBConfig{ + Interval: 1<<63 - 1, + }, + wantODCfgs: map[string]*outlierdetection.LBConfig{ + "child1": { + Interval: 1<<63 - 1, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: clusterimpl.Name, + Config: &clusterimpl.LBConfig{ + Cluster: "cluster1", + }, + }, + }, + "child2": { + Interval: 1<<63 - 1, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: clusterimpl.Name, + Config: &clusterimpl.LBConfig{ + Cluster: "cluster2", + }, + }, + }, + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got := convertClusterImplMapToOutlierDetection(test.ciCfgsMap, test.odCfg) + if diff := cmp.Diff(got, test.wantODCfgs); diff != "" { + t.Fatalf("convertClusterImplMapToOutlierDetection() diff(-got +want) %v", diff) + } + }) + } +} diff --git a/xds/internal/balancer/clusterresolver/e2e_test/aggregate_cluster_test.go b/xds/internal/balancer/clusterresolver/e2e_test/aggregate_cluster_test.go new file mode 100644 index 000000000000..f466dcca7bda --- /dev/null +++ b/xds/internal/balancer/clusterresolver/e2e_test/aggregate_cluster_test.go @@ -0,0 +1,1181 @@ +/* + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package e2e_test + +import ( + "context" + "fmt" + "net" + "sort" + "strconv" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/pickfirst" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/serviceconfig" + "google.golang.org/grpc/status" + xdstestutils "google.golang.org/grpc/xds/internal/testutils" + "google.golang.org/grpc/xds/internal/xdsclient" + "google.golang.org/grpc/xds/internal/xdsclient/bootstrap" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource/version" + "google.golang.org/protobuf/types/known/wrapperspb" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3aggregateclusterpb "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/aggregate/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +// makeAggregateClusterResource returns an aggregate cluster resource with the +// given name and list of child names. +func makeAggregateClusterResource(name string, childNames []string) *v3clusterpb.Cluster { + return &v3clusterpb.Cluster{ + Name: name, + ClusterDiscoveryType: &v3clusterpb.Cluster_ClusterType{ + ClusterType: &v3clusterpb.Cluster_CustomClusterType{ + Name: "envoy.clusters.aggregate", + TypedConfig: testutils.MarshalAny(&v3aggregateclusterpb.ClusterConfig{ + Clusters: childNames, + }), + }, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + } +} + +// makeLogicalDNSClusterResource returns a LOGICAL_DNS cluster resource with the +// given name and given DNS host and port. +func makeLogicalDNSClusterResource(name, dnsHost string, dnsPort uint32) *v3clusterpb.Cluster { + return &v3clusterpb.Cluster{ + Name: name, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_LOGICAL_DNS}, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + LoadAssignment: &v3endpointpb.ClusterLoadAssignment{ + Endpoints: []*v3endpointpb.LocalityLbEndpoints{{ + LbEndpoints: []*v3endpointpb.LbEndpoint{{ + HostIdentifier: &v3endpointpb.LbEndpoint_Endpoint{ + Endpoint: &v3endpointpb.Endpoint{ + Address: &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: dnsHost, + PortSpecifier: &v3corepb.SocketAddress_PortValue{ + PortValue: dnsPort, + }, + }, + }, + }, + }, + }, + }}, + }}, + }, + } +} + +// setupDNS unregisters the DNS resolver and registers a manual resolver for the +// same scheme. This allows the test to mock the DNS resolution by supplying the +// addresses of the test backends. +// +// Returns the following: +// - a channel onto which the DNS target being resolved is written to by the +// mock DNS resolver +// - a channel to notify close of the DNS resolver +// - a channel to notify re-resolution requests to the DNS resolver +// - a manual resolver which is used to mock the actual DNS resolution +// - a cleanup function which re-registers the original DNS resolver +func setupDNS() (chan resolver.Target, chan struct{}, chan resolver.ResolveNowOptions, *manual.Resolver, func()) { + targetCh := make(chan resolver.Target, 1) + closeCh := make(chan struct{}, 1) + resolveNowCh := make(chan resolver.ResolveNowOptions, 1) + + mr := manual.NewBuilderWithScheme("dns") + mr.BuildCallback = func(target resolver.Target, _ resolver.ClientConn, _ resolver.BuildOptions) { targetCh <- target } + mr.CloseCallback = func() { closeCh <- struct{}{} } + mr.ResolveNowCallback = func(opts resolver.ResolveNowOptions) { resolveNowCh <- opts } + + dnsResolverBuilder := resolver.Get("dns") + resolver.Register(mr) + + return targetCh, closeCh, resolveNowCh, mr, func() { resolver.Register(dnsResolverBuilder) } +} + +// TestAggregateCluster_WithTwoEDSClusters tests the case where the top-level +// cluster resource is an aggregate cluster. It verifies that RPCs fail when the +// management server has not responded to all requested EDS resources, and also +// that RPCs are routed to the highest priority cluster once all requested EDS +// resources have been sent by the management server. +func (s) TestAggregateCluster_WithTwoEDSClusters(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Start an xDS management server that pushes the EDS resource names onto a + // channel when requested. + edsResourceNameCh := make(chan []string, 1) + managementServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{ + OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + if req.GetTypeUrl() != version.V3EndpointsURL { + return nil + } + if len(req.GetResourceNames()) == 0 { + // This is the case for ACKs. Do nothing here. + return nil + } + select { + case edsResourceNameCh <- req.GetResourceNames(): + case <-ctx.Done(): + } + return nil + }, + AllowResourceSubset: true, + }) + defer cleanup() + + // Start two test backends and extract their host and port. The first + // backend belongs to EDS cluster "cluster-1", while the second backend + // belongs to EDS cluster "cluster-2". + servers, cleanup2 := startTestServiceBackends(t, 2) + defer cleanup2() + addrs, ports := backendAddressesAndPorts(t, servers) + + // Configure an aggregate cluster, two EDS clusters and only one endpoints + // resource (corresponding to the first EDS cluster) in the management + // server. + const clusterName1 = clusterName + "-cluster-1" + const clusterName2 = clusterName + "-cluster-2" + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{ + makeAggregateClusterResource(clusterName, []string{clusterName1, clusterName2}), + e2e.DefaultCluster(clusterName1, "", e2e.SecurityLevelNone), + e2e.DefaultCluster(clusterName2, "", e2e.SecurityLevelNone), + }, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(clusterName1, "localhost", []uint32{uint32(ports[0])})}, + SkipValidation: true, + } + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create xDS client, configure cds_experimental LB policy with a manual + // resolver, and dial the test backends. + cc, cleanup := setupAndDial(t, bootstrapContents) + defer cleanup() + + // Wait for both EDS resources to be requested. + func() { + for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) { + select { + case names := <-edsResourceNameCh: + // Copy and sort the sortedNames to avoid racing with an + // OnStreamRequest call. + sortedNames := make([]string, len(names)) + copy(sortedNames, names) + sort.Strings(sortedNames) + if cmp.Equal(sortedNames, []string{clusterName1, clusterName2}) { + return + } + default: + } + } + }() + if ctx.Err() != nil { + t.Fatalf("Timeout when waiting for all EDS resources %v to be requested", []string{clusterName1, clusterName2}) + } + + // Make an RPC with a short deadline. We expect this RPC to not succeed + // because the management server has not responded with all EDS resources + // requested. + client := testgrpc.NewTestServiceClient(cc) + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := client.EmptyCall(sCtx, &testpb.Empty{}); status.Code(err) != codes.DeadlineExceeded { + t.Fatalf("EmptyCall() code %s, want %s", status.Code(err), codes.DeadlineExceeded) + } + + // Update the management server with the second EDS resource. + resources.Endpoints = append(resources.Endpoints, e2e.DefaultEndpoint(clusterName2, "localhost", []uint32{uint32(ports[1])})) + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Make an RPC and ensure that it gets routed to cluster-1, implicitly + // higher priority than cluster-2. + peer := &peer.Peer{} + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer), grpc.WaitForReady(true)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + if peer.Addr.String() != addrs[0].Addr { + t.Fatalf("EmptyCall() routed to backend %q, want %q", peer.Addr, addrs[0].Addr) + } +} + +// TestAggregateCluster_WithTwoEDSClusters_PrioritiesChange tests the case where +// the top-level cluster resource is an aggregate cluster. It verifies that RPCs +// are routed to the highest priority EDS cluster. +func (s) TestAggregateCluster_WithTwoEDSClusters_PrioritiesChange(t *testing.T) { + // Start an xDS management server. + managementServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + defer cleanup() + + // Start two test backends and extract their host and port. The first + // backend belongs to EDS cluster "cluster-1", while the second backend + // belongs to EDS cluster "cluster-2". + servers, cleanup2 := startTestServiceBackends(t, 2) + defer cleanup2() + addrs, ports := backendAddressesAndPorts(t, servers) + + // Configure an aggregate cluster, two EDS clusters and the corresponding + // endpoints resources in the management server. + const clusterName1 = clusterName + "cluster-1" + const clusterName2 = clusterName + "cluster-2" + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{ + makeAggregateClusterResource(clusterName, []string{clusterName1, clusterName2}), + e2e.DefaultCluster(clusterName1, "", e2e.SecurityLevelNone), + e2e.DefaultCluster(clusterName2, "", e2e.SecurityLevelNone), + }, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{ + e2e.DefaultEndpoint(clusterName1, "localhost", []uint32{uint32(ports[0])}), + e2e.DefaultEndpoint(clusterName2, "localhost", []uint32{uint32(ports[1])}), + }, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create xDS client, configure cds_experimental LB policy with a manual + // resolver, and dial the test backends. + cc, cleanup := setupAndDial(t, bootstrapContents) + defer cleanup() + + // Make an RPC and ensure that it gets routed to cluster-1, implicitly + // higher priority than cluster-2. + client := testgrpc.NewTestServiceClient(cc) + peer := &peer.Peer{} + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer), grpc.WaitForReady(true)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + if peer.Addr.String() != addrs[0].Addr { + t.Fatalf("EmptyCall() routed to backend %q, want %q", peer.Addr, addrs[0].Addr) + } + + // Swap the priorities of the EDS clusters in the aggregate cluster. + resources.Clusters = []*v3clusterpb.Cluster{ + makeAggregateClusterResource(clusterName, []string{clusterName2, clusterName1}), + e2e.DefaultCluster(clusterName1, "", e2e.SecurityLevelNone), + e2e.DefaultCluster(clusterName2, "", e2e.SecurityLevelNone), + } + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Wait for RPCs to get routed to cluster-2, which is now implicitly higher + // priority than cluster-1, after the priority switch above. + for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) { + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer), grpc.WaitForReady(true)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + if peer.Addr.String() == addrs[1].Addr { + break + } + } + if ctx.Err() != nil { + t.Fatal("Timeout waiting for RPCs to be routed to cluster-2 after priority switch") + } +} + +func hostAndPortFromAddress(t *testing.T, addr string) (string, uint32) { + t.Helper() + + host, p, err := net.SplitHostPort(addr) + if err != nil { + t.Fatalf("Invalid serving address: %v", addr) + } + port, err := strconv.ParseUint(p, 10, 32) + if err != nil { + t.Fatalf("Invalid serving port %q: %v", p, err) + } + return host, uint32(port) +} + +// TestAggregateCluster_WithOneDNSCluster tests the case where the top-level +// cluster resource is an aggregate cluster that resolves to a single +// LOGICAL_DNS cluster. The test verifies that RPCs can be made to backends that +// make up the LOGICAL_DNS cluster. +func (s) TestAggregateCluster_WithOneDNSCluster(t *testing.T) { + // Start an xDS management server. + managementServer, nodeID, bootstrapContents, _, cleanup2 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + defer cleanup2() + + // Start a test service backend. + server := stubserver.StartTestService(t, nil) + defer server.Stop() + host, port := hostAndPortFromAddress(t, server.Address) + + // Configure an aggregate cluster pointing to a single LOGICAL_DNS cluster. + const dnsClusterName = clusterName + "-dns" + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{ + makeAggregateClusterResource(clusterName, []string{dnsClusterName}), + makeLogicalDNSClusterResource(dnsClusterName, host, uint32(port)), + }, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create xDS client, configure cds_experimental LB policy with a manual + // resolver, and dial the test backends. + cc, cleanup := setupAndDial(t, bootstrapContents) + defer cleanup() + + // Make an RPC and ensure that it gets routed to the first backend since the + // child policy for a LOGICAL_DNS cluster is pick_first by default. + client := testgrpc.NewTestServiceClient(cc) + peer := &peer.Peer{} + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer), grpc.WaitForReady(true)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + if peer.Addr.String() != server.Address { + t.Fatalf("EmptyCall() routed to backend %q, want %q", peer.Addr, server.Address) + } +} + +// Tests the case where the top-level cluster resource is an aggregate cluster +// that resolves to a single LOGICAL_DNS cluster. The specified dns hostname is +// expected to fail url parsing. The test verifies that the channel moves to +// TRANSIENT_FAILURE. +func (s) TestAggregateCluster_WithOneDNSCluster_ParseFailure(t *testing.T) { + // Start an xDS management server. + managementServer, nodeID, bootstrapContents, _, cleanup2 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + defer cleanup2() + + // Configure an aggregate cluster pointing to a single LOGICAL_DNS cluster. + const dnsClusterName = clusterName + "-dns" + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{ + makeAggregateClusterResource(clusterName, []string{dnsClusterName}), + makeLogicalDNSClusterResource(dnsClusterName, "%gh&%ij", uint32(8080)), + }, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create xDS client, configure cds_experimental LB policy with a manual + // resolver, and dial the test backends. + cc, cleanup := setupAndDial(t, bootstrapContents) + defer cleanup() + + // Ensure that the ClientConn moves to TransientFailure. + for state := cc.GetState(); state != connectivity.TransientFailure; state = cc.GetState() { + if !cc.WaitForStateChange(ctx, state) { + t.Fatalf("Timed out waiting for state change. got %v; want %v", state, connectivity.TransientFailure) + } + } +} + +// Tests the case where the top-level cluster resource is an aggregate cluster +// that resolves to a single LOGICAL_DNS cluster. The test verifies that RPCs +// can be made to backends that make up the LOGICAL_DNS cluster. The hostname of +// the LOGICAL_DNS cluster is updated, and the test verifies that RPCs can be +// made to backends that the new hostname resolves to. +func (s) TestAggregateCluster_WithOneDNSCluster_HostnameChange(t *testing.T) { + // Start an xDS management server. + managementServer, nodeID, bootstrapContents, _, cleanup1 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + defer cleanup1() + + // Start two test backends and extract their host and port. The first + // backend is used initially for the LOGICAL_DNS cluster and an update + // switches the cluster to use the second backend. + servers, cleanup2 := startTestServiceBackends(t, 2) + defer cleanup2() + + // Configure an aggregate cluster pointing to a single LOGICAL_DNS cluster. + const dnsClusterName = clusterName + "-dns" + dnsHostName, dnsPort := hostAndPortFromAddress(t, servers[0].Address) + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{ + makeAggregateClusterResource(clusterName, []string{dnsClusterName}), + makeLogicalDNSClusterResource(dnsClusterName, dnsHostName, dnsPort), + }, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create xDS client, configure cds_experimental LB policy with a manual + // resolver, and dial the test backends. + cc, cleanup := setupAndDial(t, bootstrapContents) + defer cleanup() + + // Make an RPC and ensure that it gets routed to the first backend since the + // child policy for a LOGICAL_DNS cluster is pick_first by default. + client := testgrpc.NewTestServiceClient(cc) + peer := &peer.Peer{} + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer), grpc.WaitForReady(true)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + if peer.Addr.String() != servers[0].Address { + t.Fatalf("EmptyCall() routed to backend %q, want %q", peer.Addr, servers[0].Address) + } + + // Update the LOGICAL_DNS cluster's hostname to point to the second backend. + dnsHostName, dnsPort = hostAndPortFromAddress(t, servers[1].Address) + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{ + makeAggregateClusterResource(clusterName, []string{dnsClusterName}), + makeLogicalDNSClusterResource(dnsClusterName, dnsHostName, dnsPort), + }, + SkipValidation: true, + } + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Ensure that traffic moves to the second backend eventually. + for ctx.Err() == nil { + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + if peer.Addr.String() == servers[1].Address { + break + } + } + if ctx.Err() != nil { + t.Fatal("Timeout when waiting for RPCs to switch to the second backend") + } +} + +// TestAggregateCluster_WithEDSAndDNS tests the case where the top-level cluster +// resource is an aggregate cluster that resolves to an EDS and a LOGICAL_DNS +// cluster. The test verifies that RPCs fail until both clusters are resolved to +// endpoints, and RPCs are routed to the higher priority EDS cluster. +func (s) TestAggregateCluster_WithEDSAndDNS(t *testing.T) { + dnsTargetCh, _, _, dnsR, cleanup1 := setupDNS() + defer cleanup1() + + // Start an xDS management server that pushes the name of the requested EDS + // resource onto a channel. + edsResourceCh := make(chan string, 1) + managementServer, nodeID, bootstrapContents, _, cleanup2 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{ + OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + if req.GetTypeUrl() != version.V3EndpointsURL { + return nil + } + if len(req.GetResourceNames()) > 0 { + select { + case edsResourceCh <- req.GetResourceNames()[0]: + default: + } + } + return nil + }, + AllowResourceSubset: true, + }) + defer cleanup2() + + // Start two test backends and extract their host and port. The first + // backend is used for the EDS cluster and the second backend is used for + // the LOGICAL_DNS cluster. + servers, cleanup3 := startTestServiceBackends(t, 2) + defer cleanup3() + addrs, ports := backendAddressesAndPorts(t, servers) + + // Configure an aggregate cluster pointing to an EDS and DNS cluster. Also + // configure an endpoints resource for the EDS cluster. + const ( + edsClusterName = clusterName + "-eds" + dnsClusterName = clusterName + "-dns" + dnsHostName = "dns_host" + dnsPort = uint32(8080) + ) + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{ + makeAggregateClusterResource(clusterName, []string{edsClusterName, dnsClusterName}), + e2e.DefaultCluster(edsClusterName, "", e2e.SecurityLevelNone), + makeLogicalDNSClusterResource(dnsClusterName, dnsHostName, dnsPort), + }, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(edsClusterName, "localhost", []uint32{uint32(ports[0])})}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create xDS client, configure cds_experimental LB policy with a manual + // resolver, and dial the test backends. + cc, cleanup := setupAndDial(t, bootstrapContents) + defer cleanup() + + // Ensure that an EDS request is sent for the expected resource name. + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for EDS request to be received on the management server") + case name := <-edsResourceCh: + if name != edsClusterName { + t.Fatalf("Received EDS request with resource name %q, want %q", name, edsClusterName) + } + } + + // Ensure that the DNS resolver is started for the expected target. + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for DNS resolver to be started") + case target := <-dnsTargetCh: + got, want := target.Endpoint(), fmt.Sprintf("%s:%d", dnsHostName, dnsPort) + if got != want { + t.Fatalf("DNS resolution started for target %q, want %q", got, want) + } + } + + // Make an RPC with a short deadline. We expect this RPC to not succeed + // because the DNS resolver has not responded with endpoint addresses. + client := testgrpc.NewTestServiceClient(cc) + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := client.EmptyCall(sCtx, &testpb.Empty{}); status.Code(err) != codes.DeadlineExceeded { + t.Fatalf("EmptyCall() code %s, want %s", status.Code(err), codes.DeadlineExceeded) + } + + // Update DNS resolver with test backend addresses. + dnsR.UpdateState(resolver.State{Addresses: addrs[1:]}) + + // Make an RPC and ensure that it gets routed to the first backend since the + // EDS cluster is of higher priority than the LOGICAL_DNS cluster. + peer := &peer.Peer{} + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer), grpc.WaitForReady(true)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + if peer.Addr.String() != addrs[0].Addr { + t.Fatalf("EmptyCall() routed to backend %q, want %q", peer.Addr, addrs[0].Addr) + } +} + +// TestAggregateCluster_SwitchEDSAndDNS tests the case where the top-level +// cluster resource is an aggregate cluster. It initially resolves to a single +// EDS cluster. The test verifies that RPCs are routed to backends in the EDS +// cluster. Subsequently, the aggregate cluster resolves to a single DNS +// cluster. The test verifies that RPCs are successful, this time to backends in +// the DNS cluster. +func (s) TestAggregateCluster_SwitchEDSAndDNS(t *testing.T) { + // Start an xDS management server. + managementServer, nodeID, bootstrapContents, _, cleanup2 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + defer cleanup2() + + // Start two test backends and extract their host and port. The first + // backend is used for the EDS cluster and the second backend is used for + // the LOGICAL_DNS cluster. + servers, cleanup3 := startTestServiceBackends(t, 2) + defer cleanup3() + addrs, ports := backendAddressesAndPorts(t, servers) + dnsHostName, dnsPort := hostAndPortFromAddress(t, addrs[1].Addr) + + // Configure an aggregate cluster pointing to a single EDS cluster. Also, + // configure the underlying EDS cluster (and the corresponding endpoints + // resource) and DNS cluster (will be used later in the test). + const dnsClusterName = clusterName + "-dns" + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{ + makeAggregateClusterResource(clusterName, []string{edsServiceName}), + e2e.DefaultCluster(edsServiceName, "", e2e.SecurityLevelNone), + makeLogicalDNSClusterResource(dnsClusterName, dnsHostName, dnsPort), + }, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(edsServiceName, "localhost", []uint32{uint32(ports[0])})}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create xDS client, configure cds_experimental LB policy with a manual + // resolver, and dial the test backends. + cc, cleanup := setupAndDial(t, bootstrapContents) + defer cleanup() + + // Ensure that the RPC is routed to the appropriate backend. + client := testgrpc.NewTestServiceClient(cc) + peer := &peer.Peer{} + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer), grpc.WaitForReady(true)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + if peer.Addr.String() != addrs[0].Addr { + t.Fatalf("EmptyCall() routed to backend %q, want %q", peer.Addr, addrs[0].Addr) + } + + // Update the aggregate cluster to point to a single DNS cluster. + resources.Clusters = []*v3clusterpb.Cluster{ + makeAggregateClusterResource(clusterName, []string{dnsClusterName}), + e2e.DefaultCluster(edsServiceName, "", e2e.SecurityLevelNone), + makeLogicalDNSClusterResource(dnsClusterName, dnsHostName, dnsPort), + } + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Ensure that start getting routed to the backend corresponding to the + // LOGICAL_DNS cluster. + for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) { + client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer)) + if peer.Addr.String() == addrs[1].Addr { + break + } + } + if ctx.Err() != nil { + t.Fatalf("Timeout when waiting for RPCs to be routed to backend %q in the DNS cluster", addrs[1].Addr) + } +} + +// TestAggregateCluster_BadEDS_GoodToBadDNS tests the case where the top-level +// cluster is an aggregate cluster that resolves to an EDS and LOGICAL_DNS +// cluster. The test first asserts that no RPCs can be made after receiving an +// EDS response with zero endpoints because no update has been received from the +// DNS resolver yet. Once the DNS resolver pushes an update, the test verifies +// that we switch to the DNS cluster and can make a successful RPC. At this +// point when the DNS cluster returns an error, the test verifies that RPCs are +// still successful. This is the expected behavior because the cluster resolver +// policy eats errors from DNS Resolver after it has returned an error. +func (s) TestAggregateCluster_BadEDS_GoodToBadDNS(t *testing.T) { + dnsTargetCh, _, _, dnsR, cleanup1 := setupDNS() + defer cleanup1() + + // Start an xDS management server. + managementServer, nodeID, bootstrapContents, _, cleanup2 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + defer cleanup2() + + // Start two test backends. + servers, cleanup3 := startTestServiceBackends(t, 2) + defer cleanup3() + addrs, _ := backendAddressesAndPorts(t, servers) + + // Configure an aggregate cluster pointing to an EDS and LOGICAL_DNS + // cluster. Also configure an endpoints resource for the EDS cluster which + // triggers a NACK. + const ( + edsClusterName = clusterName + "-eds" + dnsClusterName = clusterName + "-dns" + dnsHostName = "dns_host" + dnsPort = uint32(8080) + ) + emptyEndpointResource := e2e.DefaultEndpoint(edsServiceName, "localhost", nil) + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{ + makeAggregateClusterResource(clusterName, []string{edsClusterName, dnsClusterName}), + e2e.DefaultCluster(edsClusterName, edsServiceName, e2e.SecurityLevelNone), + makeLogicalDNSClusterResource(dnsClusterName, dnsHostName, dnsPort), + }, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{emptyEndpointResource}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create xDS client, configure cds_experimental LB policy with a manual + // resolver, and dial the test backends. + cc, cleanup := setupAndDial(t, bootstrapContents) + defer cleanup() + + // Make an RPC with a short deadline. We expect this RPC to not succeed + // because the EDS resource came back with no endpoints, and we are yet to + // push an update through the DNS resolver. + client := testgrpc.NewTestServiceClient(cc) + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := client.EmptyCall(sCtx, &testpb.Empty{}); status.Code(err) != codes.DeadlineExceeded { + t.Fatalf("EmptyCall() code %s, want %s", status.Code(err), codes.DeadlineExceeded) + } + + // Ensure that the DNS resolver is started for the expected target. + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for DNS resolver to be started") + case target := <-dnsTargetCh: + got, want := target.Endpoint(), fmt.Sprintf("%s:%d", dnsHostName, dnsPort) + if got != want { + t.Fatalf("DNS resolution started for target %q, want %q", got, want) + } + } + + // Update DNS resolver with test backend addresses. + dnsR.UpdateState(resolver.State{Addresses: addrs}) + + // Ensure that RPCs start getting routed to the first backend since the + // child policy for a LOGICAL_DNS cluster is pick_first by default. + for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) { + peer := &peer.Peer{} + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer)); err != nil { + t.Logf("EmptyCall() failed: %v", err) + continue + } + if peer.Addr.String() == addrs[0].Addr { + break + } + } + if ctx.Err() != nil { + t.Fatalf("Timeout when waiting for RPCs to be routed to backend %q in the DNS cluster", addrs[0].Addr) + } + + // Push an error from the DNS resolver as well. + dnsErr := fmt.Errorf("DNS error") + dnsR.ReportError(dnsErr) + + // Ensure that RPCs continue to succeed for the next second. + for end := time.Now().Add(time.Second); time.Now().Before(end); <-time.After(defaultTestShortTimeout) { + peer := &peer.Peer{} + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + if peer.Addr.String() != addrs[0].Addr { + t.Fatalf("EmptyCall() routed to backend %q, want %q", peer.Addr, addrs[0].Addr) + } + } +} + +// TestAggregateCluster_BadEDS_GoodToBadDNS tests the case where the top-level +// cluster is an aggregate cluster that resolves to an EDS and LOGICAL_DNS +// cluster. The test first sends an EDS response which triggers an NACK. Once +// the DNS resolver pushes an update, the test verifies that we switch to the +// DNS cluster and can make a successful RPC. +func (s) TestAggregateCluster_BadEDSFromError_GoodToBadDNS(t *testing.T) { + // Start an xDS management server. + managementServer, nodeID, bootstrapContents, _, cleanup2 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + defer cleanup2() + + // Start a test service backend. + server := stubserver.StartTestService(t, nil) + defer server.Stop() + dnsHostName, dnsPort := hostAndPortFromAddress(t, server.Address) + + // Configure an aggregate cluster pointing to an EDS and LOGICAL_DNS + // cluster. Also configure an empty endpoints resource for the EDS cluster + // that contains no endpoints. + const ( + edsClusterName = clusterName + "-eds" + dnsClusterName = clusterName + "-dns" + ) + nackEndpointResource := e2e.DefaultEndpoint(edsServiceName, "localhost", nil) + nackEndpointResource.Endpoints = []*v3endpointpb.LocalityLbEndpoints{ + { + LoadBalancingWeight: &wrapperspb.UInt32Value{ + Value: 0, // causes an NACK + }, + }, + } + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{ + makeAggregateClusterResource(clusterName, []string{edsClusterName, dnsClusterName}), + e2e.DefaultCluster(edsClusterName, edsServiceName, e2e.SecurityLevelNone), + makeLogicalDNSClusterResource(dnsClusterName, dnsHostName, dnsPort), + }, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{nackEndpointResource}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create xDS client, configure cds_experimental LB policy with a manual + // resolver, and dial the test backends. + cc, cleanup := setupAndDial(t, bootstrapContents) + defer cleanup() + + // Ensure that RPCs start getting routed to the first backend since the + // child policy for a LOGICAL_DNS cluster is pick_first by default. + pickfirst.CheckRPCsToBackend(ctx, cc, resolver.Address{Addr: server.Address}) +} + +// TestAggregateCluster_BadDNS_GoodEDS tests the case where the top-level +// cluster is an aggregate cluster that resolves to an LOGICAL_DNS and EDS +// cluster. When the DNS Resolver returns an error and EDS cluster returns a +// good update, this test verifies the cluster_resolver balancer correctly falls +// back from the LOGICAL_DNS cluster to the EDS cluster. +func (s) TestAggregateCluster_BadDNS_GoodEDS(t *testing.T) { + // Start an xDS management server. + managementServer, nodeID, bootstrapContents, _, cleanup2 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + defer cleanup2() + + // Start a test service backend. + server := stubserver.StartTestService(t, nil) + defer server.Stop() + _, edsPort := hostAndPortFromAddress(t, server.Address) + + // Configure an aggregate cluster pointing to an LOGICAL_DNS and EDS + // cluster. Also configure an endpoints resource for the EDS cluster. + const ( + edsClusterName = clusterName + "-eds" + dnsClusterName = clusterName + "-dns" + ) + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{ + makeAggregateClusterResource(clusterName, []string{dnsClusterName, edsClusterName}), + makeLogicalDNSClusterResource(dnsClusterName, "bad.ip.v4.address", 8080), + e2e.DefaultCluster(edsClusterName, edsServiceName, e2e.SecurityLevelNone), + }, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(edsServiceName, "localhost", []uint32{uint32(edsPort)})}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create xDS client, configure cds_experimental LB policy with a manual + // resolver, and dial the test backends. + cc, cleanup := setupAndDial(t, bootstrapContents) + defer cleanup() + + // RPCs should work, higher level DNS cluster errors so should fallback to + // EDS cluster. + client := testgrpc.NewTestServiceClient(cc) + peer := &peer.Peer{} + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer), grpc.WaitForReady(true)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + if peer.Addr.String() != server.Address { + t.Fatalf("EmptyCall() routed to backend %q, want %q", peer.Addr, server.Address) + } +} + +// TestAggregateCluster_BadEDS_BadDNS tests the case where the top-level cluster +// is an aggregate cluster that resolves to an EDS and LOGICAL_DNS cluster. When +// the EDS request returns a resource that contains no endpoints, the test +// verifies that we switch to the DNS cluster. When the DNS cluster returns an +// error, the test verifies that RPCs fail with the error triggered by the DNS +// Discovery Mechanism (from sending an empty address list down). +func (s) TestAggregateCluster_BadEDS_BadDNS(t *testing.T) { + // Start an xDS management server. + managementServer, nodeID, bootstrapContents, _, cleanup2 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + defer cleanup2() + + // Configure an aggregate cluster pointing to an EDS and LOGICAL_DNS + // cluster. Also configure an empty endpoints resource for the EDS cluster + // that contains no endpoints. + const ( + edsClusterName = clusterName + "-eds" + dnsClusterName = clusterName + "-dns" + ) + emptyEndpointResource := e2e.DefaultEndpoint(edsServiceName, "localhost", nil) + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{ + makeAggregateClusterResource(clusterName, []string{edsClusterName, dnsClusterName}), + e2e.DefaultCluster(edsClusterName, edsServiceName, e2e.SecurityLevelNone), + makeLogicalDNSClusterResource(dnsClusterName, "bad.ip.v4.address", 8080), + }, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{emptyEndpointResource}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create xDS client, configure cds_experimental LB policy with a manual + // resolver, and dial the test backends. + cc, cleanup := setupAndDial(t, bootstrapContents) + defer cleanup() + + // Ensure that the error from the DNS Resolver leads to an empty address + // update for both priorities. + client := testgrpc.NewTestServiceClient(cc) + for ctx.Err() == nil { + _, err := client.EmptyCall(ctx, &testpb.Empty{}) + if err == nil { + t.Fatal("EmptyCall() succeeded when expected to fail") + } + if status.Code(err) == codes.Unavailable && strings.Contains(err.Error(), "produced zero addresses") { + break + } + } + if ctx.Err() != nil { + t.Fatalf("Timeout when waiting for RPCs to fail with expected code and error") + } +} + +// TestAggregateCluster_NoFallback_EDSNackedWithPreviousGoodUpdate tests the +// scenario where the top-level cluster is an aggregate cluster that resolves to +// an EDS and LOGICAL_DNS cluster. The management server first sends a good EDS +// response for the EDS cluster and the test verifies that RPCs get routed to +// the EDS cluster. The management server then sends a bad EDS response. The +// test verifies that the cluster_resolver LB policy continues to use the +// previously received good update and that RPCs still get routed to the EDS +// cluster. +func (s) TestAggregateCluster_NoFallback_EDSNackedWithPreviousGoodUpdate(t *testing.T) { + // Start an xDS management server. + mgmtServer, nodeID, bootstrapContents, _, cleanup2 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + defer cleanup2() + + // Start two test backends and extract their host and port. The first + // backend is used for the EDS cluster and the second backend is used for + // the LOGICAL_DNS cluster. + servers, cleanup3 := startTestServiceBackends(t, 2) + defer cleanup3() + addrs, ports := backendAddressesAndPorts(t, servers) + dnsHostName, dnsPort := hostAndPortFromAddress(t, servers[1].Address) + + // Configure an aggregate cluster pointing to an EDS and DNS cluster. Also + // configure an endpoints resource for the EDS cluster. + const ( + edsClusterName = clusterName + "-eds" + dnsClusterName = clusterName + "-dns" + ) + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{ + makeAggregateClusterResource(clusterName, []string{edsClusterName, dnsClusterName}), + e2e.DefaultCluster(edsClusterName, "", e2e.SecurityLevelNone), + makeLogicalDNSClusterResource(dnsClusterName, dnsHostName, dnsPort), + }, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(edsClusterName, "localhost", []uint32{uint32(ports[0])})}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create xDS client, configure cds_experimental LB policy with a manual + // resolver, and dial the test backends. + cc, cleanup := setupAndDial(t, bootstrapContents) + defer cleanup() + + // Make an RPC and ensure that it gets routed to the first backend since the + // EDS cluster is of higher priority than the LOGICAL_DNS cluster. + client := testgrpc.NewTestServiceClient(cc) + peer := &peer.Peer{} + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer), grpc.WaitForReady(true)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + if peer.Addr.String() != addrs[0].Addr { + t.Fatalf("EmptyCall() routed to backend %q, want %q", peer.Addr, addrs[0].Addr) + } + + // Push an EDS resource from the management server that is expected to be + // NACKed by the xDS client. Since the cluster_resolver LB policy has a + // previously received good EDS resource, it will continue to use that. + resources.Endpoints[0].Endpoints[0].LbEndpoints[0].LoadBalancingWeight = &wrapperspb.UInt32Value{Value: 0} + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Ensure that RPCs continue to get routed to the EDS cluster for the next + // second. + for end := time.Now().Add(time.Second); time.Now().Before(end); <-time.After(defaultTestShortTimeout) { + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + if peer.Addr.String() != addrs[0].Addr { + t.Fatalf("EmptyCall() routed to backend %q, want %q", peer.Addr, addrs[0].Addr) + } + } +} + +// TestAggregateCluster_Fallback_EDSNackedWithoutPreviousGoodUpdate tests the +// scenario where the top-level cluster is an aggregate cluster that resolves to +// an EDS and LOGICAL_DNS cluster. The management server sends a bad EDS +// response. The test verifies that the cluster_resolver LB policy falls back to +// the LOGICAL_DNS cluster, because it is supposed to treat the bad EDS response +// as though it received an update with no endpoints. +func (s) TestAggregateCluster_Fallback_EDSNackedWithoutPreviousGoodUpdate(t *testing.T) { + // Start an xDS management server. + mgmtServer, nodeID, bootstrapContents, _, cleanup2 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + defer cleanup2() + + // Start two test backends and extract their host and port. The first + // backend is used for the EDS cluster and the second backend is used for + // the LOGICAL_DNS cluster. + servers, cleanup3 := startTestServiceBackends(t, 2) + defer cleanup3() + addrs, ports := backendAddressesAndPorts(t, servers) + dnsHostName, dnsPort := hostAndPortFromAddress(t, servers[1].Address) + + // Configure an aggregate cluster pointing to an EDS and DNS cluster. + const ( + edsClusterName = clusterName + "-eds" + dnsClusterName = clusterName + "-dns" + ) + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{ + makeAggregateClusterResource(clusterName, []string{edsClusterName, dnsClusterName}), + e2e.DefaultCluster(edsClusterName, "", e2e.SecurityLevelNone), + makeLogicalDNSClusterResource(dnsClusterName, dnsHostName, dnsPort), + }, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(edsClusterName, "localhost", []uint32{uint32(ports[0])})}, + SkipValidation: true, + } + + // Set a load balancing weight of 0 for the backend in the EDS resource. + // This is expected to be NACKed by the xDS client. Since the + // cluster_resolver LB policy has no previously received good EDS resource, + // it will treat this as though it received an update with no endpoints. + resources.Endpoints[0].Endpoints[0].LbEndpoints[0].LoadBalancingWeight = &wrapperspb.UInt32Value{Value: 0} + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create xDS client, configure cds_experimental LB policy with a manual + // resolver, and dial the test backends. + cc, cleanup := setupAndDial(t, bootstrapContents) + defer cleanup() + + // Make an RPC and ensure that it gets routed to the LOGICAL_DNS cluster. + peer := &peer.Peer{} + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer), grpc.WaitForReady(true)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + if peer.Addr.String() != addrs[1].Addr { + t.Fatalf("EmptyCall() routed to backend %q, want %q", peer.Addr, addrs[1].Addr) + } +} + +// TestAggregateCluster_Fallback_EDS_ResourceNotFound tests the scenario where +// the top-level cluster is an aggregate cluster that resolves to an EDS and +// LOGICAL_DNS cluster. The management server does not respond with the EDS +// cluster. The test verifies that the cluster_resolver LB policy falls back to +// the LOGICAL_DNS cluster in this case. +func (s) TestAggregateCluster_Fallback_EDS_ResourceNotFound(t *testing.T) { + // Start an xDS management server. + mgmtServer, nodeID, _, _, cleanup2 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + defer cleanup2() + + // Start a test backend for the LOGICAL_DNS cluster. + server := stubserver.StartTestService(t, nil) + defer server.Stop() + dnsHostName, dnsPort := hostAndPortFromAddress(t, server.Address) + + // Configure an aggregate cluster pointing to an EDS and DNS cluster. No + // endpoints are configured for the EDS cluster. + const ( + edsClusterName = clusterName + "-eds" + dnsClusterName = clusterName + "-dns" + ) + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{ + makeAggregateClusterResource(clusterName, []string{edsClusterName, dnsClusterName}), + e2e.DefaultCluster(edsClusterName, "", e2e.SecurityLevelNone), + makeLogicalDNSClusterResource(dnsClusterName, dnsHostName, dnsPort), + }, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create an xDS client talking to the above management server, configured + // with a short watch expiry timeout. + xdsClient, close, err := xdsclient.NewWithConfigForTesting(&bootstrap.Config{ + XDSServer: xdstestutils.ServerConfigForAddress(t, mgmtServer.Address), + NodeProto: &v3corepb.Node{Id: nodeID}, + }, defaultTestWatchExpiryTimeout, time.Duration(0)) + if err != nil { + t.Fatalf("failed to create xds client: %v", err) + } + defer close() + + // Create a manual resolver and push a service config specifying the use of + // the cds LB policy as the top-level LB policy, and a corresponding config + // with a single cluster. + r := manual.NewBuilderWithScheme("whatever") + jsonSC := fmt.Sprintf(`{ + "loadBalancingConfig":[{ + "cds_experimental":{ + "cluster": "%s" + } + }] + }`, clusterName) + scpr := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(jsonSC) + r.InitialState(xdsclient.SetClient(resolver.State{ServiceConfig: scpr}, xdsClient)) + + // Create a ClientConn. + cc, err := grpc.Dial(r.Scheme()+":///test.service", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + // Make an RPC and ensure that it gets routed to the LOGICAL_DNS cluster. + // Even though the EDS cluster is of higher priority, since the management + // server does not respond with an EDS resource, the cluster_resolver LB + // policy is expected to fallback to the LOGICAL_DNS cluster once the watch + // timeout expires. + peer := &peer.Peer{} + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer), grpc.WaitForReady(true)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + if peer.Addr.String() != server.Address { + t.Fatalf("EmptyCall() routed to backend %q, want %q", peer.Addr, server.Address) + } +} diff --git a/xds/internal/balancer/clusterresolver/e2e_test/balancer_test.go b/xds/internal/balancer/clusterresolver/e2e_test/balancer_test.go new file mode 100644 index 000000000000..4dbec18229f4 --- /dev/null +++ b/xds/internal/balancer/clusterresolver/e2e_test/balancer_test.go @@ -0,0 +1,432 @@ +/* + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package e2e_test + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/roundrobin" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/balancer/stub" + iserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/serviceconfig" + "google.golang.org/grpc/status" + "google.golang.org/grpc/xds/internal/balancer/clusterimpl" + "google.golang.org/grpc/xds/internal/balancer/outlierdetection" + "google.golang.org/grpc/xds/internal/balancer/priority" + "google.golang.org/grpc/xds/internal/balancer/wrrlocality" + "google.golang.org/grpc/xds/internal/xdsclient" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource/version" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/wrapperspb" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + + _ "google.golang.org/grpc/xds/internal/balancer/cdsbalancer" // Register the "cds_experimental" LB policy. +) + +// setupAndDial performs common setup across all tests +// +// - creates an xDS client with the passed in bootstrap contents +// - creates a manual resolver that configures `cds_experimental` as the +// top-level LB policy. +// - creates a ClientConn to talk to the test backends +// +// Returns a function to close the ClientConn and the xDS client. +func setupAndDial(t *testing.T, bootstrapContents []byte) (*grpc.ClientConn, func()) { + t.Helper() + + // Create an xDS client for use by the cluster_resolver LB policy. + xdsC, xdsClose, err := xdsclient.NewWithBootstrapContentsForTesting(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + + // Create a manual resolver and push a service config specifying the use of + // the cds LB policy as the top-level LB policy, and a corresponding config + // with a single cluster. + r := manual.NewBuilderWithScheme("whatever") + jsonSC := fmt.Sprintf(`{ + "loadBalancingConfig":[{ + "cds_experimental":{ + "cluster": "%s" + } + }] + }`, clusterName) + scpr := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(jsonSC) + r.InitialState(xdsclient.SetClient(resolver.State{ServiceConfig: scpr}, xdsC)) + + // Create a ClientConn and make a successful RPC. + cc, err := grpc.Dial(r.Scheme()+":///test.service", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + if err != nil { + xdsClose() + t.Fatalf("Failed to dial local test server: %v", err) + } + return cc, func() { + xdsClose() + cc.Close() + } +} + +// TestErrorFromParentLB_ConnectionError tests the case where the parent of the +// clusterresolver LB policy sends its a connection error. The parent policy, +// CDS LB policy, sends a connection error when the ADS stream to the management +// server breaks. The test verifies that there is no perceivable effect because +// of this connection error, and that RPCs continue to work (because the LB +// policies are expected to use previously received xDS resources). +func (s) TestErrorFromParentLB_ConnectionError(t *testing.T) { + // Create a listener to be used by the management server. The test will + // close this listener to simulate ADS stream breakage. + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("testutils.LocalTCPListener() failed: %v", err) + } + + // Start an xDS management server with the above restartable listener, and + // push a channel when the stream is closed. + streamClosedCh := make(chan struct{}, 1) + managementServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{ + Listener: lis, + OnStreamClosed: func(int64, *v3corepb.Node) { + select { + case streamClosedCh <- struct{}{}: + default: + } + }, + }) + defer cleanup() + + server := stubserver.StartTestService(t, nil) + defer server.Stop() + + // Configure cluster and endpoints resources in the management server. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, edsServiceName, e2e.SecurityLevelNone)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(edsServiceName, "localhost", []uint32{testutils.ParsePort(t, server.Address)})}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create xDS client, configure cds_experimental LB policy with a manual + // resolver, and dial the test backends. + cc, cleanup := setupAndDial(t, bootstrapContents) + defer cleanup() + + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + + // Close the listener and ensure that the ADS stream breaks. + lis.Close() + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for ADS stream to close") + default: + } + + // Ensure that RPCs continue to succeed for the next second. + for end := time.Now().Add(time.Second); time.Now().Before(end); <-time.After(defaultTestShortTimeout) { + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + } +} + +// TestErrorFromParentLB_ResourceNotFound tests the case where the parent of the +// clusterresolver LB policy sends it a resource-not-found error. The parent +// policy, CDS LB policy, sends a resource-not-found error when the cluster +// resource associated with these LB policies is removed by the management +// server. The test verifies that the associated EDS is canceled and RPCs fail. +// It also ensures that when the Cluster resource is added back, the EDS +// resource is re-requested and RPCs being to succeed. +func (s) TestErrorFromParentLB_ResourceNotFound(t *testing.T) { + // Start an xDS management server that uses a couple of channels to + // notify the test about the following events: + // - an EDS requested with the expected resource name is requested + // - EDS resource is unrequested, i.e, an EDS request with no resource name + // is received, which indicates that we are not longer interested in that + // resource. + edsResourceRequestedCh := make(chan struct{}, 1) + edsResourceCanceledCh := make(chan struct{}, 1) + managementServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{ + OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + if req.GetTypeUrl() == version.V3EndpointsURL { + switch len(req.GetResourceNames()) { + case 0: + select { + case edsResourceCanceledCh <- struct{}{}: + default: + } + case 1: + if req.GetResourceNames()[0] == edsServiceName { + select { + case edsResourceRequestedCh <- struct{}{}: + default: + } + } + default: + t.Errorf("Unexpected number of resources, %d, in an EDS request", len(req.GetResourceNames())) + } + } + return nil + }, + }) + defer cleanup() + + server := stubserver.StartTestService(t, nil) + defer server.Stop() + + // Configure cluster and endpoints resources in the management server. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, edsServiceName, e2e.SecurityLevelNone)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(edsServiceName, "localhost", []uint32{testutils.ParsePort(t, server.Address)})}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create xDS client, configure cds_experimental LB policy with a manual + // resolver, and dial the test backends. + cc, cleanup := setupAndDial(t, bootstrapContents) + defer cleanup() + + // Wait for the EDS resource to be requested. + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for EDS resource to be requested") + case <-edsResourceRequestedCh: + } + + // Ensure that a successful RPC can be made. + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + + // Delete the cluster resource from the mangement server. + resources.Clusters = nil + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Wait for the EDS resource to be not requested anymore. + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for EDS resource to not requested") + case <-edsResourceCanceledCh: + } + + // Ensure that RPCs start to fail with expected error. + for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) { + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + _, err := client.EmptyCall(sCtx, &testpb.Empty{}) + if status.Code(err) == codes.Unavailable && strings.Contains(err.Error(), "all priorities are removed") { + break + } + if err != nil { + t.Logf("EmptyCall failed: %v", err) + } + } + if ctx.Err() != nil { + t.Fatalf("RPCs did not fail after removal of Cluster resource") + } + + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + + // Configure cluster and endpoints resources in the management server. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, edsServiceName, e2e.SecurityLevelNone)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(edsServiceName, "localhost", []uint32{testutils.ParsePort(t, server.Address)})}, + SkipValidation: true, + } + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Wait for the EDS resource to be requested again. + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for EDS resource to be requested") + case <-edsResourceRequestedCh: + } + + // Ensure that a successful RPC can be made. + for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) { + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := client.EmptyCall(sCtx, &testpb.Empty{}); err != nil { + t.Logf("EmptyCall failed: %v", err) + continue + } + break + } + if ctx.Err() != nil { + t.Fatalf("RPCs did not fail after removal of Cluster resource") + } +} + +// Test verifies that when the received Cluster resource contains outlier +// detection configuration, the LB config pushed to the child policy contains +// the appropriate configuration for the outlier detection LB policy. +func (s) TestOutlierDetectionConfigPropagationToChildPolicy(t *testing.T) { + // Unregister the priority balancer builder for the duration of this test, + // and register a policy under the same name that makes the LB config + // pushed to it available to the test. + priorityBuilder := balancer.Get(priority.Name) + internal.BalancerUnregister(priorityBuilder.Name()) + lbCfgCh := make(chan serviceconfig.LoadBalancingConfig, 1) + stub.Register(priority.Name, stub.BalancerFuncs{ + Init: func(bd *stub.BalancerData) { + bd.Data = priorityBuilder.Build(bd.ClientConn, bd.BuildOptions) + }, + ParseConfig: func(lbCfg json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + return priorityBuilder.(balancer.ConfigParser).ParseConfig(lbCfg) + }, + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + select { + case lbCfgCh <- ccs.BalancerConfig: + default: + } + bal := bd.Data.(balancer.Balancer) + return bal.UpdateClientConnState(ccs) + }, + Close: func(bd *stub.BalancerData) { + bal := bd.Data.(balancer.Balancer) + bal.Close() + }, + }) + defer balancer.Register(priorityBuilder) + + managementServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup() + + server := stubserver.StartTestService(t, nil) + defer server.Stop() + + // Configure cluster and endpoints resources in the management server. + cluster := e2e.DefaultCluster(clusterName, edsServiceName, e2e.SecurityLevelNone) + cluster.OutlierDetection = &v3clusterpb.OutlierDetection{ + Interval: durationpb.New(10 * time.Second), + BaseEjectionTime: durationpb.New(30 * time.Second), + MaxEjectionTime: durationpb.New(300 * time.Second), + MaxEjectionPercent: wrapperspb.UInt32(10), + SuccessRateStdevFactor: wrapperspb.UInt32(2000), + EnforcingSuccessRate: wrapperspb.UInt32(50), + SuccessRateMinimumHosts: wrapperspb.UInt32(10), + SuccessRateRequestVolume: wrapperspb.UInt32(50), + } + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{cluster}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(edsServiceName, "localhost", []uint32{testutils.ParsePort(t, server.Address)})}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create xDS client, configure cds_experimental LB policy with a manual + // resolver, and dial the test backends. + _, cleanup = setupAndDial(t, bootstrapContents) + defer cleanup() + + // The priority configuration generated should have Outlier Detection as a + // direct child due to Outlier Detection being turned on. + wantCfg := &priority.LBConfig{ + Children: map[string]*priority.Child{ + "priority-0-0": { + Config: &iserviceconfig.BalancerConfig{ + Name: outlierdetection.Name, + Config: &outlierdetection.LBConfig{ + Interval: iserviceconfig.Duration(10 * time.Second), // default interval + BaseEjectionTime: iserviceconfig.Duration(30 * time.Second), + MaxEjectionTime: iserviceconfig.Duration(300 * time.Second), + MaxEjectionPercent: 10, + SuccessRateEjection: &outlierdetection.SuccessRateEjection{ + StdevFactor: 2000, + EnforcementPercentage: 50, + MinimumHosts: 10, + RequestVolume: 50, + }, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: clusterimpl.Name, + Config: &clusterimpl.LBConfig{ + Cluster: clusterName, + EDSServiceName: edsServiceName, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: wrrlocality.Name, + Config: &wrrlocality.LBConfig{ + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: roundrobin.Name, + }, + }, + }, + }, + }, + }, + }, + IgnoreReresolutionRequests: true, + }, + }, + Priorities: []string{"priority-0-0"}, + } + + select { + case lbCfg := <-lbCfgCh: + gotCfg := lbCfg.(*priority.LBConfig) + if diff := cmp.Diff(wantCfg, gotCfg); diff != "" { + t.Fatalf("Child policy received unexpected diff in config (-want +got):\n%s", diff) + } + case <-ctx.Done(): + t.Fatalf("Timeout when waiting for child policy to receive its configuration") + } +} diff --git a/xds/internal/balancer/clusterresolver/e2e_test/eds_impl_test.go b/xds/internal/balancer/clusterresolver/e2e_test/eds_impl_test.go new file mode 100644 index 000000000000..89c95a2fb809 --- /dev/null +++ b/xds/internal/balancer/clusterresolver/e2e_test/eds_impl_test.go @@ -0,0 +1,1099 @@ +/* + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package e2e_test + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/uuid" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/internal/testutils" + rrutil "google.golang.org/grpc/internal/testutils/roundrobin" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/serviceconfig" + "google.golang.org/grpc/status" + xdstestutils "google.golang.org/grpc/xds/internal/testutils" + "google.golang.org/grpc/xds/internal/xdsclient" + "google.golang.org/grpc/xds/internal/xdsclient/bootstrap" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource/version" + "google.golang.org/protobuf/types/known/wrapperspb" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + + _ "google.golang.org/grpc/xds/internal/balancer/clusterresolver" // Register the "cluster_resolver_experimental" LB policy. + "google.golang.org/grpc/xds/internal/balancer/priority" +) + +const ( + clusterName = "cluster-my-service-client-side-xds" + edsServiceName = "endpoints-my-service-client-side-xds" + localityName1 = "my-locality-1" + localityName2 = "my-locality-2" + localityName3 = "my-locality-3" + + defaultTestTimeout = 5 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond + defaultTestWatchExpiryTimeout = 500 * time.Millisecond +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// backendAddressesAndPorts extracts the address and port of each of the +// StubServers passed in and returns them. Fails the test if any of the +// StubServers passed have an invalid address. +func backendAddressesAndPorts(t *testing.T, servers []*stubserver.StubServer) ([]resolver.Address, []uint32) { + addrs := make([]resolver.Address, len(servers)) + ports := make([]uint32, len(servers)) + for i := 0; i < len(servers); i++ { + addrs[i] = resolver.Address{Addr: servers[i].Address} + ports[i] = testutils.ParsePort(t, servers[i].Address) + } + return addrs, ports +} + +func startTestServiceBackends(t *testing.T, numBackends int) ([]*stubserver.StubServer, func()) { + var servers []*stubserver.StubServer + for i := 0; i < numBackends; i++ { + servers = append(servers, stubserver.StartTestService(t, nil)) + servers[i].StartServer() + } + + return servers, func() { + for _, server := range servers { + server.Stop() + } + } +} + +// clientEndpointsResource returns an EDS resource for the specified nodeID, +// service name and localities. +func clientEndpointsResource(nodeID, edsServiceName string, localities []e2e.LocalityOptions) e2e.UpdateOptions { + return e2e.UpdateOptions{ + NodeID: nodeID, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.EndpointResourceWithOptions(e2e.EndpointOptions{ + ClusterName: edsServiceName, + Host: "localhost", + Localities: localities, + })}, + SkipValidation: true, + } +} + +// TestEDS_OneLocality tests the cluster_resolver LB policy using an EDS +// resource with one locality. The following scenarios are tested: +// 1. Single backend. Test verifies that RPCs reach this backend. +// 2. Add a backend. Test verifies that RPCs are roundrobined across the two +// backends. +// 3. Remove one backend. Test verifies that all RPCs reach the other backend. +// 4. Replace the backend. Test verifies that all RPCs reach the new backend. +func (s) TestEDS_OneLocality(t *testing.T) { + // Spin up a management server to receive xDS resources from. + managementServer, nodeID, bootstrapContents, _, cleanup1 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup1() + + // Start backend servers which provide an implementation of the TestService. + servers, cleanup2 := startTestServiceBackends(t, 3) + defer cleanup2() + addrs, ports := backendAddressesAndPorts(t, servers) + + // Create xDS resources for consumption by the test. We start off with a + // single backend in a single EDS locality. + resources := clientEndpointsResource(nodeID, edsServiceName, []e2e.LocalityOptions{{ + Name: localityName1, + Weight: 1, + Backends: []e2e.BackendOptions{{Port: ports[0]}}, + }}) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create an xDS client for use by the cluster_resolver LB policy. + client, close, err := xdsclient.NewWithBootstrapContentsForTesting(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Create a manual resolver and push a service config specifying the use of + // the cluster_resolver LB policy with a single discovery mechanism. + r := manual.NewBuilderWithScheme("whatever") + jsonSC := fmt.Sprintf(`{ + "loadBalancingConfig":[{ + "cluster_resolver_experimental":{ + "discoveryMechanisms": [{ + "cluster": "%s", + "type": "EDS", + "edsServiceName": "%s", + "outlierDetection": {} + }], + "xdsLbPolicy":[{"round_robin":{}}] + } + }] + }`, clusterName, edsServiceName) + scpr := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(jsonSC) + r.InitialState(xdsclient.SetClient(resolver.State{ServiceConfig: scpr}, client)) + + // Create a ClientConn and make a successful RPC. + cc, err := grpc.Dial(r.Scheme()+":///test.service", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + // Ensure RPCs are being roundrobined across the single backend. + testClient := testgrpc.NewTestServiceClient(cc) + if err := rrutil.CheckRoundRobinRPCs(ctx, testClient, addrs[:1]); err != nil { + t.Fatal(err) + } + + // Add a backend to the same locality, and ensure RPCs are sent in a + // roundrobin fashion across the two backends. + resources = clientEndpointsResource(nodeID, edsServiceName, []e2e.LocalityOptions{{ + Name: localityName1, + Weight: 1, + Backends: []e2e.BackendOptions{{Port: ports[0]}, {Port: ports[1]}}, + }}) + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + if err := rrutil.CheckRoundRobinRPCs(ctx, testClient, addrs[:2]); err != nil { + t.Fatal(err) + } + + // Remove the first backend, and ensure all RPCs are sent to the second + // backend. + resources = clientEndpointsResource(nodeID, edsServiceName, []e2e.LocalityOptions{{ + Name: localityName1, + Weight: 1, + Backends: []e2e.BackendOptions{{Port: ports[1]}}, + }}) + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + if err := rrutil.CheckRoundRobinRPCs(ctx, testClient, addrs[1:2]); err != nil { + t.Fatal(err) + } + + // Replace the backend, and ensure all RPCs are sent to the new backend. + resources = clientEndpointsResource(nodeID, edsServiceName, []e2e.LocalityOptions{{ + Name: localityName1, + Weight: 1, + Backends: []e2e.BackendOptions{{Port: ports[2]}}, + }}) + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + if err := rrutil.CheckRoundRobinRPCs(ctx, testClient, addrs[2:3]); err != nil { + t.Fatal(err) + } +} + +// TestEDS_MultipleLocalities tests the cluster_resolver LB policy using an EDS +// resource with multiple localities. The following scenarios are tested: +// 1. Two localities, each with a single backend. Test verifies that RPCs are +// weighted roundrobined across these two backends. +// 2. Add another locality, with a single backend. Test verifies that RPCs are +// weighted roundrobined across all the backends. +// 3. Remove one locality. Test verifies that RPCs are weighted roundrobined +// across backends from the remaining localities. +// 4. Add a backend to one locality. Test verifies that RPCs are weighted +// roundrobined across localities. +// 5. Change the weight of one of the localities. Test verifies that RPCs are +// weighted roundrobined across the localities. +// +// In our LB policy tree, one of the descendents of the "cluster_resolver" LB +// policy is the "weighted_target" LB policy which performs weighted roundrobin +// across localities (and this has a randomness component associated with it). +// Therefore, the moment we have backends from more than one locality, RPCs are +// weighted roundrobined across them. +func (s) TestEDS_MultipleLocalities(t *testing.T) { + // Spin up a management server to receive xDS resources from. + managementServer, nodeID, bootstrapContents, _, cleanup1 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup1() + + // Start backend servers which provide an implementation of the TestService. + servers, cleanup2 := startTestServiceBackends(t, 4) + defer cleanup2() + addrs, ports := backendAddressesAndPorts(t, servers) + + // Create xDS resources for consumption by the test. We start off with two + // localities, and single backend in each of them. + resources := clientEndpointsResource(nodeID, edsServiceName, []e2e.LocalityOptions{ + { + Name: localityName1, + Weight: 1, + Backends: []e2e.BackendOptions{{Port: ports[0]}}, + }, + { + Name: localityName2, + Weight: 1, + Backends: []e2e.BackendOptions{{Port: ports[1]}}, + }, + }) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create an xDS client for use by the cluster_resolver LB policy. + client, close, err := xdsclient.NewWithBootstrapContentsForTesting(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Create a manual resolver and push service config specifying the use of + // the cluster_resolver LB policy with a single discovery mechanism. + r := manual.NewBuilderWithScheme("whatever") + jsonSC := fmt.Sprintf(`{ + "loadBalancingConfig":[{ + "cluster_resolver_experimental":{ + "discoveryMechanisms": [{ + "cluster": "%s", + "type": "EDS", + "edsServiceName": "%s", + "outlierDetection": {} + }], + "xdsLbPolicy":[{"round_robin":{}}] + } + }] + }`, clusterName, edsServiceName) + scpr := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(jsonSC) + r.InitialState(xdsclient.SetClient(resolver.State{ServiceConfig: scpr}, client)) + + // Create a ClientConn and make a successful RPC. + cc, err := grpc.Dial(r.Scheme()+":///test.service", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + // Ensure RPCs are being weighted roundrobined across the two backends. + testClient := testgrpc.NewTestServiceClient(cc) + if err := rrutil.CheckWeightedRoundRobinRPCs(ctx, testClient, addrs[0:2]); err != nil { + t.Fatal(err) + } + + // Add another locality with a single backend, and ensure RPCs are being + // weighted roundrobined across the three backends. + resources = clientEndpointsResource(nodeID, edsServiceName, []e2e.LocalityOptions{ + { + Name: localityName1, + Weight: 1, + Backends: []e2e.BackendOptions{{Port: ports[0]}}, + }, + { + Name: localityName2, + Weight: 1, + Backends: []e2e.BackendOptions{{Port: ports[1]}}, + }, + { + Name: localityName3, + Weight: 1, + Backends: []e2e.BackendOptions{{Port: ports[2]}}, + }, + }) + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + if err := rrutil.CheckWeightedRoundRobinRPCs(ctx, testClient, addrs[0:3]); err != nil { + t.Fatal(err) + } + + // Remove the first locality, and ensure RPCs are being weighted + // roundrobined across the remaining two backends. + resources = clientEndpointsResource(nodeID, edsServiceName, []e2e.LocalityOptions{ + { + Name: localityName2, + Weight: 1, + Backends: []e2e.BackendOptions{{Port: ports[1]}}, + }, + { + Name: localityName3, + Weight: 1, + Backends: []e2e.BackendOptions{{Port: ports[2]}}, + }, + }) + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + if err := rrutil.CheckWeightedRoundRobinRPCs(ctx, testClient, addrs[1:3]); err != nil { + t.Fatal(err) + } + + // Add a backend to one locality, and ensure weighted roundrobin. Since RPCs + // are roundrobined across localities, locality2's backend will receive + // twice the traffic. + resources = clientEndpointsResource(nodeID, edsServiceName, []e2e.LocalityOptions{ + { + Name: localityName2, + Weight: 1, + Backends: []e2e.BackendOptions{{Port: ports[1]}}, + }, + { + Name: localityName3, + Weight: 1, + Backends: []e2e.BackendOptions{{Port: ports[2]}, {Port: ports[3]}}, + }, + }) + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + wantAddrs := []resolver.Address{addrs[1], addrs[1], addrs[2], addrs[3]} + if err := rrutil.CheckWeightedRoundRobinRPCs(ctx, testClient, wantAddrs); err != nil { + t.Fatal(err) + } +} + +// TestEDS_EndpointsHealth tests the cluster_resolver LB policy using an EDS +// resource which specifies endpoint health information and verifies that +// traffic is routed only to backends deemed capable of receiving traffic. +func (s) TestEDS_EndpointsHealth(t *testing.T) { + // Spin up a management server to receive xDS resources from. + managementServer, nodeID, bootstrapContents, _, cleanup1 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup1() + + // Start backend servers which provide an implementation of the TestService. + servers, cleanup2 := startTestServiceBackends(t, 12) + defer cleanup2() + addrs, ports := backendAddressesAndPorts(t, servers) + + // Create xDS resources for consumption by the test. Two localities with + // six backends each, with two of the six backends being healthy. Both + // UNKNOWN and HEALTHY are considered by gRPC for load balancing. + resources := clientEndpointsResource(nodeID, edsServiceName, []e2e.LocalityOptions{ + { + Name: localityName1, + Weight: 1, + Backends: []e2e.BackendOptions{ + {Port: ports[0], HealthStatus: v3corepb.HealthStatus_UNKNOWN}, + {Port: ports[1], HealthStatus: v3corepb.HealthStatus_HEALTHY}, + {Port: ports[2], HealthStatus: v3corepb.HealthStatus_UNHEALTHY}, + {Port: ports[3], HealthStatus: v3corepb.HealthStatus_DRAINING}, + {Port: ports[4], HealthStatus: v3corepb.HealthStatus_TIMEOUT}, + {Port: ports[5], HealthStatus: v3corepb.HealthStatus_DEGRADED}, + }, + }, + { + Name: localityName2, + Weight: 1, + Backends: []e2e.BackendOptions{ + {Port: ports[6], HealthStatus: v3corepb.HealthStatus_UNKNOWN}, + {Port: ports[7], HealthStatus: v3corepb.HealthStatus_HEALTHY}, + {Port: ports[8], HealthStatus: v3corepb.HealthStatus_UNHEALTHY}, + {Port: ports[9], HealthStatus: v3corepb.HealthStatus_DRAINING}, + {Port: ports[10], HealthStatus: v3corepb.HealthStatus_TIMEOUT}, + {Port: ports[11], HealthStatus: v3corepb.HealthStatus_DEGRADED}, + }, + }, + }) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create an xDS client for use by the cluster_resolver LB policy. + client, close, err := xdsclient.NewWithBootstrapContentsForTesting(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Create a manual resolver and push service config specifying the use of + // the cluster_resolver LB policy with a single discovery mechanism. + r := manual.NewBuilderWithScheme("whatever") + jsonSC := fmt.Sprintf(`{ + "loadBalancingConfig":[{ + "cluster_resolver_experimental":{ + "discoveryMechanisms": [{ + "cluster": "%s", + "type": "EDS", + "edsServiceName": "%s", + "outlierDetection": {} + }], + "xdsLbPolicy":[{"round_robin":{}}] + } + }] + }`, clusterName, edsServiceName) + scpr := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(jsonSC) + r.InitialState(xdsclient.SetClient(resolver.State{ServiceConfig: scpr}, client)) + + // Create a ClientConn and make a successful RPC. + cc, err := grpc.Dial(r.Scheme()+":///test.service", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + // Ensure RPCs are being weighted roundrobined across healthy backends from + // both localities. + testClient := testgrpc.NewTestServiceClient(cc) + if err := rrutil.CheckWeightedRoundRobinRPCs(ctx, testClient, append(addrs[0:2], addrs[6:8]...)); err != nil { + t.Fatal(err) + } +} + +// TestEDS_EmptyUpdate tests the cluster_resolver LB policy using an EDS +// resource with no localities and verifies that RPCs fail with "all priorities +// removed" error. +func (s) TestEDS_EmptyUpdate(t *testing.T) { + // Spin up a management server to receive xDS resources from. + managementServer, nodeID, bootstrapContents, _, cleanup1 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup1() + + // Start backend servers which provide an implementation of the TestService. + servers, cleanup2 := startTestServiceBackends(t, 4) + defer cleanup2() + addrs, ports := backendAddressesAndPorts(t, servers) + + oldCacheTimeout := priority.DefaultSubBalancerCloseTimeout + priority.DefaultSubBalancerCloseTimeout = 100 * time.Microsecond + defer func() { priority.DefaultSubBalancerCloseTimeout = oldCacheTimeout }() + + // Create xDS resources for consumption by the test. The first update is an + // empty update. This should put the channel in TRANSIENT_FAILURE. + resources := clientEndpointsResource(nodeID, edsServiceName, nil) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create an xDS client for use by the cluster_resolver LB policy. + client, close, err := xdsclient.NewWithBootstrapContentsForTesting(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Create a manual resolver and push service config specifying the use of + // the cluster_resolver LB policy with a single discovery mechanism. + r := manual.NewBuilderWithScheme("whatever") + jsonSC := fmt.Sprintf(`{ + "loadBalancingConfig":[{ + "cluster_resolver_experimental":{ + "discoveryMechanisms": [{ + "cluster": "%s", + "type": "EDS", + "edsServiceName": "%s", + "outlierDetection": {} + }], + "xdsLbPolicy":[{"round_robin":{}}] + } + }] + }`, clusterName, edsServiceName) + scpr := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(jsonSC) + r.InitialState(xdsclient.SetClient(resolver.State{ServiceConfig: scpr}, client)) + + // Create a ClientConn and ensure that RPCs fail with "all priorities + // removed" error. This is the expected error when the cluster_resolver LB + // policy receives an EDS update with no localities. + cc, err := grpc.Dial(r.Scheme()+":///test.service", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + testClient := testgrpc.NewTestServiceClient(cc) + if err := waitForProducedZeroAddressesError(ctx, t, testClient); err != nil { + t.Fatal(err) + } + + // Add a locality with one backend and ensure RPCs are successful. + resources = clientEndpointsResource(nodeID, edsServiceName, []e2e.LocalityOptions{{ + Name: localityName1, + Weight: 1, + Backends: []e2e.BackendOptions{{Port: ports[0]}}, + }}) + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + if err := rrutil.CheckRoundRobinRPCs(ctx, testClient, addrs[:1]); err != nil { + t.Fatal(err) + } + + // Push another empty update and ensure that RPCs fail with "all priorities + // removed" error again. + resources = clientEndpointsResource(nodeID, edsServiceName, nil) + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + if err := waitForProducedZeroAddressesError(ctx, t, testClient); err != nil { + t.Fatal(err) + } +} + +// TestEDS_ResourceRemoved tests the case where the EDS resource requested by +// the clusterresolver LB policy is removed from the management server. The test +// verifies that the EDS watch is not canceled and that RPCs continue to succeed +// with the previously received configuration. +func (s) TestEDS_ResourceRemoved(t *testing.T) { + // Start an xDS management server that uses a couple of channels to + // notify the test about the following events: + // - an EDS requested with the expected resource name is requested + // - EDS resource is unrequested, i.e, an EDS request with no resource name + // is received, which indicates that we are not longer interested in that + // resource. + edsResourceRequestedCh := make(chan struct{}, 1) + edsResourceCanceledCh := make(chan struct{}, 1) + managementServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{ + OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + if req.GetTypeUrl() == version.V3EndpointsURL { + switch len(req.GetResourceNames()) { + case 0: + select { + case edsResourceCanceledCh <- struct{}{}: + default: + } + case 1: + if req.GetResourceNames()[0] == edsServiceName { + select { + case edsResourceRequestedCh <- struct{}{}: + default: + } + } + default: + t.Errorf("Unexpected number of resources, %d, in an EDS request", len(req.GetResourceNames())) + } + } + return nil + }, + }) + defer cleanup() + + server := stubserver.StartTestService(t, nil) + defer server.Stop() + + // Configure cluster and endpoints resources in the management server. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, edsServiceName, e2e.SecurityLevelNone)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(edsServiceName, "localhost", []uint32{testutils.ParsePort(t, server.Address)})}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create xDS client, configure cds_experimental LB policy with a manual + // resolver, and dial the test backends. + cc, cleanup := setupAndDial(t, bootstrapContents) + defer cleanup() + + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + + // Delete the endpoints resource from the mangement server. + resources.Endpoints = nil + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Ensure that RPCs continue to succeed for the next second, and that the + // EDS watch is not canceled. + for end := time.Now().Add(time.Second); time.Now().Before(end); <-time.After(defaultTestShortTimeout) { + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + select { + case <-edsResourceCanceledCh: + t.Fatal("EDS watch canceled when not expected to be canceled") + default: + } + } +} + +// TestEDS_ClusterResourceDoesNotContainEDSServiceName tests the case where the +// Cluster resource sent by the management server does not contain an EDS +// service name. The test verifies that the cluster_resolver LB policy uses the +// cluster name for the EDS resource. +func (s) TestEDS_ClusterResourceDoesNotContainEDSServiceName(t *testing.T) { + edsResourceCh := make(chan string, 1) + managementServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{ + OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + if req.GetTypeUrl() != version.V3EndpointsURL { + return nil + } + if len(req.GetResourceNames()) > 0 { + select { + case edsResourceCh <- req.GetResourceNames()[0]: + default: + } + } + return nil + }, + }) + defer cleanup() + + server := stubserver.StartTestService(t, nil) + defer server.Stop() + + // Configure cluster and endpoints resources with the same name in the management server. The cluster resource does not specify an EDS service name. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, "", e2e.SecurityLevelNone)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(clusterName, "localhost", []uint32{testutils.ParsePort(t, server.Address)})}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create xDS client, configure cds_experimental LB policy with a manual + // resolver, and dial the test backends. + cc, cleanup := setupAndDial(t, bootstrapContents) + defer cleanup() + + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for EDS request to be received on the management server") + case name := <-edsResourceCh: + if name != clusterName { + t.Fatalf("Received EDS request with resource name %q, want %q", name, clusterName) + } + } +} + +// TestEDS_ClusterResourceUpdates verifies different scenarios with regards to +// cluster resource updates. +// +// - The first cluster resource contains an eds_service_name. The test verifies +// that an EDS request is sent for the received eds_service_name. It also +// verifies that a subsequent RPC gets routed to a backend belonging to that +// service name. +// - The next cluster resource update contains no eds_service_name. The test +// verifies that a subsequent EDS request is sent for the cluster_name and +// that the previously received eds_service_name is no longer requested. It +// also verifies that a subsequent RPC gets routed to a backend belonging to +// the service represented by the cluster_name. +// - The next cluster resource update changes the circuit breaking +// configuration, but does not change the service name. The test verifies +// that a subsequent RPC gets routed to the same backend as before. +func (s) TestEDS_ClusterResourceUpdates(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Start an xDS management server that pushes the EDS resource names onto a + // channel. + edsResourceNameCh := make(chan []string, 1) + managementServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{ + OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + if req.GetTypeUrl() != version.V3EndpointsURL { + return nil + } + if len(req.GetResourceNames()) == 0 { + // This is the case for ACKs. Do nothing here. + return nil + } + select { + case <-ctx.Done(): + case edsResourceNameCh <- req.GetResourceNames(): + } + return nil + }, + AllowResourceSubset: true, + }) + defer cleanup() + + // Start two test backends and extract their host and port. The first + // backend is used for the EDS resource identified by the eds_service_name, + // and the second backend is used for the EDS resource identified by the + // cluster_name. + servers, cleanup2 := startTestServiceBackends(t, 2) + defer cleanup2() + addrs, ports := backendAddressesAndPorts(t, servers) + + // Configure cluster and endpoints resources in the management server. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, edsServiceName, e2e.SecurityLevelNone)}, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{ + e2e.DefaultEndpoint(edsServiceName, "localhost", []uint32{uint32(ports[0])}), + e2e.DefaultEndpoint(clusterName, "localhost", []uint32{uint32(ports[1])}), + }, + SkipValidation: true, + } + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create xDS client, configure cds_experimental LB policy with a manual + // resolver, and dial the test backends. + cc, cleanup := setupAndDial(t, bootstrapContents) + defer cleanup() + + client := testgrpc.NewTestServiceClient(cc) + peer := &peer.Peer{} + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + if peer.Addr.String() != addrs[0].Addr { + t.Fatalf("EmptyCall() routed to backend %q, want %q", peer.Addr, addrs[0].Addr) + } + + // Ensure EDS watch is registered for eds_service_name. + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for EDS request to be received on the management server") + case names := <-edsResourceNameCh: + if !cmp.Equal(names, []string{edsServiceName}) { + t.Fatalf("Received EDS request with resource names %v, want %v", names, []string{edsServiceName}) + } + } + + // Change the cluster resource to not contain an eds_service_name. + resources.Clusters = []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, "", e2e.SecurityLevelNone)} + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Ensure that an EDS watch for eds_service_name is canceled and new watch + // for cluster_name is registered. The actual order in which this happens is + // not deterministic, i.e the watch for old resource could be canceled + // before the new one is registered or vice-versa. In either case, + // eventually, we want to see a request to the management server for just + // the cluster_name. + for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) { + names := <-edsResourceNameCh + if cmp.Equal(names, []string{clusterName}) { + break + } + } + if ctx.Err() != nil { + t.Fatalf("Timeout when waiting for old EDS watch %q to be canceled and new one %q to be registered", edsServiceName, clusterName) + } + + // Make a RPC, and ensure that it gets routed to second backend, + // corresponding to the cluster_name. + for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) { + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer)); err != nil { + continue + } + if peer.Addr.String() == addrs[1].Addr { + break + } + } + if ctx.Err() != nil { + t.Fatalf("Timeout when waiting for EmptyCall() to be routed to correct backend %q", addrs[1].Addr) + } + + // Change cluster resource circuit breaking count. + resources.Clusters[0].CircuitBreakers = &v3clusterpb.CircuitBreakers{ + Thresholds: []*v3clusterpb.CircuitBreakers_Thresholds{ + { + Priority: v3corepb.RoutingPriority_DEFAULT, + MaxRequests: wrapperspb.UInt32(512), + }, + }, + } + if err := managementServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Ensure that RPCs continue to get routed to the second backend for the + // next second. + for end := time.Now().Add(time.Second); time.Now().Before(end); <-time.After(defaultTestShortTimeout) { + if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(peer)); err != nil { + t.Fatalf("EmptyCall() failed: %v", err) + } + if peer.Addr.String() != addrs[1].Addr { + t.Fatalf("EmptyCall() routed to backend %q, want %q", peer.Addr, addrs[1].Addr) + } + } +} + +// TestEDS_BadUpdateWithoutPreviousGoodUpdate tests the case where the +// management server sends a bad update (one that is NACKed by the xDS client). +// Since the cluster_resolver LB policy does not have a previously received good +// update, it is expected to treat this bad update as though it received an +// update with no endpoints. Hence RPCs are expected to fail with "all +// priorities removed" error. +func (s) TestEDS_BadUpdateWithoutPreviousGoodUpdate(t *testing.T) { + // Spin up a management server to receive xDS resources from. + mgmtServer, nodeID, bootstrapContents, _, cleanup1 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup1() + + // Start a backend server that implements the TestService. + server := stubserver.StartTestService(t, nil) + defer server.Stop() + + // Create an EDS resource with a load balancing weight of 0. This will + // result in the resource being NACKed by the xDS client. Since the + // cluster_resolver LB policy does not have a previously received good EDS + // update, it should treat this update as an empty EDS update. + resources := clientEndpointsResource(nodeID, edsServiceName, []e2e.LocalityOptions{{ + Name: localityName1, + Weight: 1, + Backends: []e2e.BackendOptions{{Port: testutils.ParsePort(t, server.Address)}}, + }}) + resources.Endpoints[0].Endpoints[0].LbEndpoints[0].LoadBalancingWeight = &wrapperspb.UInt32Value{Value: 0} + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create an xDS client for use by the cluster_resolver LB policy. + xdsClient, close, err := xdsclient.NewWithBootstrapContentsForTesting(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Create a manual resolver and push a service config specifying the use of + // the cluster_resolver LB policy with a single discovery mechanism. + r := manual.NewBuilderWithScheme("whatever") + jsonSC := fmt.Sprintf(`{ + "loadBalancingConfig":[{ + "cluster_resolver_experimental":{ + "discoveryMechanisms": [{ + "cluster": "%s", + "type": "EDS", + "edsServiceName": "%s", + "outlierDetection": {} + }], + "xdsLbPolicy":[{"round_robin":{}}] + } + }] + }`, clusterName, edsServiceName) + scpr := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(jsonSC) + r.InitialState(xdsclient.SetClient(resolver.State{ServiceConfig: scpr}, xdsClient)) + + // Create a ClientConn and verify that RPCs fail with "all priorities + // removed" error. + cc, err := grpc.Dial(r.Scheme()+":///test.service", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + client := testgrpc.NewTestServiceClient(cc) + if err := waitForProducedZeroAddressesError(ctx, t, client); err != nil { + t.Fatal(err) + } +} + +// TestEDS_BadUpdateWithPreviousGoodUpdate tests the case where the +// cluster_resolver LB policy receives a good EDS update from the management +// server and the test verifies that RPCs are successful. Then, a bad update is +// received from the management server (one that is NACKed by the xDS client). +// The test verifies that the previously received good update is still being +// used and that RPCs are still successful. +func (s) TestEDS_BadUpdateWithPreviousGoodUpdate(t *testing.T) { + // Spin up a management server to receive xDS resources from. + mgmtServer, nodeID, bootstrapContents, _, cleanup1 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup1() + + // Start a backend server that implements the TestService. + server := stubserver.StartTestService(t, nil) + defer server.Stop() + + // Create an EDS resource for consumption by the test. + resources := clientEndpointsResource(nodeID, edsServiceName, []e2e.LocalityOptions{{ + Name: localityName1, + Weight: 1, + Backends: []e2e.BackendOptions{{Port: testutils.ParsePort(t, server.Address)}}, + }}) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create an xDS client for use by the cluster_resolver LB policy. + xdsClient, close, err := xdsclient.NewWithBootstrapContentsForTesting(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Create a manual resolver and push a service config specifying the use of + // the cluster_resolver LB policy with a single discovery mechanism. + r := manual.NewBuilderWithScheme("whatever") + jsonSC := fmt.Sprintf(`{ + "loadBalancingConfig":[{ + "cluster_resolver_experimental":{ + "discoveryMechanisms": [{ + "cluster": "%s", + "type": "EDS", + "edsServiceName": "%s", + "outlierDetection": {} + }], + "xdsLbPolicy":[{"round_robin":{}}] + } + }] + }`, clusterName, edsServiceName) + scpr := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(jsonSC) + r.InitialState(xdsclient.SetClient(resolver.State{ServiceConfig: scpr}, xdsClient)) + + // Create a ClientConn and make a successful RPC. + cc, err := grpc.Dial(r.Scheme()+":///test.service", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + // Ensure RPCs are being roundrobined across the single backend. + client := testgrpc.NewTestServiceClient(cc) + if err := rrutil.CheckRoundRobinRPCs(ctx, client, []resolver.Address{{Addr: server.Address}}); err != nil { + t.Fatal(err) + } + + // Update the endpoints resource in the management server with a load + // balancing weight of 0. This will result in the resource being NACKed by + // the xDS client. But since the cluster_resolver LB policy has a previously + // received good EDS update, it should continue using it. + resources.Endpoints[0].Endpoints[0].LbEndpoints[0].LoadBalancingWeight = &wrapperspb.UInt32Value{Value: 0} + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Ensure that RPCs continue to succeed for the next second. + for end := time.Now().Add(time.Second); time.Now().Before(end); <-time.After(defaultTestShortTimeout) { + if err := rrutil.CheckRoundRobinRPCs(ctx, client, []resolver.Address{{Addr: server.Address}}); err != nil { + t.Fatal(err) + } + } +} + +// TestEDS_ResourceNotFound tests the case where the requested EDS resource does +// not exist on the management server. Once the watch timer associated with the +// requested resource expires, the cluster_resolver LB policy receives a +// "resource-not-found" callback from the xDS client and is expected to treat it +// as though it received an update with no endpoints. Hence RPCs are expected to +// fail with "all priorities removed" error. +func (s) TestEDS_ResourceNotFound(t *testing.T) { + // Spin up a management server to receive xDS resources from. + mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) + if err != nil { + t.Fatalf("Failed to spin up the xDS management server: %v", err) + } + defer mgmtServer.Stop() + + // Create an xDS client talking to the above management server, configured + // with a short watch expiry timeout. + nodeID := uuid.New().String() + xdsClient, close, err := xdsclient.NewWithConfigForTesting(&bootstrap.Config{ + XDSServer: xdstestutils.ServerConfigForAddress(t, mgmtServer.Address), + NodeProto: &v3corepb.Node{Id: nodeID}, + }, defaultTestWatchExpiryTimeout, time.Duration(0)) + if err != nil { + t.Fatalf("failed to create xds client: %v", err) + } + defer close() + + // Configure no resources on the management server. + resources := e2e.UpdateOptions{NodeID: nodeID, SkipValidation: true} + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Create a manual resolver and push a service config specifying the use of + // the cluster_resolver LB policy with a single discovery mechanism. + r := manual.NewBuilderWithScheme("whatever") + jsonSC := fmt.Sprintf(`{ + "loadBalancingConfig":[{ + "cluster_resolver_experimental":{ + "discoveryMechanisms": [{ + "cluster": "%s", + "type": "EDS", + "edsServiceName": "%s", + "outlierDetection": {} + }], + "xdsLbPolicy":[{"round_robin":{}}] + } + }] + }`, clusterName, edsServiceName) + scpr := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(jsonSC) + r.InitialState(xdsclient.SetClient(resolver.State{ServiceConfig: scpr}, xdsClient)) + + // Create a ClientConn and verify that RPCs fail with "all priorities + // removed" error. + cc, err := grpc.Dial(r.Scheme()+":///test.service", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithResolvers(r)) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + client := testgrpc.NewTestServiceClient(cc) + if err := waitForProducedZeroAddressesError(ctx, t, client); err != nil { + t.Fatal(err) + } +} + +// waitForAllPrioritiesRemovedError repeatedly makes RPCs using the +// TestServiceClient until they fail with an error which indicates that no +// resolver addresses have been produced. A non-nil error is returned if the +// context expires before RPCs fail with the expected error. +func waitForProducedZeroAddressesError(ctx context.Context, t *testing.T, client testgrpc.TestServiceClient) error { + for ; ctx.Err() == nil; <-time.After(time.Millisecond) { + _, err := client.EmptyCall(ctx, &testpb.Empty{}) + if err == nil { + t.Log("EmptyCall() succeeded after error in Discovery Mechanism") + continue + } + if code := status.Code(err); code != codes.Unavailable { + t.Logf("EmptyCall() returned code: %v, want: %v", code, codes.Unavailable) + continue + } + if !strings.Contains(err.Error(), "produced zero addresses") { + t.Logf("EmptyCall() = %v, want %v", err, "produced zero addresses") + continue + } + return nil + } + return errors.New("timeout when waiting for RPCs to fail with UNAVAILABLE status and produced zero addresses") +} diff --git a/xds/internal/balancer/edsbalancer/logging.go b/xds/internal/balancer/clusterresolver/logging.go similarity index 84% rename from xds/internal/balancer/edsbalancer/logging.go rename to xds/internal/balancer/clusterresolver/logging.go index be4d0a512d16..728f1f709c28 100644 --- a/xds/internal/balancer/edsbalancer/logging.go +++ b/xds/internal/balancer/clusterresolver/logging.go @@ -16,7 +16,7 @@ * */ -package edsbalancer +package clusterresolver import ( "fmt" @@ -25,10 +25,10 @@ import ( internalgrpclog "google.golang.org/grpc/internal/grpclog" ) -const prefix = "[eds-lb %p] " +const prefix = "[xds-cluster-resolver-lb %p] " var logger = grpclog.Component("xds") -func prefixLogger(p *edsBalancer) *internalgrpclog.PrefixLogger { +func prefixLogger(p *clusterResolverBalancer) *internalgrpclog.PrefixLogger { return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p)) } diff --git a/xds/internal/balancer/clusterresolver/resource_resolver.go b/xds/internal/balancer/clusterresolver/resource_resolver.go new file mode 100644 index 000000000000..b9a81e9ba829 --- /dev/null +++ b/xds/internal/balancer/clusterresolver/resource_resolver.go @@ -0,0 +1,295 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package clusterresolver + +import ( + "context" + "sync" + + "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" +) + +// resourceUpdate is a combined update from all the resources, in the order of +// priority. For example, it can be {EDS, EDS, DNS}. +type resourceUpdate struct { + priorities []priorityConfig + err error +} + +// topLevelResolver is used by concrete endpointsResolver implementations for +// reporting updates and errors. The `resourceResolver` type implements this +// interface and takes appropriate actions upon receipt of updates and errors +// from underlying concrete resolvers. +type topLevelResolver interface { + onUpdate() +} + +// endpointsResolver wraps the functionality to resolve a given resource name to +// a set of endpoints. The mechanism used by concrete implementations depend on +// the supported discovery mechanism type. +type endpointsResolver interface { + // lastUpdate returns endpoint results from the most recent resolution. + // + // The type of the first return result is dependent on the resolver + // implementation. + // + // The second return result indicates whether the resolver was able to + // successfully resolve the resource name to endpoints. If set to false, the + // first return result is invalid and must not be used. + lastUpdate() (any, bool) + + // resolverNow triggers re-resolution of the resource. + resolveNow() + + // stop stops resolution of the resource. Implementations must not invoke + // any methods on the topLevelResolver interface once `stop()` returns. + stop() +} + +// discoveryMechanismKey is {type+resource_name}, it's used as the map key, so +// that the same resource resolver can be reused (e.g. when there are two +// mechanisms, both for the same EDS resource, but has different circuit +// breaking config. +type discoveryMechanismKey struct { + typ DiscoveryMechanismType + name string +} + +// discoveryMechanismAndResolver is needed to keep the resolver and the +// discovery mechanism together, because resolvers can be shared. And we need +// the mechanism for fields like circuit breaking, LRS etc when generating the +// balancer config. +type discoveryMechanismAndResolver struct { + dm DiscoveryMechanism + r endpointsResolver + + childNameGen *nameGenerator +} + +type resourceResolver struct { + parent *clusterResolverBalancer + logger *grpclog.PrefixLogger + updateChannel chan *resourceUpdate + serializer *grpcsync.CallbackSerializer + serializerCancel context.CancelFunc + + // mu protects the slice and map, and content of the resolvers in the slice. + mu sync.Mutex + mechanisms []DiscoveryMechanism + children []discoveryMechanismAndResolver + // childrenMap's value only needs the resolver implementation (type + // discoveryMechanism) and the childNameGen. The other two fields are not + // used. + // + // TODO(cleanup): maybe we can make a new type with just the necessary + // fields, and use it here instead. + childrenMap map[discoveryMechanismKey]discoveryMechanismAndResolver + // Each new discovery mechanism needs a child name generator to reuse child + // policy names. But to make sure the names across discover mechanism + // doesn't conflict, we need a seq ID. This ID is incremented for each new + // discover mechanism. + childNameGeneratorSeqID uint64 +} + +func newResourceResolver(parent *clusterResolverBalancer, logger *grpclog.PrefixLogger) *resourceResolver { + rr := &resourceResolver{ + parent: parent, + logger: logger, + updateChannel: make(chan *resourceUpdate, 1), + childrenMap: make(map[discoveryMechanismKey]discoveryMechanismAndResolver), + } + ctx, cancel := context.WithCancel(context.Background()) + rr.serializer = grpcsync.NewCallbackSerializer(ctx) + rr.serializerCancel = cancel + return rr +} + +func equalDiscoveryMechanisms(a, b []DiscoveryMechanism) bool { + if len(a) != len(b) { + return false + } + for i, aa := range a { + bb := b[i] + if !aa.Equal(bb) { + return false + } + } + return true +} + +func discoveryMechanismToKey(dm DiscoveryMechanism) discoveryMechanismKey { + switch dm.Type { + case DiscoveryMechanismTypeEDS: + nameToWatch := dm.EDSServiceName + if nameToWatch == "" { + nameToWatch = dm.Cluster + } + return discoveryMechanismKey{typ: dm.Type, name: nameToWatch} + case DiscoveryMechanismTypeLogicalDNS: + return discoveryMechanismKey{typ: dm.Type, name: dm.DNSHostname} + default: + return discoveryMechanismKey{} + } +} + +func (rr *resourceResolver) updateMechanisms(mechanisms []DiscoveryMechanism) { + rr.mu.Lock() + defer rr.mu.Unlock() + if equalDiscoveryMechanisms(rr.mechanisms, mechanisms) { + return + } + rr.mechanisms = mechanisms + rr.children = make([]discoveryMechanismAndResolver, len(mechanisms)) + newDMs := make(map[discoveryMechanismKey]bool) + + // Start one watch for each new discover mechanism {type+resource_name}. + for i, dm := range mechanisms { + dmKey := discoveryMechanismToKey(dm) + newDMs[dmKey] = true + dmAndResolver, ok := rr.childrenMap[dmKey] + if ok { + // If this is not new, keep the fields (especially childNameGen), + // and only update the DiscoveryMechanism. + // + // Note that the same dmKey doesn't mean the same + // DiscoveryMechanism. There are fields (e.g. + // MaxConcurrentRequests) in DiscoveryMechanism that are not copied + // to dmKey, we need to keep those updated. + dmAndResolver.dm = dm + rr.children[i] = dmAndResolver + continue + } + + // Create resolver for a newly seen resource. + var resolver endpointsResolver + switch dm.Type { + case DiscoveryMechanismTypeEDS: + resolver = newEDSResolver(dmKey.name, rr.parent.xdsClient, rr, rr.logger) + case DiscoveryMechanismTypeLogicalDNS: + resolver = newDNSResolver(dmKey.name, rr, rr.logger) + } + dmAndResolver = discoveryMechanismAndResolver{ + dm: dm, + r: resolver, + childNameGen: newNameGenerator(rr.childNameGeneratorSeqID), + } + rr.childrenMap[dmKey] = dmAndResolver + rr.children[i] = dmAndResolver + rr.childNameGeneratorSeqID++ + } + + // Stop the resources that were removed. + for dm, r := range rr.childrenMap { + if !newDMs[dm] { + delete(rr.childrenMap, dm) + r.r.stop() + } + } + // Regenerate even if there's no change in discovery mechanism, in case + // priority order changed. + rr.generateLocked() +} + +// resolveNow is typically called to trigger re-resolve of DNS. The EDS +// resolveNow() is a noop. +func (rr *resourceResolver) resolveNow() { + rr.mu.Lock() + defer rr.mu.Unlock() + for _, r := range rr.childrenMap { + r.r.resolveNow() + } +} + +func (rr *resourceResolver) stop(closing bool) { + rr.mu.Lock() + + // Save the previous childrenMap to stop the children outside the mutex, + // and reinitialize the map. We only need to reinitialize to allow for the + // policy to be reused if the resource comes back. In practice, this does + // not happen as the parent LB policy will also be closed, causing this to + // be removed entirely, but a future use case might want to reuse the + // policy instead. + cm := rr.childrenMap + rr.childrenMap = make(map[discoveryMechanismKey]discoveryMechanismAndResolver) + rr.mechanisms = nil + rr.children = nil + + rr.mu.Unlock() + + for _, r := range cm { + r.r.stop() + } + + if closing { + rr.serializerCancel() + <-rr.serializer.Done() + } + + // stop() is called when the LB policy is closed or when the underlying + // cluster resource is removed by the management server. In the latter case, + // an empty config update needs to be pushed to the child policy to ensure + // that a picker that fails RPCs is sent up to the channel. + // + // Resource resolver implementations are expected to not send any updates + // after they are stopped. Therefore, we don't have to worry about another + // write to this channel happening at the same time as this one. + select { + case <-rr.updateChannel: + default: + } + rr.updateChannel <- &resourceUpdate{} +} + +// generateLocked collects updates from all resolvers. It pushes the combined +// result on the update channel if all child resolvers have received at least +// one update. Otherwise it returns early. +// +// caller must hold rr.mu. +func (rr *resourceResolver) generateLocked() { + var ret []priorityConfig + for _, rDM := range rr.children { + u, ok := rDM.r.lastUpdate() + if !ok { + // Don't send updates to parent until all resolvers have update to + // send. + return + } + switch uu := u.(type) { + case xdsresource.EndpointsUpdate: + ret = append(ret, priorityConfig{mechanism: rDM.dm, edsResp: uu, childNameGen: rDM.childNameGen}) + case []string: + ret = append(ret, priorityConfig{mechanism: rDM.dm, addresses: uu, childNameGen: rDM.childNameGen}) + } + } + select { + case <-rr.updateChannel: + default: + } + rr.updateChannel <- &resourceUpdate{priorities: ret} +} + +func (rr *resourceResolver) onUpdate() { + rr.serializer.Schedule(func(context.Context) { + rr.mu.Lock() + rr.generateLocked() + rr.mu.Unlock() + }) +} diff --git a/xds/internal/balancer/clusterresolver/resource_resolver_dns.go b/xds/internal/balancer/clusterresolver/resource_resolver_dns.go new file mode 100644 index 000000000000..9052190b0ff0 --- /dev/null +++ b/xds/internal/balancer/clusterresolver/resource_resolver_dns.go @@ -0,0 +1,192 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package clusterresolver + +import ( + "fmt" + "net/url" + "sync" + + "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" +) + +var ( + newDNS = func(target resolver.Target, cc resolver.ClientConn, opts resolver.BuildOptions) (resolver.Resolver, error) { + // The dns resolver is registered by the grpc package. So, this call to + // resolver.Get() is never expected to return nil. + return resolver.Get("dns").Build(target, cc, opts) + } +) + +// dnsDiscoveryMechanism watches updates for the given DNS hostname. +// +// It implements resolver.ClientConn interface to work with the DNS resolver. +type dnsDiscoveryMechanism struct { + target string + topLevelResolver topLevelResolver + dnsR resolver.Resolver + logger *grpclog.PrefixLogger + + mu sync.Mutex + addrs []string + updateReceived bool +} + +// newDNSResolver creates an endpoints resolver which uses a DNS resolver under +// the hood. +// +// An error in parsing the provided target string or an error in creating a DNS +// resolver means that we will never be able to resolve the provided target +// strings to endpoints. The topLevelResolver propagates address updates to the +// clusterresolver LB policy **only** after it receives updates from all its +// child resolvers. Therefore, an error here means that the topLevelResolver +// will never send address updates to the clusterresolver LB policy. +// +// Calling the onError() callback will ensure that this error is +// propagated to the child policy which eventually move the channel to +// transient failure. +// +// The `dnsR` field is unset if we run into erros in this function. Therefore, a +// nil check is required wherever we access that field. +func newDNSResolver(target string, topLevelResolver topLevelResolver, logger *grpclog.PrefixLogger) *dnsDiscoveryMechanism { + ret := &dnsDiscoveryMechanism{ + target: target, + topLevelResolver: topLevelResolver, + logger: logger, + } + u, err := url.Parse("dns:///" + target) + if err != nil { + if ret.logger.V(2) { + ret.logger.Infof("Failed to parse dns hostname %q in clusterresolver LB policy", target) + } + ret.updateReceived = true + ret.topLevelResolver.onUpdate() + return ret + } + + r, err := newDNS(resolver.Target{URL: *u}, ret, resolver.BuildOptions{}) + if err != nil { + if ret.logger.V(2) { + ret.logger.Infof("Failed to build DNS resolver for target %q: %v", target, err) + } + ret.updateReceived = true + ret.topLevelResolver.onUpdate() + return ret + } + ret.dnsR = r + return ret +} + +func (dr *dnsDiscoveryMechanism) lastUpdate() (any, bool) { + dr.mu.Lock() + defer dr.mu.Unlock() + + if !dr.updateReceived { + return nil, false + } + return dr.addrs, true +} + +func (dr *dnsDiscoveryMechanism) resolveNow() { + if dr.dnsR != nil { + dr.dnsR.ResolveNow(resolver.ResolveNowOptions{}) + } +} + +// The definition of stop() mentions that implementations must not invoke any +// methods on the topLevelResolver once the call to `stop()` returns. The +// underlying dns resolver does not send any updates to the resolver.ClientConn +// interface passed to it (implemented by dnsDiscoveryMechanism in this case) +// after its `Close()` returns. Therefore, we can guarantee that no methods of +// the topLevelResolver are invoked after we return from this method. +func (dr *dnsDiscoveryMechanism) stop() { + if dr.dnsR != nil { + dr.dnsR.Close() + } +} + +// dnsDiscoveryMechanism needs to implement resolver.ClientConn interface to receive +// updates from the real DNS resolver. + +func (dr *dnsDiscoveryMechanism) UpdateState(state resolver.State) error { + if dr.logger.V(2) { + dr.logger.Infof("DNS discovery mechanism for resource %q reported an update: %s", dr.target, pretty.ToJSON(state)) + } + + dr.mu.Lock() + var addrs []string + if len(state.Endpoints) > 0 { + // Assume 1 address per endpoint, which is how DNS is expected to + // behave. The slice will grow as needed, however. + addrs = make([]string, 0, len(state.Endpoints)) + for _, e := range state.Endpoints { + for _, a := range e.Addresses { + addrs = append(addrs, a.Addr) + } + } + } else { + addrs = make([]string, len(state.Addresses)) + for i, a := range state.Addresses { + addrs[i] = a.Addr + } + } + dr.addrs = addrs + dr.updateReceived = true + dr.mu.Unlock() + + dr.topLevelResolver.onUpdate() + return nil +} + +func (dr *dnsDiscoveryMechanism) ReportError(err error) { + if dr.logger.V(2) { + dr.logger.Infof("DNS discovery mechanism for resource %q reported error: %v", dr.target, err) + } + + dr.mu.Lock() + // If a previous good update was received, suppress the error and continue + // using the previous update. If RPCs were succeeding prior to this, they + // will continue to do so. Also suppress errors if we previously received an + // error, since there will be no downstream effects of propagating this + // error. + if dr.updateReceived { + dr.mu.Unlock() + return + } + dr.addrs = nil + dr.updateReceived = true + dr.mu.Unlock() + + dr.topLevelResolver.onUpdate() +} + +func (dr *dnsDiscoveryMechanism) NewAddress(addresses []resolver.Address) { + dr.UpdateState(resolver.State{Addresses: addresses}) +} + +func (dr *dnsDiscoveryMechanism) NewServiceConfig(string) { + // This method is deprecated, and service config isn't supported. +} + +func (dr *dnsDiscoveryMechanism) ParseServiceConfig(string) *serviceconfig.ParseResult { + return &serviceconfig.ParseResult{Err: fmt.Errorf("service config not supported")} +} diff --git a/xds/internal/balancer/clusterresolver/resource_resolver_eds.go b/xds/internal/balancer/clusterresolver/resource_resolver_eds.go new file mode 100644 index 000000000000..3d0ec356e93a --- /dev/null +++ b/xds/internal/balancer/clusterresolver/resource_resolver_eds.go @@ -0,0 +1,140 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package clusterresolver + +import ( + "sync" + + "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" +) + +type edsDiscoveryMechanism struct { + nameToWatch string + cancelWatch func() + topLevelResolver topLevelResolver + stopped *grpcsync.Event + logger *grpclog.PrefixLogger + + mu sync.Mutex + update *xdsresource.EndpointsUpdate // Nil indicates no update received so far. +} + +func (er *edsDiscoveryMechanism) lastUpdate() (any, bool) { + er.mu.Lock() + defer er.mu.Unlock() + + if er.update == nil { + return nil, false + } + return *er.update, true +} + +func (er *edsDiscoveryMechanism) resolveNow() { +} + +// The definition of stop() mentions that implementations must not invoke any +// methods on the topLevelResolver once the call to `stop()` returns. +func (er *edsDiscoveryMechanism) stop() { + // Canceling a watch with the xDS client can race with an xDS response + // received around the same time, and can result in the watch callback being + // invoked after the watch is canceled. Callers need to handle this race, + // and we fire the stopped event here to ensure that a watch callback + // invocation around the same time becomes a no-op. + er.stopped.Fire() + er.cancelWatch() +} + +// newEDSResolver returns an implementation of the endpointsResolver interface +// that uses EDS to resolve the given name to endpoints. +func newEDSResolver(nameToWatch string, producer xdsresource.Producer, topLevelResolver topLevelResolver, logger *grpclog.PrefixLogger) *edsDiscoveryMechanism { + ret := &edsDiscoveryMechanism{ + nameToWatch: nameToWatch, + topLevelResolver: topLevelResolver, + logger: logger, + stopped: grpcsync.NewEvent(), + } + ret.cancelWatch = xdsresource.WatchEndpoints(producer, nameToWatch, ret) + return ret +} + +// OnUpdate is invoked to report an update for the resource being watched. +func (er *edsDiscoveryMechanism) OnUpdate(update *xdsresource.EndpointsResourceData) { + if er.stopped.HasFired() { + return + } + + er.mu.Lock() + er.update = &update.Resource + er.mu.Unlock() + + er.topLevelResolver.onUpdate() +} + +func (er *edsDiscoveryMechanism) OnError(err error) { + if er.stopped.HasFired() { + return + } + + if er.logger.V(2) { + er.logger.Infof("EDS discovery mechanism for resource %q reported error: %v", er.nameToWatch, err) + } + + er.mu.Lock() + if er.update != nil { + // Continue using a previously received good configuration if one + // exists. + er.mu.Unlock() + return + } + + // Else report an empty update that would result in no priority child being + // created for this discovery mechanism. This would result in the priority + // LB policy reporting TRANSIENT_FAILURE (as there would be no priorities or + // localities) if this was the only discovery mechanism, or would result in + // the priority LB policy using a lower priority discovery mechanism when + // that becomes available. + er.update = &xdsresource.EndpointsUpdate{} + er.mu.Unlock() + + er.topLevelResolver.onUpdate() +} + +func (er *edsDiscoveryMechanism) OnResourceDoesNotExist() { + if er.stopped.HasFired() { + return + } + + if er.logger.V(2) { + er.logger.Infof("EDS discovery mechanism for resource %q reported resource-does-not-exist error", er.nameToWatch) + } + + // Report an empty update that would result in no priority child being + // created for this discovery mechanism. This would result in the priority + // LB policy reporting TRANSIENT_FAILURE (as there would be no priorities or + // localities) if this was the only discovery mechanism, or would result in + // the priority LB policy using a lower priority discovery mechanism when + // that becomes available. + er.mu.Lock() + er.update = &xdsresource.EndpointsUpdate{} + er.mu.Unlock() + + er.topLevelResolver.onUpdate() +} diff --git a/xds/internal/balancer/edsbalancer/config.go b/xds/internal/balancer/edsbalancer/config.go deleted file mode 100644 index 95fa6d5b6aa4..000000000000 --- a/xds/internal/balancer/edsbalancer/config.go +++ /dev/null @@ -1,117 +0,0 @@ -/* - * - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package edsbalancer - -import ( - "encoding/json" - "fmt" - - "google.golang.org/grpc/balancer" - "google.golang.org/grpc/serviceconfig" -) - -// EDSConfig represents the loadBalancingConfig section of the service config -// for EDS balancers. -type EDSConfig struct { - serviceconfig.LoadBalancingConfig - // BalancerName represents the load balancer to use. - BalancerName string - // ChildPolicy represents the load balancing config for the child - // policy. - ChildPolicy *loadBalancingConfig - // FallBackPolicy represents the load balancing config for the - // fallback. - FallBackPolicy *loadBalancingConfig - // Name to use in EDS query. If not present, defaults to the server - // name from the target URI. - EDSServiceName string - // LRS server to send load reports to. If not present, load reporting - // will be disabled. If set to the empty string, load reporting will - // be sent to the same server that we obtained CDS data from. - LrsLoadReportingServerName *string -} - -// edsConfigJSON is the intermediate unmarshal result of EDSConfig. ChildPolicy -// and Fallbackspolicy are post-processed, and for each, the first installed -// policy is kept. -type edsConfigJSON struct { - BalancerName string - ChildPolicy []*loadBalancingConfig - FallbackPolicy []*loadBalancingConfig - EDSServiceName string - LRSLoadReportingServerName *string -} - -// UnmarshalJSON parses the JSON-encoded byte slice in data and stores it in l. -// When unmarshalling, we iterate through the childPolicy/fallbackPolicy lists -// and select the first LB policy which has been registered. -func (l *EDSConfig) UnmarshalJSON(data []byte) error { - var configJSON edsConfigJSON - if err := json.Unmarshal(data, &configJSON); err != nil { - return err - } - - l.BalancerName = configJSON.BalancerName - l.EDSServiceName = configJSON.EDSServiceName - l.LrsLoadReportingServerName = configJSON.LRSLoadReportingServerName - - for _, lbcfg := range configJSON.ChildPolicy { - if balancer.Get(lbcfg.Name) != nil { - l.ChildPolicy = lbcfg - break - } - } - - for _, lbcfg := range configJSON.FallbackPolicy { - if balancer.Get(lbcfg.Name) != nil { - l.FallBackPolicy = lbcfg - break - } - } - return nil -} - -// MarshalJSON returns a JSON encoding of l. -func (l *EDSConfig) MarshalJSON() ([]byte, error) { - return nil, fmt.Errorf("EDSConfig.MarshalJSON() is unimplemented") -} - -// loadBalancingConfig represents a single load balancing config, -// stored in JSON format. -type loadBalancingConfig struct { - Name string - Config json.RawMessage -} - -// MarshalJSON returns a JSON encoding of l. -func (l *loadBalancingConfig) MarshalJSON() ([]byte, error) { - return nil, fmt.Errorf("loadBalancingConfig.MarshalJSON() is unimplemented") -} - -// UnmarshalJSON parses the JSON-encoded byte slice in data and stores it in l. -func (l *loadBalancingConfig) UnmarshalJSON(data []byte) error { - var cfg map[string]json.RawMessage - if err := json.Unmarshal(data, &cfg); err != nil { - return err - } - for name, config := range cfg { - l.Name = name - l.Config = config - } - return nil -} diff --git a/xds/internal/balancer/edsbalancer/eds.go b/xds/internal/balancer/edsbalancer/eds.go deleted file mode 100644 index 5dc4a6a458ac..000000000000 --- a/xds/internal/balancer/edsbalancer/eds.go +++ /dev/null @@ -1,278 +0,0 @@ -/* - * - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -// Package edsbalancer contains EDS balancer implementation. -package edsbalancer - -import ( - "context" - "encoding/json" - "fmt" - - "github.com/google/go-cmp/cmp" - "google.golang.org/grpc/balancer" - "google.golang.org/grpc/balancer/roundrobin" - "google.golang.org/grpc/connectivity" - "google.golang.org/grpc/internal/buffer" - "google.golang.org/grpc/internal/grpclog" - "google.golang.org/grpc/serviceconfig" - "google.golang.org/grpc/xds/internal/balancer/lrs" - xdsclient "google.golang.org/grpc/xds/internal/client" -) - -const ( - edsName = "eds_experimental" -) - -var ( - newEDSBalancer = func(cc balancer.ClientConn, enqueueState func(priorityType, balancer.State), loadStore lrs.Store, logger *grpclog.PrefixLogger) edsBalancerImplInterface { - return newEDSBalancerImpl(cc, enqueueState, loadStore, logger) - } -) - -func init() { - balancer.Register(&edsBalancerBuilder{}) -} - -type edsBalancerBuilder struct{} - -// Build helps implement the balancer.Builder interface. -func (b *edsBalancerBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { - ctx, cancel := context.WithCancel(context.Background()) - x := &edsBalancer{ - ctx: ctx, - cancel: cancel, - cc: cc, - buildOpts: opts, - grpcUpdate: make(chan interface{}), - xdsClientUpdate: make(chan *edsUpdate), - childPolicyUpdate: buffer.NewUnbounded(), - } - loadStore := lrs.NewStore() - x.logger = prefixLogger((x)) - x.edsImpl = newEDSBalancer(x.cc, x.enqueueChildBalancerState, loadStore, x.logger) - x.client = newXDSClientWrapper(x.handleEDSUpdate, x.buildOpts, loadStore, x.logger) - x.logger.Infof("Created") - go x.run() - return x -} - -func (b *edsBalancerBuilder) Name() string { - return edsName -} - -func (b *edsBalancerBuilder) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { - var cfg EDSConfig - if err := json.Unmarshal(c, &cfg); err != nil { - return nil, fmt.Errorf("unable to unmarshal balancer config %s into EDSConfig, error: %v", string(c), err) - } - return &cfg, nil -} - -// edsBalancerImplInterface defines the interface that edsBalancerImpl must -// implement to communicate with edsBalancer. -// -// It's implemented by the real eds balancer and a fake testing eds balancer. -type edsBalancerImplInterface interface { - // handleEDSResponse passes the received EDS message from traffic director to eds balancer. - handleEDSResponse(edsResp xdsclient.EndpointsUpdate) - // handleChildPolicy updates the eds balancer the intra-cluster load balancing policy to use. - handleChildPolicy(name string, config json.RawMessage) - // handleSubConnStateChange handles state change for SubConn. - handleSubConnStateChange(sc balancer.SubConn, state connectivity.State) - // updateState handle a balancer state update from the priority. - updateState(priority priorityType, s balancer.State) - // close closes the eds balancer. - close() -} - -// edsBalancer manages xdsClient and the actual EDS balancer implementation that -// does load balancing. -// -// It currently has only an edsBalancer. Later, we may add fallback. -type edsBalancer struct { - cc balancer.ClientConn // *xdsClientConn - buildOpts balancer.BuildOptions - ctx context.Context - cancel context.CancelFunc - - logger *grpclog.PrefixLogger - - // edsBalancer continuously monitor the channels below, and will handle events from them in sync. - grpcUpdate chan interface{} - xdsClientUpdate chan *edsUpdate - childPolicyUpdate *buffer.Unbounded - - client *xdsclientWrapper // may change when passed a different service config - config *EDSConfig // may change when passed a different service config - edsImpl edsBalancerImplInterface -} - -// run gets executed in a goroutine once edsBalancer is created. It monitors updates from grpc, -// xdsClient and load balancer. It synchronizes the operations that happen inside edsBalancer. It -// exits when edsBalancer is closed. -func (x *edsBalancer) run() { - for { - select { - case update := <-x.grpcUpdate: - x.handleGRPCUpdate(update) - case update := <-x.xdsClientUpdate: - x.handleXDSClientUpdate(update) - case update := <-x.childPolicyUpdate.Get(): - x.childPolicyUpdate.Load() - u := update.(*balancerStateWithPriority) - x.edsImpl.updateState(u.priority, u.s) - case <-x.ctx.Done(): - x.client.close() - x.edsImpl.close() - return - } - } -} - -// handleErrorFromUpdate handles both the error from parent ClientConn (from CDS -// balancer) and the error from xds client (from the watcher). fromParent is -// true if error is from parent ClientConn. -// -// If the error is connection error, it should be handled for fallback purposes. -// -// If the error is resource-not-found: -// - If it's from CDS balancer (shows as a resolver error), it means LDS or CDS -// resources were removed. The EDS watch should be canceled. -// - If it's from xds client, it means EDS resource were removed. The EDS -// watcher should keep watching. -// In both cases, the sub-balancers will be closed, and the future picks will -// fail. -func (x *edsBalancer) handleErrorFromUpdate(err error, fromParent bool) { - if xdsclient.ErrType(err) == xdsclient.ErrorTypeResourceNotFound { - if fromParent { - // This is an error from the parent ClientConn (can be the parent - // CDS balancer), and is a resource-not-found error. This means the - // resource (can be either LDS or CDS) was removed. Stop the EDS - // watch. - x.client.cancelWatch() - } - x.edsImpl.handleEDSResponse(xdsclient.EndpointsUpdate{}) - } -} - -func (x *edsBalancer) handleGRPCUpdate(update interface{}) { - switch u := update.(type) { - case *subConnStateUpdate: - x.edsImpl.handleSubConnStateChange(u.sc, u.state.ConnectivityState) - case *balancer.ClientConnState: - x.logger.Infof("Receive update from resolver, balancer config: %+v", u.BalancerConfig) - cfg, _ := u.BalancerConfig.(*EDSConfig) - if cfg == nil { - // service config parsing failed. should never happen. - return - } - - x.client.handleUpdate(cfg, u.ResolverState.Attributes) - - if x.config == nil { - x.config = cfg - return - } - - // We will update the edsImpl with the new child policy, if we got a - // different one. - if !cmp.Equal(cfg.ChildPolicy, x.config.ChildPolicy) { - if cfg.ChildPolicy != nil { - x.edsImpl.handleChildPolicy(cfg.ChildPolicy.Name, cfg.ChildPolicy.Config) - } else { - x.edsImpl.handleChildPolicy(roundrobin.Name, nil) - } - } - - x.config = cfg - case error: - x.handleErrorFromUpdate(u, true) - default: - // unreachable path - panic("wrong update type") - } -} - -func (x *edsBalancer) handleXDSClientUpdate(update *edsUpdate) { - if err := update.err; err != nil { - x.handleErrorFromUpdate(err, false) - return - } - x.edsImpl.handleEDSResponse(update.resp) -} - -type subConnStateUpdate struct { - sc balancer.SubConn - state balancer.SubConnState -} - -func (x *edsBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { - update := &subConnStateUpdate{ - sc: sc, - state: state, - } - select { - case x.grpcUpdate <- update: - case <-x.ctx.Done(): - } -} - -func (x *edsBalancer) ResolverError(err error) { - select { - case x.grpcUpdate <- err: - case <-x.ctx.Done(): - } -} - -func (x *edsBalancer) UpdateClientConnState(s balancer.ClientConnState) error { - select { - case x.grpcUpdate <- &s: - case <-x.ctx.Done(): - } - return nil -} - -type edsUpdate struct { - resp xdsclient.EndpointsUpdate - err error -} - -func (x *edsBalancer) handleEDSUpdate(resp xdsclient.EndpointsUpdate, err error) { - select { - case x.xdsClientUpdate <- &edsUpdate{resp: resp, err: err}: - case <-x.ctx.Done(): - } -} - -type balancerStateWithPriority struct { - priority priorityType - s balancer.State -} - -func (x *edsBalancer) enqueueChildBalancerState(p priorityType, s balancer.State) { - x.childPolicyUpdate.Put(&balancerStateWithPriority{ - priority: p, - s: s, - }) -} - -func (x *edsBalancer) Close() { - x.cancel() - x.logger.Infof("Shutdown") -} diff --git a/xds/internal/balancer/edsbalancer/eds_impl.go b/xds/internal/balancer/edsbalancer/eds_impl.go deleted file mode 100644 index 95172cfb0ab9..000000000000 --- a/xds/internal/balancer/edsbalancer/eds_impl.go +++ /dev/null @@ -1,489 +0,0 @@ -/* - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package edsbalancer - -import ( - "encoding/json" - "reflect" - "sync" - "time" - - "github.com/google/go-cmp/cmp" - "google.golang.org/grpc/balancer" - "google.golang.org/grpc/balancer/base" - "google.golang.org/grpc/balancer/roundrobin" - "google.golang.org/grpc/balancer/weightedroundrobin" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/connectivity" - "google.golang.org/grpc/internal/grpclog" - "google.golang.org/grpc/resolver" - "google.golang.org/grpc/status" - "google.golang.org/grpc/xds/internal" - "google.golang.org/grpc/xds/internal/balancer/balancergroup" - "google.golang.org/grpc/xds/internal/balancer/lrs" - "google.golang.org/grpc/xds/internal/balancer/weightedtarget/weightedaggregator" - xdsclient "google.golang.org/grpc/xds/internal/client" -) - -// TODO: make this a environment variable? -var defaultPriorityInitTimeout = 10 * time.Second - -type localityConfig struct { - weight uint32 - addrs []resolver.Address -} - -// balancerGroupWithConfig contains the localities with the same priority. It -// manages all localities using a balancerGroup. -type balancerGroupWithConfig struct { - bg *balancergroup.BalancerGroup - stateAggregator *weightedaggregator.Aggregator - configs map[internal.LocalityID]*localityConfig -} - -// edsBalancerImpl does load balancing based on the EDS responses. Note that it -// doesn't implement the balancer interface. It's intended to be used by a high -// level balancer implementation. -// -// The localities are picked as weighted round robin. A configurable child -// policy is used to manage endpoints in each locality. -type edsBalancerImpl struct { - cc balancer.ClientConn - logger *grpclog.PrefixLogger - - enqueueChildBalancerStateUpdate func(priorityType, balancer.State) - - subBalancerBuilder balancer.Builder - loadStore lrs.Store - priorityToLocalities map[priorityType]*balancerGroupWithConfig - respReceived bool - - // There's no need to hold any mutexes at the same time. The order to take - // mutex should be: priorityMu > subConnMu, but this is implicit via - // balancers (starting balancer with next priority while holding priorityMu, - // and the balancer may create new SubConn). - - priorityMu sync.Mutex - // priorities are pointers, and will be nil when EDS returns empty result. - priorityInUse priorityType - priorityLowest priorityType - priorityToState map[priorityType]*balancer.State - // The timer to give a priority 10 seconds to connect. And if the priority - // doesn't go into Ready/Failure, start the next priority. - // - // One timer is enough because there can be at most one priority in init - // state. - priorityInitTimer *time.Timer - - subConnMu sync.Mutex - subConnToPriority map[balancer.SubConn]priorityType - - pickerMu sync.Mutex - dropConfig []xdsclient.OverloadDropConfig - drops []*dropper - innerState balancer.State // The state of the picker without drop support. -} - -// newEDSBalancerImpl create a new edsBalancerImpl. -func newEDSBalancerImpl(cc balancer.ClientConn, enqueueState func(priorityType, balancer.State), loadStore lrs.Store, logger *grpclog.PrefixLogger) *edsBalancerImpl { - edsImpl := &edsBalancerImpl{ - cc: cc, - logger: logger, - subBalancerBuilder: balancer.Get(roundrobin.Name), - - enqueueChildBalancerStateUpdate: enqueueState, - - priorityToLocalities: make(map[priorityType]*balancerGroupWithConfig), - priorityToState: make(map[priorityType]*balancer.State), - subConnToPriority: make(map[balancer.SubConn]priorityType), - loadStore: loadStore, - } - // Don't start balancer group here. Start it when handling the first EDS - // response. Otherwise the balancer group will be started with round-robin, - // and if users specify a different sub-balancer, all balancers in balancer - // group will be closed and recreated when sub-balancer update happens. - return edsImpl -} - -// handleChildPolicy updates the child balancers handling endpoints. Child -// policy is roundrobin by default. If the specified balancer is not installed, -// the old child balancer will be used. -// -// HandleChildPolicy and HandleEDSResponse must be called by the same goroutine. -func (edsImpl *edsBalancerImpl) handleChildPolicy(name string, config json.RawMessage) { - if edsImpl.subBalancerBuilder.Name() == name { - return - } - newSubBalancerBuilder := balancer.Get(name) - if newSubBalancerBuilder == nil { - edsImpl.logger.Infof("edsBalancerImpl: failed to find balancer with name %q, keep using %q", name, edsImpl.subBalancerBuilder.Name()) - return - } - edsImpl.subBalancerBuilder = newSubBalancerBuilder - for _, bgwc := range edsImpl.priorityToLocalities { - if bgwc == nil { - continue - } - for id, config := range bgwc.configs { - // TODO: (eds) add support to balancer group to support smoothly - // switching sub-balancers (keep old balancer around until new - // balancer becomes ready). - bgwc.bg.Remove(id) - bgwc.bg.Add(id, edsImpl.subBalancerBuilder) - bgwc.bg.UpdateClientConnState(id, balancer.ClientConnState{ - ResolverState: resolver.State{Addresses: config.addrs}, - }) - // This doesn't need to manually update picker, because the new - // sub-balancer will send it's picker later. - } - } -} - -// updateDrops compares new drop policies with the old. If they are different, -// it updates the drop policies and send ClientConn an updated picker. -func (edsImpl *edsBalancerImpl) updateDrops(dropConfig []xdsclient.OverloadDropConfig) { - if cmp.Equal(dropConfig, edsImpl.dropConfig) { - return - } - edsImpl.pickerMu.Lock() - edsImpl.dropConfig = dropConfig - var newDrops []*dropper - for _, c := range edsImpl.dropConfig { - newDrops = append(newDrops, newDropper(c)) - } - edsImpl.drops = newDrops - if edsImpl.innerState.Picker != nil { - // Update picker with old inner picker, new drops. - edsImpl.cc.UpdateState(balancer.State{ - ConnectivityState: edsImpl.innerState.ConnectivityState, - Picker: newDropPicker(edsImpl.innerState.Picker, newDrops, edsImpl.loadStore)}, - ) - } - edsImpl.pickerMu.Unlock() -} - -// handleEDSResponse handles the EDS response and creates/deletes localities and -// SubConns. It also handles drops. -// -// HandleChildPolicy and HandleEDSResponse must be called by the same goroutine. -func (edsImpl *edsBalancerImpl) handleEDSResponse(edsResp xdsclient.EndpointsUpdate) { - // TODO: Unhandled fields from EDS response: - // - edsResp.GetPolicy().GetOverprovisioningFactor() - // - locality.GetPriority() - // - lbEndpoint.GetMetadata(): contains BNS name, send to sub-balancers - // - as service config or as resolved address - // - if socketAddress is not ip:port - // - socketAddress.GetNamedPort(), socketAddress.GetResolverName() - // - resolve endpoint's name with another resolver - - // If the first EDS update is an empty update, nothing is changing from the - // previous update (which is the default empty value). We need to explicitly - // handle first update being empty, and send a transient failure picker. - // - // TODO: define Equal() on type EndpointUpdate to avoid DeepEqual. And do - // the same for the other types. - if !edsImpl.respReceived && reflect.DeepEqual(edsResp, xdsclient.EndpointsUpdate{}) { - edsImpl.cc.UpdateState(balancer.State{ConnectivityState: connectivity.TransientFailure, Picker: base.NewErrPicker(errAllPrioritiesRemoved)}) - } - edsImpl.respReceived = true - - edsImpl.updateDrops(edsResp.Drops) - - // Filter out all localities with weight 0. - // - // Locality weighted load balancer can be enabled by setting an option in - // CDS, and the weight of each locality. Currently, without the guarantee - // that CDS is always sent, we assume locality weighted load balance is - // always enabled, and ignore all weight 0 localities. - // - // In the future, we should look at the config in CDS response and decide - // whether locality weight matters. - newLocalitiesWithPriority := make(map[priorityType][]xdsclient.Locality) - for _, locality := range edsResp.Localities { - if locality.Weight == 0 { - continue - } - priority := newPriorityType(locality.Priority) - newLocalitiesWithPriority[priority] = append(newLocalitiesWithPriority[priority], locality) - } - - var ( - priorityLowest priorityType - priorityChanged bool - ) - - for priority, newLocalities := range newLocalitiesWithPriority { - if !priorityLowest.isSet() || priorityLowest.higherThan(priority) { - priorityLowest = priority - } - - bgwc, ok := edsImpl.priorityToLocalities[priority] - if !ok { - // Create balancer group if it's never created (this is the first - // time this priority is received). We don't start it here. It may - // be started when necessary (e.g. when higher is down, or if it's a - // new lowest priority). - ccPriorityWrapper := edsImpl.ccWrapperWithPriority(priority) - stateAggregator := weightedaggregator.New(ccPriorityWrapper, edsImpl.logger, newRandomWRR) - bgwc = &balancerGroupWithConfig{ - bg: balancergroup.New(ccPriorityWrapper, stateAggregator, edsImpl.loadStore, edsImpl.logger), - stateAggregator: stateAggregator, - configs: make(map[internal.LocalityID]*localityConfig), - } - edsImpl.priorityToLocalities[priority] = bgwc - priorityChanged = true - edsImpl.logger.Infof("New priority %v added", priority) - } - edsImpl.handleEDSResponsePerPriority(bgwc, newLocalities) - } - edsImpl.priorityLowest = priorityLowest - - // Delete priorities that are removed in the latest response, and also close - // the balancer group. - for p, bgwc := range edsImpl.priorityToLocalities { - if _, ok := newLocalitiesWithPriority[p]; !ok { - delete(edsImpl.priorityToLocalities, p) - bgwc.bg.Close() - delete(edsImpl.priorityToState, p) - priorityChanged = true - edsImpl.logger.Infof("Priority %v deleted", p) - } - } - - // If priority was added/removed, it may affect the balancer group to use. - // E.g. priorityInUse was removed, or all priorities are down, and a new - // lower priority was added. - if priorityChanged { - edsImpl.handlePriorityChange() - } -} - -func (edsImpl *edsBalancerImpl) handleEDSResponsePerPriority(bgwc *balancerGroupWithConfig, newLocalities []xdsclient.Locality) { - // newLocalitiesSet contains all names of localities in the new EDS response - // for the same priority. It's used to delete localities that are removed in - // the new EDS response. - newLocalitiesSet := make(map[internal.LocalityID]struct{}) - var rebuildStateAndPicker bool - for _, locality := range newLocalities { - // One balancer for each locality. - - lid := locality.ID - newLocalitiesSet[lid] = struct{}{} - - newWeight := locality.Weight - var newAddrs []resolver.Address - for _, lbEndpoint := range locality.Endpoints { - // Filter out all "unhealthy" endpoints (unknown and - // healthy are both considered to be healthy: - // https://www.envoyproxy.io/docs/envoy/latest/api-v2/api/v2/core/health_check.proto#envoy-api-enum-core-healthstatus). - if lbEndpoint.HealthStatus != xdsclient.EndpointHealthStatusHealthy && - lbEndpoint.HealthStatus != xdsclient.EndpointHealthStatusUnknown { - continue - } - - address := resolver.Address{ - Addr: lbEndpoint.Address, - } - if edsImpl.subBalancerBuilder.Name() == weightedroundrobin.Name && lbEndpoint.Weight != 0 { - ai := weightedroundrobin.AddrInfo{Weight: lbEndpoint.Weight} - address = weightedroundrobin.SetAddrInfo(address, ai) - // Metadata field in resolver.Address is deprecated. The - // attributes field should be used to specify arbitrary - // attributes about the address. We still need to populate the - // Metadata field here to allow users of this field to migrate - // to the new one. - // TODO(easwars): Remove this once all users have migrated. - // See https://github.com/grpc/grpc-go/issues/3563. - address.Metadata = &ai - } - newAddrs = append(newAddrs, address) - } - var weightChanged, addrsChanged bool - config, ok := bgwc.configs[lid] - if !ok { - // A new balancer, add it to balancer group and balancer map. - bgwc.stateAggregator.Add(lid, newWeight) - bgwc.bg.Add(lid, edsImpl.subBalancerBuilder) - config = &localityConfig{ - weight: newWeight, - } - bgwc.configs[lid] = config - - // weightChanged is false for new locality, because there's no need - // to update weight in bg. - addrsChanged = true - edsImpl.logger.Infof("New locality %v added", lid) - } else { - // Compare weight and addrs. - if config.weight != newWeight { - weightChanged = true - } - if !cmp.Equal(config.addrs, newAddrs) { - addrsChanged = true - } - edsImpl.logger.Infof("Locality %v updated, weightedChanged: %v, addrsChanged: %v", lid, weightChanged, addrsChanged) - } - - if weightChanged { - config.weight = newWeight - bgwc.stateAggregator.UpdateWeight(lid, newWeight) - rebuildStateAndPicker = true - } - - if addrsChanged { - config.addrs = newAddrs - bgwc.bg.UpdateClientConnState(lid, balancer.ClientConnState{ - ResolverState: resolver.State{Addresses: newAddrs}, - }) - } - } - - // Delete localities that are removed in the latest response. - for lid := range bgwc.configs { - if _, ok := newLocalitiesSet[lid]; !ok { - bgwc.stateAggregator.Remove(lid) - bgwc.bg.Remove(lid) - delete(bgwc.configs, lid) - edsImpl.logger.Infof("Locality %v deleted", lid) - rebuildStateAndPicker = true - } - } - - if rebuildStateAndPicker { - bgwc.stateAggregator.BuildAndUpdate() - } -} - -// handleSubConnStateChange handles the state change and update pickers accordingly. -func (edsImpl *edsBalancerImpl) handleSubConnStateChange(sc balancer.SubConn, s connectivity.State) { - edsImpl.subConnMu.Lock() - var bgwc *balancerGroupWithConfig - if p, ok := edsImpl.subConnToPriority[sc]; ok { - if s == connectivity.Shutdown { - // Only delete sc from the map when state changed to Shutdown. - delete(edsImpl.subConnToPriority, sc) - } - bgwc = edsImpl.priorityToLocalities[p] - } - edsImpl.subConnMu.Unlock() - if bgwc == nil { - edsImpl.logger.Infof("edsBalancerImpl: priority not found for sc state change") - return - } - if bg := bgwc.bg; bg != nil { - bg.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: s}) - } -} - -// updateState first handles priority, and then wraps picker in a drop picker -// before forwarding the update. -func (edsImpl *edsBalancerImpl) updateState(priority priorityType, s balancer.State) { - _, ok := edsImpl.priorityToLocalities[priority] - if !ok { - edsImpl.logger.Infof("eds: received picker update from unknown priority") - return - } - - if edsImpl.handlePriorityWithNewState(priority, s) { - edsImpl.pickerMu.Lock() - defer edsImpl.pickerMu.Unlock() - edsImpl.innerState = s - // Don't reset drops when it's a state change. - edsImpl.cc.UpdateState(balancer.State{ConnectivityState: s.ConnectivityState, Picker: newDropPicker(s.Picker, edsImpl.drops, edsImpl.loadStore)}) - } -} - -func (edsImpl *edsBalancerImpl) ccWrapperWithPriority(priority priorityType) *edsBalancerWrapperCC { - return &edsBalancerWrapperCC{ - ClientConn: edsImpl.cc, - priority: priority, - parent: edsImpl, - } -} - -// edsBalancerWrapperCC implements the balancer.ClientConn API and get passed to -// each balancer group. It contains the locality priority. -type edsBalancerWrapperCC struct { - balancer.ClientConn - priority priorityType - parent *edsBalancerImpl -} - -func (ebwcc *edsBalancerWrapperCC) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) { - return ebwcc.parent.newSubConn(ebwcc.priority, addrs, opts) -} -func (ebwcc *edsBalancerWrapperCC) UpdateState(state balancer.State) { - ebwcc.parent.enqueueChildBalancerStateUpdate(ebwcc.priority, state) -} - -func (edsImpl *edsBalancerImpl) newSubConn(priority priorityType, addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) { - sc, err := edsImpl.cc.NewSubConn(addrs, opts) - if err != nil { - return nil, err - } - edsImpl.subConnMu.Lock() - edsImpl.subConnToPriority[sc] = priority - edsImpl.subConnMu.Unlock() - return sc, nil -} - -// close closes the balancer. -func (edsImpl *edsBalancerImpl) close() { - for _, bgwc := range edsImpl.priorityToLocalities { - if bg := bgwc.bg; bg != nil { - bgwc.stateAggregator.Stop() - bg.Close() - } - } -} - -type dropPicker struct { - drops []*dropper - p balancer.Picker - loadStore lrs.Store -} - -func newDropPicker(p balancer.Picker, drops []*dropper, loadStore lrs.Store) *dropPicker { - return &dropPicker{ - drops: drops, - p: p, - loadStore: loadStore, - } -} - -func (d *dropPicker) Pick(info balancer.PickInfo) (balancer.PickResult, error) { - var ( - drop bool - category string - ) - for _, dp := range d.drops { - if dp.drop() { - drop = true - category = dp.c.Category - break - } - } - if drop { - if d.loadStore != nil { - d.loadStore.CallDropped(category) - } - return balancer.PickResult{}, status.Errorf(codes.Unavailable, "RPC is dropped") - } - // TODO: (eds) don't drop unless the inner picker is READY. Similar to - // https://github.com/grpc/grpc-go/issues/2622. - return d.p.Pick(info) -} diff --git a/xds/internal/balancer/edsbalancer/eds_impl_priority.go b/xds/internal/balancer/edsbalancer/eds_impl_priority.go deleted file mode 100644 index 53ac6ef5e873..000000000000 --- a/xds/internal/balancer/edsbalancer/eds_impl_priority.go +++ /dev/null @@ -1,358 +0,0 @@ -/* - * - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package edsbalancer - -import ( - "errors" - "fmt" - "time" - - "google.golang.org/grpc/balancer" - "google.golang.org/grpc/balancer/base" - "google.golang.org/grpc/connectivity" -) - -var errAllPrioritiesRemoved = errors.New("eds: no locality is provided, all priorities are removed") - -// handlePriorityChange handles priority after EDS adds/removes a -// priority. -// -// - If all priorities were deleted, unset priorityInUse, and set parent -// ClientConn to TransientFailure -// - If priorityInUse wasn't set, this is either the first EDS resp, or the -// previous EDS resp deleted everything. Set priorityInUse to 0, and start 0. -// - If priorityInUse was deleted, send the picker from the new lowest priority -// to parent ClientConn, and set priorityInUse to the new lowest. -// - If priorityInUse has a non-Ready state, and also there's a priority lower -// than priorityInUse (which means a lower priority was added), set the next -// priority as new priorityInUse, and start the bg. -func (edsImpl *edsBalancerImpl) handlePriorityChange() { - edsImpl.priorityMu.Lock() - defer edsImpl.priorityMu.Unlock() - - // Everything was removed by EDS. - if !edsImpl.priorityLowest.isSet() { - edsImpl.priorityInUse = newPriorityTypeUnset() - // Stop the init timer. This can happen if the only priority is removed - // shortly after it's added. - if timer := edsImpl.priorityInitTimer; timer != nil { - timer.Stop() - edsImpl.priorityInitTimer = nil - } - edsImpl.cc.UpdateState(balancer.State{ConnectivityState: connectivity.TransientFailure, Picker: base.NewErrPicker(errAllPrioritiesRemoved)}) - return - } - - // priorityInUse wasn't set, use 0. - if !edsImpl.priorityInUse.isSet() { - edsImpl.logger.Infof("Switching priority from unset to %v", 0) - edsImpl.startPriority(newPriorityType(0)) - return - } - - // priorityInUse was deleted, use the new lowest. - if _, ok := edsImpl.priorityToLocalities[edsImpl.priorityInUse]; !ok { - oldP := edsImpl.priorityInUse - edsImpl.priorityInUse = edsImpl.priorityLowest - edsImpl.logger.Infof("Switching priority from %v to %v, because former was deleted", oldP, edsImpl.priorityInUse) - if s, ok := edsImpl.priorityToState[edsImpl.priorityLowest]; ok { - edsImpl.cc.UpdateState(*s) - } else { - // If state for priorityLowest is not found, this means priorityLowest was - // started, but never sent any update. The init timer fired and - // triggered the next priority. The old_priorityInUse (that was just - // deleted EDS) was picked later. - // - // We don't have an old state to send to parent, but we also don't - // want parent to keep using picker from old_priorityInUse. Send an - // update to trigger block picks until a new picker is ready. - edsImpl.cc.UpdateState(balancer.State{ConnectivityState: connectivity.Connecting, Picker: base.NewErrPicker(balancer.ErrNoSubConnAvailable)}) - } - return - } - - // priorityInUse is not ready, look for next priority, and use if found. - if s, ok := edsImpl.priorityToState[edsImpl.priorityInUse]; ok && s.ConnectivityState != connectivity.Ready { - pNext := edsImpl.priorityInUse.nextLower() - if _, ok := edsImpl.priorityToLocalities[pNext]; ok { - edsImpl.logger.Infof("Switching priority from %v to %v, because latter was added, and former wasn't Ready") - edsImpl.startPriority(pNext) - } - } -} - -// startPriority sets priorityInUse to p, and starts the balancer group for p. -// It also starts a timer to fall to next priority after timeout. -// -// Caller must hold priorityMu, priority must exist, and edsImpl.priorityInUse -// must be non-nil. -func (edsImpl *edsBalancerImpl) startPriority(priority priorityType) { - edsImpl.priorityInUse = priority - p := edsImpl.priorityToLocalities[priority] - // NOTE: this will eventually send addresses to sub-balancers. If the - // sub-balancer tries to update picker, it will result in a deadlock on - // priorityMu in the update is handled synchronously. The deadlock is - // currently avoided by handling balancer update in a goroutine (the run - // goroutine in the parent eds balancer). When priority balancer is split - // into its own, this asynchronous state handling needs to be copied. - p.stateAggregator.Start() - p.bg.Start() - // startPriority can be called when - // 1. first EDS resp, start p0 - // 2. a high priority goes Failure, start next - // 3. a high priority init timeout, start next - // - // In all the cases, the existing init timer is either closed, also already - // expired. There's no need to close the old timer. - edsImpl.priorityInitTimer = time.AfterFunc(defaultPriorityInitTimeout, func() { - edsImpl.priorityMu.Lock() - defer edsImpl.priorityMu.Unlock() - if !edsImpl.priorityInUse.isSet() || !edsImpl.priorityInUse.equal(priority) { - return - } - edsImpl.priorityInitTimer = nil - pNext := priority.nextLower() - if _, ok := edsImpl.priorityToLocalities[pNext]; ok { - edsImpl.startPriority(pNext) - } - }) -} - -// handlePriorityWithNewState start/close priorities based on the connectivity -// state. It returns whether the state should be forwarded to parent ClientConn. -func (edsImpl *edsBalancerImpl) handlePriorityWithNewState(priority priorityType, s balancer.State) bool { - edsImpl.priorityMu.Lock() - defer edsImpl.priorityMu.Unlock() - - if !edsImpl.priorityInUse.isSet() { - edsImpl.logger.Infof("eds: received picker update when no priority is in use (EDS returned an empty list)") - return false - } - - if edsImpl.priorityInUse.higherThan(priority) { - // Lower priorities should all be closed, this is an unexpected update. - edsImpl.logger.Infof("eds: received picker update from priority lower then priorityInUse") - return false - } - - bState, ok := edsImpl.priorityToState[priority] - if !ok { - bState = &balancer.State{} - edsImpl.priorityToState[priority] = bState - } - oldState := bState.ConnectivityState - *bState = s - - switch s.ConnectivityState { - case connectivity.Ready: - return edsImpl.handlePriorityWithNewStateReady(priority) - case connectivity.TransientFailure: - return edsImpl.handlePriorityWithNewStateTransientFailure(priority) - case connectivity.Connecting: - return edsImpl.handlePriorityWithNewStateConnecting(priority, oldState) - default: - // New state is Idle, should never happen. Don't forward. - return false - } -} - -// handlePriorityWithNewStateReady handles state Ready and decides whether to -// forward update or not. -// -// An update with state Ready: -// - If it's from higher priority: -// - Forward the update -// - Set the priority as priorityInUse -// - Close all priorities lower than this one -// - If it's from priorityInUse: -// - Forward and do nothing else -// -// Caller must make sure priorityInUse is not higher than priority. -// -// Caller must hold priorityMu. -func (edsImpl *edsBalancerImpl) handlePriorityWithNewStateReady(priority priorityType) bool { - // If one priority higher or equal to priorityInUse goes Ready, stop the - // init timer. If update is from higher than priorityInUse, - // priorityInUse will be closed, and the init timer will become useless. - if timer := edsImpl.priorityInitTimer; timer != nil { - timer.Stop() - edsImpl.priorityInitTimer = nil - } - - if edsImpl.priorityInUse.lowerThan(priority) { - edsImpl.logger.Infof("Switching priority from %v to %v, because latter became Ready", edsImpl.priorityInUse, priority) - edsImpl.priorityInUse = priority - for i := priority.nextLower(); !i.lowerThan(edsImpl.priorityLowest); i = i.nextLower() { - bgwc := edsImpl.priorityToLocalities[i] - bgwc.stateAggregator.Stop() - bgwc.bg.Close() - } - return true - } - return true -} - -// handlePriorityWithNewStateTransientFailure handles state TransientFailure and -// decides whether to forward update or not. -// -// An update with state Failure: -// - If it's from a higher priority: -// - Do not forward, and do nothing -// - If it's from priorityInUse: -// - If there's no lower: -// - Forward and do nothing else -// - If there's a lower priority: -// - Forward -// - Set lower as priorityInUse -// - Start lower -// -// Caller must make sure priorityInUse is not higher than priority. -// -// Caller must hold priorityMu. -func (edsImpl *edsBalancerImpl) handlePriorityWithNewStateTransientFailure(priority priorityType) bool { - if edsImpl.priorityInUse.lowerThan(priority) { - return false - } - // priorityInUse sends a failure. Stop its init timer. - if timer := edsImpl.priorityInitTimer; timer != nil { - timer.Stop() - edsImpl.priorityInitTimer = nil - } - pNext := priority.nextLower() - if _, okNext := edsImpl.priorityToLocalities[pNext]; !okNext { - return true - } - edsImpl.logger.Infof("Switching priority from %v to %v, because former became TransientFailure", priority, pNext) - edsImpl.startPriority(pNext) - return true -} - -// handlePriorityWithNewStateConnecting handles state Connecting and decides -// whether to forward update or not. -// -// An update with state Connecting: -// - If it's from a higher priority -// - Do nothing -// - If it's from priorityInUse, the behavior depends on previous state. -// -// When new state is Connecting, the behavior depends on previous state. If the -// previous state was Ready, this is a transition out from Ready to Connecting. -// Assuming there are multiple backends in the same priority, this mean we are -// in a bad situation and we should failover to the next priority (Side note: -// the current connectivity state aggregating algorhtim (e.g. round-robin) is -// not handling this right, because if many backends all go from Ready to -// Connecting, the overall situation is more like TransientFailure, not -// Connecting). -// -// If the previous state was Idle, we don't do anything special with failure, -// and simply forward the update. The init timer should be in process, will -// handle failover if it timeouts. If the previous state was TransientFailure, -// we do not forward, because the lower priority is in use. -// -// Caller must make sure priorityInUse is not higher than priority. -// -// Caller must hold priorityMu. -func (edsImpl *edsBalancerImpl) handlePriorityWithNewStateConnecting(priority priorityType, oldState connectivity.State) bool { - if edsImpl.priorityInUse.lowerThan(priority) { - return false - } - - switch oldState { - case connectivity.Ready: - pNext := priority.nextLower() - if _, okNext := edsImpl.priorityToLocalities[pNext]; !okNext { - return true - } - edsImpl.logger.Infof("Switching priority from %v to %v, because former became Connecting from Ready", priority, pNext) - edsImpl.startPriority(pNext) - return true - case connectivity.Idle: - return true - case connectivity.TransientFailure: - return false - default: - // Old state is Connecting or Shutdown. Don't forward. - return false - } -} - -// priorityType represents the priority from EDS response. -// -// 0 is the highest priority. The bigger the number, the lower the priority. -type priorityType struct { - set bool - p uint32 -} - -func newPriorityType(p uint32) priorityType { - return priorityType{ - set: true, - p: p, - } -} - -func newPriorityTypeUnset() priorityType { - return priorityType{} -} - -func (p priorityType) isSet() bool { - return p.set -} - -func (p priorityType) equal(p2 priorityType) bool { - if !p.isSet() && !p2.isSet() { - return true - } - if !p.isSet() || !p2.isSet() { - return false - } - return p == p2 -} - -func (p priorityType) higherThan(p2 priorityType) bool { - if !p.isSet() || !p2.isSet() { - // TODO(menghanl): return an appropriate value instead of panic. - panic("priority unset") - } - return p.p < p2.p -} - -func (p priorityType) lowerThan(p2 priorityType) bool { - if !p.isSet() || !p2.isSet() { - // TODO(menghanl): return an appropriate value instead of panic. - panic("priority unset") - } - return p.p > p2.p -} - -func (p priorityType) nextLower() priorityType { - if !p.isSet() { - panic("priority unset") - } - return priorityType{ - set: true, - p: p.p + 1, - } -} - -func (p priorityType) String() string { - if !p.set { - return "Nil" - } - return fmt.Sprint(p.p) -} diff --git a/xds/internal/balancer/edsbalancer/eds_impl_priority_test.go b/xds/internal/balancer/edsbalancer/eds_impl_priority_test.go deleted file mode 100644 index 255665e3d4c1..000000000000 --- a/xds/internal/balancer/edsbalancer/eds_impl_priority_test.go +++ /dev/null @@ -1,841 +0,0 @@ -/* - * - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package edsbalancer - -import ( - "testing" - "time" - - corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" - "github.com/google/go-cmp/cmp" - "google.golang.org/grpc/balancer" - "google.golang.org/grpc/connectivity" - "google.golang.org/grpc/xds/internal/testutils" -) - -// When a high priority is ready, adding/removing lower locality doesn't cause -// changes. -// -// Init 0 and 1; 0 is up, use 0; add 2, use 0; remove 2, use 0. -func (s) TestEDSPriority_HighPriorityReady(t *testing.T) { - cc := testutils.NewTestClientConn(t) - edsb := newEDSBalancerImpl(cc, nil, nil, nil) - edsb.enqueueChildBalancerStateUpdate = edsb.updateState - - // Two localities, with priorities [0, 1], each with one backend. - clab1 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) - clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil) - clab1.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil) - edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab1.Build())) - - addrs1 := <-cc.NewSubConnAddrsCh - if got, want := addrs1[0].Addr, testEndpointAddrs[0]; got != want { - t.Fatalf("sc is created with addr %v, want %v", got, want) - } - sc1 := <-cc.NewSubConnCh - - // p0 is ready. - edsb.handleSubConnStateChange(sc1, connectivity.Connecting) - edsb.handleSubConnStateChange(sc1, connectivity.Ready) - - // Test roundrobin with only p0 subconns. - p1 := <-cc.NewPickerCh - want := []balancer.SubConn{sc1} - if err := testutils.IsRoundRobin(want, subConnFromPicker(p1)); err != nil { - t.Fatalf("want %v, got %v", want, err) - } - - // Add p2, it shouldn't cause any udpates. - clab2 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) - clab2.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil) - clab2.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil) - clab2.AddLocality(testSubZones[2], 1, 2, testEndpointAddrs[2:3], nil) - edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab2.Build())) - - select { - case <-cc.NewPickerCh: - t.Fatalf("got unexpected new picker") - case <-cc.NewSubConnCh: - t.Fatalf("got unexpected new SubConn") - case <-cc.RemoveSubConnCh: - t.Fatalf("got unexpected remove SubConn") - case <-time.After(time.Millisecond * 100): - } - - // Remove p2, no updates. - clab3 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) - clab3.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil) - clab3.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil) - edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab3.Build())) - - select { - case <-cc.NewPickerCh: - t.Fatalf("got unexpected new picker") - case <-cc.NewSubConnCh: - t.Fatalf("got unexpected new SubConn") - case <-cc.RemoveSubConnCh: - t.Fatalf("got unexpected remove SubConn") - case <-time.After(time.Millisecond * 100): - } -} - -// Lower priority is used when higher priority is not ready. -// -// Init 0 and 1; 0 is up, use 0; 0 is down, 1 is up, use 1; add 2, use 1; 1 is -// down, use 2; remove 2, use 1. -func (s) TestEDSPriority_SwitchPriority(t *testing.T) { - cc := testutils.NewTestClientConn(t) - edsb := newEDSBalancerImpl(cc, nil, nil, nil) - edsb.enqueueChildBalancerStateUpdate = edsb.updateState - - // Two localities, with priorities [0, 1], each with one backend. - clab1 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) - clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil) - clab1.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil) - edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab1.Build())) - - addrs0 := <-cc.NewSubConnAddrsCh - if got, want := addrs0[0].Addr, testEndpointAddrs[0]; got != want { - t.Fatalf("sc is created with addr %v, want %v", got, want) - } - sc0 := <-cc.NewSubConnCh - - // p0 is ready. - edsb.handleSubConnStateChange(sc0, connectivity.Connecting) - edsb.handleSubConnStateChange(sc0, connectivity.Ready) - - // Test roundrobin with only p0 subconns. - p0 := <-cc.NewPickerCh - want := []balancer.SubConn{sc0} - if err := testutils.IsRoundRobin(want, subConnFromPicker(p0)); err != nil { - t.Fatalf("want %v, got %v", want, err) - } - - // Turn down 0, 1 is used. - edsb.handleSubConnStateChange(sc0, connectivity.TransientFailure) - addrs1 := <-cc.NewSubConnAddrsCh - if got, want := addrs1[0].Addr, testEndpointAddrs[1]; got != want { - t.Fatalf("sc is created with addr %v, want %v", got, want) - } - sc1 := <-cc.NewSubConnCh - edsb.handleSubConnStateChange(sc1, connectivity.Connecting) - edsb.handleSubConnStateChange(sc1, connectivity.Ready) - - // Test pick with 1. - p1 := <-cc.NewPickerCh - for i := 0; i < 5; i++ { - gotSCSt, _ := p1.Pick(balancer.PickInfo{}) - if !cmp.Equal(gotSCSt.SubConn, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) { - t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc1) - } - } - - // Add p2, it shouldn't cause any udpates. - clab2 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) - clab2.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil) - clab2.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil) - clab2.AddLocality(testSubZones[2], 1, 2, testEndpointAddrs[2:3], nil) - edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab2.Build())) - - select { - case <-cc.NewPickerCh: - t.Fatalf("got unexpected new picker") - case <-cc.NewSubConnCh: - t.Fatalf("got unexpected new SubConn") - case <-cc.RemoveSubConnCh: - t.Fatalf("got unexpected remove SubConn") - case <-time.After(time.Millisecond * 100): - } - - // Turn down 1, use 2 - edsb.handleSubConnStateChange(sc1, connectivity.TransientFailure) - addrs2 := <-cc.NewSubConnAddrsCh - if got, want := addrs2[0].Addr, testEndpointAddrs[2]; got != want { - t.Fatalf("sc is created with addr %v, want %v", got, want) - } - sc2 := <-cc.NewSubConnCh - edsb.handleSubConnStateChange(sc2, connectivity.Connecting) - edsb.handleSubConnStateChange(sc2, connectivity.Ready) - - // Test pick with 2. - p2 := <-cc.NewPickerCh - for i := 0; i < 5; i++ { - gotSCSt, _ := p2.Pick(balancer.PickInfo{}) - if !cmp.Equal(gotSCSt.SubConn, sc2, cmp.AllowUnexported(testutils.TestSubConn{})) { - t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc2) - } - } - - // Remove 2, use 1. - clab3 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) - clab3.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil) - clab3.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil) - edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab3.Build())) - - // p2 SubConns are removed. - scToRemove := <-cc.RemoveSubConnCh - if !cmp.Equal(scToRemove, sc2, cmp.AllowUnexported(testutils.TestSubConn{})) { - t.Fatalf("RemoveSubConn, want %v, got %v", sc2, scToRemove) - } - - // Should get an update with 1's old picker, to override 2's old picker. - p3 := <-cc.NewPickerCh - for i := 0; i < 5; i++ { - if _, err := p3.Pick(balancer.PickInfo{}); err != balancer.ErrTransientFailure { - t.Fatalf("want pick error %v, got %v", balancer.ErrTransientFailure, err) - } - } -} - -// Add a lower priority while the higher priority is down. -// -// Init 0 and 1; 0 and 1 both down; add 2, use 2. -func (s) TestEDSPriority_HigherDownWhileAddingLower(t *testing.T) { - cc := testutils.NewTestClientConn(t) - edsb := newEDSBalancerImpl(cc, nil, nil, nil) - edsb.enqueueChildBalancerStateUpdate = edsb.updateState - - // Two localities, with different priorities, each with one backend. - clab1 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) - clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil) - clab1.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil) - edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab1.Build())) - - addrs0 := <-cc.NewSubConnAddrsCh - if got, want := addrs0[0].Addr, testEndpointAddrs[0]; got != want { - t.Fatalf("sc is created with addr %v, want %v", got, want) - } - sc0 := <-cc.NewSubConnCh - - // Turn down 0, 1 is used. - edsb.handleSubConnStateChange(sc0, connectivity.TransientFailure) - addrs1 := <-cc.NewSubConnAddrsCh - if got, want := addrs1[0].Addr, testEndpointAddrs[1]; got != want { - t.Fatalf("sc is created with addr %v, want %v", got, want) - } - sc1 := <-cc.NewSubConnCh - // Turn down 1, pick should error. - edsb.handleSubConnStateChange(sc1, connectivity.TransientFailure) - - // Test pick failure. - pFail := <-cc.NewPickerCh - for i := 0; i < 5; i++ { - if _, err := pFail.Pick(balancer.PickInfo{}); err != balancer.ErrTransientFailure { - t.Fatalf("want pick error %v, got %v", balancer.ErrTransientFailure, err) - } - } - - // Add p2, it should create a new SubConn. - clab2 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) - clab2.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil) - clab2.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil) - clab2.AddLocality(testSubZones[2], 1, 2, testEndpointAddrs[2:3], nil) - edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab2.Build())) - - addrs2 := <-cc.NewSubConnAddrsCh - if got, want := addrs2[0].Addr, testEndpointAddrs[2]; got != want { - t.Fatalf("sc is created with addr %v, want %v", got, want) - } - sc2 := <-cc.NewSubConnCh - edsb.handleSubConnStateChange(sc2, connectivity.Connecting) - edsb.handleSubConnStateChange(sc2, connectivity.Ready) - - // Test pick with 2. - p2 := <-cc.NewPickerCh - for i := 0; i < 5; i++ { - gotSCSt, _ := p2.Pick(balancer.PickInfo{}) - if !cmp.Equal(gotSCSt.SubConn, sc2, cmp.AllowUnexported(testutils.TestSubConn{})) { - t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc2) - } - } -} - -// When a higher priority becomes available, all lower priorities are closed. -// -// Init 0,1,2; 0 and 1 down, use 2; 0 up, close 1 and 2. -func (s) TestEDSPriority_HigherReadyCloseAllLower(t *testing.T) { - defer time.Sleep(10 * time.Millisecond) - - cc := testutils.NewTestClientConn(t) - edsb := newEDSBalancerImpl(cc, nil, nil, nil) - edsb.enqueueChildBalancerStateUpdate = edsb.updateState - - // Two localities, with priorities [0,1,2], each with one backend. - clab1 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) - clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil) - clab1.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil) - clab1.AddLocality(testSubZones[2], 1, 2, testEndpointAddrs[2:3], nil) - edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab1.Build())) - - addrs0 := <-cc.NewSubConnAddrsCh - if got, want := addrs0[0].Addr, testEndpointAddrs[0]; got != want { - t.Fatalf("sc is created with addr %v, want %v", got, want) - } - sc0 := <-cc.NewSubConnCh - - // Turn down 0, 1 is used. - edsb.handleSubConnStateChange(sc0, connectivity.TransientFailure) - addrs1 := <-cc.NewSubConnAddrsCh - if got, want := addrs1[0].Addr, testEndpointAddrs[1]; got != want { - t.Fatalf("sc is created with addr %v, want %v", got, want) - } - sc1 := <-cc.NewSubConnCh - // Turn down 1, 2 is used. - edsb.handleSubConnStateChange(sc1, connectivity.TransientFailure) - addrs2 := <-cc.NewSubConnAddrsCh - if got, want := addrs2[0].Addr, testEndpointAddrs[2]; got != want { - t.Fatalf("sc is created with addr %v, want %v", got, want) - } - sc2 := <-cc.NewSubConnCh - edsb.handleSubConnStateChange(sc2, connectivity.Connecting) - edsb.handleSubConnStateChange(sc2, connectivity.Ready) - - // Test pick with 2. - p2 := <-cc.NewPickerCh - for i := 0; i < 5; i++ { - gotSCSt, _ := p2.Pick(balancer.PickInfo{}) - if !cmp.Equal(gotSCSt.SubConn, sc2, cmp.AllowUnexported(testutils.TestSubConn{})) { - t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc2) - } - } - - // When 0 becomes ready, 0 should be used, 1 and 2 should all be closed. - edsb.handleSubConnStateChange(sc0, connectivity.Ready) - - // sc1 and sc2 should be removed. - // - // With localities caching, the lower priorities are closed after a timeout, - // in goroutines. The order is no longer guaranteed. - scToRemove := []balancer.SubConn{<-cc.RemoveSubConnCh, <-cc.RemoveSubConnCh} - if !(cmp.Equal(scToRemove[0], sc1, cmp.AllowUnexported(testutils.TestSubConn{})) && - cmp.Equal(scToRemove[1], sc2, cmp.AllowUnexported(testutils.TestSubConn{}))) && - !(cmp.Equal(scToRemove[0], sc2, cmp.AllowUnexported(testutils.TestSubConn{})) && - cmp.Equal(scToRemove[1], sc1, cmp.AllowUnexported(testutils.TestSubConn{}))) { - t.Errorf("RemoveSubConn, want [%v, %v], got %v", sc1, sc2, scToRemove) - } - - // Test pick with 0. - p0 := <-cc.NewPickerCh - for i := 0; i < 5; i++ { - gotSCSt, _ := p0.Pick(balancer.PickInfo{}) - if !cmp.Equal(gotSCSt.SubConn, sc0, cmp.AllowUnexported(testutils.TestSubConn{})) { - t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc0) - } - } -} - -// At init, start the next lower priority after timeout if the higher priority -// doesn't get ready. -// -// Init 0,1; 0 is not ready (in connecting), after timeout, use 1. -func (s) TestEDSPriority_InitTimeout(t *testing.T) { - const testPriorityInitTimeout = time.Second - defer func() func() { - old := defaultPriorityInitTimeout - defaultPriorityInitTimeout = testPriorityInitTimeout - return func() { - defaultPriorityInitTimeout = old - } - }()() - - cc := testutils.NewTestClientConn(t) - edsb := newEDSBalancerImpl(cc, nil, nil, nil) - edsb.enqueueChildBalancerStateUpdate = edsb.updateState - - // Two localities, with different priorities, each with one backend. - clab1 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) - clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil) - clab1.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil) - edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab1.Build())) - - addrs0 := <-cc.NewSubConnAddrsCh - if got, want := addrs0[0].Addr, testEndpointAddrs[0]; got != want { - t.Fatalf("sc is created with addr %v, want %v", got, want) - } - sc0 := <-cc.NewSubConnCh - - // Keep 0 in connecting, 1 will be used after init timeout. - edsb.handleSubConnStateChange(sc0, connectivity.Connecting) - - // Make sure new SubConn is created before timeout. - select { - case <-time.After(testPriorityInitTimeout * 3 / 4): - case <-cc.NewSubConnAddrsCh: - t.Fatalf("Got a new SubConn too early (Within timeout). Expect a new SubConn only after timeout") - } - - addrs1 := <-cc.NewSubConnAddrsCh - if got, want := addrs1[0].Addr, testEndpointAddrs[1]; got != want { - t.Fatalf("sc is created with addr %v, want %v", got, want) - } - sc1 := <-cc.NewSubConnCh - - edsb.handleSubConnStateChange(sc1, connectivity.Connecting) - edsb.handleSubConnStateChange(sc1, connectivity.Ready) - - // Test pick with 1. - p1 := <-cc.NewPickerCh - for i := 0; i < 5; i++ { - gotSCSt, _ := p1.Pick(balancer.PickInfo{}) - if !cmp.Equal(gotSCSt.SubConn, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) { - t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc1) - } - } -} - -// Add localities to existing priorities. -// -// - start with 2 locality with p0 and p1 -// - add localities to existing p0 and p1 -func (s) TestEDSPriority_MultipleLocalities(t *testing.T) { - cc := testutils.NewTestClientConn(t) - edsb := newEDSBalancerImpl(cc, nil, nil, nil) - edsb.enqueueChildBalancerStateUpdate = edsb.updateState - - // Two localities, with different priorities, each with one backend. - clab0 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) - clab0.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil) - clab0.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil) - edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab0.Build())) - - addrs0 := <-cc.NewSubConnAddrsCh - if got, want := addrs0[0].Addr, testEndpointAddrs[0]; got != want { - t.Fatalf("sc is created with addr %v, want %v", got, want) - } - sc0 := <-cc.NewSubConnCh - edsb.handleSubConnStateChange(sc0, connectivity.Connecting) - edsb.handleSubConnStateChange(sc0, connectivity.Ready) - - // Test roundrobin with only p0 subconns. - p0 := <-cc.NewPickerCh - want := []balancer.SubConn{sc0} - if err := testutils.IsRoundRobin(want, subConnFromPicker(p0)); err != nil { - t.Fatalf("want %v, got %v", want, err) - } - - // Turn down p0 subconns, p1 subconns will be created. - edsb.handleSubConnStateChange(sc0, connectivity.TransientFailure) - - addrs1 := <-cc.NewSubConnAddrsCh - if got, want := addrs1[0].Addr, testEndpointAddrs[1]; got != want { - t.Fatalf("sc is created with addr %v, want %v", got, want) - } - sc1 := <-cc.NewSubConnCh - edsb.handleSubConnStateChange(sc1, connectivity.Connecting) - edsb.handleSubConnStateChange(sc1, connectivity.Ready) - - // Test roundrobin with only p1 subconns. - p1 := <-cc.NewPickerCh - want = []balancer.SubConn{sc1} - if err := testutils.IsRoundRobin(want, subConnFromPicker(p1)); err != nil { - t.Fatalf("want %v, got %v", want, err) - } - - // Reconnect p0 subconns, p1 subconn will be closed. - edsb.handleSubConnStateChange(sc0, connectivity.Ready) - - scToRemove := <-cc.RemoveSubConnCh - if !cmp.Equal(scToRemove, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) { - t.Fatalf("RemoveSubConn, want %v, got %v", sc1, scToRemove) - } - - // Test roundrobin with only p0 subconns. - p2 := <-cc.NewPickerCh - want = []balancer.SubConn{sc0} - if err := testutils.IsRoundRobin(want, subConnFromPicker(p2)); err != nil { - t.Fatalf("want %v, got %v", want, err) - } - - // Add two localities, with two priorities, with one backend. - clab1 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) - clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil) - clab1.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil) - clab1.AddLocality(testSubZones[2], 1, 0, testEndpointAddrs[2:3], nil) - clab1.AddLocality(testSubZones[3], 1, 1, testEndpointAddrs[3:4], nil) - edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab1.Build())) - - addrs2 := <-cc.NewSubConnAddrsCh - if got, want := addrs2[0].Addr, testEndpointAddrs[2]; got != want { - t.Fatalf("sc is created with addr %v, want %v", got, want) - } - sc2 := <-cc.NewSubConnCh - edsb.handleSubConnStateChange(sc2, connectivity.Connecting) - edsb.handleSubConnStateChange(sc2, connectivity.Ready) - - // Test roundrobin with only two p0 subconns. - p3 := <-cc.NewPickerCh - want = []balancer.SubConn{sc0, sc2} - if err := testutils.IsRoundRobin(want, subConnFromPicker(p3)); err != nil { - t.Fatalf("want %v, got %v", want, err) - } - - // Turn down p0 subconns, p1 subconns will be created. - edsb.handleSubConnStateChange(sc0, connectivity.TransientFailure) - edsb.handleSubConnStateChange(sc2, connectivity.TransientFailure) - - sc3 := <-cc.NewSubConnCh - edsb.handleSubConnStateChange(sc3, connectivity.Connecting) - edsb.handleSubConnStateChange(sc3, connectivity.Ready) - sc4 := <-cc.NewSubConnCh - edsb.handleSubConnStateChange(sc4, connectivity.Connecting) - edsb.handleSubConnStateChange(sc4, connectivity.Ready) - - // Test roundrobin with only p1 subconns. - p4 := <-cc.NewPickerCh - want = []balancer.SubConn{sc3, sc4} - if err := testutils.IsRoundRobin(want, subConnFromPicker(p4)); err != nil { - t.Fatalf("want %v, got %v", want, err) - } -} - -// EDS removes all localities, and re-adds them. -func (s) TestEDSPriority_RemovesAllLocalities(t *testing.T) { - const testPriorityInitTimeout = time.Second - defer func() func() { - old := defaultPriorityInitTimeout - defaultPriorityInitTimeout = testPriorityInitTimeout - return func() { - defaultPriorityInitTimeout = old - } - }()() - - cc := testutils.NewTestClientConn(t) - edsb := newEDSBalancerImpl(cc, nil, nil, nil) - edsb.enqueueChildBalancerStateUpdate = edsb.updateState - - // Two localities, with different priorities, each with one backend. - clab0 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) - clab0.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil) - clab0.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[1:2], nil) - edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab0.Build())) - - addrs0 := <-cc.NewSubConnAddrsCh - if got, want := addrs0[0].Addr, testEndpointAddrs[0]; got != want { - t.Fatalf("sc is created with addr %v, want %v", got, want) - } - sc0 := <-cc.NewSubConnCh - edsb.handleSubConnStateChange(sc0, connectivity.Connecting) - edsb.handleSubConnStateChange(sc0, connectivity.Ready) - - // Test roundrobin with only p0 subconns. - p0 := <-cc.NewPickerCh - want := []balancer.SubConn{sc0} - if err := testutils.IsRoundRobin(want, subConnFromPicker(p0)); err != nil { - t.Fatalf("want %v, got %v", want, err) - } - - // Remove all priorities. - clab1 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) - edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab1.Build())) - - // p0 subconn should be removed. - scToRemove := <-cc.RemoveSubConnCh - if !cmp.Equal(scToRemove, sc0, cmp.AllowUnexported(testutils.TestSubConn{})) { - t.Fatalf("RemoveSubConn, want %v, got %v", sc0, scToRemove) - } - - // Test pick return TransientFailure. - pFail := <-cc.NewPickerCh - for i := 0; i < 5; i++ { - if _, err := pFail.Pick(balancer.PickInfo{}); err != errAllPrioritiesRemoved { - t.Fatalf("want pick error %v, got %v", errAllPrioritiesRemoved, err) - } - } - - // Re-add two localities, with previous priorities, but different backends. - clab2 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) - clab2.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[2:3], nil) - clab2.AddLocality(testSubZones[1], 1, 1, testEndpointAddrs[3:4], nil) - edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab2.Build())) - - addrs01 := <-cc.NewSubConnAddrsCh - if got, want := addrs01[0].Addr, testEndpointAddrs[2]; got != want { - t.Fatalf("sc is created with addr %v, want %v", got, want) - } - sc01 := <-cc.NewSubConnCh - - // Don't send any update to p0, so to not override the old state of p0. - // Later, connect to p1 and then remove p1. This will fallback to p0, and - // will send p0's old picker if they are not correctly removed. - - // p1 will be used after priority init timeout. - addrs11 := <-cc.NewSubConnAddrsCh - if got, want := addrs11[0].Addr, testEndpointAddrs[3]; got != want { - t.Fatalf("sc is created with addr %v, want %v", got, want) - } - sc11 := <-cc.NewSubConnCh - edsb.handleSubConnStateChange(sc11, connectivity.Connecting) - edsb.handleSubConnStateChange(sc11, connectivity.Ready) - - // Test roundrobin with only p1 subconns. - p1 := <-cc.NewPickerCh - want = []balancer.SubConn{sc11} - if err := testutils.IsRoundRobin(want, subConnFromPicker(p1)); err != nil { - t.Fatalf("want %v, got %v", want, err) - } - - // Remove p1 from EDS, to fallback to p0. - clab3 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) - clab3.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[2:3], nil) - edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab3.Build())) - - // p1 subconn should be removed. - scToRemove1 := <-cc.RemoveSubConnCh - if !cmp.Equal(scToRemove1, sc11, cmp.AllowUnexported(testutils.TestSubConn{})) { - t.Fatalf("RemoveSubConn, want %v, got %v", sc11, scToRemove1) - } - - // Test pick return TransientFailure. - pFail1 := <-cc.NewPickerCh - for i := 0; i < 5; i++ { - if scst, err := pFail1.Pick(balancer.PickInfo{}); err != balancer.ErrNoSubConnAvailable { - t.Fatalf("want pick error _, %v, got %v, _ ,%v", balancer.ErrTransientFailure, scst, err) - } - } - - // Send an ready update for the p0 sc that was received when re-adding - // localities to EDS. - edsb.handleSubConnStateChange(sc01, connectivity.Connecting) - edsb.handleSubConnStateChange(sc01, connectivity.Ready) - - // Test roundrobin with only p0 subconns. - p2 := <-cc.NewPickerCh - want = []balancer.SubConn{sc01} - if err := testutils.IsRoundRobin(want, subConnFromPicker(p2)); err != nil { - t.Fatalf("want %v, got %v", want, err) - } - - select { - case <-cc.NewPickerCh: - t.Fatalf("got unexpected new picker") - case <-cc.NewSubConnCh: - t.Fatalf("got unexpected new SubConn") - case <-cc.RemoveSubConnCh: - t.Fatalf("got unexpected remove SubConn") - case <-time.After(time.Millisecond * 100): - } -} - -func (s) TestPriorityType(t *testing.T) { - p0 := newPriorityType(0) - p1 := newPriorityType(1) - p2 := newPriorityType(2) - - if !p0.higherThan(p1) || !p0.higherThan(p2) { - t.Errorf("want p0 to be higher than p1 and p2, got p0>p1: %v, p0>p2: %v", !p0.higherThan(p1), !p0.higherThan(p2)) - } - if !p1.lowerThan(p0) || !p1.higherThan(p2) { - t.Errorf("want p1 to be between p0 and p2, got p1p2: %v", !p1.lowerThan(p0), !p1.higherThan(p2)) - } - if !p2.lowerThan(p0) || !p2.lowerThan(p1) { - t.Errorf("want p2 to be lower than p0 and p1, got p2") - } else if i > 50 && err != nil { - t.Errorf("The second 50%% picks should be non-drops, got error %v", err) - } - } - - // The same locality, remove drops. - clab6 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) - clab6.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[2:3], nil) - edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab6.Build())) - - // Pick without drops. - p6 := <-cc.NewPickerCh - for i := 0; i < 5; i++ { - gotSCSt, _ := p6.Pick(balancer.PickInfo{}) - if !cmp.Equal(gotSCSt.SubConn, sc3, cmp.AllowUnexported(testutils.TestSubConn{})) { - t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc3) - } - } -} - -// 2 locality -// - start with 2 locality -// - add locality -// - remove locality -// - address change for the locality -// - update locality weight -func (s) TestEDS_TwoLocalities(t *testing.T) { - cc := testutils.NewTestClientConn(t) - edsb := newEDSBalancerImpl(cc, nil, nil, nil) - edsb.enqueueChildBalancerStateUpdate = edsb.updateState - - // Two localities, each with one backend. - clab1 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) - clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil) - edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab1.Build())) - sc1 := <-cc.NewSubConnCh - edsb.handleSubConnStateChange(sc1, connectivity.Connecting) - edsb.handleSubConnStateChange(sc1, connectivity.Ready) - - // Add the second locality later to make sure sc2 belongs to the second - // locality. Otherwise the test is flaky because of a map is used in EDS to - // keep localities. - clab1.AddLocality(testSubZones[1], 1, 0, testEndpointAddrs[1:2], nil) - edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab1.Build())) - sc2 := <-cc.NewSubConnCh - edsb.handleSubConnStateChange(sc2, connectivity.Connecting) - edsb.handleSubConnStateChange(sc2, connectivity.Ready) - - // Test roundrobin with two subconns. - p1 := <-cc.NewPickerCh - want := []balancer.SubConn{sc1, sc2} - if err := testutils.IsRoundRobin(want, subConnFromPicker(p1)); err != nil { - t.Fatalf("want %v, got %v", want, err) - } - - // Add another locality, with one backend. - clab2 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) - clab2.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil) - clab2.AddLocality(testSubZones[1], 1, 0, testEndpointAddrs[1:2], nil) - clab2.AddLocality(testSubZones[2], 1, 0, testEndpointAddrs[2:3], nil) - edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab2.Build())) - - sc3 := <-cc.NewSubConnCh - edsb.handleSubConnStateChange(sc3, connectivity.Connecting) - edsb.handleSubConnStateChange(sc3, connectivity.Ready) - - // Test roundrobin with three subconns. - p2 := <-cc.NewPickerCh - want = []balancer.SubConn{sc1, sc2, sc3} - if err := testutils.IsRoundRobin(want, subConnFromPicker(p2)); err != nil { - t.Fatalf("want %v, got %v", want, err) - } - - // Remove first locality. - clab3 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) - clab3.AddLocality(testSubZones[1], 1, 0, testEndpointAddrs[1:2], nil) - clab3.AddLocality(testSubZones[2], 1, 0, testEndpointAddrs[2:3], nil) - edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab3.Build())) - - scToRemove := <-cc.RemoveSubConnCh - if !cmp.Equal(scToRemove, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) { - t.Fatalf("RemoveSubConn, want %v, got %v", sc1, scToRemove) - } - edsb.handleSubConnStateChange(scToRemove, connectivity.Shutdown) - - // Test pick with two subconns (without the first one). - p3 := <-cc.NewPickerCh - want = []balancer.SubConn{sc2, sc3} - if err := testutils.IsRoundRobin(want, subConnFromPicker(p3)); err != nil { - t.Fatalf("want %v, got %v", want, err) - } - - // Add a backend to the last locality. - clab4 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) - clab4.AddLocality(testSubZones[1], 1, 0, testEndpointAddrs[1:2], nil) - clab4.AddLocality(testSubZones[2], 1, 0, testEndpointAddrs[2:4], nil) - edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab4.Build())) - - sc4 := <-cc.NewSubConnCh - edsb.handleSubConnStateChange(sc4, connectivity.Connecting) - edsb.handleSubConnStateChange(sc4, connectivity.Ready) - - // Test pick with two subconns (without the first one). - p4 := <-cc.NewPickerCh - // Locality-1 will be picked twice, and locality-2 will be picked twice. - // Locality-1 contains only sc2, locality-2 contains sc3 and sc4. So expect - // two sc2's and sc3, sc4. - want = []balancer.SubConn{sc2, sc2, sc3, sc4} - if err := testutils.IsRoundRobin(want, subConnFromPicker(p4)); err != nil { - t.Fatalf("want %v, got %v", want, err) - } - - // Change weight of the locality[1]. - clab5 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) - clab5.AddLocality(testSubZones[1], 2, 0, testEndpointAddrs[1:2], nil) - clab5.AddLocality(testSubZones[2], 1, 0, testEndpointAddrs[2:4], nil) - edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab5.Build())) - - // Test pick with two subconns different locality weight. - p5 := <-cc.NewPickerCh - // Locality-1 will be picked four times, and locality-2 will be picked twice - // (weight 2 and 1). Locality-1 contains only sc2, locality-2 contains sc3 and - // sc4. So expect four sc2's and sc3, sc4. - want = []balancer.SubConn{sc2, sc2, sc2, sc2, sc3, sc4} - if err := testutils.IsRoundRobin(want, subConnFromPicker(p5)); err != nil { - t.Fatalf("want %v, got %v", want, err) - } - - // Change weight of the locality[1] to 0, it should never be picked. - clab6 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) - clab6.AddLocality(testSubZones[1], 0, 0, testEndpointAddrs[1:2], nil) - clab6.AddLocality(testSubZones[2], 1, 0, testEndpointAddrs[2:4], nil) - edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab6.Build())) - - // Changing weight of locality[1] to 0 caused it to be removed. It's subconn - // should also be removed. - // - // NOTE: this is because we handle locality with weight 0 same as the - // locality doesn't exist. If this changes in the future, this removeSubConn - // behavior will also change. - scToRemove2 := <-cc.RemoveSubConnCh - if !cmp.Equal(scToRemove2, sc2, cmp.AllowUnexported(testutils.TestSubConn{})) { - t.Fatalf("RemoveSubConn, want %v, got %v", sc2, scToRemove2) - } - - // Test pick with two subconns different locality weight. - p6 := <-cc.NewPickerCh - // Locality-1 will be not be picked, and locality-2 will be picked. - // Locality-2 contains sc3 and sc4. So expect sc3, sc4. - want = []balancer.SubConn{sc3, sc4} - if err := testutils.IsRoundRobin(want, subConnFromPicker(p6)); err != nil { - t.Fatalf("want %v, got %v", want, err) - } -} - -// The EDS balancer gets EDS resp with unhealthy endpoints. Test that only -// healthy ones are used. -func (s) TestEDS_EndpointsHealth(t *testing.T) { - cc := testutils.NewTestClientConn(t) - edsb := newEDSBalancerImpl(cc, nil, nil, nil) - edsb.enqueueChildBalancerStateUpdate = edsb.updateState - - // Two localities, each 3 backend, one Healthy, one Unhealthy, one Unknown. - clab1 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) - clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:6], &testutils.AddLocalityOptions{ - Health: []corepb.HealthStatus{ - corepb.HealthStatus_HEALTHY, - corepb.HealthStatus_UNHEALTHY, - corepb.HealthStatus_UNKNOWN, - corepb.HealthStatus_DRAINING, - corepb.HealthStatus_TIMEOUT, - corepb.HealthStatus_DEGRADED, - }, - }) - clab1.AddLocality(testSubZones[1], 1, 0, testEndpointAddrs[6:12], &testutils.AddLocalityOptions{ - Health: []corepb.HealthStatus{ - corepb.HealthStatus_HEALTHY, - corepb.HealthStatus_UNHEALTHY, - corepb.HealthStatus_UNKNOWN, - corepb.HealthStatus_DRAINING, - corepb.HealthStatus_TIMEOUT, - corepb.HealthStatus_DEGRADED, - }, - }) - edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab1.Build())) - - var ( - readySCs []balancer.SubConn - newSubConnAddrStrs []string - ) - for i := 0; i < 4; i++ { - addr := <-cc.NewSubConnAddrsCh - newSubConnAddrStrs = append(newSubConnAddrStrs, addr[0].Addr) - sc := <-cc.NewSubConnCh - edsb.handleSubConnStateChange(sc, connectivity.Connecting) - edsb.handleSubConnStateChange(sc, connectivity.Ready) - readySCs = append(readySCs, sc) - } - - wantNewSubConnAddrStrs := []string{ - testEndpointAddrs[0], - testEndpointAddrs[2], - testEndpointAddrs[6], - testEndpointAddrs[8], - } - sortStrTrans := cmp.Transformer("Sort", func(in []string) []string { - out := append([]string(nil), in...) // Copy input to avoid mutating it. - sort.Strings(out) - return out - }) - if !cmp.Equal(newSubConnAddrStrs, wantNewSubConnAddrStrs, sortStrTrans) { - t.Fatalf("want newSubConn with address %v, got %v", wantNewSubConnAddrStrs, newSubConnAddrStrs) - } - - // There should be exactly 4 new SubConns. Check to make sure there's no - // more subconns being created. - select { - case <-cc.NewSubConnCh: - t.Fatalf("Got unexpected new subconn") - case <-time.After(time.Microsecond * 100): - } - - // Test roundrobin with the subconns. - p1 := <-cc.NewPickerCh - want := readySCs - if err := testutils.IsRoundRobin(want, subConnFromPicker(p1)); err != nil { - t.Fatalf("want %v, got %v", want, err) - } -} - -func (s) TestClose(t *testing.T) { - edsb := newEDSBalancerImpl(nil, nil, nil, nil) - // This is what could happen when switching between fallback and eds. This - // make sure it doesn't panic. - edsb.close() -} - -// TestEDS_EmptyUpdate covers the cases when eds impl receives an empty update. -// -// It should send an error picker with transient failure to the parent. -func (s) TestEDS_EmptyUpdate(t *testing.T) { - cc := testutils.NewTestClientConn(t) - edsb := newEDSBalancerImpl(cc, nil, nil, nil) - edsb.enqueueChildBalancerStateUpdate = edsb.updateState - - // The first update is an empty update. - edsb.handleEDSResponse(xdsclient.EndpointsUpdate{}) - // Pick should fail with transient failure, and all priority removed error. - perr0 := <-cc.NewPickerCh - for i := 0; i < 5; i++ { - _, err := perr0.Pick(balancer.PickInfo{}) - if !reflect.DeepEqual(err, errAllPrioritiesRemoved) { - t.Fatalf("picker.Pick, got error %v, want error %v", err, errAllPrioritiesRemoved) - } - } - - // One locality with one backend. - clab1 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) - clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil) - edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab1.Build())) - - sc1 := <-cc.NewSubConnCh - edsb.handleSubConnStateChange(sc1, connectivity.Connecting) - edsb.handleSubConnStateChange(sc1, connectivity.Ready) - - // Pick with only the first backend. - p1 := <-cc.NewPickerCh - for i := 0; i < 5; i++ { - gotSCSt, _ := p1.Pick(balancer.PickInfo{}) - if !reflect.DeepEqual(gotSCSt.SubConn, sc1) { - t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc1) - } - } - - edsb.handleEDSResponse(xdsclient.EndpointsUpdate{}) - // Pick should fail with transient failure, and all priority removed error. - perr1 := <-cc.NewPickerCh - for i := 0; i < 5; i++ { - _, err := perr1.Pick(balancer.PickInfo{}) - if !reflect.DeepEqual(err, errAllPrioritiesRemoved) { - t.Fatalf("picker.Pick, got error %v, want error %v", err, errAllPrioritiesRemoved) - } - } - - // Handle another update with priorities and localities. - edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab1.Build())) - - sc2 := <-cc.NewSubConnCh - edsb.handleSubConnStateChange(sc2, connectivity.Connecting) - edsb.handleSubConnStateChange(sc2, connectivity.Ready) - - // Pick with only the first backend. - p2 := <-cc.NewPickerCh - for i := 0; i < 5; i++ { - gotSCSt, _ := p2.Pick(balancer.PickInfo{}) - if !reflect.DeepEqual(gotSCSt.SubConn, sc2) { - t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc2) - } - } -} - -// Create XDS balancer, and update sub-balancer before handling eds responses. -// Then switch between round-robin and test-const-balancer after handling first -// eds response. -func (s) TestEDS_UpdateSubBalancerName(t *testing.T) { - cc := testutils.NewTestClientConn(t) - edsb := newEDSBalancerImpl(cc, nil, nil, nil) - edsb.enqueueChildBalancerStateUpdate = edsb.updateState - - t.Logf("update sub-balancer to test-const-balancer") - edsb.handleChildPolicy("test-const-balancer", nil) - - // Two localities, each with one backend. - clab1 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) - clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil) - clab1.AddLocality(testSubZones[1], 1, 0, testEndpointAddrs[1:2], nil) - edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab1.Build())) - - for i := 0; i < 2; i++ { - sc := <-cc.NewSubConnCh - edsb.handleSubConnStateChange(sc, connectivity.Ready) - } - - p0 := <-cc.NewPickerCh - for i := 0; i < 5; i++ { - _, err := p0.Pick(balancer.PickInfo{}) - if err != testutils.ErrTestConstPicker { - t.Fatalf("picker.Pick, got err %+v, want err %+v", err, testutils.ErrTestConstPicker) - } - } - - t.Logf("update sub-balancer to round-robin") - edsb.handleChildPolicy(roundrobin.Name, nil) - - for i := 0; i < 2; i++ { - <-cc.RemoveSubConnCh - } - - sc1 := <-cc.NewSubConnCh - edsb.handleSubConnStateChange(sc1, connectivity.Connecting) - edsb.handleSubConnStateChange(sc1, connectivity.Ready) - sc2 := <-cc.NewSubConnCh - edsb.handleSubConnStateChange(sc2, connectivity.Connecting) - edsb.handleSubConnStateChange(sc2, connectivity.Ready) - - // Test roundrobin with two subconns. - p1 := <-cc.NewPickerCh - want := []balancer.SubConn{sc1, sc2} - if err := testutils.IsRoundRobin(want, subConnFromPicker(p1)); err != nil { - t.Fatalf("want %v, got %v", want, err) - } - - t.Logf("update sub-balancer to test-const-balancer") - edsb.handleChildPolicy("test-const-balancer", nil) - - for i := 0; i < 2; i++ { - scToRemove := <-cc.RemoveSubConnCh - if !cmp.Equal(scToRemove, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) && - !cmp.Equal(scToRemove, sc2, cmp.AllowUnexported(testutils.TestSubConn{})) { - t.Fatalf("RemoveSubConn, want (%v or %v), got %v", sc1, sc2, scToRemove) - } - edsb.handleSubConnStateChange(scToRemove, connectivity.Shutdown) - } - - for i := 0; i < 2; i++ { - sc := <-cc.NewSubConnCh - edsb.handleSubConnStateChange(sc, connectivity.Ready) - } - - p2 := <-cc.NewPickerCh - for i := 0; i < 5; i++ { - _, err := p2.Pick(balancer.PickInfo{}) - if err != testutils.ErrTestConstPicker { - t.Fatalf("picker.Pick, got err %q, want err %q", err, testutils.ErrTestConstPicker) - } - } - - t.Logf("update sub-balancer to round-robin") - edsb.handleChildPolicy(roundrobin.Name, nil) - - for i := 0; i < 2; i++ { - <-cc.RemoveSubConnCh - } - - sc3 := <-cc.NewSubConnCh - edsb.handleSubConnStateChange(sc3, connectivity.Connecting) - edsb.handleSubConnStateChange(sc3, connectivity.Ready) - sc4 := <-cc.NewSubConnCh - edsb.handleSubConnStateChange(sc4, connectivity.Connecting) - edsb.handleSubConnStateChange(sc4, connectivity.Ready) - - p3 := <-cc.NewPickerCh - want = []balancer.SubConn{sc3, sc4} - if err := testutils.IsRoundRobin(want, subConnFromPicker(p3)); err != nil { - t.Fatalf("want %v, got %v", want, err) - } -} - -func init() { - balancer.Register(&testInlineUpdateBalancerBuilder{}) -} - -// A test balancer that updates balancer.State inline when handling ClientConn -// state. -type testInlineUpdateBalancerBuilder struct{} - -func (*testInlineUpdateBalancerBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { - return &testInlineUpdateBalancer{cc: cc} -} - -func (*testInlineUpdateBalancerBuilder) Name() string { - return "test-inline-update-balancer" -} - -type testInlineUpdateBalancer struct { - cc balancer.ClientConn -} - -func (tb *testInlineUpdateBalancer) ResolverError(error) { - panic("not implemented") -} - -func (tb *testInlineUpdateBalancer) UpdateSubConnState(balancer.SubConn, balancer.SubConnState) { -} - -var errTestInlineStateUpdate = fmt.Errorf("don't like addresses, empty or not") - -func (tb *testInlineUpdateBalancer) UpdateClientConnState(balancer.ClientConnState) error { - tb.cc.UpdateState(balancer.State{ - ConnectivityState: connectivity.Ready, - Picker: &testutils.TestConstPicker{Err: errTestInlineStateUpdate}, - }) - return nil -} - -func (*testInlineUpdateBalancer) Close() { -} - -// When the child policy update picker inline in a handleClientUpdate call -// (e.g., roundrobin handling empty addresses). There could be deadlock caused -// by acquiring a locked mutex. -func (s) TestEDS_ChildPolicyUpdatePickerInline(t *testing.T) { - cc := testutils.NewTestClientConn(t) - edsb := newEDSBalancerImpl(cc, nil, nil, nil) - edsb.enqueueChildBalancerStateUpdate = func(p priorityType, state balancer.State) { - // For this test, euqueue needs to happen asynchronously (like in the - // real implementation). - go edsb.updateState(p, state) - } - - edsb.handleChildPolicy("test-inline-update-balancer", nil) - - clab1 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) - clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil) - edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab1.Build())) - - p0 := <-cc.NewPickerCh - for i := 0; i < 5; i++ { - _, err := p0.Pick(balancer.PickInfo{}) - if err != errTestInlineStateUpdate { - t.Fatalf("picker.Pick, got err %q, want err %q", err, errTestInlineStateUpdate) - } - } -} - -func (s) TestDropPicker(t *testing.T) { - const pickCount = 12 - var constPicker = &testutils.TestConstPicker{ - SC: testutils.TestSubConns[0], - } - - tests := []struct { - name string - drops []*dropper - }{ - { - name: "no drop", - drops: nil, - }, - { - name: "one drop", - drops: []*dropper{ - newDropper(xdsclient.OverloadDropConfig{Numerator: 1, Denominator: 2}), - }, - }, - { - name: "two drops", - drops: []*dropper{ - newDropper(xdsclient.OverloadDropConfig{Numerator: 1, Denominator: 3}), - newDropper(xdsclient.OverloadDropConfig{Numerator: 1, Denominator: 2}), - }, - }, - { - name: "three drops", - drops: []*dropper{ - newDropper(xdsclient.OverloadDropConfig{Numerator: 1, Denominator: 3}), - newDropper(xdsclient.OverloadDropConfig{Numerator: 1, Denominator: 4}), - newDropper(xdsclient.OverloadDropConfig{Numerator: 1, Denominator: 2}), - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - - p := newDropPicker(constPicker, tt.drops, nil) - - // scCount is the number of sc's returned by pick. The opposite of - // drop-count. - var ( - scCount int - wantCount = pickCount - ) - for _, dp := range tt.drops { - wantCount = wantCount * int(dp.c.Denominator-dp.c.Numerator) / int(dp.c.Denominator) - } - - for i := 0; i < pickCount; i++ { - _, err := p.Pick(balancer.PickInfo{}) - if err == nil { - scCount++ - } - } - - if scCount != (wantCount) { - t.Errorf("drops: %+v, scCount %v, wantCount %v", tt.drops, scCount, wantCount) - } - }) - } -} - -func (s) TestEDS_LoadReport(t *testing.T) { - testLoadStore := testutils.NewTestLoadStore() - - cc := testutils.NewTestClientConn(t) - edsb := newEDSBalancerImpl(cc, nil, testLoadStore, nil) - edsb.enqueueChildBalancerStateUpdate = edsb.updateState - - backendToBalancerID := make(map[balancer.SubConn]internal.LocalityID) - - // Two localities, each with one backend. - clab1 := testutils.NewClusterLoadAssignmentBuilder(testClusterNames[0], nil) - clab1.AddLocality(testSubZones[0], 1, 0, testEndpointAddrs[:1], nil) - edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab1.Build())) - sc1 := <-cc.NewSubConnCh - edsb.handleSubConnStateChange(sc1, connectivity.Connecting) - edsb.handleSubConnStateChange(sc1, connectivity.Ready) - backendToBalancerID[sc1] = internal.LocalityID{ - SubZone: testSubZones[0], - } - - // Add the second locality later to make sure sc2 belongs to the second - // locality. Otherwise the test is flaky because of a map is used in EDS to - // keep localities. - clab1.AddLocality(testSubZones[1], 1, 0, testEndpointAddrs[1:2], nil) - edsb.handleEDSResponse(parseEDSRespProtoForTesting(clab1.Build())) - sc2 := <-cc.NewSubConnCh - edsb.handleSubConnStateChange(sc2, connectivity.Connecting) - edsb.handleSubConnStateChange(sc2, connectivity.Ready) - backendToBalancerID[sc2] = internal.LocalityID{ - SubZone: testSubZones[1], - } - - // Test roundrobin with two subconns. - p1 := <-cc.NewPickerCh - var ( - wantStart []internal.LocalityID - wantEnd []internal.LocalityID - ) - - for i := 0; i < 10; i++ { - scst, _ := p1.Pick(balancer.PickInfo{}) - locality := backendToBalancerID[scst.SubConn] - wantStart = append(wantStart, locality) - if scst.Done != nil && scst.SubConn != sc1 { - scst.Done(balancer.DoneInfo{}) - wantEnd = append(wantEnd, backendToBalancerID[scst.SubConn]) - } - } - - if !cmp.Equal(testLoadStore.CallsStarted, wantStart) { - t.Fatalf("want started: %v, got: %v", testLoadStore.CallsStarted, wantStart) - } - if !cmp.Equal(testLoadStore.CallsEnded, wantEnd) { - t.Fatalf("want ended: %v, got: %v", testLoadStore.CallsEnded, wantEnd) - } -} diff --git a/xds/internal/balancer/edsbalancer/eds_test.go b/xds/internal/balancer/edsbalancer/eds_test.go deleted file mode 100644 index 312ad9efb030..000000000000 --- a/xds/internal/balancer/edsbalancer/eds_test.go +++ /dev/null @@ -1,710 +0,0 @@ -/* - * - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package edsbalancer - -import ( - "bytes" - "encoding/json" - "fmt" - "reflect" - "testing" - - "github.com/golang/protobuf/jsonpb" - wrapperspb "github.com/golang/protobuf/ptypes/wrappers" - "github.com/google/go-cmp/cmp" - "google.golang.org/grpc" - "google.golang.org/grpc/balancer" - "google.golang.org/grpc/connectivity" - "google.golang.org/grpc/internal/grpclog" - "google.golang.org/grpc/internal/grpctest" - scpb "google.golang.org/grpc/internal/proto/grpc_service_config" - "google.golang.org/grpc/resolver" - "google.golang.org/grpc/serviceconfig" - "google.golang.org/grpc/xds/internal/balancer/lrs" - xdsclient "google.golang.org/grpc/xds/internal/client" - "google.golang.org/grpc/xds/internal/client/bootstrap" - "google.golang.org/grpc/xds/internal/testutils" - "google.golang.org/grpc/xds/internal/testutils/fakeclient" - - _ "google.golang.org/grpc/xds/internal/client/v2" // V2 client registration. -) - -func init() { - balancer.Register(&edsBalancerBuilder{}) - - bootstrapConfigNew = func() (*bootstrap.Config, error) { - return &bootstrap.Config{ - BalancerName: testBalancerNameFooBar, - Creds: grpc.WithInsecure(), - NodeProto: testutils.EmptyNodeProtoV2, - }, nil - } -} - -func subConnFromPicker(p balancer.Picker) func() balancer.SubConn { - return func() balancer.SubConn { - scst, _ := p.Pick(balancer.PickInfo{}) - return scst.SubConn - } -} - -type s struct { - grpctest.Tester -} - -func Test(t *testing.T) { - grpctest.RunSubTests(t, s{}) -} - -const testBalancerNameFooBar = "foo.bar" - -func newNoopTestClientConn() *noopTestClientConn { - return &noopTestClientConn{} -} - -// noopTestClientConn is used in EDS balancer config update tests that only -// cover the config update handling, but not SubConn/load-balancing. -type noopTestClientConn struct { - balancer.ClientConn -} - -func (t *noopTestClientConn) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) { - return nil, nil -} - -func (noopTestClientConn) Target() string { return testServiceName } - -type scStateChange struct { - sc balancer.SubConn - state connectivity.State -} - -type fakeEDSBalancer struct { - cc balancer.ClientConn - childPolicy *testutils.Channel - subconnStateChange *testutils.Channel - edsUpdate *testutils.Channel - loadStore lrs.Store -} - -func (f *fakeEDSBalancer) handleSubConnStateChange(sc balancer.SubConn, state connectivity.State) { - f.subconnStateChange.Send(&scStateChange{sc: sc, state: state}) -} - -func (f *fakeEDSBalancer) handleChildPolicy(name string, config json.RawMessage) { - f.childPolicy.Send(&loadBalancingConfig{Name: name, Config: config}) -} - -func (f *fakeEDSBalancer) handleEDSResponse(edsResp xdsclient.EndpointsUpdate) { - f.edsUpdate.Send(edsResp) -} - -func (f *fakeEDSBalancer) updateState(priority priorityType, s balancer.State) {} - -func (f *fakeEDSBalancer) close() {} - -func (f *fakeEDSBalancer) waitForChildPolicy(wantPolicy *loadBalancingConfig) error { - val, err := f.childPolicy.Receive() - if err != nil { - return fmt.Errorf("error waiting for childPolicy: %v", err) - } - gotPolicy := val.(*loadBalancingConfig) - if !cmp.Equal(gotPolicy, wantPolicy) { - return fmt.Errorf("got childPolicy %v, want %v", gotPolicy, wantPolicy) - } - return nil -} - -func (f *fakeEDSBalancer) waitForSubConnStateChange(wantState *scStateChange) error { - val, err := f.subconnStateChange.Receive() - if err != nil { - return fmt.Errorf("error waiting for subconnStateChange: %v", err) - } - gotState := val.(*scStateChange) - if !cmp.Equal(gotState, wantState, cmp.AllowUnexported(scStateChange{})) { - return fmt.Errorf("got subconnStateChange %v, want %v", gotState, wantState) - } - return nil -} - -func (f *fakeEDSBalancer) waitForEDSResponse(wantUpdate xdsclient.EndpointsUpdate) error { - val, err := f.edsUpdate.Receive() - if err != nil { - return fmt.Errorf("error waiting for edsUpdate: %v", err) - } - gotUpdate := val.(xdsclient.EndpointsUpdate) - if !reflect.DeepEqual(gotUpdate, wantUpdate) { - return fmt.Errorf("got edsUpdate %+v, want %+v", gotUpdate, wantUpdate) - } - return nil -} - -func newFakeEDSBalancer(cc balancer.ClientConn, loadStore lrs.Store) edsBalancerImplInterface { - return &fakeEDSBalancer{ - cc: cc, - childPolicy: testutils.NewChannelWithSize(10), - subconnStateChange: testutils.NewChannelWithSize(10), - edsUpdate: testutils.NewChannelWithSize(10), - loadStore: loadStore, - } -} - -type fakeSubConn struct{} - -func (*fakeSubConn) UpdateAddresses([]resolver.Address) { panic("implement me") } -func (*fakeSubConn) Connect() { panic("implement me") } - -// waitForNewXDSClientWithEDSWatch makes sure that a new xdsClient is created -// with the provided name. It also make sure that the newly created client -// registers an eds watcher. -func waitForNewXDSClientWithEDSWatch(t *testing.T, ch *testutils.Channel, wantName string) *fakeclient.Client { - t.Helper() - - val, err := ch.Receive() - if err != nil { - t.Fatalf("error when waiting for a new xds client: %v", err) - return nil - } - xdsC := val.(*fakeclient.Client) - if xdsC.Name() != wantName { - t.Fatalf("xdsClient created to balancer: %v, want %v", xdsC.Name(), wantName) - return nil - } - _, err = xdsC.WaitForWatchEDS() - if err != nil { - t.Fatalf("xdsClient.WatchEndpoints failed with error: %v", err) - return nil - } - return xdsC -} - -// waitForNewEDSLB makes sure that a new edsLB is created by the top-level -// edsBalancer. -func waitForNewEDSLB(t *testing.T, ch *testutils.Channel) *fakeEDSBalancer { - t.Helper() - - val, err := ch.Receive() - if err != nil { - t.Fatalf("error when waiting for a new edsLB: %v", err) - return nil - } - return val.(*fakeEDSBalancer) -} - -// setup overrides the functions which are used to create the xdsClient and the -// edsLB, creates fake version of them and makes them available on the provided -// channels. The returned cancel function should be called by the test for -// cleanup. -func setup(edsLBCh *testutils.Channel, xdsClientCh *testutils.Channel) func() { - origNewEDSBalancer := newEDSBalancer - newEDSBalancer = func(cc balancer.ClientConn, enqueue func(priorityType, balancer.State), loadStore lrs.Store, logger *grpclog.PrefixLogger) edsBalancerImplInterface { - edsLB := newFakeEDSBalancer(cc, loadStore) - defer func() { edsLBCh.Send(edsLB) }() - return edsLB - } - - origXdsClientNew := xdsclientNew - xdsclientNew = func(opts xdsclient.Options) (xdsClientInterface, error) { - xdsC := fakeclient.NewClientWithName(opts.Config.BalancerName) - defer func() { xdsClientCh.Send(xdsC) }() - return xdsC, nil - } - return func() { - newEDSBalancer = origNewEDSBalancer - xdsclientNew = origXdsClientNew - } -} - -// TestXDSConfigBalancerNameUpdate verifies different scenarios where the -// balancer name in the lbConfig is updated. -// -// The test does the following: -// * Builds a new xds balancer. -// * Repeatedly pushes new ClientConnState which specifies different -// balancerName in the lbConfig. We expect xdsClient objects to created -// whenever the balancerName changes. -func (s) TestXDSConfigBalancerNameUpdate(t *testing.T) { - oldBootstrapConfigNew := bootstrapConfigNew - bootstrapConfigNew = func() (*bootstrap.Config, error) { - // Return an error from bootstrap, so the eds balancer will use - // BalancerName from the config. - // - // TODO: remove this when deleting BalancerName from config. - return nil, fmt.Errorf("no bootstrap available") - } - defer func() { bootstrapConfigNew = oldBootstrapConfigNew }() - edsLBCh := testutils.NewChannel() - xdsClientCh := testutils.NewChannel() - cancel := setup(edsLBCh, xdsClientCh) - defer cancel() - - builder := balancer.Get(edsName) - cc := newNoopTestClientConn() - edsB, ok := builder.Build(cc, balancer.BuildOptions{Target: resolver.Target{Endpoint: testEDSClusterName}}).(*edsBalancer) - if !ok { - t.Fatalf("builder.Build(%s) returned type {%T}, want {*edsBalancer}", edsName, edsB) - } - defer edsB.Close() - - addrs := []resolver.Address{{Addr: "1.1.1.1:10001"}, {Addr: "2.2.2.2:10002"}, {Addr: "3.3.3.3:10003"}} - for i := 0; i < 2; i++ { - balancerName := fmt.Sprintf("balancer-%d", i) - edsB.UpdateClientConnState(balancer.ClientConnState{ - ResolverState: resolver.State{Addresses: addrs}, - BalancerConfig: &EDSConfig{ - BalancerName: balancerName, - EDSServiceName: testEDSClusterName, - }, - }) - - xdsC := waitForNewXDSClientWithEDSWatch(t, xdsClientCh, balancerName) - xdsC.InvokeWatchEDSCallback(xdsclient.EndpointsUpdate{}, nil) - } -} - -const ( - fakeBalancerA = "fake_balancer_A" - fakeBalancerB = "fake_balancer_B" -) - -// Install two fake balancers for service config update tests. -// -// ParseConfig only accepts the json if the balancer specified is registered. - -func init() { - balancer.Register(&fakeBalancerBuilder{name: fakeBalancerA}) - balancer.Register(&fakeBalancerBuilder{name: fakeBalancerB}) -} - -type fakeBalancerBuilder struct { - name string -} - -func (b *fakeBalancerBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { - return &fakeBalancer{cc: cc} -} - -func (b *fakeBalancerBuilder) Name() string { - return b.name -} - -type fakeBalancer struct { - cc balancer.ClientConn -} - -func (b *fakeBalancer) ResolverError(error) { - panic("implement me") -} - -func (b *fakeBalancer) UpdateClientConnState(balancer.ClientConnState) error { - panic("implement me") -} - -func (b *fakeBalancer) UpdateSubConnState(balancer.SubConn, balancer.SubConnState) { - panic("implement me") -} - -func (b *fakeBalancer) Close() {} - -// TestXDSConnfigChildPolicyUpdate verifies scenarios where the childPolicy -// section of the lbConfig is updated. -// -// The test does the following: -// * Builds a new xds balancer. -// * Pushes a new ClientConnState with a childPolicy set to fakeBalancerA. -// Verifies that a new xdsClient is created. It then pushes a new edsUpdate -// through the fakexds client. Verifies that a new edsLB is created and it -// receives the expected childPolicy. -// * Pushes a new ClientConnState with a childPolicy set to fakeBalancerB. -// This time around, we expect no new xdsClient or edsLB to be created. -// Instead, we expect the existing edsLB to receive the new child policy. -func (s) TestXDSConnfigChildPolicyUpdate(t *testing.T) { - edsLBCh := testutils.NewChannel() - xdsClientCh := testutils.NewChannel() - cancel := setup(edsLBCh, xdsClientCh) - defer cancel() - - builder := balancer.Get(edsName) - cc := newNoopTestClientConn() - edsB, ok := builder.Build(cc, balancer.BuildOptions{Target: resolver.Target{Endpoint: testServiceName}}).(*edsBalancer) - if !ok { - t.Fatalf("builder.Build(%s) returned type {%T}, want {*edsBalancer}", edsName, edsB) - } - defer edsB.Close() - - edsB.UpdateClientConnState(balancer.ClientConnState{ - BalancerConfig: &EDSConfig{ - BalancerName: testBalancerNameFooBar, - ChildPolicy: &loadBalancingConfig{ - Name: fakeBalancerA, - Config: json.RawMessage("{}"), - }, - EDSServiceName: testEDSClusterName, - }, - }) - xdsC := waitForNewXDSClientWithEDSWatch(t, xdsClientCh, testBalancerNameFooBar) - xdsC.InvokeWatchEDSCallback(xdsclient.EndpointsUpdate{}, nil) - edsLB := waitForNewEDSLB(t, edsLBCh) - edsLB.waitForChildPolicy(&loadBalancingConfig{ - Name: string(fakeBalancerA), - Config: json.RawMessage(`{}`), - }) - - edsB.UpdateClientConnState(balancer.ClientConnState{ - BalancerConfig: &EDSConfig{ - BalancerName: testBalancerNameFooBar, - ChildPolicy: &loadBalancingConfig{ - Name: fakeBalancerB, - Config: json.RawMessage("{}"), - }, - EDSServiceName: testEDSClusterName, - }, - }) - edsLB.waitForChildPolicy(&loadBalancingConfig{ - Name: string(fakeBalancerA), - Config: json.RawMessage(`{}`), - }) -} - -// TestXDSSubConnStateChange verifies if the top-level edsBalancer passes on -// the subConnStateChange to appropriate child balancers. -func (s) TestXDSSubConnStateChange(t *testing.T) { - edsLBCh := testutils.NewChannel() - xdsClientCh := testutils.NewChannel() - cancel := setup(edsLBCh, xdsClientCh) - defer cancel() - - builder := balancer.Get(edsName) - cc := newNoopTestClientConn() - edsB, ok := builder.Build(cc, balancer.BuildOptions{Target: resolver.Target{Endpoint: testEDSClusterName}}).(*edsBalancer) - if !ok { - t.Fatalf("builder.Build(%s) returned type {%T}, want {*edsBalancer}", edsName, edsB) - } - defer edsB.Close() - - addrs := []resolver.Address{{Addr: "1.1.1.1:10001"}, {Addr: "2.2.2.2:10002"}, {Addr: "3.3.3.3:10003"}} - edsB.UpdateClientConnState(balancer.ClientConnState{ - ResolverState: resolver.State{Addresses: addrs}, - BalancerConfig: &EDSConfig{ - BalancerName: testBalancerNameFooBar, - EDSServiceName: testEDSClusterName, - }, - }) - - xdsC := waitForNewXDSClientWithEDSWatch(t, xdsClientCh, testBalancerNameFooBar) - xdsC.InvokeWatchEDSCallback(xdsclient.EndpointsUpdate{}, nil) - edsLB := waitForNewEDSLB(t, edsLBCh) - - fsc := &fakeSubConn{} - state := connectivity.Ready - edsB.UpdateSubConnState(fsc, balancer.SubConnState{ConnectivityState: state}) - edsLB.waitForSubConnStateChange(&scStateChange{sc: fsc, state: state}) -} - -// TestErrorFromXDSClientUpdate verifies that errros from xdsclient update are -// handled correctly. -// -// If it's resource-not-found, watch will NOT be canceled, the EDS impl will -// receive an empty EDS update, and new RPCs will fail. -// -// If it's connection error, nothing will happen. This will need to change to -// handle fallback. -func (s) TestErrorFromXDSClientUpdate(t *testing.T) { - edsLBCh := testutils.NewChannel() - xdsClientCh := testutils.NewChannel() - cancel := setup(edsLBCh, xdsClientCh) - defer cancel() - - builder := balancer.Get(edsName) - cc := newNoopTestClientConn() - edsB, ok := builder.Build(cc, balancer.BuildOptions{Target: resolver.Target{Endpoint: testEDSClusterName}}).(*edsBalancer) - if !ok { - t.Fatalf("builder.Build(%s) returned type {%T}, want {*edsBalancer}", edsName, edsB) - } - defer edsB.Close() - - edsB.UpdateClientConnState(balancer.ClientConnState{ - BalancerConfig: &EDSConfig{ - BalancerName: testBalancerNameFooBar, - EDSServiceName: testEDSClusterName, - }, - }) - - xdsC := waitForNewXDSClientWithEDSWatch(t, xdsClientCh, testBalancerNameFooBar) - xdsC.InvokeWatchEDSCallback(xdsclient.EndpointsUpdate{}, nil) - edsLB := waitForNewEDSLB(t, edsLBCh) - if err := edsLB.waitForEDSResponse(xdsclient.EndpointsUpdate{}); err != nil { - t.Fatalf("EDS impl got unexpected EDS response: %v", err) - } - - connectionErr := xdsclient.NewErrorf(xdsclient.ErrorTypeConnection, "connection error") - xdsC.InvokeWatchEDSCallback(xdsclient.EndpointsUpdate{}, connectionErr) - if err := xdsC.WaitForCancelEDSWatch(); err == nil { - t.Fatal("watch was canceled, want not canceled (timeout error)") - } - if err := edsLB.waitForEDSResponse(xdsclient.EndpointsUpdate{}); err == nil { - t.Fatal("eds impl got EDS resp, want timeout error") - } - - resourceErr := xdsclient.NewErrorf(xdsclient.ErrorTypeResourceNotFound, "edsBalancer resource not found error") - xdsC.InvokeWatchEDSCallback(xdsclient.EndpointsUpdate{}, resourceErr) - // Even if error is resource not found, watch shouldn't be canceled, because - // this is an EDS resource removed (and xds client actually never sends this - // error, but we still handles it). - if err := xdsC.WaitForCancelEDSWatch(); err == nil { - t.Fatal("watch was canceled, want not canceled (timeout error)") - } - if err := edsLB.waitForEDSResponse(xdsclient.EndpointsUpdate{}); err != nil { - t.Fatalf("eds impl expecting empty update, got %v", err) - } -} - -// TestErrorFromResolver verifies that resolver errors are handled correctly. -// -// If it's resource-not-found, watch will be canceled, the EDS impl will receive -// an empty EDS update, and new RPCs will fail. -// -// If it's connection error, nothing will happen. This will need to change to -// handle fallback. -func (s) TestErrorFromResolver(t *testing.T) { - edsLBCh := testutils.NewChannel() - xdsClientCh := testutils.NewChannel() - cancel := setup(edsLBCh, xdsClientCh) - defer cancel() - - builder := balancer.Get(edsName) - cc := newNoopTestClientConn() - edsB, ok := builder.Build(cc, balancer.BuildOptions{Target: resolver.Target{Endpoint: testEDSClusterName}}).(*edsBalancer) - if !ok { - t.Fatalf("builder.Build(%s) returned type {%T}, want {*edsBalancer}", edsName, edsB) - } - defer edsB.Close() - - edsB.UpdateClientConnState(balancer.ClientConnState{ - BalancerConfig: &EDSConfig{ - BalancerName: testBalancerNameFooBar, - EDSServiceName: testEDSClusterName, - }, - }) - - xdsC := waitForNewXDSClientWithEDSWatch(t, xdsClientCh, testBalancerNameFooBar) - xdsC.InvokeWatchEDSCallback(xdsclient.EndpointsUpdate{}, nil) - edsLB := waitForNewEDSLB(t, edsLBCh) - if err := edsLB.waitForEDSResponse(xdsclient.EndpointsUpdate{}); err != nil { - t.Fatalf("EDS impl got unexpected EDS response: %v", err) - } - - connectionErr := xdsclient.NewErrorf(xdsclient.ErrorTypeConnection, "connection error") - edsB.ResolverError(connectionErr) - if err := xdsC.WaitForCancelEDSWatch(); err == nil { - t.Fatal("watch was canceled, want not canceled (timeout error)") - } - if err := edsLB.waitForEDSResponse(xdsclient.EndpointsUpdate{}); err == nil { - t.Fatal("eds impl got EDS resp, want timeout error") - } - - resourceErr := xdsclient.NewErrorf(xdsclient.ErrorTypeResourceNotFound, "edsBalancer resource not found error") - edsB.ResolverError(resourceErr) - if err := xdsC.WaitForCancelEDSWatch(); err != nil { - t.Fatalf("want watch to be canceled, waitForCancel failed: %v", err) - } - if err := edsLB.waitForEDSResponse(xdsclient.EndpointsUpdate{}); err != nil { - t.Fatalf("EDS impl got unexpected EDS response: %v", err) - } -} - -func (s) TestXDSBalancerConfigParsing(t *testing.T) { - const testEDSName = "eds.service" - var testLRSName = "lrs.server" - b := bytes.NewBuffer(nil) - if err := (&jsonpb.Marshaler{}).Marshal(b, &scpb.XdsConfig{ - ChildPolicy: []*scpb.LoadBalancingConfig{ - {Policy: &scpb.LoadBalancingConfig_Xds{}}, - {Policy: &scpb.LoadBalancingConfig_RoundRobin{ - RoundRobin: &scpb.RoundRobinConfig{}, - }}, - }, - FallbackPolicy: []*scpb.LoadBalancingConfig{ - {Policy: &scpb.LoadBalancingConfig_Xds{}}, - {Policy: &scpb.LoadBalancingConfig_PickFirst{ - PickFirst: &scpb.PickFirstConfig{}, - }}, - }, - EdsServiceName: testEDSName, - LrsLoadReportingServerName: &wrapperspb.StringValue{Value: testLRSName}, - }); err != nil { - t.Fatalf("%v", err) - } - - tests := []struct { - name string - js json.RawMessage - want serviceconfig.LoadBalancingConfig - wantErr bool - }{ - { - name: "jsonpb-generated", - js: b.Bytes(), - want: &EDSConfig{ - ChildPolicy: &loadBalancingConfig{ - Name: "round_robin", - Config: json.RawMessage("{}"), - }, - FallBackPolicy: &loadBalancingConfig{ - Name: "pick_first", - Config: json.RawMessage("{}"), - }, - EDSServiceName: testEDSName, - LrsLoadReportingServerName: &testLRSName, - }, - wantErr: false, - }, - { - // json with random balancers, and the first is not registered. - name: "manually-generated", - js: json.RawMessage(` -{ - "balancerName": "fake.foo.bar", - "childPolicy": [ - {"fake_balancer_C": {}}, - {"fake_balancer_A": {}}, - {"fake_balancer_B": {}} - ], - "fallbackPolicy": [ - {"fake_balancer_C": {}}, - {"fake_balancer_B": {}}, - {"fake_balancer_A": {}} - ], - "edsServiceName": "eds.service", - "lrsLoadReportingServerName": "lrs.server" -}`), - want: &EDSConfig{ - BalancerName: "fake.foo.bar", - ChildPolicy: &loadBalancingConfig{ - Name: "fake_balancer_A", - Config: json.RawMessage("{}"), - }, - FallBackPolicy: &loadBalancingConfig{ - Name: "fake_balancer_B", - Config: json.RawMessage("{}"), - }, - EDSServiceName: testEDSName, - LrsLoadReportingServerName: &testLRSName, - }, - wantErr: false, - }, - { - // json with no lrs server name, LrsLoadReportingServerName should - // be nil (not an empty string). - name: "no-lrs-server-name", - js: json.RawMessage(` -{ - "balancerName": "fake.foo.bar", - "edsServiceName": "eds.service" -}`), - want: &EDSConfig{ - BalancerName: "fake.foo.bar", - EDSServiceName: testEDSName, - LrsLoadReportingServerName: nil, - }, - wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - b := &edsBalancerBuilder{} - got, err := b.ParseConfig(tt.js) - if (err != nil) != tt.wantErr { - t.Errorf("edsBalancerBuilder.ParseConfig() error = %v, wantErr %v", err, tt.wantErr) - return - } - if !cmp.Equal(got, tt.want) { - t.Errorf(cmp.Diff(got, tt.want)) - } - }) - } -} -func (s) TestLoadbalancingConfigParsing(t *testing.T) { - tests := []struct { - name string - s string - want *EDSConfig - }{ - { - name: "empty", - s: "{}", - want: &EDSConfig{}, - }, - { - name: "success1", - s: `{"childPolicy":[{"pick_first":{}}]}`, - want: &EDSConfig{ - ChildPolicy: &loadBalancingConfig{ - Name: "pick_first", - Config: json.RawMessage(`{}`), - }, - }, - }, - { - name: "success2", - s: `{"childPolicy":[{"round_robin":{}},{"pick_first":{}}]}`, - want: &EDSConfig{ - ChildPolicy: &loadBalancingConfig{ - Name: "round_robin", - Config: json.RawMessage(`{}`), - }, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - var cfg EDSConfig - if err := json.Unmarshal([]byte(tt.s), &cfg); err != nil || !cmp.Equal(&cfg, tt.want) { - t.Errorf("test name: %s, parseFullServiceConfig() = %+v, err: %v, want %+v, ", tt.name, cfg, err, tt.want) - } - }) - } -} - -func (s) TestEqualStringPointers(t *testing.T) { - var ( - ta1 = "test-a" - ta2 = "test-a" - tb = "test-b" - ) - tests := []struct { - name string - a *string - b *string - want bool - }{ - {"both-nil", nil, nil, true}, - {"a-non-nil", &ta1, nil, false}, - {"b-non-nil", nil, &tb, false}, - {"equal", &ta1, &ta2, true}, - {"different", &ta1, &tb, false}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := equalStringPointers(tt.a, tt.b); got != tt.want { - t.Errorf("equalStringPointers() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/xds/internal/balancer/edsbalancer/eds_testutil.go b/xds/internal/balancer/edsbalancer/eds_testutil.go deleted file mode 100644 index 5e37cdcb47c7..000000000000 --- a/xds/internal/balancer/edsbalancer/eds_testutil.go +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package edsbalancer - -import ( - "fmt" - "net" - "strconv" - - xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2" - corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" - endpointpb "github.com/envoyproxy/go-control-plane/envoy/api/v2/endpoint" - typepb "github.com/envoyproxy/go-control-plane/envoy/type" - "google.golang.org/grpc/xds/internal" - xdsclient "google.golang.org/grpc/xds/internal/client" -) - -// parseEDSRespProtoForTesting parses EDS response, and panic if parsing fails. -// -// TODO: delete this. The EDS balancer tests should build an EndpointsUpdate -// directly, instead of building and parsing a proto message. -func parseEDSRespProtoForTesting(m *xdspb.ClusterLoadAssignment) xdsclient.EndpointsUpdate { - u, err := parseEDSRespProto(m) - if err != nil { - panic(err.Error()) - } - return u -} - -// parseEDSRespProto turns EDS response proto message to EndpointsUpdate. -func parseEDSRespProto(m *xdspb.ClusterLoadAssignment) (xdsclient.EndpointsUpdate, error) { - ret := xdsclient.EndpointsUpdate{} - for _, dropPolicy := range m.GetPolicy().GetDropOverloads() { - ret.Drops = append(ret.Drops, parseDropPolicy(dropPolicy)) - } - priorities := make(map[uint32]struct{}) - for _, locality := range m.Endpoints { - l := locality.GetLocality() - if l == nil { - return xdsclient.EndpointsUpdate{}, fmt.Errorf("EDS response contains a locality without ID, locality: %+v", locality) - } - lid := internal.LocalityID{ - Region: l.Region, - Zone: l.Zone, - SubZone: l.SubZone, - } - priority := locality.GetPriority() - priorities[priority] = struct{}{} - ret.Localities = append(ret.Localities, xdsclient.Locality{ - ID: lid, - Endpoints: parseEndpoints(locality.GetLbEndpoints()), - Weight: locality.GetLoadBalancingWeight().GetValue(), - Priority: priority, - }) - } - for i := 0; i < len(priorities); i++ { - if _, ok := priorities[uint32(i)]; !ok { - return xdsclient.EndpointsUpdate{}, fmt.Errorf("priority %v missing (with different priorities %v received)", i, priorities) - } - } - return ret, nil -} - -func parseAddress(socketAddress *corepb.SocketAddress) string { - return net.JoinHostPort(socketAddress.GetAddress(), strconv.Itoa(int(socketAddress.GetPortValue()))) -} - -func parseDropPolicy(dropPolicy *xdspb.ClusterLoadAssignment_Policy_DropOverload) xdsclient.OverloadDropConfig { - percentage := dropPolicy.GetDropPercentage() - var ( - numerator = percentage.GetNumerator() - denominator uint32 - ) - switch percentage.GetDenominator() { - case typepb.FractionalPercent_HUNDRED: - denominator = 100 - case typepb.FractionalPercent_TEN_THOUSAND: - denominator = 10000 - case typepb.FractionalPercent_MILLION: - denominator = 1000000 - } - return xdsclient.OverloadDropConfig{ - Category: dropPolicy.GetCategory(), - Numerator: numerator, - Denominator: denominator, - } -} - -func parseEndpoints(lbEndpoints []*endpointpb.LbEndpoint) []xdsclient.Endpoint { - endpoints := make([]xdsclient.Endpoint, 0, len(lbEndpoints)) - for _, lbEndpoint := range lbEndpoints { - endpoints = append(endpoints, xdsclient.Endpoint{ - HealthStatus: xdsclient.EndpointHealthStatus(lbEndpoint.GetHealthStatus()), - Address: parseAddress(lbEndpoint.GetEndpoint().GetAddress().GetSocketAddress()), - Weight: lbEndpoint.GetLoadBalancingWeight().GetValue(), - }) - } - return endpoints -} diff --git a/xds/internal/balancer/edsbalancer/util_test.go b/xds/internal/balancer/edsbalancer/util_test.go deleted file mode 100644 index 748aeffe2bb9..000000000000 --- a/xds/internal/balancer/edsbalancer/util_test.go +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package edsbalancer - -import ( - "testing" - - xdsclient "google.golang.org/grpc/xds/internal/client" - "google.golang.org/grpc/xds/internal/testutils" -) - -func init() { - newRandomWRR = testutils.NewTestWRR -} - -func (s) TestDropper(t *testing.T) { - const repeat = 2 - - type args struct { - numerator uint32 - denominator uint32 - } - tests := []struct { - name string - args args - }{ - { - name: "2_3", - args: args{ - numerator: 2, - denominator: 3, - }, - }, - { - name: "4_8", - args: args{ - numerator: 4, - denominator: 8, - }, - }, - { - name: "7_20", - args: args{ - numerator: 7, - denominator: 20, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - d := newDropper(xdsclient.OverloadDropConfig{ - Category: "", - Numerator: tt.args.numerator, - Denominator: tt.args.denominator, - }) - - var ( - dCount int - wantCount = int(tt.args.numerator) * repeat - loopCount = int(tt.args.denominator) * repeat - ) - for i := 0; i < loopCount; i++ { - if d.drop() { - dCount++ - } - } - - if dCount != (wantCount) { - t.Errorf("with numerator %v, denominator %v repeat %v, got drop count: %v, want %v", - tt.args.numerator, tt.args.denominator, repeat, dCount, wantCount) - } - }) - } -} diff --git a/xds/internal/balancer/edsbalancer/xds_client_wrapper.go b/xds/internal/balancer/edsbalancer/xds_client_wrapper.go deleted file mode 100644 index 0f987a948fb9..000000000000 --- a/xds/internal/balancer/edsbalancer/xds_client_wrapper.go +++ /dev/null @@ -1,313 +0,0 @@ -/* - * - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package edsbalancer - -import ( - "google.golang.org/grpc" - "google.golang.org/grpc/attributes" - "google.golang.org/grpc/balancer" - "google.golang.org/grpc/internal/grpclog" - xdsinternal "google.golang.org/grpc/xds/internal" - "google.golang.org/grpc/xds/internal/balancer/lrs" - xdsclient "google.golang.org/grpc/xds/internal/client" - "google.golang.org/grpc/xds/internal/client/bootstrap" -) - -// xdsClientInterface contains only the xds_client methods needed by EDS -// balancer. It's defined so we can override xdsclientNew function in tests. -type xdsClientInterface interface { - WatchEndpoints(clusterName string, edsCb func(xdsclient.EndpointsUpdate, error)) (cancel func()) - ReportLoad(server string, clusterName string, loadStore lrs.Store) (cancel func()) - Close() -} - -var ( - xdsclientNew = func(opts xdsclient.Options) (xdsClientInterface, error) { - return xdsclient.New(opts) - } - bootstrapConfigNew = bootstrap.NewConfig -) - -// xdsclientWrapper is responsible for getting the xds client from attributes or -// creating a new xds client, and start watching EDS. The given callbacks will -// be called with EDS updates or errors. -type xdsclientWrapper struct { - logger *grpclog.PrefixLogger - - newEDSUpdate func(xdsclient.EndpointsUpdate, error) - bbo balancer.BuildOptions - loadStore lrs.Store - - balancerName string - // xdsclient could come from attributes, or created with balancerName. - xdsclient xdsClientInterface - - // edsServiceName is the edsServiceName currently being watched, not - // necessary the edsServiceName from service config. - // - // If edsServiceName from service config is an empty, this will be user's - // dial target (because that's what we use to watch EDS). - // - // TODO: remove the empty string related behavior, when we switch to always - // do CDS. - edsServiceName string - cancelEndpointsWatch func() - loadReportServer *string // LRS is disabled if loadReporterServer is nil. - cancelLoadReport func() -} - -// newXDSClientWrapper creates an empty xds_client wrapper that does nothing. It -// can accept xds_client configs, to new/switch xds_client to use. -// -// The given callbacks won't be called until the underlying xds_client is -// working and sends updates. -func newXDSClientWrapper(newEDSUpdate func(xdsclient.EndpointsUpdate, error), bbo balancer.BuildOptions, loadStore lrs.Store, logger *grpclog.PrefixLogger) *xdsclientWrapper { - return &xdsclientWrapper{ - logger: logger, - newEDSUpdate: newEDSUpdate, - bbo: bbo, - loadStore: loadStore, - } -} - -// replaceXDSClient replaces xdsclient fields to the newClient if they are -// different. If xdsclient is replaced, the balancerName field will also be -// updated to newBalancerName. -// -// If the old xdsclient is replaced, and was created locally (not from -// attributes), it will be closed. -// -// It returns whether xdsclient is replaced. -func (c *xdsclientWrapper) replaceXDSClient(newClient xdsClientInterface, newBalancerName string) bool { - if c.xdsclient == newClient { - return false - } - oldClient := c.xdsclient - oldBalancerName := c.balancerName - c.xdsclient = newClient - c.balancerName = newBalancerName - if oldBalancerName != "" { - // OldBalancerName!="" means if the old client was not from attributes. - oldClient.Close() - } - return true -} - -// updateXDSClient sets xdsclient in wrapper to the correct one based on the -// attributes and service config. -// -// If client is found in attributes, it will be used, but we also need to decide -// whether to close the old client. -// - if old client was created locally (balancerName is not ""), close it and -// replace it -// - if old client was from previous attributes, only replace it, but don't -// close it -// -// If client is not found in attributes, will need to create a new one only if -// the balancerName (from bootstrap file or from service config) changed. -// - if balancer names are the same, do nothing, and return false -// - if balancer names are different, create new one, and return true -func (c *xdsclientWrapper) updateXDSClient(config *EDSConfig, attr *attributes.Attributes) bool { - if attr != nil { - if clientFromAttr, _ := attr.Value(xdsinternal.XDSClientID).(xdsClientInterface); clientFromAttr != nil { - // This will also clear balancerName, to indicate that client is - // from attributes. - return c.replaceXDSClient(clientFromAttr, "") - } - } - - clientConfig, err := bootstrapConfigNew() - if err != nil { - // TODO: propagate this error to ClientConn, and fail RPCs if necessary. - clientConfig = &bootstrap.Config{BalancerName: config.BalancerName} - } - - if c.balancerName == clientConfig.BalancerName { - return false - } - - if clientConfig.Creds == nil { - // TODO: Once we start supporting a mechanism to register credential - // types, a failure to find the credential type mentioned in the - // bootstrap file should result in a failure, and not in using - // credentials from the parent channel (passed through the - // resolver.BuildOptions). - clientConfig.Creds = c.defaultDialCreds(clientConfig.BalancerName) - } - var dopts []grpc.DialOption - if dialer := c.bbo.Dialer; dialer != nil { - dopts = []grpc.DialOption{grpc.WithContextDialer(dialer)} - } - - newClient, err := xdsclientNew(xdsclient.Options{Config: *clientConfig, DialOpts: dopts}) - if err != nil { - // This should never fail. xdsclientnew does a non-blocking dial, and - // all the config passed in should be validated. - // - // This could leave c.xdsclient as nil if this is the first update. - c.logger.Warningf("eds: failed to create xdsclient, error: %v", err) - return false - } - return c.replaceXDSClient(newClient, clientConfig.BalancerName) -} - -// startEndpointsWatch starts the EDS watch. Caller can call this when the -// xds_client is updated, or the edsServiceName is updated. -// -// Note that if there's already a watch in progress, it's not explicitly -// canceled. Because for each xds_client, there should be only one EDS watch in -// progress. So a new EDS watch implicitly cancels the previous one. -// -// This usually means load report needs to be restarted, but this function does -// NOT do that. Caller needs to call startLoadReport separately. -func (c *xdsclientWrapper) startEndpointsWatch(nameToWatch string) { - if c.xdsclient == nil { - return - } - - c.edsServiceName = nameToWatch - if c.cancelEndpointsWatch != nil { - c.cancelEndpointsWatch() - } - cancelEDSWatch := c.xdsclient.WatchEndpoints(c.edsServiceName, func(update xdsclient.EndpointsUpdate, err error) { - c.logger.Infof("Watch update from xds-client %p, content: %+v", c.xdsclient, update) - c.newEDSUpdate(update, err) - }) - c.logger.Infof("Watch started on resource name %v with xds-client %p", c.edsServiceName, c.xdsclient) - c.cancelEndpointsWatch = func() { - cancelEDSWatch() - c.logger.Infof("Watch cancelled on resource name %v with xds-client %p", c.edsServiceName, c.xdsclient) - } -} - -// startLoadReport starts load reporting. If there's already a load reporting in -// progress, it cancels that. -// -// Caller can cal this when the loadReportServer name changes, but -// edsServiceName doesn't (so we only need to restart load reporting, not EDS -// watch). -func (c *xdsclientWrapper) startLoadReport(edsServiceNameBeingWatched string, loadReportServer *string) { - if c.xdsclient == nil { - c.logger.Warningf("xds: xdsclient is nil when trying to start load reporting. This means xdsclient wasn't passed in from the resolver, and xdsclient.New failed") - return - } - if c.loadStore != nil { - if c.cancelLoadReport != nil { - c.cancelLoadReport() - } - c.loadReportServer = loadReportServer - if c.loadReportServer != nil { - c.cancelLoadReport = c.xdsclient.ReportLoad(*c.loadReportServer, edsServiceNameBeingWatched, c.loadStore) - } - } -} - -// handleUpdate applies the service config and attributes updates to the client, -// including updating the xds_client to use, and updating the EDS name to watch. -func (c *xdsclientWrapper) handleUpdate(config *EDSConfig, attr *attributes.Attributes) { - clientChanged := c.updateXDSClient(config, attr) - - var ( - restartEndpointsWatch bool - restartLoadReport bool - ) - - // The clusterName to watch should come from CDS response, via service - // config. If it's an empty string, fallback user's dial target. - nameToWatch := config.EDSServiceName - if nameToWatch == "" { - c.logger.Warningf("eds: cluster name to watch is an empty string. Fallback to user's dial target") - nameToWatch = c.bbo.Target.Endpoint - } - - // Need to restart EDS watch when one of the following happens: - // - the xds_client is updated - // - the xds_client didn't change, but the edsServiceName changed - // - // Only need to restart load reporting when: - // - no need to restart EDS, but loadReportServer name changed - if clientChanged || c.edsServiceName != nameToWatch { - restartEndpointsWatch = true - restartLoadReport = true - } else if !equalStringPointers(c.loadReportServer, config.LrsLoadReportingServerName) { - restartLoadReport = true - } - - if restartEndpointsWatch { - c.startEndpointsWatch(nameToWatch) - } - - if restartLoadReport { - c.startLoadReport(nameToWatch, config.LrsLoadReportingServerName) - } -} - -func (c *xdsclientWrapper) cancelWatch() { - c.loadReportServer = nil - if c.cancelLoadReport != nil { - c.cancelLoadReport() - } - c.edsServiceName = "" - if c.cancelEndpointsWatch != nil { - c.cancelEndpointsWatch() - } -} - -func (c *xdsclientWrapper) close() { - c.cancelWatch() - if c.xdsclient != nil && c.balancerName != "" { - // Only close xdsclient if it's not from attributes. - c.xdsclient.Close() - } -} - -// defaultDialCreds builds a DialOption containing the credentials to be used -// while talking to the xDS server (this is done only if the xds bootstrap -// process does not return any credentials to use). If the parent channel -// contains DialCreds, we use it as is. If it contains a CredsBundle, we use -// just the transport credentials from the bundle. If we don't find any -// credentials on the parent channel, we resort to using an insecure channel. -func (c *xdsclientWrapper) defaultDialCreds(balancerName string) grpc.DialOption { - switch { - case c.bbo.DialCreds != nil: - if err := c.bbo.DialCreds.OverrideServerName(balancerName); err != nil { - c.logger.Warningf("xds: failed to override server name in credentials: %v, using Insecure", err) - return grpc.WithInsecure() - } - return grpc.WithTransportCredentials(c.bbo.DialCreds) - case c.bbo.CredsBundle != nil: - return grpc.WithTransportCredentials(c.bbo.CredsBundle.TransportCredentials()) - default: - c.logger.Warningf("xds: no credentials available, using Insecure") - return grpc.WithInsecure() - } -} - -// equalStringPointers returns true if -// - a and b are both nil OR -// - *a == *b (and a and b are both non-nil) -func equalStringPointers(a, b *string) bool { - if a == nil && b == nil { - return true - } - if a == nil || b == nil { - return false - } - return *a == *b -} diff --git a/xds/internal/balancer/edsbalancer/xds_client_wrapper_test.go b/xds/internal/balancer/edsbalancer/xds_client_wrapper_test.go deleted file mode 100644 index 6ae65dc7c94b..000000000000 --- a/xds/internal/balancer/edsbalancer/xds_client_wrapper_test.go +++ /dev/null @@ -1,230 +0,0 @@ -/* - * - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package edsbalancer - -import ( - "errors" - "fmt" - "testing" - "time" - - xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2" - "github.com/golang/protobuf/proto" - "github.com/google/go-cmp/cmp" - - "google.golang.org/grpc" - "google.golang.org/grpc/attributes" - "google.golang.org/grpc/balancer" - "google.golang.org/grpc/resolver" - xdsinternal "google.golang.org/grpc/xds/internal" - xdsclient "google.golang.org/grpc/xds/internal/client" - "google.golang.org/grpc/xds/internal/client/bootstrap" - "google.golang.org/grpc/xds/internal/testutils" - "google.golang.org/grpc/xds/internal/testutils/fakeclient" - "google.golang.org/grpc/xds/internal/testutils/fakeserver" - "google.golang.org/grpc/xds/internal/version" -) - -var ( - testServiceName = "test/foo" - testEDSClusterName = "test/service/eds" -) - -// Given a list of resource names, verifies that EDS requests for the same are -// received at the fake server. -func verifyExpectedRequests(fs *fakeserver.Server, resourceNames ...string) error { - wantReq := &xdspb.DiscoveryRequest{ - TypeUrl: version.V2EndpointsURL, - Node: testutils.EmptyNodeProtoV2, - } - for _, name := range resourceNames { - if name != "" { - wantReq.ResourceNames = []string{name} - } - req, err := fs.XDSRequestChan.TimedReceive(time.Millisecond * 100) - if err != nil { - return fmt.Errorf("timed out when expecting request {%+v} at fake server", wantReq) - } - edsReq := req.(*fakeserver.Request) - if edsReq.Err != nil { - return fmt.Errorf("eds RPC failed with err: %v", edsReq.Err) - } - if !proto.Equal(edsReq.Req, wantReq) { - return fmt.Errorf("got EDS request %v, expected: %v, diff: %s", edsReq.Req, wantReq, cmp.Diff(edsReq.Req, wantReq, cmp.Comparer(proto.Equal))) - } - } - return nil -} - -// TestClientWrapperWatchEDS verifies that the clientWrapper registers an -// EDS watch for expected resource upon receiving an update from the top-level -// edsBalancer. -// -// The test does the following: -// * Starts a fake xDS server. -// * Creates a clientWrapper. -// * Sends updates with different edsServiceNames and expects new watches to be -// registered. -func (s) TestClientWrapperWatchEDS(t *testing.T) { - fakeServer, cleanup, err := fakeserver.StartServer() - if err != nil { - t.Fatalf("Failed to start fake xDS server: %v", err) - } - defer cleanup() - t.Logf("Started fake xDS server at %s...", fakeServer.Address) - - cw := newXDSClientWrapper(nil, balancer.BuildOptions{Target: resolver.Target{Endpoint: testServiceName}}, nil, nil) - defer cw.close() - t.Logf("Started xDS client wrapper for endpoint %s...", testServiceName) - - oldBootstrapConfigNew := bootstrapConfigNew - bootstrapConfigNew = func() (*bootstrap.Config, error) { - return &bootstrap.Config{ - BalancerName: fakeServer.Address, - Creds: grpc.WithInsecure(), - NodeProto: testutils.EmptyNodeProtoV2, - }, nil - } - defer func() { bootstrapConfigNew = oldBootstrapConfigNew }() - - // Update with an empty edsServiceName should trigger an EDS watch - // for the user's dial target. - cw.handleUpdate(&EDSConfig{ - BalancerName: fakeServer.Address, - EDSServiceName: "", - }, nil) - if _, err := fakeServer.NewConnChan.TimedReceive(1 * time.Second); err != nil { - t.Fatal("Failed to connect to fake server") - } - t.Log("Client connection established to fake server...") - if err := verifyExpectedRequests(fakeServer, testServiceName); err != nil { - t.Fatal(err) - } - - // Update with an non-empty edsServiceName should trigger an EDS watch for - // the same. The previously registered watch will be cancelled, which will - // result in an EDS request with no resource names being sent to the server. - cw.handleUpdate(&EDSConfig{ - BalancerName: fakeServer.Address, - EDSServiceName: "foobar-1", - }, nil) - if err := verifyExpectedRequests(fakeServer, "", "foobar-1"); err != nil { - t.Fatal(err) - } - - // Also test the case where the edsServerName changes from one - // non-empty name to another, and make sure a new watch is - // registered. - cw.handleUpdate(&EDSConfig{ - BalancerName: fakeServer.Address, - EDSServiceName: "foobar-2", - }, nil) - if err := verifyExpectedRequests(fakeServer, "", "foobar-2"); err != nil { - t.Fatal(err) - } -} - -// TestClientWrapperHandleUpdateError verifies that the clientWrapper handles -// errors from the edsWatch callback appropriately. -// -// The test does the following: -// * Creates a clientWrapper. -// * Creates a fakeclient.Client and passes it to the clientWrapper in attributes. -// * Verifies the clientWrapper registers an EDS watch. -// * Forces the fakeclient.Client to invoke the registered EDS watch callback with -// an error. Verifies that the wrapper does not invoke the top-level -// edsBalancer with the received error. -func (s) TestClientWrapperHandleUpdateError(t *testing.T) { - edsRespChan := testutils.NewChannel() - newEDS := func(update xdsclient.EndpointsUpdate, err error) { - edsRespChan.Send(&edsUpdate{resp: update, err: err}) - } - - cw := newXDSClientWrapper(newEDS, balancer.BuildOptions{Target: resolver.Target{Endpoint: testServiceName}}, nil, nil) - defer cw.close() - - xdsC := fakeclient.NewClient() - cw.handleUpdate(&EDSConfig{EDSServiceName: testEDSClusterName}, attributes.New(xdsinternal.XDSClientID, xdsC)) - gotCluster, err := xdsC.WaitForWatchEDS() - if err != nil { - t.Fatalf("xdsClient.WatchEndpoints failed with error: %v", err) - } - if gotCluster != testEDSClusterName { - t.Fatalf("xdsClient.WatchEndpoints() called with cluster: %v, want %v", gotCluster, testEDSClusterName) - } - watchErr := errors.New("EDS watch callback error") - xdsC.InvokeWatchEDSCallback(xdsclient.EndpointsUpdate{}, watchErr) - - // The callback is called with an error, expect no update from edsRespChan. - // - // TODO: check for loseContact() when errors indicating "lose contact" are - // handled correctly. - gotUpdate, err := edsRespChan.Receive() - if err != nil { - t.Fatalf("edsBalancer failed to get edsUpdate %v", err) - } - update := gotUpdate.(*edsUpdate) - if !cmp.Equal(update.resp, (xdsclient.EndpointsUpdate{})) || update.err != watchErr { - t.Fatalf("want update {nil, %v}, got %+v", watchErr, update) - } -} - -// TestClientWrapperGetsXDSClientInAttributes verfies the case where the -// clientWrapper receives the xdsClient to use in the attributes section of the -// update. -func (s) TestClientWrapperGetsXDSClientInAttributes(t *testing.T) { - oldxdsclientNew := xdsclientNew - xdsclientNew = func(_ xdsclient.Options) (xdsClientInterface, error) { - t.Fatalf("unexpected call to xdsclientNew when xds_client is set in attributes") - return nil, nil - } - defer func() { xdsclientNew = oldxdsclientNew }() - - cw := newXDSClientWrapper(nil, balancer.BuildOptions{Target: resolver.Target{Endpoint: testServiceName}}, nil, nil) - defer cw.close() - - // Verify that the eds watch is registered for the expected resource name. - xdsC1 := fakeclient.NewClient() - cw.handleUpdate(&EDSConfig{EDSServiceName: testEDSClusterName}, attributes.New(xdsinternal.XDSClientID, xdsC1)) - gotCluster, err := xdsC1.WaitForWatchEDS() - if err != nil { - t.Fatalf("xdsClient.WatchEndpoints failed with error: %v", err) - } - if gotCluster != testEDSClusterName { - t.Fatalf("xdsClient.WatchEndpoints() called with cluster: %v, want %v", gotCluster, testEDSClusterName) - } - - // Pass a new client in the attributes. Verify that the watch is - // re-registered on the new client, and that the old client is not closed - // (because clientWrapper only closes clients that it creates, it does not - // close client that are passed through attributes). - xdsC2 := fakeclient.NewClient() - cw.handleUpdate(&EDSConfig{EDSServiceName: testEDSClusterName}, attributes.New(xdsinternal.XDSClientID, xdsC2)) - gotCluster, err = xdsC2.WaitForWatchEDS() - if err != nil { - t.Fatalf("xdsClient.WatchEndpoints failed with error: %v", err) - } - if gotCluster != testEDSClusterName { - t.Fatalf("xdsClient.WatchEndpoints() called with cluster: %v, want %v", gotCluster, testEDSClusterName) - } - - if err := xdsC1.WaitForClose(); err != testutils.ErrRecvTimeout { - t.Fatalf("clientWrapper closed xdsClient received in attributes") - } -} diff --git a/xds/internal/balancer/edsbalancer/xds_lrs_test.go b/xds/internal/balancer/edsbalancer/xds_lrs_test.go deleted file mode 100644 index 7e2b28ba387a..000000000000 --- a/xds/internal/balancer/edsbalancer/xds_lrs_test.go +++ /dev/null @@ -1,64 +0,0 @@ -/* - * - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package edsbalancer - -import ( - "testing" - - "google.golang.org/grpc/attributes" - "google.golang.org/grpc/balancer" - "google.golang.org/grpc/resolver" - xdsinternal "google.golang.org/grpc/xds/internal" - "google.golang.org/grpc/xds/internal/testutils/fakeclient" -) - -// TestXDSLoadReporting verifies that the edsBalancer starts the loadReport -// stream when the lbConfig passed to it contains a valid value for the LRS -// server (empty string). -func (s) TestXDSLoadReporting(t *testing.T) { - builder := balancer.Get(edsName) - cc := newNoopTestClientConn() - edsB, ok := builder.Build(cc, balancer.BuildOptions{Target: resolver.Target{Endpoint: testEDSClusterName}}).(*edsBalancer) - if !ok { - t.Fatalf("builder.Build(%s) returned type {%T}, want {*edsBalancer}", edsName, edsB) - } - defer edsB.Close() - - xdsC := fakeclient.NewClient() - edsB.UpdateClientConnState(balancer.ClientConnState{ - ResolverState: resolver.State{Attributes: attributes.New(xdsinternal.XDSClientID, xdsC)}, - BalancerConfig: &EDSConfig{LrsLoadReportingServerName: new(string)}, - }) - - gotCluster, err := xdsC.WaitForWatchEDS() - if err != nil { - t.Fatalf("xdsClient.WatchEndpoints failed with error: %v", err) - } - if gotCluster != testEDSClusterName { - t.Fatalf("xdsClient.WatchEndpoints() called with cluster: %v, want %v", gotCluster, testEDSClusterName) - } - - got, err := xdsC.WaitForReportLoad() - if err != nil { - t.Fatalf("xdsClient.ReportLoad failed with error: %v", err) - } - if got.Server != "" || got.Cluster != testEDSClusterName { - t.Fatalf("xdsClient.ReportLoad called with {%v, %v}: want {\"\", %v}", got.Server, got.Cluster, testEDSClusterName) - } -} diff --git a/xds/internal/balancer/edsbalancer/xds_old.go b/xds/internal/balancer/edsbalancer/xds_old.go deleted file mode 100644 index 6729e6801f15..000000000000 --- a/xds/internal/balancer/edsbalancer/xds_old.go +++ /dev/null @@ -1,46 +0,0 @@ -/* - * - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package edsbalancer - -import "google.golang.org/grpc/balancer" - -// The old xds balancer implements logic for both CDS and EDS. With the new -// design, CDS is split and moved to a separate balancer, and the xds balancer -// becomes the EDS balancer. -// -// To keep the existing tests working, this file regisger EDS balancer under the -// old xds balancer name. -// -// TODO: delete this file when migration to new workflow (LDS, RDS, CDS, EDS) is -// done. - -const xdsName = "xds_experimental" - -func init() { - balancer.Register(&xdsBalancerBuilder{}) -} - -// xdsBalancerBuilder register edsBalancerBuilder (now with name -// "eds_experimental") under the old name "xds_experimental". -type xdsBalancerBuilder struct { - edsBalancerBuilder -} - -func (b *xdsBalancerBuilder) Name() string { - return xdsName -} diff --git a/xds/internal/balancer/loadstore/load_store_wrapper.go b/xds/internal/balancer/loadstore/load_store_wrapper.go new file mode 100644 index 000000000000..8ce958d71ca8 --- /dev/null +++ b/xds/internal/balancer/loadstore/load_store_wrapper.go @@ -0,0 +1,120 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package loadstore contains the loadStoreWrapper shared by the balancers. +package loadstore + +import ( + "sync" + + "google.golang.org/grpc/xds/internal/xdsclient/load" +) + +// NewWrapper creates a Wrapper. +func NewWrapper() *Wrapper { + return &Wrapper{} +} + +// Wrapper wraps a load store with cluster and edsService. +// +// It's store and cluster/edsService can be updated separately. And it will +// update its internal perCluster store so that new stats will be added to the +// correct perCluster. +// +// Note that this struct is a temporary walkaround before we implement graceful +// switch for EDS. Any update to the clusterName and serviceName is too early, +// the perfect timing is when the picker is updated with the new connection. +// This early update could cause picks for the old SubConn being reported to the +// new services. +// +// When the graceful switch in EDS is done, there should be no need for this +// struct. The policies that record/report load shouldn't need to handle update +// of lrsServerName/cluster/edsService. Its parent should do a graceful switch +// of the whole tree when one of that changes. +type Wrapper struct { + mu sync.RWMutex + cluster string + edsService string + // store and perCluster are initialized as nil. They are only set by the + // balancer when LRS is enabled. Before that, all functions to record loads + // are no-op. + store *load.Store + perCluster load.PerClusterReporter +} + +// UpdateClusterAndService updates the cluster name and eds service for this +// wrapper. If any one of them is changed from before, the perCluster store in +// this wrapper will also be updated. +func (lsw *Wrapper) UpdateClusterAndService(cluster, edsService string) { + lsw.mu.Lock() + defer lsw.mu.Unlock() + if cluster == lsw.cluster && edsService == lsw.edsService { + return + } + lsw.cluster = cluster + lsw.edsService = edsService + lsw.perCluster = lsw.store.PerCluster(lsw.cluster, lsw.edsService) +} + +// UpdateLoadStore updates the load store for this wrapper. If it is changed +// from before, the perCluster store in this wrapper will also be updated. +func (lsw *Wrapper) UpdateLoadStore(store *load.Store) { + lsw.mu.Lock() + defer lsw.mu.Unlock() + if store == lsw.store { + return + } + lsw.store = store + lsw.perCluster = lsw.store.PerCluster(lsw.cluster, lsw.edsService) +} + +// CallStarted records a call started in the store. +func (lsw *Wrapper) CallStarted(locality string) { + lsw.mu.RLock() + defer lsw.mu.RUnlock() + if lsw.perCluster != nil { + lsw.perCluster.CallStarted(locality) + } +} + +// CallFinished records a call finished in the store. +func (lsw *Wrapper) CallFinished(locality string, err error) { + lsw.mu.RLock() + defer lsw.mu.RUnlock() + if lsw.perCluster != nil { + lsw.perCluster.CallFinished(locality, err) + } +} + +// CallServerLoad records the server load in the store. +func (lsw *Wrapper) CallServerLoad(locality, name string, val float64) { + lsw.mu.RLock() + defer lsw.mu.RUnlock() + if lsw.perCluster != nil { + lsw.perCluster.CallServerLoad(locality, name, val) + } +} + +// CallDropped records a call dropped in the store. +func (lsw *Wrapper) CallDropped(category string) { + lsw.mu.RLock() + defer lsw.mu.RUnlock() + if lsw.perCluster != nil { + lsw.perCluster.CallDropped(category) + } +} diff --git a/xds/internal/balancer/lrs/balancer.go b/xds/internal/balancer/lrs/balancer.go deleted file mode 100644 index 4af91c76498a..000000000000 --- a/xds/internal/balancer/lrs/balancer.go +++ /dev/null @@ -1,216 +0,0 @@ -/* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -// Package lrs implements load reporting balancer for xds. -package lrs - -import ( - "encoding/json" - "fmt" - - "google.golang.org/grpc/attributes" - "google.golang.org/grpc/balancer" - "google.golang.org/grpc/internal/grpclog" - "google.golang.org/grpc/serviceconfig" - "google.golang.org/grpc/xds/internal" - xdsinternal "google.golang.org/grpc/xds/internal" -) - -func init() { - balancer.Register(&lrsBB{}) -} - -const lrsBalancerName = "lrs_experimental" - -type lrsBB struct{} - -func (l *lrsBB) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { - b := &lrsBalancer{ - cc: cc, - buildOpts: opts, - } - b.loadStore = NewStore() - b.client = newXDSClientWrapper(b.loadStore) - b.logger = prefixLogger(b) - b.logger.Infof("Created") - return b -} - -func (l *lrsBB) Name() string { - return lrsBalancerName -} - -func (l *lrsBB) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { - return parseConfig(c) -} - -type lrsBalancer struct { - cc balancer.ClientConn - buildOpts balancer.BuildOptions - - logger *grpclog.PrefixLogger - loadStore Store - client *xdsClientWrapper - - config *lbConfig - lb balancer.Balancer // The sub balancer. -} - -func (b *lrsBalancer) UpdateClientConnState(s balancer.ClientConnState) error { - newConfig, ok := s.BalancerConfig.(*lbConfig) - if !ok { - return fmt.Errorf("unexpected balancer config with type: %T", s.BalancerConfig) - } - - // If child policy is a different type, recreate the sub-balancer. - if b.config == nil || b.config.ChildPolicy.Name != newConfig.ChildPolicy.Name { - bb := balancer.Get(newConfig.ChildPolicy.Name) - if bb == nil { - return fmt.Errorf("balancer %q not registered", newConfig.ChildPolicy.Name) - } - if b.lb != nil { - b.lb.Close() - } - b.lb = bb.Build(newCCWrapper(b.cc, b.loadStore, newConfig.Locality), b.buildOpts) - } - // Update load reporting config or xds client. - b.client.update(newConfig, s.ResolverState.Attributes) - b.config = newConfig - - // Addresses and sub-balancer config are sent to sub-balancer. - return b.lb.UpdateClientConnState(balancer.ClientConnState{ - ResolverState: s.ResolverState, - BalancerConfig: b.config.ChildPolicy.Config, - }) -} - -func (b *lrsBalancer) ResolverError(err error) { - if b.lb != nil { - b.lb.ResolverError(err) - } -} - -func (b *lrsBalancer) UpdateSubConnState(sc balancer.SubConn, s balancer.SubConnState) { - if b.lb != nil { - b.lb.UpdateSubConnState(sc, s) - } -} - -func (b *lrsBalancer) Close() { - if b.lb != nil { - b.lb.Close() - b.lb = nil - } - b.client.close() -} - -type ccWrapper struct { - balancer.ClientConn - loadStore Store - localityID *internal.LocalityID -} - -func newCCWrapper(cc balancer.ClientConn, loadStore Store, localityID *internal.LocalityID) *ccWrapper { - return &ccWrapper{ - ClientConn: cc, - loadStore: loadStore, - localityID: localityID, - } -} - -func (ccw *ccWrapper) UpdateState(s balancer.State) { - s.Picker = newLoadReportPicker(s.Picker, *ccw.localityID, ccw.loadStore) - ccw.ClientConn.UpdateState(s) -} - -// xdsClientInterface contains only the xds_client methods needed by LRS -// balancer. It's defined so we can override xdsclient in tests. -type xdsClientInterface interface { - ReportLoad(server string, clusterName string, loadStore Store) (cancel func()) - Close() -} - -type xdsClientWrapper struct { - loadStore Store - - c xdsClientInterface - cancelLoadReport func() - clusterName string - lrsServerName string -} - -func newXDSClientWrapper(loadStore Store) *xdsClientWrapper { - return &xdsClientWrapper{ - loadStore: loadStore, - } -} - -// update checks the config and xdsclient, and decides whether it needs to -// restart the load reporting stream. -// -// TODO: refactor lrs to share one stream instead of one per EDS. -func (w *xdsClientWrapper) update(newConfig *lbConfig, attr *attributes.Attributes) { - var restartLoadReport bool - if attr != nil { - if clientFromAttr, _ := attr.Value(xdsinternal.XDSClientID).(xdsClientInterface); clientFromAttr != nil { - if w.c != clientFromAttr { - // xds client is different, restart. - restartLoadReport = true - w.c = clientFromAttr - } - } - } - - // ClusterName is different, restart. ClusterName is from ClusterName and - // EdsServiceName. - // - // TODO: LRS request actually has separate fields from these two values. - // Update lrs.Store to set both. - newClusterName := newConfig.EdsServiceName - if newClusterName == "" { - newClusterName = newConfig.ClusterName - } - if w.clusterName != newClusterName { - restartLoadReport = true - w.clusterName = newClusterName - } - - if w.lrsServerName != newConfig.LrsLoadReportingServerName { - // LrsLoadReportingServerName is different, load should be report to a - // different server, restart. - restartLoadReport = true - w.lrsServerName = newConfig.LrsLoadReportingServerName - } - - if restartLoadReport { - if w.cancelLoadReport != nil { - w.cancelLoadReport() - w.cancelLoadReport = nil - } - if w.c != nil { - w.cancelLoadReport = w.c.ReportLoad(w.lrsServerName, w.clusterName, w.loadStore) - } - } -} - -func (w *xdsClientWrapper) close() { - if w.cancelLoadReport != nil { - w.cancelLoadReport() - w.cancelLoadReport = nil - } -} diff --git a/xds/internal/balancer/lrs/balancer_test.go b/xds/internal/balancer/lrs/balancer_test.go deleted file mode 100644 index c57b5e1a127f..000000000000 --- a/xds/internal/balancer/lrs/balancer_test.go +++ /dev/null @@ -1,175 +0,0 @@ -/* - * - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package lrs - -import ( - "fmt" - "testing" - "time" - - "github.com/google/go-cmp/cmp" - "google.golang.org/grpc/attributes" - "google.golang.org/grpc/balancer" - "google.golang.org/grpc/balancer/roundrobin" - "google.golang.org/grpc/connectivity" - internalserviceconfig "google.golang.org/grpc/internal/serviceconfig" - "google.golang.org/grpc/resolver" - xdsinternal "google.golang.org/grpc/xds/internal" - "google.golang.org/grpc/xds/internal/testutils" -) - -var ( - testBackendAddrs = []resolver.Address{ - {Addr: "1.1.1.1:1"}, - } - testLocality = &xdsinternal.LocalityID{ - Region: "test-region", - Zone: "test-zone", - SubZone: "test-sub-zone", - } -) - -// This is a subset of testutils.fakeclient. Cannot use testutils.fakeclient -// because testutils imports package lrs. -// -// TODO: after refactoring xdsclient to support load reporting, the testutils -// package won't need to depend on lrs package for the store. And we can use the -// testutils for this. -type fakeXDSClient struct { - loadReportCh chan *reportLoadArgs -} - -func newFakeXDSClient() *fakeXDSClient { - return &fakeXDSClient{ - loadReportCh: make(chan *reportLoadArgs, 10), - } -} - -// reportLoadArgs wraps the arguments passed to ReportLoad. -type reportLoadArgs struct { - // server is the name of the server to which the load is reported. - server string - // cluster is the name of the cluster for which load is reported. - cluster string - // loadStore is the store where loads are stored. - loadStore interface{} -} - -// ReportLoad starts reporting load about clusterName to server. -func (xdsC *fakeXDSClient) ReportLoad(server string, clusterName string, loadStore Store) (cancel func()) { - xdsC.loadReportCh <- &reportLoadArgs{server: server, cluster: clusterName, loadStore: loadStore} - return func() {} -} - -// waitForReportLoad waits for ReportLoad to be invoked on this client within a -// reasonable timeout, and returns the arguments passed to it. -func (xdsC *fakeXDSClient) waitForReportLoad() (*reportLoadArgs, error) { - select { - case <-time.After(time.Second): - return nil, fmt.Errorf("timeout") - case a := <-xdsC.loadReportCh: - return a, nil - } -} - -// Close closes the xds client. -func (xdsC *fakeXDSClient) Close() { -} - -// TestLoadReporting verifies that the lrs balancer starts the loadReport -// stream when the lbConfig passed to it contains a valid value for the LRS -// server (empty string). -func TestLoadReporting(t *testing.T) { - builder := balancer.Get(lrsBalancerName) - cc := testutils.NewTestClientConn(t) - lrsB := builder.Build(cc, balancer.BuildOptions{}) - defer lrsB.Close() - - xdsC := newFakeXDSClient() - if err := lrsB.UpdateClientConnState(balancer.ClientConnState{ - ResolverState: resolver.State{ - Addresses: testBackendAddrs, - Attributes: attributes.New(xdsinternal.XDSClientID, xdsC), - }, - BalancerConfig: &lbConfig{ - EdsServiceName: testClusterName, - LrsLoadReportingServerName: testLRSServerName, - Locality: testLocality, - ChildPolicy: &internalserviceconfig.BalancerConfig{ - Name: roundrobin.Name, - }, - }, - }); err != nil { - t.Fatalf("unexpected error from UpdateClientConnState: %v", err) - } - - got, err := xdsC.waitForReportLoad() - if err != nil { - t.Fatalf("xdsClient.ReportLoad failed with error: %v", err) - } - if got.server != testLRSServerName || got.cluster != testClusterName { - t.Fatalf("xdsClient.ReportLoad called with {%q, %q}: want {%q, %q}", got.server, got.cluster, testLRSServerName, testClusterName) - } - - sc1 := <-cc.NewSubConnCh - lrsB.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - lrsB.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - - // Test pick with one backend. - p1 := <-cc.NewPickerCh - const successCount = 5 - for i := 0; i < successCount; i++ { - gotSCSt, _ := p1.Pick(balancer.PickInfo{}) - if !cmp.Equal(gotSCSt.SubConn, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) { - t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc1) - } - gotSCSt.Done(balancer.DoneInfo{}) - } - const errorCount = 5 - for i := 0; i < errorCount; i++ { - gotSCSt, _ := p1.Pick(balancer.PickInfo{}) - if !cmp.Equal(gotSCSt.SubConn, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) { - t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc1) - } - gotSCSt.Done(balancer.DoneInfo{Err: fmt.Errorf("error")}) - } - - loads := make(map[xdsinternal.LocalityID]*rpcCountData) - - got.loadStore.(*lrsStore).localityRPCCount.Range( - func(key, value interface{}) bool { - loads[key.(xdsinternal.LocalityID)] = value.(*rpcCountData) - return true - }, - ) - - countData, ok := loads[*testLocality] - if !ok { - t.Fatalf("loads for %v not found in store", testLocality) - } - if *countData.succeeded != successCount { - t.Errorf("got succeeded %v, want %v", *countData.succeeded, successCount) - } - if *countData.errored != errorCount { - t.Errorf("got errord %v, want %v", *countData.errored, errorCount) - } - if *countData.inProgress != 0 { - t.Errorf("got inProgress %v, want %v", *countData.inProgress, 0) - } -} diff --git a/xds/internal/balancer/lrs/config_test.go b/xds/internal/balancer/lrs/config_test.go deleted file mode 100644 index f49430569fed..000000000000 --- a/xds/internal/balancer/lrs/config_test.go +++ /dev/null @@ -1,127 +0,0 @@ -/* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package lrs - -import ( - "testing" - - "github.com/google/go-cmp/cmp" - "google.golang.org/grpc/balancer/roundrobin" - internalserviceconfig "google.golang.org/grpc/internal/serviceconfig" - xdsinternal "google.golang.org/grpc/xds/internal" -) - -const ( - testClusterName = "test-cluster" - testServiceName = "test-eds-service" - testLRSServerName = "test-lrs-name" -) - -func TestParseConfig(t *testing.T) { - tests := []struct { - name string - js string - want *lbConfig - wantErr bool - }{ - { - name: "no cluster name", - js: `{ - "edsServiceName": "test-eds-service", - "lrsLoadReportingServerName": "test-lrs-name", - "locality": { - "region": "test-region", - "zone": "test-zone", - "subZone": "test-sub-zone" - }, - "childPolicy":[{"round_robin":{}}] -} - `, - wantErr: true, - }, - { - name: "no LRS server name", - js: `{ - "clusterName": "test-cluster", - "edsServiceName": "test-eds-service", - "locality": { - "region": "test-region", - "zone": "test-zone", - "subZone": "test-sub-zone" - }, - "childPolicy":[{"round_robin":{}}] -} - `, - wantErr: true, - }, - { - name: "no locality", - js: `{ - "clusterName": "test-cluster", - "edsServiceName": "test-eds-service", - "lrsLoadReportingServerName": "test-lrs-name", - "childPolicy":[{"round_robin":{}}] -} - `, - wantErr: true, - }, - { - name: "good", - js: `{ - "clusterName": "test-cluster", - "edsServiceName": "test-eds-service", - "lrsLoadReportingServerName": "test-lrs-name", - "locality": { - "region": "test-region", - "zone": "test-zone", - "subZone": "test-sub-zone" - }, - "childPolicy":[{"round_robin":{}}] -} - `, - want: &lbConfig{ - ClusterName: testClusterName, - EdsServiceName: testServiceName, - LrsLoadReportingServerName: testLRSServerName, - Locality: &xdsinternal.LocalityID{ - Region: "test-region", - Zone: "test-zone", - SubZone: "test-sub-zone", - }, - ChildPolicy: &internalserviceconfig.BalancerConfig{ - Name: roundrobin.Name, - Config: nil, - }, - }, - wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := parseConfig([]byte(tt.js)) - if (err != nil) != tt.wantErr { - t.Errorf("parseConfig() error = %v, wantErr %v", err, tt.wantErr) - return - } - if diff := cmp.Diff(got, tt.want); diff != "" { - t.Errorf("parseConfig() got = %v, want %v, diff: %s", got, tt.want, diff) - } - }) - } -} diff --git a/xds/internal/balancer/lrs/picker.go b/xds/internal/balancer/lrs/picker.go deleted file mode 100644 index 1fcc6e9e5b31..000000000000 --- a/xds/internal/balancer/lrs/picker.go +++ /dev/null @@ -1,75 +0,0 @@ -/* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package lrs - -import ( - orcapb "github.com/cncf/udpa/go/udpa/data/orca/v1" - "google.golang.org/grpc/balancer" - "google.golang.org/grpc/xds/internal" -) - -const ( - serverLoadCPUName = "cpu_utilization" - serverLoadMemoryName = "mem_utilization" -) - -type loadReportPicker struct { - p balancer.Picker - - id internal.LocalityID - loadStore Store -} - -func newLoadReportPicker(p balancer.Picker, id internal.LocalityID, loadStore Store) *loadReportPicker { - return &loadReportPicker{ - p: p, - id: id, - loadStore: loadStore, - } -} - -func (lrp *loadReportPicker) Pick(info balancer.PickInfo) (balancer.PickResult, error) { - res, err := lrp.p.Pick(info) - if err != nil { - return res, err - } - - lrp.loadStore.CallStarted(lrp.id) - oldDone := res.Done - res.Done = func(info balancer.DoneInfo) { - if oldDone != nil { - oldDone(info) - } - lrp.loadStore.CallFinished(lrp.id, info.Err) - - load, ok := info.ServerLoad.(*orcapb.OrcaLoadReport) - if !ok { - return - } - lrp.loadStore.CallServerLoad(lrp.id, serverLoadCPUName, load.CpuUtilization) - lrp.loadStore.CallServerLoad(lrp.id, serverLoadMemoryName, load.MemUtilization) - for n, d := range load.RequestCost { - lrp.loadStore.CallServerLoad(lrp.id, n, d) - } - for n, d := range load.Utilization { - lrp.loadStore.CallServerLoad(lrp.id, n, d) - } - } - return res, err -} diff --git a/xds/internal/balancer/lrs/store.go b/xds/internal/balancer/lrs/store.go deleted file mode 100644 index 96c85f9cc9cd..000000000000 --- a/xds/internal/balancer/lrs/store.go +++ /dev/null @@ -1,376 +0,0 @@ -/* - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package lrs - -import ( - "context" - "sync" - "sync/atomic" - "time" - - corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" - endpointpb "github.com/envoyproxy/go-control-plane/envoy/api/v2/endpoint" - lrsgrpc "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v2" - lrspb "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v2" - "github.com/golang/protobuf/ptypes" - "google.golang.org/grpc" - "google.golang.org/grpc/internal/backoff" - "google.golang.org/grpc/xds/internal" -) - -const negativeOneUInt64 = ^uint64(0) - -// Store defines the interface for a load store. It keeps loads and can report -// them to a server when requested. -type Store interface { - CallDropped(category string) - CallStarted(l internal.LocalityID) - CallFinished(l internal.LocalityID, err error) - CallServerLoad(l internal.LocalityID, name string, d float64) - // Report the load of clusterName to cc. - ReportTo(ctx context.Context, cc *grpc.ClientConn, clusterName string, node *corepb.Node) -} - -type rpcCountData struct { - // Only atomic accesses are allowed for the fields. - succeeded *uint64 - errored *uint64 - inProgress *uint64 - - // Map from load name to load data (sum+count). Loading data from map is - // atomic, but updating data takes a lock, which could cause contention when - // multiple RPCs try to report loads for the same name. - // - // To fix the contention, shard this map. - serverLoads sync.Map // map[string]*rpcLoadData -} - -func newRPCCountData() *rpcCountData { - return &rpcCountData{ - succeeded: new(uint64), - errored: new(uint64), - inProgress: new(uint64), - } -} - -func (rcd *rpcCountData) incrSucceeded() { - atomic.AddUint64(rcd.succeeded, 1) -} - -func (rcd *rpcCountData) loadAndClearSucceeded() uint64 { - return atomic.SwapUint64(rcd.succeeded, 0) -} - -func (rcd *rpcCountData) incrErrored() { - atomic.AddUint64(rcd.errored, 1) -} - -func (rcd *rpcCountData) loadAndClearErrored() uint64 { - return atomic.SwapUint64(rcd.errored, 0) -} - -func (rcd *rpcCountData) incrInProgress() { - atomic.AddUint64(rcd.inProgress, 1) -} - -func (rcd *rpcCountData) decrInProgress() { - atomic.AddUint64(rcd.inProgress, negativeOneUInt64) // atomic.Add(x, -1) -} - -func (rcd *rpcCountData) loadInProgress() uint64 { - return atomic.LoadUint64(rcd.inProgress) // InProgress count is not clear when reading. -} - -func (rcd *rpcCountData) addServerLoad(name string, d float64) { - loads, ok := rcd.serverLoads.Load(name) - if !ok { - tl := newRPCLoadData() - loads, _ = rcd.serverLoads.LoadOrStore(name, tl) - } - loads.(*rpcLoadData).add(d) -} - -// Data for server loads (from trailers or oob). Fields in this struct must be -// updated consistently. -// -// The current solution is to hold a lock, which could cause contention. To fix, -// shard serverLoads map in rpcCountData. -type rpcLoadData struct { - mu sync.Mutex - sum float64 - count uint64 -} - -func newRPCLoadData() *rpcLoadData { - return &rpcLoadData{} -} - -func (rld *rpcLoadData) add(v float64) { - rld.mu.Lock() - rld.sum += v - rld.count++ - rld.mu.Unlock() -} - -func (rld *rpcLoadData) loadAndClear() (s float64, c uint64) { - rld.mu.Lock() - s = rld.sum - rld.sum = 0 - c = rld.count - rld.count = 0 - rld.mu.Unlock() - return -} - -// lrsStore collects loads from xds balancer, and periodically sends load to the -// server. -type lrsStore struct { - backoff backoff.Strategy - lastReported time.Time - - drops sync.Map // map[string]*uint64 - localityRPCCount sync.Map // map[internal.LocalityID]*rpcCountData -} - -// NewStore creates a store for load reports. -func NewStore() Store { - return &lrsStore{ - backoff: backoff.DefaultExponential, - lastReported: time.Now(), - } -} - -// Update functions are called by picker for each RPC. To avoid contention, all -// updates are done atomically. - -// CallDropped adds one drop record with the given category to store. -func (ls *lrsStore) CallDropped(category string) { - p, ok := ls.drops.Load(category) - if !ok { - tp := new(uint64) - p, _ = ls.drops.LoadOrStore(category, tp) - } - atomic.AddUint64(p.(*uint64), 1) -} - -func (ls *lrsStore) CallStarted(l internal.LocalityID) { - p, ok := ls.localityRPCCount.Load(l) - if !ok { - tp := newRPCCountData() - p, _ = ls.localityRPCCount.LoadOrStore(l, tp) - } - p.(*rpcCountData).incrInProgress() -} - -func (ls *lrsStore) CallFinished(l internal.LocalityID, err error) { - p, ok := ls.localityRPCCount.Load(l) - if !ok { - // The map is never cleared, only values in the map are reset. So the - // case where entry for call-finish is not found should never happen. - return - } - p.(*rpcCountData).decrInProgress() - if err == nil { - p.(*rpcCountData).incrSucceeded() - } else { - p.(*rpcCountData).incrErrored() - } -} - -func (ls *lrsStore) CallServerLoad(l internal.LocalityID, name string, d float64) { - p, ok := ls.localityRPCCount.Load(l) - if !ok { - // The map is never cleared, only values in the map are reset. So the - // case where entry for CallServerLoad is not found should never happen. - return - } - p.(*rpcCountData).addServerLoad(name, d) -} - -func (ls *lrsStore) buildStats(clusterName string) []*endpointpb.ClusterStats { - var ( - totalDropped uint64 - droppedReqs []*endpointpb.ClusterStats_DroppedRequests - localityStats []*endpointpb.UpstreamLocalityStats - ) - ls.drops.Range(func(category, countP interface{}) bool { - tempCount := atomic.SwapUint64(countP.(*uint64), 0) - if tempCount == 0 { - return true - } - totalDropped += tempCount - droppedReqs = append(droppedReqs, &endpointpb.ClusterStats_DroppedRequests{ - Category: category.(string), - DroppedCount: tempCount, - }) - return true - }) - ls.localityRPCCount.Range(func(locality, countP interface{}) bool { - tempLocality := locality.(internal.LocalityID) - tempCount := countP.(*rpcCountData) - - tempSucceeded := tempCount.loadAndClearSucceeded() - tempInProgress := tempCount.loadInProgress() - tempErrored := tempCount.loadAndClearErrored() - if tempSucceeded == 0 && tempInProgress == 0 && tempErrored == 0 { - return true - } - - var loadMetricStats []*endpointpb.EndpointLoadMetricStats - tempCount.serverLoads.Range(func(name, data interface{}) bool { - tempName := name.(string) - tempSum, tempCount := data.(*rpcLoadData).loadAndClear() - if tempCount == 0 { - return true - } - loadMetricStats = append(loadMetricStats, - &endpointpb.EndpointLoadMetricStats{ - MetricName: tempName, - NumRequestsFinishedWithMetric: tempCount, - TotalMetricValue: tempSum, - }, - ) - return true - }) - - localityStats = append(localityStats, &endpointpb.UpstreamLocalityStats{ - Locality: &corepb.Locality{ - Region: tempLocality.Region, - Zone: tempLocality.Zone, - SubZone: tempLocality.SubZone, - }, - TotalSuccessfulRequests: tempSucceeded, - TotalRequestsInProgress: tempInProgress, - TotalErrorRequests: tempErrored, - LoadMetricStats: loadMetricStats, - UpstreamEndpointStats: nil, // TODO: populate for per endpoint loads. - }) - return true - }) - - dur := time.Since(ls.lastReported) - ls.lastReported = time.Now() - - var ret []*endpointpb.ClusterStats - ret = append(ret, &endpointpb.ClusterStats{ - ClusterName: clusterName, - UpstreamLocalityStats: localityStats, - - TotalDroppedRequests: totalDropped, - DroppedRequests: droppedReqs, - LoadReportInterval: ptypes.DurationProto(dur), - }) - - return ret -} - -// ReportTo makes a streaming lrs call to cc and blocks. -// -// It retries the call (with backoff) until ctx is canceled. -func (ls *lrsStore) ReportTo(ctx context.Context, cc *grpc.ClientConn, clusterName string, node *corepb.Node) { - c := lrsgrpc.NewLoadReportingServiceClient(cc) - var ( - retryCount int - doBackoff bool - ) - for { - select { - case <-ctx.Done(): - return - default: - } - - if doBackoff { - backoffTimer := time.NewTimer(ls.backoff.Backoff(retryCount)) - select { - case <-backoffTimer.C: - case <-ctx.Done(): - backoffTimer.Stop() - return - } - retryCount++ - } - - doBackoff = true - stream, err := c.StreamLoadStats(ctx) - if err != nil { - logger.Warningf("lrs: failed to create stream: %v", err) - continue - } - logger.Infof("lrs: created LRS stream") - req := &lrspb.LoadStatsRequest{Node: node} - logger.Infof("lrs: sending init LoadStatsRequest: %v", req) - if err := stream.Send(req); err != nil { - logger.Warningf("lrs: failed to send first request: %v", err) - continue - } - first, err := stream.Recv() - if err != nil { - logger.Warningf("lrs: failed to receive first response: %v", err) - continue - } - logger.Infof("lrs: received first LoadStatsResponse: %+v", first) - interval, err := ptypes.Duration(first.LoadReportingInterval) - if err != nil { - logger.Warningf("lrs: failed to convert report interval: %v", err) - continue - } - // The LRS client should join the clusters it knows with the cluster - // list from response, and send loads for them. - // - // But the LRS client now only supports one cluster. TODO: extend it to - // support multiple clusters. - var clusterFoundInResponse bool - for _, c := range first.Clusters { - if c == clusterName { - clusterFoundInResponse = true - } - } - if !clusterFoundInResponse { - logger.Warningf("lrs: received clusters %v does not contain expected {%v}", first.Clusters, clusterName) - continue - } - if first.ReportEndpointGranularity { - // TODO: fixme to support per endpoint loads. - logger.Warningf("lrs: endpoint loads requested, but not supported by current implementation") - continue - } - - // No backoff afterwards. - doBackoff = false - retryCount = 0 - ls.sendLoads(ctx, stream, clusterName, interval) - } -} - -func (ls *lrsStore) sendLoads(ctx context.Context, stream lrsgrpc.LoadReportingService_StreamLoadStatsClient, clusterName string, interval time.Duration) { - tick := time.NewTicker(interval) - defer tick.Stop() - for { - select { - case <-tick.C: - case <-ctx.Done(): - return - } - req := &lrspb.LoadStatsRequest{ClusterStats: ls.buildStats(clusterName)} - logger.Infof("lrs: sending LRS loads: %+v", req) - if err := stream.Send(req); err != nil { - logger.Warningf("lrs: failed to send report: %v", err) - return - } - } -} diff --git a/xds/internal/balancer/lrs/store_test.go b/xds/internal/balancer/lrs/store_test.go deleted file mode 100644 index b18c3d7e218d..000000000000 --- a/xds/internal/balancer/lrs/store_test.go +++ /dev/null @@ -1,517 +0,0 @@ -/* - * - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package lrs - -import ( - "context" - "fmt" - "io" - "net" - "sort" - "sync" - "testing" - "time" - - corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" - endpointpb "github.com/envoyproxy/go-control-plane/envoy/api/v2/endpoint" - lrsgrpc "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v2" - lrspb "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v2" - "github.com/golang/protobuf/proto" - durationpb "github.com/golang/protobuf/ptypes/duration" - structpb "github.com/golang/protobuf/ptypes/struct" - "github.com/google/go-cmp/cmp" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - "google.golang.org/grpc/xds/internal" - "google.golang.org/grpc/xds/internal/testutils" -) - -const ( - testService = "grpc.service.test" - testHostname = "grpc.server.name" - nodeMetadataHostnameKey = "PROXYLESS_CLIENT_HOSTNAME" -) - -var ( - dropCategories = []string{"drop_for_real", "drop_for_fun"} - localities = []internal.LocalityID{{Region: "a"}, {Region: "b"}} - errTest = fmt.Errorf("test error") -) - -type rpcCountDataForTest struct { - succeeded uint64 - errored uint64 - inProgress uint64 - serverLoads map[string]float64 -} - -func newRPCCountDataForTest(succeeded, errored, inprogress uint64, serverLoads map[string]float64) *rpcCountDataForTest { - return &rpcCountDataForTest{ - succeeded: succeeded, - errored: errored, - inProgress: inprogress, - serverLoads: serverLoads, - } -} - -// Equal() is needed to compare unexported fields. -func (rcd *rpcCountDataForTest) Equal(b *rpcCountDataForTest) bool { - return rcd.inProgress == b.inProgress && - rcd.errored == b.errored && - rcd.succeeded == b.succeeded && - cmp.Equal(rcd.serverLoads, b.serverLoads) -} - -// equalClusterStats sorts requests and clear report internal before comparing. -func equalClusterStats(a, b []*endpointpb.ClusterStats) bool { - for _, t := range [][]*endpointpb.ClusterStats{a, b} { - for _, s := range t { - sort.Slice(s.DroppedRequests, func(i, j int) bool { - return s.DroppedRequests[i].Category < s.DroppedRequests[j].Category - }) - sort.Slice(s.UpstreamLocalityStats, func(i, j int) bool { - return s.UpstreamLocalityStats[i].Locality.String() < s.UpstreamLocalityStats[j].Locality.String() - }) - for _, us := range s.UpstreamLocalityStats { - sort.Slice(us.LoadMetricStats, func(i, j int) bool { - return us.LoadMetricStats[i].MetricName < us.LoadMetricStats[j].MetricName - }) - } - s.LoadReportInterval = nil - } - } - return cmp.Equal(a, b, cmp.Comparer(proto.Equal)) -} - -func Test_lrsStore_buildStats_drops(t *testing.T) { - tests := []struct { - name string - drops []map[string]uint64 - }{ - { - name: "one drop report", - drops: []map[string]uint64{{ - dropCategories[0]: 31, - dropCategories[1]: 41, - }}, - }, - { - name: "two drop reports", - drops: []map[string]uint64{{ - dropCategories[0]: 31, - dropCategories[1]: 41, - }, { - dropCategories[0]: 59, - dropCategories[1]: 26, - }}, - }, - { - name: "no empty report", - drops: []map[string]uint64{{ - dropCategories[0]: 31, - dropCategories[1]: 41, - }, { - dropCategories[0]: 0, // This shouldn't cause an empty report for category[0]. - dropCategories[1]: 26, - }}, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ls := NewStore().(*lrsStore) - - for _, ds := range tt.drops { - var ( - totalDropped uint64 - droppedReqs []*endpointpb.ClusterStats_DroppedRequests - ) - for cat, count := range ds { - if count == 0 { - continue - } - totalDropped += count - droppedReqs = append(droppedReqs, &endpointpb.ClusterStats_DroppedRequests{ - Category: cat, - DroppedCount: count, - }) - } - want := []*endpointpb.ClusterStats{ - { - ClusterName: testService, - TotalDroppedRequests: totalDropped, - DroppedRequests: droppedReqs, - }, - } - - var wg sync.WaitGroup - for c, count := range ds { - for i := 0; i < int(count); i++ { - wg.Add(1) - go func(i int, c string) { - ls.CallDropped(c) - wg.Done() - }(i, c) - } - } - wg.Wait() - - if got := ls.buildStats(testService); !equalClusterStats(got, want) { - t.Errorf("lrsStore.buildStats() = %v, want %v", got, want) - t.Errorf("%s", cmp.Diff(got, want)) - } - } - }) - } -} - -func Test_lrsStore_buildStats_rpcCounts(t *testing.T) { - tests := []struct { - name string - rpcs []map[internal.LocalityID]struct { - start, success, failure uint64 - serverData map[string]float64 // Will be reported with successful RPCs. - } - }{ - { - name: "one rpcCount report", - rpcs: []map[internal.LocalityID]struct { - start, success, failure uint64 - serverData map[string]float64 - }{{ - localities[0]: {8, 3, 1, nil}, - }}, - }, - { - name: "two localities one rpcCount report", - rpcs: []map[internal.LocalityID]struct { - start, success, failure uint64 - serverData map[string]float64 - }{{ - localities[0]: {8, 3, 1, nil}, - localities[1]: {15, 1, 5, nil}, - }}, - }, - { - name: "three rpcCount reports", - rpcs: []map[internal.LocalityID]struct { - start, success, failure uint64 - serverData map[string]float64 - }{{ - localities[0]: {8, 3, 1, nil}, - localities[1]: {15, 1, 5, nil}, - }, { - localities[0]: {8, 3, 1, nil}, - }, { - localities[1]: {15, 1, 5, nil}, - }}, - }, - { - name: "no empty report", - rpcs: []map[internal.LocalityID]struct { - start, success, failure uint64 - serverData map[string]float64 - }{{ - localities[0]: {4, 3, 1, nil}, - localities[1]: {7, 1, 5, nil}, - }, { - localities[0]: {0, 0, 0, nil}, // This shouldn't cause an empty report for locality[0]. - localities[1]: {1, 1, 0, nil}, - }}, - }, - { - name: "two localities one report with server loads", - rpcs: []map[internal.LocalityID]struct { - start, success, failure uint64 - serverData map[string]float64 - }{{ - localities[0]: {8, 3, 1, map[string]float64{"cpu": 15, "mem": 20}}, - localities[1]: {15, 4, 5, map[string]float64{"net": 5, "disk": 0.8}}, - }}, - }, - { - name: "three reports with server loads", - rpcs: []map[internal.LocalityID]struct { - start, success, failure uint64 - serverData map[string]float64 - }{{ - localities[0]: {8, 3, 1, map[string]float64{"cpu": 15, "mem": 20}}, - localities[1]: {15, 4, 5, map[string]float64{"net": 5, "disk": 0.8}}, - }, { - localities[0]: {8, 3, 1, map[string]float64{"cpu": 1, "mem": 2}}, - }, { - localities[1]: {15, 4, 5, map[string]float64{"net": 13, "disk": 1.4}}, - }}, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ls := NewStore().(*lrsStore) - - // InProgress count doesn't get cleared at each buildStats, keep - // them to carry over. - inProgressCounts := make(map[internal.LocalityID]uint64) - - for _, counts := range tt.rpcs { - var upstreamLocalityStats []*endpointpb.UpstreamLocalityStats - - for l, count := range counts { - tempInProgress := count.start - count.success - count.failure + inProgressCounts[l] - inProgressCounts[l] = tempInProgress - if count.success == 0 && tempInProgress == 0 && count.failure == 0 { - continue - } - var loadMetricStats []*endpointpb.EndpointLoadMetricStats - for n, d := range count.serverData { - loadMetricStats = append(loadMetricStats, - &endpointpb.EndpointLoadMetricStats{ - MetricName: n, - NumRequestsFinishedWithMetric: count.success, - TotalMetricValue: d * float64(count.success), - }, - ) - } - upstreamLocalityStats = append(upstreamLocalityStats, &endpointpb.UpstreamLocalityStats{ - Locality: testutils.LocalityIDToProto(l), - TotalSuccessfulRequests: count.success, - TotalRequestsInProgress: tempInProgress, - TotalErrorRequests: count.failure, - LoadMetricStats: loadMetricStats, - }) - } - // InProgress count doesn't get cleared at each buildStats, and - // needs to be carried over to the next result. - for l, c := range inProgressCounts { - if _, ok := counts[l]; !ok { - upstreamLocalityStats = append(upstreamLocalityStats, &endpointpb.UpstreamLocalityStats{ - Locality: testutils.LocalityIDToProto(l), - TotalRequestsInProgress: c, - }) - } - } - want := []*endpointpb.ClusterStats{ - { - ClusterName: testService, - UpstreamLocalityStats: upstreamLocalityStats, - }, - } - - var wg sync.WaitGroup - for l, count := range counts { - for i := 0; i < int(count.success); i++ { - wg.Add(1) - go func(l internal.LocalityID, serverData map[string]float64) { - ls.CallStarted(l) - ls.CallFinished(l, nil) - for n, d := range serverData { - ls.CallServerLoad(l, n, d) - } - wg.Done() - }(l, count.serverData) - } - for i := 0; i < int(count.failure); i++ { - wg.Add(1) - go func(l internal.LocalityID) { - ls.CallStarted(l) - ls.CallFinished(l, errTest) - wg.Done() - }(l) - } - for i := 0; i < int(count.start-count.success-count.failure); i++ { - wg.Add(1) - go func(l internal.LocalityID) { - ls.CallStarted(l) - wg.Done() - }(l) - } - } - wg.Wait() - - if got := ls.buildStats(testService); !equalClusterStats(got, want) { - t.Errorf("lrsStore.buildStats() = %v, want %v", got, want) - t.Errorf("%s", cmp.Diff(got, want)) - } - } - }) - } -} - -type lrsServer struct { - reportingInterval *durationpb.Duration - - mu sync.Mutex - dropTotal uint64 - drops map[string]uint64 - rpcs map[internal.LocalityID]*rpcCountDataForTest -} - -func (lrss *lrsServer) StreamLoadStats(stream lrsgrpc.LoadReportingService_StreamLoadStatsServer) error { - req, err := stream.Recv() - if err != nil { - return err - } - - if req.GetNode().GetMetadata().GetFields()[nodeMetadataHostnameKey].GetStringValue() != testHostname { - return status.Errorf(codes.FailedPrecondition, "unexpected req: %+v", req) - } - if err := stream.Send(&lrspb.LoadStatsResponse{ - Clusters: []string{testService, "another-cluster"}, - LoadReportingInterval: lrss.reportingInterval, - }); err != nil { - return err - } - - for { - req, err := stream.Recv() - if err != nil { - if err == io.EOF { - return nil - } - return err - } - stats := req.ClusterStats[0] - lrss.mu.Lock() - lrss.dropTotal += stats.TotalDroppedRequests - for _, d := range stats.DroppedRequests { - lrss.drops[d.Category] += d.DroppedCount - } - for _, ss := range stats.UpstreamLocalityStats { - l := internal.LocalityID{ - Region: ss.Locality.Region, - Zone: ss.Locality.Zone, - SubZone: ss.Locality.SubZone, - } - counts, ok := lrss.rpcs[l] - if !ok { - counts = newRPCCountDataForTest(0, 0, 0, nil) - lrss.rpcs[l] = counts - } - counts.succeeded += ss.TotalSuccessfulRequests - counts.inProgress = ss.TotalRequestsInProgress - counts.errored += ss.TotalErrorRequests - for _, ts := range ss.LoadMetricStats { - if counts.serverLoads == nil { - counts.serverLoads = make(map[string]float64) - } - counts.serverLoads[ts.MetricName] = ts.TotalMetricValue / float64(ts.NumRequestsFinishedWithMetric) - } - } - lrss.mu.Unlock() - } -} - -func setupServer(t *testing.T, reportingInterval *durationpb.Duration) (addr string, lrss *lrsServer, cleanup func()) { - lis, err := net.Listen("tcp", "localhost:0") - if err != nil { - t.Fatalf("listen failed due to: %v", err) - } - svr := grpc.NewServer() - lrss = &lrsServer{ - reportingInterval: reportingInterval, - drops: make(map[string]uint64), - rpcs: make(map[internal.LocalityID]*rpcCountDataForTest), - } - lrsgrpc.RegisterLoadReportingServiceServer(svr, lrss) - go svr.Serve(lis) - return lis.Addr().String(), lrss, func() { - svr.Stop() - lis.Close() - } -} - -func Test_lrsStore_ReportTo(t *testing.T) { - const intervalNano = 1000 * 1000 * 50 - addr, lrss, cleanup := setupServer(t, &durationpb.Duration{ - Seconds: 0, - Nanos: intervalNano, - }) - defer cleanup() - - ls := NewStore() - cc, err := grpc.Dial(addr, grpc.WithInsecure()) - if err != nil { - t.Fatalf("failed to dial: %v", err) - } - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - done := make(chan struct{}) - go func() { - node := &corepb.Node{ - Metadata: &structpb.Struct{ - Fields: map[string]*structpb.Value{ - nodeMetadataHostnameKey: { - Kind: &structpb.Value_StringValue{StringValue: testHostname}, - }, - }, - }, - } - ls.ReportTo(ctx, cc, testService, node) - close(done) - }() - - drops := map[string]uint64{ - dropCategories[0]: 13, - dropCategories[1]: 14, - } - for c, d := range drops { - for i := 0; i < int(d); i++ { - ls.CallDropped(c) - time.Sleep(time.Nanosecond * intervalNano / 10) - } - } - - rpcs := map[internal.LocalityID]*rpcCountDataForTest{ - localities[0]: newRPCCountDataForTest(3, 1, 4, nil), - localities[1]: newRPCCountDataForTest(1, 5, 9, map[string]float64{"pi": 3.14, "e": 2.71}), - } - for l, count := range rpcs { - for i := 0; i < int(count.succeeded); i++ { - go func(i int, l internal.LocalityID, count *rpcCountDataForTest) { - ls.CallStarted(l) - ls.CallFinished(l, nil) - for n, d := range count.serverLoads { - ls.CallServerLoad(l, n, d) - } - }(i, l, count) - } - for i := 0; i < int(count.inProgress); i++ { - go func(i int, l internal.LocalityID) { - ls.CallStarted(l) - }(i, l) - } - for i := 0; i < int(count.errored); i++ { - go func(i int, l internal.LocalityID) { - ls.CallStarted(l) - ls.CallFinished(l, errTest) - }(i, l) - } - } - - time.Sleep(time.Nanosecond * intervalNano * 2) - cancel() - <-done - - lrss.mu.Lock() - defer lrss.mu.Unlock() - if !cmp.Equal(lrss.drops, drops) { - t.Errorf("different: %v", cmp.Diff(lrss.drops, drops)) - } - if !cmp.Equal(lrss.rpcs, rpcs) { - t.Errorf("different: %v", cmp.Diff(lrss.rpcs, rpcs)) - } -} diff --git a/xds/internal/balancer/orca/orca.go b/xds/internal/balancer/orca/orca.go deleted file mode 100644 index 28016806eec4..000000000000 --- a/xds/internal/balancer/orca/orca.go +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Package orca implements Open Request Cost Aggregation. -package orca - -import ( - orcapb "github.com/cncf/udpa/go/udpa/data/orca/v1" - "github.com/golang/protobuf/proto" - "google.golang.org/grpc/grpclog" - "google.golang.org/grpc/internal/balancerload" - "google.golang.org/grpc/metadata" -) - -const mdKey = "X-Endpoint-Load-Metrics-Bin" - -var logger = grpclog.Component("xds") - -// toBytes converts a orca load report into bytes. -func toBytes(r *orcapb.OrcaLoadReport) []byte { - if r == nil { - return nil - } - - b, err := proto.Marshal(r) - if err != nil { - logger.Warningf("orca: failed to marshal load report: %v", err) - return nil - } - return b -} - -// ToMetadata converts a orca load report into grpc metadata. -func ToMetadata(r *orcapb.OrcaLoadReport) metadata.MD { - b := toBytes(r) - if b == nil { - return nil - } - return metadata.Pairs(mdKey, string(b)) -} - -// fromBytes reads load report bytes and converts it to orca. -func fromBytes(b []byte) *orcapb.OrcaLoadReport { - ret := new(orcapb.OrcaLoadReport) - if err := proto.Unmarshal(b, ret); err != nil { - logger.Warningf("orca: failed to unmarshal load report: %v", err) - return nil - } - return ret -} - -// FromMetadata reads load report from metadata and converts it to orca. -// -// It returns nil if report is not found in metadata. -func FromMetadata(md metadata.MD) *orcapb.OrcaLoadReport { - vs := md.Get(mdKey) - if len(vs) == 0 { - return nil - } - return fromBytes([]byte(vs[0])) -} - -type loadParser struct{} - -func (*loadParser) Parse(md metadata.MD) interface{} { - return FromMetadata(md) -} - -func init() { - balancerload.SetParser(&loadParser{}) -} diff --git a/xds/internal/balancer/orca/orca_test.go b/xds/internal/balancer/orca/orca_test.go deleted file mode 100644 index d7a44134e22b..000000000000 --- a/xds/internal/balancer/orca/orca_test.go +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package orca - -import ( - "strings" - "testing" - - orcapb "github.com/cncf/udpa/go/udpa/data/orca/v1" - "github.com/golang/protobuf/proto" - "github.com/google/go-cmp/cmp" - "google.golang.org/grpc/internal/grpctest" - "google.golang.org/grpc/metadata" -) - -var ( - testMessage = &orcapb.OrcaLoadReport{ - CpuUtilization: 0.1, - MemUtilization: 0.2, - RequestCost: map[string]float64{"ccc": 3.4}, - Utilization: map[string]float64{"ttt": 0.4}, - } - testBytes, _ = proto.Marshal(testMessage) -) - -type s struct { - grpctest.Tester -} - -func Test(t *testing.T) { - grpctest.RunSubTests(t, s{}) -} - -func (s) TestToMetadata(t *testing.T) { - tests := []struct { - name string - r *orcapb.OrcaLoadReport - want metadata.MD - }{{ - name: "nil", - r: nil, - want: nil, - }, { - name: "valid", - r: testMessage, - want: metadata.MD{ - strings.ToLower(mdKey): []string{string(testBytes)}, - }, - }} - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := ToMetadata(tt.r); !cmp.Equal(got, tt.want) { - t.Errorf("ToMetadata() = %v, want %v", got, tt.want) - } - }) - } -} - -func (s) TestFromMetadata(t *testing.T) { - tests := []struct { - name string - md metadata.MD - want *orcapb.OrcaLoadReport - }{{ - name: "nil", - md: nil, - want: nil, - }, { - name: "valid", - md: metadata.MD{ - strings.ToLower(mdKey): []string{string(testBytes)}, - }, - want: testMessage, - }} - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := FromMetadata(tt.md); !cmp.Equal(got, tt.want, cmp.Comparer(proto.Equal)) { - t.Errorf("FromMetadata() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/xds/internal/balancer/outlierdetection/balancer.go b/xds/internal/balancer/outlierdetection/balancer.go new file mode 100644 index 000000000000..965297a73dbc --- /dev/null +++ b/xds/internal/balancer/outlierdetection/balancer.go @@ -0,0 +1,935 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package outlierdetection provides an implementation of the outlier detection +// LB policy, as defined in +// https://github.com/grpc/proposal/blob/master/A50-xds-outlier-detection.md. +package outlierdetection + +import ( + "encoding/json" + "fmt" + "math" + "strings" + "sync" + "sync/atomic" + "time" + "unsafe" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/internal/balancer/gracefulswitch" + "google.golang.org/grpc/internal/buffer" + "google.golang.org/grpc/internal/channelz" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/grpcrand" + "google.golang.org/grpc/internal/grpcsync" + iserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" +) + +// Globals to stub out in tests. +var ( + afterFunc = time.AfterFunc + now = time.Now +) + +// Name is the name of the outlier detection balancer. +const Name = "outlier_detection_experimental" + +func init() { + if envconfig.XDSOutlierDetection { + balancer.Register(bb{}) + } +} + +type bb struct{} + +func (bb) Build(cc balancer.ClientConn, bOpts balancer.BuildOptions) balancer.Balancer { + b := &outlierDetectionBalancer{ + cc: cc, + closed: grpcsync.NewEvent(), + done: grpcsync.NewEvent(), + addrs: make(map[string]*addressInfo), + scWrappers: make(map[balancer.SubConn]*subConnWrapper), + scUpdateCh: buffer.NewUnbounded(), + pickerUpdateCh: buffer.NewUnbounded(), + channelzParentID: bOpts.ChannelzParentID, + } + b.logger = prefixLogger(b) + b.logger.Infof("Created") + b.child = gracefulswitch.NewBalancer(b, bOpts) + go b.run() + return b +} + +func (bb) ParseConfig(s json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + lbCfg := &LBConfig{ + // Default top layer values as documented in A50. + Interval: iserviceconfig.Duration(10 * time.Second), + BaseEjectionTime: iserviceconfig.Duration(30 * time.Second), + MaxEjectionTime: iserviceconfig.Duration(300 * time.Second), + MaxEjectionPercent: 10, + } + + // This unmarshalling handles underlying layers sre and fpe which have their + // own defaults for their fields if either sre or fpe are present. + if err := json.Unmarshal(s, lbCfg); err != nil { // Validates child config if present as well. + return nil, fmt.Errorf("xds: unable to unmarshal LBconfig: %s, error: %v", string(s), err) + } + + // Note: in the xds flow, these validations will never fail. The xdsclient + // performs the same validations as here on the xds Outlier Detection + // resource before parsing resource into JSON which this function gets + // called with. A50 defines two separate places for these validations to + // take place, the xdsclient and this ParseConfig method. "When parsing a + // config from JSON, if any of these requirements is violated, that should + // be treated as a parsing error." - A50 + switch { + // "The google.protobuf.Duration fields interval, base_ejection_time, and + // max_ejection_time must obey the restrictions in the + // google.protobuf.Duration documentation and they must have non-negative + // values." - A50 + // Approximately 290 years is the maximum time that time.Duration (int64) + // can represent. The restrictions on the protobuf.Duration field are to be + // within +-10000 years. Thus, just check for negative values. + case lbCfg.Interval < 0: + return nil, fmt.Errorf("OutlierDetectionLoadBalancingConfig.interval = %s; must be >= 0", lbCfg.Interval) + case lbCfg.BaseEjectionTime < 0: + return nil, fmt.Errorf("OutlierDetectionLoadBalancingConfig.base_ejection_time = %s; must be >= 0", lbCfg.BaseEjectionTime) + case lbCfg.MaxEjectionTime < 0: + return nil, fmt.Errorf("OutlierDetectionLoadBalancingConfig.max_ejection_time = %s; must be >= 0", lbCfg.MaxEjectionTime) + + // "The fields max_ejection_percent, + // success_rate_ejection.enforcement_percentage, + // failure_percentage_ejection.threshold, and + // failure_percentage.enforcement_percentage must have values less than or + // equal to 100." - A50 + case lbCfg.MaxEjectionPercent > 100: + return nil, fmt.Errorf("OutlierDetectionLoadBalancingConfig.max_ejection_percent = %v; must be <= 100", lbCfg.MaxEjectionPercent) + case lbCfg.SuccessRateEjection != nil && lbCfg.SuccessRateEjection.EnforcementPercentage > 100: + return nil, fmt.Errorf("OutlierDetectionLoadBalancingConfig.SuccessRateEjection.enforcement_percentage = %v; must be <= 100", lbCfg.SuccessRateEjection.EnforcementPercentage) + case lbCfg.FailurePercentageEjection != nil && lbCfg.FailurePercentageEjection.Threshold > 100: + return nil, fmt.Errorf("OutlierDetectionLoadBalancingConfig.FailurePercentageEjection.threshold = %v; must be <= 100", lbCfg.FailurePercentageEjection.Threshold) + case lbCfg.FailurePercentageEjection != nil && lbCfg.FailurePercentageEjection.EnforcementPercentage > 100: + return nil, fmt.Errorf("OutlierDetectionLoadBalancingConfig.FailurePercentageEjection.enforcement_percentage = %v; must be <= 100", lbCfg.FailurePercentageEjection.EnforcementPercentage) + } + return lbCfg, nil +} + +func (bb) Name() string { + return Name +} + +// scUpdate wraps a subConn update to be sent to the child balancer. +type scUpdate struct { + scw *subConnWrapper + state balancer.SubConnState +} + +type ejectionUpdate struct { + scw *subConnWrapper + isEjected bool // true for ejected, false for unejected +} + +type lbCfgUpdate struct { + lbCfg *LBConfig + // to make sure picker is updated synchronously. + done chan struct{} +} + +type outlierDetectionBalancer struct { + // These fields are safe to be accessed without holding any mutex because + // they are synchronized in run(), which makes these field accesses happen + // serially. + // + // childState is the latest balancer state received from the child. + childState balancer.State + // recentPickerNoop represents whether the most recent picker sent upward to + // the balancer.ClientConn is a noop picker, which doesn't count RPC's. Used + // to suppress redundant picker updates. + recentPickerNoop bool + + closed *grpcsync.Event + done *grpcsync.Event + cc balancer.ClientConn + logger *grpclog.PrefixLogger + channelzParentID *channelz.Identifier + + // childMu guards calls into child (to uphold the balancer.Balancer API + // guarantee of synchronous calls). + childMu sync.Mutex + child *gracefulswitch.Balancer + + // mu guards access to the following fields. It also helps to synchronize + // behaviors of the following events: config updates, firing of the interval + // timer, SubConn State updates, SubConn address updates, and child state + // updates. + // + // For example, when we receive a config update in the middle of the + // interval timer algorithm, which uses knobs present in the config, the + // balancer will wait for the interval timer algorithm to finish before + // persisting the new configuration. + // + // Another example would be the updating of the addrs map, such as from a + // SubConn address update in the middle of the interval timer algorithm + // which uses addrs. This balancer waits for the interval timer algorithm to + // finish before making the update to the addrs map. + // + // This mutex is never held at the same time as childMu (within the context + // of a single goroutine). + mu sync.Mutex + addrs map[string]*addressInfo + cfg *LBConfig + scWrappers map[balancer.SubConn]*subConnWrapper + timerStartTime time.Time + intervalTimer *time.Timer + inhibitPickerUpdates bool + updateUnconditionally bool + numAddrsEjected int // For fast calculations of percentage of addrs ejected + + scUpdateCh *buffer.Unbounded + pickerUpdateCh *buffer.Unbounded +} + +// noopConfig returns whether this balancer is configured with a logical no-op +// configuration or not. +// +// Caller must hold b.mu. +func (b *outlierDetectionBalancer) noopConfig() bool { + return b.cfg.SuccessRateEjection == nil && b.cfg.FailurePercentageEjection == nil +} + +// onIntervalConfig handles logic required specifically on the receipt of a +// configuration which specifies to count RPC's and periodically perform passive +// health checking based on heuristics defined in configuration every configured +// interval. +// +// Caller must hold b.mu. +func (b *outlierDetectionBalancer) onIntervalConfig() { + var interval time.Duration + if b.timerStartTime.IsZero() { + b.timerStartTime = time.Now() + for _, addrInfo := range b.addrs { + addrInfo.callCounter.clear() + } + interval = time.Duration(b.cfg.Interval) + } else { + interval = time.Duration(b.cfg.Interval) - now().Sub(b.timerStartTime) + if interval < 0 { + interval = 0 + } + } + b.intervalTimer = afterFunc(interval, b.intervalTimerAlgorithm) +} + +// onNoopConfig handles logic required specifically on the receipt of a +// configuration which specifies the balancer to be a noop. +// +// Caller must hold b.mu. +func (b *outlierDetectionBalancer) onNoopConfig() { + // "If a config is provided with both the `success_rate_ejection` and + // `failure_percentage_ejection` fields unset, skip starting the timer and + // do the following:" + // "Unset the timer start timestamp." + b.timerStartTime = time.Time{} + for _, addrInfo := range b.addrs { + // "Uneject all currently ejected addresses." + if !addrInfo.latestEjectionTimestamp.IsZero() { + b.unejectAddress(addrInfo) + } + // "Reset each address's ejection time multiplier to 0." + addrInfo.ejectionTimeMultiplier = 0 + } +} + +func (b *outlierDetectionBalancer) UpdateClientConnState(s balancer.ClientConnState) error { + lbCfg, ok := s.BalancerConfig.(*LBConfig) + if !ok { + b.logger.Errorf("received config with unexpected type %T: %v", s.BalancerConfig, s.BalancerConfig) + return balancer.ErrBadResolverState + } + + // Reject whole config if child policy doesn't exist, don't persist it for + // later. + bb := balancer.Get(lbCfg.ChildPolicy.Name) + if bb == nil { + return fmt.Errorf("outlier detection: child balancer %q not registered", lbCfg.ChildPolicy.Name) + } + + // It is safe to read b.cfg here without holding the mutex, as the only + // write to b.cfg happens later in this function. This function is part of + // the balancer.Balancer API, so it is guaranteed to be called in a + // synchronous manner, so it cannot race with this read. + if b.cfg == nil || b.cfg.ChildPolicy.Name != lbCfg.ChildPolicy.Name { + b.childMu.Lock() + err := b.child.SwitchTo(bb) + if err != nil { + b.childMu.Unlock() + return fmt.Errorf("outlier detection: error switching to child of type %q: %v", lbCfg.ChildPolicy.Name, err) + } + b.childMu.Unlock() + } + + b.mu.Lock() + // Inhibit child picker updates until this UpdateClientConnState() call + // completes. If needed, a picker update containing the no-op config bit + // determined from this config and most recent state from the child will be + // sent synchronously upward at the end of this UpdateClientConnState() + // call. + b.inhibitPickerUpdates = true + b.updateUnconditionally = false + b.cfg = lbCfg + + addrs := make(map[string]bool, len(s.ResolverState.Addresses)) + for _, addr := range s.ResolverState.Addresses { + addrs[addr.Addr] = true + if _, ok := b.addrs[addr.Addr]; !ok { + b.addrs[addr.Addr] = newAddressInfo() + } + } + for addr := range b.addrs { + if !addrs[addr] { + delete(b.addrs, addr) + } + } + + if b.intervalTimer != nil { + b.intervalTimer.Stop() + } + + if b.noopConfig() { + b.onNoopConfig() + } else { + b.onIntervalConfig() + } + b.mu.Unlock() + + b.childMu.Lock() + err := b.child.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: s.ResolverState, + BalancerConfig: b.cfg.ChildPolicy.Config, + }) + b.childMu.Unlock() + + done := make(chan struct{}) + b.pickerUpdateCh.Put(lbCfgUpdate{ + lbCfg: lbCfg, + done: done, + }) + <-done + + return err +} + +func (b *outlierDetectionBalancer) ResolverError(err error) { + b.childMu.Lock() + defer b.childMu.Unlock() + b.child.ResolverError(err) +} + +func (b *outlierDetectionBalancer) updateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { + b.mu.Lock() + defer b.mu.Unlock() + scw, ok := b.scWrappers[sc] + if !ok { + // Shouldn't happen if passed down a SubConnWrapper to child on SubConn + // creation. + b.logger.Errorf("UpdateSubConnState called with SubConn that has no corresponding SubConnWrapper") + return + } + if state.ConnectivityState == connectivity.Shutdown { + delete(b.scWrappers, scw.SubConn) + } + b.scUpdateCh.Put(&scUpdate{ + scw: scw, + state: state, + }) +} + +func (b *outlierDetectionBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { + b.logger.Errorf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, state) +} + +func (b *outlierDetectionBalancer) Close() { + b.closed.Fire() + <-b.done.Done() + b.childMu.Lock() + b.child.Close() + b.childMu.Unlock() + + b.scUpdateCh.Close() + b.pickerUpdateCh.Close() + + b.mu.Lock() + defer b.mu.Unlock() + if b.intervalTimer != nil { + b.intervalTimer.Stop() + } +} + +func (b *outlierDetectionBalancer) ExitIdle() { + b.childMu.Lock() + defer b.childMu.Unlock() + b.child.ExitIdle() +} + +// wrappedPicker delegates to the child policy's picker, and when the request +// finishes, it increments the corresponding counter in the map entry referenced +// by the subConnWrapper that was picked. If both the `success_rate_ejection` +// and `failure_percentage_ejection` fields are unset in the configuration, this +// picker will not count. +type wrappedPicker struct { + childPicker balancer.Picker + noopPicker bool +} + +func (wp *wrappedPicker) Pick(info balancer.PickInfo) (balancer.PickResult, error) { + pr, err := wp.childPicker.Pick(info) + if err != nil { + return balancer.PickResult{}, err + } + + done := func(di balancer.DoneInfo) { + if !wp.noopPicker { + incrementCounter(pr.SubConn, di) + } + if pr.Done != nil { + pr.Done(di) + } + } + scw, ok := pr.SubConn.(*subConnWrapper) + if !ok { + // This can never happen, but check is present for defensive + // programming. + logger.Errorf("Picked SubConn from child picker is not a SubConnWrapper") + return balancer.PickResult{ + SubConn: pr.SubConn, + Done: done, + Metadata: pr.Metadata, + }, nil + } + return balancer.PickResult{ + SubConn: scw.SubConn, + Done: done, + Metadata: pr.Metadata, + }, nil +} + +func incrementCounter(sc balancer.SubConn, info balancer.DoneInfo) { + scw, ok := sc.(*subConnWrapper) + if !ok { + // Shouldn't happen, as comes from child + return + } + + // scw.addressInfo and callCounter.activeBucket can be written to + // concurrently (the pointers themselves). Thus, protect the reads here with + // atomics to prevent data corruption. There exists a race in which you read + // the addressInfo or active bucket pointer and then that pointer points to + // deprecated memory. If this goroutine yields the processor, in between + // reading the addressInfo pointer and writing to the active bucket, + // UpdateAddresses can switch the addressInfo the scw points to. Writing to + // an outdated addresses is a very small race and tolerable. After reading + // callCounter.activeBucket in this picker a swap call can concurrently + // change what activeBucket points to. A50 says to swap the pointer, which + // will cause this race to write to deprecated memory the interval timer + // algorithm will never read, which makes this race alright. + addrInfo := (*addressInfo)(atomic.LoadPointer(&scw.addressInfo)) + if addrInfo == nil { + return + } + ab := (*bucket)(atomic.LoadPointer(&addrInfo.callCounter.activeBucket)) + + if info.Err == nil { + atomic.AddUint32(&ab.numSuccesses, 1) + } else { + atomic.AddUint32(&ab.numFailures, 1) + } +} + +func (b *outlierDetectionBalancer) UpdateState(s balancer.State) { + b.pickerUpdateCh.Put(s) +} + +func (b *outlierDetectionBalancer) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) { + var sc balancer.SubConn + oldListener := opts.StateListener + opts.StateListener = func(state balancer.SubConnState) { b.updateSubConnState(sc, state) } + sc, err := b.cc.NewSubConn(addrs, opts) + if err != nil { + return nil, err + } + scw := &subConnWrapper{ + SubConn: sc, + addresses: addrs, + scUpdateCh: b.scUpdateCh, + listener: oldListener, + } + b.mu.Lock() + defer b.mu.Unlock() + b.scWrappers[sc] = scw + if len(addrs) != 1 { + return scw, nil + } + addrInfo, ok := b.addrs[addrs[0].Addr] + if !ok { + return scw, nil + } + addrInfo.sws = append(addrInfo.sws, scw) + atomic.StorePointer(&scw.addressInfo, unsafe.Pointer(addrInfo)) + if !addrInfo.latestEjectionTimestamp.IsZero() { + scw.eject() + } + return scw, nil +} + +func (b *outlierDetectionBalancer) RemoveSubConn(sc balancer.SubConn) { + b.logger.Errorf("RemoveSubConn(%v) called unexpectedly", sc) +} + +// appendIfPresent appends the scw to the address, if the address is present in +// the Outlier Detection balancers address map. Returns nil if not present, and +// the map entry if present. +// +// Caller must hold b.mu. +func (b *outlierDetectionBalancer) appendIfPresent(addr string, scw *subConnWrapper) *addressInfo { + addrInfo, ok := b.addrs[addr] + if !ok { + return nil + } + + addrInfo.sws = append(addrInfo.sws, scw) + atomic.StorePointer(&scw.addressInfo, unsafe.Pointer(addrInfo)) + return addrInfo +} + +// removeSubConnFromAddressesMapEntry removes the scw from its map entry if +// present. +// +// Caller must hold b.mu. +func (b *outlierDetectionBalancer) removeSubConnFromAddressesMapEntry(scw *subConnWrapper) { + addrInfo := (*addressInfo)(atomic.LoadPointer(&scw.addressInfo)) + if addrInfo == nil { + return + } + for i, sw := range addrInfo.sws { + if scw == sw { + addrInfo.sws = append(addrInfo.sws[:i], addrInfo.sws[i+1:]...) + return + } + } +} + +func (b *outlierDetectionBalancer) UpdateAddresses(sc balancer.SubConn, addrs []resolver.Address) { + scw, ok := sc.(*subConnWrapper) + if !ok { + // Return, shouldn't happen if passed up scw + return + } + + b.cc.UpdateAddresses(scw.SubConn, addrs) + b.mu.Lock() + defer b.mu.Unlock() + + // Note that 0 addresses is a valid update/state for a SubConn to be in. + // This is correctly handled by this algorithm (handled as part of a non singular + // old address/new address). + switch { + case len(scw.addresses) == 1 && len(addrs) == 1: // single address to single address + // If the updated address is the same, then there is nothing to do + // past this point. + if scw.addresses[0].Addr == addrs[0].Addr { + return + } + b.removeSubConnFromAddressesMapEntry(scw) + addrInfo := b.appendIfPresent(addrs[0].Addr, scw) + if addrInfo == nil { // uneject unconditionally because could have come from an ejected address + scw.uneject() + break + } + if addrInfo.latestEjectionTimestamp.IsZero() { // relay new updated subconn state + scw.uneject() + } else { + scw.eject() + } + case len(scw.addresses) == 1: // single address to multiple/no addresses + b.removeSubConnFromAddressesMapEntry(scw) + addrInfo := (*addressInfo)(atomic.LoadPointer(&scw.addressInfo)) + if addrInfo != nil { + addrInfo.callCounter.clear() + } + scw.uneject() + case len(addrs) == 1: // multiple/no addresses to single address + addrInfo := b.appendIfPresent(addrs[0].Addr, scw) + if addrInfo != nil && !addrInfo.latestEjectionTimestamp.IsZero() { + scw.eject() + } + } // otherwise multiple/no addresses to multiple/no addresses; ignore + + scw.addresses = addrs +} + +func (b *outlierDetectionBalancer) ResolveNow(opts resolver.ResolveNowOptions) { + b.cc.ResolveNow(opts) +} + +func (b *outlierDetectionBalancer) Target() string { + return b.cc.Target() +} + +func max(x, y time.Duration) time.Duration { + if x < y { + return y + } + return x +} + +func min(x, y time.Duration) time.Duration { + if x < y { + return x + } + return y +} + +// handleSubConnUpdate stores the recent state and forward the update +// if the SubConn is not ejected. +func (b *outlierDetectionBalancer) handleSubConnUpdate(u *scUpdate) { + scw := u.scw + scw.latestState = u.state + if !scw.ejected { + if scw.listener != nil { + b.childMu.Lock() + scw.listener(u.state) + b.childMu.Unlock() + } + } +} + +// handleEjectedUpdate handles any SubConns that get ejected/unejected, and +// forwards the appropriate corresponding subConnState to the child policy. +func (b *outlierDetectionBalancer) handleEjectedUpdate(u *ejectionUpdate) { + scw := u.scw + scw.ejected = u.isEjected + // If scw.latestState has never been written to will default to connectivity + // IDLE, which is fine. + stateToUpdate := scw.latestState + if u.isEjected { + stateToUpdate = balancer.SubConnState{ + ConnectivityState: connectivity.TransientFailure, + } + } + if scw.listener != nil { + b.childMu.Lock() + scw.listener(stateToUpdate) + b.childMu.Unlock() + } +} + +// handleChildStateUpdate forwards the picker update wrapped in a wrapped picker +// with the noop picker bit present. +func (b *outlierDetectionBalancer) handleChildStateUpdate(u balancer.State) { + b.childState = u + b.mu.Lock() + if b.inhibitPickerUpdates { + // If a child's state is updated during the suppression of child + // updates, the synchronous handleLBConfigUpdate function with respect + // to UpdateClientConnState should return a picker unconditionally. + b.updateUnconditionally = true + b.mu.Unlock() + return + } + noopCfg := b.noopConfig() + b.mu.Unlock() + b.recentPickerNoop = noopCfg + b.cc.UpdateState(balancer.State{ + ConnectivityState: b.childState.ConnectivityState, + Picker: &wrappedPicker{ + childPicker: b.childState.Picker, + noopPicker: noopCfg, + }, + }) +} + +// handleLBConfigUpdate compares whether the new config is a noop config or not, +// to the noop bit in the picker if present. It updates the picker if this bit +// changed compared to the picker currently in use. +func (b *outlierDetectionBalancer) handleLBConfigUpdate(u lbCfgUpdate) { + lbCfg := u.lbCfg + noopCfg := lbCfg.SuccessRateEjection == nil && lbCfg.FailurePercentageEjection == nil + // If the child has sent it's first update and this config flips the noop + // bit compared to the most recent picker update sent upward, then a new + // picker with this updated bit needs to be forwarded upward. If a child + // update was received during the suppression of child updates within + // UpdateClientConnState(), then a new picker needs to be forwarded with + // this updated state, irregardless of whether this new configuration flips + // the bit. + if b.childState.Picker != nil && noopCfg != b.recentPickerNoop || b.updateUnconditionally { + b.recentPickerNoop = noopCfg + b.cc.UpdateState(balancer.State{ + ConnectivityState: b.childState.ConnectivityState, + Picker: &wrappedPicker{ + childPicker: b.childState.Picker, + noopPicker: noopCfg, + }, + }) + } + b.inhibitPickerUpdates = false + b.updateUnconditionally = false + close(u.done) +} + +func (b *outlierDetectionBalancer) run() { + defer b.done.Fire() + for { + select { + case update, ok := <-b.scUpdateCh.Get(): + if !ok { + return + } + b.scUpdateCh.Load() + if b.closed.HasFired() { // don't send SubConn updates to child after the balancer has been closed + return + } + switch u := update.(type) { + case *scUpdate: + b.handleSubConnUpdate(u) + case *ejectionUpdate: + b.handleEjectedUpdate(u) + } + case update, ok := <-b.pickerUpdateCh.Get(): + if !ok { + return + } + b.pickerUpdateCh.Load() + if b.closed.HasFired() { // don't send picker updates to grpc after the balancer has been closed + return + } + switch u := update.(type) { + case balancer.State: + b.handleChildStateUpdate(u) + case lbCfgUpdate: + b.handleLBConfigUpdate(u) + } + case <-b.closed.Done(): + return + } + } +} + +// intervalTimerAlgorithm ejects and unejects addresses based on the Outlier +// Detection configuration and data about each address from the previous +// interval. +func (b *outlierDetectionBalancer) intervalTimerAlgorithm() { + b.mu.Lock() + defer b.mu.Unlock() + b.timerStartTime = time.Now() + + for _, addrInfo := range b.addrs { + addrInfo.callCounter.swap() + } + + if b.cfg.SuccessRateEjection != nil { + b.successRateAlgorithm() + } + + if b.cfg.FailurePercentageEjection != nil { + b.failurePercentageAlgorithm() + } + + for _, addrInfo := range b.addrs { + if addrInfo.latestEjectionTimestamp.IsZero() && addrInfo.ejectionTimeMultiplier > 0 { + addrInfo.ejectionTimeMultiplier-- + continue + } + if addrInfo.latestEjectionTimestamp.IsZero() { + // Address is already not ejected, so no need to check for whether + // to uneject the address below. + continue + } + et := time.Duration(b.cfg.BaseEjectionTime) * time.Duration(addrInfo.ejectionTimeMultiplier) + met := max(time.Duration(b.cfg.BaseEjectionTime), time.Duration(b.cfg.MaxEjectionTime)) + uet := addrInfo.latestEjectionTimestamp.Add(min(et, met)) + if now().After(uet) { + b.unejectAddress(addrInfo) + } + } + + // This conditional only for testing (since the interval timer algorithm is + // called manually), will never hit in production. + if b.intervalTimer != nil { + b.intervalTimer.Stop() + } + b.intervalTimer = afterFunc(time.Duration(b.cfg.Interval), b.intervalTimerAlgorithm) +} + +// addrsWithAtLeastRequestVolume returns a slice of address information of all +// addresses with at least request volume passed in. +// +// Caller must hold b.mu. +func (b *outlierDetectionBalancer) addrsWithAtLeastRequestVolume(requestVolume uint32) []*addressInfo { + var addrs []*addressInfo + for _, addrInfo := range b.addrs { + bucket := addrInfo.callCounter.inactiveBucket + rv := bucket.numSuccesses + bucket.numFailures + if rv >= requestVolume { + addrs = append(addrs, addrInfo) + } + } + return addrs +} + +// meanAndStdDev returns the mean and std dev of the fractions of successful +// requests of the addresses passed in. +// +// Caller must hold b.mu. +func (b *outlierDetectionBalancer) meanAndStdDev(addrs []*addressInfo) (float64, float64) { + var totalFractionOfSuccessfulRequests float64 + var mean float64 + for _, addrInfo := range addrs { + bucket := addrInfo.callCounter.inactiveBucket + rv := bucket.numSuccesses + bucket.numFailures + totalFractionOfSuccessfulRequests += float64(bucket.numSuccesses) / float64(rv) + } + mean = totalFractionOfSuccessfulRequests / float64(len(addrs)) + var sumOfSquares float64 + for _, addrInfo := range addrs { + bucket := addrInfo.callCounter.inactiveBucket + rv := bucket.numSuccesses + bucket.numFailures + devFromMean := (float64(bucket.numSuccesses) / float64(rv)) - mean + sumOfSquares += devFromMean * devFromMean + } + variance := sumOfSquares / float64(len(addrs)) + return mean, math.Sqrt(variance) +} + +// successRateAlgorithm ejects any addresses where the success rate falls below +// the other addresses according to mean and standard deviation, and if overall +// applicable from other set heuristics. +// +// Caller must hold b.mu. +func (b *outlierDetectionBalancer) successRateAlgorithm() { + addrsToConsider := b.addrsWithAtLeastRequestVolume(b.cfg.SuccessRateEjection.RequestVolume) + if len(addrsToConsider) < int(b.cfg.SuccessRateEjection.MinimumHosts) { + return + } + mean, stddev := b.meanAndStdDev(addrsToConsider) + for _, addrInfo := range addrsToConsider { + bucket := addrInfo.callCounter.inactiveBucket + ejectionCfg := b.cfg.SuccessRateEjection + if float64(b.numAddrsEjected)/float64(len(b.addrs))*100 >= float64(b.cfg.MaxEjectionPercent) { + return + } + successRate := float64(bucket.numSuccesses) / float64(bucket.numSuccesses+bucket.numFailures) + requiredSuccessRate := mean - stddev*(float64(ejectionCfg.StdevFactor)/1000) + if successRate < requiredSuccessRate { + channelz.Infof(logger, b.channelzParentID, "SuccessRate algorithm detected outlier: %s. Parameters: successRate=%f, mean=%f, stddev=%f, requiredSuccessRate=%f", addrInfo, successRate, mean, stddev, requiredSuccessRate) + if uint32(grpcrand.Int31n(100)) < ejectionCfg.EnforcementPercentage { + b.ejectAddress(addrInfo) + } + } + } +} + +// failurePercentageAlgorithm ejects any addresses where the failure percentage +// rate exceeds a set enforcement percentage, if overall applicable from other +// set heuristics. +// +// Caller must hold b.mu. +func (b *outlierDetectionBalancer) failurePercentageAlgorithm() { + addrsToConsider := b.addrsWithAtLeastRequestVolume(b.cfg.FailurePercentageEjection.RequestVolume) + if len(addrsToConsider) < int(b.cfg.FailurePercentageEjection.MinimumHosts) { + return + } + + for _, addrInfo := range addrsToConsider { + bucket := addrInfo.callCounter.inactiveBucket + ejectionCfg := b.cfg.FailurePercentageEjection + if float64(b.numAddrsEjected)/float64(len(b.addrs))*100 >= float64(b.cfg.MaxEjectionPercent) { + return + } + failurePercentage := (float64(bucket.numFailures) / float64(bucket.numSuccesses+bucket.numFailures)) * 100 + if failurePercentage > float64(b.cfg.FailurePercentageEjection.Threshold) { + channelz.Infof(logger, b.channelzParentID, "FailurePercentage algorithm detected outlier: %s, failurePercentage=%f", addrInfo, failurePercentage) + if uint32(grpcrand.Int31n(100)) < ejectionCfg.EnforcementPercentage { + b.ejectAddress(addrInfo) + } + } + } +} + +// Caller must hold b.mu. +func (b *outlierDetectionBalancer) ejectAddress(addrInfo *addressInfo) { + b.numAddrsEjected++ + addrInfo.latestEjectionTimestamp = b.timerStartTime + addrInfo.ejectionTimeMultiplier++ + for _, sbw := range addrInfo.sws { + sbw.eject() + channelz.Infof(logger, b.channelzParentID, "Subchannel ejected: %s", sbw) + } + +} + +// Caller must hold b.mu. +func (b *outlierDetectionBalancer) unejectAddress(addrInfo *addressInfo) { + b.numAddrsEjected-- + addrInfo.latestEjectionTimestamp = time.Time{} + for _, sbw := range addrInfo.sws { + sbw.uneject() + channelz.Infof(logger, b.channelzParentID, "Subchannel unejected: %s", sbw) + } +} + +// addressInfo contains the runtime information about an address that pertains +// to Outlier Detection. This struct and all of its fields is protected by +// outlierDetectionBalancer.mu in the case where it is accessed through the +// address map. In the case of Picker callbacks, the writes to the activeBucket +// of callCounter are protected by atomically loading and storing +// unsafe.Pointers (see further explanation in incrementCounter()). +type addressInfo struct { + // The call result counter object. + callCounter *callCounter + + // The latest ejection timestamp, or zero if the address is currently not + // ejected. + latestEjectionTimestamp time.Time + + // The current ejection time multiplier, starting at 0. + ejectionTimeMultiplier int64 + + // A list of subchannel wrapper objects that correspond to this address. + sws []*subConnWrapper +} + +func (a *addressInfo) String() string { + var res strings.Builder + res.WriteString("[") + for _, sw := range a.sws { + res.WriteString(sw.String()) + } + res.WriteString("]") + return res.String() +} + +func newAddressInfo() *addressInfo { + return &addressInfo{ + callCounter: newCallCounter(), + } +} diff --git a/xds/internal/balancer/outlierdetection/balancer_test.go b/xds/internal/balancer/outlierdetection/balancer_test.go new file mode 100644 index 000000000000..c6f0ac7ee128 --- /dev/null +++ b/xds/internal/balancer/outlierdetection/balancer_test.go @@ -0,0 +1,1568 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package outlierdetection + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "math" + "strings" + "sync" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/internal/balancer/stub" + "google.golang.org/grpc/internal/channelz" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/grpctest" + iserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" + "google.golang.org/grpc/xds/internal/balancer/clusterimpl" +) + +var ( + defaultTestTimeout = 5 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// TestParseConfig verifies the ParseConfig() method in the Outlier Detection +// Balancer. +func (s) TestParseConfig(t *testing.T) { + const errParseConfigName = "errParseConfigBalancer" + stub.Register(errParseConfigName, stub.BalancerFuncs{ + ParseConfig: func(json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + return nil, errors.New("some error") + }, + }) + + parser := bb{} + const ( + defaultInterval = iserviceconfig.Duration(10 * time.Second) + defaultBaseEjectionTime = iserviceconfig.Duration(30 * time.Second) + defaultMaxEjectionTime = iserviceconfig.Duration(300 * time.Second) + defaultMaxEjectionPercent = 10 + defaultSuccessRateStdevFactor = 1900 + defaultEnforcingSuccessRate = 100 + defaultSuccessRateMinimumHosts = 5 + defaultSuccessRateRequestVolume = 100 + defaultFailurePercentageThreshold = 85 + defaultEnforcingFailurePercentage = 0 + defaultFailurePercentageMinimumHosts = 5 + defaultFailurePercentageRequestVolume = 50 + ) + tests := []struct { + name string + input string + wantCfg serviceconfig.LoadBalancingConfig + wantErr string + }{ + { + name: "no-fields-set-should-get-default", + input: `{ + "childPolicy": [ + { + "xds_cluster_impl_experimental": { + "cluster": "test_cluster" + } + } + ] + }`, + wantCfg: &LBConfig{ + Interval: defaultInterval, + BaseEjectionTime: defaultBaseEjectionTime, + MaxEjectionTime: defaultMaxEjectionTime, + MaxEjectionPercent: defaultMaxEjectionPercent, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: "xds_cluster_impl_experimental", + Config: &clusterimpl.LBConfig{ + Cluster: "test_cluster", + }, + }, + }, + }, + + { + name: "some-top-level-fields-set", + input: `{ + "interval": "15s", + "maxEjectionTime": "350s", + "childPolicy": [ + { + "xds_cluster_impl_experimental": { + "cluster": "test_cluster" + } + } + ] + }`, + // Should get set fields + defaults for unset fields. + wantCfg: &LBConfig{ + Interval: iserviceconfig.Duration(15 * time.Second), + BaseEjectionTime: defaultBaseEjectionTime, + MaxEjectionTime: iserviceconfig.Duration(350 * time.Second), + MaxEjectionPercent: defaultMaxEjectionPercent, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: "xds_cluster_impl_experimental", + Config: &clusterimpl.LBConfig{ + Cluster: "test_cluster", + }, + }, + }, + }, + { + name: "success-rate-ejection-present-but-no-fields", + input: `{ + "successRateEjection": {}, + "childPolicy": [ + { + "xds_cluster_impl_experimental": { + "cluster": "test_cluster" + } + } + ] + }`, + // Should get defaults of success-rate-ejection struct. + wantCfg: &LBConfig{ + Interval: defaultInterval, + BaseEjectionTime: defaultBaseEjectionTime, + MaxEjectionTime: defaultMaxEjectionTime, + MaxEjectionPercent: defaultMaxEjectionPercent, + SuccessRateEjection: &SuccessRateEjection{ + StdevFactor: defaultSuccessRateStdevFactor, + EnforcementPercentage: defaultEnforcingSuccessRate, + MinimumHosts: defaultSuccessRateMinimumHosts, + RequestVolume: defaultSuccessRateRequestVolume, + }, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: "xds_cluster_impl_experimental", + Config: &clusterimpl.LBConfig{ + Cluster: "test_cluster", + }, + }, + }, + }, + { + name: "success-rate-ejection-present-partially-set", + input: `{ + "successRateEjection": { + "stdevFactor": 1000, + "minimumHosts": 5 + }, + "childPolicy": [ + { + "xds_cluster_impl_experimental": { + "cluster": "test_cluster" + } + } + ] + }`, + // Should get set fields + defaults for others in success rate + // ejection layer. + wantCfg: &LBConfig{ + Interval: defaultInterval, + BaseEjectionTime: defaultBaseEjectionTime, + MaxEjectionTime: defaultMaxEjectionTime, + MaxEjectionPercent: defaultMaxEjectionPercent, + SuccessRateEjection: &SuccessRateEjection{ + StdevFactor: 1000, + EnforcementPercentage: defaultEnforcingSuccessRate, + MinimumHosts: 5, + RequestVolume: defaultSuccessRateRequestVolume, + }, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: "xds_cluster_impl_experimental", + Config: &clusterimpl.LBConfig{ + Cluster: "test_cluster", + }, + }, + }, + }, + { + name: "success-rate-ejection-present-fully-set", + input: `{ + "successRateEjection": { + "stdevFactor": 1000, + "enforcementPercentage": 50, + "minimumHosts": 5, + "requestVolume": 50 + }, + "childPolicy": [ + { + "xds_cluster_impl_experimental": { + "cluster": "test_cluster" + } + } + ] + }`, + wantCfg: &LBConfig{ + Interval: defaultInterval, + BaseEjectionTime: defaultBaseEjectionTime, + MaxEjectionTime: defaultMaxEjectionTime, + MaxEjectionPercent: defaultMaxEjectionPercent, + SuccessRateEjection: &SuccessRateEjection{ + StdevFactor: 1000, + EnforcementPercentage: 50, + MinimumHosts: 5, + RequestVolume: 50, + }, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: "xds_cluster_impl_experimental", + Config: &clusterimpl.LBConfig{ + Cluster: "test_cluster", + }, + }, + }, + }, + { + name: "failure-percentage-ejection-present-but-no-fields", + input: `{ + "failurePercentageEjection": {}, + "childPolicy": [ + { + "xds_cluster_impl_experimental": { + "cluster": "test_cluster" + } + } + ] + }`, + // Should get defaults of failure percentage ejection layer. + wantCfg: &LBConfig{ + Interval: defaultInterval, + BaseEjectionTime: defaultBaseEjectionTime, + MaxEjectionTime: defaultMaxEjectionTime, + MaxEjectionPercent: defaultMaxEjectionPercent, + FailurePercentageEjection: &FailurePercentageEjection{ + Threshold: defaultFailurePercentageThreshold, + EnforcementPercentage: defaultEnforcingFailurePercentage, + MinimumHosts: defaultFailurePercentageMinimumHosts, + RequestVolume: defaultFailurePercentageRequestVolume, + }, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: "xds_cluster_impl_experimental", + Config: &clusterimpl.LBConfig{ + Cluster: "test_cluster", + }, + }, + }, + }, + { + name: "failure-percentage-ejection-present-partially-set", + input: `{ + "failurePercentageEjection": { + "threshold": 80, + "minimumHosts": 10 + }, + "childPolicy": [ + { + "xds_cluster_impl_experimental": { + "cluster": "test_cluster" + } + } + ] + }`, + // Should get set fields + defaults for others in success rate + // ejection layer. + wantCfg: &LBConfig{ + Interval: defaultInterval, + BaseEjectionTime: defaultBaseEjectionTime, + MaxEjectionTime: defaultMaxEjectionTime, + MaxEjectionPercent: defaultMaxEjectionPercent, + FailurePercentageEjection: &FailurePercentageEjection{ + Threshold: 80, + EnforcementPercentage: defaultEnforcingFailurePercentage, + MinimumHosts: 10, + RequestVolume: defaultFailurePercentageRequestVolume, + }, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: "xds_cluster_impl_experimental", + Config: &clusterimpl.LBConfig{ + Cluster: "test_cluster", + }, + }, + }, + }, + { + name: "failure-percentage-ejection-present-fully-set", + input: `{ + "failurePercentageEjection": { + "threshold": 80, + "enforcementPercentage": 100, + "minimumHosts": 10, + "requestVolume": 40 + }, + "childPolicy": [ + { + "xds_cluster_impl_experimental": { + "cluster": "test_cluster" + } + } + ] + }`, + wantCfg: &LBConfig{ + Interval: defaultInterval, + BaseEjectionTime: defaultBaseEjectionTime, + MaxEjectionTime: defaultMaxEjectionTime, + MaxEjectionPercent: defaultMaxEjectionPercent, + FailurePercentageEjection: &FailurePercentageEjection{ + Threshold: 80, + EnforcementPercentage: 100, + MinimumHosts: 10, + RequestVolume: 40, + }, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: "xds_cluster_impl_experimental", + Config: &clusterimpl.LBConfig{ + Cluster: "test_cluster", + }, + }, + }, + }, + { // to make sure zero values aren't overwritten by defaults + name: "lb-config-every-field-set-zero-value", + input: `{ + "interval": "0s", + "baseEjectionTime": "0s", + "maxEjectionTime": "0s", + "maxEjectionPercent": 0, + "successRateEjection": { + "stdevFactor": 0, + "enforcementPercentage": 0, + "minimumHosts": 0, + "requestVolume": 0 + }, + "failurePercentageEjection": { + "threshold": 0, + "enforcementPercentage": 0, + "minimumHosts": 0, + "requestVolume": 0 + }, + "childPolicy": [ + { + "xds_cluster_impl_experimental": { + "cluster": "test_cluster" + } + } + ] + }`, + wantCfg: &LBConfig{ + SuccessRateEjection: &SuccessRateEjection{}, + FailurePercentageEjection: &FailurePercentageEjection{}, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: "xds_cluster_impl_experimental", + Config: &clusterimpl.LBConfig{ + Cluster: "test_cluster", + }, + }, + }, + }, + { + name: "lb-config-every-field-set", + input: `{ + "interval": "10s", + "baseEjectionTime": "30s", + "maxEjectionTime": "300s", + "maxEjectionPercent": 10, + "successRateEjection": { + "stdevFactor": 1900, + "enforcementPercentage": 100, + "minimumHosts": 5, + "requestVolume": 100 + }, + "failurePercentageEjection": { + "threshold": 85, + "enforcementPercentage": 5, + "minimumHosts": 5, + "requestVolume": 50 + }, + "childPolicy": [ + { + "xds_cluster_impl_experimental": { + "cluster": "test_cluster" + } + } + ] + }`, + wantCfg: &LBConfig{ + Interval: iserviceconfig.Duration(10 * time.Second), + BaseEjectionTime: iserviceconfig.Duration(30 * time.Second), + MaxEjectionTime: iserviceconfig.Duration(300 * time.Second), + MaxEjectionPercent: 10, + SuccessRateEjection: &SuccessRateEjection{ + StdevFactor: 1900, + EnforcementPercentage: 100, + MinimumHosts: 5, + RequestVolume: 100, + }, + FailurePercentageEjection: &FailurePercentageEjection{ + Threshold: 85, + EnforcementPercentage: 5, + MinimumHosts: 5, + RequestVolume: 50, + }, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: "xds_cluster_impl_experimental", + Config: &clusterimpl.LBConfig{ + Cluster: "test_cluster", + }, + }, + }, + }, + { + name: "interval-is-negative", + input: `{"interval": "-10s"}`, + wantErr: "OutlierDetectionLoadBalancingConfig.interval = -10s; must be >= 0", + }, + { + name: "base-ejection-time-is-negative", + input: `{"baseEjectionTime": "-10s"}`, + wantErr: "OutlierDetectionLoadBalancingConfig.base_ejection_time = -10s; must be >= 0", + }, + { + name: "max-ejection-time-is-negative", + input: `{"maxEjectionTime": "-10s"}`, + wantErr: "OutlierDetectionLoadBalancingConfig.max_ejection_time = -10s; must be >= 0", + }, + { + name: "max-ejection-percent-is-greater-than-100", + input: `{"maxEjectionPercent": 150}`, + wantErr: "OutlierDetectionLoadBalancingConfig.max_ejection_percent = 150; must be <= 100", + }, + { + name: "enforcement-percentage-success-rate-is-greater-than-100", + input: `{ + "successRateEjection": { + "enforcementPercentage": 150 + } + }`, + wantErr: "OutlierDetectionLoadBalancingConfig.SuccessRateEjection.enforcement_percentage = 150; must be <= 100", + }, + { + name: "failure-percentage-threshold-is-greater-than-100", + input: `{ + "failurePercentageEjection": { + "threshold": 150 + } + }`, + wantErr: "OutlierDetectionLoadBalancingConfig.FailurePercentageEjection.threshold = 150; must be <= 100", + }, + { + name: "enforcement-percentage-failure-percentage-ejection-is-greater-than-100", + input: `{ + "failurePercentageEjection": { + "enforcementPercentage": 150 + } + }`, + wantErr: "OutlierDetectionLoadBalancingConfig.FailurePercentageEjection.enforcement_percentage = 150; must be <= 100", + }, + { + name: "child-policy-present-but-parse-error", + input: `{ + "childPolicy": [ + { + "errParseConfigBalancer": { + "cluster": "test_cluster" + } + } + ] + }`, + wantErr: "error parsing loadBalancingConfig for policy \"errParseConfigBalancer\"", + }, + { + name: "no-supported-child-policy", + input: `{ + "childPolicy": [ + { + "doesNotExistBalancer": { + "cluster": "test_cluster" + } + } + ] + }`, + wantErr: "invalid loadBalancingConfig: no supported policies found", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + gotCfg, gotErr := parser.ParseConfig(json.RawMessage(test.input)) + if gotErr != nil && !strings.Contains(gotErr.Error(), test.wantErr) { + t.Fatalf("ParseConfig(%v) = %v, wantErr %v", test.input, gotErr, test.wantErr) + } + if (gotErr != nil) != (test.wantErr != "") { + t.Fatalf("ParseConfig(%v) = %v, wantErr %v", test.input, gotErr, test.wantErr) + } + if test.wantErr != "" { + return + } + if diff := cmp.Diff(gotCfg, test.wantCfg); diff != "" { + t.Fatalf("parseConfig(%v) got unexpected output, diff (-got +want): %v", string(test.input), diff) + } + }) + } +} + +func (lbc *LBConfig) Equal(lbc2 *LBConfig) bool { + if !lbc.EqualIgnoringChildPolicy(lbc2) { + return false + } + return cmp.Equal(lbc.ChildPolicy, lbc2.ChildPolicy) +} + +type subConnWithState struct { + sc balancer.SubConn + state balancer.SubConnState +} + +func setup(t *testing.T) (*outlierDetectionBalancer, *testutils.TestClientConn, func()) { + t.Helper() + builder := balancer.Get(Name) + if builder == nil { + t.Fatalf("balancer.Get(%q) returned nil", Name) + } + tcc := testutils.NewTestClientConn(t) + odB := builder.Build(tcc, balancer.BuildOptions{ChannelzParentID: channelz.NewIdentifierForTesting(channelz.RefChannel, time.Now().Unix(), nil)}) + return odB.(*outlierDetectionBalancer), tcc, odB.Close +} + +type emptyChildConfig struct { + serviceconfig.LoadBalancingConfig +} + +// TestChildBasicOperations tests basic operations of the Outlier Detection +// Balancer and it's interaction with it's child. The following scenarios are +// tested, in a step by step fashion: +// 1. The Outlier Detection Balancer receives it's first good configuration. The +// balancer is expected to create a child and sent the child it's configuration. +// 2. The Outlier Detection Balancer receives new configuration that specifies a +// child's type, and the new type immediately reports READY inline. The first +// child balancer should be closed and the second child balancer should receive +// a config update. +// 3. The Outlier Detection Balancer is closed. The second child balancer should +// be closed. +func (s) TestChildBasicOperations(t *testing.T) { + bc := emptyChildConfig{} + + ccsCh := testutils.NewChannel() + closeCh := testutils.NewChannel() + + stub.Register(t.Name()+"child1", stub.BalancerFuncs{ + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + ccsCh.Send(ccs.BalancerConfig) + return nil + }, + Close: func(bd *stub.BalancerData) { + closeCh.Send(nil) + }, + }) + + stub.Register(t.Name()+"child2", stub.BalancerFuncs{ + UpdateClientConnState: func(bd *stub.BalancerData, _ balancer.ClientConnState) error { + // UpdateState inline to READY to complete graceful switch process + // synchronously from any UpdateClientConnState call. + bd.ClientConn.UpdateState(balancer.State{ + ConnectivityState: connectivity.Ready, + Picker: &testutils.TestConstPicker{}, + }) + ccsCh.Send(nil) + return nil + }, + Close: func(bd *stub.BalancerData) { + closeCh.Send(nil) + }, + }) + + od, tcc, _ := setup(t) + + // This first config update should cause a child to be built and forwarded + // it's first update. + od.UpdateClientConnState(balancer.ClientConnState{ + BalancerConfig: &LBConfig{ + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: t.Name() + "child1", + Config: bc, + }, + }, + }) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + cr, err := ccsCh.Receive(ctx) + if err != nil { + t.Fatalf("timed out waiting for UpdateClientConnState on the first child balancer: %v", err) + } + if _, ok := cr.(emptyChildConfig); !ok { + t.Fatalf("Received child policy config of type %T, want %T", cr, emptyChildConfig{}) + } + + // This Update Client Conn State call should cause the first child balancer + // to close, and a new child to be created and also forwarded it's first + // config update. + od.UpdateClientConnState(balancer.ClientConnState{ + BalancerConfig: &LBConfig{ + Interval: math.MaxInt64, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: t.Name() + "child2", + Config: emptyChildConfig{}, + }, + }, + }) + + // Verify inline UpdateState() call from the new child eventually makes it's + // way to the Test Client Conn. + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for a UpdateState call on the ClientConn") + case state := <-tcc.NewStateCh: + if state != connectivity.Ready { + t.Fatalf("ClientConn received connectivity state %v, want %v", state, connectivity.Ready) + } + } + + // Verify the first child balancer closed. + if _, err = closeCh.Receive(ctx); err != nil { + t.Fatalf("timed out waiting for the first child balancer to be closed: %v", err) + } + // Verify the second child balancer received it's first config update. + if _, err = ccsCh.Receive(ctx); err != nil { + t.Fatalf("timed out waiting for UpdateClientConnState on the second child balancer: %v", err) + } + // Closing the Outlier Detection Balancer should close the newly created + // child. + od.Close() + if _, err = closeCh.Receive(ctx); err != nil { + t.Fatalf("timed out waiting for the second child balancer to be closed: %v", err) + } +} + +// TestUpdateAddresses tests the functionality of UpdateAddresses and any +// changes in the addresses/plurality of those addresses for a SubConn. The +// Balancer is set up with two upstreams, with one of the upstreams being +// ejected. Initially, there is one SubConn for each address. The following +// scenarios are tested, in a step by step fashion: +// 1. The SubConn not currently ejected switches addresses to the address that +// is ejected. This should cause the SubConn to get ejected. +// 2. Update this same SubConn to multiple addresses. This should cause the +// SubConn to get unejected, as it is no longer being tracked by Outlier +// Detection at that point. +// 3. Update this same SubConn to different addresses, still multiple. This +// should be a noop, as the SubConn is still no longer being tracked by Outlier +// Detection. +// 4. Update this same SubConn to the a single address which is ejected. This +// should cause the SubConn to be ejected. +func (s) TestUpdateAddresses(t *testing.T) { + scsCh := testutils.NewChannel() + var scw1, scw2 balancer.SubConn + var err error + stub.Register(t.Name(), stub.BalancerFuncs{ + UpdateClientConnState: func(bd *stub.BalancerData, _ balancer.ClientConnState) error { + scw1, err = bd.ClientConn.NewSubConn([]resolver.Address{{Addr: "address1"}}, balancer.NewSubConnOptions{ + StateListener: func(state balancer.SubConnState) { scsCh.Send(subConnWithState{sc: scw1, state: state}) }, + }) + if err != nil { + t.Errorf("error in od.NewSubConn call: %v", err) + } + scw2, err = bd.ClientConn.NewSubConn([]resolver.Address{{Addr: "address2"}}, balancer.NewSubConnOptions{ + StateListener: func(state balancer.SubConnState) { scsCh.Send(subConnWithState{sc: scw2, state: state}) }, + }) + if err != nil { + t.Errorf("error in od.NewSubConn call: %v", err) + } + bd.ClientConn.UpdateState(balancer.State{ + ConnectivityState: connectivity.Ready, + Picker: &rrPicker{ + scs: []balancer.SubConn{scw1, scw2}, + }, + }) + return nil + }, + }) + + od, tcc, cleanup := setup(t) + defer cleanup() + + od.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: []resolver.Address{ + {Addr: "address1"}, + {Addr: "address2"}, + }, + }, + BalancerConfig: &LBConfig{ + Interval: iserviceconfig.Duration(10 * time.Second), + BaseEjectionTime: iserviceconfig.Duration(30 * time.Second), + MaxEjectionTime: iserviceconfig.Duration(300 * time.Second), + MaxEjectionPercent: 10, + FailurePercentageEjection: &FailurePercentageEjection{ + Threshold: 50, + EnforcementPercentage: 100, + MinimumHosts: 2, + RequestVolume: 3, + }, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: t.Name(), + Config: emptyChildConfig{}, + }, + }, + }) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Setup the system to where one address is ejected and one address + // isn't. + select { + case <-ctx.Done(): + t.Fatal("timeout while waiting for a UpdateState call on the ClientConn") + case picker := <-tcc.NewPickerCh: + pi, err := picker.Pick(balancer.PickInfo{}) + if err != nil { + t.Fatalf("picker.Pick failed with error: %v", err) + } + // Simulate 5 successful RPC calls on the first SubConn (the first call + // to picker.Pick). + for c := 0; c < 5; c++ { + pi.Done(balancer.DoneInfo{}) + } + pi, err = picker.Pick(balancer.PickInfo{}) + if err != nil { + t.Fatalf("picker.Pick failed with error: %v", err) + } + // Simulate 5 failed RPC calls on the second SubConn (the second call to + // picker.Pick). Thus, when the interval timer algorithm is run, the + // second SubConn's address should be ejected, which will allow us to + // further test UpdateAddresses() logic. + for c := 0; c < 5; c++ { + pi.Done(balancer.DoneInfo{Err: errors.New("some error")}) + } + od.intervalTimerAlgorithm() + // verify StateListener() got called with TRANSIENT_FAILURE for child + // with address that was ejected. + gotSCWS, err := scsCh.Receive(ctx) + if err != nil { + t.Fatalf("Error waiting for Sub Conn update: %v", err) + } + if err = scwsEqual(gotSCWS.(subConnWithState), subConnWithState{ + sc: scw2, + state: balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}, + }); err != nil { + t.Fatalf("Error in Sub Conn update: %v", err) + } + } + + // Update scw1 to another address that is currently ejected. This should + // cause scw1 to get ejected. + od.UpdateAddresses(scw1, []resolver.Address{{Addr: "address2"}}) + + // Verify that update addresses gets forwarded to ClientConn. + select { + case <-ctx.Done(): + t.Fatal("timeout while waiting for a UpdateState call on the ClientConn") + case <-tcc.UpdateAddressesAddrsCh: + } + // Verify scw1 got ejected (StateListener called with TRANSIENT_FAILURE). + gotSCWS, err := scsCh.Receive(ctx) + if err != nil { + t.Fatalf("Error waiting for Sub Conn update: %v", err) + } + if err = scwsEqual(gotSCWS.(subConnWithState), subConnWithState{ + sc: scw1, + state: balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}, + }); err != nil { + t.Fatalf("Error in Sub Conn update: %v", err) + } + + // Update scw1 to multiple addresses. This should cause scw1 to get + // unejected, as is it no longer being tracked for Outlier Detection. + od.UpdateAddresses(scw1, []resolver.Address{ + {Addr: "address1"}, + {Addr: "address2"}, + }) + // Verify scw1 got unejected (StateListener called with recent state). + gotSCWS, err = scsCh.Receive(ctx) + if err != nil { + t.Fatalf("Error waiting for Sub Conn update: %v", err) + } + if err = scwsEqual(gotSCWS.(subConnWithState), subConnWithState{ + sc: scw1, + state: balancer.SubConnState{ConnectivityState: connectivity.Idle}, + }); err != nil { + t.Fatalf("Error in Sub Conn update: %v", err) + } + + // Update scw1 to a different multiple addresses list. A change of addresses + // in which the plurality goes from multiple to multiple should be a no-op, + // as the address continues to be ignored by outlier detection. + od.UpdateAddresses(scw1, []resolver.Address{ + {Addr: "address2"}, + {Addr: "address3"}, + }) + // Verify no downstream effects. + sCtx, cancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer cancel() + if _, err := scsCh.Receive(sCtx); err == nil { + t.Fatalf("no SubConn update should have been sent (no SubConn got ejected/unejected)") + } + + // Update scw1 back to a single address, which is ejected. This should cause + // the SubConn to be re-ejected. + od.UpdateAddresses(scw1, []resolver.Address{{Addr: "address2"}}) + // Verify scw1 got ejected (StateListener called with TRANSIENT FAILURE). + gotSCWS, err = scsCh.Receive(ctx) + if err != nil { + t.Fatalf("Error waiting for Sub Conn update: %v", err) + } + if err = scwsEqual(gotSCWS.(subConnWithState), subConnWithState{ + sc: scw1, + state: balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}, + }); err != nil { + t.Fatalf("Error in Sub Conn update: %v", err) + } +} + +func scwsEqual(gotSCWS subConnWithState, wantSCWS subConnWithState) error { + if gotSCWS.sc != wantSCWS.sc || !cmp.Equal(gotSCWS.state, wantSCWS.state, cmp.AllowUnexported(subConnWrapper{}, addressInfo{}), cmpopts.IgnoreFields(subConnWrapper{}, "scUpdateCh")) { + return fmt.Errorf("received SubConnState: %+v, want %+v", gotSCWS, wantSCWS) + } + return nil +} + +type rrPicker struct { + scs []balancer.SubConn + next int +} + +func (rrp *rrPicker) Pick(balancer.PickInfo) (balancer.PickResult, error) { + sc := rrp.scs[rrp.next] + rrp.next = (rrp.next + 1) % len(rrp.scs) + return balancer.PickResult{SubConn: sc}, nil +} + +// TestDurationOfInterval tests the configured interval timer. +// The following scenarios are tested: +// 1. The Outlier Detection Balancer receives it's first config. The balancer +// should configure the timer with whatever is directly specified on the config. +// 2. The Outlier Detection Balancer receives a subsequent config. The balancer +// should configure with whatever interval is configured minus the difference +// between the current time and the previous start timestamp. +// 3. The Outlier Detection Balancer receives a no-op configuration. The +// balancer should not configure a timer at all. +func (s) TestDurationOfInterval(t *testing.T) { + stub.Register(t.Name(), stub.BalancerFuncs{}) + + od, _, cleanup := setup(t) + defer func(af func(d time.Duration, f func()) *time.Timer) { + cleanup() + afterFunc = af + }(afterFunc) + + durationChan := testutils.NewChannel() + afterFunc = func(dur time.Duration, _ func()) *time.Timer { + durationChan.Send(dur) + return time.NewTimer(math.MaxInt64) + } + + od.UpdateClientConnState(balancer.ClientConnState{ + BalancerConfig: &LBConfig{ + Interval: iserviceconfig.Duration(8 * time.Second), + SuccessRateEjection: &SuccessRateEjection{ + StdevFactor: 1900, + EnforcementPercentage: 100, + MinimumHosts: 5, + RequestVolume: 100, + }, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: t.Name(), + Config: emptyChildConfig{}, + }, + }, + }) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + d, err := durationChan.Receive(ctx) + if err != nil { + t.Fatalf("Error receiving duration from afterFunc() call: %v", err) + } + dur := d.(time.Duration) + // The configured duration should be 8 seconds - what the balancer was + // configured with. + if dur != 8*time.Second { + t.Fatalf("configured duration should have been 8 seconds to start timer") + } + + // Override time.Now to time.Now() + 5 seconds. This will represent 5 + // seconds already passing for the next check in UpdateClientConnState. + defer func(n func() time.Time) { + now = n + }(now) + now = func() time.Time { + return time.Now().Add(time.Second * 5) + } + + // UpdateClientConnState with an interval of 9 seconds. Due to 5 seconds + // already passing (from overridden time.Now function), this should start an + // interval timer of ~4 seconds. + od.UpdateClientConnState(balancer.ClientConnState{ + BalancerConfig: &LBConfig{ + Interval: iserviceconfig.Duration(9 * time.Second), + SuccessRateEjection: &SuccessRateEjection{ + StdevFactor: 1900, + EnforcementPercentage: 100, + MinimumHosts: 5, + RequestVolume: 100, + }, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: t.Name(), + Config: emptyChildConfig{}, + }, + }, + }) + + d, err = durationChan.Receive(ctx) + if err != nil { + t.Fatalf("Error receiving duration from afterFunc() call: %v", err) + } + dur = d.(time.Duration) + if dur.Seconds() < 3.5 || 4.5 < dur.Seconds() { + t.Fatalf("configured duration should have been around 4 seconds to start timer") + } + + // UpdateClientConnState with a no-op config. This shouldn't configure the + // interval timer at all due to it being a no-op. + od.UpdateClientConnState(balancer.ClientConnState{ + BalancerConfig: &LBConfig{ + Interval: iserviceconfig.Duration(10 * time.Second), + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: t.Name(), + Config: emptyChildConfig{}, + }, + }, + }) + + // No timer should have been started. + sCtx, cancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer cancel() + if _, err = durationChan.Receive(sCtx); err == nil { + t.Fatal("No timer should have started.") + } +} + +// TestEjectUnejectSuccessRate tests the functionality of the interval timer +// algorithm when configured with SuccessRateEjection. The Outlier Detection +// Balancer will be set up with 3 SubConns, each with a different address. +// It tests the following scenarios, in a step by step fashion: +// 1. The three addresses each have 5 successes. The interval timer algorithm should +// not eject any of the addresses. +// 2. Two of the addresses have 5 successes, the third has five failures. The +// interval timer algorithm should eject the third address with five failures. +// 3. The interval timer algorithm is run at a later time past max ejection +// time. The interval timer algorithm should uneject the third address. +func (s) TestEjectUnejectSuccessRate(t *testing.T) { + scsCh := testutils.NewChannel() + var scw1, scw2, scw3 balancer.SubConn + var err error + stub.Register(t.Name(), stub.BalancerFuncs{ + UpdateClientConnState: func(bd *stub.BalancerData, _ balancer.ClientConnState) error { + scw1, err = bd.ClientConn.NewSubConn([]resolver.Address{{Addr: "address1"}}, balancer.NewSubConnOptions{ + StateListener: func(state balancer.SubConnState) { scsCh.Send(subConnWithState{sc: scw1, state: state}) }, + }) + if err != nil { + t.Errorf("error in od.NewSubConn call: %v", err) + } + scw2, err = bd.ClientConn.NewSubConn([]resolver.Address{{Addr: "address2"}}, balancer.NewSubConnOptions{ + StateListener: func(state balancer.SubConnState) { scsCh.Send(subConnWithState{sc: scw2, state: state}) }, + }) + if err != nil { + t.Errorf("error in od.NewSubConn call: %v", err) + } + scw3, err = bd.ClientConn.NewSubConn([]resolver.Address{{Addr: "address3"}}, balancer.NewSubConnOptions{ + StateListener: func(state balancer.SubConnState) { scsCh.Send(subConnWithState{sc: scw3, state: state}) }, + }) + if err != nil { + t.Errorf("error in od.NewSubConn call: %v", err) + } + bd.ClientConn.UpdateState(balancer.State{ + ConnectivityState: connectivity.Ready, + Picker: &rrPicker{ + scs: []balancer.SubConn{scw1, scw2, scw3}, + }, + }) + return nil + }, + }) + + od, tcc, cleanup := setup(t) + defer func() { + cleanup() + }() + + od.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: []resolver.Address{ + {Addr: "address1"}, + {Addr: "address2"}, + {Addr: "address3"}, + }, + }, + BalancerConfig: &LBConfig{ + Interval: math.MaxInt64, // so the interval will never run unless called manually in test. + BaseEjectionTime: iserviceconfig.Duration(30 * time.Second), + MaxEjectionTime: iserviceconfig.Duration(300 * time.Second), + MaxEjectionPercent: 10, + FailurePercentageEjection: &FailurePercentageEjection{ + Threshold: 50, + EnforcementPercentage: 100, + MinimumHosts: 3, + RequestVolume: 3, + }, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: t.Name(), + Config: emptyChildConfig{}, + }, + }, + }) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for a UpdateState call on the ClientConn") + case picker := <-tcc.NewPickerCh: + // Set each of the three upstream addresses to have five successes each. + // This should cause none of the addresses to be ejected as none of them + // are outliers according to the success rate algorithm. + for i := 0; i < 3; i++ { + pi, err := picker.Pick(balancer.PickInfo{}) + if err != nil { + t.Fatalf("picker.Pick failed with error: %v", err) + } + for c := 0; c < 5; c++ { + pi.Done(balancer.DoneInfo{}) + } + } + + od.intervalTimerAlgorithm() + + // verify no StateListener() call on the child, as no addresses got + // ejected (ejected address will cause an StateListener call). + sCtx, cancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer cancel() + if _, err := scsCh.Receive(sCtx); err == nil { + t.Fatalf("no SubConn update should have been sent (no SubConn got ejected)") + } + + // Since no addresses are ejected, a SubConn update should forward down + // to the child. + od.updateSubConnState(scw1.(*subConnWrapper).SubConn, balancer.SubConnState{ + ConnectivityState: connectivity.Connecting, + }) + + gotSCWS, err := scsCh.Receive(ctx) + if err != nil { + t.Fatalf("Error waiting for Sub Conn update: %v", err) + } + if err = scwsEqual(gotSCWS.(subConnWithState), subConnWithState{ + sc: scw1, + state: balancer.SubConnState{ConnectivityState: connectivity.Connecting}, + }); err != nil { + t.Fatalf("Error in Sub Conn update: %v", err) + } + + // Set two of the upstream addresses to have five successes each, and + // one of the upstream addresses to have five failures. This should + // cause the address which has five failures to be ejected according to + // the SuccessRateAlgorithm. + for i := 0; i < 2; i++ { + pi, err := picker.Pick(balancer.PickInfo{}) + if err != nil { + t.Fatalf("picker.Pick failed with error: %v", err) + } + for c := 0; c < 5; c++ { + pi.Done(balancer.DoneInfo{}) + } + } + pi, err := picker.Pick(balancer.PickInfo{}) + if err != nil { + t.Fatalf("picker.Pick failed with error: %v", err) + } + for c := 0; c < 5; c++ { + pi.Done(balancer.DoneInfo{Err: errors.New("some error")}) + } + + // should eject address that always errored. + od.intervalTimerAlgorithm() + // Due to the address being ejected, the SubConn with that address + // should be ejected, meaning a TRANSIENT_FAILURE connectivity state + // gets reported to the child. + gotSCWS, err = scsCh.Receive(ctx) + if err != nil { + t.Fatalf("Error waiting for Sub Conn update: %v", err) + } + if err = scwsEqual(gotSCWS.(subConnWithState), subConnWithState{ + sc: scw3, + state: balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}, + }); err != nil { + t.Fatalf("Error in Sub Conn update: %v", err) + } + // Only one address should be ejected. + sCtx, cancel = context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer cancel() + if _, err := scsCh.Receive(sCtx); err == nil { + t.Fatalf("Only one SubConn update should have been sent (only one SubConn got ejected)") + } + + // Now that an address is ejected, SubConn updates for SubConns using + // that address should not be forwarded downward. These SubConn updates + // will be cached to update the child sometime in the future when the + // address gets unejected. + od.updateSubConnState(pi.SubConn, balancer.SubConnState{ + ConnectivityState: connectivity.Connecting, + }) + sCtx, cancel = context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer cancel() + if _, err := scsCh.Receive(sCtx); err == nil { + t.Fatalf("SubConn update should not have been forwarded (the SubConn is ejected)") + } + + // Override now to cause the interval timer algorithm to always uneject + // the ejected address. This will always uneject the ejected address + // because this time is set way past the max ejection time set in the + // configuration, which will make the next interval timer algorithm run + // uneject any ejected addresses. + defer func(n func() time.Time) { + now = n + }(now) + now = func() time.Time { + return time.Now().Add(time.Second * 1000) + } + od.intervalTimerAlgorithm() + + // unejected SubConn should report latest persisted state - which is + // connecting from earlier. + gotSCWS, err = scsCh.Receive(ctx) + if err != nil { + t.Fatalf("Error waiting for Sub Conn update: %v", err) + } + if err = scwsEqual(gotSCWS.(subConnWithState), subConnWithState{ + sc: scw3, + state: balancer.SubConnState{ConnectivityState: connectivity.Connecting}, + }); err != nil { + t.Fatalf("Error in Sub Conn update: %v", err) + } + } +} + +// TestEjectFailureRate tests the functionality of the interval timer algorithm +// when configured with FailurePercentageEjection, and also the functionality of +// noop configuration. The Outlier Detection Balancer will be set up with 3 +// SubConns, each with a different address. It tests the following scenarios, in +// a step by step fashion: +// 1. The three addresses each have 5 successes. The interval timer algorithm +// should not eject any of the addresses. +// 2. Two of the addresses have 5 successes, the third has five failures. The +// interval timer algorithm should eject the third address with five failures. +// 3. The Outlier Detection Balancer receives a subsequent noop config update. +// The balancer should uneject all ejected addresses. +func (s) TestEjectFailureRate(t *testing.T) { + scsCh := testutils.NewChannel() + var scw1, scw2, scw3 balancer.SubConn + var err error + stub.Register(t.Name(), stub.BalancerFuncs{ + UpdateClientConnState: func(bd *stub.BalancerData, _ balancer.ClientConnState) error { + if scw1 != nil { // UpdateClientConnState was already called, no need to recreate SubConns. + return nil + } + scw1, err = bd.ClientConn.NewSubConn([]resolver.Address{{Addr: "address1"}}, balancer.NewSubConnOptions{ + StateListener: func(state balancer.SubConnState) { scsCh.Send(subConnWithState{sc: scw1, state: state}) }, + }) + if err != nil { + t.Errorf("error in od.NewSubConn call: %v", err) + } + scw2, err = bd.ClientConn.NewSubConn([]resolver.Address{{Addr: "address2"}}, balancer.NewSubConnOptions{ + StateListener: func(state balancer.SubConnState) { scsCh.Send(subConnWithState{sc: scw2, state: state}) }, + }) + if err != nil { + t.Errorf("error in od.NewSubConn call: %v", err) + } + scw3, err = bd.ClientConn.NewSubConn([]resolver.Address{{Addr: "address3"}}, balancer.NewSubConnOptions{ + StateListener: func(state balancer.SubConnState) { scsCh.Send(subConnWithState{sc: scw3, state: state}) }, + }) + if err != nil { + t.Errorf("error in od.NewSubConn call: %v", err) + } + return nil + }, + }) + + od, tcc, cleanup := setup(t) + defer func() { + cleanup() + }() + + od.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: []resolver.Address{ + {Addr: "address1"}, + {Addr: "address2"}, + {Addr: "address3"}, + }, + }, + BalancerConfig: &LBConfig{ + Interval: math.MaxInt64, // so the interval will never run unless called manually in test. + BaseEjectionTime: iserviceconfig.Duration(30 * time.Second), + MaxEjectionTime: iserviceconfig.Duration(300 * time.Second), + MaxEjectionPercent: 10, + SuccessRateEjection: &SuccessRateEjection{ + StdevFactor: 500, + EnforcementPercentage: 100, + MinimumHosts: 3, + RequestVolume: 3, + }, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: t.Name(), + Config: emptyChildConfig{}, + }, + }, + }) + + od.UpdateState(balancer.State{ + ConnectivityState: connectivity.Ready, + Picker: &rrPicker{ + scs: []balancer.SubConn{scw1, scw2, scw3}, + }, + }) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for a UpdateState call on the ClientConn") + case picker := <-tcc.NewPickerCh: + // Set each upstream address to have five successes each. This should + // cause none of the addresses to be ejected as none of them are below + // the failure percentage threshold. + for i := 0; i < 3; i++ { + pi, err := picker.Pick(balancer.PickInfo{}) + if err != nil { + t.Fatalf("picker.Pick failed with error: %v", err) + } + for c := 0; c < 5; c++ { + pi.Done(balancer.DoneInfo{}) + } + } + + od.intervalTimerAlgorithm() + sCtx, cancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer cancel() + if _, err := scsCh.Receive(sCtx); err == nil { + t.Fatalf("no SubConn update should have been sent (no SubConn got ejected)") + } + + // Set two upstream addresses to have five successes each, and one + // upstream address to have five failures. This should cause the address + // with five failures to be ejected according to the Failure Percentage + // Algorithm. + for i := 0; i < 2; i++ { + pi, err := picker.Pick(balancer.PickInfo{}) + if err != nil { + t.Fatalf("picker.Pick failed with error: %v", err) + } + for c := 0; c < 5; c++ { + pi.Done(balancer.DoneInfo{}) + } + } + pi, err := picker.Pick(balancer.PickInfo{}) + if err != nil { + t.Fatalf("picker.Pick failed with error: %v", err) + } + for c := 0; c < 5; c++ { + pi.Done(balancer.DoneInfo{Err: errors.New("some error")}) + } + + // should eject address that always errored. + od.intervalTimerAlgorithm() + + // verify StateListener() got called with TRANSIENT_FAILURE for child + // in address that was ejected. + gotSCWS, err := scsCh.Receive(ctx) + if err != nil { + t.Fatalf("Error waiting for Sub Conn update: %v", err) + } + if err = scwsEqual(gotSCWS.(subConnWithState), subConnWithState{ + sc: scw3, + state: balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}, + }); err != nil { + t.Fatalf("Error in Sub Conn update: %v", err) + } + + // verify only one address got ejected. + sCtx, cancel = context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer cancel() + if _, err := scsCh.Receive(sCtx); err == nil { + t.Fatalf("Only one SubConn update should have been sent (only one SubConn got ejected)") + } + + // upon the Outlier Detection balancer being reconfigured with a noop + // configuration, every ejected SubConn should be unejected. + od.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: []resolver.Address{ + {Addr: "address1"}, + {Addr: "address2"}, + {Addr: "address3"}, + }, + }, + BalancerConfig: &LBConfig{ + Interval: math.MaxInt64, + BaseEjectionTime: iserviceconfig.Duration(30 * time.Second), + MaxEjectionTime: iserviceconfig.Duration(300 * time.Second), + MaxEjectionPercent: 10, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: t.Name(), + Config: emptyChildConfig{}, + }, + }, + }) + gotSCWS, err = scsCh.Receive(ctx) + if err != nil { + t.Fatalf("Error waiting for Sub Conn update: %v", err) + } + if err = scwsEqual(gotSCWS.(subConnWithState), subConnWithState{ + sc: scw3, + state: balancer.SubConnState{ConnectivityState: connectivity.Idle}, + }); err != nil { + t.Fatalf("Error in Sub Conn update: %v", err) + } + } +} + +// TestConcurrentOperations calls different operations on the balancer in +// separate goroutines to test for any race conditions and deadlocks. It also +// uses a child balancer which verifies that no operations on the child get +// called after the child balancer is closed. +func (s) TestConcurrentOperations(t *testing.T) { + closed := grpcsync.NewEvent() + stub.Register(t.Name(), stub.BalancerFuncs{ + UpdateClientConnState: func(*stub.BalancerData, balancer.ClientConnState) error { + if closed.HasFired() { + t.Error("UpdateClientConnState was called after Close(), which breaks the balancer API") + } + return nil + }, + ResolverError: func(*stub.BalancerData, error) { + if closed.HasFired() { + t.Error("ResolverError was called after Close(), which breaks the balancer API") + } + }, + Close: func(*stub.BalancerData) { + closed.Fire() + }, + ExitIdle: func(*stub.BalancerData) { + if closed.HasFired() { + t.Error("ExitIdle was called after Close(), which breaks the balancer API") + } + }, + }) + + od, tcc, cleanup := setup(t) + defer func() { + cleanup() + }() + + od.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: []resolver.Address{ + {Addr: "address1"}, + {Addr: "address2"}, + {Addr: "address3"}, + }, + }, + BalancerConfig: &LBConfig{ + Interval: math.MaxInt64, // so the interval will never run unless called manually in test. + BaseEjectionTime: iserviceconfig.Duration(30 * time.Second), + MaxEjectionTime: iserviceconfig.Duration(300 * time.Second), + MaxEjectionPercent: 10, + SuccessRateEjection: &SuccessRateEjection{ // Have both Success Rate and Failure Percentage to step through all the interval timer code + StdevFactor: 500, + EnforcementPercentage: 100, + MinimumHosts: 3, + RequestVolume: 3, + }, + FailurePercentageEjection: &FailurePercentageEjection{ + Threshold: 50, + EnforcementPercentage: 100, + MinimumHosts: 3, + RequestVolume: 3, + }, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: t.Name(), + Config: emptyChildConfig{}, + }, + }, + }) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + scw1, err := od.NewSubConn([]resolver.Address{{Addr: "address1"}}, balancer.NewSubConnOptions{}) + if err != nil { + t.Fatalf("error in od.NewSubConn call: %v", err) + } + if err != nil { + t.Fatalf("error in od.NewSubConn call: %v", err) + } + + scw2, err := od.NewSubConn([]resolver.Address{{Addr: "address2"}}, balancer.NewSubConnOptions{}) + if err != nil { + t.Fatalf("error in od.NewSubConn call: %v", err) + } + + scw3, err := od.NewSubConn([]resolver.Address{{Addr: "address3"}}, balancer.NewSubConnOptions{}) + if err != nil { + t.Fatalf("error in od.NewSubConn call: %v", err) + } + + od.UpdateState(balancer.State{ + ConnectivityState: connectivity.Ready, + Picker: &rrPicker{ + scs: []balancer.SubConn{scw2, scw3}, + }, + }) + + var picker balancer.Picker + select { + case <-ctx.Done(): + t.Fatalf("timeout while waiting for a UpdateState call on the ClientConn") + case picker = <-tcc.NewPickerCh: + } + + finished := make(chan struct{}) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + for { + select { + case <-finished: + return + default: + } + pi, err := picker.Pick(balancer.PickInfo{}) + if err != nil { + continue + } + pi.Done(balancer.DoneInfo{}) + pi.Done(balancer.DoneInfo{Err: errors.New("some error")}) + time.Sleep(1 * time.Nanosecond) + } + }() + + wg.Add(1) + go func() { + defer wg.Done() + for { + select { + case <-finished: + return + default: + } + od.intervalTimerAlgorithm() + } + }() + + // call Outlier Detection's balancer.ClientConn operations asynchronously. + // balancer.ClientConn operations have no guarantee from the API to be + // called synchronously. + wg.Add(1) + go func() { + defer wg.Done() + for { + select { + case <-finished: + return + default: + } + od.UpdateState(balancer.State{ + ConnectivityState: connectivity.Ready, + Picker: &rrPicker{ + scs: []balancer.SubConn{scw2, scw3}, + }, + }) + time.Sleep(1 * time.Nanosecond) + } + }() + + wg.Add(1) + go func() { + defer wg.Done() + od.NewSubConn([]resolver.Address{{Addr: "address4"}}, balancer.NewSubConnOptions{}) + }() + + wg.Add(1) + go func() { + defer wg.Done() + scw1.Shutdown() + }() + + wg.Add(1) + go func() { + defer wg.Done() + od.UpdateAddresses(scw2, []resolver.Address{{Addr: "address3"}}) + }() + + // Call balancer.Balancers synchronously in this goroutine, upholding the + // balancer.Balancer API guarantee of synchronous calls. + od.UpdateClientConnState(balancer.ClientConnState{ // This will delete addresses and flip to no op + ResolverState: resolver.State{ + Addresses: []resolver.Address{{Addr: "address1"}}, + }, + BalancerConfig: &LBConfig{ + Interval: math.MaxInt64, + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: t.Name(), + Config: emptyChildConfig{}, + }, + }, + }) + + // Call balancer.Balancers synchronously in this goroutine, upholding the + // balancer.Balancer API guarantee. + od.updateSubConnState(scw1.(*subConnWrapper).SubConn, balancer.SubConnState{ + ConnectivityState: connectivity.Connecting, + }) + od.ResolverError(errors.New("some error")) + od.ExitIdle() + od.Close() + close(finished) + wg.Wait() +} diff --git a/xds/internal/balancer/outlierdetection/callcounter.go b/xds/internal/balancer/outlierdetection/callcounter.go new file mode 100644 index 000000000000..4597f727b6e0 --- /dev/null +++ b/xds/internal/balancer/outlierdetection/callcounter.go @@ -0,0 +1,66 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package outlierdetection + +import ( + "sync/atomic" + "unsafe" +) + +type bucket struct { + numSuccesses uint32 + numFailures uint32 +} + +func newCallCounter() *callCounter { + return &callCounter{ + activeBucket: unsafe.Pointer(&bucket{}), + inactiveBucket: &bucket{}, + } +} + +// callCounter has two buckets, which each count successful and failing RPC's. +// The activeBucket is used to actively count any finished RPC's, and the +// inactiveBucket is populated with this activeBucket's data every interval for +// use by the Outlier Detection algorithm. +type callCounter struct { + // activeBucket updates every time a call finishes (from picker passed to + // Client Conn), so protect pointer read with atomic load of unsafe.Pointer + // so picker does not have to grab a mutex per RPC, the critical path. + activeBucket unsafe.Pointer // bucket + inactiveBucket *bucket +} + +func (cc *callCounter) clear() { + atomic.StorePointer(&cc.activeBucket, unsafe.Pointer(&bucket{})) + cc.inactiveBucket = &bucket{} +} + +// "When the timer triggers, the inactive bucket is zeroed and swapped with the +// active bucket. Then the inactive bucket contains the number of successes and +// failures since the last time the timer triggered. Those numbers are used to +// evaluate the ejection criteria." - A50. +func (cc *callCounter) swap() { + ib := cc.inactiveBucket + *ib = bucket{} + ab := (*bucket)(atomic.SwapPointer(&cc.activeBucket, unsafe.Pointer(ib))) + cc.inactiveBucket = &bucket{ + numSuccesses: atomic.LoadUint32(&ab.numSuccesses), + numFailures: atomic.LoadUint32(&ab.numFailures), + } +} diff --git a/xds/internal/balancer/outlierdetection/callcounter_test.go b/xds/internal/balancer/outlierdetection/callcounter_test.go new file mode 100644 index 000000000000..8e4f5f29b5f8 --- /dev/null +++ b/xds/internal/balancer/outlierdetection/callcounter_test.go @@ -0,0 +1,94 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package outlierdetection + +import ( + "sync/atomic" + "testing" + "unsafe" + + "github.com/google/go-cmp/cmp" +) + +func (b1 *bucket) Equal(b2 *bucket) bool { + if b1 == nil && b2 == nil { + return true + } + if (b1 != nil) != (b2 != nil) { + return false + } + if b1.numSuccesses != b2.numSuccesses { + return false + } + return b1.numFailures == b2.numFailures +} + +func (cc1 *callCounter) Equal(cc2 *callCounter) bool { + if cc1 == nil && cc2 == nil { + return true + } + if (cc1 != nil) != (cc2 != nil) { + return false + } + ab1 := (*bucket)(atomic.LoadPointer(&cc1.activeBucket)) + ab2 := (*bucket)(atomic.LoadPointer(&cc2.activeBucket)) + if !ab1.Equal(ab2) { + return false + } + return cc1.inactiveBucket.Equal(cc2.inactiveBucket) +} + +// TestClear tests that clear on the call counter clears (everything set to 0) +// the active and inactive buckets. +func (s) TestClear(t *testing.T) { + cc := newCallCounter() + ab := (*bucket)(atomic.LoadPointer(&cc.activeBucket)) + ab.numSuccesses = 1 + ab.numFailures = 2 + cc.inactiveBucket.numSuccesses = 4 + cc.inactiveBucket.numFailures = 5 + cc.clear() + // Both the active and inactive buckets should be cleared. + ccWant := newCallCounter() + if diff := cmp.Diff(cc, ccWant); diff != "" { + t.Fatalf("callCounter is different than expected, diff (-got +want): %v", diff) + } +} + +// TestSwap tests that swap() on the callCounter successfully has the desired +// end result of inactive bucket containing the previous active buckets data, +// and the active bucket being cleared. +func (s) TestSwap(t *testing.T) { + cc := newCallCounter() + ab := (*bucket)(atomic.LoadPointer(&cc.activeBucket)) + ab.numSuccesses = 1 + ab.numFailures = 2 + cc.inactiveBucket.numSuccesses = 4 + cc.inactiveBucket.numFailures = 5 + ib := cc.inactiveBucket + cc.swap() + // Inactive should pick up active's data, active should be swapped to zeroed + // inactive. + ccWant := newCallCounter() + ccWant.inactiveBucket.numSuccesses = 1 + ccWant.inactiveBucket.numFailures = 2 + atomic.StorePointer(&ccWant.activeBucket, unsafe.Pointer(ib)) + if diff := cmp.Diff(cc, ccWant); diff != "" { + t.Fatalf("callCounter is different than expected, diff (-got +want): %v", diff) + } +} diff --git a/xds/internal/balancer/outlierdetection/config.go b/xds/internal/balancer/outlierdetection/config.go new file mode 100644 index 000000000000..196a562ed69d --- /dev/null +++ b/xds/internal/balancer/outlierdetection/config.go @@ -0,0 +1,240 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package outlierdetection + +import ( + "encoding/json" + "time" + + iserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/serviceconfig" +) + +// SuccessRateEjection is parameters for the success rate ejection algorithm. +// This algorithm monitors the request success rate for all endpoints and ejects +// individual endpoints whose success rates are statistical outliers. +type SuccessRateEjection struct { + // StddevFactor is used to determine the ejection threshold for + // success rate outlier ejection. The ejection threshold is the difference + // between the mean success rate, and the product of this factor and the + // standard deviation of the mean success rate: mean - (stdev * + // success_rate_stdev_factor). This factor is divided by a thousand to get a + // double. That is, if the desired factor is 1.9, the runtime value should + // be 1900. Defaults to 1900. + StdevFactor uint32 `json:"stdevFactor,omitempty"` + // EnforcementPercentage is the % chance that a host will be actually ejected + // when an outlier status is detected through success rate statistics. This + // setting can be used to disable ejection or to ramp it up slowly. Defaults + // to 100. + EnforcementPercentage uint32 `json:"enforcementPercentage,omitempty"` + // MinimumHosts is the number of hosts in a cluster that must have enough + // request volume to detect success rate outliers. If the number of hosts is + // less than this setting, outlier detection via success rate statistics is + // not performed for any host in the cluster. Defaults to 5. + MinimumHosts uint32 `json:"minimumHosts,omitempty"` + // RequestVolume is the minimum number of total requests that must be + // collected in one interval (as defined by the interval duration above) to + // include this host in success rate based outlier detection. If the volume + // is lower than this setting, outlier detection via success rate statistics + // is not performed for that host. Defaults to 100. + RequestVolume uint32 `json:"requestVolume,omitempty"` +} + +// For UnmarshalJSON to work correctly and set defaults without infinite +// recursion. +type successRateEjection SuccessRateEjection + +// UnmarshalJSON unmarshals JSON into SuccessRateEjection. If a +// SuccessRateEjection field is not set, that field will get its default value. +func (sre *SuccessRateEjection) UnmarshalJSON(j []byte) error { + sre.StdevFactor = 1900 + sre.EnforcementPercentage = 100 + sre.MinimumHosts = 5 + sre.RequestVolume = 100 + // Unmarshal JSON on a type with zero values for methods, including + // UnmarshalJSON. Overwrites defaults, leaves alone if not. typecast to + // avoid infinite recursion by not recalling this function and causing stack + // overflow. + return json.Unmarshal(j, (*successRateEjection)(sre)) +} + +// Equal returns whether the SuccessRateEjection is the same with the parameter. +func (sre *SuccessRateEjection) Equal(sre2 *SuccessRateEjection) bool { + if sre == nil && sre2 == nil { + return true + } + if (sre != nil) != (sre2 != nil) { + return false + } + if sre.StdevFactor != sre2.StdevFactor { + return false + } + if sre.EnforcementPercentage != sre2.EnforcementPercentage { + return false + } + if sre.MinimumHosts != sre2.MinimumHosts { + return false + } + return sre.RequestVolume == sre2.RequestVolume +} + +// FailurePercentageEjection is parameters for the failure percentage algorithm. +// This algorithm ejects individual endpoints whose failure rate is greater than +// some threshold, independently of any other endpoint. +type FailurePercentageEjection struct { + // Threshold is the failure percentage to use when determining failure + // percentage-based outlier detection. If the failure percentage of a given + // host is greater than or equal to this value, it will be ejected. Defaults + // to 85. + Threshold uint32 `json:"threshold,omitempty"` + // EnforcementPercentage is the % chance that a host will be actually + // ejected when an outlier status is detected through failure percentage + // statistics. This setting can be used to disable ejection or to ramp it up + // slowly. Defaults to 0. + EnforcementPercentage uint32 `json:"enforcementPercentage,omitempty"` + // MinimumHosts is the minimum number of hosts in a cluster in order to + // perform failure percentage-based ejection. If the total number of hosts + // in the cluster is less than this value, failure percentage-based ejection + // will not be performed. Defaults to 5. + MinimumHosts uint32 `json:"minimumHosts,omitempty"` + // RequestVolume is the minimum number of total requests that must be + // collected in one interval (as defined by the interval duration above) to + // perform failure percentage-based ejection for this host. If the volume is + // lower than this setting, failure percentage-based ejection will not be + // performed for this host. Defaults to 50. + RequestVolume uint32 `json:"requestVolume,omitempty"` +} + +// For UnmarshalJSON to work correctly and set defaults without infinite +// recursion. +type failurePercentageEjection FailurePercentageEjection + +// UnmarshalJSON unmarshals JSON into FailurePercentageEjection. If a +// FailurePercentageEjection field is not set, that field will get its default +// value. +func (fpe *FailurePercentageEjection) UnmarshalJSON(j []byte) error { + fpe.Threshold = 85 + fpe.EnforcementPercentage = 0 + fpe.MinimumHosts = 5 + fpe.RequestVolume = 50 + // Unmarshal JSON on a type with zero values for methods, including + // UnmarshalJSON. Overwrites defaults, leaves alone if not. typecast to + // avoid infinite recursion by not recalling this function and causing stack + // overflow. + return json.Unmarshal(j, (*failurePercentageEjection)(fpe)) +} + +// Equal returns whether the FailurePercentageEjection is the same with the +// parameter. +func (fpe *FailurePercentageEjection) Equal(fpe2 *FailurePercentageEjection) bool { + if fpe == nil && fpe2 == nil { + return true + } + if (fpe != nil) != (fpe2 != nil) { + return false + } + if fpe.Threshold != fpe2.Threshold { + return false + } + if fpe.EnforcementPercentage != fpe2.EnforcementPercentage { + return false + } + if fpe.MinimumHosts != fpe2.MinimumHosts { + return false + } + return fpe.RequestVolume == fpe2.RequestVolume +} + +// LBConfig is the config for the outlier detection balancer. +type LBConfig struct { + serviceconfig.LoadBalancingConfig `json:"-"` + // Interval is the time interval between ejection analysis sweeps. This can + // result in both new ejections as well as addresses being returned to + // service. Defaults to 10s. + Interval iserviceconfig.Duration `json:"interval,omitempty"` + // BaseEjectionTime is the base time that a host is ejected for. The real + // time is equal to the base time multiplied by the number of times the host + // has been ejected and is capped by MaxEjectionTime. Defaults to 30s. + BaseEjectionTime iserviceconfig.Duration `json:"baseEjectionTime,omitempty"` + // MaxEjectionTime is the maximum time that an address is ejected for. If + // not specified, the default value (300s) or the BaseEjectionTime value is + // applied, whichever is larger. + MaxEjectionTime iserviceconfig.Duration `json:"maxEjectionTime,omitempty"` + // MaxEjectionPercent is the maximum % of an upstream cluster that can be + // ejected due to outlier detection. Defaults to 10% but will eject at least + // one host regardless of the value. + MaxEjectionPercent uint32 `json:"maxEjectionPercent,omitempty"` + // SuccessRateEjection is the parameters for the success rate ejection + // algorithm. If set, success rate ejections will be performed. + SuccessRateEjection *SuccessRateEjection `json:"successRateEjection,omitempty"` + // FailurePercentageEjection is the parameters for the failure percentage + // algorithm. If set, failure rate ejections will be performed. + FailurePercentageEjection *FailurePercentageEjection `json:"failurePercentageEjection,omitempty"` + // ChildPolicy is the config for the child policy. + ChildPolicy *iserviceconfig.BalancerConfig `json:"childPolicy,omitempty"` +} + +// For UnmarshalJSON to work correctly and set defaults without infinite +// recursion. +type lbConfig LBConfig + +// UnmarshalJSON unmarshals JSON into LBConfig. If a top level LBConfig field +// (i.e. not next layer sre or fpe) is not set, that field will get its default +// value. If sre or fpe is not set, it will stay unset, otherwise it will +// unmarshal on those types populating with default values for their fields if +// needed. +func (lbc *LBConfig) UnmarshalJSON(j []byte) error { + // Default top layer values as documented in A50. + lbc.Interval = iserviceconfig.Duration(10 * time.Second) + lbc.BaseEjectionTime = iserviceconfig.Duration(30 * time.Second) + lbc.MaxEjectionTime = iserviceconfig.Duration(300 * time.Second) + lbc.MaxEjectionPercent = 10 + // Unmarshal JSON on a type with zero values for methods, including + // UnmarshalJSON. Overwrites defaults, leaves alone if not. typecast to + // avoid infinite recursion by not recalling this function and causing stack + // overflow. + return json.Unmarshal(j, (*lbConfig)(lbc)) +} + +// EqualIgnoringChildPolicy returns whether the LBConfig is same with the +// parameter outside of the child policy, only comparing the Outlier Detection +// specific configuration. +func (lbc *LBConfig) EqualIgnoringChildPolicy(lbc2 *LBConfig) bool { + if lbc == nil && lbc2 == nil { + return true + } + if (lbc != nil) != (lbc2 != nil) { + return false + } + if lbc.Interval != lbc2.Interval { + return false + } + if lbc.BaseEjectionTime != lbc2.BaseEjectionTime { + return false + } + if lbc.MaxEjectionTime != lbc2.MaxEjectionTime { + return false + } + if lbc.MaxEjectionPercent != lbc2.MaxEjectionPercent { + return false + } + if !lbc.SuccessRateEjection.Equal(lbc2.SuccessRateEjection) { + return false + } + return lbc.FailurePercentageEjection.Equal(lbc2.FailurePercentageEjection) +} diff --git a/xds/internal/balancer/outlierdetection/config_test.go b/xds/internal/balancer/outlierdetection/config_test.go new file mode 100644 index 000000000000..ce924dca1bc6 --- /dev/null +++ b/xds/internal/balancer/outlierdetection/config_test.go @@ -0,0 +1,72 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package outlierdetection + +import ( + "reflect" + "testing" +) + +func TestSuccessRateEjection(t *testing.T) { + fields := map[string]bool{ + "StdevFactor": true, + "EnforcementPercentage": true, + "MinimumHosts": true, + "RequestVolume": true, + } + typ := reflect.TypeOf(SuccessRateEjection{}) + for i := 0; i < typ.NumField(); i++ { + if n := typ.Field(i).Name; !fields[n] { + t.Errorf("New field in SuccessRateEjection %q, update this test and Equal", n) + } + } +} + +func TestEqualFieldsFailurePercentageEjection(t *testing.T) { + fields := map[string]bool{ + "Threshold": true, + "EnforcementPercentage": true, + "MinimumHosts": true, + "RequestVolume": true, + } + typ := reflect.TypeOf(FailurePercentageEjection{}) + for i := 0; i < typ.NumField(); i++ { + if n := typ.Field(i).Name; !fields[n] { + t.Errorf("New field in FailurePercentageEjection %q, update this test and Equal", n) + } + } +} + +func TestEqualFieldsLBConfig(t *testing.T) { + fields := map[string]bool{ + "LoadBalancingConfig": true, + "Interval": true, + "BaseEjectionTime": true, + "MaxEjectionTime": true, + "MaxEjectionPercent": true, + "SuccessRateEjection": true, + "FailurePercentageEjection": true, + "ChildPolicy": true, + } + typ := reflect.TypeOf(LBConfig{}) + for i := 0; i < typ.NumField(); i++ { + if n := typ.Field(i).Name; !fields[n] { + t.Errorf("New field in LBConfig %q, update this test and EqualIgnoringChildPolicy", n) + } + } +} diff --git a/xds/internal/balancer/outlierdetection/e2e_test/outlierdetection_test.go b/xds/internal/balancer/outlierdetection/e2e_test/outlierdetection_test.go new file mode 100644 index 000000000000..b1f59a3c545c --- /dev/null +++ b/xds/internal/balancer/outlierdetection/e2e_test/outlierdetection_test.go @@ -0,0 +1,369 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package e2e_test contains e2e test cases for the Outlier Detection LB Policy. +package e2e_test + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/stubserver" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + "google.golang.org/grpc/serviceconfig" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + + _ "google.golang.org/grpc/xds/internal/balancer/outlierdetection" // To register helper functions which register/unregister Outlier Detection LB Policy. +) + +var defaultTestTimeout = 5 * time.Second + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// Setup spins up three test backends, each listening on a port on localhost. +// Two of the backends are configured to always reply with an empty response and +// no error and one is configured to always return an error. +func setupBackends(t *testing.T) ([]string, func()) { + t.Helper() + + backends := make([]*stubserver.StubServer, 3) + addresses := make([]string, 3) + // Construct and start 2 working backends. + for i := 0; i < 2; i++ { + backend := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil + }, + } + if err := backend.StartServer(); err != nil { + t.Fatalf("Failed to start backend: %v", err) + } + t.Logf("Started good TestService backend at: %q", backend.Address) + backends[i] = backend + addresses[i] = backend.Address + } + + // Construct and start a failing backend. + backend := &stubserver.StubServer{ + EmptyCallF: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { + return nil, errors.New("some error") + }, + } + if err := backend.StartServer(); err != nil { + t.Fatalf("Failed to start backend: %v", err) + } + t.Logf("Started bad TestService backend at: %q", backend.Address) + backends[2] = backend + addresses[2] = backend.Address + cancel := func() { + for _, backend := range backends { + backend.Stop() + } + } + return addresses, cancel +} + +// checkRoundRobinRPCs verifies that EmptyCall RPCs on the given ClientConn, +// connected to a server exposing the test.grpc_testing.TestService, are +// roundrobined across the given backend addresses. +// +// Returns a non-nil error if context deadline expires before RPCs start to get +// roundrobined across the given backends. +func checkRoundRobinRPCs(ctx context.Context, client testgrpc.TestServiceClient, addrs []resolver.Address) error { + wantAddrCount := make(map[string]int) + for _, addr := range addrs { + wantAddrCount[addr.Addr]++ + } + gotAddrCount := make(map[string]int) + for ; ctx.Err() == nil; <-time.After(time.Millisecond) { + gotAddrCount = make(map[string]int) + // Perform 3 iterations. + var iterations [][]string + for i := 0; i < 3; i++ { + iteration := make([]string, len(addrs)) + for c := 0; c < len(addrs); c++ { + var peer peer.Peer + client.EmptyCall(ctx, &testpb.Empty{}, grpc.Peer(&peer)) + if peer.Addr != nil { + iteration[c] = peer.Addr.String() + } + } + iterations = append(iterations, iteration) + } + // Ensure the the first iteration contains all addresses in addrs. + for _, addr := range iterations[0] { + gotAddrCount[addr]++ + } + if diff := cmp.Diff(gotAddrCount, wantAddrCount); diff != "" { + continue + } + // Ensure all three iterations contain the same addresses. + if !cmp.Equal(iterations[0], iterations[1]) || !cmp.Equal(iterations[0], iterations[2]) { + continue + } + return nil + } + return fmt.Errorf("timeout when waiting for roundrobin distribution of RPCs across addresses: %v; got: %v", addrs, gotAddrCount) +} + +// TestOutlierDetectionAlgorithmsE2E tests the Outlier Detection Success Rate +// and Failure Percentage algorithms in an e2e fashion. The Outlier Detection +// Balancer is configured as the top level LB Policy of the channel with a Round +// Robin child, and connects to three upstreams. Two of the upstreams are healthy and +// one is unhealthy. The two algorithms should at some point eject the failing +// upstream, causing RPC's to not be routed to that upstream, and only be +// Round Robined across the two healthy upstreams. Other than the intervals the +// unhealthy upstream is ejected, RPC's should regularly round robin +// across all three upstreams. +func (s) TestOutlierDetectionAlgorithmsE2E(t *testing.T) { + tests := []struct { + name string + odscJSON string + }{ + { + name: "Success Rate Algorithm", + odscJSON: ` +{ + "loadBalancingConfig": [ + { + "outlier_detection_experimental": { + "interval": "0.050s", + "baseEjectionTime": "0.100s", + "maxEjectionTime": "300s", + "maxEjectionPercent": 33, + "successRateEjection": { + "stdevFactor": 50, + "enforcementPercentage": 100, + "minimumHosts": 3, + "requestVolume": 5 + }, + "childPolicy": [{"round_robin": {}}] + } + } + ] +}`, + }, + { + name: "Failure Percentage Algorithm", + odscJSON: ` +{ + "loadBalancingConfig": [ + { + "outlier_detection_experimental": { + "interval": "0.050s", + "baseEjectionTime": "0.100s", + "maxEjectionTime": "300s", + "maxEjectionPercent": 33, + "failurePercentageEjection": { + "threshold": 50, + "enforcementPercentage": 100, + "minimumHosts": 3, + "requestVolume": 5 + }, + "childPolicy": [{"round_robin": {}} + ] + } + } + ] +}`, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + addresses, cancel := setupBackends(t) + defer cancel() + + mr := manual.NewBuilderWithScheme("od-e2e") + defer mr.Close() + + sc := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(test.odscJSON) + // The full list of addresses. + fullAddresses := []resolver.Address{ + {Addr: addresses[0]}, + {Addr: addresses[1]}, + {Addr: addresses[2]}, + } + mr.InitialState(resolver.State{ + Addresses: fullAddresses, + ServiceConfig: sc, + }) + + cc, err := grpc.Dial(mr.Scheme()+":///", grpc.WithResolvers(mr), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) + } + defer cc.Close() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testServiceClient := testgrpc.NewTestServiceClient(cc) + + // At first, due to no statistics on each of the backends, the 3 + // upstreams should all be round robined across. + if err = checkRoundRobinRPCs(ctx, testServiceClient, fullAddresses); err != nil { + t.Fatalf("error in expected round robin: %v", err) + } + + // The addresses which don't return errors. + okAddresses := []resolver.Address{ + {Addr: addresses[0]}, + {Addr: addresses[1]}, + } + // After calling the three upstreams, one of them constantly error + // and should eventually be ejected for a period of time. This + // period of time should cause the RPC's to be round robined only + // across the two that are healthy. + if err = checkRoundRobinRPCs(ctx, testServiceClient, okAddresses); err != nil { + t.Fatalf("error in expected round robin: %v", err) + } + + // The failing upstream isn't ejected indefinitely, and eventually + // should be unejected in subsequent iterations of the interval + // algorithm as per the spec for the two specific algorithms. + if err = checkRoundRobinRPCs(ctx, testServiceClient, fullAddresses); err != nil { + t.Fatalf("error in expected round robin: %v", err) + } + }) + } +} + +// TestNoopConfiguration tests the Outlier Detection Balancer configured with a +// noop configuration. The noop configuration should cause the Outlier Detection +// Balancer to not count RPC's, and thus never eject any upstreams and continue +// to route to every upstream connected to, even if they continuously error. +// Once the Outlier Detection Balancer gets reconfigured with configuration +// requiring counting RPC's, the Outlier Detection Balancer should start +// ejecting any upstreams as specified in the configuration. +func (s) TestNoopConfiguration(t *testing.T) { + addresses, cancel := setupBackends(t) + defer cancel() + + mr := manual.NewBuilderWithScheme("od-e2e") + defer mr.Close() + + noopODServiceConfigJSON := ` +{ + "loadBalancingConfig": [ + { + "outlier_detection_experimental": { + "interval": "0.050s", + "baseEjectionTime": "0.100s", + "maxEjectionTime": "300s", + "maxEjectionPercent": 33, + "childPolicy": [{"round_robin": {}}] + } + } + ] +}` + sc := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(noopODServiceConfigJSON) + // The full list of addresses. + fullAddresses := []resolver.Address{ + {Addr: addresses[0]}, + {Addr: addresses[1]}, + {Addr: addresses[2]}, + } + mr.InitialState(resolver.State{ + Addresses: fullAddresses, + ServiceConfig: sc, + }) + cc, err := grpc.Dial(mr.Scheme()+":///", grpc.WithResolvers(mr), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("grpc.Dial() failed: %v", err) + } + defer cc.Close() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + testServiceClient := testgrpc.NewTestServiceClient(cc) + + for i := 0; i < 2; i++ { + // Since the Outlier Detection Balancer starts with a noop + // configuration, it shouldn't count RPCs or eject any upstreams. Thus, + // even though an upstream it connects to constantly errors, it should + // continue to Round Robin across every upstream. + if err := checkRoundRobinRPCs(ctx, testServiceClient, fullAddresses); err != nil { + t.Fatalf("error in expected round robin: %v", err) + } + } + + // Reconfigure the Outlier Detection Balancer with a configuration that + // specifies to count RPC's and eject upstreams. Due to the balancer no + // longer being a noop, it should eject any unhealthy addresses as specified + // by the failure percentage portion of the configuration. + countingODServiceConfigJSON := ` +{ + "loadBalancingConfig": [ + { + "outlier_detection_experimental": { + "interval": "0.050s", + "baseEjectionTime": "0.100s", + "maxEjectionTime": "300s", + "maxEjectionPercent": 33, + "failurePercentageEjection": { + "threshold": 50, + "enforcementPercentage": 100, + "minimumHosts": 3, + "requestVolume": 5 + }, + "childPolicy": [{"round_robin": {}}] + } + } + ] +}` + sc = internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(countingODServiceConfigJSON) + + mr.UpdateState(resolver.State{ + Addresses: fullAddresses, + ServiceConfig: sc, + }) + + // At first on the reconfigured balancer, the balancer has no stats + // collected about upstreams. Thus, it should at first route across the full + // upstream list. + if err = checkRoundRobinRPCs(ctx, testServiceClient, fullAddresses); err != nil { + t.Fatalf("error in expected round robin: %v", err) + } + + // The addresses which don't return errors. + okAddresses := []resolver.Address{ + {Addr: addresses[0]}, + {Addr: addresses[1]}, + } + // Now that the reconfigured balancer has data about the failing upstream, + // it should eject the upstream and only route across the two healthy + // upstreams. + if err = checkRoundRobinRPCs(ctx, testServiceClient, okAddresses); err != nil { + t.Fatalf("error in expected round robin: %v", err) + } +} diff --git a/xds/internal/balancer/outlierdetection/logging.go b/xds/internal/balancer/outlierdetection/logging.go new file mode 100644 index 000000000000..705b0cb6918d --- /dev/null +++ b/xds/internal/balancer/outlierdetection/logging.go @@ -0,0 +1,34 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package outlierdetection + +import ( + "fmt" + + "google.golang.org/grpc/grpclog" + internalgrpclog "google.golang.org/grpc/internal/grpclog" +) + +const prefix = "[outlier-detection-lb %p] " + +var logger = grpclog.Component("xds") + +func prefixLogger(p *outlierDetectionBalancer) *internalgrpclog.PrefixLogger { + return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p)) +} diff --git a/xds/internal/balancer/outlierdetection/subconn_wrapper.go b/xds/internal/balancer/outlierdetection/subconn_wrapper.go new file mode 100644 index 000000000000..0fa422d8f262 --- /dev/null +++ b/xds/internal/balancer/outlierdetection/subconn_wrapper.go @@ -0,0 +1,74 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package outlierdetection + +import ( + "fmt" + "unsafe" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/internal/buffer" + "google.golang.org/grpc/resolver" +) + +// subConnWrapper wraps every created SubConn in the Outlier Detection Balancer, +// to help track the latest state update from the underlying SubConn, and also +// whether or not this SubConn is ejected. +type subConnWrapper struct { + balancer.SubConn + listener func(balancer.SubConnState) + + // addressInfo is a pointer to the subConnWrapper's corresponding address + // map entry, if the map entry exists. + addressInfo unsafe.Pointer // *addressInfo + // These two pieces of state will reach eventual consistency due to sync in + // run(), and child will always have the correctly updated SubConnState. + // latestState is the latest state update from the underlying SubConn. This + // is used whenever a SubConn gets unejected. + latestState balancer.SubConnState + ejected bool + + scUpdateCh *buffer.Unbounded + + // addresses is the list of address(es) this SubConn was created with to + // help support any change in address(es) + addresses []resolver.Address +} + +// eject causes the wrapper to report a state update with the TRANSIENT_FAILURE +// state, and to stop passing along updates from the underlying subchannel. +func (scw *subConnWrapper) eject() { + scw.scUpdateCh.Put(&ejectionUpdate{ + scw: scw, + isEjected: true, + }) +} + +// uneject causes the wrapper to report a state update with the latest update +// from the underlying subchannel, and resume passing along updates from the +// underlying subchannel. +func (scw *subConnWrapper) uneject() { + scw.scUpdateCh.Put(&ejectionUpdate{ + scw: scw, + isEjected: false, + }) +} + +func (scw *subConnWrapper) String() string { + return fmt.Sprintf("%+v", scw.addresses) +} diff --git a/xds/internal/balancer/priority/balancer.go b/xds/internal/balancer/priority/balancer.go new file mode 100644 index 000000000000..7efbe402a8e5 --- /dev/null +++ b/xds/internal/balancer/priority/balancer.go @@ -0,0 +1,286 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package priority implements the priority balancer. +// +// This balancer will be kept in internal until we use it in the xds balancers, +// and are confident its functionalities are stable. It will then be exported +// for more users. +package priority + +import ( + "encoding/json" + "fmt" + "sync" + "time" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/base" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/internal/balancergroup" + "google.golang.org/grpc/internal/buffer" + "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/hierarchy" + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" +) + +// Name is the name of the priority balancer. +const Name = "priority_experimental" + +// DefaultSubBalancerCloseTimeout is defined as a variable instead of const for +// testing. +var DefaultSubBalancerCloseTimeout = 15 * time.Minute + +func init() { + balancer.Register(bb{}) +} + +type bb struct{} + +func (bb) Build(cc balancer.ClientConn, bOpts balancer.BuildOptions) balancer.Balancer { + b := &priorityBalancer{ + cc: cc, + done: grpcsync.NewEvent(), + children: make(map[string]*childBalancer), + childBalancerStateUpdate: buffer.NewUnbounded(), + } + + b.logger = prefixLogger(b) + b.bg = balancergroup.New(balancergroup.Options{ + CC: cc, + BuildOpts: bOpts, + StateAggregator: b, + Logger: b.logger, + SubBalancerCloseTimeout: DefaultSubBalancerCloseTimeout, + }) + b.bg.Start() + go b.run() + b.logger.Infof("Created") + return b +} + +func (b bb) ParseConfig(s json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + return parseConfig(s) +} + +func (bb) Name() string { + return Name +} + +// timerWrapper wraps a timer with a boolean. So that when a race happens +// between AfterFunc and Stop, the func is guaranteed to not execute. +type timerWrapper struct { + stopped bool + timer *time.Timer +} + +type priorityBalancer struct { + logger *grpclog.PrefixLogger + cc balancer.ClientConn + bg *balancergroup.BalancerGroup + done *grpcsync.Event + childBalancerStateUpdate *buffer.Unbounded + + mu sync.Mutex + childInUse string + // priorities is a list of child names from higher to lower priority. + priorities []string + // children is a map from child name to sub-balancers. + children map[string]*childBalancer + + // Set during UpdateClientConnState when calling into sub-balancers. + // Prevents child updates from recomputing the active priority or sending + // an update of the aggregated picker to the parent. Cleared after all + // sub-balancers have finished UpdateClientConnState, after which + // syncPriority is called manually. + inhibitPickerUpdates bool +} + +func (b *priorityBalancer) UpdateClientConnState(s balancer.ClientConnState) error { + b.logger.Debugf("Received an update with balancer config: %+v", pretty.ToJSON(s.BalancerConfig)) + newConfig, ok := s.BalancerConfig.(*LBConfig) + if !ok { + return fmt.Errorf("unexpected balancer config with type: %T", s.BalancerConfig) + } + addressesSplit := hierarchy.Group(s.ResolverState.Addresses) + + b.mu.Lock() + // Create and remove children, since we know all children from the config + // are used by some priority. + for name, newSubConfig := range newConfig.Children { + bb := balancer.Get(newSubConfig.Config.Name) + if bb == nil { + b.logger.Errorf("balancer name %v from config is not registered", newSubConfig.Config.Name) + continue + } + + currentChild, ok := b.children[name] + if !ok { + // This is a new child, add it to the children list. But note that + // the balancer isn't built, because this child can be a low + // priority. If necessary, it will be built when syncing priorities. + cb := newChildBalancer(name, b, bb.Name(), b.cc) + cb.updateConfig(newSubConfig, resolver.State{ + Addresses: addressesSplit[name], + ServiceConfig: s.ResolverState.ServiceConfig, + Attributes: s.ResolverState.Attributes, + }) + b.children[name] = cb + continue + } + + // This is not a new child. But the config/addresses could change. + + // The balancing policy name is changed, close the old child. But don't + // rebuild, rebuild will happen when syncing priorities. + if currentChild.balancerName != bb.Name() { + currentChild.stop() + currentChild.updateBalancerName(bb.Name()) + } + + // Update config and address, but note that this doesn't send the + // updates to non-started child balancers (the child balancer might not + // be built, if it's a low priority). + currentChild.updateConfig(newSubConfig, resolver.State{ + Addresses: addressesSplit[name], + ServiceConfig: s.ResolverState.ServiceConfig, + Attributes: s.ResolverState.Attributes, + }) + } + // Cleanup resources used by children removed from the config. + for name, oldChild := range b.children { + if _, ok := newConfig.Children[name]; !ok { + oldChild.stop() + delete(b.children, name) + } + } + + // Update priorities and handle priority changes. + b.priorities = newConfig.Priorities + + // Everything was removed by the update. + if len(b.priorities) == 0 { + b.childInUse = "" + b.cc.UpdateState(balancer.State{ + ConnectivityState: connectivity.TransientFailure, + Picker: base.NewErrPicker(ErrAllPrioritiesRemoved), + }) + b.mu.Unlock() + return nil + } + + // This will sync the states of all children to the new updated + // priorities. Includes starting/stopping child balancers when necessary. + // Block picker updates until all children have had a chance to call + // UpdateState to prevent races where, e.g., the active priority reports + // transient failure but a higher priority may have reported something that + // made it active, and if the transient failure update is handled first, + // RPCs could fail. + b.inhibitPickerUpdates = true + // Add an item to queue to notify us when the current items in the queue + // are done and syncPriority has been called. + done := make(chan struct{}) + b.childBalancerStateUpdate.Put(resumePickerUpdates{done: done}) + b.mu.Unlock() + <-done + + return nil +} + +func (b *priorityBalancer) ResolverError(err error) { + b.bg.ResolverError(err) +} + +func (b *priorityBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { + b.logger.Errorf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, state) +} + +func (b *priorityBalancer) Close() { + b.bg.Close() + b.childBalancerStateUpdate.Close() + + b.mu.Lock() + defer b.mu.Unlock() + b.done.Fire() + // Clear states of the current child in use, so if there's a race in picker + // update, it will be dropped. + b.childInUse = "" + // Stop the child policies, this is necessary to stop the init timers in the + // children. + for _, child := range b.children { + child.stop() + } +} + +func (b *priorityBalancer) ExitIdle() { + b.bg.ExitIdle() +} + +// UpdateState implements balancergroup.BalancerStateAggregator interface. The +// balancer group sends new connectivity state and picker here. +func (b *priorityBalancer) UpdateState(childName string, state balancer.State) { + b.childBalancerStateUpdate.Put(childBalancerState{ + name: childName, + s: state, + }) +} + +type childBalancerState struct { + name string + s balancer.State +} + +type resumePickerUpdates struct { + done chan struct{} +} + +// run handles child update in a separate goroutine, so if the child sends +// updates inline (when called by parent), it won't cause deadlocks (by trying +// to hold the same mutex). +func (b *priorityBalancer) run() { + for { + select { + case u, ok := <-b.childBalancerStateUpdate.Get(): + if !ok { + return + } + b.childBalancerStateUpdate.Load() + // Needs to handle state update in a goroutine, because each state + // update needs to start/close child policy, could result in + // deadlock. + b.mu.Lock() + if b.done.HasFired() { + return + } + switch s := u.(type) { + case childBalancerState: + b.handleChildStateUpdate(s.name, s.s) + case resumePickerUpdates: + b.inhibitPickerUpdates = false + b.syncPriority(b.childInUse) + close(s.done) + } + b.mu.Unlock() + case <-b.done.Done(): + return + } + } +} diff --git a/xds/internal/balancer/priority/balancer_child.go b/xds/internal/balancer/priority/balancer_child.go new file mode 100644 index 000000000000..7e8ccbd335e9 --- /dev/null +++ b/xds/internal/balancer/priority/balancer_child.go @@ -0,0 +1,173 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package priority + +import ( + "time" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/base" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" +) + +type childBalancer struct { + name string + parent *priorityBalancer + parentCC balancer.ClientConn + balancerName string + cc *ignoreResolveNowClientConn + + ignoreReresolutionRequests bool + config serviceconfig.LoadBalancingConfig + rState resolver.State + + started bool + // This is set when the child reports TransientFailure, and unset when it + // reports Ready or Idle. It is used to decide whether the failover timer + // should start when the child is transitioning into Connecting. The timer + // will be restarted if the child has not reported TF more recently than it + // reported Ready or Idle. + reportedTF bool + // The latest state the child balancer provided. + state balancer.State + // The timer to give a priority some time to connect. And if the priority + // doesn't go into Ready/Failure, the next priority will be started. + initTimer *timerWrapper +} + +// newChildBalancer creates a child balancer place holder, but doesn't +// build/start the child balancer. +func newChildBalancer(name string, parent *priorityBalancer, balancerName string, cc balancer.ClientConn) *childBalancer { + return &childBalancer{ + name: name, + parent: parent, + parentCC: cc, + balancerName: balancerName, + cc: newIgnoreResolveNowClientConn(cc, false), + started: false, + // Start with the connecting state and picker with re-pick error, so + // that when a priority switch causes this child picked before it's + // balancing policy is created, a re-pick will happen. + state: balancer.State{ + ConnectivityState: connectivity.Connecting, + Picker: base.NewErrPicker(balancer.ErrNoSubConnAvailable), + }, + } +} + +// updateBalancerName updates balancer name for the child, but doesn't build a +// new one. The parent priority LB always closes the child policy before +// updating the balancer name, and the new balancer is built when it gets added +// to the balancergroup as part of start(). +func (cb *childBalancer) updateBalancerName(balancerName string) { + cb.balancerName = balancerName + cb.cc = newIgnoreResolveNowClientConn(cb.parentCC, cb.ignoreReresolutionRequests) +} + +// updateConfig sets childBalancer's config and state, but doesn't send update to +// the child balancer unless it is started. +func (cb *childBalancer) updateConfig(child *Child, rState resolver.State) { + cb.ignoreReresolutionRequests = child.IgnoreReresolutionRequests + cb.config = child.Config.Config + cb.rState = rState + if cb.started { + cb.sendUpdate() + } +} + +// start builds the child balancer if it's not already started. +// +// It doesn't do it directly. It asks the balancer group to build it. +func (cb *childBalancer) start() { + if cb.started { + return + } + cb.started = true + cb.parent.bg.AddWithClientConn(cb.name, cb.balancerName, cb.cc) + cb.startInitTimer() + cb.sendUpdate() +} + +// sendUpdate sends the addresses and config to the child balancer. +func (cb *childBalancer) sendUpdate() { + cb.cc.updateIgnoreResolveNow(cb.ignoreReresolutionRequests) + // TODO: return and aggregate the returned error in the parent. + err := cb.parent.bg.UpdateClientConnState(cb.name, balancer.ClientConnState{ + ResolverState: cb.rState, + BalancerConfig: cb.config, + }) + if err != nil { + cb.parent.logger.Warningf("Failed to update state for child policy %q: %v", cb.name, err) + } +} + +// stop stops the child balancer and resets the state. +// +// It doesn't do it directly. It asks the balancer group to remove it. +// +// Note that the underlying balancer group could keep the child in a cache. +func (cb *childBalancer) stop() { + if !cb.started { + return + } + cb.stopInitTimer() + cb.parent.bg.Remove(cb.name) + cb.started = false + cb.state = balancer.State{ + ConnectivityState: connectivity.Connecting, + Picker: base.NewErrPicker(balancer.ErrNoSubConnAvailable), + } + // Clear child.reportedTF, so that if this child is started later, it will + // be given time to connect. + cb.reportedTF = false +} + +func (cb *childBalancer) startInitTimer() { + if cb.initTimer != nil { + return + } + // Need this local variable to capture timerW in the AfterFunc closure + // to check the stopped boolean. + timerW := &timerWrapper{} + cb.initTimer = timerW + timerW.timer = time.AfterFunc(DefaultPriorityInitTimeout, func() { + cb.parent.mu.Lock() + defer cb.parent.mu.Unlock() + if timerW.stopped { + return + } + cb.initTimer = nil + // Re-sync the priority. This will switch to the next priority if + // there's any. Note that it's important sync() is called after setting + // initTimer to nil. + cb.parent.syncPriority("") + }) +} + +func (cb *childBalancer) stopInitTimer() { + timerW := cb.initTimer + if timerW == nil { + return + } + cb.initTimer = nil + timerW.stopped = true + timerW.timer.Stop() +} diff --git a/xds/internal/balancer/priority/balancer_priority.go b/xds/internal/balancer/priority/balancer_priority.go new file mode 100644 index 000000000000..4655bf418474 --- /dev/null +++ b/xds/internal/balancer/priority/balancer_priority.go @@ -0,0 +1,204 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package priority + +import ( + "errors" + "time" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/connectivity" +) + +var ( + // ErrAllPrioritiesRemoved is returned by the picker when there's no priority available. + ErrAllPrioritiesRemoved = errors.New("no priority is provided, all priorities are removed") + // DefaultPriorityInitTimeout is the timeout after which if a priority is + // not READY, the next will be started. It's exported to be overridden by + // tests. + DefaultPriorityInitTimeout = 10 * time.Second +) + +// syncPriority handles priority after a config update or a child balancer +// connectivity state update. It makes sure the balancer state (started or not) +// is in sync with the priorities (even in tricky cases where a child is moved +// from a priority to another). +// +// It's guaranteed that after this function returns: +// +// If some child is READY, it is childInUse, and all lower priorities are +// closed. +// +// If some child is newly started(in Connecting for the first time), it is +// childInUse, and all lower priorities are closed. +// +// Otherwise, the lowest priority is childInUse (none of the children is +// ready, and the overall state is not ready). +// +// Steps: +// +// If all priorities were deleted, unset childInUse (to an empty string), and +// set parent ClientConn to TransientFailure +// +// Otherwise, Scan all children from p0, and check balancer stats: +// +// For any of the following cases: +// +// If balancer is not started (not built), this is either a new child with +// high priority, or a new builder for an existing child. +// +// If balancer is Connecting and has non-nil initTimer (meaning it +// transitioned from Ready or Idle to connecting, not from TF, so we +// should give it init-time to connect). +// +// If balancer is READY or IDLE +// +// If this is the lowest priority +// +// do the following: +// +// if this is not the old childInUse, override picker so old picker is no +// longer used. +// +// switch to it (because all higher priorities are neither new or Ready) +// +// forward the new addresses and config +// +// Caller must hold b.mu. +func (b *priorityBalancer) syncPriority(childUpdating string) { + if b.inhibitPickerUpdates { + b.logger.Debugf("Skipping update from child policy %q", childUpdating) + return + } + for p, name := range b.priorities { + child, ok := b.children[name] + if !ok { + b.logger.Warningf("Priority name %q is not found in list of child policies", name) + continue + } + + if !child.started || + child.state.ConnectivityState == connectivity.Ready || + child.state.ConnectivityState == connectivity.Idle || + (child.state.ConnectivityState == connectivity.Connecting && child.initTimer != nil) || + p == len(b.priorities)-1 { + if b.childInUse != child.name || child.name == childUpdating { + b.logger.Debugf("childInUse, childUpdating: %q, %q", b.childInUse, child.name) + // If we switch children or the child in use just updated its + // picker, push the child's picker to the parent. + b.cc.UpdateState(child.state) + } + b.logger.Debugf("Switching to (%q, %v) in syncPriority", child.name, p) + b.switchToChild(child, p) + break + } + } +} + +// Stop priorities [p+1, lowest]. +// +// Caller must hold b.mu. +func (b *priorityBalancer) stopSubBalancersLowerThanPriority(p int) { + for i := p + 1; i < len(b.priorities); i++ { + name := b.priorities[i] + child, ok := b.children[name] + if !ok { + b.logger.Warningf("Priority name %q is not found in list of child policies", name) + continue + } + child.stop() + } +} + +// switchToChild does the following: +// - stop all child with lower priorities +// - if childInUse is not this child +// - set childInUse to this child +// - if this child is not started, start it +// +// Note that it does NOT send the current child state (picker) to the parent +// ClientConn. The caller needs to send it if necessary. +// +// this can be called when +// 1. first update, start p0 +// 2. an update moves a READY child from a lower priority to higher +// 2. a different builder is updated for this child +// 3. a high priority goes Failure, start next +// 4. a high priority init timeout, start next +// +// Caller must hold b.mu. +func (b *priorityBalancer) switchToChild(child *childBalancer, priority int) { + // Stop lower priorities even if childInUse is same as this child. It's + // possible this child was moved from a priority to another. + b.stopSubBalancersLowerThanPriority(priority) + + // If this child is already in use, do nothing. + // + // This can happen: + // - all priorities are not READY, an config update always triggers switch + // to the lowest. In this case, the lowest child could still be connecting, + // so we don't stop the init timer. + // - a high priority is READY, an config update always triggers switch to + // it. + if b.childInUse == child.name && child.started { + return + } + b.childInUse = child.name + + if !child.started { + child.start() + } +} + +// handleChildStateUpdate start/close priorities based on the connectivity +// state. +func (b *priorityBalancer) handleChildStateUpdate(childName string, s balancer.State) { + // Update state in child. The updated picker will be sent to parent later if + // necessary. + child, ok := b.children[childName] + if !ok { + b.logger.Warningf("Child policy not found for %q", childName) + return + } + if !child.started { + b.logger.Warningf("Ignoring update from child policy %q which is not in started state: %+v", childName, s) + return + } + child.state = s + + // We start/stop the init timer of this child based on the new connectivity + // state. syncPriority() later will need the init timer (to check if it's + // nil or not) to decide which child to switch to. + switch s.ConnectivityState { + case connectivity.Ready, connectivity.Idle: + child.reportedTF = false + child.stopInitTimer() + case connectivity.TransientFailure: + child.reportedTF = true + child.stopInitTimer() + case connectivity.Connecting: + if !child.reportedTF { + child.startInitTimer() + } + default: + // New state is Shutdown, should never happen. Don't forward. + } + + child.parent.syncPriority(childName) +} diff --git a/xds/internal/balancer/priority/balancer_test.go b/xds/internal/balancer/priority/balancer_test.go new file mode 100644 index 000000000000..efdd280d029e --- /dev/null +++ b/xds/internal/balancer/priority/balancer_test.go @@ -0,0 +1,2034 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package priority + +import ( + "context" + "fmt" + "testing" + "time" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/roundrobin" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/internal/balancer/stub" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/hierarchy" + internalserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/resolver" +) + +const ( + defaultTestTimeout = 5 * time.Second + defaultTestShortTimeout = 100 * time.Millisecond +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +var testBackendAddrStrs []string + +const ( + testBackendAddrsCount = 12 + testRRBalancerName = "another-round-robin" +) + +type anotherRR struct { + balancer.Builder +} + +func (*anotherRR) Name() string { + return testRRBalancerName +} + +func init() { + for i := 0; i < testBackendAddrsCount; i++ { + testBackendAddrStrs = append(testBackendAddrStrs, fmt.Sprintf("%d.%d.%d.%d:%d", i, i, i, i, i)) + } + // Disable sub-balancer caching for all but the tests which exercise the + // caching behavior. + DefaultSubBalancerCloseTimeout = time.Duration(0) + balancer.Register(&anotherRR{Builder: balancer.Get(roundrobin.Name)}) +} + +// When a high priority is ready, adding/removing lower locality doesn't cause +// changes. +// +// Init 0 and 1; 0 is up, use 0; add 2, use 0; remove 2, use 0. +func (s) TestPriority_HighPriorityReady(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + cc := testutils.NewTestClientConn(t) + bb := balancer.Get(Name) + pb := bb.Build(cc, balancer.BuildOptions{}) + defer pb.Close() + + // Two children, with priorities [0, 1], each with one backend. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: []resolver.Address{ + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}), + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0", "child-1"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + addrs1 := <-cc.NewSubConnAddrsCh + if got, want := addrs1[0].Addr, testBackendAddrStrs[0]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc1 := <-cc.NewSubConnCh + + // p0 is ready. + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // Test roundrobin with only p0 subconns. + if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil { + t.Fatal(err.Error()) + } + + // Add p2, it shouldn't cause any updates. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: []resolver.Address{ + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}), + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}), + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[2]}, []string{"child-2"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-2": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0", "child-1", "child-2"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + select { + case sc := <-cc.NewSubConnCh: + t.Fatalf("got unexpected new SubConn: %s", sc) + case sc := <-cc.ShutdownSubConnCh: + t.Fatalf("got unexpected shutdown SubConn: %v", sc) + case <-time.After(time.Millisecond * 100): + } + + // Test roundrobin with only p0 subconns. + if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil { + t.Fatal(err.Error()) + } + + // Remove p2, no updates. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: []resolver.Address{ + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}), + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0", "child-1"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + select { + case <-cc.NewSubConnCh: + t.Fatalf("got unexpected new SubConn") + case <-cc.ShutdownSubConnCh: + t.Fatalf("got unexpected shutdown SubConn") + case <-time.After(time.Millisecond * 100): + } + + // Test roundrobin with only p0 subconns. + if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil { + t.Fatal(err.Error()) + } +} + +// Lower priority is used when higher priority is not ready. +// +// Init 0 and 1; 0 is up, use 0; 0 is down, 1 is up, use 1; add 2, use 1; 1 is +// down, use 2; remove 2, use 1. +func (s) TestPriority_SwitchPriority(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + cc := testutils.NewTestClientConn(t) + bb := balancer.Get(Name) + pb := bb.Build(cc, balancer.BuildOptions{}) + defer pb.Close() + + t.Log("Two localities, with priorities [0, 1], each with one backend.") + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: []resolver.Address{ + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}), + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0", "child-1"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + addrs0 := <-cc.NewSubConnAddrsCh + if got, want := addrs0[0].Addr, testBackendAddrStrs[0]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc0 := <-cc.NewSubConnCh + + t.Log("Make p0 ready.") + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // Test roundrobin with only p0 subconns. + if err := cc.WaitForRoundRobinPicker(ctx, sc0); err != nil { + t.Fatal(err.Error()) + } + + t.Log("Turn down 0, will start and use 1.") + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) + // Before 1 gets READY, picker should return NoSubConnAvailable, so RPCs + // will retry. + if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil { + t.Fatal(err.Error()) + } + + t.Log("Handle SubConn creation from 1.") + addrs1 := <-cc.NewSubConnAddrsCh + if got, want := addrs1[0].Addr, testBackendAddrStrs[1]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc1 := <-cc.NewSubConnCh + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // Test pick with 1. + if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil { + t.Fatal(err.Error()) + } + + t.Log("Add p2, it shouldn't cause any udpates.") + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: []resolver.Address{ + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}), + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}), + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[2]}, []string{"child-2"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-2": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0", "child-1", "child-2"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + select { + case sc := <-cc.NewSubConnCh: + t.Fatalf("got unexpected new SubConn, %s", sc) + case <-cc.ShutdownSubConnCh: + t.Fatalf("got unexpected shutdown SubConn") + case <-time.After(time.Millisecond * 100): + } + + t.Log("Turn down 1, use 2.") + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) + + // Before 2 gets READY, picker should return NoSubConnAvailable, so RPCs + // will retry. + if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil { + t.Fatal(err.Error()) + } + + addrs2 := <-cc.NewSubConnAddrsCh + if got, want := addrs2[0].Addr, testBackendAddrStrs[2]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc2 := <-cc.NewSubConnCh + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // Test pick with 2. + if err := cc.WaitForRoundRobinPicker(ctx, sc2); err != nil { + t.Fatal(err.Error()) + } + + t.Log("Remove 2, use 1.") + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: []resolver.Address{ + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}), + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0", "child-1"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // p2 SubConns are shut down. + scToShutdown := <-cc.ShutdownSubConnCh + if scToShutdown != sc2 { + t.Fatalf("ShutdownSubConn, want %v, got %v", sc2, scToShutdown) + } + + // Should get an update with 1's old transient failure picker, to override + // 2's old picker. + if err := cc.WaitForErrPicker(ctx); err != nil { + t.Fatal(err.Error()) + } + <-cc.NewStateCh // Drain to match picker + + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + // Does not change the aggregate state, because round robin does not leave + // TRANIENT_FAILURE if a subconn goes CONNECTING. + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil { + t.Fatal(err.Error()) + } +} + +// Lower priority is used when higher priority turns Connecting from Ready. +// Because changing from Ready to Connecting is a failure. +// +// Init 0 and 1; 0 is up, use 0; 0 is connecting, 1 is up, use 1; 0 is ready, +// use 0. +func (s) TestPriority_HighPriorityToConnectingFromReady(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + cc := testutils.NewTestClientConn(t) + bb := balancer.Get(Name) + pb := bb.Build(cc, balancer.BuildOptions{}) + defer pb.Close() + + // Two localities, with priorities [0, 1], each with one backend. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: []resolver.Address{ + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}), + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0", "child-1"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + addrs0 := <-cc.NewSubConnAddrsCh + if got, want := addrs0[0].Addr, testBackendAddrStrs[0]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc0 := <-cc.NewSubConnCh + + // p0 is ready. + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // Test roundrobin with only p0 subconns. + if err := cc.WaitForRoundRobinPicker(ctx, sc0); err != nil { + t.Fatal(err.Error()) + } + + // Turn 0 to TransientFailure, will start and use 1. + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) + + // Before 1 gets READY, picker should return NoSubConnAvailable, so RPCs + // will retry. + if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil { + t.Fatal(err.Error()) + } + + // Handle SubConn creation from 1. + addrs1 := <-cc.NewSubConnAddrsCh + if got, want := addrs1[0].Addr, testBackendAddrStrs[1]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc1 := <-cc.NewSubConnCh + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // Test pick with 1. + if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil { + t.Fatal(err.Error()) + } + + // Turn 0 back to Ready. + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // p1 subconn should be shut down. + scToShutdown := <-cc.ShutdownSubConnCh + if scToShutdown != sc1 { + t.Fatalf("ShutdownSubConn, want %v, got %v", sc0, scToShutdown) + } + + if err := cc.WaitForRoundRobinPicker(ctx, sc0); err != nil { + t.Fatal(err.Error()) + } +} + +// Add a lower priority while the higher priority is down. +// +// Init 0 and 1; 0 and 1 both down; add 2, use 2. +func (s) TestPriority_HigherDownWhileAddingLower(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + cc := testutils.NewTestClientConn(t) + bb := balancer.Get(Name) + pb := bb.Build(cc, balancer.BuildOptions{}) + defer pb.Close() + + // Two localities, with different priorities, each with one backend. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: []resolver.Address{ + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}), + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0", "child-1"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + addrs0 := <-cc.NewSubConnAddrsCh + if got, want := addrs0[0].Addr, testBackendAddrStrs[0]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc0 := <-cc.NewSubConnCh + + t.Log("Turn down 0, 1 is used.") + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) + + // Before 1 gets READY, picker should return NoSubConnAvailable, so RPCs + // will retry. + if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil { + t.Fatal(err.Error()) + } + + addrs1 := <-cc.NewSubConnAddrsCh + if got, want := addrs1[0].Addr, testBackendAddrStrs[1]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc1 := <-cc.NewSubConnCh + + t.Log("Turn down 1, pick should error.") + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) + + // Test pick failure. + if err := cc.WaitForErrPicker(ctx); err != nil { + t.Fatal(err.Error()) + } + <-cc.NewStateCh // Drain to match picker + + t.Log("Add p2, it should create a new SubConn.") + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: []resolver.Address{ + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}), + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}), + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[2]}, []string{"child-2"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-2": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0", "child-1", "child-2"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // A new connecting picker should be updated for the new priority. + if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil { + t.Fatal(err.Error()) + } + + addrs2 := <-cc.NewSubConnAddrsCh + if got, want := addrs2[0].Addr, testBackendAddrStrs[2]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc2 := <-cc.NewSubConnCh + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // Test pick with 2. + if err := cc.WaitForRoundRobinPicker(ctx, sc2); err != nil { + t.Fatal(err.Error()) + } +} + +// When a higher priority becomes available, all lower priorities are closed. +// +// Init 0,1,2; 0 and 1 down, use 2; 0 up, close 1 and 2. +func (s) TestPriority_HigherReadyCloseAllLower(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + cc := testutils.NewTestClientConn(t) + bb := balancer.Get(Name) + pb := bb.Build(cc, balancer.BuildOptions{}) + defer pb.Close() + + // Three localities, with priorities [0,1,2], each with one backend. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: []resolver.Address{ + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}), + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}), + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[2]}, []string{"child-2"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-2": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0", "child-1", "child-2"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + addrs0 := <-cc.NewSubConnAddrsCh + if got, want := addrs0[0].Addr, testBackendAddrStrs[0]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc0 := <-cc.NewSubConnCh + + // Turn down 0, 1 is used. + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) + // Before 1 gets READY, picker should return NoSubConnAvailable, so RPCs + // will retry. + if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil { + t.Fatal(err.Error()) + } + + addrs1 := <-cc.NewSubConnAddrsCh + if got, want := addrs1[0].Addr, testBackendAddrStrs[1]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc1 := <-cc.NewSubConnCh + + // Turn down 1, 2 is used. + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) + // Before 2 gets READY, picker should return NoSubConnAvailable, so RPCs + // will retry. + if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil { + t.Fatal(err.Error()) + } + + addrs2 := <-cc.NewSubConnAddrsCh + if got, want := addrs2[0].Addr, testBackendAddrStrs[2]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc2 := <-cc.NewSubConnCh + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // Test pick with 2. + if err := cc.WaitForRoundRobinPicker(ctx, sc2); err != nil { + t.Fatal(err.Error()) + } + + // When 0 becomes ready, 0 should be used, 1 and 2 should all be closed. + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // sc1 and sc2 should be shut down. + // + // With localities caching, the lower priorities are closed after a timeout, + // in goroutines. The order is no longer guaranteed. + scToShutdown := []balancer.SubConn{<-cc.ShutdownSubConnCh, <-cc.ShutdownSubConnCh} + if !(scToShutdown[0] == sc1 && scToShutdown[1] == sc2) && !(scToShutdown[0] == sc2 && scToShutdown[1] == sc1) { + t.Errorf("ShutdownSubConn, want [%v, %v], got %v", sc1, sc2, scToShutdown) + } + + // Test pick with 0. + if err := cc.WaitForRoundRobinPicker(ctx, sc0); err != nil { + t.Fatal(err.Error()) + } +} + +// At init, start the next lower priority after timeout if the higher priority +// doesn't get ready. +// +// Init 0,1; 0 is not ready (in connecting), after timeout, use 1. +func (s) TestPriority_InitTimeout(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + const testPriorityInitTimeout = 200 * time.Millisecond + defer func() func() { + old := DefaultPriorityInitTimeout + DefaultPriorityInitTimeout = testPriorityInitTimeout + return func() { + DefaultPriorityInitTimeout = old + } + }()() + + cc := testutils.NewTestClientConn(t) + bb := balancer.Get(Name) + pb := bb.Build(cc, balancer.BuildOptions{}) + defer pb.Close() + + // Two localities, with different priorities, each with one backend. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: []resolver.Address{ + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}), + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0", "child-1"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + addrs0 := <-cc.NewSubConnAddrsCh + if got, want := addrs0[0].Addr, testBackendAddrStrs[0]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc0 := <-cc.NewSubConnCh + + // Keep 0 in connecting, 1 will be used after init timeout. + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + + // Make sure new SubConn is created before timeout. + select { + case <-time.After(testPriorityInitTimeout * 3 / 4): + case <-cc.NewSubConnAddrsCh: + t.Fatalf("Got a new SubConn too early (Within timeout). Expect a new SubConn only after timeout") + } + + addrs1 := <-cc.NewSubConnAddrsCh + if got, want := addrs1[0].Addr, testBackendAddrStrs[1]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc1 := <-cc.NewSubConnCh + + // After the init timer of p0, when switching to p1, a connecting picker + // will be sent to the parent. Clear it here. + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // Test pick with 1. + if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil { + t.Fatal(err.Error()) + } +} + +// EDS removes all priorities, and re-adds them. +func (s) TestPriority_RemovesAllPriorities(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + const testPriorityInitTimeout = 200 * time.Millisecond + defer func() func() { + old := DefaultPriorityInitTimeout + DefaultPriorityInitTimeout = testPriorityInitTimeout + return func() { + DefaultPriorityInitTimeout = old + } + }()() + + cc := testutils.NewTestClientConn(t) + bb := balancer.Get(Name) + pb := bb.Build(cc, balancer.BuildOptions{}) + defer pb.Close() + + // Two localities, with different priorities, each with one backend. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: []resolver.Address{ + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}), + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0", "child-1"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + addrs0 := <-cc.NewSubConnAddrsCh + if got, want := addrs0[0].Addr, testBackendAddrStrs[0]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc0 := <-cc.NewSubConnCh + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // Test roundrobin with only p0 subconns. + if err := cc.WaitForRoundRobinPicker(ctx, sc0); err != nil { + t.Fatal(err.Error()) + } + + // Remove all priorities. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: nil, + }, + BalancerConfig: &LBConfig{ + Children: nil, + Priorities: nil, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // p0 subconn should be shut down. + scToShutdown := <-cc.ShutdownSubConnCh + if scToShutdown != sc0 { + t.Fatalf("ShutdownSubConn, want %v, got %v", sc0, scToShutdown) + } + + // Test pick return TransientFailure. + if err := cc.WaitForPickerWithErr(ctx, ErrAllPrioritiesRemoved); err != nil { + t.Fatal(err.Error()) + } + + // Re-add two localities, with previous priorities, but different backends. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: []resolver.Address{ + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[2]}, []string{"child-0"}), + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[3]}, []string{"child-1"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0", "child-1"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + addrs01 := <-cc.NewSubConnAddrsCh + if got, want := addrs01[0].Addr, testBackendAddrStrs[2]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc01 := <-cc.NewSubConnCh + + // Don't send any update to p0, so to not override the old state of p0. + // Later, connect to p1 and then remove p1. This will fallback to p0, and + // will send p0's old picker if they are not correctly removed. + + // p1 will be used after priority init timeout. + addrs11 := <-cc.NewSubConnAddrsCh + if got, want := addrs11[0].Addr, testBackendAddrStrs[3]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc11 := <-cc.NewSubConnCh + sc11.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc11.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // Test roundrobin with only p1 subconns. + if err := cc.WaitForRoundRobinPicker(ctx, sc11); err != nil { + t.Fatal(err.Error()) + } + + // Remove p1, to fallback to p0. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: []resolver.Address{ + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[2]}, []string{"child-0"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // p1 subconn should be shut down. + scToShutdown1 := <-cc.ShutdownSubConnCh + if scToShutdown1 != sc11 { + t.Fatalf("ShutdownSubConn, want %v, got %v", sc11, scToShutdown1) + } + + // Test pick return NoSubConn. + if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil { + t.Fatal(err.Error()) + } + + // Send an ready update for the p0 sc that was received when re-adding + // priorities. + sc01.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc01.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // Test roundrobin with only p0 subconns. + if err := cc.WaitForRoundRobinPicker(ctx, sc01); err != nil { + t.Fatal(err.Error()) + } + + select { + case <-cc.NewPickerCh: + t.Fatalf("got unexpected new picker") + case <-cc.NewSubConnCh: + t.Fatalf("got unexpected new SubConn") + case <-cc.ShutdownSubConnCh: + t.Fatalf("got unexpected shutdown SubConn") + case <-time.After(time.Millisecond * 100): + } +} + +// Test the case where the high priority contains no backends. The low priority +// will be used. +func (s) TestPriority_HighPriorityNoEndpoints(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + cc := testutils.NewTestClientConn(t) + bb := balancer.Get(Name) + pb := bb.Build(cc, balancer.BuildOptions{}) + defer pb.Close() + + // Two localities, with priorities [0, 1], each with one backend. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: []resolver.Address{ + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}), + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0", "child-1"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + addrs1 := <-cc.NewSubConnAddrsCh + if got, want := addrs1[0].Addr, testBackendAddrStrs[0]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc1 := <-cc.NewSubConnCh + + // p0 is ready. + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // Test roundrobin with only p0 subconns. + if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil { + t.Fatal(err.Error()) + } + + // Remove addresses from priority 0, should use p1. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: []resolver.Address{ + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0", "child-1"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // p0 will shutdown the subconn, and ClientConn will send a sc update to + // shutdown. + scToShutdown := <-cc.ShutdownSubConnCh + scToShutdown.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Shutdown}) + + addrs2 := <-cc.NewSubConnAddrsCh + if got, want := addrs2[0].Addr, testBackendAddrStrs[1]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc2 := <-cc.NewSubConnCh + + // Before 1 gets READY, picker should return NoSubConnAvailable, so RPCs + // will retry. + if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil { + t.Fatal(err.Error()) + } + + // p1 is ready. + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // Test roundrobin with only p1 subconns. + if err := cc.WaitForRoundRobinPicker(ctx, sc2); err != nil { + t.Fatal(err.Error()) + } +} + +// Test the case where the first and only priority is removed. +func (s) TestPriority_FirstPriorityUnavailable(t *testing.T) { + const testPriorityInitTimeout = 200 * time.Millisecond + defer func(t time.Duration) { + DefaultPriorityInitTimeout = t + }(DefaultPriorityInitTimeout) + DefaultPriorityInitTimeout = testPriorityInitTimeout + + cc := testutils.NewTestClientConn(t) + bb := balancer.Get(Name) + pb := bb.Build(cc, balancer.BuildOptions{}) + defer pb.Close() + + // One localities, with priorities [0], each with one backend. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: []resolver.Address{ + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // Remove the only localities. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: nil, + }, + BalancerConfig: &LBConfig{ + Children: nil, + Priorities: nil, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // Wait after double the init timer timeout, to ensure it doesn't panic. + time.Sleep(testPriorityInitTimeout * 2) +} + +// When a child is moved from low priority to high. +// +// Init a(p0) and b(p1); a(p0) is up, use a; move b to p0, a to p1, use b. +func (s) TestPriority_MoveChildToHigherPriority(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + cc := testutils.NewTestClientConn(t) + bb := balancer.Get(Name) + pb := bb.Build(cc, balancer.BuildOptions{}) + defer pb.Close() + + // Two children, with priorities [0, 1], each with one backend. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: []resolver.Address{ + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}), + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0", "child-1"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + addrs1 := <-cc.NewSubConnAddrsCh + if got, want := addrs1[0].Addr, testBackendAddrStrs[0]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc1 := <-cc.NewSubConnCh + + // p0 is ready. + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // Test roundrobin with only p0 subconns. + if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil { + t.Fatal(err.Error()) + } + + // Swap child with p0 and p1, the child at lower priority should now be the + // higher priority, and be used. The old SubConn should be closed. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: []resolver.Address{ + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}), + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-1", "child-0"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // When the new child for p0 is changed from the previous child, the + // balancer should immediately update the picker so the picker from old + // child is not used. In this case, the picker becomes a + // no-subconn-available picker because this child is just started. + if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil { + t.Fatal(err.Error()) + } + + // Old subconn should be shut down. + scToShutdown := <-cc.ShutdownSubConnCh + if scToShutdown != sc1 { + t.Fatalf("ShutdownSubConn, want %v, got %v", sc1, scToShutdown) + } + + addrs2 := <-cc.NewSubConnAddrsCh + if got, want := addrs2[0].Addr, testBackendAddrStrs[1]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc2 := <-cc.NewSubConnCh + + // New p0 child is ready. + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // Test roundrobin with only new subconns. + if err := cc.WaitForRoundRobinPicker(ctx, sc2); err != nil { + t.Fatal(err.Error()) + } +} + +// When a child is in lower priority, and in use (because higher is down), +// move it from low priority to high. +// +// Init a(p0) and b(p1); a(p0) is down, use b; move b to p0, a to p1, use b. +func (s) TestPriority_MoveReadyChildToHigherPriority(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + cc := testutils.NewTestClientConn(t) + bb := balancer.Get(Name) + pb := bb.Build(cc, balancer.BuildOptions{}) + defer pb.Close() + + // Two children, with priorities [0, 1], each with one backend. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: []resolver.Address{ + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}), + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0", "child-1"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + addrs0 := <-cc.NewSubConnAddrsCh + if got, want := addrs0[0].Addr, testBackendAddrStrs[0]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc0 := <-cc.NewSubConnCh + + // p0 is down. + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) + // Before 1 gets READY, picker should return NoSubConnAvailable, so RPCs + // will retry. + if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil { + t.Fatal(err.Error()) + } + + addrs1 := <-cc.NewSubConnAddrsCh + if got, want := addrs1[0].Addr, testBackendAddrStrs[1]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc1 := <-cc.NewSubConnCh + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // Test roundrobin with only p1 subconns. + if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil { + t.Fatal(err.Error()) + } + + // Swap child with p0 and p1, the child at lower priority should now be the + // higher priority, and be used. The old SubConn should be closed. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: []resolver.Address{ + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}), + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-1", "child-0"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // Old subconn from child-0 should be removed. + scToShutdown := <-cc.ShutdownSubConnCh + if scToShutdown != sc0 { + t.Fatalf("ShutdownSubConn, want %v, got %v", sc0, scToShutdown) + } + + // Because this was a ready child moved to a higher priority, no new subconn + // or picker should be updated. + select { + case <-cc.NewSubConnCh: + t.Fatalf("got unexpected new SubConn") + case <-cc.ShutdownSubConnCh: + t.Fatalf("got unexpected shutdown SubConn") + case <-time.After(time.Millisecond * 100): + } +} + +// When the lowest child is in use, and is removed, should use the higher +// priority child even though it's not ready. +// +// Init a(p0) and b(p1); a(p0) is down, use b; move b to p0, a to p1, use b. +func (s) TestPriority_RemoveReadyLowestChild(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + cc := testutils.NewTestClientConn(t) + bb := balancer.Get(Name) + pb := bb.Build(cc, balancer.BuildOptions{}) + defer pb.Close() + + // Two children, with priorities [0, 1], each with one backend. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: []resolver.Address{ + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}), + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0", "child-1"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + addrs0 := <-cc.NewSubConnAddrsCh + if got, want := addrs0[0].Addr, testBackendAddrStrs[0]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc0 := <-cc.NewSubConnCh + + // p0 is down. + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) + // Before 1 gets READY, picker should return NoSubConnAvailable, so RPCs + // will retry. + if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil { + t.Fatal(err.Error()) + } + + addrs1 := <-cc.NewSubConnAddrsCh + if got, want := addrs1[0].Addr, testBackendAddrStrs[1]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc1 := <-cc.NewSubConnCh + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // Test roundrobin with only p1 subconns. + if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil { + t.Fatal(err.Error()) + } + + // Remove child with p1, the child at higher priority should now be used. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: []resolver.Address{ + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // Old subconn from child-1 should be shut down. + scToShutdown := <-cc.ShutdownSubConnCh + if scToShutdown != sc1 { + t.Fatalf("ShutdownSubConn, want %v, got %v", sc1, scToShutdown) + } + + if err := cc.WaitForErrPicker(ctx); err != nil { + t.Fatal(err.Error()) + } + <-cc.NewStateCh // Drain to match picker + + // Because there was no new child, no new subconn should be created. + select { + case <-cc.NewSubConnCh: + t.Fatalf("got unexpected new SubConn") + case <-time.After(time.Millisecond * 100): + } +} + +// When a ready child is removed, it's kept in cache. Re-adding doesn't create subconns. +// +// Init 0; 0 is up, use 0; remove 0, only picker is updated, no subconn is +// removed; re-add 0, picker is updated. +func (s) TestPriority_ReadyChildRemovedButInCache(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + const testChildCacheTimeout = time.Second + defer func() func() { + old := DefaultSubBalancerCloseTimeout + DefaultSubBalancerCloseTimeout = testChildCacheTimeout + return func() { + DefaultSubBalancerCloseTimeout = old + } + }()() + + cc := testutils.NewTestClientConn(t) + bb := balancer.Get(Name) + pb := bb.Build(cc, balancer.BuildOptions{}) + defer pb.Close() + + // One children, with priorities [0], with one backend. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: []resolver.Address{ + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + addrs1 := <-cc.NewSubConnAddrsCh + if got, want := addrs1[0].Addr, testBackendAddrStrs[0]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc1 := <-cc.NewSubConnCh + + // p0 is ready. + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // Test roundrobin with only p0 subconns. + if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil { + t.Fatal(err.Error()) + } + + // Remove the child, it shouldn't cause any conn changed, but picker should + // be different. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{}, + BalancerConfig: &LBConfig{}, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + if err := cc.WaitForPickerWithErr(ctx, ErrAllPrioritiesRemoved); err != nil { + t.Fatal(err.Error()) + } + + // But no conn changes should happen. Child balancer is in cache. + select { + case sc := <-cc.NewSubConnCh: + t.Fatalf("got unexpected new SubConn: %s", sc) + case sc := <-cc.ShutdownSubConnCh: + t.Fatalf("got unexpected shutdown SubConn: %v", sc) + case <-time.After(time.Millisecond * 100): + } + + // Re-add the child, shouldn't create new connections. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: []resolver.Address{ + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // Test roundrobin with only p0 subconns. + if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil { + t.Fatal(err.Error()) + } + + // But no conn changes should happen. Child balancer is just taken out from + // the cache. + select { + case sc := <-cc.NewSubConnCh: + t.Fatalf("got unexpected new SubConn: %s", sc) + case sc := <-cc.ShutdownSubConnCh: + t.Fatalf("got unexpected shutdown SubConn: %v", sc) + case <-time.After(time.Millisecond * 100): + } +} + +// When the policy of a child is changed. +// +// Init 0; 0 is up, use 0; change 0's policy, 0 is used. +func (s) TestPriority_ChildPolicyChange(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + cc := testutils.NewTestClientConn(t) + bb := balancer.Get(Name) + pb := bb.Build(cc, balancer.BuildOptions{}) + defer pb.Close() + + // One children, with priorities [0], with one backend. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: []resolver.Address{ + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + addrs1 := <-cc.NewSubConnAddrsCh + if got, want := addrs1[0].Addr, testBackendAddrStrs[0]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc1 := <-cc.NewSubConnCh + + // p0 is ready. + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // Test roundrobin with only p0 subconns. + if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil { + t.Fatal(err.Error()) + } + + // Change the policy for the child (still roundrobin, but with a different + // name). + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: []resolver.Address{ + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: testRRBalancerName}}, + }, + Priorities: []string{"child-0"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // Old subconn should be shut down. + scToShutdown := <-cc.ShutdownSubConnCh + if scToShutdown != sc1 { + t.Fatalf("ShutdownSubConn, want %v, got %v", sc1, scToShutdown) + } + + // A new subconn should be created. + addrs2 := <-cc.NewSubConnAddrsCh + if got, want := addrs2[0].Addr, testBackendAddrStrs[0]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc2 := <-cc.NewSubConnCh + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // Test pickfirst with the new subconns. + if err := cc.WaitForRoundRobinPicker(ctx, sc2); err != nil { + t.Fatal(err.Error()) + } +} + +const inlineUpdateBalancerName = "test-inline-update-balancer" + +var errTestInlineStateUpdate = fmt.Errorf("don't like addresses, empty or not") + +func init() { + stub.Register(inlineUpdateBalancerName, stub.BalancerFuncs{ + UpdateClientConnState: func(bd *stub.BalancerData, opts balancer.ClientConnState) error { + bd.ClientConn.UpdateState(balancer.State{ + ConnectivityState: connectivity.Ready, + Picker: &testutils.TestConstPicker{Err: errTestInlineStateUpdate}, + }) + return nil + }, + }) +} + +// When the child policy update picker inline in a handleClientUpdate call +// (e.g., roundrobin handling empty addresses). There could be deadlock caused +// by acquiring a locked mutex. +func (s) TestPriority_ChildPolicyUpdatePickerInline(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + cc := testutils.NewTestClientConn(t) + bb := balancer.Get(Name) + pb := bb.Build(cc, balancer.BuildOptions{}) + defer pb.Close() + + // One children, with priorities [0], with one backend. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: []resolver.Address{ + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: inlineUpdateBalancerName}}, + }, + Priorities: []string{"child-0"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + if err := cc.WaitForPickerWithErr(ctx, errTestInlineStateUpdate); err != nil { + t.Fatal(err.Error()) + } +} + +// TestPriority_IgnoreReresolutionRequest tests the case where the priority +// policy has a single child policy. The test verifies that ResolveNow() calls +// from the child policy are ignored based on the value of the +// IgnoreReresolutionRequests field in the configuration. +func (s) TestPriority_IgnoreReresolutionRequest(t *testing.T) { + // Register a stub balancer to act the child policy of the priority policy. + // Provide an init function to the stub balancer to capture the ClientConn + // passed to the child policy. + ccCh := testutils.NewChannel() + childPolicyName := t.Name() + stub.Register(childPolicyName, stub.BalancerFuncs{ + Init: func(data *stub.BalancerData) { + ccCh.Send(data.ClientConn) + }, + }) + + cc := testutils.NewTestClientConn(t) + bb := balancer.Get(Name) + pb := bb.Build(cc, balancer.BuildOptions{}) + defer pb.Close() + + // One children, with priorities [0], with one backend, reresolution is + // ignored. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: []resolver.Address{ + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": { + Config: &internalserviceconfig.BalancerConfig{Name: childPolicyName}, + IgnoreReresolutionRequests: true, + }, + }, + Priorities: []string{"child-0"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // Retrieve the ClientConn passed to the child policy. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + val, err := ccCh.Receive(ctx) + if err != nil { + t.Fatalf("timeout waiting for ClientConn from the child policy") + } + balancerCC := val.(balancer.ClientConn) + + // Since IgnoreReresolutionRequests was set to true, all ResolveNow() calls + // should be ignored. + for i := 0; i < 5; i++ { + balancerCC.ResolveNow(resolver.ResolveNowOptions{}) + } + select { + case <-cc.ResolveNowCh: + t.Fatalf("got unexpected ResolveNow() call") + case <-time.After(defaultTestShortTimeout): + } + + // Send another update to set IgnoreReresolutionRequests to false. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: []resolver.Address{ + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": { + Config: &internalserviceconfig.BalancerConfig{Name: childPolicyName}, + IgnoreReresolutionRequests: false, + }, + }, + Priorities: []string{"child-0"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // Call ResolveNow() on the CC, it should be forwarded. + balancerCC.ResolveNow(resolver.ResolveNowOptions{}) + select { + case <-cc.ResolveNowCh: + case <-time.After(time.Second): + t.Fatalf("timeout waiting for ResolveNow()") + } + +} + +// TestPriority_IgnoreReresolutionRequestTwoChildren tests the case where the +// priority policy has two child policies, one of them has the +// IgnoreReresolutionRequests field set to true while the other one has it set +// to false. The test verifies that ResolveNow() calls from the child which is +// set to ignore reresolution requests are ignored, while calls from the other +// child are processed. +func (s) TestPriority_IgnoreReresolutionRequestTwoChildren(t *testing.T) { + // Register a stub balancer to act the child policy of the priority policy. + // Provide an init function to the stub balancer to capture the ClientConn + // passed to the child policy. + ccCh := testutils.NewChannel() + childPolicyName := t.Name() + stub.Register(childPolicyName, stub.BalancerFuncs{ + Init: func(bd *stub.BalancerData) { + ccCh.Send(bd.ClientConn) + bd.Data = balancer.Get(roundrobin.Name).Build(bd.ClientConn, bd.BuildOptions) + }, + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + bal := bd.Data.(balancer.Balancer) + return bal.UpdateClientConnState(ccs) + }, + }) + + cc := testutils.NewTestClientConn(t) + bb := balancer.Get(Name) + pb := bb.Build(cc, balancer.BuildOptions{}) + defer pb.Close() + + // One children, with priorities [0, 1], each with one backend. + // Reresolution is ignored for p0. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: []resolver.Address{ + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}), + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": { + Config: &internalserviceconfig.BalancerConfig{Name: childPolicyName}, + IgnoreReresolutionRequests: true, + }, + "child-1": { + Config: &internalserviceconfig.BalancerConfig{Name: childPolicyName}, + }, + }, + Priorities: []string{"child-0", "child-1"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // Retrieve the ClientConn passed to the child policy from p0. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + val, err := ccCh.Receive(ctx) + if err != nil { + t.Fatalf("timeout waiting for ClientConn from the child policy") + } + balancerCC0 := val.(balancer.ClientConn) + + // Set p0 to transient failure, p1 will be started. + addrs0 := <-cc.NewSubConnAddrsCh + if got, want := addrs0[0].Addr, testBackendAddrStrs[0]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc0 := <-cc.NewSubConnCh + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) + + // Retrieve the ClientConn passed to the child policy from p1. + val, err = ccCh.Receive(ctx) + if err != nil { + t.Fatalf("timeout waiting for ClientConn from the child policy") + } + balancerCC1 := val.(balancer.ClientConn) + + // Since IgnoreReresolutionRequests was set to true for p0, ResolveNow() + // from p0 should all be ignored. + for i := 0; i < 5; i++ { + balancerCC0.ResolveNow(resolver.ResolveNowOptions{}) + } + select { + case <-cc.ResolveNowCh: + t.Fatalf("got unexpected ResolveNow() call") + case <-time.After(defaultTestShortTimeout): + } + + // But IgnoreReresolutionRequests was false for p1, ResolveNow() from p1 + // should be forwarded. + balancerCC1.ResolveNow(resolver.ResolveNowOptions{}) + select { + case <-cc.ResolveNowCh: + case <-time.After(defaultTestShortTimeout): + t.Fatalf("timeout waiting for ResolveNow()") + } +} + +const initIdleBalancerName = "test-init-Idle-balancer" + +var errsTestInitIdle = []error{ + fmt.Errorf("init Idle balancer error 0"), + fmt.Errorf("init Idle balancer error 1"), +} + +func init() { + for i := 0; i < 2; i++ { + ii := i + stub.Register(fmt.Sprintf("%s-%d", initIdleBalancerName, ii), stub.BalancerFuncs{ + UpdateClientConnState: func(bd *stub.BalancerData, opts balancer.ClientConnState) error { + lis := func(state balancer.SubConnState) { + err := fmt.Errorf("wrong picker error") + if state.ConnectivityState == connectivity.Idle { + err = errsTestInitIdle[ii] + } + bd.ClientConn.UpdateState(balancer.State{ + ConnectivityState: state.ConnectivityState, + Picker: &testutils.TestConstPicker{Err: err}, + }) + } + + sc, err := bd.ClientConn.NewSubConn(opts.ResolverState.Addresses, balancer.NewSubConnOptions{StateListener: lis}) + if err != nil { + return err + } + sc.Connect() + bd.ClientConn.UpdateState(balancer.State{ + ConnectivityState: connectivity.Connecting, + Picker: &testutils.TestConstPicker{Err: balancer.ErrNoSubConnAvailable}, + }) + return nil + }, + }) + } +} + +// If the high priorities send initial pickers with Idle state, their pickers +// should get picks, because policies like ringhash starts in Idle, and doesn't +// connect. +// +// Init 0, 1; 0 is Idle, use 0; 0 is down, start 1; 1 is Idle, use 1. +func (s) TestPriority_HighPriorityInitIdle(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + cc := testutils.NewTestClientConn(t) + bb := balancer.Get(Name) + pb := bb.Build(cc, balancer.BuildOptions{}) + defer pb.Close() + + // Two children, with priorities [0, 1], each with one backend. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: []resolver.Address{ + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}), + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: fmt.Sprintf("%s-%d", initIdleBalancerName, 0)}}, + "child-1": {Config: &internalserviceconfig.BalancerConfig{Name: fmt.Sprintf("%s-%d", initIdleBalancerName, 1)}}, + }, + Priorities: []string{"child-0", "child-1"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + addrs0 := <-cc.NewSubConnAddrsCh + if got, want := addrs0[0].Addr, testBackendAddrStrs[0]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc0 := <-cc.NewSubConnCh + + // Send an Idle state update to trigger an Idle picker update. + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Idle}) + if err := cc.WaitForPickerWithErr(ctx, errsTestInitIdle[0]); err != nil { + t.Fatal(err.Error()) + } + + // Turn p0 down, to start p1. + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) + // Before 1 gets READY, picker should return NoSubConnAvailable, so RPCs + // will retry. + if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil { + t.Fatal(err.Error()) + } + + addrs1 := <-cc.NewSubConnAddrsCh + if got, want := addrs1[0].Addr, testBackendAddrStrs[1]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc1 := <-cc.NewSubConnCh + // Idle picker from p1 should also be forwarded. + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Idle}) + if err := cc.WaitForPickerWithErr(ctx, errsTestInitIdle[1]); err != nil { + t.Fatal(err.Error()) + } +} + +// If the high priorities send initial pickers with Idle state, their pickers +// should get picks, because policies like ringhash starts in Idle, and doesn't +// connect. In this case, if a lower priority is added, it shouldn't switch to +// the lower priority. +// +// Init 0; 0 is Idle, use 0; add 1, use 0. +func (s) TestPriority_AddLowPriorityWhenHighIsInIdle(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + cc := testutils.NewTestClientConn(t) + bb := balancer.Get(Name) + pb := bb.Build(cc, balancer.BuildOptions{}) + defer pb.Close() + + // One child, with priorities [0], one backend. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: []resolver.Address{ + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: fmt.Sprintf("%s-%d", initIdleBalancerName, 0)}}, + }, + Priorities: []string{"child-0"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + addrs0 := <-cc.NewSubConnAddrsCh + if got, want := addrs0[0].Addr, testBackendAddrStrs[0]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc0 := <-cc.NewSubConnCh + + // Send an Idle state update to trigger an Idle picker update. + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Idle}) + if err := cc.WaitForPickerWithErr(ctx, errsTestInitIdle[0]); err != nil { + t.Fatal(err.Error()) + } + + // Add 1, should keep using 0. + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: []resolver.Address{ + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}), + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: fmt.Sprintf("%s-%d", initIdleBalancerName, 0)}}, + "child-1": {Config: &internalserviceconfig.BalancerConfig{Name: fmt.Sprintf("%s-%d", initIdleBalancerName, 1)}}, + }, + Priorities: []string{"child-0", "child-1"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // The ClientConn state update triggers a priority switch, from p0 -> p0 + // (since p0 is still in use). Along with this the update, p0 also gets a + // ClientConn state update, with the addresses, which didn't change in this + // test (this update to the child is necessary in case the addresses are + // different). + // + // The test child policy, initIdleBalancer, blindly calls NewSubConn with + // all the addresses it receives, so this will trigger a NewSubConn with the + // old p0 addresses. (Note that in a real balancer, like roundrobin, no new + // SubConn will be created because the addresses didn't change). + // + // The check below makes sure that the addresses are still from p0, and not + // from p1. This is good enough for the purpose of this test. + addrsNew := <-cc.NewSubConnAddrsCh + if got, want := addrsNew[0].Addr, testBackendAddrStrs[0]; got != want { + // Fail if p1 is started and creates a SubConn. + t.Fatalf("got unexpected call to NewSubConn with addr: %v, want %v", addrsNew, want) + } +} + +// Lower priority is used when higher priority is not ready; higher priority +// still gets updates. +// +// Init 0 and 1; 0 is down, 1 is up, use 1; update 0; 0 is up, use 0 +func (s) TestPriority_HighPriorityUpdatesWhenLowInUse(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + cc := testutils.NewTestClientConn(t) + bb := balancer.Get(Name) + pb := bb.Build(cc, balancer.BuildOptions{}) + defer pb.Close() + + t.Log("Two localities, with priorities [0, 1], each with one backend.") + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: []resolver.Address{ + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}), + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0", "child-1"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + addrs0 := <-cc.NewSubConnAddrsCh + if got, want := addrs0[0].Addr, testBackendAddrStrs[0]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc0 := <-cc.NewSubConnCh + + t.Log("Make p0 fail.") + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) + + // Before 1 gets READY, picker should return NoSubConnAvailable, so RPCs + // will retry. + if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil { + t.Fatal(err.Error()) + } + + t.Log("Make p1 ready.") + addrs1 := <-cc.NewSubConnAddrsCh + if got, want := addrs1[0].Addr, testBackendAddrStrs[1]; got != want { + t.Fatalf("sc is created with addr %v, want %v", got, want) + } + sc1 := <-cc.NewSubConnCh + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // Test pick with 1. + if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil { + t.Fatal(err.Error()) + } + + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + // Does not change the aggregate state, because round robin does not leave + // TRANIENT_FAILURE if a subconn goes CONNECTING. + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil { + t.Fatal(err.Error()) + } + + t.Log("Change p0 to use new address.") + if err := pb.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: []resolver.Address{ + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[2]}, []string{"child-0"}), + hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[3]}, []string{"child-1"}), + }, + }, + BalancerConfig: &LBConfig{ + Children: map[string]*Child{ + "child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + "child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}}, + }, + Priorities: []string{"child-0", "child-1"}, + }, + }); err != nil { + t.Fatalf("failed to update ClientConn state: %v", err) + } + + // Two new subconns are created by the previous update; one by p0 and one + // by p1. They don't happen concurrently, but they could happen in any + // order. + t.Log("Make p0 and p1 both ready; p0 should be used.") + var sc2, sc3 balancer.SubConn + for i := 0; i < 2; i++ { + addr := <-cc.NewSubConnAddrsCh + sc := <-cc.NewSubConnCh + switch addr[0].Addr { + case testBackendAddrStrs[2]: + sc2 = sc + case testBackendAddrStrs[3]: + sc3 = sc + default: + t.Fatalf("sc is created with addr %v, want %v or %v", addr[0].Addr, testBackendAddrStrs[2], testBackendAddrStrs[3]) + } + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + } + if sc2 == nil { + t.Fatalf("sc not created with addr %v", testBackendAddrStrs[2]) + } + if sc3 == nil { + t.Fatalf("sc not created with addr %v", testBackendAddrStrs[3]) + } + + // Test pick with 0. + if err := cc.WaitForRoundRobinPicker(ctx, sc2); err != nil { + t.Fatal(err.Error()) + } +} diff --git a/xds/internal/balancer/priority/config.go b/xds/internal/balancer/priority/config.go new file mode 100644 index 000000000000..37f1c9a829a8 --- /dev/null +++ b/xds/internal/balancer/priority/config.go @@ -0,0 +1,67 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package priority + +import ( + "encoding/json" + "fmt" + + internalserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/serviceconfig" +) + +// Child is a child of priority balancer. +type Child struct { + Config *internalserviceconfig.BalancerConfig `json:"config,omitempty"` + IgnoreReresolutionRequests bool `json:"ignoreReresolutionRequests,omitempty"` +} + +// LBConfig represents priority balancer's config. +type LBConfig struct { + serviceconfig.LoadBalancingConfig `json:"-"` + + // Children is a map from the child balancer names to their configs. Child + // names can be found in field Priorities. + Children map[string]*Child `json:"children,omitempty"` + // Priorities is a list of child balancer names. They are sorted from + // highest priority to low. The type/config for each child can be found in + // field Children, with the balancer name as the key. + Priorities []string `json:"priorities,omitempty"` +} + +func parseConfig(c json.RawMessage) (*LBConfig, error) { + var cfg LBConfig + if err := json.Unmarshal(c, &cfg); err != nil { + return nil, err + } + + prioritiesSet := make(map[string]bool) + for _, name := range cfg.Priorities { + if _, ok := cfg.Children[name]; !ok { + return nil, fmt.Errorf("LB policy name %q found in Priorities field (%v) is not found in Children field (%+v)", name, cfg.Priorities, cfg.Children) + } + prioritiesSet[name] = true + } + for name := range cfg.Children { + if _, ok := prioritiesSet[name]; !ok { + return nil, fmt.Errorf("LB policy name %q found in Children field (%v) is not found in Priorities field (%+v)", name, cfg.Children, cfg.Priorities) + } + } + return &cfg, nil +} diff --git a/xds/internal/balancer/priority/config_test.go b/xds/internal/balancer/priority/config_test.go new file mode 100644 index 000000000000..8316224c91be --- /dev/null +++ b/xds/internal/balancer/priority/config_test.go @@ -0,0 +1,108 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package priority + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/balancer/roundrobin" + internalserviceconfig "google.golang.org/grpc/internal/serviceconfig" +) + +func TestParseConfig(t *testing.T) { + tests := []struct { + name string + js string + want *LBConfig + wantErr bool + }{ + { + name: "child not found", + js: `{ + "priorities": ["child-1", "child-2", "child-3"], + "children": { + "child-1": {"config": [{"round_robin":{}}]}, + "child-3": {"config": [{"round_robin":{}}]} + } +} + `, + wantErr: true, + }, + { + name: "child not used", + js: `{ + "priorities": ["child-1", "child-2"], + "children": { + "child-1": {"config": [{"round_robin":{}}]}, + "child-2": {"config": [{"round_robin":{}}]}, + "child-3": {"config": [{"round_robin":{}}]} + } +} + `, + wantErr: true, + }, + { + name: "good", + js: `{ + "priorities": ["child-1", "child-2", "child-3"], + "children": { + "child-1": {"config": [{"round_robin":{}}], "ignoreReresolutionRequests": true}, + "child-2": {"config": [{"round_robin":{}}]}, + "child-3": {"config": [{"round_robin":{}}]} + } +} + `, + want: &LBConfig{ + Children: map[string]*Child{ + "child-1": { + Config: &internalserviceconfig.BalancerConfig{ + Name: roundrobin.Name, + }, + IgnoreReresolutionRequests: true, + }, + "child-2": { + Config: &internalserviceconfig.BalancerConfig{ + Name: roundrobin.Name, + }, + }, + "child-3": { + Config: &internalserviceconfig.BalancerConfig{ + Name: roundrobin.Name, + }, + }, + }, + Priorities: []string{"child-1", "child-2", "child-3"}, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parseConfig([]byte(tt.js)) + if (err != nil) != tt.wantErr { + t.Errorf("parseConfig() error = %v, wantErr %v", err, tt.wantErr) + return + } + if diff := cmp.Diff(got, tt.want); diff != "" { + t.Errorf("parseConfig() got = %v, want %v, diff: %s", got, tt.want, diff) + } + }) + } +} diff --git a/xds/internal/balancer/priority/ignore_resolve_now.go b/xds/internal/balancer/priority/ignore_resolve_now.go new file mode 100644 index 000000000000..792ee4b3f242 --- /dev/null +++ b/xds/internal/balancer/priority/ignore_resolve_now.go @@ -0,0 +1,58 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package priority + +import ( + "sync/atomic" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/resolver" +) + +// ignoreResolveNowClientConn wraps a balancer.ClientConn and overrides the +// ResolveNow() method to ignore those calls if the ignoreResolveNow bit is set. +type ignoreResolveNowClientConn struct { + balancer.ClientConn + ignoreResolveNow *uint32 +} + +func newIgnoreResolveNowClientConn(cc balancer.ClientConn, ignore bool) *ignoreResolveNowClientConn { + ret := &ignoreResolveNowClientConn{ + ClientConn: cc, + ignoreResolveNow: new(uint32), + } + ret.updateIgnoreResolveNow(ignore) + return ret +} + +func (i *ignoreResolveNowClientConn) updateIgnoreResolveNow(b bool) { + if b { + atomic.StoreUint32(i.ignoreResolveNow, 1) + return + } + atomic.StoreUint32(i.ignoreResolveNow, 0) + +} + +func (i ignoreResolveNowClientConn) ResolveNow(o resolver.ResolveNowOptions) { + if atomic.LoadUint32(i.ignoreResolveNow) != 0 { + return + } + i.ClientConn.ResolveNow(o) +} diff --git a/xds/internal/balancer/priority/ignore_resolve_now_test.go b/xds/internal/balancer/priority/ignore_resolve_now_test.go new file mode 100644 index 000000000000..5a0083147888 --- /dev/null +++ b/xds/internal/balancer/priority/ignore_resolve_now_test.go @@ -0,0 +1,66 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package priority + +import ( + "context" + "testing" + "time" + + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/resolver" +) + +func (s) TestIgnoreResolveNowClientConn(t *testing.T) { + cc := testutils.NewTestClientConn(t) + ignoreCC := newIgnoreResolveNowClientConn(cc, false) + + // Call ResolveNow() on the CC, it should be forwarded. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + ignoreCC.ResolveNow(resolver.ResolveNowOptions{}) + select { + case <-cc.ResolveNowCh: + case <-ctx.Done(): + t.Fatalf("Timeout waiting for ResolveNow()") + } + + // Update ignoreResolveNow to true, call ResolveNow() on the CC, they should + // all be ignored. + ignoreCC.updateIgnoreResolveNow(true) + for i := 0; i < 5; i++ { + ignoreCC.ResolveNow(resolver.ResolveNowOptions{}) + } + select { + case <-cc.ResolveNowCh: + t.Fatalf("got unexpected ResolveNow() call") + case <-time.After(defaultTestShortTimeout): + } + + // Update ignoreResolveNow to false, new ResolveNow() calls should be + // forwarded. + ignoreCC.updateIgnoreResolveNow(false) + ignoreCC.ResolveNow(resolver.ResolveNowOptions{}) + select { + case <-cc.ResolveNowCh: + case <-ctx.Done(): + t.Fatalf("timeout waiting for ResolveNow()") + } +} diff --git a/xds/internal/client/client_logging.go b/xds/internal/balancer/priority/logging.go similarity index 83% rename from xds/internal/client/client_logging.go rename to xds/internal/balancer/priority/logging.go index a47e5247fe22..2fb8d2d204c5 100644 --- a/xds/internal/client/client_logging.go +++ b/xds/internal/balancer/priority/logging.go @@ -1,6 +1,6 @@ /* * - * Copyright 2020 gRPC authors. + * Copyright 2021 gRPC authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,7 +16,7 @@ * */ -package client +package priority import ( "fmt" @@ -25,10 +25,10 @@ import ( internalgrpclog "google.golang.org/grpc/internal/grpclog" ) -const prefix = "[xds-client %p] " +const prefix = "[priority-lb %p] " var logger = grpclog.Component("xds") -func prefixLogger(p *Client) *internalgrpclog.PrefixLogger { +func prefixLogger(p *priorityBalancer) *internalgrpclog.PrefixLogger { return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p)) } diff --git a/xds/internal/balancer/priority/utils.go b/xds/internal/balancer/priority/utils.go new file mode 100644 index 000000000000..45fbe764434a --- /dev/null +++ b/xds/internal/balancer/priority/utils.go @@ -0,0 +1,31 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package priority + +func equalStringSlice(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} diff --git a/xds/internal/balancer/priority/utils_test.go b/xds/internal/balancer/priority/utils_test.go new file mode 100644 index 000000000000..c80a89b080f9 --- /dev/null +++ b/xds/internal/balancer/priority/utils_test.go @@ -0,0 +1,62 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package priority + +import "testing" + +func TestCompareStringSlice(t *testing.T) { + tests := []struct { + name string + a []string + b []string + want bool + }{ + { + name: "equal", + a: []string{"a", "b"}, + b: []string{"a", "b"}, + want: true, + }, + { + name: "not equal", + a: []string{"a", "b"}, + b: []string{"a", "b", "c"}, + want: false, + }, + { + name: "both empty", + a: nil, + b: nil, + want: true, + }, + { + name: "one empty", + a: []string{"a", "b"}, + b: nil, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := equalStringSlice(tt.a, tt.b); got != tt.want { + t.Errorf("equalStringSlice(%v, %v) = %v, want %v", tt.a, tt.b, got, tt.want) + } + }) + } +} diff --git a/xds/internal/balancer/ringhash/config.go b/xds/internal/balancer/ringhash/config.go new file mode 100644 index 000000000000..b4afcf100132 --- /dev/null +++ b/xds/internal/balancer/ringhash/config.go @@ -0,0 +1,70 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package ringhash + +import ( + "encoding/json" + "fmt" + + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/serviceconfig" +) + +// LBConfig is the balancer config for ring_hash balancer. +type LBConfig struct { + serviceconfig.LoadBalancingConfig `json:"-"` + + MinRingSize uint64 `json:"minRingSize,omitempty"` + MaxRingSize uint64 `json:"maxRingSize,omitempty"` +} + +const ( + defaultMinSize = 1024 + defaultMaxSize = 4096 + ringHashSizeUpperBound = 8 * 1024 * 1024 // 8M +) + +func parseConfig(c json.RawMessage) (*LBConfig, error) { + var cfg LBConfig + if err := json.Unmarshal(c, &cfg); err != nil { + return nil, err + } + if cfg.MinRingSize > ringHashSizeUpperBound { + return nil, fmt.Errorf("min_ring_size value of %d is greater than max supported value %d for this field", cfg.MinRingSize, ringHashSizeUpperBound) + } + if cfg.MaxRingSize > ringHashSizeUpperBound { + return nil, fmt.Errorf("max_ring_size value of %d is greater than max supported value %d for this field", cfg.MaxRingSize, ringHashSizeUpperBound) + } + if cfg.MinRingSize == 0 { + cfg.MinRingSize = defaultMinSize + } + if cfg.MaxRingSize == 0 { + cfg.MaxRingSize = defaultMaxSize + } + if cfg.MinRingSize > cfg.MaxRingSize { + return nil, fmt.Errorf("min %v is greater than max %v", cfg.MinRingSize, cfg.MaxRingSize) + } + if cfg.MinRingSize > envconfig.RingHashCap { + cfg.MinRingSize = envconfig.RingHashCap + } + if cfg.MaxRingSize > envconfig.RingHashCap { + cfg.MaxRingSize = envconfig.RingHashCap + } + return &cfg, nil +} diff --git a/xds/internal/balancer/ringhash/config_test.go b/xds/internal/balancer/ringhash/config_test.go new file mode 100644 index 000000000000..1077d3e7dafb --- /dev/null +++ b/xds/internal/balancer/ringhash/config_test.go @@ -0,0 +1,115 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package ringhash + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/internal/envconfig" +) + +func (s) TestParseConfig(t *testing.T) { + tests := []struct { + name string + js string + envConfigCap uint64 + want *LBConfig + wantErr bool + }{ + { + name: "OK", + js: `{"minRingSize": 1, "maxRingSize": 2}`, + want: &LBConfig{MinRingSize: 1, MaxRingSize: 2}, + }, + { + name: "OK with default min", + js: `{"maxRingSize": 2000}`, + want: &LBConfig{MinRingSize: defaultMinSize, MaxRingSize: 2000}, + }, + { + name: "OK with default max", + js: `{"minRingSize": 2000}`, + want: &LBConfig{MinRingSize: 2000, MaxRingSize: defaultMaxSize}, + }, + { + name: "min greater than max", + js: `{"minRingSize": 10, "maxRingSize": 2}`, + want: nil, + wantErr: true, + }, + { + name: "min greater than max greater than global limit", + js: `{"minRingSize": 6000, "maxRingSize": 5000}`, + want: nil, + wantErr: true, + }, + { + name: "max greater than global limit", + js: `{"minRingSize": 1, "maxRingSize": 6000}`, + want: &LBConfig{MinRingSize: 1, MaxRingSize: 4096}, + }, + { + name: "min and max greater than global limit", + js: `{"minRingSize": 5000, "maxRingSize": 6000}`, + want: &LBConfig{MinRingSize: 4096, MaxRingSize: 4096}, + }, + { + name: "min and max less than raised global limit", + js: `{"minRingSize": 5000, "maxRingSize": 6000}`, + envConfigCap: 8000, + want: &LBConfig{MinRingSize: 5000, MaxRingSize: 6000}, + }, + { + name: "min and max greater than raised global limit", + js: `{"minRingSize": 10000, "maxRingSize": 10000}`, + envConfigCap: 8000, + want: &LBConfig{MinRingSize: 8000, MaxRingSize: 8000}, + }, + { + name: "min greater than upper bound", + js: `{"minRingSize": 8388610, "maxRingSize": 10}`, + want: nil, + wantErr: true, + }, + { + name: "max greater than upper bound", + js: `{"minRingSize": 10, "maxRingSize": 8388610}`, + want: nil, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.envConfigCap != 0 { + old := envconfig.RingHashCap + defer func() { envconfig.RingHashCap = old }() + envconfig.RingHashCap = tt.envConfigCap + } + got, err := parseConfig([]byte(tt.js)) + if (err != nil) != tt.wantErr { + t.Errorf("parseConfig() error = %v, wantErr %v", err, tt.wantErr) + return + } + if diff := cmp.Diff(got, tt.want); diff != "" { + t.Errorf("parseConfig() got unexpected output, diff (-got +want): %v", diff) + } + }) + } +} diff --git a/xds/internal/balancer/ringhash/e2e/ringhash_balancer_test.go b/xds/internal/balancer/ringhash/e2e/ringhash_balancer_test.go new file mode 100644 index 000000000000..5eb8ffd16bd3 --- /dev/null +++ b/xds/internal/balancer/ringhash/e2e/ringhash_balancer_test.go @@ -0,0 +1,144 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package ringhash_test + +import ( + "context" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/resolver/manual" + + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + + _ "google.golang.org/grpc/xds/internal/balancer/ringhash" // Register the ring_hash_experimental LB policy. +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +const ( + defaultTestTimeout = 10 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond // For events expected to *not* happen. +) + +type testService struct { + testgrpc.TestServiceServer +} + +func (*testService) EmptyCall(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil +} + +// TestRingHash_ReconnectToMoveOutOfTransientFailure tests the case where the +// ring contains a single subConn, and verifies that when the server goes down, +// the LB policy on the client automatically reconnects until the subChannel +// moves out of TRANSIENT_FAILURE. +func (s) TestRingHash_ReconnectToMoveOutOfTransientFailure(t *testing.T) { + // Create a restartable listener to simulate server being down. + l, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("testutils.LocalTCPListener() failed: %v", err) + } + lis := testutils.NewRestartableListener(l) + + // Start a server backend exposing the test service. + server := grpc.NewServer() + defer server.Stop() + testgrpc.RegisterTestServiceServer(server, &testService{}) + go func() { + if err := server.Serve(lis); err != nil { + t.Errorf("Serve() failed: %v", err) + } + }() + + // Create a clientConn with a manual resolver (which is used to push the + // address of the test backend), and a default service config pointing to + // the use of the ring_hash_experimental LB policy. + const ringHashServiceConfig = `{"loadBalancingConfig": [{"ring_hash_experimental":{}}]}` + r := manual.NewBuilderWithScheme("whatever") + dopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(r), + grpc.WithDefaultServiceConfig(ringHashServiceConfig), + } + cc, err := grpc.Dial(r.Scheme()+":///test.server", dopts...) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + // Push the address of the test backend through the manual resolver. + r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: lis.Addr().String()}}}) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + client := testgrpc.NewTestServiceClient(cc) + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("rpc EmptyCall() failed: %v", err) + } + + // Stopping the server listener will close the transport on the client, + // which will lead to the channel eventually moving to IDLE. The ring_hash + // LB policy is not expected to reconnect by itself at this point. + lis.Stop() + + testutils.AwaitState(ctx, t, cc, connectivity.Idle) + + // Make an RPC to get the ring_hash LB policy to reconnect and thereby move + // to TRANSIENT_FAILURE upon connection failure. + client.EmptyCall(ctx, &testpb.Empty{}) + + testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure) + + // An RPC at this point is expected to fail. + if _, err = client.EmptyCall(ctx, &testpb.Empty{}); err == nil { + t.Fatal("EmptyCall RPC succeeded when the channel is in TRANSIENT_FAILURE") + } + + // Restart the server listener. The ring_hash LB polcy is expected to + // attempt to reconnect on its own and come out of TRANSIENT_FAILURE, even + // without an RPC attempt. + lis.Restart() + for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) { + if cc.GetState() == connectivity.Ready { + break + } + } + if err := ctx.Err(); err != nil { + t.Fatalf("Timeout waiting for channel to reach READT after server restart: %v", err) + } + + // An RPC at this point is expected to fail. + if _, err := client.EmptyCall(ctx, &testpb.Empty{}); err != nil { + t.Fatalf("rpc EmptyCall() failed: %v", err) + } +} diff --git a/xds/internal/balancer/ringhash/logging.go b/xds/internal/balancer/ringhash/logging.go new file mode 100644 index 000000000000..3e0f0adf58eb --- /dev/null +++ b/xds/internal/balancer/ringhash/logging.go @@ -0,0 +1,38 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package ringhash + +import ( + "fmt" + + "google.golang.org/grpc/grpclog" + internalgrpclog "google.golang.org/grpc/internal/grpclog" +) + +const prefix = "[ring-hash-lb %p] " + +var logger = grpclog.Component("xds") + +func prefixLogger(p *ringhashBalancer) *internalgrpclog.PrefixLogger { + return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p)) +} + +func subConnPrefixLogger(p *ringhashBalancer, sc *subConn) *internalgrpclog.PrefixLogger { + return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p)+fmt.Sprintf("[subConn %p] ", sc)) +} diff --git a/xds/internal/balancer/ringhash/picker.go b/xds/internal/balancer/ringhash/picker.go new file mode 100644 index 000000000000..b450716fa0f0 --- /dev/null +++ b/xds/internal/balancer/ringhash/picker.go @@ -0,0 +1,186 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package ringhash + +import ( + "fmt" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/status" +) + +type picker struct { + ring *ring + logger *grpclog.PrefixLogger + subConnStates map[*subConn]connectivity.State +} + +func newPicker(ring *ring, logger *grpclog.PrefixLogger) *picker { + states := make(map[*subConn]connectivity.State) + for _, e := range ring.items { + states[e.sc] = e.sc.effectiveState() + } + return &picker{ring: ring, logger: logger, subConnStates: states} +} + +// handleRICSResult is the return type of handleRICS. It's needed to wrap the +// returned error from Pick() in a struct. With this, if the return values are +// `balancer.PickResult, error, bool`, linter complains because error is not the +// last return value. +type handleRICSResult struct { + pr balancer.PickResult + err error +} + +// handleRICS generates pick result if the entry is in Ready, Idle, Connecting +// or Shutdown. TransientFailure will be handled specifically after this +// function returns. +// +// The first return value indicates if the state is in Ready, Idle, Connecting +// or Shutdown. If it's true, the PickResult and error should be returned from +// Pick() as is. +func (p *picker) handleRICS(e *ringEntry) (handleRICSResult, bool) { + switch state := p.subConnStates[e.sc]; state { + case connectivity.Ready: + return handleRICSResult{pr: balancer.PickResult{SubConn: e.sc.sc}}, true + case connectivity.Idle: + // Trigger Connect() and queue the pick. + e.sc.queueConnect() + return handleRICSResult{err: balancer.ErrNoSubConnAvailable}, true + case connectivity.Connecting: + return handleRICSResult{err: balancer.ErrNoSubConnAvailable}, true + case connectivity.TransientFailure: + // Return ok==false, so TransientFailure will be handled afterwards. + return handleRICSResult{}, false + case connectivity.Shutdown: + // Shutdown can happen in a race where the old picker is called. A new + // picker should already be sent. + return handleRICSResult{err: balancer.ErrNoSubConnAvailable}, true + default: + // Should never reach this. All the connectivity states are already + // handled in the cases. + p.logger.Errorf("SubConn has undefined connectivity state: %v", state) + return handleRICSResult{err: status.Errorf(codes.Unavailable, "SubConn has undefined connectivity state: %v", state)}, true + } +} + +func (p *picker) Pick(info balancer.PickInfo) (balancer.PickResult, error) { + e := p.ring.pick(getRequestHash(info.Ctx)) + if hr, ok := p.handleRICS(e); ok { + return hr.pr, hr.err + } + // ok was false, the entry is in transient failure. + return p.handleTransientFailure(e) +} + +func (p *picker) handleTransientFailure(e *ringEntry) (balancer.PickResult, error) { + // Queue a connect on the first picked SubConn. + e.sc.queueConnect() + + // Find next entry in the ring, skipping duplicate SubConns. + e2 := nextSkippingDuplicates(p.ring, e) + if e2 == nil { + // There's no next entry available, fail the pick. + return balancer.PickResult{}, fmt.Errorf("the only SubConn is in Transient Failure") + } + + // For the second SubConn, also check Ready/Idle/Connecting as if it's the + // first entry. + if hr, ok := p.handleRICS(e2); ok { + return hr.pr, hr.err + } + + // The second SubConn is also in TransientFailure. Queue a connect on it. + e2.sc.queueConnect() + + // If it gets here, this is after the second SubConn, and the second SubConn + // was in TransientFailure. + // + // Loop over all other SubConns: + // - If all SubConns so far are all TransientFailure, trigger Connect() on + // the TransientFailure SubConns, and keep going. + // - If there's one SubConn that's not in TransientFailure, keep checking + // the remaining SubConns (in case there's a Ready, which will be returned), + // but don't not trigger Connect() on the other SubConns. + var firstNonFailedFound bool + for ee := nextSkippingDuplicates(p.ring, e2); ee != e; ee = nextSkippingDuplicates(p.ring, ee) { + scState := p.subConnStates[ee.sc] + if scState == connectivity.Ready { + return balancer.PickResult{SubConn: ee.sc.sc}, nil + } + if firstNonFailedFound { + continue + } + if scState == connectivity.TransientFailure { + // This will queue a connect. + ee.sc.queueConnect() + continue + } + // This is a SubConn in a non-failure state. We continue to check the + // other SubConns, but remember that there was a non-failed SubConn + // seen. After this, Pick() will never trigger any SubConn to Connect(). + firstNonFailedFound = true + if scState == connectivity.Idle { + // This is the first non-failed SubConn, and it is in a real Idle + // state. Trigger it to Connect(). + ee.sc.queueConnect() + } + } + return balancer.PickResult{}, fmt.Errorf("no connection is Ready") +} + +// nextSkippingDuplicates finds the next entry in the ring, with a different +// subconn from the given entry. +func nextSkippingDuplicates(ring *ring, entry *ringEntry) *ringEntry { + for next := ring.next(entry); next != entry; next = ring.next(next) { + if next.sc != entry.sc { + return next + } + } + // There's no qualifying next entry. + return nil +} + +// nextSkippingDuplicatesSubConn finds the next subconn in the ring, that's +// different from the given subconn. +func nextSkippingDuplicatesSubConn(ring *ring, sc *subConn) *subConn { + var entry *ringEntry + for _, it := range ring.items { + if it.sc == sc { + entry = it + break + } + } + if entry == nil { + // If the given subconn is not in the ring (e.g. it was deleted), return + // the first one. + if len(ring.items) > 0 { + return ring.items[0].sc + } + return nil + } + ee := nextSkippingDuplicates(ring, entry) + if ee == nil { + return nil + } + return ee.sc +} diff --git a/xds/internal/balancer/ringhash/picker_test.go b/xds/internal/balancer/ringhash/picker_test.go new file mode 100644 index 000000000000..f1dbaf2e5ed8 --- /dev/null +++ b/xds/internal/balancer/ringhash/picker_test.go @@ -0,0 +1,296 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package ringhash + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/grpclog" + igrpclog "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/testutils" +) + +var testSubConns []*testutils.TestSubConn + +func init() { + for i := 0; i < 8; i++ { + testSubConns = append(testSubConns, testutils.NewTestSubConn(fmt.Sprint(i))) + } +} + +func newTestRing(cStats []connectivity.State) *ring { + var items []*ringEntry + for i, st := range cStats { + testSC := testSubConns[i] + items = append(items, &ringEntry{ + idx: i, + hash: uint64((i + 1) * 10), + sc: &subConn{ + addr: testSC.String(), + sc: testSC, + state: st, + }, + }) + } + return &ring{items: items} +} + +func (s) TestPickerPickFirstTwo(t *testing.T) { + tests := []struct { + name string + ring *ring + hash uint64 + wantSC balancer.SubConn + wantErr error + wantSCToConnect balancer.SubConn + }{ + { + name: "picked is Ready", + ring: newTestRing([]connectivity.State{connectivity.Ready, connectivity.Idle}), + hash: 5, + wantSC: testSubConns[0], + }, + { + name: "picked is connecting, queue", + ring: newTestRing([]connectivity.State{connectivity.Connecting, connectivity.Idle}), + hash: 5, + wantErr: balancer.ErrNoSubConnAvailable, + }, + { + name: "picked is Idle, connect and queue", + ring: newTestRing([]connectivity.State{connectivity.Idle, connectivity.Idle}), + hash: 5, + wantErr: balancer.ErrNoSubConnAvailable, + wantSCToConnect: testSubConns[0], + }, + { + name: "picked is TransientFailure, next is ready, return", + ring: newTestRing([]connectivity.State{connectivity.TransientFailure, connectivity.Ready}), + hash: 5, + wantSC: testSubConns[1], + }, + { + name: "picked is TransientFailure, next is connecting, queue", + ring: newTestRing([]connectivity.State{connectivity.TransientFailure, connectivity.Connecting}), + hash: 5, + wantErr: balancer.ErrNoSubConnAvailable, + }, + { + name: "picked is TransientFailure, next is Idle, connect and queue", + ring: newTestRing([]connectivity.State{connectivity.TransientFailure, connectivity.Idle}), + hash: 5, + wantErr: balancer.ErrNoSubConnAvailable, + wantSCToConnect: testSubConns[1], + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := newPicker(tt.ring, igrpclog.NewPrefixLogger(grpclog.Component("xds"), "rh_test")) + got, err := p.Pick(balancer.PickInfo{ + Ctx: SetRequestHash(context.Background(), tt.hash), + }) + if err != tt.wantErr { + t.Errorf("Pick() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got.SubConn != tt.wantSC { + t.Errorf("Pick() got = %v, want picked SubConn: %v", got, tt.wantSC) + } + if sc := tt.wantSCToConnect; sc != nil { + select { + case <-sc.(*testutils.TestSubConn).ConnectCh: + case <-time.After(defaultTestShortTimeout): + t.Errorf("timeout waiting for Connect() from SubConn %v", sc) + } + } + }) + } +} + +// TestPickerPickTriggerTFConnect covers that if the picked SubConn is +// TransientFailures, all SubConns until a non-TransientFailure are queued for +// Connect(). +func (s) TestPickerPickTriggerTFConnect(t *testing.T) { + ring := newTestRing([]connectivity.State{ + connectivity.TransientFailure, connectivity.TransientFailure, connectivity.TransientFailure, connectivity.TransientFailure, + connectivity.Idle, connectivity.TransientFailure, connectivity.TransientFailure, connectivity.TransientFailure, + }) + p := newPicker(ring, igrpclog.NewPrefixLogger(grpclog.Component("xds"), "rh_test")) + _, err := p.Pick(balancer.PickInfo{Ctx: SetRequestHash(context.Background(), 5)}) + if err == nil { + t.Fatalf("Pick() error = %v, want non-nil", err) + } + // The first 4 SubConns, all in TransientFailure, should be queued to + // connect. + for i := 0; i < 4; i++ { + it := ring.items[i] + if !it.sc.connectQueued { + t.Errorf("the %d-th SubConn is not queued for connect", i) + } + } + // The other SubConns, after the first Idle, should not be queued to + // connect. + for i := 5; i < len(ring.items); i++ { + it := ring.items[i] + if it.sc.connectQueued { + t.Errorf("the %d-th SubConn is unexpected queued for connect", i) + } + } +} + +// TestPickerPickTriggerTFReturnReady covers that if the picked SubConn is +// TransientFailure, SubConn 2 and 3 are TransientFailure, 4 is Ready. SubConn 2 +// and 3 will Connect(), and 4 will be returned. +func (s) TestPickerPickTriggerTFReturnReady(t *testing.T) { + ring := newTestRing([]connectivity.State{ + connectivity.TransientFailure, connectivity.TransientFailure, connectivity.TransientFailure, connectivity.Ready, + }) + p := newPicker(ring, igrpclog.NewPrefixLogger(grpclog.Component("xds"), "rh_test")) + pr, err := p.Pick(balancer.PickInfo{Ctx: SetRequestHash(context.Background(), 5)}) + if err != nil { + t.Fatalf("Pick() error = %v, want nil", err) + } + if wantSC := testSubConns[3]; pr.SubConn != wantSC { + t.Fatalf("Pick() = %v, want %v", pr.SubConn, wantSC) + } + // The first 3 SubConns, all in TransientFailure, should be queued to + // connect. + for i := 0; i < 3; i++ { + it := ring.items[i] + if !it.sc.connectQueued { + t.Errorf("the %d-th SubConn is not queued for connect", i) + } + } +} + +// TestPickerPickTriggerTFWithIdle covers that if the picked SubConn is +// TransientFailure, SubConn 2 is TransientFailure, 3 is Idle (init Idle). Pick +// will be queue, SubConn 3 will Connect(), SubConn 4 and 5 (in TransientFailre) +// will not queue a Connect. +func (s) TestPickerPickTriggerTFWithIdle(t *testing.T) { + ring := newTestRing([]connectivity.State{ + connectivity.TransientFailure, connectivity.TransientFailure, connectivity.Idle, connectivity.TransientFailure, connectivity.TransientFailure, + }) + p := newPicker(ring, igrpclog.NewPrefixLogger(grpclog.Component("xds"), "rh_test")) + _, err := p.Pick(balancer.PickInfo{Ctx: SetRequestHash(context.Background(), 5)}) + if err == balancer.ErrNoSubConnAvailable { + t.Fatalf("Pick() error = %v, want %v", err, balancer.ErrNoSubConnAvailable) + } + // The first 2 SubConns, all in TransientFailure, should be queued to + // connect. + for i := 0; i < 2; i++ { + it := ring.items[i] + if !it.sc.connectQueued { + t.Errorf("the %d-th SubConn is not queued for connect", i) + } + } + // SubConn 3 was in Idle, so should Connect() + select { + case <-testSubConns[2].ConnectCh: + case <-time.After(defaultTestShortTimeout): + t.Errorf("timeout waiting for Connect() from SubConn %v", testSubConns[2]) + } + // The other SubConns, after the first Idle, should not be queued to + // connect. + for i := 3; i < len(ring.items); i++ { + it := ring.items[i] + if it.sc.connectQueued { + t.Errorf("the %d-th SubConn is unexpected queued for connect", i) + } + } +} + +func (s) TestNextSkippingDuplicatesNoDup(t *testing.T) { + testRing := newTestRing([]connectivity.State{connectivity.Idle, connectivity.Idle}) + tests := []struct { + name string + ring *ring + cur *ringEntry + want *ringEntry + }{ + { + name: "no dup", + ring: testRing, + cur: testRing.items[0], + want: testRing.items[1], + }, + { + name: "only one entry", + ring: &ring{items: []*ringEntry{testRing.items[0]}}, + cur: testRing.items[0], + want: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := nextSkippingDuplicates(tt.ring, tt.cur); !cmp.Equal(got, tt.want, cmpOpts) { + t.Errorf("nextSkippingDuplicates() = %v, want %v", got, tt.want) + } + }) + } +} + +// addDups adds duplicates of items[0] to the ring. +func addDups(r *ring, count int) *ring { + var ( + items []*ringEntry + idx int + ) + for i, it := range r.items { + itt := *it + itt.idx = idx + items = append(items, &itt) + idx++ + if i == 0 { + // Add duplicate of items[0] to the ring + for j := 0; j < count; j++ { + itt2 := *it + itt2.idx = idx + items = append(items, &itt2) + idx++ + } + } + } + return &ring{items: items} +} + +func (s) TestNextSkippingDuplicatesMoreDup(t *testing.T) { + testRing := newTestRing([]connectivity.State{connectivity.Idle, connectivity.Idle}) + // Make a new ring with duplicate SubConns. + dupTestRing := addDups(testRing, 3) + if got := nextSkippingDuplicates(dupTestRing, dupTestRing.items[0]); !cmp.Equal(got, dupTestRing.items[len(dupTestRing.items)-1], cmpOpts) { + t.Errorf("nextSkippingDuplicates() = %v, want %v", got, dupTestRing.items[len(dupTestRing.items)-1]) + } +} + +func (s) TestNextSkippingDuplicatesOnlyDup(t *testing.T) { + testRing := newTestRing([]connectivity.State{connectivity.Idle}) + // Make a new ring with only duplicate SubConns. + dupTestRing := addDups(testRing, 3) + // This ring only has duplicates of items[0], should return nil. + if got := nextSkippingDuplicates(dupTestRing, dupTestRing.items[0]); got != nil { + t.Errorf("nextSkippingDuplicates() = %v, want nil", got) + } +} diff --git a/xds/internal/balancer/ringhash/ring.go b/xds/internal/balancer/ringhash/ring.go new file mode 100644 index 000000000000..4d7fdb35e722 --- /dev/null +++ b/xds/internal/balancer/ringhash/ring.go @@ -0,0 +1,169 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package ringhash + +import ( + "math" + "sort" + "strconv" + + xxhash "github.com/cespare/xxhash/v2" + "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/resolver" +) + +type ring struct { + items []*ringEntry +} + +type subConnWithWeight struct { + sc *subConn + weight float64 +} + +type ringEntry struct { + idx int + hash uint64 + sc *subConn +} + +// newRing creates a ring from the subConns stored in the AddressMap. The ring +// size is limited by the passed in max/min. +// +// ring entries will be created for each subConn, and subConn with high weight +// (specified by the address) may have multiple entries. +// +// For example, for subConns with weights {a:3, b:3, c:4}, a generated ring of +// size 10 could be: +// - {idx:0 hash:3689675255460411075 b} +// - {idx:1 hash:4262906501694543955 c} +// - {idx:2 hash:5712155492001633497 c} +// - {idx:3 hash:8050519350657643659 b} +// - {idx:4 hash:8723022065838381142 b} +// - {idx:5 hash:11532782514799973195 a} +// - {idx:6 hash:13157034721563383607 c} +// - {idx:7 hash:14468677667651225770 c} +// - {idx:8 hash:17336016884672388720 a} +// - {idx:9 hash:18151002094784932496 a} +// +// To pick from a ring, a binary search will be done for the given target hash, +// and first item with hash >= given hash will be returned. +// +// Must be called with a non-empty subConns map. +func newRing(subConns *resolver.AddressMap, minRingSize, maxRingSize uint64, logger *grpclog.PrefixLogger) *ring { + logger.Debugf("newRing: number of subConns is %d, minRingSize is %d, maxRingSize is %d", subConns.Len(), minRingSize, maxRingSize) + + // https://github.com/envoyproxy/envoy/blob/765c970f06a4c962961a0e03a467e165b276d50f/source/common/upstream/ring_hash_lb.cc#L114 + normalizedWeights, minWeight := normalizeWeights(subConns) + logger.Debugf("newRing: normalized subConn weights is %v", normalizedWeights) + + // Normalized weights for {3,3,4} is {0.3,0.3,0.4}. + + // Scale up the size of the ring such that the least-weighted host gets a + // whole number of hashes on the ring. + // + // Note that size is limited by the input max/min. + scale := math.Min(math.Ceil(minWeight*float64(minRingSize))/minWeight, float64(maxRingSize)) + ringSize := math.Ceil(scale) + items := make([]*ringEntry, 0, int(ringSize)) + logger.Debugf("newRing: creating new ring of size %v", ringSize) + + // For each entry, scale*weight nodes are generated in the ring. + // + // Not all of these are whole numbers. E.g. for weights {a:3,b:3,c:4}, if + // ring size is 7, scale is 6.66. The numbers of nodes will be + // {a,a,b,b,c,c,c}. + // + // A hash is generated for each item, and later the results will be sorted + // based on the hash. + var currentHashes, targetHashes float64 + for _, scw := range normalizedWeights { + targetHashes += scale * scw.weight + // This index ensures that ring entries corresponding to the same + // address hash to different values. And since this index is + // per-address, these entries hash to the same value across address + // updates. + idx := 0 + for currentHashes < targetHashes { + h := xxhash.Sum64String(scw.sc.addr + "_" + strconv.Itoa(idx)) + items = append(items, &ringEntry{hash: h, sc: scw.sc}) + idx++ + currentHashes++ + } + } + + // Sort items based on hash, to prepare for binary search. + sort.Slice(items, func(i, j int) bool { return items[i].hash < items[j].hash }) + for i, ii := range items { + ii.idx = i + } + return &ring{items: items} +} + +// normalizeWeights divides all the weights by the sum, so that the total weight +// is 1. +// +// Must be called with a non-empty subConns map. +func normalizeWeights(subConns *resolver.AddressMap) ([]subConnWithWeight, float64) { + var weightSum uint32 + keys := subConns.Keys() + for _, a := range keys { + weightSum += getWeightAttribute(a) + } + ret := make([]subConnWithWeight, 0, len(keys)) + min := float64(1.0) + for _, a := range keys { + v, _ := subConns.Get(a) + scInfo := v.(*subConn) + // getWeightAttribute() returns 1 if the weight attribute is not found + // on the address. And since this function is guaranteed to be called + // with a non-empty subConns map, weightSum is guaranteed to be + // non-zero. So, we need not worry about divide a by zero error here. + nw := float64(getWeightAttribute(a)) / float64(weightSum) + ret = append(ret, subConnWithWeight{sc: scInfo, weight: nw}) + if nw < min { + min = nw + } + } + // Sort the addresses to return consistent results. + // + // Note: this might not be necessary, but this makes sure the ring is + // consistent as long as the addresses are the same, for example, in cases + // where an address is added and then removed, the RPCs will still pick the + // same old SubConn. + sort.Slice(ret, func(i, j int) bool { return ret[i].sc.addr < ret[j].sc.addr }) + return ret, min +} + +// pick does a binary search. It returns the item with smallest index i that +// r.items[i].hash >= h. +func (r *ring) pick(h uint64) *ringEntry { + i := sort.Search(len(r.items), func(i int) bool { return r.items[i].hash >= h }) + if i == len(r.items) { + // If not found, and h is greater than the largest hash, return the + // first item. + i = 0 + } + return r.items[i] +} + +// next returns the next entry. +func (r *ring) next(e *ringEntry) *ringEntry { + return r.items[(e.idx+1)%len(r.items)] +} diff --git a/xds/internal/balancer/ringhash/ring_test.go b/xds/internal/balancer/ringhash/ring_test.go new file mode 100644 index 000000000000..9c6eb0c242ff --- /dev/null +++ b/xds/internal/balancer/ringhash/ring_test.go @@ -0,0 +1,111 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package ringhash + +import ( + "fmt" + "math" + "testing" + + xxhash "github.com/cespare/xxhash/v2" + "google.golang.org/grpc/balancer/weightedroundrobin" + "google.golang.org/grpc/resolver" +) + +var testAddrs []resolver.Address +var testSubConnMap *resolver.AddressMap + +func init() { + testAddrs = []resolver.Address{ + testAddr("a", 3), + testAddr("b", 3), + testAddr("c", 4), + } + testSubConnMap = resolver.NewAddressMap() + testSubConnMap.Set(testAddrs[0], &subConn{addr: "a"}) + testSubConnMap.Set(testAddrs[1], &subConn{addr: "b"}) + testSubConnMap.Set(testAddrs[2], &subConn{addr: "c"}) +} + +func testAddr(addr string, weight uint32) resolver.Address { + return weightedroundrobin.SetAddrInfo(resolver.Address{Addr: addr}, weightedroundrobin.AddrInfo{Weight: weight}) +} + +func (s) TestRingNew(t *testing.T) { + var totalWeight float64 = 10 + for _, min := range []uint64{3, 4, 6, 8} { + for _, max := range []uint64{20, 8} { + t.Run(fmt.Sprintf("size-min-%v-max-%v", min, max), func(t *testing.T) { + r := newRing(testSubConnMap, min, max, nil) + totalCount := len(r.items) + if totalCount < int(min) || totalCount > int(max) { + t.Fatalf("unexpect size %v, want min %v, max %v", totalCount, min, max) + } + for _, a := range testAddrs { + var count int + for _, ii := range r.items { + if ii.sc.addr == a.Addr { + count++ + } + } + got := float64(count) / float64(totalCount) + want := float64(getWeightAttribute(a)) / totalWeight + if !equalApproximately(got, want) { + t.Fatalf("unexpected item weight in ring: %v != %v", got, want) + } + } + }) + } + } +} + +func equalApproximately(x, y float64) bool { + delta := math.Abs(x - y) + mean := math.Abs(x+y) / 2.0 + return delta/mean < 0.25 +} + +func (s) TestRingPick(t *testing.T) { + r := newRing(testSubConnMap, 10, 20, nil) + for _, h := range []uint64{xxhash.Sum64String("1"), xxhash.Sum64String("2"), xxhash.Sum64String("3"), xxhash.Sum64String("4")} { + t.Run(fmt.Sprintf("picking-hash-%v", h), func(t *testing.T) { + e := r.pick(h) + var low uint64 + if e.idx > 0 { + low = r.items[e.idx-1].hash + } + high := e.hash + // h should be in [low, high). + if h < low || h >= high { + t.Fatalf("unexpected item picked, hash: %v, low: %v, high: %v", h, low, high) + } + }) + } +} + +func (s) TestRingNext(t *testing.T) { + r := newRing(testSubConnMap, 10, 20, nil) + + for _, e := range r.items { + ne := r.next(e) + if ne.idx != (e.idx+1)%len(r.items) { + t.Fatalf("next(%+v) returned unexpected %+v", e, ne) + } + } +} diff --git a/xds/internal/balancer/ringhash/ringhash.go b/xds/internal/balancer/ringhash/ringhash.go new file mode 100644 index 000000000000..e63c6f653904 --- /dev/null +++ b/xds/internal/balancer/ringhash/ringhash.go @@ -0,0 +1,518 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package ringhash implements the ringhash balancer. +package ringhash + +import ( + "encoding/json" + "errors" + "fmt" + "sync" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/base" + "google.golang.org/grpc/balancer/weightedroundrobin" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" +) + +// Name is the name of the ring_hash balancer. +const Name = "ring_hash_experimental" + +func init() { + balancer.Register(bb{}) +} + +type bb struct{} + +func (bb) Build(cc balancer.ClientConn, bOpts balancer.BuildOptions) balancer.Balancer { + b := &ringhashBalancer{ + cc: cc, + subConns: resolver.NewAddressMap(), + scStates: make(map[balancer.SubConn]*subConn), + csEvltr: &connectivityStateEvaluator{}, + } + b.logger = prefixLogger(b) + b.logger.Infof("Created") + return b +} + +func (bb) Name() string { + return Name +} + +func (bb) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + return parseConfig(c) +} + +type subConn struct { + addr string + weight uint32 + sc balancer.SubConn + logger *grpclog.PrefixLogger + + mu sync.RWMutex + // This is the actual state of this SubConn (as updated by the ClientConn). + // The effective state can be different, see comment of attemptedToConnect. + state connectivity.State + // failing is whether this SubConn is in a failing state. A subConn is + // considered to be in a failing state if it was previously in + // TransientFailure. + // + // This affects the effective connectivity state of this SubConn, e.g. + // - if the actual state is Idle or Connecting, but this SubConn is failing, + // the effective state is TransientFailure. + // + // This is used in pick(). E.g. if a subConn is Idle, but has failing as + // true, pick() will + // - consider this SubConn as TransientFailure, and check the state of the + // next SubConn. + // - trigger Connect() (note that normally a SubConn in real + // TransientFailure cannot Connect()) + // + // A subConn starts in non-failing (failing is false). A transition to + // TransientFailure sets failing to true (and it stays true). A transition + // to Ready sets failing to false. + failing bool + // connectQueued is true if a Connect() was queued for this SubConn while + // it's not in Idle (most likely was in TransientFailure). A Connect() will + // be triggered on this SubConn when it turns Idle. + // + // When connectivity state is updated to Idle for this SubConn, if + // connectQueued is true, Connect() will be called on the SubConn. + connectQueued bool + // attemptingToConnect indicates if this subconn is attempting to connect. + // It's set when queueConnect is called. It's unset when the state is + // changed to Ready/Shutdown, or Idle (and if connectQueued is false). + attemptingToConnect bool +} + +// setState updates the state of this SubConn. +// +// It also handles the queued Connect(). If the new state is Idle, and a +// Connect() was queued, this SubConn will be triggered to Connect(). +func (sc *subConn) setState(s connectivity.State) { + sc.mu.Lock() + defer sc.mu.Unlock() + switch s { + case connectivity.Idle: + // Trigger Connect() if new state is Idle, and there is a queued connect. + if sc.connectQueued { + sc.connectQueued = false + sc.logger.Infof("Executing a queued connect for subConn moving to state: %v", sc.state) + sc.sc.Connect() + } else { + sc.attemptingToConnect = false + } + case connectivity.Connecting: + // Clear connectQueued if the SubConn isn't failing. This state + // transition is unlikely to happen, but handle this just in case. + sc.connectQueued = false + case connectivity.Ready: + // Clear connectQueued if the SubConn isn't failing. This state + // transition is unlikely to happen, but handle this just in case. + sc.connectQueued = false + sc.attemptingToConnect = false + // Set to a non-failing state. + sc.failing = false + case connectivity.TransientFailure: + // Set to a failing state. + sc.failing = true + case connectivity.Shutdown: + sc.attemptingToConnect = false + } + sc.state = s +} + +// effectiveState returns the effective state of this SubConn. It can be +// different from the actual state, e.g. Idle while the subConn is failing is +// considered TransientFailure. Read comment of field failing for other cases. +func (sc *subConn) effectiveState() connectivity.State { + sc.mu.RLock() + defer sc.mu.RUnlock() + if sc.failing && (sc.state == connectivity.Idle || sc.state == connectivity.Connecting) { + return connectivity.TransientFailure + } + return sc.state +} + +// queueConnect sets a boolean so that when the SubConn state changes to Idle, +// it's Connect() will be triggered. If the SubConn state is already Idle, it +// will just call Connect(). +func (sc *subConn) queueConnect() { + sc.mu.Lock() + defer sc.mu.Unlock() + sc.attemptingToConnect = true + if sc.state == connectivity.Idle { + sc.logger.Infof("Executing a queued connect for subConn in state: %v", sc.state) + sc.sc.Connect() + return + } + // Queue this connect, and when this SubConn switches back to Idle (happens + // after backoff in TransientFailure), it will Connect(). + sc.logger.Infof("Queueing a connect for subConn in state: %v", sc.state) + sc.connectQueued = true +} + +func (sc *subConn) isAttemptingToConnect() bool { + sc.mu.Lock() + defer sc.mu.Unlock() + return sc.attemptingToConnect +} + +type ringhashBalancer struct { + cc balancer.ClientConn + logger *grpclog.PrefixLogger + + config *LBConfig + subConns *resolver.AddressMap // Map from resolver.Address to `*subConn`. + scStates map[balancer.SubConn]*subConn + + // ring is always in sync with subConns. When subConns change, a new ring is + // generated. Note that address weights updates (they are keys in the + // subConns map) also regenerates the ring. + ring *ring + picker balancer.Picker + csEvltr *connectivityStateEvaluator + state connectivity.State + + resolverErr error // the last error reported by the resolver; cleared on successful resolution + connErr error // the last connection error; cleared upon leaving TransientFailure +} + +// updateAddresses creates new SubConns and removes SubConns, based on the +// address update. +// +// The return value is whether the new address list is different from the +// previous. True if +// - an address was added +// - an address was removed +// - an address's weight was updated +// +// Note that this function doesn't trigger SubConn connecting, so all the new +// SubConn states are Idle. +func (b *ringhashBalancer) updateAddresses(addrs []resolver.Address) bool { + var addrsUpdated bool + // addrsSet is the set converted from addrs, used for quick lookup. + addrsSet := resolver.NewAddressMap() + for _, addr := range addrs { + addrsSet.Set(addr, true) + newWeight := getWeightAttribute(addr) + if val, ok := b.subConns.Get(addr); !ok { + var sc balancer.SubConn + opts := balancer.NewSubConnOptions{ + HealthCheckEnabled: true, + StateListener: func(state balancer.SubConnState) { b.updateSubConnState(sc, state) }, + } + sc, err := b.cc.NewSubConn([]resolver.Address{addr}, opts) + if err != nil { + b.logger.Warningf("Failed to create new SubConn: %v", err) + continue + } + scs := &subConn{addr: addr.Addr, weight: newWeight, sc: sc} + scs.logger = subConnPrefixLogger(b, scs) + scs.setState(connectivity.Idle) + b.state = b.csEvltr.recordTransition(connectivity.Shutdown, connectivity.Idle) + b.subConns.Set(addr, scs) + b.scStates[sc] = scs + addrsUpdated = true + } else { + // We have seen this address before and created a subConn for it. If the + // weight associated with the address has changed, update the subConns map + // with the new weight. This will be used when a new ring is created. + // + // There is no need to call UpdateAddresses on the subConn at this point + // since *only* the weight attribute has changed, and that does not affect + // subConn uniqueness. + scInfo := val.(*subConn) + if oldWeight := scInfo.weight; oldWeight != newWeight { + scInfo.weight = newWeight + b.subConns.Set(addr, scInfo) + // Return true to force recreation of the ring. + addrsUpdated = true + } + } + } + for _, addr := range b.subConns.Keys() { + // addr was removed by resolver. + if _, ok := addrsSet.Get(addr); !ok { + v, _ := b.subConns.Get(addr) + scInfo := v.(*subConn) + scInfo.sc.Shutdown() + b.subConns.Delete(addr) + addrsUpdated = true + // Keep the state of this sc in b.scStates until sc's state becomes Shutdown. + // The entry will be deleted in updateSubConnState. + } + } + return addrsUpdated +} + +func (b *ringhashBalancer) UpdateClientConnState(s balancer.ClientConnState) error { + b.logger.Infof("Received update from resolver, balancer config: %+v", pretty.ToJSON(s.BalancerConfig)) + newConfig, ok := s.BalancerConfig.(*LBConfig) + if !ok { + return fmt.Errorf("unexpected balancer config with type: %T", s.BalancerConfig) + } + + // If addresses were updated, whether it resulted in SubConn + // creation/deletion, or just weight update, we need to regenerate the ring + // and send a new picker. + regenerateRing := b.updateAddresses(s.ResolverState.Addresses) + + // If the ring configuration has changed, we need to regenerate the ring and + // send a new picker. + if b.config == nil || b.config.MinRingSize != newConfig.MinRingSize || b.config.MaxRingSize != newConfig.MaxRingSize { + regenerateRing = true + } + b.config = newConfig + + // If resolver state contains no addresses, return an error so ClientConn + // will trigger re-resolve. Also records this as an resolver error, so when + // the overall state turns transient failure, the error message will have + // the zero address information. + if len(s.ResolverState.Addresses) == 0 { + b.ResolverError(errors.New("produced zero addresses")) + return balancer.ErrBadResolverState + } + + if regenerateRing { + // Ring creation is guaranteed to not fail because we call newRing() + // with a non-empty subConns map. + b.ring = newRing(b.subConns, b.config.MinRingSize, b.config.MaxRingSize, b.logger) + b.regeneratePicker() + b.cc.UpdateState(balancer.State{ConnectivityState: b.state, Picker: b.picker}) + } + + // Successful resolution; clear resolver error and return nil. + b.resolverErr = nil + return nil +} + +func (b *ringhashBalancer) ResolverError(err error) { + b.resolverErr = err + if b.subConns.Len() == 0 { + b.state = connectivity.TransientFailure + } + + if b.state != connectivity.TransientFailure { + // The picker will not change since the balancer does not currently + // report an error. + return + } + b.regeneratePicker() + b.cc.UpdateState(balancer.State{ + ConnectivityState: b.state, + Picker: b.picker, + }) +} + +func (b *ringhashBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { + b.logger.Errorf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, state) +} + +// updateSubConnState updates the per-SubConn state stored in the ring, and also +// the aggregated state. +// +// It triggers an update to cc when: +// - the new state is TransientFailure, to update the error message +// - it's possible that this is a noop, but sending an extra update is easier +// than comparing errors +// +// - the aggregated state is changed +// - the same picker will be sent again, but this update may trigger a re-pick +// for some RPCs. +func (b *ringhashBalancer) updateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { + s := state.ConnectivityState + if logger.V(2) { + b.logger.Infof("Handle SubConn state change: %p, %v", sc, s) + } + scs, ok := b.scStates[sc] + if !ok { + b.logger.Infof("Received state change for an unknown SubConn: %p, %v", sc, s) + return + } + oldSCState := scs.effectiveState() + scs.setState(s) + newSCState := scs.effectiveState() + b.logger.Infof("SubConn's effective old state was: %v, new state is %v", oldSCState, newSCState) + + b.state = b.csEvltr.recordTransition(oldSCState, newSCState) + + switch s { + case connectivity.TransientFailure: + // Save error to be reported via picker. + b.connErr = state.ConnectionError + case connectivity.Shutdown: + // When an address was removed by resolver, b called Shutdown but kept + // the sc's state in scStates. Remove state for this sc here. + delete(b.scStates, sc) + } + + if oldSCState != newSCState { + // Because the picker caches the state of the subconns, we always + // regenerate and update the picker when the effective SubConn state + // changes. + b.regeneratePicker() + b.logger.Infof("Pushing new state %v and picker %p", b.state, b.picker) + b.cc.UpdateState(balancer.State{ConnectivityState: b.state, Picker: b.picker}) + } + + switch b.state { + case connectivity.Connecting, connectivity.TransientFailure: + // When overall state is TransientFailure, we need to make sure at least + // one SubConn is attempting to connect, otherwise this balancer may + // never get picks if the parent is priority. + // + // Because we report Connecting as the overall state when only one + // SubConn is in TransientFailure, we do the same check for Connecting + // here. + // + // Note that this check also covers deleting SubConns due to address + // change. E.g. if the SubConn attempting to connect is deleted, and the + // overall state is TF. Since there must be at least one SubConn + // attempting to connect, we need to trigger one. But since the deleted + // SubConn will eventually send a shutdown update, this code will run + // and trigger the next SubConn to connect. + for _, v := range b.subConns.Values() { + sc := v.(*subConn) + if sc.isAttemptingToConnect() { + return + } + } + // Trigger a SubConn (this updated SubConn's next SubConn in the ring) + // to connect if nobody is attempting to connect. + sc := nextSkippingDuplicatesSubConn(b.ring, scs) + if sc != nil { + sc.queueConnect() + return + } + // This handles the edge case where we have a single subConn in the + // ring. nextSkippingDuplicatesSubCon() would have returned nil. We + // still need to ensure that some subConn is attempting to connect, in + // order to give the LB policy a chance to move out of + // TRANSIENT_FAILURE. Hence, we try connecting on the current subConn. + scs.queueConnect() + } +} + +// mergeErrors builds an error from the last connection error and the last +// resolver error. Must only be called if b.state is TransientFailure. +func (b *ringhashBalancer) mergeErrors() error { + // connErr must always be non-nil unless there are no SubConns, in which + // case resolverErr must be non-nil. + if b.connErr == nil { + return fmt.Errorf("last resolver error: %v", b.resolverErr) + } + if b.resolverErr == nil { + return fmt.Errorf("last connection error: %v", b.connErr) + } + return fmt.Errorf("last connection error: %v; last resolver error: %v", b.connErr, b.resolverErr) +} + +func (b *ringhashBalancer) regeneratePicker() { + if b.state == connectivity.TransientFailure { + b.picker = base.NewErrPicker(b.mergeErrors()) + return + } + b.picker = newPicker(b.ring, b.logger) +} + +func (b *ringhashBalancer) Close() { + b.logger.Infof("Shutdown") +} + +func (b *ringhashBalancer) ExitIdle() { + // ExitIdle implementation is a no-op because connections are either + // triggers from picks or from subConn state changes. +} + +// connectivityStateEvaluator takes the connectivity states of multiple SubConns +// and returns one aggregated connectivity state. +// +// It's not thread safe. +type connectivityStateEvaluator struct { + sum uint64 + nums [5]uint64 +} + +// recordTransition records state change happening in subConn and based on that +// it evaluates what aggregated state should be. +// +// - If there is at least one subchannel in READY state, report READY. +// - If there are 2 or more subchannels in TRANSIENT_FAILURE state, report TRANSIENT_FAILURE. +// - If there is at least one subchannel in CONNECTING state, report CONNECTING. +// - If there is one subchannel in TRANSIENT_FAILURE and there is more than one subchannel, report state CONNECTING. +// - If there is at least one subchannel in Idle state, report Idle. +// - Otherwise, report TRANSIENT_FAILURE. +// +// Note that if there are 1 connecting, 2 transient failure, the overall state +// is transient failure. This is because the second transient failure is a +// fallback of the first failing SubConn, and we want to report transient +// failure to failover to the lower priority. +func (cse *connectivityStateEvaluator) recordTransition(oldState, newState connectivity.State) connectivity.State { + // Update counters. + for idx, state := range []connectivity.State{oldState, newState} { + updateVal := 2*uint64(idx) - 1 // -1 for oldState and +1 for new. + cse.nums[state] += updateVal + } + if oldState == connectivity.Shutdown { + // There's technically no transition from Shutdown. But we record a + // Shutdown->Idle transition when a new SubConn is created. + cse.sum++ + } + if newState == connectivity.Shutdown { + cse.sum-- + } + + if cse.nums[connectivity.Ready] > 0 { + return connectivity.Ready + } + if cse.nums[connectivity.TransientFailure] > 1 { + return connectivity.TransientFailure + } + if cse.nums[connectivity.Connecting] > 0 { + return connectivity.Connecting + } + if cse.nums[connectivity.TransientFailure] > 0 && cse.sum > 1 { + return connectivity.Connecting + } + if cse.nums[connectivity.Idle] > 0 { + return connectivity.Idle + } + return connectivity.TransientFailure +} + +// getWeightAttribute is a convenience function which returns the value of the +// weight attribute stored in the BalancerAttributes field of addr, using the +// weightedroundrobin package. +// +// When used in the xDS context, the weight attribute is guaranteed to be +// non-zero. But, when used in a non-xDS context, the weight attribute could be +// unset. A Default of 1 is used in the latter case. +func getWeightAttribute(addr resolver.Address) uint32 { + w := weightedroundrobin.GetAddrInfo(addr).Weight + if w == 0 { + return 1 + } + return w +} diff --git a/xds/internal/balancer/ringhash/ringhash_test.go b/xds/internal/balancer/ringhash/ringhash_test.go new file mode 100644 index 000000000000..16872dd346b1 --- /dev/null +++ b/xds/internal/balancer/ringhash/ringhash_test.go @@ -0,0 +1,556 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package ringhash + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "google.golang.org/grpc/attributes" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/weightedroundrobin" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/xds/internal" +) + +var ( + cmpOpts = cmp.Options{ + cmp.AllowUnexported(testutils.TestSubConn{}, ringEntry{}, subConn{}), + cmpopts.IgnoreFields(subConn{}, "mu"), + cmpopts.IgnoreFields(testutils.TestSubConn{}, "connectCalled"), + } +) + +const ( + defaultTestTimeout = 10 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond + + testBackendAddrsCount = 12 +) + +var ( + testBackendAddrStrs []string + testConfig = &LBConfig{MinRingSize: 1, MaxRingSize: 10} +) + +func init() { + for i := 0; i < testBackendAddrsCount; i++ { + testBackendAddrStrs = append(testBackendAddrStrs, fmt.Sprintf("%d.%d.%d.%d:%d", i, i, i, i, i)) + } +} + +func ctxWithHash(h uint64) context.Context { + return SetRequestHash(context.Background(), h) +} + +// setupTest creates the balancer, and does an initial sanity check. +func setupTest(t *testing.T, addrs []resolver.Address) (*testutils.TestClientConn, balancer.Balancer, balancer.Picker) { + t.Helper() + cc := testutils.NewTestClientConn(t) + builder := balancer.Get(Name) + b := builder.Build(cc, balancer.BuildOptions{}) + if b == nil { + t.Fatalf("builder.Build(%s) failed and returned nil", Name) + } + if err := b.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Addresses: addrs}, + BalancerConfig: testConfig, + }); err != nil { + t.Fatalf("UpdateClientConnState returned err: %v", err) + } + + for _, addr := range addrs { + addr1 := <-cc.NewSubConnAddrsCh + if want := []resolver.Address{addr}; !cmp.Equal(addr1, want, cmp.AllowUnexported(attributes.Attributes{})) { + t.Fatalf("got unexpected new subconn addrs: %v", cmp.Diff(addr1, want, cmp.AllowUnexported(attributes.Attributes{}))) + } + sc1 := <-cc.NewSubConnCh + // All the SubConns start in Idle, and should not Connect(). + select { + case <-sc1.ConnectCh: + t.Errorf("unexpected Connect() from SubConn %v", sc1) + case <-time.After(defaultTestShortTimeout): + } + } + + // Should also have a picker, with all SubConns in Idle. + p1 := <-cc.NewPickerCh + return cc, b, p1 +} + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// TestUpdateClientConnState_NewRingSize tests the scenario where the ringhash +// LB policy receives new configuration which specifies new values for the ring +// min and max sizes. The test verifies that a new ring is created and a new +// picker is sent to the ClientConn. +func (s) TestUpdateClientConnState_NewRingSize(t *testing.T) { + origMinRingSize, origMaxRingSize := 1, 10 // Configured from `testConfig` in `setupTest` + newMinRingSize, newMaxRingSize := 20, 100 + + addrs := []resolver.Address{{Addr: testBackendAddrStrs[0]}} + cc, b, p1 := setupTest(t, addrs) + ring1 := p1.(*picker).ring + if ringSize := len(ring1.items); ringSize < origMinRingSize || ringSize > origMaxRingSize { + t.Fatalf("Ring created with size %d, want between [%d, %d]", ringSize, origMinRingSize, origMaxRingSize) + } + + if err := b.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Addresses: addrs}, + BalancerConfig: &LBConfig{MinRingSize: uint64(newMinRingSize), MaxRingSize: uint64(newMaxRingSize)}, + }); err != nil { + t.Fatalf("UpdateClientConnState returned err: %v", err) + } + + var ring2 *ring + select { + case <-time.After(defaultTestTimeout): + t.Fatal("Timeout when waiting for a picker update after a configuration update") + case p2 := <-cc.NewPickerCh: + ring2 = p2.(*picker).ring + } + if ringSize := len(ring2.items); ringSize < newMinRingSize || ringSize > newMaxRingSize { + t.Fatalf("Ring created with size %d, want between [%d, %d]", ringSize, newMinRingSize, newMaxRingSize) + } +} + +func (s) TestOneSubConn(t *testing.T) { + wantAddr1 := resolver.Address{Addr: testBackendAddrStrs[0]} + cc, _, p0 := setupTest(t, []resolver.Address{wantAddr1}) + ring0 := p0.(*picker).ring + + firstHash := ring0.items[0].hash + // firstHash-1 will pick the first (and only) SubConn from the ring. + testHash := firstHash - 1 + // The first pick should be queued, and should trigger Connect() on the only + // SubConn. + if _, err := p0.Pick(balancer.PickInfo{Ctx: ctxWithHash(testHash)}); err != balancer.ErrNoSubConnAvailable { + t.Fatalf("first pick returned err %v, want %v", err, balancer.ErrNoSubConnAvailable) + } + sc0 := ring0.items[0].sc.sc.(*testutils.TestSubConn) + select { + case <-sc0.ConnectCh: + case <-time.After(defaultTestTimeout): + t.Errorf("timeout waiting for Connect() from SubConn %v", sc0) + } + + // Send state updates to Ready. + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // Test pick with one backend. + p1 := <-cc.NewPickerCh + for i := 0; i < 5; i++ { + gotSCSt, _ := p1.Pick(balancer.PickInfo{Ctx: ctxWithHash(testHash)}) + if gotSCSt.SubConn != sc0 { + t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc0) + } + } +} + +// TestThreeBackendsAffinity covers that there are 3 SubConns, RPCs with the +// same hash always pick the same SubConn. When the one picked is down, another +// one will be picked. +func (s) TestThreeSubConnsAffinity(t *testing.T) { + wantAddrs := []resolver.Address{ + {Addr: testBackendAddrStrs[0]}, + {Addr: testBackendAddrStrs[1]}, + {Addr: testBackendAddrStrs[2]}, + } + cc, _, p0 := setupTest(t, wantAddrs) + // This test doesn't update addresses, so this ring will be used by all the + // pickers. + ring0 := p0.(*picker).ring + + firstHash := ring0.items[0].hash + // firstHash+1 will pick the second SubConn from the ring. + testHash := firstHash + 1 + // The first pick should be queued, and should trigger Connect() on the only + // SubConn. + if _, err := p0.Pick(balancer.PickInfo{Ctx: ctxWithHash(testHash)}); err != balancer.ErrNoSubConnAvailable { + t.Fatalf("first pick returned err %v, want %v", err, balancer.ErrNoSubConnAvailable) + } + // The picked SubConn should be the second in the ring. + sc0 := ring0.items[1].sc.sc.(*testutils.TestSubConn) + select { + case <-sc0.ConnectCh: + case <-time.After(defaultTestTimeout): + t.Errorf("timeout waiting for Connect() from SubConn %v", sc0) + } + + // Send state updates to Ready. + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + p1 := <-cc.NewPickerCh + for i := 0; i < 5; i++ { + gotSCSt, _ := p1.Pick(balancer.PickInfo{Ctx: ctxWithHash(testHash)}) + if gotSCSt.SubConn != sc0 { + t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc0) + } + } + + // Turn down the subConn in use. + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) + p2 := <-cc.NewPickerCh + // Pick with the same hash should be queued, because the SubConn after the + // first picked is Idle. + if _, err := p2.Pick(balancer.PickInfo{Ctx: ctxWithHash(testHash)}); err != balancer.ErrNoSubConnAvailable { + t.Fatalf("first pick returned err %v, want %v", err, balancer.ErrNoSubConnAvailable) + } + + // The third SubConn in the ring should connect. + sc1 := ring0.items[2].sc.sc.(*testutils.TestSubConn) + select { + case <-sc1.ConnectCh: + case <-time.After(defaultTestTimeout): + t.Errorf("timeout waiting for Connect() from SubConn %v", sc1) + } + + // Send state updates to Ready. + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + // New picks should all return this SubConn. + p3 := <-cc.NewPickerCh + for i := 0; i < 5; i++ { + gotSCSt, _ := p3.Pick(balancer.PickInfo{Ctx: ctxWithHash(testHash)}) + if gotSCSt.SubConn != sc1 { + t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc1) + } + } + + // Now, after backoff, the first picked SubConn will turn Idle. + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Idle}) + // The picks above should have queued Connect() for the first picked + // SubConn, so this Idle state change will trigger a Connect(). + select { + case <-sc0.ConnectCh: + case <-time.After(defaultTestTimeout): + t.Errorf("timeout waiting for Connect() from SubConn %v", sc0) + } + + // After the first picked SubConn turn Ready, new picks should return it + // again (even though the second picked SubConn is also Ready). + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + p4 := <-cc.NewPickerCh + for i := 0; i < 5; i++ { + gotSCSt, _ := p4.Pick(balancer.PickInfo{Ctx: ctxWithHash(testHash)}) + if gotSCSt.SubConn != sc0 { + t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc0) + } + } +} + +// TestThreeBackendsAffinity covers that there are 3 SubConns, RPCs with the +// same hash always pick the same SubConn. Then try different hash to pick +// another backend, and verify the first hash still picks the first backend. +func (s) TestThreeSubConnsAffinityMultiple(t *testing.T) { + wantAddrs := []resolver.Address{ + {Addr: testBackendAddrStrs[0]}, + {Addr: testBackendAddrStrs[1]}, + {Addr: testBackendAddrStrs[2]}, + } + cc, _, p0 := setupTest(t, wantAddrs) + // This test doesn't update addresses, so this ring will be used by all the + // pickers. + ring0 := p0.(*picker).ring + + firstHash := ring0.items[0].hash + // firstHash+1 will pick the second SubConn from the ring. + testHash := firstHash + 1 + // The first pick should be queued, and should trigger Connect() on the only + // SubConn. + if _, err := p0.Pick(balancer.PickInfo{Ctx: ctxWithHash(testHash)}); err != balancer.ErrNoSubConnAvailable { + t.Fatalf("first pick returned err %v, want %v", err, balancer.ErrNoSubConnAvailable) + } + sc0 := ring0.items[1].sc.sc.(*testutils.TestSubConn) + select { + case <-sc0.ConnectCh: + case <-time.After(defaultTestTimeout): + t.Errorf("timeout waiting for Connect() from SubConn %v", sc0) + } + + // Send state updates to Ready. + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // First hash should always pick sc0. + p1 := <-cc.NewPickerCh + for i := 0; i < 5; i++ { + gotSCSt, _ := p1.Pick(balancer.PickInfo{Ctx: ctxWithHash(testHash)}) + if gotSCSt.SubConn != sc0 { + t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc0) + } + } + + secondHash := ring0.items[1].hash + // secondHash+1 will pick the third SubConn from the ring. + testHash2 := secondHash + 1 + if _, err := p0.Pick(balancer.PickInfo{Ctx: ctxWithHash(testHash2)}); err != balancer.ErrNoSubConnAvailable { + t.Fatalf("first pick returned err %v, want %v", err, balancer.ErrNoSubConnAvailable) + } + sc1 := ring0.items[2].sc.sc.(*testutils.TestSubConn) + select { + case <-sc1.ConnectCh: + case <-time.After(defaultTestTimeout): + t.Errorf("timeout waiting for Connect() from SubConn %v", sc1) + } + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting}) + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready}) + + // With the new generated picker, hash2 always picks sc1. + p2 := <-cc.NewPickerCh + for i := 0; i < 5; i++ { + gotSCSt, _ := p2.Pick(balancer.PickInfo{Ctx: ctxWithHash(testHash2)}) + if gotSCSt.SubConn != sc1 { + t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc1) + } + } + // But the first hash still picks sc0. + for i := 0; i < 5; i++ { + gotSCSt, _ := p2.Pick(balancer.PickInfo{Ctx: ctxWithHash(testHash)}) + if gotSCSt.SubConn != sc0 { + t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc0) + } + } +} + +func (s) TestAddrWeightChange(t *testing.T) { + wantAddrs := []resolver.Address{ + {Addr: testBackendAddrStrs[0]}, + {Addr: testBackendAddrStrs[1]}, + {Addr: testBackendAddrStrs[2]}, + } + cc, b, p0 := setupTest(t, wantAddrs) + ring0 := p0.(*picker).ring + + if err := b.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Addresses: wantAddrs}, + BalancerConfig: testConfig, + }); err != nil { + t.Fatalf("UpdateClientConnState returned err: %v", err) + } + select { + case <-cc.NewPickerCh: + t.Fatalf("unexpected picker after UpdateClientConn with the same addresses") + case <-time.After(defaultTestShortTimeout): + } + + // Delete an address, should send a new Picker. + if err := b.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Addresses: []resolver.Address{ + {Addr: testBackendAddrStrs[0]}, + {Addr: testBackendAddrStrs[1]}, + }}, + BalancerConfig: testConfig, + }); err != nil { + t.Fatalf("UpdateClientConnState returned err: %v", err) + } + var p1 balancer.Picker + select { + case p1 = <-cc.NewPickerCh: + case <-time.After(defaultTestTimeout): + t.Fatalf("timeout waiting for picker after UpdateClientConn with different addresses") + } + ring1 := p1.(*picker).ring + if ring1 == ring0 { + t.Fatalf("new picker after removing address has the same ring as before, want different") + } + + // Another update with the same addresses, but different weight. + if err := b.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Addresses: []resolver.Address{ + {Addr: testBackendAddrStrs[0]}, + weightedroundrobin.SetAddrInfo( + resolver.Address{Addr: testBackendAddrStrs[1]}, + weightedroundrobin.AddrInfo{Weight: 2}), + }}, + BalancerConfig: testConfig, + }); err != nil { + t.Fatalf("UpdateClientConnState returned err: %v", err) + } + var p2 balancer.Picker + select { + case p2 = <-cc.NewPickerCh: + case <-time.After(defaultTestTimeout): + t.Fatalf("timeout waiting for picker after UpdateClientConn with different addresses") + } + if p2.(*picker).ring == ring1 { + t.Fatalf("new picker after changing address weight has the same ring as before, want different") + } +} + +// TestSubConnToConnectWhenOverallTransientFailure covers the situation when the +// overall state is TransientFailure, the SubConns turning Idle will trigger the +// next SubConn in the ring to Connect(). But not when the overall state is not +// TransientFailure. +func (s) TestSubConnToConnectWhenOverallTransientFailure(t *testing.T) { + wantAddrs := []resolver.Address{ + {Addr: testBackendAddrStrs[0]}, + {Addr: testBackendAddrStrs[1]}, + {Addr: testBackendAddrStrs[2]}, + } + _, _, p0 := setupTest(t, wantAddrs) + ring0 := p0.(*picker).ring + + // ringhash won't tell SCs to connect until there is an RPC, so simulate + // one now. + p0.Pick(balancer.PickInfo{Ctx: context.Background()}) + + // Turn the first subconn to transient failure. + sc0 := ring0.items[0].sc.sc.(*testutils.TestSubConn) + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) + sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Idle}) + + // It will trigger the second subconn to connect (because overall state is + // Connect (when one subconn is TF)). + sc1 := ring0.items[1].sc.sc.(*testutils.TestSubConn) + select { + case <-sc1.ConnectCh: + case <-time.After(defaultTestShortTimeout): + t.Fatalf("timeout waiting for Connect() from SubConn %v", sc1) + } + + // Turn the second subconn to TF. This will set the overall state to TF. + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) + sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Idle}) + + // It will trigger the third subconn to connect. + sc2 := ring0.items[2].sc.sc.(*testutils.TestSubConn) + select { + case <-sc2.ConnectCh: + case <-time.After(defaultTestShortTimeout): + t.Fatalf("timeout waiting for Connect() from SubConn %v", sc2) + } + + // Turn the third subconn to TF. This will set the overall state to TF. + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Idle}) + + // It will trigger the first subconn to connect. + select { + case <-sc0.ConnectCh: + case <-time.After(defaultTestShortTimeout): + t.Fatalf("timeout waiting for Connect() from SubConn %v", sc0) + } + + // Turn the third subconn to TF again. + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure}) + sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Idle}) + + // This will not trigger any new Connect() on the SubConns, because sc0 is + // still attempting to connect, and we only need one SubConn to connect. + select { + case <-sc0.ConnectCh: + t.Fatalf("unexpected Connect() from SubConn %v", sc0) + case <-sc1.ConnectCh: + t.Fatalf("unexpected Connect() from SubConn %v", sc1) + case <-sc2.ConnectCh: + t.Fatalf("unexpected Connect() from SubConn %v", sc2) + case <-time.After(defaultTestShortTimeout): + } +} + +func (s) TestConnectivityStateEvaluatorRecordTransition(t *testing.T) { + tests := []struct { + name string + from, to []connectivity.State + want connectivity.State + }{ + { + name: "one ready", + from: []connectivity.State{connectivity.Idle}, + to: []connectivity.State{connectivity.Ready}, + want: connectivity.Ready, + }, + { + name: "one connecting", + from: []connectivity.State{connectivity.Idle}, + to: []connectivity.State{connectivity.Connecting}, + want: connectivity.Connecting, + }, + { + name: "one ready one transient failure", + from: []connectivity.State{connectivity.Idle, connectivity.Idle}, + to: []connectivity.State{connectivity.Ready, connectivity.TransientFailure}, + want: connectivity.Ready, + }, + { + name: "one connecting one transient failure", + from: []connectivity.State{connectivity.Idle, connectivity.Idle}, + to: []connectivity.State{connectivity.Connecting, connectivity.TransientFailure}, + want: connectivity.Connecting, + }, + { + name: "one connecting two transient failure", + from: []connectivity.State{connectivity.Idle, connectivity.Idle, connectivity.Idle}, + to: []connectivity.State{connectivity.Connecting, connectivity.TransientFailure, connectivity.TransientFailure}, + want: connectivity.TransientFailure, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cse := &connectivityStateEvaluator{} + var got connectivity.State + for i, fff := range tt.from { + ttt := tt.to[i] + got = cse.recordTransition(fff, ttt) + } + if got != tt.want { + t.Errorf("recordTransition() = %v, want %v", got, tt.want) + } + }) + } +} + +// TestAddrBalancerAttributesChange tests the case where the ringhash balancer +// receives a ClientConnUpdate with the same config and addresses as received in +// the previous update. Although the `BalancerAttributes` contents are the same, +// the pointer is different. This test verifies that subConns are not recreated +// in this scenario. +func (s) TestAddrBalancerAttributesChange(t *testing.T) { + addrs1 := []resolver.Address{internal.SetLocalityID(resolver.Address{Addr: testBackendAddrStrs[0]}, internal.LocalityID{Region: "americas"})} + cc, b, _ := setupTest(t, addrs1) + + addrs2 := []resolver.Address{internal.SetLocalityID(resolver.Address{Addr: testBackendAddrStrs[0]}, internal.LocalityID{Region: "americas"})} + if err := b.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{Addresses: addrs2}, + BalancerConfig: testConfig, + }); err != nil { + t.Fatalf("UpdateClientConnState returned err: %v", err) + } + select { + case <-cc.NewSubConnCh: + t.Fatal("new subConn created for an update with the same addresses") + case <-time.After(defaultTestShortTimeout): + } +} diff --git a/xds/internal/balancer/ringhash/util.go b/xds/internal/balancer/ringhash/util.go new file mode 100644 index 000000000000..92bb3ae5b791 --- /dev/null +++ b/xds/internal/balancer/ringhash/util.go @@ -0,0 +1,40 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package ringhash + +import "context" + +type clusterKey struct{} + +func getRequestHash(ctx context.Context) uint64 { + requestHash, _ := ctx.Value(clusterKey{}).(uint64) + return requestHash +} + +// GetRequestHashForTesting returns the request hash in the context; to be used +// for testing only. +func GetRequestHashForTesting(ctx context.Context) uint64 { + return getRequestHash(ctx) +} + +// SetRequestHash adds the request hash to the context for use in Ring Hash Load +// Balancing. +func SetRequestHash(ctx context.Context, requestHash uint64) context.Context { + return context.WithValue(ctx, clusterKey{}, requestHash) +} diff --git a/xds/internal/balancer/weightedtarget/weightedtarget.go b/xds/internal/balancer/weightedtarget/weightedtarget.go deleted file mode 100644 index 4172a4033641..000000000000 --- a/xds/internal/balancer/weightedtarget/weightedtarget.go +++ /dev/null @@ -1,170 +0,0 @@ -/* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -// Package weightedtarget implements the weighted_target balancer. -package weightedtarget - -import ( - "encoding/json" - "fmt" - - "google.golang.org/grpc/balancer" - "google.golang.org/grpc/internal/grpclog" - "google.golang.org/grpc/internal/hierarchy" - "google.golang.org/grpc/internal/wrr" - "google.golang.org/grpc/resolver" - "google.golang.org/grpc/serviceconfig" - "google.golang.org/grpc/xds/internal" - "google.golang.org/grpc/xds/internal/balancer/balancergroup" - "google.golang.org/grpc/xds/internal/balancer/weightedtarget/weightedaggregator" -) - -const weightedTargetName = "weighted_target_experimental" - -// newRandomWRR is the WRR constructor used to pick sub-pickers from -// sub-balancers. It's to be modified in tests. -var newRandomWRR = wrr.NewRandom - -func init() { - balancer.Register(&weightedTargetBB{}) -} - -type weightedTargetBB struct{} - -func (wt *weightedTargetBB) Build(cc balancer.ClientConn, _ balancer.BuildOptions) balancer.Balancer { - b := &weightedTargetBalancer{} - b.logger = prefixLogger(b) - b.stateAggregator = weightedaggregator.New(cc, b.logger, newRandomWRR) - b.stateAggregator.Start() - b.bg = balancergroup.New(cc, b.stateAggregator, nil, b.logger) - b.bg.Start() - b.logger.Infof("Created") - return b -} - -func (wt *weightedTargetBB) Name() string { - return weightedTargetName -} - -func (wt *weightedTargetBB) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { - return parseConfig(c) -} - -type weightedTargetBalancer struct { - logger *grpclog.PrefixLogger - - // TODO: Make this package not dependent on any xds specific code. - // BalancerGroup uses xdsinternal.LocalityID as the key in the map of child - // policies that it maintains and reports load using LRS. Once these two - // dependencies are removed from the balancerGroup, this package will not - // have any dependencies on xds code. - bg *balancergroup.BalancerGroup - stateAggregator *weightedaggregator.Aggregator - - targets map[string]target -} - -// TODO: remove this and use strings directly as keys for balancer group. -func makeLocalityFromName(name string) internal.LocalityID { - return internal.LocalityID{Region: name} -} - -// UpdateClientConnState takes the new targets in balancer group, -// creates/deletes sub-balancers and sends them update. Addresses are split into -// groups based on hierarchy path. -func (w *weightedTargetBalancer) UpdateClientConnState(s balancer.ClientConnState) error { - newConfig, ok := s.BalancerConfig.(*lbConfig) - if !ok { - return fmt.Errorf("unexpected balancer config with type: %T", s.BalancerConfig) - } - addressesSplit := hierarchy.Group(s.ResolverState.Addresses) - - var rebuildStateAndPicker bool - - // Remove sub-pickers and sub-balancers that are not in the new config. - for name := range w.targets { - if _, ok := newConfig.Targets[name]; !ok { - l := makeLocalityFromName(name) - w.stateAggregator.Remove(l) - w.bg.Remove(l) - // Trigger a state/picker update, because we don't want `ClientConn` - // to pick this sub-balancer anymore. - rebuildStateAndPicker = true - } - } - - // For sub-balancers in the new config - // - if it's new. add to balancer group, - // - if it's old, but has a new weight, update weight in balancer group. - // - // For all sub-balancers, forward the address/balancer config update. - for name, newT := range newConfig.Targets { - l := makeLocalityFromName(name) - - oldT, ok := w.targets[name] - if !ok { - // If this is a new sub-balancer, add weights to the picker map. - w.stateAggregator.Add(l, newT.Weight) - // Then add to the balancer group. - w.bg.Add(l, balancer.Get(newT.ChildPolicy.Name)) - // Not trigger a state/picker update. Wait for the new sub-balancer - // to send its updates. - } else if newT.Weight != oldT.Weight { - // If this is an existing sub-balancer, update weight if necessary. - w.stateAggregator.UpdateWeight(l, newT.Weight) - // Trigger a state/picker update, because we don't want `ClientConn` - // should do picks with the new weights now. - rebuildStateAndPicker = true - } - - // Forwards all the update: - // - Addresses are from the map after splitting with hierarchy path, - // - Top level service config and attributes are the same, - // - Balancer config comes from the targets map. - // - // TODO: handle error? How to aggregate errors and return? - _ = w.bg.UpdateClientConnState(l, balancer.ClientConnState{ - ResolverState: resolver.State{ - Addresses: addressesSplit[name], - ServiceConfig: s.ResolverState.ServiceConfig, - Attributes: s.ResolverState.Attributes, - }, - BalancerConfig: newT.ChildPolicy.Config, - }) - } - - w.targets = newConfig.Targets - - if rebuildStateAndPicker { - w.stateAggregator.BuildAndUpdate() - } - return nil -} - -func (w *weightedTargetBalancer) ResolverError(err error) { - w.bg.ResolverError(err) -} - -func (w *weightedTargetBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { - w.bg.UpdateSubConnState(sc, state) -} - -func (w *weightedTargetBalancer) Close() { - w.stateAggregator.Stop() - w.bg.Close() -} diff --git a/xds/internal/balancer/weightedtarget/weightedtarget_test.go b/xds/internal/balancer/weightedtarget/weightedtarget_test.go deleted file mode 100644 index 7f9e566ca5b5..000000000000 --- a/xds/internal/balancer/weightedtarget/weightedtarget_test.go +++ /dev/null @@ -1,225 +0,0 @@ -/* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package weightedtarget - -import ( - "encoding/json" - "fmt" - "testing" - "time" - - "github.com/google/go-cmp/cmp" - "google.golang.org/grpc/attributes" - "google.golang.org/grpc/balancer" - "google.golang.org/grpc/balancer/roundrobin" - "google.golang.org/grpc/connectivity" - "google.golang.org/grpc/internal/hierarchy" - "google.golang.org/grpc/resolver" - "google.golang.org/grpc/serviceconfig" - "google.golang.org/grpc/xds/internal/balancer/balancergroup" - "google.golang.org/grpc/xds/internal/testutils" -) - -type testConfigBalancerBuilder struct { - balancer.Builder -} - -func newTestConfigBalancerBuilder() *testConfigBalancerBuilder { - return &testConfigBalancerBuilder{ - Builder: balancer.Get(roundrobin.Name), - } -} - -func (t *testConfigBalancerBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { - rr := t.Builder.Build(cc, opts) - return &testConfigBalancer{ - Balancer: rr, - } -} - -const testConfigBalancerName = "test_config_balancer" - -func (t *testConfigBalancerBuilder) Name() string { - return testConfigBalancerName -} - -type stringBalancerConfig struct { - serviceconfig.LoadBalancingConfig - s string -} - -func (t *testConfigBalancerBuilder) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { - // Return string without quotes. - return stringBalancerConfig{s: string(c[1 : len(c)-1])}, nil -} - -// testConfigBalancer is a roundrobin balancer, but it takes the balancer config -// string and append it to the backend addresses. -type testConfigBalancer struct { - balancer.Balancer -} - -func (b *testConfigBalancer) UpdateClientConnState(s balancer.ClientConnState) error { - c, ok := s.BalancerConfig.(stringBalancerConfig) - if !ok { - return fmt.Errorf("unexpected balancer config with type %T", s.BalancerConfig) - } - oneMoreAddr := resolver.Address{Addr: c.s} - s.BalancerConfig = nil - s.ResolverState.Addresses = append(s.ResolverState.Addresses, oneMoreAddr) - return b.Balancer.UpdateClientConnState(s) -} - -func (b *testConfigBalancer) Close() { - b.Balancer.Close() -} - -var ( - wtbBuilder balancer.Builder - wtbParser balancer.ConfigParser - testBackendAddrStrs []string -) - -const testBackendAddrsCount = 12 - -func init() { - balancer.Register(newTestConfigBalancerBuilder()) - for i := 0; i < testBackendAddrsCount; i++ { - testBackendAddrStrs = append(testBackendAddrStrs, fmt.Sprintf("%d.%d.%d.%d:%d", i, i, i, i, i)) - } - wtbBuilder = balancer.Get(weightedTargetName) - wtbParser = wtbBuilder.(balancer.ConfigParser) - - balancergroup.DefaultSubBalancerCloseTimeout = time.Millisecond -} - -// TestWeightedTarget covers the cases that a sub-balancer is added and a -// sub-balancer is removed. It verifies that the addresses and balancer configs -// are forwarded to the right sub-balancer. -// -// This test is intended to test the glue code in weighted_target. Most of the -// functionality tests are covered by the balancer group tests. -func TestWeightedTarget(t *testing.T) { - cc := testutils.NewTestClientConn(t) - wtb := wtbBuilder.Build(cc, balancer.BuildOptions{}) - - // Start with "cluster_1: round_robin". - config1, err := wtbParser.ParseConfig([]byte(`{"targets":{"cluster_1":{"weight":1,"childPolicy":[{"round_robin":""}]}}}`)) - if err != nil { - t.Fatalf("failed to parse balancer config: %v", err) - } - - // Send the config, and an address with hierarchy path ["cluster_1"]. - wantAddr1 := resolver.Address{Addr: testBackendAddrStrs[0], Attributes: nil} - if err := wtb.UpdateClientConnState(balancer.ClientConnState{ - ResolverState: resolver.State{Addresses: []resolver.Address{ - hierarchy.Set(wantAddr1, []string{"cluster_1"}), - }}, - BalancerConfig: config1, - }); err != nil { - t.Fatalf("failed to update ClientConn state: %v", err) - } - - // Verify that a subconn is created with the address, and the hierarchy path - // in the address is cleared. - addr1 := <-cc.NewSubConnAddrsCh - if want := []resolver.Address{ - hierarchy.Set(wantAddr1, []string{}), - }; !cmp.Equal(addr1, want, cmp.AllowUnexported(attributes.Attributes{})) { - t.Fatalf("got unexpected new subconn addrs: %v", cmp.Diff(addr1, want, cmp.AllowUnexported(attributes.Attributes{}))) - } - - // Send subconn state change. - sc1 := <-cc.NewSubConnCh - wtb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - wtb.UpdateSubConnState(sc1, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - - // Test pick with one backend. - p1 := <-cc.NewPickerCh - for i := 0; i < 5; i++ { - gotSCSt, _ := p1.Pick(balancer.PickInfo{}) - if !cmp.Equal(gotSCSt.SubConn, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) { - t.Fatalf("picker.Pick, got %v, want SubConn=%v", gotSCSt, sc1) - } - } - - // Remove cluster_1, and add "cluster_2: test_config_balancer". - wantAddr3Str := testBackendAddrStrs[2] - config2, err := wtbParser.ParseConfig([]byte( - fmt.Sprintf(`{"targets":{"cluster_2":{"weight":1,"childPolicy":[{%q:%q}]}}}`, testConfigBalancerName, wantAddr3Str), - )) - if err != nil { - t.Fatalf("failed to parse balancer config: %v", err) - } - - // Send the config, and one address with hierarchy path "cluster_2". - wantAddr2 := resolver.Address{Addr: testBackendAddrStrs[1], Attributes: nil} - if err := wtb.UpdateClientConnState(balancer.ClientConnState{ - ResolverState: resolver.State{Addresses: []resolver.Address{ - hierarchy.Set(wantAddr2, []string{"cluster_2"}), - }}, - BalancerConfig: config2, - }); err != nil { - t.Fatalf("failed to update ClientConn state: %v", err) - } - - // Expect the address sent in the address list. The hierarchy path should be - // cleared. - addr2 := <-cc.NewSubConnAddrsCh - if want := []resolver.Address{ - hierarchy.Set(wantAddr2, []string{}), - }; !cmp.Equal(addr2, want, cmp.AllowUnexported(attributes.Attributes{})) { - t.Fatalf("got unexpected new subconn addrs: %v", cmp.Diff(addr2, want, cmp.AllowUnexported(attributes.Attributes{}))) - } - // Expect the other address sent as balancer config. This address doesn't - // have hierarchy path. - wantAddr3 := resolver.Address{Addr: wantAddr3Str, Attributes: nil} - addr3 := <-cc.NewSubConnAddrsCh - if want := []resolver.Address{wantAddr3}; !cmp.Equal(addr3, want, cmp.AllowUnexported(attributes.Attributes{})) { - t.Fatalf("got unexpected new subconn addrs: %v", cmp.Diff(addr3, want, cmp.AllowUnexported(attributes.Attributes{}))) - } - - // The subconn for cluster_1 should be removed. - scToRemove := <-cc.RemoveSubConnCh - if !cmp.Equal(scToRemove, sc1, cmp.AllowUnexported(testutils.TestSubConn{})) { - t.Fatalf("RemoveSubConn, want %v, got %v", sc1, scToRemove) - } - wtb.UpdateSubConnState(scToRemove, balancer.SubConnState{ConnectivityState: connectivity.Shutdown}) - - sc2 := <-cc.NewSubConnCh - wtb.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - wtb.UpdateSubConnState(sc2, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - sc3 := <-cc.NewSubConnCh - wtb.UpdateSubConnState(sc3, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - wtb.UpdateSubConnState(sc3, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - - // Test roundrobin pick with backends in cluster_2. - p2 := <-cc.NewPickerCh - want := []balancer.SubConn{sc2, sc3} - if err := testutils.IsRoundRobin(want, subConnFromPicker(p2)); err != nil { - t.Fatalf("want %v, got %v", want, err) - } -} - -func subConnFromPicker(p balancer.Picker) func() balancer.SubConn { - return func() balancer.SubConn { - scst, _ := p.Pick(balancer.PickInfo{}) - return scst.SubConn - } -} diff --git a/xds/internal/balancer/wrrlocality/balancer.go b/xds/internal/balancer/wrrlocality/balancer.go new file mode 100644 index 000000000000..943ee7806ba1 --- /dev/null +++ b/xds/internal/balancer/wrrlocality/balancer.go @@ -0,0 +1,201 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package wrrlocality provides an implementation of the wrr locality LB policy, +// as defined in [A52 - xDS Custom LB Policies]. +// +// [A52 - xDS Custom LB Policies]: https://github.com/grpc/proposal/blob/master/A52-xds-custom-lb-policies.md +package wrrlocality + +import ( + "encoding/json" + "errors" + "fmt" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/weightedtarget" + "google.golang.org/grpc/internal/grpclog" + internalserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" + "google.golang.org/grpc/xds/internal" +) + +// Name is the name of wrr_locality balancer. +const Name = "xds_wrr_locality_experimental" + +func init() { + balancer.Register(bb{}) +} + +type bb struct{} + +func (bb) Name() string { + return Name +} + +// LBConfig is the config for the wrr locality balancer. +type LBConfig struct { + serviceconfig.LoadBalancingConfig `json:"-"` + // ChildPolicy is the config for the child policy. + ChildPolicy *internalserviceconfig.BalancerConfig `json:"childPolicy,omitempty"` +} + +// To plumb in a different child in tests. +var weightedTargetName = weightedtarget.Name + +func (bb) Build(cc balancer.ClientConn, bOpts balancer.BuildOptions) balancer.Balancer { + builder := balancer.Get(weightedTargetName) + if builder == nil { + // Shouldn't happen, registered through imported weighted target, + // defensive programming. + return nil + } + + // Doesn't need to intercept any balancer.ClientConn operations; pass + // through by just giving cc to child balancer. + wtb := builder.Build(cc, bOpts) + if wtb == nil { + // shouldn't happen, defensive programming. + return nil + } + wtbCfgParser, ok := builder.(balancer.ConfigParser) + if !ok { + // Shouldn't happen, imported weighted target builder has this method. + return nil + } + wrrL := &wrrLocalityBalancer{ + child: wtb, + childParser: wtbCfgParser, + } + + wrrL.logger = prefixLogger(wrrL) + wrrL.logger.Infof("Created") + return wrrL +} + +func (bb) ParseConfig(s json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + var lbCfg *LBConfig + if err := json.Unmarshal(s, &lbCfg); err != nil { + return nil, fmt.Errorf("xds_wrr_locality: invalid LBConfig: %s, error: %v", string(s), err) + } + if lbCfg == nil || lbCfg.ChildPolicy == nil { + return nil, errors.New("xds_wrr_locality: invalid LBConfig: child policy field must be set") + } + return lbCfg, nil +} + +type attributeKey struct{} + +// Equal allows the values to be compared by Attributes.Equal. +func (a AddrInfo) Equal(o any) bool { + oa, ok := o.(AddrInfo) + return ok && oa.LocalityWeight == a.LocalityWeight +} + +// AddrInfo is the locality weight of the locality an address is a part of. +type AddrInfo struct { + LocalityWeight uint32 +} + +// SetAddrInfo returns a copy of addr in which the BalancerAttributes field is +// updated with AddrInfo. +func SetAddrInfo(addr resolver.Address, addrInfo AddrInfo) resolver.Address { + addr.BalancerAttributes = addr.BalancerAttributes.WithValue(attributeKey{}, addrInfo) + return addr +} + +func (a AddrInfo) String() string { + return fmt.Sprintf("Locality Weight: %d", a.LocalityWeight) +} + +// getAddrInfo returns the AddrInfo stored in the BalancerAttributes field of +// addr. Returns false if no AddrInfo found. +func getAddrInfo(addr resolver.Address) (AddrInfo, bool) { + v := addr.BalancerAttributes.Value(attributeKey{}) + ai, ok := v.(AddrInfo) + return ai, ok +} + +// wrrLocalityBalancer wraps a weighted target balancer, and builds +// configuration for the weighted target once it receives configuration +// specifying the weighted target child balancer and locality weight +// information. +type wrrLocalityBalancer struct { + // child will be a weighted target balancer, and will be built it at + // wrrLocalityBalancer build time. Other than preparing configuration, other + // balancer operations are simply pass through. + child balancer.Balancer + + childParser balancer.ConfigParser + + logger *grpclog.PrefixLogger +} + +func (b *wrrLocalityBalancer) UpdateClientConnState(s balancer.ClientConnState) error { + lbCfg, ok := s.BalancerConfig.(*LBConfig) + if !ok { + b.logger.Errorf("Received config with unexpected type %T: %v", s.BalancerConfig, s.BalancerConfig) + return balancer.ErrBadResolverState + } + + weightedTargets := make(map[string]weightedtarget.Target) + for _, addr := range s.ResolverState.Addresses { + // This get of LocalityID could potentially return a zero value. This + // shouldn't happen though (this attribute that is set actually gets + // used to build localities in the first place), and thus don't error + // out, and just build a weighted target with undefined behavior. + locality, err := internal.GetLocalityID(addr).ToString() + if err != nil { + // Should never happen. + logger.Errorf("Failed to marshal LocalityID: %v, skipping this locality in weighted target") + } + ai, ok := getAddrInfo(addr) + if !ok { + return fmt.Errorf("xds_wrr_locality: missing locality weight information in address %q", addr) + } + weightedTargets[locality] = weightedtarget.Target{Weight: ai.LocalityWeight, ChildPolicy: lbCfg.ChildPolicy} + } + wtCfg := &weightedtarget.LBConfig{Targets: weightedTargets} + wtCfgJSON, err := json.Marshal(wtCfg) + if err != nil { + // Shouldn't happen. + return fmt.Errorf("xds_wrr_locality: error marshalling prepared config: %v", wtCfg) + } + var sc serviceconfig.LoadBalancingConfig + if sc, err = b.childParser.ParseConfig(wtCfgJSON); err != nil { + return fmt.Errorf("xds_wrr_locality: config generated %v is invalid: %v", wtCfgJSON, err) + } + + return b.child.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: s.ResolverState, + BalancerConfig: sc, + }) +} + +func (b *wrrLocalityBalancer) ResolverError(err error) { + b.child.ResolverError(err) +} + +func (b *wrrLocalityBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { + b.logger.Errorf("UpdateSubConnState(%v, %+v) called unexpectedly", sc, state) +} + +func (b *wrrLocalityBalancer) Close() { + b.child.Close() +} diff --git a/xds/internal/balancer/wrrlocality/balancer_test.go b/xds/internal/balancer/wrrlocality/balancer_test.go new file mode 100644 index 000000000000..f0da7413bdb8 --- /dev/null +++ b/xds/internal/balancer/wrrlocality/balancer_test.go @@ -0,0 +1,252 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package wrrlocality + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/roundrobin" + "google.golang.org/grpc/balancer/weightedtarget" + "google.golang.org/grpc/internal/balancer/stub" + "google.golang.org/grpc/internal/grpctest" + internalserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" + "google.golang.org/grpc/xds/internal" +) + +const ( + defaultTestTimeout = 5 * time.Second +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func (s) TestParseConfig(t *testing.T) { + const errParseConfigName = "errParseConfigBalancer" + stub.Register(errParseConfigName, stub.BalancerFuncs{ + ParseConfig: func(json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + return nil, errors.New("some error") + }, + }) + + parser := bb{} + tests := []struct { + name string + input string + wantCfg serviceconfig.LoadBalancingConfig + wantErr string + }{ + { + name: "happy-case-round robin-child", + input: `{"childPolicy": [{"round_robin": {}}]}`, + wantCfg: &LBConfig{ + ChildPolicy: &internalserviceconfig.BalancerConfig{ + Name: roundrobin.Name, + }, + }, + }, + { + name: "invalid-json", + input: "{{invalidjson{{", + wantErr: "invalid character", + }, + + { + name: "child-policy-field-isn't-set", + input: `{}`, + wantErr: "child policy field must be set", + }, + { + name: "child-policy-type-is-empty", + input: `{"childPolicy": []}`, + wantErr: "invalid loadBalancingConfig: no supported policies found in []", + }, + { + name: "child-policy-empty-config", + input: `{"childPolicy": [{"": {}}]}`, + wantErr: "invalid loadBalancingConfig: no supported policies found in []", + }, + { + name: "child-policy-type-isn't-registered", + input: `{"childPolicy": [{"doesNotExistBalancer": {"cluster": "test_cluster"}}]}`, + wantErr: "invalid loadBalancingConfig: no supported policies found in [doesNotExistBalancer]", + }, + { + name: "child-policy-config-is-invalid", + input: `{"childPolicy": [{"errParseConfigBalancer": {"cluster": "test_cluster"}}]}`, + wantErr: "error parsing loadBalancingConfig for policy \"errParseConfigBalancer\"", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + gotCfg, gotErr := parser.ParseConfig(json.RawMessage(test.input)) + // Substring match makes this very tightly coupled to the + // internalserviceconfig.BalancerConfig error strings. However, it + // is important to distinguish the different types of error messages + // possible as the parser has a few defined buckets of ways it can + // error out. + if (gotErr != nil) != (test.wantErr != "") { + t.Fatalf("ParseConfig(%v) = %v, wantErr %v", test.input, gotErr, test.wantErr) + } + if gotErr != nil && !strings.Contains(gotErr.Error(), test.wantErr) { + t.Fatalf("ParseConfig(%v) = %v, wantErr %v", test.input, gotErr, test.wantErr) + } + if test.wantErr != "" { + return + } + if diff := cmp.Diff(gotCfg, test.wantCfg); diff != "" { + t.Fatalf("ParseConfig(%v) got unexpected output, diff (-got +want): %v", test.input, diff) + } + }) + } +} + +// TestUpdateClientConnState tests the UpdateClientConnState method of the +// wrr_locality_experimental balancer. This UpdateClientConn operation should +// take the localities and their weights in the addresses passed in, alongside +// the endpoint picking policy defined in the Balancer Config and construct a +// weighted target configuration corresponding to these inputs. +func (s) TestUpdateClientConnState(t *testing.T) { + // Configure the stub balancer defined below as the child policy of + // wrrLocalityBalancer. + cfgCh := testutils.NewChannel() + oldWeightedTargetName := weightedTargetName + defer func() { + weightedTargetName = oldWeightedTargetName + }() + weightedTargetName = "fake_weighted_target" + stub.Register("fake_weighted_target", stub.BalancerFuncs{ + ParseConfig: func(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + var cfg weightedtarget.LBConfig + if err := json.Unmarshal(c, &cfg); err != nil { + return nil, err + } + return &cfg, nil + }, + UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error { + wtCfg, ok := ccs.BalancerConfig.(*weightedtarget.LBConfig) + if !ok { + return errors.New("child received config that was not a weighted target config") + } + defer cfgCh.Send(wtCfg) + return nil + }, + }) + + builder := balancer.Get(Name) + if builder == nil { + t.Fatalf("balancer.Get(%q) returned nil", Name) + } + tcc := testutils.NewTestClientConn(t) + bal := builder.Build(tcc, balancer.BuildOptions{}) + defer bal.Close() + wrrL := bal.(*wrrLocalityBalancer) + + // Create the addresses with two localities with certain locality weights. + // This represents what addresses the wrr_locality balancer will receive in + // UpdateClientConnState. + addr1 := resolver.Address{ + Addr: "locality-1", + } + addr1 = internal.SetLocalityID(addr1, internal.LocalityID{ + Region: "region-1", + Zone: "zone-1", + SubZone: "subzone-1", + }) + addr1 = SetAddrInfo(addr1, AddrInfo{LocalityWeight: 2}) + + addr2 := resolver.Address{ + Addr: "locality-2", + } + addr2 = internal.SetLocalityID(addr2, internal.LocalityID{ + Region: "region-2", + Zone: "zone-2", + SubZone: "subzone-2", + }) + addr2 = SetAddrInfo(addr2, AddrInfo{LocalityWeight: 1}) + addrs := []resolver.Address{addr1, addr2} + + err := wrrL.UpdateClientConnState(balancer.ClientConnState{ + BalancerConfig: &LBConfig{ + ChildPolicy: &internalserviceconfig.BalancerConfig{ + Name: "round_robin", + }, + }, + ResolverState: resolver.State{ + Addresses: addrs, + }, + }) + if err != nil { + t.Fatalf("Unexpected error from UpdateClientConnState: %v", err) + } + + // Note that these inline strings declared as the key in Targets built from + // Locality ID are not exactly what is shown in the example in the gRFC. + // However, this is an implementation detail that does not affect + // correctness (confirmed with Java team). The important thing is to get + // those three pieces of information region, zone, and subzone down to the + // child layer. + wantWtCfg := &weightedtarget.LBConfig{ + Targets: map[string]weightedtarget.Target{ + "{\"region\":\"region-1\",\"zone\":\"zone-1\",\"subZone\":\"subzone-1\"}": { + Weight: 2, + ChildPolicy: &internalserviceconfig.BalancerConfig{ + Name: "round_robin", + }, + }, + "{\"region\":\"region-2\",\"zone\":\"zone-2\",\"subZone\":\"subzone-2\"}": { + Weight: 1, + ChildPolicy: &internalserviceconfig.BalancerConfig{ + Name: "round_robin", + }, + }, + }, + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + cfg, err := cfgCh.Receive(ctx) + if err != nil { + t.Fatalf("No signal received from UpdateClientConnState() on the child: %v", err) + } + + gotWtCfg, ok := cfg.(*weightedtarget.LBConfig) + if !ok { + // Shouldn't happen - only sends a config on this channel. + t.Fatalf("Unexpected config type: %T", gotWtCfg) + } + + if diff := cmp.Diff(gotWtCfg, wantWtCfg); diff != "" { + t.Fatalf("Child received unexpected config, diff (-got, +want): %v", diff) + } +} diff --git a/xds/internal/balancer/xdsrouting/logging.go b/xds/internal/balancer/wrrlocality/logging.go similarity index 82% rename from xds/internal/balancer/xdsrouting/logging.go rename to xds/internal/balancer/wrrlocality/logging.go index 5c4a6b3cb410..42ccea0a92b2 100644 --- a/xds/internal/balancer/xdsrouting/logging.go +++ b/xds/internal/balancer/wrrlocality/logging.go @@ -1,6 +1,6 @@ /* * - * Copyright 2020 gRPC authors. + * Copyright 2023 gRPC authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,7 +16,7 @@ * */ -package xdsrouting +package wrrlocality import ( "fmt" @@ -25,10 +25,10 @@ import ( internalgrpclog "google.golang.org/grpc/internal/grpclog" ) -const prefix = "[xds-routing-lb %p] " +const prefix = "[wrrlocality-lb %p] " var logger = grpclog.Component("xds") -func prefixLogger(p *routingBalancer) *internalgrpclog.PrefixLogger { +func prefixLogger(p *wrrLocalityBalancer) *internalgrpclog.PrefixLogger { return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p)) } diff --git a/xds/internal/balancer/xdsrouting/matcher.go b/xds/internal/balancer/xdsrouting/matcher.go deleted file mode 100644 index 196aefae564b..000000000000 --- a/xds/internal/balancer/xdsrouting/matcher.go +++ /dev/null @@ -1,146 +0,0 @@ -/* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package xdsrouting - -import ( - "fmt" - "strings" - - "google.golang.org/grpc/balancer" - "google.golang.org/grpc/internal/grpcrand" - "google.golang.org/grpc/internal/grpcutil" - "google.golang.org/grpc/metadata" -) - -// compositeMatcher.match returns true if all matchers return true. -type compositeMatcher struct { - pm pathMatcherInterface - hms []headerMatcherInterface - fm *fractionMatcher -} - -func newCompositeMatcher(pm pathMatcherInterface, hms []headerMatcherInterface, fm *fractionMatcher) *compositeMatcher { - return &compositeMatcher{pm: pm, hms: hms, fm: fm} -} - -func (a *compositeMatcher) match(info balancer.PickInfo) bool { - if a.pm != nil && !a.pm.match(info.FullMethodName) { - return false - } - - // Call headerMatchers even if md is nil, because routes may match - // non-presence of some headers. - var md metadata.MD - if info.Ctx != nil { - md, _ = metadata.FromOutgoingContext(info.Ctx) - if extraMD, ok := grpcutil.ExtraMetadata(info.Ctx); ok { - md = metadata.Join(md, extraMD) - // Remove all binary headers. They are hard to match with. May need - // to add back if asked by users. - for k := range md { - if strings.HasSuffix(k, "-bin") { - delete(md, k) - } - } - } - } - for _, m := range a.hms { - if !m.match(md) { - return false - } - } - - if a.fm != nil && !a.fm.match() { - return false - } - return true -} - -func (a *compositeMatcher) equal(mm *compositeMatcher) bool { - if a == mm { - return true - } - - if a == nil || mm == nil { - return false - } - - if (a.pm != nil || mm.pm != nil) && (a.pm == nil || !a.pm.equal(mm.pm)) { - return false - } - - if len(a.hms) != len(mm.hms) { - return false - } - for i := range a.hms { - if !a.hms[i].equal(mm.hms[i]) { - return false - } - } - - if (a.fm != nil || mm.fm != nil) && (a.fm == nil || !a.fm.equal(mm.fm)) { - return false - } - - return true -} - -func (a *compositeMatcher) String() string { - var ret string - if a.pm != nil { - ret += a.pm.String() - } - for _, m := range a.hms { - ret += m.String() - } - if a.fm != nil { - ret += a.fm.String() - } - return ret -} - -type fractionMatcher struct { - fraction int64 // real fraction is fraction/1,000,000. -} - -func newFractionMatcher(fraction uint32) *fractionMatcher { - return &fractionMatcher{fraction: int64(fraction)} -} - -var grpcrandInt63n = grpcrand.Int63n - -func (fm *fractionMatcher) match() bool { - t := grpcrandInt63n(1000000) - return t <= fm.fraction -} - -func (fm *fractionMatcher) equal(m *fractionMatcher) bool { - if fm == m { - return true - } - if fm == nil || m == nil { - return false - } - - return fm.fraction == m.fraction -} - -func (fm *fractionMatcher) String() string { - return fmt.Sprintf("fraction:%v", fm.fraction) -} diff --git a/xds/internal/balancer/xdsrouting/matcher_header.go b/xds/internal/balancer/xdsrouting/matcher_header.go deleted file mode 100644 index a900e43f5492..000000000000 --- a/xds/internal/balancer/xdsrouting/matcher_header.go +++ /dev/null @@ -1,245 +0,0 @@ -/* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package xdsrouting - -import ( - "fmt" - "regexp" - "strconv" - "strings" - - "google.golang.org/grpc/metadata" -) - -type headerMatcherInterface interface { - match(metadata.MD) bool - equal(headerMatcherInterface) bool - String() string -} - -// mdValuesFromOutgoingCtx retrieves metadata from context. If there are -// multiple values, the values are concatenated with "," (comma and no space). -// -// All header matchers only match against the comma-concatenated string. -func mdValuesFromOutgoingCtx(md metadata.MD, key string) (string, bool) { - vs, ok := md[key] - if !ok { - return "", false - } - return strings.Join(vs, ","), true -} - -type headerExactMatcher struct { - key string - exact string -} - -func newHeaderExactMatcher(key, exact string) *headerExactMatcher { - return &headerExactMatcher{key: key, exact: exact} -} - -func (hem *headerExactMatcher) match(md metadata.MD) bool { - v, ok := mdValuesFromOutgoingCtx(md, hem.key) - if !ok { - return false - } - return v == hem.exact -} - -func (hem *headerExactMatcher) equal(m headerMatcherInterface) bool { - mm, ok := m.(*headerExactMatcher) - if !ok { - return false - } - return hem.key == mm.key && hem.exact == mm.exact -} - -func (hem *headerExactMatcher) String() string { - return fmt.Sprintf("headerExact:%v:%v", hem.key, hem.exact) -} - -type headerRegexMatcher struct { - key string - re *regexp.Regexp -} - -func newHeaderRegexMatcher(key string, re *regexp.Regexp) *headerRegexMatcher { - return &headerRegexMatcher{key: key, re: re} -} - -func (hrm *headerRegexMatcher) match(md metadata.MD) bool { - v, ok := mdValuesFromOutgoingCtx(md, hrm.key) - if !ok { - return false - } - return hrm.re.MatchString(v) -} - -func (hrm *headerRegexMatcher) equal(m headerMatcherInterface) bool { - mm, ok := m.(*headerRegexMatcher) - if !ok { - return false - } - return hrm.key == mm.key && hrm.re.String() == mm.re.String() -} - -func (hrm *headerRegexMatcher) String() string { - return fmt.Sprintf("headerRegex:%v:%v", hrm.key, hrm.re.String()) -} - -type headerRangeMatcher struct { - key string - start, end int64 // represents [start, end). -} - -func newHeaderRangeMatcher(key string, start, end int64) *headerRangeMatcher { - return &headerRangeMatcher{key: key, start: start, end: end} -} - -func (hrm *headerRangeMatcher) match(md metadata.MD) bool { - v, ok := mdValuesFromOutgoingCtx(md, hrm.key) - if !ok { - return false - } - if i, err := strconv.ParseInt(v, 10, 64); err == nil && i >= hrm.start && i < hrm.end { - return true - } - return false -} - -func (hrm *headerRangeMatcher) equal(m headerMatcherInterface) bool { - mm, ok := m.(*headerRangeMatcher) - if !ok { - return false - } - return hrm.key == mm.key && hrm.start == mm.start && hrm.end == mm.end -} - -func (hrm *headerRangeMatcher) String() string { - return fmt.Sprintf("headerRange:%v:[%d,%d)", hrm.key, hrm.start, hrm.end) -} - -type headerPresentMatcher struct { - key string - present bool -} - -func newHeaderPresentMatcher(key string, present bool) *headerPresentMatcher { - return &headerPresentMatcher{key: key, present: present} -} - -func (hpm *headerPresentMatcher) match(md metadata.MD) bool { - vs, ok := mdValuesFromOutgoingCtx(md, hpm.key) - present := ok && len(vs) > 0 - return present == hpm.present -} - -func (hpm *headerPresentMatcher) equal(m headerMatcherInterface) bool { - mm, ok := m.(*headerPresentMatcher) - if !ok { - return false - } - return hpm.key == mm.key && hpm.present == mm.present -} - -func (hpm *headerPresentMatcher) String() string { - return fmt.Sprintf("headerPresent:%v:%v", hpm.key, hpm.present) -} - -type headerPrefixMatcher struct { - key string - prefix string -} - -func newHeaderPrefixMatcher(key string, prefix string) *headerPrefixMatcher { - return &headerPrefixMatcher{key: key, prefix: prefix} -} - -func (hpm *headerPrefixMatcher) match(md metadata.MD) bool { - v, ok := mdValuesFromOutgoingCtx(md, hpm.key) - if !ok { - return false - } - return strings.HasPrefix(v, hpm.prefix) -} - -func (hpm *headerPrefixMatcher) equal(m headerMatcherInterface) bool { - mm, ok := m.(*headerPrefixMatcher) - if !ok { - return false - } - return hpm.key == mm.key && hpm.prefix == mm.prefix -} - -func (hpm *headerPrefixMatcher) String() string { - return fmt.Sprintf("headerPrefix:%v:%v", hpm.key, hpm.prefix) -} - -type headerSuffixMatcher struct { - key string - suffix string -} - -func newHeaderSuffixMatcher(key string, suffix string) *headerSuffixMatcher { - return &headerSuffixMatcher{key: key, suffix: suffix} -} - -func (hsm *headerSuffixMatcher) match(md metadata.MD) bool { - v, ok := mdValuesFromOutgoingCtx(md, hsm.key) - if !ok { - return false - } - return strings.HasSuffix(v, hsm.suffix) -} - -func (hsm *headerSuffixMatcher) equal(m headerMatcherInterface) bool { - mm, ok := m.(*headerSuffixMatcher) - if !ok { - return false - } - return hsm.key == mm.key && hsm.suffix == mm.suffix -} - -func (hsm *headerSuffixMatcher) String() string { - return fmt.Sprintf("headerSuffix:%v:%v", hsm.key, hsm.suffix) -} - -type invertMatcher struct { - m headerMatcherInterface -} - -func newInvertMatcher(m headerMatcherInterface) *invertMatcher { - return &invertMatcher{m: m} -} - -func (i *invertMatcher) match(md metadata.MD) bool { - return !i.m.match(md) -} - -func (i *invertMatcher) equal(m headerMatcherInterface) bool { - mm, ok := m.(*invertMatcher) - if !ok { - return false - } - return i.m.equal(mm.m) -} - -func (i *invertMatcher) String() string { - return fmt.Sprintf("invert{%s}", i.m) -} diff --git a/xds/internal/balancer/xdsrouting/matcher_test.go b/xds/internal/balancer/xdsrouting/matcher_test.go deleted file mode 100644 index e7d76e27469f..000000000000 --- a/xds/internal/balancer/xdsrouting/matcher_test.go +++ /dev/null @@ -1,175 +0,0 @@ -/* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package xdsrouting - -import ( - "context" - "testing" - - "google.golang.org/grpc/balancer" - "google.golang.org/grpc/internal/grpcrand" - "google.golang.org/grpc/internal/grpcutil" - "google.golang.org/grpc/metadata" -) - -func TestAndMatcherMatch(t *testing.T) { - tests := []struct { - name string - pm pathMatcherInterface - hm headerMatcherInterface - info balancer.PickInfo - want bool - }{ - { - name: "both match", - pm: newPathExactMatcher("/a/b"), - hm: newHeaderExactMatcher("th", "tv"), - info: balancer.PickInfo{ - FullMethodName: "/a/b", - Ctx: metadata.NewOutgoingContext(context.Background(), metadata.Pairs("th", "tv")), - }, - want: true, - }, - { - name: "only one match", - pm: newPathExactMatcher("/a/b"), - hm: newHeaderExactMatcher("th", "tv"), - info: balancer.PickInfo{ - FullMethodName: "/z/y", - Ctx: metadata.NewOutgoingContext(context.Background(), metadata.Pairs("th", "tv")), - }, - want: false, - }, - { - name: "both not match", - pm: newPathExactMatcher("/z/y"), - hm: newHeaderExactMatcher("th", "abc"), - info: balancer.PickInfo{ - FullMethodName: "/a/b", - Ctx: metadata.NewOutgoingContext(context.Background(), metadata.Pairs("th", "tv")), - }, - want: false, - }, - { - name: "fake header", - pm: newPathPrefixMatcher("/"), - hm: newHeaderExactMatcher("content-type", "fake"), - info: balancer.PickInfo{ - FullMethodName: "/a/b", - Ctx: grpcutil.WithExtraMetadata(context.Background(), metadata.Pairs( - "content-type", "fake", - )), - }, - want: true, - }, - { - name: "binary header", - pm: newPathPrefixMatcher("/"), - hm: newHeaderPresentMatcher("t-bin", true), - info: balancer.PickInfo{ - FullMethodName: "/a/b", - Ctx: grpcutil.WithExtraMetadata( - metadata.NewOutgoingContext(context.Background(), metadata.Pairs("t-bin", "123")), metadata.Pairs( - "content-type", "fake", - )), - }, - // Shouldn't match binary header, even though it's in metadata. - want: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - a := newCompositeMatcher(tt.pm, []headerMatcherInterface{tt.hm}, nil) - if got := a.match(tt.info); got != tt.want { - t.Errorf("match() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestFractionMatcherMatch(t *testing.T) { - const fraction = 500000 - fm := newFractionMatcher(fraction) - defer func() { - grpcrandInt63n = grpcrand.Int63n - }() - - // rand > fraction, should return false. - grpcrandInt63n = func(n int64) int64 { - return fraction + 1 - } - if matched := fm.match(); matched { - t.Errorf("match() = %v, want not match", matched) - } - - // rand == fraction, should return true. - grpcrandInt63n = func(n int64) int64 { - return fraction - } - if matched := fm.match(); !matched { - t.Errorf("match() = %v, want match", matched) - } - - // rand < fraction, should return true. - grpcrandInt63n = func(n int64) int64 { - return fraction - 1 - } - if matched := fm.match(); !matched { - t.Errorf("match() = %v, want match", matched) - } -} - -func TestCompositeMatcherEqual(t *testing.T) { - tests := []struct { - name string - pm pathMatcherInterface - hms []headerMatcherInterface - fm *fractionMatcher - mm *compositeMatcher - want bool - }{ - { - name: "equal", - pm: newPathExactMatcher("/a/b"), - mm: newCompositeMatcher(newPathExactMatcher("/a/b"), nil, nil), - want: true, - }, - { - name: "no path matcher", - pm: nil, - mm: newCompositeMatcher(nil, nil, nil), - want: true, - }, - { - name: "not equal", - pm: newPathExactMatcher("/a/b"), - fm: newFractionMatcher(123), - mm: newCompositeMatcher(newPathExactMatcher("/a/b"), nil, nil), - want: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - a := newCompositeMatcher(tt.pm, tt.hms, tt.fm) - if got := a.equal(tt.mm); got != tt.want { - t.Errorf("equal() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/xds/internal/balancer/xdsrouting/routing.go b/xds/internal/balancer/xdsrouting/routing.go deleted file mode 100644 index ebbb2b8cf9d7..000000000000 --- a/xds/internal/balancer/xdsrouting/routing.go +++ /dev/null @@ -1,256 +0,0 @@ -/* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package xdsrouting - -import ( - "encoding/json" - "fmt" - "regexp" - - "google.golang.org/grpc/balancer" - "google.golang.org/grpc/internal/grpclog" - "google.golang.org/grpc/internal/hierarchy" - "google.golang.org/grpc/resolver" - "google.golang.org/grpc/serviceconfig" - "google.golang.org/grpc/xds/internal" - "google.golang.org/grpc/xds/internal/balancer/balancergroup" -) - -const xdsRoutingName = "xds_routing_experimental" - -func init() { - balancer.Register(&routingBB{}) -} - -type routingBB struct{} - -func (rbb *routingBB) Build(cc balancer.ClientConn, _ balancer.BuildOptions) balancer.Balancer { - b := &routingBalancer{} - b.logger = prefixLogger(b) - b.stateAggregator = newBalancerStateAggregator(cc, b.logger) - b.stateAggregator.start() - b.bg = balancergroup.New(cc, b.stateAggregator, nil, b.logger) - b.bg.Start() - b.logger.Infof("Created") - return b -} - -func (rbb *routingBB) Name() string { - return xdsRoutingName -} - -func (rbb *routingBB) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { - return parseConfig(c) -} - -type route struct { - action string - m *compositeMatcher -} - -func (r route) String() string { - return r.m.String() + "->" + r.action -} - -type routingBalancer struct { - logger *grpclog.PrefixLogger - - // TODO: make this package not dependent on xds specific code. Same as for - // weighted target balancer. - bg *balancergroup.BalancerGroup - stateAggregator *balancerStateAggregator - - actions map[string]actionConfig - routes []route -} - -// TODO: remove this and use strings directly as keys for balancer group. -func makeLocalityFromName(name string) internal.LocalityID { - return internal.LocalityID{Region: name} -} - -// TODO: remove this and use strings directly as keys for balancer group. -func getNameFromLocality(id internal.LocalityID) string { - return id.Region -} - -func (rb *routingBalancer) updateActions(s balancer.ClientConnState, newConfig *lbConfig) (needRebuild bool) { - addressesSplit := hierarchy.Group(s.ResolverState.Addresses) - var rebuildStateAndPicker bool - - // Remove sub-pickers and sub-balancers that are not in the new action list. - for name := range rb.actions { - if _, ok := newConfig.actions[name]; !ok { - l := makeLocalityFromName(name) - rb.stateAggregator.remove(l) - rb.bg.Remove(l) - // Trigger a state/picker update, because we don't want `ClientConn` - // to pick this sub-balancer anymore. - rebuildStateAndPicker = true - } - } - - // For sub-balancers in the new action list, - // - add to balancer group if it's new, - // - forward the address/balancer config update. - for name, newT := range newConfig.actions { - l := makeLocalityFromName(name) - if _, ok := rb.actions[name]; !ok { - // If this is a new sub-balancer, add weights to the picker map. - rb.stateAggregator.add(l) - // Then add to the balancer group. - rb.bg.Add(l, balancer.Get(newT.ChildPolicy.Name)) - // Not trigger a state/picker update. Wait for the new sub-balancer - // to send its updates. - } - // Forwards all the update: - // - Addresses are from the map after splitting with hierarchy path, - // - Top level service config and attributes are the same, - // - Balancer config comes from the targets map. - // - // TODO: handle error? How to aggregate errors and return? - _ = rb.bg.UpdateClientConnState(l, balancer.ClientConnState{ - ResolverState: resolver.State{ - Addresses: addressesSplit[name], - ServiceConfig: s.ResolverState.ServiceConfig, - Attributes: s.ResolverState.Attributes, - }, - BalancerConfig: newT.ChildPolicy.Config, - }) - } - - rb.actions = newConfig.actions - return rebuildStateAndPicker -} - -func routeToMatcher(r routeConfig) (*compositeMatcher, error) { - var pathMatcher pathMatcherInterface - switch { - case r.regex != "": - re, err := regexp.Compile(r.regex) - if err != nil { - return nil, fmt.Errorf("failed to compile regex %q", r.regex) - } - pathMatcher = newPathRegexMatcher(re) - case r.path != "": - pathMatcher = newPathExactMatcher(r.path) - default: - pathMatcher = newPathPrefixMatcher(r.prefix) - } - - var headerMatchers []headerMatcherInterface - for _, h := range r.headers { - var matcherT headerMatcherInterface - switch { - case h.exactMatch != "": - matcherT = newHeaderExactMatcher(h.name, h.exactMatch) - case h.regexMatch != "": - re, err := regexp.Compile(h.regexMatch) - if err != nil { - return nil, fmt.Errorf("failed to compile regex %q, skipping this matcher", h.regexMatch) - } - matcherT = newHeaderRegexMatcher(h.name, re) - case h.prefixMatch != "": - matcherT = newHeaderPrefixMatcher(h.name, h.prefixMatch) - case h.suffixMatch != "": - matcherT = newHeaderSuffixMatcher(h.name, h.suffixMatch) - case h.rangeMatch != nil: - matcherT = newHeaderRangeMatcher(h.name, h.rangeMatch.start, h.rangeMatch.end) - default: - matcherT = newHeaderPresentMatcher(h.name, h.presentMatch) - } - if h.invertMatch { - matcherT = newInvertMatcher(matcherT) - } - headerMatchers = append(headerMatchers, matcherT) - } - - var fractionMatcher *fractionMatcher - if r.fraction != nil { - fractionMatcher = newFractionMatcher(*r.fraction) - } - return newCompositeMatcher(pathMatcher, headerMatchers, fractionMatcher), nil -} - -func routesEqual(a, b []route) bool { - if len(a) != len(b) { - return false - } - for i := range a { - aa := a[i] - bb := b[i] - if aa.action != bb.action { - return false - } - if !aa.m.equal(bb.m) { - return false - } - } - return true -} - -func (rb *routingBalancer) updateRoutes(newConfig *lbConfig) (needRebuild bool, _ error) { - var newRoutes []route - for _, rt := range newConfig.routes { - newMatcher, err := routeToMatcher(rt) - if err != nil { - return false, err - } - newRoutes = append(newRoutes, route{action: rt.action, m: newMatcher}) - } - rebuildStateAndPicker := !routesEqual(newRoutes, rb.routes) - rb.routes = newRoutes - - if rebuildStateAndPicker { - rb.stateAggregator.updateRoutes(rb.routes) - } - return rebuildStateAndPicker, nil -} - -func (rb *routingBalancer) UpdateClientConnState(s balancer.ClientConnState) error { - newConfig, ok := s.BalancerConfig.(*lbConfig) - if !ok { - return fmt.Errorf("unexpected balancer config with type: %T", s.BalancerConfig) - } - rb.logger.Infof("update with config %+v, resolver state %+v", s.BalancerConfig, s.ResolverState) - - rebuildForActions := rb.updateActions(s, newConfig) - rebuildForRoutes, err := rb.updateRoutes(newConfig) - if err != nil { - return fmt.Errorf("xds_routing balancer: failed to update routes: %v", err) - } - - if rebuildForActions || rebuildForRoutes { - rb.stateAggregator.buildAndUpdate() - } - return nil -} - -func (rb *routingBalancer) ResolverError(err error) { - rb.bg.ResolverError(err) -} - -func (rb *routingBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { - rb.bg.UpdateSubConnState(sc, state) -} - -func (rb *routingBalancer) Close() { - rb.stateAggregator.close() - rb.bg.Close() -} diff --git a/xds/internal/balancer/xdsrouting/routing_config.go b/xds/internal/balancer/xdsrouting/routing_config.go deleted file mode 100644 index 78716a098136..000000000000 --- a/xds/internal/balancer/xdsrouting/routing_config.go +++ /dev/null @@ -1,202 +0,0 @@ -/* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package xdsrouting - -import ( - "encoding/json" - "fmt" - - wrapperspb "github.com/golang/protobuf/ptypes/wrappers" - internalserviceconfig "google.golang.org/grpc/internal/serviceconfig" - "google.golang.org/grpc/serviceconfig" - xdsclient "google.golang.org/grpc/xds/internal/client" -) - -type actionConfig struct { - // ChildPolicy is the child policy and it's config. - ChildPolicy *internalserviceconfig.BalancerConfig -} - -type int64Range struct { - start, end int64 -} - -type headerMatcher struct { - name string - // matchSpecifiers - invertMatch bool - - // At most one of the following has non-default value. - exactMatch, regexMatch, prefixMatch, suffixMatch string - rangeMatch *int64Range - presentMatch bool -} - -type routeConfig struct { - // Path, Prefix and Regex can have at most one set. This is guaranteed by - // config parsing. - path, prefix, regex string - - headers []headerMatcher - fraction *uint32 - - // Action is the name from the action list. - action string -} - -// lbConfig is the balancer config for xds routing policy. -type lbConfig struct { - serviceconfig.LoadBalancingConfig - routes []routeConfig - actions map[string]actionConfig -} - -// The following structs with `JSON` in name are temporary structs to unmarshal -// json into. The fields will be read into lbConfig, to be used by the balancer. - -// routeJSON is temporary struct for json unmarshal. -type routeJSON struct { - // Path, Prefix and Regex can have at most one non-nil. - Path, Prefix, Regex *string - // Zero or more header matchers. - Headers []*xdsclient.HeaderMatcher - MatchFraction *wrapperspb.UInt32Value - // Action is the name from the action list. - Action string -} - -// lbConfigJSON is temporary struct for json unmarshal. -type lbConfigJSON struct { - Route []routeJSON - Action map[string]actionConfig -} - -func (jc lbConfigJSON) toLBConfig() *lbConfig { - var ret lbConfig - for _, r := range jc.Route { - var tempR routeConfig - switch { - case r.Path != nil: - tempR.path = *r.Path - case r.Prefix != nil: - tempR.prefix = *r.Prefix - case r.Regex != nil: - tempR.regex = *r.Regex - } - for _, h := range r.Headers { - var tempHeader headerMatcher - switch { - case h.ExactMatch != nil: - tempHeader.exactMatch = *h.ExactMatch - case h.RegexMatch != nil: - tempHeader.regexMatch = *h.RegexMatch - case h.PrefixMatch != nil: - tempHeader.prefixMatch = *h.PrefixMatch - case h.SuffixMatch != nil: - tempHeader.suffixMatch = *h.SuffixMatch - case h.RangeMatch != nil: - tempHeader.rangeMatch = &int64Range{ - start: h.RangeMatch.Start, - end: h.RangeMatch.End, - } - case h.PresentMatch != nil: - tempHeader.presentMatch = *h.PresentMatch - } - tempHeader.name = h.Name - if h.InvertMatch != nil { - tempHeader.invertMatch = *h.InvertMatch - } - tempR.headers = append(tempR.headers, tempHeader) - } - if r.MatchFraction != nil { - tempR.fraction = &r.MatchFraction.Value - } - tempR.action = r.Action - ret.routes = append(ret.routes, tempR) - } - ret.actions = jc.Action - return &ret -} - -func parseConfig(c json.RawMessage) (*lbConfig, error) { - var tempConfig lbConfigJSON - if err := json.Unmarshal(c, &tempConfig); err != nil { - return nil, err - } - - // For each route: - // - at most one of path/prefix/regex. - // - action is in action list. - - allRouteActions := make(map[string]bool) - for _, r := range tempConfig.Route { - var oneOfCount int - if r.Path != nil { - oneOfCount++ - } - if r.Prefix != nil { - oneOfCount++ - } - if r.Regex != nil { - oneOfCount++ - } - if oneOfCount != 1 { - return nil, fmt.Errorf("%d (not exactly one) of path/prefix/regex is set in route %+v", oneOfCount, r) - } - - for _, h := range r.Headers { - var oneOfCountH int - if h.ExactMatch != nil { - oneOfCountH++ - } - if h.RegexMatch != nil { - oneOfCountH++ - } - if h.PrefixMatch != nil { - oneOfCountH++ - } - if h.SuffixMatch != nil { - oneOfCountH++ - } - if h.RangeMatch != nil { - oneOfCountH++ - } - if h.PresentMatch != nil { - oneOfCountH++ - } - if oneOfCountH != 1 { - return nil, fmt.Errorf("%d (not exactly one) of header matcher specifier is set in route %+v", oneOfCountH, h) - } - } - - if _, ok := tempConfig.Action[r.Action]; !ok { - return nil, fmt.Errorf("action %q from route %+v is not found in action list", r.Action, r) - } - allRouteActions[r.Action] = true - } - - // Verify that actions are used by at least one route. - for n := range tempConfig.Action { - if _, ok := allRouteActions[n]; !ok { - return nil, fmt.Errorf("action %q is not used by any route", n) - } - } - - return tempConfig.toLBConfig(), nil -} diff --git a/xds/internal/balancer/xdsrouting/routing_config_test.go b/xds/internal/balancer/xdsrouting/routing_config_test.go deleted file mode 100644 index 8cb4d22724e3..000000000000 --- a/xds/internal/balancer/xdsrouting/routing_config_test.go +++ /dev/null @@ -1,369 +0,0 @@ -/* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package xdsrouting - -import ( - "testing" - - "github.com/google/go-cmp/cmp" - "google.golang.org/grpc/balancer" - internalserviceconfig "google.golang.org/grpc/internal/serviceconfig" - _ "google.golang.org/grpc/xds/internal/balancer/cdsbalancer" - _ "google.golang.org/grpc/xds/internal/balancer/weightedtarget" -) - -const ( - testJSONConfig = `{ - "action":{ - "cds:cluster_1":{ - "childPolicy":[{ - "cds_experimental":{"cluster":"cluster_1"} - }] - }, - "weighted:cluster_1_cluster_2_1":{ - "childPolicy":[{ - "weighted_target_experimental":{ - "targets": { - "cluster_1" : { - "weight":75, - "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}] - }, - "cluster_2" : { - "weight":25, - "childPolicy":[{"cds_experimental":{"cluster":"cluster_2"}}] - } - } - } - }] - }, - "weighted:cluster_1_cluster_3_1":{ - "childPolicy":[{ - "weighted_target_experimental":{ - "targets": { - "cluster_1": { - "weight":99, - "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}] - }, - "cluster_3": { - "weight":1, - "childPolicy":[{"cds_experimental":{"cluster":"cluster_3"}}] - } - } - } - }] - } - }, - - "route":[{ - "path":"/service_1/method_1", - "action":"cds:cluster_1" - }, - { - "path":"/service_1/method_2", - "action":"cds:cluster_1" - }, - { - "prefix":"/service_2/method_1", - "action":"weighted:cluster_1_cluster_2_1" - }, - { - "prefix":"/service_2", - "action":"weighted:cluster_1_cluster_2_1" - }, - { - "regex":"^/service_2/method_3$", - "action":"weighted:cluster_1_cluster_3_1" - }] - } -` - - testJSONConfigWithAllMatchers = `{ - "action":{ - "cds:cluster_1":{ - "childPolicy":[{ - "cds_experimental":{"cluster":"cluster_1"} - }] - }, - "cds:cluster_2":{ - "childPolicy":[{ - "cds_experimental":{"cluster":"cluster_2"} - }] - }, - "cds:cluster_3":{ - "childPolicy":[{ - "cds_experimental":{"cluster":"cluster_3"} - }] - } - }, - - "route":[{ - "path":"/service_1/method_1", - "action":"cds:cluster_1" - }, - { - "prefix":"/service_2/method_1", - "action":"cds:cluster_1" - }, - { - "regex":"^/service_2/method_3$", - "action":"cds:cluster_1" - }, - { - "prefix":"", - "headers":[{"name":"header-1", "exactMatch":"value-1", "invertMatch":true}], - "action":"cds:cluster_2" - }, - { - "prefix":"", - "headers":[{"name":"header-1", "regexMatch":"^value-1$"}], - "action":"cds:cluster_2" - }, - { - "prefix":"", - "headers":[{"name":"header-1", "rangeMatch":{"start":-1, "end":7}}], - "action":"cds:cluster_3" - }, - { - "prefix":"", - "headers":[{"name":"header-1", "presentMatch":true}], - "action":"cds:cluster_3" - }, - { - "prefix":"", - "headers":[{"name":"header-1", "prefixMatch":"value-1"}], - "action":"cds:cluster_2" - }, - { - "prefix":"", - "headers":[{"name":"header-1", "suffixMatch":"value-1"}], - "action":"cds:cluster_2" - }, - { - "prefix":"", - "matchFraction":{"value": 31415}, - "action":"cds:cluster_3" - }] - } -` - - cdsName = "cds_experimental" - wtName = "weighted_target_experimental" -) - -var ( - cdsConfigParser = balancer.Get(cdsName).(balancer.ConfigParser) - cdsConfigJSON1 = `{"cluster":"cluster_1"}` - cdsConfig1, _ = cdsConfigParser.ParseConfig([]byte(cdsConfigJSON1)) - cdsConfigJSON2 = `{"cluster":"cluster_2"}` - cdsConfig2, _ = cdsConfigParser.ParseConfig([]byte(cdsConfigJSON2)) - cdsConfigJSON3 = `{"cluster":"cluster_3"}` - cdsConfig3, _ = cdsConfigParser.ParseConfig([]byte(cdsConfigJSON3)) - - wtConfigParser = balancer.Get(wtName).(balancer.ConfigParser) - wtConfigJSON1 = `{ - "targets": { - "cluster_1" : { "weight":75, "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}] }, - "cluster_2" : { "weight":25, "childPolicy":[{"cds_experimental":{"cluster":"cluster_2"}}] } - } }` - wtConfig1, _ = wtConfigParser.ParseConfig([]byte(wtConfigJSON1)) - wtConfigJSON2 = `{ - "targets": { - "cluster_1": { "weight":99, "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}] }, - "cluster_3": { "weight":1, "childPolicy":[{"cds_experimental":{"cluster":"cluster_3"}}] } - } }` - wtConfig2, _ = wtConfigParser.ParseConfig([]byte(wtConfigJSON2)) -) - -func Test_parseConfig(t *testing.T) { - tests := []struct { - name string - js string - want *lbConfig - wantErr bool - }{ - { - name: "empty json", - js: "", - want: nil, - wantErr: true, - }, - { - name: "more than one path matcher", // Path matcher is oneof, so this is an error. - js: `{ - "Action":{ - "cds:cluster_1":{ "childPolicy":[{ "cds_experimental":{"cluster":"cluster_1"} }]} - }, - "Route": [{ - "path":"/service_1/method_1", - "prefix":"/service_1/", - "action":"cds:cluster_1" - }] - }`, - want: nil, - wantErr: true, - }, - { - name: "no path matcher", - js: `{ - "Action":{ - "cds:cluster_1":{ "childPolicy":[{ "cds_experimental":{"cluster":"cluster_1"} }]} - }, - "Route": [{ - "action":"cds:cluster_1" - }] - }`, - want: nil, - wantErr: true, - }, - { - name: "route action not found in action list", - js: `{ - "Action":{}, - "Route": [{ - "path":"/service_1/method_1", - "action":"cds:cluster_1" - }] - }`, - want: nil, - wantErr: true, - }, - { - name: "action list contains action not used", - js: `{ - "Action":{ - "cds:cluster_1":{ "childPolicy":[{ "cds_experimental":{"cluster":"cluster_1"} }]}, - "cds:cluster_not_used":{ "childPolicy":[{ "cds_experimental":{"cluster":"cluster_1"} }]} - }, - "Route": [{ - "path":"/service_1/method_1", - "action":"cds:cluster_1" - }] - }`, - want: nil, - wantErr: true, - }, - - { - name: "no header specifier in header matcher", - js: `{ - "Action":{ - "cds:cluster_1":{ "childPolicy":[{ "cds_experimental":{"cluster":"cluster_1"} }]} - }, - "Route": [{ - "path":"/service_1/method_1", - "headers":[{"name":"header-1"}], - "action":"cds:cluster_1" - }] - }`, - want: nil, - wantErr: true, - }, - { - name: "more than one header specifier in header matcher", - js: `{ - "Action":{ - "cds:cluster_1":{ "childPolicy":[{ "cds_experimental":{"cluster":"cluster_1"} }]} - }, - "Route": [{ - "path":"/service_1/method_1", - "headers":[{"name":"header-1", "prefixMatch":"a", "suffixMatch":"b"}], - "action":"cds:cluster_1" - }] - }`, - want: nil, - wantErr: true, - }, - - { - name: "OK with path matchers only", - js: testJSONConfig, - want: &lbConfig{ - routes: []routeConfig{ - {path: "/service_1/method_1", action: "cds:cluster_1"}, - {path: "/service_1/method_2", action: "cds:cluster_1"}, - {prefix: "/service_2/method_1", action: "weighted:cluster_1_cluster_2_1"}, - {prefix: "/service_2", action: "weighted:cluster_1_cluster_2_1"}, - {regex: "^/service_2/method_3$", action: "weighted:cluster_1_cluster_3_1"}, - }, - actions: map[string]actionConfig{ - "cds:cluster_1": {ChildPolicy: &internalserviceconfig.BalancerConfig{ - Name: cdsName, Config: cdsConfig1}, - }, - "weighted:cluster_1_cluster_2_1": {ChildPolicy: &internalserviceconfig.BalancerConfig{ - Name: wtName, Config: wtConfig1}, - }, - "weighted:cluster_1_cluster_3_1": {ChildPolicy: &internalserviceconfig.BalancerConfig{ - Name: wtName, Config: wtConfig2}, - }, - }, - }, - wantErr: false, - }, - { - name: "OK with all matchers", - js: testJSONConfigWithAllMatchers, - want: &lbConfig{ - routes: []routeConfig{ - {path: "/service_1/method_1", action: "cds:cluster_1"}, - {prefix: "/service_2/method_1", action: "cds:cluster_1"}, - {regex: "^/service_2/method_3$", action: "cds:cluster_1"}, - - {prefix: "", headers: []headerMatcher{{name: "header-1", exactMatch: "value-1", invertMatch: true}}, action: "cds:cluster_2"}, - {prefix: "", headers: []headerMatcher{{name: "header-1", regexMatch: "^value-1$"}}, action: "cds:cluster_2"}, - {prefix: "", headers: []headerMatcher{{name: "header-1", rangeMatch: &int64Range{start: -1, end: 7}}}, action: "cds:cluster_3"}, - {prefix: "", headers: []headerMatcher{{name: "header-1", presentMatch: true}}, action: "cds:cluster_3"}, - {prefix: "", headers: []headerMatcher{{name: "header-1", prefixMatch: "value-1"}}, action: "cds:cluster_2"}, - {prefix: "", headers: []headerMatcher{{name: "header-1", suffixMatch: "value-1"}}, action: "cds:cluster_2"}, - {prefix: "", fraction: newUInt32P(31415), action: "cds:cluster_3"}, - }, - actions: map[string]actionConfig{ - "cds:cluster_1": {ChildPolicy: &internalserviceconfig.BalancerConfig{ - Name: cdsName, Config: cdsConfig1}, - }, - "cds:cluster_2": {ChildPolicy: &internalserviceconfig.BalancerConfig{ - Name: cdsName, Config: cdsConfig2}, - }, - "cds:cluster_3": {ChildPolicy: &internalserviceconfig.BalancerConfig{ - Name: cdsName, Config: cdsConfig3}, - }, - }, - }, - wantErr: false, - }, - } - - cmpOptions := []cmp.Option{cmp.AllowUnexported(lbConfig{}, routeConfig{}, headerMatcher{}, int64Range{})} - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := parseConfig([]byte(tt.js)) - if (err != nil) != tt.wantErr { - t.Errorf("parseConfig() error = %v, wantErr %v", err, tt.wantErr) - return - } - if !cmp.Equal(got, tt.want, cmpOptions...) { - t.Errorf("parseConfig() got unexpected result, diff: %v", cmp.Diff(got, tt.want, cmpOptions...)) - } - }) - } -} - -func newUInt32P(i uint32) *uint32 { - return &i -} diff --git a/xds/internal/balancer/xdsrouting/routing_picker.go b/xds/internal/balancer/xdsrouting/routing_picker.go deleted file mode 100644 index 36fdebd4ad2a..000000000000 --- a/xds/internal/balancer/xdsrouting/routing_picker.go +++ /dev/null @@ -1,61 +0,0 @@ -/* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package xdsrouting - -import ( - "google.golang.org/grpc/balancer" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - "google.golang.org/grpc/xds/internal" -) - -// pickerGroup contains a list of route matchers and their corresponding -// pickers. For each pick, the first matched picker is used. If the picker isn't -// ready, the pick will be queued. -type pickerGroup struct { - routes []route - pickers map[string]balancer.Picker -} - -func newPickerGroup(routes []route, idToPickerState map[internal.LocalityID]*subBalancerState) *pickerGroup { - pickers := make(map[string]balancer.Picker) - for id, st := range idToPickerState { - pickers[getNameFromLocality(id)] = st.state.Picker - } - return &pickerGroup{ - routes: routes, - pickers: pickers, - } -} - -var errNoMatchedRouteFound = status.Errorf(codes.Unavailable, "no matched route was found") - -func (pg *pickerGroup) Pick(info balancer.PickInfo) (balancer.PickResult, error) { - for _, rt := range pg.routes { - if rt.m.match(info) { - // action from route is the ID for the sub-balancer to use. - p, ok := pg.pickers[rt.action] - if !ok { - return balancer.PickResult{}, balancer.ErrNoSubConnAvailable - } - return p.Pick(info) - } - } - return balancer.PickResult{}, errNoMatchedRouteFound -} diff --git a/xds/internal/balancer/xdsrouting/routing_picker_test.go b/xds/internal/balancer/xdsrouting/routing_picker_test.go deleted file mode 100644 index 2de40757a4f7..000000000000 --- a/xds/internal/balancer/xdsrouting/routing_picker_test.go +++ /dev/null @@ -1,177 +0,0 @@ -/* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package xdsrouting - -import ( - "testing" - - "github.com/google/go-cmp/cmp" - "google.golang.org/grpc/balancer" - "google.golang.org/grpc/connectivity" - "google.golang.org/grpc/xds/internal" - "google.golang.org/grpc/xds/internal/testutils" -) - -var ( - testPickers = []*testutils.TestConstPicker{ - {SC: testutils.TestSubConns[0]}, - {SC: testutils.TestSubConns[1]}, - } -) - -func (s) TestRoutingPickerGroupPick(t *testing.T) { - tests := []struct { - name string - - routes []route - pickers map[internal.LocalityID]*subBalancerState - info balancer.PickInfo - - want balancer.PickResult - wantErr error - }{ - { - name: "empty", - wantErr: errNoMatchedRouteFound, - }, - { - name: "one route no match", - routes: []route{ - {m: newCompositeMatcher(newPathPrefixMatcher("/a/"), nil, nil), action: "action-0"}, - }, - pickers: map[internal.LocalityID]*subBalancerState{ - makeLocalityFromName("action-0"): {state: balancer.State{ - ConnectivityState: connectivity.Ready, - Picker: testPickers[0], - }}, - }, - info: balancer.PickInfo{FullMethodName: "/z/y"}, - wantErr: errNoMatchedRouteFound, - }, - { - name: "one route one match", - routes: []route{ - {m: newCompositeMatcher(newPathPrefixMatcher("/a/"), nil, nil), action: "action-0"}, - }, - pickers: map[internal.LocalityID]*subBalancerState{ - makeLocalityFromName("action-0"): {state: balancer.State{ - ConnectivityState: connectivity.Ready, - Picker: testPickers[0], - }}, - }, - info: balancer.PickInfo{FullMethodName: "/a/b"}, - want: balancer.PickResult{SubConn: testutils.TestSubConns[0]}, - }, - { - name: "two routes first match", - routes: []route{ - {m: newCompositeMatcher(newPathPrefixMatcher("/a/"), nil, nil), action: "action-0"}, - {m: newCompositeMatcher(newPathPrefixMatcher("/z/"), nil, nil), action: "action-1"}, - }, - pickers: map[internal.LocalityID]*subBalancerState{ - makeLocalityFromName("action-0"): {state: balancer.State{ - ConnectivityState: connectivity.Ready, - Picker: testPickers[0], - }}, - makeLocalityFromName("action-1"): {state: balancer.State{ - ConnectivityState: connectivity.Ready, - Picker: testPickers[1], - }}, - }, - info: balancer.PickInfo{FullMethodName: "/a/b"}, - want: balancer.PickResult{SubConn: testutils.TestSubConns[0]}, - }, - { - name: "two routes second match", - routes: []route{ - {m: newCompositeMatcher(newPathPrefixMatcher("/a/"), nil, nil), action: "action-0"}, - {m: newCompositeMatcher(newPathPrefixMatcher("/z/"), nil, nil), action: "action-1"}, - }, - pickers: map[internal.LocalityID]*subBalancerState{ - makeLocalityFromName("action-0"): {state: balancer.State{ - ConnectivityState: connectivity.Ready, - Picker: testPickers[0], - }}, - makeLocalityFromName("action-1"): {state: balancer.State{ - ConnectivityState: connectivity.Ready, - Picker: testPickers[1], - }}, - }, - info: balancer.PickInfo{FullMethodName: "/z/y"}, - want: balancer.PickResult{SubConn: testutils.TestSubConns[1]}, - }, - { - name: "two routes both match former more specific", - routes: []route{ - {m: newCompositeMatcher(newPathExactMatcher("/a/b"), nil, nil), action: "action-0"}, - {m: newCompositeMatcher(newPathPrefixMatcher("/a/"), nil, nil), action: "action-1"}, - }, - pickers: map[internal.LocalityID]*subBalancerState{ - makeLocalityFromName("action-0"): {state: balancer.State{ - ConnectivityState: connectivity.Ready, - Picker: testPickers[0], - }}, - makeLocalityFromName("action-1"): {state: balancer.State{ - ConnectivityState: connectivity.Ready, - Picker: testPickers[1], - }}, - }, - info: balancer.PickInfo{FullMethodName: "/a/b"}, - // First route is a match, so first action is picked. - want: balancer.PickResult{SubConn: testutils.TestSubConns[0]}, - }, - { - name: "tow routes both match latter more specific", - routes: []route{ - {m: newCompositeMatcher(newPathPrefixMatcher("/a/"), nil, nil), action: "action-0"}, - {m: newCompositeMatcher(newPathExactMatcher("/a/b"), nil, nil), action: "action-1"}, - }, - pickers: map[internal.LocalityID]*subBalancerState{ - makeLocalityFromName("action-0"): {state: balancer.State{ - ConnectivityState: connectivity.Ready, - Picker: testPickers[0], - }}, - makeLocalityFromName("action-1"): {state: balancer.State{ - ConnectivityState: connectivity.Ready, - Picker: testPickers[1], - }}, - }, - info: balancer.PickInfo{FullMethodName: "/a/b"}, - // First route is a match, so first action is picked, even though - // second is an exact match. - want: balancer.PickResult{SubConn: testutils.TestSubConns[0]}, - }, - } - cmpOpts := []cmp.Option{cmp.AllowUnexported(testutils.TestSubConn{})} - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - pg := newPickerGroup(tt.routes, tt.pickers) - got, err := pg.Pick(tt.info) - t.Logf("Pick(%+v) = {%+v, %+v}", tt.info, got, err) - if err != tt.wantErr { - t.Errorf("Pick() error = %v, wantErr %v", err, tt.wantErr) - return - } - if !cmp.Equal(got, tt.want, cmpOpts...) { - t.Errorf("Pick() got = %v, want %v, diff %s", got, tt.want, cmp.Diff(got, tt.want, cmpOpts...)) - } - }) - } - -} diff --git a/xds/internal/balancer/xdsrouting/routing_test.go b/xds/internal/balancer/xdsrouting/routing_test.go deleted file mode 100644 index 90607bc18607..000000000000 --- a/xds/internal/balancer/xdsrouting/routing_test.go +++ /dev/null @@ -1,710 +0,0 @@ -/* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package xdsrouting - -import ( - "context" - "fmt" - "testing" - "time" - - "github.com/google/go-cmp/cmp" - "google.golang.org/grpc/balancer" - "google.golang.org/grpc/balancer/roundrobin" - "google.golang.org/grpc/connectivity" - "google.golang.org/grpc/internal/grpctest" - "google.golang.org/grpc/internal/hierarchy" - "google.golang.org/grpc/metadata" - "google.golang.org/grpc/resolver" - "google.golang.org/grpc/xds/internal/balancer/balancergroup" - "google.golang.org/grpc/xds/internal/testutils" -) - -type s struct { - grpctest.Tester -} - -func Test(t *testing.T) { - grpctest.RunSubTests(t, s{}) -} - -var ( - rtBuilder balancer.Builder - rtParser balancer.ConfigParser - testBackendAddrStrs []string -) - -const ignoreAttrsRRName = "ignore_attrs_round_robin" - -type ignoreAttrsRRBuilder struct { - balancer.Builder -} - -func (trrb *ignoreAttrsRRBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { - return &ignoreAttrsRRBalancer{trrb.Builder.Build(cc, opts)} -} - -func (*ignoreAttrsRRBuilder) Name() string { - return ignoreAttrsRRName -} - -// ignoreAttrsRRBalancer clears attributes from all addresses. -// -// It's necessary in this tests because hierarchy modifies address.Attributes. -// Even if rr gets addresses with empty hierarchy, the attributes fields are -// different. This is a temporary walkaround for the tests to ignore attributes. -// Eventually, we need a way for roundrobin to know that two addresses with -// empty attributes are equal. -// -// TODO: delete this when the issue is resolved: -// https://github.com/grpc/grpc-go/issues/3611. -type ignoreAttrsRRBalancer struct { - balancer.Balancer -} - -func (trrb *ignoreAttrsRRBalancer) UpdateClientConnState(s balancer.ClientConnState) error { - var newAddrs []resolver.Address - for _, a := range s.ResolverState.Addresses { - a.Attributes = nil - newAddrs = append(newAddrs, a) - } - s.ResolverState.Addresses = newAddrs - return trrb.Balancer.UpdateClientConnState(s) -} - -const testBackendAddrsCount = 12 - -func init() { - for i := 0; i < testBackendAddrsCount; i++ { - testBackendAddrStrs = append(testBackendAddrStrs, fmt.Sprintf("%d.%d.%d.%d:%d", i, i, i, i, i)) - } - rtBuilder = balancer.Get(xdsRoutingName) - rtParser = rtBuilder.(balancer.ConfigParser) - - balancer.Register(&ignoreAttrsRRBuilder{balancer.Get(roundrobin.Name)}) - - balancergroup.DefaultSubBalancerCloseTimeout = time.Millisecond -} - -func testPick(t *testing.T, p balancer.Picker, info balancer.PickInfo, wantSC balancer.SubConn, wantErr error) { - t.Helper() - for i := 0; i < 5; i++ { - gotSCSt, err := p.Pick(info) - if err != wantErr { - t.Fatalf("picker.Pick(%+v), got error %v, want %v", info, err, wantErr) - } - if !cmp.Equal(gotSCSt.SubConn, wantSC, cmp.AllowUnexported(testutils.TestSubConn{})) { - t.Fatalf("picker.Pick(%+v), got %v, want SubConn=%v", info, gotSCSt, wantSC) - } - } -} - -func TestRouting(t *testing.T) { - cc := testutils.NewTestClientConn(t) - rtb := rtBuilder.Build(cc, balancer.BuildOptions{}) - - configJSON1 := `{ -"Action": { - "cds:cluster_1":{ "childPolicy": [{"ignore_attrs_round_robin":""}] }, - "cds:cluster_2":{ "childPolicy": [{"ignore_attrs_round_robin":""}] } -}, -"Route": [ - {"prefix":"/a/", "action":"cds:cluster_1"}, - {"prefix":"", "headers":[{"name":"header-1", "exactMatch":"value-1"}], "action":"cds:cluster_2"} -] -}` - - config1, err := rtParser.ParseConfig([]byte(configJSON1)) - if err != nil { - t.Fatalf("failed to parse balancer config: %v", err) - } - - // Send the config, and an address with hierarchy path ["cluster_1"]. - wantAddrs := []resolver.Address{ - {Addr: testBackendAddrStrs[0], Attributes: nil}, - {Addr: testBackendAddrStrs[1], Attributes: nil}, - } - if err := rtb.UpdateClientConnState(balancer.ClientConnState{ - ResolverState: resolver.State{Addresses: []resolver.Address{ - hierarchy.Set(wantAddrs[0], []string{"cds:cluster_1"}), - hierarchy.Set(wantAddrs[1], []string{"cds:cluster_2"}), - }}, - BalancerConfig: config1, - }); err != nil { - t.Fatalf("failed to update ClientConn state: %v", err) - } - - m1 := make(map[resolver.Address]balancer.SubConn) - // Verify that a subconn is created with the address, and the hierarchy path - // in the address is cleared. - for range wantAddrs { - addrs := <-cc.NewSubConnAddrsCh - if len(hierarchy.Get(addrs[0])) != 0 { - t.Fatalf("NewSubConn with address %+v, attrs %+v, want address with hierarchy cleared", addrs[0], addrs[0].Attributes) - } - sc := <-cc.NewSubConnCh - // Clear the attributes before adding to map. - addrs[0].Attributes = nil - m1[addrs[0]] = sc - rtb.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - rtb.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - } - - p1 := <-cc.NewPickerCh - for _, tt := range []struct { - pickInfo balancer.PickInfo - wantSC balancer.SubConn - wantErr error - }{ - { - pickInfo: balancer.PickInfo{FullMethodName: "/a/0"}, - wantSC: m1[wantAddrs[0]], - wantErr: nil, - }, - { - pickInfo: balancer.PickInfo{FullMethodName: "/a/1"}, - wantSC: m1[wantAddrs[0]], - wantErr: nil, - }, - { - pickInfo: balancer.PickInfo{ - FullMethodName: "/z/y", - Ctx: metadata.NewOutgoingContext(context.Background(), metadata.Pairs("header-1", "value-1")), - }, - wantSC: m1[wantAddrs[1]], - wantErr: nil, - }, - { - pickInfo: balancer.PickInfo{ - FullMethodName: "/z/y", - Ctx: metadata.NewOutgoingContext(context.Background(), metadata.Pairs("h", "v")), - }, - wantSC: nil, - wantErr: errNoMatchedRouteFound, - }, - } { - testPick(t, p1, tt.pickInfo, tt.wantSC, tt.wantErr) - } -} - -// TestRoutingConfigUpdateAddRoute covers the cases the routing balancer -// receives config update with extra route, but the same actions. -func TestRoutingConfigUpdateAddRoute(t *testing.T) { - cc := testutils.NewTestClientConn(t) - rtb := rtBuilder.Build(cc, balancer.BuildOptions{}) - - configJSON1 := `{ -"Action": { - "cds:cluster_1":{ "childPolicy": [{"ignore_attrs_round_robin":""}] }, - "cds:cluster_2":{ "childPolicy": [{"ignore_attrs_round_robin":""}] } -}, -"Route": [ - {"prefix":"/a/", "action":"cds:cluster_1"}, - {"path":"/z/y", "action":"cds:cluster_2"} -] -}` - - config1, err := rtParser.ParseConfig([]byte(configJSON1)) - if err != nil { - t.Fatalf("failed to parse balancer config: %v", err) - } - - // Send the config, and an address with hierarchy path ["cluster_1"]. - wantAddrs := []resolver.Address{ - {Addr: testBackendAddrStrs[0], Attributes: nil}, - {Addr: testBackendAddrStrs[1], Attributes: nil}, - } - if err := rtb.UpdateClientConnState(balancer.ClientConnState{ - ResolverState: resolver.State{Addresses: []resolver.Address{ - hierarchy.Set(wantAddrs[0], []string{"cds:cluster_1"}), - hierarchy.Set(wantAddrs[1], []string{"cds:cluster_2"}), - }}, - BalancerConfig: config1, - }); err != nil { - t.Fatalf("failed to update ClientConn state: %v", err) - } - - m1 := make(map[resolver.Address]balancer.SubConn) - // Verify that a subconn is created with the address, and the hierarchy path - // in the address is cleared. - for range wantAddrs { - addrs := <-cc.NewSubConnAddrsCh - if len(hierarchy.Get(addrs[0])) != 0 { - t.Fatalf("NewSubConn with address %+v, attrs %+v, want address with hierarchy cleared", addrs[0], addrs[0].Attributes) - } - sc := <-cc.NewSubConnCh - // Clear the attributes before adding to map. - addrs[0].Attributes = nil - m1[addrs[0]] = sc - rtb.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - rtb.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - } - - p1 := <-cc.NewPickerCh - for _, tt := range []struct { - pickInfo balancer.PickInfo - wantSC balancer.SubConn - wantErr error - }{ - { - pickInfo: balancer.PickInfo{FullMethodName: "/a/0"}, - wantSC: m1[wantAddrs[0]], - wantErr: nil, - }, - { - pickInfo: balancer.PickInfo{FullMethodName: "/a/1"}, - wantSC: m1[wantAddrs[0]], - wantErr: nil, - }, - { - pickInfo: balancer.PickInfo{FullMethodName: "/z/y"}, - wantSC: m1[wantAddrs[1]], - wantErr: nil, - }, - { - pickInfo: balancer.PickInfo{FullMethodName: "/c/d"}, - wantSC: nil, - wantErr: errNoMatchedRouteFound, - }, - } { - testPick(t, p1, tt.pickInfo, tt.wantSC, tt.wantErr) - } - - // A config update with different routes, but the same actions. Expect a - // picker update, but no subconn changes. - configJSON2 := `{ -"Action": { - "cds:cluster_1":{ "childPolicy": [{"ignore_attrs_round_robin":""}] }, - "cds:cluster_2":{ "childPolicy": [{"ignore_attrs_round_robin":""}] } -}, -"Route": [ - {"prefix":"", "headers":[{"name":"header-1", "presentMatch":true}], "action":"cds:cluster_2"}, - {"prefix":"/a/", "action":"cds:cluster_1"}, - {"path":"/z/y", "action":"cds:cluster_2"} -] -}` - config2, err := rtParser.ParseConfig([]byte(configJSON2)) - if err != nil { - t.Fatalf("failed to parse balancer config: %v", err) - } - // Send update with the same addresses. - if err := rtb.UpdateClientConnState(balancer.ClientConnState{ - ResolverState: resolver.State{Addresses: []resolver.Address{ - hierarchy.Set(wantAddrs[0], []string{"cds:cluster_1"}), - hierarchy.Set(wantAddrs[1], []string{"cds:cluster_2"}), - }}, - BalancerConfig: config2, - }); err != nil { - t.Fatalf("failed to update ClientConn state: %v", err) - } - - // New change to actions, expect no newSubConn. - select { - case <-time.After(time.Millisecond * 500): - case <-cc.NewSubConnCh: - addrs := <-cc.NewSubConnAddrsCh - t.Fatalf("unexpected NewSubConn with address %v", addrs) - } - - p2 := <-cc.NewPickerCh - for _, tt := range []struct { - pickInfo balancer.PickInfo - wantSC balancer.SubConn - wantErr error - }{ - { - pickInfo: balancer.PickInfo{FullMethodName: "/a/0"}, - wantSC: m1[wantAddrs[0]], - wantErr: nil, - }, - { - pickInfo: balancer.PickInfo{FullMethodName: "/a/1"}, - wantSC: m1[wantAddrs[0]], - wantErr: nil, - }, - { - pickInfo: balancer.PickInfo{FullMethodName: "/z/y"}, - wantSC: m1[wantAddrs[1]], - wantErr: nil, - }, - { - pickInfo: balancer.PickInfo{ - FullMethodName: "/a/z", - Ctx: metadata.NewOutgoingContext(context.Background(), metadata.Pairs("header-1", "value-1")), - }, - wantSC: m1[wantAddrs[1]], - wantErr: nil, - }, - { - pickInfo: balancer.PickInfo{ - FullMethodName: "/c/d", - Ctx: metadata.NewOutgoingContext(context.Background(), metadata.Pairs("h", "v")), - }, - wantSC: nil, - wantErr: errNoMatchedRouteFound, - }, - } { - testPick(t, p2, tt.pickInfo, tt.wantSC, tt.wantErr) - } -} - -// TestRoutingConfigUpdateAddRouteAndAction covers the cases the routing -// balancer receives config update with extra route and actions. -func TestRoutingConfigUpdateAddRouteAndAction(t *testing.T) { - cc := testutils.NewTestClientConn(t) - rtb := rtBuilder.Build(cc, balancer.BuildOptions{}) - - configJSON1 := `{ -"Action": { - "cds:cluster_1":{ "childPolicy": [{"ignore_attrs_round_robin":""}] }, - "cds:cluster_2":{ "childPolicy": [{"ignore_attrs_round_robin":""}] } -}, -"Route": [ - {"prefix":"/a/", "action":"cds:cluster_1"}, - {"path":"/z/y", "action":"cds:cluster_2"} -] -}` - - config1, err := rtParser.ParseConfig([]byte(configJSON1)) - if err != nil { - t.Fatalf("failed to parse balancer config: %v", err) - } - - // Send the config, and an address with hierarchy path ["cluster_1"]. - wantAddrs := []resolver.Address{ - {Addr: testBackendAddrStrs[0], Attributes: nil}, - {Addr: testBackendAddrStrs[1], Attributes: nil}, - } - if err := rtb.UpdateClientConnState(balancer.ClientConnState{ - ResolverState: resolver.State{Addresses: []resolver.Address{ - hierarchy.Set(wantAddrs[0], []string{"cds:cluster_1"}), - hierarchy.Set(wantAddrs[1], []string{"cds:cluster_2"}), - }}, - BalancerConfig: config1, - }); err != nil { - t.Fatalf("failed to update ClientConn state: %v", err) - } - - m1 := make(map[resolver.Address]balancer.SubConn) - // Verify that a subconn is created with the address, and the hierarchy path - // in the address is cleared. - for range wantAddrs { - addrs := <-cc.NewSubConnAddrsCh - if len(hierarchy.Get(addrs[0])) != 0 { - t.Fatalf("NewSubConn with address %+v, attrs %+v, want address with hierarchy cleared", addrs[0], addrs[0].Attributes) - } - sc := <-cc.NewSubConnCh - // Clear the attributes before adding to map. - addrs[0].Attributes = nil - m1[addrs[0]] = sc - rtb.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - rtb.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - } - - p1 := <-cc.NewPickerCh - for _, tt := range []struct { - pickInfo balancer.PickInfo - wantSC balancer.SubConn - wantErr error - }{ - { - pickInfo: balancer.PickInfo{FullMethodName: "/a/0"}, - wantSC: m1[wantAddrs[0]], - wantErr: nil, - }, - { - pickInfo: balancer.PickInfo{FullMethodName: "/a/1"}, - wantSC: m1[wantAddrs[0]], - wantErr: nil, - }, - { - pickInfo: balancer.PickInfo{FullMethodName: "/z/y"}, - wantSC: m1[wantAddrs[1]], - wantErr: nil, - }, - { - pickInfo: balancer.PickInfo{FullMethodName: "/c/d"}, - wantSC: nil, - wantErr: errNoMatchedRouteFound, - }, - } { - testPick(t, p1, tt.pickInfo, tt.wantSC, tt.wantErr) - } - - // A config update with different routes, and different actions. Expect a - // new subconn and a picker update. - configJSON2 := `{ -"Action": { - "cds:cluster_1":{ "childPolicy": [{"ignore_attrs_round_robin":""}] }, - "cds:cluster_2":{ "childPolicy": [{"ignore_attrs_round_robin":""}] }, - "cds:cluster_3":{ "childPolicy": [{"ignore_attrs_round_robin":""}] } -}, -"Route": [ - {"prefix":"", "headers":[{"name":"header-1", "presentMatch":false, "invertMatch":true}], "action":"cds:cluster_3"}, - {"prefix":"/a/", "action":"cds:cluster_1"}, - {"path":"/z/y", "action":"cds:cluster_2"} -] -}` - config2, err := rtParser.ParseConfig([]byte(configJSON2)) - if err != nil { - t.Fatalf("failed to parse balancer config: %v", err) - } - wantAddrs = append(wantAddrs, resolver.Address{Addr: testBackendAddrStrs[2], Attributes: nil}) - if err := rtb.UpdateClientConnState(balancer.ClientConnState{ - ResolverState: resolver.State{Addresses: []resolver.Address{ - hierarchy.Set(wantAddrs[0], []string{"cds:cluster_1"}), - hierarchy.Set(wantAddrs[1], []string{"cds:cluster_2"}), - hierarchy.Set(wantAddrs[2], []string{"cds:cluster_3"}), - }}, - BalancerConfig: config2, - }); err != nil { - t.Fatalf("failed to update ClientConn state: %v", err) - } - - // Expect exactly one new subconn. - addrs := <-cc.NewSubConnAddrsCh - if len(hierarchy.Get(addrs[0])) != 0 { - t.Fatalf("NewSubConn with address %+v, attrs %+v, want address with hierarchy cleared", addrs[0], addrs[0].Attributes) - } - sc := <-cc.NewSubConnCh - // Clear the attributes before adding to map. - addrs[0].Attributes = nil - m1[addrs[0]] = sc - rtb.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - rtb.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - - // Should have no more newSubConn. - select { - case <-time.After(time.Millisecond * 500): - case <-cc.NewSubConnCh: - addrs := <-cc.NewSubConnAddrsCh - t.Fatalf("unexpected NewSubConn with address %v", addrs) - } - - p2 := <-cc.NewPickerCh - for _, tt := range []struct { - pickInfo balancer.PickInfo - wantSC balancer.SubConn - wantErr error - }{ - { - pickInfo: balancer.PickInfo{FullMethodName: "/a/0"}, - wantSC: m1[wantAddrs[0]], - wantErr: nil, - }, - { - pickInfo: balancer.PickInfo{FullMethodName: "/a/1"}, - wantSC: m1[wantAddrs[0]], - wantErr: nil, - }, - { - pickInfo: balancer.PickInfo{FullMethodName: "/z/y"}, - wantSC: m1[wantAddrs[1]], - wantErr: nil, - }, - { - pickInfo: balancer.PickInfo{ - FullMethodName: "/a/z", - Ctx: metadata.NewOutgoingContext(context.Background(), metadata.Pairs("header-1", "value-1")), - }, - wantSC: m1[wantAddrs[2]], - wantErr: nil, - }, - { - pickInfo: balancer.PickInfo{ - FullMethodName: "/c/d", - Ctx: metadata.NewOutgoingContext(context.Background(), metadata.Pairs("h", "v")), - }, - wantSC: nil, - wantErr: errNoMatchedRouteFound, - }, - } { - testPick(t, p2, tt.pickInfo, tt.wantSC, tt.wantErr) - } -} - -// TestRoutingConfigUpdateDeleteAll covers the cases the routing balancer receives config -// update with no routes. Pick should fail with details in error. -func TestRoutingConfigUpdateDeleteAll(t *testing.T) { - cc := testutils.NewTestClientConn(t) - rtb := rtBuilder.Build(cc, balancer.BuildOptions{}) - - configJSON1 := `{ -"Action": { - "cds:cluster_1":{ "childPolicy": [{"ignore_attrs_round_robin":""}] }, - "cds:cluster_2":{ "childPolicy": [{"ignore_attrs_round_robin":""}] } -}, -"Route": [ - {"prefix":"/a/", "action":"cds:cluster_1"}, - {"path":"/z/y", "action":"cds:cluster_2"} -] -}` - - config1, err := rtParser.ParseConfig([]byte(configJSON1)) - if err != nil { - t.Fatalf("failed to parse balancer config: %v", err) - } - - // Send the config, and an address with hierarchy path ["cluster_1"]. - wantAddrs := []resolver.Address{ - {Addr: testBackendAddrStrs[0], Attributes: nil}, - {Addr: testBackendAddrStrs[1], Attributes: nil}, - } - if err := rtb.UpdateClientConnState(balancer.ClientConnState{ - ResolverState: resolver.State{Addresses: []resolver.Address{ - hierarchy.Set(wantAddrs[0], []string{"cds:cluster_1"}), - hierarchy.Set(wantAddrs[1], []string{"cds:cluster_2"}), - }}, - BalancerConfig: config1, - }); err != nil { - t.Fatalf("failed to update ClientConn state: %v", err) - } - - m1 := make(map[resolver.Address]balancer.SubConn) - // Verify that a subconn is created with the address, and the hierarchy path - // in the address is cleared. - for range wantAddrs { - addrs := <-cc.NewSubConnAddrsCh - if len(hierarchy.Get(addrs[0])) != 0 { - t.Fatalf("NewSubConn with address %+v, attrs %+v, want address with hierarchy cleared", addrs[0], addrs[0].Attributes) - } - sc := <-cc.NewSubConnCh - // Clear the attributes before adding to map. - addrs[0].Attributes = nil - m1[addrs[0]] = sc - rtb.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - rtb.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - } - - p1 := <-cc.NewPickerCh - for _, tt := range []struct { - pickInfo balancer.PickInfo - wantSC balancer.SubConn - wantErr error - }{ - { - pickInfo: balancer.PickInfo{FullMethodName: "/a/0"}, - wantSC: m1[wantAddrs[0]], - wantErr: nil, - }, - { - pickInfo: balancer.PickInfo{FullMethodName: "/a/1"}, - wantSC: m1[wantAddrs[0]], - wantErr: nil, - }, - { - pickInfo: balancer.PickInfo{FullMethodName: "/z/y"}, - wantSC: m1[wantAddrs[1]], - wantErr: nil, - }, - { - pickInfo: balancer.PickInfo{FullMethodName: "/c/d"}, - wantSC: nil, - wantErr: errNoMatchedRouteFound, - }, - } { - testPick(t, p1, tt.pickInfo, tt.wantSC, tt.wantErr) - } - - // A config update with no routes. - configJSON2 := `{}` - config2, err := rtParser.ParseConfig([]byte(configJSON2)) - if err != nil { - t.Fatalf("failed to parse balancer config: %v", err) - } - if err := rtb.UpdateClientConnState(balancer.ClientConnState{ - BalancerConfig: config2, - }); err != nil { - t.Fatalf("failed to update ClientConn state: %v", err) - } - - // Expect two remove subconn. - for range wantAddrs { - select { - case <-time.After(time.Millisecond * 500): - t.Fatalf("timeout waiting for remove subconn") - case <-cc.RemoveSubConnCh: - } - } - - p2 := <-cc.NewPickerCh - for i := 0; i < 5; i++ { - gotSCSt, err := p2.Pick(balancer.PickInfo{}) - if err != errNoMatchedRouteFound { - t.Fatalf("picker.Pick, got %v, %v, want error %v", gotSCSt, err, errNoMatchedRouteFound) - } - } - - // Resend the previous config with routes and actions. - if err := rtb.UpdateClientConnState(balancer.ClientConnState{ - ResolverState: resolver.State{Addresses: []resolver.Address{ - hierarchy.Set(wantAddrs[0], []string{"cds:cluster_1"}), - hierarchy.Set(wantAddrs[1], []string{"cds:cluster_2"}), - }}, - BalancerConfig: config1, - }); err != nil { - t.Fatalf("failed to update ClientConn state: %v", err) - } - - m2 := make(map[resolver.Address]balancer.SubConn) - // Verify that a subconn is created with the address, and the hierarchy path - // in the address is cleared. - for range wantAddrs { - addrs := <-cc.NewSubConnAddrsCh - if len(hierarchy.Get(addrs[0])) != 0 { - t.Fatalf("NewSubConn with address %+v, attrs %+v, want address with hierarchy cleared", addrs[0], addrs[0].Attributes) - } - sc := <-cc.NewSubConnCh - // Clear the attributes before adding to map. - addrs[0].Attributes = nil - m2[addrs[0]] = sc - rtb.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: connectivity.Connecting}) - rtb.UpdateSubConnState(sc, balancer.SubConnState{ConnectivityState: connectivity.Ready}) - } - - p3 := <-cc.NewPickerCh - for _, tt := range []struct { - pickInfo balancer.PickInfo - wantSC balancer.SubConn - wantErr error - }{ - { - pickInfo: balancer.PickInfo{FullMethodName: "/a/0"}, - wantSC: m2[wantAddrs[0]], - wantErr: nil, - }, - { - pickInfo: balancer.PickInfo{FullMethodName: "/a/1"}, - wantSC: m2[wantAddrs[0]], - wantErr: nil, - }, - { - pickInfo: balancer.PickInfo{FullMethodName: "/z/y"}, - wantSC: m2[wantAddrs[1]], - wantErr: nil, - }, - { - pickInfo: balancer.PickInfo{FullMethodName: "/c/d"}, - wantSC: nil, - wantErr: errNoMatchedRouteFound, - }, - } { - testPick(t, p3, tt.pickInfo, tt.wantSC, tt.wantErr) - } -} diff --git a/xds/internal/client/bootstrap/bootstrap.go b/xds/internal/client/bootstrap/bootstrap.go deleted file mode 100644 index b2805bf73720..000000000000 --- a/xds/internal/client/bootstrap/bootstrap.go +++ /dev/null @@ -1,260 +0,0 @@ -/* - * - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -// Package bootstrap provides the functionality to initialize certain aspects -// of an xDS client by reading a bootstrap file. -package bootstrap - -import ( - "bytes" - "encoding/json" - "fmt" - "io/ioutil" - "os" - - v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" - v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" - "github.com/golang/protobuf/jsonpb" - "github.com/golang/protobuf/proto" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/google" - "google.golang.org/grpc/xds/internal/version" -) - -const ( - // Environment variable which holds the name of the xDS bootstrap file. - bootstrapFileEnv = "GRPC_XDS_BOOTSTRAP" - // Environment variable which controls the use of xDS v3 API. - v3SupportEnv = "GRPC_XDS_EXPERIMENTAL_V3_SUPPORT" - // The "server_features" field in the bootstrap file contains a list of - // features supported by the server. A value of "xds_v3" indicates that the - // server supports the v3 version of the xDS transport protocol. - serverFeaturesV3 = "xds_v3" - - // Type name for Google default credentials. - googleDefaultCreds = "google_default" - gRPCUserAgentName = "gRPC Go" - clientFeatureNoOverprovisioning = "envoy.lb.does_not_support_overprovisioning" -) - -var gRPCVersion = fmt.Sprintf("%s %s", gRPCUserAgentName, grpc.Version) - -// For overriding in unit tests. -var bootstrapFileReadFunc = ioutil.ReadFile - -// Config provides the xDS client with several key bits of information that it -// requires in its interaction with an xDS server. The Config is initialized -// from the bootstrap file. -type Config struct { - // BalancerName is the name of the xDS server to connect to. - // - // The bootstrap file contains a list of servers (with name+creds), but we - // pick the first one. - BalancerName string - // Creds contains the credentials to be used while talking to the xDS - // server, as a grpc.DialOption. - Creds grpc.DialOption - // TransportAPI indicates the API version of xDS transport protocol to use. - // This describes the xDS gRPC endpoint and version of - // DiscoveryRequest/Response used on the wire. - TransportAPI version.TransportAPI - // NodeProto contains the Node proto to be used in xDS requests. The actual - // type depends on the transport protocol version used. - NodeProto proto.Message -} - -type channelCreds struct { - Type string `json:"type"` - Config json.RawMessage `json:"config"` -} - -type xdsServer struct { - ServerURI string `json:"server_uri"` - ChannelCreds []channelCreds `json:"channel_creds"` -} - -// NewConfig returns a new instance of Config initialized by reading the -// bootstrap file found at ${GRPC_XDS_BOOTSTRAP}. -// -// The format of the bootstrap file will be as follows: -// { -// "xds_server": { -// "server_uri": , -// "channel_creds": [ -// { -// "type": , -// "config": -// } -// ], -// "server_features": [ ... ] -// }, -// "node": -// } -// -// Currently, we support exactly one type of credential, which is -// "google_default", where we use the host's default certs for transport -// credentials and a Google oauth token for call credentials. -// -// This function tries to process as much of the bootstrap file as possible (in -// the presence of the errors) and may return a Config object with certain -// fields left unspecified, in which case the caller should use some sane -// defaults. -func NewConfig() (*Config, error) { - config := &Config{} - - fName, ok := os.LookupEnv(bootstrapFileEnv) - if !ok { - return nil, fmt.Errorf("xds: Environment variable %v not defined", bootstrapFileEnv) - } - logger.Infof("Got bootstrap file location from %v environment variable: %v", bootstrapFileEnv, fName) - - data, err := bootstrapFileReadFunc(fName) - if err != nil { - return nil, fmt.Errorf("xds: Failed to read bootstrap file %s with error %v", fName, err) - } - logger.Debugf("Bootstrap content: %s", data) - - var jsonData map[string]json.RawMessage - if err := json.Unmarshal(data, &jsonData); err != nil { - return nil, fmt.Errorf("xds: Failed to parse file %s (content %v) with error: %v", fName, string(data), err) - } - - serverSupportsV3 := false - m := jsonpb.Unmarshaler{AllowUnknownFields: true} - for k, v := range jsonData { - switch k { - case "node": - // We unconditionally convert the JSON into a v3.Node proto. The v3 - // proto does not contain the deprecated field "build_version" from - // the v2 proto. We do not expect the bootstrap file to contain the - // "build_version" field. In any case, the unmarshal will succeed - // because we have set the `AllowUnknownFields` option on the - // unmarshaler. - n := &v3corepb.Node{} - if err := m.Unmarshal(bytes.NewReader(v), n); err != nil { - return nil, fmt.Errorf("xds: jsonpb.Unmarshal(%v) for field %q failed during bootstrap: %v", string(v), k, err) - } - config.NodeProto = n - case "xds_servers": - var servers []*xdsServer - if err := json.Unmarshal(v, &servers); err != nil { - return nil, fmt.Errorf("xds: json.Unmarshal(%v) for field %q failed during bootstrap: %v", string(v), k, err) - } - if len(servers) < 1 { - return nil, fmt.Errorf("xds: bootstrap file parsing failed during bootstrap: file doesn't contain any xds server to connect to") - } - xs := servers[0] - config.BalancerName = xs.ServerURI - for _, cc := range xs.ChannelCreds { - if cc.Type == googleDefaultCreds { - config.Creds = grpc.WithCredentialsBundle(google.NewDefaultCredentials()) - // We stop at the first credential type that we support. - break - } - } - case "server_features": - var features []string - if err := json.Unmarshal(v, &features); err != nil { - return nil, fmt.Errorf("xds: json.Unmarshal(%v) for field %q failed during bootstrap: %v", string(v), k, err) - } - for _, f := range features { - switch f { - case serverFeaturesV3: - serverSupportsV3 = true - } - } - } - // Do not fail the xDS bootstrap when an unknown field is seen. This can - // happen when an older version client reads a newer version bootstrap - // file with new fields. - } - - if config.BalancerName == "" { - return nil, fmt.Errorf("xds: Required field %q not found in bootstrap", "xds_servers.server_uri") - } - - // We end up using v3 transport protocol version only if the following - // conditions are met: - // 1. Server supports v3, indicated by the presence of "xds_v3" in - // server_features. - // 2. Environment variable "GRPC_XDS_EXPERIMENTAL_V3_SUPPORT" is set to - // true. - // The default value of the enum type "version.TransportAPI" is v2. - // - // TODO: there are multiple env variables, GRPC_XDS_BOOTSTRAP and - // GRPC_XDS_EXPERIMENTAL_V3_SUPPORT. Move all env variables into a separate - // package. - if v3Env := os.Getenv(v3SupportEnv); v3Env == "true" { - if serverSupportsV3 { - config.TransportAPI = version.TransportV3 - } - } - - if err := config.updateNodeProto(); err != nil { - return nil, err - } - logger.Infof("Bootstrap config for creating xds-client: %+v", config) - return config, nil -} - -// updateNodeProto updates the node proto read from the bootstrap file. -// -// Node proto in Config contains a v3.Node protobuf message corresponding to the -// JSON contents found in the bootstrap file. This method performs some post -// processing on it: -// 1. If we don't find a nodeProto in the bootstrap file, we create an empty one -// here. That way, callers of this function can always expect that the NodeProto -// field is non-nil. -// 2. If the transport protocol version to be used is not v3, we convert the -// current v3.Node proto in a v2.Node proto. -// 3. Some additional fields which are not expected to be set in the bootstrap -// file are populated here. -func (c *Config) updateNodeProto() error { - if c.TransportAPI == version.TransportV3 { - v3, _ := c.NodeProto.(*v3corepb.Node) - if v3 == nil { - v3 = &v3corepb.Node{} - } - v3.UserAgentName = gRPCUserAgentName - v3.UserAgentVersionType = &v3corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version} - v3.ClientFeatures = append(v3.ClientFeatures, clientFeatureNoOverprovisioning) - c.NodeProto = v3 - return nil - } - - v2 := &v2corepb.Node{} - if c.NodeProto != nil { - v3, err := proto.Marshal(c.NodeProto) - if err != nil { - return fmt.Errorf("xds: proto.Marshal(%v): %v", c.NodeProto, err) - } - if err := proto.Unmarshal(v3, v2); err != nil { - return fmt.Errorf("xds: proto.Unmarshal(%v): %v", v3, err) - } - } - c.NodeProto = v2 - - // BuildVersion is deprecated, and is replaced by user_agent_name and - // user_agent_version. But the management servers are still using the old - // field, so we will keep both set. - v2.BuildVersion = gRPCVersion - v2.UserAgentName = gRPCUserAgentName - v2.UserAgentVersionType = &v2corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version} - v2.ClientFeatures = append(v2.ClientFeatures, clientFeatureNoOverprovisioning) - return nil -} diff --git a/xds/internal/client/bootstrap/bootstrap_test.go b/xds/internal/client/bootstrap/bootstrap_test.go deleted file mode 100644 index 18c28db346be..000000000000 --- a/xds/internal/client/bootstrap/bootstrap_test.go +++ /dev/null @@ -1,449 +0,0 @@ -/* - * - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package bootstrap - -import ( - "fmt" - "os" - "testing" - - v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" - v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" - "github.com/golang/protobuf/proto" - structpb "github.com/golang/protobuf/ptypes/struct" - "github.com/google/go-cmp/cmp" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/google" - "google.golang.org/grpc/xds/internal/version" -) - -var ( - v2BootstrapFileMap = map[string]string{ - "emptyNodeProto": ` - { - "xds_servers" : [{ - "server_uri": "trafficdirector.googleapis.com:443" - }] - }`, - "unknownTopLevelFieldInFile": ` - { - "node": { - "id": "ENVOY_NODE_ID", - "metadata": { - "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" - } - }, - "xds_servers" : [{ - "server_uri": "trafficdirector.googleapis.com:443", - "channel_creds": [ - { "type": "not-google-default" } - ] - }], - "unknownField": "foobar" - }`, - "unknownFieldInNodeProto": ` - { - "node": { - "id": "ENVOY_NODE_ID", - "unknownField": "foobar", - "metadata": { - "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" - } - }, - "xds_servers" : [{ - "server_uri": "trafficdirector.googleapis.com:443" - }] - }`, - "unknownFieldInXdsServer": ` - { - "node": { - "id": "ENVOY_NODE_ID", - "metadata": { - "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" - } - }, - "xds_servers" : [{ - "server_uri": "trafficdirector.googleapis.com:443", - "channel_creds": [ - { "type": "not-google-default" } - ], - "unknownField": "foobar" - }] - }`, - "emptyChannelCreds": ` - { - "node": { - "id": "ENVOY_NODE_ID", - "metadata": { - "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" - } - }, - "xds_servers" : [{ - "server_uri": "trafficdirector.googleapis.com:443" - }] - }`, - "nonGoogleDefaultCreds": ` - { - "node": { - "id": "ENVOY_NODE_ID", - "metadata": { - "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" - } - }, - "xds_servers" : [{ - "server_uri": "trafficdirector.googleapis.com:443", - "channel_creds": [ - { "type": "not-google-default" } - ] - }] - }`, - "multipleChannelCreds": ` - { - "node": { - "id": "ENVOY_NODE_ID", - "metadata": { - "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" - } - }, - "xds_servers" : [{ - "server_uri": "trafficdirector.googleapis.com:443", - "channel_creds": [ - { "type": "not-google-default" }, - { "type": "google_default" } - ] - }] - }`, - "goodBootstrap": ` - { - "node": { - "id": "ENVOY_NODE_ID", - "metadata": { - "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" - } - }, - "xds_servers" : [{ - "server_uri": "trafficdirector.googleapis.com:443", - "channel_creds": [ - { "type": "google_default" } - ] - }] - }`, - "multipleXDSServers": ` - { - "node": { - "id": "ENVOY_NODE_ID", - "metadata": { - "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" - } - }, - "xds_servers" : [ - { - "server_uri": "trafficdirector.googleapis.com:443", - "channel_creds": [{ "type": "google_default" }] - }, - { - "server_uri": "backup.never.use.com:1234", - "channel_creds": [{ "type": "not-google-default" }] - } - ] - }`, - } - v3BootstrapFileMap = map[string]string{ - "serverDoesNotSupportsV3": ` - { - "node": { - "id": "ENVOY_NODE_ID", - "metadata": { - "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" - } - }, - "xds_servers" : [{ - "server_uri": "trafficdirector.googleapis.com:443", - "channel_creds": [ - { "type": "google_default" } - ] - }], - "server_features" : ["foo", "bar"] - }`, - "serverSupportsV3": ` - { - "node": { - "id": "ENVOY_NODE_ID", - "metadata": { - "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" - } - }, - "xds_servers" : [{ - "server_uri": "trafficdirector.googleapis.com:443", - "channel_creds": [ - { "type": "google_default" } - ] - }], - "server_features" : ["foo", "bar", "xds_v3"] - }`, - } - metadata = &structpb.Struct{ - Fields: map[string]*structpb.Value{ - "TRAFFICDIRECTOR_GRPC_HOSTNAME": { - Kind: &structpb.Value_StringValue{StringValue: "trafficdirector"}, - }, - }, - } - v2NodeProto = &v2corepb.Node{ - Id: "ENVOY_NODE_ID", - Metadata: metadata, - BuildVersion: gRPCVersion, - UserAgentName: gRPCUserAgentName, - UserAgentVersionType: &v2corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version}, - ClientFeatures: []string{clientFeatureNoOverprovisioning}, - } - v3NodeProto = &v3corepb.Node{ - Id: "ENVOY_NODE_ID", - Metadata: metadata, - UserAgentName: gRPCUserAgentName, - UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version}, - ClientFeatures: []string{clientFeatureNoOverprovisioning}, - } - nilCredsConfigV2 = &Config{ - BalancerName: "trafficdirector.googleapis.com:443", - Creds: nil, - NodeProto: v2NodeProto, - } - nonNilCredsConfigV2 = &Config{ - BalancerName: "trafficdirector.googleapis.com:443", - Creds: grpc.WithCredentialsBundle(google.NewComputeEngineCredentials()), - NodeProto: v2NodeProto, - } - nonNilCredsConfigV3 = &Config{ - BalancerName: "trafficdirector.googleapis.com:443", - Creds: grpc.WithCredentialsBundle(google.NewComputeEngineCredentials()), - TransportAPI: version.TransportV3, - NodeProto: v3NodeProto, - } -) - -func (c *Config) compare(want *Config) error { - if c.BalancerName != want.BalancerName { - return fmt.Errorf("config.BalancerName is %s, want %s", c.BalancerName, want.BalancerName) - } - // Since Creds is of type grpc.DialOption interface, where the - // implementation is provided by a function, it is not possible to compare. - if (c.Creds != nil) != (want.Creds != nil) { - return fmt.Errorf("config.Creds is %#v, want %#v", c.Creds, want.Creds) - } - if c.TransportAPI != want.TransportAPI { - return fmt.Errorf("config.TransportAPI is %v, want %v", c.TransportAPI, want.TransportAPI) - - } - if diff := cmp.Diff(want.NodeProto, c.NodeProto, cmp.Comparer(proto.Equal)); diff != "" { - return fmt.Errorf("config.NodeProto diff (-want, +got):\n%s", diff) - } - return nil -} - -func setupBootstrapOverride(bootstrapFileMap map[string]string) func() { - oldFileReadFunc := bootstrapFileReadFunc - bootstrapFileReadFunc = func(name string) ([]byte, error) { - if b, ok := bootstrapFileMap[name]; ok { - return []byte(b), nil - } - return nil, os.ErrNotExist - } - return func() { - bootstrapFileReadFunc = oldFileReadFunc - os.Unsetenv(bootstrapFileEnv) - } -} - -// TODO: enable leak check for this package when -// https://github.com/googleapis/google-cloud-go/issues/2417 is fixed. - -// TestNewConfigV2ProtoFailure exercises the functionality in NewConfig with -// different bootstrap file contents which are expected to fail. -func TestNewConfigV2ProtoFailure(t *testing.T) { - bootstrapFileMap := map[string]string{ - "empty": "", - "badJSON": `["test": 123]`, - "noBalancerName": `{"node": {"id": "ENVOY_NODE_ID"}}`, - "emptyXdsServer": ` - { - "node": { - "id": "ENVOY_NODE_ID", - "metadata": { - "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" - } - } - }`, - } - cancel := setupBootstrapOverride(bootstrapFileMap) - defer cancel() - - tests := []struct { - name string - wantError bool - }{ - {"nonExistentBootstrapFile", true}, - {"empty", true}, - {"badJSON", true}, - {"noBalancerName", true}, - {"emptyXdsServer", true}, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - if err := os.Setenv(bootstrapFileEnv, test.name); err != nil { - t.Fatalf("os.Setenv(%s, %s) failed with error: %v", bootstrapFileEnv, test.name, err) - } - if _, err := NewConfig(); err == nil { - t.Fatalf("NewConfig() returned nil error, expected to fail") - } - }) - } -} - -// TestNewConfigV2ProtoSuccess exercises the functionality in NewConfig with -// different bootstrap file contents. It overrides the fileReadFunc by returning -// bootstrap file contents defined in this test, instead of reading from a file. -func TestNewConfigV2ProtoSuccess(t *testing.T) { - cancel := setupBootstrapOverride(v2BootstrapFileMap) - defer cancel() - - tests := []struct { - name string - wantConfig *Config - }{ - { - "emptyNodeProto", &Config{ - BalancerName: "trafficdirector.googleapis.com:443", - NodeProto: &v2corepb.Node{ - BuildVersion: gRPCVersion, - UserAgentName: gRPCUserAgentName, - UserAgentVersionType: &v2corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version}, - ClientFeatures: []string{clientFeatureNoOverprovisioning}, - }, - }, - }, - {"unknownTopLevelFieldInFile", nilCredsConfigV2}, - {"unknownFieldInNodeProto", nilCredsConfigV2}, - {"unknownFieldInXdsServer", nilCredsConfigV2}, - {"emptyChannelCreds", nilCredsConfigV2}, - {"nonGoogleDefaultCreds", nilCredsConfigV2}, - {"multipleChannelCreds", nonNilCredsConfigV2}, - {"goodBootstrap", nonNilCredsConfigV2}, - {"multipleXDSServers", nonNilCredsConfigV2}, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - if err := os.Setenv(bootstrapFileEnv, test.name); err != nil { - t.Fatalf("os.Setenv(%s, %s) failed with error: %v", bootstrapFileEnv, test.name, err) - } - c, err := NewConfig() - if err != nil { - t.Fatalf("NewConfig() failed: %v", err) - } - if err := c.compare(test.wantConfig); err != nil { - t.Fatal(err) - } - }) - } -} - -// TestNewConfigV3SupportNotEnabledOnClient verifies bootstrap functionality -// when the GRPC_XDS_EXPERIMENTAL_V3_SUPPORT environment variable is not enabled -// on the client. In this case, whether the server supports v3 or not, the -// client will end up using v2. -func TestNewConfigV3SupportNotEnabledOnClient(t *testing.T) { - if err := os.Setenv(v3SupportEnv, "false"); err != nil { - t.Fatalf("os.Setenv(%s, %s) failed with error: %v", v3SupportEnv, "true", err) - } - defer os.Unsetenv(v3SupportEnv) - - cancel := setupBootstrapOverride(v3BootstrapFileMap) - defer cancel() - - tests := []struct { - name string - wantConfig *Config - }{ - {"serverDoesNotSupportsV3", nonNilCredsConfigV2}, - {"serverSupportsV3", nonNilCredsConfigV2}, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - if err := os.Setenv(bootstrapFileEnv, test.name); err != nil { - t.Fatalf("os.Setenv(%s, %s) failed with error: %v", bootstrapFileEnv, test.name, err) - } - c, err := NewConfig() - if err != nil { - t.Fatalf("NewConfig() failed: %v", err) - } - if err := c.compare(test.wantConfig); err != nil { - t.Fatal(err) - } - }) - } -} - -// TestNewConfigV3SupportEnabledOnClient verifies bootstrap functionality when -// the GRPC_XDS_EXPERIMENTAL_V3_SUPPORT environment variable is enabled on the -// client. Here the client ends up using v2 or v3 based on what the server -// supports. -func TestNewConfigV3SupportEnabledOnClient(t *testing.T) { - if err := os.Setenv(v3SupportEnv, "true"); err != nil { - t.Fatalf("os.Setenv(%s, %s) failed with error: %v", v3SupportEnv, "true", err) - } - defer os.Unsetenv(v3SupportEnv) - - cancel := setupBootstrapOverride(v3BootstrapFileMap) - defer cancel() - - tests := []struct { - name string - wantConfig *Config - }{ - {"serverDoesNotSupportsV3", nonNilCredsConfigV2}, - {"serverSupportsV3", nonNilCredsConfigV3}, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - if err := os.Setenv(bootstrapFileEnv, test.name); err != nil { - t.Fatalf("os.Setenv(%s, %s) failed with error: %v", bootstrapFileEnv, test.name, err) - } - c, err := NewConfig() - if err != nil { - t.Fatalf("NewConfig() failed: %v", err) - } - if err := c.compare(test.wantConfig); err != nil { - t.Fatal(err) - } - }) - } -} - -// TestNewConfigBootstrapFileEnvNotSet tests the case where the bootstrap file -// environment variable is not set. -func TestNewConfigBootstrapFileEnvNotSet(t *testing.T) { - os.Unsetenv(bootstrapFileEnv) - if _, err := NewConfig(); err == nil { - t.Errorf("NewConfig() returned nil error, expected to fail") - } -} diff --git a/xds/internal/client/client.go b/xds/internal/client/client.go deleted file mode 100644 index 2f260643777b..000000000000 --- a/xds/internal/client/client.go +++ /dev/null @@ -1,460 +0,0 @@ -/* - * - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -// Package client implementation a full fledged gRPC client for the xDS API -// used by the xds resolver and balancer implementations. -package client - -import ( - "errors" - "fmt" - "sync" - "time" - - v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" - v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" - "github.com/golang/protobuf/proto" - - "google.golang.org/grpc" - "google.golang.org/grpc/internal/backoff" - "google.golang.org/grpc/internal/buffer" - "google.golang.org/grpc/internal/grpclog" - "google.golang.org/grpc/internal/grpcsync" - "google.golang.org/grpc/keepalive" - "google.golang.org/grpc/xds/internal" - "google.golang.org/grpc/xds/internal/client/bootstrap" - "google.golang.org/grpc/xds/internal/version" -) - -var ( - m = make(map[version.TransportAPI]APIClientBuilder) -) - -// RegisterAPIClientBuilder registers a client builder for xDS transport protocol -// version specified by b.Version(). -// -// NOTE: this function must only be called during initialization time (i.e. in -// an init() function), and is not thread-safe. If multiple builders are -// registered for the same version, the one registered last will take effect. -func RegisterAPIClientBuilder(b APIClientBuilder) { - m[b.Version()] = b -} - -// getAPIClientBuilder returns the client builder registered for the provided -// xDS transport API version. -func getAPIClientBuilder(version version.TransportAPI) APIClientBuilder { - if b, ok := m[version]; ok { - return b - } - return nil -} - -// BuildOptions contains options to be passed to client builders. -type BuildOptions struct { - // Parent is a top-level xDS client or server which has the intelligence to - // take appropriate action based on xDS responses received from the - // management server. - Parent UpdateHandler - // NodeProto contains the Node proto to be used in xDS requests. The actual - // type depends on the transport protocol version used. - NodeProto proto.Message - // Backoff returns the amount of time to backoff before retrying broken - // streams. - Backoff func(int) time.Duration - // Logger provides enhanced logging capabilities. - Logger *grpclog.PrefixLogger -} - -// APIClientBuilder creates an xDS client for a specific xDS transport protocol -// version. -type APIClientBuilder interface { - // Build builds a transport protocol specific implementation of the xDS - // client based on the provided clientConn to the management server and the - // provided options. - Build(*grpc.ClientConn, BuildOptions) (APIClient, error) - // Version returns the xDS transport protocol version used by clients build - // using this builder. - Version() version.TransportAPI -} - -// APIClient represents the functionality provided by transport protocol -// version specific implementations of the xDS client. -type APIClient interface { - // AddWatch adds a watch for an xDS resource given its type and name. - AddWatch(ResourceType, string) - // RemoveWatch cancels an already registered watch for an xDS resource - // given its type and name. - RemoveWatch(ResourceType, string) - // Close cleans up resources allocated by the API client. - Close() -} - -// UpdateHandler receives and processes (by taking appropriate actions) xDS -// resource updates from an APIClient for a specific version. -type UpdateHandler interface { - // NewListeners handles updates to xDS listener resources. - NewListeners(map[string]ListenerUpdate) - // NewRouteConfigs handles updates to xDS RouteConfiguration resources. - NewRouteConfigs(map[string]RouteConfigUpdate) - // NewClusters handles updates to xDS Cluster resources. - NewClusters(map[string]ClusterUpdate) - // NewEndpoints handles updates to xDS ClusterLoadAssignment (or tersely - // referred to as Endpoints) resources. - NewEndpoints(map[string]EndpointsUpdate) -} - -// ListenerUpdate contains information received in an LDS response, which is of -// interest to the registered LDS watcher. -type ListenerUpdate struct { - // RouteConfigName is the route configuration name corresponding to the - // target which is being watched through LDS. - RouteConfigName string -} - -// RouteConfigUpdate contains information received in an RDS response, which is -// of interest to the registered RDS watcher. -type RouteConfigUpdate struct { - // Routes contains a list of routes, each containing matchers and - // corresponding action. - Routes []*Route -} - -// Route is both a specification of how to match a request as well as an -// indication of the action to take upon match. -type Route struct { - Path, Prefix, Regex *string - Headers []*HeaderMatcher - Fraction *uint32 - Action map[string]uint32 // action is weighted clusters. -} - -// HeaderMatcher represents header matchers. -type HeaderMatcher struct { - Name string `json:"name"` - InvertMatch *bool `json:"invertMatch,omitempty"` - ExactMatch *string `json:"exactMatch,omitempty"` - RegexMatch *string `json:"regexMatch,omitempty"` - PrefixMatch *string `json:"prefixMatch,omitempty"` - SuffixMatch *string `json:"suffixMatch,omitempty"` - RangeMatch *Int64Range `json:"rangeMatch,omitempty"` - PresentMatch *bool `json:"presentMatch,omitempty"` -} - -// Int64Range is a range for header range match. -type Int64Range struct { - Start int64 `json:"start"` - End int64 `json:"end"` -} - -// ServiceUpdate contains information received from LDS and RDS responses, -// which is of interest to the registered service watcher. -type ServiceUpdate struct { - // Routes contain matchers+actions to route RPCs. - Routes []*Route -} - -// ClusterUpdate contains information from a received CDS response, which is of -// interest to the registered CDS watcher. -type ClusterUpdate struct { - // ServiceName is the service name corresponding to the clusterName which - // is being watched for through CDS. - ServiceName string - // EnableLRS indicates whether or not load should be reported through LRS. - EnableLRS bool -} - -// OverloadDropConfig contains the config to drop overloads. -type OverloadDropConfig struct { - Category string - Numerator uint32 - Denominator uint32 -} - -// EndpointHealthStatus represents the health status of an endpoint. -type EndpointHealthStatus int32 - -const ( - // EndpointHealthStatusUnknown represents HealthStatus UNKNOWN. - EndpointHealthStatusUnknown EndpointHealthStatus = iota - // EndpointHealthStatusHealthy represents HealthStatus HEALTHY. - EndpointHealthStatusHealthy - // EndpointHealthStatusUnhealthy represents HealthStatus UNHEALTHY. - EndpointHealthStatusUnhealthy - // EndpointHealthStatusDraining represents HealthStatus DRAINING. - EndpointHealthStatusDraining - // EndpointHealthStatusTimeout represents HealthStatus TIMEOUT. - EndpointHealthStatusTimeout - // EndpointHealthStatusDegraded represents HealthStatus DEGRADED. - EndpointHealthStatusDegraded -) - -// Endpoint contains information of an endpoint. -type Endpoint struct { - Address string - HealthStatus EndpointHealthStatus - Weight uint32 -} - -// Locality contains information of a locality. -type Locality struct { - Endpoints []Endpoint - ID internal.LocalityID - Priority uint32 - Weight uint32 -} - -// EndpointsUpdate contains an EDS update. -type EndpointsUpdate struct { - Drops []OverloadDropConfig - Localities []Locality -} - -// Options provides all parameters required for the creation of an xDS client. -type Options struct { - // Config contains a fully populated bootstrap config. It is the - // responsibility of the caller to use some sane defaults here if the - // bootstrap process returned with certain fields left unspecified. - Config bootstrap.Config - // DialOpts contains dial options to be used when dialing the xDS server. - DialOpts []grpc.DialOption - // TargetName is the target of the parent ClientConn. - TargetName string - // WatchExpiryTimeout is the amount of time the client is willing to wait - // for the first response from the server for any resource being watched. - // Expiry will not cause cancellation of the watch. It will only trigger the - // invocation of the registered callback and it is left up to the caller to - // decide whether or not they want to cancel the watch. - // - // If this field is left unspecified, a default value of 15 seconds will be - // used. This is based on the default value of the initial_fetch_timeout - // field in corepb.ConfigSource proto. - WatchExpiryTimeout time.Duration -} - -// Function to be overridden in tests. -var newAPIClient = func(apiVersion version.TransportAPI, cc *grpc.ClientConn, opts BuildOptions) (APIClient, error) { - cb := getAPIClientBuilder(apiVersion) - if cb == nil { - return nil, fmt.Errorf("no client builder for xDS API version: %v", apiVersion) - } - return cb.Build(cc, opts) -} - -// Client is a full fledged gRPC client which queries a set of discovery APIs -// (collectively termed as xDS) on a remote management server, to discover -// various dynamic resources. -// -// A single client object will be shared by the xds resolver and balancer -// implementations. But the same client can only be shared by the same parent -// ClientConn. -// -// Implements UpdateHandler interface. -// TODO(easwars): Make a wrapper struct which implements this interface in the -// style of ccBalancerWrapper so that the Client type does not implement these -// exported methods. -type Client struct { - done *grpcsync.Event - opts Options - cc *grpc.ClientConn // Connection to the xDS server - apiClient APIClient - - logger *grpclog.PrefixLogger - - updateCh *buffer.Unbounded // chan *watcherInfoWithUpdate - mu sync.Mutex - ldsWatchers map[string]map[*watchInfo]bool - ldsCache map[string]ListenerUpdate - rdsWatchers map[string]map[*watchInfo]bool - rdsCache map[string]RouteConfigUpdate - cdsWatchers map[string]map[*watchInfo]bool - cdsCache map[string]ClusterUpdate - edsWatchers map[string]map[*watchInfo]bool - edsCache map[string]EndpointsUpdate -} - -// New returns a new xdsClient configured with opts. -func New(opts Options) (*Client, error) { - switch { - case opts.Config.BalancerName == "": - return nil, errors.New("xds: no xds_server name provided in options") - case opts.Config.Creds == nil: - return nil, errors.New("xds: no credentials provided in options") - case opts.Config.NodeProto == nil: - return nil, errors.New("xds: no node_proto provided in options") - } - - switch opts.Config.TransportAPI { - case version.TransportV2: - if _, ok := opts.Config.NodeProto.(*v2corepb.Node); !ok { - return nil, fmt.Errorf("xds: Node proto type (%T) does not match API version: %v", opts.Config.NodeProto, opts.Config.TransportAPI) - } - case version.TransportV3: - if _, ok := opts.Config.NodeProto.(*v3corepb.Node); !ok { - return nil, fmt.Errorf("xds: Node proto type (%T) does not match API version: %v", opts.Config.NodeProto, opts.Config.TransportAPI) - } - } - - dopts := []grpc.DialOption{ - opts.Config.Creds, - grpc.WithKeepaliveParams(keepalive.ClientParameters{ - Time: 5 * time.Minute, - Timeout: 20 * time.Second, - }), - } - dopts = append(dopts, opts.DialOpts...) - - if opts.WatchExpiryTimeout == 0 { - // This is based on the default value of the initial_fetch_timeout field - // in corepb.ConfigSource proto. - opts.WatchExpiryTimeout = 15 * time.Second - } - - c := &Client{ - done: grpcsync.NewEvent(), - opts: opts, - - updateCh: buffer.NewUnbounded(), - ldsWatchers: make(map[string]map[*watchInfo]bool), - ldsCache: make(map[string]ListenerUpdate), - rdsWatchers: make(map[string]map[*watchInfo]bool), - rdsCache: make(map[string]RouteConfigUpdate), - cdsWatchers: make(map[string]map[*watchInfo]bool), - cdsCache: make(map[string]ClusterUpdate), - edsWatchers: make(map[string]map[*watchInfo]bool), - edsCache: make(map[string]EndpointsUpdate), - } - - cc, err := grpc.Dial(opts.Config.BalancerName, dopts...) - if err != nil { - // An error from a non-blocking dial indicates something serious. - return nil, fmt.Errorf("xds: failed to dial balancer {%s}: %v", opts.Config.BalancerName, err) - } - c.cc = cc - c.logger = prefixLogger((c)) - c.logger.Infof("Created ClientConn to xDS server: %s", opts.Config.BalancerName) - - apiClient, err := newAPIClient(opts.Config.TransportAPI, cc, BuildOptions{ - Parent: c, - NodeProto: opts.Config.NodeProto, - Backoff: backoff.DefaultExponential.Backoff, - Logger: c.logger, - }) - if err != nil { - return nil, err - } - c.apiClient = apiClient - c.logger.Infof("Created") - go c.run() - return c, nil -} - -// run is a goroutine for all the callbacks. -// -// Callback can be called in watch(), if an item is found in cache. Without this -// goroutine, the callback will be called inline, which might cause a deadlock -// in user's code. Callbacks also cannot be simple `go callback()` because the -// order matters. -func (c *Client) run() { - for { - select { - case t := <-c.updateCh.Get(): - c.updateCh.Load() - if c.done.HasFired() { - return - } - c.callCallback(t.(*watcherInfoWithUpdate)) - case <-c.done.Done(): - return - } - } -} - -// Close closes the gRPC connection to the xDS server. -func (c *Client) Close() { - if c.done.HasFired() { - return - } - c.done.Fire() - // TODO: Should we invoke the registered callbacks here with an error that - // the client is closed? - c.apiClient.Close() - c.cc.Close() - c.logger.Infof("Shutdown") -} - -// ResourceType identifies resources in a transport protocol agnostic way. These -// will be used in transport version agnostic code, while the versioned API -// clients will map these to appropriate version URLs. -type ResourceType int - -// Version agnostic resource type constants. -const ( - UnknownResource ResourceType = iota - ListenerResource - HTTPConnManagerResource - RouteConfigResource - ClusterResource - EndpointsResource -) - -func (r ResourceType) String() string { - switch r { - case ListenerResource: - return "ListenerResource" - case HTTPConnManagerResource: - return "HTTPConnManagerResource" - case RouteConfigResource: - return "RouteConfigResource" - case ClusterResource: - return "ClusterResource" - case EndpointsResource: - return "EndpointsResource" - default: - return "UnknownResource" - } -} - -// IsListenerResource returns true if the provider URL corresponds to an xDS -// Listener resource. -func IsListenerResource(url string) bool { - return url == version.V2ListenerURL || url == version.V3ListenerURL -} - -// IsHTTPConnManagerResource returns true if the provider URL corresponds to an xDS -// HTTPConnManager resource. -func IsHTTPConnManagerResource(url string) bool { - return url == version.V2HTTPConnManagerURL || url == version.V3HTTPConnManagerURL -} - -// IsRouteConfigResource returns true if the provider URL corresponds to an xDS -// RouteConfig resource. -func IsRouteConfigResource(url string) bool { - return url == version.V2RouteConfigURL || url == version.V3RouteConfigURL -} - -// IsClusterResource returns true if the provider URL corresponds to an xDS -// Cluster resource. -func IsClusterResource(url string) bool { - return url == version.V2ClusterURL || url == version.V3ClusterURL -} - -// IsEndpointsResource returns true if the provider URL corresponds to an xDS -// Endpoints resource. -func IsEndpointsResource(url string) bool { - return url == version.V2EndpointsURL || url == version.V3EndpointsURL -} diff --git a/xds/internal/client/client_callback.go b/xds/internal/client/client_callback.go deleted file mode 100644 index a135dae745b9..000000000000 --- a/xds/internal/client/client_callback.go +++ /dev/null @@ -1,182 +0,0 @@ -/* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package client - -type watcherInfoWithUpdate struct { - wi *watchInfo - update interface{} - err error -} - -// scheduleCallback should only be called by methods of watchInfo, which checks -// for watcher states and maintain consistency. -func (c *Client) scheduleCallback(wi *watchInfo, update interface{}, err error) { - c.updateCh.Put(&watcherInfoWithUpdate{ - wi: wi, - update: update, - err: err, - }) -} - -func (c *Client) callCallback(wiu *watcherInfoWithUpdate) { - c.mu.Lock() - // Use a closure to capture the callback and type assertion, to save one - // more switch case. - // - // The callback must be called without c.mu. Otherwise if the callback calls - // another watch() inline, it will cause a deadlock. This leaves a small - // window that a watcher's callback could be called after the watcher is - // canceled, and the user needs to take care of it. - var ccb func() - switch wiu.wi.rType { - case ListenerResource: - if s, ok := c.ldsWatchers[wiu.wi.target]; ok && s[wiu.wi] { - ccb = func() { wiu.wi.ldsCallback(wiu.update.(ListenerUpdate), wiu.err) } - } - case RouteConfigResource: - if s, ok := c.rdsWatchers[wiu.wi.target]; ok && s[wiu.wi] { - ccb = func() { wiu.wi.rdsCallback(wiu.update.(RouteConfigUpdate), wiu.err) } - } - case ClusterResource: - if s, ok := c.cdsWatchers[wiu.wi.target]; ok && s[wiu.wi] { - ccb = func() { wiu.wi.cdsCallback(wiu.update.(ClusterUpdate), wiu.err) } - } - case EndpointsResource: - if s, ok := c.edsWatchers[wiu.wi.target]; ok && s[wiu.wi] { - ccb = func() { wiu.wi.edsCallback(wiu.update.(EndpointsUpdate), wiu.err) } - } - } - c.mu.Unlock() - - if ccb != nil { - ccb() - } -} - -// NewListeners is called by the underlying xdsAPIClient when it receives an -// xDS response. -// -// A response can contain multiple resources. They will be parsed and put in a -// map from resource name to the resource content. -func (c *Client) NewListeners(updates map[string]ListenerUpdate) { - c.mu.Lock() - defer c.mu.Unlock() - - for name, update := range updates { - if s, ok := c.ldsWatchers[name]; ok { - for wi := range s { - wi.newUpdate(update) - } - // Sync cache. - c.logger.Debugf("LDS resource with name %v, value %+v added to cache", name, update) - c.ldsCache[name] = update - } - } - for name := range c.ldsCache { - if _, ok := updates[name]; !ok { - // If resource exists in cache, but not in the new update, delete it - // from cache, and also send an resource not found error to indicate - // resource removed. - delete(c.ldsCache, name) - for wi := range c.ldsWatchers[name] { - wi.resourceNotFound() - } - } - } - // When LDS resource is removed, we don't delete corresponding RDS cached - // data. The RDS watch will be canceled, and cache entry is removed when the - // last watch is canceled. -} - -// NewRouteConfigs is called by the underlying xdsAPIClient when it receives an -// xDS response. -// -// A response can contain multiple resources. They will be parsed and put in a -// map from resource name to the resource content. -func (c *Client) NewRouteConfigs(updates map[string]RouteConfigUpdate) { - c.mu.Lock() - defer c.mu.Unlock() - - for name, update := range updates { - if s, ok := c.rdsWatchers[name]; ok { - for wi := range s { - wi.newUpdate(update) - } - // Sync cache. - c.logger.Debugf("RDS resource with name %v, value %+v added to cache", name, update) - c.rdsCache[name] = update - } - } -} - -// NewClusters is called by the underlying xdsAPIClient when it receives an xDS -// response. -// -// A response can contain multiple resources. They will be parsed and put in a -// map from resource name to the resource content. -func (c *Client) NewClusters(updates map[string]ClusterUpdate) { - c.mu.Lock() - defer c.mu.Unlock() - - for name, update := range updates { - if s, ok := c.cdsWatchers[name]; ok { - for wi := range s { - wi.newUpdate(update) - } - // Sync cache. - c.logger.Debugf("CDS resource with name %v, value %+v added to cache", name, update) - c.cdsCache[name] = update - } - } - for name := range c.cdsCache { - if _, ok := updates[name]; !ok { - // If resource exists in cache, but not in the new update, delete it - // from cache, and also send an resource not found error to indicate - // resource removed. - delete(c.cdsCache, name) - for wi := range c.cdsWatchers[name] { - wi.resourceNotFound() - } - } - } - // When CDS resource is removed, we don't delete corresponding EDS cached - // data. The EDS watch will be canceled, and cache entry is removed when the - // last watch is canceled. -} - -// NewEndpoints is called by the underlying xdsAPIClient when it receives an -// xDS response. -// -// A response can contain multiple resources. They will be parsed and put in a -// map from resource name to the resource content. -func (c *Client) NewEndpoints(updates map[string]EndpointsUpdate) { - c.mu.Lock() - defer c.mu.Unlock() - - for name, update := range updates { - if s, ok := c.edsWatchers[name]; ok { - for wi := range s { - wi.newUpdate(update) - } - // Sync cache. - c.logger.Debugf("EDS resource with name %v, value %+v added to cache", name, update) - c.edsCache[name] = update - } - } -} diff --git a/xds/internal/client/client_cds_test.go b/xds/internal/client/client_cds_test.go deleted file mode 100644 index d45baa0e823f..000000000000 --- a/xds/internal/client/client_cds_test.go +++ /dev/null @@ -1,312 +0,0 @@ -/* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package client - -import ( - "testing" - - v2xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2" - v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" - v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" - v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" - "github.com/golang/protobuf/proto" - anypb "github.com/golang/protobuf/ptypes/any" - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" - "google.golang.org/grpc/xds/internal/version" -) - -func (s) TestValidateCluster(t *testing.T) { - const ( - clusterName = "clusterName" - serviceName = "service" - ) - var ( - emptyUpdate = ClusterUpdate{ServiceName: "", EnableLRS: false} - ) - - tests := []struct { - name string - cluster *v3clusterpb.Cluster - wantUpdate ClusterUpdate - wantErr bool - }{ - { - name: "non-eds-cluster-type", - cluster: &v3clusterpb.Cluster{ - ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_STATIC}, - EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ - EdsConfig: &v3corepb.ConfigSource{ - ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ - Ads: &v3corepb.AggregatedConfigSource{}, - }, - }, - }, - LbPolicy: v3clusterpb.Cluster_LEAST_REQUEST, - }, - wantUpdate: emptyUpdate, - wantErr: true, - }, - { - name: "no-eds-config", - cluster: &v3clusterpb.Cluster{ - ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, - LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, - }, - wantUpdate: emptyUpdate, - wantErr: true, - }, - { - name: "no-ads-config-source", - cluster: &v3clusterpb.Cluster{ - ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, - EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{}, - LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, - }, - wantUpdate: emptyUpdate, - wantErr: true, - }, - { - name: "non-round-robin-lb-policy", - cluster: &v3clusterpb.Cluster{ - ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, - EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ - EdsConfig: &v3corepb.ConfigSource{ - ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ - Ads: &v3corepb.AggregatedConfigSource{}, - }, - }, - }, - LbPolicy: v3clusterpb.Cluster_LEAST_REQUEST, - }, - wantUpdate: emptyUpdate, - wantErr: true, - }, - { - name: "happy-case-no-service-name-no-lrs", - cluster: &v3clusterpb.Cluster{ - ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, - EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ - EdsConfig: &v3corepb.ConfigSource{ - ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ - Ads: &v3corepb.AggregatedConfigSource{}, - }, - }, - }, - LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, - }, - wantUpdate: emptyUpdate, - }, - { - name: "happy-case-no-lrs", - cluster: &v3clusterpb.Cluster{ - ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, - EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ - EdsConfig: &v3corepb.ConfigSource{ - ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ - Ads: &v3corepb.AggregatedConfigSource{}, - }, - }, - ServiceName: serviceName, - }, - LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, - }, - wantUpdate: ClusterUpdate{ServiceName: serviceName, EnableLRS: false}, - }, - { - name: "happiest-case", - cluster: &v3clusterpb.Cluster{ - Name: clusterName, - ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, - EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ - EdsConfig: &v3corepb.ConfigSource{ - ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ - Ads: &v3corepb.AggregatedConfigSource{}, - }, - }, - ServiceName: serviceName, - }, - LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, - LrsServer: &v3corepb.ConfigSource{ - ConfigSourceSpecifier: &v3corepb.ConfigSource_Self{ - Self: &v3corepb.SelfConfigSource{}, - }, - }, - }, - wantUpdate: ClusterUpdate{ServiceName: serviceName, EnableLRS: true}, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - update, err := validateCluster(test.cluster) - if ((err != nil) != test.wantErr) || !cmp.Equal(update, test.wantUpdate, cmpopts.EquateEmpty()) { - t.Errorf("validateCluster(%+v) = (%v, %v), wantErr: (%v, %v)", test.cluster, update, err, test.wantUpdate, test.wantErr) - } - }) - } -} - -func (s) TestUnmarshalCluster(t *testing.T) { - const ( - v2ClusterName = "v2clusterName" - v3ClusterName = "v3clusterName" - v2Service = "v2Service" - v3Service = "v2Service" - ) - var ( - v2Cluster = &v2xdspb.Cluster{ - Name: v2ClusterName, - ClusterDiscoveryType: &v2xdspb.Cluster_Type{Type: v2xdspb.Cluster_EDS}, - EdsClusterConfig: &v2xdspb.Cluster_EdsClusterConfig{ - EdsConfig: &v2corepb.ConfigSource{ - ConfigSourceSpecifier: &v2corepb.ConfigSource_Ads{ - Ads: &v2corepb.AggregatedConfigSource{}, - }, - }, - ServiceName: v2Service, - }, - LbPolicy: v2xdspb.Cluster_ROUND_ROBIN, - LrsServer: &v2corepb.ConfigSource{ - ConfigSourceSpecifier: &v2corepb.ConfigSource_Self{ - Self: &v2corepb.SelfConfigSource{}, - }, - }, - } - - v3Cluster = &v3clusterpb.Cluster{ - Name: v3ClusterName, - ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, - EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ - EdsConfig: &v3corepb.ConfigSource{ - ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ - Ads: &v3corepb.AggregatedConfigSource{}, - }, - }, - ServiceName: v3Service, - }, - LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, - LrsServer: &v3corepb.ConfigSource{ - ConfigSourceSpecifier: &v3corepb.ConfigSource_Self{ - Self: &v3corepb.SelfConfigSource{}, - }, - }, - } - ) - - tests := []struct { - name string - resources []*anypb.Any - wantUpdate map[string]ClusterUpdate - wantErr bool - }{ - { - name: "non-cluster resource type", - resources: []*anypb.Any{{TypeUrl: version.V3HTTPConnManagerURL}}, - wantErr: true, - }, - { - name: "badly marshaled cluster resource", - resources: []*anypb.Any{ - { - TypeUrl: version.V3ClusterURL, - Value: []byte{1, 2, 3, 4}, - }, - }, - wantErr: true, - }, - { - name: "bad cluster resource", - resources: []*anypb.Any{ - { - TypeUrl: version.V3ClusterURL, - Value: func() []byte { - cl := &v3clusterpb.Cluster{ - ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_STATIC}, - } - mcl, _ := proto.Marshal(cl) - return mcl - }(), - }, - }, - wantErr: true, - }, - { - name: "v2 cluster", - resources: []*anypb.Any{ - { - TypeUrl: version.V3ClusterURL, - Value: func() []byte { - mcl, _ := proto.Marshal(v2Cluster) - return mcl - }(), - }, - }, - wantUpdate: map[string]ClusterUpdate{ - v2ClusterName: {ServiceName: v2Service, EnableLRS: true}, - }, - }, - { - name: "v3 cluster", - resources: []*anypb.Any{ - { - TypeUrl: version.V3ClusterURL, - Value: func() []byte { - mcl, _ := proto.Marshal(v3Cluster) - return mcl - }(), - }, - }, - wantUpdate: map[string]ClusterUpdate{ - v3ClusterName: {ServiceName: v3Service, EnableLRS: true}, - }, - }, - { - name: "multiple clusters", - resources: []*anypb.Any{ - { - TypeUrl: version.V3ClusterURL, - Value: func() []byte { - mcl, _ := proto.Marshal(v2Cluster) - return mcl - }(), - }, - { - TypeUrl: version.V3ClusterURL, - Value: func() []byte { - mcl, _ := proto.Marshal(v3Cluster) - return mcl - }(), - }, - }, - wantUpdate: map[string]ClusterUpdate{ - v2ClusterName: {ServiceName: v2Service, EnableLRS: true}, - v3ClusterName: {ServiceName: v3Service, EnableLRS: true}, - }, - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - update, err := UnmarshalCluster(test.resources, nil) - if ((err != nil) != test.wantErr) || !cmp.Equal(update, test.wantUpdate, cmpopts.EquateEmpty()) { - t.Errorf("UnmarshalCluster(%v) = (%+v, %v) want (%+v, %v)", test.resources, update, err, test.wantUpdate, test.wantErr) - } - }) - } -} diff --git a/xds/internal/client/client_eds_test.go b/xds/internal/client/client_eds_test.go deleted file mode 100644 index 01df225083b5..000000000000 --- a/xds/internal/client/client_eds_test.go +++ /dev/null @@ -1,312 +0,0 @@ -/* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package client - -import ( - "fmt" - "net" - "strconv" - "testing" - - v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" - v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" - v3typepb "github.com/envoyproxy/go-control-plane/envoy/type/v3" - "github.com/golang/protobuf/proto" - anypb "github.com/golang/protobuf/ptypes/any" - wrapperspb "github.com/golang/protobuf/ptypes/wrappers" - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" - "google.golang.org/grpc/xds/internal" - "google.golang.org/grpc/xds/internal/version" -) - -func (s) TestEDSParseRespProto(t *testing.T) { - tests := []struct { - name string - m *v3endpointpb.ClusterLoadAssignment - want EndpointsUpdate - wantErr bool - }{ - { - name: "missing-priority", - m: func() *v3endpointpb.ClusterLoadAssignment { - clab0 := newClaBuilder("test", nil) - clab0.addLocality("locality-1", 1, 0, []string{"addr1:314"}, nil) - clab0.addLocality("locality-2", 1, 2, []string{"addr2:159"}, nil) - return clab0.Build() - }(), - want: EndpointsUpdate{}, - wantErr: true, - }, - { - name: "missing-locality-ID", - m: func() *v3endpointpb.ClusterLoadAssignment { - clab0 := newClaBuilder("test", nil) - clab0.addLocality("", 1, 0, []string{"addr1:314"}, nil) - return clab0.Build() - }(), - want: EndpointsUpdate{}, - wantErr: true, - }, - { - name: "good", - m: func() *v3endpointpb.ClusterLoadAssignment { - clab0 := newClaBuilder("test", nil) - clab0.addLocality("locality-1", 1, 1, []string{"addr1:314"}, &addLocalityOptions{ - Health: []v3corepb.HealthStatus{v3corepb.HealthStatus_UNHEALTHY}, - Weight: []uint32{271}, - }) - clab0.addLocality("locality-2", 1, 0, []string{"addr2:159"}, &addLocalityOptions{ - Health: []v3corepb.HealthStatus{v3corepb.HealthStatus_DRAINING}, - Weight: []uint32{828}, - }) - return clab0.Build() - }(), - want: EndpointsUpdate{ - Drops: nil, - Localities: []Locality{ - { - Endpoints: []Endpoint{{ - Address: "addr1:314", - HealthStatus: EndpointHealthStatusUnhealthy, - Weight: 271, - }}, - ID: internal.LocalityID{SubZone: "locality-1"}, - Priority: 1, - Weight: 1, - }, - { - Endpoints: []Endpoint{{ - Address: "addr2:159", - HealthStatus: EndpointHealthStatusDraining, - Weight: 828, - }}, - ID: internal.LocalityID{SubZone: "locality-2"}, - Priority: 0, - Weight: 1, - }, - }, - }, - wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := parseEDSRespProto(tt.m) - if (err != nil) != tt.wantErr { - t.Errorf("parseEDSRespProto() error = %v, wantErr %v", err, tt.wantErr) - return - } - if d := cmp.Diff(got, tt.want); d != "" { - t.Errorf("parseEDSRespProto() got = %v, want %v, diff: %v", got, tt.want, d) - } - }) - } -} - -func (s) TestUnmarshalEndpoints(t *testing.T) { - tests := []struct { - name string - resources []*anypb.Any - wantUpdate map[string]EndpointsUpdate - wantErr bool - }{ - { - name: "non-clusterLoadAssignment resource type", - resources: []*anypb.Any{{TypeUrl: version.V3HTTPConnManagerURL}}, - wantErr: true, - }, - { - name: "badly marshaled clusterLoadAssignment resource", - resources: []*anypb.Any{ - { - TypeUrl: version.V3EndpointsURL, - Value: []byte{1, 2, 3, 4}, - }, - }, - wantErr: true, - }, - { - name: "bad endpoints resource", - resources: []*anypb.Any{ - { - TypeUrl: version.V3EndpointsURL, - Value: func() []byte { - clab0 := newClaBuilder("test", nil) - clab0.addLocality("locality-1", 1, 0, []string{"addr1:314"}, nil) - clab0.addLocality("locality-2", 1, 2, []string{"addr2:159"}, nil) - e := clab0.Build() - me, _ := proto.Marshal(e) - return me - }(), - }, - }, - wantErr: true, - }, - { - name: "v3 endpoints", - resources: []*anypb.Any{ - { - TypeUrl: version.V3EndpointsURL, - Value: func() []byte { - clab0 := newClaBuilder("test", nil) - clab0.addLocality("locality-1", 1, 1, []string{"addr1:314"}, &addLocalityOptions{ - Health: []v3corepb.HealthStatus{v3corepb.HealthStatus_UNHEALTHY}, - Weight: []uint32{271}, - }) - clab0.addLocality("locality-2", 1, 0, []string{"addr2:159"}, &addLocalityOptions{ - Health: []v3corepb.HealthStatus{v3corepb.HealthStatus_DRAINING}, - Weight: []uint32{828}, - }) - e := clab0.Build() - me, _ := proto.Marshal(e) - return me - }(), - }, - }, - wantUpdate: map[string]EndpointsUpdate{ - "test": { - Drops: nil, - Localities: []Locality{ - { - Endpoints: []Endpoint{{ - Address: "addr1:314", - HealthStatus: EndpointHealthStatusUnhealthy, - Weight: 271, - }}, - ID: internal.LocalityID{SubZone: "locality-1"}, - Priority: 1, - Weight: 1, - }, - { - Endpoints: []Endpoint{{ - Address: "addr2:159", - HealthStatus: EndpointHealthStatusDraining, - Weight: 828, - }}, - ID: internal.LocalityID{SubZone: "locality-2"}, - Priority: 0, - Weight: 1, - }, - }, - }, - }, - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - update, err := UnmarshalEndpoints(test.resources, nil) - if ((err != nil) != test.wantErr) || !cmp.Equal(update, test.wantUpdate, cmpopts.EquateEmpty()) { - t.Errorf("UnmarshalEndpoints(%v) = (%+v, %v) want (%+v, %v)", test.resources, update, err, test.wantUpdate, test.wantErr) - } - }) - } -} - -// claBuilder builds a ClusterLoadAssignment, aka EDS -// response. -type claBuilder struct { - v *v3endpointpb.ClusterLoadAssignment -} - -// newClaBuilder creates a claBuilder. -func newClaBuilder(clusterName string, dropPercents []uint32) *claBuilder { - var drops []*v3endpointpb.ClusterLoadAssignment_Policy_DropOverload - for i, d := range dropPercents { - drops = append(drops, &v3endpointpb.ClusterLoadAssignment_Policy_DropOverload{ - Category: fmt.Sprintf("test-drop-%d", i), - DropPercentage: &v3typepb.FractionalPercent{ - Numerator: d, - Denominator: v3typepb.FractionalPercent_HUNDRED, - }, - }) - } - - return &claBuilder{ - v: &v3endpointpb.ClusterLoadAssignment{ - ClusterName: clusterName, - Policy: &v3endpointpb.ClusterLoadAssignment_Policy{ - DropOverloads: drops, - }, - }, - } -} - -// addLocalityOptions contains options when adding locality to the builder. -type addLocalityOptions struct { - Health []v3corepb.HealthStatus - Weight []uint32 -} - -// addLocality adds a locality to the builder. -func (clab *claBuilder) addLocality(subzone string, weight uint32, priority uint32, addrsWithPort []string, opts *addLocalityOptions) { - var lbEndPoints []*v3endpointpb.LbEndpoint - for i, a := range addrsWithPort { - host, portStr, err := net.SplitHostPort(a) - if err != nil { - panic("failed to split " + a) - } - port, err := strconv.Atoi(portStr) - if err != nil { - panic("failed to atoi " + portStr) - } - - lbe := &v3endpointpb.LbEndpoint{ - HostIdentifier: &v3endpointpb.LbEndpoint_Endpoint{ - Endpoint: &v3endpointpb.Endpoint{ - Address: &v3corepb.Address{ - Address: &v3corepb.Address_SocketAddress{ - SocketAddress: &v3corepb.SocketAddress{ - Protocol: v3corepb.SocketAddress_TCP, - Address: host, - PortSpecifier: &v3corepb.SocketAddress_PortValue{ - PortValue: uint32(port)}}}}}}, - } - if opts != nil { - if i < len(opts.Health) { - lbe.HealthStatus = opts.Health[i] - } - if i < len(opts.Weight) { - lbe.LoadBalancingWeight = &wrapperspb.UInt32Value{Value: opts.Weight[i]} - } - } - lbEndPoints = append(lbEndPoints, lbe) - } - - var localityID *v3corepb.Locality - if subzone != "" { - localityID = &v3corepb.Locality{ - Region: "", - Zone: "", - SubZone: subzone, - } - } - - clab.v.Endpoints = append(clab.v.Endpoints, &v3endpointpb.LocalityLbEndpoints{ - Locality: localityID, - LbEndpoints: lbEndPoints, - LoadBalancingWeight: &wrapperspb.UInt32Value{Value: weight}, - Priority: priority, - }) -} - -// Build builds ClusterLoadAssignment. -func (clab *claBuilder) Build() *v3endpointpb.ClusterLoadAssignment { - return clab.v -} diff --git a/xds/internal/client/client_lds_test.go b/xds/internal/client/client_lds_test.go deleted file mode 100644 index eedfe72f94e2..000000000000 --- a/xds/internal/client/client_lds_test.go +++ /dev/null @@ -1,171 +0,0 @@ -/* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package client - -import ( - "testing" - - v2xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2" - v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" - v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" - v2httppb "github.com/envoyproxy/go-control-plane/envoy/config/filter/network/http_connection_manager/v2" - v2listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v2" - v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" - v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" - "github.com/golang/protobuf/proto" - anypb "github.com/golang/protobuf/ptypes/any" - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" - "google.golang.org/grpc/xds/internal/version" -) - -func (s) TestUnmarshalListener(t *testing.T) { - const ( - v2LDSTarget = "lds.target.good:2222" - v3LDSTarget = "lds.target.good:3333" - v2RouteConfigName = "v2RouteConfig" - v3RouteConfigName = "v3RouteConfig" - ) - - var ( - v2Lis = &anypb.Any{ - TypeUrl: version.V2ListenerURL, - Value: func() []byte { - cm := &v2httppb.HttpConnectionManager{ - RouteSpecifier: &v2httppb.HttpConnectionManager_Rds{ - Rds: &v2httppb.Rds{ - ConfigSource: &v2corepb.ConfigSource{ - ConfigSourceSpecifier: &v2corepb.ConfigSource_Ads{Ads: &v2corepb.AggregatedConfigSource{}}, - }, - RouteConfigName: v2RouteConfigName, - }, - }, - } - mcm, _ := proto.Marshal(cm) - lis := &v2xdspb.Listener{ - Name: v2LDSTarget, - ApiListener: &v2listenerpb.ApiListener{ - ApiListener: &anypb.Any{ - TypeUrl: version.V2HTTPConnManagerURL, - Value: mcm, - }, - }, - } - mLis, _ := proto.Marshal(lis) - return mLis - }(), - } - v3Lis = &anypb.Any{ - TypeUrl: version.V3ListenerURL, - Value: func() []byte { - cm := &v3httppb.HttpConnectionManager{ - RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{ - Rds: &v3httppb.Rds{ - ConfigSource: &v3corepb.ConfigSource{ - ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}, - }, - RouteConfigName: v3RouteConfigName, - }, - }, - } - mcm, _ := proto.Marshal(cm) - lis := &v3listenerpb.Listener{ - Name: v3LDSTarget, - ApiListener: &v3listenerpb.ApiListener{ - ApiListener: &anypb.Any{ - TypeUrl: version.V3HTTPConnManagerURL, - Value: mcm, - }, - }, - } - mLis, _ := proto.Marshal(lis) - return mLis - }(), - } - ) - - tests := []struct { - name string - resources []*anypb.Any - wantUpdate map[string]ListenerUpdate - wantErr bool - }{ - { - name: "non-listener resource", - resources: []*anypb.Any{{TypeUrl: version.V3HTTPConnManagerURL}}, - wantErr: true, - }, - { - name: "badly marshaled listener resource", - resources: []*anypb.Any{ - { - TypeUrl: version.V3ListenerURL, - Value: func() []byte { - lis := &v3listenerpb.Listener{ - Name: v3LDSTarget, - ApiListener: &v3listenerpb.ApiListener{ - ApiListener: &anypb.Any{ - TypeUrl: version.V3HTTPConnManagerURL, - Value: []byte{1, 2, 3, 4}, - }, - }, - } - mLis, _ := proto.Marshal(lis) - return mLis - }(), - }, - }, - wantErr: true, - }, - { - name: "empty resource list", - }, - { - name: "v2 listener resource", - resources: []*anypb.Any{v2Lis}, - wantUpdate: map[string]ListenerUpdate{ - v2LDSTarget: {RouteConfigName: v2RouteConfigName}, - }, - }, - { - name: "v3 listener resource", - resources: []*anypb.Any{v3Lis}, - wantUpdate: map[string]ListenerUpdate{ - v3LDSTarget: {RouteConfigName: v3RouteConfigName}, - }, - }, - { - name: "multiple listener resources", - resources: []*anypb.Any{v2Lis, v3Lis}, - wantUpdate: map[string]ListenerUpdate{ - v2LDSTarget: {RouteConfigName: v2RouteConfigName}, - v3LDSTarget: {RouteConfigName: v3RouteConfigName}, - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - update, err := UnmarshalListener(test.resources, nil) - if ((err != nil) != test.wantErr) || !cmp.Equal(update, test.wantUpdate, cmpopts.EquateEmpty()) { - t.Errorf("UnmarshalListener(%v) = (%v, %v) want (%v, %v)", test.resources, update, err, test.wantUpdate, test.wantErr) - } - }) - } -} diff --git a/xds/internal/client/client_loadreport.go b/xds/internal/client/client_loadreport.go deleted file mode 100644 index 8539e973511c..000000000000 --- a/xds/internal/client/client_loadreport.go +++ /dev/null @@ -1,83 +0,0 @@ -/* - * - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package client - -import ( - "context" - - corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" - "github.com/golang/protobuf/proto" - structpb "github.com/golang/protobuf/ptypes/struct" - "google.golang.org/grpc" - "google.golang.org/grpc/xds/internal/balancer/lrs" -) - -const nodeMetadataHostnameKey = "PROXYLESS_CLIENT_HOSTNAME" - -// ReportLoad sends the load of the given clusterName from loadStore to the -// given server. If the server is not an empty string, and is different from the -// xds server, a new ClientConn will be created. -// -// The same options used for creating the Client will be used (including -// NodeProto, and dial options if necessary). -// -// It returns a function to cancel the load reporting stream. If server is -// different from xds server, the ClientConn will also be closed. -func (c *Client) ReportLoad(server string, clusterName string, loadStore lrs.Store) func() { - var ( - cc *grpc.ClientConn - closeCC bool - ) - c.logger.Infof("Starting load report to server: %s", server) - if server == "" || server == c.opts.Config.BalancerName { - cc = c.cc - } else { - c.logger.Infof("LRS server is different from xDS server, starting a new ClientConn") - dopts := append([]grpc.DialOption{c.opts.Config.Creds}, c.opts.DialOpts...) - ccNew, err := grpc.Dial(server, dopts...) - if err != nil { - // An error from a non-blocking dial indicates something serious. - c.logger.Infof("xds: failed to dial load report server {%s}: %v", server, err) - return func() {} - } - cc = ccNew - closeCC = true - } - ctx, cancel := context.WithCancel(context.Background()) - - nodeTemp := proto.Clone(c.opts.Config.NodeProto).(*corepb.Node) - if nodeTemp == nil { - nodeTemp = &corepb.Node{} - } - if nodeTemp.Metadata == nil { - nodeTemp.Metadata = &structpb.Struct{} - } - if nodeTemp.Metadata.Fields == nil { - nodeTemp.Metadata.Fields = make(map[string]*structpb.Value) - } - nodeTemp.Metadata.Fields[nodeMetadataHostnameKey] = &structpb.Value{ - Kind: &structpb.Value_StringValue{StringValue: c.opts.TargetName}, - } - go loadStore.ReportTo(ctx, c.cc, clusterName, nodeTemp) - return func() { - cancel() - if closeCC { - cc.Close() - } - } -} diff --git a/xds/internal/client/client_rds_test.go b/xds/internal/client/client_rds_test.go deleted file mode 100644 index 44e2cfc5dde8..000000000000 --- a/xds/internal/client/client_rds_test.go +++ /dev/null @@ -1,854 +0,0 @@ -/* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package client - -import ( - "testing" - - v2xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2" - v2routepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/route" - v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" - v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" - v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" - v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" - v3typepb "github.com/envoyproxy/go-control-plane/envoy/type/v3" - "github.com/golang/protobuf/proto" - anypb "github.com/golang/protobuf/ptypes/any" - wrapperspb "github.com/golang/protobuf/ptypes/wrappers" - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" - "google.golang.org/grpc/xds/internal/version" -) - -func (s) TestGetRouteConfigFromListener(t *testing.T) { - const ( - goodLDSTarget = "lds.target.good:1111" - goodRouteConfigName = "GoodRouteConfig" - ) - - tests := []struct { - name string - lis *v3listenerpb.Listener - wantRoute string - wantErr bool - }{ - { - name: "no-apiListener-field", - lis: &v3listenerpb.Listener{}, - wantRoute: "", - wantErr: true, - }, - { - name: "badly-marshaled-apiListener", - lis: &v3listenerpb.Listener{ - Name: goodLDSTarget, - ApiListener: &v3listenerpb.ApiListener{ - ApiListener: &anypb.Any{ - TypeUrl: version.V3HTTPConnManagerURL, - Value: []byte{1, 2, 3, 4}, - }, - }, - }, - wantRoute: "", - wantErr: true, - }, - { - name: "wrong-type-in-apiListener", - lis: &v3listenerpb.Listener{ - Name: goodLDSTarget, - ApiListener: &v3listenerpb.ApiListener{ - ApiListener: &anypb.Any{ - TypeUrl: version.V2ListenerURL, - Value: func() []byte { - cm := &v3httppb.HttpConnectionManager{ - RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{ - Rds: &v3httppb.Rds{ - ConfigSource: &v3corepb.ConfigSource{ - ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}, - }, - RouteConfigName: goodRouteConfigName}}} - mcm, _ := proto.Marshal(cm) - return mcm - }()}}}, - wantRoute: "", - wantErr: true, - }, - { - name: "empty-httpConnMgr-in-apiListener", - lis: &v3listenerpb.Listener{ - Name: goodLDSTarget, - ApiListener: &v3listenerpb.ApiListener{ - ApiListener: &anypb.Any{ - TypeUrl: version.V3HTTPConnManagerURL, - Value: func() []byte { - cm := &v3httppb.HttpConnectionManager{ - RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{ - Rds: &v3httppb.Rds{}, - }, - } - mcm, _ := proto.Marshal(cm) - return mcm - }()}}}, - wantRoute: "", - wantErr: true, - }, - { - name: "scopedRoutes-routeConfig-in-apiListener", - lis: &v3listenerpb.Listener{ - Name: goodLDSTarget, - ApiListener: &v3listenerpb.ApiListener{ - ApiListener: &anypb.Any{ - TypeUrl: version.V3HTTPConnManagerURL, - Value: func() []byte { - cm := &v3httppb.HttpConnectionManager{ - RouteSpecifier: &v3httppb.HttpConnectionManager_ScopedRoutes{}, - } - mcm, _ := proto.Marshal(cm) - return mcm - }()}}}, - wantRoute: "", - wantErr: true, - }, - { - name: "rds.ConfigSource-in-apiListener-is-not-ADS", - lis: &v3listenerpb.Listener{ - Name: goodLDSTarget, - ApiListener: &v3listenerpb.ApiListener{ - ApiListener: &anypb.Any{ - TypeUrl: version.V3HTTPConnManagerURL, - Value: func() []byte { - cm := &v3httppb.HttpConnectionManager{ - RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{ - Rds: &v3httppb.Rds{ - ConfigSource: &v3corepb.ConfigSource{ - ConfigSourceSpecifier: &v3corepb.ConfigSource_Path{ - Path: "/some/path", - }, - }, - RouteConfigName: goodRouteConfigName}}} - mcm, _ := proto.Marshal(cm) - return mcm - }()}}}, - wantRoute: "", - wantErr: true, - }, - { - name: "goodListener", - lis: &v3listenerpb.Listener{ - Name: goodLDSTarget, - ApiListener: &v3listenerpb.ApiListener{ - ApiListener: &anypb.Any{ - TypeUrl: version.V3HTTPConnManagerURL, - Value: func() []byte { - cm := &v3httppb.HttpConnectionManager{ - RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{ - Rds: &v3httppb.Rds{ - ConfigSource: &v3corepb.ConfigSource{ - ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}, - }, - RouteConfigName: goodRouteConfigName}}} - mcm, _ := proto.Marshal(cm) - return mcm - }()}}}, - wantRoute: goodRouteConfigName, - wantErr: false, - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - gotRoute, err := getRouteConfigNameFromListener(test.lis, nil) - if (err != nil) != test.wantErr || gotRoute != test.wantRoute { - t.Errorf("getRouteConfigNameFromListener(%+v) = (%s, %v), want (%s, %v)", test.lis, gotRoute, err, test.wantRoute, test.wantErr) - } - }) - } -} - -func (s) TestRDSGenerateRDSUpdateFromRouteConfiguration(t *testing.T) { - const ( - uninterestingDomain = "uninteresting.domain" - uninterestingClusterName = "uninterestingClusterName" - ldsTarget = "lds.target.good:1111" - routeName = "routeName" - clusterName = "clusterName" - ) - - tests := []struct { - name string - rc *v3routepb.RouteConfiguration - wantUpdate RouteConfigUpdate - wantError bool - }{ - { - name: "no-virtual-hosts-in-rc", - rc: &v3routepb.RouteConfiguration{}, - wantError: true, - }, - { - name: "no-domains-in-rc", - rc: &v3routepb.RouteConfiguration{ - VirtualHosts: []*v3routepb.VirtualHost{{}}, - }, - wantError: true, - }, - { - name: "non-matching-domain-in-rc", - rc: &v3routepb.RouteConfiguration{ - VirtualHosts: []*v3routepb.VirtualHost{ - {Domains: []string{uninterestingDomain}}, - }, - }, - wantError: true, - }, - { - name: "no-routes-in-rc", - rc: &v3routepb.RouteConfiguration{ - VirtualHosts: []*v3routepb.VirtualHost{ - {Domains: []string{ldsTarget}}, - }, - }, - wantError: true, - }, - { - name: "default-route-match-field-is-nil", - rc: &v3routepb.RouteConfiguration{ - VirtualHosts: []*v3routepb.VirtualHost{ - { - Domains: []string{ldsTarget}, - Routes: []*v3routepb.Route{ - { - Action: &v3routepb.Route_Route{ - Route: &v3routepb.RouteAction{ - ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName}, - }, - }, - }, - }, - }, - }, - }, - wantError: true, - }, - { - name: "default-route-match-field-is-non-nil", - rc: &v3routepb.RouteConfiguration{ - VirtualHosts: []*v3routepb.VirtualHost{ - { - Domains: []string{ldsTarget}, - Routes: []*v3routepb.Route{ - { - Match: &v3routepb.RouteMatch{}, - Action: &v3routepb.Route_Route{}, - }, - }, - }, - }, - }, - wantError: true, - }, - { - name: "default-route-routeaction-field-is-nil", - rc: &v3routepb.RouteConfiguration{ - VirtualHosts: []*v3routepb.VirtualHost{ - { - Domains: []string{ldsTarget}, - Routes: []*v3routepb.Route{{}}, - }, - }, - }, - wantError: true, - }, - { - name: "default-route-cluster-field-is-empty", - rc: &v3routepb.RouteConfiguration{ - VirtualHosts: []*v3routepb.VirtualHost{ - { - Domains: []string{ldsTarget}, - Routes: []*v3routepb.Route{ - { - Action: &v3routepb.Route_Route{ - Route: &v3routepb.RouteAction{ - ClusterSpecifier: &v3routepb.RouteAction_ClusterHeader{}, - }, - }, - }, - }, - }, - }, - }, - wantError: true, - }, - { - // default route's match sets case-sensitive to false. - name: "good-route-config-but-with-casesensitive-false", - rc: &v3routepb.RouteConfiguration{ - Name: routeName, - VirtualHosts: []*v3routepb.VirtualHost{{ - Domains: []string{ldsTarget}, - Routes: []*v3routepb.Route{{ - Match: &v3routepb.RouteMatch{ - PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}, - CaseSensitive: &wrapperspb.BoolValue{Value: false}, - }, - Action: &v3routepb.Route_Route{ - Route: &v3routepb.RouteAction{ - ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName}, - }}}}}}}, - wantError: true, - }, - { - name: "good-route-config-with-empty-string-route", - rc: &v3routepb.RouteConfiguration{ - Name: routeName, - VirtualHosts: []*v3routepb.VirtualHost{ - { - Domains: []string{uninterestingDomain}, - Routes: []*v3routepb.Route{ - { - Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: ""}}, - Action: &v3routepb.Route_Route{ - Route: &v3routepb.RouteAction{ - ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: uninterestingClusterName}, - }, - }, - }, - }, - }, - { - Domains: []string{ldsTarget}, - Routes: []*v3routepb.Route{ - { - Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: ""}}, - Action: &v3routepb.Route_Route{ - Route: &v3routepb.RouteAction{ - ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName}, - }, - }, - }, - }, - }, - }, - }, - wantUpdate: RouteConfigUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{clusterName: 1}}}}, - }, - { - // default route's match is not empty string, but "/". - name: "good-route-config-with-slash-string-route", - rc: &v3routepb.RouteConfiguration{ - Name: routeName, - VirtualHosts: []*v3routepb.VirtualHost{ - { - Domains: []string{ldsTarget}, - Routes: []*v3routepb.Route{ - { - Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}}, - Action: &v3routepb.Route_Route{ - Route: &v3routepb.RouteAction{ - ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName}, - }, - }, - }, - }, - }, - }, - }, - wantUpdate: RouteConfigUpdate{Routes: []*Route{{Prefix: newStringP("/"), Action: map[string]uint32{clusterName: 1}}}}, - }, - { - // weights not add up to total-weight. - name: "route-config-with-weighted_clusters_weights_not_add_up", - rc: &v3routepb.RouteConfiguration{ - Name: routeName, - VirtualHosts: []*v3routepb.VirtualHost{ - { - Domains: []string{ldsTarget}, - Routes: []*v3routepb.Route{ - { - Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}}, - Action: &v3routepb.Route_Route{ - Route: &v3routepb.RouteAction{ - ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{ - WeightedClusters: &v3routepb.WeightedCluster{ - Clusters: []*v3routepb.WeightedCluster_ClusterWeight{ - {Name: "a", Weight: &wrapperspb.UInt32Value{Value: 2}}, - {Name: "b", Weight: &wrapperspb.UInt32Value{Value: 3}}, - {Name: "c", Weight: &wrapperspb.UInt32Value{Value: 5}}, - }, - TotalWeight: &wrapperspb.UInt32Value{Value: 30}, - }, - }, - }, - }, - }, - }, - }, - }, - }, - wantError: true, - }, - { - name: "good-route-config-with-weighted_clusters", - rc: &v3routepb.RouteConfiguration{ - Name: routeName, - VirtualHosts: []*v3routepb.VirtualHost{ - { - Domains: []string{ldsTarget}, - Routes: []*v3routepb.Route{ - { - Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}}, - Action: &v3routepb.Route_Route{ - Route: &v3routepb.RouteAction{ - ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{ - WeightedClusters: &v3routepb.WeightedCluster{ - Clusters: []*v3routepb.WeightedCluster_ClusterWeight{ - {Name: "a", Weight: &wrapperspb.UInt32Value{Value: 2}}, - {Name: "b", Weight: &wrapperspb.UInt32Value{Value: 3}}, - {Name: "c", Weight: &wrapperspb.UInt32Value{Value: 5}}, - }, - TotalWeight: &wrapperspb.UInt32Value{Value: 10}, - }, - }, - }, - }, - }, - }, - }, - }, - }, - wantUpdate: RouteConfigUpdate{Routes: []*Route{{Prefix: newStringP("/"), Action: map[string]uint32{"a": 2, "b": 3, "c": 5}}}}, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - gotUpdate, gotError := generateRDSUpdateFromRouteConfiguration(test.rc, ldsTarget, nil) - if (gotError != nil) != test.wantError || !cmp.Equal(gotUpdate, test.wantUpdate, cmpopts.EquateEmpty()) { - t.Errorf("generateRDSUpdateFromRouteConfiguration(%+v, %v) = %v, want %v", test.rc, ldsTarget, gotUpdate, test.wantUpdate) - } - }) - } -} - -func (s) TestUnmarshalRouteConfig(t *testing.T) { - const ( - ldsTarget = "lds.target.good:1111" - uninterestingDomain = "uninteresting.domain" - uninterestingClusterName = "uninterestingClusterName" - v2RouteConfigName = "v2RouteConfig" - v3RouteConfigName = "v3RouteConfig" - v2ClusterName = "v2Cluster" - v3ClusterName = "v3Cluster" - ) - - var ( - v2VirtualHost = []*v2routepb.VirtualHost{ - { - Domains: []string{uninterestingDomain}, - Routes: []*v2routepb.Route{ - { - Match: &v2routepb.RouteMatch{PathSpecifier: &v2routepb.RouteMatch_Prefix{Prefix: ""}}, - Action: &v2routepb.Route_Route{ - Route: &v2routepb.RouteAction{ - ClusterSpecifier: &v2routepb.RouteAction_Cluster{Cluster: uninterestingClusterName}, - }, - }, - }, - }, - }, - { - Domains: []string{ldsTarget}, - Routes: []*v2routepb.Route{ - { - Match: &v2routepb.RouteMatch{PathSpecifier: &v2routepb.RouteMatch_Prefix{Prefix: ""}}, - Action: &v2routepb.Route_Route{ - Route: &v2routepb.RouteAction{ - ClusterSpecifier: &v2routepb.RouteAction_Cluster{Cluster: v2ClusterName}, - }, - }, - }, - }, - }, - } - v2RouteConfig = &anypb.Any{ - TypeUrl: version.V2RouteConfigURL, - Value: func() []byte { - rc := &v2xdspb.RouteConfiguration{ - Name: v2RouteConfigName, - VirtualHosts: v2VirtualHost, - } - m, _ := proto.Marshal(rc) - return m - }(), - } - v3VirtualHost = []*v3routepb.VirtualHost{ - { - Domains: []string{uninterestingDomain}, - Routes: []*v3routepb.Route{ - { - Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: ""}}, - Action: &v3routepb.Route_Route{ - Route: &v3routepb.RouteAction{ - ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: uninterestingClusterName}, - }, - }, - }, - }, - }, - { - Domains: []string{ldsTarget}, - Routes: []*v3routepb.Route{ - { - Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: ""}}, - Action: &v3routepb.Route_Route{ - Route: &v3routepb.RouteAction{ - ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: v3ClusterName}, - }, - }, - }, - }, - }, - } - v3RouteConfig = &anypb.Any{ - TypeUrl: version.V2RouteConfigURL, - Value: func() []byte { - rc := &v3routepb.RouteConfiguration{ - Name: v3RouteConfigName, - VirtualHosts: v3VirtualHost, - } - m, _ := proto.Marshal(rc) - return m - }(), - } - ) - - tests := []struct { - name string - resources []*anypb.Any - wantUpdate map[string]RouteConfigUpdate - wantErr bool - }{ - { - name: "non-routeConfig resource type", - resources: []*anypb.Any{{TypeUrl: version.V3HTTPConnManagerURL}}, - wantErr: true, - }, - { - name: "badly marshaled routeconfig resource", - resources: []*anypb.Any{ - { - TypeUrl: version.V3RouteConfigURL, - Value: []byte{1, 2, 3, 4}, - }, - }, - wantErr: true, - }, - { - name: "bad routeConfig resource", - resources: []*anypb.Any{ - { - TypeUrl: version.V3RouteConfigURL, - Value: func() []byte { - rc := &v3routepb.RouteConfiguration{ - VirtualHosts: []*v3routepb.VirtualHost{ - {Domains: []string{uninterestingDomain}}, - }, - } - m, _ := proto.Marshal(rc) - return m - }(), - }, - }, - wantErr: true, - }, - { - name: "empty resource list", - }, - { - name: "v2 routeConfig resource", - resources: []*anypb.Any{v2RouteConfig}, - wantUpdate: map[string]RouteConfigUpdate{ - v2RouteConfigName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{v2ClusterName: 1}}}}, - }, - }, - { - name: "v3 routeConfig resource", - resources: []*anypb.Any{v3RouteConfig}, - wantUpdate: map[string]RouteConfigUpdate{ - v3RouteConfigName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{v3ClusterName: 1}}}}, - }, - }, - { - name: "multiple routeConfig resources", - resources: []*anypb.Any{v2RouteConfig, v3RouteConfig}, - wantUpdate: map[string]RouteConfigUpdate{ - v3RouteConfigName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{v3ClusterName: 1}}}}, - v2RouteConfigName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{v2ClusterName: 1}}}}, - }, - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - update, err := UnmarshalRouteConfig(test.resources, ldsTarget, nil) - if ((err != nil) != test.wantErr) || !cmp.Equal(update, test.wantUpdate, cmpopts.EquateEmpty()) { - t.Errorf("UnmarshalRouteConfig(%v, %v) = (%v, %v) want (%v, %v)", test.resources, ldsTarget, update, err, test.wantUpdate, test.wantErr) - } - }) - } -} - -func (s) TestMatchTypeForDomain(t *testing.T) { - tests := []struct { - d string - want domainMatchType - }{ - {d: "", want: domainMatchTypeInvalid}, - {d: "*", want: domainMatchTypeUniversal}, - {d: "bar.*", want: domainMatchTypePrefix}, - {d: "*.abc.com", want: domainMatchTypeSuffix}, - {d: "foo.bar.com", want: domainMatchTypeExact}, - {d: "foo.*.com", want: domainMatchTypeInvalid}, - } - for _, tt := range tests { - if got := matchTypeForDomain(tt.d); got != tt.want { - t.Errorf("matchTypeForDomain(%q) = %v, want %v", tt.d, got, tt.want) - } - } -} - -func (s) TestMatch(t *testing.T) { - tests := []struct { - name string - domain string - host string - wantTyp domainMatchType - wantMatched bool - }{ - {name: "invalid-empty", domain: "", host: "", wantTyp: domainMatchTypeInvalid, wantMatched: false}, - {name: "invalid", domain: "a.*.b", host: "", wantTyp: domainMatchTypeInvalid, wantMatched: false}, - {name: "universal", domain: "*", host: "abc.com", wantTyp: domainMatchTypeUniversal, wantMatched: true}, - {name: "prefix-match", domain: "abc.*", host: "abc.123", wantTyp: domainMatchTypePrefix, wantMatched: true}, - {name: "prefix-no-match", domain: "abc.*", host: "abcd.123", wantTyp: domainMatchTypePrefix, wantMatched: false}, - {name: "suffix-match", domain: "*.123", host: "abc.123", wantTyp: domainMatchTypeSuffix, wantMatched: true}, - {name: "suffix-no-match", domain: "*.123", host: "abc.1234", wantTyp: domainMatchTypeSuffix, wantMatched: false}, - {name: "exact-match", domain: "foo.bar", host: "foo.bar", wantTyp: domainMatchTypeExact, wantMatched: true}, - {name: "exact-no-match", domain: "foo.bar.com", host: "foo.bar", wantTyp: domainMatchTypeExact, wantMatched: false}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if gotTyp, gotMatched := match(tt.domain, tt.host); gotTyp != tt.wantTyp || gotMatched != tt.wantMatched { - t.Errorf("match() = %v, %v, want %v, %v", gotTyp, gotMatched, tt.wantTyp, tt.wantMatched) - } - }) - } -} - -func (s) TestFindBestMatchingVirtualHost(t *testing.T) { - var ( - oneExactMatch = &v3routepb.VirtualHost{ - Name: "one-exact-match", - Domains: []string{"foo.bar.com"}, - } - oneSuffixMatch = &v3routepb.VirtualHost{ - Name: "one-suffix-match", - Domains: []string{"*.bar.com"}, - } - onePrefixMatch = &v3routepb.VirtualHost{ - Name: "one-prefix-match", - Domains: []string{"foo.bar.*"}, - } - oneUniversalMatch = &v3routepb.VirtualHost{ - Name: "one-universal-match", - Domains: []string{"*"}, - } - longExactMatch = &v3routepb.VirtualHost{ - Name: "one-exact-match", - Domains: []string{"v2.foo.bar.com"}, - } - multipleMatch = &v3routepb.VirtualHost{ - Name: "multiple-match", - Domains: []string{"pi.foo.bar.com", "314.*", "*.159"}, - } - vhs = []*v3routepb.VirtualHost{oneExactMatch, oneSuffixMatch, onePrefixMatch, oneUniversalMatch, longExactMatch, multipleMatch} - ) - - tests := []struct { - name string - host string - vHosts []*v3routepb.VirtualHost - want *v3routepb.VirtualHost - }{ - {name: "exact-match", host: "foo.bar.com", vHosts: vhs, want: oneExactMatch}, - {name: "suffix-match", host: "123.bar.com", vHosts: vhs, want: oneSuffixMatch}, - {name: "prefix-match", host: "foo.bar.org", vHosts: vhs, want: onePrefixMatch}, - {name: "universal-match", host: "abc.123", vHosts: vhs, want: oneUniversalMatch}, - {name: "long-exact-match", host: "v2.foo.bar.com", vHosts: vhs, want: longExactMatch}, - // Matches suffix "*.bar.com" and exact "pi.foo.bar.com". Takes exact. - {name: "multiple-match-exact", host: "pi.foo.bar.com", vHosts: vhs, want: multipleMatch}, - // Matches suffix "*.159" and prefix "foo.bar.*". Takes suffix. - {name: "multiple-match-suffix", host: "foo.bar.159", vHosts: vhs, want: multipleMatch}, - // Matches suffix "*.bar.com" and prefix "314.*". Takes suffix. - {name: "multiple-match-prefix", host: "314.bar.com", vHosts: vhs, want: oneSuffixMatch}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := findBestMatchingVirtualHost(tt.host, tt.vHosts); !cmp.Equal(got, tt.want, cmp.Comparer(proto.Equal)) { - t.Errorf("findBestMatchingVirtualHost() = %v, want %v", got, tt.want) - } - }) - } -} - -func (s) TestRoutesProtoToSlice(t *testing.T) { - tests := []struct { - name string - routes []*v3routepb.Route - wantRoutes []*Route - wantErr bool - }{ - { - name: "no path", - routes: []*v3routepb.Route{{ - Match: &v3routepb.RouteMatch{}, - }}, - wantErr: true, - }, - { - name: "case_sensitive is false", - routes: []*v3routepb.Route{{ - Match: &v3routepb.RouteMatch{ - PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}, - CaseSensitive: &wrapperspb.BoolValue{Value: false}, - }, - }}, - wantErr: true, - }, - { - name: "good", - routes: []*v3routepb.Route{ - { - Match: &v3routepb.RouteMatch{ - PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/a/"}, - Headers: []*v3routepb.HeaderMatcher{ - { - Name: "th", - HeaderMatchSpecifier: &v3routepb.HeaderMatcher_PrefixMatch{ - PrefixMatch: "tv", - }, - InvertMatch: true, - }, - }, - RuntimeFraction: &v3corepb.RuntimeFractionalPercent{ - DefaultValue: &v3typepb.FractionalPercent{ - Numerator: 1, - Denominator: v3typepb.FractionalPercent_HUNDRED, - }, - }, - }, - Action: &v3routepb.Route_Route{ - Route: &v3routepb.RouteAction{ - ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{ - WeightedClusters: &v3routepb.WeightedCluster{ - Clusters: []*v3routepb.WeightedCluster_ClusterWeight{ - {Name: "B", Weight: &wrapperspb.UInt32Value{Value: 60}}, - {Name: "A", Weight: &wrapperspb.UInt32Value{Value: 40}}, - }, - TotalWeight: &wrapperspb.UInt32Value{Value: 100}, - }}}}, - }, - }, - wantRoutes: []*Route{{ - Prefix: newStringP("/a/"), - Headers: []*HeaderMatcher{ - { - Name: "th", - InvertMatch: newBoolP(true), - PrefixMatch: newStringP("tv"), - }, - }, - Fraction: newUInt32P(10000), - Action: map[string]uint32{"A": 40, "B": 60}, - }}, - wantErr: false, - }, - { - name: "query is ignored", - routes: []*v3routepb.Route{ - { - Match: &v3routepb.RouteMatch{ - PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/a/"}, - }, - Action: &v3routepb.Route_Route{ - Route: &v3routepb.RouteAction{ - ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{ - WeightedClusters: &v3routepb.WeightedCluster{ - Clusters: []*v3routepb.WeightedCluster_ClusterWeight{ - {Name: "B", Weight: &wrapperspb.UInt32Value{Value: 60}}, - {Name: "A", Weight: &wrapperspb.UInt32Value{Value: 40}}, - }, - TotalWeight: &wrapperspb.UInt32Value{Value: 100}, - }}}}, - }, - { - Name: "with_query", - Match: &v3routepb.RouteMatch{ - PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/b/"}, - QueryParameters: []*v3routepb.QueryParameterMatcher{{Name: "route_will_be_ignored"}}, - }, - }, - }, - // Only one route in the result, because the second one with query - // parameters is ignored. - wantRoutes: []*Route{{ - Prefix: newStringP("/a/"), - Action: map[string]uint32{"A": 40, "B": 60}, - }}, - wantErr: false, - }, - } - - cmpOpts := []cmp.Option{ - cmp.AllowUnexported(Route{}, HeaderMatcher{}, Int64Range{}), - cmpopts.EquateEmpty(), - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := routesProtoToSlice(tt.routes, nil) - if (err != nil) != tt.wantErr { - t.Errorf("routesProtoToSlice() error = %v, wantErr %v", err, tt.wantErr) - return - } - if !cmp.Equal(got, tt.wantRoutes, cmpOpts...) { - t.Errorf("routesProtoToSlice() got = %v, want %v, diff: %v", got, tt.wantRoutes, cmp.Diff(got, tt.wantRoutes, cmpOpts...)) - } - }) - } -} - -func newStringP(s string) *string { - return &s -} - -func newUInt32P(i uint32) *uint32 { - return &i -} - -func newBoolP(b bool) *bool { - return &b -} diff --git a/xds/internal/client/client_test.go b/xds/internal/client/client_test.go deleted file mode 100644 index 729be4cc0726..000000000000 --- a/xds/internal/client/client_test.go +++ /dev/null @@ -1,161 +0,0 @@ -/* - * - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package client - -import ( - "testing" - "time" - - "google.golang.org/grpc" - "google.golang.org/grpc/internal/grpctest" - "google.golang.org/grpc/xds/internal/client/bootstrap" - "google.golang.org/grpc/xds/internal/testutils" - "google.golang.org/grpc/xds/internal/version" -) - -type s struct { - grpctest.Tester -} - -func Test(t *testing.T) { - grpctest.RunSubTests(t, s{}) -} - -const ( - testXDSServer = "xds-server" - chanRecvTimeout = 100 * time.Millisecond - - testLDSName = "test-lds" - testRDSName = "test-rds" - testCDSName = "test-cds" - testEDSName = "test-eds" - - defaultTestWatchExpiryTimeout = 500 * time.Millisecond -) - -func clientOpts(balancerName string, overrideWatchExpiryTImeout bool) Options { - watchExpiryTimeout := time.Duration(0) - if overrideWatchExpiryTImeout { - watchExpiryTimeout = defaultTestWatchExpiryTimeout - } - return Options{ - Config: bootstrap.Config{ - BalancerName: balancerName, - Creds: grpc.WithInsecure(), - NodeProto: testutils.EmptyNodeProtoV2, - }, - WatchExpiryTimeout: watchExpiryTimeout, - } -} - -type testAPIClient struct { - r UpdateHandler - - addWatches map[ResourceType]*testutils.Channel - removeWatches map[ResourceType]*testutils.Channel -} - -func overrideNewAPIClient() (<-chan *testAPIClient, func()) { - origNewAPIClient := newAPIClient - ch := make(chan *testAPIClient, 1) - newAPIClient = func(apiVersion version.TransportAPI, cc *grpc.ClientConn, opts BuildOptions) (APIClient, error) { - ret := newTestAPIClient(opts.Parent) - ch <- ret - return ret, nil - } - return ch, func() { newAPIClient = origNewAPIClient } -} - -func newTestAPIClient(r UpdateHandler) *testAPIClient { - addWatches := map[ResourceType]*testutils.Channel{ - ListenerResource: testutils.NewChannel(), - RouteConfigResource: testutils.NewChannel(), - ClusterResource: testutils.NewChannel(), - EndpointsResource: testutils.NewChannel(), - } - removeWatches := map[ResourceType]*testutils.Channel{ - ListenerResource: testutils.NewChannel(), - RouteConfigResource: testutils.NewChannel(), - ClusterResource: testutils.NewChannel(), - EndpointsResource: testutils.NewChannel(), - } - return &testAPIClient{ - r: r, - addWatches: addWatches, - removeWatches: removeWatches, - } -} - -func (c *testAPIClient) AddWatch(resourceType ResourceType, resourceName string) { - c.addWatches[resourceType].Send(resourceName) -} - -func (c *testAPIClient) RemoveWatch(resourceType ResourceType, resourceName string) { - c.removeWatches[resourceType].Send(resourceName) -} - -func (c *testAPIClient) Close() {} - -// TestWatchCallAnotherWatch covers the case where watch() is called inline by a -// callback. It makes sure it doesn't cause a deadlock. -func (s) TestWatchCallAnotherWatch(t *testing.T) { - v2ClientCh, cleanup := overrideNewAPIClient() - defer cleanup() - - c, err := New(clientOpts(testXDSServer, false)) - if err != nil { - t.Fatalf("failed to create client: %v", err) - } - defer c.Close() - - v2Client := <-v2ClientCh - - clusterUpdateCh := testutils.NewChannel() - firstTime := true - c.WatchCluster(testCDSName, func(update ClusterUpdate, err error) { - clusterUpdateCh.Send(clusterUpdateErr{u: update, err: err}) - // Calls another watch inline, to ensure there's deadlock. - c.WatchCluster("another-random-name", func(ClusterUpdate, error) {}) - if _, err := v2Client.addWatches[ClusterResource].Receive(); firstTime && err != nil { - t.Fatalf("want new watch to start, got error %v", err) - } - firstTime = false - }) - if _, err := v2Client.addWatches[ClusterResource].Receive(); err != nil { - t.Fatalf("want new watch to start, got error %v", err) - } - - wantUpdate := ClusterUpdate{ServiceName: testEDSName} - v2Client.r.NewClusters(map[string]ClusterUpdate{ - testCDSName: wantUpdate, - }) - - if u, err := clusterUpdateCh.Receive(); err != nil || u != (clusterUpdateErr{wantUpdate, nil}) { - t.Errorf("unexpected clusterUpdate: %v, error receiving from channel: %v", u, err) - } - - wantUpdate2 := ClusterUpdate{ServiceName: testEDSName + "2"} - v2Client.r.NewClusters(map[string]ClusterUpdate{ - testCDSName: wantUpdate2, - }) - - if u, err := clusterUpdateCh.Receive(); err != nil || u != (clusterUpdateErr{wantUpdate2, nil}) { - t.Errorf("unexpected clusterUpdate: %v, error receiving from channel: %v", u, err) - } -} diff --git a/xds/internal/client/client_watchers.go b/xds/internal/client/client_watchers.go deleted file mode 100644 index 7f67be894226..000000000000 --- a/xds/internal/client/client_watchers.go +++ /dev/null @@ -1,402 +0,0 @@ -/* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package client - -import ( - "fmt" - "sync" - "time" -) - -type watchInfoState int - -const ( - watchInfoStateStarted watchInfoState = iota - watchInfoStateRespReceived - watchInfoStateTimeout - watchInfoStateCanceled -) - -// watchInfo holds all the information from a watch() call. -type watchInfo struct { - c *Client - rType ResourceType - target string - - ldsCallback func(ListenerUpdate, error) - rdsCallback func(RouteConfigUpdate, error) - cdsCallback func(ClusterUpdate, error) - edsCallback func(EndpointsUpdate, error) - - expiryTimer *time.Timer - - // mu protects state, and c.scheduleCallback(). - // - No callback should be scheduled after watchInfo is canceled. - // - No timeout error should be scheduled after watchInfo is resp received. - mu sync.Mutex - state watchInfoState -} - -func (wi *watchInfo) newUpdate(update interface{}) { - wi.mu.Lock() - defer wi.mu.Unlock() - if wi.state == watchInfoStateCanceled { - return - } - wi.state = watchInfoStateRespReceived - wi.expiryTimer.Stop() - wi.c.scheduleCallback(wi, update, nil) -} - -func (wi *watchInfo) resourceNotFound() { - wi.mu.Lock() - defer wi.mu.Unlock() - if wi.state == watchInfoStateCanceled { - return - } - wi.state = watchInfoStateRespReceived - wi.expiryTimer.Stop() - wi.sendErrorLocked(NewErrorf(ErrorTypeResourceNotFound, "xds: %v target %s not found in received response", wi.rType, wi.target)) -} - -func (wi *watchInfo) timeout() { - wi.mu.Lock() - defer wi.mu.Unlock() - if wi.state == watchInfoStateCanceled || wi.state == watchInfoStateRespReceived { - return - } - wi.state = watchInfoStateTimeout - wi.sendErrorLocked(fmt.Errorf("xds: %v target %s not found, watcher timeout", wi.rType, wi.target)) -} - -// Caller must hold wi.mu. -func (wi *watchInfo) sendErrorLocked(err error) { - var ( - u interface{} - ) - switch wi.rType { - case ListenerResource: - u = ListenerUpdate{} - case RouteConfigResource: - u = RouteConfigUpdate{} - case ClusterResource: - u = ClusterUpdate{} - case EndpointsResource: - u = EndpointsUpdate{} - } - wi.c.scheduleCallback(wi, u, err) -} - -func (wi *watchInfo) cancel() { - wi.mu.Lock() - defer wi.mu.Unlock() - if wi.state == watchInfoStateCanceled { - return - } - wi.expiryTimer.Stop() - wi.state = watchInfoStateCanceled -} - -func (c *Client) watch(wi *watchInfo) (cancel func()) { - c.mu.Lock() - defer c.mu.Unlock() - c.logger.Debugf("new watch for type %v, resource name %v", wi.rType, wi.target) - var watchers map[string]map[*watchInfo]bool - switch wi.rType { - case ListenerResource: - watchers = c.ldsWatchers - case RouteConfigResource: - watchers = c.rdsWatchers - case ClusterResource: - watchers = c.cdsWatchers - case EndpointsResource: - watchers = c.edsWatchers - } - - resourceName := wi.target - s, ok := watchers[wi.target] - if !ok { - // If this is a new watcher, will ask lower level to send a new request - // with the resource name. - // - // If this (type+name) is already being watched, will not notify the - // underlying versioned apiClient. - c.logger.Debugf("first watch for type %v, resource name %v, will send a new xDS request", wi.rType, wi.target) - s = make(map[*watchInfo]bool) - watchers[resourceName] = s - c.apiClient.AddWatch(wi.rType, resourceName) - } - // No matter what, add the new watcher to the set, so it's callback will be - // call for new responses. - s[wi] = true - - // If the resource is in cache, call the callback with the value. - switch wi.rType { - case ListenerResource: - if v, ok := c.ldsCache[resourceName]; ok { - c.logger.Debugf("LDS resource with name %v found in cache: %+v", wi.target, v) - wi.newUpdate(v) - } - case RouteConfigResource: - if v, ok := c.rdsCache[resourceName]; ok { - c.logger.Debugf("RDS resource with name %v found in cache: %+v", wi.target, v) - wi.newUpdate(v) - } - case ClusterResource: - if v, ok := c.cdsCache[resourceName]; ok { - c.logger.Debugf("CDS resource with name %v found in cache: %+v", wi.target, v) - wi.newUpdate(v) - } - case EndpointsResource: - if v, ok := c.edsCache[resourceName]; ok { - c.logger.Debugf("EDS resource with name %v found in cache: %+v", wi.target, v) - wi.newUpdate(v) - } - } - - return func() { - c.logger.Debugf("watch for type %v, resource name %v canceled", wi.rType, wi.target) - wi.cancel() - c.mu.Lock() - defer c.mu.Unlock() - if s := watchers[resourceName]; s != nil { - // Remove this watcher, so it's callback will not be called in the - // future. - delete(s, wi) - if len(s) == 0 { - c.logger.Debugf("last watch for type %v, resource name %v canceled, will send a new xDS request", wi.rType, wi.target) - // If this was the last watcher, also tell xdsv2Client to stop - // watching this resource. - delete(watchers, resourceName) - c.apiClient.RemoveWatch(wi.rType, resourceName) - // Remove the resource from cache. When a watch for this - // resource is added later, it will trigger a xDS request with - // resource names, and client will receive new xDS responses. - switch wi.rType { - case ListenerResource: - delete(c.ldsCache, resourceName) - case RouteConfigResource: - delete(c.rdsCache, resourceName) - case ClusterResource: - delete(c.cdsCache, resourceName) - case EndpointsResource: - delete(c.edsCache, resourceName) - } - } - } - } -} - -// WatchListener uses LDS to discover information about the provided listener. -// -// WatchListener is expected to called only from the server side implementation -// of xDS. Clients will use WatchService instead. -// -// Note that during race (e.g. an xDS response is received while the user is -// calling cancel()), there's a small window where the callback can be called -// after the watcher is canceled. The caller needs to handle this case. -func (c *Client) WatchListener(listener string, cb func(ListenerUpdate, error)) (cancel func()) { - return c.watchLDS(listener, cb) -} - -// watchLDS starts a listener watcher for the service.. -// -// Note that during race (e.g. an xDS response is received while the user is -// calling cancel()), there's a small window where the callback can be called -// after the watcher is canceled. The caller needs to handle this case. -func (c *Client) watchLDS(serviceName string, cb func(ListenerUpdate, error)) (cancel func()) { - wi := &watchInfo{ - c: c, - rType: ListenerResource, - target: serviceName, - ldsCallback: cb, - } - - wi.expiryTimer = time.AfterFunc(c.opts.WatchExpiryTimeout, func() { - wi.timeout() - }) - return c.watch(wi) -} - -// watchRDS starts a listener watcher for the service.. -// -// Note that during race (e.g. an xDS response is received while the user is -// calling cancel()), there's a small window where the callback can be called -// after the watcher is canceled. The caller needs to handle this case. -func (c *Client) watchRDS(routeName string, cb func(RouteConfigUpdate, error)) (cancel func()) { - wi := &watchInfo{ - c: c, - rType: RouteConfigResource, - target: routeName, - rdsCallback: cb, - } - - wi.expiryTimer = time.AfterFunc(c.opts.WatchExpiryTimeout, func() { - wi.timeout() - }) - return c.watch(wi) -} - -// WatchService uses LDS and RDS to discover information about the provided -// serviceName. -// -// WatchService can only be called once. The second call will not start a -// watcher and the callback will get an error. It's this case because an xDS -// client is expected to be used only by one ClientConn. -// -// Note that during race (e.g. an xDS response is received while the user is -// calling cancel()), there's a small window where the callback can be called -// after the watcher is canceled. The caller needs to handle this case. -func (c *Client) WatchService(serviceName string, cb func(ServiceUpdate, error)) (cancel func()) { - c.mu.Lock() - if len(c.ldsWatchers) != 0 { - go cb(ServiceUpdate{}, fmt.Errorf("unexpected WatchService when there's another service being watched")) - c.mu.Unlock() - return func() {} - } - c.mu.Unlock() - - w := &serviceUpdateWatcher{c: c, serviceCb: cb} - w.ldsCancel = c.watchLDS(serviceName, w.handleLDSResp) - - return w.close -} - -// serviceUpdateWatcher handles LDS and RDS response, and calls the service -// callback at the right time. -type serviceUpdateWatcher struct { - c *Client - ldsCancel func() - serviceCb func(ServiceUpdate, error) - - mu sync.Mutex - closed bool - rdsName string - rdsCancel func() -} - -func (w *serviceUpdateWatcher) handleLDSResp(update ListenerUpdate, err error) { - w.c.logger.Infof("xds: client received LDS update: %+v, err: %v", update, err) - w.mu.Lock() - defer w.mu.Unlock() - if w.closed { - return - } - if err != nil { - // We check the error type and do different things. For now, the only - // type we check is ResourceNotFound, which indicates the LDS resource - // was removed, and besides sending the error to callback, we also - // cancel the RDS watch. - if ErrType(err) == ErrorTypeResourceNotFound && w.rdsCancel != nil { - w.rdsCancel() - w.rdsName = "" - w.rdsCancel = nil - } - // The other error cases still return early without canceling the - // existing RDS watch. - w.serviceCb(ServiceUpdate{}, err) - return - } - - if w.rdsName == update.RouteConfigName { - // If the new RouteConfigName is same as the previous, don't cancel and - // restart the RDS watch. - return - } - w.rdsName = update.RouteConfigName - if w.rdsCancel != nil { - w.rdsCancel() - } - w.rdsCancel = w.c.watchRDS(update.RouteConfigName, w.handleRDSResp) -} - -func (w *serviceUpdateWatcher) handleRDSResp(update RouteConfigUpdate, err error) { - w.c.logger.Infof("xds: client received RDS update: %+v, err: %v", update, err) - w.mu.Lock() - defer w.mu.Unlock() - if w.closed { - return - } - if w.rdsCancel == nil { - // This mean only the RDS watch is canceled, can happen if the LDS - // resource is removed. - return - } - if err != nil { - w.serviceCb(ServiceUpdate{}, err) - return - } - w.serviceCb(ServiceUpdate(update), nil) -} - -func (w *serviceUpdateWatcher) close() { - w.mu.Lock() - defer w.mu.Unlock() - w.closed = true - w.ldsCancel() - if w.rdsCancel != nil { - w.rdsCancel() - w.rdsCancel = nil - } -} - -// WatchCluster uses CDS to discover information about the provided -// clusterName. -// -// WatchCluster can be called multiple times, with same or different -// clusterNames. Each call will start an independent watcher for the resource. -// -// Note that during race (e.g. an xDS response is received while the user is -// calling cancel()), there's a small window where the callback can be called -// after the watcher is canceled. The caller needs to handle this case. -func (c *Client) WatchCluster(clusterName string, cb func(ClusterUpdate, error)) (cancel func()) { - wi := &watchInfo{ - c: c, - rType: ClusterResource, - target: clusterName, - cdsCallback: cb, - } - - wi.expiryTimer = time.AfterFunc(c.opts.WatchExpiryTimeout, func() { - wi.timeout() - }) - return c.watch(wi) -} - -// WatchEndpoints uses EDS to discover endpoints in the provided clusterName. -// -// WatchEndpoints can be called multiple times, with same or different -// clusterNames. Each call will start an independent watcher for the resource. -// -// Note that during race (e.g. an xDS response is received while the user is -// calling cancel()), there's a small window where the callback can be called -// after the watcher is canceled. The caller needs to handle this case. -func (c *Client) WatchEndpoints(clusterName string, cb func(EndpointsUpdate, error)) (cancel func()) { - wi := &watchInfo{ - c: c, - rType: EndpointsResource, - target: clusterName, - edsCallback: cb, - } - - wi.expiryTimer = time.AfterFunc(c.opts.WatchExpiryTimeout, func() { - wi.timeout() - }) - return c.watch(wi) -} diff --git a/xds/internal/client/client_watchers_cluster_test.go b/xds/internal/client/client_watchers_cluster_test.go deleted file mode 100644 index eedda3e0a2d5..000000000000 --- a/xds/internal/client/client_watchers_cluster_test.go +++ /dev/null @@ -1,412 +0,0 @@ -/* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package client - -import ( - "testing" - - "google.golang.org/grpc/xds/internal/testutils" -) - -type clusterUpdateErr struct { - u ClusterUpdate - err error -} - -// TestClusterWatch covers the cases: -// - an update is received after a watch() -// - an update for another resource name -// - an update is received after cancel() -func (s) TestClusterWatch(t *testing.T) { - v2ClientCh, cleanup := overrideNewAPIClient() - defer cleanup() - - c, err := New(clientOpts(testXDSServer, false)) - if err != nil { - t.Fatalf("failed to create client: %v", err) - } - defer c.Close() - - // TODO: add a timeout to this recv. - // Note that this won't be necessary if we finish the TODO below to call - // Client directly instead of v2Client.r. - v2Client := <-v2ClientCh - - clusterUpdateCh := testutils.NewChannel() - cancelWatch := c.WatchCluster(testCDSName, func(update ClusterUpdate, err error) { - clusterUpdateCh.Send(clusterUpdateErr{u: update, err: err}) - }) - if _, err := v2Client.addWatches[ClusterResource].Receive(); err != nil { - t.Fatalf("want new watch to start, got error %v", err) - } - - wantUpdate := ClusterUpdate{ServiceName: testEDSName} - // This is calling v2Client.r to send the update, but r is set to Client, so - // this is same as calling Client to update. The one thing this covers is - // that `NewXDSV2Client` is called with the right parent. - // - // TODO: in a future cleanup, this (and the same thing in other tests) can - // be changed call Client directly. - v2Client.r.NewClusters(map[string]ClusterUpdate{ - testCDSName: wantUpdate, - }) - - if u, err := clusterUpdateCh.Receive(); err != nil || u != (clusterUpdateErr{wantUpdate, nil}) { - t.Errorf("unexpected clusterUpdate: %v, error receiving from channel: %v", u, err) - } - - // Another update, with an extra resource for a different resource name. - v2Client.r.NewClusters(map[string]ClusterUpdate{ - testCDSName: wantUpdate, - "randomName": {}, - }) - - if u, err := clusterUpdateCh.Receive(); err != nil || u != (clusterUpdateErr{wantUpdate, nil}) { - t.Errorf("unexpected clusterUpdate: %+v, %v, want channel recv timeout", u, err) - } - - // Cancel watch, and send update again. - cancelWatch() - v2Client.r.NewClusters(map[string]ClusterUpdate{ - testCDSName: wantUpdate, - }) - - if u, err := clusterUpdateCh.TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout { - t.Errorf("unexpected clusterUpdate: %v, %v, want channel recv timeout", u, err) - } -} - -// TestClusterTwoWatchSameResourceName covers the case where an update is received -// after two watch() for the same resource name. -func (s) TestClusterTwoWatchSameResourceName(t *testing.T) { - v2ClientCh, cleanup := overrideNewAPIClient() - defer cleanup() - - c, err := New(clientOpts(testXDSServer, false)) - if err != nil { - t.Fatalf("failed to create client: %v", err) - } - defer c.Close() - - v2Client := <-v2ClientCh - - var clusterUpdateChs []*testutils.Channel - var cancelLastWatch func() - - const count = 2 - for i := 0; i < count; i++ { - clusterUpdateCh := testutils.NewChannel() - clusterUpdateChs = append(clusterUpdateChs, clusterUpdateCh) - cancelLastWatch = c.WatchCluster(testCDSName, func(update ClusterUpdate, err error) { - clusterUpdateCh.Send(clusterUpdateErr{u: update, err: err}) - }) - if _, err := v2Client.addWatches[ClusterResource].Receive(); i == 0 && err != nil { - t.Fatalf("want new watch to start, got error %v", err) - } - } - - wantUpdate := ClusterUpdate{ServiceName: testEDSName} - v2Client.r.NewClusters(map[string]ClusterUpdate{ - testCDSName: wantUpdate, - }) - - for i := 0; i < count; i++ { - if u, err := clusterUpdateChs[i].Receive(); err != nil || u != (clusterUpdateErr{wantUpdate, nil}) { - t.Errorf("i=%v, unexpected clusterUpdate: %v, error receiving from channel: %v", i, u, err) - } - } - - // Cancel the last watch, and send update again. - cancelLastWatch() - v2Client.r.NewClusters(map[string]ClusterUpdate{ - testCDSName: wantUpdate, - }) - - for i := 0; i < count-1; i++ { - if u, err := clusterUpdateChs[i].Receive(); err != nil || u != (clusterUpdateErr{wantUpdate, nil}) { - t.Errorf("i=%v, unexpected clusterUpdate: %v, error receiving from channel: %v", i, u, err) - } - } - - if u, err := clusterUpdateChs[count-1].TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout { - t.Errorf("unexpected clusterUpdate: %v, %v, want channel recv timeout", u, err) - } -} - -// TestClusterThreeWatchDifferentResourceName covers the case where an update is -// received after three watch() for different resource names. -func (s) TestClusterThreeWatchDifferentResourceName(t *testing.T) { - v2ClientCh, cleanup := overrideNewAPIClient() - defer cleanup() - - c, err := New(clientOpts(testXDSServer, false)) - if err != nil { - t.Fatalf("failed to create client: %v", err) - } - defer c.Close() - - v2Client := <-v2ClientCh - - var clusterUpdateChs []*testutils.Channel - const count = 2 - - // Two watches for the same name. - for i := 0; i < count; i++ { - clusterUpdateCh := testutils.NewChannel() - clusterUpdateChs = append(clusterUpdateChs, clusterUpdateCh) - c.WatchCluster(testCDSName+"1", func(update ClusterUpdate, err error) { - clusterUpdateCh.Send(clusterUpdateErr{u: update, err: err}) - }) - if _, err := v2Client.addWatches[ClusterResource].Receive(); i == 0 && err != nil { - t.Fatalf("want new watch to start, got error %v", err) - } - } - - // Third watch for a different name. - clusterUpdateCh2 := testutils.NewChannel() - c.WatchCluster(testCDSName+"2", func(update ClusterUpdate, err error) { - clusterUpdateCh2.Send(clusterUpdateErr{u: update, err: err}) - }) - if _, err := v2Client.addWatches[ClusterResource].Receive(); err != nil { - t.Fatalf("want new watch to start, got error %v", err) - } - - wantUpdate1 := ClusterUpdate{ServiceName: testEDSName + "1"} - wantUpdate2 := ClusterUpdate{ServiceName: testEDSName + "2"} - v2Client.r.NewClusters(map[string]ClusterUpdate{ - testCDSName + "1": wantUpdate1, - testCDSName + "2": wantUpdate2, - }) - - for i := 0; i < count; i++ { - if u, err := clusterUpdateChs[i].Receive(); err != nil || u != (clusterUpdateErr{wantUpdate1, nil}) { - t.Errorf("i=%v, unexpected clusterUpdate: %v, error receiving from channel: %v", i, u, err) - } - } - - if u, err := clusterUpdateCh2.Receive(); err != nil || u != (clusterUpdateErr{wantUpdate2, nil}) { - t.Errorf("unexpected clusterUpdate: %v, error receiving from channel: %v", u, err) - } -} - -// TestClusterWatchAfterCache covers the case where watch is called after the update -// is in cache. -func (s) TestClusterWatchAfterCache(t *testing.T) { - v2ClientCh, cleanup := overrideNewAPIClient() - defer cleanup() - - c, err := New(clientOpts(testXDSServer, false)) - if err != nil { - t.Fatalf("failed to create client: %v", err) - } - defer c.Close() - - v2Client := <-v2ClientCh - - clusterUpdateCh := testutils.NewChannel() - c.WatchCluster(testCDSName, func(update ClusterUpdate, err error) { - clusterUpdateCh.Send(clusterUpdateErr{u: update, err: err}) - }) - if _, err := v2Client.addWatches[ClusterResource].Receive(); err != nil { - t.Fatalf("want new watch to start, got error %v", err) - } - - wantUpdate := ClusterUpdate{ServiceName: testEDSName} - v2Client.r.NewClusters(map[string]ClusterUpdate{ - testCDSName: wantUpdate, - }) - - if u, err := clusterUpdateCh.Receive(); err != nil || u != (clusterUpdateErr{wantUpdate, nil}) { - t.Errorf("unexpected clusterUpdate: %v, error receiving from channel: %v", u, err) - } - - // Another watch for the resource in cache. - clusterUpdateCh2 := testutils.NewChannel() - c.WatchCluster(testCDSName, func(update ClusterUpdate, err error) { - clusterUpdateCh2.Send(clusterUpdateErr{u: update, err: err}) - }) - if n, err := v2Client.addWatches[ClusterResource].Receive(); err == nil { - t.Fatalf("want no new watch to start (recv timeout), got resource name: %v error %v", n, err) - } - - // New watch should receives the update. - if u, err := clusterUpdateCh2.Receive(); err != nil || u != (clusterUpdateErr{wantUpdate, nil}) { - t.Errorf("unexpected clusterUpdate: %v, error receiving from channel: %v", u, err) - } - - // Old watch should see nothing. - if u, err := clusterUpdateCh.TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout { - t.Errorf("unexpected clusterUpdate: %v, %v, want channel recv timeout", u, err) - } -} - -// TestClusterWatchExpiryTimer tests the case where the client does not receive -// an CDS response for the request that it sends out. We want the watch callback -// to be invoked with an error once the watchExpiryTimer fires. -func (s) TestClusterWatchExpiryTimer(t *testing.T) { - v2ClientCh, cleanup := overrideNewAPIClient() - defer cleanup() - - c, err := New(clientOpts(testXDSServer, true)) - if err != nil { - t.Fatalf("failed to create client: %v", err) - } - defer c.Close() - - v2Client := <-v2ClientCh - - clusterUpdateCh := testutils.NewChannel() - c.WatchCluster(testCDSName, func(u ClusterUpdate, err error) { - clusterUpdateCh.Send(clusterUpdateErr{u: u, err: err}) - }) - if _, err := v2Client.addWatches[ClusterResource].Receive(); err != nil { - t.Fatalf("want new watch to start, got error %v", err) - } - - u, err := clusterUpdateCh.TimedReceive(defaultTestWatchExpiryTimeout * 2) - if err != nil { - t.Fatalf("failed to get clusterUpdate: %v", err) - } - uu := u.(clusterUpdateErr) - if uu.u != (ClusterUpdate{}) { - t.Errorf("unexpected clusterUpdate: %v, want %v", uu.u, ClusterUpdate{}) - } - if uu.err == nil { - t.Errorf("unexpected clusterError: , want error watcher timeout") - } -} - -// TestClusterWatchExpiryTimerStop tests the case where the client does receive -// an CDS response for the request that it sends out. We want no error even -// after expiry timeout. -func (s) TestClusterWatchExpiryTimerStop(t *testing.T) { - v2ClientCh, cleanup := overrideNewAPIClient() - defer cleanup() - - c, err := New(clientOpts(testXDSServer, true)) - if err != nil { - t.Fatalf("failed to create client: %v", err) - } - defer c.Close() - - v2Client := <-v2ClientCh - - clusterUpdateCh := testutils.NewChannel() - c.WatchCluster(testCDSName, func(u ClusterUpdate, err error) { - clusterUpdateCh.Send(clusterUpdateErr{u: u, err: err}) - }) - if _, err := v2Client.addWatches[ClusterResource].Receive(); err != nil { - t.Fatalf("want new watch to start, got error %v", err) - } - - wantUpdate := ClusterUpdate{ServiceName: testEDSName} - v2Client.r.NewClusters(map[string]ClusterUpdate{ - testCDSName: wantUpdate, - }) - - if u, err := clusterUpdateCh.Receive(); err != nil || u != (clusterUpdateErr{wantUpdate, nil}) { - t.Errorf("unexpected clusterUpdate: %v, error receiving from channel: %v", u, err) - } - - // Wait for an error, the error should never happen. - u, err := clusterUpdateCh.TimedReceive(defaultTestWatchExpiryTimeout * 2) - if err != testutils.ErrRecvTimeout { - t.Fatalf("got unexpected: %v, %v, want recv timeout", u.(clusterUpdateErr).u, u.(clusterUpdateErr).err) - } -} - -// TestClusterResourceRemoved covers the cases: -// - an update is received after a watch() -// - another update is received, with one resource removed -// - this should trigger callback with resource removed error -// - one more update without the removed resource -// - the callback (above) shouldn't receive any update -func (s) TestClusterResourceRemoved(t *testing.T) { - v2ClientCh, cleanup := overrideNewAPIClient() - defer cleanup() - - c, err := New(clientOpts(testXDSServer, false)) - if err != nil { - t.Fatalf("failed to create client: %v", err) - } - defer c.Close() - - v2Client := <-v2ClientCh - - clusterUpdateCh1 := testutils.NewChannel() - c.WatchCluster(testCDSName+"1", func(update ClusterUpdate, err error) { - clusterUpdateCh1.Send(clusterUpdateErr{u: update, err: err}) - }) - if _, err := v2Client.addWatches[ClusterResource].Receive(); err != nil { - t.Fatalf("want new watch to start, got error %v", err) - } - // Another watch for a different name. - clusterUpdateCh2 := testutils.NewChannel() - c.WatchCluster(testCDSName+"2", func(update ClusterUpdate, err error) { - clusterUpdateCh2.Send(clusterUpdateErr{u: update, err: err}) - }) - if _, err := v2Client.addWatches[ClusterResource].Receive(); err != nil { - t.Fatalf("want new watch to start, got error %v", err) - } - - wantUpdate1 := ClusterUpdate{ServiceName: testEDSName + "1"} - wantUpdate2 := ClusterUpdate{ServiceName: testEDSName + "2"} - v2Client.r.NewClusters(map[string]ClusterUpdate{ - testCDSName + "1": wantUpdate1, - testCDSName + "2": wantUpdate2, - }) - - if u, err := clusterUpdateCh1.Receive(); err != nil || u != (clusterUpdateErr{wantUpdate1, nil}) { - t.Errorf("unexpected clusterUpdate: %v, error receiving from channel: %v", u, err) - } - - if u, err := clusterUpdateCh2.Receive(); err != nil || u != (clusterUpdateErr{wantUpdate2, nil}) { - t.Errorf("unexpected clusterUpdate: %v, error receiving from channel: %v", u, err) - } - - // Send another update to remove resource 1. - v2Client.r.NewClusters(map[string]ClusterUpdate{ - testCDSName + "2": wantUpdate2, - }) - - // watcher 1 should get an error. - if u, err := clusterUpdateCh1.Receive(); err != nil || ErrType(u.(clusterUpdateErr).err) != ErrorTypeResourceNotFound { - t.Errorf("unexpected clusterUpdate: %v, error receiving from channel: %v, want update with error resource not found", u, err) - } - - // watcher 2 should get the same update again. - if u, err := clusterUpdateCh2.Receive(); err != nil || u != (clusterUpdateErr{wantUpdate2, nil}) { - t.Errorf("unexpected clusterUpdate: %v, error receiving from channel: %v", u, err) - } - - // Send one more update without resource 1. - v2Client.r.NewClusters(map[string]ClusterUpdate{ - testCDSName + "2": wantUpdate2, - }) - - // watcher 1 should get an error. - if u, err := clusterUpdateCh1.Receive(); err != testutils.ErrRecvTimeout { - t.Errorf("unexpected clusterUpdate: %v, want receiving from channel timeout", u) - } - - // watcher 2 should get the same update again. - if u, err := clusterUpdateCh2.Receive(); err != nil || u != (clusterUpdateErr{wantUpdate2, nil}) { - t.Errorf("unexpected clusterUpdate: %v, error receiving from channel: %v", u, err) - } -} diff --git a/xds/internal/client/client_watchers_endpoints_test.go b/xds/internal/client/client_watchers_endpoints_test.go deleted file mode 100644 index 09dc80817284..000000000000 --- a/xds/internal/client/client_watchers_endpoints_test.go +++ /dev/null @@ -1,306 +0,0 @@ -/* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package client - -import ( - "testing" - - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" - - "google.golang.org/grpc/xds/internal" - "google.golang.org/grpc/xds/internal/testutils" -) - -var ( - testLocalities = []Locality{ - { - Endpoints: []Endpoint{{Address: "addr1:314"}}, - ID: internal.LocalityID{SubZone: "locality-1"}, - Priority: 1, - Weight: 1, - }, - { - Endpoints: []Endpoint{{Address: "addr2:159"}}, - ID: internal.LocalityID{SubZone: "locality-2"}, - Priority: 0, - Weight: 1, - }, - } - endpointsCmpOpts = []cmp.Option{cmp.AllowUnexported(endpointsUpdateErr{}), cmpopts.EquateEmpty()} -) - -type endpointsUpdateErr struct { - u EndpointsUpdate - err error -} - -// TestEndpointsWatch covers the cases: -// - an update is received after a watch() -// - an update for another resource name (which doesn't trigger callback) -// - an update is received after cancel() -func (s) TestEndpointsWatch(t *testing.T) { - v2ClientCh, cleanup := overrideNewAPIClient() - defer cleanup() - - c, err := New(clientOpts(testXDSServer, false)) - if err != nil { - t.Fatalf("failed to create client: %v", err) - } - defer c.Close() - - v2Client := <-v2ClientCh - - endpointsUpdateCh := testutils.NewChannel() - cancelWatch := c.WatchEndpoints(testCDSName, func(update EndpointsUpdate, err error) { - endpointsUpdateCh.Send(endpointsUpdateErr{u: update, err: err}) - }) - if _, err := v2Client.addWatches[EndpointsResource].Receive(); err != nil { - t.Fatalf("want new watch to start, got error %v", err) - } - - wantUpdate := EndpointsUpdate{Localities: []Locality{testLocalities[0]}} - v2Client.r.NewEndpoints(map[string]EndpointsUpdate{ - testCDSName: wantUpdate, - }) - - if u, err := endpointsUpdateCh.Receive(); err != nil || !cmp.Equal(u, endpointsUpdateErr{wantUpdate, nil}, endpointsCmpOpts...) { - t.Errorf("unexpected endpointsUpdate: %v, error receiving from channel: %v", u, err) - } - - // Another update for a different resource name. - v2Client.r.NewEndpoints(map[string]EndpointsUpdate{ - "randomName": {}, - }) - - if u, err := endpointsUpdateCh.TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout { - t.Errorf("unexpected endpointsUpdate: %v, %v, want channel recv timeout", u, err) - } - - // Cancel watch, and send update again. - cancelWatch() - v2Client.r.NewEndpoints(map[string]EndpointsUpdate{ - testCDSName: wantUpdate, - }) - - if u, err := endpointsUpdateCh.TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout { - t.Errorf("unexpected endpointsUpdate: %v, %v, want channel recv timeout", u, err) - } -} - -// TestEndpointsTwoWatchSameResourceName covers the case where an update is received -// after two watch() for the same resource name. -func (s) TestEndpointsTwoWatchSameResourceName(t *testing.T) { - v2ClientCh, cleanup := overrideNewAPIClient() - defer cleanup() - - c, err := New(clientOpts(testXDSServer, false)) - if err != nil { - t.Fatalf("failed to create client: %v", err) - } - defer c.Close() - - v2Client := <-v2ClientCh - - var endpointsUpdateChs []*testutils.Channel - const count = 2 - - var cancelLastWatch func() - - for i := 0; i < count; i++ { - endpointsUpdateCh := testutils.NewChannel() - endpointsUpdateChs = append(endpointsUpdateChs, endpointsUpdateCh) - cancelLastWatch = c.WatchEndpoints(testCDSName, func(update EndpointsUpdate, err error) { - endpointsUpdateCh.Send(endpointsUpdateErr{u: update, err: err}) - }) - if _, err := v2Client.addWatches[EndpointsResource].Receive(); i == 0 && err != nil { - t.Fatalf("want new watch to start, got error %v", err) - } - } - - wantUpdate := EndpointsUpdate{Localities: []Locality{testLocalities[0]}} - v2Client.r.NewEndpoints(map[string]EndpointsUpdate{ - testCDSName: wantUpdate, - }) - - for i := 0; i < count; i++ { - if u, err := endpointsUpdateChs[i].Receive(); err != nil || !cmp.Equal(u, endpointsUpdateErr{wantUpdate, nil}, endpointsCmpOpts...) { - t.Errorf("i=%v, unexpected endpointsUpdate: %v, error receiving from channel: %v", i, u, err) - } - } - - // Cancel the last watch, and send update again. - cancelLastWatch() - v2Client.r.NewEndpoints(map[string]EndpointsUpdate{ - testCDSName: wantUpdate, - }) - - for i := 0; i < count-1; i++ { - if u, err := endpointsUpdateChs[i].Receive(); err != nil || !cmp.Equal(u, endpointsUpdateErr{wantUpdate, nil}, endpointsCmpOpts...) { - t.Errorf("i=%v, unexpected endpointsUpdate: %v, error receiving from channel: %v", i, u, err) - } - } - - if u, err := endpointsUpdateChs[count-1].TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout { - t.Errorf("unexpected endpointsUpdate: %v, %v, want channel recv timeout", u, err) - } -} - -// TestEndpointsThreeWatchDifferentResourceName covers the case where an update is -// received after three watch() for different resource names. -func (s) TestEndpointsThreeWatchDifferentResourceName(t *testing.T) { - v2ClientCh, cleanup := overrideNewAPIClient() - defer cleanup() - - c, err := New(clientOpts(testXDSServer, false)) - if err != nil { - t.Fatalf("failed to create client: %v", err) - } - defer c.Close() - - v2Client := <-v2ClientCh - - var endpointsUpdateChs []*testutils.Channel - const count = 2 - - // Two watches for the same name. - for i := 0; i < count; i++ { - endpointsUpdateCh := testutils.NewChannel() - endpointsUpdateChs = append(endpointsUpdateChs, endpointsUpdateCh) - c.WatchEndpoints(testCDSName+"1", func(update EndpointsUpdate, err error) { - endpointsUpdateCh.Send(endpointsUpdateErr{u: update, err: err}) - }) - if _, err := v2Client.addWatches[EndpointsResource].Receive(); i == 0 && err != nil { - t.Fatalf("want new watch to start, got error %v", err) - } - } - - // Third watch for a different name. - endpointsUpdateCh2 := testutils.NewChannel() - c.WatchEndpoints(testCDSName+"2", func(update EndpointsUpdate, err error) { - endpointsUpdateCh2.Send(endpointsUpdateErr{u: update, err: err}) - }) - if _, err := v2Client.addWatches[EndpointsResource].Receive(); err != nil { - t.Fatalf("want new watch to start, got error %v", err) - } - - wantUpdate1 := EndpointsUpdate{Localities: []Locality{testLocalities[0]}} - wantUpdate2 := EndpointsUpdate{Localities: []Locality{testLocalities[1]}} - v2Client.r.NewEndpoints(map[string]EndpointsUpdate{ - testCDSName + "1": wantUpdate1, - testCDSName + "2": wantUpdate2, - }) - - for i := 0; i < count; i++ { - if u, err := endpointsUpdateChs[i].Receive(); err != nil || !cmp.Equal(u, endpointsUpdateErr{wantUpdate1, nil}, endpointsCmpOpts...) { - t.Errorf("i=%v, unexpected endpointsUpdate: %v, error receiving from channel: %v", i, u, err) - } - } - - if u, err := endpointsUpdateCh2.Receive(); err != nil || !cmp.Equal(u, endpointsUpdateErr{wantUpdate2, nil}, endpointsCmpOpts...) { - t.Errorf("unexpected endpointsUpdate: %v, error receiving from channel: %v", u, err) - } -} - -// TestEndpointsWatchAfterCache covers the case where watch is called after the update -// is in cache. -func (s) TestEndpointsWatchAfterCache(t *testing.T) { - v2ClientCh, cleanup := overrideNewAPIClient() - defer cleanup() - - c, err := New(clientOpts(testXDSServer, false)) - if err != nil { - t.Fatalf("failed to create client: %v", err) - } - defer c.Close() - - v2Client := <-v2ClientCh - - endpointsUpdateCh := testutils.NewChannel() - c.WatchEndpoints(testCDSName, func(update EndpointsUpdate, err error) { - endpointsUpdateCh.Send(endpointsUpdateErr{u: update, err: err}) - }) - if _, err := v2Client.addWatches[EndpointsResource].Receive(); err != nil { - t.Fatalf("want new watch to start, got error %v", err) - } - - wantUpdate := EndpointsUpdate{Localities: []Locality{testLocalities[0]}} - v2Client.r.NewEndpoints(map[string]EndpointsUpdate{ - testCDSName: wantUpdate, - }) - - if u, err := endpointsUpdateCh.Receive(); err != nil || !cmp.Equal(u, endpointsUpdateErr{wantUpdate, nil}, endpointsCmpOpts...) { - t.Errorf("unexpected endpointsUpdate: %v, error receiving from channel: %v", u, err) - } - - // Another watch for the resource in cache. - endpointsUpdateCh2 := testutils.NewChannel() - c.WatchEndpoints(testCDSName, func(update EndpointsUpdate, err error) { - endpointsUpdateCh2.Send(endpointsUpdateErr{u: update, err: err}) - }) - if n, err := v2Client.addWatches[EndpointsResource].Receive(); err == nil { - t.Fatalf("want no new watch to start (recv timeout), got resource name: %v error %v", n, err) - } - - // New watch should receives the update. - if u, err := endpointsUpdateCh2.Receive(); err != nil || !cmp.Equal(u, endpointsUpdateErr{wantUpdate, nil}, endpointsCmpOpts...) { - t.Errorf("unexpected endpointsUpdate: %v, error receiving from channel: %v", u, err) - } - - // Old watch should see nothing. - if u, err := endpointsUpdateCh.TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout { - t.Errorf("unexpected endpointsUpdate: %v, %v, want channel recv timeout", u, err) - } -} - -// TestEndpointsWatchExpiryTimer tests the case where the client does not receive -// an CDS response for the request that it sends out. We want the watch callback -// to be invoked with an error once the watchExpiryTimer fires. -func (s) TestEndpointsWatchExpiryTimer(t *testing.T) { - v2ClientCh, cleanup := overrideNewAPIClient() - defer cleanup() - - c, err := New(clientOpts(testXDSServer, true)) - if err != nil { - t.Fatalf("failed to create client: %v", err) - } - defer c.Close() - - v2Client := <-v2ClientCh - - endpointsUpdateCh := testutils.NewChannel() - c.WatchEndpoints(testCDSName, func(update EndpointsUpdate, err error) { - endpointsUpdateCh.Send(endpointsUpdateErr{u: update, err: err}) - }) - if _, err := v2Client.addWatches[EndpointsResource].Receive(); err != nil { - t.Fatalf("want new watch to start, got error %v", err) - } - - u, err := endpointsUpdateCh.TimedReceive(defaultTestWatchExpiryTimeout * 2) - if err != nil { - t.Fatalf("failed to get endpointsUpdate: %v", err) - } - uu := u.(endpointsUpdateErr) - if !cmp.Equal(uu.u, EndpointsUpdate{}, endpointsCmpOpts...) { - t.Errorf("unexpected endpointsUpdate: %v, want %v", uu.u, EndpointsUpdate{}) - } - if uu.err == nil { - t.Errorf("unexpected endpointsError: , want error watcher timeout") - } -} diff --git a/xds/internal/client/client_watchers_lds_test.go b/xds/internal/client/client_watchers_lds_test.go deleted file mode 100644 index b2bcef8c7237..000000000000 --- a/xds/internal/client/client_watchers_lds_test.go +++ /dev/null @@ -1,329 +0,0 @@ -/* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package client - -import ( - "testing" - - "google.golang.org/grpc/xds/internal/testutils" -) - -type ldsUpdateErr struct { - u ListenerUpdate - err error -} - -// TestLDSWatch covers the cases: -// - an update is received after a watch() -// - an update for another resource name -// - an update is received after cancel() -func (s) TestLDSWatch(t *testing.T) { - v2ClientCh, cleanup := overrideNewAPIClient() - defer cleanup() - - c, err := New(clientOpts(testXDSServer, false)) - if err != nil { - t.Fatalf("failed to create client: %v", err) - } - defer c.Close() - - v2Client := <-v2ClientCh - - ldsUpdateCh := testutils.NewChannel() - cancelWatch := c.watchLDS(testLDSName, func(update ListenerUpdate, err error) { - ldsUpdateCh.Send(ldsUpdateErr{u: update, err: err}) - }) - if _, err := v2Client.addWatches[ListenerResource].Receive(); err != nil { - t.Fatalf("want new watch to start, got error %v", err) - } - - wantUpdate := ListenerUpdate{RouteConfigName: testRDSName} - v2Client.r.NewListeners(map[string]ListenerUpdate{ - testLDSName: wantUpdate, - }) - - if u, err := ldsUpdateCh.Receive(); err != nil || u != (ldsUpdateErr{wantUpdate, nil}) { - t.Errorf("unexpected ListenerUpdate: %v, error receiving from channel: %v", u, err) - } - - // Another update, with an extra resource for a different resource name. - v2Client.r.NewListeners(map[string]ListenerUpdate{ - testLDSName: wantUpdate, - "randomName": {}, - }) - - if u, err := ldsUpdateCh.Receive(); err != nil || u != (ldsUpdateErr{wantUpdate, nil}) { - t.Errorf("unexpected ListenerUpdate: %v, %v, want channel recv timeout", u, err) - } - - // Cancel watch, and send update again. - cancelWatch() - v2Client.r.NewListeners(map[string]ListenerUpdate{ - testLDSName: wantUpdate, - }) - - if u, err := ldsUpdateCh.TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout { - t.Errorf("unexpected ListenerUpdate: %v, %v, want channel recv timeout", u, err) - } -} - -// TestLDSTwoWatchSameResourceName covers the case where an update is received -// after two watch() for the same resource name. -func (s) TestLDSTwoWatchSameResourceName(t *testing.T) { - v2ClientCh, cleanup := overrideNewAPIClient() - defer cleanup() - - c, err := New(clientOpts(testXDSServer, false)) - if err != nil { - t.Fatalf("failed to create client: %v", err) - } - defer c.Close() - - v2Client := <-v2ClientCh - - var ldsUpdateChs []*testutils.Channel - const count = 2 - - var cancelLastWatch func() - - for i := 0; i < count; i++ { - ldsUpdateCh := testutils.NewChannel() - ldsUpdateChs = append(ldsUpdateChs, ldsUpdateCh) - cancelLastWatch = c.watchLDS(testLDSName, func(update ListenerUpdate, err error) { - ldsUpdateCh.Send(ldsUpdateErr{u: update, err: err}) - }) - if _, err := v2Client.addWatches[ListenerResource].Receive(); i == 0 && err != nil { - t.Fatalf("want new watch to start, got error %v", err) - } - } - - wantUpdate := ListenerUpdate{RouteConfigName: testRDSName} - v2Client.r.NewListeners(map[string]ListenerUpdate{ - testLDSName: wantUpdate, - }) - - for i := 0; i < count; i++ { - if u, err := ldsUpdateChs[i].Receive(); err != nil || u != (ldsUpdateErr{wantUpdate, nil}) { - t.Errorf("i=%v, unexpected ListenerUpdate: %v, error receiving from channel: %v", i, u, err) - } - } - - // Cancel the last watch, and send update again. - cancelLastWatch() - v2Client.r.NewListeners(map[string]ListenerUpdate{ - testLDSName: wantUpdate, - }) - - for i := 0; i < count-1; i++ { - if u, err := ldsUpdateChs[i].Receive(); err != nil || u != (ldsUpdateErr{wantUpdate, nil}) { - t.Errorf("i=%v, unexpected ListenerUpdate: %v, error receiving from channel: %v", i, u, err) - } - } - - if u, err := ldsUpdateChs[count-1].TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout { - t.Errorf("unexpected ListenerUpdate: %v, %v, want channel recv timeout", u, err) - } -} - -// TestLDSThreeWatchDifferentResourceName covers the case where an update is -// received after three watch() for different resource names. -func (s) TestLDSThreeWatchDifferentResourceName(t *testing.T) { - v2ClientCh, cleanup := overrideNewAPIClient() - defer cleanup() - - c, err := New(clientOpts(testXDSServer, false)) - if err != nil { - t.Fatalf("failed to create client: %v", err) - } - defer c.Close() - - v2Client := <-v2ClientCh - - var ldsUpdateChs []*testutils.Channel - const count = 2 - - // Two watches for the same name. - for i := 0; i < count; i++ { - ldsUpdateCh := testutils.NewChannel() - ldsUpdateChs = append(ldsUpdateChs, ldsUpdateCh) - c.watchLDS(testLDSName+"1", func(update ListenerUpdate, err error) { - ldsUpdateCh.Send(ldsUpdateErr{u: update, err: err}) - }) - if _, err := v2Client.addWatches[ListenerResource].Receive(); i == 0 && err != nil { - t.Fatalf("want new watch to start, got error %v", err) - } - } - - // Third watch for a different name. - ldsUpdateCh2 := testutils.NewChannel() - c.watchLDS(testLDSName+"2", func(update ListenerUpdate, err error) { - ldsUpdateCh2.Send(ldsUpdateErr{u: update, err: err}) - }) - if _, err := v2Client.addWatches[ListenerResource].Receive(); err != nil { - t.Fatalf("want new watch to start, got error %v", err) - } - - wantUpdate1 := ListenerUpdate{RouteConfigName: testRDSName + "1"} - wantUpdate2 := ListenerUpdate{RouteConfigName: testRDSName + "2"} - v2Client.r.NewListeners(map[string]ListenerUpdate{ - testLDSName + "1": wantUpdate1, - testLDSName + "2": wantUpdate2, - }) - - for i := 0; i < count; i++ { - if u, err := ldsUpdateChs[i].Receive(); err != nil || u != (ldsUpdateErr{wantUpdate1, nil}) { - t.Errorf("i=%v, unexpected ListenerUpdate: %v, error receiving from channel: %v", i, u, err) - } - } - - if u, err := ldsUpdateCh2.Receive(); err != nil || u != (ldsUpdateErr{wantUpdate2, nil}) { - t.Errorf("unexpected ListenerUpdate: %v, error receiving from channel: %v", u, err) - } -} - -// TestLDSWatchAfterCache covers the case where watch is called after the update -// is in cache. -func (s) TestLDSWatchAfterCache(t *testing.T) { - v2ClientCh, cleanup := overrideNewAPIClient() - defer cleanup() - - c, err := New(clientOpts(testXDSServer, false)) - if err != nil { - t.Fatalf("failed to create client: %v", err) - } - defer c.Close() - - v2Client := <-v2ClientCh - - ldsUpdateCh := testutils.NewChannel() - c.watchLDS(testLDSName, func(update ListenerUpdate, err error) { - ldsUpdateCh.Send(ldsUpdateErr{u: update, err: err}) - }) - if _, err := v2Client.addWatches[ListenerResource].Receive(); err != nil { - t.Fatalf("want new watch to start, got error %v", err) - } - - wantUpdate := ListenerUpdate{RouteConfigName: testRDSName} - v2Client.r.NewListeners(map[string]ListenerUpdate{ - testLDSName: wantUpdate, - }) - - if u, err := ldsUpdateCh.Receive(); err != nil || u != (ldsUpdateErr{wantUpdate, nil}) { - t.Errorf("unexpected ListenerUpdate: %v, error receiving from channel: %v", u, err) - } - - // Another watch for the resource in cache. - ldsUpdateCh2 := testutils.NewChannel() - c.watchLDS(testLDSName, func(update ListenerUpdate, err error) { - ldsUpdateCh2.Send(ldsUpdateErr{u: update, err: err}) - }) - if n, err := v2Client.addWatches[ListenerResource].Receive(); err == nil { - t.Fatalf("want no new watch to start (recv timeout), got resource name: %v error %v", n, err) - } - - // New watch should receives the update. - if u, err := ldsUpdateCh2.Receive(); err != nil || u != (ldsUpdateErr{wantUpdate, nil}) { - t.Errorf("unexpected ListenerUpdate: %v, error receiving from channel: %v", u, err) - } - - // Old watch should see nothing. - if u, err := ldsUpdateCh.TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout { - t.Errorf("unexpected ListenerUpdate: %v, %v, want channel recv timeout", u, err) - } -} - -// TestLDSResourceRemoved covers the cases: -// - an update is received after a watch() -// - another update is received, with one resource removed -// - this should trigger callback with resource removed error -// - one more update without the removed resource -// - the callback (above) shouldn't receive any update -func (s) TestLDSResourceRemoved(t *testing.T) { - v2ClientCh, cleanup := overrideNewAPIClient() - defer cleanup() - - c, err := New(clientOpts(testXDSServer, false)) - if err != nil { - t.Fatalf("failed to create client: %v", err) - } - defer c.Close() - - v2Client := <-v2ClientCh - - ldsUpdateCh1 := testutils.NewChannel() - c.watchLDS(testLDSName+"1", func(update ListenerUpdate, err error) { - ldsUpdateCh1.Send(ldsUpdateErr{u: update, err: err}) - }) - if _, err := v2Client.addWatches[ListenerResource].Receive(); err != nil { - t.Fatalf("want new watch to start, got error %v", err) - } - // Another watch for a different name. - ldsUpdateCh2 := testutils.NewChannel() - c.watchLDS(testLDSName+"2", func(update ListenerUpdate, err error) { - ldsUpdateCh2.Send(ldsUpdateErr{u: update, err: err}) - }) - if _, err := v2Client.addWatches[ListenerResource].Receive(); err != nil { - t.Fatalf("want new watch to start, got error %v", err) - } - - wantUpdate1 := ListenerUpdate{RouteConfigName: testEDSName + "1"} - wantUpdate2 := ListenerUpdate{RouteConfigName: testEDSName + "2"} - v2Client.r.NewListeners(map[string]ListenerUpdate{ - testLDSName + "1": wantUpdate1, - testLDSName + "2": wantUpdate2, - }) - - if u, err := ldsUpdateCh1.Receive(); err != nil || u != (ldsUpdateErr{wantUpdate1, nil}) { - t.Errorf("unexpected ListenerUpdate: %v, error receiving from channel: %v", u, err) - } - - if u, err := ldsUpdateCh2.Receive(); err != nil || u != (ldsUpdateErr{wantUpdate2, nil}) { - t.Errorf("unexpected ListenerUpdate: %v, error receiving from channel: %v", u, err) - } - - // Send another update to remove resource 1. - v2Client.r.NewListeners(map[string]ListenerUpdate{ - testLDSName + "2": wantUpdate2, - }) - - // watcher 1 should get an error. - if u, err := ldsUpdateCh1.Receive(); err != nil || ErrType(u.(ldsUpdateErr).err) != ErrorTypeResourceNotFound { - t.Errorf("unexpected ListenerUpdate: %v, error receiving from channel: %v, want update with error resource not found", u, err) - } - - // watcher 2 should get the same update again. - if u, err := ldsUpdateCh2.Receive(); err != nil || u != (ldsUpdateErr{wantUpdate2, nil}) { - t.Errorf("unexpected ListenerUpdate: %v, error receiving from channel: %v", u, err) - } - - // Send one more update without resource 1. - v2Client.r.NewListeners(map[string]ListenerUpdate{ - testLDSName + "2": wantUpdate2, - }) - - // watcher 1 should get an error. - if u, err := ldsUpdateCh1.Receive(); err != testutils.ErrRecvTimeout { - t.Errorf("unexpected ListenerUpdate: %v, want receiving from channel timeout", u) - } - - // watcher 2 should get the same update again. - if u, err := ldsUpdateCh2.Receive(); err != nil || u != (ldsUpdateErr{wantUpdate2, nil}) { - t.Errorf("unexpected ListenerUpdate: %v, error receiving from channel: %v", u, err) - } -} diff --git a/xds/internal/client/client_watchers_rds_test.go b/xds/internal/client/client_watchers_rds_test.go deleted file mode 100644 index f9bb30d559f4..000000000000 --- a/xds/internal/client/client_watchers_rds_test.go +++ /dev/null @@ -1,249 +0,0 @@ -/* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package client - -import ( - "testing" - - "github.com/google/go-cmp/cmp" - "google.golang.org/grpc/xds/internal/testutils" -) - -type rdsUpdateErr struct { - u RouteConfigUpdate - err error -} - -// TestRDSWatch covers the cases: -// - an update is received after a watch() -// - an update for another resource name (which doesn't trigger callback) -// - an update is received after cancel() -func (s) TestRDSWatch(t *testing.T) { - v2ClientCh, cleanup := overrideNewAPIClient() - defer cleanup() - - c, err := New(clientOpts(testXDSServer, false)) - if err != nil { - t.Fatalf("failed to create client: %v", err) - } - defer c.Close() - - v2Client := <-v2ClientCh - - rdsUpdateCh := testutils.NewChannel() - cancelWatch := c.watchRDS(testRDSName, func(update RouteConfigUpdate, err error) { - rdsUpdateCh.Send(rdsUpdateErr{u: update, err: err}) - }) - if _, err := v2Client.addWatches[RouteConfigResource].Receive(); err != nil { - t.Fatalf("want new watch to start, got error %v", err) - } - - wantUpdate := RouteConfigUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}} - v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{ - testRDSName: wantUpdate, - }) - - if u, err := rdsUpdateCh.Receive(); err != nil || !cmp.Equal(u, rdsUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(rdsUpdateErr{})) { - t.Errorf("unexpected RouteConfigUpdate: %v, error receiving from channel: %v", u, err) - } - - // Another update for a different resource name. - v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{ - "randomName": {}, - }) - - if u, err := rdsUpdateCh.TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout { - t.Errorf("unexpected RouteConfigUpdate: %v, %v, want channel recv timeout", u, err) - } - - // Cancel watch, and send update again. - cancelWatch() - v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{ - testRDSName: wantUpdate, - }) - - if u, err := rdsUpdateCh.TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout { - t.Errorf("unexpected RouteConfigUpdate: %v, %v, want channel recv timeout", u, err) - } -} - -// TestRDSTwoWatchSameResourceName covers the case where an update is received -// after two watch() for the same resource name. -func (s) TestRDSTwoWatchSameResourceName(t *testing.T) { - v2ClientCh, cleanup := overrideNewAPIClient() - defer cleanup() - - c, err := New(clientOpts(testXDSServer, false)) - if err != nil { - t.Fatalf("failed to create client: %v", err) - } - defer c.Close() - - v2Client := <-v2ClientCh - - var rdsUpdateChs []*testutils.Channel - const count = 2 - - var cancelLastWatch func() - - for i := 0; i < count; i++ { - rdsUpdateCh := testutils.NewChannel() - rdsUpdateChs = append(rdsUpdateChs, rdsUpdateCh) - cancelLastWatch = c.watchRDS(testRDSName, func(update RouteConfigUpdate, err error) { - rdsUpdateCh.Send(rdsUpdateErr{u: update, err: err}) - }) - if _, err := v2Client.addWatches[RouteConfigResource].Receive(); i == 0 && err != nil { - t.Fatalf("want new watch to start, got error %v", err) - } - } - - wantUpdate := RouteConfigUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}} - v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{ - testRDSName: wantUpdate, - }) - - for i := 0; i < count; i++ { - if u, err := rdsUpdateChs[i].Receive(); err != nil || !cmp.Equal(u, rdsUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(rdsUpdateErr{})) { - t.Errorf("i=%v, unexpected RouteConfigUpdate: %v, error receiving from channel: %v", i, u, err) - } - } - - // Cancel the last watch, and send update again. - cancelLastWatch() - v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{ - testRDSName: wantUpdate, - }) - - for i := 0; i < count-1; i++ { - if u, err := rdsUpdateChs[i].Receive(); err != nil || !cmp.Equal(u, rdsUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(rdsUpdateErr{})) { - t.Errorf("i=%v, unexpected RouteConfigUpdate: %v, error receiving from channel: %v", i, u, err) - } - } - - if u, err := rdsUpdateChs[count-1].TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout { - t.Errorf("unexpected RouteConfigUpdate: %v, %v, want channel recv timeout", u, err) - } -} - -// TestRDSThreeWatchDifferentResourceName covers the case where an update is -// received after three watch() for different resource names. -func (s) TestRDSThreeWatchDifferentResourceName(t *testing.T) { - v2ClientCh, cleanup := overrideNewAPIClient() - defer cleanup() - - c, err := New(clientOpts(testXDSServer, false)) - if err != nil { - t.Fatalf("failed to create client: %v", err) - } - defer c.Close() - - v2Client := <-v2ClientCh - - var rdsUpdateChs []*testutils.Channel - const count = 2 - - // Two watches for the same name. - for i := 0; i < count; i++ { - rdsUpdateCh := testutils.NewChannel() - rdsUpdateChs = append(rdsUpdateChs, rdsUpdateCh) - c.watchRDS(testRDSName+"1", func(update RouteConfigUpdate, err error) { - rdsUpdateCh.Send(rdsUpdateErr{u: update, err: err}) - }) - if _, err := v2Client.addWatches[RouteConfigResource].Receive(); i == 0 && err != nil { - t.Fatalf("want new watch to start, got error %v", err) - } - } - - // Third watch for a different name. - rdsUpdateCh2 := testutils.NewChannel() - c.watchRDS(testRDSName+"2", func(update RouteConfigUpdate, err error) { - rdsUpdateCh2.Send(rdsUpdateErr{u: update, err: err}) - }) - if _, err := v2Client.addWatches[RouteConfigResource].Receive(); err != nil { - t.Fatalf("want new watch to start, got error %v", err) - } - - wantUpdate1 := RouteConfigUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "1": 1}}}} - wantUpdate2 := RouteConfigUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "2": 1}}}} - v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{ - testRDSName + "1": wantUpdate1, - testRDSName + "2": wantUpdate2, - }) - - for i := 0; i < count; i++ { - if u, err := rdsUpdateChs[i].Receive(); err != nil || !cmp.Equal(u, rdsUpdateErr{wantUpdate1, nil}, cmp.AllowUnexported(rdsUpdateErr{})) { - t.Errorf("i=%v, unexpected RouteConfigUpdate: %v, error receiving from channel: %v", i, u, err) - } - } - - if u, err := rdsUpdateCh2.Receive(); err != nil || !cmp.Equal(u, rdsUpdateErr{wantUpdate2, nil}, cmp.AllowUnexported(rdsUpdateErr{})) { - t.Errorf("unexpected RouteConfigUpdate: %v, error receiving from channel: %v", u, err) - } -} - -// TestRDSWatchAfterCache covers the case where watch is called after the update -// is in cache. -func (s) TestRDSWatchAfterCache(t *testing.T) { - v2ClientCh, cleanup := overrideNewAPIClient() - defer cleanup() - - c, err := New(clientOpts(testXDSServer, false)) - if err != nil { - t.Fatalf("failed to create client: %v", err) - } - defer c.Close() - - v2Client := <-v2ClientCh - - rdsUpdateCh := testutils.NewChannel() - c.watchRDS(testRDSName, func(update RouteConfigUpdate, err error) { - rdsUpdateCh.Send(rdsUpdateErr{u: update, err: err}) - }) - if _, err := v2Client.addWatches[RouteConfigResource].Receive(); err != nil { - t.Fatalf("want new watch to start, got error %v", err) - } - - wantUpdate := RouteConfigUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}} - v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{ - testRDSName: wantUpdate, - }) - - if u, err := rdsUpdateCh.Receive(); err != nil || !cmp.Equal(u, rdsUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(rdsUpdateErr{})) { - t.Errorf("unexpected RouteConfigUpdate: %v, error receiving from channel: %v", u, err) - } - - // Another watch for the resource in cache. - rdsUpdateCh2 := testutils.NewChannel() - c.watchRDS(testRDSName, func(update RouteConfigUpdate, err error) { - rdsUpdateCh2.Send(rdsUpdateErr{u: update, err: err}) - }) - if n, err := v2Client.addWatches[RouteConfigResource].Receive(); err == nil { - t.Fatalf("want no new watch to start (recv timeout), got resource name: %v error %v", n, err) - } - - // New watch should receives the update. - if u, err := rdsUpdateCh2.Receive(); err != nil || !cmp.Equal(u, rdsUpdateErr{wantUpdate, nil}, cmp.AllowUnexported(rdsUpdateErr{})) { - t.Errorf("unexpected RouteConfigUpdate: %v, error receiving from channel: %v", u, err) - } - - // Old watch should see nothing. - if u, err := rdsUpdateCh.TimedReceive(chanRecvTimeout); err != testutils.ErrRecvTimeout { - t.Errorf("unexpected RouteConfigUpdate: %v, %v, want channel recv timeout", u, err) - } -} diff --git a/xds/internal/client/client_watchers_service_test.go b/xds/internal/client/client_watchers_service_test.go deleted file mode 100644 index 6f994686b8e4..000000000000 --- a/xds/internal/client/client_watchers_service_test.go +++ /dev/null @@ -1,478 +0,0 @@ -/* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package client - -import ( - "testing" - - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" - - "google.golang.org/grpc/xds/internal/testutils" -) - -type serviceUpdateErr struct { - u ServiceUpdate - err error -} - -var serviceCmpOpts = []cmp.Option{cmp.AllowUnexported(serviceUpdateErr{}), cmpopts.EquateEmpty()} - -// TestServiceWatch covers the cases: -// - an update is received after a watch() -// - an update with routes received -func (s) TestServiceWatch(t *testing.T) { - v2ClientCh, cleanup := overrideNewAPIClient() - defer cleanup() - - c, err := New(clientOpts(testXDSServer, false)) - if err != nil { - t.Fatalf("failed to create client: %v", err) - } - defer c.Close() - - v2Client := <-v2ClientCh - - serviceUpdateCh := testutils.NewChannel() - c.WatchService(testLDSName, func(update ServiceUpdate, err error) { - serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err}) - }) - - wantUpdate := ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}} - - if _, err := v2Client.addWatches[ListenerResource].Receive(); err != nil { - t.Fatalf("want new watch to start, got error %v", err) - } - v2Client.r.NewListeners(map[string]ListenerUpdate{ - testLDSName: {RouteConfigName: testRDSName}, - }) - if _, err := v2Client.addWatches[RouteConfigResource].Receive(); err != nil { - t.Fatalf("want new watch to start, got error %v", err) - } - v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{ - testRDSName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}, - }) - - if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, serviceCmpOpts...) { - t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err) - } - - wantUpdate2 := ServiceUpdate{ - Routes: []*Route{{ - Prefix: newStringP(""), - Action: map[string]uint32{testCDSName: 1}, - }}, - } - v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{ - testRDSName: { - Routes: []*Route{{ - Prefix: newStringP(""), - Action: map[string]uint32{testCDSName: 1}, - }}, - }, - }) - if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate2, nil}, serviceCmpOpts...) { - t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err) - } -} - -// TestServiceWatchLDSUpdate covers the case that after first LDS and first RDS -// response, the second LDS response trigger an new RDS watch, and an update of -// the old RDS watch doesn't trigger update to service callback. -func (s) TestServiceWatchLDSUpdate(t *testing.T) { - v2ClientCh, cleanup := overrideNewAPIClient() - defer cleanup() - - c, err := New(clientOpts(testXDSServer, false)) - if err != nil { - t.Fatalf("failed to create client: %v", err) - } - defer c.Close() - - v2Client := <-v2ClientCh - - serviceUpdateCh := testutils.NewChannel() - c.WatchService(testLDSName, func(update ServiceUpdate, err error) { - serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err}) - }) - - wantUpdate := ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}} - - if _, err := v2Client.addWatches[ListenerResource].Receive(); err != nil { - t.Fatalf("want new watch to start, got error %v", err) - } - v2Client.r.NewListeners(map[string]ListenerUpdate{ - testLDSName: {RouteConfigName: testRDSName}, - }) - if _, err := v2Client.addWatches[RouteConfigResource].Receive(); err != nil { - t.Fatalf("want new watch to start, got error %v", err) - } - v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{ - testRDSName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}, - }) - - if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, serviceCmpOpts...) { - t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err) - } - - // Another LDS update with a different RDS_name. - v2Client.r.NewListeners(map[string]ListenerUpdate{ - testLDSName: {RouteConfigName: testRDSName + "2"}, - }) - if _, err := v2Client.addWatches[RouteConfigResource].Receive(); err != nil { - t.Fatalf("want new watch to start, got error %v", err) - } - - // Another update for the old name. - v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{ - testRDSName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}, - }) - - if u, err := serviceUpdateCh.Receive(); err != testutils.ErrRecvTimeout { - t.Errorf("unexpected serviceUpdate: %v, %v, want channel recv timeout", u, err) - } - - wantUpdate2 := ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "2": 1}}}} - // RDS update for the new name. - v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{ - testRDSName + "2": {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "2": 1}}}}, - }) - - if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate2, nil}, serviceCmpOpts...) { - t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err) - } -} - -// TestServiceWatchSecond covers the case where a second WatchService() gets an -// error (because only one is allowed). But the first watch still receives -// updates. -func (s) TestServiceWatchSecond(t *testing.T) { - v2ClientCh, cleanup := overrideNewAPIClient() - defer cleanup() - - c, err := New(clientOpts(testXDSServer, false)) - if err != nil { - t.Fatalf("failed to create client: %v", err) - } - defer c.Close() - - v2Client := <-v2ClientCh - - serviceUpdateCh := testutils.NewChannel() - c.WatchService(testLDSName, func(update ServiceUpdate, err error) { - serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err}) - }) - - wantUpdate := ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}} - - if _, err := v2Client.addWatches[ListenerResource].Receive(); err != nil { - t.Fatalf("want new watch to start, got error %v", err) - } - v2Client.r.NewListeners(map[string]ListenerUpdate{ - testLDSName: {RouteConfigName: testRDSName}, - }) - if _, err := v2Client.addWatches[RouteConfigResource].Receive(); err != nil { - t.Fatalf("want new watch to start, got error %v", err) - } - v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{ - testRDSName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}, - }) - - if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, serviceCmpOpts...) { - t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err) - } - - serviceUpdateCh2 := testutils.NewChannel() - // Call WatchService() again, with the same or different name. - c.WatchService(testLDSName, func(update ServiceUpdate, err error) { - serviceUpdateCh2.Send(serviceUpdateErr{u: update, err: err}) - }) - - u, err := serviceUpdateCh2.Receive() - if err != nil { - t.Fatalf("failed to get serviceUpdate: %v", err) - } - uu := u.(serviceUpdateErr) - if !cmp.Equal(uu.u, ServiceUpdate{}) { - t.Errorf("unexpected serviceUpdate: %v, want %v", uu.u, ServiceUpdate{}) - } - if uu.err == nil { - t.Errorf("unexpected serviceError: , want error watcher timeout") - } - - // Send update again, first callback should be called, second should - // timeout. - v2Client.r.NewListeners(map[string]ListenerUpdate{ - testLDSName: {RouteConfigName: testRDSName}, - }) - v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{ - testRDSName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}, - }) - - if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, serviceCmpOpts...) { - t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err) - } - - if u, err := serviceUpdateCh2.Receive(); err != testutils.ErrRecvTimeout { - t.Errorf("unexpected serviceUpdate: %v, %v, want channel recv timeout", u, err) - } -} - -// TestServiceWatchWithNoResponseFromServer tests the case where the xDS server -// does not respond to the requests being sent out as part of registering a -// service update watcher. The callback will get an error. -func (s) TestServiceWatchWithNoResponseFromServer(t *testing.T) { - v2ClientCh, cleanup := overrideNewAPIClient() - defer cleanup() - - c, err := New(clientOpts(testXDSServer, true)) - if err != nil { - t.Fatalf("failed to create client: %v", err) - } - defer c.Close() - - v2Client := <-v2ClientCh - - serviceUpdateCh := testutils.NewChannel() - c.WatchService(testLDSName, func(update ServiceUpdate, err error) { - serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err}) - }) - if _, err := v2Client.addWatches[ListenerResource].Receive(); err != nil { - t.Fatalf("want new watch to start, got error %v", err) - } - u, err := serviceUpdateCh.TimedReceive(defaultTestWatchExpiryTimeout * 2) - if err != nil { - t.Fatalf("failed to get serviceUpdate: %v", err) - } - uu := u.(serviceUpdateErr) - if !cmp.Equal(uu.u, ServiceUpdate{}) { - t.Errorf("unexpected serviceUpdate: %v, want %v", uu.u, ServiceUpdate{}) - } - if uu.err == nil { - t.Errorf("unexpected serviceError: , want error watcher timeout") - } -} - -// TestServiceWatchEmptyRDS tests the case where the underlying v2Client -// receives an empty RDS response. The callback will get an error. -func (s) TestServiceWatchEmptyRDS(t *testing.T) { - v2ClientCh, cleanup := overrideNewAPIClient() - defer cleanup() - - c, err := New(clientOpts(testXDSServer, true)) - if err != nil { - t.Fatalf("failed to create client: %v", err) - } - defer c.Close() - - v2Client := <-v2ClientCh - - serviceUpdateCh := testutils.NewChannel() - c.WatchService(testLDSName, func(update ServiceUpdate, err error) { - serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err}) - }) - - if _, err := v2Client.addWatches[ListenerResource].Receive(); err != nil { - t.Fatalf("want new watch to start, got error %v", err) - } - v2Client.r.NewListeners(map[string]ListenerUpdate{ - testLDSName: {RouteConfigName: testRDSName}, - }) - if _, err := v2Client.addWatches[RouteConfigResource].Receive(); err != nil { - t.Fatalf("want new watch to start, got error %v", err) - } - v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{}) - u, err := serviceUpdateCh.TimedReceive(defaultTestWatchExpiryTimeout * 2) - if err != nil { - t.Fatalf("failed to get serviceUpdate: %v", err) - } - uu := u.(serviceUpdateErr) - if !cmp.Equal(uu.u, ServiceUpdate{}) { - t.Errorf("unexpected serviceUpdate: %v, want %v", uu.u, ServiceUpdate{}) - } - if uu.err == nil { - t.Errorf("unexpected serviceError: , want error watcher timeout") - } -} - -// TestServiceWatchWithClientClose tests the case where xDS responses are -// received after the client is closed, and we make sure that the registered -// watcher callback is not invoked. -func (s) TestServiceWatchWithClientClose(t *testing.T) { - v2ClientCh, cleanup := overrideNewAPIClient() - defer cleanup() - - c, err := New(clientOpts(testXDSServer, true)) - if err != nil { - t.Fatalf("failed to create client: %v", err) - } - defer c.Close() - - v2Client := <-v2ClientCh - - serviceUpdateCh := testutils.NewChannel() - c.WatchService(testLDSName, func(update ServiceUpdate, err error) { - serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err}) - }) - - if _, err := v2Client.addWatches[ListenerResource].Receive(); err != nil { - t.Fatalf("want new watch to start, got error %v", err) - } - v2Client.r.NewListeners(map[string]ListenerUpdate{ - testLDSName: {RouteConfigName: testRDSName}, - }) - if _, err := v2Client.addWatches[RouteConfigResource].Receive(); err != nil { - t.Fatalf("want new watch to start, got error %v", err) - } - // Client is closed before it receives the RDS response. - c.Close() - if u, err := serviceUpdateCh.TimedReceive(defaultTestWatchExpiryTimeout * 2); err != testutils.ErrRecvTimeout { - t.Errorf("unexpected serviceUpdate: %v, %v, want channel recv timeout", u, err) - } -} - -// TestServiceNotCancelRDSOnSameLDSUpdate covers the case that if the second LDS -// update contains the same RDS name as the previous, the RDS watch isn't -// canceled and restarted. -func (s) TestServiceNotCancelRDSOnSameLDSUpdate(t *testing.T) { - v2ClientCh, cleanup := overrideNewAPIClient() - defer cleanup() - - c, err := New(clientOpts(testXDSServer, false)) - if err != nil { - t.Fatalf("failed to create client: %v", err) - } - defer c.Close() - - v2Client := <-v2ClientCh - - serviceUpdateCh := testutils.NewChannel() - c.WatchService(testLDSName, func(update ServiceUpdate, err error) { - serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err}) - }) - - wantUpdate := ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}} - - if _, err := v2Client.addWatches[ListenerResource].Receive(); err != nil { - t.Fatalf("want new watch to start, got error %v", err) - } - v2Client.r.NewListeners(map[string]ListenerUpdate{ - testLDSName: {RouteConfigName: testRDSName}, - }) - if _, err := v2Client.addWatches[RouteConfigResource].Receive(); err != nil { - t.Fatalf("want new watch to start, got error %v", err) - } - v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{ - testRDSName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}, - }) - - if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, serviceCmpOpts...) { - t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err) - } - - // Another LDS update with a the same RDS_name. - v2Client.r.NewListeners(map[string]ListenerUpdate{ - testLDSName: {RouteConfigName: testRDSName}, - }) - if v, err := v2Client.removeWatches[RouteConfigResource].Receive(); err == nil { - t.Fatalf("unexpected rds watch cancel: %v", v) - } -} - -// TestServiceResourceRemoved covers the cases: -// - an update is received after a watch() -// - another update is received, with one resource removed -// - this should trigger callback with resource removed error -// - one more update without the removed resource -// - the callback (above) shouldn't receive any update -func (s) TestServiceResourceRemoved(t *testing.T) { - v2ClientCh, cleanup := overrideNewAPIClient() - defer cleanup() - - c, err := New(clientOpts(testXDSServer, false)) - if err != nil { - t.Fatalf("failed to create client: %v", err) - } - defer c.Close() - - v2Client := <-v2ClientCh - - serviceUpdateCh := testutils.NewChannel() - c.WatchService(testLDSName, func(update ServiceUpdate, err error) { - serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err}) - }) - - wantUpdate := ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}} - - if _, err := v2Client.addWatches[ListenerResource].Receive(); err != nil { - t.Fatalf("want new watch to start, got error %v", err) - } - v2Client.r.NewListeners(map[string]ListenerUpdate{ - testLDSName: {RouteConfigName: testRDSName}, - }) - if _, err := v2Client.addWatches[RouteConfigResource].Receive(); err != nil { - t.Fatalf("want new watch to start, got error %v", err) - } - v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{ - testRDSName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName: 1}}}}, - }) - - if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{wantUpdate, nil}, serviceCmpOpts...) { - t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err) - } - - // Remove LDS resource, should cancel the RDS watch, and trigger resource - // removed error. - v2Client.r.NewListeners(map[string]ListenerUpdate{}) - if _, err := v2Client.removeWatches[RouteConfigResource].Receive(); err != nil { - t.Fatalf("want watch to be canceled, got error %v", err) - } - if u, err := serviceUpdateCh.Receive(); err != nil || ErrType(u.(serviceUpdateErr).err) != ErrorTypeResourceNotFound { - t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v, want update with error resource not found", u, err) - } - - // Send RDS update for the removed LDS resource, expect no updates to - // callback, because RDS should be canceled. - v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{ - testRDSName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "new": 1}}}}, - }) - if u, err := serviceUpdateCh.Receive(); err != testutils.ErrRecvTimeout { - t.Errorf("unexpected serviceUpdate: %v, want receiving from channel timeout", u) - } - - // Add LDS resource, but not RDS resource, should - // - start a new RDS watch - // - timeout on service channel, because RDS cache was cleared - v2Client.r.NewListeners(map[string]ListenerUpdate{ - testLDSName: {RouteConfigName: testRDSName}, - }) - if _, err := v2Client.addWatches[RouteConfigResource].Receive(); err != nil { - t.Fatalf("want new watch to start, got error %v", err) - } - if u, err := serviceUpdateCh.Receive(); err != testutils.ErrRecvTimeout { - t.Errorf("unexpected serviceUpdate: %v, want receiving from channel timeout", u) - } - - v2Client.r.NewRouteConfigs(map[string]RouteConfigUpdate{ - testRDSName: {Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "new2": 1}}}}, - }) - if u, err := serviceUpdateCh.Receive(); err != nil || !cmp.Equal(u, serviceUpdateErr{ServiceUpdate{Routes: []*Route{{Prefix: newStringP(""), Action: map[string]uint32{testCDSName + "new2": 1}}}}, nil}, serviceCmpOpts...) { - t.Errorf("unexpected serviceUpdate: %v, error receiving from channel: %v", u, err) - } -} diff --git a/xds/internal/client/client_xds.go b/xds/internal/client/client_xds.go deleted file mode 100644 index f5f60a4f9460..000000000000 --- a/xds/internal/client/client_xds.go +++ /dev/null @@ -1,507 +0,0 @@ -/* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package client - -import ( - "fmt" - "net" - "strconv" - "strings" - - v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" - v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" - v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" - v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" - v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" - v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" - v3typepb "github.com/envoyproxy/go-control-plane/envoy/type/v3" - "github.com/golang/protobuf/proto" - anypb "github.com/golang/protobuf/ptypes/any" - - "google.golang.org/grpc/internal/grpclog" - "google.golang.org/grpc/xds/internal" -) - -// UnmarshalListener processes resources received in an LDS response, validates -// them, and transforms them into a native struct which contains only fields we -// are interested in. -func UnmarshalListener(resources []*anypb.Any, logger *grpclog.PrefixLogger) (map[string]ListenerUpdate, error) { - update := make(map[string]ListenerUpdate) - for _, r := range resources { - if !IsListenerResource(r.GetTypeUrl()) { - return nil, fmt.Errorf("xds: unexpected resource type: %s in LDS response", r.GetTypeUrl()) - } - lis := &v3listenerpb.Listener{} - if err := proto.Unmarshal(r.GetValue(), lis); err != nil { - return nil, fmt.Errorf("xds: failed to unmarshal resource in LDS response: %v", err) - } - logger.Infof("Resource with name: %v, type: %T, contains: %v", lis.GetName(), lis, lis) - routeName, err := getRouteConfigNameFromListener(lis, logger) - if err != nil { - return nil, err - } - update[lis.GetName()] = ListenerUpdate{RouteConfigName: routeName} - } - return update, nil -} - -// getRouteConfigNameFromListener checks if the provided Listener proto meets -// the expected criteria. If so, it returns a non-empty routeConfigName. -func getRouteConfigNameFromListener(lis *v3listenerpb.Listener, logger *grpclog.PrefixLogger) (string, error) { - if lis.GetApiListener() == nil { - return "", fmt.Errorf("xds: no api_listener field in LDS response %+v", lis) - } - apiLisAny := lis.GetApiListener().GetApiListener() - if !IsHTTPConnManagerResource(apiLisAny.GetTypeUrl()) { - return "", fmt.Errorf("xds: unexpected resource type: %s in LDS response", apiLisAny.GetTypeUrl()) - } - apiLis := &v3httppb.HttpConnectionManager{} - if err := proto.Unmarshal(apiLisAny.GetValue(), apiLis); err != nil { - return "", fmt.Errorf("xds: failed to unmarshal api_listner in LDS response: %v", err) - } - - logger.Infof("Resource with type %T, contains %v", apiLis, apiLis) - switch apiLis.RouteSpecifier.(type) { - case *v3httppb.HttpConnectionManager_Rds: - if apiLis.GetRds().GetConfigSource().GetAds() == nil { - return "", fmt.Errorf("xds: ConfigSource is not ADS in LDS response: %+v", lis) - } - name := apiLis.GetRds().GetRouteConfigName() - if name == "" { - return "", fmt.Errorf("xds: empty route_config_name in LDS response: %+v", lis) - } - return name, nil - case *v3httppb.HttpConnectionManager_RouteConfig: - // TODO: Add support for specifying the RouteConfiguration inline - // in the LDS response. - return "", fmt.Errorf("xds: LDS response contains RDS config inline. Not supported for now: %+v", apiLis) - case nil: - return "", fmt.Errorf("xds: no RouteSpecifier in received LDS response: %+v", apiLis) - default: - return "", fmt.Errorf("xds: unsupported type %T for RouteSpecifier in received LDS response", apiLis.RouteSpecifier) - } -} - -// UnmarshalRouteConfig processes resources received in an RDS response, -// validates them, and transforms them into a native struct which contains only -// fields we are interested in. The provided hostname determines the route -// configuration resources of interest. -func UnmarshalRouteConfig(resources []*anypb.Any, hostname string, logger *grpclog.PrefixLogger) (map[string]RouteConfigUpdate, error) { - update := make(map[string]RouteConfigUpdate) - for _, r := range resources { - if !IsRouteConfigResource(r.GetTypeUrl()) { - return nil, fmt.Errorf("xds: unexpected resource type: %s in RDS response", r.GetTypeUrl()) - } - rc := &v3routepb.RouteConfiguration{} - if err := proto.Unmarshal(r.GetValue(), rc); err != nil { - return nil, fmt.Errorf("xds: failed to unmarshal resource in RDS response: %v", err) - } - logger.Infof("Resource with name: %v, type: %T, contains: %v. Picking routes for current watching hostname %v", rc.GetName(), rc, rc, hostname) - - // Use the hostname (resourceName for LDS) to find the routes. - u, err := generateRDSUpdateFromRouteConfiguration(rc, hostname, logger) - if err != nil { - return nil, fmt.Errorf("xds: received invalid RouteConfiguration in RDS response: %+v with err: %v", rc, err) - } - update[rc.GetName()] = u - } - return update, nil -} - -// generateRDSUpdateFromRouteConfiguration checks if the provided -// RouteConfiguration meets the expected criteria. If so, it returns a -// RouteConfigUpdate with nil error. -// -// A RouteConfiguration resource is considered valid when only if it contains a -// VirtualHost whose domain field matches the server name from the URI passed -// to the gRPC channel, and it contains a clusterName or a weighted cluster. -// -// The RouteConfiguration includes a list of VirtualHosts, which may have zero -// or more elements. We are interested in the element whose domains field -// matches the server name specified in the "xds:" URI. The only field in the -// VirtualHost proto that the we are interested in is the list of routes. We -// only look at the last route in the list (the default route), whose match -// field must be empty and whose route field must be set. Inside that route -// message, the cluster field will contain the clusterName or weighted clusters -// we are looking for. -func generateRDSUpdateFromRouteConfiguration(rc *v3routepb.RouteConfiguration, host string, logger *grpclog.PrefixLogger) (RouteConfigUpdate, error) { - // - // Currently this returns "" on error, and the caller will return an error. - // But the error doesn't contain details of why the response is invalid - // (mismatch domain or empty route). - // - // For logging purposes, we can log in line. But if we want to populate - // error details for nack, a detailed error needs to be returned. - vh := findBestMatchingVirtualHost(host, rc.GetVirtualHosts()) - if vh == nil { - // No matching virtual host found. - return RouteConfigUpdate{}, fmt.Errorf("no matching virtual host found") - } - if len(vh.Routes) == 0 { - // The matched virtual host has no routes, this is invalid because there - // should be at least one default route. - return RouteConfigUpdate{}, fmt.Errorf("matched virtual host has no routes") - } - - routes, err := routesProtoToSlice(vh.Routes, logger) - if err != nil { - return RouteConfigUpdate{}, fmt.Errorf("received route is invalid: %v", err) - } - return RouteConfigUpdate{Routes: routes}, nil -} - -func routesProtoToSlice(routes []*v3routepb.Route, logger *grpclog.PrefixLogger) ([]*Route, error) { - var routesRet []*Route - - for _, r := range routes { - match := r.GetMatch() - if match == nil { - return nil, fmt.Errorf("route %+v doesn't have a match", r) - } - - if len(match.GetQueryParameters()) != 0 { - // Ignore route with query parameters. - logger.Warningf("route %+v has query parameter matchers, the route will be ignored", r) - continue - } - - if caseSensitive := match.GetCaseSensitive(); caseSensitive != nil && !caseSensitive.Value { - return nil, fmt.Errorf("route %+v has case-sensitive false", r) - } - - pathSp := match.GetPathSpecifier() - if pathSp == nil { - return nil, fmt.Errorf("route %+v doesn't have a path specifier", r) - } - - var route Route - switch pt := pathSp.(type) { - case *v3routepb.RouteMatch_Prefix: - route.Prefix = &pt.Prefix - case *v3routepb.RouteMatch_Path: - route.Path = &pt.Path - case *v3routepb.RouteMatch_SafeRegex: - route.Regex = &pt.SafeRegex.Regex - default: - logger.Warningf("route %+v has an unrecognized path specifier: %+v", r, pt) - continue - } - - for _, h := range match.GetHeaders() { - var header HeaderMatcher - switch ht := h.GetHeaderMatchSpecifier().(type) { - case *v3routepb.HeaderMatcher_ExactMatch: - header.ExactMatch = &ht.ExactMatch - case *v3routepb.HeaderMatcher_SafeRegexMatch: - header.RegexMatch = &ht.SafeRegexMatch.Regex - case *v3routepb.HeaderMatcher_RangeMatch: - header.RangeMatch = &Int64Range{ - Start: ht.RangeMatch.Start, - End: ht.RangeMatch.End, - } - case *v3routepb.HeaderMatcher_PresentMatch: - header.PresentMatch = &ht.PresentMatch - case *v3routepb.HeaderMatcher_PrefixMatch: - header.PrefixMatch = &ht.PrefixMatch - case *v3routepb.HeaderMatcher_SuffixMatch: - header.SuffixMatch = &ht.SuffixMatch - default: - logger.Warningf("route %+v has an unrecognized header matcher: %+v", r, ht) - continue - } - header.Name = h.GetName() - invert := h.GetInvertMatch() - header.InvertMatch = &invert - route.Headers = append(route.Headers, &header) - } - - if fr := match.GetRuntimeFraction(); fr != nil { - d := fr.GetDefaultValue() - n := d.GetNumerator() - switch d.GetDenominator() { - case v3typepb.FractionalPercent_HUNDRED: - n *= 10000 - case v3typepb.FractionalPercent_TEN_THOUSAND: - n *= 100 - case v3typepb.FractionalPercent_MILLION: - } - route.Fraction = &n - } - - clusters := make(map[string]uint32) - switch a := r.GetRoute().GetClusterSpecifier().(type) { - case *v3routepb.RouteAction_Cluster: - clusters[a.Cluster] = 1 - case *v3routepb.RouteAction_WeightedClusters: - wcs := a.WeightedClusters - var totalWeight uint32 - for _, c := range wcs.Clusters { - w := c.GetWeight().GetValue() - clusters[c.GetName()] = w - totalWeight += w - } - if totalWeight != wcs.GetTotalWeight().GetValue() { - return nil, fmt.Errorf("route %+v, action %+v, weights of clusters do not add up to total total weight, got: %v, want %v", r, a, wcs.GetTotalWeight().GetValue(), totalWeight) - } - case *v3routepb.RouteAction_ClusterHeader: - continue - } - - route.Action = clusters - routesRet = append(routesRet, &route) - } - return routesRet, nil -} - -type domainMatchType int - -const ( - domainMatchTypeInvalid domainMatchType = iota - domainMatchTypeUniversal - domainMatchTypePrefix - domainMatchTypeSuffix - domainMatchTypeExact -) - -// Exact > Suffix > Prefix > Universal > Invalid. -func (t domainMatchType) betterThan(b domainMatchType) bool { - return t > b -} - -func matchTypeForDomain(d string) domainMatchType { - if d == "" { - return domainMatchTypeInvalid - } - if d == "*" { - return domainMatchTypeUniversal - } - if strings.HasPrefix(d, "*") { - return domainMatchTypeSuffix - } - if strings.HasSuffix(d, "*") { - return domainMatchTypePrefix - } - if strings.Contains(d, "*") { - return domainMatchTypeInvalid - } - return domainMatchTypeExact -} - -func match(domain, host string) (domainMatchType, bool) { - switch typ := matchTypeForDomain(domain); typ { - case domainMatchTypeInvalid: - return typ, false - case domainMatchTypeUniversal: - return typ, true - case domainMatchTypePrefix: - // abc.* - return typ, strings.HasPrefix(host, strings.TrimSuffix(domain, "*")) - case domainMatchTypeSuffix: - // *.123 - return typ, strings.HasSuffix(host, strings.TrimPrefix(domain, "*")) - case domainMatchTypeExact: - return typ, domain == host - default: - return domainMatchTypeInvalid, false - } -} - -// findBestMatchingVirtualHost returns the virtual host whose domains field best -// matches host -// -// The domains field support 4 different matching pattern types: -// - Exact match -// - Suffix match (e.g. “*ABC”) -// - Prefix match (e.g. “ABC*) -// - Universal match (e.g. “*”) -// -// The best match is defined as: -// - A match is better if it’s matching pattern type is better -// - Exact match > suffix match > prefix match > universal match -// - If two matches are of the same pattern type, the longer match is better -// - This is to compare the length of the matching pattern, e.g. “*ABCDE” > -// “*ABC” -func findBestMatchingVirtualHost(host string, vHosts []*v3routepb.VirtualHost) *v3routepb.VirtualHost { - var ( - matchVh *v3routepb.VirtualHost - matchType = domainMatchTypeInvalid - matchLen int - ) - for _, vh := range vHosts { - for _, domain := range vh.GetDomains() { - typ, matched := match(domain, host) - if typ == domainMatchTypeInvalid { - // The rds response is invalid. - return nil - } - if matchType.betterThan(typ) || matchType == typ && matchLen >= len(domain) || !matched { - // The previous match has better type, or the previous match has - // better length, or this domain isn't a match. - continue - } - matchVh = vh - matchType = typ - matchLen = len(domain) - } - } - return matchVh -} - -// UnmarshalCluster processes resources received in an CDS response, validates -// them, and transforms them into a native struct which contains only fields we -// are interested in. -func UnmarshalCluster(resources []*anypb.Any, logger *grpclog.PrefixLogger) (map[string]ClusterUpdate, error) { - update := make(map[string]ClusterUpdate) - for _, r := range resources { - if !IsClusterResource(r.GetTypeUrl()) { - return nil, fmt.Errorf("xds: unexpected resource type: %s in CDS response", r.GetTypeUrl()) - } - - cluster := &v3clusterpb.Cluster{} - if err := proto.Unmarshal(r.GetValue(), cluster); err != nil { - return nil, fmt.Errorf("xds: failed to unmarshal resource in CDS response: %v", err) - } - logger.Infof("Resource with name: %v, type: %T, contains: %v", cluster.GetName(), cluster, cluster) - cu, err := validateCluster(cluster) - if err != nil { - return nil, err - } - - // If the Cluster message in the CDS response did not contain a - // serviceName, we will just use the clusterName for EDS. - if cu.ServiceName == "" { - cu.ServiceName = cluster.GetName() - } - logger.Debugf("Resource with name %v, value %+v added to cache", cluster.GetName(), cu) - update[cluster.GetName()] = cu - } - return update, nil -} - -func validateCluster(cluster *v3clusterpb.Cluster) (ClusterUpdate, error) { - emptyUpdate := ClusterUpdate{ServiceName: "", EnableLRS: false} - switch { - case cluster.GetType() != v3clusterpb.Cluster_EDS: - return emptyUpdate, fmt.Errorf("xds: unexpected cluster type %v in response: %+v", cluster.GetType(), cluster) - case cluster.GetEdsClusterConfig().GetEdsConfig().GetAds() == nil: - return emptyUpdate, fmt.Errorf("xds: unexpected edsConfig in response: %+v", cluster) - case cluster.GetLbPolicy() != v3clusterpb.Cluster_ROUND_ROBIN: - return emptyUpdate, fmt.Errorf("xds: unexpected lbPolicy %v in response: %+v", cluster.GetLbPolicy(), cluster) - } - - return ClusterUpdate{ - ServiceName: cluster.GetEdsClusterConfig().GetServiceName(), - EnableLRS: cluster.GetLrsServer().GetSelf() != nil, - }, nil -} - -// UnmarshalEndpoints processes resources received in an EDS response, -// validates them, and transforms them into a native struct which contains only -// fields we are interested in. -func UnmarshalEndpoints(resources []*anypb.Any, logger *grpclog.PrefixLogger) (map[string]EndpointsUpdate, error) { - update := make(map[string]EndpointsUpdate) - for _, r := range resources { - if !IsEndpointsResource(r.GetTypeUrl()) { - return nil, fmt.Errorf("xds: unexpected resource type: %s in EDS response", r.GetTypeUrl()) - } - - cla := &v3endpointpb.ClusterLoadAssignment{} - if err := proto.Unmarshal(r.GetValue(), cla); err != nil { - return nil, fmt.Errorf("xds: failed to unmarshal resource in EDS response: %v", err) - } - logger.Infof("Resource with name: %v, type: %T, contains: %v", cla.GetClusterName(), cla, cla) - - u, err := parseEDSRespProto(cla) - if err != nil { - return nil, err - } - update[cla.GetClusterName()] = u - } - return update, nil -} - -func parseAddress(socketAddress *v3corepb.SocketAddress) string { - return net.JoinHostPort(socketAddress.GetAddress(), strconv.Itoa(int(socketAddress.GetPortValue()))) -} - -func parseDropPolicy(dropPolicy *v3endpointpb.ClusterLoadAssignment_Policy_DropOverload) OverloadDropConfig { - percentage := dropPolicy.GetDropPercentage() - var ( - numerator = percentage.GetNumerator() - denominator uint32 - ) - switch percentage.GetDenominator() { - case v3typepb.FractionalPercent_HUNDRED: - denominator = 100 - case v3typepb.FractionalPercent_TEN_THOUSAND: - denominator = 10000 - case v3typepb.FractionalPercent_MILLION: - denominator = 1000000 - } - return OverloadDropConfig{ - Category: dropPolicy.GetCategory(), - Numerator: numerator, - Denominator: denominator, - } -} - -func parseEndpoints(lbEndpoints []*v3endpointpb.LbEndpoint) []Endpoint { - endpoints := make([]Endpoint, 0, len(lbEndpoints)) - for _, lbEndpoint := range lbEndpoints { - endpoints = append(endpoints, Endpoint{ - HealthStatus: EndpointHealthStatus(lbEndpoint.GetHealthStatus()), - Address: parseAddress(lbEndpoint.GetEndpoint().GetAddress().GetSocketAddress()), - Weight: lbEndpoint.GetLoadBalancingWeight().GetValue(), - }) - } - return endpoints -} - -func parseEDSRespProto(m *v3endpointpb.ClusterLoadAssignment) (EndpointsUpdate, error) { - ret := EndpointsUpdate{} - for _, dropPolicy := range m.GetPolicy().GetDropOverloads() { - ret.Drops = append(ret.Drops, parseDropPolicy(dropPolicy)) - } - priorities := make(map[uint32]struct{}) - for _, locality := range m.Endpoints { - l := locality.GetLocality() - if l == nil { - return EndpointsUpdate{}, fmt.Errorf("EDS response contains a locality without ID, locality: %+v", locality) - } - lid := internal.LocalityID{ - Region: l.Region, - Zone: l.Zone, - SubZone: l.SubZone, - } - priority := locality.GetPriority() - priorities[priority] = struct{}{} - ret.Localities = append(ret.Localities, Locality{ - ID: lid, - Endpoints: parseEndpoints(locality.GetLbEndpoints()), - Weight: locality.GetLoadBalancingWeight().GetValue(), - Priority: priority, - }) - } - for i := 0; i < len(priorities); i++ { - if _, ok := priorities[uint32(i)]; !ok { - return EndpointsUpdate{}, fmt.Errorf("priority %v missing (with different priorities %v received)", i, priorities) - } - } - return ret, nil -} diff --git a/xds/internal/client/tests/client_test.go b/xds/internal/client/tests/client_test.go deleted file mode 100644 index 3c96175b0d89..000000000000 --- a/xds/internal/client/tests/client_test.go +++ /dev/null @@ -1,122 +0,0 @@ -/* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package tests - -import ( - "testing" - - "google.golang.org/grpc" - "google.golang.org/grpc/internal/grpctest" - xdsclient "google.golang.org/grpc/xds/internal/client" - "google.golang.org/grpc/xds/internal/client/bootstrap" - _ "google.golang.org/grpc/xds/internal/client/v2" // Register the v2 API client. - "google.golang.org/grpc/xds/internal/testutils" - "google.golang.org/grpc/xds/internal/version" -) - -type s struct { - grpctest.Tester -} - -func Test(t *testing.T) { - grpctest.RunSubTests(t, s{}) -} - -const ( - testXDSServer = "xds-server" -) - -func clientOpts(balancerName string) xdsclient.Options { - return xdsclient.Options{ - Config: bootstrap.Config{ - BalancerName: balancerName, - Creds: grpc.WithInsecure(), - NodeProto: testutils.EmptyNodeProtoV2, - }, - } -} - -func (s) TestNew(t *testing.T) { - tests := []struct { - name string - opts xdsclient.Options - wantErr bool - }{ - {name: "empty-opts", opts: xdsclient.Options{}, wantErr: true}, - { - name: "empty-balancer-name", - opts: xdsclient.Options{ - Config: bootstrap.Config{ - Creds: grpc.WithInsecure(), - NodeProto: testutils.EmptyNodeProtoV2, - }, - }, - wantErr: true, - }, - { - name: "empty-dial-creds", - opts: xdsclient.Options{ - Config: bootstrap.Config{ - BalancerName: testXDSServer, - NodeProto: testutils.EmptyNodeProtoV2, - }, - }, - wantErr: true, - }, - { - name: "empty-node-proto", - opts: xdsclient.Options{ - Config: bootstrap.Config{ - BalancerName: testXDSServer, - Creds: grpc.WithInsecure(), - }, - }, - wantErr: true, - }, - { - name: "node-proto-version-mismatch", - opts: xdsclient.Options{ - Config: bootstrap.Config{ - BalancerName: testXDSServer, - Creds: grpc.WithInsecure(), - NodeProto: testutils.EmptyNodeProtoV3, - TransportAPI: version.TransportV2, - }, - }, - wantErr: true, - }, - // TODO(easwars): Add cases for v3 API client. - { - name: "happy-case", - opts: clientOpts(testXDSServer), - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - c, err := xdsclient.New(test.opts) - if (err != nil) != test.wantErr { - t.Fatalf("New(%+v) = %v, wantErr: %v", test.opts, err, test.wantErr) - } - if c != nil { - c.Close() - } - }) - } -} diff --git a/xds/internal/client/transport_helper.go b/xds/internal/client/transport_helper.go deleted file mode 100644 index 976003605041..000000000000 --- a/xds/internal/client/transport_helper.go +++ /dev/null @@ -1,421 +0,0 @@ -/* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package client - -import ( - "context" - "sync" - "time" - - "github.com/golang/protobuf/proto" - "google.golang.org/grpc" - "google.golang.org/grpc/internal/buffer" - "google.golang.org/grpc/internal/grpclog" -) - -// ErrResourceTypeUnsupported is an error used to indicate an unsupported xDS -// resource type. The wrapped ErrStr contains the details. -type ErrResourceTypeUnsupported struct { - ErrStr string -} - -// Error helps implements the error interface. -func (e ErrResourceTypeUnsupported) Error() string { - return e.ErrStr -} - -// VersionedClient is the interface to be provided by the transport protocol -// specific client implementations. This mainly deals with the actual sending -// and receiving of messages. -type VersionedClient interface { - // NewStream returns a new grpc.ClientStream specific to the underlying - // transport protocol version. - NewStream(ctx context.Context) (grpc.ClientStream, error) - - // SendRequest constructs and sends out a DiscoveryRequest message specific - // to the underlying transport protocol version. - SendRequest(s grpc.ClientStream, resourceNames []string, rType ResourceType, version string, nonce string) error - - // RecvResponse uses the provided stream to receive a response specific to - // the underlying transport protocol version. - RecvResponse(stream grpc.ClientStream) (proto.Message, error) - - // HandleResponse parses and validates the received response and notifies - // the top-level client which in turn notifies the registered watchers. - // - // Return values are: resourceType, version, nonce, error. - // If the provided protobuf message contains a resource type which is not - // supported, implementations must return an error of type - // ErrResourceTypeUnsupported. - HandleResponse(proto.Message) (ResourceType, string, string, error) -} - -// TransportHelper contains all xDS transport protocol related functionality -// which is common across different versioned client implementations. -// -// TransportHelper takes care of sending and receiving xDS requests and -// responses on an ADS stream. It also takes care of ACK/NACK handling. It -// delegates to the actual versioned client implementations wherever -// appropriate. -// -// Implements the APIClient interface which makes it possible for versioned -// client implementations to embed this type, and thereby satisfy the interface -// requirements. -type TransportHelper struct { - cancelCtx context.CancelFunc - - vClient VersionedClient - logger *grpclog.PrefixLogger - backoff func(int) time.Duration - streamCh chan grpc.ClientStream - sendCh *buffer.Unbounded - - mu sync.Mutex - // Message specific watch infos, protected by the above mutex. These are - // written to, after successfully reading from the update channel, and are - // read from when recovering from a broken stream to resend the xDS - // messages. When the user of this client object cancels a watch call, - // these are set to nil. All accesses to the map protected and any value - // inside the map should be protected with the above mutex. - watchMap map[ResourceType]map[string]bool - // versionMap contains the version that was acked (the version in the ack - // request that was sent on wire). The key is rType, the value is the - // version string, becaues the versions for different resource types should - // be independent. - versionMap map[ResourceType]string - // nonceMap contains the nonce from the most recent received response. - nonceMap map[ResourceType]string -} - -// NewTransportHelper creates a new transport helper to be used by versioned -// client implementations. -func NewTransportHelper(vc VersionedClient, logger *grpclog.PrefixLogger, backoff func(int) time.Duration) *TransportHelper { - ctx, cancelCtx := context.WithCancel(context.Background()) - t := &TransportHelper{ - cancelCtx: cancelCtx, - vClient: vc, - logger: logger, - backoff: backoff, - - streamCh: make(chan grpc.ClientStream, 1), - sendCh: buffer.NewUnbounded(), - watchMap: make(map[ResourceType]map[string]bool), - versionMap: make(map[ResourceType]string), - nonceMap: make(map[ResourceType]string), - } - - go t.run(ctx) - return t -} - -// AddWatch adds a watch for an xDS resource given its type and name. -func (t *TransportHelper) AddWatch(rType ResourceType, resourceName string) { - t.sendCh.Put(&watchAction{ - rType: rType, - remove: false, - resource: resourceName, - }) -} - -// RemoveWatch cancels an already registered watch for an xDS resource -// given its type and name. -func (t *TransportHelper) RemoveWatch(rType ResourceType, resourceName string) { - t.sendCh.Put(&watchAction{ - rType: rType, - remove: true, - resource: resourceName, - }) -} - -// Close closes the transport helper. -func (t *TransportHelper) Close() { - t.cancelCtx() -} - -// run starts an ADS stream (and backs off exponentially, if the previous -// stream failed without receiving a single reply) and runs the sender and -// receiver routines to send and receive data from the stream respectively. -func (t *TransportHelper) run(ctx context.Context) { - go t.send(ctx) - // TODO: start a goroutine monitoring ClientConn's connectivity state, and - // report error (and log) when stats is transient failure. - - retries := 0 - for { - select { - case <-ctx.Done(): - return - default: - } - - if retries != 0 { - timer := time.NewTimer(t.backoff(retries)) - select { - case <-timer.C: - case <-ctx.Done(): - if !timer.Stop() { - <-timer.C - } - return - } - } - - retries++ - stream, err := t.vClient.NewStream(ctx) - if err != nil { - t.logger.Warningf("xds: ADS stream creation failed: %v", err) - continue - } - t.logger.Infof("ADS stream created") - - select { - case <-t.streamCh: - default: - } - t.streamCh <- stream - if t.recv(stream) { - retries = 0 - } - } -} - -// send is a separate goroutine for sending watch requests on the xds stream. -// -// It watches the stream channel for new streams, and the request channel for -// new requests to send on the stream. -// -// For each new request (watchAction), it's -// - processed and added to the watch map -// - so resend will pick them up when there are new streams -// - sent on the current stream if there's one -// - the current stream is cleared when any send on it fails -// -// For each new stream, all the existing requests will be resent. -// -// Note that this goroutine doesn't do anything to the old stream when there's a -// new one. In fact, there should be only one stream in progress, and new one -// should only be created when the old one fails (recv returns an error). -func (t *TransportHelper) send(ctx context.Context) { - var stream grpc.ClientStream - for { - select { - case <-ctx.Done(): - return - case stream = <-t.streamCh: - if !t.sendExisting(stream) { - // send failed, clear the current stream. - stream = nil - } - case u := <-t.sendCh.Get(): - t.sendCh.Load() - - var ( - target []string - rType ResourceType - version, nonce string - send bool - ) - switch update := u.(type) { - case *watchAction: - target, rType, version, nonce = t.processWatchInfo(update) - case *ackAction: - target, rType, version, nonce, send = t.processAckInfo(update, stream) - if !send { - continue - } - } - if stream == nil { - // There's no stream yet. Skip the request. This request - // will be resent to the new streams. If no stream is - // created, the watcher will timeout (same as server not - // sending response back). - continue - } - if err := t.vClient.SendRequest(stream, target, rType, version, nonce); err != nil { - t.logger.Warningf("ADS request for {target: %q, type: %v, version: %q, nonce: %q} failed: %v", target, rType, version, nonce, err) - // send failed, clear the current stream. - stream = nil - } - } - } -} - -// sendExisting sends out xDS requests for registered watchers when recovering -// from a broken stream. -// -// We call stream.Send() here with the lock being held. It should be OK to do -// that here because the stream has just started and Send() usually returns -// quickly (once it pushes the message onto the transport layer) and is only -// ever blocked if we don't have enough flow control quota. -func (t *TransportHelper) sendExisting(stream grpc.ClientStream) bool { - t.mu.Lock() - defer t.mu.Unlock() - - // Reset the ack versions when the stream restarts. - t.versionMap = make(map[ResourceType]string) - t.nonceMap = make(map[ResourceType]string) - - for rType, s := range t.watchMap { - if err := t.vClient.SendRequest(stream, mapToSlice(s), rType, "", ""); err != nil { - t.logger.Errorf("ADS request failed: %v", err) - return false - } - } - - return true -} - -// recv receives xDS responses on the provided ADS stream and branches out to -// message specific handlers. -func (t *TransportHelper) recv(stream grpc.ClientStream) bool { - success := false - for { - resp, err := t.vClient.RecvResponse(stream) - if err != nil { - t.logger.Warningf("ADS stream is closed with error: %v", err) - return success - } - rType, version, nonce, err := t.vClient.HandleResponse(resp) - if e, ok := err.(ErrResourceTypeUnsupported); ok { - t.logger.Warningf("%s", e.ErrStr) - continue - } - if err != nil { - t.sendCh.Put(&ackAction{ - rType: rType, - version: "", - nonce: nonce, - stream: stream, - }) - t.logger.Warningf("Sending NACK for response type: %v, version: %v, nonce: %v, reason: %v", rType, version, nonce, err) - continue - } - t.sendCh.Put(&ackAction{ - rType: rType, - version: version, - nonce: nonce, - stream: stream, - }) - t.logger.Infof("Sending ACK for response type: %v, version: %v, nonce: %v", rType, version, nonce) - success = true - } -} - -func mapToSlice(m map[string]bool) (ret []string) { - for i := range m { - ret = append(ret, i) - } - return -} - -type watchAction struct { - rType ResourceType - remove bool // Whether this is to remove watch for the resource. - resource string -} - -// processWatchInfo pulls the fields needed by the request from a watchAction. -// -// It also updates the watch map. -func (t *TransportHelper) processWatchInfo(w *watchAction) (target []string, rType ResourceType, ver, nonce string) { - t.mu.Lock() - defer t.mu.Unlock() - - var current map[string]bool - current, ok := t.watchMap[w.rType] - if !ok { - current = make(map[string]bool) - t.watchMap[w.rType] = current - } - - if w.remove { - delete(current, w.resource) - if len(current) == 0 { - delete(t.watchMap, w.rType) - } - } else { - current[w.resource] = true - } - - rType = w.rType - target = mapToSlice(current) - // We don't reset version or nonce when a new watch is started. The version - // and nonce from previous response are carried by the request unless the - // stream is recreated. - ver = t.versionMap[rType] - nonce = t.nonceMap[rType] - return target, rType, ver, nonce -} - -type ackAction struct { - rType ResourceType - version string // NACK if version is an empty string. - nonce string - // ACK/NACK are tagged with the stream it's for. When the stream is down, - // all the ACK/NACK for this stream will be dropped, and the version/nonce - // won't be updated. - stream grpc.ClientStream -} - -// processAckInfo pulls the fields needed by the ack request from a ackAction. -// -// If no active watch is found for this ack, it returns false for send. -func (t *TransportHelper) processAckInfo(ack *ackAction, stream grpc.ClientStream) (target []string, rType ResourceType, version, nonce string, send bool) { - if ack.stream != stream { - // If ACK's stream isn't the current sending stream, this means the ACK - // was pushed to queue before the old stream broke, and a new stream has - // been started since. Return immediately here so we don't update the - // nonce for the new stream. - return nil, UnknownResource, "", "", false - } - rType = ack.rType - - t.mu.Lock() - defer t.mu.Unlock() - - // Update the nonce no matter if we are going to send the ACK request on - // wire. We may not send the request if the watch is canceled. But the nonce - // needs to be updated so the next request will have the right nonce. - nonce = ack.nonce - t.nonceMap[rType] = nonce - - s, ok := t.watchMap[rType] - if !ok || len(s) == 0 { - // We don't send the request ack if there's no active watch (this can be - // either the server sends responses before any request, or the watch is - // canceled while the ackAction is in queue), because there's no resource - // name. And if we send a request with empty resource name list, the - // server may treat it as a wild card and send us everything. - return nil, UnknownResource, "", "", false - } - send = true - target = mapToSlice(s) - - version = ack.version - if version == "" { - // This is a nack, get the previous acked version. - version = t.versionMap[rType] - // version will still be an empty string if rType isn't - // found in versionMap, this can happen if there wasn't any ack - // before. - } else { - t.versionMap[rType] = version - } - return target, rType, version, nonce, send -} diff --git a/xds/internal/client/v2/client.go b/xds/internal/client/v2/client.go deleted file mode 100644 index 96bd5e9b5686..000000000000 --- a/xds/internal/client/v2/client.go +++ /dev/null @@ -1,272 +0,0 @@ -/* - * - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -// Package v2 provides xDS v2 transport protocol specific functionality. -package v2 - -import ( - "context" - "fmt" - "sync" - - "github.com/golang/protobuf/proto" - "google.golang.org/grpc" - "google.golang.org/grpc/internal/grpclog" - xdsclient "google.golang.org/grpc/xds/internal/client" - "google.golang.org/grpc/xds/internal/version" - - v2xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2" - v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" - v2adsgrpc "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v2" -) - -func init() { - xdsclient.RegisterAPIClientBuilder(clientBuilder{}) -} - -var ( - resourceTypeToURL = map[xdsclient.ResourceType]string{ - xdsclient.ListenerResource: version.V2ListenerURL, - xdsclient.RouteConfigResource: version.V2RouteConfigURL, - xdsclient.ClusterResource: version.V2ClusterURL, - xdsclient.EndpointsResource: version.V2EndpointsURL, - } -) - -type clientBuilder struct{} - -func (clientBuilder) Build(cc *grpc.ClientConn, opts xdsclient.BuildOptions) (xdsclient.APIClient, error) { - return newClient(cc, opts) -} - -func (clientBuilder) Version() version.TransportAPI { - return version.TransportV2 -} - -func newClient(cc *grpc.ClientConn, opts xdsclient.BuildOptions) (xdsclient.APIClient, error) { - nodeProto, ok := opts.NodeProto.(*v2corepb.Node) - if !ok { - return nil, fmt.Errorf("xds: unsupported Node proto type: %T, want %T", opts.NodeProto, (*v2corepb.Node)(nil)) - } - v2c := &client{ - cc: cc, - parent: opts.Parent, - nodeProto: nodeProto, - logger: opts.Logger, - } - v2c.ctx, v2c.cancelCtx = context.WithCancel(context.Background()) - v2c.TransportHelper = xdsclient.NewTransportHelper(v2c, opts.Logger, opts.Backoff) - return v2c, nil -} - -type adsStream v2adsgrpc.AggregatedDiscoveryService_StreamAggregatedResourcesClient - -// client performs the actual xDS RPCs using the xDS v2 API. It creates a -// single ADS stream on which the different types of xDS requests and responses -// are multiplexed. -type client struct { - *xdsclient.TransportHelper - - ctx context.Context - cancelCtx context.CancelFunc - parent xdsclient.UpdateHandler - logger *grpclog.PrefixLogger - - // ClientConn to the xDS gRPC server. Owned by the parent xdsClient. - cc *grpc.ClientConn - nodeProto *v2corepb.Node - - mu sync.Mutex - // ldsResourceName is the LDS resource_name to watch. It is set to the first - // LDS resource_name to watch, and removed when the LDS watch is canceled. - // - // It's from the dial target of the parent ClientConn. RDS resource - // processing needs this to do the host matching. - ldsResourceName string - ldsWatchCount int -} - -// AddWatch overrides the transport helper's AddWatch to save the LDS -// resource_name. This is required when handling an RDS response to perform host -// matching. -func (v2c *client) AddWatch(rType xdsclient.ResourceType, rName string) { - v2c.mu.Lock() - // Special handling for LDS, because RDS needs the LDS resource_name for - // response host matching. - if rType == xdsclient.ListenerResource { - // Set hostname to the first LDS resource_name, and reset it when the - // last LDS watch is removed. The upper level Client isn't expected to - // watchLDS more than once. - v2c.ldsWatchCount++ - if v2c.ldsWatchCount == 1 { - v2c.ldsResourceName = rName - } - } - v2c.mu.Unlock() - v2c.TransportHelper.AddWatch(rType, rName) -} - -// RemoveWatch overrides the transport helper's RemoveWatch to clear the LDS -// resource_name when the last watch is removed. -func (v2c *client) RemoveWatch(rType xdsclient.ResourceType, rName string) { - v2c.mu.Lock() - // Special handling for LDS, because RDS needs the LDS resource_name for - // response host matching. - if rType == xdsclient.ListenerResource { - // Set hostname to the first LDS resource_name, and reset it when the - // last LDS watch is removed. The upper level Client isn't expected to - // watchLDS more than once. - v2c.ldsWatchCount-- - if v2c.ldsWatchCount == 0 { - v2c.ldsResourceName = "" - } - } - v2c.mu.Unlock() - v2c.TransportHelper.RemoveWatch(rType, rName) -} - -func (v2c *client) NewStream(ctx context.Context) (grpc.ClientStream, error) { - return v2adsgrpc.NewAggregatedDiscoveryServiceClient(v2c.cc).StreamAggregatedResources(v2c.ctx, grpc.WaitForReady(true)) -} - -// sendRequest sends out a DiscoveryRequest for the given resourceNames, of type -// rType, on the provided stream. -// -// version is the ack version to be sent with the request -// - If this is the new request (not an ack/nack), version will be empty. -// - If this is an ack, version will be the version from the response. -// - If this is a nack, version will be the previous acked version (from -// versionMap). If there was no ack before, it will be empty. -func (v2c *client) SendRequest(s grpc.ClientStream, resourceNames []string, rType xdsclient.ResourceType, version, nonce string) error { - stream, ok := s.(adsStream) - if !ok { - return fmt.Errorf("xds: Attempt to send request on unsupported stream type: %T", s) - } - req := &v2xdspb.DiscoveryRequest{ - Node: v2c.nodeProto, - TypeUrl: resourceTypeToURL[rType], - ResourceNames: resourceNames, - VersionInfo: version, - ResponseNonce: nonce, - // TODO: populate ErrorDetails for nack. - } - if err := stream.Send(req); err != nil { - return fmt.Errorf("xds: stream.Send(%+v) failed: %v", req, err) - } - v2c.logger.Debugf("ADS request sent: %v", req) - return nil -} - -// RecvResponse blocks on the receipt of one response message on the provided -// stream. -func (v2c *client) RecvResponse(s grpc.ClientStream) (proto.Message, error) { - stream, ok := s.(adsStream) - if !ok { - return nil, fmt.Errorf("xds: Attempt to receive response on unsupported stream type: %T", s) - } - - resp, err := stream.Recv() - if err != nil { - // TODO: call watch callbacks with error when stream is broken. - return nil, fmt.Errorf("xds: stream.Recv() failed: %v", err) - } - v2c.logger.Infof("ADS response received, type: %v", resp.GetTypeUrl()) - v2c.logger.Debugf("ADS response received: %v", resp) - return resp, nil -} - -func (v2c *client) HandleResponse(r proto.Message) (xdsclient.ResourceType, string, string, error) { - rType := xdsclient.UnknownResource - resp, ok := r.(*v2xdspb.DiscoveryResponse) - if !ok { - return rType, "", "", fmt.Errorf("xds: unsupported message type: %T", resp) - } - - // Note that the xDS transport protocol is versioned independently of - // the resource types, and it is supported to transfer older versions - // of resource types using new versions of the transport protocol, or - // vice-versa. Hence we need to handle v3 type_urls as well here. - var err error - url := resp.GetTypeUrl() - switch { - case xdsclient.IsListenerResource(url): - err = v2c.handleLDSResponse(resp) - rType = xdsclient.ListenerResource - case xdsclient.IsRouteConfigResource(url): - err = v2c.handleRDSResponse(resp) - rType = xdsclient.RouteConfigResource - case xdsclient.IsClusterResource(url): - err = v2c.handleCDSResponse(resp) - rType = xdsclient.ClusterResource - case xdsclient.IsEndpointsResource(url): - err = v2c.handleEDSResponse(resp) - rType = xdsclient.EndpointsResource - default: - return rType, "", "", xdsclient.ErrResourceTypeUnsupported{ - ErrStr: fmt.Sprintf("Resource type %v unknown in response from server", resp.GetTypeUrl()), - } - } - return rType, resp.GetVersionInfo(), resp.GetNonce(), err -} - -// handleLDSResponse processes an LDS response received from the xDS server. On -// receipt of a good response, it also invokes the registered watcher callback. -func (v2c *client) handleLDSResponse(resp *v2xdspb.DiscoveryResponse) error { - update, err := xdsclient.UnmarshalListener(resp.GetResources(), v2c.logger) - if err != nil { - return err - } - v2c.parent.NewListeners(update) - return nil -} - -// handleRDSResponse processes an RDS response received from the xDS server. On -// receipt of a good response, it caches validated resources and also invokes -// the registered watcher callback. -func (v2c *client) handleRDSResponse(resp *v2xdspb.DiscoveryResponse) error { - v2c.mu.Lock() - hostname := v2c.ldsResourceName - v2c.mu.Unlock() - - update, err := xdsclient.UnmarshalRouteConfig(resp.GetResources(), hostname, v2c.logger) - if err != nil { - return err - } - v2c.parent.NewRouteConfigs(update) - return nil -} - -// handleCDSResponse processes an CDS response received from the xDS server. On -// receipt of a good response, it also invokes the registered watcher callback. -func (v2c *client) handleCDSResponse(resp *v2xdspb.DiscoveryResponse) error { - update, err := xdsclient.UnmarshalCluster(resp.GetResources(), v2c.logger) - if err != nil { - return err - } - v2c.parent.NewClusters(update) - return nil -} - -func (v2c *client) handleEDSResponse(resp *v2xdspb.DiscoveryResponse) error { - update, err := xdsclient.UnmarshalEndpoints(resp.GetResources(), v2c.logger) - if err != nil { - return err - } - v2c.parent.NewEndpoints(update) - return nil -} diff --git a/xds/internal/client/v2/client_ack_test.go b/xds/internal/client/v2/client_ack_test.go deleted file mode 100644 index 40f5668297e2..000000000000 --- a/xds/internal/client/v2/client_ack_test.go +++ /dev/null @@ -1,446 +0,0 @@ -/* - * - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package v2 - -import ( - "fmt" - "strconv" - "testing" - "time" - - xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2" - "github.com/golang/protobuf/proto" - anypb "github.com/golang/protobuf/ptypes/any" - "github.com/google/go-cmp/cmp" - "google.golang.org/grpc" - xdsclient "google.golang.org/grpc/xds/internal/client" - "google.golang.org/grpc/xds/internal/testutils" - "google.golang.org/grpc/xds/internal/testutils/fakeserver" - "google.golang.org/grpc/xds/internal/version" -) - -func startXDSV2Client(t *testing.T, cc *grpc.ClientConn) (v2c *client, cbLDS, cbRDS, cbCDS, cbEDS *testutils.Channel, cleanup func()) { - cbLDS = testutils.NewChannel() - cbRDS = testutils.NewChannel() - cbCDS = testutils.NewChannel() - cbEDS = testutils.NewChannel() - v2c, err := newV2Client(&testUpdateReceiver{ - f: func(rType xdsclient.ResourceType, d map[string]interface{}) { - t.Logf("Received %v callback with {%+v}", rType, d) - switch rType { - case xdsclient.ListenerResource: - if _, ok := d[goodLDSTarget1]; ok { - cbLDS.Send(struct{}{}) - } - case xdsclient.RouteConfigResource: - if _, ok := d[goodRouteName1]; ok { - cbRDS.Send(struct{}{}) - } - case xdsclient.ClusterResource: - if _, ok := d[goodClusterName1]; ok { - cbCDS.Send(struct{}{}) - } - case xdsclient.EndpointsResource: - if _, ok := d[goodEDSName]; ok { - cbEDS.Send(struct{}{}) - } - } - }, - }, cc, goodNodeProto, func(int) time.Duration { return 0 }, nil) - if err != nil { - t.Fatal(err) - } - t.Log("Started xds client...") - return v2c, cbLDS, cbRDS, cbCDS, cbEDS, v2c.Close -} - -// compareXDSRequest reads requests from channel, compare it with want. -func compareXDSRequest(ch *testutils.Channel, want *xdspb.DiscoveryRequest, ver, nonce string) error { - val, err := ch.Receive() - if err != nil { - return err - } - req := val.(*fakeserver.Request) - if req.Err != nil { - return fmt.Errorf("unexpected error from request: %v", req.Err) - } - wantClone := proto.Clone(want).(*xdspb.DiscoveryRequest) - wantClone.VersionInfo = ver - wantClone.ResponseNonce = nonce - if !cmp.Equal(req.Req, wantClone, cmp.Comparer(proto.Equal)) { - return fmt.Errorf("received request different from want, diff: %s", cmp.Diff(req.Req, wantClone)) - } - return nil -} - -func sendXDSRespWithVersion(ch chan<- *fakeserver.Response, respWithoutVersion *xdspb.DiscoveryResponse, ver int) (nonce string) { - respToSend := proto.Clone(respWithoutVersion).(*xdspb.DiscoveryResponse) - respToSend.VersionInfo = strconv.Itoa(ver) - nonce = strconv.Itoa(int(time.Now().UnixNano())) - respToSend.Nonce = nonce - ch <- &fakeserver.Response{Resp: respToSend} - return -} - -// startXDS calls watch to send the first request. It then sends a good response -// and checks for ack. -func startXDS(t *testing.T, rType xdsclient.ResourceType, v2c *client, reqChan *testutils.Channel, req *xdspb.DiscoveryRequest, preVersion string, preNonce string) { - nameToWatch := "" - switch rType { - case xdsclient.ListenerResource: - nameToWatch = goodLDSTarget1 - case xdsclient.RouteConfigResource: - nameToWatch = goodRouteName1 - case xdsclient.ClusterResource: - nameToWatch = goodClusterName1 - case xdsclient.EndpointsResource: - nameToWatch = goodEDSName - } - v2c.AddWatch(rType, nameToWatch) - - if err := compareXDSRequest(reqChan, req, preVersion, preNonce); err != nil { - t.Fatalf("Failed to receive %v request: %v", rType, err) - } - t.Logf("FakeServer received %v request...", rType) -} - -// sendGoodResp sends the good response, with the given version, and a random -// nonce. -// -// It also waits and checks that the ack request contains the given version, and -// the generated nonce. -func sendGoodResp(t *testing.T, rType xdsclient.ResourceType, fakeServer *fakeserver.Server, ver int, goodResp *xdspb.DiscoveryResponse, wantReq *xdspb.DiscoveryRequest, callbackCh *testutils.Channel) (string, error) { - nonce := sendXDSRespWithVersion(fakeServer.XDSResponseChan, goodResp, ver) - t.Logf("Good %v response pushed to fakeServer...", rType) - - if err := compareXDSRequest(fakeServer.XDSRequestChan, wantReq, strconv.Itoa(ver), nonce); err != nil { - return "", fmt.Errorf("failed to receive %v request: %v", rType, err) - } - t.Logf("Good %v response acked", rType) - - if _, err := callbackCh.Receive(); err != nil { - return "", fmt.Errorf("timeout when expecting %v update", rType) - } - t.Logf("Good %v response callback executed", rType) - return nonce, nil -} - -// sendBadResp sends a bad response with the given version. This response will -// be nacked, so we expect a request with the previous version (version-1). -// -// But the nonce in request should be the new nonce. -func sendBadResp(t *testing.T, rType xdsclient.ResourceType, fakeServer *fakeserver.Server, ver int, wantReq *xdspb.DiscoveryRequest) error { - var typeURL string - switch rType { - case xdsclient.ListenerResource: - typeURL = version.V2ListenerURL - case xdsclient.RouteConfigResource: - typeURL = version.V2RouteConfigURL - case xdsclient.ClusterResource: - typeURL = version.V2ClusterURL - case xdsclient.EndpointsResource: - typeURL = version.V2EndpointsURL - } - nonce := sendXDSRespWithVersion(fakeServer.XDSResponseChan, &xdspb.DiscoveryResponse{ - Resources: []*anypb.Any{{}}, - TypeUrl: typeURL, - }, ver) - t.Logf("Bad %v response pushed to fakeServer...", rType) - if err := compareXDSRequest(fakeServer.XDSRequestChan, wantReq, strconv.Itoa(ver-1), nonce); err != nil { - return fmt.Errorf("failed to receive %v request: %v", rType, err) - } - t.Logf("Bad %v response nacked", rType) - return nil -} - -// TestV2ClientAck verifies that valid responses are acked, and invalid ones -// are nacked. -// -// This test also verifies the version for different types are independent. -func (s) TestV2ClientAck(t *testing.T) { - var ( - versionLDS = 1000 - versionRDS = 2000 - versionCDS = 3000 - versionEDS = 4000 - ) - - fakeServer, cc, cleanup := startServerAndGetCC(t) - defer cleanup() - - v2c, cbLDS, cbRDS, cbCDS, cbEDS, v2cCleanup := startXDSV2Client(t, cc) - defer v2cCleanup() - - // Start the watch, send a good response, and check for ack. - startXDS(t, xdsclient.ListenerResource, v2c, fakeServer.XDSRequestChan, goodLDSRequest, "", "") - if _, err := sendGoodResp(t, xdsclient.ListenerResource, fakeServer, versionLDS, goodLDSResponse1, goodLDSRequest, cbLDS); err != nil { - t.Fatal(err) - } - versionLDS++ - startXDS(t, xdsclient.RouteConfigResource, v2c, fakeServer.XDSRequestChan, goodRDSRequest, "", "") - if _, err := sendGoodResp(t, xdsclient.RouteConfigResource, fakeServer, versionRDS, goodRDSResponse1, goodRDSRequest, cbRDS); err != nil { - t.Fatal(err) - } - versionRDS++ - startXDS(t, xdsclient.ClusterResource, v2c, fakeServer.XDSRequestChan, goodCDSRequest, "", "") - if _, err := sendGoodResp(t, xdsclient.ClusterResource, fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, cbCDS); err != nil { - t.Fatal(err) - } - versionCDS++ - startXDS(t, xdsclient.EndpointsResource, v2c, fakeServer.XDSRequestChan, goodEDSRequest, "", "") - if _, err := sendGoodResp(t, xdsclient.EndpointsResource, fakeServer, versionEDS, goodEDSResponse1, goodEDSRequest, cbEDS); err != nil { - t.Fatal(err) - } - versionEDS++ - - // Send a bad response, and check for nack. - if err := sendBadResp(t, xdsclient.ListenerResource, fakeServer, versionLDS, goodLDSRequest); err != nil { - t.Fatal(err) - } - versionLDS++ - if err := sendBadResp(t, xdsclient.RouteConfigResource, fakeServer, versionRDS, goodRDSRequest); err != nil { - t.Fatal(err) - } - versionRDS++ - if err := sendBadResp(t, xdsclient.ClusterResource, fakeServer, versionCDS, goodCDSRequest); err != nil { - t.Fatal(err) - } - versionCDS++ - if err := sendBadResp(t, xdsclient.EndpointsResource, fakeServer, versionEDS, goodEDSRequest); err != nil { - t.Fatal(err) - } - versionEDS++ - - // send another good response, and check for ack, with the new version. - if _, err := sendGoodResp(t, xdsclient.ListenerResource, fakeServer, versionLDS, goodLDSResponse1, goodLDSRequest, cbLDS); err != nil { - t.Fatal(err) - } - versionLDS++ - if _, err := sendGoodResp(t, xdsclient.RouteConfigResource, fakeServer, versionRDS, goodRDSResponse1, goodRDSRequest, cbRDS); err != nil { - t.Fatal(err) - } - versionRDS++ - if _, err := sendGoodResp(t, xdsclient.ClusterResource, fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, cbCDS); err != nil { - t.Fatal(err) - } - versionCDS++ - if _, err := sendGoodResp(t, xdsclient.EndpointsResource, fakeServer, versionEDS, goodEDSResponse1, goodEDSRequest, cbEDS); err != nil { - t.Fatal(err) - } - versionEDS++ -} - -// Test when the first response is invalid, and is nacked, the nack requests -// should have an empty version string. -func (s) TestV2ClientAckFirstIsNack(t *testing.T) { - var versionLDS = 1000 - - fakeServer, cc, cleanup := startServerAndGetCC(t) - defer cleanup() - - v2c, cbLDS, _, _, _, v2cCleanup := startXDSV2Client(t, cc) - defer v2cCleanup() - - // Start the watch, send a good response, and check for ack. - startXDS(t, xdsclient.ListenerResource, v2c, fakeServer.XDSRequestChan, goodLDSRequest, "", "") - - nonce := sendXDSRespWithVersion(fakeServer.XDSResponseChan, &xdspb.DiscoveryResponse{ - Resources: []*anypb.Any{{}}, - TypeUrl: version.V2ListenerURL, - }, versionLDS) - t.Logf("Bad response pushed to fakeServer...") - - // The expected version string is an empty string, because this is the first - // response, and it's nacked (so there's no previous ack version). - if err := compareXDSRequest(fakeServer.XDSRequestChan, goodLDSRequest, "", nonce); err != nil { - t.Errorf("Failed to receive request: %v", err) - } - t.Logf("Bad response nacked") - versionLDS++ - - sendGoodResp(t, xdsclient.ListenerResource, fakeServer, versionLDS, goodLDSResponse1, goodLDSRequest, cbLDS) - versionLDS++ -} - -// Test when a nack is sent after a new watch, we nack with the previous acked -// version (instead of resetting to empty string). -func (s) TestV2ClientAckNackAfterNewWatch(t *testing.T) { - var versionLDS = 1000 - - fakeServer, cc, cleanup := startServerAndGetCC(t) - defer cleanup() - - v2c, cbLDS, _, _, _, v2cCleanup := startXDSV2Client(t, cc) - defer v2cCleanup() - - // Start the watch, send a good response, and check for ack. - startXDS(t, xdsclient.ListenerResource, v2c, fakeServer.XDSRequestChan, goodLDSRequest, "", "") - nonce, err := sendGoodResp(t, xdsclient.ListenerResource, fakeServer, versionLDS, goodLDSResponse1, goodLDSRequest, cbLDS) - if err != nil { - t.Fatal(err) - } - // Start a new watch. The version in the new request should be the version - // from the previous response, thus versionLDS before ++. - startXDS(t, xdsclient.ListenerResource, v2c, fakeServer.XDSRequestChan, goodLDSRequest, strconv.Itoa(versionLDS), nonce) - versionLDS++ - - // This is an invalid response after the new watch. - nonce = sendXDSRespWithVersion(fakeServer.XDSResponseChan, &xdspb.DiscoveryResponse{ - Resources: []*anypb.Any{{}}, - TypeUrl: version.V2ListenerURL, - }, versionLDS) - t.Logf("Bad response pushed to fakeServer...") - - // The expected version string is the previous acked version. - if err := compareXDSRequest(fakeServer.XDSRequestChan, goodLDSRequest, strconv.Itoa(versionLDS-1), nonce); err != nil { - t.Errorf("Failed to receive request: %v", err) - } - t.Logf("Bad response nacked") - versionLDS++ - - if _, err := sendGoodResp(t, xdsclient.ListenerResource, fakeServer, versionLDS, goodLDSResponse1, goodLDSRequest, cbLDS); err != nil { - t.Fatal(err) - } - versionLDS++ -} - -// TestV2ClientAckNewWatchAfterCancel verifies the new request for a new watch -// after the previous watch is canceled, has the right version. -func (s) TestV2ClientAckNewWatchAfterCancel(t *testing.T) { - var versionCDS = 3000 - - fakeServer, cc, cleanup := startServerAndGetCC(t) - defer cleanup() - - v2c, _, _, cbCDS, _, v2cCleanup := startXDSV2Client(t, cc) - defer v2cCleanup() - - // Start a CDS watch. - v2c.AddWatch(xdsclient.ClusterResource, goodClusterName1) - if err := compareXDSRequest(fakeServer.XDSRequestChan, goodCDSRequest, "", ""); err != nil { - t.Fatal(err) - } - t.Logf("FakeServer received %v request...", xdsclient.ClusterResource) - - // Send a good CDS response, this function waits for the ACK with the right - // version. - nonce, err := sendGoodResp(t, xdsclient.ClusterResource, fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, cbCDS) - if err != nil { - t.Fatal(err) - } - // Cancel the CDS watch, and start a new one. The new watch should have the - // version from the response above. - v2c.RemoveWatch(xdsclient.ClusterResource, goodClusterName1) - // Wait for a request with no resource names, because the only watch was - // removed. - emptyReq := &xdspb.DiscoveryRequest{Node: goodNodeProto, TypeUrl: version.V2ClusterURL} - if err := compareXDSRequest(fakeServer.XDSRequestChan, emptyReq, strconv.Itoa(versionCDS), nonce); err != nil { - t.Fatalf("Failed to receive %v request: %v", xdsclient.ClusterResource, err) - } - v2c.AddWatch(xdsclient.ClusterResource, goodClusterName1) - // Wait for a request with correct resource names and version. - if err := compareXDSRequest(fakeServer.XDSRequestChan, goodCDSRequest, strconv.Itoa(versionCDS), nonce); err != nil { - t.Fatalf("Failed to receive %v request: %v", xdsclient.ClusterResource, err) - } - versionCDS++ - - // Send a bad response with the next version. - if err := sendBadResp(t, xdsclient.ClusterResource, fakeServer, versionCDS, goodCDSRequest); err != nil { - t.Fatal(err) - } - versionCDS++ - - // send another good response, and check for ack, with the new version. - if _, err := sendGoodResp(t, xdsclient.ClusterResource, fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, cbCDS); err != nil { - t.Fatal(err) - } - versionCDS++ -} - -// TestV2ClientAckCancelResponseRace verifies if the response and ACK request -// race with cancel (which means the ACK request will not be sent on wire, -// because there's no active watch), the nonce will still be updated, and the -// new request with the new watch will have the correct nonce. -func (s) TestV2ClientAckCancelResponseRace(t *testing.T) { - var versionCDS = 3000 - - fakeServer, cc, cleanup := startServerAndGetCC(t) - defer cleanup() - - v2c, _, _, cbCDS, _, v2cCleanup := startXDSV2Client(t, cc) - defer v2cCleanup() - - // Start a CDS watch. - v2c.AddWatch(xdsclient.ClusterResource, goodClusterName1) - if err := compareXDSRequest(fakeServer.XDSRequestChan, goodCDSRequest, "", ""); err != nil { - t.Fatalf("Failed to receive %v request: %v", xdsclient.ClusterResource, err) - } - t.Logf("FakeServer received %v request...", xdsclient.ClusterResource) - - // send a good response, and check for ack, with the new version. - nonce, err := sendGoodResp(t, xdsclient.ClusterResource, fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, cbCDS) - if err != nil { - t.Fatal(err) - } - // Cancel the watch before the next response is sent. This mimics the case - // watch is canceled while response is on wire. - v2c.RemoveWatch(xdsclient.ClusterResource, goodClusterName1) - // Wait for a request with no resource names, because the only watch was - // removed. - emptyReq := &xdspb.DiscoveryRequest{Node: goodNodeProto, TypeUrl: version.V2ClusterURL} - if err := compareXDSRequest(fakeServer.XDSRequestChan, emptyReq, strconv.Itoa(versionCDS), nonce); err != nil { - t.Fatalf("Failed to receive %v request: %v", xdsclient.ClusterResource, err) - } - versionCDS++ - - if req, err := fakeServer.XDSRequestChan.Receive(); err != testutils.ErrRecvTimeout { - t.Fatalf("Got unexpected xds request after watch is canceled: %v", req) - } - - // Send a good response. - nonce = sendXDSRespWithVersion(fakeServer.XDSResponseChan, goodCDSResponse1, versionCDS) - t.Logf("Good %v response pushed to fakeServer...", xdsclient.ClusterResource) - - // Expect no ACK because watch was canceled. - if req, err := fakeServer.XDSRequestChan.Receive(); err != testutils.ErrRecvTimeout { - t.Fatalf("Got unexpected xds request after watch is canceled: %v", req) - } - // Still expected an callback update, because response was good. - if _, err := cbCDS.Receive(); err != nil { - t.Fatalf("Timeout when expecting %v update", xdsclient.ClusterResource) - } - - // Start a new watch. The new watch should have the nonce from the response - // above, and version from the first good response. - v2c.AddWatch(xdsclient.ClusterResource, goodClusterName1) - if err := compareXDSRequest(fakeServer.XDSRequestChan, goodCDSRequest, strconv.Itoa(versionCDS-1), nonce); err != nil { - t.Fatalf("Failed to receive %v request: %v", xdsclient.ClusterResource, err) - } - - // Send a bad response with the next version. - if err := sendBadResp(t, xdsclient.ClusterResource, fakeServer, versionCDS, goodCDSRequest); err != nil { - t.Fatal(err) - } - versionCDS++ - - // send another good response, and check for ack, with the new version. - if _, err := sendGoodResp(t, xdsclient.ClusterResource, fakeServer, versionCDS, goodCDSResponse1, goodCDSRequest, cbCDS); err != nil { - t.Fatal(err) - } - versionCDS++ -} diff --git a/xds/internal/client/v2/client_cds_test.go b/xds/internal/client/v2/client_cds_test.go deleted file mode 100644 index 8538f4e46afc..000000000000 --- a/xds/internal/client/v2/client_cds_test.go +++ /dev/null @@ -1,189 +0,0 @@ -/* - * - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package v2 - -import ( - "testing" - "time" - - xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2" - corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" - "github.com/golang/protobuf/proto" - anypb "github.com/golang/protobuf/ptypes/any" - xdsclient "google.golang.org/grpc/xds/internal/client" - "google.golang.org/grpc/xds/internal/version" -) - -const ( - serviceName1 = "foo-service" - serviceName2 = "bar-service" -) - -var ( - badlyMarshaledCDSResponse = &xdspb.DiscoveryResponse{ - Resources: []*anypb.Any{ - { - TypeUrl: version.V2ClusterURL, - Value: []byte{1, 2, 3, 4}, - }, - }, - TypeUrl: version.V2ClusterURL, - } - goodCluster1 = &xdspb.Cluster{ - Name: goodClusterName1, - ClusterDiscoveryType: &xdspb.Cluster_Type{Type: xdspb.Cluster_EDS}, - EdsClusterConfig: &xdspb.Cluster_EdsClusterConfig{ - EdsConfig: &corepb.ConfigSource{ - ConfigSourceSpecifier: &corepb.ConfigSource_Ads{ - Ads: &corepb.AggregatedConfigSource{}, - }, - }, - ServiceName: serviceName1, - }, - LbPolicy: xdspb.Cluster_ROUND_ROBIN, - LrsServer: &corepb.ConfigSource{ - ConfigSourceSpecifier: &corepb.ConfigSource_Self{ - Self: &corepb.SelfConfigSource{}, - }, - }, - } - marshaledCluster1, _ = proto.Marshal(goodCluster1) - goodCluster2 = &xdspb.Cluster{ - Name: goodClusterName2, - ClusterDiscoveryType: &xdspb.Cluster_Type{Type: xdspb.Cluster_EDS}, - EdsClusterConfig: &xdspb.Cluster_EdsClusterConfig{ - EdsConfig: &corepb.ConfigSource{ - ConfigSourceSpecifier: &corepb.ConfigSource_Ads{ - Ads: &corepb.AggregatedConfigSource{}, - }, - }, - ServiceName: serviceName2, - }, - LbPolicy: xdspb.Cluster_ROUND_ROBIN, - } - marshaledCluster2, _ = proto.Marshal(goodCluster2) - goodCDSResponse1 = &xdspb.DiscoveryResponse{ - Resources: []*anypb.Any{ - { - TypeUrl: version.V2ClusterURL, - Value: marshaledCluster1, - }, - }, - TypeUrl: version.V2ClusterURL, - } - goodCDSResponse2 = &xdspb.DiscoveryResponse{ - Resources: []*anypb.Any{ - { - TypeUrl: version.V2ClusterURL, - Value: marshaledCluster2, - }, - }, - TypeUrl: version.V2ClusterURL, - } -) - -// TestCDSHandleResponse starts a fake xDS server, makes a ClientConn to it, -// and creates a v2Client using it. Then, it registers a CDS watcher and tests -// different CDS responses. -func (s) TestCDSHandleResponse(t *testing.T) { - tests := []struct { - name string - cdsResponse *xdspb.DiscoveryResponse - wantErr bool - wantUpdate *xdsclient.ClusterUpdate - wantUpdateErr bool - }{ - // Badly marshaled CDS response. - { - name: "badly-marshaled-response", - cdsResponse: badlyMarshaledCDSResponse, - wantErr: true, - wantUpdate: nil, - wantUpdateErr: false, - }, - // Response does not contain Cluster proto. - { - name: "no-cluster-proto-in-response", - cdsResponse: badResourceTypeInLDSResponse, - wantErr: true, - wantUpdate: nil, - wantUpdateErr: false, - }, - // Response contains no clusters. - { - name: "no-cluster", - cdsResponse: &xdspb.DiscoveryResponse{}, - wantErr: false, - wantUpdate: nil, - wantUpdateErr: false, - }, - // Response contains one good cluster we are not interested in. - { - name: "one-uninteresting-cluster", - cdsResponse: goodCDSResponse2, - wantErr: false, - wantUpdate: nil, - wantUpdateErr: false, - }, - // Response contains one cluster and it is good. - { - name: "one-good-cluster", - cdsResponse: goodCDSResponse1, - wantErr: false, - wantUpdate: &xdsclient.ClusterUpdate{ServiceName: serviceName1, EnableLRS: true}, - wantUpdateErr: false, - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - testWatchHandle(t, &watchHandleTestcase{ - rType: xdsclient.ClusterResource, - resourceName: goodClusterName1, - - responseToHandle: test.cdsResponse, - wantHandleErr: test.wantErr, - wantUpdate: test.wantUpdate, - wantUpdateErr: test.wantUpdateErr, - }) - }) - } -} - -// TestCDSHandleResponseWithoutWatch tests the case where the v2Client receives -// a CDS response without a registered watcher. -func (s) TestCDSHandleResponseWithoutWatch(t *testing.T) { - _, cc, cleanup := startServerAndGetCC(t) - defer cleanup() - - v2c, err := newV2Client(&testUpdateReceiver{ - f: func(xdsclient.ResourceType, map[string]interface{}) {}, - }, cc, goodNodeProto, func(int) time.Duration { return 0 }, nil) - if err != nil { - t.Fatal(err) - } - defer v2c.Close() - - if v2c.handleCDSResponse(badResourceTypeInLDSResponse) == nil { - t.Fatal("v2c.handleCDSResponse() succeeded, should have failed") - } - - if v2c.handleCDSResponse(goodCDSResponse1) != nil { - t.Fatal("v2c.handleCDSResponse() succeeded, should have failed") - } -} diff --git a/xds/internal/client/v2/client_eds_test.go b/xds/internal/client/v2/client_eds_test.go deleted file mode 100644 index 7af74f0b6393..000000000000 --- a/xds/internal/client/v2/client_eds_test.go +++ /dev/null @@ -1,170 +0,0 @@ -/* - * - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package v2 - -import ( - "testing" - "time" - - v2xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2" - "github.com/golang/protobuf/ptypes" - anypb "github.com/golang/protobuf/ptypes/any" - "google.golang.org/grpc/xds/internal" - xdsclient "google.golang.org/grpc/xds/internal/client" - "google.golang.org/grpc/xds/internal/testutils" - "google.golang.org/grpc/xds/internal/version" -) - -var ( - badlyMarshaledEDSResponse = &v2xdspb.DiscoveryResponse{ - Resources: []*anypb.Any{ - { - TypeUrl: version.V2EndpointsURL, - Value: []byte{1, 2, 3, 4}, - }, - }, - TypeUrl: version.V2EndpointsURL, - } - badResourceTypeInEDSResponse = &v2xdspb.DiscoveryResponse{ - Resources: []*anypb.Any{ - { - TypeUrl: httpConnManagerURL, - Value: marshaledConnMgr1, - }, - }, - TypeUrl: version.V2EndpointsURL, - } - goodEDSResponse1 = &v2xdspb.DiscoveryResponse{ - Resources: []*anypb.Any{ - func() *anypb.Any { - clab0 := testutils.NewClusterLoadAssignmentBuilder(goodEDSName, nil) - clab0.AddLocality("locality-1", 1, 1, []string{"addr1:314"}, nil) - clab0.AddLocality("locality-2", 1, 0, []string{"addr2:159"}, nil) - a, _ := ptypes.MarshalAny(clab0.Build()) - return a - }(), - }, - TypeUrl: version.V2EndpointsURL, - } - goodEDSResponse2 = &v2xdspb.DiscoveryResponse{ - Resources: []*anypb.Any{ - func() *anypb.Any { - clab0 := testutils.NewClusterLoadAssignmentBuilder("not-goodEDSName", nil) - clab0.AddLocality("locality-1", 1, 1, []string{"addr1:314"}, nil) - clab0.AddLocality("locality-2", 1, 0, []string{"addr2:159"}, nil) - a, _ := ptypes.MarshalAny(clab0.Build()) - return a - }(), - }, - TypeUrl: version.V2EndpointsURL, - } -) - -func (s) TestEDSHandleResponse(t *testing.T) { - tests := []struct { - name string - edsResponse *v2xdspb.DiscoveryResponse - wantErr bool - wantUpdate *xdsclient.EndpointsUpdate - wantUpdateErr bool - }{ - // Any in resource is badly marshaled. - { - name: "badly-marshaled_response", - edsResponse: badlyMarshaledEDSResponse, - wantErr: true, - wantUpdate: nil, - wantUpdateErr: false, - }, - // Response doesn't contain resource with the right type. - { - name: "no-config-in-response", - edsResponse: badResourceTypeInEDSResponse, - wantErr: true, - wantUpdate: nil, - wantUpdateErr: false, - }, - // Response contains one uninteresting ClusterLoadAssignment. - { - name: "one-uninterestring-assignment", - edsResponse: goodEDSResponse2, - wantErr: false, - wantUpdate: nil, - wantUpdateErr: false, - }, - // Response contains one good ClusterLoadAssignment. - { - name: "one-good-assignment", - edsResponse: goodEDSResponse1, - wantErr: false, - wantUpdate: &xdsclient.EndpointsUpdate{ - Localities: []xdsclient.Locality{ - { - Endpoints: []xdsclient.Endpoint{{Address: "addr1:314"}}, - ID: internal.LocalityID{SubZone: "locality-1"}, - Priority: 1, - Weight: 1, - }, - { - Endpoints: []xdsclient.Endpoint{{Address: "addr2:159"}}, - ID: internal.LocalityID{SubZone: "locality-2"}, - Priority: 0, - Weight: 1, - }, - }, - }, - wantUpdateErr: false, - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - testWatchHandle(t, &watchHandleTestcase{ - rType: xdsclient.EndpointsResource, - resourceName: goodEDSName, - responseToHandle: test.edsResponse, - wantHandleErr: test.wantErr, - wantUpdate: test.wantUpdate, - wantUpdateErr: test.wantUpdateErr, - }) - }) - } -} - -// TestEDSHandleResponseWithoutWatch tests the case where the v2Client -// receives an EDS response without a registered EDS watcher. -func (s) TestEDSHandleResponseWithoutWatch(t *testing.T) { - _, cc, cleanup := startServerAndGetCC(t) - defer cleanup() - - v2c, err := newV2Client(&testUpdateReceiver{ - f: func(xdsclient.ResourceType, map[string]interface{}) {}, - }, cc, goodNodeProto, func(int) time.Duration { return 0 }, nil) - if err != nil { - t.Fatal(err) - } - defer v2c.Close() - - if v2c.handleEDSResponse(badResourceTypeInEDSResponse) == nil { - t.Fatal("v2c.handleEDSResponse() succeeded, should have failed") - } - - if v2c.handleEDSResponse(goodEDSResponse1) != nil { - t.Fatal("v2c.handleEDSResponse() succeeded, should have failed") - } -} diff --git a/xds/internal/client/v2/client_lds_test.go b/xds/internal/client/v2/client_lds_test.go deleted file mode 100644 index 854cf3ccee73..000000000000 --- a/xds/internal/client/v2/client_lds_test.go +++ /dev/null @@ -1,148 +0,0 @@ -/* - * - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package v2 - -import ( - "testing" - "time" - - v2xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2" - - xdsclient "google.golang.org/grpc/xds/internal/client" -) - -// TestLDSHandleResponse starts a fake xDS server, makes a ClientConn to it, -// and creates a client using it. Then, it registers a watchLDS and tests -// different LDS responses. -func (s) TestLDSHandleResponse(t *testing.T) { - tests := []struct { - name string - ldsResponse *v2xdspb.DiscoveryResponse - wantErr bool - wantUpdate *xdsclient.ListenerUpdate - wantUpdateErr bool - }{ - // Badly marshaled LDS response. - { - name: "badly-marshaled-response", - ldsResponse: badlyMarshaledLDSResponse, - wantErr: true, - wantUpdate: nil, - wantUpdateErr: false, - }, - // Response does not contain Listener proto. - { - name: "no-listener-proto-in-response", - ldsResponse: badResourceTypeInLDSResponse, - wantErr: true, - wantUpdate: nil, - wantUpdateErr: false, - }, - // No APIListener in the response. Just one test case here for a bad - // ApiListener, since the others are covered in - // TestGetRouteConfigNameFromListener. - { - name: "no-apiListener-in-response", - ldsResponse: noAPIListenerLDSResponse, - wantErr: true, - wantUpdate: nil, - wantUpdateErr: false, - }, - // Response contains one listener and it is good. - { - name: "one-good-listener", - ldsResponse: goodLDSResponse1, - wantErr: false, - wantUpdate: &xdsclient.ListenerUpdate{RouteConfigName: goodRouteName1}, - wantUpdateErr: false, - }, - // Response contains multiple good listeners, including the one we are - // interested in. - { - name: "multiple-good-listener", - ldsResponse: ldsResponseWithMultipleResources, - wantErr: false, - wantUpdate: &xdsclient.ListenerUpdate{RouteConfigName: goodRouteName1}, - wantUpdateErr: false, - }, - // Response contains two good listeners (one interesting and one - // uninteresting), and one badly marshaled listener. This will cause a - // nack because the uninteresting listener will still be parsed. - { - name: "good-bad-ugly-listeners", - ldsResponse: goodBadUglyLDSResponse, - wantErr: true, - wantUpdate: nil, - wantUpdateErr: false, - }, - // Response contains one listener, but we are not interested in it. - { - name: "one-uninteresting-listener", - ldsResponse: goodLDSResponse2, - wantErr: false, - wantUpdate: nil, - wantUpdateErr: false, - }, - // Response constains no resources. This is the case where the server - // does not know about the target we are interested in. - { - name: "empty-response", - ldsResponse: emptyLDSResponse, - wantErr: false, - wantUpdate: nil, - wantUpdateErr: false, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - testWatchHandle(t, &watchHandleTestcase{ - rType: xdsclient.ListenerResource, - resourceName: goodLDSTarget1, - responseToHandle: test.ldsResponse, - wantHandleErr: test.wantErr, - wantUpdate: test.wantUpdate, - wantUpdateErr: test.wantUpdateErr, - }) - }) - } -} - -// TestLDSHandleResponseWithoutWatch tests the case where the client receives -// an LDS response without a registered watcher. -func (s) TestLDSHandleResponseWithoutWatch(t *testing.T) { - _, cc, cleanup := startServerAndGetCC(t) - defer cleanup() - - v2c, err := newV2Client(&testUpdateReceiver{ - f: func(xdsclient.ResourceType, map[string]interface{}) {}, - }, cc, goodNodeProto, func(int) time.Duration { return 0 }, nil) - if err != nil { - t.Fatal(err) - } - defer v2c.Close() - - if v2c.handleLDSResponse(badResourceTypeInLDSResponse) == nil { - t.Fatal("v2c.handleLDSResponse() succeeded, should have failed") - } - - if v2c.handleLDSResponse(goodLDSResponse1) != nil { - t.Fatal("v2c.handleLDSResponse() succeeded, should have failed") - } -} diff --git a/xds/internal/client/v2/client_rds_test.go b/xds/internal/client/v2/client_rds_test.go deleted file mode 100644 index 59e63d83e287..000000000000 --- a/xds/internal/client/v2/client_rds_test.go +++ /dev/null @@ -1,167 +0,0 @@ -/* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package v2 - -import ( - "testing" - "time" - - xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2" - - xdsclient "google.golang.org/grpc/xds/internal/client" - "google.golang.org/grpc/xds/internal/testutils/fakeserver" -) - -// doLDS makes a LDS watch, and waits for the response and ack to finish. -// -// This is called by RDS tests to start LDS first, because LDS is a -// pre-requirement for RDS, and RDS handle would fail without an existing LDS -// watch. -func doLDS(t *testing.T, v2c xdsclient.APIClient, fakeServer *fakeserver.Server) { - v2c.AddWatch(xdsclient.ListenerResource, goodLDSTarget1) - if _, err := fakeServer.XDSRequestChan.Receive(); err != nil { - t.Fatalf("Timeout waiting for LDS request: %v", err) - } -} - -// TestRDSHandleResponseWithRouting starts a fake xDS server, makes a ClientConn -// to it, and creates a v2Client using it. Then, it registers an LDS and RDS -// watcher and tests different RDS responses. -func (s) TestRDSHandleResponseWithRouting(t *testing.T) { - tests := []struct { - name string - rdsResponse *xdspb.DiscoveryResponse - wantErr bool - wantUpdate *xdsclient.RouteConfigUpdate - wantUpdateErr bool - }{ - // Badly marshaled RDS response. - { - name: "badly-marshaled-response", - rdsResponse: badlyMarshaledRDSResponse, - wantErr: true, - wantUpdate: nil, - wantUpdateErr: false, - }, - // Response does not contain RouteConfiguration proto. - { - name: "no-route-config-in-response", - rdsResponse: badResourceTypeInRDSResponse, - wantErr: true, - wantUpdate: nil, - wantUpdateErr: false, - }, - // No VirtualHosts in the response. Just one test case here for a bad - // RouteConfiguration, since the others are covered in - // TestGetClusterFromRouteConfiguration. - { - name: "no-virtual-hosts-in-response", - rdsResponse: noVirtualHostsInRDSResponse, - wantErr: true, - wantUpdate: nil, - wantUpdateErr: false, - }, - // Response contains one good RouteConfiguration, uninteresting though. - { - name: "one-uninteresting-route-config", - rdsResponse: goodRDSResponse2, - wantErr: false, - wantUpdate: nil, - wantUpdateErr: false, - }, - // Response contains one good interesting RouteConfiguration. - { - name: "one-good-route-config", - rdsResponse: goodRDSResponse1, - wantErr: false, - wantUpdate: &xdsclient.RouteConfigUpdate{Routes: []*xdsclient.Route{{Prefix: newStringP(""), Action: map[string]uint32{goodClusterName1: 1}}}}, - wantUpdateErr: false, - }, - { - name: "one-good-route-config with routes", - rdsResponse: goodRDSResponse1, - wantErr: false, - wantUpdate: &xdsclient.RouteConfigUpdate{ - // Instead of just weighted targets when routing is disabled, - // this result contains a route with perfix "", and action as - // weighted targets. - Routes: []*xdsclient.Route{{ - Prefix: newStringP(""), - Action: map[string]uint32{goodClusterName1: 1}, - }}, - }, - wantUpdateErr: false, - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - testWatchHandle(t, &watchHandleTestcase{ - rType: xdsclient.RouteConfigResource, - resourceName: goodRouteName1, - responseToHandle: test.rdsResponse, - wantHandleErr: test.wantErr, - wantUpdate: test.wantUpdate, - wantUpdateErr: test.wantUpdateErr, - }) - }) - } -} - -// TestRDSHandleResponseWithoutLDSWatch tests the case where the v2Client -// receives an RDS response without a registered LDS watcher. -func (s) TestRDSHandleResponseWithoutLDSWatch(t *testing.T) { - _, cc, cleanup := startServerAndGetCC(t) - defer cleanup() - - v2c, err := newV2Client(&testUpdateReceiver{ - f: func(xdsclient.ResourceType, map[string]interface{}) {}, - }, cc, goodNodeProto, func(int) time.Duration { return 0 }, nil) - if err != nil { - t.Fatal(err) - } - defer v2c.Close() - - if v2c.handleRDSResponse(goodRDSResponse1) == nil { - t.Fatal("v2c.handleRDSResponse() succeeded, should have failed") - } -} - -// TestRDSHandleResponseWithoutRDSWatch tests the case where the v2Client -// receives an RDS response without a registered RDS watcher. -func (s) TestRDSHandleResponseWithoutRDSWatch(t *testing.T) { - fakeServer, cc, cleanup := startServerAndGetCC(t) - defer cleanup() - - v2c, err := newV2Client(&testUpdateReceiver{ - f: func(xdsclient.ResourceType, map[string]interface{}) {}, - }, cc, goodNodeProto, func(int) time.Duration { return 0 }, nil) - if err != nil { - t.Fatal(err) - } - defer v2c.Close() - doLDS(t, v2c, fakeServer) - - if v2c.handleRDSResponse(badResourceTypeInRDSResponse) == nil { - t.Fatal("v2c.handleRDSResponse() succeeded, should have failed") - } - - if v2c.handleRDSResponse(goodRDSResponse1) != nil { - t.Fatal("v2c.handleRDSResponse() succeeded, should have failed") - } -} diff --git a/xds/internal/client/v2/client_test.go b/xds/internal/client/v2/client_test.go deleted file mode 100644 index fecda6ad06d9..000000000000 --- a/xds/internal/client/v2/client_test.go +++ /dev/null @@ -1,687 +0,0 @@ -/* - * - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package v2 - -import ( - "errors" - "reflect" - "testing" - "time" - - "github.com/golang/protobuf/proto" - "github.com/google/go-cmp/cmp" - "google.golang.org/grpc" - "google.golang.org/grpc/internal/grpclog" - "google.golang.org/grpc/internal/grpctest" - "google.golang.org/grpc/resolver" - "google.golang.org/grpc/resolver/manual" - xdsclient "google.golang.org/grpc/xds/internal/client" - "google.golang.org/grpc/xds/internal/testutils" - "google.golang.org/grpc/xds/internal/testutils/fakeserver" - "google.golang.org/grpc/xds/internal/version" - - xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2" - basepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" - routepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/route" - httppb "github.com/envoyproxy/go-control-plane/envoy/config/filter/network/http_connection_manager/v2" - listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v2" - anypb "github.com/golang/protobuf/ptypes/any" - structpb "github.com/golang/protobuf/ptypes/struct" -) - -type s struct { - grpctest.Tester -} - -func Test(t *testing.T) { - grpctest.RunSubTests(t, s{}) -} - -const ( - goodLDSTarget1 = "lds.target.good:1111" - goodLDSTarget2 = "lds.target.good:2222" - goodRouteName1 = "GoodRouteConfig1" - goodRouteName2 = "GoodRouteConfig2" - goodEDSName = "GoodClusterAssignment1" - uninterestingRouteName = "UninterestingRouteName" - uninterestingDomain = "uninteresting.domain" - goodClusterName1 = "GoodClusterName1" - goodClusterName2 = "GoodClusterName2" - uninterestingClusterName = "UninterestingClusterName" - httpConnManagerURL = "type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager" -) - -var ( - goodNodeProto = &basepb.Node{ - Id: "ENVOY_NODE_ID", - Metadata: &structpb.Struct{ - Fields: map[string]*structpb.Value{ - "TRAFFICDIRECTOR_GRPC_HOSTNAME": { - Kind: &structpb.Value_StringValue{StringValue: "trafficdirector"}, - }, - }, - }, - } - goodLDSRequest = &xdspb.DiscoveryRequest{ - Node: goodNodeProto, - TypeUrl: version.V2ListenerURL, - ResourceNames: []string{goodLDSTarget1}, - } - goodRDSRequest = &xdspb.DiscoveryRequest{ - Node: goodNodeProto, - TypeUrl: version.V2RouteConfigURL, - ResourceNames: []string{goodRouteName1}, - } - goodCDSRequest = &xdspb.DiscoveryRequest{ - Node: goodNodeProto, - TypeUrl: version.V2ClusterURL, - ResourceNames: []string{goodClusterName1}, - } - goodEDSRequest = &xdspb.DiscoveryRequest{ - Node: goodNodeProto, - TypeUrl: version.V2EndpointsURL, - ResourceNames: []string{goodEDSName}, - } - goodHTTPConnManager1 = &httppb.HttpConnectionManager{ - RouteSpecifier: &httppb.HttpConnectionManager_Rds{ - Rds: &httppb.Rds{ - ConfigSource: &basepb.ConfigSource{ - ConfigSourceSpecifier: &basepb.ConfigSource_Ads{Ads: &basepb.AggregatedConfigSource{}}, - }, - RouteConfigName: goodRouteName1, - }, - }, - } - marshaledConnMgr1, _ = proto.Marshal(goodHTTPConnManager1) - goodListener1 = &xdspb.Listener{ - Name: goodLDSTarget1, - ApiListener: &listenerpb.ApiListener{ - ApiListener: &anypb.Any{ - TypeUrl: httpConnManagerURL, - Value: marshaledConnMgr1, - }, - }, - } - marshaledListener1, _ = proto.Marshal(goodListener1) - goodListener2 = &xdspb.Listener{ - Name: goodLDSTarget2, - ApiListener: &listenerpb.ApiListener{ - ApiListener: &anypb.Any{ - TypeUrl: httpConnManagerURL, - Value: marshaledConnMgr1, - }, - }, - } - marshaledListener2, _ = proto.Marshal(goodListener2) - noAPIListener = &xdspb.Listener{Name: goodLDSTarget1} - marshaledNoAPIListener, _ = proto.Marshal(noAPIListener) - badAPIListener2 = &xdspb.Listener{ - Name: goodLDSTarget2, - ApiListener: &listenerpb.ApiListener{ - ApiListener: &anypb.Any{ - TypeUrl: httpConnManagerURL, - Value: []byte{1, 2, 3, 4}, - }, - }, - } - badlyMarshaledAPIListener2, _ = proto.Marshal(badAPIListener2) - goodLDSResponse1 = &xdspb.DiscoveryResponse{ - Resources: []*anypb.Any{ - { - TypeUrl: version.V2ListenerURL, - Value: marshaledListener1, - }, - }, - TypeUrl: version.V2ListenerURL, - } - goodLDSResponse2 = &xdspb.DiscoveryResponse{ - Resources: []*anypb.Any{ - { - TypeUrl: version.V2ListenerURL, - Value: marshaledListener2, - }, - }, - TypeUrl: version.V2ListenerURL, - } - emptyLDSResponse = &xdspb.DiscoveryResponse{TypeUrl: version.V2ListenerURL} - badlyMarshaledLDSResponse = &xdspb.DiscoveryResponse{ - Resources: []*anypb.Any{ - { - TypeUrl: version.V2ListenerURL, - Value: []byte{1, 2, 3, 4}, - }, - }, - TypeUrl: version.V2ListenerURL, - } - badResourceTypeInLDSResponse = &xdspb.DiscoveryResponse{ - Resources: []*anypb.Any{ - { - TypeUrl: httpConnManagerURL, - Value: marshaledConnMgr1, - }, - }, - TypeUrl: version.V2ListenerURL, - } - ldsResponseWithMultipleResources = &xdspb.DiscoveryResponse{ - Resources: []*anypb.Any{ - { - TypeUrl: version.V2ListenerURL, - Value: marshaledListener2, - }, - { - TypeUrl: version.V2ListenerURL, - Value: marshaledListener1, - }, - }, - TypeUrl: version.V2ListenerURL, - } - noAPIListenerLDSResponse = &xdspb.DiscoveryResponse{ - Resources: []*anypb.Any{ - { - TypeUrl: version.V2ListenerURL, - Value: marshaledNoAPIListener, - }, - }, - TypeUrl: version.V2ListenerURL, - } - goodBadUglyLDSResponse = &xdspb.DiscoveryResponse{ - Resources: []*anypb.Any{ - { - TypeUrl: version.V2ListenerURL, - Value: marshaledListener2, - }, - { - TypeUrl: version.V2ListenerURL, - Value: marshaledListener1, - }, - { - TypeUrl: version.V2ListenerURL, - Value: badlyMarshaledAPIListener2, - }, - }, - TypeUrl: version.V2ListenerURL, - } - badlyMarshaledRDSResponse = &xdspb.DiscoveryResponse{ - Resources: []*anypb.Any{ - { - TypeUrl: version.V2RouteConfigURL, - Value: []byte{1, 2, 3, 4}, - }, - }, - TypeUrl: version.V2RouteConfigURL, - } - badResourceTypeInRDSResponse = &xdspb.DiscoveryResponse{ - Resources: []*anypb.Any{ - { - TypeUrl: httpConnManagerURL, - Value: marshaledConnMgr1, - }, - }, - TypeUrl: version.V2RouteConfigURL, - } - emptyRouteConfig = &xdspb.RouteConfiguration{} - marshaledEmptyRouteConfig, _ = proto.Marshal(emptyRouteConfig) - noVirtualHostsInRDSResponse = &xdspb.DiscoveryResponse{ - Resources: []*anypb.Any{ - { - TypeUrl: version.V2RouteConfigURL, - Value: marshaledEmptyRouteConfig, - }, - }, - TypeUrl: version.V2RouteConfigURL, - } - goodRouteConfig1 = &xdspb.RouteConfiguration{ - Name: goodRouteName1, - VirtualHosts: []*routepb.VirtualHost{ - { - Domains: []string{uninterestingDomain}, - Routes: []*routepb.Route{ - { - Match: &routepb.RouteMatch{PathSpecifier: &routepb.RouteMatch_Prefix{Prefix: ""}}, - Action: &routepb.Route_Route{ - Route: &routepb.RouteAction{ - ClusterSpecifier: &routepb.RouteAction_Cluster{Cluster: uninterestingClusterName}, - }, - }, - }, - }, - }, - { - Domains: []string{goodLDSTarget1}, - Routes: []*routepb.Route{ - { - Match: &routepb.RouteMatch{PathSpecifier: &routepb.RouteMatch_Prefix{Prefix: ""}}, - Action: &routepb.Route_Route{ - Route: &routepb.RouteAction{ - ClusterSpecifier: &routepb.RouteAction_Cluster{Cluster: goodClusterName1}, - }, - }, - }, - }, - }, - }, - } - marshaledGoodRouteConfig1, _ = proto.Marshal(goodRouteConfig1) - goodRouteConfig2 = &xdspb.RouteConfiguration{ - Name: goodRouteName2, - VirtualHosts: []*routepb.VirtualHost{ - { - Domains: []string{uninterestingDomain}, - Routes: []*routepb.Route{ - { - Match: &routepb.RouteMatch{PathSpecifier: &routepb.RouteMatch_Prefix{Prefix: ""}}, - Action: &routepb.Route_Route{ - Route: &routepb.RouteAction{ - ClusterSpecifier: &routepb.RouteAction_Cluster{Cluster: uninterestingClusterName}, - }, - }, - }, - }, - }, - { - Domains: []string{goodLDSTarget1}, - Routes: []*routepb.Route{ - { - Match: &routepb.RouteMatch{PathSpecifier: &routepb.RouteMatch_Prefix{Prefix: ""}}, - Action: &routepb.Route_Route{ - Route: &routepb.RouteAction{ - ClusterSpecifier: &routepb.RouteAction_Cluster{Cluster: goodClusterName2}, - }, - }, - }, - }, - }, - }, - } - marshaledGoodRouteConfig2, _ = proto.Marshal(goodRouteConfig2) - goodRDSResponse1 = &xdspb.DiscoveryResponse{ - Resources: []*anypb.Any{ - { - TypeUrl: version.V2RouteConfigURL, - Value: marshaledGoodRouteConfig1, - }, - }, - TypeUrl: version.V2RouteConfigURL, - } - goodRDSResponse2 = &xdspb.DiscoveryResponse{ - Resources: []*anypb.Any{ - { - TypeUrl: version.V2RouteConfigURL, - Value: marshaledGoodRouteConfig2, - }, - }, - TypeUrl: version.V2RouteConfigURL, - } -) - -type watchHandleTestcase struct { - rType xdsclient.ResourceType - resourceName string - - responseToHandle *xdspb.DiscoveryResponse - wantHandleErr bool - wantUpdate interface{} - wantUpdateErr bool -} - -type testUpdateReceiver struct { - f func(rType xdsclient.ResourceType, d map[string]interface{}) -} - -func (t *testUpdateReceiver) NewListeners(d map[string]xdsclient.ListenerUpdate) { - dd := make(map[string]interface{}) - for k, v := range d { - dd[k] = v - } - t.newUpdate(xdsclient.ListenerResource, dd) -} - -func (t *testUpdateReceiver) NewRouteConfigs(d map[string]xdsclient.RouteConfigUpdate) { - dd := make(map[string]interface{}) - for k, v := range d { - dd[k] = v - } - t.newUpdate(xdsclient.RouteConfigResource, dd) -} - -func (t *testUpdateReceiver) NewClusters(d map[string]xdsclient.ClusterUpdate) { - dd := make(map[string]interface{}) - for k, v := range d { - dd[k] = v - } - t.newUpdate(xdsclient.ClusterResource, dd) -} - -func (t *testUpdateReceiver) NewEndpoints(d map[string]xdsclient.EndpointsUpdate) { - dd := make(map[string]interface{}) - for k, v := range d { - dd[k] = v - } - t.newUpdate(xdsclient.EndpointsResource, dd) -} - -func (t *testUpdateReceiver) newUpdate(rType xdsclient.ResourceType, d map[string]interface{}) { - t.f(rType, d) -} - -// testWatchHandle is called to test response handling for each xDS. -// -// It starts the xDS watch as configured in test, waits for the fake xds server -// to receive the request (so watch callback is installed), and calls -// handleXDSResp with responseToHandle (if it's set). It then compares the -// update received by watch callback with the expected results. -func testWatchHandle(t *testing.T, test *watchHandleTestcase) { - fakeServer, cc, cleanup := startServerAndGetCC(t) - defer cleanup() - - type updateErr struct { - u interface{} - err error - } - gotUpdateCh := testutils.NewChannel() - - v2c, err := newV2Client(&testUpdateReceiver{ - f: func(rType xdsclient.ResourceType, d map[string]interface{}) { - if rType == test.rType { - if u, ok := d[test.resourceName]; ok { - gotUpdateCh.Send(updateErr{u, nil}) - } - } - }, - }, cc, goodNodeProto, func(int) time.Duration { return 0 }, nil) - if err != nil { - t.Fatal(err) - } - defer v2c.Close() - - // RDS needs an existing LDS watch for the hostname. - if test.rType == xdsclient.RouteConfigResource { - doLDS(t, v2c, fakeServer) - } - - // Register the watcher, this will also trigger the v2Client to send the xDS - // request. - v2c.AddWatch(test.rType, test.resourceName) - - // Wait till the request makes it to the fakeServer. This ensures that - // the watch request has been processed by the v2Client. - if _, err := fakeServer.XDSRequestChan.Receive(); err != nil { - t.Fatalf("Timeout waiting for an xDS request: %v", err) - } - - // Directly push the response through a call to handleXDSResp. This bypasses - // the fakeServer, so it's only testing the handle logic. Client response - // processing is covered elsewhere. - // - // Also note that this won't trigger ACK, so there's no need to clear the - // request channel afterwards. - var handleXDSResp func(response *xdspb.DiscoveryResponse) error - switch test.rType { - case xdsclient.ListenerResource: - handleXDSResp = v2c.handleLDSResponse - case xdsclient.RouteConfigResource: - handleXDSResp = v2c.handleRDSResponse - case xdsclient.ClusterResource: - handleXDSResp = v2c.handleCDSResponse - case xdsclient.EndpointsResource: - handleXDSResp = v2c.handleEDSResponse - } - if err := handleXDSResp(test.responseToHandle); (err != nil) != test.wantHandleErr { - t.Fatalf("v2c.handleRDSResponse() returned err: %v, wantErr: %v", err, test.wantHandleErr) - } - - // If the test doesn't expect the callback to be invoked, verify that no - // update or error is pushed to the callback. - // - // Cannot directly compare test.wantUpdate with nil (typed vs non-typed nil: - // https://golang.org/doc/faq#nil_error). - if c := test.wantUpdate; c == nil || (reflect.ValueOf(c).Kind() == reflect.Ptr && reflect.ValueOf(c).IsNil()) { - update, err := gotUpdateCh.Receive() - if err == testutils.ErrRecvTimeout { - return - } - t.Fatalf("Unexpected update: +%v", update) - } - - wantUpdate := reflect.ValueOf(test.wantUpdate).Elem().Interface() - uErr, err := gotUpdateCh.Receive() - if err == testutils.ErrRecvTimeout { - t.Fatal("Timeout expecting xDS update") - } - gotUpdate := uErr.(updateErr).u - if diff := cmp.Diff(gotUpdate, wantUpdate); diff != "" { - t.Fatalf("got update : %+v, want %+v, diff: %s", gotUpdate, wantUpdate, diff) - } - gotUpdateErr := uErr.(updateErr).err - if (gotUpdateErr != nil) != test.wantUpdateErr { - t.Fatalf("got xDS update error {%v}, wantErr: %v", gotUpdateErr, test.wantUpdateErr) - } -} - -// startServerAndGetCC starts a fake XDS server and also returns a ClientConn -// connected to it. -func startServerAndGetCC(t *testing.T) (*fakeserver.Server, *grpc.ClientConn, func()) { - t.Helper() - - fs, sCleanup, err := fakeserver.StartServer() - if err != nil { - t.Fatalf("Failed to start fake xDS server: %v", err) - } - - cc, ccCleanup, err := fs.XDSClientConn() - if err != nil { - sCleanup() - t.Fatalf("Failed to get a clientConn to the fake xDS server: %v", err) - } - return fs, cc, func() { - sCleanup() - ccCleanup() - } -} - -func newV2Client(p xdsclient.UpdateHandler, cc *grpc.ClientConn, n *basepb.Node, b func(int) time.Duration, l *grpclog.PrefixLogger) (*client, error) { - c, err := newClient(cc, xdsclient.BuildOptions{ - Parent: p, - NodeProto: n, - Backoff: b, - Logger: l, - }) - if err != nil { - return nil, err - } - return c.(*client), nil -} - -// TestV2ClientBackoffAfterRecvError verifies if the v2Client backs off when it -// encounters a Recv error while receiving an LDS response. -func (s) TestV2ClientBackoffAfterRecvError(t *testing.T) { - fakeServer, cc, cleanup := startServerAndGetCC(t) - defer cleanup() - - // Override the v2Client backoff function with this, so that we can verify - // that a backoff actually was triggered. - boCh := make(chan int, 1) - clientBackoff := func(v int) time.Duration { - boCh <- v - return 0 - } - - callbackCh := make(chan struct{}) - v2c, err := newV2Client(&testUpdateReceiver{ - f: func(xdsclient.ResourceType, map[string]interface{}) { close(callbackCh) }, - }, cc, goodNodeProto, clientBackoff, nil) - if err != nil { - t.Fatal(err) - } - defer v2c.Close() - t.Log("Started xds v2Client...") - - v2c.AddWatch(xdsclient.ListenerResource, goodLDSTarget1) - if _, err := fakeServer.XDSRequestChan.Receive(); err != nil { - t.Fatalf("Timeout expired when expecting an LDS request") - } - t.Log("FakeServer received request...") - - fakeServer.XDSResponseChan <- &fakeserver.Response{Err: errors.New("RPC error")} - t.Log("Bad LDS response pushed to fakeServer...") - - timer := time.NewTimer(1 * time.Second) - select { - case <-timer.C: - t.Fatal("Timeout when expecting LDS update") - case <-boCh: - timer.Stop() - t.Log("v2Client backed off before retrying...") - case <-callbackCh: - t.Fatal("Received unexpected LDS callback") - } - - if _, err := fakeServer.XDSRequestChan.Receive(); err != nil { - t.Fatalf("Timeout expired when expecting an LDS request") - } - t.Log("FakeServer received request after backoff...") -} - -// TestV2ClientRetriesAfterBrokenStream verifies the case where a stream -// encountered a Recv() error, and is expected to send out xDS requests for -// registered watchers once it comes back up again. -func (s) TestV2ClientRetriesAfterBrokenStream(t *testing.T) { - fakeServer, cc, cleanup := startServerAndGetCC(t) - defer cleanup() - - callbackCh := testutils.NewChannel() - v2c, err := newV2Client(&testUpdateReceiver{ - f: func(rType xdsclient.ResourceType, d map[string]interface{}) { - if rType == xdsclient.ListenerResource { - if u, ok := d[goodLDSTarget1]; ok { - t.Logf("Received LDS callback with ldsUpdate {%+v}", u) - callbackCh.Send(struct{}{}) - } - } - }, - }, cc, goodNodeProto, func(int) time.Duration { return 0 }, nil) - if err != nil { - t.Fatal(err) - } - defer v2c.Close() - t.Log("Started xds v2Client...") - - v2c.AddWatch(xdsclient.ListenerResource, goodLDSTarget1) - if _, err := fakeServer.XDSRequestChan.Receive(); err != nil { - t.Fatalf("Timeout expired when expecting an LDS request") - } - t.Log("FakeServer received request...") - - fakeServer.XDSResponseChan <- &fakeserver.Response{Resp: goodLDSResponse1} - t.Log("Good LDS response pushed to fakeServer...") - - if _, err := callbackCh.Receive(); err != nil { - t.Fatal("Timeout when expecting LDS update") - } - - // Read the ack, so the next request is sent after stream re-creation. - if _, err := fakeServer.XDSRequestChan.Receive(); err != nil { - t.Fatalf("Timeout expired when expecting an LDS ACK") - } - - fakeServer.XDSResponseChan <- &fakeserver.Response{Err: errors.New("RPC error")} - t.Log("Bad LDS response pushed to fakeServer...") - - val, err := fakeServer.XDSRequestChan.Receive() - if err == testutils.ErrRecvTimeout { - t.Fatalf("Timeout expired when expecting LDS update") - } - gotRequest := val.(*fakeserver.Request) - if !proto.Equal(gotRequest.Req, goodLDSRequest) { - t.Fatalf("gotRequest: %+v, wantRequest: %+v", gotRequest.Req, goodLDSRequest) - } -} - -// TestV2ClientWatchWithoutStream verifies the case where a watch is started -// when the xds stream is not created. The watcher should not receive any update -// (because there won't be any xds response, and timeout is done at a upper -// level). And when the stream is re-created, the watcher should get future -// updates. -func (s) TestV2ClientWatchWithoutStream(t *testing.T) { - fakeServer, sCleanup, err := fakeserver.StartServer() - if err != nil { - t.Fatalf("Failed to start fake xDS server: %v", err) - } - defer sCleanup() - - const scheme = "xds_client_test_whatever" - rb := manual.NewBuilderWithScheme(scheme) - rb.InitialState(resolver.State{Addresses: []resolver.Address{{Addr: "no.such.server"}}}) - - cc, err := grpc.Dial(scheme+":///whatever", grpc.WithInsecure(), grpc.WithResolvers(rb)) - if err != nil { - t.Fatalf("Failed to dial ClientConn: %v", err) - } - defer cc.Close() - - callbackCh := testutils.NewChannel() - v2c, err := newV2Client(&testUpdateReceiver{ - f: func(rType xdsclient.ResourceType, d map[string]interface{}) { - if rType == xdsclient.ListenerResource { - if u, ok := d[goodLDSTarget1]; ok { - t.Logf("Received LDS callback with ldsUpdate {%+v}", u) - callbackCh.Send(u) - } - } - }, - }, cc, goodNodeProto, func(int) time.Duration { return 0 }, nil) - if err != nil { - t.Fatal(err) - } - defer v2c.Close() - t.Log("Started xds v2Client...") - - // This watch is started when the xds-ClientConn is in Transient Failure, - // and no xds stream is created. - v2c.AddWatch(xdsclient.ListenerResource, goodLDSTarget1) - - // The watcher should receive an update, with a timeout error in it. - if v, err := callbackCh.TimedReceive(100 * time.Millisecond); err == nil { - t.Fatalf("Expect an timeout error from watcher, got %v", v) - } - - // Send the real server address to the ClientConn, the stream should be - // created, and the previous watch should be sent. - rb.UpdateState(resolver.State{ - Addresses: []resolver.Address{{Addr: fakeServer.Address}}, - }) - - if _, err := fakeServer.XDSRequestChan.Receive(); err != nil { - t.Fatalf("Timeout expired when expecting an LDS request") - } - t.Log("FakeServer received request...") - - fakeServer.XDSResponseChan <- &fakeserver.Response{Resp: goodLDSResponse1} - t.Log("Good LDS response pushed to fakeServer...") - - if v, err := callbackCh.Receive(); err != nil { - t.Fatal("Timeout when expecting LDS update") - } else if _, ok := v.(xdsclient.ListenerUpdate); !ok { - t.Fatalf("Expect an LDS update from watcher, got %v", v) - } -} - -func newStringP(s string) *string { - return &s -} diff --git a/xds/internal/client/v3/client.go b/xds/internal/client/v3/client.go deleted file mode 100644 index a1103f8f8b85..000000000000 --- a/xds/internal/client/v3/client.go +++ /dev/null @@ -1,270 +0,0 @@ -/* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -// Package v3 provides xDS v3 transport protocol specific functionality. -package v3 - -import ( - "context" - "fmt" - "sync" - - "github.com/golang/protobuf/proto" - "google.golang.org/grpc" - "google.golang.org/grpc/internal/grpclog" - xdsclient "google.golang.org/grpc/xds/internal/client" - "google.golang.org/grpc/xds/internal/version" - - v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" - v3adsgrpc "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" - v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" -) - -func init() { - xdsclient.RegisterAPIClientBuilder(clientBuilder{}) -} - -var ( - resourceTypeToURL = map[xdsclient.ResourceType]string{ - xdsclient.ListenerResource: version.V2ListenerURL, - xdsclient.RouteConfigResource: version.V2RouteConfigURL, - xdsclient.ClusterResource: version.V2ClusterURL, - xdsclient.EndpointsResource: version.V2EndpointsURL, - } -) - -type clientBuilder struct{} - -func (clientBuilder) Build(cc *grpc.ClientConn, opts xdsclient.BuildOptions) (xdsclient.APIClient, error) { - return newClient(cc, opts) -} - -func (clientBuilder) Version() version.TransportAPI { - return version.TransportV3 -} - -func newClient(cc *grpc.ClientConn, opts xdsclient.BuildOptions) (xdsclient.APIClient, error) { - nodeProto, ok := opts.NodeProto.(*v3corepb.Node) - if !ok { - return nil, fmt.Errorf("xds: unsupported Node proto type: %T, want %T", opts.NodeProto, v3corepb.Node{}) - } - v3c := &client{ - cc: cc, - parent: opts.Parent, - nodeProto: nodeProto, - logger: opts.Logger, - } - v3c.ctx, v3c.cancelCtx = context.WithCancel(context.Background()) - v3c.TransportHelper = xdsclient.NewTransportHelper(v3c, opts.Logger, opts.Backoff) - return v3c, nil -} - -type adsStream v3adsgrpc.AggregatedDiscoveryService_StreamAggregatedResourcesClient - -// client performs the actual xDS RPCs using the xDS v3 API. It creates a -// single ADS stream on which the different types of xDS requests and responses -// are multiplexed. -type client struct { - *xdsclient.TransportHelper - - ctx context.Context - cancelCtx context.CancelFunc - parent xdsclient.UpdateHandler - logger *grpclog.PrefixLogger - - // ClientConn to the xDS gRPC server. Owned by the parent xdsClient. - cc *grpc.ClientConn - nodeProto *v3corepb.Node - - mu sync.Mutex - // ldsResourceName is the LDS resource_name to watch. It is set to the first - // LDS resource_name to watch, and removed when the LDS watch is canceled. - // - // It's from the dial target of the parent ClientConn. RDS resource - // processing needs this to do the host matching. - ldsResourceName string - ldsWatchCount int -} - -// AddWatch overrides the transport helper's AddWatch to save the LDS -// resource_name. This is required when handling an RDS response to perform host -// matching. -func (v3c *client) AddWatch(rType xdsclient.ResourceType, rName string) { - v3c.mu.Lock() - // Special handling for LDS, because RDS needs the LDS resource_name for - // response host matching. - if rType == xdsclient.ListenerResource { - // Set hostname to the first LDS resource_name, and reset it when the - // last LDS watch is removed. The upper level Client isn't expected to - // watchLDS more than once. - v3c.ldsWatchCount++ - if v3c.ldsWatchCount == 1 { - v3c.ldsResourceName = rName - } - } - v3c.mu.Unlock() - v3c.TransportHelper.AddWatch(rType, rName) -} - -func (v3c *client) RemoveWatch(rType xdsclient.ResourceType, rName string) { - v3c.mu.Lock() - // Special handling for LDS, because RDS needs the LDS resource_name for - // response host matching. - if rType == xdsclient.ListenerResource { - // Set hostname to the first LDS resource_name, and reset it when the - // last LDS watch is removed. The upper level Client isn't expected to - // watchLDS more than once. - v3c.ldsWatchCount-- - if v3c.ldsWatchCount == 0 { - v3c.ldsResourceName = "" - } - } - v3c.mu.Unlock() - v3c.TransportHelper.RemoveWatch(rType, rName) -} - -func (v3c *client) NewStream(ctx context.Context) (grpc.ClientStream, error) { - return v3adsgrpc.NewAggregatedDiscoveryServiceClient(v3c.cc).StreamAggregatedResources(v3c.ctx, grpc.WaitForReady(true)) -} - -// sendRequest sends out a DiscoveryRequest for the given resourceNames, of type -// rType, on the provided stream. -// -// version is the ack version to be sent with the request -// - If this is the new request (not an ack/nack), version will be empty. -// - If this is an ack, version will be the version from the response. -// - If this is a nack, version will be the previous acked version (from -// versionMap). If there was no ack before, it will be empty. -func (v3c *client) SendRequest(s grpc.ClientStream, resourceNames []string, rType xdsclient.ResourceType, version, nonce string) error { - stream, ok := s.(adsStream) - if !ok { - return fmt.Errorf("xds: Attempt to send request on unsupported stream type: %T", s) - } - req := &v3discoverypb.DiscoveryRequest{ - Node: v3c.nodeProto, - TypeUrl: resourceTypeToURL[rType], - ResourceNames: resourceNames, - VersionInfo: version, - ResponseNonce: nonce, - // TODO: populate ErrorDetails for nack. - } - if err := stream.Send(req); err != nil { - return fmt.Errorf("xds: stream.Send(%+v) failed: %v", req, err) - } - v3c.logger.Debugf("ADS request sent: %v", req) - return nil -} - -// RecvResponse blocks on the receipt of one response message on the provided -// stream. -func (v3c *client) RecvResponse(s grpc.ClientStream) (proto.Message, error) { - stream, ok := s.(adsStream) - if !ok { - return nil, fmt.Errorf("xds: Attempt to receive response on unsupported stream type: %T", s) - } - - resp, err := stream.Recv() - if err != nil { - // TODO: call watch callbacks with error when stream is broken. - return nil, fmt.Errorf("xds: stream.Recv() failed: %v", err) - } - v3c.logger.Infof("ADS response received, type: %v", resp.GetTypeUrl()) - v3c.logger.Debugf("ADS response received: %v", resp) - return resp, nil -} - -func (v3c *client) HandleResponse(r proto.Message) (xdsclient.ResourceType, string, string, error) { - rType := xdsclient.UnknownResource - resp, ok := r.(*v3discoverypb.DiscoveryResponse) - if !ok { - return rType, "", "", fmt.Errorf("xds: unsupported message type: %T", resp) - } - - // Note that the xDS transport protocol is versioned independently of - // the resource types, and it is supported to transfer older versions - // of resource types using new versions of the transport protocol, or - // vice-versa. Hence we need to handle v3 type_urls as well here. - var err error - url := resp.GetTypeUrl() - switch { - case xdsclient.IsListenerResource(url): - err = v3c.handleLDSResponse(resp) - rType = xdsclient.ListenerResource - case xdsclient.IsRouteConfigResource(url): - err = v3c.handleRDSResponse(resp) - rType = xdsclient.RouteConfigResource - case xdsclient.IsClusterResource(url): - err = v3c.handleCDSResponse(resp) - rType = xdsclient.ClusterResource - case xdsclient.IsEndpointsResource(url): - err = v3c.handleEDSResponse(resp) - rType = xdsclient.EndpointsResource - default: - return rType, "", "", xdsclient.ErrResourceTypeUnsupported{ - ErrStr: fmt.Sprintf("Resource type %v unknown in response from server", resp.GetTypeUrl()), - } - } - return rType, resp.GetVersionInfo(), resp.GetNonce(), err -} - -// handleLDSResponse processes an LDS response received from the xDS server. On -// receipt of a good response, it also invokes the registered watcher callback. -func (v3c *client) handleLDSResponse(resp *v3discoverypb.DiscoveryResponse) error { - update, err := xdsclient.UnmarshalListener(resp.GetResources(), v3c.logger) - if err != nil { - return err - } - v3c.parent.NewListeners(update) - return nil -} - -// handleRDSResponse processes an RDS response received from the xDS server. On -// receipt of a good response, it caches validated resources and also invokes -// the registered watcher callback. -func (v3c *client) handleRDSResponse(resp *v3discoverypb.DiscoveryResponse) error { - v3c.mu.Lock() - hostname := v3c.ldsResourceName - v3c.mu.Unlock() - - update, err := xdsclient.UnmarshalRouteConfig(resp.GetResources(), hostname, v3c.logger) - if err != nil { - return err - } - v3c.parent.NewRouteConfigs(update) - return nil -} - -// handleCDSResponse processes an CDS response received from the xDS server. On -// receipt of a good response, it also invokes the registered watcher callback. -func (v3c *client) handleCDSResponse(resp *v3discoverypb.DiscoveryResponse) error { - update, err := xdsclient.UnmarshalCluster(resp.GetResources(), v3c.logger) - if err != nil { - return err - } - v3c.parent.NewClusters(update) - return nil -} - -func (v3c *client) handleEDSResponse(resp *v3discoverypb.DiscoveryResponse) error { - update, err := xdsclient.UnmarshalEndpoints(resp.GetResources(), v3c.logger) - if err != nil { - return err - } - v3c.parent.NewEndpoints(update) - return nil -} diff --git a/xds/internal/clusterspecifier/cluster_specifier.go b/xds/internal/clusterspecifier/cluster_specifier.go new file mode 100644 index 000000000000..8fcb83cdbb1e --- /dev/null +++ b/xds/internal/clusterspecifier/cluster_specifier.go @@ -0,0 +1,72 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package clusterspecifier contains the ClusterSpecifier interface and a registry for +// storing and retrieving their implementations. +package clusterspecifier + +import ( + "github.com/golang/protobuf/proto" +) + +// BalancerConfig is the Go Native JSON representation of a balancer +// configuration. +type BalancerConfig []map[string]any + +// ClusterSpecifier defines the parsing functionality of a Cluster Specifier. +type ClusterSpecifier interface { + // TypeURLs are the proto message types supported by this + // ClusterSpecifierPlugin. A ClusterSpecifierPlugin will be registered by + // each of its supported message types. + TypeURLs() []string + // ParseClusterSpecifierConfig parses the provided configuration + // proto.Message from the top level RDS configuration. The resulting + // BalancerConfig will be used as configuration for a child LB Policy of the + // Cluster Manager LB Policy. A nil BalancerConfig is invalid. + ParseClusterSpecifierConfig(proto.Message) (BalancerConfig, error) +} + +var ( + // m is a map from scheme to filter. + m = make(map[string]ClusterSpecifier) +) + +// Register registers the ClusterSpecifierPlugin to the ClusterSpecifier map. +// cs.TypeURLs() will be used as the types for this ClusterSpecifierPlugin. +// +// NOTE: this function must only be called during initialization time (i.e. in +// an init() function), and is not thread-safe. If multiple cluster specifier +// plugins are registered with the same type URL, the one registered last will +// take effect. +func Register(cs ClusterSpecifier) { + for _, u := range cs.TypeURLs() { + m[u] = cs + } +} + +// Get returns the ClusterSpecifier registered with typeURL. +// +// If no cluster specifier is registered with typeURL, nil will be returned. +func Get(typeURL string) ClusterSpecifier { + return m[typeURL] +} + +// UnregisterForTesting unregisters the ClusterSpecifier for testing purposes. +func UnregisterForTesting(typeURL string) { + delete(m, typeURL) +} diff --git a/xds/internal/clusterspecifier/rls/rls.go b/xds/internal/clusterspecifier/rls/rls.go new file mode 100644 index 000000000000..4c39e85739db --- /dev/null +++ b/xds/internal/clusterspecifier/rls/rls.go @@ -0,0 +1,109 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package rls implements the RLS cluster specifier plugin. +package rls + +import ( + "encoding/json" + "fmt" + + "github.com/golang/protobuf/proto" + "github.com/golang/protobuf/ptypes" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/envconfig" + rlspb "google.golang.org/grpc/internal/proto/grpc_lookup_v1" + "google.golang.org/grpc/xds/internal/clusterspecifier" + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/types/known/anypb" +) + +func init() { + if envconfig.XDSRLS { + clusterspecifier.Register(rls{}) + } + + // TODO: Remove these once the RLS env var is removed. + internal.RegisterRLSClusterSpecifierPluginForTesting = func() { + clusterspecifier.Register(rls{}) + } + internal.UnregisterRLSClusterSpecifierPluginForTesting = func() { + for _, typeURL := range rls.TypeURLs(rls{}) { + clusterspecifier.UnregisterForTesting(typeURL) + } + } +} + +type rls struct{} + +func (rls) TypeURLs() []string { + return []string{"type.googleapis.com/grpc.lookup.v1.RouteLookupClusterSpecifier"} +} + +// lbConfigJSON is the RLS LB Policies configuration in JSON format. +// RouteLookupConfig will be a raw JSON string from the passed in proto +// configuration, and the other fields will be hardcoded. +type lbConfigJSON struct { + RouteLookupConfig json.RawMessage `json:"routeLookupConfig"` + ChildPolicy []map[string]json.RawMessage `json:"childPolicy"` + ChildPolicyConfigTargetFieldName string `json:"childPolicyConfigTargetFieldName"` +} + +func (rls) ParseClusterSpecifierConfig(cfg proto.Message) (clusterspecifier.BalancerConfig, error) { + if cfg == nil { + return nil, fmt.Errorf("rls_csp: nil configuration message provided") + } + any, ok := cfg.(*anypb.Any) + if !ok { + return nil, fmt.Errorf("rls_csp: error parsing config %v: unknown type %T", cfg, cfg) + } + rlcs := new(rlspb.RouteLookupClusterSpecifier) + + if err := ptypes.UnmarshalAny(any, rlcs); err != nil { + return nil, fmt.Errorf("rls_csp: error parsing config %v: %v", cfg, err) + } + rlcJSON, err := protojson.Marshal(rlcs.GetRouteLookupConfig()) + if err != nil { + return nil, fmt.Errorf("rls_csp: error marshaling route lookup config: %v: %v", rlcs.GetRouteLookupConfig(), err) + } + lbCfgJSON := &lbConfigJSON{ + RouteLookupConfig: rlcJSON, // "JSON form of RouteLookupClusterSpecifier.config" - RLS in xDS Design Doc + ChildPolicy: []map[string]json.RawMessage{ + { + "cds_experimental": json.RawMessage("{}"), + }, + }, + ChildPolicyConfigTargetFieldName: "cluster", + } + + rawJSON, err := json.Marshal(lbCfgJSON) + if err != nil { + return nil, fmt.Errorf("rls_csp: error marshaling load balancing config %v: %v", lbCfgJSON, err) + } + + rlsBB := balancer.Get(internal.RLSLoadBalancingPolicyName) + if rlsBB == nil { + return nil, fmt.Errorf("RLS LB policy not registered") + } + if _, err = rlsBB.(balancer.ConfigParser).ParseConfig(rawJSON); err != nil { + return nil, fmt.Errorf("rls_csp: validation error from rls lb policy parsing: %v", err) + } + + return clusterspecifier.BalancerConfig{{internal.RLSLoadBalancingPolicyName: lbCfgJSON}}, nil +} diff --git a/xds/internal/clusterspecifier/rls/rls_test.go b/xds/internal/clusterspecifier/rls/rls_test.go new file mode 100644 index 000000000000..0be52f4db406 --- /dev/null +++ b/xds/internal/clusterspecifier/rls/rls_test.go @@ -0,0 +1,169 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package rls + +import ( + "encoding/json" + "testing" + + "github.com/golang/protobuf/proto" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "google.golang.org/grpc/internal/grpctest" + rlspb "google.golang.org/grpc/internal/proto/grpc_lookup_v1" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/xds/internal/clusterspecifier" + "google.golang.org/protobuf/types/known/durationpb" + + _ "google.golang.org/grpc/balancer/rls" // Register the RLS LB policy. + _ "google.golang.org/grpc/xds/internal/balancer/cdsbalancer" // Register the CDS LB policy. +) + +func init() { + clusterspecifier.Register(rls{}) +} + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +// TestParseClusterSpecifierConfig tests the parsing functionality of the RLS +// Cluster Specifier Plugin. +func (s) TestParseClusterSpecifierConfig(t *testing.T) { + tests := []struct { + name string + rlcs proto.Message + wantConfig clusterspecifier.BalancerConfig + wantErr bool + }{ + { + name: "invalid-rls-cluster-specifier", + rlcs: rlsClusterSpecifierConfigError, + wantErr: true, + }, + { + name: "valid-rls-cluster-specifier", + rlcs: rlsClusterSpecifierConfigWithoutTransformations, + wantConfig: configWithoutTransformationsWant, + }, + } + for _, test := range tests { + cs := clusterspecifier.Get("type.googleapis.com/grpc.lookup.v1.RouteLookupClusterSpecifier") + if cs == nil { + t.Fatal("Error getting cluster specifier") + } + lbCfg, err := cs.ParseClusterSpecifierConfig(test.rlcs) + + if (err != nil) != test.wantErr { + t.Fatalf("ParseClusterSpecifierConfig(%+v) returned err: %v, wantErr: %v", test.rlcs, err, test.wantErr) + } + if test.wantErr { // Successfully received an error. + return + } + // Marshal and then unmarshal into any to get rid of nondeterministic + // protojson Marshaling. + lbCfgJSON, err := json.Marshal(lbCfg) + if err != nil { + t.Fatalf("json.Marshal(%+v) returned err %v", lbCfg, err) + } + var got any + err = json.Unmarshal(lbCfgJSON, got) + if err != nil { + t.Fatalf("json.Unmarshal(%+v) returned err %v", lbCfgJSON, err) + } + wantCfgJSON, err := json.Marshal(test.wantConfig) + if err != nil { + t.Fatalf("json.Marshal(%+v) returned err %v", test.wantConfig, err) + } + var want any + err = json.Unmarshal(wantCfgJSON, want) + if err != nil { + t.Fatalf("json.Unmarshal(%+v) returned err %v", lbCfgJSON, err) + } + if diff := cmp.Diff(want, got, cmpopts.EquateEmpty()); diff != "" { + t.Fatalf("ParseClusterSpecifierConfig(%+v) returned expected, diff (-want +got) %v", test.rlcs, diff) + } + } +} + +// This will error because the required match field is set in grpc key builder. +var rlsClusterSpecifierConfigError = testutils.MarshalAny(&rlspb.RouteLookupClusterSpecifier{ + RouteLookupConfig: &rlspb.RouteLookupConfig{ + GrpcKeybuilders: []*rlspb.GrpcKeyBuilder{ + { + Names: []*rlspb.GrpcKeyBuilder_Name{ + { + Service: "service", + Method: "method", + }, + }, + Headers: []*rlspb.NameMatcher{ + { + Key: "k1", + RequiredMatch: true, + Names: []string{"v1"}, + }, + }, + }, + }, + }, +}) + +// Corresponds to the rls unit test case in +// balancer/rls/internal/config_test.go. +var rlsClusterSpecifierConfigWithoutTransformations = testutils.MarshalAny(&rlspb.RouteLookupClusterSpecifier{ + RouteLookupConfig: &rlspb.RouteLookupConfig{ + GrpcKeybuilders: []*rlspb.GrpcKeyBuilder{ + { + Names: []*rlspb.GrpcKeyBuilder_Name{ + { + Service: "service", + Method: "method", + }, + }, + Headers: []*rlspb.NameMatcher{ + { + Key: "k1", + Names: []string{"v1"}, + }, + }, + }, + }, + LookupService: "target", + LookupServiceTimeout: &durationpb.Duration{Seconds: 100}, + MaxAge: &durationpb.Duration{Seconds: 60}, + StaleAge: &durationpb.Duration{Seconds: 50}, + CacheSizeBytes: 1000, + DefaultTarget: "passthrough:///default", + }, +}) + +var configWithoutTransformationsWant = clusterspecifier.BalancerConfig{{"rls_experimental": &lbConfigJSON{ + RouteLookupConfig: []byte(`{"grpcKeybuilders":[{"names":[{"service":"service","method":"method"}],"headers":[{"key":"k1","names":["v1"]}]}],"lookupService":"target","lookupServiceTimeout":"100s","maxAge":"60s","staleAge":"50s","cacheSizeBytes":"1000","defaultTarget":"passthrough:///default"}`), + ChildPolicy: []map[string]json.RawMessage{ + { + "cds_experimental": []byte(`{}`), + }, + }, + ChildPolicyConfigTargetFieldName: "cluster", +}}} diff --git a/xds/internal/httpfilter/fault/fault.go b/xds/internal/httpfilter/fault/fault.go new file mode 100644 index 000000000000..aa329d13ac30 --- /dev/null +++ b/xds/internal/httpfilter/fault/fault.go @@ -0,0 +1,301 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package fault implements the Envoy Fault Injection HTTP filter. +package fault + +import ( + "context" + "errors" + "fmt" + "io" + "strconv" + "sync/atomic" + "time" + + "github.com/golang/protobuf/proto" + "github.com/golang/protobuf/ptypes" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal/grpcrand" + iresolver "google.golang.org/grpc/internal/resolver" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/grpc/xds/internal/httpfilter" + "google.golang.org/protobuf/types/known/anypb" + + cpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/common/fault/v3" + fpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/fault/v3" + tpb "github.com/envoyproxy/go-control-plane/envoy/type/v3" +) + +const headerAbortHTTPStatus = "x-envoy-fault-abort-request" +const headerAbortGRPCStatus = "x-envoy-fault-abort-grpc-request" +const headerAbortPercentage = "x-envoy-fault-abort-request-percentage" + +const headerDelayPercentage = "x-envoy-fault-delay-request-percentage" +const headerDelayDuration = "x-envoy-fault-delay-request" + +var statusMap = map[int]codes.Code{ + 400: codes.Internal, + 401: codes.Unauthenticated, + 403: codes.PermissionDenied, + 404: codes.Unimplemented, + 429: codes.Unavailable, + 502: codes.Unavailable, + 503: codes.Unavailable, + 504: codes.Unavailable, +} + +func init() { + httpfilter.Register(builder{}) +} + +type builder struct { +} + +type config struct { + httpfilter.FilterConfig + config *fpb.HTTPFault +} + +func (builder) TypeURLs() []string { + return []string{"type.googleapis.com/envoy.extensions.filters.http.fault.v3.HTTPFault"} +} + +// Parsing is the same for the base config and the override config. +func parseConfig(cfg proto.Message) (httpfilter.FilterConfig, error) { + if cfg == nil { + return nil, fmt.Errorf("fault: nil configuration message provided") + } + any, ok := cfg.(*anypb.Any) + if !ok { + return nil, fmt.Errorf("fault: error parsing config %v: unknown type %T", cfg, cfg) + } + msg := new(fpb.HTTPFault) + if err := ptypes.UnmarshalAny(any, msg); err != nil { + return nil, fmt.Errorf("fault: error parsing config %v: %v", cfg, err) + } + return config{config: msg}, nil +} + +func (builder) ParseFilterConfig(cfg proto.Message) (httpfilter.FilterConfig, error) { + return parseConfig(cfg) +} + +func (builder) ParseFilterConfigOverride(override proto.Message) (httpfilter.FilterConfig, error) { + return parseConfig(override) +} + +func (builder) IsTerminal() bool { + return false +} + +var _ httpfilter.ClientInterceptorBuilder = builder{} + +func (builder) BuildClientInterceptor(cfg, override httpfilter.FilterConfig) (iresolver.ClientInterceptor, error) { + if cfg == nil { + return nil, fmt.Errorf("fault: nil config provided") + } + + c, ok := cfg.(config) + if !ok { + return nil, fmt.Errorf("fault: incorrect config type provided (%T): %v", cfg, cfg) + } + + if override != nil { + // override completely replaces the listener configuration; but we + // still validate the listener config type. + c, ok = override.(config) + if !ok { + return nil, fmt.Errorf("fault: incorrect override config type provided (%T): %v", override, override) + } + } + + icfg := c.config + if (icfg.GetMaxActiveFaults() != nil && icfg.GetMaxActiveFaults().GetValue() == 0) || + (icfg.GetDelay() == nil && icfg.GetAbort() == nil) { + return nil, nil + } + return &interceptor{config: icfg}, nil +} + +type interceptor struct { + config *fpb.HTTPFault +} + +var activeFaults uint32 // global active faults; accessed atomically + +func (i *interceptor) NewStream(ctx context.Context, ri iresolver.RPCInfo, done func(), newStream func(ctx context.Context, done func()) (iresolver.ClientStream, error)) (iresolver.ClientStream, error) { + if maxAF := i.config.GetMaxActiveFaults(); maxAF != nil { + defer atomic.AddUint32(&activeFaults, ^uint32(0)) // decrement counter + if af := atomic.AddUint32(&activeFaults, 1); af > maxAF.GetValue() { + // Would exceed maximum active fault limit. + return newStream(ctx, done) + } + } + + if err := injectDelay(ctx, i.config.GetDelay()); err != nil { + return nil, err + } + + if err := injectAbort(ctx, i.config.GetAbort()); err != nil { + if err == errOKStream { + return &okStream{ctx: ctx}, nil + } + return nil, err + } + return newStream(ctx, done) +} + +// For overriding in tests +var randIntn = grpcrand.Intn +var newTimer = time.NewTimer + +func injectDelay(ctx context.Context, delayCfg *cpb.FaultDelay) error { + numerator, denominator := splitPct(delayCfg.GetPercentage()) + var delay time.Duration + switch delayType := delayCfg.GetFaultDelaySecifier().(type) { + case *cpb.FaultDelay_FixedDelay: + delay = delayType.FixedDelay.AsDuration() + case *cpb.FaultDelay_HeaderDelay_: + md, _ := metadata.FromOutgoingContext(ctx) + v := md[headerDelayDuration] + if v == nil { + // No delay configured for this RPC. + return nil + } + ms, ok := parseIntFromMD(v) + if !ok { + // Malformed header; no delay. + return nil + } + delay = time.Duration(ms) * time.Millisecond + if v := md[headerDelayPercentage]; v != nil { + if num, ok := parseIntFromMD(v); ok && num < numerator { + numerator = num + } + } + } + if delay == 0 || randIntn(denominator) >= numerator { + return nil + } + t := newTimer(delay) + select { + case <-t.C: + case <-ctx.Done(): + t.Stop() + return ctx.Err() + } + return nil +} + +func injectAbort(ctx context.Context, abortCfg *fpb.FaultAbort) error { + numerator, denominator := splitPct(abortCfg.GetPercentage()) + code := codes.OK + okCode := false + switch errType := abortCfg.GetErrorType().(type) { + case *fpb.FaultAbort_HttpStatus: + code, okCode = grpcFromHTTP(int(errType.HttpStatus)) + case *fpb.FaultAbort_GrpcStatus: + code, okCode = sanitizeGRPCCode(codes.Code(errType.GrpcStatus)), true + case *fpb.FaultAbort_HeaderAbort_: + md, _ := metadata.FromOutgoingContext(ctx) + if v := md[headerAbortHTTPStatus]; v != nil { + // HTTP status has priority over gRPC status. + if httpStatus, ok := parseIntFromMD(v); ok { + code, okCode = grpcFromHTTP(httpStatus) + } + } else if v := md[headerAbortGRPCStatus]; v != nil { + if grpcStatus, ok := parseIntFromMD(v); ok { + code, okCode = sanitizeGRPCCode(codes.Code(grpcStatus)), true + } + } + if v := md[headerAbortPercentage]; v != nil { + if num, ok := parseIntFromMD(v); ok && num < numerator { + numerator = num + } + } + } + if !okCode || randIntn(denominator) >= numerator { + return nil + } + if code == codes.OK { + return errOKStream + } + return status.Errorf(code, "RPC terminated due to fault injection") +} + +var errOKStream = errors.New("stream terminated early with OK status") + +// parseIntFromMD returns the integer in the last header or nil if parsing +// failed. +func parseIntFromMD(header []string) (int, bool) { + if len(header) == 0 { + return 0, false + } + v, err := strconv.Atoi(header[len(header)-1]) + return v, err == nil +} + +func splitPct(fp *tpb.FractionalPercent) (num int, den int) { + if fp == nil { + return 0, 100 + } + num = int(fp.GetNumerator()) + switch fp.GetDenominator() { + case tpb.FractionalPercent_HUNDRED: + return num, 100 + case tpb.FractionalPercent_TEN_THOUSAND: + return num, 10 * 1000 + case tpb.FractionalPercent_MILLION: + return num, 1000 * 1000 + } + return num, 100 +} + +func grpcFromHTTP(httpStatus int) (codes.Code, bool) { + if httpStatus < 200 || httpStatus >= 600 { + // Malformed; ignore this fault type. + return codes.OK, false + } + if c := statusMap[httpStatus]; c != codes.OK { + // OK = 0/the default for the map. + return c, true + } + // All undefined HTTP status codes convert to Unknown. HTTP status of 200 + // is "success", but gRPC converts to Unknown due to missing grpc status. + return codes.Unknown, true +} + +func sanitizeGRPCCode(c codes.Code) codes.Code { + if c > codes.Code(16) { + return codes.Unknown + } + return c +} + +type okStream struct { + ctx context.Context +} + +func (*okStream) Header() (metadata.MD, error) { return nil, nil } +func (*okStream) Trailer() metadata.MD { return nil } +func (*okStream) CloseSend() error { return nil } +func (o *okStream) Context() context.Context { return o.ctx } +func (*okStream) SendMsg(m any) error { return io.EOF } +func (*okStream) RecvMsg(m any) error { return io.EOF } diff --git a/xds/internal/httpfilter/fault/fault_test.go b/xds/internal/httpfilter/fault/fault_test.go new file mode 100644 index 000000000000..48fac43c46d7 --- /dev/null +++ b/xds/internal/httpfilter/fault/fault_test.go @@ -0,0 +1,671 @@ +//go:build !386 +// +build !386 + +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package xds_test contains e2e tests for xDS use. +package fault + +import ( + "context" + "fmt" + "io" + "net" + "reflect" + "testing" + "time" + + "github.com/golang/protobuf/ptypes" + "github.com/google/uuid" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/internal/grpcrand" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/bootstrap" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/wrapperspb" + + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + cpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/common/fault/v3" + fpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/fault/v3" + v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + tpb "github.com/envoyproxy/go-control-plane/envoy/type/v3" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" + + _ "google.golang.org/grpc/xds/internal/balancer" // Register the balancers. + _ "google.golang.org/grpc/xds/internal/httpfilter/router" // Register the router filter. + _ "google.golang.org/grpc/xds/internal/resolver" // Register the xds_resolver. +) + +const defaultTestTimeout = 10 * time.Second + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +type testService struct { + testgrpc.TestServiceServer +} + +func (*testService) EmptyCall(context.Context, *testpb.Empty) (*testpb.Empty, error) { + return &testpb.Empty{}, nil +} + +func (*testService) FullDuplexCall(stream testgrpc.TestService_FullDuplexCallServer) error { + // End RPC after client does a CloseSend. + for { + if _, err := stream.Recv(); err == io.EOF { + return nil + } else if err != nil { + return err + } + } +} + +// clientSetup performs a bunch of steps common to all xDS server tests here: +// - spin up an xDS management server on a local port +// - spin up a gRPC server and register the test service on it +// - create a local TCP listener and start serving on it +// +// Returns the following: +// - the management server: tests use this to configure resources +// - nodeID expected by the management server: this is set in the Node proto +// sent by the xdsClient for queries. +// - the port the server is listening on +// - cleanup function to be invoked by the tests when done +func clientSetup(t *testing.T) (*e2e.ManagementServer, string, uint32, func()) { + // Spin up a xDS management server on a local port. + nodeID := uuid.New().String() + fs, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) + if err != nil { + t.Fatal(err) + } + + // Create a bootstrap file in a temporary directory. + bootstrapCleanup, err := bootstrap.CreateFile(bootstrap.Options{ + NodeID: nodeID, + ServerURI: fs.Address, + ServerListenerResourceNameTemplate: "grpc/server", + }) + if err != nil { + t.Fatal(err) + } + + // Initialize a gRPC server and register the stubServer on it. + server := grpc.NewServer() + testgrpc.RegisterTestServiceServer(server, &testService{}) + + // Create a local listener and pass it to Serve(). + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("testutils.LocalTCPListener() failed: %v", err) + } + + go func() { + if err := server.Serve(lis); err != nil { + t.Errorf("Serve() failed: %v", err) + } + }() + + return fs, nodeID, uint32(lis.Addr().(*net.TCPAddr).Port), func() { + fs.Stop() + bootstrapCleanup() + server.Stop() + } +} + +func (s) TestFaultInjection_Unary(t *testing.T) { + type subcase struct { + name string + code codes.Code + repeat int + randIn []int // Intn calls per-repeat (not per-subcase) + delays []time.Duration // NewTimer calls per-repeat (not per-subcase) + md metadata.MD + } + testCases := []struct { + name string + cfgs []*fpb.HTTPFault + randOutInc int + want []subcase + }{{ + name: "max faults zero", + cfgs: []*fpb.HTTPFault{{ + MaxActiveFaults: wrapperspb.UInt32(0), + Abort: &fpb.FaultAbort{ + Percentage: &tpb.FractionalPercent{Numerator: 100, Denominator: tpb.FractionalPercent_HUNDRED}, + ErrorType: &fpb.FaultAbort_GrpcStatus{GrpcStatus: uint32(codes.Aborted)}, + }, + }}, + randOutInc: 5, + want: []subcase{{ + code: codes.OK, + repeat: 25, + }}, + }, { + name: "no abort or delay", + cfgs: []*fpb.HTTPFault{{}}, + randOutInc: 5, + want: []subcase{{ + code: codes.OK, + repeat: 25, + }}, + }, { + name: "abort always", + cfgs: []*fpb.HTTPFault{{ + Abort: &fpb.FaultAbort{ + Percentage: &tpb.FractionalPercent{Numerator: 100, Denominator: tpb.FractionalPercent_HUNDRED}, + ErrorType: &fpb.FaultAbort_GrpcStatus{GrpcStatus: uint32(codes.Aborted)}, + }, + }}, + randOutInc: 5, + want: []subcase{{ + code: codes.Aborted, + randIn: []int{100}, + repeat: 25, + }}, + }, { + name: "abort 10%", + cfgs: []*fpb.HTTPFault{{ + Abort: &fpb.FaultAbort{ + Percentage: &tpb.FractionalPercent{Numerator: 100000, Denominator: tpb.FractionalPercent_MILLION}, + ErrorType: &fpb.FaultAbort_GrpcStatus{GrpcStatus: uint32(codes.Aborted)}, + }, + }}, + randOutInc: 50000, + want: []subcase{{ + name: "[0,10]%", + code: codes.Aborted, + randIn: []int{1000000}, + repeat: 2, + }, { + name: "(10,100]%", + code: codes.OK, + randIn: []int{1000000}, + repeat: 18, + }, { + name: "[0,10]% again", + code: codes.Aborted, + randIn: []int{1000000}, + repeat: 2, + }}, + }, { + name: "delay always", + cfgs: []*fpb.HTTPFault{{ + Delay: &cpb.FaultDelay{ + Percentage: &tpb.FractionalPercent{Numerator: 100, Denominator: tpb.FractionalPercent_HUNDRED}, + FaultDelaySecifier: &cpb.FaultDelay_FixedDelay{FixedDelay: ptypes.DurationProto(time.Second)}, + }, + }}, + randOutInc: 5, + want: []subcase{{ + randIn: []int{100}, + repeat: 25, + delays: []time.Duration{time.Second}, + }}, + }, { + name: "delay 10%", + cfgs: []*fpb.HTTPFault{{ + Delay: &cpb.FaultDelay{ + Percentage: &tpb.FractionalPercent{Numerator: 1000, Denominator: tpb.FractionalPercent_TEN_THOUSAND}, + FaultDelaySecifier: &cpb.FaultDelay_FixedDelay{FixedDelay: ptypes.DurationProto(time.Second)}, + }, + }}, + randOutInc: 500, + want: []subcase{{ + name: "[0,10]%", + randIn: []int{10000}, + repeat: 2, + delays: []time.Duration{time.Second}, + }, { + name: "(10,100]%", + randIn: []int{10000}, + repeat: 18, + }, { + name: "[0,10]% again", + randIn: []int{10000}, + repeat: 2, + delays: []time.Duration{time.Second}, + }}, + }, { + name: "delay 80%, abort 50%", + cfgs: []*fpb.HTTPFault{{ + Delay: &cpb.FaultDelay{ + Percentage: &tpb.FractionalPercent{Numerator: 80, Denominator: tpb.FractionalPercent_HUNDRED}, + FaultDelaySecifier: &cpb.FaultDelay_FixedDelay{FixedDelay: ptypes.DurationProto(3 * time.Second)}, + }, + Abort: &fpb.FaultAbort{ + Percentage: &tpb.FractionalPercent{Numerator: 50, Denominator: tpb.FractionalPercent_HUNDRED}, + ErrorType: &fpb.FaultAbort_GrpcStatus{GrpcStatus: uint32(codes.Unimplemented)}, + }, + }}, + randOutInc: 5, + want: []subcase{{ + name: "50% delay and abort", + code: codes.Unimplemented, + randIn: []int{100, 100}, + repeat: 10, + delays: []time.Duration{3 * time.Second}, + }, { + name: "30% delay, no abort", + randIn: []int{100, 100}, + repeat: 6, + delays: []time.Duration{3 * time.Second}, + }, { + name: "20% success", + randIn: []int{100, 100}, + repeat: 4, + }, { + name: "50% delay and abort again", + code: codes.Unimplemented, + randIn: []int{100, 100}, + repeat: 10, + delays: []time.Duration{3 * time.Second}, + }}, + }, { + name: "header abort", + cfgs: []*fpb.HTTPFault{{ + Abort: &fpb.FaultAbort{ + Percentage: &tpb.FractionalPercent{Numerator: 80, Denominator: tpb.FractionalPercent_HUNDRED}, + ErrorType: &fpb.FaultAbort_HeaderAbort_{}, + }, + }}, + randOutInc: 10, + want: []subcase{{ + name: "30% abort; [0,30]%", + md: metadata.MD{ + headerAbortGRPCStatus: []string{fmt.Sprintf("%d", codes.DataLoss)}, + headerAbortPercentage: []string{"30"}, + }, + code: codes.DataLoss, + randIn: []int{100}, + repeat: 3, + }, { + name: "30% abort; (30,60]%", + md: metadata.MD{ + headerAbortGRPCStatus: []string{fmt.Sprintf("%d", codes.DataLoss)}, + headerAbortPercentage: []string{"30"}, + }, + randIn: []int{100}, + repeat: 3, + }, { + name: "80% abort; (60,80]%", + md: metadata.MD{ + headerAbortGRPCStatus: []string{fmt.Sprintf("%d", codes.DataLoss)}, + headerAbortPercentage: []string{"80"}, + }, + code: codes.DataLoss, + randIn: []int{100}, + repeat: 2, + }, { + name: "cannot exceed percentage in filter", + md: metadata.MD{ + headerAbortGRPCStatus: []string{fmt.Sprintf("%d", codes.DataLoss)}, + headerAbortPercentage: []string{"100"}, + }, + randIn: []int{100}, + repeat: 2, + }, { + name: "HTTP Status 404", + md: metadata.MD{ + headerAbortHTTPStatus: []string{"404"}, + headerAbortPercentage: []string{"100"}, + }, + code: codes.Unimplemented, + randIn: []int{100}, + repeat: 1, + }, { + name: "HTTP Status 429", + md: metadata.MD{ + headerAbortHTTPStatus: []string{"429"}, + headerAbortPercentage: []string{"100"}, + }, + code: codes.Unavailable, + randIn: []int{100}, + repeat: 1, + }, { + name: "HTTP Status 200", + md: metadata.MD{ + headerAbortHTTPStatus: []string{"200"}, + headerAbortPercentage: []string{"100"}, + }, + // No GRPC status, but HTTP Status of 200 translates to Unknown, + // per spec in statuscodes.md. + code: codes.Unknown, + randIn: []int{100}, + repeat: 1, + }, { + name: "gRPC Status OK", + md: metadata.MD{ + headerAbortGRPCStatus: []string{fmt.Sprintf("%d", codes.OK)}, + headerAbortPercentage: []string{"100"}, + }, + // This should be Unimplemented (mismatched request/response + // count), per spec in statuscodes.md, but grpc-go currently + // returns io.EOF which status.Code() converts to Unknown + code: codes.Unknown, + randIn: []int{100}, + repeat: 1, + }, { + name: "invalid header results in no abort", + md: metadata.MD{ + headerAbortGRPCStatus: []string{"error"}, + headerAbortPercentage: []string{"100"}, + }, + repeat: 1, + }, { + name: "invalid header results in default percentage", + md: metadata.MD{ + headerAbortGRPCStatus: []string{fmt.Sprintf("%d", codes.DataLoss)}, + headerAbortPercentage: []string{"error"}, + }, + code: codes.DataLoss, + randIn: []int{100}, + repeat: 1, + }}, + }, { + name: "header delay", + cfgs: []*fpb.HTTPFault{{ + Delay: &cpb.FaultDelay{ + Percentage: &tpb.FractionalPercent{Numerator: 80, Denominator: tpb.FractionalPercent_HUNDRED}, + FaultDelaySecifier: &cpb.FaultDelay_HeaderDelay_{}, + }, + }}, + randOutInc: 10, + want: []subcase{{ + name: "30% delay; [0,30]%", + md: metadata.MD{ + headerDelayDuration: []string{"2"}, + headerDelayPercentage: []string{"30"}, + }, + randIn: []int{100}, + delays: []time.Duration{2 * time.Millisecond}, + repeat: 3, + }, { + name: "30% delay; (30, 60]%", + md: metadata.MD{ + headerDelayDuration: []string{"2"}, + headerDelayPercentage: []string{"30"}, + }, + randIn: []int{100}, + repeat: 3, + }, { + name: "invalid header results in no delay", + md: metadata.MD{ + headerDelayDuration: []string{"error"}, + headerDelayPercentage: []string{"80"}, + }, + repeat: 1, + }, { + name: "invalid header results in default percentage", + md: metadata.MD{ + headerDelayDuration: []string{"2"}, + headerDelayPercentage: []string{"error"}, + }, + randIn: []int{100}, + delays: []time.Duration{2 * time.Millisecond}, + repeat: 1, + }, { + name: "invalid header results in default percentage", + md: metadata.MD{ + headerDelayDuration: []string{"2"}, + headerDelayPercentage: []string{"error"}, + }, + randIn: []int{100}, + repeat: 1, + }, { + name: "cannot exceed percentage in filter", + md: metadata.MD{ + headerDelayDuration: []string{"2"}, + headerDelayPercentage: []string{"100"}, + }, + randIn: []int{100}, + repeat: 1, + }}, + }, { + name: "abort then delay filters", + cfgs: []*fpb.HTTPFault{{ + Abort: &fpb.FaultAbort{ + Percentage: &tpb.FractionalPercent{Numerator: 50, Denominator: tpb.FractionalPercent_HUNDRED}, + ErrorType: &fpb.FaultAbort_GrpcStatus{GrpcStatus: uint32(codes.Unimplemented)}, + }, + }, { + Delay: &cpb.FaultDelay{ + Percentage: &tpb.FractionalPercent{Numerator: 80, Denominator: tpb.FractionalPercent_HUNDRED}, + FaultDelaySecifier: &cpb.FaultDelay_FixedDelay{FixedDelay: ptypes.DurationProto(time.Second)}, + }, + }}, + randOutInc: 10, + want: []subcase{{ + name: "50% delay and abort (abort skips delay)", + code: codes.Unimplemented, + randIn: []int{100}, + repeat: 5, + }, { + name: "30% delay, no abort", + randIn: []int{100, 100}, + repeat: 3, + delays: []time.Duration{time.Second}, + }, { + name: "20% success", + randIn: []int{100, 100}, + repeat: 2, + }}, + }} + + fs, nodeID, port, cleanup := clientSetup(t) + defer cleanup() + + for tcNum, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + defer func() { randIntn = grpcrand.Intn; newTimer = time.NewTimer }() + var intnCalls []int + var newTimerCalls []time.Duration + randOut := 0 + randIntn = func(n int) int { + intnCalls = append(intnCalls, n) + return randOut % n + } + + newTimer = func(d time.Duration) *time.Timer { + newTimerCalls = append(newTimerCalls, d) + return time.NewTimer(0) + } + + serviceName := fmt.Sprintf("myservice%d", tcNum) + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: serviceName, + NodeID: nodeID, + Host: "localhost", + Port: port, + SecLevel: e2e.SecurityLevelNone, + }) + hcm := new(v3httppb.HttpConnectionManager) + err := ptypes.UnmarshalAny(resources.Listeners[0].GetApiListener().GetApiListener(), hcm) + if err != nil { + t.Fatal(err) + } + routerFilter := hcm.HttpFilters[len(hcm.HttpFilters)-1] + + hcm.HttpFilters = nil + for i, cfg := range tc.cfgs { + hcm.HttpFilters = append(hcm.HttpFilters, e2e.HTTPFilter(fmt.Sprintf("fault%d", i), cfg)) + } + hcm.HttpFilters = append(hcm.HttpFilters, routerFilter) + hcmAny := testutils.MarshalAny(hcm) + resources.Listeners[0].ApiListener.ApiListener = hcmAny + resources.Listeners[0].FilterChains[0].Filters[0].ConfigType = &v3listenerpb.Filter_TypedConfig{TypedConfig: hcmAny} + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := fs.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create a ClientConn and run the test case. + cc, err := grpc.Dial("xds:///"+serviceName, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + count := 0 + for _, want := range tc.want { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + if want.repeat == 0 { + t.Fatalf("invalid repeat count") + } + for n := 0; n < want.repeat; n++ { + intnCalls = nil + newTimerCalls = nil + ctx = metadata.NewOutgoingContext(ctx, want.md) + _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)) + t.Logf("%v: RPC %d: err: %v, intnCalls: %v, newTimerCalls: %v", want.name, count, err, intnCalls, newTimerCalls) + if status.Code(err) != want.code || !reflect.DeepEqual(intnCalls, want.randIn) || !reflect.DeepEqual(newTimerCalls, want.delays) { + t.Fatalf("WANTED code: %v, intnCalls: %v, newTimerCalls: %v", want.code, want.randIn, want.delays) + } + randOut += tc.randOutInc + count++ + } + } + }) + } +} + +func (s) TestFaultInjection_MaxActiveFaults(t *testing.T) { + fs, nodeID, port, cleanup := clientSetup(t) + defer cleanup() + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: "myservice", + NodeID: nodeID, + Host: "localhost", + Port: port, + SecLevel: e2e.SecurityLevelNone, + }) + hcm := new(v3httppb.HttpConnectionManager) + err := ptypes.UnmarshalAny(resources.Listeners[0].GetApiListener().GetApiListener(), hcm) + if err != nil { + t.Fatal(err) + } + + defer func() { newTimer = time.NewTimer }() + timers := make(chan *time.Timer, 2) + newTimer = func(d time.Duration) *time.Timer { + t := time.NewTimer(24 * time.Hour) // Will reset to fire. + timers <- t + return t + } + + hcm.HttpFilters = append([]*v3httppb.HttpFilter{ + e2e.HTTPFilter("fault", &fpb.HTTPFault{ + MaxActiveFaults: wrapperspb.UInt32(2), + Delay: &cpb.FaultDelay{ + Percentage: &tpb.FractionalPercent{Numerator: 100, Denominator: tpb.FractionalPercent_HUNDRED}, + FaultDelaySecifier: &cpb.FaultDelay_FixedDelay{FixedDelay: ptypes.DurationProto(time.Second)}, + }, + })}, + hcm.HttpFilters...) + hcmAny := testutils.MarshalAny(hcm) + resources.Listeners[0].ApiListener.ApiListener = hcmAny + resources.Listeners[0].FilterChains[0].Filters[0].ConfigType = &v3listenerpb.Filter_TypedConfig{TypedConfig: hcmAny} + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := fs.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Create a ClientConn + cc, err := grpc.Dial("xds:///myservice", grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("failed to dial local test server: %v", err) + } + defer cc.Close() + + client := testgrpc.NewTestServiceClient(cc) + + streams := make(chan testgrpc.TestService_FullDuplexCallClient, 5) // startStream() is called 5 times + startStream := func() { + str, err := client.FullDuplexCall(ctx) + if err != nil { + t.Error("RPC error:", err) + } + streams <- str + } + endStream := func() { + str := <-streams + str.CloseSend() + if _, err := str.Recv(); err != io.EOF { + t.Error("stream error:", err) + } + } + releaseStream := func() { + timer := <-timers + timer.Reset(0) + } + + // Start three streams; two should delay. + go startStream() + go startStream() + go startStream() + + // End one of the streams. Ensure the others are blocked on creation. + endStream() + + select { + case <-streams: + t.Errorf("unexpected second stream created before delay expires") + case <-time.After(50 * time.Millisecond): + // Wait a short time to ensure no other streams were started yet. + } + + // Start one more; it should not be blocked. + go startStream() + endStream() + + // Expire one stream's delay; it should be created. + releaseStream() + endStream() + + // Another new stream should delay. + go startStream() + select { + case <-streams: + t.Errorf("unexpected second stream created before delay expires") + case <-time.After(50 * time.Millisecond): + // Wait a short time to ensure no other streams were started yet. + } + + // Expire both pending timers and end the two streams. + releaseStream() + releaseStream() + endStream() + endStream() +} diff --git a/xds/internal/httpfilter/httpfilter.go b/xds/internal/httpfilter/httpfilter.go new file mode 100644 index 000000000000..dd9a278389b5 --- /dev/null +++ b/xds/internal/httpfilter/httpfilter.go @@ -0,0 +1,108 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package httpfilter contains the HTTPFilter interface and a registry for +// storing and retrieving their implementations. +package httpfilter + +import ( + "github.com/golang/protobuf/proto" + iresolver "google.golang.org/grpc/internal/resolver" +) + +// FilterConfig represents an opaque data structure holding configuration for a +// filter. Embed this interface to implement it. +type FilterConfig interface { + isFilterConfig() +} + +// Filter defines the parsing functionality of an HTTP filter. A Filter may +// optionally implement either ClientInterceptorBuilder or +// ServerInterceptorBuilder or both, indicating it is capable of working on the +// client side or server side or both, respectively. +type Filter interface { + // TypeURLs are the proto message types supported by this filter. A filter + // will be registered by each of its supported message types. + TypeURLs() []string + // ParseFilterConfig parses the provided configuration proto.Message from + // the LDS configuration of this filter. This may be an anypb.Any, a + // udpa.type.v1.TypedStruct, or an xds.type.v3.TypedStruct for filters that + // do not accept a custom type. The resulting FilterConfig will later be + // passed to Build. + ParseFilterConfig(proto.Message) (FilterConfig, error) + // ParseFilterConfigOverride parses the provided override configuration + // proto.Message from the RDS override configuration of this filter. This + // may be an anypb.Any, a udpa.type.v1.TypedStruct, or an + // xds.type.v3.TypedStruct for filters that do not accept a custom type. + // The resulting FilterConfig will later be passed to Build. + ParseFilterConfigOverride(proto.Message) (FilterConfig, error) + // IsTerminal returns whether this Filter is terminal or not (i.e. it must + // be last filter in the filter chain). + IsTerminal() bool +} + +// ClientInterceptorBuilder constructs a Client Interceptor. If this type is +// implemented by a Filter, it is capable of working on a client. +type ClientInterceptorBuilder interface { + // BuildClientInterceptor uses the FilterConfigs produced above to produce + // an HTTP filter interceptor for clients. config will always be non-nil, + // but override may be nil if no override config exists for the filter. It + // is valid for Build to return a nil Interceptor and a nil error. In this + // case, the RPC will not be intercepted by this filter. + BuildClientInterceptor(config, override FilterConfig) (iresolver.ClientInterceptor, error) +} + +// ServerInterceptorBuilder constructs a Server Interceptor. If this type is +// implemented by a Filter, it is capable of working on a server. +type ServerInterceptorBuilder interface { + // BuildServerInterceptor uses the FilterConfigs produced above to produce + // an HTTP filter interceptor for servers. config will always be non-nil, + // but override may be nil if no override config exists for the filter. It + // is valid for Build to return a nil Interceptor and a nil error. In this + // case, the RPC will not be intercepted by this filter. + BuildServerInterceptor(config, override FilterConfig) (iresolver.ServerInterceptor, error) +} + +var ( + // m is a map from scheme to filter. + m = make(map[string]Filter) +) + +// Register registers the HTTP filter Builder to the filter map. b.TypeURLs() +// will be used as the types for this filter. +// +// NOTE: this function must only be called during initialization time (i.e. in +// an init() function), and is not thread-safe. If multiple filters are +// registered with the same type URL, the one registered last will take effect. +func Register(b Filter) { + for _, u := range b.TypeURLs() { + m[u] = b + } +} + +// UnregisterForTesting unregisters the HTTP Filter for testing purposes. +func UnregisterForTesting(typeURL string) { + delete(m, typeURL) +} + +// Get returns the HTTPFilter registered with typeURL. +// +// If no filter is register with typeURL, nil will be returned. +func Get(typeURL string) Filter { + return m[typeURL] +} diff --git a/xds/internal/httpfilter/rbac/rbac.go b/xds/internal/httpfilter/rbac/rbac.go new file mode 100644 index 000000000000..277fcfc5927a --- /dev/null +++ b/xds/internal/httpfilter/rbac/rbac.go @@ -0,0 +1,218 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package rbac implements the Envoy RBAC HTTP filter. +package rbac + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/golang/protobuf/proto" + "github.com/golang/protobuf/ptypes" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/resolver" + "google.golang.org/grpc/internal/xds/rbac" + "google.golang.org/grpc/xds/internal/httpfilter" + "google.golang.org/protobuf/types/known/anypb" + + v3rbacpb "github.com/envoyproxy/go-control-plane/envoy/config/rbac/v3" + rpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/rbac/v3" +) + +func init() { + if envconfig.XDSRBAC { + httpfilter.Register(builder{}) + } + + // TODO: Remove these once the RBAC env var is removed. + internal.RegisterRBACHTTPFilterForTesting = func() { + httpfilter.Register(builder{}) + } + internal.UnregisterRBACHTTPFilterForTesting = func() { + for _, typeURL := range builder.TypeURLs(builder{}) { + httpfilter.UnregisterForTesting(typeURL) + } + } +} + +type builder struct { +} + +type config struct { + httpfilter.FilterConfig + chainEngine *rbac.ChainEngine +} + +func (builder) TypeURLs() []string { + return []string{ + "type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBAC", + "type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute", + } +} + +// Parsing is the same for the base config and the override config. +func parseConfig(rbacCfg *rpb.RBAC) (httpfilter.FilterConfig, error) { + // All the validation logic described in A41. + for _, policy := range rbacCfg.GetRules().GetPolicies() { + // "Policy.condition and Policy.checked_condition must cause a + // validation failure if present." - A41 + if policy.Condition != nil { + return nil, errors.New("rbac: Policy.condition is present") + } + if policy.CheckedCondition != nil { + return nil, errors.New("rbac: policy.CheckedCondition is present") + } + + // "It is also a validation failure if Permission or Principal has a + // header matcher for a grpc- prefixed header name or :scheme." - A41 + for _, principal := range policy.Principals { + name := principal.GetHeader().GetName() + if name == ":scheme" || strings.HasPrefix(name, "grpc-") { + return nil, fmt.Errorf("rbac: principal header matcher for %v is :scheme or starts with grpc", name) + } + } + for _, permission := range policy.Permissions { + name := permission.GetHeader().GetName() + if name == ":scheme" || strings.HasPrefix(name, "grpc-") { + return nil, fmt.Errorf("rbac: permission header matcher for %v is :scheme or starts with grpc", name) + } + } + } + + // "Envoy aliases :authority and Host in its header map implementation, so + // they should be treated equivalent for the RBAC matchers; there must be no + // behavior change depending on which of the two header names is used in the + // RBAC policy." - A41. Loop through config's principals and policies, change + // any header matcher with value "host" to :authority", as that is what + // grpc-go shifts both headers to in transport layer. + for _, policy := range rbacCfg.GetRules().GetPolicies() { + for _, principal := range policy.Principals { + if principal.GetHeader().GetName() == "host" { + principal.GetHeader().Name = ":authority" + } + } + for _, permission := range policy.Permissions { + if permission.GetHeader().GetName() == "host" { + permission.GetHeader().Name = ":authority" + } + } + } + + // Two cases where this HTTP Filter is a no op: + // "If absent, no enforcing RBAC policy will be applied" - RBAC + // Documentation for Rules field. + // "At this time, if the RBAC.action is Action.LOG then the policy will be + // completely ignored, as if RBAC was not configurated." - A41 + if rbacCfg.Rules == nil || rbacCfg.GetRules().GetAction() == v3rbacpb.RBAC_LOG { + return config{}, nil + } + + // TODO(gregorycooke) - change the call chain to here so we have the filter + // name to input here instead of an empty string. It will come from here: + // https://github.com/grpc/grpc-go/blob/eff0942e95d93112921414aee758e619ec86f26f/xds/internal/xdsclient/xdsresource/unmarshal_lds.go#L199 + ce, err := rbac.NewChainEngine([]*v3rbacpb.RBAC{rbacCfg.GetRules()}, "") + if err != nil { + // "At this time, if the RBAC.action is Action.LOG then the policy will be + // completely ignored, as if RBAC was not configurated." - A41 + if rbacCfg.GetRules().GetAction() != v3rbacpb.RBAC_LOG { + return nil, fmt.Errorf("rbac: error constructing matching engine: %v", err) + } + } + + return config{chainEngine: ce}, nil +} + +func (builder) ParseFilterConfig(cfg proto.Message) (httpfilter.FilterConfig, error) { + if cfg == nil { + return nil, fmt.Errorf("rbac: nil configuration message provided") + } + any, ok := cfg.(*anypb.Any) + if !ok { + return nil, fmt.Errorf("rbac: error parsing config %v: unknown type %T", cfg, cfg) + } + msg := new(rpb.RBAC) + if err := ptypes.UnmarshalAny(any, msg); err != nil { + return nil, fmt.Errorf("rbac: error parsing config %v: %v", cfg, err) + } + return parseConfig(msg) +} + +func (builder) ParseFilterConfigOverride(override proto.Message) (httpfilter.FilterConfig, error) { + if override == nil { + return nil, fmt.Errorf("rbac: nil configuration message provided") + } + any, ok := override.(*anypb.Any) + if !ok { + return nil, fmt.Errorf("rbac: error parsing override config %v: unknown type %T", override, override) + } + msg := new(rpb.RBACPerRoute) + if err := ptypes.UnmarshalAny(any, msg); err != nil { + return nil, fmt.Errorf("rbac: error parsing override config %v: %v", override, err) + } + return parseConfig(msg.Rbac) +} + +func (builder) IsTerminal() bool { + return false +} + +var _ httpfilter.ServerInterceptorBuilder = builder{} + +// BuildServerInterceptor is an optional interface builder implements in order +// to signify it works server side. +func (builder) BuildServerInterceptor(cfg httpfilter.FilterConfig, override httpfilter.FilterConfig) (resolver.ServerInterceptor, error) { + if cfg == nil { + return nil, fmt.Errorf("rbac: nil config provided") + } + + c, ok := cfg.(config) + if !ok { + return nil, fmt.Errorf("rbac: incorrect config type provided (%T): %v", cfg, cfg) + } + + if override != nil { + // override completely replaces the listener configuration; but we + // still validate the listener config type. + c, ok = override.(config) + if !ok { + return nil, fmt.Errorf("rbac: incorrect override config type provided (%T): %v", override, override) + } + } + + // RBAC HTTP Filter is a no op from one of these two cases: + // "If absent, no enforcing RBAC policy will be applied" - RBAC + // Documentation for Rules field. + // "At this time, if the RBAC.action is Action.LOG then the policy will be + // completely ignored, as if RBAC was not configurated." - A41 + if c.chainEngine == nil { + return nil, nil + } + return &interceptor{chainEngine: c.chainEngine}, nil +} + +type interceptor struct { + chainEngine *rbac.ChainEngine +} + +func (i *interceptor) AllowRPC(ctx context.Context) error { + return i.chainEngine.IsAuthorized(ctx) +} diff --git a/xds/internal/httpfilter/router/router.go b/xds/internal/httpfilter/router/router.go new file mode 100644 index 000000000000..1ac6518170fc --- /dev/null +++ b/xds/internal/httpfilter/router/router.go @@ -0,0 +1,114 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package router implements the Envoy Router HTTP filter. +package router + +import ( + "fmt" + + "github.com/golang/protobuf/proto" + "github.com/golang/protobuf/ptypes" + iresolver "google.golang.org/grpc/internal/resolver" + "google.golang.org/grpc/xds/internal/httpfilter" + "google.golang.org/protobuf/types/known/anypb" + + pb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3" +) + +// TypeURL is the message type for the Router configuration. +const TypeURL = "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router" + +func init() { + httpfilter.Register(builder{}) +} + +// IsRouterFilter returns true iff a HTTP filter is a Router filter. +func IsRouterFilter(b httpfilter.Filter) bool { + _, ok := b.(builder) + return ok +} + +type builder struct { +} + +func (builder) TypeURLs() []string { return []string{TypeURL} } + +func (builder) ParseFilterConfig(cfg proto.Message) (httpfilter.FilterConfig, error) { + // The gRPC router filter does not currently use any fields from the + // config. Verify type only. + if cfg == nil { + return nil, fmt.Errorf("router: nil configuration message provided") + } + any, ok := cfg.(*anypb.Any) + if !ok { + return nil, fmt.Errorf("router: error parsing config %v: unknown type %T", cfg, cfg) + } + msg := new(pb.Router) + if err := ptypes.UnmarshalAny(any, msg); err != nil { + return nil, fmt.Errorf("router: error parsing config %v: %v", cfg, err) + } + return config{}, nil +} + +func (builder) ParseFilterConfigOverride(override proto.Message) (httpfilter.FilterConfig, error) { + if override != nil { + return nil, fmt.Errorf("router: unexpected config override specified: %v", override) + } + return config{}, nil +} + +func (builder) IsTerminal() bool { + return true +} + +var ( + _ httpfilter.ClientInterceptorBuilder = builder{} + _ httpfilter.ServerInterceptorBuilder = builder{} +) + +func (builder) BuildClientInterceptor(cfg, override httpfilter.FilterConfig) (iresolver.ClientInterceptor, error) { + if _, ok := cfg.(config); !ok { + return nil, fmt.Errorf("router: incorrect config type provided (%T): %v", cfg, cfg) + } + if override != nil { + return nil, fmt.Errorf("router: unexpected override configuration specified: %v", override) + } + // The gRPC router is implemented within the xds resolver's config + // selector, not as a separate plugin. So we return a nil HTTPFilter, + // which will not be invoked. + return nil, nil +} + +func (builder) BuildServerInterceptor(cfg, override httpfilter.FilterConfig) (iresolver.ServerInterceptor, error) { + if _, ok := cfg.(config); !ok { + return nil, fmt.Errorf("router: incorrect config type provided (%T): %v", cfg, cfg) + } + if override != nil { + return nil, fmt.Errorf("router: unexpected override configuration specified: %v", override) + } + // The gRPC router is currently unimplemented on the server side. So we + // return a nil HTTPFilter, which will not be invoked. + return nil, nil +} + +// The gRPC router filter does not currently support any configuration. Verify +// type only. +type config struct { + httpfilter.FilterConfig +} diff --git a/xds/internal/internal.go b/xds/internal/internal.go index 462a8bac59b2..fda4c7f56106 100644 --- a/xds/internal/internal.go +++ b/xds/internal/internal.go @@ -20,15 +20,11 @@ package internal import ( + "encoding/json" "fmt" -) - -type clientID string -// XDSClientID is the attributes key used to pass the address of the xdsClient -// object shared between the resolver and the balancer. The xdsClient object is -// created by the resolver and passed to the balancer. -const XDSClientID = clientID("xdsClientID") + "google.golang.org/grpc/resolver" +) // LocalityID is xds.Locality without XXX fields, so it can be used as map // keys. @@ -40,6 +36,50 @@ type LocalityID struct { SubZone string `json:"subZone,omitempty"` } -func (l LocalityID) String() string { - return fmt.Sprintf("%s-%s-%s", l.Region, l.Zone, l.SubZone) +// ToString generates a string representation of LocalityID by marshalling it into +// json. Not calling it String() so printf won't call it. +func (l LocalityID) ToString() (string, error) { + b, err := json.Marshal(l) + if err != nil { + return "", err + } + return string(b), nil +} + +// Equal allows the values to be compared by Attributes.Equal. +func (l LocalityID) Equal(o any) bool { + ol, ok := o.(LocalityID) + if !ok { + return false + } + return l.Region == ol.Region && l.Zone == ol.Zone && l.SubZone == ol.SubZone +} + +// LocalityIDFromString converts a json representation of locality, into a +// LocalityID struct. +func LocalityIDFromString(s string) (ret LocalityID, _ error) { + err := json.Unmarshal([]byte(s), &ret) + if err != nil { + return LocalityID{}, fmt.Errorf("%s is not a well formatted locality ID, error: %v", s, err) + } + return ret, nil } + +type localityKeyType string + +const localityKey = localityKeyType("grpc.xds.internal.address.locality") + +// GetLocalityID returns the locality ID of addr. +func GetLocalityID(addr resolver.Address) LocalityID { + path, _ := addr.BalancerAttributes.Value(localityKey).(LocalityID) + return path +} + +// SetLocalityID sets locality ID in addr to l. +func SetLocalityID(addr resolver.Address, l LocalityID) resolver.Address { + addr.BalancerAttributes = addr.BalancerAttributes.WithValue(localityKey, l) + return addr +} + +// ResourceTypeMapForTesting maps TypeUrl to corresponding ResourceType. +var ResourceTypeMapForTesting map[string]any diff --git a/xds/internal/internal_test.go b/xds/internal/internal_test.go index bdd6e85cba97..903b9db23c48 100644 --- a/xds/internal/internal_test.go +++ b/xds/internal/internal_test.go @@ -70,3 +70,48 @@ func (s) TestLocalityMatchProtoMessage(t *testing.T) { t.Fatalf("internal type and proto message have different fields: (-got +want):\n%+v", diff) } } + +func TestLocalityToAndFromJSON(t *testing.T) { + tests := []struct { + name string + localityID LocalityID + str string + wantErr bool + }{ + { + name: "3 fields", + localityID: LocalityID{Region: "r:r", Zone: "z#z", SubZone: "s^s"}, + str: `{"region":"r:r","zone":"z#z","subZone":"s^s"}`, + }, + { + name: "2 fields", + localityID: LocalityID{Region: "r:r", Zone: "z#z"}, + str: `{"region":"r:r","zone":"z#z"}`, + }, + { + name: "1 field", + localityID: LocalityID{Region: "r:r"}, + str: `{"region":"r:r"}`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotStr, err := tt.localityID.ToString() + if err != nil { + t.Errorf("failed to marshal LocalityID: %#v", tt.localityID) + } + if gotStr != tt.str { + t.Errorf("%#v.String() = %q, want %q", tt.localityID, gotStr, tt.str) + } + + gotID, err := LocalityIDFromString(tt.str) + if (err != nil) != tt.wantErr { + t.Errorf("LocalityIDFromString(%q) error = %v, wantErr %v", tt.str, err, tt.wantErr) + return + } + if diff := cmp.Diff(gotID, tt.localityID); diff != "" { + t.Errorf("LocalityIDFromString() got = %v, want %v, diff: %s", gotID, tt.localityID, diff) + } + }) + } +} diff --git a/xds/internal/resolver/cluster_specifier_plugin_test.go b/xds/internal/resolver/cluster_specifier_plugin_test.go new file mode 100644 index 000000000000..38ab9acf5b9f --- /dev/null +++ b/xds/internal/resolver/cluster_specifier_plugin_test.go @@ -0,0 +1,483 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package resolver + +import ( + "context" + "encoding/json" + "fmt" + "testing" + + "github.com/golang/protobuf/proto" + "github.com/golang/protobuf/ptypes" + "github.com/google/go-cmp/cmp" + "github.com/google/uuid" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/envconfig" + iresolver "google.golang.org/grpc/internal/resolver" + "google.golang.org/grpc/internal/testutils" + xdsbootstrap "google.golang.org/grpc/internal/testutils/xds/bootstrap" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" + "google.golang.org/grpc/xds/internal/balancer/clustermanager" + "google.golang.org/grpc/xds/internal/clusterspecifier" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/wrapperspb" + + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" +) + +func init() { + balancer.Register(cspBalancerBuilder{}) + clusterspecifier.Register(testClusterSpecifierPlugin{}) +} + +// cspBalancerBuilder is a no-op LB policy which is referenced by the +// testClusterSpecifierPlugin. +type cspBalancerBuilder struct{} + +func (cspBalancerBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { + return nil +} + +func (cspBalancerBuilder) Name() string { + return "csp_experimental" +} + +type cspBalancerConfig struct { + serviceconfig.LoadBalancingConfig + ArbitraryField string `json:"arbitrary_field"` +} + +func (cspBalancerBuilder) ParseConfig(lbCfg json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + cfg := &cspBalancerConfig{} + if err := json.Unmarshal(lbCfg, cfg); err != nil { + return nil, err + } + return cfg, nil + +} + +// testClusterSpecifierPlugin is a test cluster specifier plugin which returns +// an LB policy configuration specifying the cspBalancer. +type testClusterSpecifierPlugin struct { +} + +func (testClusterSpecifierPlugin) TypeURLs() []string { + // The config for this plugin contains a wrapperspb.StringValue, and since + // we marshal that proto as an Any proto, the type URL on the latter gets + // set to "type.googleapis.com/google.protobuf.StringValue". If we wanted a + // more descriptive type URL for this test plugin, we would have to define a + // proto package with a message for the configuration. That would be + // overkill for a test. Therefore, this seems to be an acceptable tradeoff. + return []string{"type.googleapis.com/google.protobuf.StringValue"} +} + +func (testClusterSpecifierPlugin) ParseClusterSpecifierConfig(cfg proto.Message) (clusterspecifier.BalancerConfig, error) { + if cfg == nil { + return nil, fmt.Errorf("testClusterSpecifierPlugin: nil configuration message provided") + } + anyp, ok := cfg.(*anypb.Any) + if !ok { + return nil, fmt.Errorf("testClusterSpecifierPlugin: error parsing config %v: got type %T, want *anypb.Any", cfg, cfg) + } + lbCfg := new(wrapperspb.StringValue) + if err := ptypes.UnmarshalAny(anyp, lbCfg); err != nil { + return nil, fmt.Errorf("testClusterSpecifierPlugin: error parsing config %v: %v", cfg, err) + } + return []map[string]any{{"csp_experimental": cspBalancerConfig{ArbitraryField: lbCfg.GetValue()}}}, nil +} + +// TestResolverClusterSpecifierPlugin tests the case where a route configuration +// containing cluster specifier plugins is sent by the management server. The +// test verifies that the service config output by the resolver contains the LB +// policy specified by the cluster specifier plugin, and the config selector +// returns the cluster associated with the cluster specifier plugin. +// +// The test also verifies that a change in the cluster specifier plugin config +// result in appropriate change in the service config pushed by the resolver. +func (s) TestResolverClusterSpecifierPlugin(t *testing.T) { + // Env var GRPC_EXPERIMENTAL_XDS_RLS_LB controls whether the xDS client + // allows routes with cluster specifier plugin as their route action. + oldRLS := envconfig.XDSRLS + envconfig.XDSRLS = true + defer func() { + envconfig.XDSRLS = oldRLS + }() + + mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) + if err != nil { + t.Fatalf("Failed to start xDS management server: %v", err) + } + defer mgmtServer.Stop() + + // Create a bootstrap configuration specifying the above management server. + nodeID := uuid.New().String() + cleanup, err := xdsbootstrap.CreateFile(xdsbootstrap.Options{ + NodeID: nodeID, + ServerURI: mgmtServer.Address, + }) + if err != nil { + t.Fatal(err) + } + defer cleanup() + + // Configure listener and route configuration resources on the management + // server. + const serviceName = "my-service-client-side-xds" + rdsName := "route-" + serviceName + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, rdsName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.RouteConfigResourceWithOptions(e2e.RouteConfigOptions{ + RouteConfigName: rdsName, + ListenerName: serviceName, + ClusterSpecifierType: e2e.RouteConfigClusterSpecifierTypeClusterSpecifierPlugin, + ClusterSpecifierPluginName: "cspA", + ClusterSpecifierPluginConfig: testutils.MarshalAny(&wrapperspb.StringValue{Value: "anything"}), + })}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + tcc, rClose := buildResolverForTarget(t, resolver.Target{URL: *testutils.MustParseURL("xds:///" + serviceName)}) + defer rClose() + + // Wait for an update from the resolver, and verify the service config. + val, err := tcc.stateCh.Receive(ctx) + if err != nil { + t.Fatalf("Timeout waiting for an update from the resolver: %v", err) + } + rState := val.(resolver.State) + if err := rState.ServiceConfig.Err; err != nil { + t.Fatalf("Received error in service config: %v", rState.ServiceConfig.Err) + } + wantSCParsed := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(` +{ + "loadBalancingConfig": [ + { + "xds_cluster_manager_experimental": { + "children": { + "cluster_specifier_plugin:cspA": { + "childPolicy": [ + { + "csp_experimental": { + "arbitrary_field": "anything" + } + } + ] + } + } + } + } + ] +}`) + if !internal.EqualServiceConfigForTesting(rState.ServiceConfig.Config, wantSCParsed.Config) { + t.Fatalf("Got service config:\n%s \nWant service config:\n%s", cmp.Diff(nil, rState.ServiceConfig.Config), cmp.Diff(nil, wantSCParsed.Config)) + } + + cs := iresolver.GetConfigSelector(rState) + if cs == nil { + t.Fatal("Received nil config selector in update from resolver") + } + res, err := cs.SelectConfig(iresolver.RPCInfo{Context: ctx, Method: "/service/method"}) + if err != nil { + t.Fatalf("cs.SelectConfig(): %v", err) + } + + gotCluster := clustermanager.GetPickedClusterForTesting(res.Context) + wantCluster := "cluster_specifier_plugin:cspA" + if gotCluster != wantCluster { + t.Fatalf("config selector returned cluster: %v, want: %v", gotCluster, wantCluster) + } + + // Change the cluster specifier plugin configuration. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, rdsName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.RouteConfigResourceWithOptions(e2e.RouteConfigOptions{ + RouteConfigName: rdsName, + ListenerName: serviceName, + ClusterSpecifierType: e2e.RouteConfigClusterSpecifierTypeClusterSpecifierPlugin, + ClusterSpecifierPluginName: "cspA", + ClusterSpecifierPluginConfig: testutils.MarshalAny(&wrapperspb.StringValue{Value: "changed"}), + })}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Wait for an update from the resolver, and verify the service config. + val, err = tcc.stateCh.Receive(ctx) + if err != nil { + t.Fatalf("Timeout waiting for an update from the resolver: %v", err) + } + rState = val.(resolver.State) + if err := rState.ServiceConfig.Err; err != nil { + t.Fatalf("Received error in service config: %v", rState.ServiceConfig.Err) + } + wantSCParsed = internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(` +{ + "loadBalancingConfig": [ + { + "xds_cluster_manager_experimental": { + "children": { + "cluster_specifier_plugin:cspA": { + "childPolicy": [ + { + "csp_experimental": { + "arbitrary_field": "changed" + } + } + ] + } + } + } + } + ] +}`) + if !internal.EqualServiceConfigForTesting(rState.ServiceConfig.Config, wantSCParsed.Config) { + t.Fatalf("Got service config:\n%s \nWant service config:\n%s", cmp.Diff(nil, rState.ServiceConfig.Config), cmp.Diff(nil, wantSCParsed.Config)) + } +} + +// TestXDSResolverDelayedOnCommittedCSP tests that cluster specifier plugins and +// their corresponding configurations remain in service config if RPCs are in +// flight. +func (s) TestXDSResolverDelayedOnCommittedCSP(t *testing.T) { + // Env var GRPC_EXPERIMENTAL_XDS_RLS_LB controls whether the xDS client + // allows routes with cluster specifier plugin as their route action. + oldRLS := envconfig.XDSRLS + envconfig.XDSRLS = true + defer func() { + envconfig.XDSRLS = oldRLS + }() + + mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) + if err != nil { + t.Fatalf("Failed to start xDS management server: %v", err) + } + defer mgmtServer.Stop() + + // Create a bootstrap configuration specifying the above management server. + nodeID := uuid.New().String() + cleanup, err := xdsbootstrap.CreateFile(xdsbootstrap.Options{ + NodeID: nodeID, + ServerURI: mgmtServer.Address, + }) + if err != nil { + t.Fatal(err) + } + defer cleanup() + + // Configure listener and route configuration resources on the management + // server. + const serviceName = "my-service-client-side-xds" + rdsName := "route-" + serviceName + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, rdsName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.RouteConfigResourceWithOptions(e2e.RouteConfigOptions{ + RouteConfigName: rdsName, + ListenerName: serviceName, + ClusterSpecifierType: e2e.RouteConfigClusterSpecifierTypeClusterSpecifierPlugin, + ClusterSpecifierPluginName: "cspA", + ClusterSpecifierPluginConfig: testutils.MarshalAny(&wrapperspb.StringValue{Value: "anythingA"}), + })}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + tcc, rClose := buildResolverForTarget(t, resolver.Target{URL: *testutils.MustParseURL("xds:///" + serviceName)}) + defer rClose() + + // Wait for an update from the resolver, and verify the service config. + val, err := tcc.stateCh.Receive(ctx) + if err != nil { + t.Fatalf("Timeout waiting for an update from the resolver: %v", err) + } + rState := val.(resolver.State) + if err := rState.ServiceConfig.Err; err != nil { + t.Fatalf("Received error in service config: %v", rState.ServiceConfig.Err) + } + wantSCParsed := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(` +{ + "loadBalancingConfig": [ + { + "xds_cluster_manager_experimental": { + "children": { + "cluster_specifier_plugin:cspA": { + "childPolicy": [ + { + "csp_experimental": { + "arbitrary_field": "anythingA" + } + } + ] + } + } + } + } + ] +}`) + if !internal.EqualServiceConfigForTesting(rState.ServiceConfig.Config, wantSCParsed.Config) { + t.Fatalf("Got service config:\n%s \nWant service config:\n%s", cmp.Diff(nil, rState.ServiceConfig.Config), cmp.Diff(nil, wantSCParsed.Config)) + } + + cs := iresolver.GetConfigSelector(rState) + if cs == nil { + t.Fatal("Received nil config selector in update from resolver") + } + resOld, err := cs.SelectConfig(iresolver.RPCInfo{Context: ctx, Method: "/service/method"}) + if err != nil { + t.Fatalf("cs.SelectConfig(): %v", err) + } + + gotCluster := clustermanager.GetPickedClusterForTesting(resOld.Context) + wantCluster := "cluster_specifier_plugin:cspA" + if gotCluster != wantCluster { + t.Fatalf("config selector returned cluster: %v, want: %v", gotCluster, wantCluster) + } + + // Delay resOld.OnCommitted(). As long as there are pending RPCs to removed + // clusters, they still appear in the service config. + + // Change the cluster specifier plugin configuration. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, rdsName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.RouteConfigResourceWithOptions(e2e.RouteConfigOptions{ + RouteConfigName: rdsName, + ListenerName: serviceName, + ClusterSpecifierType: e2e.RouteConfigClusterSpecifierTypeClusterSpecifierPlugin, + ClusterSpecifierPluginName: "cspB", + ClusterSpecifierPluginConfig: testutils.MarshalAny(&wrapperspb.StringValue{Value: "anythingB"}), + })}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Wait for an update from the resolver, and verify the service config. + val, err = tcc.stateCh.Receive(ctx) + if err != nil { + t.Fatalf("Timeout waiting for an update from the resolver: %v", err) + } + rState = val.(resolver.State) + if err := rState.ServiceConfig.Err; err != nil { + t.Fatalf("Received error in service config: %v", rState.ServiceConfig.Err) + } + wantSCParsed = internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(` +{ + "loadBalancingConfig": [ + { + "xds_cluster_manager_experimental": { + "children": { + "cluster_specifier_plugin:cspA": { + "childPolicy": [ + { + "csp_experimental": { + "arbitrary_field": "anythingA" + } + } + ] + }, + "cluster_specifier_plugin:cspB": { + "childPolicy": [ + { + "csp_experimental": { + "arbitrary_field": "anythingB" + } + } + ] + } + } + } + } + ] +}`) + if !internal.EqualServiceConfigForTesting(rState.ServiceConfig.Config, wantSCParsed.Config) { + t.Fatalf("Got service config:\n%s \nWant service config:\n%s", cmp.Diff(nil, rState.ServiceConfig.Config), cmp.Diff(nil, wantSCParsed.Config)) + } + + // Perform an RPC and ensure that it is routed to the new cluster. + cs = iresolver.GetConfigSelector(rState) + if cs == nil { + t.Fatal("Received nil config selector in update from resolver") + } + resNew, err := cs.SelectConfig(iresolver.RPCInfo{Context: ctx, Method: "/service/method"}) + if err != nil { + t.Fatalf("cs.SelectConfig(): %v", err) + } + + gotCluster = clustermanager.GetPickedClusterForTesting(resNew.Context) + wantCluster = "cluster_specifier_plugin:cspB" + if gotCluster != wantCluster { + t.Fatalf("config selector returned cluster: %v, want: %v", gotCluster, wantCluster) + } + + // Invoke resOld.OnCommitted; should lead to a service config update that deletes + // cspA. + resOld.OnCommitted() + + val, err = tcc.stateCh.Receive(ctx) + if err != nil { + t.Fatalf("Timeout waiting for an update from the resolver: %v", err) + } + rState = val.(resolver.State) + if err := rState.ServiceConfig.Err; err != nil { + t.Fatalf("Received error in service config: %v", rState.ServiceConfig.Err) + } + wantSCParsed = internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(` +{ + "loadBalancingConfig": [ + { + "xds_cluster_manager_experimental": { + "children": { + "cluster_specifier_plugin:cspB": { + "childPolicy": [ + { + "csp_experimental": { + "arbitrary_field": "anythingB" + } + } + ] + } + } + } + } + ] +}`) + if !internal.EqualServiceConfigForTesting(rState.ServiceConfig.Config, wantSCParsed.Config) { + t.Fatalf("Got service config:\n%s \nWant service config:\n%s", cmp.Diff(nil, rState.ServiceConfig.Config), cmp.Diff(nil, wantSCParsed.Config)) + } +} diff --git a/xds/internal/resolver/serviceconfig.go b/xds/internal/resolver/serviceconfig.go index 805d8d41104c..02470ddca5e4 100644 --- a/xds/internal/resolver/serviceconfig.go +++ b/xds/internal/resolver/serviceconfig.go @@ -19,124 +19,423 @@ package resolver import ( + "context" "encoding/json" "fmt" + "math/bits" + "strings" + "sync/atomic" + "time" - wrapperspb "github.com/golang/protobuf/ptypes/wrappers" - xdsclient "google.golang.org/grpc/xds/internal/client" + xxhash "github.com/cespare/xxhash/v2" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/grpcrand" + iresolver "google.golang.org/grpc/internal/resolver" + "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/internal/wrr" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/grpc/xds/internal/balancer/clustermanager" + "google.golang.org/grpc/xds/internal/balancer/ringhash" + "google.golang.org/grpc/xds/internal/httpfilter" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" ) const ( - cdsName = "cds_experimental" - weightedTargetName = "weighted_target_experimental" - xdsRoutingName = "xds_routing_experimental" + cdsName = "cds_experimental" + xdsClusterManagerName = "xds_cluster_manager_experimental" + clusterPrefix = "cluster:" + clusterSpecifierPluginPrefix = "cluster_specifier_plugin:" ) type serviceConfig struct { LoadBalancingConfig balancerConfig `json:"loadBalancingConfig"` } -type balancerConfig []map[string]interface{} +type balancerConfig []map[string]any -func newBalancerConfig(name string, config interface{}) balancerConfig { - return []map[string]interface{}{{name: config}} +func newBalancerConfig(name string, config any) balancerConfig { + return []map[string]any{{name: config}} } -type weightedCDSBalancerConfig struct { - Targets map[string]cdsWithWeight `json:"targets"` +type cdsBalancerConfig struct { + Cluster string `json:"cluster"` } -type cdsWithWeight struct { - Weight uint32 `json:"weight"` +type xdsChildConfig struct { ChildPolicy balancerConfig `json:"childPolicy"` } -type cdsBalancerConfig struct { - Cluster string `json:"cluster"` +type xdsClusterManagerConfig struct { + Children map[string]xdsChildConfig `json:"children"` +} + +// pruneActiveClusters deletes entries in r.activeClusters with zero +// references. +func (r *xdsResolver) pruneActiveClusters() { + for cluster, ci := range r.activeClusters { + if atomic.LoadInt32(&ci.refCount) == 0 { + delete(r.activeClusters, cluster) + } + } +} + +// serviceConfigJSON produces a service config in JSON format representing all +// the clusters referenced in activeClusters. This includes clusters with zero +// references, so they must be pruned first. +func serviceConfigJSON(activeClusters map[string]*clusterInfo) ([]byte, error) { + // Generate children (all entries in activeClusters). + children := make(map[string]xdsChildConfig) + for cluster, ci := range activeClusters { + children[cluster] = ci.cfg + } + + sc := serviceConfig{ + LoadBalancingConfig: newBalancerConfig( + xdsClusterManagerName, xdsClusterManagerConfig{Children: children}, + ), + } + + bs, err := json.Marshal(sc) + if err != nil { + return nil, fmt.Errorf("failed to marshal json: %v", err) + } + return bs, nil +} + +type virtualHost struct { + // map from filter name to its config + httpFilterConfigOverride map[string]httpfilter.FilterConfig + // retry policy present in virtual host + retryConfig *xdsresource.RetryConfig +} + +// routeCluster holds information about a cluster as referenced by a route. +type routeCluster struct { + name string + // map from filter name to its config + httpFilterConfigOverride map[string]httpfilter.FilterConfig } type route struct { - Path *string `json:"path,omitempty"` - Prefix *string `json:"prefix,omitempty"` - Regex *string `json:"regex,omitempty"` - Headers []*xdsclient.HeaderMatcher `json:"headers,omitempty"` - Fraction *wrapperspb.UInt32Value `json:"matchFraction,omitempty"` - Action string `json:"action"` + m *xdsresource.CompositeMatcher // converted from route matchers + actionType xdsresource.RouteActionType // holds route action type + clusters wrr.WRR // holds *routeCluster entries + maxStreamDuration time.Duration + // map from filter name to its config + httpFilterConfigOverride map[string]httpfilter.FilterConfig + retryConfig *xdsresource.RetryConfig + hashPolicies []*xdsresource.HashPolicy } -type xdsActionConfig struct { - ChildPolicy balancerConfig `json:"childPolicy"` +func (r route) String() string { + return fmt.Sprintf("%s -> { clusters: %v, maxStreamDuration: %v }", r.m.String(), r.clusters, r.maxStreamDuration) } -type xdsRoutingBalancerConfig struct { - Action map[string]xdsActionConfig `json:"action"` - Route []*route `json:"route"` +type configSelector struct { + r *xdsResolver + virtualHost virtualHost + routes []route + clusters map[string]*clusterInfo + httpFilterConfig []xdsresource.HTTPFilter } -func (r *xdsResolver) routesToJSON(routes []*xdsclient.Route) (string, error) { - r.updateActions(newActionsFromRoutes(routes)) +var errNoMatchedRouteFound = status.Errorf(codes.Unavailable, "no matched route was found") +var errUnsupportedClientRouteAction = status.Errorf(codes.Unavailable, "matched route does not have a supported route action type") - // Generate routes. - var rts []*route - for _, rt := range routes { - t := &route{ - Path: rt.Path, - Prefix: rt.Prefix, - Regex: rt.Regex, - Headers: rt.Headers, +func (cs *configSelector) SelectConfig(rpcInfo iresolver.RPCInfo) (*iresolver.RPCConfig, error) { + if cs == nil { + return nil, status.Errorf(codes.Unavailable, "no valid clusters") + } + var rt *route + // Loop through routes in order and select first match. + for _, r := range cs.routes { + if r.m.Match(rpcInfo) { + rt = &r + break } + } - if f := rt.Fraction; f != nil { - t.Fraction = &wrapperspb.UInt32Value{Value: *f} - } + if rt == nil || rt.clusters == nil { + return nil, errNoMatchedRouteFound + } + + if rt.actionType != xdsresource.RouteActionRoute { + return nil, errUnsupportedClientRouteAction + } + + cluster, ok := rt.clusters.Next().(*routeCluster) + if !ok { + return nil, status.Errorf(codes.Internal, "error retrieving cluster for match: %v (%T)", cluster, cluster) + } + + // Add a ref to the selected cluster, as this RPC needs this cluster until + // it is committed. + ref := &cs.clusters[cluster.name].refCount + atomic.AddInt32(ref, 1) + + interceptor, err := cs.newInterceptor(rt, cluster) + if err != nil { + return nil, err + } + + lbCtx := clustermanager.SetPickedCluster(rpcInfo.Context, cluster.name) + // Request Hashes are only applicable for a Ring Hash LB. + if envconfig.XDSRingHash { + lbCtx = ringhash.SetRequestHash(lbCtx, cs.generateHash(rpcInfo, rt.hashPolicies)) + } + + config := &iresolver.RPCConfig{ + // Communicate to the LB policy the chosen cluster and request hash, if Ring Hash LB policy. + Context: lbCtx, + OnCommitted: func() { + // When the RPC is committed, the cluster is no longer required. + // Decrease its ref. + if v := atomic.AddInt32(ref, -1); v == 0 { + // This entry will be removed from activeClusters when + // producing the service config for the empty update. + select { + case cs.r.updateCh <- suWithError{emptyUpdate: true}: + default: + } + } + }, + Interceptor: interceptor, + } + + if rt.maxStreamDuration != 0 { + config.MethodConfig.Timeout = &rt.maxStreamDuration + } + if rt.retryConfig != nil { + config.MethodConfig.RetryPolicy = retryConfigToPolicy(rt.retryConfig) + } else if cs.virtualHost.retryConfig != nil { + config.MethodConfig.RetryPolicy = retryConfigToPolicy(cs.virtualHost.retryConfig) + } - t.Action = r.getActionAssignedName(rt.Action) - rts = append(rts, t) + return config, nil +} + +func retryConfigToPolicy(config *xdsresource.RetryConfig) *serviceconfig.RetryPolicy { + return &serviceconfig.RetryPolicy{ + MaxAttempts: int(config.NumRetries) + 1, + InitialBackoff: config.RetryBackoff.BaseInterval, + MaxBackoff: config.RetryBackoff.MaxInterval, + BackoffMultiplier: 2, + RetryableStatusCodes: config.RetryOn, } +} + +func (cs *configSelector) generateHash(rpcInfo iresolver.RPCInfo, hashPolicies []*xdsresource.HashPolicy) uint64 { + var hash uint64 + var generatedHash bool + for _, policy := range hashPolicies { + var policyHash uint64 + var generatedPolicyHash bool + switch policy.HashPolicyType { + case xdsresource.HashPolicyTypeHeader: + md, ok := metadata.FromOutgoingContext(rpcInfo.Context) + if !ok { + continue + } + values := md.Get(policy.HeaderName) + // If the header isn't present, no-op. + if len(values) == 0 { + continue + } + joinedValues := strings.Join(values, ",") + if policy.Regex != nil { + joinedValues = policy.Regex.ReplaceAllString(joinedValues, policy.RegexSubstitution) + } + policyHash = xxhash.Sum64String(joinedValues) + generatedHash = true + generatedPolicyHash = true + case xdsresource.HashPolicyTypeChannelID: + // Use the static channel ID as the hash for this policy. + policyHash = cs.r.channelID + generatedHash = true + generatedPolicyHash = true + } + + // Deterministically combine the hash policies. Rotating prevents + // duplicate hash policies from cancelling each other out and preserves + // the 64 bits of entropy. + if generatedPolicyHash { + hash = bits.RotateLeft64(hash, 1) + hash = hash ^ policyHash + } - // Generate actions. - action := make(map[string]xdsActionConfig) - for _, act := range r.actions { - action[act.assignedName] = xdsActionConfig{ - ChildPolicy: weightedClusterToBalancerConfig(act.clustersWithWeights), + // If terminal policy and a hash has already been generated, ignore the + // rest of the policies and use that hash already generated. + if policy.Terminal && generatedHash { + break } } - sc := serviceConfig{ - LoadBalancingConfig: newBalancerConfig( - xdsRoutingName, xdsRoutingBalancerConfig{ - Route: rts, - Action: action, - }, - ), + if generatedHash { + return hash } + // If no generated hash return a random long. In the grand scheme of things + // this logically will map to choosing a random backend to route request to. + return grpcrand.Uint64() +} - bs, err := json.Marshal(sc) - if err != nil { - return "", fmt.Errorf("failed to marshal json: %v", err) +func (cs *configSelector) newInterceptor(rt *route, cluster *routeCluster) (iresolver.ClientInterceptor, error) { + if len(cs.httpFilterConfig) == 0 { + return nil, nil } - return string(bs), nil + interceptors := make([]iresolver.ClientInterceptor, 0, len(cs.httpFilterConfig)) + for _, filter := range cs.httpFilterConfig { + override := cluster.httpFilterConfigOverride[filter.Name] // cluster is highest priority + if override == nil { + override = rt.httpFilterConfigOverride[filter.Name] // route is second priority + } + if override == nil { + override = cs.virtualHost.httpFilterConfigOverride[filter.Name] // VH is third & lowest priority + } + ib, ok := filter.Filter.(httpfilter.ClientInterceptorBuilder) + if !ok { + // Should not happen if it passed xdsClient validation. + return nil, fmt.Errorf("filter does not support use in client") + } + i, err := ib.BuildClientInterceptor(filter.Config, override) + if err != nil { + return nil, fmt.Errorf("error constructing filter: %v", err) + } + if i != nil { + interceptors = append(interceptors, i) + } + } + return &interceptorList{interceptors: interceptors}, nil } -func weightedClusterToBalancerConfig(wc map[string]uint32) balancerConfig { - // Even if WeightedCluster has only one entry, we still use weighted_target - // as top level balancer, to avoid switching top policy between CDS and - // weighted_target, causing TCP connection to be recreated. - targets := make(map[string]cdsWithWeight) - for name, weight := range wc { - targets[name] = cdsWithWeight{ - Weight: weight, - ChildPolicy: newBalancerConfig(cdsName, cdsBalancerConfig{Cluster: name}), +// stop decrements refs of all clusters referenced by this config selector. +func (cs *configSelector) stop() { + // The resolver's old configSelector may be nil. Handle that here. + if cs == nil { + return + } + // If any refs drop to zero, we'll need a service config update to delete + // the cluster. + needUpdate := false + // Loops over cs.clusters, but these are pointers to entries in + // activeClusters. + for _, ci := range cs.clusters { + if v := atomic.AddInt32(&ci.refCount, -1); v == 0 { + needUpdate = true } } - bc := newBalancerConfig( - weightedTargetName, weightedCDSBalancerConfig{ - Targets: targets, + // We stop the old config selector immediately after sending a new config + // selector; we need another update to delete clusters from the config (if + // we don't have another update pending already). + if needUpdate { + select { + case cs.r.updateCh <- suWithError{emptyUpdate: true}: + default: + } + } +} + +// A global for testing. +var newWRR = wrr.NewRandom + +// newConfigSelector creates the config selector for su; may add entries to +// r.activeClusters for previously-unseen clusters. +func (r *xdsResolver) newConfigSelector(su serviceUpdate) (*configSelector, error) { + cs := &configSelector{ + r: r, + virtualHost: virtualHost{ + httpFilterConfigOverride: su.virtualHost.HTTPFilterConfigOverride, + retryConfig: su.virtualHost.RetryConfig, }, - ) - return bc + routes: make([]route, len(su.virtualHost.Routes)), + clusters: make(map[string]*clusterInfo), + httpFilterConfig: su.ldsConfig.httpFilterConfig, + } + + for i, rt := range su.virtualHost.Routes { + clusters := newWRR() + if rt.ClusterSpecifierPlugin != "" { + clusterName := clusterSpecifierPluginPrefix + rt.ClusterSpecifierPlugin + clusters.Add(&routeCluster{ + name: clusterName, + }, 1) + cs.initializeCluster(clusterName, xdsChildConfig{ + ChildPolicy: balancerConfig(su.clusterSpecifierPlugins[rt.ClusterSpecifierPlugin]), + }) + } else { + for cluster, wc := range rt.WeightedClusters { + clusterName := clusterPrefix + cluster + clusters.Add(&routeCluster{ + name: clusterName, + httpFilterConfigOverride: wc.HTTPFilterConfigOverride, + }, int64(wc.Weight)) + cs.initializeCluster(clusterName, xdsChildConfig{ + ChildPolicy: newBalancerConfig(cdsName, cdsBalancerConfig{Cluster: cluster}), + }) + } + } + cs.routes[i].clusters = clusters + + var err error + cs.routes[i].m, err = xdsresource.RouteToMatcher(rt) + if err != nil { + return nil, err + } + cs.routes[i].actionType = rt.ActionType + if rt.MaxStreamDuration == nil { + cs.routes[i].maxStreamDuration = su.ldsConfig.maxStreamDuration + } else { + cs.routes[i].maxStreamDuration = *rt.MaxStreamDuration + } + + cs.routes[i].httpFilterConfigOverride = rt.HTTPFilterConfigOverride + cs.routes[i].retryConfig = rt.RetryConfig + cs.routes[i].hashPolicies = rt.HashPolicies + } + + // Account for this config selector's clusters. Do this after no further + // errors may occur. Note: cs.clusters are pointers to entries in + // activeClusters. + for _, ci := range cs.clusters { + atomic.AddInt32(&ci.refCount, 1) + } + + return cs, nil +} + +// initializeCluster initializes entries in cs.clusters map, creating entries in +// r.activeClusters as necessary. Any created entries will have a ref count set +// to zero as their ref count will be incremented by incRefs. +func (cs *configSelector) initializeCluster(clusterName string, cfg xdsChildConfig) { + ci := cs.r.activeClusters[clusterName] + if ci == nil { + ci = &clusterInfo{refCount: 0} + cs.r.activeClusters[clusterName] = ci + } + cs.clusters[clusterName] = ci + cs.clusters[clusterName].cfg = cfg } -func (r *xdsResolver) serviceUpdateToJSON(su xdsclient.ServiceUpdate) (string, error) { - return r.routesToJSON(su.Routes) +type clusterInfo struct { + // number of references to this cluster; accessed atomically + refCount int32 + // cfg is the child configuration for this cluster, containing either the + // csp config or the cds cluster config. + cfg xdsChildConfig +} + +type interceptorList struct { + interceptors []iresolver.ClientInterceptor +} + +func (il *interceptorList) NewStream(ctx context.Context, ri iresolver.RPCInfo, done func(), newStream func(ctx context.Context, done func()) (iresolver.ClientStream, error)) (iresolver.ClientStream, error) { + for i := len(il.interceptors) - 1; i >= 0; i-- { + ns := newStream + interceptor := il.interceptors[i] + newStream = func(ctx context.Context, done func()) (iresolver.ClientStream, error) { + return interceptor.NewStream(ctx, ri, done, ns) + } + } + return newStream(ctx, func() {}) } diff --git a/xds/internal/resolver/serviceconfig_action.go b/xds/internal/resolver/serviceconfig_action.go deleted file mode 100644 index d582048fda09..000000000000 --- a/xds/internal/resolver/serviceconfig_action.go +++ /dev/null @@ -1,186 +0,0 @@ -/* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package resolver - -import ( - "fmt" - "math" - "sort" - "strconv" - - "google.golang.org/grpc/internal/grpcrand" - xdsclient "google.golang.org/grpc/xds/internal/client" -) - -type actionWithAssignedName struct { - // cluster:weight, "A":40, "B":60 - clustersWithWeights map[string]uint32 - // clusterNames, without weights, sorted and hashed, "A_B_" - clusterNames string - // The assigned name, clusters plus a random number, "A_B_1" - assignedName string - // randomNumber is the number appended to assignedName. - randomNumber int64 -} - -// newActionsFromRoutes gets actions from the routes, and turns them into a map -// keyed by the hash of the clusters. -// -// In the returned map, all actions don't have assignedName. The assignedName -// will be filled in after comparing the new actions with the existing actions, -// so when a new and old action only diff in weights, the new action can reuse -// the old action's name. -// -// from -// {B:60, A:40}, {A:30, B:70}, {B:90, C:10} -// -// to -// A40_B60_: {{A:40, B:60}, "A_B_", ""} -// A30_B70_: {{A:30, B:70}, "A_B_", ""} -// B90_C10_: {{B:90, C:10}, "B_C_", ""} -func newActionsFromRoutes(routes []*xdsclient.Route) map[string]actionWithAssignedName { - newActions := make(map[string]actionWithAssignedName) - for _, route := range routes { - var clusterNames []string - for n := range route.Action { - clusterNames = append(clusterNames, n) - } - - // Sort names to be consistent. - sort.Strings(clusterNames) - clustersOnly := "" - clustersWithWeight := "" - for _, c := range clusterNames { - // Generates A_B_ - clustersOnly = clustersOnly + c + "_" - // Generates A40_B60_ - clustersWithWeight = clustersWithWeight + c + strconv.FormatUint(uint64(route.Action[c]), 10) + "_" - } - - if _, ok := newActions[clustersWithWeight]; !ok { - newActions[clustersWithWeight] = actionWithAssignedName{ - clustersWithWeights: route.Action, - clusterNames: clustersOnly, - } - } - } - return newActions -} - -// updateActions takes a new map of actions, and updates the existing action map in the resolver. -// -// In the old map, all actions have assignedName set. -// In the new map, all actions have no assignedName. -// -// After the update, the action map is updated to have all actions from the new -// map, with assignedName: -// - if the new action exists in old, get the old name -// - if the new action doesn't exist in old -// - if there is an old action that will be removed, and has the same set of -// clusters, reuse the old action's name -// - otherwise, generate a new name -func (r *xdsResolver) updateActions(newActions map[string]actionWithAssignedName) { - if r.actions == nil { - r.actions = make(map[string]actionWithAssignedName) - } - - // Delete actions from existingActions if they are not in newActions. Keep - // the removed actions in a map, with key as clusterNames without weights, - // so their assigned names can be reused. - existingActions := r.actions - actionsRemoved := make(map[string][]string) - for actionHash, act := range existingActions { - if _, ok := newActions[actionHash]; !ok { - actionsRemoved[act.clusterNames] = append(actionsRemoved[act.clusterNames], act.assignedName) - delete(existingActions, actionHash) - } - } - - // Find actions in newActions but not in oldActions. Add them, and try to - // reuse assigned names from actionsRemoved. - if r.usedActionNameRandomNumber == nil { - r.usedActionNameRandomNumber = make(map[int64]bool) - } - for actionHash, act := range newActions { - if _, ok := existingActions[actionHash]; !ok { - if assignedNamed, ok := actionsRemoved[act.clusterNames]; ok { - // Reuse the first assigned name from actionsRemoved. - act.assignedName = assignedNamed[0] - // If there are more names to reuse after this, update the slice - // in the map. Otherwise, remove the entry from the map. - if len(assignedNamed) > 1 { - actionsRemoved[act.clusterNames] = assignedNamed[1:] - } else { - delete(actionsRemoved, act.clusterNames) - } - existingActions[actionHash] = act - continue - } - // Generate a new name. - act.randomNumber = r.nextAssignedNameRandomNumber() - act.assignedName = fmt.Sprintf("%s%d", act.clusterNames, act.randomNumber) - existingActions[actionHash] = act - } - } - - // Delete entry from nextIndex if all actions with the clusters are removed. - remainingRandomNumbers := make(map[int64]bool) - for _, act := range existingActions { - remainingRandomNumbers[act.randomNumber] = true - } - r.usedActionNameRandomNumber = remainingRandomNumbers -} - -var grpcrandInt63n = grpcrand.Int63n - -func (r *xdsResolver) nextAssignedNameRandomNumber() int64 { - for { - t := grpcrandInt63n(math.MaxInt32) - if !r.usedActionNameRandomNumber[t] { - return t - } - } -} - -// getActionAssignedName hashes the clusters from the action, and find the -// assigned action name. The assigned action names are kept in r.actions, with -// the clusters name hash as map key. -// -// The assigned action name is not simply the hash. For example, the hash can be -// "A40_B60_", but the assigned name can be "A_B_0". It's this way so the action -// can be reused if only weights are changing. -func (r *xdsResolver) getActionAssignedName(action map[string]uint32) string { - var clusterNames []string - for n := range action { - clusterNames = append(clusterNames, n) - } - // Hash cluster names. Sort names to be consistent. - sort.Strings(clusterNames) - clustersWithWeight := "" - for _, c := range clusterNames { - // Generates hash "A40_B60_". - clustersWithWeight = clustersWithWeight + c + strconv.FormatUint(uint64(action[c]), 10) + "_" - } - // Look in r.actions for the assigned action name. - if act, ok := r.actions[clustersWithWeight]; ok { - return act.assignedName - } - r.logger.Warningf("no assigned name found for action %v", action) - return "" -} diff --git a/xds/internal/resolver/serviceconfig_action_test.go b/xds/internal/resolver/serviceconfig_action_test.go deleted file mode 100644 index bfc0e6830155..000000000000 --- a/xds/internal/resolver/serviceconfig_action_test.go +++ /dev/null @@ -1,356 +0,0 @@ -/* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package resolver - -import ( - "testing" - - "github.com/google/go-cmp/cmp" - xdsclient "google.golang.org/grpc/xds/internal/client" -) - -func TestNewActionsFromRoutes(t *testing.T) { - tests := []struct { - name string - routes []*xdsclient.Route - want map[string]actionWithAssignedName - }{ - { - name: "temp", - routes: []*xdsclient.Route{ - {Action: map[string]uint32{"B": 60, "A": 40}}, - {Action: map[string]uint32{"A": 30, "B": 70}}, - {Action: map[string]uint32{"B": 90, "C": 10}}, - }, - want: map[string]actionWithAssignedName{ - "A40_B60_": {map[string]uint32{"A": 40, "B": 60}, "A_B_", "", 0}, - "A30_B70_": {map[string]uint32{"A": 30, "B": 70}, "A_B_", "", 0}, - "B90_C10_": {map[string]uint32{"B": 90, "C": 10}, "B_C_", "", 0}, - }, - }, - } - - cmpOpts := []cmp.Option{cmp.AllowUnexported(actionWithAssignedName{})} - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := newActionsFromRoutes(tt.routes); !cmp.Equal(got, tt.want, cmpOpts...) { - t.Errorf("newActionsFromRoutes() got unexpected result, diff %v", cmp.Diff(got, tt.want, cmpOpts...)) - } - }) - } -} - -func TestRemoveOrReuseName(t *testing.T) { - tests := []struct { - name string - oldActions map[string]actionWithAssignedName - oldRandNums map[int64]bool - newActions map[string]actionWithAssignedName - wantActions map[string]actionWithAssignedName - wantRandNums map[int64]bool - }{ - { - name: "add same cluster", - oldActions: map[string]actionWithAssignedName{ - "a20_b30_c50_": { - clustersWithWeights: map[string]uint32{"a": 20, "b": 30, "c": 50}, - clusterNames: "a_b_c_", - assignedName: "a_b_c_0", - randomNumber: 0, - }, - }, - oldRandNums: map[int64]bool{ - 0: true, - }, - newActions: map[string]actionWithAssignedName{ - "a20_b30_c50_": { - clustersWithWeights: map[string]uint32{"a": 20, "b": 30, "c": 50}, - clusterNames: "a_b_c_", - }, - "a10_b50_c40_": { - clustersWithWeights: map[string]uint32{"a": 10, "b": 50, "c": 40}, - clusterNames: "a_b_c_", - }, - }, - wantActions: map[string]actionWithAssignedName{ - "a20_b30_c50_": { - clustersWithWeights: map[string]uint32{"a": 20, "b": 30, "c": 50}, - clusterNames: "a_b_c_", - assignedName: "a_b_c_0", - randomNumber: 0, - }, - "a10_b50_c40_": { - clustersWithWeights: map[string]uint32{"a": 10, "b": 50, "c": 40}, - clusterNames: "a_b_c_", - assignedName: "a_b_c_1000", - randomNumber: 1000, - }, - }, - wantRandNums: map[int64]bool{ - 0: true, - 1000: true, - }, - }, - { - name: "delete same cluster", - oldActions: map[string]actionWithAssignedName{ - "a20_b30_c50_": { - clustersWithWeights: map[string]uint32{"a": 20, "b": 30, "c": 50}, - clusterNames: "a_b_c_", - assignedName: "a_b_c_0", - randomNumber: 0, - }, - "a10_b50_c40_": { - clustersWithWeights: map[string]uint32{"a": 10, "b": 50, "c": 40}, - clusterNames: "a_b_c_", - assignedName: "a_b_c_1", - randomNumber: 1, - }, - }, - oldRandNums: map[int64]bool{ - 0: true, - 1: true, - }, - newActions: map[string]actionWithAssignedName{ - "a20_b30_c50_": { - clustersWithWeights: map[string]uint32{"a": 20, "b": 30, "c": 50}, - clusterNames: "a_b_c_", - }, - }, - wantActions: map[string]actionWithAssignedName{ - "a20_b30_c50_": { - clustersWithWeights: map[string]uint32{"a": 20, "b": 30, "c": 50}, - clusterNames: "a_b_c_", - assignedName: "a_b_c_0", - randomNumber: 0, - }, - }, - wantRandNums: map[int64]bool{ - 0: true, - }, - }, - { - name: "add new clusters", - oldActions: map[string]actionWithAssignedName{ - "a20_b30_c50_": { - clustersWithWeights: map[string]uint32{"a": 20, "b": 30, "c": 50}, - clusterNames: "a_b_c_", - assignedName: "a_b_c_0", - randomNumber: 0, - }, - }, - oldRandNums: map[int64]bool{ - 0: true, - }, - newActions: map[string]actionWithAssignedName{ - "a20_b30_c50_": { - clustersWithWeights: map[string]uint32{"a": 20, "b": 30, "c": 50}, - clusterNames: "a_b_c_", - }, - "a50_b50_": { - clustersWithWeights: map[string]uint32{"a": 50, "b": 50}, - clusterNames: "a_b_", - }, - }, - wantActions: map[string]actionWithAssignedName{ - "a20_b30_c50_": { - clustersWithWeights: map[string]uint32{"a": 20, "b": 30, "c": 50}, - clusterNames: "a_b_c_", - assignedName: "a_b_c_0", - randomNumber: 0, - }, - "a50_b50_": { - clustersWithWeights: map[string]uint32{"a": 50, "b": 50}, - clusterNames: "a_b_", - assignedName: "a_b_1000", - randomNumber: 1000, - }, - }, - wantRandNums: map[int64]bool{ - 0: true, - 1000: true, - }, - }, - { - name: "reuse", - oldActions: map[string]actionWithAssignedName{ - "a20_b30_c50_": { - clustersWithWeights: map[string]uint32{"a": 20, "b": 30, "c": 50}, - clusterNames: "a_b_c_", - assignedName: "a_b_c_0", - randomNumber: 0, - }, - }, - oldRandNums: map[int64]bool{ - 0: true, - }, - newActions: map[string]actionWithAssignedName{ - "a10_b50_c40_": { - clustersWithWeights: map[string]uint32{"a": 10, "b": 50, "c": 40}, - clusterNames: "a_b_c_", - }, - }, - wantActions: map[string]actionWithAssignedName{ - "a10_b50_c40_": { - clustersWithWeights: map[string]uint32{"a": 10, "b": 50, "c": 40}, - clusterNames: "a_b_c_", - assignedName: "a_b_c_0", - randomNumber: 0, - }, - }, - wantRandNums: map[int64]bool{ - 0: true, - }, - }, - { - name: "add and reuse", - oldActions: map[string]actionWithAssignedName{ - "a20_b30_c50_": { - clustersWithWeights: map[string]uint32{"a": 20, "b": 30, "c": 50}, - clusterNames: "a_b_c_", - assignedName: "a_b_c_0", - randomNumber: 0, - }, - "a10_b50_c40_": { - clustersWithWeights: map[string]uint32{"a": 10, "b": 50, "c": 40}, - clusterNames: "a_b_c_", - assignedName: "a_b_c_1", - randomNumber: 1, - }, - "a50_b50_": { - clustersWithWeights: map[string]uint32{"a": 50, "b": 50}, - clusterNames: "a_b_", - assignedName: "a_b_2", - randomNumber: 2, - }, - }, - oldRandNums: map[int64]bool{ - 0: true, - 1: true, - 2: true, - }, - newActions: map[string]actionWithAssignedName{ - "a10_b50_c40_": { - clustersWithWeights: map[string]uint32{"a": 10, "b": 50, "c": 40}, - clusterNames: "a_b_c_", - }, - "a30_b30_c40_": { - clustersWithWeights: map[string]uint32{"a": 30, "b": 30, "c": 40}, - clusterNames: "a_b_c_", - }, - "c50_d50_": { - clustersWithWeights: map[string]uint32{"c": 50, "d": 50}, - clusterNames: "c_d_", - }, - }, - wantActions: map[string]actionWithAssignedName{ - "a10_b50_c40_": { - clustersWithWeights: map[string]uint32{"a": 10, "b": 50, "c": 40}, - clusterNames: "a_b_c_", - assignedName: "a_b_c_1", - randomNumber: 1, - }, - "a30_b30_c40_": { - clustersWithWeights: map[string]uint32{"a": 30, "b": 30, "c": 40}, - clusterNames: "a_b_c_", - assignedName: "a_b_c_0", - randomNumber: 0, - }, - "c50_d50_": { - clustersWithWeights: map[string]uint32{"c": 50, "d": 50}, - clusterNames: "c_d_", - assignedName: "c_d_1000", - randomNumber: 1000, - }, - }, - wantRandNums: map[int64]bool{ - 0: true, - 1: true, - 1000: true, - }, - }, - } - cmpOpts := []cmp.Option{cmp.AllowUnexported(actionWithAssignedName{})} - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - defer replaceRandNumGenerator(1000)() - r := &xdsResolver{ - actions: tt.oldActions, - usedActionNameRandomNumber: tt.oldRandNums, - } - r.updateActions(tt.newActions) - if !cmp.Equal(r.actions, tt.wantActions, cmpOpts...) { - t.Errorf("removeOrReuseName() got unexpected actions, diff %v", cmp.Diff(r.actions, tt.wantActions, cmpOpts...)) - } - if !cmp.Equal(r.usedActionNameRandomNumber, tt.wantRandNums) { - t.Errorf("removeOrReuseName() got unexpected nextIndex, diff %v", cmp.Diff(r.usedActionNameRandomNumber, tt.wantRandNums)) - } - }) - } -} - -func TestGetActionAssignedName(t *testing.T) { - tests := []struct { - name string - actions map[string]actionWithAssignedName - action map[string]uint32 - want string - }{ - { - name: "good", - actions: map[string]actionWithAssignedName{ - "a20_b30_c50_": { - clustersWithWeights: map[string]uint32{"a": 20, "b": 30, "c": 50}, - clusterNames: "a_b_c_", - assignedName: "a_b_c_0", - }, - }, - action: map[string]uint32{"a": 20, "b": 30, "c": 50}, - want: "a_b_c_0", - }, - { - name: "two", - actions: map[string]actionWithAssignedName{ - "a20_b30_c50_": { - clustersWithWeights: map[string]uint32{"a": 20, "b": 30, "c": 50}, - clusterNames: "a_b_c_", - assignedName: "a_b_c_0", - }, - "c50_d50_": { - clustersWithWeights: map[string]uint32{"c": 50, "d": 50}, - clusterNames: "c_d_", - assignedName: "c_d_0", - }, - }, - action: map[string]uint32{"c": 50, "d": 50}, - want: "c_d_0", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - r := &xdsResolver{ - actions: tt.actions, - } - if got := r.getActionAssignedName(tt.action); got != tt.want { - t.Errorf("getActionAssignedName() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/xds/internal/resolver/serviceconfig_test.go b/xds/internal/resolver/serviceconfig_test.go index 1fc03d1fac16..786b003c0154 100644 --- a/xds/internal/resolver/serviceconfig_test.go +++ b/xds/internal/resolver/serviceconfig_test.go @@ -19,347 +19,100 @@ package resolver import ( + "context" + "regexp" "testing" + xxhash "github.com/cespare/xxhash/v2" "github.com/google/go-cmp/cmp" - "google.golang.org/grpc/internal" - "google.golang.org/grpc/internal/grpcrand" - "google.golang.org/grpc/serviceconfig" - _ "google.golang.org/grpc/xds/internal/balancer/weightedtarget" - _ "google.golang.org/grpc/xds/internal/balancer/xdsrouting" - xdsclient "google.golang.org/grpc/xds/internal/client" + iresolver "google.golang.org/grpc/internal/resolver" + "google.golang.org/grpc/metadata" + _ "google.golang.org/grpc/xds/internal/balancer/cdsbalancer" // To parse LB config + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" ) -const ( - testCluster1 = "test-cluster-1" - testOneClusterOnlyJSON = `{"loadBalancingConfig":[{ - "xds_routing_experimental":{ - "action":{ - "test-cluster-1_0":{ - "childPolicy":[{ - "weighted_target_experimental":{ - "targets":{ - "test-cluster-1":{ - "weight":1, - "childPolicy":[{"cds_experimental":{"cluster":"test-cluster-1"}}] - } - }}}] - } - }, - "route":[{"prefix":"","action":"test-cluster-1_0"}] - }}]}` - testWeightedCDSJSON = `{"loadBalancingConfig":[{ - "xds_routing_experimental":{ - "action":{ - "cluster_1_cluster_2_1":{ - "childPolicy":[{ - "weighted_target_experimental":{ - "targets":{ - "cluster_1":{ - "weight":75, - "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}] - }, - "cluster_2":{ - "weight":25, - "childPolicy":[{"cds_experimental":{"cluster":"cluster_2"}}] - } - }}}] - } - }, - "route":[{"prefix":"","action":"cluster_1_cluster_2_1"}] - }}]}` - - testRoutingJSON = `{"loadBalancingConfig":[{ - "xds_routing_experimental": { - "action":{ - "cluster_1_cluster_2_0":{ - "childPolicy":[{ - "weighted_target_experimental": { - "targets": { - "cluster_1" : { - "weight":75, - "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}] - }, - "cluster_2" : { - "weight":25, - "childPolicy":[{"cds_experimental":{"cluster":"cluster_2"}}] - } - } - } - }] - } - }, - - "route":[{ - "path":"/service_1/method_1", - "action":"cluster_1_cluster_2_0" - }] - } - }]} -` - testRoutingAllMatchersJSON = `{"loadBalancingConfig":[{ - "xds_routing_experimental": { - "action":{ - "cluster_1_0":{ - "childPolicy":[{ - "weighted_target_experimental": { - "targets": { - "cluster_1" : { - "weight":1, - "childPolicy":[{"cds_experimental":{"cluster":"cluster_1"}}] - } - } - } - }] - }, - "cluster_2_0":{ - "childPolicy":[{ - "weighted_target_experimental": { - "targets": { - "cluster_2" : { - "weight":1, - "childPolicy":[{"cds_experimental":{"cluster":"cluster_2"}}] - } - } - } - }] - }, - "cluster_3_0":{ - "childPolicy":[{ - "weighted_target_experimental": { - "targets": { - "cluster_3" : { - "weight":1, - "childPolicy":[{"cds_experimental":{"cluster":"cluster_3"}}] - } - } - } - }] - } - }, - - "route":[{ - "path":"/service_1/method_1", - "action":"cluster_1_0" - }, - { - "prefix":"/service_2/method_1", - "action":"cluster_1_0" - }, - { - "regex":"^/service_2/method_3$", - "action":"cluster_1_0" - }, - { - "prefix":"", - "headers":[{"name":"header-1", "exactMatch":"value-1", "invertMatch":true}], - "action":"cluster_2_0" - }, - { - "prefix":"", - "headers":[{"name":"header-1", "regexMatch":"^value-1$"}], - "action":"cluster_2_0" - }, - { - "prefix":"", - "headers":[{"name":"header-1", "rangeMatch":{"start":-1, "end":7}}], - "action":"cluster_3_0" - }, - { - "prefix":"", - "headers":[{"name":"header-1", "presentMatch":true}], - "action":"cluster_3_0" - }, - { - "prefix":"", - "headers":[{"name":"header-1", "prefixMatch":"value-1"}], - "action":"cluster_2_0" - }, - { - "prefix":"", - "headers":[{"name":"header-1", "suffixMatch":"value-1"}], - "action":"cluster_2_0" - }, - { - "prefix":"", - "matchFraction":{"value": 31415}, - "action":"cluster_3_0" - }] - } - }]} -` -) +func (s) TestPruneActiveClusters(t *testing.T) { + r := &xdsResolver{activeClusters: map[string]*clusterInfo{ + "zero": {refCount: 0}, + "one": {refCount: 1}, + "two": {refCount: 2}, + "anotherzero": {refCount: 0}, + }} + want := map[string]*clusterInfo{ + "one": {refCount: 1}, + "two": {refCount: 2}, + } + r.pruneActiveClusters() + if d := cmp.Diff(r.activeClusters, want, cmp.AllowUnexported(clusterInfo{})); d != "" { + t.Fatalf("r.activeClusters = %v; want %v\nDiffs: %v", r.activeClusters, want, d) + } +} -func TestRoutesToJSON(t *testing.T) { +func (s) TestGenerateRequestHash(t *testing.T) { + const channelID = 12378921 + cs := &configSelector{ + r: &xdsResolver{ + cc: &testClientConn{}, + channelID: channelID, + }, + } tests := []struct { - name string - routes []*xdsclient.Route - wantJSON string - wantErr bool + name string + hashPolicies []*xdsresource.HashPolicy + requestHashWant uint64 + rpcInfo iresolver.RPCInfo }{ + // TestGenerateRequestHashHeaders tests generating request hashes for + // hash policies that specify to hash headers. { - name: "one route", - routes: []*xdsclient.Route{{ - Path: newStringP("/service_1/method_1"), - Action: map[string]uint32{"cluster_1": 75, "cluster_2": 25}, + name: "test-generate-request-hash-headers", + hashPolicies: []*xdsresource.HashPolicy{{ + HashPolicyType: xdsresource.HashPolicyTypeHeader, + HeaderName: ":path", + Regex: func() *regexp.Regexp { return regexp.MustCompile("/products") }(), // Will replace /products with /new-products, to test find and replace functionality. + RegexSubstitution: "/new-products", }}, - wantJSON: testRoutingJSON, - wantErr: false, + requestHashWant: xxhash.Sum64String("/new-products"), + rpcInfo: iresolver.RPCInfo{ + Context: metadata.NewOutgoingContext(context.Background(), metadata.Pairs(":path", "/products")), + Method: "/some-method", + }, }, + // TestGenerateHashChannelID tests generating request hashes for hash + // policies that specify to hash something that uniquely identifies the + // ClientConn (the pointer). { - name: "all matchers", - routes: []*xdsclient.Route{ - { - Path: newStringP("/service_1/method_1"), - Action: map[string]uint32{"cluster_1": 1}, - }, - { - Prefix: newStringP("/service_2/method_1"), - Action: map[string]uint32{"cluster_1": 1}, - }, - { - Regex: newStringP("^/service_2/method_3$"), - Action: map[string]uint32{"cluster_1": 1}, - }, - { - Prefix: newStringP(""), - Headers: []*xdsclient.HeaderMatcher{{ - Name: "header-1", - InvertMatch: newBoolP(true), - ExactMatch: newStringP("value-1"), - }}, - Action: map[string]uint32{"cluster_2": 1}, - }, - { - Prefix: newStringP(""), - Headers: []*xdsclient.HeaderMatcher{{ - Name: "header-1", - RegexMatch: newStringP("^value-1$"), - }}, - Action: map[string]uint32{"cluster_2": 1}, - }, - { - Prefix: newStringP(""), - Headers: []*xdsclient.HeaderMatcher{{ - Name: "header-1", - RangeMatch: &xdsclient.Int64Range{Start: -1, End: 7}, - }}, - Action: map[string]uint32{"cluster_3": 1}, - }, - { - Prefix: newStringP(""), - Headers: []*xdsclient.HeaderMatcher{{ - Name: "header-1", - PresentMatch: newBoolP(true), - }}, - Action: map[string]uint32{"cluster_3": 1}, - }, - { - Prefix: newStringP(""), - Headers: []*xdsclient.HeaderMatcher{{ - Name: "header-1", - PrefixMatch: newStringP("value-1"), - }}, - Action: map[string]uint32{"cluster_2": 1}, - }, - { - Prefix: newStringP(""), - Headers: []*xdsclient.HeaderMatcher{{ - Name: "header-1", - SuffixMatch: newStringP("value-1"), - }}, - Action: map[string]uint32{"cluster_2": 1}, - }, - { - Prefix: newStringP(""), - Fraction: newUint32P(31415), - Action: map[string]uint32{"cluster_3": 1}, - }, - }, - wantJSON: testRoutingAllMatchersJSON, - wantErr: false, + name: "test-generate-request-hash-channel-id", + hashPolicies: []*xdsresource.HashPolicy{{ + HashPolicyType: xdsresource.HashPolicyTypeChannelID, + }}, + requestHashWant: channelID, + rpcInfo: iresolver.RPCInfo{}, }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // Note this random number function only generates 0. This is - // because the test doesn't handle action update, and there's only - // one action for each cluster bundle. - // - // This is necessary so the output is deterministic. - grpcrandInt63n = func(int64) int64 { return 0 } - defer func() { grpcrandInt63n = grpcrand.Int63n }() - - gotJSON, err := (&xdsResolver{}).routesToJSON(tt.routes) - if err != nil { - t.Errorf("routesToJSON returned error: %v", err) - return - } - - gotParsed := internal.ParseServiceConfigForTesting.(func(string) *serviceconfig.ParseResult)(gotJSON) - wantParsed := internal.ParseServiceConfigForTesting.(func(string) *serviceconfig.ParseResult)(tt.wantJSON) - - if !internal.EqualServiceConfigForTesting(gotParsed.Config, wantParsed.Config) { - t.Errorf("serviceUpdateToJSON() = %v, want %v", gotJSON, tt.wantJSON) - t.Error("gotParsed: ", cmp.Diff(nil, gotParsed)) - t.Error("wantParsed: ", cmp.Diff(nil, wantParsed)) - } - }) - } -} - -func TestServiceUpdateToJSON(t *testing.T) { - tests := []struct { - name string - su xdsclient.ServiceUpdate - wantJSON string - wantErr bool - }{ + // TestGenerateRequestHashEmptyString tests generating request hashes + // for hash policies that specify to hash headers and replace empty + // strings in the headers. { - name: "routing", - su: xdsclient.ServiceUpdate{ - Routes: []*xdsclient.Route{{ - Path: newStringP("/service_1/method_1"), - Action: map[string]uint32{"cluster_1": 75, "cluster_2": 25}, - }}, + name: "test-generate-request-hash-empty-string", + hashPolicies: []*xdsresource.HashPolicy{{ + HashPolicyType: xdsresource.HashPolicyTypeHeader, + HeaderName: ":path", + Regex: func() *regexp.Regexp { return regexp.MustCompile("") }(), + RegexSubstitution: "e", + }}, + requestHashWant: xxhash.Sum64String("eaebece"), + rpcInfo: iresolver.RPCInfo{ + Context: metadata.NewOutgoingContext(context.Background(), metadata.Pairs(":path", "abc")), + Method: "/some-method", }, - wantJSON: testRoutingJSON, - wantErr: false, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - defer replaceRandNumGenerator(0)() - gotJSON, err := (&xdsResolver{}).serviceUpdateToJSON(tt.su) - if err != nil { - t.Errorf("serviceUpdateToJSON returned error: %v", err) - return - } - - gotParsed := internal.ParseServiceConfigForTesting.(func(string) *serviceconfig.ParseResult)(gotJSON) - wantParsed := internal.ParseServiceConfigForTesting.(func(string) *serviceconfig.ParseResult)(tt.wantJSON) - - if !internal.EqualServiceConfigForTesting(gotParsed.Config, wantParsed.Config) { - t.Errorf("serviceUpdateToJSON() = %v, want %v", gotJSON, tt.wantJSON) - t.Error("gotParsed: ", cmp.Diff(nil, gotParsed)) - t.Error("wantParsed: ", cmp.Diff(nil, wantParsed)) + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + requestHashGot := cs.generateHash(test.rpcInfo, test.hashPolicies) + if requestHashGot != test.requestHashWant { + t.Fatalf("requestHashGot = %v, requestHashWant = %v", requestHashGot, test.requestHashWant) } }) } } - -// Two updates to the same resolver, test that action names are reused. -func TestServiceUpdateToJSON_TwoConfig_UpdateActions(t *testing.T) { -} - -func newStringP(s string) *string { - return &s -} - -func newBoolP(b bool) *bool { - return &b -} - -func newUint32P(i uint32) *uint32 { - return &i -} diff --git a/xds/internal/resolver/watch_service.go b/xds/internal/resolver/watch_service.go new file mode 100644 index 000000000000..4f8609ce9df5 --- /dev/null +++ b/xds/internal/resolver/watch_service.go @@ -0,0 +1,203 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package resolver + +import ( + "fmt" + "sync" + "time" + + "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/xds/internal/clusterspecifier" + "google.golang.org/grpc/xds/internal/xdsclient" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" +) + +// serviceUpdate contains information received from the LDS/RDS responses which +// are of interest to the xds resolver. The RDS request is built by first +// making a LDS to get the RouteConfig name. +type serviceUpdate struct { + // virtualHost contains routes and other configuration to route RPCs. + virtualHost *xdsresource.VirtualHost + // clusterSpecifierPlugins contains the configurations for any cluster + // specifier plugins emitted by the xdsclient. + clusterSpecifierPlugins map[string]clusterspecifier.BalancerConfig + // ldsConfig contains configuration that applies to all routes. + ldsConfig ldsConfig +} + +// ldsConfig contains information received from the LDS responses which are of +// interest to the xds resolver. +type ldsConfig struct { + // maxStreamDuration is from the HTTP connection manager's + // common_http_protocol_options field. + maxStreamDuration time.Duration + httpFilterConfig []xdsresource.HTTPFilter +} + +// watchService uses LDS and RDS to discover information about the provided +// serviceName. +// +// Note that during race (e.g. an xDS response is received while the user is +// calling cancel()), there's a small window where the callback can be called +// after the watcher is canceled. The caller needs to handle this case. +// +// TODO(easwars): Make this function a method on the xdsResolver type. +// Currently, there is a single call site for this function, and all arguments +// passed to it are fields of the xdsResolver type. +func watchService(c xdsclient.XDSClient, serviceName string, cb func(serviceUpdate, error), logger *grpclog.PrefixLogger) (cancel func()) { + w := &serviceUpdateWatcher{ + logger: logger, + c: c, + serviceName: serviceName, + serviceCb: cb, + } + w.ldsCancel = c.WatchListener(serviceName, w.handleLDSResp) + + return w.close +} + +// serviceUpdateWatcher handles LDS and RDS response, and calls the service +// callback at the right time. +type serviceUpdateWatcher struct { + logger *grpclog.PrefixLogger + c xdsclient.XDSClient + serviceName string + ldsCancel func() + serviceCb func(serviceUpdate, error) + lastUpdate serviceUpdate + + mu sync.Mutex + closed bool + rdsName string + rdsCancel func() +} + +func (w *serviceUpdateWatcher) handleLDSResp(update xdsresource.ListenerUpdate, err error) { + w.logger.Infof("received LDS update: %+v, err: %v", pretty.ToJSON(update), err) + w.mu.Lock() + defer w.mu.Unlock() + if w.closed { + return + } + if err != nil { + // We check the error type and do different things. For now, the only + // type we check is ResourceNotFound, which indicates the LDS resource + // was removed, and besides sending the error to callback, we also + // cancel the RDS watch. + if xdsresource.ErrType(err) == xdsresource.ErrorTypeResourceNotFound && w.rdsCancel != nil { + w.rdsCancel() + w.rdsName = "" + w.rdsCancel = nil + w.lastUpdate = serviceUpdate{} + } + // The other error cases still return early without canceling the + // existing RDS watch. + w.serviceCb(serviceUpdate{}, err) + return + } + + w.lastUpdate.ldsConfig = ldsConfig{ + maxStreamDuration: update.MaxStreamDuration, + httpFilterConfig: update.HTTPFilters, + } + + if update.InlineRouteConfig != nil { + // If there was an RDS watch, cancel it. + w.rdsName = "" + if w.rdsCancel != nil { + w.rdsCancel() + w.rdsCancel = nil + } + + // Handle the inline RDS update as if it's from an RDS watch. + w.applyRouteConfigUpdate(*update.InlineRouteConfig) + return + } + + // RDS name from update is not an empty string, need RDS to fetch the + // routes. + + if w.rdsName == update.RouteConfigName { + // If the new RouteConfigName is same as the previous, don't cancel and + // restart the RDS watch. + // + // If the route name did change, then we must wait until the first RDS + // update before reporting this LDS config. + if w.lastUpdate.virtualHost != nil { + // We want to send an update with the new fields from the new LDS + // (e.g. max stream duration), and old fields from the previous + // RDS. + // + // But note that this should only happen when virtual host is set, + // which means an RDS was received. + w.serviceCb(w.lastUpdate, nil) + } + return + } + w.rdsName = update.RouteConfigName + if w.rdsCancel != nil { + w.rdsCancel() + } + w.rdsCancel = w.c.WatchRouteConfig(update.RouteConfigName, w.handleRDSResp) +} + +func (w *serviceUpdateWatcher) applyRouteConfigUpdate(update xdsresource.RouteConfigUpdate) { + matchVh := xdsresource.FindBestMatchingVirtualHost(w.serviceName, update.VirtualHosts) + if matchVh == nil { + // No matching virtual host found. + w.serviceCb(serviceUpdate{}, fmt.Errorf("no matching virtual host found for %q", w.serviceName)) + return + } + + w.lastUpdate.virtualHost = matchVh + w.lastUpdate.clusterSpecifierPlugins = update.ClusterSpecifierPlugins + w.serviceCb(w.lastUpdate, nil) +} + +func (w *serviceUpdateWatcher) handleRDSResp(update xdsresource.RouteConfigUpdate, err error) { + w.logger.Infof("received RDS update: %+v, err: %v", pretty.ToJSON(update), err) + w.mu.Lock() + defer w.mu.Unlock() + if w.closed { + return + } + if w.rdsCancel == nil { + // This mean only the RDS watch is canceled, can happen if the LDS + // resource is removed. + return + } + if err != nil { + w.serviceCb(serviceUpdate{}, err) + return + } + w.applyRouteConfigUpdate(update) +} + +func (w *serviceUpdateWatcher) close() { + w.mu.Lock() + defer w.mu.Unlock() + w.closed = true + w.ldsCancel() + if w.rdsCancel != nil { + w.rdsCancel() + w.rdsCancel = nil + } +} diff --git a/xds/internal/resolver/watch_service_test.go b/xds/internal/resolver/watch_service_test.go new file mode 100644 index 000000000000..1a4b45bc8ad2 --- /dev/null +++ b/xds/internal/resolver/watch_service_test.go @@ -0,0 +1,390 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package resolver + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/xds/internal/testutils/fakeclient" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" + "google.golang.org/protobuf/proto" +) + +func (s) TestFindBestMatchingVirtualHost(t *testing.T) { + var ( + oneExactMatch = &xdsresource.VirtualHost{ + Domains: []string{"foo.bar.com"}, + } + oneSuffixMatch = &xdsresource.VirtualHost{ + Domains: []string{"*.bar.com"}, + } + onePrefixMatch = &xdsresource.VirtualHost{ + Domains: []string{"foo.bar.*"}, + } + oneUniversalMatch = &xdsresource.VirtualHost{ + Domains: []string{"*"}, + } + longExactMatch = &xdsresource.VirtualHost{ + Domains: []string{"v2.foo.bar.com"}, + } + multipleMatch = &xdsresource.VirtualHost{ + Domains: []string{"pi.foo.bar.com", "314.*", "*.159"}, + } + vhs = []*xdsresource.VirtualHost{oneExactMatch, oneSuffixMatch, onePrefixMatch, oneUniversalMatch, longExactMatch, multipleMatch} + ) + + tests := []struct { + name string + host string + vHosts []*xdsresource.VirtualHost + want *xdsresource.VirtualHost + }{ + {name: "exact-match", host: "foo.bar.com", vHosts: vhs, want: oneExactMatch}, + {name: "suffix-match", host: "123.bar.com", vHosts: vhs, want: oneSuffixMatch}, + {name: "prefix-match", host: "foo.bar.org", vHosts: vhs, want: onePrefixMatch}, + {name: "universal-match", host: "abc.123", vHosts: vhs, want: oneUniversalMatch}, + {name: "long-exact-match", host: "v2.foo.bar.com", vHosts: vhs, want: longExactMatch}, + // Matches suffix "*.bar.com" and exact "pi.foo.bar.com". Takes exact. + {name: "multiple-match-exact", host: "pi.foo.bar.com", vHosts: vhs, want: multipleMatch}, + // Matches suffix "*.159" and prefix "foo.bar.*". Takes suffix. + {name: "multiple-match-suffix", host: "foo.bar.159", vHosts: vhs, want: multipleMatch}, + // Matches suffix "*.bar.com" and prefix "314.*". Takes suffix. + {name: "multiple-match-prefix", host: "314.bar.com", vHosts: vhs, want: oneSuffixMatch}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := xdsresource.FindBestMatchingVirtualHost(tt.host, tt.vHosts); !cmp.Equal(got, tt.want, cmp.Comparer(proto.Equal)) { + t.Errorf("findBestMatchingxdsclient.VirtualHost() = %v, want %v", got, tt.want) + } + }) + } +} + +type serviceUpdateErr struct { + u serviceUpdate + err error +} + +func verifyServiceUpdate(ctx context.Context, updateCh *testutils.Channel, wantUpdate serviceUpdate) error { + u, err := updateCh.Receive(ctx) + if err != nil { + return fmt.Errorf("timeout when waiting for service update: %v", err) + } + gotUpdate := u.(serviceUpdateErr) + if gotUpdate.err != nil || !cmp.Equal(gotUpdate.u, wantUpdate, cmpopts.EquateEmpty(), cmp.AllowUnexported(serviceUpdate{}, ldsConfig{})) { + return fmt.Errorf("unexpected service update: (%v, %v), want: (%v, nil), diff (-want +got):\n%s", gotUpdate.u, gotUpdate.err, wantUpdate, cmp.Diff(gotUpdate.u, wantUpdate, cmpopts.EquateEmpty(), cmp.AllowUnexported(serviceUpdate{}, ldsConfig{}))) + } + return nil +} + +func newStringP(s string) *string { + return &s +} + +// TestServiceWatch covers the cases: +// - an update is received after a watch() +// - an update with routes received +func (s) TestServiceWatch(t *testing.T) { + serviceUpdateCh := testutils.NewChannel() + xdsC := fakeclient.NewClient() + cancelWatch := watchService(xdsC, targetStr, func(update serviceUpdate, err error) { + serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err}) + }, nil) + defer cancelWatch() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + waitForWatchListener(ctx, t, xdsC, targetStr) + xdsC.InvokeWatchListenerCallback(xdsresource.ListenerUpdate{RouteConfigName: routeStr}, nil) + waitForWatchRouteConfig(ctx, t, xdsC, routeStr) + + wantUpdate := serviceUpdate{virtualHost: &xdsresource.VirtualHost{Domains: []string{"target"}, Routes: []*xdsresource.Route{{Prefix: newStringP(""), WeightedClusters: map[string]xdsresource.WeightedCluster{cluster: {Weight: 1}}}}}} + xdsC.InvokeWatchRouteConfigCallback("", xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{ + { + Domains: []string{targetStr}, + Routes: []*xdsresource.Route{{Prefix: newStringP(""), WeightedClusters: map[string]xdsresource.WeightedCluster{cluster: {Weight: 1}}}}, + }, + }, + }, nil) + if err := verifyServiceUpdate(ctx, serviceUpdateCh, wantUpdate); err != nil { + t.Fatal(err) + } + + wantUpdate2 := serviceUpdate{virtualHost: &xdsresource.VirtualHost{Domains: []string{"target"}, + Routes: []*xdsresource.Route{{ + Path: newStringP(""), + WeightedClusters: map[string]xdsresource.WeightedCluster{cluster: {Weight: 1}}, + }}, + }} + xdsC.InvokeWatchRouteConfigCallback("", xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{ + { + Domains: []string{targetStr}, + Routes: []*xdsresource.Route{{Path: newStringP(""), WeightedClusters: map[string]xdsresource.WeightedCluster{cluster: {Weight: 1}}}}, + }, + { + // Another virtual host, with different domains. + Domains: []string{"random"}, + Routes: []*xdsresource.Route{{Prefix: newStringP(""), WeightedClusters: map[string]xdsresource.WeightedCluster{cluster: {Weight: 1}}}}, + }, + }, + }, nil) + if err := verifyServiceUpdate(ctx, serviceUpdateCh, wantUpdate2); err != nil { + t.Fatal(err) + } +} + +// TestServiceWatchLDSUpdate covers the case that after first LDS and first RDS +// response, the second LDS response trigger an new RDS watch, and an update of +// the old RDS watch doesn't trigger update to service callback. +func (s) TestServiceWatchLDSUpdate(t *testing.T) { + serviceUpdateCh := testutils.NewChannel() + xdsC := fakeclient.NewClient() + cancelWatch := watchService(xdsC, targetStr, func(update serviceUpdate, err error) { + serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err}) + }, nil) + defer cancelWatch() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + waitForWatchListener(ctx, t, xdsC, targetStr) + xdsC.InvokeWatchListenerCallback(xdsresource.ListenerUpdate{RouteConfigName: routeStr}, nil) + waitForWatchRouteConfig(ctx, t, xdsC, routeStr) + + wantUpdate := serviceUpdate{virtualHost: &xdsresource.VirtualHost{Domains: []string{"target"}, Routes: []*xdsresource.Route{{Prefix: newStringP(""), WeightedClusters: map[string]xdsresource.WeightedCluster{cluster: {Weight: 1}}}}}} + xdsC.InvokeWatchRouteConfigCallback("", xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{ + { + Domains: []string{targetStr}, + Routes: []*xdsresource.Route{{Prefix: newStringP(""), WeightedClusters: map[string]xdsresource.WeightedCluster{cluster: {Weight: 1}}}}, + }, + }, + }, nil) + if err := verifyServiceUpdate(ctx, serviceUpdateCh, wantUpdate); err != nil { + t.Fatal(err) + } + + // Another LDS update with a different RDS_name. + xdsC.InvokeWatchListenerCallback(xdsresource.ListenerUpdate{RouteConfigName: routeStr + "2"}, nil) + if _, err := xdsC.WaitForCancelRouteConfigWatch(ctx); err != nil { + t.Fatalf("wait for cancel route watch failed: %v, want nil", err) + } + waitForWatchRouteConfig(ctx, t, xdsC, routeStr+"2") + + // RDS update for the new name. + wantUpdate2 := serviceUpdate{virtualHost: &xdsresource.VirtualHost{Domains: []string{"target"}, Routes: []*xdsresource.Route{{Prefix: newStringP(""), WeightedClusters: map[string]xdsresource.WeightedCluster{cluster + "2": {Weight: 1}}}}}} + xdsC.InvokeWatchRouteConfigCallback(routeStr+"2", xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{ + { + Domains: []string{targetStr}, + Routes: []*xdsresource.Route{{Prefix: newStringP(""), WeightedClusters: map[string]xdsresource.WeightedCluster{cluster + "2": {Weight: 1}}}}, + }, + }, + }, nil) + if err := verifyServiceUpdate(ctx, serviceUpdateCh, wantUpdate2); err != nil { + t.Fatal(err) + } +} + +// TestServiceWatchLDSUpdate covers the case that after first LDS and first RDS +// response, the second LDS response includes a new MaxStreamDuration. It also +// verifies this is reported in subsequent RDS updates. +func (s) TestServiceWatchLDSUpdateMaxStreamDuration(t *testing.T) { + serviceUpdateCh := testutils.NewChannel() + xdsC := fakeclient.NewClient() + cancelWatch := watchService(xdsC, targetStr, func(update serviceUpdate, err error) { + serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err}) + }, nil) + defer cancelWatch() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + waitForWatchListener(ctx, t, xdsC, targetStr) + xdsC.InvokeWatchListenerCallback(xdsresource.ListenerUpdate{RouteConfigName: routeStr, MaxStreamDuration: time.Second}, nil) + waitForWatchRouteConfig(ctx, t, xdsC, routeStr) + + wantUpdate := serviceUpdate{virtualHost: &xdsresource.VirtualHost{Domains: []string{"target"}, Routes: []*xdsresource.Route{{ + Prefix: newStringP(""), + WeightedClusters: map[string]xdsresource.WeightedCluster{cluster: {Weight: 1}}}}}, + ldsConfig: ldsConfig{maxStreamDuration: time.Second}, + } + xdsC.InvokeWatchRouteConfigCallback("", xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{ + { + Domains: []string{targetStr}, + Routes: []*xdsresource.Route{{Prefix: newStringP(""), WeightedClusters: map[string]xdsresource.WeightedCluster{cluster: {Weight: 1}}}}, + }, + }, + }, nil) + if err := verifyServiceUpdate(ctx, serviceUpdateCh, wantUpdate); err != nil { + t.Fatal(err) + } + + // Another LDS update with the same RDS_name but different MaxStreamDuration (zero in this case). + wantUpdate2 := serviceUpdate{virtualHost: &xdsresource.VirtualHost{Domains: []string{"target"}, Routes: []*xdsresource.Route{{Prefix: newStringP(""), WeightedClusters: map[string]xdsresource.WeightedCluster{cluster: {Weight: 1}}}}}} + xdsC.InvokeWatchListenerCallback(xdsresource.ListenerUpdate{RouteConfigName: routeStr}, nil) + if err := verifyServiceUpdate(ctx, serviceUpdateCh, wantUpdate2); err != nil { + t.Fatal(err) + } + + // RDS update. + wantUpdate3 := serviceUpdate{virtualHost: &xdsresource.VirtualHost{Domains: []string{"target"}, Routes: []*xdsresource.Route{{ + Prefix: newStringP(""), + WeightedClusters: map[string]xdsresource.WeightedCluster{cluster + "2": {Weight: 1}}}}, + }} + xdsC.InvokeWatchRouteConfigCallback("", xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{ + { + Domains: []string{targetStr}, + Routes: []*xdsresource.Route{{Prefix: newStringP(""), WeightedClusters: map[string]xdsresource.WeightedCluster{cluster + "2": {Weight: 1}}}}, + }, + }, + }, nil) + if err := verifyServiceUpdate(ctx, serviceUpdateCh, wantUpdate3); err != nil { + t.Fatal(err) + } +} + +// TestServiceNotCancelRDSOnSameLDSUpdate covers the case that if the second LDS +// update contains the same RDS name as the previous, the RDS watch isn't +// canceled and restarted. +func (s) TestServiceNotCancelRDSOnSameLDSUpdate(t *testing.T) { + serviceUpdateCh := testutils.NewChannel() + xdsC := fakeclient.NewClient() + cancelWatch := watchService(xdsC, targetStr, func(update serviceUpdate, err error) { + serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err}) + }, nil) + defer cancelWatch() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + waitForWatchListener(ctx, t, xdsC, targetStr) + xdsC.InvokeWatchListenerCallback(xdsresource.ListenerUpdate{RouteConfigName: routeStr}, nil) + waitForWatchRouteConfig(ctx, t, xdsC, routeStr) + + wantUpdate := serviceUpdate{virtualHost: &xdsresource.VirtualHost{Domains: []string{"target"}, Routes: []*xdsresource.Route{{ + Prefix: newStringP(""), + WeightedClusters: map[string]xdsresource.WeightedCluster{cluster: {Weight: 1}}}}, + }} + xdsC.InvokeWatchRouteConfigCallback("", xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{ + { + Domains: []string{targetStr}, + Routes: []*xdsresource.Route{{Prefix: newStringP(""), WeightedClusters: map[string]xdsresource.WeightedCluster{cluster: {Weight: 1}}}}, + }, + }, + }, nil) + + if err := verifyServiceUpdate(ctx, serviceUpdateCh, wantUpdate); err != nil { + t.Fatal(err) + } + + // Another LDS update with a the same RDS_name. + xdsC.InvokeWatchListenerCallback(xdsresource.ListenerUpdate{RouteConfigName: routeStr}, nil) + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := xdsC.WaitForCancelRouteConfigWatch(sCtx); err != context.DeadlineExceeded { + t.Fatalf("wait for cancel route watch failed: %v, want nil", err) + } +} + +// TestServiceWatchInlineRDS covers the cases switching between: +// - LDS update contains RDS name to watch +// - LDS update contains inline RDS resource +func (s) TestServiceWatchInlineRDS(t *testing.T) { + serviceUpdateCh := testutils.NewChannel() + xdsC := fakeclient.NewClient() + cancelWatch := watchService(xdsC, targetStr, func(update serviceUpdate, err error) { + serviceUpdateCh.Send(serviceUpdateErr{u: update, err: err}) + }, nil) + defer cancelWatch() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // First LDS update is LDS with RDS name to watch. + waitForWatchListener(ctx, t, xdsC, targetStr) + xdsC.InvokeWatchListenerCallback(xdsresource.ListenerUpdate{RouteConfigName: routeStr}, nil) + waitForWatchRouteConfig(ctx, t, xdsC, routeStr) + wantUpdate := serviceUpdate{virtualHost: &xdsresource.VirtualHost{Domains: []string{"target"}, Routes: []*xdsresource.Route{{Prefix: newStringP(""), WeightedClusters: map[string]xdsresource.WeightedCluster{cluster: {Weight: 1}}}}}} + xdsC.InvokeWatchRouteConfigCallback("", xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{ + { + Domains: []string{targetStr}, + Routes: []*xdsresource.Route{{Prefix: newStringP(""), WeightedClusters: map[string]xdsresource.WeightedCluster{cluster: {Weight: 1}}}}, + }, + }, + }, nil) + if err := verifyServiceUpdate(ctx, serviceUpdateCh, wantUpdate); err != nil { + t.Fatal(err) + } + + // Switch LDS resp to a LDS with inline RDS resource + wantVirtualHosts2 := &xdsresource.VirtualHost{Domains: []string{"target"}, + Routes: []*xdsresource.Route{{ + Path: newStringP(""), + WeightedClusters: map[string]xdsresource.WeightedCluster{cluster: {Weight: 1}}, + }}, + } + wantUpdate2 := serviceUpdate{virtualHost: wantVirtualHosts2} + xdsC.InvokeWatchListenerCallback(xdsresource.ListenerUpdate{InlineRouteConfig: &xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{wantVirtualHosts2}, + }}, nil) + // This inline RDS resource should cause the RDS watch to be canceled. + if _, err := xdsC.WaitForCancelRouteConfigWatch(ctx); err != nil { + t.Fatalf("wait for cancel route watch failed: %v, want nil", err) + } + if err := verifyServiceUpdate(ctx, serviceUpdateCh, wantUpdate2); err != nil { + t.Fatal(err) + } + + // Switch LDS update back to LDS with RDS name to watch. + xdsC.InvokeWatchListenerCallback(xdsresource.ListenerUpdate{RouteConfigName: routeStr}, nil) + waitForWatchRouteConfig(ctx, t, xdsC, routeStr) + xdsC.InvokeWatchRouteConfigCallback("", xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{ + { + Domains: []string{targetStr}, + Routes: []*xdsresource.Route{{Prefix: newStringP(""), WeightedClusters: map[string]xdsresource.WeightedCluster{cluster: {Weight: 1}}}}, + }, + }, + }, nil) + if err := verifyServiceUpdate(ctx, serviceUpdateCh, wantUpdate); err != nil { + t.Fatal(err) + } + + // Switch LDS resp to a LDS with inline RDS resource again. + xdsC.InvokeWatchListenerCallback(xdsresource.ListenerUpdate{InlineRouteConfig: &xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{wantVirtualHosts2}, + }}, nil) + // This inline RDS resource should cause the RDS watch to be canceled. + if _, err := xdsC.WaitForCancelRouteConfigWatch(ctx); err != nil { + t.Fatalf("wait for cancel route watch failed: %v, want nil", err) + } + if err := verifyServiceUpdate(ctx, serviceUpdateCh, wantUpdate2); err != nil { + t.Fatal(err) + } +} diff --git a/xds/internal/resolver/xds_resolver.go b/xds/internal/resolver/xds_resolver.go index cdd103ef7dc3..09b3356301db 100644 --- a/xds/internal/resolver/xds_resolver.go +++ b/xds/internal/resolver/xds_resolver.go @@ -20,123 +20,147 @@ package resolver import ( - "context" + "errors" "fmt" + "strings" - "google.golang.org/grpc" - "google.golang.org/grpc/attributes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/internal" "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/grpcrand" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/pretty" + iresolver "google.golang.org/grpc/internal/resolver" "google.golang.org/grpc/resolver" - - xdsinternal "google.golang.org/grpc/xds/internal" - xdsclient "google.golang.org/grpc/xds/internal/client" - "google.golang.org/grpc/xds/internal/client/bootstrap" + "google.golang.org/grpc/xds/internal/xdsclient" + "google.golang.org/grpc/xds/internal/xdsclient/bootstrap" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" ) const xdsScheme = "xds" +// newBuilderForTesting creates a new xds resolver builder using a specific xds +// bootstrap config, so tests can use multiple xds clients in different +// ClientConns at the same time. +func newBuilderForTesting(config []byte) (resolver.Builder, error) { + return &xdsResolverBuilder{ + newXDSClient: func() (xdsclient.XDSClient, func(), error) { + return xdsclient.NewWithBootstrapContentsForTesting(config) + }, + }, nil +} + // For overriding in unittests. -var ( - newXDSClient = func(opts xdsclient.Options) (xdsClientInterface, error) { - return xdsclient.New(opts) - } - newXDSConfig = bootstrap.NewConfig -) +var newXDSClient = func() (xdsclient.XDSClient, func(), error) { return xdsclient.New() } func init() { resolver.Register(&xdsResolverBuilder{}) + internal.NewXDSResolverWithConfigForTesting = newBuilderForTesting } -type xdsResolverBuilder struct{} +type xdsResolverBuilder struct { + newXDSClient func() (xdsclient.XDSClient, func(), error) +} // Build helps implement the resolver.Builder interface. // // The xds bootstrap process is performed (and a new xds client is built) every // time an xds resolver is built. -func (b *xdsResolverBuilder) Build(t resolver.Target, cc resolver.ClientConn, rbo resolver.BuildOptions) (resolver.Resolver, error) { - config, err := newXDSConfig() - if err != nil { - return nil, fmt.Errorf("xds: failed to read bootstrap file: %v", err) +func (b *xdsResolverBuilder) Build(target resolver.Target, cc resolver.ClientConn, opts resolver.BuildOptions) (_ resolver.Resolver, retErr error) { + r := &xdsResolver{ + cc: cc, + closed: grpcsync.NewEvent(), + updateCh: make(chan suWithError, 1), + activeClusters: make(map[string]*clusterInfo), + channelID: grpcrand.Uint64(), } + defer func() { + if retErr != nil { + r.Close() + } + }() + r.logger = prefixLogger(r) + r.logger.Infof("Creating resolver for target: %+v", target) - r := &xdsResolver{ - target: t, - cc: cc, - updateCh: make(chan suWithError, 1), + newXDSClient := newXDSClient + if b.newXDSClient != nil { + newXDSClient = b.newXDSClient } - r.logger = prefixLogger((r)) - r.logger.Infof("Creating resolver for target: %+v", t) - if config.Creds == nil { - // TODO: Once we start supporting a mechanism to register credential - // types, a failure to find the credential type mentioned in the - // bootstrap file should result in a failure, and not in using - // credentials from the parent channel (passed through the - // resolver.BuildOptions). - config.Creds = r.defaultDialCreds(config.BalancerName, rbo) + client, close, err := newXDSClient() + if err != nil { + return nil, fmt.Errorf("xds: failed to create xds-client: %v", err) + } + r.xdsClient = client + r.xdsClientClose = close + bootstrapConfig := client.BootstrapConfig() + if bootstrapConfig == nil { + return nil, errors.New("bootstrap configuration is empty") } - var dopts []grpc.DialOption - if rbo.Dialer != nil { - dopts = []grpc.DialOption{grpc.WithContextDialer(rbo.Dialer)} + // If xds credentials were specified by the user, but bootstrap configs do + // not contain any certificate provider configuration, it is better to fail + // right now rather than failing when attempting to create certificate + // providers after receiving an CDS response with security configuration. + var creds credentials.TransportCredentials + switch { + case opts.DialCreds != nil: + creds = opts.DialCreds + case opts.CredsBundle != nil: + creds = opts.CredsBundle.TransportCredentials() + } + if xc, ok := creds.(interface{ UsesXDS() bool }); ok && xc.UsesXDS() { + if len(bootstrapConfig.CertProviderConfigs) == 0 { + return nil, errors.New("xds: xdsCreds specified but certificate_providers config missing in bootstrap file") + } } - client, err := newXDSClient(xdsclient.Options{Config: *config, DialOpts: dopts, TargetName: t.Endpoint}) - if err != nil { - return nil, fmt.Errorf("xds: failed to create xds-client: %v", err) + // Find the client listener template to use from the bootstrap config: + // - If authority is not set in the target, use the top level template + // - If authority is set, use the template from the authority map. + template := bootstrapConfig.ClientDefaultListenerResourceNameTemplate + if authority := target.URL.Host; authority != "" { + a := bootstrapConfig.Authorities[authority] + if a == nil { + return nil, fmt.Errorf("xds: authority %q is not found in the bootstrap file", authority) + } + if a.ClientListenerResourceNameTemplate != "" { + // This check will never be false, because + // ClientListenerResourceNameTemplate is required to start with + // xdstp://, and has a default value (not an empty string) if unset. + template = a.ClientListenerResourceNameTemplate + } } - r.client = client - r.ctx, r.cancelCtx = context.WithCancel(context.Background()) - cancelWatch := r.client.WatchService(r.target.Endpoint, r.handleServiceUpdate) - r.logger.Infof("Watch started on resource name %v with xds-client %p", r.target.Endpoint, r.client) + endpoint := target.URL.Path + if endpoint == "" { + endpoint = target.URL.Opaque + } + endpoint = strings.TrimPrefix(endpoint, "/") + r.ldsResourceName = bootstrap.PopulateResourceTemplate(template, endpoint) + + // Register a watch on the xdsClient for the resource name determined above. + cancelWatch := watchService(r.xdsClient, r.ldsResourceName, r.handleServiceUpdate, r.logger) + r.logger.Infof("Watch started on resource name %v with xds-client %p", r.ldsResourceName, r.xdsClient) r.cancelWatch = func() { cancelWatch() - r.logger.Infof("Watch cancel on resource name %v with xds-client %p", r.target.Endpoint, r.client) + r.logger.Infof("Watch cancel on resource name %v with xds-client %p", r.ldsResourceName, r.xdsClient) } go r.run() return r, nil } -// defaultDialCreds builds a DialOption containing the credentials to be used -// while talking to the xDS server (this is done only if the xds bootstrap -// process does not return any credentials to use). If the parent channel -// contains DialCreds, we use it as is. If it contains a CredsBundle, we use -// just the transport credentials from the bundle. If we don't find any -// credentials on the parent channel, we resort to using an insecure channel. -func (r *xdsResolver) defaultDialCreds(balancerName string, rbo resolver.BuildOptions) grpc.DialOption { - switch { - case rbo.DialCreds != nil: - if err := rbo.DialCreds.OverrideServerName(balancerName); err != nil { - r.logger.Errorf("Failed to override server name in credentials: %v, using Insecure", err) - return grpc.WithInsecure() - } - return grpc.WithTransportCredentials(rbo.DialCreds) - case rbo.CredsBundle != nil: - return grpc.WithTransportCredentials(rbo.CredsBundle.TransportCredentials()) - default: - r.logger.Warningf("No credentials available, using Insecure") - return grpc.WithInsecure() - } -} - // Name helps implement the resolver.Builder interface. func (*xdsResolverBuilder) Scheme() string { return xdsScheme } -// xdsClientInterface contains methods from xdsClient.Client which are used by -// the resolver. This will be faked out in unittests. -type xdsClientInterface interface { - WatchService(string, func(xdsclient.ServiceUpdate, error)) func() - Close() -} - // suWithError wraps the ServiceUpdate and error received through a watch API // callback, so that it can pushed onto the update channel as a single entity. type suWithError struct { - su xdsclient.ServiceUpdate - err error + su serviceUpdate + emptyUpdate bool + err error } // xdsResolver implements the resolver.Resolver interface. @@ -145,30 +169,65 @@ type suWithError struct { // (which performs LDS/RDS queries for the same), and passes the received // updates to the ClientConn. type xdsResolver struct { - ctx context.Context - cancelCtx context.CancelFunc - target resolver.Target - cc resolver.ClientConn - - logger *grpclog.PrefixLogger + cc resolver.ClientConn + closed *grpcsync.Event + logger *grpclog.PrefixLogger + ldsResourceName string // The underlying xdsClient which performs all xDS requests and responses. - client xdsClientInterface + xdsClient xdsclient.XDSClient + xdsClientClose func() // A channel for the watch API callback to write service updates on to. The // updates are read by the run goroutine and passed on to the ClientConn. updateCh chan suWithError // cancelWatch is the function to cancel the watcher. cancelWatch func() - // actions is a map from hash of weighted cluster, to the weighted cluster - // map, and it's assigned name. E.g. - // "A40_B60_": {{A:40, B:60}, "A_B_", "A_B_0"} - // "A30_B70_": {{A:30, B:70}, "A_B_", "A_B_1"} - // "B90_C10_": {{B:90, C:10}, "B_C_", "B_C_0"} - actions map[string]actionWithAssignedName - // usedActionNameRandomNumber contains random numbers that have been used in - // assigned names, to avoid collision. - usedActionNameRandomNumber map[int64]bool + // activeClusters is a map from cluster name to a ref count. Only read or + // written during a service update (synchronous). + activeClusters map[string]*clusterInfo + + curConfigSelector *configSelector + + // A random number which uniquely identifies the channel which owns this + // resolver. + channelID uint64 +} + +// sendNewServiceConfig prunes active clusters, generates a new service config +// based on the current set of active clusters, and sends an update to the +// channel with that service config and the provided config selector. Returns +// false if an error occurs while generating the service config and the update +// cannot be sent. +func (r *xdsResolver) sendNewServiceConfig(cs *configSelector) bool { + // Delete entries from r.activeClusters with zero references; + // otherwise serviceConfigJSON will generate a config including + // them. + r.pruneActiveClusters() + + if cs == nil && len(r.activeClusters) == 0 { + // There are no clusters and we are sending a failing configSelector. + // Send an empty config, which picks pick-first, with no address, and + // puts the ClientConn into transient failure. + r.cc.UpdateState(resolver.State{ServiceConfig: r.cc.ParseServiceConfig("{}")}) + return true + } + + sc, err := serviceConfigJSON(r.activeClusters) + if err != nil { + // JSON marshal error; should never happen. + r.logger.Errorf("%v", err) + r.cc.ReportError(err) + return false + } + r.logger.Infof("Received update on resource %v from xds-client %p, generated service config: %v", r.ldsResourceName, r.xdsClient, pretty.FormatJSON(sc)) + + // Send the update to the ClientConn. + state := iresolver.SetConfigSelector(resolver.State{ + ServiceConfig: r.cc.ParseServiceConfig(string(sc)), + }, cs) + r.cc.UpdateState(xdsclient.SetClient(state, r.xdsClient)) + return true } // run is a long running goroutine which blocks on receiving service updates @@ -176,36 +235,55 @@ type xdsResolver struct { func (r *xdsResolver) run() { for { select { - case <-r.ctx.Done(): + case <-r.closed.Done(): + return case update := <-r.updateCh: if update.err != nil { - r.logger.Warningf("Watch error on resource %v from xds-client %p, %v", r.target.Endpoint, r.client, update.err) - if xdsclient.ErrType(update.err) == xdsclient.ErrorTypeResourceNotFound { - // If error is resource-not-found, it means the LDS resource - // was removed. Send an empty service config, which picks - // pick-first, with no address, and puts the ClientConn into - // transient failure.. - r.cc.UpdateState(resolver.State{ - ServiceConfig: r.cc.ParseServiceConfig("{}"), - }) + r.logger.Warningf("Watch error on resource %v from xds-client %p, %v", r.ldsResourceName, r.xdsClient, update.err) + if xdsresource.ErrType(update.err) == xdsresource.ErrorTypeResourceNotFound { + // If error is resource-not-found, it means the LDS + // resource was removed. Ultimately send an empty service + // config, which picks pick-first, with no address, and + // puts the ClientConn into transient failure. Before we + // can do that, we may need to send a normal service config + // along with an erroring (nil) config selector. + r.sendNewServiceConfig(nil) + // Stop and dereference the active config selector, if one exists. + r.curConfigSelector.stop() + r.curConfigSelector = nil continue } // Send error to ClientConn, and balancers, if error is not - // resource not found. + // resource not found. No need to update resolver state if we + // can keep using the old config. r.cc.ReportError(update.err) continue } - sc, err := r.serviceUpdateToJSON(update.su) + if update.emptyUpdate { + r.sendNewServiceConfig(r.curConfigSelector) + continue + } + + // Create the config selector for this update. + cs, err := r.newConfigSelector(update.su) if err != nil { - r.logger.Warningf("failed to convert update to service config: %v", err) + r.logger.Warningf("Error parsing update on resource %v from xds-client %p: %v", r.ldsResourceName, r.xdsClient, err) r.cc.ReportError(err) continue } - r.logger.Infof("Received update on resource %v from xds-client %p, generated service config: %v", r.target.Endpoint, r.client, sc) - r.cc.UpdateState(resolver.State{ - ServiceConfig: r.cc.ParseServiceConfig(sc), - Attributes: attributes.New(xdsinternal.XDSClientID, r.client), - }) + + if !r.sendNewServiceConfig(cs) { + // JSON error creating the service config (unexpected); erase + // this config selector and ignore this update, continuing with + // the previous config selector. + cs.stop() + continue + } + + // Decrement references to the old config selector and assign the + // new one as the current one. + r.curConfigSelector.stop() + r.curConfigSelector = cs } } } @@ -213,12 +291,17 @@ func (r *xdsResolver) run() { // handleServiceUpdate is the callback which handles service updates. It writes // the received update to the update channel, which is picked by the run // goroutine. -func (r *xdsResolver) handleServiceUpdate(su xdsclient.ServiceUpdate, err error) { - if r.ctx.Err() != nil { +func (r *xdsResolver) handleServiceUpdate(su serviceUpdate, err error) { + if r.closed.HasFired() { // Do not pass updates to the ClientConn once the resolver is closed. return } - r.updateCh <- suWithError{su, err} + // Remove any existing entry in updateCh and replace with the new one. + select { + case <-r.updateCh: + default: + } + r.updateCh <- suWithError{su: su, err: err} } // ResolveNow is a no-op at this point. @@ -226,8 +309,15 @@ func (*xdsResolver) ResolveNow(o resolver.ResolveNowOptions) {} // Close closes the resolver, and also closes the underlying xdsClient. func (r *xdsResolver) Close() { - r.cancelWatch() - r.client.Close() - r.cancelCtx() + // Note that Close needs to check for nils even if some of them are always + // set in the constructor. This is because the constructor defers Close() in + // error cases, and the fields might not be set when the error happens. + if r.cancelWatch != nil { + r.cancelWatch() + } + if r.xdsClientClose != nil { + r.xdsClientClose() + } + r.closed.Fire() r.logger.Infof("Shutdown") } diff --git a/xds/internal/resolver/xds_resolver_test.go b/xds/internal/resolver/xds_resolver_test.go index 33f2e2695a0c..4e0782646f4c 100644 --- a/xds/internal/resolver/xds_resolver_test.go +++ b/xds/internal/resolver/xds_resolver_test.go @@ -21,66 +21,112 @@ package resolver import ( "context" "errors" - "fmt" - "net" + "net/url" + "reflect" + "strings" "testing" + "time" + xxhash "github.com/cespare/xxhash/v2" + "github.com/envoyproxy/go-control-plane/pkg/wellknown" "github.com/google/go-cmp/cmp" - "google.golang.org/grpc" + "github.com/google/uuid" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + xdscreds "google.golang.org/grpc/credentials/xds" "google.golang.org/grpc/internal" - "google.golang.org/grpc/internal/grpcrand" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/grpctest" + iresolver "google.golang.org/grpc/internal/resolver" + "google.golang.org/grpc/internal/testutils" + xdsbootstrap "google.golang.org/grpc/internal/testutils/xds/bootstrap" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/wrr" + "google.golang.org/grpc/metadata" "google.golang.org/grpc/resolver" "google.golang.org/grpc/serviceconfig" - xdsinternal "google.golang.org/grpc/xds/internal" - _ "google.golang.org/grpc/xds/internal/balancer/cdsbalancer" // To parse LB config - "google.golang.org/grpc/xds/internal/client" - xdsclient "google.golang.org/grpc/xds/internal/client" - "google.golang.org/grpc/xds/internal/client/bootstrap" - "google.golang.org/grpc/xds/internal/testutils" + "google.golang.org/grpc/status" + "google.golang.org/grpc/xds/internal/balancer/clustermanager" + "google.golang.org/grpc/xds/internal/balancer/ringhash" + "google.golang.org/grpc/xds/internal/httpfilter" + "google.golang.org/grpc/xds/internal/httpfilter/router" + xdstestutils "google.golang.org/grpc/xds/internal/testutils" "google.golang.org/grpc/xds/internal/testutils/fakeclient" + "google.golang.org/grpc/xds/internal/xdsclient" + "google.golang.org/grpc/xds/internal/xdsclient/bootstrap" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource/version" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/wrapperspb" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3routerpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3" + v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + + _ "google.golang.org/grpc/xds/internal/balancer/cdsbalancer" // To parse LB config ) const ( - targetStr = "target" - cluster = "cluster" - balancerName = "dummyBalancer" + targetStr = "target" + routeStr = "route" + cluster = "cluster" + defaultTestTimeout = 10 * time.Second + defaultTestShortTimeout = 100 * time.Microsecond ) var ( - validConfig = bootstrap.Config{ - BalancerName: balancerName, - Creds: grpc.WithInsecure(), - NodeProto: testutils.EmptyNodeProtoV2, - } - target = resolver.Target{Endpoint: targetStr} + target = resolver.Target{URL: *testutils.MustParseURL("xds:///" + targetStr)} + + routerHTTPFilter = httpfilter.Get(router.TypeURL) + routerConfig, _ = routerHTTPFilter.ParseFilterConfig(testutils.MarshalAny(&v3routerpb.Router{})) + routerFilter = xdsresource.HTTPFilter{Name: "rtr", Filter: routerHTTPFilter, Config: routerConfig} ) -func TestRegister(t *testing.T) { - b := resolver.Get(xdsScheme) - if b == nil { +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func (s) TestRegister(t *testing.T) { + if resolver.Get(xdsScheme) == nil { t.Errorf("scheme %v is not registered", xdsScheme) } } -// testClientConn is a fake implemetation of resolver.ClientConn. All is does -// is to store the state received from the resolver locally and signal that -// event through a channel. +// testClientConn is a fake implemetation of resolver.ClientConn that pushes +// state updates and errors returned by the resolver on to channels for +// consumption by tests. type testClientConn struct { resolver.ClientConn stateCh *testutils.Channel errorCh *testutils.Channel } -func (t *testClientConn) UpdateState(s resolver.State) { +func (t *testClientConn) UpdateState(s resolver.State) error { + // Tests should ideally consume all state updates, and if one happens + // unexpectedly, tests should catch it. Hence using `Send()` here. t.stateCh.Send(s) + return nil } func (t *testClientConn) ReportError(err error) { - t.errorCh.Send(err) + // When used with a go-control-plane management server that continuously + // resends resources which are NACKed by the xDS client, using a `Replace()` + // here simplifies tests which will have access to the most recently + // received error. + t.errorCh.Replace(err) } func (t *testClientConn) ParseServiceConfig(jsonSC string) *serviceconfig.ParseResult { - return internal.ParseServiceConfigForTesting.(func(string) *serviceconfig.ParseResult)(jsonSC) + return internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(jsonSC) } func newTestClientConn() *testClientConn { @@ -90,108 +136,100 @@ func newTestClientConn() *testClientConn { } } -func getXDSClientMakerFunc(wantOpts xdsclient.Options) func(xdsclient.Options) (xdsClientInterface, error) { - return func(gotOpts xdsclient.Options) (xdsClientInterface, error) { - if gotOpts.Config.BalancerName != wantOpts.Config.BalancerName { - return nil, fmt.Errorf("got balancerName: %s, want: %s", gotOpts.Config.BalancerName, wantOpts.Config.BalancerName) - } - // We cannot compare two DialOption objects to see if they are equal - // because each of these is a function pointer. So, the only thing we - // can do here is to check if the got option is nil or not based on - // what the want option is. We should be able to do extensive - // credential testing in e2e tests. - if (gotOpts.Config.Creds != nil) != (wantOpts.Config.Creds != nil) { - return nil, fmt.Errorf("got len(creds): %s, want: %s", gotOpts.Config.Creds, wantOpts.Config.Creds) - } - if len(gotOpts.DialOpts) != len(wantOpts.DialOpts) { - return nil, fmt.Errorf("got len(DialOpts): %v, want: %v", len(gotOpts.DialOpts), len(wantOpts.DialOpts)) - } - return fakeclient.NewClient(), nil +// TestResolverBuilder_ClientCreationFails tests the case where xDS client +// creation fails, and verifies that xDS resolver build fails as well. +func (s) TestResolverBuilder_ClientCreationFails(t *testing.T) { + // Override xDS client creation function and return an error. + origNewClient := newXDSClient + newXDSClient = func() (xdsclient.XDSClient, func(), error) { + return nil, nil, errors.New("failed to create xDS client") } -} + defer func() { + newXDSClient = origNewClient + }() -func errorDialer(_ context.Context, _ string) (net.Conn, error) { - return nil, errors.New("dial error") + // Build an xDS resolver and expect it to fail. + builder := resolver.Get(xdsScheme) + if builder == nil { + t.Fatalf("resolver.Get(%v) returned nil", xdsScheme) + } + if _, err := builder.Build(target, newTestClientConn(), resolver.BuildOptions{}); err == nil { + t.Fatalf("builder.Build(%v) succeeded when expected to fail", target) + } } -// TestResolverBuilder tests the xdsResolverBuilder's Build method with -// different parameters. -func TestResolverBuilder(t *testing.T) { +// TestResolverBuilder_DifferentBootstrapConfigs tests the resolver builder's +// Build() method with different xDS bootstrap configurations. +func (s) TestResolverBuilder_DifferentBootstrapConfigs(t *testing.T) { tests := []struct { - name string - rbo resolver.BuildOptions - config bootstrap.Config - xdsClientFunc func(xdsclient.Options) (xdsClientInterface, error) - wantErr bool + name string + bootstrapCfg *bootstrap.Config // Empty top-level xDS server config, will be set by test logic. + target resolver.Target + buildOpts resolver.BuildOptions + wantErr string }{ { - name: "empty-config", - rbo: resolver.BuildOptions{}, - config: bootstrap.Config{}, - wantErr: true, + name: "good", + bootstrapCfg: &bootstrap.Config{}, + target: target, }, { - name: "no-balancer-name-in-config", - rbo: resolver.BuildOptions{}, - config: bootstrap.Config{ - Creds: grpc.WithInsecure(), - NodeProto: testutils.EmptyNodeProtoV2, + name: "authority not defined in bootstrap", + bootstrapCfg: &bootstrap.Config{ + ClientDefaultListenerResourceNameTemplate: "%s", + Authorities: map[string]*bootstrap.Authority{ + "test-authority": { + ClientListenerResourceNameTemplate: "xdstp://test-authority/%s", + }, + }, }, - wantErr: true, - }, - { - name: "no-creds-in-config", - rbo: resolver.BuildOptions{}, - config: bootstrap.Config{ - BalancerName: balancerName, - NodeProto: testutils.EmptyNodeProtoV2, + target: resolver.Target{ + URL: url.URL{ + Host: "non-existing-authority", + Path: "/" + targetStr, + }, }, - xdsClientFunc: getXDSClientMakerFunc(xdsclient.Options{Config: validConfig}), - wantErr: false, - }, - { - name: "error-dialer-in-rbo", - rbo: resolver.BuildOptions{Dialer: errorDialer}, - config: validConfig, - xdsClientFunc: getXDSClientMakerFunc(xdsclient.Options{ - Config: validConfig, - DialOpts: []grpc.DialOption{grpc.WithContextDialer(errorDialer)}, - }), - wantErr: false, + wantErr: `authority "non-existing-authority" is not found in the bootstrap file`, }, { - name: "simple-good", - rbo: resolver.BuildOptions{}, - config: validConfig, - xdsClientFunc: getXDSClientMakerFunc(xdsclient.Options{Config: validConfig}), - wantErr: false, - }, - { - name: "newXDSClient-throws-error", - rbo: resolver.BuildOptions{}, - config: validConfig, - xdsClientFunc: func(_ xdsclient.Options) (xdsClientInterface, error) { - return nil, errors.New("newXDSClient-throws-error") + name: "xDS creds specified without certificate providers in bootstrap", + bootstrapCfg: &bootstrap.Config{}, + target: target, + buildOpts: resolver.BuildOptions{ + DialCreds: func() credentials.TransportCredentials { + creds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()}) + if err != nil { + t.Fatalf("xds.NewClientCredentials() failed: %v", err) + } + return creds + }(), }, - wantErr: true, + wantErr: `xdsCreds specified but certificate_providers config missing in bootstrap file`, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - // Fake out the bootstrap process by providing our own config. - oldConfigMaker := newXDSConfig - newXDSConfig = func() (*bootstrap.Config, error) { - if test.config.BalancerName == "" { - return nil, fmt.Errorf("no balancer name found in config") - } - return &test.config, nil + mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) + if err != nil { + t.Fatalf("Starting xDS management server: %v", err) + } + defer mgmtServer.Stop() + + // Add top-level xDS server config corresponding to the above + // management server. + test.bootstrapCfg.XDSServer = xdstestutils.ServerConfigForAddress(t, mgmtServer.Address) + + // Override xDS client creation to use bootstrap configuration + // specified by the test. + origNewClient := newXDSClient + newXDSClient = func() (xdsclient.XDSClient, func(), error) { + // The watch timeout and idle authority timeout values passed to + // NewWithConfigForTesing() are immaterial for this test, as we + // are only testing the resolver build functionality. + return xdsclient.NewWithConfigForTesting(test.bootstrapCfg, defaultTestTimeout, defaultTestTimeout) } - // Fake out the xdsClient creation process by providing a fake. - oldClientMaker := newXDSClient - newXDSClient = test.xdsClientFunc defer func() { - newXDSConfig = oldConfigMaker - newXDSClient = oldClientMaker + newXDSClient = origNewClient }() builder := resolver.Get(xdsScheme) @@ -199,8 +237,11 @@ func TestResolverBuilder(t *testing.T) { t.Fatalf("resolver.Get(%v) returned nil", xdsScheme) } - r, err := builder.Build(target, newTestClientConn(), test.rbo) - if (err != nil) != test.wantErr { + r, err := builder.Build(test.target, newTestClientConn(), test.buildOpts) + if gotErr, wantErr := err != nil, test.wantErr != ""; gotErr != wantErr { + t.Fatalf("builder.Build(%v) returned err: %v, wantErr: %v", target, err, test.wantErr) + } + if test.wantErr != "" && !strings.Contains(err.Error(), test.wantErr) { t.Fatalf("builder.Build(%v) returned err: %v, wantErr: %v", target, err, test.wantErr) } if err != nil { @@ -213,42 +254,55 @@ func TestResolverBuilder(t *testing.T) { } type setupOpts struct { - config *bootstrap.Config - xdsClientFunc func(xdsclient.Options) (xdsClientInterface, error) + bootstrapC *bootstrap.Config + target resolver.Target } -func testSetup(t *testing.T, opts setupOpts) (*xdsResolver, *testClientConn, func()) { +func testSetup(t *testing.T, opts setupOpts) (*xdsResolver, *fakeclient.Client, *testClientConn, func()) { t.Helper() - oldConfigMaker := newXDSConfig - newXDSConfig = func() (*bootstrap.Config, error) { return opts.config, nil } + fc := fakeclient.NewClient() + if opts.bootstrapC != nil { + fc.SetBootstrapConfig(opts.bootstrapC) + } oldClientMaker := newXDSClient - newXDSClient = opts.xdsClientFunc + closeCh := make(chan struct{}) + newXDSClient = func() (xdsclient.XDSClient, func(), error) { + return fc, grpcsync.OnceFunc(func() { close(closeCh) }), nil + } cancel := func() { - newXDSConfig = oldConfigMaker + // Make sure the xDS client is closed, in all (successful or failed) + // cases. + select { + case <-time.After(defaultTestTimeout): + t.Fatalf("timeout waiting for close") + case <-closeCh: + } newXDSClient = oldClientMaker } - builder := resolver.Get(xdsScheme) if builder == nil { t.Fatalf("resolver.Get(%v) returned nil", xdsScheme) } tcc := newTestClientConn() - r, err := builder.Build(target, tcc, resolver.BuildOptions{}) + r, err := builder.Build(opts.target, tcc, resolver.BuildOptions{}) if err != nil { t.Fatalf("builder.Build(%v) returned err: %v", target, err) } - return r.(*xdsResolver), tcc, cancel + return r.(*xdsResolver), fc, tcc, func() { + r.Close() + cancel() + } } -// waitForWatchService waits for the WatchService method to be called on the +// waitForWatchListener waits for the WatchListener method to be called on the // xdsClient within a reasonable amount of time, and also verifies that the // watch is called with the expected target. -func waitForWatchService(t *testing.T, xdsC *fakeclient.Client, wantTarget string) { +func waitForWatchListener(ctx context.Context, t *testing.T, xdsC *fakeclient.Client, wantTarget string) { t.Helper() - gotTarget, err := xdsC.WaitForWatchService() + gotTarget, err := xdsC.WaitForWatchListener(ctx) if err != nil { t.Fatalf("xdsClient.WatchService failed with error: %v", err) } @@ -257,206 +311,1736 @@ func waitForWatchService(t *testing.T, xdsC *fakeclient.Client, wantTarget strin } } -// TestXDSResolverWatchCallbackAfterClose tests the case where a service update -// from the underlying xdsClient is received after the resolver is closed. -func TestXDSResolverWatchCallbackAfterClose(t *testing.T) { - xdsC := fakeclient.NewClient() - xdsR, tcc, cancel := testSetup(t, setupOpts{ - config: &validConfig, - xdsClientFunc: func(_ xdsclient.Options) (xdsClientInterface, error) { return xdsC, nil }, +// waitForWatchRouteConfig waits for the WatchRoute method to be called on the +// xdsClient within a reasonable amount of time, and also verifies that the +// watch is called with the expected target. +func waitForWatchRouteConfig(ctx context.Context, t *testing.T, xdsC *fakeclient.Client, wantTarget string) { + t.Helper() + + gotTarget, err := xdsC.WaitForWatchRouteConfig(ctx) + if err != nil { + t.Fatalf("xdsClient.WatchService failed with error: %v", err) + } + if gotTarget != wantTarget { + t.Fatalf("xdsClient.WatchService() called with target: %v, want %v", gotTarget, wantTarget) + } +} + +// buildResolverForTarget builds an xDS resolver for the given target. It +// returns a testClientConn which allows inspection of resolver updates, and a +// function to close the resolver once the test is complete. +func buildResolverForTarget(t *testing.T, target resolver.Target) (*testClientConn, func()) { + builder := resolver.Get(xdsScheme) + if builder == nil { + t.Fatalf("resolver.Get(%v) returned nil", xdsScheme) + } + + tcc := newTestClientConn() + r, err := builder.Build(target, tcc, resolver.BuildOptions{}) + if err != nil { + t.Fatalf("builder.Build(%v) returned err: %v", target, err) + } + return tcc, r.Close +} + +// TestResolverResourceName builds an xDS resolver and verifies that the +// resource name specified in the discovery request matches expectations. +func (s) TestResolverResourceName(t *testing.T) { + // Federation support is required when new style names are used. + oldXDSFederation := envconfig.XDSFederation + envconfig.XDSFederation = true + defer func() { envconfig.XDSFederation = oldXDSFederation }() + + tests := []struct { + name string + listenerResourceNameTemplate string + extraAuthority string + dialTarget string + wantResourceName string + }{ + { + name: "default %s old style", + listenerResourceNameTemplate: "%s", + dialTarget: "xds:///target", + wantResourceName: "target", + }, + { + name: "old style no percent encoding", + listenerResourceNameTemplate: "/path/to/%s", + dialTarget: "xds:///target", + wantResourceName: "/path/to/target", + }, + { + name: "new style with %s", + listenerResourceNameTemplate: "xdstp://authority.com/%s", + dialTarget: "xds:///0.0.0.0:8080", + wantResourceName: "xdstp://authority.com/0.0.0.0:8080", + }, + { + name: "new style percent encoding", + listenerResourceNameTemplate: "xdstp://authority.com/%s", + dialTarget: "xds:///[::1]:8080", + wantResourceName: "xdstp://authority.com/%5B::1%5D:8080", + }, + { + name: "new style different authority", + listenerResourceNameTemplate: "xdstp://authority.com/%s", + extraAuthority: "test-authority", + dialTarget: "xds://test-authority/target", + wantResourceName: "xdstp://test-authority/envoy.config.listener.v3.Listener/target", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Setup the management server to push the requested resource name + // on to a channel. No resources are configured on the management + // server as part of this test, as we are only interested in the + // resource name being requested. + resourceNameCh := make(chan string, 1) + mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{ + OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + // When the resolver is being closed, the watch associated + // with the listener resource will be cancelled, and it + // might result in a discovery request with no resource + // names. Hence, we only consider requests which contain a + // resource name. + var name string + if len(req.GetResourceNames()) == 1 { + name = req.GetResourceNames()[0] + } + select { + case resourceNameCh <- name: + default: + } + return nil + }, + }) + if err != nil { + t.Fatalf("Failed to start xDS management server: %v", err) + } + defer mgmtServer.Stop() + + // Create a bootstrap configuration with test options. + opts := xdsbootstrap.Options{ + ServerURI: mgmtServer.Address, + ClientDefaultListenerResourceNameTemplate: tt.listenerResourceNameTemplate, + } + if tt.extraAuthority != "" { + // In this test, we really don't care about having multiple + // management servers. All we need to verify is whether the + // resource name matches expectation. + opts.Authorities = map[string]string{ + tt.extraAuthority: mgmtServer.Address, + } + } + cleanup, err := xdsbootstrap.CreateFile(opts) + if err != nil { + t.Fatal(err) + } + defer cleanup() + + _, rClose := buildResolverForTarget(t, resolver.Target{URL: *testutils.MustParseURL(tt.dialTarget)}) + defer rClose() + + // Verify the resource name in the discovery request being sent out. + select { + case gotResourceName := <-resourceNameCh: + if gotResourceName != tt.wantResourceName { + t.Fatalf("Received discovery request with resource name: %v, want %v", gotResourceName, tt.wantResourceName) + } + case <-time.After(defaultTestTimeout): + t.Fatalf("Timeout when waiting for discovery request") + } + }) + } +} + +// TestResolverWatchCallbackAfterClose tests the case where a service update +// from the underlying xDS client is received after the resolver is closed, and +// verifies that the update is not propagated to the ClientConn. +func (s) TestResolverWatchCallbackAfterClose(t *testing.T) { + // Setup the management server that synchronizes with the test goroutine + // using two channels. The management server signals the test goroutine when + // it receives a discovery request for a route configuration resource. And + // the test goroutine signals the management server when the resolver is + // closed. + waitForRouteConfigDiscoveryReqCh := make(chan struct{}, 1) + waitForResolverCloseCh := make(chan struct{}) + mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{ + OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + if req.GetTypeUrl() == version.V3RouteConfigURL { + select { + case waitForRouteConfigDiscoveryReqCh <- struct{}{}: + default: + } + <-waitForResolverCloseCh + } + return nil + }, }) + if err != nil { + t.Fatalf("Failed to start xDS management server: %v", err) + } + defer mgmtServer.Stop() + + // Create a bootstrap configuration specifying the above management server. + nodeID := uuid.New().String() + cleanup, err := xdsbootstrap.CreateFile(xdsbootstrap.Options{ + NodeID: nodeID, + ServerURI: mgmtServer.Address, + }) + if err != nil { + t.Fatal(err) + } + defer cleanup() + + // Configure listener and route configuration resources on the management + // server. + const serviceName = "my-service-client-side-xds" + rdsName := "route-" + serviceName + cdsName := "cluster-" + serviceName + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, rdsName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(rdsName, serviceName, cdsName)}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + tcc, rClose := buildResolverForTarget(t, resolver.Target{URL: *testutils.MustParseURL("xds:///" + serviceName)}) + defer rClose() - waitForWatchService(t, xdsC, targetStr) + // Wait for a discovery request for a route configuration resource. + select { + case <-waitForRouteConfigDiscoveryReqCh: + case <-ctx.Done(): + t.Fatal("Timeout when waiting for a discovery request for a route configuration resource") + } + + // Close the resolver and unblock the management server. + rClose() + close(waitForResolverCloseCh) - // Call the watchAPI callback after closing the resolver, and make sure no - // update is triggerred on the ClientConn. - xdsR.Close() - xdsC.InvokeWatchServiceCallback(xdsclient.ServiceUpdate{Routes: []*client.Route{{Prefix: newStringP(""), Action: map[string]uint32{cluster: 1}}}}, nil) - if gotVal, gotErr := tcc.stateCh.Receive(); gotErr != testutils.ErrRecvTimeout { - t.Fatalf("ClientConn.UpdateState called after xdsResolver is closed: %v", gotVal) + // Verify that the update from the management server is not propagated to + // the ClientConn. The xDS resolver, once closed, is expected to drop + // updates from the xDS client. + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := tcc.stateCh.Receive(sCtx); err != context.DeadlineExceeded { + t.Fatalf("ClientConn received an update from the resolver that was closed: %v", err) } } -// TestXDSResolverBadServiceUpdate tests the case the xdsClient returns a bad -// service update. -func TestXDSResolverBadServiceUpdate(t *testing.T) { - xdsC := fakeclient.NewClient() - xdsR, tcc, cancel := testSetup(t, setupOpts{ - config: &validConfig, - xdsClientFunc: func(_ xdsclient.Options) (xdsClientInterface, error) { return xdsC, nil }, - }) +// TestResolverCloseClosesXDSClient tests that the xDS resolver's Close method +// closes the xDS client. +func (s) TestResolverCloseClosesXDSClient(t *testing.T) { + bootstrapCfg := &bootstrap.Config{ + XDSServer: xdstestutils.ServerConfigForAddress(t, "dummy-management-server-address"), + } + + // Override xDS client creation to use bootstrap configuration pointing to a + // dummy management server. Also close a channel when the returned xDS + // client is closed. + closeCh := make(chan struct{}) + origNewClient := newXDSClient + newXDSClient = func() (xdsclient.XDSClient, func(), error) { + c, cancel, err := xdsclient.NewWithConfigForTesting(bootstrapCfg, defaultTestTimeout, defaultTestTimeout) + return c, func() { + close(closeCh) + cancel() + }, err + } defer func() { - cancel() - xdsR.Close() + newXDSClient = origNewClient }() - waitForWatchService(t, xdsC, targetStr) + _, rClose := buildResolverForTarget(t, resolver.Target{URL: *testutils.MustParseURL("xds:///my-service-client-side-xds")}) + rClose() - // Invoke the watchAPI callback with a bad service update and wait for the - // ReportError method to be called on the ClientConn. - suErr := errors.New("bad serviceupdate") - xdsC.InvokeWatchServiceCallback(xdsclient.ServiceUpdate{}, suErr) - if gotErrVal, gotErr := tcc.errorCh.Receive(); gotErr != nil || gotErrVal != suErr { - t.Fatalf("ClientConn.ReportError() received %v, want %v", gotErrVal, suErr) + select { + case <-closeCh: + case <-time.After(defaultTestTimeout): + t.Fatal("Timeout when waiting for xDS client to be closed") } } -// TestXDSResolverGoodServiceUpdate tests the happy case where the resolver -// gets a good service update from the xdsClient. -func TestXDSResolverGoodServiceUpdate(t *testing.T) { - xdsC := fakeclient.NewClient() - xdsR, tcc, cancel := testSetup(t, setupOpts{ - config: &validConfig, - xdsClientFunc: func(_ xdsclient.Options) (xdsClientInterface, error) { return xdsC, nil }, +// TestResolverBadServiceUpdate tests the case where a resource returned by the +// management server is NACKed by the xDS client, which then returns an update +// containing an error to the resolver. Verifies that the update is propagated +// to the ClientConn by the resolver. It also tests the cases where the resolver +// gets a good update subsequently, and another error after the good update. +func (s) TestResolverBadServiceUpdate(t *testing.T) { + mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) + if err != nil { + t.Fatal(err) + } + defer mgmtServer.Stop() + + // Create a bootstrap configuration specifying the above management server. + nodeID := uuid.New().String() + cleanup, err := xdsbootstrap.CreateFile(xdsbootstrap.Options{ + NodeID: nodeID, + ServerURI: mgmtServer.Address, }) - defer func() { - cancel() - xdsR.Close() - }() + if err != nil { + t.Fatal(err) + } + defer cleanup() + + const serviceName = "my-service-client-side-xds" + tcc, rClose := buildResolverForTarget(t, resolver.Target{URL: *testutils.MustParseURL("xds:///" + serviceName)}) + defer rClose() + + // Configure a listener resource that is expected to be NACKed because it + // does not contain the `RouteSpecifier` field in the HTTPConnectionManager. + hcm := testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + HttpFilters: []*v3httppb.HttpFilter{e2e.HTTPFilter("router", &v3routerpb.Router{})}, + }) + lis := &v3listenerpb.Listener{ + Name: serviceName, + ApiListener: &v3listenerpb.ApiListener{ApiListener: hcm}, + FilterChains: []*v3listenerpb.FilterChain{{ + Name: "filter-chain-name", + Filters: []*v3listenerpb.Filter{{ + Name: wellknown.HTTPConnectionManager, + ConfigType: &v3listenerpb.Filter_TypedConfig{TypedConfig: hcm}, + }}, + }}, + } + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{lis}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + wantErr := "no RouteSpecifier" + val, err := tcc.errorCh.Receive(ctx) + if err != nil { + t.Fatal("Timeout when waiting for error to be propagated to the ClientConn") + } + gotErr := val.(error) + if gotErr == nil || !strings.Contains(gotErr.Error(), wantErr) { + t.Fatalf("Received error from resolver %q, want %q", gotErr, wantErr) + } + + // Configure good listener and route configuration resources on the + // management server. + rdsName := "route-" + serviceName + cdsName := "cluster-" + serviceName + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, rdsName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(rdsName, serviceName, cdsName)}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Expect a good update from the resolver. + val, err = tcc.stateCh.Receive(ctx) + if err != nil { + t.Fatalf("Timeout waiting for an update from the resolver: %v", err) + } + rState := val.(resolver.State) + if err := rState.ServiceConfig.Err; err != nil { + t.Fatalf("Received error in service config: %v", rState.ServiceConfig.Err) + } + + // Configure another bad resource on the management server. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{lis}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Expect an error update from the resolver. + val, err = tcc.errorCh.Receive(ctx) + if err != nil { + t.Fatal("Timeout when waiting for error to be propagated to the ClientConn") + } + gotErr = val.(error) + if gotErr == nil || !strings.Contains(gotErr.Error(), wantErr) { + t.Fatalf("Received error from resolver %q, want %q", gotErr, wantErr) + } +} - waitForWatchService(t, xdsC, targetStr) - defer replaceRandNumGenerator(0)() +// TestResolverGoodServiceUpdate tests the case where the resource returned by +// the management server is ACKed by the xDS client, which then returns a good +// service update to the resolver. The test verifies that the service config +// returned by the resolver matches expectations, and that the config selector +// returned by the resolver picks clusters based on the route configuration +// received from the management server. +func (s) TestResolverGoodServiceUpdate(t *testing.T) { + mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) + if err != nil { + t.Fatal(err) + } + defer mgmtServer.Stop() + // Create a bootstrap configuration specifying the above management server. + nodeID := uuid.New().String() + cleanup, err := xdsbootstrap.CreateFile(xdsbootstrap.Options{ + NodeID: nodeID, + ServerURI: mgmtServer.Address, + }) + if err != nil { + t.Fatal(err) + } + defer cleanup() + + const serviceName = "my-service-client-side-xds" + tcc, rClose := buildResolverForTarget(t, resolver.Target{URL: *testutils.MustParseURL("xds:///" + serviceName)}) + defer rClose() + + ldsName := serviceName + rdsName := "route-" + serviceName for _, tt := range []struct { - su xdsclient.ServiceUpdate - wantJSON string + routeConfig *v3routepb.RouteConfiguration + wantServiceConfig string + wantClusters map[string]bool }{ { - su: xdsclient.ServiceUpdate{Routes: []*client.Route{{Prefix: newStringP(""), Action: map[string]uint32{testCluster1: 1}}}}, - wantJSON: testOneClusterOnlyJSON, + // A route configuration with a single cluster. + routeConfig: e2e.RouteConfigResourceWithOptions(e2e.RouteConfigOptions{ + RouteConfigName: rdsName, + ListenerName: ldsName, + ClusterSpecifierType: e2e.RouteConfigClusterSpecifierTypeCluster, + ClusterName: "test-cluster-1", + }), + wantServiceConfig: ` +{ + "loadBalancingConfig": [{ + "xds_cluster_manager_experimental": { + "children": { + "cluster:test-cluster-1": { + "childPolicy": [{ + "cds_experimental": { + "cluster": "test-cluster-1" + } + }] + } + } + } + }] +}`, + wantClusters: map[string]bool{"cluster:test-cluster-1": true}, }, { - su: client.ServiceUpdate{Routes: []*client.Route{{Prefix: newStringP(""), Action: map[string]uint32{ - "cluster_1": 75, - "cluster_2": 25, - }}}}, - wantJSON: testWeightedCDSJSON, + // A route configuration with a two new clusters. + routeConfig: e2e.RouteConfigResourceWithOptions(e2e.RouteConfigOptions{ + RouteConfigName: rdsName, + ListenerName: ldsName, + ClusterSpecifierType: e2e.RouteConfigClusterSpecifierTypeWeightedCluster, + WeightedClusters: map[string]int{"cluster_1": 75, "cluster_2": 25}, + }), + // This update contains the cluster from the previous update as well + // as this update, as the previous config selector still references + // the old cluster when the new one is pushed. + wantServiceConfig: ` +{ + "loadBalancingConfig": [{ + "xds_cluster_manager_experimental": { + "children": { + "cluster:test-cluster-1": { + "childPolicy": [{ + "cds_experimental": { + "cluster": "test-cluster-1" + } + }] + }, + "cluster:cluster_1": { + "childPolicy": [{ + "cds_experimental": { + "cluster": "cluster_1" + } + }] + }, + "cluster:cluster_2": { + "childPolicy": [{ + "cds_experimental": { + "cluster": "cluster_2" + } + }] + } + } + } + }] +}`, + wantClusters: map[string]bool{"cluster:cluster_1": true, "cluster:cluster_2": true}, }, } { - // Invoke the watchAPI callback with a good service update and wait for the - // UpdateState method to be called on the ClientConn. - xdsC.InvokeWatchServiceCallback(tt.su, nil) - gotState, err := tcc.stateCh.Receive() - if err != nil { - t.Fatalf("ClientConn.UpdateState returned error: %v", err) + // Configure the management server with a good listener resource and a + // route configuration resource, as specified by the test case. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(ldsName, rdsName)}, + Routes: []*v3routepb.RouteConfiguration{tt.routeConfig}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) } - rState := gotState.(resolver.State) - if gotClient := rState.Attributes.Value(xdsinternal.XDSClientID); gotClient != xdsC { - t.Fatalf("ClientConn.UpdateState got xdsClient: %v, want %v", gotClient, xdsC) + + // Read the update pushed by the resolver to the ClientConn. + val, err := tcc.stateCh.Receive(ctx) + if err != nil { + t.Fatalf("Timeout waiting for an update from the resolver: %v", err) } + rState := val.(resolver.State) if err := rState.ServiceConfig.Err; err != nil { - t.Fatalf("ClientConn.UpdateState received error in service config: %v", rState.ServiceConfig.Err) + t.Fatalf("Received error in service config: %v", rState.ServiceConfig.Err) } - wantSCParsed := internal.ParseServiceConfigForTesting.(func(string) *serviceconfig.ParseResult)(tt.wantJSON) + wantSCParsed := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(tt.wantServiceConfig) if !internal.EqualServiceConfigForTesting(rState.ServiceConfig.Config, wantSCParsed.Config) { - t.Errorf("ClientConn.UpdateState received different service config") - t.Error("got: ", cmp.Diff(nil, rState.ServiceConfig.Config)) - t.Error("want: ", cmp.Diff(nil, wantSCParsed.Config)) + t.Fatalf("Got service config:\n%s \nWant service config:\n%s", cmp.Diff(nil, rState.ServiceConfig.Config), cmp.Diff(nil, wantSCParsed.Config)) + } + + cs := iresolver.GetConfigSelector(rState) + if cs == nil { + t.Fatal("Received nil config selector in update from resolver") + } + + pickedClusters := make(map[string]bool) + // Odds of picking 75% cluster 100 times in a row: 1 in 3E-13. And + // with the random number generator stubbed out, we can rely on this + // to be 100% reproducible. + for i := 0; i < 100; i++ { + res, err := cs.SelectConfig(iresolver.RPCInfo{Context: ctx, Method: "/service/method"}) + if err != nil { + t.Fatalf("cs.SelectConfig(): %v", err) + } + cluster := clustermanager.GetPickedClusterForTesting(res.Context) + pickedClusters[cluster] = true + res.OnCommitted() + } + if !cmp.Equal(pickedClusters, tt.wantClusters) { + t.Errorf("Picked clusters: %v; want: %v", pickedClusters, tt.wantClusters) } } } -// TestXDSResolverUpdates tests the cases where the resolver gets a good update -// after an error, and an error after the good update. -func TestXDSResolverGoodUpdateAfterError(t *testing.T) { - xdsC := fakeclient.NewClient() - xdsR, tcc, cancel := testSetup(t, setupOpts{ - config: &validConfig, - xdsClientFunc: func(_ xdsclient.Options) (xdsClientInterface, error) { return xdsC, nil }, +// TestResolverRequestHash tests a case where a resolver receives a RouteConfig update +// with a HashPolicy specifying to generate a hash. The configSelector generated should +// successfully generate a Hash. +func (s) TestResolverRequestHash(t *testing.T) { + oldRH := envconfig.XDSRingHash + envconfig.XDSRingHash = true + defer func() { envconfig.XDSRingHash = oldRH }() + + mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) + if err != nil { + t.Fatal(err) + } + defer mgmtServer.Stop() + + // Create a bootstrap configuration specifying the above management server. + nodeID := uuid.New().String() + cleanup, err := xdsbootstrap.CreateFile(xdsbootstrap.Options{ + NodeID: nodeID, + ServerURI: mgmtServer.Address, }) - defer func() { - cancel() - xdsR.Close() - }() + if err != nil { + t.Fatal(err) + } + defer cleanup() - waitForWatchService(t, xdsC, targetStr) + const serviceName = "my-service-client-side-xds" + tcc, rClose := buildResolverForTarget(t, resolver.Target{URL: *testutils.MustParseURL("xds:///" + serviceName)}) + defer rClose() - // Invoke the watchAPI callback with a bad service update and wait for the - // ReportError method to be called on the ClientConn. - suErr := errors.New("bad serviceupdate") - xdsC.InvokeWatchServiceCallback(xdsclient.ServiceUpdate{}, suErr) - if gotErrVal, gotErr := tcc.errorCh.Receive(); gotErr != nil || gotErrVal != suErr { - t.Fatalf("ClientConn.ReportError() received %v, want %v", gotErrVal, suErr) + ldsName := serviceName + rdsName := "route-" + serviceName + // Configure the management server with a good listener resource and a + // route configuration resource that specifies a hash policy. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(ldsName, rdsName)}, + Routes: []*v3routepb.RouteConfiguration{{ + Name: rdsName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{ldsName}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}}, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{WeightedClusters: &v3routepb.WeightedCluster{ + Clusters: []*v3routepb.WeightedCluster_ClusterWeight{ + { + Name: "test-cluster-1", + Weight: &wrapperspb.UInt32Value{Value: 100}, + }, + }, + }}, + HashPolicy: []*v3routepb.RouteAction_HashPolicy{{ + PolicySpecifier: &v3routepb.RouteAction_HashPolicy_Header_{ + Header: &v3routepb.RouteAction_HashPolicy_Header{ + HeaderName: ":path", + }, + }, + Terminal: true, + }}, + }}, + }}, + }}, + }}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) } - // Invoke the watchAPI callback with a good service update and wait for the - // UpdateState method to be called on the ClientConn. - xdsC.InvokeWatchServiceCallback(xdsclient.ServiceUpdate{Routes: []*client.Route{{Prefix: newStringP(""), Action: map[string]uint32{cluster: 1}}}}, nil) - gotState, err := tcc.stateCh.Receive() + // Read the update pushed by the resolver to the ClientConn. + val, err := tcc.stateCh.Receive(ctx) if err != nil { - t.Fatalf("ClientConn.UpdateState returned error: %v", err) + t.Fatalf("Timeout waiting for an update from the resolver: %v", err) } - rState := gotState.(resolver.State) - if gotClient := rState.Attributes.Value(xdsinternal.XDSClientID); gotClient != xdsC { - t.Fatalf("ClientConn.UpdateState got xdsClient: %v, want %v", gotClient, xdsC) + rState := val.(resolver.State) + if err := rState.ServiceConfig.Err; err != nil { + t.Fatalf("Received error in service config: %v", rState.ServiceConfig.Err) + } + cs := iresolver.GetConfigSelector(rState) + if cs == nil { + t.Fatal("Received nil config selector in update from resolver") + } + + // Selecting a config when there was a hash policy specified in the route + // that will be selected should put a request hash in the config's context. + res, err := cs.SelectConfig(iresolver.RPCInfo{ + Context: metadata.NewOutgoingContext(ctx, metadata.Pairs(":path", "/products")), + Method: "/service/method", + }) + if err != nil { + t.Fatalf("cs.SelectConfig(): %v", err) + } + gotHash := ringhash.GetRequestHashForTesting(res.Context) + wantHash := xxhash.Sum64String("/products") + if gotHash != wantHash { + t.Fatalf("Got request hash: %v, want: %v", gotHash, wantHash) + } +} + +// TestResolverRemovedWithRPCs tests the case where resources are removed from +// the management server, causing it to send an empty update to the xDS client, +// which returns a resource-not-found error to the xDS resolver. The test +// verifies that an ongoing RPC is handled properly when this happens. +func (s) TestResolverRemovedWithRPCs(t *testing.T) { + mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) + if err != nil { + t.Fatal(err) + } + defer mgmtServer.Stop() + + // Create a bootstrap configuration specifying the above management server. + nodeID := uuid.New().String() + cleanup, err := xdsbootstrap.CreateFile(xdsbootstrap.Options{ + NodeID: nodeID, + ServerURI: mgmtServer.Address, + }) + if err != nil { + t.Fatal(err) + } + defer cleanup() + + const serviceName = "my-service-client-side-xds" + tcc, rClose := buildResolverForTarget(t, resolver.Target{URL: *testutils.MustParseURL("xds:///" + serviceName)}) + defer rClose() + + ldsName := serviceName + rdsName := "route-" + serviceName + // Configure the management server with a good listener and route + // configuration resource. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(ldsName, rdsName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(rdsName, ldsName, "test-cluster-1")}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Read the update pushed by the resolver to the ClientConn. + val, err := tcc.stateCh.Receive(ctx) + if err != nil { + t.Fatalf("Timeout waiting for an update from the resolver: %v", err) + } + rState := val.(resolver.State) + if err := rState.ServiceConfig.Err; err != nil { + t.Fatalf("Received error in service config: %v", rState.ServiceConfig.Err) + } + wantSCParsed := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(` +{ + "loadBalancingConfig": [ + { + "xds_cluster_manager_experimental": { + "children": { + "cluster:test-cluster-1": { + "childPolicy": [ + { + "cds_experimental": { + "cluster": "test-cluster-1" + } + } + ] + } + } + } + } + ] +}`) + if !internal.EqualServiceConfigForTesting(rState.ServiceConfig.Config, wantSCParsed.Config) { + t.Fatalf("Got service config:\n%s \nWant service config:\n%s", cmp.Diff(nil, rState.ServiceConfig.Config), cmp.Diff(nil, wantSCParsed.Config)) + } + + cs := iresolver.GetConfigSelector(rState) + if cs == nil { + t.Fatal("Received nil config selector in update from resolver") + } + res, err := cs.SelectConfig(iresolver.RPCInfo{Context: ctx, Method: "/service/method"}) + if err != nil { + t.Fatalf("cs.SelectConfig(): %v", err) + } + + // Delete the resources on the management server. This should result in a + // resource-not-found error from the xDS client. + if err := mgmtServer.Update(ctx, e2e.UpdateOptions{NodeID: nodeID}); err != nil { + t.Fatal(err) + } + + // The RPC started earlier is still in progress. So, the xDS resolver will + // not produce an empty service config at this point. Instead it will retain + // the cluster to which the RPC is ongoing in the service config, but will + // return an erroring config selector which will fail new RPCs. + val, err = tcc.stateCh.Receive(ctx) + if err != nil { + t.Fatalf("Timeout waiting for an update from the resolver: %v", err) + } + rState = val.(resolver.State) + if err := rState.ServiceConfig.Err; err != nil { + t.Fatalf("Received error in service config: %v", rState.ServiceConfig.Err) + } + if !internal.EqualServiceConfigForTesting(rState.ServiceConfig.Config, wantSCParsed.Config) { + t.Fatalf("Got service config:\n%s \nWant service config:\n%s", cmp.Diff(nil, rState.ServiceConfig.Config), cmp.Diff(nil, wantSCParsed.Config)) + } + cs = iresolver.GetConfigSelector(rState) + if cs == nil { + t.Fatal("Received nil config selector in update from resolver") + } + _, err = cs.SelectConfig(iresolver.RPCInfo{Context: ctx, Method: "/service/method"}) + if err == nil || status.Code(err) != codes.Unavailable { + t.Fatalf("cs.SelectConfig() returned: %v, want: %v", err, codes.Unavailable) + } + + // "Finish the RPC"; this could cause a panic if the resolver doesn't + // handle it correctly. + res.OnCommitted() + + // Now that the RPC is committed, the xDS resolver is expected to send an + // update with an empty service config. + val, err = tcc.stateCh.Receive(ctx) + if err != nil { + t.Fatalf("Timeout waiting for an update from the resolver: %v", err) + } + rState = val.(resolver.State) + if err := rState.ServiceConfig.Err; err != nil { + t.Fatalf("Received error in service config: %v", rState.ServiceConfig.Err) + } + wantSCParsed = internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(`{}`) + if !internal.EqualServiceConfigForTesting(rState.ServiceConfig.Config, wantSCParsed.Config) { + t.Fatalf("Got service config:\n%s \nWant service config:\n%s", cmp.Diff(nil, rState.ServiceConfig.Config), cmp.Diff(nil, wantSCParsed.Config)) + } +} + +// TestResolverRemovedResource tests the case where resources returned by the +// management server are removed. The test verifies that the resolver pushes the +// expected config selector and service config in this case. +func (s) TestResolverRemovedResource(t *testing.T) { + mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) + if err != nil { + t.Fatal(err) + } + defer mgmtServer.Stop() + + // Create a bootstrap configuration specifying the above management server. + nodeID := uuid.New().String() + cleanup, err := xdsbootstrap.CreateFile(xdsbootstrap.Options{ + NodeID: nodeID, + ServerURI: mgmtServer.Address, + }) + if err != nil { + t.Fatal(err) + } + defer cleanup() + + const serviceName = "my-service-client-side-xds" + tcc, rClose := buildResolverForTarget(t, resolver.Target{URL: *testutils.MustParseURL("xds:///" + serviceName)}) + defer rClose() + + // Configure the management server with a good listener and route + // configuration resource. + ldsName := serviceName + rdsName := "route-" + serviceName + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(ldsName, rdsName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(rdsName, ldsName, "test-cluster-1")}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Read the update pushed by the resolver to the ClientConn. + val, err := tcc.stateCh.Receive(ctx) + if err != nil { + t.Fatalf("Timeout waiting for an update from the resolver: %v", err) + } + rState := val.(resolver.State) + if err := rState.ServiceConfig.Err; err != nil { + t.Fatalf("Received error in service config: %v", rState.ServiceConfig.Err) + } + wantSCParsed := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(` +{ + "loadBalancingConfig": [ + { + "xds_cluster_manager_experimental": { + "children": { + "cluster:test-cluster-1": { + "childPolicy": [ + { + "cds_experimental": { + "cluster": "test-cluster-1" + } + } + ] + } + } + } + } + ] +}`) + if !internal.EqualServiceConfigForTesting(rState.ServiceConfig.Config, wantSCParsed.Config) { + t.Fatalf("Got service config:\n%s \nWant service config:\n%s", cmp.Diff(nil, rState.ServiceConfig.Config), cmp.Diff(nil, wantSCParsed.Config)) + } + + // "Make an RPC" by invoking the config selector. + cs := iresolver.GetConfigSelector(rState) + if cs == nil { + t.Fatal("Received nil config selector in update from resolver") + } + + res, err := cs.SelectConfig(iresolver.RPCInfo{Context: ctx, Method: "/service/method"}) + if err != nil { + t.Fatalf("cs.SelectConfig(): %v", err) } + + // "Finish the RPC"; this could cause a panic if the resolver doesn't + // handle it correctly. + res.OnCommitted() + + // Delete the resources on the management server, resulting in a + // resource-not-found error from the xDS client. + if err := mgmtServer.Update(ctx, e2e.UpdateOptions{NodeID: nodeID}); err != nil { + t.Fatal(err) + } + + // The channel should receive the existing service config with the original + // cluster but with an erroring config selector. + val, err = tcc.stateCh.Receive(ctx) + if err != nil { + t.Fatalf("Timeout waiting for an update from the resolver: %v", err) + } + rState = val.(resolver.State) if err := rState.ServiceConfig.Err; err != nil { - t.Fatalf("ClientConn.UpdateState received error in service config: %v", rState.ServiceConfig.Err) + t.Fatalf("Received error in service config: %v", rState.ServiceConfig.Err) + } + if !internal.EqualServiceConfigForTesting(rState.ServiceConfig.Config, wantSCParsed.Config) { + t.Fatalf("Got service config:\n%s \nWant service config:\n%s", cmp.Diff(nil, rState.ServiceConfig.Config), cmp.Diff(nil, wantSCParsed.Config)) } - // Invoke the watchAPI callback with a bad service update and wait for the - // ReportError method to be called on the ClientConn. - suErr2 := errors.New("bad serviceupdate 2") - xdsC.InvokeWatchServiceCallback(xdsclient.ServiceUpdate{}, suErr2) - if gotErrVal, gotErr := tcc.errorCh.Receive(); gotErr != nil || gotErrVal != suErr2 { - t.Fatalf("ClientConn.ReportError() received %v, want %v", gotErrVal, suErr2) + // "Make another RPC" by invoking the config selector. + cs = iresolver.GetConfigSelector(rState) + if cs == nil { + t.Fatal("Received nil config selector in update from resolver") + } + + res, err = cs.SelectConfig(iresolver.RPCInfo{Context: ctx, Method: "/service/method"}) + if err == nil || status.Code(err) != codes.Unavailable { + t.Fatalf("cs.SelectConfig() got %v, %v, expected UNAVAILABLE error", res, err) + } + + // In the meantime, an empty ServiceConfig update should have been sent. + val, err = tcc.stateCh.Receive(ctx) + if err != nil { + t.Fatalf("Timeout waiting for an update from the resolver: %v", err) + } + rState = val.(resolver.State) + if err := rState.ServiceConfig.Err; err != nil { + t.Fatalf("Received error in service config: %v", rState.ServiceConfig.Err) + } + wantSCParsed = internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)("{}") + if !internal.EqualServiceConfigForTesting(rState.ServiceConfig.Config, wantSCParsed.Config) { + t.Fatalf("Got service config:\n%s \nWant service config:\n%s", cmp.Diff(nil, rState.ServiceConfig.Config), cmp.Diff(nil, wantSCParsed.Config)) } } -// TestXDSResolverResourceNotFoundError tests the cases where the resolver gets -// a ResourceNotFoundError. It should generate a service config picking -// weighted_target, but no child balancers. -func TestXDSResolverResourceNotFoundError(t *testing.T) { - xdsC := fakeclient.NewClient() - xdsR, tcc, cancel := testSetup(t, setupOpts{ - config: &validConfig, - xdsClientFunc: func(_ xdsclient.Options) (xdsClientInterface, error) { return xdsC, nil }, +// TestResolverWRR tests the case where the route configuration returned by the +// management server contains a set of weighted clusters. The test performs a +// bunch of RPCs using the cluster specifier returned by the resolver, and +// verifies the cluster distribution. +func (s) TestResolverWRR(t *testing.T) { + defer func(oldNewWRR func() wrr.WRR) { newWRR = oldNewWRR }(newWRR) + newWRR = testutils.NewTestWRR + + mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) + if err != nil { + t.Fatal(err) + } + defer mgmtServer.Stop() + + // Create a bootstrap configuration specifying the above management server. + nodeID := uuid.New().String() + cleanup, err := xdsbootstrap.CreateFile(xdsbootstrap.Options{ + NodeID: nodeID, + ServerURI: mgmtServer.Address, }) - defer func() { - cancel() - xdsR.Close() - }() + if err != nil { + t.Fatal(err) + } + defer cleanup() - waitForWatchService(t, xdsC, targetStr) + const serviceName = "my-service-client-side-xds" + tcc, rClose := buildResolverForTarget(t, resolver.Target{URL: *testutils.MustParseURL("xds:///" + serviceName)}) + defer rClose() - // Invoke the watchAPI callback with a bad service update and wait for the - // ReportError method to be called on the ClientConn. - suErr := xdsclient.NewErrorf(xdsclient.ErrorTypeResourceNotFound, "resource removed error") - xdsC.InvokeWatchServiceCallback(xdsclient.ServiceUpdate{}, suErr) - if gotErrVal, gotErr := tcc.errorCh.Receive(); gotErr != testutils.ErrRecvTimeout { - t.Fatalf("ClientConn.ReportError() received %v, %v, want channel recv timeout", gotErrVal, gotErr) + ldsName := serviceName + rdsName := "route-" + serviceName + // Configure the management server with a good listener resource and a + // route configuration resource. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(ldsName, rdsName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.RouteConfigResourceWithOptions(e2e.RouteConfigOptions{ + RouteConfigName: rdsName, + ListenerName: ldsName, + ClusterSpecifierType: e2e.RouteConfigClusterSpecifierTypeWeightedCluster, + WeightedClusters: map[string]int{"A": 75, "B": 25}, + })}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) } - gotState, err := tcc.stateCh.Receive() + + // Read the update pushed by the resolver to the ClientConn. + gotState, err := tcc.stateCh.Receive(ctx) if err != nil { - t.Fatalf("ClientConn.UpdateState returned error: %v", err) + t.Fatalf("Timeout waiting for an update from the resolver: %v", err) } rState := gotState.(resolver.State) - // This update shouldn't have xds-client in it, because it doesn't pick an - // xds balancer. - if gotClient := rState.Attributes.Value(xdsinternal.XDSClientID); gotClient != nil { - t.Fatalf("ClientConn.UpdateState got xdsClient: %v, want ", gotClient) + if err := rState.ServiceConfig.Err; err != nil { + t.Fatalf("Received error in service config: %v", rState.ServiceConfig.Err) + } + cs := iresolver.GetConfigSelector(rState) + if cs == nil { + t.Fatal("Received nil config selector in update from resolver") + } + + // Make RPCs are verify WRR behavior in the cluster specifier. + picks := map[string]int{} + for i := 0; i < 100; i++ { + res, err := cs.SelectConfig(iresolver.RPCInfo{Context: ctx, Method: "/service/method"}) + if err != nil { + t.Fatalf("cs.SelectConfig(): %v", err) + } + picks[clustermanager.GetPickedClusterForTesting(res.Context)]++ + res.OnCommitted() } - wantParsedConfig := internal.ParseServiceConfigForTesting.(func(string) *serviceconfig.ParseResult)("{}") - if !internal.EqualServiceConfigForTesting(rState.ServiceConfig.Config, wantParsedConfig.Config) { - t.Error("ClientConn.UpdateState got wrong service config") - t.Errorf("gotParsed: %s", cmp.Diff(nil, rState.ServiceConfig.Config)) - t.Errorf("wantParsed: %s", cmp.Diff(nil, wantParsedConfig.Config)) + want := map[string]int{"cluster:A": 75, "cluster:B": 25} + if !cmp.Equal(picks, want) { + t.Errorf("Picked clusters: %v; want: %v", picks, want) } +} + +// TestResolverMaxStreamDuration tests the case where the resolver receives max +// stream duration as part of the listener and route configuration resources. +// The test verifies that the RPC timeout returned by the config selector +// matches expectations. A non-nil max stream duration (this includes an +// explicit zero value) in a matching route overrides the value specified in the +// listener resource. +func (s) TestResolverMaxStreamDuration(t *testing.T) { + mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) + if err != nil { + t.Fatal(err) + } + defer mgmtServer.Stop() + + // Create a bootstrap configuration specifying the above management server. + nodeID := uuid.New().String() + cleanup, err := xdsbootstrap.CreateFile(xdsbootstrap.Options{ + NodeID: nodeID, + ServerURI: mgmtServer.Address, + }) + if err != nil { + t.Fatal(err) + } + defer cleanup() + + const serviceName = "my-service-client-side-xds" + tcc, rClose := buildResolverForTarget(t, resolver.Target{URL: *testutils.MustParseURL("xds:///" + serviceName)}) + defer rClose() + + // Configure the management server with a listener resource that specifies a + // max stream duration as part of its HTTP connection manager. Also + // configure a route configuration resource, which has multiple routes with + // different values of max stream duration. + ldsName := serviceName + rdsName := "route-" + serviceName + hcm := testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{Rds: &v3httppb.Rds{ + ConfigSource: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}, + }, + RouteConfigName: rdsName, + }}, + HttpFilters: []*v3httppb.HttpFilter{e2e.RouterHTTPFilter}, + CommonHttpProtocolOptions: &v3corepb.HttpProtocolOptions{ + MaxStreamDuration: durationpb.New(1 * time.Second), + }, + }) + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{{ + Name: ldsName, + ApiListener: &v3listenerpb.ApiListener{ApiListener: hcm}, + FilterChains: []*v3listenerpb.FilterChain{{ + Name: "filter-chain-name", + Filters: []*v3listenerpb.Filter{{ + Name: wellknown.HTTPConnectionManager, + ConfigType: &v3listenerpb.Filter_TypedConfig{TypedConfig: hcm}, + }}, + }}, + }}, + Routes: []*v3routepb.RouteConfiguration{{ + Name: rdsName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{ldsName}, + Routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/foo"}}, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{WeightedClusters: &v3routepb.WeightedCluster{ + Clusters: []*v3routepb.WeightedCluster_ClusterWeight{ + { + Name: "A", + Weight: &wrapperspb.UInt32Value{Value: 100}, + }, + }}, + }, + MaxStreamDuration: &v3routepb.RouteAction_MaxStreamDuration{ + MaxStreamDuration: durationpb.New(5 * time.Second), + }, + }}, + }, + { + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/bar"}}, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{WeightedClusters: &v3routepb.WeightedCluster{ + Clusters: []*v3routepb.WeightedCluster_ClusterWeight{ + { + Name: "B", + Weight: &wrapperspb.UInt32Value{Value: 100}, + }, + }}, + }, + MaxStreamDuration: &v3routepb.RouteAction_MaxStreamDuration{ + MaxStreamDuration: durationpb.New(0 * time.Second), + }, + }}, + }, + { + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}}, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{WeightedClusters: &v3routepb.WeightedCluster{ + Clusters: []*v3routepb.WeightedCluster_ClusterWeight{ + { + Name: "C", + Weight: &wrapperspb.UInt32Value{Value: 100}, + }, + }}, + }, + }}, + }, + }, + }}, + }}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Read the update pushed by the resolver to the ClientConn. + gotState, err := tcc.stateCh.Receive(ctx) + if err != nil { + t.Fatalf("Timeout waiting for an update from the resolver: %v", err) + } + rState := gotState.(resolver.State) if err := rState.ServiceConfig.Err; err != nil { - t.Fatalf("ClientConn.UpdateState received error in service config: %v", rState.ServiceConfig.Err) + t.Fatalf("Received error in service config: %v", rState.ServiceConfig.Err) + } + cs := iresolver.GetConfigSelector(rState) + if cs == nil { + t.Fatal("Received nil config selector in update from resolver") + } + + testCases := []struct { + name string + method string + want *time.Duration + }{{ + name: "RDS setting", + method: "/foo/method", + want: newDurationP(5 * time.Second), + }, { + name: "explicit zero in RDS; ignore LDS", + method: "/bar/method", + want: nil, + }, { + name: "no config in RDS; fallback to LDS", + method: "/baz/method", + want: newDurationP(time.Second), + }} + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + req := iresolver.RPCInfo{ + Method: tc.method, + Context: ctx, + } + res, err := cs.SelectConfig(req) + if err != nil { + t.Errorf("cs.SelectConfig(%v): %v", req, err) + return + } + res.OnCommitted() + got := res.MethodConfig.Timeout + if !cmp.Equal(got, tc.want) { + t.Errorf("For method %q: res.MethodConfig.Timeout = %v; want %v", tc.method, got, tc.want) + } + }) } } -func replaceRandNumGenerator(start int64) func() { - nextInt := start - grpcrandInt63n = func(int64) (ret int64) { - ret = nextInt - nextInt++ - return +// TestResolverDelayedOnCommitted tests that clusters remain in service +// config if RPCs are in flight. +func (s) TestResolverDelayedOnCommitted(t *testing.T) { + mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) + if err != nil { + t.Fatal(err) } - return func() { - grpcrandInt63n = grpcrand.Int63n + defer mgmtServer.Stop() + + // Create a bootstrap configuration specifying the above management server. + nodeID := uuid.New().String() + cleanup, err := xdsbootstrap.CreateFile(xdsbootstrap.Options{ + NodeID: nodeID, + ServerURI: mgmtServer.Address, + }) + if err != nil { + t.Fatal(err) + } + defer cleanup() + + const serviceName = "my-service-client-side-xds" + tcc, rClose := buildResolverForTarget(t, resolver.Target{URL: *testutils.MustParseURL("xds:///" + serviceName)}) + defer rClose() + + // Configure the management server with a good listener and route + // configuration resource. + ldsName := serviceName + rdsName := "route-" + serviceName + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(ldsName, rdsName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(rdsName, ldsName, "old-cluster")}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Read the update pushed by the resolver to the ClientConn. + val, err := tcc.stateCh.Receive(ctx) + if err != nil { + t.Fatalf("Timeout waiting for an update from the resolver: %v", err) + } + rState := val.(resolver.State) + if err := rState.ServiceConfig.Err; err != nil { + t.Fatalf("Received error in service config: %v", rState.ServiceConfig.Err) + } + wantSCParsed := internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(` +{ + "loadBalancingConfig": [ + { + "xds_cluster_manager_experimental": { + "children": { + "cluster:old-cluster": { + "childPolicy": [ + { + "cds_experimental": { + "cluster": "old-cluster" + } + } + ] + } + } + } + } + ] +}`) + if !internal.EqualServiceConfigForTesting(rState.ServiceConfig.Config, wantSCParsed.Config) { + t.Fatalf("Got service config:\n%s \nWant service config:\n%s", cmp.Diff(nil, rState.ServiceConfig.Config), cmp.Diff(nil, wantSCParsed.Config)) + } + + // Make an RPC, but do not commit it yet. + cs := iresolver.GetConfigSelector(rState) + if cs == nil { + t.Fatal("Received nil config selector in update from resolver") + } + resOld, err := cs.SelectConfig(iresolver.RPCInfo{Context: ctx, Method: "/service/method"}) + if err != nil { + t.Fatalf("cs.SelectConfig(): %v", err) } + if cluster := clustermanager.GetPickedClusterForTesting(resOld.Context); cluster != "cluster:old-cluster" { + t.Fatalf("Picked cluster is %q, want %q", cluster, "cluster:old-cluster") + } + + // Delay resOld.OnCommitted(). As long as there are pending RPCs to removed + // clusters, they still appear in the service config. + + // Update the route configuration resource on the management server to + // return a new cluster. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(ldsName, rdsName)}, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(rdsName, ldsName, "new-cluster")}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Read the update pushed by the resolver to the ClientConn and ensure the + // old cluster is present in the service config. Also ensure that the newly + // returned config selector does not hold a reference to the old cluster. + val, err = tcc.stateCh.Receive(ctx) + if err != nil { + t.Fatalf("Timeout waiting for an update from the resolver: %v", err) + } + rState = val.(resolver.State) + if err := rState.ServiceConfig.Err; err != nil { + t.Fatalf("Received error in service config: %v", rState.ServiceConfig.Err) + } + wantSCParsed = internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(` +{ + "loadBalancingConfig": [ + { + "xds_cluster_manager_experimental": { + "children": { + "cluster:old-cluster": { + "childPolicy": [ + { + "cds_experimental": { + "cluster": "old-cluster" + } + } + ] + }, + "cluster:new-cluster": { + "childPolicy": [ + { + "cds_experimental": { + "cluster": "new-cluster" + } + } + ] + } + } + } + } + ] +}`) + if !internal.EqualServiceConfigForTesting(rState.ServiceConfig.Config, wantSCParsed.Config) { + t.Fatalf("Got service config:\n%s\nWant service config:\n%s", cmp.Diff(nil, rState.ServiceConfig.Config), cmp.Diff(nil, wantSCParsed.Config)) + } + + cs = iresolver.GetConfigSelector(rState) + if cs == nil { + t.Fatal("Received nil config selector in update from resolver") + } + resNew, err := cs.SelectConfig(iresolver.RPCInfo{Context: ctx, Method: "/service/method"}) + if err != nil { + t.Fatalf("cs.SelectConfig(): %v", err) + } + if cluster := clustermanager.GetPickedClusterForTesting(resNew.Context); cluster != "cluster:new-cluster" { + t.Fatalf("Picked cluster is %q, want %q", cluster, "cluster:new-cluster") + } + + // Invoke OnCommitted on the old RPC; should lead to a service config update + // that deletes the old cluster, as the old cluster no longer has any + // pending RPCs. + resOld.OnCommitted() + + val, err = tcc.stateCh.Receive(ctx) + if err != nil { + t.Fatalf("Timeout waiting for an update from the resolver: %v", err) + } + rState = val.(resolver.State) + if err := rState.ServiceConfig.Err; err != nil { + t.Fatalf("Received error in service config: %v", rState.ServiceConfig.Err) + } + wantSCParsed = internal.ParseServiceConfig.(func(string) *serviceconfig.ParseResult)(` +{ + "loadBalancingConfig": [ + { + "xds_cluster_manager_experimental": { + "children": { + "cluster:new-cluster": { + "childPolicy": [ + { + "cds_experimental": { + "cluster": "new-cluster" + } + } + ] + } + } + } + } + ] +}`) + if !internal.EqualServiceConfigForTesting(rState.ServiceConfig.Config, wantSCParsed.Config) { + t.Fatalf("Got service config:\n%s \nWant service config:\n%s", cmp.Diff(nil, rState.ServiceConfig.Config), cmp.Diff(nil, wantSCParsed.Config)) + } +} + +// TestResolverMultipleLDSUpdates tests the case where two LDS updates with the +// same RDS name to watch are received without an RDS in between. Those LDS +// updates shouldn't trigger a service config update. +func (s) TestResolverMultipleLDSUpdates(t *testing.T) { + mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) + if err != nil { + t.Fatal(err) + } + defer mgmtServer.Stop() + + // Create a bootstrap configuration specifying the above management server. + nodeID := uuid.New().String() + cleanup, err := xdsbootstrap.CreateFile(xdsbootstrap.Options{ + NodeID: nodeID, + ServerURI: mgmtServer.Address, + }) + if err != nil { + t.Fatal(err) + } + defer cleanup() + + // Build an xDS resolver that uses the above bootstrap configuration + // Creating the xDS resolver should result in creation of the xDS client. + const serviceName = "my-service-client-side-xds" + tcc, rClose := buildResolverForTarget(t, resolver.Target{URL: *testutils.MustParseURL("xds:///" + serviceName)}) + defer rClose() + + // Configure the management server with a listener resource, but no route + // configuration resource. + ldsName := serviceName + rdsName := "route-" + serviceName + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(ldsName, rdsName)}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Ensure there is no update from the resolver. + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + gotState, err := tcc.stateCh.Receive(sCtx) + if err == nil { + t.Fatalf("Received update from resolver %v when none expected", gotState) + } + + // Configure the management server with a listener resource that points to + // the same route configuration resource but has different values for some + // other fields. There is still no route configuration resource on the + // management server. + hcm := testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{Rds: &v3httppb.Rds{ + ConfigSource: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}, + }, + RouteConfigName: rdsName, + }}, + HttpFilters: []*v3httppb.HttpFilter{e2e.RouterHTTPFilter}, + CommonHttpProtocolOptions: &v3corepb.HttpProtocolOptions{ + MaxStreamDuration: durationpb.New(1 * time.Second), + }, + }) + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{{ + Name: ldsName, + ApiListener: &v3listenerpb.ApiListener{ApiListener: hcm}, + FilterChains: []*v3listenerpb.FilterChain{{ + Name: "filter-chain-name", + Filters: []*v3listenerpb.Filter{{ + Name: wellknown.HTTPConnectionManager, + ConfigType: &v3listenerpb.Filter_TypedConfig{TypedConfig: hcm}, + }}, + }}, + }}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Ensure that there is no update from the resolver. + sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + gotState, err = tcc.stateCh.Receive(sCtx) + if err == nil { + t.Fatalf("Received update from resolver %v when none expected", gotState) + } +} + +type filterBuilder struct { + httpfilter.Filter // embedded as we do not need to implement registry / parsing in this test. + path *[]string +} + +var _ httpfilter.ClientInterceptorBuilder = &filterBuilder{} + +func (fb *filterBuilder) BuildClientInterceptor(config, override httpfilter.FilterConfig) (iresolver.ClientInterceptor, error) { + if config == nil { + panic("unexpected missing config") + } + *fb.path = append(*fb.path, "build:"+config.(filterCfg).s) + err := config.(filterCfg).newStreamErr + if override != nil { + *fb.path = append(*fb.path, "override:"+override.(filterCfg).s) + err = override.(filterCfg).newStreamErr + } + + return &filterInterceptor{path: fb.path, s: config.(filterCfg).s, err: err}, nil +} + +type filterInterceptor struct { + path *[]string + s string + err error +} + +func (fi *filterInterceptor) NewStream(ctx context.Context, ri iresolver.RPCInfo, done func(), newStream func(ctx context.Context, done func()) (iresolver.ClientStream, error)) (iresolver.ClientStream, error) { + *fi.path = append(*fi.path, "newstream:"+fi.s) + if fi.err != nil { + return nil, fi.err + } + d := func() { + *fi.path = append(*fi.path, "done:"+fi.s) + done() + } + cs, err := newStream(ctx, d) + if err != nil { + return nil, err + } + return &clientStream{ClientStream: cs, path: fi.path, s: fi.s}, nil +} + +type clientStream struct { + iresolver.ClientStream + path *[]string + s string +} + +type filterCfg struct { + httpfilter.FilterConfig + s string + newStreamErr error +} + +func (s) TestXDSResolverHTTPFilters(t *testing.T) { + var path []string + testCases := []struct { + name string + ldsFilters []xdsresource.HTTPFilter + rtCfgUpdate xdsresource.RouteConfigUpdate + rpcRes map[string][][]string + selectErr string + newStreamErr string + }{ + + { + name: "route type RouteActionUnsupported invalid for client", + ldsFilters: []xdsresource.HTTPFilter{ + {Name: "foo", Filter: &filterBuilder{path: &path}, Config: filterCfg{s: "foo1"}}, + }, + rtCfgUpdate: xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{ + { + Domains: []string{targetStr}, + Routes: []*xdsresource.Route{{ + Prefix: newStringP("1"), + WeightedClusters: map[string]xdsresource.WeightedCluster{ + "A": {Weight: 1}, + "B": {Weight: 1}, + }, + ActionType: xdsresource.RouteActionUnsupported, + }}, + }, + }, + }, + rpcRes: map[string][][]string{ + "1": { + {"build:foo1", "override:foo2", "build:bar1", "override:bar2", "newstream:foo1", "newstream:bar1", "done:bar1", "done:foo1"}, + }, + }, + selectErr: errUnsupportedClientRouteAction.Error(), + }, + { + name: "route type RouteActionNonForwardingAction invalid for client", + ldsFilters: []xdsresource.HTTPFilter{ + {Name: "foo", Filter: &filterBuilder{path: &path}, Config: filterCfg{s: "foo1"}}, + }, + rtCfgUpdate: xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{ + { + Domains: []string{targetStr}, + Routes: []*xdsresource.Route{{ + Prefix: newStringP("1"), + WeightedClusters: map[string]xdsresource.WeightedCluster{ + "A": {Weight: 1}, + "B": {Weight: 1}, + }, + ActionType: xdsresource.RouteActionNonForwardingAction, + }}, + }, + }, + }, + rpcRes: map[string][][]string{ + "1": { + {"build:foo1", "override:foo2", "build:bar1", "override:bar2", "newstream:foo1", "newstream:bar1", "done:bar1", "done:foo1"}, + }, + }, + selectErr: errUnsupportedClientRouteAction.Error(), + }, + { + name: "NewStream error; ensure earlier interceptor Done is still called", + ldsFilters: []xdsresource.HTTPFilter{ + {Name: "foo", Filter: &filterBuilder{path: &path}, Config: filterCfg{s: "foo1"}}, + {Name: "bar", Filter: &filterBuilder{path: &path}, Config: filterCfg{s: "bar1", newStreamErr: errors.New("bar newstream err")}}, + routerFilter, + }, + rtCfgUpdate: xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{ + { + Domains: []string{targetStr}, + Routes: []*xdsresource.Route{{ + Prefix: newStringP("1"), + WeightedClusters: map[string]xdsresource.WeightedCluster{ + "A": {Weight: 1}, + "B": {Weight: 1}, + }, + ActionType: xdsresource.RouteActionRoute, + }}, + }, + }, + }, + rpcRes: map[string][][]string{ + "1": { + {"build:foo1", "build:bar1", "newstream:foo1", "newstream:bar1" /* */, "done:foo1"}, + }, + }, + newStreamErr: "bar newstream err", + }, + { + name: "all overrides", + ldsFilters: []xdsresource.HTTPFilter{ + {Name: "foo", Filter: &filterBuilder{path: &path}, Config: filterCfg{s: "foo1", newStreamErr: errors.New("this is overridden to nil")}}, + {Name: "bar", Filter: &filterBuilder{path: &path}, Config: filterCfg{s: "bar1"}}, + routerFilter, + }, + rtCfgUpdate: xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{ + { + Domains: []string{targetStr}, + Routes: []*xdsresource.Route{{ + Prefix: newStringP("1"), + WeightedClusters: map[string]xdsresource.WeightedCluster{ + "A": {Weight: 1}, + "B": {Weight: 1}, + }, + ActionType: xdsresource.RouteActionRoute, + }, { + Prefix: newStringP("2"), + WeightedClusters: map[string]xdsresource.WeightedCluster{ + "A": {Weight: 1}, + "B": {Weight: 1, HTTPFilterConfigOverride: map[string]httpfilter.FilterConfig{"foo": filterCfg{s: "foo4"}, "bar": filterCfg{s: "bar4"}}}, + }, + HTTPFilterConfigOverride: map[string]httpfilter.FilterConfig{"foo": filterCfg{s: "foo3"}, "bar": filterCfg{s: "bar3"}}, + ActionType: xdsresource.RouteActionRoute, + }}, + HTTPFilterConfigOverride: map[string]httpfilter.FilterConfig{"foo": filterCfg{s: "foo2"}, "bar": filterCfg{s: "bar2"}}, + }, + }, + }, + rpcRes: map[string][][]string{ + "1": { + {"build:foo1", "override:foo2", "build:bar1", "override:bar2", "newstream:foo1", "newstream:bar1", "done:bar1", "done:foo1"}, + {"build:foo1", "override:foo2", "build:bar1", "override:bar2", "newstream:foo1", "newstream:bar1", "done:bar1", "done:foo1"}, + }, + "2": { + {"build:foo1", "override:foo3", "build:bar1", "override:bar3", "newstream:foo1", "newstream:bar1", "done:bar1", "done:foo1"}, + {"build:foo1", "override:foo4", "build:bar1", "override:bar4", "newstream:foo1", "newstream:bar1", "done:bar1", "done:foo1"}, + {"build:foo1", "override:foo3", "build:bar1", "override:bar3", "newstream:foo1", "newstream:bar1", "done:bar1", "done:foo1"}, + {"build:foo1", "override:foo4", "build:bar1", "override:bar4", "newstream:foo1", "newstream:bar1", "done:bar1", "done:foo1"}, + }, + }, + }, + } + + for i, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + xdsR, xdsC, tcc, cancel := testSetup(t, setupOpts{target: target}) + defer xdsR.Close() + defer cancel() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + waitForWatchListener(ctx, t, xdsC, targetStr) + + xdsC.InvokeWatchListenerCallback(xdsresource.ListenerUpdate{ + RouteConfigName: routeStr, + HTTPFilters: tc.ldsFilters, + }, nil) + if i == 0 { + waitForWatchRouteConfig(ctx, t, xdsC, routeStr) + } + + defer func(oldNewWRR func() wrr.WRR) { newWRR = oldNewWRR }(newWRR) + newWRR = testutils.NewTestWRR + + // Invoke the watchAPI callback with a good service update and wait for the + // UpdateState method to be called on the ClientConn. + xdsC.InvokeWatchRouteConfigCallback("", tc.rtCfgUpdate, nil) + + gotState, err := tcc.stateCh.Receive(ctx) + if err != nil { + t.Fatalf("Error waiting for UpdateState to be called: %v", err) + } + rState := gotState.(resolver.State) + if err := rState.ServiceConfig.Err; err != nil { + t.Fatalf("ClientConn.UpdateState received error in service config: %v", rState.ServiceConfig.Err) + } + + cs := iresolver.GetConfigSelector(rState) + if cs == nil { + t.Fatal("received nil config selector") + } + + for method, wants := range tc.rpcRes { + // Order of wants is non-deterministic. + remainingWant := make([][]string, len(wants)) + copy(remainingWant, wants) + for n := range wants { + path = nil + + res, err := cs.SelectConfig(iresolver.RPCInfo{Method: method, Context: context.Background()}) + if tc.selectErr != "" { + if err == nil || !strings.Contains(err.Error(), tc.selectErr) { + t.Errorf("SelectConfig(_) = _, %v; want _, Contains(%v)", err, tc.selectErr) + } + if err == nil { + res.OnCommitted() + } + continue + } + if err != nil { + t.Fatalf("Unexpected error from cs.SelectConfig(_): %v", err) + } + var doneFunc func() + _, err = res.Interceptor.NewStream(context.Background(), iresolver.RPCInfo{}, func() {}, func(ctx context.Context, done func()) (iresolver.ClientStream, error) { + doneFunc = done + return nil, nil + }) + if tc.newStreamErr != "" { + if err == nil || !strings.Contains(err.Error(), tc.newStreamErr) { + t.Errorf("NewStream(...) = _, %v; want _, Contains(%v)", err, tc.newStreamErr) + } + if err == nil { + res.OnCommitted() + doneFunc() + } + continue + } + if err != nil { + t.Fatalf("unexpected error from Interceptor.NewStream: %v", err) + + } + res.OnCommitted() + doneFunc() + + // Confirm the desired path is found in remainingWant, and remove it. + pass := false + for i := range remainingWant { + if reflect.DeepEqual(path, remainingWant[i]) { + remainingWant[i] = remainingWant[len(remainingWant)-1] + remainingWant = remainingWant[:len(remainingWant)-1] + pass = true + break + } + } + if !pass { + t.Errorf("%q:%v - path:\n%v\nwant one of:\n%v", method, n, path, remainingWant) + } + } + } + }) + } +} + +func newDurationP(d time.Duration) *time.Duration { + return &d } diff --git a/xds/internal/server/conn_wrapper.go b/xds/internal/server/conn_wrapper.go new file mode 100644 index 000000000000..ec6da32fad18 --- /dev/null +++ b/xds/internal/server/conn_wrapper.go @@ -0,0 +1,165 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package server + +import ( + "errors" + "fmt" + "net" + "sync" + "time" + + "google.golang.org/grpc/credentials/tls/certprovider" + xdsinternal "google.golang.org/grpc/internal/credentials/xds" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" +) + +// connWrapper is a thin wrapper around a net.Conn returned by Accept(). It +// provides the following additional functionality: +// 1. A way to retrieve the configured deadline. This is required by the +// ServerHandshake() method of the xdsCredentials when it attempts to read +// key material from the certificate providers. +// 2. Implements the XDSHandshakeInfo() method used by the xdsCredentials to +// retrieve the configured certificate providers. +// 3. xDS filter_chain matching logic to select appropriate security +// configuration for the incoming connection. +type connWrapper struct { + net.Conn + + // The specific filter chain picked for handling this connection. + filterChain *xdsresource.FilterChain + + // A reference fo the listenerWrapper on which this connection was accepted. + parent *listenerWrapper + + // The certificate providers created for this connection. + rootProvider, identityProvider certprovider.Provider + + // The connection deadline as configured by the grpc.Server on the rawConn + // that is returned by a call to Accept(). This is set to the connection + // timeout value configured by the user (or to a default value) before + // initiating the transport credential handshake, and set to zero after + // completing the HTTP2 handshake. + deadlineMu sync.Mutex + deadline time.Time + + // The virtual hosts with matchable routes and instantiated HTTP Filters per + // route. + virtualHosts []xdsresource.VirtualHostWithInterceptors +} + +// VirtualHosts returns the virtual hosts to be used for server side routing. +func (c *connWrapper) VirtualHosts() []xdsresource.VirtualHostWithInterceptors { + return c.virtualHosts +} + +// SetDeadline makes a copy of the passed in deadline and forwards the call to +// the underlying rawConn. +func (c *connWrapper) SetDeadline(t time.Time) error { + c.deadlineMu.Lock() + c.deadline = t + c.deadlineMu.Unlock() + return c.Conn.SetDeadline(t) +} + +// GetDeadline returns the configured deadline. This will be invoked by the +// ServerHandshake() method of the XdsCredentials, which needs a deadline to +// pass to the certificate provider. +func (c *connWrapper) GetDeadline() time.Time { + c.deadlineMu.Lock() + t := c.deadline + c.deadlineMu.Unlock() + return t +} + +// XDSHandshakeInfo returns a HandshakeInfo with appropriate security +// configuration for this connection. This method is invoked by the +// ServerHandshake() method of the XdsCredentials. +func (c *connWrapper) XDSHandshakeInfo() (*xdsinternal.HandshakeInfo, error) { + // Ideally this should never happen, since xdsCredentials are the only ones + // which will invoke this method at handshake time. But to be on the safe + // side, we avoid acting on the security configuration received from the + // control plane when the user has not configured the use of xDS + // credentials, by checking the value of this flag. + if !c.parent.xdsCredsInUse { + return nil, errors.New("user has not configured xDS credentials") + } + + if c.filterChain.SecurityCfg == nil { + // If the security config is empty, this means that the control plane + // did not provide any security configuration and therefore we should + // return an empty HandshakeInfo here so that the xdsCreds can use the + // configured fallback credentials. + return xdsinternal.NewHandshakeInfo(nil, nil), nil + } + + cpc := c.parent.xdsC.BootstrapConfig().CertProviderConfigs + // Identity provider name is mandatory on the server-side, and this is + // enforced when the resource is received at the XDSClient layer. + secCfg := c.filterChain.SecurityCfg + ip, err := buildProviderFunc(cpc, secCfg.IdentityInstanceName, secCfg.IdentityCertName, true, false) + if err != nil { + return nil, err + } + // Root provider name is optional and required only when doing mTLS. + var rp certprovider.Provider + if instance, cert := secCfg.RootInstanceName, secCfg.RootCertName; instance != "" { + rp, err = buildProviderFunc(cpc, instance, cert, false, true) + if err != nil { + return nil, err + } + } + c.identityProvider = ip + c.rootProvider = rp + + xdsHI := xdsinternal.NewHandshakeInfo(c.rootProvider, c.identityProvider) + xdsHI.SetRequireClientCert(secCfg.RequireClientCert) + return xdsHI, nil +} + +// Close closes the providers and the underlying connection. +func (c *connWrapper) Close() error { + if c.identityProvider != nil { + c.identityProvider.Close() + } + if c.rootProvider != nil { + c.rootProvider.Close() + } + return c.Conn.Close() +} + +func buildProviderFunc(configs map[string]*certprovider.BuildableConfig, instanceName, certName string, wantIdentity, wantRoot bool) (certprovider.Provider, error) { + cfg, ok := configs[instanceName] + if !ok { + return nil, fmt.Errorf("certificate provider instance %q not found in bootstrap file", instanceName) + } + provider, err := cfg.Build(certprovider.BuildOptions{ + CertName: certName, + WantIdentity: wantIdentity, + WantRoot: wantRoot, + }) + if err != nil { + // This error is not expected since the bootstrap process parses the + // config and makes sure that it is acceptable to the plugin. Still, it + // is possible that the plugin parses the config successfully, but its + // Build() method errors out. + return nil, fmt.Errorf("failed to get security plugin instance (%+v): %v", cfg, err) + } + return provider, nil +} diff --git a/xds/internal/server/listener_wrapper.go b/xds/internal/server/listener_wrapper.go new file mode 100644 index 000000000000..9f5b2ecafe5f --- /dev/null +++ b/xds/internal/server/listener_wrapper.go @@ -0,0 +1,442 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package server contains internal server-side functionality used by the public +// facing xds package. +package server + +import ( + "errors" + "fmt" + "net" + "sync" + "sync/atomic" + "time" + "unsafe" + + "google.golang.org/grpc/backoff" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/grpclog" + internalbackoff "google.golang.org/grpc/internal/backoff" + "google.golang.org/grpc/internal/envconfig" + internalgrpclog "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/xds/internal/xdsclient/bootstrap" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" +) + +var ( + logger = grpclog.Component("xds") + + // Backoff strategy for temporary errors received from Accept(). If this + // needs to be configurable, we can inject it through ListenerWrapperParams. + bs = internalbackoff.Exponential{Config: backoff.Config{ + BaseDelay: 5 * time.Millisecond, + Multiplier: 2.0, + MaxDelay: 1 * time.Second, + }} + backoffFunc = bs.Backoff +) + +// ServingModeCallback is the callback that users can register to get notified +// about the server's serving mode changes. The callback is invoked with the +// address of the listener and its new mode. The err parameter is set to a +// non-nil error if the server has transitioned into not-serving mode. +type ServingModeCallback func(addr net.Addr, mode connectivity.ServingMode, err error) + +// DrainCallback is the callback that an xDS-enabled server registers to get +// notified about updates to the Listener configuration. The server is expected +// to gracefully shutdown existing connections, thereby forcing clients to +// reconnect and have the new configuration applied to the newly created +// connections. +type DrainCallback func(addr net.Addr) + +// XDSClient wraps the methods on the XDSClient which are required by +// the listenerWrapper. +type XDSClient interface { + WatchListener(string, func(xdsresource.ListenerUpdate, error)) func() + WatchRouteConfig(string, func(xdsresource.RouteConfigUpdate, error)) func() + BootstrapConfig() *bootstrap.Config +} + +// ListenerWrapperParams wraps parameters required to create a listenerWrapper. +type ListenerWrapperParams struct { + // Listener is the net.Listener passed by the user that is to be wrapped. + Listener net.Listener + // ListenerResourceName is the xDS Listener resource to request. + ListenerResourceName string + // XDSCredsInUse specifies whether or not the user expressed interest to + // receive security configuration from the control plane. + XDSCredsInUse bool + // XDSClient provides the functionality from the XDSClient required here. + XDSClient XDSClient + // ModeCallback is the callback to invoke when the serving mode changes. + ModeCallback ServingModeCallback + // DrainCallback is the callback to invoke when the Listener gets a LDS + // update. + DrainCallback DrainCallback +} + +// NewListenerWrapper creates a new listenerWrapper with params. It returns a +// net.Listener and a channel which is written to, indicating that the former is +// ready to be passed to grpc.Serve(). +// +// Only TCP listeners are supported. +func NewListenerWrapper(params ListenerWrapperParams) (net.Listener, <-chan struct{}) { + lw := &listenerWrapper{ + Listener: params.Listener, + name: params.ListenerResourceName, + xdsCredsInUse: params.XDSCredsInUse, + xdsC: params.XDSClient, + modeCallback: params.ModeCallback, + drainCallback: params.DrainCallback, + isUnspecifiedAddr: params.Listener.Addr().(*net.TCPAddr).IP.IsUnspecified(), + + mode: connectivity.ServingModeStarting, + closed: grpcsync.NewEvent(), + goodUpdate: grpcsync.NewEvent(), + ldsUpdateCh: make(chan ldsUpdateWithError, 1), + rdsUpdateCh: make(chan rdsHandlerUpdate, 1), + } + lw.logger = internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf("[xds-server-listener %p] ", lw)) + + // Serve() verifies that Addr() returns a valid TCPAddr. So, it is safe to + // ignore the error from SplitHostPort(). + lisAddr := lw.Listener.Addr().String() + lw.addr, lw.port, _ = net.SplitHostPort(lisAddr) + + lw.rdsHandler = newRDSHandler(lw.xdsC, lw.rdsUpdateCh) + lw.cancelWatch = lw.xdsC.WatchListener(lw.name, lw.handleListenerUpdate) + go lw.run() + return lw, lw.goodUpdate.Done() +} + +type ldsUpdateWithError struct { + update xdsresource.ListenerUpdate + err error +} + +// listenerWrapper wraps the net.Listener associated with the listening address +// passed to Serve(). It also contains all other state associated with this +// particular invocation of Serve(). +type listenerWrapper struct { + net.Listener + logger *internalgrpclog.PrefixLogger + + name string + xdsCredsInUse bool + xdsC XDSClient + cancelWatch func() + modeCallback ServingModeCallback + drainCallback DrainCallback + + // Set to true if the listener is bound to the IP_ANY address (which is + // "0.0.0.0" for IPv4 and "::" for IPv6). + isUnspecifiedAddr bool + // Listening address and port. Used to validate the socket address in the + // Listener resource received from the control plane. + addr, port string + + // This is used to notify that a good update has been received and that + // Serve() can be invoked on the underlying gRPC server. Using an event + // instead of a vanilla channel simplifies the update handler as it need not + // keep track of whether the received update is the first one or not. + goodUpdate *grpcsync.Event + // A small race exists in the XDSClient code between the receipt of an xDS + // response and the user cancelling the associated watch. In this window, + // the registered callback may be invoked after the watch is canceled, and + // the user is expected to work around this. This event signifies that the + // listener is closed (and hence the watch is cancelled), and we drop any + // updates received in the callback if this event has fired. + closed *grpcsync.Event + + // mu guards access to the current serving mode and the filter chains. The + // reason for using an rw lock here is that these fields are read in + // Accept() for all incoming connections, but writes happen rarely (when we + // get a Listener resource update). + mu sync.RWMutex + // Current serving mode. + mode connectivity.ServingMode + // Filter chains received as part of the last good update. + filterChains *xdsresource.FilterChainManager + + // rdsHandler is used for any dynamic RDS resources specified in a LDS + // update. + rdsHandler *rdsHandler + // rdsUpdates are the RDS resources received from the management + // server, keyed on the RouteName of the RDS resource. + rdsUpdates unsafe.Pointer // map[string]xdsclient.RouteConfigUpdate + // ldsUpdateCh is a channel for XDSClient LDS updates. + ldsUpdateCh chan ldsUpdateWithError + // rdsUpdateCh is a channel for XDSClient RDS updates. + rdsUpdateCh chan rdsHandlerUpdate +} + +// Accept blocks on an Accept() on the underlying listener, and wraps the +// returned net.connWrapper with the configured certificate providers. +func (l *listenerWrapper) Accept() (net.Conn, error) { + var retries int + for { + conn, err := l.Listener.Accept() + if err != nil { + // Temporary() method is implemented by certain error types returned + // from the net package, and it is useful for us to not shutdown the + // server in these conditions. The listen queue being full is one + // such case. + if ne, ok := err.(interface{ Temporary() bool }); !ok || !ne.Temporary() { + return nil, err + } + retries++ + timer := time.NewTimer(backoffFunc(retries)) + select { + case <-timer.C: + case <-l.closed.Done(): + timer.Stop() + // Continuing here will cause us to call Accept() again + // which will return a non-temporary error. + continue + } + continue + } + // Reset retries after a successful Accept(). + retries = 0 + + // Since the net.Conn represents an incoming connection, the source and + // destination address can be retrieved from the local address and + // remote address of the net.Conn respectively. + destAddr, ok1 := conn.LocalAddr().(*net.TCPAddr) + srcAddr, ok2 := conn.RemoteAddr().(*net.TCPAddr) + if !ok1 || !ok2 { + // If the incoming connection is not a TCP connection, which is + // really unexpected since we check whether the provided listener is + // a TCP listener in Serve(), we return an error which would cause + // us to stop serving. + return nil, fmt.Errorf("received connection with non-TCP address (local: %T, remote %T)", conn.LocalAddr(), conn.RemoteAddr()) + } + + l.mu.RLock() + if l.mode == connectivity.ServingModeNotServing { + // Close connections as soon as we accept them when we are in + // "not-serving" mode. Since we accept a net.Listener from the user + // in Serve(), we cannot close the listener when we move to + // "not-serving". Closing the connection immediately upon accepting + // is one of the other ways to implement the "not-serving" mode as + // outlined in gRFC A36. + l.mu.RUnlock() + conn.Close() + continue + } + fc, err := l.filterChains.Lookup(xdsresource.FilterChainLookupParams{ + IsUnspecifiedListener: l.isUnspecifiedAddr, + DestAddr: destAddr.IP, + SourceAddr: srcAddr.IP, + SourcePort: srcAddr.Port, + }) + l.mu.RUnlock() + if err != nil { + // When a matching filter chain is not found, we close the + // connection right away, but do not return an error back to + // `grpc.Serve()` from where this Accept() was invoked. Returning an + // error to `grpc.Serve()` causes the server to shutdown. If we want + // to avoid the server from shutting down, we would need to return + // an error type which implements the `Temporary() bool` method, + // which is invoked by `grpc.Serve()` to see if the returned error + // represents a temporary condition. In the case of a temporary + // error, `grpc.Serve()` method sleeps for a small duration and + // therefore ends up blocking all connection attempts during that + // time frame, which is also not ideal for an error like this. + l.logger.Warningf("Connection from %s to %s failed to find any matching filter chain", conn.RemoteAddr().String(), conn.LocalAddr().String()) + conn.Close() + continue + } + if !envconfig.XDSRBAC { + return &connWrapper{Conn: conn, filterChain: fc, parent: l}, nil + } + var rc xdsresource.RouteConfigUpdate + if fc.InlineRouteConfig != nil { + rc = *fc.InlineRouteConfig + } else { + rcPtr := atomic.LoadPointer(&l.rdsUpdates) + rcuPtr := (*map[string]xdsresource.RouteConfigUpdate)(rcPtr) + // This shouldn't happen, but this error protects against a panic. + if rcuPtr == nil { + return nil, errors.New("route configuration pointer is nil") + } + rcu := *rcuPtr + rc = rcu[fc.RouteConfigName] + } + // The filter chain will construct a usuable route table on each + // connection accept. This is done because preinstantiating every route + // table before it is needed for a connection would potentially lead to + // a lot of cpu time and memory allocated for route tables that will + // never be used. There was also a thought to cache this configuration, + // and reuse it for the next accepted connection. However, this would + // lead to a lot of code complexity (RDS Updates for a given route name + // can come it at any time), and connections aren't accepted too often, + // so this reinstantation of the Route Configuration is an acceptable + // tradeoff for simplicity. + vhswi, err := fc.ConstructUsableRouteConfiguration(rc) + if err != nil { + l.logger.Warningf("Failed to construct usable route configuration: %v", err) + conn.Close() + continue + } + return &connWrapper{Conn: conn, filterChain: fc, parent: l, virtualHosts: vhswi}, nil + } +} + +// Close closes the underlying listener. It also cancels the xDS watch +// registered in Serve() and closes any certificate provider instances created +// based on security configuration received in the LDS response. +func (l *listenerWrapper) Close() error { + l.closed.Fire() + l.Listener.Close() + if l.cancelWatch != nil { + l.cancelWatch() + } + l.rdsHandler.close() + return nil +} + +// run is a long running goroutine which handles all xds updates. LDS and RDS +// push updates onto a channel which is read and acted upon from this goroutine. +func (l *listenerWrapper) run() { + for { + select { + case <-l.closed.Done(): + return + case u := <-l.ldsUpdateCh: + l.handleLDSUpdate(u) + case u := <-l.rdsUpdateCh: + l.handleRDSUpdate(u) + } + } +} + +// handleLDSUpdate is the callback which handles LDS Updates. It writes the +// received update to the update channel, which is picked up by the run +// goroutine. +func (l *listenerWrapper) handleListenerUpdate(update xdsresource.ListenerUpdate, err error) { + if l.closed.HasFired() { + l.logger.Warningf("Resource %q received update: %v with error: %v, after listener was closed", l.name, update, err) + return + } + // Remove any existing entry in ldsUpdateCh and replace with the new one, as the only update + // listener cares about is most recent update. + select { + case <-l.ldsUpdateCh: + default: + } + l.ldsUpdateCh <- ldsUpdateWithError{update: update, err: err} +} + +// handleRDSUpdate handles a full rds update from rds handler. On a successful +// update, the server will switch to ServingModeServing as the full +// configuration (both LDS and RDS) has been received. +func (l *listenerWrapper) handleRDSUpdate(update rdsHandlerUpdate) { + if l.closed.HasFired() { + l.logger.Warningf("RDS received update: %v with error: %v, after listener was closed", update.updates, update.err) + return + } + if update.err != nil { + l.logger.Warningf("Received error for rds names specified in resource %q: %+v", l.name, update.err) + if xdsresource.ErrType(update.err) == xdsresource.ErrorTypeResourceNotFound { + l.switchMode(nil, connectivity.ServingModeNotServing, update.err) + } + // For errors which are anything other than "resource-not-found", we + // continue to use the old configuration. + return + } + atomic.StorePointer(&l.rdsUpdates, unsafe.Pointer(&update.updates)) + + l.switchMode(l.filterChains, connectivity.ServingModeServing, nil) + l.goodUpdate.Fire() +} + +func (l *listenerWrapper) handleLDSUpdate(update ldsUpdateWithError) { + if update.err != nil { + l.logger.Warningf("Received error for resource %q: %+v", l.name, update.err) + if xdsresource.ErrType(update.err) == xdsresource.ErrorTypeResourceNotFound { + l.switchMode(nil, connectivity.ServingModeNotServing, update.err) + } + // For errors which are anything other than "resource-not-found", we + // continue to use the old configuration. + return + } + + // Make sure that the socket address on the received Listener resource + // matches the address of the net.Listener passed to us by the user. This + // check is done here instead of at the XDSClient layer because of the + // following couple of reasons: + // - XDSClient cannot know the listening address of every listener in the + // system, and hence cannot perform this check. + // - this is a very context-dependent check and only the server has the + // appropriate context to perform this check. + // + // What this means is that the XDSClient has ACKed a resource which can push + // the server into a "not serving" mode. This is not ideal, but this is + // what we have decided to do. See gRPC A36 for more details. + ilc := update.update.InboundListenerCfg + if ilc.Address != l.addr || ilc.Port != l.port { + l.switchMode(nil, connectivity.ServingModeNotServing, fmt.Errorf("address (%s:%s) in Listener update does not match listening address: (%s:%s)", ilc.Address, ilc.Port, l.addr, l.port)) + return + } + + // "Updates to a Listener cause all older connections on that Listener to be + // gracefully shut down with a grace period of 10 minutes for long-lived + // RPC's, such that clients will reconnect and have the updated + // configuration apply." - A36 Note that this is not the same as moving the + // Server's state to ServingModeNotServing. That prevents new connections + // from being accepted, whereas here we simply want the clients to reconnect + // to get the updated configuration. + if envconfig.XDSRBAC { + if l.drainCallback != nil { + l.drainCallback(l.Listener.Addr()) + } + } + l.rdsHandler.updateRouteNamesToWatch(ilc.FilterChains.RouteConfigNames) + // If there are no dynamic RDS Configurations still needed to be received + // from the management server, this listener has all the configuration + // needed, and is ready to serve. + if len(ilc.FilterChains.RouteConfigNames) == 0 { + l.switchMode(ilc.FilterChains, connectivity.ServingModeServing, nil) + l.goodUpdate.Fire() + } +} + +// switchMode updates the value of serving mode and filter chains stored in the +// listenerWrapper. And if the serving mode has changed, it invokes the +// registered mode change callback. +func (l *listenerWrapper) switchMode(fcs *xdsresource.FilterChainManager, newMode connectivity.ServingMode, err error) { + l.mu.Lock() + defer l.mu.Unlock() + + l.filterChains = fcs + if l.mode == newMode && l.mode == connectivity.ServingModeServing { + // Redundant updates are suppressed only when we are SERVING and the new + // mode is also SERVING. In the other case (where we are NOT_SERVING and the + // new mode is also NOT_SERVING), the update is not suppressed as: + // 1. the error may have change + // 2. it provides a timestamp of the last backoff attempt + return + } + l.mode = newMode + if l.modeCallback != nil { + l.modeCallback(l.Listener.Addr(), newMode, err) + } +} diff --git a/xds/internal/server/listener_wrapper_test.go b/xds/internal/server/listener_wrapper_test.go new file mode 100644 index 000000000000..7d246f6373eb --- /dev/null +++ b/xds/internal/server/listener_wrapper_test.go @@ -0,0 +1,486 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package server + +import ( + "context" + "errors" + "net" + "strconv" + "testing" + "time" + + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/xds/internal/testutils/fakeclient" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" + wrapperspb "github.com/golang/protobuf/ptypes/wrappers" + + _ "google.golang.org/grpc/xds/internal/httpfilter/router" +) + +const ( + fakeListenerHost = "0.0.0.0" + fakeListenerPort = 50051 + testListenerResourceName = "lds.target.1.2.3.4:1111" + defaultTestTimeout = 1 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond +) + +var listenerWithRouteConfiguration = &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{ + { + AddressPrefix: "192.168.0.0", + PrefixLen: &wrapperspb.UInt32Value{ + Value: uint32(16), + }, + }, + }, + SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK, + SourcePrefixRanges: []*v3corepb.CidrRange{ + { + AddressPrefix: "192.168.0.0", + PrefixLen: &wrapperspb.UInt32Value{ + Value: uint32(16), + }, + }, + }, + SourcePorts: []uint32{80}, + }, + Filters: []*v3listenerpb.Filter{ + { + Name: "filter-1", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{ + Rds: &v3httppb.Rds{ + ConfigSource: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}, + }, + RouteConfigName: "route-1", + }, + }, + HttpFilters: []*v3httppb.HttpFilter{e2e.RouterHTTPFilter}, + }), + }, + }, + }, + }, + }, +} + +var listenerWithFilterChains = &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{ + { + AddressPrefix: "192.168.0.0", + PrefixLen: &wrapperspb.UInt32Value{ + Value: uint32(16), + }, + }, + }, + SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK, + SourcePrefixRanges: []*v3corepb.CidrRange{ + { + AddressPrefix: "192.168.0.0", + PrefixLen: &wrapperspb.UInt32Value{ + Value: uint32(16), + }, + }, + }, + SourcePorts: []uint32{80}, + }, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: "identityPluginInstance", + CertificateName: "identityCertName", + }, + }, + }), + }, + }, + Filters: []*v3listenerpb.Filter{ + { + Name: "filter-1", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: &v3routepb.RouteConfiguration{ + Name: "routeName", + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{"lds.target.good:3333"}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}, + }, + Action: &v3routepb.Route_NonForwardingAction{}, + }}}}}, + }, + HttpFilters: []*v3httppb.HttpFilter{e2e.RouterHTTPFilter}, + }), + }, + }, + }, + }, + }, +} + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +type tempError struct{} + +func (tempError) Error() string { + return "listenerWrapper test temporary error" +} + +func (tempError) Temporary() bool { + return true +} + +// connAndErr wraps a net.Conn and an error. +type connAndErr struct { + conn net.Conn + err error +} + +// fakeListener allows the user to inject conns returned by Accept(). +type fakeListener struct { + acceptCh chan connAndErr + closeCh *testutils.Channel +} + +func (fl *fakeListener) Accept() (net.Conn, error) { + cne, ok := <-fl.acceptCh + if !ok { + return nil, errors.New("a non-temporary error") + } + return cne.conn, cne.err +} + +func (fl *fakeListener) Close() error { + fl.closeCh.Send(nil) + return nil +} + +func (fl *fakeListener) Addr() net.Addr { + return &net.TCPAddr{ + IP: net.IPv4(0, 0, 0, 0), + Port: fakeListenerPort, + } +} + +// fakeConn overrides LocalAddr, RemoteAddr and Close methods. +type fakeConn struct { + net.Conn + local, remote net.Addr + closeCh *testutils.Channel +} + +func (fc *fakeConn) LocalAddr() net.Addr { + return fc.local +} + +func (fc *fakeConn) RemoteAddr() net.Addr { + return fc.remote +} + +func (fc *fakeConn) Close() error { + fc.closeCh.Send(nil) + return nil +} + +func newListenerWrapper(t *testing.T) (*listenerWrapper, <-chan struct{}, *fakeclient.Client, *fakeListener, func()) { + t.Helper() + + // Create a listener wrapper with a fake listener and fake XDSClient and + // verify that it extracts the host and port from the passed in listener. + lis := &fakeListener{ + acceptCh: make(chan connAndErr, 1), + closeCh: testutils.NewChannel(), + } + xdsC := fakeclient.NewClient() + lParams := ListenerWrapperParams{ + Listener: lis, + ListenerResourceName: testListenerResourceName, + XDSClient: xdsC, + } + l, readyCh := NewListenerWrapper(lParams) + if l == nil { + t.Fatalf("NewListenerWrapper(%+v) returned nil", lParams) + } + lw, ok := l.(*listenerWrapper) + if !ok { + t.Fatalf("NewListenerWrapper(%+v) returned listener of type %T want *listenerWrapper", lParams, l) + } + if lw.addr != fakeListenerHost || lw.port != strconv.Itoa(fakeListenerPort) { + t.Fatalf("listenerWrapper has host:port %s:%s, want %s:%d", lw.addr, lw.port, fakeListenerHost, fakeListenerPort) + } + return lw, readyCh, xdsC, lis, func() { l.Close() } +} + +func (s) TestNewListenerWrapper(t *testing.T) { + _, readyCh, xdsC, _, cleanup := newListenerWrapper(t) + defer cleanup() + + // Verify that the listener wrapper registers a listener watch for the + // expected Listener resource name. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + name, err := xdsC.WaitForWatchListener(ctx) + if err != nil { + t.Fatalf("error when waiting for a watch on a Listener resource: %v", err) + } + if name != testListenerResourceName { + t.Fatalf("listenerWrapper registered a lds watch on %s, want %s", name, testListenerResourceName) + } + + // Push an error to the listener update handler. + xdsC.InvokeWatchListenerCallback(xdsresource.ListenerUpdate{}, errors.New("bad listener update")) + timer := time.NewTimer(defaultTestShortTimeout) + select { + case <-timer.C: + timer.Stop() + case <-readyCh: + t.Fatalf("ready channel written to after receipt of a bad Listener update") + } + + fcm, err := xdsresource.NewFilterChainManager(listenerWithFilterChains) + if err != nil { + t.Fatalf("xdsclient.NewFilterChainManager() failed with error: %v", err) + } + + // Push an update whose address does not match the address to which our + // listener is bound, and verify that the ready channel is not written to. + xdsC.InvokeWatchListenerCallback(xdsresource.ListenerUpdate{ + InboundListenerCfg: &xdsresource.InboundListenerConfig{ + Address: "10.0.0.1", + Port: "50051", + FilterChains: fcm, + }}, nil) + timer = time.NewTimer(defaultTestShortTimeout) + select { + case <-timer.C: + timer.Stop() + case <-readyCh: + t.Fatalf("ready channel written to after receipt of a bad Listener update") + } + + // Push a good update, and verify that the ready channel is written to. + // Since there are no dynamic RDS updates needed to be received, the + // ListenerWrapper does not have to wait for anything else before telling + // that it is ready. + xdsC.InvokeWatchListenerCallback(xdsresource.ListenerUpdate{ + InboundListenerCfg: &xdsresource.InboundListenerConfig{ + Address: fakeListenerHost, + Port: strconv.Itoa(fakeListenerPort), + FilterChains: fcm, + }}, nil) + + select { + case <-ctx.Done(): + t.Fatalf("timeout waiting for the ready channel to be written to after receipt of a good Listener update") + case <-readyCh: + } +} + +// TestNewListenerWrapperWithRouteUpdate tests the scenario where the listener +// gets built, starts a watch, that watch returns a list of Route Names to +// return, than receives an update from the rds handler. Only after receiving +// the update from the rds handler should it move the server to +// ServingModeServing. +func (s) TestNewListenerWrapperWithRouteUpdate(t *testing.T) { + oldRBAC := envconfig.XDSRBAC + envconfig.XDSRBAC = true + defer func() { + envconfig.XDSRBAC = oldRBAC + }() + _, readyCh, xdsC, _, cleanup := newListenerWrapper(t) + defer cleanup() + + // Verify that the listener wrapper registers a listener watch for the + // expected Listener resource name. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + name, err := xdsC.WaitForWatchListener(ctx) + if err != nil { + t.Fatalf("error when waiting for a watch on a Listener resource: %v", err) + } + if name != testListenerResourceName { + t.Fatalf("listenerWrapper registered a lds watch on %s, want %s", name, testListenerResourceName) + } + fcm, err := xdsresource.NewFilterChainManager(listenerWithRouteConfiguration) + if err != nil { + t.Fatalf("xdsclient.NewFilterChainManager() failed with error: %v", err) + } + + // Push a good update which contains a Filter Chain that specifies dynamic + // RDS Resources that need to be received. This should ping rds handler + // about which rds names to start, which will eventually start a watch on + // xds client for rds name "route-1". + xdsC.InvokeWatchListenerCallback(xdsresource.ListenerUpdate{ + InboundListenerCfg: &xdsresource.InboundListenerConfig{ + Address: fakeListenerHost, + Port: strconv.Itoa(fakeListenerPort), + FilterChains: fcm, + }}, nil) + + // This should start a watch on xds client for rds name "route-1". + routeName, err := xdsC.WaitForWatchRouteConfig(ctx) + if err != nil { + t.Fatalf("error when waiting for a watch on a Route resource: %v", err) + } + if routeName != "route-1" { + t.Fatalf("listenerWrapper registered a lds watch on %s, want %s", routeName, "route-1") + } + + // This shouldn't invoke good update channel, as has not received rds updates yet. + timer := time.NewTimer(defaultTestShortTimeout) + select { + case <-timer.C: + timer.Stop() + case <-readyCh: + t.Fatalf("ready channel written to without rds configuration specified") + } + + // Invoke rds callback for the started rds watch. This valid rds callback + // should trigger the listener wrapper to fire GoodUpdate, as it has + // received both it's LDS Configuration and also RDS Configuration, + // specified in LDS Configuration. + xdsC.InvokeWatchRouteConfigCallback("route-1", xdsresource.RouteConfigUpdate{}, nil) + + // All of the xDS updates have completed, so can expect to send a ping on + // good update channel. + select { + case <-ctx.Done(): + t.Fatalf("timeout waiting for the ready channel to be written to after receipt of a good rds update") + case <-readyCh: + } +} + +func (s) TestListenerWrapper_Accept(t *testing.T) { + boCh := testutils.NewChannel() + origBackoffFunc := backoffFunc + backoffFunc = func(v int) time.Duration { + boCh.Send(v) + return 0 + } + defer func() { backoffFunc = origBackoffFunc }() + + lw, readyCh, xdsC, lis, cleanup := newListenerWrapper(t) + defer cleanup() + + // Push a good update with a filter chain which accepts local connections on + // 192.168.0.0/16 subnet and port 80. + fcm, err := xdsresource.NewFilterChainManager(listenerWithFilterChains) + if err != nil { + t.Fatalf("xdsclient.NewFilterChainManager() failed with error: %v", err) + } + xdsC.InvokeWatchListenerCallback(xdsresource.ListenerUpdate{ + InboundListenerCfg: &xdsresource.InboundListenerConfig{ + Address: fakeListenerHost, + Port: strconv.Itoa(fakeListenerPort), + FilterChains: fcm, + }}, nil) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + defer close(lis.acceptCh) + select { + case <-ctx.Done(): + t.Fatalf("timeout waiting for the ready channel to be written to after receipt of a good Listener update") + case <-readyCh: + } + + // Push a non-temporary error into Accept(). + nonTempErr := errors.New("a non-temporary error") + lis.acceptCh <- connAndErr{err: nonTempErr} + if _, err := lw.Accept(); err != nonTempErr { + t.Fatalf("listenerWrapper.Accept() returned error: %v, want: %v", err, nonTempErr) + } + + // Invoke Accept() in a goroutine since we expect it to swallow: + // 1. temporary errors returned from the underlying listener + // 2. errors related to finding a matching filter chain for the incoming + // connection. + errCh := testutils.NewChannel() + go func() { + conn, err := lw.Accept() + if err != nil { + errCh.Send(err) + return + } + if _, ok := conn.(*connWrapper); !ok { + errCh.Send(errors.New("listenerWrapper.Accept() returned a Conn of type %T, want *connWrapper")) + return + } + errCh.Send(nil) + }() + + // Push a temporary error into Accept() and verify that it backs off. + lis.acceptCh <- connAndErr{err: tempError{}} + if _, err := boCh.Receive(ctx); err != nil { + t.Fatalf("error when waiting for Accept() to backoff on temporary errors: %v", err) + } + + // Push a fakeConn which does not match any filter chains configured on the + // received Listener resource. Verify that the conn is closed. + fc := &fakeConn{ + local: &net.TCPAddr{IP: net.IPv4(192, 168, 1, 2), Port: 79}, + remote: &net.TCPAddr{IP: net.IPv4(10, 1, 1, 1), Port: 80}, + closeCh: testutils.NewChannel(), + } + lis.acceptCh <- connAndErr{conn: fc} + if _, err := fc.closeCh.Receive(ctx); err != nil { + t.Fatalf("error when waiting for conn to be closed on no filter chain match: %v", err) + } + + // Push a fakeConn which matches the filter chains configured on the + // received Listener resource. Verify that Accept() returns. + fc = &fakeConn{ + local: &net.TCPAddr{IP: net.IPv4(192, 168, 1, 2)}, + remote: &net.TCPAddr{IP: net.IPv4(192, 168, 1, 2), Port: 80}, + closeCh: testutils.NewChannel(), + } + lis.acceptCh <- connAndErr{conn: fc} + if _, err := errCh.Receive(ctx); err != nil { + t.Fatalf("error when waiting for Accept() to return the conn on filter chain match: %v", err) + } +} diff --git a/xds/internal/server/rds_handler.go b/xds/internal/server/rds_handler.go new file mode 100644 index 000000000000..722748cbd526 --- /dev/null +++ b/xds/internal/server/rds_handler.go @@ -0,0 +1,133 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package server + +import ( + "sync" + + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" +) + +// rdsHandlerUpdate wraps the full RouteConfigUpdate that are dynamically +// queried for a given server side listener. +type rdsHandlerUpdate struct { + updates map[string]xdsresource.RouteConfigUpdate + err error +} + +// rdsHandler handles any RDS queries that need to be started for a given server +// side listeners Filter Chains (i.e. not inline). +type rdsHandler struct { + xdsC XDSClient + + mu sync.Mutex + updates map[string]xdsresource.RouteConfigUpdate + cancels map[string]func() + + // For a rdsHandler update, the only update wrapped listener cares about is + // most recent one, so this channel will be opportunistically drained before + // sending any new updates. + updateChannel chan rdsHandlerUpdate +} + +// newRDSHandler creates a new rdsHandler to watch for RDS resources. +// listenerWrapper updates the list of route names to watch by calling +// updateRouteNamesToWatch() upon receipt of new Listener configuration. +func newRDSHandler(xdsC XDSClient, ch chan rdsHandlerUpdate) *rdsHandler { + return &rdsHandler{ + xdsC: xdsC, + updateChannel: ch, + updates: make(map[string]xdsresource.RouteConfigUpdate), + cancels: make(map[string]func()), + } +} + +// updateRouteNamesToWatch handles a list of route names to watch for a given +// server side listener (if a filter chain specifies dynamic RDS configuration). +// This function handles all the logic with respect to any routes that may have +// been added or deleted as compared to what was previously present. +func (rh *rdsHandler) updateRouteNamesToWatch(routeNamesToWatch map[string]bool) { + rh.mu.Lock() + defer rh.mu.Unlock() + // Add and start watches for any routes for any new routes in + // routeNamesToWatch. + for routeName := range routeNamesToWatch { + if _, ok := rh.cancels[routeName]; !ok { + func(routeName string) { + rh.cancels[routeName] = rh.xdsC.WatchRouteConfig(routeName, func(update xdsresource.RouteConfigUpdate, err error) { + rh.handleRouteUpdate(routeName, update, err) + }) + }(routeName) + } + } + + // Delete and cancel watches for any routes from persisted routeNamesToWatch + // that are no longer present. + for routeName := range rh.cancels { + if _, ok := routeNamesToWatch[routeName]; !ok { + rh.cancels[routeName]() + delete(rh.cancels, routeName) + delete(rh.updates, routeName) + } + } + + // If the full list (determined by length) of updates are now successfully + // updated, the listener is ready to be updated. + if len(rh.updates) == len(rh.cancels) && len(routeNamesToWatch) != 0 { + drainAndPush(rh.updateChannel, rdsHandlerUpdate{updates: rh.updates}) + } +} + +// handleRouteUpdate persists the route config for a given route name, and also +// sends an update to the Listener Wrapper on an error received or if the rds +// handler has a full collection of updates. +func (rh *rdsHandler) handleRouteUpdate(routeName string, update xdsresource.RouteConfigUpdate, err error) { + if err != nil { + drainAndPush(rh.updateChannel, rdsHandlerUpdate{err: err}) + return + } + rh.mu.Lock() + defer rh.mu.Unlock() + rh.updates[routeName] = update + + // If the full list (determined by length) of updates have successfully + // updated, the listener is ready to be updated. + if len(rh.updates) == len(rh.cancels) { + drainAndPush(rh.updateChannel, rdsHandlerUpdate{updates: rh.updates}) + } +} + +func drainAndPush(ch chan rdsHandlerUpdate, update rdsHandlerUpdate) { + select { + case <-ch: + default: + } + ch <- update +} + +// close() is meant to be called by wrapped listener when the wrapped listener +// is closed, and it cleans up resources by canceling all the active RDS +// watches. +func (rh *rdsHandler) close() { + rh.mu.Lock() + defer rh.mu.Unlock() + for _, cancel := range rh.cancels { + cancel() + } +} diff --git a/xds/internal/server/rds_handler_test.go b/xds/internal/server/rds_handler_test.go new file mode 100644 index 000000000000..fc622851cfa2 --- /dev/null +++ b/xds/internal/server/rds_handler_test.go @@ -0,0 +1,401 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package server + +import ( + "context" + "errors" + "fmt" + "testing" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/xds/internal/testutils/fakeclient" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" +) + +const ( + route1 = "route1" + route2 = "route2" + route3 = "route3" +) + +// setupTests creates a rds handler with a fake xds client for control over the +// xds client. +func setupTests() (*rdsHandler, *fakeclient.Client, chan rdsHandlerUpdate) { + xdsC := fakeclient.NewClient() + ch := make(chan rdsHandlerUpdate, 1) + rh := newRDSHandler(xdsC, ch) + return rh, xdsC, ch +} + +// waitForFuncWithNames makes sure that a blocking function returns the correct +// set of names, where order doesn't matter. This takes away nondeterminism from +// ranging through a map. +func waitForFuncWithNames(ctx context.Context, f func(context.Context) (string, error), names ...string) error { + wantNames := make(map[string]bool, len(names)) + for _, name := range names { + wantNames[name] = true + } + gotNames := make(map[string]bool, len(names)) + for range wantNames { + name, err := f(ctx) + if err != nil { + return err + } + gotNames[name] = true + } + if !cmp.Equal(gotNames, wantNames) { + return fmt.Errorf("got routeNames %v, want %v", gotNames, wantNames) + } + return nil +} + +// TestSuccessCaseOneRDSWatch tests the simplest scenario: the rds handler +// receives a single route name, starts a watch for that route name, gets a +// successful update, and then writes an update to the update channel for +// listener to pick up. +func (s) TestSuccessCaseOneRDSWatch(t *testing.T) { + rh, fakeClient, ch := setupTests() + // When you first update the rds handler with a list of a single Route names + // that needs dynamic RDS Configuration, this Route name has not been seen + // before, so the RDS Handler should start a watch on that RouteName. + rh.updateRouteNamesToWatch(map[string]bool{route1: true}) + // The RDS Handler should start a watch for that routeName. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + gotRoute, err := fakeClient.WaitForWatchRouteConfig(ctx) + if err != nil { + t.Fatalf("xdsClient.WatchRDS failed with error: %v", err) + } + if gotRoute != route1 { + t.Fatalf("xdsClient.WatchRDS called for route: %v, want %v", gotRoute, route1) + } + rdsUpdate := xdsresource.RouteConfigUpdate{} + // Invoke callback with the xds client with a certain route update. Due to + // this route update updating every route name that rds handler handles, + // this should write to the update channel to send to the listener. + fakeClient.InvokeWatchRouteConfigCallback(route1, rdsUpdate, nil) + rhuWant := map[string]xdsresource.RouteConfigUpdate{route1: rdsUpdate} + select { + case rhu := <-ch: + if diff := cmp.Diff(rhu.updates, rhuWant); diff != "" { + t.Fatalf("got unexpected route update, diff (-got, +want): %v", diff) + } + case <-ctx.Done(): + t.Fatal("Timed out waiting for update from update channel.") + } + // Close the rds handler. This is meant to be called when the lis wrapper is + // closed, and the call should cancel all the watches present (for this + // test, a single watch). + rh.close() + routeNameDeleted, err := fakeClient.WaitForCancelRouteConfigWatch(ctx) + if err != nil { + t.Fatalf("xdsClient.CancelRDS failed with error: %v", err) + } + if routeNameDeleted != route1 { + t.Fatalf("xdsClient.CancelRDS called for route %v, want %v", routeNameDeleted, route1) + } +} + +// TestSuccessCaseTwoUpdates tests the case where the rds handler receives an +// update with a single Route, then receives a second update with two routes. +// The handler should start a watch for the added route, and if received a RDS +// update for that route it should send an update with both RDS updates present. +func (s) TestSuccessCaseTwoUpdates(t *testing.T) { + rh, fakeClient, ch := setupTests() + + rh.updateRouteNamesToWatch(map[string]bool{route1: true}) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + gotRoute, err := fakeClient.WaitForWatchRouteConfig(ctx) + if err != nil { + t.Fatalf("xdsClient.WatchRDS failed with error: %v", err) + } + if gotRoute != route1 { + t.Fatalf("xdsClient.WatchRDS called for route: %v, want %v", gotRoute, route1) + } + + // Update the RDSHandler with route names which adds a route name to watch. + // This should trigger the RDSHandler to start a watch for the added route + // name to watch. + rh.updateRouteNamesToWatch(map[string]bool{route1: true, route2: true}) + gotRoute, err = fakeClient.WaitForWatchRouteConfig(ctx) + if err != nil { + t.Fatalf("xdsClient.WatchRDS failed with error: %v", err) + } + if gotRoute != route2 { + t.Fatalf("xdsClient.WatchRDS called for route: %v, want %v", gotRoute, route2) + } + + // Invoke the callback with an update for route 1. This shouldn't cause the + // handler to write an update, as it has not received RouteConfigurations + // for every RouteName. + rdsUpdate1 := xdsresource.RouteConfigUpdate{} + fakeClient.InvokeWatchRouteConfigCallback(route1, rdsUpdate1, nil) + + // The RDS Handler should not send an update. + sCtx, sCtxCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCtxCancel() + select { + case <-ch: + t.Fatal("RDS Handler wrote an update to updateChannel when it shouldn't have, as each route name has not received an update yet") + case <-sCtx.Done(): + } + + // Invoke the callback with an update for route 2. This should cause the + // handler to write an update, as it has received RouteConfigurations for + // every RouteName. + rdsUpdate2 := xdsresource.RouteConfigUpdate{} + fakeClient.InvokeWatchRouteConfigCallback(route2, rdsUpdate2, nil) + // The RDS Handler should then update the listener wrapper with an update + // with two route configurations, as both route names the RDS Handler handles + // have received an update. + rhuWant := map[string]xdsresource.RouteConfigUpdate{route1: rdsUpdate1, route2: rdsUpdate2} + select { + case rhu := <-ch: + if diff := cmp.Diff(rhu.updates, rhuWant); diff != "" { + t.Fatalf("got unexpected route update, diff (-got, +want): %v", diff) + } + case <-ctx.Done(): + t.Fatal("Timed out waiting for the rds handler update to be written to the update buffer.") + } + + // Close the rds handler. This is meant to be called when the lis wrapper is + // closed, and the call should cancel all the watches present (for this + // test, two watches on route1 and route2). + rh.close() + if err = waitForFuncWithNames(ctx, fakeClient.WaitForCancelRouteConfigWatch, route1, route2); err != nil { + t.Fatalf("Error while waiting for names: %v", err) + } +} + +// TestSuccessCaseDeletedRoute tests the case where the rds handler receives an +// update with two routes, then receives an update with only one route. The RDS +// Handler is expected to cancel the watch for the route no longer present. +func (s) TestSuccessCaseDeletedRoute(t *testing.T) { + rh, fakeClient, ch := setupTests() + + rh.updateRouteNamesToWatch(map[string]bool{route1: true, route2: true}) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + // Will start two watches. + if err := waitForFuncWithNames(ctx, fakeClient.WaitForWatchRouteConfig, route1, route2); err != nil { + t.Fatalf("Error while waiting for names: %v", err) + } + + // Update the RDSHandler with route names which deletes a route name to + // watch. This should trigger the RDSHandler to cancel the watch for the + // deleted route name to watch. + rh.updateRouteNamesToWatch(map[string]bool{route1: true}) + // This should delete the watch for route2. + routeNameDeleted, err := fakeClient.WaitForCancelRouteConfigWatch(ctx) + if err != nil { + t.Fatalf("xdsClient.CancelRDS failed with error %v", err) + } + if routeNameDeleted != route2 { + t.Fatalf("xdsClient.CancelRDS called for route %v, want %v", routeNameDeleted, route2) + } + + rdsUpdate := xdsresource.RouteConfigUpdate{} + // Invoke callback with the xds client with a certain route update. Due to + // this route update updating every route name that rds handler handles, + // this should write to the update channel to send to the listener. + fakeClient.InvokeWatchRouteConfigCallback(route1, rdsUpdate, nil) + rhuWant := map[string]xdsresource.RouteConfigUpdate{route1: rdsUpdate} + select { + case rhu := <-ch: + if diff := cmp.Diff(rhu.updates, rhuWant); diff != "" { + t.Fatalf("got unexpected route update, diff (-got, +want): %v", diff) + } + case <-ctx.Done(): + t.Fatal("Timed out waiting for update from update channel.") + } + + rh.close() + routeNameDeleted, err = fakeClient.WaitForCancelRouteConfigWatch(ctx) + if err != nil { + t.Fatalf("xdsClient.CancelRDS failed with error: %v", err) + } + if routeNameDeleted != route1 { + t.Fatalf("xdsClient.CancelRDS called for route %v, want %v", routeNameDeleted, route1) + } +} + +// TestSuccessCaseTwoUpdatesAddAndDeleteRoute tests the case where the rds +// handler receives an update with two routes, and then receives an update with +// two routes, one previously there and one added (i.e. 12 -> 23). This should +// cause the route that is no longer there to be deleted and cancelled, and the +// route that was added should have a watch started for it. +func (s) TestSuccessCaseTwoUpdatesAddAndDeleteRoute(t *testing.T) { + rh, fakeClient, ch := setupTests() + + rh.updateRouteNamesToWatch(map[string]bool{route1: true, route2: true}) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := waitForFuncWithNames(ctx, fakeClient.WaitForWatchRouteConfig, route1, route2); err != nil { + t.Fatalf("Error while waiting for names: %v", err) + } + + // Update the rds handler with two routes, one which was already there and a new route. + // This should cause the rds handler to delete/cancel watch for route 1 and start a watch + // for route 3. + rh.updateRouteNamesToWatch(map[string]bool{route2: true, route3: true}) + + // Start watch comes first, which should be for route3 as was just added. + gotRoute, err := fakeClient.WaitForWatchRouteConfig(ctx) + if err != nil { + t.Fatalf("xdsClient.WatchRDS failed with error: %v", err) + } + if gotRoute != route3 { + t.Fatalf("xdsClient.WatchRDS called for route: %v, want %v", gotRoute, route3) + } + + // Then route 1 should be deleted/cancelled watch for, as it is no longer present + // in the new RouteName to watch map. + routeNameDeleted, err := fakeClient.WaitForCancelRouteConfigWatch(ctx) + if err != nil { + t.Fatalf("xdsClient.CancelRDS failed with error: %v", err) + } + if routeNameDeleted != route1 { + t.Fatalf("xdsClient.CancelRDS called for route %v, want %v", routeNameDeleted, route1) + } + + // Invoke the callback with an update for route 2. This shouldn't cause the + // handler to write an update, as it has not received RouteConfigurations + // for every RouteName. + rdsUpdate2 := xdsresource.RouteConfigUpdate{} + fakeClient.InvokeWatchRouteConfigCallback(route2, rdsUpdate2, nil) + + // The RDS Handler should not send an update. + sCtx, sCtxCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCtxCancel() + select { + case <-ch: + t.Fatalf("RDS Handler wrote an update to updateChannel when it shouldn't have, as each route name has not received an update yet") + case <-sCtx.Done(): + } + + // Invoke the callback with an update for route 3. This should cause the + // handler to write an update, as it has received RouteConfigurations for + // every RouteName. + rdsUpdate3 := xdsresource.RouteConfigUpdate{} + fakeClient.InvokeWatchRouteConfigCallback(route3, rdsUpdate3, nil) + // The RDS Handler should then update the listener wrapper with an update + // with two route configurations, as both route names the RDS Handler handles + // have received an update. + rhuWant := map[string]xdsresource.RouteConfigUpdate{route2: rdsUpdate2, route3: rdsUpdate3} + select { + case rhu := <-rh.updateChannel: + if diff := cmp.Diff(rhu.updates, rhuWant); diff != "" { + t.Fatalf("got unexpected route update, diff (-got, +want): %v", diff) + } + case <-ctx.Done(): + t.Fatal("Timed out waiting for the rds handler update to be written to the update buffer.") + } + // Close the rds handler. This is meant to be called when the lis wrapper is + // closed, and the call should cancel all the watches present (for this + // test, two watches on route2 and route3). + rh.close() + if err = waitForFuncWithNames(ctx, fakeClient.WaitForCancelRouteConfigWatch, route2, route3); err != nil { + t.Fatalf("Error while waiting for names: %v", err) + } +} + +// TestSuccessCaseSecondUpdateMakesRouteFull tests the scenario where the rds handler gets +// told to watch three rds configurations, gets two successful updates, then gets told to watch +// only those two. The rds handler should then write an update to update buffer. +func (s) TestSuccessCaseSecondUpdateMakesRouteFull(t *testing.T) { + rh, fakeClient, ch := setupTests() + + rh.updateRouteNamesToWatch(map[string]bool{route1: true, route2: true, route3: true}) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := waitForFuncWithNames(ctx, fakeClient.WaitForWatchRouteConfig, route1, route2, route3); err != nil { + t.Fatalf("Error while waiting for names: %v", err) + } + + // Invoke the callbacks for two of the three watches. Since RDS is not full, + // this shouldn't trigger rds handler to write an update to update buffer. + fakeClient.InvokeWatchRouteConfigCallback(route1, xdsresource.RouteConfigUpdate{}, nil) + fakeClient.InvokeWatchRouteConfigCallback(route2, xdsresource.RouteConfigUpdate{}, nil) + + // The RDS Handler should not send an update. + sCtx, sCtxCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCtxCancel() + select { + case <-rh.updateChannel: + t.Fatalf("RDS Handler wrote an update to updateChannel when it shouldn't have, as each route name has not received an update yet") + case <-sCtx.Done(): + } + + // Tell the rds handler to now only watch Route 1 and Route 2. This should + // trigger the rds handler to write an update to the update buffer as it now + // has full rds configuration. + rh.updateRouteNamesToWatch(map[string]bool{route1: true, route2: true}) + // Route 3 should be deleted/cancelled watch for, as it is no longer present + // in the new RouteName to watch map. + routeNameDeleted, err := fakeClient.WaitForCancelRouteConfigWatch(ctx) + if err != nil { + t.Fatalf("xdsClient.CancelRDS failed with error: %v", err) + } + if routeNameDeleted != route3 { + t.Fatalf("xdsClient.CancelRDS called for route %v, want %v", routeNameDeleted, route1) + } + rhuWant := map[string]xdsresource.RouteConfigUpdate{route1: {}, route2: {}} + select { + case rhu := <-ch: + if diff := cmp.Diff(rhu.updates, rhuWant); diff != "" { + t.Fatalf("got unexpected route update, diff (-got, +want): %v", diff) + } + case <-ctx.Done(): + t.Fatal("Timed out waiting for the rds handler update to be written to the update buffer.") + } +} + +// TestErrorReceived tests the case where the rds handler receives a route name +// to watch, then receives an update with an error. This error should be then +// written to the update channel. +func (s) TestErrorReceived(t *testing.T) { + rh, fakeClient, ch := setupTests() + + rh.updateRouteNamesToWatch(map[string]bool{route1: true}) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + gotRoute, err := fakeClient.WaitForWatchRouteConfig(ctx) + if err != nil { + t.Fatalf("xdsClient.WatchRDS failed with error %v", err) + } + if gotRoute != route1 { + t.Fatalf("xdsClient.WatchRDS called for route: %v, want %v", gotRoute, route1) + } + + rdsErr := errors.New("some error") + fakeClient.InvokeWatchRouteConfigCallback(route1, xdsresource.RouteConfigUpdate{}, rdsErr) + select { + case rhu := <-ch: + if rhu.err.Error() != "some error" { + t.Fatalf("Did not receive the expected error, instead received: %v", rhu.err.Error()) + } + case <-ctx.Done(): + t.Fatal("Timed out waiting for update from update channel") + } +} diff --git a/xds/internal/test/e2e/README.md b/xds/internal/test/e2e/README.md new file mode 100644 index 000000000000..33cffa0da56f --- /dev/null +++ b/xds/internal/test/e2e/README.md @@ -0,0 +1,19 @@ +Build client and server binaries. + +```sh +go build -o ./binaries/client ../../../../interop/xds/client/ +go build -o ./binaries/server ../../../../interop/xds/server/ +``` + +Run the test + +```sh +go test . -v +``` + +The client/server paths are flags + +```sh +go test . -v -client=$HOME/grpc-java/interop-testing/build/install/grpc-interop-testing/bin/xds-test-client +``` +Note that grpc logs are only turned on for Go. diff --git a/xds/internal/test/e2e/controlplane.go b/xds/internal/test/e2e/controlplane.go new file mode 100644 index 000000000000..98030dd448f5 --- /dev/null +++ b/xds/internal/test/e2e/controlplane.go @@ -0,0 +1,61 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package e2e + +import ( + "fmt" + + "github.com/google/uuid" + "google.golang.org/grpc/internal/testutils/xds/bootstrap" + "google.golang.org/grpc/internal/testutils/xds/e2e" +) + +type controlPlane struct { + server *e2e.ManagementServer + nodeID string + bootstrapContent string +} + +func newControlPlane() (*controlPlane, error) { + // Spin up an xDS management server on a local port. + server, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to spin up the xDS management server: %v", err) + } + + nodeID := uuid.New().String() + bootstrapContentBytes, err := bootstrap.Contents(bootstrap.Options{ + NodeID: nodeID, + ServerURI: server.Address, + ServerListenerResourceNameTemplate: e2e.ServerListenerResourceNameTemplate, + }) + if err != nil { + server.Stop() + return nil, fmt.Errorf("failed to create bootstrap file: %v", err) + } + + return &controlPlane{ + server: server, + nodeID: nodeID, + bootstrapContent: string(bootstrapContentBytes), + }, nil +} + +func (cp *controlPlane) stop() { + cp.server.Stop() +} diff --git a/xds/internal/test/e2e/e2e.go b/xds/internal/test/e2e/e2e.go new file mode 100644 index 000000000000..30b125b787a1 --- /dev/null +++ b/xds/internal/test/e2e/e2e.go @@ -0,0 +1,173 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package e2e implements xds e2e tests using go-control-plane. +package e2e + +import ( + "context" + "fmt" + "io" + "os" + "os/exec" + + "google.golang.org/grpc" + channelzgrpc "google.golang.org/grpc/channelz/grpc_channelz_v1" + channelzpb "google.golang.org/grpc/channelz/grpc_channelz_v1" + "google.golang.org/grpc/credentials/insecure" + testgrpc "google.golang.org/grpc/interop/grpc_testing" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +func cmd(path string, logger io.Writer, args []string, env []string) *exec.Cmd { + cmd := exec.Command(path, args...) + cmd.Env = append(os.Environ(), env...) + cmd.Stdout = logger + cmd.Stderr = logger + return cmd +} + +const ( + clientStatsPort = 60363 // TODO: make this different per-test, only needed for parallel tests. +) + +type client struct { + cmd *exec.Cmd + + target string + statsCC *grpc.ClientConn +} + +// newClient create a client with the given target and bootstrap content. +func newClient(target, binaryPath, bootstrap string, logger io.Writer, flags ...string) (*client, error) { + cmd := cmd( + binaryPath, + logger, + append([]string{ + "--server=" + target, + "--print_response=true", + "--qps=100", + fmt.Sprintf("--stats_port=%d", clientStatsPort), + }, flags...), // Append any flags from caller. + []string{ + "GRPC_GO_LOG_VERBOSITY_LEVEL=99", + "GRPC_GO_LOG_SEVERITY_LEVEL=info", + "GRPC_XDS_BOOTSTRAP_CONFIG=" + bootstrap, // The bootstrap content doesn't need to be quoted. + }, + ) + cmd.Start() + + cc, err := grpc.Dial(fmt.Sprintf("localhost:%d", clientStatsPort), grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultCallOptions(grpc.WaitForReady(true))) + if err != nil { + return nil, err + } + return &client{ + cmd: cmd, + target: target, + statsCC: cc, + }, nil +} + +func (c *client) clientStats(ctx context.Context) (*testpb.LoadBalancerStatsResponse, error) { + ccc := testgrpc.NewLoadBalancerStatsServiceClient(c.statsCC) + return ccc.GetClientStats(ctx, &testpb.LoadBalancerStatsRequest{ + NumRpcs: 100, + TimeoutSec: 10, + }) +} + +func (c *client) configRPCs(ctx context.Context, req *testpb.ClientConfigureRequest) error { + ccc := testgrpc.NewXdsUpdateClientConfigureServiceClient(c.statsCC) + _, err := ccc.Configure(ctx, req) + return err +} + +func (c *client) channelzSubChannels(ctx context.Context) ([]*channelzpb.Subchannel, error) { + ccc := channelzgrpc.NewChannelzClient(c.statsCC) + r, err := ccc.GetTopChannels(ctx, &channelzpb.GetTopChannelsRequest{}) + if err != nil { + return nil, err + } + + var ret []*channelzpb.Subchannel + for _, cc := range r.Channel { + if cc.Data.Target != c.target { + continue + } + for _, sc := range cc.SubchannelRef { + rr, err := ccc.GetSubchannel(ctx, &channelzpb.GetSubchannelRequest{SubchannelId: sc.SubchannelId}) + if err != nil { + return nil, err + } + ret = append(ret, rr.Subchannel) + } + } + return ret, nil +} + +func (c *client) stop() { + c.cmd.Process.Kill() + c.cmd.Wait() +} + +const ( + serverPort = 50051 // TODO: make this different per-test, only needed for parallel tests. +) + +type server struct { + cmd *exec.Cmd + port int +} + +// newServer creates multiple servers with the given bootstrap content. +// +// Each server gets a different hostname, in the format of +// -. +func newServers(hostnamePrefix, binaryPath, bootstrap string, logger io.Writer, count int) (_ []*server, err error) { + var ret []*server + defer func() { + if err != nil { + for _, s := range ret { + s.stop() + } + } + }() + for i := 0; i < count; i++ { + port := serverPort + i + cmd := cmd( + binaryPath, + logger, + []string{ + fmt.Sprintf("--port=%d", port), + fmt.Sprintf("--host_name_override=%s-%d", hostnamePrefix, i), + }, + []string{ + "GRPC_GO_LOG_VERBOSITY_LEVEL=99", + "GRPC_GO_LOG_SEVERITY_LEVEL=info", + "GRPC_XDS_BOOTSTRAP_CONFIG=" + bootstrap, // The bootstrap content doesn't need to be quoted., + }, + ) + cmd.Start() + ret = append(ret, &server{cmd: cmd, port: port}) + } + return ret, nil +} + +func (s *server) stop() { + s.cmd.Process.Kill() + s.cmd.Wait() +} diff --git a/xds/internal/test/e2e/e2e_test.go b/xds/internal/test/e2e/e2e_test.go new file mode 100644 index 000000000000..be8af2b0a26a --- /dev/null +++ b/xds/internal/test/e2e/e2e_test.go @@ -0,0 +1,259 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package e2e + +import ( + "bytes" + "context" + "flag" + "fmt" + "os" + "strconv" + "testing" + "time" + + "google.golang.org/grpc/internal/testutils/xds/e2e" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + channelzpb "google.golang.org/grpc/channelz/grpc_channelz_v1" + testpb "google.golang.org/grpc/interop/grpc_testing" +) + +var ( + clientPath = flag.String("client", "./binaries/client", "The interop client") + serverPath = flag.String("server", "./binaries/server", "The interop server") +) + +type testOpts struct { + testName string + backendCount int + clientFlags []string +} + +func setup(t *testing.T, opts testOpts) (*controlPlane, *client, []*server) { + t.Helper() + if _, err := os.Stat(*clientPath); os.IsNotExist(err) { + t.Skip("skipped because client is not found") + } + if _, err := os.Stat(*serverPath); os.IsNotExist(err) { + t.Skip("skipped because server is not found") + } + backendCount := 1 + if opts.backendCount != 0 { + backendCount = opts.backendCount + } + + cp, err := newControlPlane() + if err != nil { + t.Fatalf("failed to start control-plane: %v", err) + } + t.Cleanup(cp.stop) + + var clientLog bytes.Buffer + c, err := newClient(fmt.Sprintf("xds:///%s", opts.testName), *clientPath, cp.bootstrapContent, &clientLog, opts.clientFlags...) + if err != nil { + t.Fatalf("failed to start client: %v", err) + } + t.Cleanup(c.stop) + + var serverLog bytes.Buffer + servers, err := newServers(opts.testName, *serverPath, cp.bootstrapContent, &serverLog, backendCount) + if err != nil { + t.Fatalf("failed to start server: %v", err) + } + t.Cleanup(func() { + for _, s := range servers { + s.stop() + } + }) + t.Cleanup(func() { + // TODO: find a better way to print the log. They are long, and hide the failure. + t.Logf("\n----- client logs -----\n%v", clientLog.String()) + t.Logf("\n----- server logs -----\n%v", serverLog.String()) + }) + return cp, c, servers +} + +func TestPingPong(t *testing.T) { + const testName = "pingpong" + cp, c, _ := setup(t, testOpts{testName: testName}) + + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: testName, + NodeID: cp.nodeID, + Host: "localhost", + Port: serverPort, + SecLevel: e2e.SecurityLevelNone, + }) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := cp.server.Update(ctx, resources); err != nil { + t.Fatalf("failed to update control plane resources: %v", err) + } + + st, err := c.clientStats(ctx) + if err != nil { + t.Fatalf("failed to get client stats: %v", err) + } + if st.NumFailures != 0 { + t.Fatalf("Got %v failures: %+v", st.NumFailures, st) + } +} + +// TestAffinity covers the affinity tests with ringhash policy. +// - client is configured to use ringhash, with 3 backends +// - all RPCs will hash a specific metadata header +// - verify that +// - all RPCs with the same metadata value are sent to the same backend +// - only one backend is Ready +// +// - send more RPCs with different metadata values until a new backend is picked, and verify that +// - only two backends are in Ready +func TestAffinity(t *testing.T) { + const ( + testName = "affinity" + backendCount = 3 + testMDKey = "xds_md" + testMDValue = "unary_yranu" + ) + cp, c, servers := setup(t, testOpts{ + testName: testName, + backendCount: backendCount, + clientFlags: []string{"--rpc=EmptyCall", fmt.Sprintf("--metadata=EmptyCall:%s:%s", testMDKey, testMDValue)}, + }) + + resources := e2e.DefaultClientResources(e2e.ResourceParams{ + DialTarget: testName, + NodeID: cp.nodeID, + Host: "localhost", + Port: serverPort, + SecLevel: e2e.SecurityLevelNone, + }) + + // Update EDS to multiple backends. + var ports []uint32 + for _, s := range servers { + ports = append(ports, uint32(s.port)) + } + edsMsg := resources.Endpoints[0] + resources.Endpoints[0] = e2e.DefaultEndpoint( + edsMsg.ClusterName, + "localhost", + ports, + ) + + // Update CDS lbpolicy to ringhash. + cdsMsg := resources.Clusters[0] + cdsMsg.LbPolicy = v3clusterpb.Cluster_RING_HASH + + // Update RDS to hash the header. + rdsMsg := resources.Routes[0] + rdsMsg.VirtualHosts[0].Routes[0].Action = &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: cdsMsg.Name}, + HashPolicy: []*v3routepb.RouteAction_HashPolicy{{ + PolicySpecifier: &v3routepb.RouteAction_HashPolicy_Header_{ + Header: &v3routepb.RouteAction_HashPolicy_Header{ + HeaderName: testMDKey, + }, + }, + }}, + }} + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := cp.server.Update(ctx, resources); err != nil { + t.Fatalf("failed to update control plane resources: %v", err) + } + + // Note: We can skip CSDS check because there's no long delay as in TD. + // + // The client stats check doesn't race with the xds resource update because + // there's only one version of xds resource, updated at the beginning of the + // test. So there's no need to retry the stats call. + // + // In the future, we may add tests that update xds in the middle. Then we + // either need to retry clientStats(), or make a CSDS check before so the + // result is stable. + + st, err := c.clientStats(ctx) + if err != nil { + t.Fatalf("failed to get client stats: %v", err) + } + if st.NumFailures != 0 { + t.Fatalf("Got %v failures: %+v", st.NumFailures, st) + } + if len(st.RpcsByPeer) != 1 { + t.Fatalf("more than 1 backends got traffic: %v, want 1", st.RpcsByPeer) + } + + // Call channelz to verify that only one subchannel is in state Ready. + scs, err := c.channelzSubChannels(ctx) + if err != nil { + t.Fatalf("failed to fetch channelz: %v", err) + } + verifySubConnStates(t, scs, map[channelzpb.ChannelConnectivityState_State]int{ + channelzpb.ChannelConnectivityState_READY: 1, + channelzpb.ChannelConnectivityState_IDLE: 2, + }) + + // Send Unary call with different metadata value with integers starting from + // 0. Stop when a second peer is picked. + var ( + diffPeerPicked bool + mdValue int + ) + for !diffPeerPicked { + if err := c.configRPCs(ctx, &testpb.ClientConfigureRequest{ + Types: []testpb.ClientConfigureRequest_RpcType{ + testpb.ClientConfigureRequest_EMPTY_CALL, + testpb.ClientConfigureRequest_UNARY_CALL, + }, + Metadata: []*testpb.ClientConfigureRequest_Metadata{ + {Type: testpb.ClientConfigureRequest_EMPTY_CALL, Key: testMDKey, Value: testMDValue}, + {Type: testpb.ClientConfigureRequest_UNARY_CALL, Key: testMDKey, Value: strconv.Itoa(mdValue)}, + }, + }); err != nil { + t.Fatalf("failed to configure RPC: %v", err) + } + + st, err := c.clientStats(ctx) + if err != nil { + t.Fatalf("failed to get client stats: %v", err) + } + if st.NumFailures != 0 { + t.Fatalf("Got %v failures: %+v", st.NumFailures, st) + } + if len(st.RpcsByPeer) == 2 { + break + } + + mdValue++ + } + + // Call channelz to verify that only one subchannel is in state Ready. + scs2, err := c.channelzSubChannels(ctx) + if err != nil { + t.Fatalf("failed to fetch channelz: %v", err) + } + verifySubConnStates(t, scs2, map[channelzpb.ChannelConnectivityState_State]int{ + channelzpb.ChannelConnectivityState_READY: 2, + channelzpb.ChannelConnectivityState_IDLE: 1, + }) +} diff --git a/xds/internal/test/e2e/e2e_utils.go b/xds/internal/test/e2e/e2e_utils.go new file mode 100644 index 000000000000..34b0ee9eb092 --- /dev/null +++ b/xds/internal/test/e2e/e2e_utils.go @@ -0,0 +1,36 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package e2e + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + channelzpb "google.golang.org/grpc/channelz/grpc_channelz_v1" +) + +func verifySubConnStates(t *testing.T, scs []*channelzpb.Subchannel, want map[channelzpb.ChannelConnectivityState_State]int) { + t.Helper() + var scStatsCount = map[channelzpb.ChannelConnectivityState_State]int{} + for _, sc := range scs { + scStatsCount[sc.Data.State.State]++ + } + if diff := cmp.Diff(scStatsCount, want); diff != "" { + t.Fatalf("got unexpected number of subchannels in state Ready, %v, scs: %v", diff, scs) + } +} diff --git a/xds/internal/test/e2e/run.sh b/xds/internal/test/e2e/run.sh new file mode 100755 index 000000000000..4363d6cbd94f --- /dev/null +++ b/xds/internal/test/e2e/run.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +mkdir binaries +go build -o ./binaries/client ../../../../interop/xds/client/ +go build -o ./binaries/server ../../../../interop/xds/server/ +go test . -v diff --git a/xds/internal/testutils/balancer.go b/xds/internal/testutils/balancer.go deleted file mode 100644 index ec3c4b5c6e91..000000000000 --- a/xds/internal/testutils/balancer.go +++ /dev/null @@ -1,362 +0,0 @@ -/* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package testutils - -import ( - "context" - "fmt" - "sync" - "testing" - - corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" - "google.golang.org/grpc" - "google.golang.org/grpc/balancer" - "google.golang.org/grpc/connectivity" - "google.golang.org/grpc/internal/wrr" - "google.golang.org/grpc/resolver" - "google.golang.org/grpc/xds/internal" -) - -const testSubConnsCount = 16 - -// TestSubConns contains a list of SubConns to be used in tests. -var TestSubConns []*TestSubConn - -func init() { - for i := 0; i < testSubConnsCount; i++ { - TestSubConns = append(TestSubConns, &TestSubConn{ - id: fmt.Sprintf("sc%d", i), - }) - } -} - -// TestSubConn implements the SubConn interface, to be used in tests. -type TestSubConn struct { - id string -} - -// UpdateAddresses panics. -func (tsc *TestSubConn) UpdateAddresses([]resolver.Address) { panic("not implemented") } - -// Connect is a no-op. -func (tsc *TestSubConn) Connect() {} - -// String implements stringer to print human friendly error message. -func (tsc *TestSubConn) String() string { - return tsc.id -} - -// TestClientConn is a mock balancer.ClientConn used in tests. -type TestClientConn struct { - t *testing.T // For logging only. - - NewSubConnAddrsCh chan []resolver.Address // the last 10 []Address to create subconn. - NewSubConnCh chan balancer.SubConn // the last 10 subconn created. - RemoveSubConnCh chan balancer.SubConn // the last 10 subconn removed. - - NewPickerCh chan balancer.Picker // the last picker updated. - NewStateCh chan connectivity.State // the last state. - - subConnIdx int -} - -// NewTestClientConn creates a TestClientConn. -func NewTestClientConn(t *testing.T) *TestClientConn { - return &TestClientConn{ - t: t, - - NewSubConnAddrsCh: make(chan []resolver.Address, 10), - NewSubConnCh: make(chan balancer.SubConn, 10), - RemoveSubConnCh: make(chan balancer.SubConn, 10), - - NewPickerCh: make(chan balancer.Picker, 1), - NewStateCh: make(chan connectivity.State, 1), - } -} - -// NewSubConn creates a new SubConn. -func (tcc *TestClientConn) NewSubConn(a []resolver.Address, o balancer.NewSubConnOptions) (balancer.SubConn, error) { - sc := TestSubConns[tcc.subConnIdx] - tcc.subConnIdx++ - - tcc.t.Logf("testClientConn: NewSubConn(%v, %+v) => %s", a, o, sc) - select { - case tcc.NewSubConnAddrsCh <- a: - default: - } - - select { - case tcc.NewSubConnCh <- sc: - default: - } - - return sc, nil -} - -// RemoveSubConn removes the SubConn. -func (tcc *TestClientConn) RemoveSubConn(sc balancer.SubConn) { - tcc.t.Logf("testClientCOnn: RemoveSubConn(%p)", sc) - select { - case tcc.RemoveSubConnCh <- sc: - default: - } -} - -// UpdateBalancerState implements balancer.Balancer API. It will be removed when -// switching to the new balancer interface. -func (tcc *TestClientConn) UpdateBalancerState(s connectivity.State, p balancer.Picker) { - tcc.t.Fatal("not implemented") -} - -// UpdateState updates connectivity state and picker. -func (tcc *TestClientConn) UpdateState(bs balancer.State) { - tcc.t.Logf("testClientConn: UpdateState(%v)", bs) - select { - case <-tcc.NewStateCh: - default: - } - tcc.NewStateCh <- bs.ConnectivityState - - select { - case <-tcc.NewPickerCh: - default: - } - tcc.NewPickerCh <- bs.Picker -} - -// ResolveNow panics. -func (tcc *TestClientConn) ResolveNow(resolver.ResolveNowOptions) { - panic("not implemented") -} - -// Target panics. -func (tcc *TestClientConn) Target() string { - panic("not implemented") -} - -// TestServerLoad is testing Load for testing LRS. -type TestServerLoad struct { - Name string - D float64 -} - -// TestLoadStore is a load store to be used in tests. -type TestLoadStore struct { - CallsStarted []internal.LocalityID - CallsEnded []internal.LocalityID - CallsCost []TestServerLoad -} - -// NewTestLoadStore creates a new TestLoadStore. -func NewTestLoadStore() *TestLoadStore { - return &TestLoadStore{} -} - -// CallDropped records a call dropped. -func (*TestLoadStore) CallDropped(category string) { - panic("not implemented") -} - -// CallStarted records a call started. -func (tls *TestLoadStore) CallStarted(l internal.LocalityID) { - tls.CallsStarted = append(tls.CallsStarted, l) -} - -// CallFinished records a call finished. -func (tls *TestLoadStore) CallFinished(l internal.LocalityID, err error) { - tls.CallsEnded = append(tls.CallsEnded, l) -} - -// CallServerLoad records a call server load. -func (tls *TestLoadStore) CallServerLoad(l internal.LocalityID, name string, d float64) { - tls.CallsCost = append(tls.CallsCost, TestServerLoad{Name: name, D: d}) -} - -// ReportTo panics. -func (*TestLoadStore) ReportTo(ctx context.Context, cc *grpc.ClientConn, clusterName string, node *corepb.Node) { - panic("not implemented") -} - -// IsRoundRobin checks whether f's return value is roundrobin of elements from -// want. But it doesn't check for the order. Note that want can contain -// duplicate items, which makes it weight-round-robin. -// -// Step 1. the return values of f should form a permutation of all elements in -// want, but not necessary in the same order. E.g. if want is {a,a,b}, the check -// fails if f returns: -// - {a,a,a}: third a is returned before b -// - {a,b,b}: second b is returned before the second a -// -// If error is found in this step, the returned error contains only the first -// iteration until where it goes wrong. -// -// Step 2. the return values of f should be repetitions of the same permutation. -// E.g. if want is {a,a,b}, the check failes if f returns: -// - {a,b,a,b,a,a}: though it satisfies step 1, the second iteration is not -// repeating the first iteration. -// -// If error is found in this step, the returned error contains the first -// iteration + the second iteration until where it goes wrong. -func IsRoundRobin(want []balancer.SubConn, f func() balancer.SubConn) error { - wantSet := make(map[balancer.SubConn]int) // SubConn -> count, for weighted RR. - for _, sc := range want { - wantSet[sc]++ - } - - // The first iteration: makes sure f's return values form a permutation of - // elements in want. - // - // Also keep the returns values in a slice, so we can compare the order in - // the second iteration. - gotSliceFirstIteration := make([]balancer.SubConn, 0, len(want)) - for range want { - got := f() - gotSliceFirstIteration = append(gotSliceFirstIteration, got) - wantSet[got]-- - if wantSet[got] < 0 { - return fmt.Errorf("non-roundrobin want: %v, result: %v", want, gotSliceFirstIteration) - } - } - - // The second iteration should repeat the first iteration. - var gotSliceSecondIteration []balancer.SubConn - for i := 0; i < 2; i++ { - for _, w := range gotSliceFirstIteration { - g := f() - gotSliceSecondIteration = append(gotSliceSecondIteration, g) - if w != g { - return fmt.Errorf("non-roundrobin, first iter: %v, second iter: %v", gotSliceFirstIteration, gotSliceSecondIteration) - } - } - } - - return nil -} - -// testClosure is a test util for TestIsRoundRobin. -type testClosure struct { - r []balancer.SubConn - i int -} - -func (tc *testClosure) next() balancer.SubConn { - ret := tc.r[tc.i] - tc.i = (tc.i + 1) % len(tc.r) - return ret -} - -func init() { - balancer.Register(&TestConstBalancerBuilder{}) -} - -// ErrTestConstPicker is error returned by test const picker. -var ErrTestConstPicker = fmt.Errorf("const picker error") - -// TestConstBalancerBuilder is a balancer builder for tests. -type TestConstBalancerBuilder struct{} - -// Build builds a test const balancer. -func (*TestConstBalancerBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { - return &testConstBalancer{cc: cc} -} - -// Name returns test-const-balancer name. -func (*TestConstBalancerBuilder) Name() string { - return "test-const-balancer" -} - -type testConstBalancer struct { - cc balancer.ClientConn -} - -func (tb *testConstBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { - tb.cc.UpdateState(balancer.State{ConnectivityState: connectivity.Ready, Picker: &TestConstPicker{Err: ErrTestConstPicker}}) -} - -func (tb *testConstBalancer) ResolverError(error) { - panic("not implemented") -} - -func (tb *testConstBalancer) UpdateClientConnState(s balancer.ClientConnState) error { - if len(s.ResolverState.Addresses) == 0 { - return nil - } - tb.cc.NewSubConn(s.ResolverState.Addresses, balancer.NewSubConnOptions{}) - return nil -} - -func (*testConstBalancer) Close() { -} - -// TestConstPicker is a const picker for tests. -type TestConstPicker struct { - Err error - SC balancer.SubConn -} - -// Pick returns the const SubConn or the error. -func (tcp *TestConstPicker) Pick(info balancer.PickInfo) (balancer.PickResult, error) { - if tcp.Err != nil { - return balancer.PickResult{}, tcp.Err - } - return balancer.PickResult{SubConn: tcp.SC}, nil -} - -// testWRR is a deterministic WRR implementation. -// -// The real implementation does random WRR. testWRR makes the balancer behavior -// deterministic and easier to test. -// -// With {a: 2, b: 3}, the Next() results will be {a, a, b, b, b}. -type testWRR struct { - itemsWithWeight []struct { - item interface{} - weight int64 - } - length int - - mu sync.Mutex - idx int // The index of the item that will be picked - count int64 // The number of times the current item has been picked. -} - -// NewTestWRR return a WRR for testing. It's deterministic instead random. -func NewTestWRR() wrr.WRR { - return &testWRR{} -} - -func (twrr *testWRR) Add(item interface{}, weight int64) { - twrr.itemsWithWeight = append(twrr.itemsWithWeight, struct { - item interface{} - weight int64 - }{item: item, weight: weight}) - twrr.length++ -} - -func (twrr *testWRR) Next() interface{} { - twrr.mu.Lock() - iww := twrr.itemsWithWeight[twrr.idx] - twrr.count++ - if twrr.count >= iww.weight { - twrr.idx = (twrr.idx + 1) % twrr.length - twrr.count = 0 - } - twrr.mu.Unlock() - return iww.item -} diff --git a/xds/internal/testutils/balancer_test.go b/xds/internal/testutils/balancer_test.go index 4891eb9cdadf..73e0d1c12433 100644 --- a/xds/internal/testutils/balancer_test.go +++ b/xds/internal/testutils/balancer_test.go @@ -22,13 +22,14 @@ import ( "testing" "google.golang.org/grpc/balancer" + "google.golang.org/grpc/internal/testutils" ) func TestIsRoundRobin(t *testing.T) { var ( - sc1 = TestSubConns[0] - sc2 = TestSubConns[1] - sc3 = TestSubConns[2] + sc1 = &testutils.TestSubConn{} + sc2 = &testutils.TestSubConn{} + sc3 = &testutils.TestSubConn{} ) testCases := []struct { @@ -125,10 +126,22 @@ func TestIsRoundRobin(t *testing.T) { } for _, tC := range testCases { t.Run(tC.desc, func(t *testing.T) { - err := IsRoundRobin(tC.want, (&testClosure{r: tC.got}).next) + err := testutils.IsRoundRobin(tC.want, (&testClosure{r: tC.got}).next) if err == nil != tC.pass { t.Errorf("want pass %v, want %v, got err %v", tC.pass, tC.want, err) } }) } } + +// testClosure is a test util for TestIsRoundRobin. +type testClosure struct { + r []balancer.SubConn + i int +} + +func (tc *testClosure) next() balancer.SubConn { + ret := tc.r[tc.i] + tc.i = (tc.i + 1) % len(tc.r) + return ret +} diff --git a/xds/internal/testutils/channel.go b/xds/internal/testutils/channel.go deleted file mode 100644 index 1715a01ff68f..000000000000 --- a/xds/internal/testutils/channel.go +++ /dev/null @@ -1,87 +0,0 @@ -/* - * - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Package testutils provides utility types, for use in xds tests. -package testutils - -import ( - "errors" - "time" -) - -// ErrRecvTimeout is an error to indicate that a receive operation on the -// channel timed out. -var ErrRecvTimeout = errors.New("timed out when waiting for value on channel") - -const ( - // DefaultChanRecvTimeout is the default timeout for receive operations on the - // underlying channel. - DefaultChanRecvTimeout = 1 * time.Second - // DefaultChanBufferSize is the default buffer size of the underlying channel. - DefaultChanBufferSize = 1 -) - -// Channel wraps a generic channel and provides a timed receive operation. -type Channel struct { - ch chan interface{} -} - -// Send sends value on the underlying channel. -func (cwt *Channel) Send(value interface{}) { - cwt.ch <- value -} - -// Replace clears the value on the underlying channel, and sends the new value. -// -// It's expected to be used with a size-1 channel, to only keep the most -// up-to-date item. -func (cwt *Channel) Replace(value interface{}) { - select { - case <-cwt.ch: - default: - } - cwt.ch <- value -} - -// TimedReceive returns the value received on the underlying channel, or -// ErrRecvTimeout if timeout amount of time elapsed. -func (cwt *Channel) TimedReceive(timeout time.Duration) (interface{}, error) { - timer := time.NewTimer(timeout) - select { - case <-timer.C: - return nil, ErrRecvTimeout - case got := <-cwt.ch: - timer.Stop() - return got, nil - } -} - -// Receive returns the value received on the underlying channel, or -// ErrRecvTimeout if DefaultChanRecvTimeout amount of time elapses. -func (cwt *Channel) Receive() (interface{}, error) { - return cwt.TimedReceive(DefaultChanRecvTimeout) -} - -// NewChannel returns a new Channel. -func NewChannel() *Channel { - return NewChannelWithSize(DefaultChanBufferSize) -} - -// NewChannelWithSize returns a new Channel with a buffer of bufSize. -func NewChannelWithSize(bufSize int) *Channel { - return &Channel{ch: make(chan interface{}, bufSize)} -} diff --git a/xds/internal/testutils/fakeclient/client.go b/xds/internal/testutils/fakeclient/client.go index aacade2fbd0a..9794425c501f 100644 --- a/xds/internal/testutils/fakeclient/client.go +++ b/xds/internal/testutils/fakeclient/client.go @@ -20,78 +20,140 @@ package fakeclient import ( - "sync" + "context" - "google.golang.org/grpc/xds/internal/balancer/lrs" - xdsclient "google.golang.org/grpc/xds/internal/client" - "google.golang.org/grpc/xds/internal/testutils" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/xds/internal/xdsclient" + "google.golang.org/grpc/xds/internal/xdsclient/bootstrap" + "google.golang.org/grpc/xds/internal/xdsclient/load" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" ) // Client is a fake implementation of an xds client. It exposes a bunch of // channels to signal the occurrence of various events. type Client struct { + // Embed XDSClient so this fake client implements the interface, but it's + // never set (it's always nil). This may cause nil panic since not all the + // methods are implemented. + xdsclient.XDSClient + name string - suWatchCh *testutils.Channel + ldsWatchCh *testutils.Channel + rdsWatchCh *testutils.Channel cdsWatchCh *testutils.Channel edsWatchCh *testutils.Channel - suCancelCh *testutils.Channel + ldsCancelCh *testutils.Channel + rdsCancelCh *testutils.Channel cdsCancelCh *testutils.Channel edsCancelCh *testutils.Channel loadReportCh *testutils.Channel - closeCh *testutils.Channel + lrsCancelCh *testutils.Channel + loadStore *load.Store + bootstrapCfg *bootstrap.Config + + ldsCb func(xdsresource.ListenerUpdate, error) + rdsCbs map[string]func(xdsresource.RouteConfigUpdate, error) + cdsCbs map[string]func(xdsresource.ClusterUpdate, error) + edsCbs map[string]func(xdsresource.EndpointsUpdate, error) +} + +// WatchListener registers a LDS watch. +func (xdsC *Client) WatchListener(serviceName string, callback func(xdsresource.ListenerUpdate, error)) func() { + xdsC.ldsCb = callback + xdsC.ldsWatchCh.Send(serviceName) + return func() { + xdsC.ldsCancelCh.Send(nil) + } +} - mu sync.Mutex - serviceCb func(xdsclient.ServiceUpdate, error) - cdsCb func(xdsclient.ClusterUpdate, error) - edsCb func(xdsclient.EndpointsUpdate, error) +// WaitForWatchListener waits for WatchCluster to be invoked on this client and +// returns the serviceName being watched. +func (xdsC *Client) WaitForWatchListener(ctx context.Context) (string, error) { + val, err := xdsC.ldsWatchCh.Receive(ctx) + if err != nil { + return "", err + } + return val.(string), err } -// WatchService registers a LDS/RDS watch. -func (xdsC *Client) WatchService(target string, callback func(xdsclient.ServiceUpdate, error)) func() { - xdsC.mu.Lock() - defer xdsC.mu.Unlock() +// InvokeWatchListenerCallback invokes the registered ldsWatch callback. +// +// Not thread safe with WatchListener. Only call this after +// WaitForWatchListener. +func (xdsC *Client) InvokeWatchListenerCallback(update xdsresource.ListenerUpdate, err error) { + xdsC.ldsCb(update, err) +} - xdsC.serviceCb = callback - xdsC.suWatchCh.Send(target) +// WaitForCancelListenerWatch waits for a LDS watch to be cancelled and returns +// context.DeadlineExceeded otherwise. +func (xdsC *Client) WaitForCancelListenerWatch(ctx context.Context) error { + _, err := xdsC.ldsCancelCh.Receive(ctx) + return err +} + +// WatchRouteConfig registers a RDS watch. +func (xdsC *Client) WatchRouteConfig(routeName string, callback func(xdsresource.RouteConfigUpdate, error)) func() { + xdsC.rdsCbs[routeName] = callback + xdsC.rdsWatchCh.Send(routeName) return func() { - xdsC.suCancelCh.Send(nil) + xdsC.rdsCancelCh.Send(routeName) } } -// WaitForWatchService waits for WatchService to be invoked on this client -// within a reasonable timeout, and returns the serviceName being watched. -func (xdsC *Client) WaitForWatchService() (string, error) { - val, err := xdsC.suWatchCh.Receive() +// WaitForWatchRouteConfig waits for WatchCluster to be invoked on this client and +// returns the routeName being watched. +func (xdsC *Client) WaitForWatchRouteConfig(ctx context.Context) (string, error) { + val, err := xdsC.rdsWatchCh.Receive(ctx) if err != nil { return "", err } return val.(string), err } -// InvokeWatchServiceCallback invokes the registered service watch callback. -func (xdsC *Client) InvokeWatchServiceCallback(u xdsclient.ServiceUpdate, err error) { - xdsC.mu.Lock() - defer xdsC.mu.Unlock() +// InvokeWatchRouteConfigCallback invokes the registered rdsWatch callback. +// +// Not thread safe with WatchRouteConfig. Only call this after +// WaitForWatchRouteConfig. +func (xdsC *Client) InvokeWatchRouteConfigCallback(name string, update xdsresource.RouteConfigUpdate, err error) { + if len(xdsC.rdsCbs) != 1 { + xdsC.rdsCbs[name](update, err) + return + } + // Keeps functionality with previous usage of this on client side, if single + // callback call that callback. + var routeName string + for route := range xdsC.rdsCbs { + routeName = route + } + xdsC.rdsCbs[routeName](update, err) +} - xdsC.serviceCb(u, err) +// WaitForCancelRouteConfigWatch waits for a RDS watch to be cancelled and returns +// context.DeadlineExceeded otherwise. +func (xdsC *Client) WaitForCancelRouteConfigWatch(ctx context.Context) (string, error) { + val, err := xdsC.rdsCancelCh.Receive(ctx) + if err != nil { + return "", err + } + return val.(string), err } // WatchCluster registers a CDS watch. -func (xdsC *Client) WatchCluster(clusterName string, callback func(xdsclient.ClusterUpdate, error)) func() { - xdsC.mu.Lock() - defer xdsC.mu.Unlock() - - xdsC.cdsCb = callback +func (xdsC *Client) WatchCluster(clusterName string, callback func(xdsresource.ClusterUpdate, error)) func() { + // Due to the tree like structure of aggregate clusters, there can be multiple callbacks persisted for each cluster + // node. However, the client doesn't care about the parent child relationship between the nodes, only that it invokes + // the right callback for a particular cluster. + xdsC.cdsCbs[clusterName] = callback xdsC.cdsWatchCh.Send(clusterName) return func() { - xdsC.cdsCancelCh.Send(nil) + xdsC.cdsCancelCh.Send(clusterName) } } -// WaitForWatchCluster waits for WatchCluster to be invoked on this client -// within a reasonable timeout, and returns the clusterName being watched. -func (xdsC *Client) WaitForWatchCluster() (string, error) { - val, err := xdsC.cdsWatchCh.Receive() +// WaitForWatchCluster waits for WatchCluster to be invoked on this client and +// returns the clusterName being watched. +func (xdsC *Client) WaitForWatchCluster(ctx context.Context) (string, error) { + val, err := xdsC.cdsWatchCh.Receive(ctx) if err != nil { return "", err } @@ -99,36 +161,47 @@ func (xdsC *Client) WaitForWatchCluster() (string, error) { } // InvokeWatchClusterCallback invokes the registered cdsWatch callback. -func (xdsC *Client) InvokeWatchClusterCallback(update xdsclient.ClusterUpdate, err error) { - xdsC.mu.Lock() - defer xdsC.mu.Unlock() - - xdsC.cdsCb(update, err) +// +// Not thread safe with WatchCluster. Only call this after +// WaitForWatchCluster. +func (xdsC *Client) InvokeWatchClusterCallback(update xdsresource.ClusterUpdate, err error) { + // Keeps functionality with previous usage of this, if single callback call that callback. + if len(xdsC.cdsCbs) == 1 { + var clusterName string + for cluster := range xdsC.cdsCbs { + clusterName = cluster + } + xdsC.cdsCbs[clusterName](update, err) + } else { + // Have what callback you call with the update determined by the service name in the ClusterUpdate. Left up to the + // caller to make sure the cluster update matches with a persisted callback. + xdsC.cdsCbs[update.ClusterName](update, err) + } } -// WaitForCancelClusterWatch waits for a CDS watch to be cancelled within a -// reasonable timeout, and returns testutils.ErrRecvTimeout otherwise. -func (xdsC *Client) WaitForCancelClusterWatch() error { - _, err := xdsC.cdsCancelCh.Receive() - return err +// WaitForCancelClusterWatch waits for a CDS watch to be cancelled and returns +// context.DeadlineExceeded otherwise. +func (xdsC *Client) WaitForCancelClusterWatch(ctx context.Context) (string, error) { + clusterNameReceived, err := xdsC.cdsCancelCh.Receive(ctx) + if err != nil { + return "", err + } + return clusterNameReceived.(string), err } // WatchEndpoints registers an EDS watch for provided clusterName. -func (xdsC *Client) WatchEndpoints(clusterName string, callback func(xdsclient.EndpointsUpdate, error)) (cancel func()) { - xdsC.mu.Lock() - defer xdsC.mu.Unlock() - - xdsC.edsCb = callback +func (xdsC *Client) WatchEndpoints(clusterName string, callback func(xdsresource.EndpointsUpdate, error)) (cancel func()) { + xdsC.edsCbs[clusterName] = callback xdsC.edsWatchCh.Send(clusterName) return func() { - xdsC.edsCancelCh.Send(nil) + xdsC.edsCancelCh.Send(clusterName) } } -// WaitForWatchEDS waits for WatchEndpoints to be invoked on this client within a -// reasonable timeout, and returns the clusterName being watched. -func (xdsC *Client) WaitForWatchEDS() (string, error) { - val, err := xdsC.edsWatchCh.Receive() +// WaitForWatchEDS waits for WatchEndpoints to be invoked on this client and +// returns the clusterName being watched. +func (xdsC *Client) WaitForWatchEDS(ctx context.Context) (string, error) { + val, err := xdsC.edsWatchCh.Receive(ctx) if err != nil { return "", err } @@ -136,51 +209,77 @@ func (xdsC *Client) WaitForWatchEDS() (string, error) { } // InvokeWatchEDSCallback invokes the registered edsWatch callback. -func (xdsC *Client) InvokeWatchEDSCallback(update xdsclient.EndpointsUpdate, err error) { - xdsC.mu.Lock() - defer xdsC.mu.Unlock() - - xdsC.edsCb(update, err) +// +// Not thread safe with WatchEndpoints. Only call this after +// WaitForWatchEDS. +func (xdsC *Client) InvokeWatchEDSCallback(name string, update xdsresource.EndpointsUpdate, err error) { + if len(xdsC.edsCbs) != 1 { + // This may panic if name isn't found. But it's fine for tests. + xdsC.edsCbs[name](update, err) + return + } + // Keeps functionality with previous usage of this, if single callback call + // that callback. + for n := range xdsC.edsCbs { + name = n + } + xdsC.edsCbs[name](update, err) } -// WaitForCancelEDSWatch waits for a EDS watch to be cancelled within a -// reasonable timeout, and returns testutils.ErrRecvTimeout otherwise. -func (xdsC *Client) WaitForCancelEDSWatch() error { - _, err := xdsC.edsCancelCh.Receive() - return err +// WaitForCancelEDSWatch waits for a EDS watch to be cancelled and returns +// context.DeadlineExceeded otherwise. +func (xdsC *Client) WaitForCancelEDSWatch(ctx context.Context) (string, error) { + edsNameReceived, err := xdsC.edsCancelCh.Receive(ctx) + if err != nil { + return "", err + } + return edsNameReceived.(string), err } // ReportLoadArgs wraps the arguments passed to ReportLoad. type ReportLoadArgs struct { // Server is the name of the server to which the load is reported. - Server string - // Cluster is the name of the cluster for which load is reported. - Cluster string + Server *bootstrap.ServerConfig } // ReportLoad starts reporting load about clusterName to server. -func (xdsC *Client) ReportLoad(server string, clusterName string, loadStore lrs.Store) (cancel func()) { - xdsC.loadReportCh.Send(ReportLoadArgs{Server: server, Cluster: clusterName}) - return func() {} +func (xdsC *Client) ReportLoad(server *bootstrap.ServerConfig) (loadStore *load.Store, cancel func()) { + xdsC.loadReportCh.Send(ReportLoadArgs{Server: server}) + return xdsC.loadStore, func() { + xdsC.lrsCancelCh.Send(nil) + } } -// WaitForReportLoad waits for ReportLoad to be invoked on this client within a -// reasonable timeout, and returns the arguments passed to it. -func (xdsC *Client) WaitForReportLoad() (ReportLoadArgs, error) { - val, err := xdsC.loadReportCh.Receive() - return val.(ReportLoadArgs), err +// WaitForCancelReportLoad waits for a load report to be cancelled and returns +// context.DeadlineExceeded otherwise. +func (xdsC *Client) WaitForCancelReportLoad(ctx context.Context) error { + _, err := xdsC.lrsCancelCh.Receive(ctx) + return err } -// Close closes the xds client. -func (xdsC *Client) Close() { - xdsC.closeCh.Send(nil) +// LoadStore returns the underlying load data store. +func (xdsC *Client) LoadStore() *load.Store { + return xdsC.loadStore } -// WaitForClose waits for Close to be invoked on this client within a -// reasonable timeout, and returns testutils.ErrRecvTimeout otherwise. -func (xdsC *Client) WaitForClose() error { - _, err := xdsC.closeCh.Receive() - return err +// WaitForReportLoad waits for ReportLoad to be invoked on this client and +// returns the arguments passed to it. +func (xdsC *Client) WaitForReportLoad(ctx context.Context) (ReportLoadArgs, error) { + val, err := xdsC.loadReportCh.Receive(ctx) + if err != nil { + return ReportLoadArgs{}, err + } + return val.(ReportLoadArgs), nil +} + +// BootstrapConfig returns the bootstrap config. +func (xdsC *Client) BootstrapConfig() *bootstrap.Config { + return xdsC.bootstrapCfg +} + +// SetBootstrapConfig updates the bootstrap config. +func (xdsC *Client) SetBootstrapConfig(cfg *bootstrap.Config) { + xdsC.bootstrapCfg = cfg } // Name returns the name of the xds client. @@ -199,13 +298,20 @@ func NewClient() *Client { func NewClientWithName(name string) *Client { return &Client{ name: name, - suWatchCh: testutils.NewChannel(), - cdsWatchCh: testutils.NewChannel(), - edsWatchCh: testutils.NewChannel(), - suCancelCh: testutils.NewChannel(), - cdsCancelCh: testutils.NewChannel(), - edsCancelCh: testutils.NewChannel(), + ldsWatchCh: testutils.NewChannelWithSize(10), + rdsWatchCh: testutils.NewChannelWithSize(10), + cdsWatchCh: testutils.NewChannelWithSize(10), + edsWatchCh: testutils.NewChannelWithSize(10), + ldsCancelCh: testutils.NewChannelWithSize(10), + rdsCancelCh: testutils.NewChannelWithSize(10), + cdsCancelCh: testutils.NewChannelWithSize(10), + edsCancelCh: testutils.NewChannelWithSize(10), loadReportCh: testutils.NewChannel(), - closeCh: testutils.NewChannel(), + lrsCancelCh: testutils.NewChannel(), + loadStore: load.NewStore(), + bootstrapCfg: &bootstrap.Config{ClientDefaultListenerResourceNameTemplate: "%s"}, + rdsCbs: make(map[string]func(xdsresource.RouteConfigUpdate, error)), + cdsCbs: make(map[string]func(xdsresource.ClusterUpdate, error)), + edsCbs: make(map[string]func(xdsresource.EndpointsUpdate, error)), } } diff --git a/xds/internal/testutils/protos.go b/xds/internal/testutils/protos.go index 25a0944d96dc..fc3cdf307fcd 100644 --- a/xds/internal/testutils/protos.go +++ b/xds/internal/testutils/protos.go @@ -18,7 +18,6 @@ package testutils import ( - "fmt" "net" "strconv" @@ -59,11 +58,11 @@ type ClusterLoadAssignmentBuilder struct { } // NewClusterLoadAssignmentBuilder creates a ClusterLoadAssignmentBuilder. -func NewClusterLoadAssignmentBuilder(clusterName string, dropPercents []uint32) *ClusterLoadAssignmentBuilder { - var drops []*v2xdspb.ClusterLoadAssignment_Policy_DropOverload - for i, d := range dropPercents { +func NewClusterLoadAssignmentBuilder(clusterName string, dropPercents map[string]uint32) *ClusterLoadAssignmentBuilder { + drops := make([]*v2xdspb.ClusterLoadAssignment_Policy_DropOverload, 0, len(dropPercents)) + for n, d := range dropPercents { drops = append(drops, &v2xdspb.ClusterLoadAssignment_Policy_DropOverload{ - Category: fmt.Sprintf("test-drop-%d", i), + Category: n, DropPercentage: &v2typepb.FractionalPercent{ Numerator: d, Denominator: v2typepb.FractionalPercent_HUNDRED, @@ -89,7 +88,7 @@ type AddLocalityOptions struct { // AddLocality adds a locality to the builder. func (clab *ClusterLoadAssignmentBuilder) AddLocality(subzone string, weight uint32, priority uint32, addrsWithPort []string, opts *AddLocalityOptions) { - var lbEndPoints []*v2endpointpb.LbEndpoint + lbEndPoints := make([]*v2endpointpb.LbEndpoint, 0, len(addrsWithPort)) for i, a := range addrsWithPort { host, portStr, err := net.SplitHostPort(a) if err != nil { diff --git a/xds/internal/testutils/resource_watcher.go b/xds/internal/testutils/resource_watcher.go new file mode 100644 index 000000000000..aac9c1464774 --- /dev/null +++ b/xds/internal/testutils/resource_watcher.go @@ -0,0 +1,75 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package testutils + +import "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" + +// TestResourceWatcher implements the xdsresource.ResourceWatcher interface, +// used to receive updates on watches registered with the xDS client, when using +// the resource-type agnostic WatchResource API. +// +// Tests can the channels provided by this tyep to get access to updates and +// errors sent by the xDS client. +type TestResourceWatcher struct { + // UpdateCh is the channel on which xDS client updates are delivered. + UpdateCh chan *xdsresource.ResourceData + // ErrorCh is the channel on which errors from the xDS client are delivered. + ErrorCh chan error + // ResourceDoesNotExistCh is the channel used to indicate calls to OnResourceDoesNotExist + ResourceDoesNotExistCh chan struct{} +} + +// OnUpdate is invoked by the xDS client to report the latest update on the resource +// being watched. +func (w *TestResourceWatcher) OnUpdate(data xdsresource.ResourceData) { + select { + case <-w.UpdateCh: + default: + } + w.UpdateCh <- &data +} + +// OnError is invoked by the xDS client to report the latest error. +func (w *TestResourceWatcher) OnError(err error) { + select { + case <-w.ErrorCh: + default: + } + w.ErrorCh <- err +} + +// OnResourceDoesNotExist is used by the xDS client to report that the resource +// being watched no longer exists. +func (w *TestResourceWatcher) OnResourceDoesNotExist() { + select { + case <-w.ResourceDoesNotExistCh: + default: + } + w.ResourceDoesNotExistCh <- struct{}{} +} + +// NewTestResourceWatcher returns a TestResourceWatcher to watch for resources +// via the xDS client. +func NewTestResourceWatcher() *TestResourceWatcher { + return &TestResourceWatcher{ + UpdateCh: make(chan *xdsresource.ResourceData, 1), + ErrorCh: make(chan error, 1), + ResourceDoesNotExistCh: make(chan struct{}, 1), + } +} diff --git a/xds/internal/testutils/testutils.go b/xds/internal/testutils/testutils.go new file mode 100644 index 000000000000..44891780e0c4 --- /dev/null +++ b/xds/internal/testutils/testutils.go @@ -0,0 +1,72 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package testutils provides utility types, for use in xds tests. +package testutils + +import ( + "fmt" + "testing" + + "google.golang.org/grpc/xds/internal/xdsclient/bootstrap" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource/version" +) + +// BuildResourceName returns the resource name in the format of an xdstp:// +// resource. +func BuildResourceName(typeName, auth, id string, ctxParams map[string]string) string { + var typS string + switch typeName { + case xdsresource.ListenerResourceTypeName: + typS = version.V3ListenerType + case xdsresource.RouteConfigTypeName: + typS = version.V3RouteConfigType + case xdsresource.ClusterResourceTypeName: + typS = version.V3ClusterType + case xdsresource.EndpointsResourceTypeName: + typS = version.V3EndpointsType + default: + // If the name doesn't match any of the standard resources fallback + // to the type name. + typS = typeName + } + return (&xdsresource.Name{ + Scheme: "xdstp", + Authority: auth, + Type: typS, + ID: id, + ContextParams: ctxParams, + }).String() +} + +// ServerConfigForAddress returns a bootstrap.ServerConfig for the given address +// with default values of insecure channel_creds and v3 server_features. +func ServerConfigForAddress(t *testing.T, addr string) *bootstrap.ServerConfig { + t.Helper() + + jsonCfg := fmt.Sprintf(`{ + "server_uri": "%s", + "channel_creds": [{"type": "insecure"}], + "server_features": ["xds_v3"] + }`, addr) + sc, err := bootstrap.ServerConfigFromJSON([]byte(jsonCfg)) + if err != nil { + t.Fatalf("Failed to create server config from JSON %s: %v", jsonCfg, err) + } + return sc +} diff --git a/xds/internal/version/version.go b/xds/internal/version/version.go deleted file mode 100644 index d1ca46f658a1..000000000000 --- a/xds/internal/version/version.go +++ /dev/null @@ -1,49 +0,0 @@ -/* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -// Package version defines constants to distinguish between supported xDS API -// versions. -package version - -// TransportAPI refers to the API version for xDS transport protocol. This -// describes the xDS gRPC endpoint and version of DiscoveryRequest/Response used -// on the wire. -type TransportAPI int - -const ( - // TransportV2 refers to the v2 xDS transport protocol. - TransportV2 TransportAPI = iota - // TransportV3 refers to the v3 xDS transport protocol. - TransportV3 -) - -// Resource URLs. We need to be able to accept either version of the resource -// regardless of the version of the transport protocol in use. -const ( - V2ListenerURL = "type.googleapis.com/envoy.api.v2.Listener" - V2RouteConfigURL = "type.googleapis.com/envoy.api.v2.RouteConfiguration" - V2ClusterURL = "type.googleapis.com/envoy.api.v2.Cluster" - V2EndpointsURL = "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment" - V2HTTPConnManagerURL = "type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager" - - V3ListenerURL = "type.googleapis.com/envoy.config.listener.v3.Listener" - V3RouteConfigURL = "type.googleapis.com/envoy.config.route.v3.RouteConfiguration" - V3ClusterURL = "type.googleapis.com/envoy.config.cluster.v3.Cluster" - V3EndpointsURL = "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment" - V3HTTPConnManagerURL = "type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager" -) diff --git a/xds/internal/xdsclient/attributes.go b/xds/internal/xdsclient/attributes.go new file mode 100644 index 000000000000..9076a76fd0dc --- /dev/null +++ b/xds/internal/xdsclient/attributes.go @@ -0,0 +1,36 @@ +/* + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient + +import "google.golang.org/grpc/resolver" + +type clientKeyType string + +const clientKey = clientKeyType("grpc.xds.internal.client.Client") + +// FromResolverState returns the Client from state, or nil if not present. +func FromResolverState(state resolver.State) XDSClient { + cs, _ := state.Attributes.Value(clientKey).(XDSClient) + return cs +} + +// SetClient sets c in state and returns the new state. +func SetClient(state resolver.State, c XDSClient) resolver.State { + state.Attributes = state.Attributes.WithValue(clientKey, c) + return state +} diff --git a/xds/internal/xdsclient/authority.go b/xds/internal/xdsclient/authority.go new file mode 100644 index 000000000000..6ad61dae4ae4 --- /dev/null +++ b/xds/internal/xdsclient/authority.go @@ -0,0 +1,609 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsclient + +import ( + "context" + "errors" + "fmt" + "strings" + "sync" + "time" + + "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/xds/internal/xdsclient/bootstrap" + "google.golang.org/grpc/xds/internal/xdsclient/load" + "google.golang.org/grpc/xds/internal/xdsclient/transport" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" + "google.golang.org/protobuf/types/known/anypb" +) + +type watchState int + +const ( + watchStateStarted watchState = iota // Watch started, request not yet set. + watchStateRequested // Request sent for resource being watched. + watchStateReceived // Response received for resource being watched. + watchStateTimeout // Watch timer expired, no response. + watchStateCanceled // Watch cancelled. +) + +type resourceState struct { + watchers map[xdsresource.ResourceWatcher]bool // Set of watchers for this resource + cache xdsresource.ResourceData // Most recent ACKed update for this resource + md xdsresource.UpdateMetadata // Metadata for the most recent update + deletionIgnored bool // True if resource deletion was ignored for a prior update + + // Common watch state for all watchers of this resource. + wTimer *time.Timer // Expiry timer + wState watchState // State of the watch +} + +// authority wraps all state associated with a single management server. It +// contains the transport used to communicate with the management server and a +// cache of resource state for resources requested from the management server. +// +// Bootstrap configuration could contain multiple entries in the authorities map +// that share the same server config (server address and credentials to use). We +// share the same authority instance amongst these entries, and the reference +// counting is taken care of by the `clientImpl` type. +type authority struct { + serverCfg *bootstrap.ServerConfig // Server config for this authority + bootstrapCfg *bootstrap.Config // Full bootstrap configuration + refCount int // Reference count of watches referring to this authority + serializer *grpcsync.CallbackSerializer // Callback serializer for invoking watch callbacks + resourceTypeGetter func(string) xdsresource.Type // ResourceType registry lookup + transport *transport.Transport // Underlying xDS transport to the management server + watchExpiryTimeout time.Duration // Resource watch expiry timeout + logger *grpclog.PrefixLogger + + // A two level map containing the state of all the resources being watched. + // + // The first level map key is the ResourceType (Listener, Route etc). This + // allows us to have a single map for all resources instead of having per + // resource-type maps. + // + // The second level map key is the resource name, with the value being the + // actual state of the resource. + resourcesMu sync.Mutex + resources map[xdsresource.Type]map[string]*resourceState + closed bool +} + +// authorityArgs is a convenience struct to wrap arguments required to create a +// new authority. All fields here correspond directly to appropriate fields +// stored in the authority struct. +type authorityArgs struct { + // The reason for passing server config and bootstrap config separately + // (although the former is part of the latter) is because authorities in the + // bootstrap config might contain an empty server config, and in this case, + // the top-level server config is to be used. + // + // There are two code paths from where a new authority struct might be + // created. One is when a watch is registered for a resource, and one is + // when load reporting needs to be started. We have the authority name in + // the first case, but do in the second. We only have the server config in + // the second case. + serverCfg *bootstrap.ServerConfig + bootstrapCfg *bootstrap.Config + serializer *grpcsync.CallbackSerializer + resourceTypeGetter func(string) xdsresource.Type + watchExpiryTimeout time.Duration + logger *grpclog.PrefixLogger +} + +func newAuthority(args authorityArgs) (*authority, error) { + ret := &authority{ + serverCfg: args.serverCfg, + bootstrapCfg: args.bootstrapCfg, + serializer: args.serializer, + resourceTypeGetter: args.resourceTypeGetter, + watchExpiryTimeout: args.watchExpiryTimeout, + logger: args.logger, + resources: make(map[xdsresource.Type]map[string]*resourceState), + } + + tr, err := transport.New(transport.Options{ + ServerCfg: *args.serverCfg, + OnRecvHandler: ret.handleResourceUpdate, + OnErrorHandler: ret.newConnectionError, + OnSendHandler: ret.transportOnSendHandler, + Logger: args.logger, + NodeProto: args.bootstrapCfg.NodeProto, + }) + if err != nil { + return nil, fmt.Errorf("creating new transport to %q: %v", args.serverCfg, err) + } + ret.transport = tr + return ret, nil +} + +// transportOnSendHandler is called by the underlying transport when it sends a +// resource request successfully. Timers are activated for resources waiting for +// a response. +func (a *authority) transportOnSendHandler(u *transport.ResourceSendInfo) { + rType := a.resourceTypeGetter(u.URL) + // Resource type not found is not expected under normal circumstances, since + // the resource type url passed to the transport is determined by the authority. + if rType == nil { + a.logger.Warningf("Unknown resource type url: %s.", u.URL) + return + } + a.resourcesMu.Lock() + defer a.resourcesMu.Unlock() + a.startWatchTimersLocked(rType, u.ResourceNames) +} + +func (a *authority) handleResourceUpdate(resourceUpdate transport.ResourceUpdate) error { + rType := a.resourceTypeGetter(resourceUpdate.URL) + if rType == nil { + return xdsresource.NewErrorf(xdsresource.ErrorTypeResourceTypeUnsupported, "Resource URL %v unknown in response from server", resourceUpdate.URL) + } + + opts := &xdsresource.DecodeOptions{BootstrapConfig: a.bootstrapCfg} + updates, md, err := decodeAllResources(opts, rType, resourceUpdate) + a.updateResourceStateAndScheduleCallbacks(rType, updates, md) + return err +} + +func (a *authority) updateResourceStateAndScheduleCallbacks(rType xdsresource.Type, updates map[string]resourceDataErrTuple, md xdsresource.UpdateMetadata) { + a.resourcesMu.Lock() + defer a.resourcesMu.Unlock() + + resourceStates := a.resources[rType] + for name, uErr := range updates { + if state, ok := resourceStates[name]; ok { + // Cancel the expiry timer associated with the resource once a + // response is received, irrespective of whether the update is a + // good one or not. + // + // We check for watch states `started` and `requested` here to + // accommodate for a race which can happen in the following + // scenario: + // - When a watch is registered, it is possible that the ADS stream + // is not yet created. In this case, the request for the resource + // is not sent out immediately. An entry in the `resourceStates` + // map is created with a watch state of `started`. + // - Once the stream is created, it is possible that the management + // server might respond with the requested resource before we send + // out request for the same. If we don't check for `started` here, + // and move the state to `received`, we will end up starting the + // timer when the request gets sent out. And since the mangement + // server already sent us the resource, there is a good chance + // that it will not send it again. This would eventually lead to + // the timer firing, even though we have the resource in the + // cache. + if state.wState == watchStateStarted || state.wState == watchStateRequested { + // It is OK to ignore the return value from Stop() here because + // if the timer has already fired, it means that the timer watch + // expiry callback is blocked on the same lock that we currently + // hold. Since we move the state to `received` here, the timer + // callback will be a no-op. + if state.wTimer != nil { + state.wTimer.Stop() + } + state.wState = watchStateReceived + } + + if uErr.err != nil { + // On error, keep previous version of the resource. But update + // status and error. + state.md.ErrState = md.ErrState + state.md.Status = md.Status + for watcher := range state.watchers { + watcher := watcher + err := uErr.err + a.serializer.Schedule(func(context.Context) { watcher.OnError(err) }) + } + continue + } + + if state.deletionIgnored { + state.deletionIgnored = false + a.logger.Infof("A valid update was received for resource %q of type %q after previously ignoring a deletion", name, rType.TypeName()) + } + // Notify watchers only if this is a first time update or it is different + // from the one currently cached. + if state.cache == nil || !state.cache.Equal(uErr.resource) { + for watcher := range state.watchers { + watcher := watcher + resource := uErr.resource + a.serializer.Schedule(func(context.Context) { watcher.OnUpdate(resource) }) + } + } + // Sync cache. + a.logger.Debugf("Resource type %q with name %q added to cache", rType.TypeName(), name) + state.cache = uErr.resource + // Set status to ACK, and clear error state. The metadata might be a + // NACK metadata because some other resources in the same response + // are invalid. + state.md = md + state.md.ErrState = nil + state.md.Status = xdsresource.ServiceStatusACKed + if md.ErrState != nil { + state.md.Version = md.ErrState.Version + } + } + } + + // If this resource type requires that all resources be present in every + // SotW response from the server, a response that does not include a + // previously seen resource will be interpreted as a deletion of that + // resource unless ignore_resource_deletion option was set in the server + // config. + if !rType.AllResourcesRequiredInSotW() { + return + } + for name, state := range resourceStates { + if state.cache == nil { + // If the resource state does not contain a cached update, which can + // happen when: + // - resource was newly requested but has not yet been received, or, + // - resource was removed as part of a previous update, + // we don't want to generate an error for the watchers. + // + // For the first of the above two conditions, this ADS response may + // be in reaction to an earlier request that did not yet request the + // new resource, so its absence from the response does not + // necessarily indicate that the resource does not exist. For that + // case, we rely on the request timeout instead. + // + // For the second of the above two conditions, we already generated + // an error when we received the first response which removed this + // resource. So, there is no need to generate another one. + continue + } + if _, ok := updates[name]; !ok { + // The metadata status is set to "ServiceStatusNotExist" if a + // previous update deleted this resource, in which case we do not + // want to repeatedly call the watch callbacks with a + // "resource-not-found" error. + if state.md.Status == xdsresource.ServiceStatusNotExist { + continue + } + // Per A53, resource deletions are ignored if the `ignore_resource_deletion` + // server feature is enabled through the bootstrap configuration. If the + // resource deletion is to be ignored, the resource is not removed from + // the cache and the corresponding OnResourceDoesNotExist() callback is + // not invoked on the watchers. + if a.serverCfg.IgnoreResourceDeletion { + if !state.deletionIgnored { + state.deletionIgnored = true + a.logger.Warningf("Ignoring resource deletion for resource %q of type %q", name, rType.TypeName()) + } + continue + } + // If resource exists in cache, but not in the new update, delete + // the resource from cache, and also send a resource not found error + // to indicate resource removed. Metadata for the resource is still + // maintained, as this is required by CSDS. + state.cache = nil + state.md = xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusNotExist} + for watcher := range state.watchers { + watcher := watcher + a.serializer.Schedule(func(context.Context) { watcher.OnResourceDoesNotExist() }) + } + } + } +} + +type resourceDataErrTuple struct { + resource xdsresource.ResourceData + err error +} + +func decodeAllResources(opts *xdsresource.DecodeOptions, rType xdsresource.Type, update transport.ResourceUpdate) (map[string]resourceDataErrTuple, xdsresource.UpdateMetadata, error) { + timestamp := time.Now() + md := xdsresource.UpdateMetadata{ + Version: update.Version, + Timestamp: timestamp, + } + + topLevelErrors := make([]error, 0) // Tracks deserialization errors, where we don't have a resource name. + perResourceErrors := make(map[string]error) // Tracks resource validation errors, where we have a resource name. + ret := make(map[string]resourceDataErrTuple) // Return result, a map from resource name to either resource data or error. + for _, r := range update.Resources { + result, err := rType.Decode(opts, r) + + // Name field of the result is left unpopulated only when resource + // deserialization fails. + name := "" + if result != nil { + name = xdsresource.ParseName(result.Name).String() + } + if err == nil { + ret[name] = resourceDataErrTuple{resource: result.Resource} + continue + } + if name == "" { + topLevelErrors = append(topLevelErrors, err) + continue + } + perResourceErrors[name] = err + // Add place holder in the map so we know this resource name was in + // the response. + ret[name] = resourceDataErrTuple{err: err} + } + + if len(topLevelErrors) == 0 && len(perResourceErrors) == 0 { + md.Status = xdsresource.ServiceStatusACKed + return ret, md, nil + } + + md.Status = xdsresource.ServiceStatusNACKed + errRet := combineErrors(rType.TypeName(), topLevelErrors, perResourceErrors) + md.ErrState = &xdsresource.UpdateErrorMetadata{ + Version: update.Version, + Err: errRet, + Timestamp: timestamp, + } + return ret, md, errRet +} + +// startWatchTimersLocked is invoked upon transport.OnSend() callback with resources +// requested on the underlying ADS stream. This satisfies the conditions to start +// watch timers per A57 [https://github.com/grpc/proposal/blob/master/A57-xds-client-failure-mode-behavior.md#handling-resources-that-do-not-exist] +// +// Caller must hold a.resourcesMu. +func (a *authority) startWatchTimersLocked(rType xdsresource.Type, resourceNames []string) { + resourceStates := a.resources[rType] + for _, resourceName := range resourceNames { + if state, ok := resourceStates[resourceName]; ok { + if state.wState != watchStateStarted { + continue + } + state.wTimer = time.AfterFunc(a.watchExpiryTimeout, func() { + a.handleWatchTimerExpiry(rType, resourceName, state) + }) + state.wState = watchStateRequested + } + } +} + +// stopWatchTimersLocked is invoked upon connection errors to stops watch timers +// for resources that have been requested, but not yet responded to by the management +// server. +// +// Caller must hold a.resourcesMu. +func (a *authority) stopWatchTimersLocked() { + for _, rType := range a.resources { + for resourceName, state := range rType { + if state.wState != watchStateRequested { + continue + } + if !state.wTimer.Stop() { + // If the timer has already fired, it means that the timer watch expiry + // callback is blocked on the same lock that we currently hold. Don't change + // the watch state and instead let the watch expiry callback handle it. + a.logger.Warningf("Watch timer for resource %v already fired. Ignoring here.", resourceName) + continue + } + state.wTimer = nil + state.wState = watchStateStarted + } + } +} + +// newConnectionError is called by the underlying transport when it receives a +// connection error. The error will be forwarded to all the resource watchers. +func (a *authority) newConnectionError(err error) { + a.resourcesMu.Lock() + defer a.resourcesMu.Unlock() + + a.stopWatchTimersLocked() + + // We do not consider it an error if the ADS stream was closed after having received + // a response on the stream. This is because there are legitimate reasons why the server + // may need to close the stream during normal operations, such as needing to rebalance + // load or the underlying connection hitting its max connection age limit. + // See gRFC A57 for more details. + if xdsresource.ErrType(err) == xdsresource.ErrTypeStreamFailedAfterRecv { + a.logger.Warningf("Watchers not notified since ADS stream failed after having received at least one response: %v", err) + return + } + + for _, rType := range a.resources { + for _, state := range rType { + // Propagate the connection error from the transport layer to all watchers. + for watcher := range state.watchers { + watcher := watcher + a.serializer.Schedule(func(context.Context) { + watcher.OnError(xdsresource.NewErrorf(xdsresource.ErrorTypeConnection, "xds: error received from xDS stream: %v", err)) + }) + } + } + } +} + +// Increments the reference count. Caller must hold parent's authorityMu. +func (a *authority) refLocked() { + a.refCount++ +} + +// Decrements the reference count. Caller must hold parent's authorityMu. +func (a *authority) unrefLocked() int { + a.refCount-- + return a.refCount +} + +func (a *authority) close() { + a.transport.Close() + + a.resourcesMu.Lock() + a.closed = true + a.resourcesMu.Unlock() +} + +func (a *authority) watchResource(rType xdsresource.Type, resourceName string, watcher xdsresource.ResourceWatcher) func() { + a.logger.Debugf("New watch for type %q, resource name %q", rType.TypeName(), resourceName) + a.resourcesMu.Lock() + defer a.resourcesMu.Unlock() + + // Lookup the ResourceType specific resources from the top-level map. If + // there is no entry for this ResourceType, create one. + resources := a.resources[rType] + if resources == nil { + resources = make(map[string]*resourceState) + a.resources[rType] = resources + } + + // Lookup the resourceState for the particular resource that the watch is + // being registered for. If this is the first watch for this resource, + // instruct the transport layer to send a DiscoveryRequest for the same. + state := resources[resourceName] + if state == nil { + a.logger.Debugf("First watch for type %q, resource name %q", rType.TypeName(), resourceName) + state = &resourceState{ + watchers: make(map[xdsresource.ResourceWatcher]bool), + md: xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusRequested}, + wState: watchStateStarted, + } + resources[resourceName] = state + a.sendDiscoveryRequestLocked(rType, resources) + } + // Always add the new watcher to the set of watchers. + state.watchers[watcher] = true + + // If we have a cached copy of the resource, notify the new watcher. + if state.cache != nil { + a.logger.Debugf("Resource type %q with resource name %q found in cache: %s", rType.TypeName(), resourceName, state.cache.ToJSON()) + resource := state.cache + a.serializer.Schedule(func(context.Context) { watcher.OnUpdate(resource) }) + } + + return func() { + a.resourcesMu.Lock() + defer a.resourcesMu.Unlock() + + // We already have a reference to the resourceState for this particular + // resource. Avoid indexing into the two-level map to figure this out. + + // Delete this particular watcher from the list of watchers, so that its + // callback will not be invoked in the future. + state.wState = watchStateCanceled + delete(state.watchers, watcher) + if len(state.watchers) > 0 { + return + } + + // There are no more watchers for this resource, delete the state + // associated with it, and instruct the transport to send a request + // which does not include this resource name. + a.logger.Debugf("Removing last watch for type %q, resource name %q", rType.TypeName(), resourceName) + delete(resources, resourceName) + a.sendDiscoveryRequestLocked(rType, resources) + } +} + +func (a *authority) handleWatchTimerExpiry(rType xdsresource.Type, resourceName string, state *resourceState) { + a.resourcesMu.Lock() + defer a.resourcesMu.Unlock() + + if a.closed { + return + } + a.logger.Warningf("Watch for resource %q of type %s timed out", resourceName, rType.TypeName()) + + switch state.wState { + case watchStateRequested: + // This is the only state where we need to handle the timer expiry by + // invoking appropriate watch callbacks. This is handled outside the switch. + case watchStateCanceled: + return + default: + a.logger.Warningf("Unexpected watch state %q for resource %q.", state.wState, resourceName) + return + } + + state.wState = watchStateTimeout + // With the watch timer firing, it is safe to assume that the resource does + // not exist on the management server. + state.cache = nil + state.md = xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusNotExist} + for watcher := range state.watchers { + watcher := watcher + a.serializer.Schedule(func(context.Context) { watcher.OnResourceDoesNotExist() }) + } +} + +// sendDiscoveryRequestLocked sends a discovery request for the specified +// resource type and resource names. Even though this method does not directly +// access the resource cache, it is important that `resourcesMu` be beld when +// calling this method to ensure that a consistent snapshot of resource names is +// being requested. +func (a *authority) sendDiscoveryRequestLocked(rType xdsresource.Type, resources map[string]*resourceState) { + resourcesToRequest := make([]string, len(resources)) + i := 0 + for name := range resources { + resourcesToRequest[i] = name + i++ + } + a.transport.SendRequest(rType.TypeURL(), resourcesToRequest) +} + +func (a *authority) reportLoad() (*load.Store, func()) { + return a.transport.ReportLoad() +} + +func (a *authority) dumpResources() map[string]map[string]xdsresource.UpdateWithMD { + a.resourcesMu.Lock() + defer a.resourcesMu.Unlock() + + dump := make(map[string]map[string]xdsresource.UpdateWithMD) + for rType, resourceStates := range a.resources { + states := make(map[string]xdsresource.UpdateWithMD) + for name, state := range resourceStates { + var raw *anypb.Any + if state.cache != nil { + raw = state.cache.Raw() + } + states[name] = xdsresource.UpdateWithMD{ + MD: state.md, + Raw: raw, + } + } + dump[rType.TypeURL()] = states + } + return dump +} + +func combineErrors(rType string, topLevelErrors []error, perResourceErrors map[string]error) error { + var errStrB strings.Builder + errStrB.WriteString(fmt.Sprintf("error parsing %q response: ", rType)) + if len(topLevelErrors) > 0 { + errStrB.WriteString("top level errors: ") + for i, err := range topLevelErrors { + if i != 0 { + errStrB.WriteString(";\n") + } + errStrB.WriteString(err.Error()) + } + } + if len(perResourceErrors) > 0 { + var i int + for name, err := range perResourceErrors { + if i != 0 { + errStrB.WriteString(";\n") + } + i++ + errStrB.WriteString(fmt.Sprintf("resource %q: %v", name, err.Error())) + } + } + return errors.New(errStrB.String()) +} diff --git a/xds/internal/xdsclient/authority_test.go b/xds/internal/xdsclient/authority_test.go new file mode 100644 index 000000000000..09a81759a1f3 --- /dev/null +++ b/xds/internal/xdsclient/authority_test.go @@ -0,0 +1,297 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/google/uuid" + "google.golang.org/grpc/internal/grpcsync" + util "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/xds/internal" + + "google.golang.org/grpc/xds/internal/testutils" + xdstestutils "google.golang.org/grpc/xds/internal/testutils" + "google.golang.org/grpc/xds/internal/xdsclient/bootstrap" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource/version" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + _ "google.golang.org/grpc/xds/internal/httpfilter/router" // Register the router filter. +) + +var emptyServerOpts = e2e.ManagementServerOptions{} + +var ( + // Listener resource type implementation retrieved from the resource type map + // in the internal package, which is initialized when the individual resource + // types are created. + listenerResourceType = internal.ResourceTypeMapForTesting[version.V3ListenerURL].(xdsresource.Type) + rtRegistry = newResourceTypeRegistry() +) + +func init() { + // Simulating maybeRegister for listenerResourceType. The getter to this registry + // is passed to the authority for accessing the resource type. + rtRegistry.types[listenerResourceType.TypeURL()] = listenerResourceType +} + +func setupTest(ctx context.Context, t *testing.T, opts e2e.ManagementServerOptions, watchExpiryTimeout time.Duration) (*authority, *e2e.ManagementServer, string) { + t.Helper() + nodeID := uuid.New().String() + ms, err := e2e.StartManagementServer(opts) + if err != nil { + t.Fatalf("Failed to spin up the xDS management server: %q", err) + } + + a, err := newAuthority(authorityArgs{ + serverCfg: xdstestutils.ServerConfigForAddress(t, ms.Address), + bootstrapCfg: &bootstrap.Config{ + NodeProto: &v3corepb.Node{Id: nodeID}, + }, + serializer: grpcsync.NewCallbackSerializer(ctx), + resourceTypeGetter: rtRegistry.get, + watchExpiryTimeout: watchExpiryTimeout, + logger: nil, + }) + if err != nil { + t.Fatalf("Failed to create authority: %q", err) + } + return a, ms, nodeID +} + +// This tests verifies watch and timer state for the scenario where a watch for +// an LDS resource is registered and the management server sends an update the +// same resource. +func (s) TestTimerAndWatchStateOnSendCallback(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + a, ms, nodeID := setupTest(ctx, t, emptyServerOpts, defaultTestTimeout) + defer ms.Stop() + defer a.close() + + rn := "xdsclient-test-lds-resource" + w := testutils.NewTestResourceWatcher() + cancelResource := a.watchResource(listenerResourceType, rn, w) + defer cancelResource() + + // Looping until the underlying transport has successfully sent the request to + // the server, which would call the onSend callback and transition the watchState + // to `watchStateRequested`. + for ctx.Err() == nil { + if err := compareWatchState(a, rn, watchStateRequested); err == nil { + break + } + } + if ctx.Err() != nil { + t.Fatalf("Test timed out before state transiton to %q was verified.", watchStateRequested) + } + + // Updating mgmt server with the same lds resource. Blocking on watcher's update + // ch to verify the watch state transition to `watchStateReceived`. + if err := updateResourceInServer(ctx, ms, rn, nodeID); err != nil { + t.Fatalf("Failed to update server with resource: %q; err: %q", rn, err) + } + for { + select { + case <-ctx.Done(): + t.Fatal("Test timed out before watcher received an update from server.") + case <-w.ErrorCh: + case <-w.UpdateCh: + // This means the OnUpdate callback was invoked and the watcher was notified. + if err := compareWatchState(a, rn, watchStateReceived); err != nil { + t.Fatal(err) + } + return + } + } +} + +// This tests the resource's watch state transition when the ADS stream is closed +// by the management server. After the test calls `watchResource` api to register +// a watch for a resource, it stops the management server, and verifies the resource's +// watch state transitions to `watchStateStarted` and timer ready to be restarted. +func (s) TestTimerAndWatchStateOnErrorCallback(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + a, ms, _ := setupTest(ctx, t, emptyServerOpts, defaultTestTimeout) + defer a.close() + + rn := "xdsclient-test-lds-resource" + w := testutils.NewTestResourceWatcher() + cancelResource := a.watchResource(listenerResourceType, rn, w) + defer cancelResource() + + // Stopping the server and blocking on watcher's err channel to be notified. + // This means the onErr callback should be invoked which transitions the watch + // state to `watchStateStarted`. + ms.Stop() + + select { + case <-ctx.Done(): + t.Fatal("Test timed out before verifying error propagation.") + case err := <-w.ErrorCh: + if xdsresource.ErrType(err) != xdsresource.ErrorTypeConnection { + t.Fatal("Connection error not propagated to watchers.") + } + } + + if err := compareWatchState(a, rn, watchStateStarted); err != nil { + t.Fatal(err) + } +} + +// This tests the case where the ADS stream breaks after successfully receiving +// a message on the stream. The test performs the following: +// - configures the management server with the ability to dropRequests based on +// a boolean flag. +// - update the mgmt server with resourceA. +// - registers a watch for resourceA and verifies that the watcher's update +// callback is invoked. +// - registers a watch for resourceB and verifies that the watcher's update +// callback is not invoked. This is because the management server does not +// contain resourceB. +// - force mgmt server to drop requests. Verify that watcher for resourceB gets +// connection error. +// - resume mgmt server to accept requests. +// - update the mgmt server with resourceB and verifies that the watcher's +// update callback is invoked. +func (s) TestWatchResourceTimerCanRestartOnIgnoredADSRecvError(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + // Create a restartable listener which can close existing connections. + l, err := util.LocalTCPListener() + if err != nil { + t.Fatalf("testutils.LocalTCPListener() failed: %v", err) + } + lis := util.NewRestartableListener(l) + defer lis.Close() + streamRestarted := grpcsync.NewEvent() + serverOpt := e2e.ManagementServerOptions{ + Listener: lis, + OnStreamClosed: func(int64, *v3corepb.Node) { + streamRestarted.Fire() + }, + } + + a, ms, nodeID := setupTest(ctx, t, serverOpt, defaultTestTimeout) + defer ms.Stop() + defer a.close() + + nameA := "xdsclient-test-lds-resourceA" + watcherA := testutils.NewTestResourceWatcher() + cancelA := a.watchResource(listenerResourceType, nameA, watcherA) + + if err := updateResourceInServer(ctx, ms, nameA, nodeID); err != nil { + t.Fatalf("Failed to update server with resource: %q; err: %q", nameA, err) + } + + // Blocking on resource A watcher's update Channel to verify that there is + // more than one msg(s) received the ADS stream. + select { + case <-ctx.Done(): + t.Fatal("Test timed out before watcher received the update.") + case err := <-watcherA.ErrorCh: + t.Fatalf("Watch got an unexpected error update: %q; want: valid update.", err) + case <-watcherA.UpdateCh: + } + + cancelA() + lis.Stop() + + nameB := "xdsclient-test-lds-resourceB" + watcherB := testutils.NewTestResourceWatcher() + cancelB := a.watchResource(listenerResourceType, nameB, watcherB) + defer cancelB() + + // Blocking on resource B watcher's error channel. This error should be due to + // connectivity issue when reconnecting because the mgmt server was already been + // stopped. Also verifying that OnResourceDoesNotExist() method was not invoked + // on the watcher. + select { + case <-ctx.Done(): + t.Fatal("Test timed out before mgmt server got the request.") + case u := <-watcherB.UpdateCh: + t.Fatalf("Watch got an unexpected resource update: %v.", u) + case <-watcherB.ResourceDoesNotExistCh: + t.Fatalf("Illegal invocation of OnResourceDoesNotExist() method on the watcher.") + case gotErr := <-watcherB.ErrorCh: + wantErr := xdsresource.ErrorTypeConnection + if xdsresource.ErrType(gotErr) != wantErr { + t.Fatalf("Watch got an unexpected error:%q. Want: %q.", gotErr, wantErr) + } + } + + // Updating server with resource B and also re-enabling requests on the server. + if err := updateResourceInServer(ctx, ms, nameB, nodeID); err != nil { + t.Fatalf("Failed to update server with resource: %q; err: %q", nameB, err) + } + lis.Restart() + + for { + select { + case <-ctx.Done(): + t.Fatal("Test timed out before watcher received the update.") + case <-watcherB.UpdateCh: + return + } + } +} + +func compareWatchState(a *authority, rn string, wantState watchState) error { + a.resourcesMu.Lock() + defer a.resourcesMu.Unlock() + gotState := a.resources[listenerResourceType][rn].wState + if gotState != wantState { + return fmt.Errorf("Got %v. Want: %v", gotState, wantState) + } + + wTimer := a.resources[listenerResourceType][rn].wTimer + switch gotState { + case watchStateRequested: + if wTimer == nil { + return fmt.Errorf("got nil timer, want active timer") + } + case watchStateStarted: + if wTimer != nil { + return fmt.Errorf("got active timer, want nil timer") + } + default: + if wTimer.Stop() { + // This means that the timer was running but could be successfully stopped. + return fmt.Errorf("got active timer, want stopped timer") + } + } + return nil +} + +func updateResourceInServer(ctx context.Context, ms *e2e.ManagementServer, rn string, nID string) error { + l := e2e.DefaultClientListener(rn, "new-rds-resource") + resources := e2e.UpdateOptions{ + NodeID: nID, + Listeners: []*v3listenerpb.Listener{l}, + SkipValidation: true, + } + return ms.Update(ctx, resources) +} diff --git a/xds/internal/xdsclient/bootstrap/bootstrap.go b/xds/internal/xdsclient/bootstrap/bootstrap.go new file mode 100644 index 000000000000..aec2fa51f30f --- /dev/null +++ b/xds/internal/xdsclient/bootstrap/bootstrap.go @@ -0,0 +1,564 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package bootstrap provides the functionality to initialize certain aspects +// of an xDS client by reading a bootstrap file. +package bootstrap + +import ( + "bytes" + "encoding/json" + "fmt" + "net/url" + "os" + "strings" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + "github.com/golang/protobuf/jsonpb" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/google" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/credentials/tls/certprovider" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/xds/bootstrap" +) + +const ( + // The "server_features" field in the bootstrap file contains a list of + // features supported by the server: + // - A value of "xds_v3" indicates that the server supports the v3 version of + // the xDS transport protocol. + // - A value of "ignore_resource_deletion" indicates that the client should + // ignore deletion of Listener and Cluster resources in updates from the + // server. + serverFeaturesV3 = "xds_v3" + serverFeaturesIgnoreResourceDeletion = "ignore_resource_deletion" + + gRPCUserAgentName = "gRPC Go" + clientFeatureNoOverprovisioning = "envoy.lb.does_not_support_overprovisioning" + clientFeatureResourceWrapper = "xds.config.resource-in-sotw" +) + +func init() { + bootstrap.RegisterCredentials(&insecureCredsBuilder{}) + bootstrap.RegisterCredentials(&googleDefaultCredsBuilder{}) +} + +// For overriding in unit tests. +var bootstrapFileReadFunc = os.ReadFile + +// insecureCredsBuilder implements the `Credentials` interface defined in +// package `xds/bootstrap` and encapsulates an insecure credential. +type insecureCredsBuilder struct{} + +func (i *insecureCredsBuilder) Build(json.RawMessage) (credentials.Bundle, error) { + return insecure.NewBundle(), nil +} + +func (i *insecureCredsBuilder) Name() string { + return "insecure" +} + +// googleDefaultCredsBuilder implements the `Credentials` interface defined in +// package `xds/boostrap` and encapsulates a Google Default credential. +type googleDefaultCredsBuilder struct{} + +func (d *googleDefaultCredsBuilder) Build(json.RawMessage) (credentials.Bundle, error) { + return google.NewDefaultCredentials(), nil +} + +func (d *googleDefaultCredsBuilder) Name() string { + return "google_default" +} + +// ChannelCreds contains the credentials to be used while communicating with an +// xDS server. It is also used to dedup servers with the same server URI. +type ChannelCreds struct { + // Type contains a unique name identifying the credentials type. The only + // supported types currently are "google_default" and "insecure". + Type string + // Config contains the JSON configuration associated with the credentials. + Config json.RawMessage +} + +// Equal reports whether cc and other are considered equal. +func (cc ChannelCreds) Equal(other ChannelCreds) bool { + return cc.Type == other.Type && bytes.Equal(cc.Config, other.Config) +} + +// String returns a string representation of the credentials. It contains the +// type and the config (if non-nil) separated by a "-". +func (cc ChannelCreds) String() string { + if cc.Config == nil { + return cc.Type + } + + // We do not expect the Marshal call to fail since we wrote to cc.Config + // after a successful unmarshaling from JSON configuration. Therefore, + // it is safe to ignore the error here. + b, _ := json.Marshal(cc.Config) + return cc.Type + "-" + string(b) +} + +// ServerConfig contains the configuration to connect to a server, including +// URI, creds, and transport API version (e.g. v2 or v3). +// +// It contains unexported fields that are initialized when unmarshaled from JSON +// using either the UnmarshalJSON() method or the ServerConfigFromJSON() +// function. Hence users are strongly encouraged not to use a literal struct +// initialization to create an instance of this type, but instead unmarshal from +// JSON using one of the two available options. +type ServerConfig struct { + // ServerURI is the management server to connect to. + // + // The bootstrap file contains an ordered list of xDS servers to contact for + // this authority. The first one is picked. + ServerURI string + // Creds contains the credentials to be used while communicationg with this + // xDS server. It is also used to dedup servers with the same server URI. + Creds ChannelCreds + // ServerFeatures contains a list of features supported by this xDS server. + // It is also used to dedup servers with the same server URI and creds. + ServerFeatures []string + + // As part of unmarshaling the JSON config into this struct, we ensure that + // the credentials config is valid by building an instance of the specified + // credentials and store it here as a grpc.DialOption for easy access when + // dialing this xDS server. + credsDialOption grpc.DialOption + + // IgnoreResourceDeletion controls the behavior of the xDS client when the + // server deletes a previously sent Listener or Cluster resource. If set, the + // xDS client will not invoke the watchers' OnResourceDoesNotExist() method + // when a resource is deleted, nor will it remove the existing resource value + // from its cache. + IgnoreResourceDeletion bool +} + +// CredsDialOption returns the configured credentials as a grpc dial option. +func (sc *ServerConfig) CredsDialOption() grpc.DialOption { + return sc.credsDialOption +} + +// String returns the string representation of the ServerConfig. +// +// This string representation will be used as map keys in federation +// (`map[ServerConfig]authority`), so that the xDS ClientConn and stream will be +// shared by authorities with different names but the same server config. +// +// It covers (almost) all the fields so the string can represent the config +// content. It doesn't cover NodeProto because NodeProto isn't used by +// federation. +func (sc *ServerConfig) String() string { + features := strings.Join(sc.ServerFeatures, "-") + return strings.Join([]string{sc.ServerURI, sc.Creds.String(), features}, "-") +} + +// MarshalJSON marshals the ServerConfig to json. +func (sc ServerConfig) MarshalJSON() ([]byte, error) { + server := xdsServer{ + ServerURI: sc.ServerURI, + ChannelCreds: []channelCreds{{Type: sc.Creds.Type, Config: sc.Creds.Config}}, + ServerFeatures: sc.ServerFeatures, + } + server.ServerFeatures = []string{serverFeaturesV3} + if sc.IgnoreResourceDeletion { + server.ServerFeatures = append(server.ServerFeatures, serverFeaturesIgnoreResourceDeletion) + } + return json.Marshal(server) +} + +// UnmarshalJSON takes the json data (a server) and unmarshals it to the struct. +func (sc *ServerConfig) UnmarshalJSON(data []byte) error { + var server xdsServer + if err := json.Unmarshal(data, &server); err != nil { + return fmt.Errorf("xds: json.Unmarshal(data) for field ServerConfig failed during bootstrap: %v", err) + } + + sc.ServerURI = server.ServerURI + sc.ServerFeatures = server.ServerFeatures + for _, f := range server.ServerFeatures { + if f == serverFeaturesIgnoreResourceDeletion { + sc.IgnoreResourceDeletion = true + } + } + for _, cc := range server.ChannelCreds { + // We stop at the first credential type that we support. + c := bootstrap.GetCredentials(cc.Type) + if c == nil { + continue + } + bundle, err := c.Build(cc.Config) + if err != nil { + return fmt.Errorf("failed to build credentials bundle from bootstrap for %q: %v", cc.Type, err) + } + sc.Creds = ChannelCreds(cc) + sc.credsDialOption = grpc.WithCredentialsBundle(bundle) + break + } + return nil +} + +// ServerConfigFromJSON creates a new ServerConfig from the given JSON +// configuration. This is the preferred way of creating a ServerConfig when +// hand-crafting the JSON configuration. +func ServerConfigFromJSON(data []byte) (*ServerConfig, error) { + sc := new(ServerConfig) + if err := sc.UnmarshalJSON(data); err != nil { + return nil, err + } + return sc, nil +} + +// Equal reports whether sc and other are considered equal. +func (sc *ServerConfig) Equal(other *ServerConfig) bool { + switch { + case sc == nil && other == nil: + return true + case (sc != nil) != (other != nil): + return false + case sc.ServerURI != other.ServerURI: + return false + case !sc.Creds.Equal(other.Creds): + return false + case !equalStringSlice(sc.ServerFeatures, other.ServerFeatures): + return false + } + return true +} + +func equalStringSlice(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +// unmarshalJSONServerConfigSlice unmarshals JSON to a slice. +func unmarshalJSONServerConfigSlice(data []byte) ([]*ServerConfig, error) { + var servers []*ServerConfig + if err := json.Unmarshal(data, &servers); err != nil { + return nil, fmt.Errorf("failed to unmarshal JSON to []*ServerConfig: %v", err) + } + if len(servers) < 1 { + return nil, fmt.Errorf("no management server found in JSON") + } + return servers, nil +} + +// Authority contains configuration for an Authority for an xDS control plane +// server. See the Authorities field in the Config struct for how it's used. +type Authority struct { + // ClientListenerResourceNameTemplate is template for the name of the + // Listener resource to subscribe to for a gRPC client channel. Used only + // when the channel is created using an "xds:" URI with this authority name. + // + // The token "%s", if present in this string, will be replaced + // with %-encoded service authority (i.e., the path part of the target + // URI used to create the gRPC channel). + // + // Must start with "xdstp:///". If it does not, + // that is considered a bootstrap file parsing error. + // + // If not present in the bootstrap file, defaults to + // "xdstp:///envoy.config.listener.v3.Listener/%s". + ClientListenerResourceNameTemplate string + // XDSServer contains the management server and config to connect to for + // this authority. + XDSServer *ServerConfig +} + +// UnmarshalJSON implement json unmarshaller. +func (a *Authority) UnmarshalJSON(data []byte) error { + var jsonData map[string]json.RawMessage + if err := json.Unmarshal(data, &jsonData); err != nil { + return fmt.Errorf("xds: failed to parse authority: %v", err) + } + + for k, v := range jsonData { + switch k { + case "xds_servers": + servers, err := unmarshalJSONServerConfigSlice(v) + if err != nil { + return fmt.Errorf("xds: json.Unmarshal(data) for field %q failed during bootstrap: %v", k, err) + } + a.XDSServer = servers[0] + case "client_listener_resource_name_template": + if err := json.Unmarshal(v, &a.ClientListenerResourceNameTemplate); err != nil { + return fmt.Errorf("xds: json.Unmarshal(%v) for field %q failed during bootstrap: %v", string(v), k, err) + } + } + } + return nil +} + +// Config provides the xDS client with several key bits of information that it +// requires in its interaction with the management server. The Config is +// initialized from the bootstrap file. +type Config struct { + // XDSServer is the management server to connect to. + // + // The bootstrap file contains a list of servers (with name+creds), but we + // pick the first one. + XDSServer *ServerConfig + // CertProviderConfigs contains a mapping from certificate provider plugin + // instance names to parsed buildable configs. + CertProviderConfigs map[string]*certprovider.BuildableConfig + // ServerListenerResourceNameTemplate is a template for the name of the + // Listener resource to subscribe to for a gRPC server. + // + // If starts with "xdstp:", will be interpreted as a new-style name, + // in which case the authority of the URI will be used to select the + // relevant configuration in the "authorities" map. + // + // The token "%s", if present in this string, will be replaced with the IP + // and port on which the server is listening. (e.g., "0.0.0.0:8080", + // "[::]:8080"). For example, a value of "example/resource/%s" could become + // "example/resource/0.0.0.0:8080". If the template starts with "xdstp:", + // the replaced string will be %-encoded. + // + // There is no default; if unset, xDS-based server creation fails. + ServerListenerResourceNameTemplate string + // A template for the name of the Listener resource to subscribe to + // for a gRPC client channel. Used only when the channel is created + // with an "xds:" URI with no authority. + // + // If starts with "xdstp:", will be interpreted as a new-style name, + // in which case the authority of the URI will be used to select the + // relevant configuration in the "authorities" map. + // + // The token "%s", if present in this string, will be replaced with + // the service authority (i.e., the path part of the target URI + // used to create the gRPC channel). If the template starts with + // "xdstp:", the replaced string will be %-encoded. + // + // Defaults to "%s". + ClientDefaultListenerResourceNameTemplate string + // Authorities is a map of authority name to corresponding configuration. + // + // This is used in the following cases: + // - A gRPC client channel is created using an "xds:" URI that includes + // an authority. + // - A gRPC client channel is created using an "xds:" URI with no + // authority, but the "client_default_listener_resource_name_template" + // field above turns it into an "xdstp:" URI. + // - A gRPC server is created and the + // "server_listener_resource_name_template" field is an "xdstp:" URI. + // + // In any of those cases, it is an error if the specified authority is + // not present in this map. + Authorities map[string]*Authority + // NodeProto contains the Node proto to be used in xDS requests. This will be + // of type *v3corepb.Node. + NodeProto *v3corepb.Node +} + +type channelCreds struct { + Type string `json:"type"` + Config json.RawMessage `json:"config,omitempty"` +} + +type xdsServer struct { + ServerURI string `json:"server_uri"` + ChannelCreds []channelCreds `json:"channel_creds"` + ServerFeatures []string `json:"server_features"` +} + +func bootstrapConfigFromEnvVariable() ([]byte, error) { + fName := envconfig.XDSBootstrapFileName + fContent := envconfig.XDSBootstrapFileContent + + // Bootstrap file name has higher priority than bootstrap content. + if fName != "" { + // If file name is set + // - If file not found (or other errors), fail + // - Otherwise, use the content. + // + // Note that even if the content is invalid, we don't failover to the + // file content env variable. + logger.Debugf("Using bootstrap file with name %q", fName) + return bootstrapFileReadFunc(fName) + } + + if fContent != "" { + return []byte(fContent), nil + } + + return nil, fmt.Errorf("none of the bootstrap environment variables (%q or %q) defined", + envconfig.XDSBootstrapFileNameEnv, envconfig.XDSBootstrapFileContentEnv) +} + +// NewConfig returns a new instance of Config initialized by reading the +// bootstrap file found at ${GRPC_XDS_BOOTSTRAP} or bootstrap contents specified +// at ${GRPC_XDS_BOOTSTRAP_CONFIG}. If both env vars are set, the former is +// preferred. +// +// We support a credential registration mechanism and only credentials +// registered through that mechanism will be accepted here. See package +// `xds/bootstrap` for details. +// +// This function tries to process as much of the bootstrap file as possible (in +// the presence of the errors) and may return a Config object with certain +// fields left unspecified, in which case the caller should use some sane +// defaults. +func NewConfig() (*Config, error) { + // Examples of the bootstrap json can be found in the generator tests + // https://github.com/GoogleCloudPlatform/traffic-director-grpc-bootstrap/blob/master/main_test.go. + data, err := bootstrapConfigFromEnvVariable() + if err != nil { + return nil, fmt.Errorf("xds: Failed to read bootstrap config: %v", err) + } + return newConfigFromContents(data) +} + +// NewConfigFromContentsForTesting returns a new Config using the specified +// bootstrap file contents instead of reading the environment variable. +// +// This is only suitable for testing purposes. +func NewConfigFromContentsForTesting(data []byte) (*Config, error) { + return newConfigFromContents(data) +} + +func newConfigFromContents(data []byte) (*Config, error) { + config := &Config{} + + var jsonData map[string]json.RawMessage + if err := json.Unmarshal(data, &jsonData); err != nil { + return nil, fmt.Errorf("xds: failed to parse bootstrap config: %v", err) + } + + var node *v3corepb.Node + m := jsonpb.Unmarshaler{AllowUnknownFields: true} + for k, v := range jsonData { + switch k { + case "node": + node = &v3corepb.Node{} + if err := m.Unmarshal(bytes.NewReader(v), node); err != nil { + return nil, fmt.Errorf("xds: jsonpb.Unmarshal(%v) for field %q failed during bootstrap: %v", string(v), k, err) + } + case "xds_servers": + servers, err := unmarshalJSONServerConfigSlice(v) + if err != nil { + return nil, fmt.Errorf("xds: json.Unmarshal(data) for field %q failed during bootstrap: %v", k, err) + } + config.XDSServer = servers[0] + case "certificate_providers": + var providerInstances map[string]json.RawMessage + if err := json.Unmarshal(v, &providerInstances); err != nil { + return nil, fmt.Errorf("xds: json.Unmarshal(%v) for field %q failed during bootstrap: %v", string(v), k, err) + } + configs := make(map[string]*certprovider.BuildableConfig) + getBuilder := internal.GetCertificateProviderBuilder.(func(string) certprovider.Builder) + for instance, data := range providerInstances { + var nameAndConfig struct { + PluginName string `json:"plugin_name"` + Config json.RawMessage `json:"config"` + } + if err := json.Unmarshal(data, &nameAndConfig); err != nil { + return nil, fmt.Errorf("xds: json.Unmarshal(%v) for field %q failed during bootstrap: %v", string(v), instance, err) + } + + name := nameAndConfig.PluginName + parser := getBuilder(nameAndConfig.PluginName) + if parser == nil { + // We ignore plugins that we do not know about. + continue + } + bc, err := parser.ParseConfig(nameAndConfig.Config) + if err != nil { + return nil, fmt.Errorf("xds: config parsing for plugin %q failed: %v", name, err) + } + configs[instance] = bc + } + config.CertProviderConfigs = configs + case "server_listener_resource_name_template": + if err := json.Unmarshal(v, &config.ServerListenerResourceNameTemplate); err != nil { + return nil, fmt.Errorf("xds: json.Unmarshal(%v) for field %q failed during bootstrap: %v", string(v), k, err) + } + case "client_default_listener_resource_name_template": + if !envconfig.XDSFederation { + logger.Warningf("Bootstrap field %v is not support when Federation is disabled", k) + continue + } + if err := json.Unmarshal(v, &config.ClientDefaultListenerResourceNameTemplate); err != nil { + return nil, fmt.Errorf("xds: json.Unmarshal(%v) for field %q failed during bootstrap: %v", string(v), k, err) + } + case "authorities": + if !envconfig.XDSFederation { + logger.Warningf("Bootstrap field %v is not support when Federation is disabled", k) + continue + } + if err := json.Unmarshal(v, &config.Authorities); err != nil { + return nil, fmt.Errorf("xds: json.Unmarshal(%v) for field %q failed during bootstrap: %v", string(v), k, err) + } + default: + logger.Warningf("Bootstrap content has unknown field: %s", k) + } + // Do not fail the xDS bootstrap when an unknown field is seen. This can + // happen when an older version client reads a newer version bootstrap + // file with new fields. + } + + if config.ClientDefaultListenerResourceNameTemplate == "" { + // Default value of the default client listener name template is "%s". + config.ClientDefaultListenerResourceNameTemplate = "%s" + } + if config.XDSServer == nil { + return nil, fmt.Errorf("xds: required field %q not found in bootstrap %s", "xds_servers", jsonData["xds_servers"]) + } + if config.XDSServer.ServerURI == "" { + return nil, fmt.Errorf("xds: required field %q not found in bootstrap %s", "xds_servers.server_uri", jsonData["xds_servers"]) + } + if config.XDSServer.CredsDialOption() == nil { + return nil, fmt.Errorf("xds: required field %q doesn't contain valid value in bootstrap %s", "xds_servers.channel_creds", jsonData["xds_servers"]) + } + // Post-process the authorities' client listener resource template field: + // - if set, it must start with "xdstp:///" + // - if not set, it defaults to "xdstp:///envoy.config.listener.v3.Listener/%s" + for name, authority := range config.Authorities { + prefix := fmt.Sprintf("xdstp://%s", url.PathEscape(name)) + if authority.ClientListenerResourceNameTemplate == "" { + authority.ClientListenerResourceNameTemplate = prefix + "/envoy.config.listener.v3.Listener/%s" + continue + } + if !strings.HasPrefix(authority.ClientListenerResourceNameTemplate, prefix) { + return nil, fmt.Errorf("xds: field ClientListenerResourceNameTemplate %q of authority %q doesn't start with prefix %q", authority.ClientListenerResourceNameTemplate, name, prefix) + } + } + + // Performing post-production on the node information. Some additional fields + // which are not expected to be set in the bootstrap file are populated here. + if node == nil { + node = &v3corepb.Node{} + } + node.UserAgentName = gRPCUserAgentName + node.UserAgentVersionType = &v3corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version} + node.ClientFeatures = append(node.ClientFeatures, clientFeatureNoOverprovisioning, clientFeatureResourceWrapper) + config.NodeProto = node + + logger.Debugf("Bootstrap config for creating xds-client: %v", pretty.ToJSON(config)) + return config, nil +} diff --git a/xds/internal/xdsclient/bootstrap/bootstrap_test.go b/xds/internal/xdsclient/bootstrap/bootstrap_test.go new file mode 100644 index 000000000000..d9eb786bbe11 --- /dev/null +++ b/xds/internal/xdsclient/bootstrap/bootstrap_test.go @@ -0,0 +1,1041 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package bootstrap + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "testing" + + "github.com/golang/protobuf/proto" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/tls/certprovider" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/xds/bootstrap" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + structpb "github.com/golang/protobuf/ptypes/struct" +) + +var ( + v3BootstrapFileMap = map[string]string{ + "serverFeaturesIncludesXDSV3": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ + { "type": "google_default" } + ], + "server_features" : ["xds_v3"] + }] + }`, + "serverFeaturesExcludesXDSV3": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ + { "type": "google_default" } + ] + }] + }`, + "emptyNodeProto": ` + { + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ + { "type": "insecure" } + ] + }] + }`, + "unknownTopLevelFieldInFile": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ + { "type": "insecure" } + ] + }], + "unknownField": "foobar" + }`, + "unknownFieldInNodeProto": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "unknownField": "foobar", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ + { "type": "insecure" } + ] + }] + }`, + "unknownFieldInXdsServer": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ + { "type": "insecure" } + ], + "unknownField": "foobar" + }] + }`, + "multipleChannelCreds": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ + { "type": "not-google-default" }, + { "type": "google_default" } + ], + "server_features": ["xds_v3"] + }] + }`, + "goodBootstrap": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ + { "type": "google_default" } + ], + "server_features": ["xds_v3"] + }] + }`, + "multipleXDSServers": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [ + { + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [{ "type": "google_default" }], + "server_features": ["xds_v3"] + }, + { + "server_uri": "backup.never.use.com:1234", + "channel_creds": [{ "type": "not-google-default" }] + } + ] + }`, + "serverSupportsIgnoreResourceDeletion": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ + { "type": "google_default" } + ], + "server_features" : ["ignore_resource_deletion", "xds_v3"] + }] + }`, + } + metadata = &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "TRAFFICDIRECTOR_GRPC_HOSTNAME": { + Kind: &structpb.Value_StringValue{StringValue: "trafficdirector"}, + }, + }, + } + v3NodeProto = &v3corepb.Node{ + Id: "ENVOY_NODE_ID", + Metadata: metadata, + UserAgentName: gRPCUserAgentName, + UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version}, + ClientFeatures: []string{clientFeatureNoOverprovisioning, clientFeatureResourceWrapper}, + } + nilCredsConfigNoServerFeatures = &Config{ + XDSServer: &ServerConfig{ + ServerURI: "trafficdirector.googleapis.com:443", + Creds: ChannelCreds{Type: "insecure"}, + }, + NodeProto: v3NodeProto, + ClientDefaultListenerResourceNameTemplate: "%s", + } + nonNilCredsConfigV3 = &Config{ + XDSServer: &ServerConfig{ + ServerURI: "trafficdirector.googleapis.com:443", + Creds: ChannelCreds{Type: "google_default"}, + ServerFeatures: []string{"xds_v3"}, + }, + NodeProto: v3NodeProto, + ClientDefaultListenerResourceNameTemplate: "%s", + } + nonNilCredsConfigWithDeletionIgnored = &Config{ + XDSServer: &ServerConfig{ + ServerURI: "trafficdirector.googleapis.com:443", + Creds: ChannelCreds{Type: "google_default"}, + IgnoreResourceDeletion: true, + ServerFeatures: []string{"ignore_resource_deletion", "xds_v3"}, + }, + NodeProto: v3NodeProto, + ClientDefaultListenerResourceNameTemplate: "%s", + } + nonNilCredsConfigNoServerFeatures = &Config{ + XDSServer: &ServerConfig{ + ServerURI: "trafficdirector.googleapis.com:443", + Creds: ChannelCreds{Type: "google_default"}, + }, + NodeProto: v3NodeProto, + ClientDefaultListenerResourceNameTemplate: "%s", + } +) + +func (c *Config) compare(want *Config) error { + if diff := cmp.Diff(want, c, + cmpopts.EquateEmpty(), + cmp.Comparer(proto.Equal), + cmp.Comparer(func(a, b grpc.DialOption) bool { return (a != nil) == (b != nil) }), + cmp.Transformer("certproviderconfigstring", func(a *certprovider.BuildableConfig) string { return a.String() }), + ); diff != "" { + return fmt.Errorf("unexpected diff in config (-want, +got):\n%s", diff) + } + return nil +} + +func fileReadFromFileMap(bootstrapFileMap map[string]string, name string) ([]byte, error) { + if b, ok := bootstrapFileMap[name]; ok { + return []byte(b), nil + } + return nil, os.ErrNotExist +} + +func setupBootstrapOverride(bootstrapFileMap map[string]string) func() { + oldFileReadFunc := bootstrapFileReadFunc + bootstrapFileReadFunc = func(filename string) ([]byte, error) { + return fileReadFromFileMap(bootstrapFileMap, filename) + } + return func() { bootstrapFileReadFunc = oldFileReadFunc } +} + +// TODO: enable leak check for this package when +// https://github.com/googleapis/google-cloud-go/issues/2417 is fixed. + +// This function overrides the bootstrap file NAME env variable, to test the +// code that reads file with the given fileName. +func testNewConfigWithFileNameEnv(t *testing.T, fileName string, wantError bool, wantConfig *Config) { + origBootstrapFileName := envconfig.XDSBootstrapFileName + envconfig.XDSBootstrapFileName = fileName + defer func() { envconfig.XDSBootstrapFileName = origBootstrapFileName }() + + c, err := NewConfig() + if (err != nil) != wantError { + t.Fatalf("NewConfig() returned error %v, wantError: %v", err, wantError) + } + if wantError { + return + } + if err := c.compare(wantConfig); err != nil { + t.Fatal(err) + } +} + +// This function overrides the bootstrap file CONTENT env variable, to test the +// code that uses the content from env directly. +func testNewConfigWithFileContentEnv(t *testing.T, fileName string, wantError bool, wantConfig *Config) { + t.Helper() + b, err := bootstrapFileReadFunc(fileName) + if err != nil { + t.Skip(err) + } + origBootstrapContent := envconfig.XDSBootstrapFileContent + envconfig.XDSBootstrapFileContent = string(b) + defer func() { envconfig.XDSBootstrapFileContent = origBootstrapContent }() + + c, err := NewConfig() + if (err != nil) != wantError { + t.Fatalf("NewConfig() returned error %v, wantError: %v", err, wantError) + } + if wantError { + return + } + if err := c.compare(wantConfig); err != nil { + t.Fatal(err) + } +} + +// TestNewConfigV3ProtoFailure exercises the functionality in NewConfig with +// different bootstrap file contents which are expected to fail. +func TestNewConfigV3ProtoFailure(t *testing.T) { + bootstrapFileMap := map[string]string{ + "empty": "", + "badJSON": `["test": 123]`, + "noBalancerName": `{"node": {"id": "ENVOY_NODE_ID"}}`, + "emptyXdsServer": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + } + }`, + "emptyChannelCreds": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443" + }] + }`, + "nonGoogleDefaultCreds": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ + { "type": "not-google-default" } + ] + }] + }`, + } + cancel := setupBootstrapOverride(bootstrapFileMap) + defer cancel() + + tests := []struct { + name string + wantError bool + }{ + {"nonExistentBootstrapFile", true}, + {"empty", true}, + {"badJSON", true}, + {"noBalancerName", true}, + {"emptyXdsServer", true}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + testNewConfigWithFileNameEnv(t, test.name, true, nil) + testNewConfigWithFileContentEnv(t, test.name, true, nil) + }) + } +} + +// TestNewConfigV3ProtoSuccess exercises the functionality in NewConfig with +// different bootstrap file contents. It overrides the fileReadFunc by returning +// bootstrap file contents defined in this test, instead of reading from a file. +func TestNewConfigV3ProtoSuccess(t *testing.T) { + cancel := setupBootstrapOverride(v3BootstrapFileMap) + defer cancel() + + tests := []struct { + name string + wantConfig *Config + }{ + { + "emptyNodeProto", &Config{ + XDSServer: &ServerConfig{ + ServerURI: "trafficdirector.googleapis.com:443", + Creds: ChannelCreds{Type: "insecure"}, + }, + NodeProto: &v3corepb.Node{ + UserAgentName: gRPCUserAgentName, + UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version}, + ClientFeatures: []string{clientFeatureNoOverprovisioning, clientFeatureResourceWrapper}, + }, + ClientDefaultListenerResourceNameTemplate: "%s", + }, + }, + {"unknownTopLevelFieldInFile", nilCredsConfigNoServerFeatures}, + {"unknownFieldInNodeProto", nilCredsConfigNoServerFeatures}, + {"unknownFieldInXdsServer", nilCredsConfigNoServerFeatures}, + {"multipleChannelCreds", nonNilCredsConfigV3}, + {"goodBootstrap", nonNilCredsConfigV3}, + {"multipleXDSServers", nonNilCredsConfigV3}, + {"serverSupportsIgnoreResourceDeletion", nonNilCredsConfigWithDeletionIgnored}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + testNewConfigWithFileNameEnv(t, test.name, false, test.wantConfig) + testNewConfigWithFileContentEnv(t, test.name, false, test.wantConfig) + }) + } +} + +// TestNewConfigBootstrapEnvPriority tests that the two env variables are read +// in correct priority. +// +// "GRPC_XDS_BOOTSTRAP" which specifies the file name containing the bootstrap +// configuration takes precedence over "GRPC_XDS_BOOTSTRAP_CONFIG", which +// directly specifies the bootstrap configuration in itself. +func TestNewConfigBootstrapEnvPriority(t *testing.T) { + oldFileReadFunc := bootstrapFileReadFunc + bootstrapFileReadFunc = func(filename string) ([]byte, error) { + return fileReadFromFileMap(v3BootstrapFileMap, filename) + } + defer func() { bootstrapFileReadFunc = oldFileReadFunc }() + + goodFileName1 := "serverFeaturesIncludesXDSV3" + goodConfig1 := nonNilCredsConfigV3 + + goodFileName2 := "serverFeaturesExcludesXDSV3" + goodFileContent2 := v3BootstrapFileMap[goodFileName2] + goodConfig2 := nonNilCredsConfigNoServerFeatures + + origBootstrapFileName := envconfig.XDSBootstrapFileName + envconfig.XDSBootstrapFileName = "" + defer func() { envconfig.XDSBootstrapFileName = origBootstrapFileName }() + + origBootstrapContent := envconfig.XDSBootstrapFileContent + envconfig.XDSBootstrapFileContent = "" + defer func() { envconfig.XDSBootstrapFileContent = origBootstrapContent }() + + // When both env variables are empty, NewConfig should fail. + if _, err := NewConfig(); err == nil { + t.Errorf("NewConfig() returned nil error, expected to fail") + } + + // When one of them is set, it should be used. + envconfig.XDSBootstrapFileName = goodFileName1 + envconfig.XDSBootstrapFileContent = "" + c, err := NewConfig() + if err != nil { + t.Errorf("NewConfig() failed: %v", err) + } + if err := c.compare(goodConfig1); err != nil { + t.Error(err) + } + + envconfig.XDSBootstrapFileName = "" + envconfig.XDSBootstrapFileContent = goodFileContent2 + c, err = NewConfig() + if err != nil { + t.Errorf("NewConfig() failed: %v", err) + } + if err := c.compare(goodConfig2); err != nil { + t.Error(err) + } + + // Set both, file name should be read. + envconfig.XDSBootstrapFileName = goodFileName1 + envconfig.XDSBootstrapFileContent = goodFileContent2 + c, err = NewConfig() + if err != nil { + t.Errorf("NewConfig() failed: %v", err) + } + if err := c.compare(goodConfig1); err != nil { + t.Error(err) + } +} + +func init() { + certprovider.Register(&fakeCertProviderBuilder{}) +} + +const fakeCertProviderName = "fake-certificate-provider" + +// fakeCertProviderBuilder builds new instances of fakeCertProvider and +// interprets the config provided to it as JSON with a single key and value. +type fakeCertProviderBuilder struct{} + +// ParseConfig expects input in JSON format containing a map from string to +// string, with a single entry and mapKey being "configKey". +func (b *fakeCertProviderBuilder) ParseConfig(cfg any) (*certprovider.BuildableConfig, error) { + config, ok := cfg.(json.RawMessage) + if !ok { + return nil, fmt.Errorf("fakeCertProviderBuilder received config of type %T, want []byte", config) + } + var cfgData map[string]string + if err := json.Unmarshal(config, &cfgData); err != nil { + return nil, fmt.Errorf("fakeCertProviderBuilder config parsing failed: %v", err) + } + if len(cfgData) != 1 || cfgData["configKey"] == "" { + return nil, errors.New("fakeCertProviderBuilder received invalid config") + } + fc := &fakeStableConfig{config: cfgData} + return certprovider.NewBuildableConfig(fakeCertProviderName, fc.canonical(), func(certprovider.BuildOptions) certprovider.Provider { + return &fakeCertProvider{} + }), nil +} + +func (b *fakeCertProviderBuilder) Name() string { + return fakeCertProviderName +} + +type fakeStableConfig struct { + config map[string]string +} + +func (c *fakeStableConfig) canonical() []byte { + var cfg string + for k, v := range c.config { + cfg = fmt.Sprintf("%s:%s", k, v) + } + return []byte(cfg) +} + +// fakeCertProvider is an empty implementation of the Provider interface. +type fakeCertProvider struct { + certprovider.Provider +} + +func TestNewConfigWithCertificateProviders(t *testing.T) { + bootstrapFileMap := map[string]string{ + "badJSONCertProviderConfig": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ + { "type": "google_default" } + ], + "server_features" : ["foo", "bar", "xds_v3"], + }], + "certificate_providers": "bad JSON" + }`, + "allUnknownCertProviders": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ + { "type": "google_default" } + ], + "server_features" : ["xds_v3"] + }], + "certificate_providers": { + "unknownProviderInstance1": { + "plugin_name": "foo", + "config": {"foo": "bar"} + }, + "unknownProviderInstance2": { + "plugin_name": "bar", + "config": {"foo": "bar"} + } + } + }`, + "badCertProviderConfig": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ + { "type": "google_default" } + ], + "server_features" : ["xds_v3"], + }], + "certificate_providers": { + "unknownProviderInstance": { + "plugin_name": "foo", + "config": {"foo": "bar"} + }, + "fakeProviderInstanceBad": { + "plugin_name": "fake-certificate-provider", + "config": {"configKey": 666} + } + } + }`, + "goodCertProviderConfig": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ + { "type": "insecure" } + ], + "server_features" : ["xds_v3"] + }], + "certificate_providers": { + "unknownProviderInstance": { + "plugin_name": "foo", + "config": {"foo": "bar"} + }, + "fakeProviderInstance": { + "plugin_name": "fake-certificate-provider", + "config": {"configKey": "configValue"} + } + } + }`, + } + + getBuilder := internal.GetCertificateProviderBuilder.(func(string) certprovider.Builder) + parser := getBuilder(fakeCertProviderName) + if parser == nil { + t.Fatalf("missing certprovider plugin %q", fakeCertProviderName) + } + wantCfg, err := parser.ParseConfig(json.RawMessage(`{"configKey": "configValue"}`)) + if err != nil { + t.Fatalf("config parsing for plugin %q failed: %v", fakeCertProviderName, err) + } + + cancel := setupBootstrapOverride(bootstrapFileMap) + defer cancel() + + // Cannot use xdstestutils.ServerConfigForAddress here, as it would lead to + // a cyclic dependency. + jsonCfg := `{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [{"type": "insecure"}], + "server_features": ["xds_v3"] + }` + serverCfg, err := ServerConfigFromJSON([]byte(jsonCfg)) + if err != nil { + t.Fatalf("Failed to create server config from JSON %s: %v", jsonCfg, err) + } + goodConfig := &Config{ + XDSServer: serverCfg, + NodeProto: v3NodeProto, + CertProviderConfigs: map[string]*certprovider.BuildableConfig{ + "fakeProviderInstance": wantCfg, + }, + ClientDefaultListenerResourceNameTemplate: "%s", + } + tests := []struct { + name string + wantConfig *Config + wantErr bool + }{ + { + name: "badJSONCertProviderConfig", + wantErr: true, + }, + { + + name: "badCertProviderConfig", + wantErr: true, + }, + { + + name: "allUnknownCertProviders", + wantConfig: nonNilCredsConfigV3, + }, + { + name: "goodCertProviderConfig", + wantConfig: goodConfig, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + testNewConfigWithFileNameEnv(t, test.name, test.wantErr, test.wantConfig) + testNewConfigWithFileContentEnv(t, test.name, test.wantErr, test.wantConfig) + }) + } +} + +func TestNewConfigWithServerListenerResourceNameTemplate(t *testing.T) { + cancel := setupBootstrapOverride(map[string]string{ + "badServerListenerResourceNameTemplate:": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ + { "type": "google_default" } + ] + }], + "server_listener_resource_name_template": 123456789 + }`, + "goodServerListenerResourceNameTemplate": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ + { "type": "google_default" } + ] + }], + "server_listener_resource_name_template": "grpc/server?xds.resource.listening_address=%s" + }`, + }) + defer cancel() + + tests := []struct { + name string + wantConfig *Config + wantErr bool + }{ + { + name: "badServerListenerResourceNameTemplate", + wantErr: true, + }, + { + name: "goodServerListenerResourceNameTemplate", + wantConfig: &Config{ + XDSServer: &ServerConfig{ + ServerURI: "trafficdirector.googleapis.com:443", + Creds: ChannelCreds{Type: "google_default"}, + }, + NodeProto: v3NodeProto, + ServerListenerResourceNameTemplate: "grpc/server?xds.resource.listening_address=%s", + ClientDefaultListenerResourceNameTemplate: "%s", + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + testNewConfigWithFileNameEnv(t, test.name, test.wantErr, test.wantConfig) + testNewConfigWithFileContentEnv(t, test.name, test.wantErr, test.wantConfig) + }) + } +} + +func TestNewConfigWithFederation(t *testing.T) { + cancel := setupBootstrapOverride(map[string]string{ + "badClientListenerResourceNameTemplate": ` + { + "node": { "id": "ENVOY_NODE_ID" }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443" + }], + "client_default_listener_resource_name_template": 123456789 + }`, + "badClientListenerResourceNameTemplatePerAuthority": ` + { + "node": { "id": "ENVOY_NODE_ID" }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ { "type": "google_default" } ] + }], + "authorities": { + "xds.td.com": { + "client_listener_resource_name_template": "some/template/%s", + "xds_servers": [{ + "server_uri": "td.com", + "channel_creds": [ { "type": "google_default" } ], + "server_features" : ["foo", "bar", "xds_v3"] + }] + } + } + }`, + "good": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ { "type": "google_default" } ] + }], + "server_listener_resource_name_template": "xdstp://xds.example.com/envoy.config.listener.v3.Listener/grpc/server?listening_address=%s", + "client_default_listener_resource_name_template": "xdstp://xds.example.com/envoy.config.listener.v3.Listener/%s", + "authorities": { + "xds.td.com": { + "client_listener_resource_name_template": "xdstp://xds.td.com/envoy.config.listener.v3.Listener/%s", + "xds_servers": [{ + "server_uri": "td.com", + "channel_creds": [ { "type": "google_default" } ], + "server_features" : ["xds_v3"] + }] + } + } + }`, + // If client_default_listener_resource_name_template is not set, it + // defaults to "%s". + "goodWithDefaultDefaultClientListenerTemplate": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ { "type": "google_default" } ] + }] + }`, + // If client_listener_resource_name_template in authority is not set, it + // defaults to + // "xdstp:///envoy.config.listener.v3.Listener/%s". + "goodWithDefaultClientListenerTemplatePerAuthority": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ { "type": "google_default" } ] + }], + "client_default_listener_resource_name_template": "xdstp://xds.example.com/envoy.config.listener.v3.Listener/%s", + "authorities": { + "xds.td.com": { }, + "#.com": { } + } + }`, + // It's OK for an authority to not have servers. The top-level server + // will be used. + "goodWithNoServerPerAuthority": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ { "type": "google_default" } ] + }], + "client_default_listener_resource_name_template": "xdstp://xds.example.com/envoy.config.listener.v3.Listener/%s", + "authorities": { + "xds.td.com": { + "client_listener_resource_name_template": "xdstp://xds.td.com/envoy.config.listener.v3.Listener/%s" + } + } + }`, + }) + defer cancel() + + tests := []struct { + name string + wantConfig *Config + wantErr bool + }{ + { + name: "badClientListenerResourceNameTemplate", + wantErr: true, + }, + { + name: "badClientListenerResourceNameTemplatePerAuthority", + wantErr: true, + }, + { + name: "good", + wantConfig: &Config{ + XDSServer: &ServerConfig{ + ServerURI: "trafficdirector.googleapis.com:443", + Creds: ChannelCreds{Type: "google_default"}, + }, + NodeProto: v3NodeProto, + ServerListenerResourceNameTemplate: "xdstp://xds.example.com/envoy.config.listener.v3.Listener/grpc/server?listening_address=%s", + ClientDefaultListenerResourceNameTemplate: "xdstp://xds.example.com/envoy.config.listener.v3.Listener/%s", + Authorities: map[string]*Authority{ + "xds.td.com": { + ClientListenerResourceNameTemplate: "xdstp://xds.td.com/envoy.config.listener.v3.Listener/%s", + XDSServer: &ServerConfig{ + ServerURI: "td.com", + Creds: ChannelCreds{Type: "google_default"}, + ServerFeatures: []string{"xds_v3"}, + }, + }, + }, + }, + }, + { + name: "goodWithDefaultDefaultClientListenerTemplate", + wantConfig: &Config{ + XDSServer: &ServerConfig{ + ServerURI: "trafficdirector.googleapis.com:443", + Creds: ChannelCreds{Type: "google_default"}, + }, + NodeProto: v3NodeProto, + ClientDefaultListenerResourceNameTemplate: "%s", + }, + }, + { + name: "goodWithDefaultClientListenerTemplatePerAuthority", + wantConfig: &Config{ + XDSServer: &ServerConfig{ + ServerURI: "trafficdirector.googleapis.com:443", + Creds: ChannelCreds{Type: "google_default"}, + }, + NodeProto: v3NodeProto, + ClientDefaultListenerResourceNameTemplate: "xdstp://xds.example.com/envoy.config.listener.v3.Listener/%s", + Authorities: map[string]*Authority{ + "xds.td.com": { + ClientListenerResourceNameTemplate: "xdstp://xds.td.com/envoy.config.listener.v3.Listener/%s", + }, + "#.com": { + ClientListenerResourceNameTemplate: "xdstp://%23.com/envoy.config.listener.v3.Listener/%s", + }, + }, + }, + }, + { + name: "goodWithNoServerPerAuthority", + wantConfig: &Config{ + XDSServer: &ServerConfig{ + ServerURI: "trafficdirector.googleapis.com:443", + Creds: ChannelCreds{Type: "google_default"}, + }, + NodeProto: v3NodeProto, + ClientDefaultListenerResourceNameTemplate: "xdstp://xds.example.com/envoy.config.listener.v3.Listener/%s", + Authorities: map[string]*Authority{ + "xds.td.com": { + ClientListenerResourceNameTemplate: "xdstp://xds.td.com/envoy.config.listener.v3.Listener/%s", + }, + }, + }, + }, + } + + oldFederationSupport := envconfig.XDSFederation + envconfig.XDSFederation = true + defer func() { envconfig.XDSFederation = oldFederationSupport }() + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + testNewConfigWithFileNameEnv(t, test.name, test.wantErr, test.wantConfig) + testNewConfigWithFileContentEnv(t, test.name, test.wantErr, test.wantConfig) + }) + } +} + +func TestServerConfigMarshalAndUnmarshal(t *testing.T) { + jsonCfg := `{ + "server_uri": "test-server", + "channel_creds": [{"type": "insecure"}], + "server_features": ["xds_v3"] + }` + origConfig, err := ServerConfigFromJSON([]byte(jsonCfg)) + if err != nil { + t.Fatalf("Failed to create server config from JSON %s: %v", jsonCfg, err) + } + bs, err := json.Marshal(origConfig) + if err != nil { + t.Fatalf("failed to marshal: %v", err) + } + + unmarshaledConfig := new(ServerConfig) + if err := json.Unmarshal(bs, unmarshaledConfig); err != nil { + t.Fatalf("failed to unmarshal: %v", err) + } + if diff := cmp.Diff(origConfig, unmarshaledConfig); diff != "" { + t.Fatalf("Unexpected diff in server config (-want, +got):\n%s", diff) + } +} + +func TestDefaultBundles(t *testing.T) { + if c := bootstrap.GetCredentials("google_default"); c == nil { + t.Errorf(`bootstrap.GetCredentials("google_default") credential is nil, want non-nil`) + } + + if c := bootstrap.GetCredentials("insecure"); c == nil { + t.Errorf(`bootstrap.GetCredentials("insecure") credential is nil, want non-nil`) + } +} + +func TestCredsBuilders(t *testing.T) { + b := &googleDefaultCredsBuilder{} + if _, err := b.Build(nil); err != nil { + t.Errorf("googleDefaultCredsBuilder.Build failed: %v", err) + } + if got, want := b.Name(), "google_default"; got != want { + t.Errorf("googleDefaultCredsBuilder.Name = %v, want %v", got, want) + } + + i := &insecureCredsBuilder{} + if _, err := i.Build(nil); err != nil { + t.Errorf("insecureCredsBuilder.Build failed: %v", err) + } + + if got, want := i.Name(), "insecure"; got != want { + t.Errorf("insecureCredsBuilder.Name = %v, want %v", got, want) + } +} diff --git a/xds/internal/client/bootstrap/logging.go b/xds/internal/xdsclient/bootstrap/logging.go similarity index 100% rename from xds/internal/client/bootstrap/logging.go rename to xds/internal/xdsclient/bootstrap/logging.go diff --git a/xds/internal/xdsclient/bootstrap/template.go b/xds/internal/xdsclient/bootstrap/template.go new file mode 100644 index 000000000000..9b51fcc83972 --- /dev/null +++ b/xds/internal/xdsclient/bootstrap/template.go @@ -0,0 +1,47 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package bootstrap + +import ( + "net/url" + "strings" +) + +// PopulateResourceTemplate populates the given template using the target +// string. "%s", if exists in the template, will be replaced with target. +// +// If the template starts with "xdstp:", the replaced string will be %-encoded. +// But note that "/" is not percent encoded. +func PopulateResourceTemplate(template, target string) string { + if !strings.Contains(template, "%s") { + return template + } + if strings.HasPrefix(template, "xdstp:") { + target = percentEncode(target) + } + return strings.Replace(template, "%s", target, -1) +} + +// percentEncode percent encode t, except for "/". See the tests for examples. +func percentEncode(t string) string { + segs := strings.Split(t, "/") + for i := range segs { + segs[i] = url.PathEscape(segs[i]) + } + return strings.Join(segs, "/") +} diff --git a/xds/internal/xdsclient/bootstrap/template_test.go b/xds/internal/xdsclient/bootstrap/template_test.go new file mode 100644 index 000000000000..bc12eb42991f --- /dev/null +++ b/xds/internal/xdsclient/bootstrap/template_test.go @@ -0,0 +1,97 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package bootstrap + +import "testing" + +func Test_percentEncode(t *testing.T) { + tests := []struct { + name string + target string + want string + }{ + { + name: "normal name", + target: "server.example.com", + want: "server.example.com", + }, + { + name: "ipv4", + target: "0.0.0.0:8080", + want: "0.0.0.0:8080", + }, + { + name: "ipv6", + target: "[::1]:8080", + want: "%5B::1%5D:8080", // [ and ] are percent encoded. + }, + { + name: "/ should not be percent encoded", + target: "my/service/region", + want: "my/service/region", // "/"s are kept. + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := percentEncode(tt.target); got != tt.want { + t.Errorf("percentEncode() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestPopulateResourceTemplate(t *testing.T) { + tests := []struct { + name string + template string + target string + want string + }{ + { + name: "no %s", + template: "/name/template", + target: "[::1]:8080", + want: "/name/template", + }, + { + name: "with %s, no xdstp: prefix, ipv6", + template: "/name/template/%s", + target: "[::1]:8080", + want: "/name/template/[::1]:8080", + }, + { + name: "with %s, with xdstp: prefix", + template: "xdstp://authority.com/%s", + target: "0.0.0.0:8080", + want: "xdstp://authority.com/0.0.0.0:8080", + }, + { + name: "with %s, with xdstp: prefix, and ipv6", + template: "xdstp://authority.com/%s", + target: "[::1]:8080", + want: "xdstp://authority.com/%5B::1%5D:8080", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := PopulateResourceTemplate(tt.template, tt.target); got != tt.want { + t.Errorf("PopulateResourceTemplate() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/xds/internal/xdsclient/client.go b/xds/internal/xdsclient/client.go new file mode 100644 index 000000000000..44f6d3bc0a1c --- /dev/null +++ b/xds/internal/xdsclient/client.go @@ -0,0 +1,63 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package xdsclient implements a full fledged gRPC client for the xDS API used +// by the xds resolver and balancer implementations. +package xdsclient + +import ( + "google.golang.org/grpc/xds/internal/xdsclient/bootstrap" + "google.golang.org/grpc/xds/internal/xdsclient/load" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" +) + +// XDSClient is a full fledged gRPC client which queries a set of discovery APIs +// (collectively termed as xDS) on a remote management server, to discover +// various dynamic resources. +type XDSClient interface { + WatchListener(string, func(xdsresource.ListenerUpdate, error)) func() + WatchRouteConfig(string, func(xdsresource.RouteConfigUpdate, error)) func() + WatchCluster(string, func(xdsresource.ClusterUpdate, error)) func() + + // WatchResource uses xDS to discover the resource associated with the + // provided resource name. The resource type implementation determines how + // xDS requests are sent out and how responses are deserialized and + // validated. Upon receipt of a response from the management server, an + // appropriate callback on the watcher is invoked. + // + // Most callers will not have a need to use this API directly. They will + // instead use a resource-type-specific wrapper API provided by the relevant + // resource type implementation. + // + // + // During a race (e.g. an xDS response is received while the user is calling + // cancel()), there's a small window where the callback can be called after + // the watcher is canceled. Callers need to handle this case. + // + // TODO: Once this generic client API is fully implemented and integrated, + // delete the resource type specific watch APIs on this interface. + WatchResource(rType xdsresource.Type, resourceName string, watcher xdsresource.ResourceWatcher) (cancel func()) + + // DumpResources returns the status of the xDS resources. Returns a map of + // resource type URLs to a map of resource names to resource state. + DumpResources() map[string]map[string]xdsresource.UpdateWithMD + + ReportLoad(*bootstrap.ServerConfig) (*load.Store, func()) + + BootstrapConfig() *bootstrap.Config +} diff --git a/xds/internal/xdsclient/client_new.go b/xds/internal/xdsclient/client_new.go new file mode 100644 index 000000000000..b330c19dfd17 --- /dev/null +++ b/xds/internal/xdsclient/client_new.go @@ -0,0 +1,166 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "sync" + "time" + + "google.golang.org/grpc/internal/cache" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/xds/internal/xdsclient/bootstrap" +) + +// New returns a new xDS client configured by the bootstrap file specified in env +// variable GRPC_XDS_BOOTSTRAP or GRPC_XDS_BOOTSTRAP_CONFIG. +// +// The returned client is a reference counted singleton instance. This function +// creates a new client only when one doesn't already exist. +// +// The second return value represents a close function which releases the +// caller's reference on the returned client. The caller is expected to invoke +// it once they are done using the client. The underlying client will be closed +// only when all references are released, and it is safe for the caller to +// invoke this close function multiple times. +func New() (XDSClient, func(), error) { + return newRefCountedWithConfig(nil) +} + +// NewWithConfig returns a new xDS client configured by the given config. +// +// The second return value represents a close function which releases the +// caller's reference on the returned client. The caller is expected to invoke +// it once they are done using the client. The underlying client will be closed +// only when all references are released, and it is safe for the caller to +// invoke this close function multiple times. +// +// # Internal/Testing Only +// +// This function should ONLY be used for internal (c2p resolver) and/or testing +// purposese. DO NOT use this elsewhere. Use New() instead. +func NewWithConfig(config *bootstrap.Config) (XDSClient, func(), error) { + return newRefCountedWithConfig(config) +} + +// newWithConfig returns a new xdsClient with the given config. +func newWithConfig(config *bootstrap.Config, watchExpiryTimeout time.Duration, idleAuthorityDeleteTimeout time.Duration) (*clientImpl, error) { + ctx, cancel := context.WithCancel(context.Background()) + c := &clientImpl{ + done: grpcsync.NewEvent(), + config: config, + watchExpiryTimeout: watchExpiryTimeout, + serializer: grpcsync.NewCallbackSerializer(ctx), + serializerClose: cancel, + resourceTypes: newResourceTypeRegistry(), + authorities: make(map[string]*authority), + idleAuthorities: cache.NewTimeoutCache(idleAuthorityDeleteTimeout), + } + + c.logger = prefixLogger(c) + c.logger.Infof("Created client to xDS management server: %s", config.XDSServer) + return c, nil +} + +// NewWithConfigForTesting returns an xDS client for the specified bootstrap +// config, separate from the global singleton. +// +// The second return value represents a close function which the caller is +// expected to invoke once they are done using the client. It is safe for the +// caller to invoke this close function multiple times. +// +// # Testing Only +// +// This function should ONLY be used for testing purposes. +// TODO(easwars): Document the new close func. +func NewWithConfigForTesting(config *bootstrap.Config, watchExpiryTimeout, authorityIdleTimeout time.Duration) (XDSClient, func(), error) { + cl, err := newWithConfig(config, watchExpiryTimeout, authorityIdleTimeout) + if err != nil { + return nil, nil, err + } + return cl, grpcsync.OnceFunc(cl.close), nil +} + +// NewWithBootstrapContentsForTesting returns an xDS client for this config, +// separate from the global singleton. +// +// The second return value represents a close function which the caller is +// expected to invoke once they are done using the client. It is safe for the +// caller to invoke this close function multiple times. +// +// # Testing Only +// +// This function should ONLY be used for testing purposes. +func NewWithBootstrapContentsForTesting(contents []byte) (XDSClient, func(), error) { + // Normalize the contents + buf := bytes.Buffer{} + err := json.Indent(&buf, contents, "", "") + if err != nil { + return nil, nil, fmt.Errorf("xds: error normalizing JSON: %v", err) + } + contents = bytes.TrimSpace(buf.Bytes()) + + c, err := getOrMakeClientForTesting(contents) + if err != nil { + return nil, nil, err + } + return c, grpcsync.OnceFunc(func() { + clientsMu.Lock() + defer clientsMu.Unlock() + if c.decrRef() == 0 { + c.close() + delete(clients, string(contents)) + } + }), nil +} + +// getOrMakeClientForTesting creates a new reference counted client (separate +// from the global singleton) for the given config, or returns an existing one. +// It takes care of incrementing the reference count for the returned client, +// and leaves the caller responsible for decrementing the reference count once +// the client is no longer needed. +func getOrMakeClientForTesting(config []byte) (*clientRefCounted, error) { + clientsMu.Lock() + defer clientsMu.Unlock() + + if c := clients[string(config)]; c != nil { + c.incrRef() + return c, nil + } + + bcfg, err := bootstrap.NewConfigFromContentsForTesting(config) + if err != nil { + return nil, fmt.Errorf("bootstrap config %s: %v", string(config), err) + } + cImpl, err := newWithConfig(bcfg, defaultWatchExpiryTimeout, defaultIdleAuthorityDeleteTimeout) + if err != nil { + return nil, fmt.Errorf("creating xDS client: %v", err) + } + c := &clientRefCounted{clientImpl: cImpl, refCount: 1} + clients[string(config)] = c + return c, nil +} + +var ( + clients = map[string]*clientRefCounted{} + clientsMu sync.Mutex +) diff --git a/credentials/go12.go b/xds/internal/xdsclient/client_test.go similarity index 60% rename from credentials/go12.go rename to xds/internal/xdsclient/client_test.go index ccbf35b33125..f32688850e4f 100644 --- a/credentials/go12.go +++ b/xds/internal/xdsclient/client_test.go @@ -1,5 +1,3 @@ -// +build go1.12 - /* * * Copyright 2019 gRPC authors. @@ -18,13 +16,25 @@ * */ -package credentials +package xdsclient + +import ( + "testing" + "time" -import "crypto/tls" + "google.golang.org/grpc/internal/grpctest" +) -// This init function adds cipher suite constants only defined in Go 1.12. -func init() { - cipherSuiteLookup[tls.TLS_AES_128_GCM_SHA256] = "TLS_AES_128_GCM_SHA256" - cipherSuiteLookup[tls.TLS_AES_256_GCM_SHA384] = "TLS_AES_256_GCM_SHA384" - cipherSuiteLookup[tls.TLS_CHACHA20_POLY1305_SHA256] = "TLS_CHACHA20_POLY1305_SHA256" +type s struct { + grpctest.Tester } + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +const ( + defaultTestWatchExpiryTimeout = 100 * time.Millisecond + defaultTestTimeout = 5 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond // For events expected to *not* happen. +) diff --git a/xds/internal/xdsclient/clientimpl.go b/xds/internal/xdsclient/clientimpl.go new file mode 100644 index 000000000000..2c05ea66f5f9 --- /dev/null +++ b/xds/internal/xdsclient/clientimpl.go @@ -0,0 +1,89 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient + +import ( + "sync" + "time" + + "google.golang.org/grpc/internal/cache" + "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/xds/internal/xdsclient/bootstrap" +) + +var _ XDSClient = &clientImpl{} + +// clientImpl is the real implementation of the xds client. The exported Client +// is a wrapper of this struct with a ref count. +type clientImpl struct { + done *grpcsync.Event + config *bootstrap.Config + logger *grpclog.PrefixLogger + watchExpiryTimeout time.Duration + serializer *grpcsync.CallbackSerializer + serializerClose func() + resourceTypes *resourceTypeRegistry + + // authorityMu protects the authority fields. It's necessary because an + // authority is created when it's used. + authorityMu sync.Mutex + // authorities is a map from ServerConfig to authority. So that + // different authorities sharing the same ServerConfig can share the + // authority. + // + // The key is **ServerConfig.String()**, not the authority name. + // + // An authority is either in authorities, or idleAuthorities, + // never both. + authorities map[string]*authority + // idleAuthorities keeps the authorities that are not used (the last + // watch on it was canceled). They are kept in the cache and will be deleted + // after a timeout. The key is ServerConfig.String(). + // + // An authority is either in authorities, or idleAuthorities, + // never both. + idleAuthorities *cache.TimeoutCache +} + +// BootstrapConfig returns the configuration read from the bootstrap file. +// Callers must treat the return value as read-only. +func (c *clientImpl) BootstrapConfig() *bootstrap.Config { + return c.config +} + +// close closes the gRPC connection to the management server. +func (c *clientImpl) close() { + if c.done.HasFired() { + return + } + c.done.Fire() + // TODO: Should we invoke the registered callbacks here with an error that + // the client is closed? + + c.authorityMu.Lock() + for _, a := range c.authorities { + a.close() + } + c.idleAuthorities.Clear(true) + c.authorityMu.Unlock() + c.serializerClose() + + c.logger.Infof("Shutdown") +} diff --git a/xds/internal/xdsclient/clientimpl_authority.go b/xds/internal/xdsclient/clientimpl_authority.go new file mode 100644 index 000000000000..925566cf44f3 --- /dev/null +++ b/xds/internal/xdsclient/clientimpl_authority.go @@ -0,0 +1,141 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsclient + +import ( + "errors" + "fmt" + + "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/xds/internal/xdsclient/bootstrap" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" +) + +// findAuthority returns the authority for this name. If it doesn't already +// exist, one will be created. +// +// Note that this doesn't always create new authority. authorities with the same +// config but different names are shared. +// +// The returned unref function must be called when the caller is done using this +// authority, without holding c.authorityMu. +// +// Caller must not hold c.authorityMu. +func (c *clientImpl) findAuthority(n *xdsresource.Name) (_ *authority, unref func(), _ error) { + scheme, authority := n.Scheme, n.Authority + + c.authorityMu.Lock() + defer c.authorityMu.Unlock() + if c.done.HasFired() { + return nil, nil, errors.New("the xds-client is closed") + } + + config := c.config.XDSServer + if scheme == xdsresource.FederationScheme { + cfg, ok := c.config.Authorities[authority] + if !ok { + return nil, nil, fmt.Errorf("xds: failed to find authority %q", authority) + } + if cfg.XDSServer != nil { + config = cfg.XDSServer + } + } + + a, err := c.newAuthorityLocked(config) + if err != nil { + return nil, nil, fmt.Errorf("xds: failed to connect to the control plane for authority %q: %v", authority, err) + } + // All returned authority from this function will be used by a watch, + // holding the ref here. + // + // Note that this must be done while c.authorityMu is held, to avoid the + // race that an authority is returned, but before the watch starts, the + // old last watch is canceled (in another goroutine), causing this + // authority to be removed, and then a watch will start on a removed + // authority. + // + // unref() will be done when the watch is canceled. + a.refLocked() + return a, func() { c.unrefAuthority(a) }, nil +} + +// newAuthorityLocked creates a new authority for the given config. If an +// authority for the given config exists in the cache, it is returned instead of +// creating a new one. +// +// The caller must take a reference of the returned authority before using, and +// unref afterwards. +// +// caller must hold c.authorityMu +func (c *clientImpl) newAuthorityLocked(config *bootstrap.ServerConfig) (_ *authority, retErr error) { + // First check if there's already an authority for this config. If found, it + // means this authority is used by other watches (could be the same + // authority name, or a different authority name but the same server + // config). Return it. + configStr := config.String() + if a, ok := c.authorities[configStr]; ok { + return a, nil + } + // Second check if there's an authority in the idle cache. If found, it + // means this authority was created, but moved to the idle cache because the + // watch was canceled. Move it from idle cache to the authority cache, and + // return. + if old, ok := c.idleAuthorities.Remove(configStr); ok { + oldA, _ := old.(*authority) + if oldA != nil { + c.authorities[configStr] = oldA + return oldA, nil + } + } + + // Make a new authority since there's no existing authority for this config. + ret, err := newAuthority(authorityArgs{ + serverCfg: config, + bootstrapCfg: c.config, + serializer: c.serializer, + resourceTypeGetter: c.resourceTypes.get, + watchExpiryTimeout: c.watchExpiryTimeout, + logger: grpclog.NewPrefixLogger(logger, authorityPrefix(c, config.ServerURI)), + }) + if err != nil { + return nil, fmt.Errorf("creating new authority for config %q: %v", config.String(), err) + } + // Add it to the cache, so it will be reused. + c.authorities[configStr] = ret + return ret, nil +} + +// unrefAuthority unrefs the authority. It also moves the authority to idle +// cache if it's ref count is 0. +// +// This function doesn't need to called explicitly. It's called by the returned +// unref from findAuthority(). +// +// Caller must not hold c.authorityMu. +func (c *clientImpl) unrefAuthority(a *authority) { + c.authorityMu.Lock() + defer c.authorityMu.Unlock() + if a.unrefLocked() > 0 { + return + } + configStr := a.serverCfg.String() + delete(c.authorities, configStr) + c.idleAuthorities.Add(configStr, a, func() { + a.close() + }) +} diff --git a/xds/internal/xdsclient/clientimpl_dump.go b/xds/internal/xdsclient/clientimpl_dump.go new file mode 100644 index 000000000000..b9d0499301a2 --- /dev/null +++ b/xds/internal/xdsclient/clientimpl_dump.go @@ -0,0 +1,53 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient + +import ( + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" +) + +func appendMaps(dst, src map[string]map[string]xdsresource.UpdateWithMD) { + // Iterate through the resource types. + for rType, srcResources := range src { + // Lookup/create the resource type specific map in the destination. + dstResources := dst[rType] + if dstResources == nil { + dstResources = make(map[string]xdsresource.UpdateWithMD) + dst[rType] = dstResources + } + + // Iterate through the resources within the resource type in the source, + // and copy them over to the destination. + for name, update := range srcResources { + dstResources[name] = update + } + } +} + +// DumpResources returns the status and contents of all xDS resources. +func (c *clientImpl) DumpResources() map[string]map[string]xdsresource.UpdateWithMD { + c.authorityMu.Lock() + defer c.authorityMu.Unlock() + dumps := make(map[string]map[string]xdsresource.UpdateWithMD) + for _, a := range c.authorities { + dump := a.dumpResources() + appendMaps(dumps, dump) + } + return dumps +} diff --git a/xds/internal/xdsclient/clientimpl_loadreport.go b/xds/internal/xdsclient/clientimpl_loadreport.go new file mode 100644 index 000000000000..e53df3f1edd9 --- /dev/null +++ b/xds/internal/xdsclient/clientimpl_loadreport.go @@ -0,0 +1,47 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsclient + +import ( + "google.golang.org/grpc/xds/internal/xdsclient/bootstrap" + "google.golang.org/grpc/xds/internal/xdsclient/load" +) + +// ReportLoad starts a load reporting stream to the given server. All load +// reports to the same server share the LRS stream. +// +// It returns a Store for the user to report loads, a function to cancel the +// load reporting stream. +func (c *clientImpl) ReportLoad(server *bootstrap.ServerConfig) (*load.Store, func()) { + c.authorityMu.Lock() + a, err := c.newAuthorityLocked(server) + if err != nil { + c.authorityMu.Unlock() + c.logger.Infof("xds: failed to connect to the control plane to do load reporting for authority %q: %v", server, err) + return nil, func() {} + } + // Hold the ref before starting load reporting. + a.refLocked() + c.authorityMu.Unlock() + + store, cancelF := a.reportLoad() + return store, func() { + cancelF() + c.unrefAuthority(a) + } +} diff --git a/xds/internal/xdsclient/clientimpl_watchers.go b/xds/internal/xdsclient/clientimpl_watchers.go new file mode 100644 index 000000000000..e503349dbc29 --- /dev/null +++ b/xds/internal/xdsclient/clientimpl_watchers.go @@ -0,0 +1,192 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsclient + +import ( + "context" + "fmt" + "sync" + + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" +) + +// This is only required temporarily, while we modify the +// clientImpl.WatchListener API to be implemented via the wrapper +// WatchListener() API which calls the WatchResource() API. +type listenerWatcher struct { + resourceName string + cb func(xdsresource.ListenerUpdate, error) +} + +func (l *listenerWatcher) OnUpdate(update *xdsresource.ListenerResourceData) { + l.cb(update.Resource, nil) +} + +func (l *listenerWatcher) OnError(err error) { + l.cb(xdsresource.ListenerUpdate{}, err) +} + +func (l *listenerWatcher) OnResourceDoesNotExist() { + err := xdsresource.NewErrorf(xdsresource.ErrorTypeResourceNotFound, "resource name %q of type Listener not found in received response", l.resourceName) + l.cb(xdsresource.ListenerUpdate{}, err) +} + +// WatchListener uses LDS to discover information about the Listener resource +// identified by resourceName. +func (c *clientImpl) WatchListener(resourceName string, cb func(xdsresource.ListenerUpdate, error)) (cancel func()) { + watcher := &listenerWatcher{resourceName: resourceName, cb: cb} + return xdsresource.WatchListener(c, resourceName, watcher) +} + +// This is only required temporarily, while we modify the +// clientImpl.WatchRouteConfig API to be implemented via the wrapper +// WatchRouteConfig() API which calls the WatchResource() API. +type routeConfigWatcher struct { + resourceName string + cb func(xdsresource.RouteConfigUpdate, error) +} + +func (r *routeConfigWatcher) OnUpdate(update *xdsresource.RouteConfigResourceData) { + r.cb(update.Resource, nil) +} + +func (r *routeConfigWatcher) OnError(err error) { + r.cb(xdsresource.RouteConfigUpdate{}, err) +} + +func (r *routeConfigWatcher) OnResourceDoesNotExist() { + err := xdsresource.NewErrorf(xdsresource.ErrorTypeResourceNotFound, "resource name %q of type RouteConfiguration not found in received response", r.resourceName) + r.cb(xdsresource.RouteConfigUpdate{}, err) +} + +// WatchRouteConfig uses RDS to discover information about the +// RouteConfiguration resource identified by resourceName. +func (c *clientImpl) WatchRouteConfig(resourceName string, cb func(xdsresource.RouteConfigUpdate, error)) (cancel func()) { + watcher := &routeConfigWatcher{resourceName: resourceName, cb: cb} + return xdsresource.WatchRouteConfig(c, resourceName, watcher) +} + +// This is only required temporarily, while we modify the +// clientImpl.WatchCluster API to be implemented via the wrapper WatchCluster() +// API which calls the WatchResource() API. +type clusterWatcher struct { + resourceName string + cb func(xdsresource.ClusterUpdate, error) +} + +func (c *clusterWatcher) OnUpdate(update *xdsresource.ClusterResourceData) { + c.cb(update.Resource, nil) +} + +func (c *clusterWatcher) OnError(err error) { + c.cb(xdsresource.ClusterUpdate{}, err) +} + +func (c *clusterWatcher) OnResourceDoesNotExist() { + err := xdsresource.NewErrorf(xdsresource.ErrorTypeResourceNotFound, "resource name %q of type Cluster not found in received response", c.resourceName) + c.cb(xdsresource.ClusterUpdate{}, err) +} + +// WatchCluster uses CDS to discover information about the Cluster resource +// identified by resourceName. +// +// WatchCluster can be called multiple times, with same or different +// clusterNames. Each call will start an independent watcher for the resource. +func (c *clientImpl) WatchCluster(resourceName string, cb func(xdsresource.ClusterUpdate, error)) (cancel func()) { + watcher := &clusterWatcher{resourceName: resourceName, cb: cb} + return xdsresource.WatchCluster(c, resourceName, watcher) +} + +// WatchResource uses xDS to discover the resource associated with the provided +// resource name. The resource type implementation determines how xDS requests +// are sent out and how responses are deserialized and validated. Upon receipt +// of a response from the management server, an appropriate callback on the +// watcher is invoked. +func (c *clientImpl) WatchResource(rType xdsresource.Type, resourceName string, watcher xdsresource.ResourceWatcher) (cancel func()) { + // Return early if the client is already closed. + // + // The client returned from the top-level API is a ref-counted client which + // contains a pointer to `clientImpl`. When all references are released, the + // ref-counted client sets its pointer to `nil`. And if any watch APIs are + // made on such a closed client, we will get here with a `nil` receiver. + if c == nil || c.done.HasFired() { + logger.Warningf("Watch registered for name %q of type %q, but client is closed", rType.TypeName(), resourceName) + return func() {} + } + + if err := c.resourceTypes.maybeRegister(rType); err != nil { + logger.Warningf("Watch registered for name %q of type %q which is already registered", rType.TypeName(), resourceName) + c.serializer.Schedule(func(context.Context) { watcher.OnError(err) }) + return func() {} + } + + // TODO: replace this with the code does the following when we have + // implemented generic watch API on the authority: + // - Parse the resource name and extract the authority. + // - Locate the corresponding authority object and acquire a reference to + // it. If the authority is not found, error out. + // - Call the watchResource() method on the authority. + // - Return a cancel function to cancel the watch on the authority and to + // release the reference. + + // TODO: Make ParseName return an error if parsing fails, and + // schedule the OnError callback in that case. + n := xdsresource.ParseName(resourceName) + a, unref, err := c.findAuthority(n) + if err != nil { + logger.Warningf("Watch registered for name %q of type %q, authority %q is not found", rType.TypeName(), resourceName, n.Authority) + c.serializer.Schedule(func(context.Context) { watcher.OnError(err) }) + return func() {} + } + cancelF := a.watchResource(rType, n.String(), watcher) + return func() { + cancelF() + unref() + } +} + +// A registry of xdsresource.Type implementations indexed by their corresponding +// type URLs. Registration of an xdsresource.Type happens the first time a watch +// for a resource of that type is invoked. +type resourceTypeRegistry struct { + mu sync.Mutex + types map[string]xdsresource.Type +} + +func newResourceTypeRegistry() *resourceTypeRegistry { + return &resourceTypeRegistry{types: make(map[string]xdsresource.Type)} +} + +func (r *resourceTypeRegistry) get(url string) xdsresource.Type { + r.mu.Lock() + defer r.mu.Unlock() + return r.types[url] +} + +func (r *resourceTypeRegistry) maybeRegister(rType xdsresource.Type) error { + r.mu.Lock() + defer r.mu.Unlock() + + url := rType.TypeURL() + typ, ok := r.types[url] + if ok && typ != rType { + return fmt.Errorf("attempt to re-register a resource type implementation for %v", rType.TypeName()) + } + r.types[url] = rType + return nil +} diff --git a/xds/internal/xdsclient/load/reporter.go b/xds/internal/xdsclient/load/reporter.go new file mode 100644 index 000000000000..67e29e5bae13 --- /dev/null +++ b/xds/internal/xdsclient/load/reporter.go @@ -0,0 +1,27 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package load + +// PerClusterReporter wraps the methods from the loadStore that are used here. +type PerClusterReporter interface { + CallStarted(locality string) + CallFinished(locality string, err error) + CallServerLoad(locality, name string, val float64) + CallDropped(category string) +} diff --git a/xds/internal/xdsclient/load/store.go b/xds/internal/xdsclient/load/store.go new file mode 100644 index 000000000000..1f266ae20185 --- /dev/null +++ b/xds/internal/xdsclient/load/store.go @@ -0,0 +1,426 @@ +/* + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package load provides functionality to record and maintain load data. +package load + +import ( + "sync" + "sync/atomic" + "time" +) + +const negativeOneUInt64 = ^uint64(0) + +// Store keeps the loads for multiple clusters and services to be reported via +// LRS. It contains loads to reported to one LRS server. Create multiple stores +// for multiple servers. +// +// It is safe for concurrent use. +type Store struct { + // mu only protects the map (2 layers). The read/write to *perClusterStore + // doesn't need to hold the mu. + mu sync.Mutex + // clusters is a map with cluster name as the key. The second layer is a map + // with service name as the key. Each value (perClusterStore) contains data + // for a (cluster, service) pair. + // + // Note that new entries are added to this map, but never removed. This is + // potentially a memory leak. But the memory is allocated for each new + // (cluster,service) pair, and the memory allocated is just pointers and + // maps. So this shouldn't get too bad. + clusters map[string]map[string]*perClusterStore +} + +// NewStore creates a Store. +func NewStore() *Store { + return &Store{ + clusters: make(map[string]map[string]*perClusterStore), + } +} + +// Stats returns the load data for the given cluster names. Data is returned in +// a slice with no specific order. +// +// If no clusterName is given (an empty slice), all data for all known clusters +// is returned. +// +// If a cluster's Data is empty (no load to report), it's not appended to the +// returned slice. +func (s *Store) Stats(clusterNames []string) []*Data { + var ret []*Data + s.mu.Lock() + defer s.mu.Unlock() + + if len(clusterNames) == 0 { + for _, c := range s.clusters { + ret = appendClusterStats(ret, c) + } + return ret + } + + for _, n := range clusterNames { + if c, ok := s.clusters[n]; ok { + ret = appendClusterStats(ret, c) + } + } + return ret +} + +// appendClusterStats gets Data for the given cluster, append to ret, and return +// the new slice. +// +// Data is only appended to ret if it's not empty. +func appendClusterStats(ret []*Data, cluster map[string]*perClusterStore) []*Data { + for _, d := range cluster { + data := d.stats() + if data == nil { + // Skip this data if it doesn't contain any information. + continue + } + ret = append(ret, data) + } + return ret +} + +// PerCluster returns the perClusterStore for the given clusterName + +// serviceName. +func (s *Store) PerCluster(clusterName, serviceName string) PerClusterReporter { + if s == nil { + return nil + } + + s.mu.Lock() + defer s.mu.Unlock() + c, ok := s.clusters[clusterName] + if !ok { + c = make(map[string]*perClusterStore) + s.clusters[clusterName] = c + } + + if p, ok := c[serviceName]; ok { + return p + } + p := &perClusterStore{ + cluster: clusterName, + service: serviceName, + } + c[serviceName] = p + return p +} + +// perClusterStore is a repository for LB policy implementations to report store +// load data. It contains load for a (cluster, edsService) pair. +// +// It is safe for concurrent use. +// +// TODO(easwars): Use regular maps with mutexes instead of sync.Map here. The +// latter is optimized for two common use cases: (1) when the entry for a given +// key is only ever written once but read many times, as in caches that only +// grow, or (2) when multiple goroutines read, write, and overwrite entries for +// disjoint sets of keys. In these two cases, use of a Map may significantly +// reduce lock contention compared to a Go map paired with a separate Mutex or +// RWMutex. +// Neither of these conditions are met here, and we should transition to a +// regular map with a mutex for better type safety. +type perClusterStore struct { + cluster, service string + drops sync.Map // map[string]*uint64 + localityRPCCount sync.Map // map[string]*rpcCountData + + mu sync.Mutex + lastLoadReportAt time.Time +} + +// Update functions are called by picker for each RPC. To avoid contention, all +// updates are done atomically. + +// CallDropped adds one drop record with the given category to store. +func (ls *perClusterStore) CallDropped(category string) { + if ls == nil { + return + } + + p, ok := ls.drops.Load(category) + if !ok { + tp := new(uint64) + p, _ = ls.drops.LoadOrStore(category, tp) + } + atomic.AddUint64(p.(*uint64), 1) +} + +// CallStarted adds one call started record for the given locality. +func (ls *perClusterStore) CallStarted(locality string) { + if ls == nil { + return + } + + p, ok := ls.localityRPCCount.Load(locality) + if !ok { + tp := newRPCCountData() + p, _ = ls.localityRPCCount.LoadOrStore(locality, tp) + } + p.(*rpcCountData).incrInProgress() +} + +// CallFinished adds one call finished record for the given locality. +// For successful calls, err needs to be nil. +func (ls *perClusterStore) CallFinished(locality string, err error) { + if ls == nil { + return + } + + p, ok := ls.localityRPCCount.Load(locality) + if !ok { + // The map is never cleared, only values in the map are reset. So the + // case where entry for call-finish is not found should never happen. + return + } + p.(*rpcCountData).decrInProgress() + if err == nil { + p.(*rpcCountData).incrSucceeded() + } else { + p.(*rpcCountData).incrErrored() + } +} + +// CallServerLoad adds one server load record for the given locality. The +// load type is specified by desc, and its value by val. +func (ls *perClusterStore) CallServerLoad(locality, name string, d float64) { + if ls == nil { + return + } + + p, ok := ls.localityRPCCount.Load(locality) + if !ok { + // The map is never cleared, only values in the map are reset. So the + // case where entry for callServerLoad is not found should never happen. + return + } + p.(*rpcCountData).addServerLoad(name, d) +} + +// Data contains all load data reported to the Store since the most recent call +// to stats(). +type Data struct { + // Cluster is the name of the cluster this data is for. + Cluster string + // Service is the name of the EDS service this data is for. + Service string + // TotalDrops is the total number of dropped requests. + TotalDrops uint64 + // Drops is the number of dropped requests per category. + Drops map[string]uint64 + // LocalityStats contains load reports per locality. + LocalityStats map[string]LocalityData + // ReportInternal is the duration since last time load was reported (stats() + // was called). + ReportInterval time.Duration +} + +// LocalityData contains load data for a single locality. +type LocalityData struct { + // RequestStats contains counts of requests made to the locality. + RequestStats RequestData + // LoadStats contains server load data for requests made to the locality, + // indexed by the load type. + LoadStats map[string]ServerLoadData +} + +// RequestData contains request counts. +type RequestData struct { + // Succeeded is the number of succeeded requests. + Succeeded uint64 + // Errored is the number of requests which ran into errors. + Errored uint64 + // InProgress is the number of requests in flight. + InProgress uint64 +} + +// ServerLoadData contains server load data. +type ServerLoadData struct { + // Count is the number of load reports. + Count uint64 + // Sum is the total value of all load reports. + Sum float64 +} + +func newData(cluster, service string) *Data { + return &Data{ + Cluster: cluster, + Service: service, + Drops: make(map[string]uint64), + LocalityStats: make(map[string]LocalityData), + } +} + +// stats returns and resets all loads reported to the store, except inProgress +// rpc counts. +// +// It returns nil if the store doesn't contain any (new) data. +func (ls *perClusterStore) stats() *Data { + if ls == nil { + return nil + } + + sd := newData(ls.cluster, ls.service) + ls.drops.Range(func(key, val any) bool { + d := atomic.SwapUint64(val.(*uint64), 0) + if d == 0 { + return true + } + sd.TotalDrops += d + keyStr := key.(string) + if keyStr != "" { + // Skip drops without category. They are counted in total_drops, but + // not in per category. One example is drops by circuit breaking. + sd.Drops[keyStr] = d + } + return true + }) + ls.localityRPCCount.Range(func(key, val any) bool { + countData := val.(*rpcCountData) + succeeded := countData.loadAndClearSucceeded() + inProgress := countData.loadInProgress() + errored := countData.loadAndClearErrored() + if succeeded == 0 && inProgress == 0 && errored == 0 { + return true + } + + ld := LocalityData{ + RequestStats: RequestData{ + Succeeded: succeeded, + Errored: errored, + InProgress: inProgress, + }, + LoadStats: make(map[string]ServerLoadData), + } + countData.serverLoads.Range(func(key, val any) bool { + sum, count := val.(*rpcLoadData).loadAndClear() + if count == 0 { + return true + } + ld.LoadStats[key.(string)] = ServerLoadData{ + Count: count, + Sum: sum, + } + return true + }) + sd.LocalityStats[key.(string)] = ld + return true + }) + + ls.mu.Lock() + sd.ReportInterval = time.Since(ls.lastLoadReportAt) + ls.lastLoadReportAt = time.Now() + ls.mu.Unlock() + + if sd.TotalDrops == 0 && len(sd.Drops) == 0 && len(sd.LocalityStats) == 0 { + return nil + } + return sd +} + +type rpcCountData struct { + // Only atomic accesses are allowed for the fields. + succeeded *uint64 + errored *uint64 + inProgress *uint64 + + // Map from load desc to load data (sum+count). Loading data from map is + // atomic, but updating data takes a lock, which could cause contention when + // multiple RPCs try to report loads for the same desc. + // + // To fix the contention, shard this map. + serverLoads sync.Map // map[string]*rpcLoadData +} + +func newRPCCountData() *rpcCountData { + return &rpcCountData{ + succeeded: new(uint64), + errored: new(uint64), + inProgress: new(uint64), + } +} + +func (rcd *rpcCountData) incrSucceeded() { + atomic.AddUint64(rcd.succeeded, 1) +} + +func (rcd *rpcCountData) loadAndClearSucceeded() uint64 { + return atomic.SwapUint64(rcd.succeeded, 0) +} + +func (rcd *rpcCountData) incrErrored() { + atomic.AddUint64(rcd.errored, 1) +} + +func (rcd *rpcCountData) loadAndClearErrored() uint64 { + return atomic.SwapUint64(rcd.errored, 0) +} + +func (rcd *rpcCountData) incrInProgress() { + atomic.AddUint64(rcd.inProgress, 1) +} + +func (rcd *rpcCountData) decrInProgress() { + atomic.AddUint64(rcd.inProgress, negativeOneUInt64) // atomic.Add(x, -1) +} + +func (rcd *rpcCountData) loadInProgress() uint64 { + return atomic.LoadUint64(rcd.inProgress) // InProgress count is not clear when reading. +} + +func (rcd *rpcCountData) addServerLoad(name string, d float64) { + loads, ok := rcd.serverLoads.Load(name) + if !ok { + tl := newRPCLoadData() + loads, _ = rcd.serverLoads.LoadOrStore(name, tl) + } + loads.(*rpcLoadData).add(d) +} + +// Data for server loads (from trailers or oob). Fields in this struct must be +// updated consistently. +// +// The current solution is to hold a lock, which could cause contention. To fix, +// shard serverLoads map in rpcCountData. +type rpcLoadData struct { + mu sync.Mutex + sum float64 + count uint64 +} + +func newRPCLoadData() *rpcLoadData { + return &rpcLoadData{} +} + +func (rld *rpcLoadData) add(v float64) { + rld.mu.Lock() + rld.sum += v + rld.count++ + rld.mu.Unlock() +} + +func (rld *rpcLoadData) loadAndClear() (s float64, c uint64) { + rld.mu.Lock() + s = rld.sum + rld.sum = 0 + c = rld.count + rld.count = 0 + rld.mu.Unlock() + return +} diff --git a/xds/internal/xdsclient/load/store_test.go b/xds/internal/xdsclient/load/store_test.go new file mode 100644 index 000000000000..46568591f9e4 --- /dev/null +++ b/xds/internal/xdsclient/load/store_test.go @@ -0,0 +1,446 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package load + +import ( + "fmt" + "sort" + "sync" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" +) + +var ( + dropCategories = []string{"drop_for_real", "drop_for_fun"} + localities = []string{"locality-A", "locality-B"} + errTest = fmt.Errorf("test error") +) + +// rpcData wraps the rpc counts and load data to be pushed to the store. +type rpcData struct { + start, success, failure int + serverData map[string]float64 // Will be reported with successful RPCs. +} + +// TestDrops spawns a bunch of goroutines which report drop data. After the +// goroutines have exited, the test dumps the stats from the Store and makes +// sure they are as expected. +func TestDrops(t *testing.T) { + var ( + drops = map[string]int{ + dropCategories[0]: 30, + dropCategories[1]: 40, + "": 10, + } + wantStoreData = &Data{ + TotalDrops: 80, + Drops: map[string]uint64{ + dropCategories[0]: 30, + dropCategories[1]: 40, + }, + } + ) + + ls := perClusterStore{} + var wg sync.WaitGroup + for category, count := range drops { + for i := 0; i < count; i++ { + wg.Add(1) + go func(c string) { + ls.CallDropped(c) + wg.Done() + }(category) + } + } + wg.Wait() + + gotStoreData := ls.stats() + if diff := cmp.Diff(wantStoreData, gotStoreData, cmpopts.EquateEmpty(), cmpopts.IgnoreFields(Data{}, "ReportInterval")); diff != "" { + t.Errorf("store.stats() returned unexpected diff (-want +got):\n%s", diff) + } +} + +// TestLocalityStats spawns a bunch of goroutines which report rpc and load +// data. After the goroutines have exited, the test dumps the stats from the +// Store and makes sure they are as expected. +func TestLocalityStats(t *testing.T) { + var ( + localityData = map[string]rpcData{ + localities[0]: { + start: 40, + success: 20, + failure: 10, + serverData: map[string]float64{"net": 1, "disk": 2, "cpu": 3, "mem": 4}, + }, + localities[1]: { + start: 80, + success: 40, + failure: 20, + serverData: map[string]float64{"net": 1, "disk": 2, "cpu": 3, "mem": 4}, + }, + } + wantStoreData = &Data{ + LocalityStats: map[string]LocalityData{ + localities[0]: { + RequestStats: RequestData{Succeeded: 20, Errored: 10, InProgress: 10}, + LoadStats: map[string]ServerLoadData{ + "net": {Count: 20, Sum: 20}, + "disk": {Count: 20, Sum: 40}, + "cpu": {Count: 20, Sum: 60}, + "mem": {Count: 20, Sum: 80}, + }, + }, + localities[1]: { + RequestStats: RequestData{Succeeded: 40, Errored: 20, InProgress: 20}, + LoadStats: map[string]ServerLoadData{ + "net": {Count: 40, Sum: 40}, + "disk": {Count: 40, Sum: 80}, + "cpu": {Count: 40, Sum: 120}, + "mem": {Count: 40, Sum: 160}, + }, + }, + }, + } + ) + + ls := perClusterStore{} + var wg sync.WaitGroup + for locality, data := range localityData { + wg.Add(data.start) + for i := 0; i < data.start; i++ { + go func(l string) { + ls.CallStarted(l) + wg.Done() + }(locality) + } + // The calls to callStarted() need to happen before the other calls are + // made. Hence the wait here. + wg.Wait() + + wg.Add(data.success) + for i := 0; i < data.success; i++ { + go func(l string, serverData map[string]float64) { + ls.CallFinished(l, nil) + for n, d := range serverData { + ls.CallServerLoad(l, n, d) + } + wg.Done() + }(locality, data.serverData) + } + wg.Add(data.failure) + for i := 0; i < data.failure; i++ { + go func(l string) { + ls.CallFinished(l, errTest) + wg.Done() + }(locality) + } + wg.Wait() + } + + gotStoreData := ls.stats() + if diff := cmp.Diff(wantStoreData, gotStoreData, cmpopts.EquateEmpty(), cmpopts.IgnoreFields(Data{}, "ReportInterval")); diff != "" { + t.Errorf("store.stats() returned unexpected diff (-want +got):\n%s", diff) + } +} + +func TestResetAfterStats(t *testing.T) { + // Push a bunch of drops, call stats and load stats, and leave inProgress to be non-zero. + // Dump the stats. Verify expexted + // Push the same set of loads as before + // Now dump and verify the newly expected ones. + var ( + drops = map[string]int{ + dropCategories[0]: 30, + dropCategories[1]: 40, + } + localityData = map[string]rpcData{ + localities[0]: { + start: 40, + success: 20, + failure: 10, + serverData: map[string]float64{"net": 1, "disk": 2, "cpu": 3, "mem": 4}, + }, + localities[1]: { + start: 80, + success: 40, + failure: 20, + serverData: map[string]float64{"net": 1, "disk": 2, "cpu": 3, "mem": 4}, + }, + } + wantStoreData = &Data{ + TotalDrops: 70, + Drops: map[string]uint64{ + dropCategories[0]: 30, + dropCategories[1]: 40, + }, + LocalityStats: map[string]LocalityData{ + localities[0]: { + RequestStats: RequestData{Succeeded: 20, Errored: 10, InProgress: 10}, + LoadStats: map[string]ServerLoadData{ + "net": {Count: 20, Sum: 20}, + "disk": {Count: 20, Sum: 40}, + "cpu": {Count: 20, Sum: 60}, + "mem": {Count: 20, Sum: 80}, + }, + }, + localities[1]: { + RequestStats: RequestData{Succeeded: 40, Errored: 20, InProgress: 20}, + LoadStats: map[string]ServerLoadData{ + "net": {Count: 40, Sum: 40}, + "disk": {Count: 40, Sum: 80}, + "cpu": {Count: 40, Sum: 120}, + "mem": {Count: 40, Sum: 160}, + }, + }, + }, + } + ) + + reportLoad := func(ls *perClusterStore) { + for category, count := range drops { + for i := 0; i < count; i++ { + ls.CallDropped(category) + } + } + for locality, data := range localityData { + for i := 0; i < data.start; i++ { + ls.CallStarted(locality) + } + for i := 0; i < data.success; i++ { + ls.CallFinished(locality, nil) + for n, d := range data.serverData { + ls.CallServerLoad(locality, n, d) + } + } + for i := 0; i < data.failure; i++ { + ls.CallFinished(locality, errTest) + } + } + } + + ls := perClusterStore{} + reportLoad(&ls) + gotStoreData := ls.stats() + if diff := cmp.Diff(wantStoreData, gotStoreData, cmpopts.EquateEmpty(), cmpopts.IgnoreFields(Data{}, "ReportInterval")); diff != "" { + t.Errorf("store.stats() returned unexpected diff (-want +got):\n%s", diff) + } + + // The above call to stats() should have reset all load reports except the + // inProgress rpc count. We are now going to push the same load data into + // the store. So, we should expect to see twice the count for inProgress. + for _, l := range localities { + ls := wantStoreData.LocalityStats[l] + ls.RequestStats.InProgress *= 2 + wantStoreData.LocalityStats[l] = ls + } + reportLoad(&ls) + gotStoreData = ls.stats() + if diff := cmp.Diff(wantStoreData, gotStoreData, cmpopts.EquateEmpty(), cmpopts.IgnoreFields(Data{}, "ReportInterval")); diff != "" { + t.Errorf("store.stats() returned unexpected diff (-want +got):\n%s", diff) + } +} + +var sortDataSlice = cmp.Transformer("SortDataSlice", func(in []*Data) []*Data { + out := append([]*Data(nil), in...) // Copy input to avoid mutating it + sort.Slice(out, + func(i, j int) bool { + if out[i].Cluster < out[j].Cluster { + return true + } + if out[i].Cluster == out[j].Cluster { + return out[i].Service < out[j].Service + } + return false + }, + ) + return out +}) + +// Test all load are returned for the given clusters, and all clusters are +// reported if no cluster is specified. +func TestStoreStats(t *testing.T) { + var ( + testClusters = []string{"c0", "c1", "c2"} + testServices = []string{"s0", "s1"} + testLocality = "test-locality" + ) + + store := NewStore() + for _, c := range testClusters { + for _, s := range testServices { + store.PerCluster(c, s).CallStarted(testLocality) + store.PerCluster(c, s).CallServerLoad(testLocality, "abc", 123) + store.PerCluster(c, s).CallDropped("dropped") + store.PerCluster(c, s).CallFinished(testLocality, nil) + } + } + + wantC0 := []*Data{ + { + Cluster: "c0", Service: "s0", + TotalDrops: 1, Drops: map[string]uint64{"dropped": 1}, + LocalityStats: map[string]LocalityData{ + "test-locality": { + RequestStats: RequestData{Succeeded: 1}, + LoadStats: map[string]ServerLoadData{"abc": {Count: 1, Sum: 123}}, + }, + }, + }, + { + Cluster: "c0", Service: "s1", + TotalDrops: 1, Drops: map[string]uint64{"dropped": 1}, + LocalityStats: map[string]LocalityData{ + "test-locality": { + RequestStats: RequestData{Succeeded: 1}, + LoadStats: map[string]ServerLoadData{"abc": {Count: 1, Sum: 123}}, + }, + }, + }, + } + // Call Stats with just "c0", this should return data for "c0", and not + // touch data for other clusters. + gotC0 := store.Stats([]string{"c0"}) + if diff := cmp.Diff(wantC0, gotC0, cmpopts.EquateEmpty(), cmpopts.IgnoreFields(Data{}, "ReportInterval"), sortDataSlice); diff != "" { + t.Errorf("store.stats() returned unexpected diff (-want +got):\n%s", diff) + } + + wantOther := []*Data{ + { + Cluster: "c1", Service: "s0", + TotalDrops: 1, Drops: map[string]uint64{"dropped": 1}, + LocalityStats: map[string]LocalityData{ + "test-locality": { + RequestStats: RequestData{Succeeded: 1}, + LoadStats: map[string]ServerLoadData{"abc": {Count: 1, Sum: 123}}, + }, + }, + }, + { + Cluster: "c1", Service: "s1", + TotalDrops: 1, Drops: map[string]uint64{"dropped": 1}, + LocalityStats: map[string]LocalityData{ + "test-locality": { + RequestStats: RequestData{Succeeded: 1}, + LoadStats: map[string]ServerLoadData{"abc": {Count: 1, Sum: 123}}, + }, + }, + }, + { + Cluster: "c2", Service: "s0", + TotalDrops: 1, Drops: map[string]uint64{"dropped": 1}, + LocalityStats: map[string]LocalityData{ + "test-locality": { + RequestStats: RequestData{Succeeded: 1}, + LoadStats: map[string]ServerLoadData{"abc": {Count: 1, Sum: 123}}, + }, + }, + }, + { + Cluster: "c2", Service: "s1", + TotalDrops: 1, Drops: map[string]uint64{"dropped": 1}, + LocalityStats: map[string]LocalityData{ + "test-locality": { + RequestStats: RequestData{Succeeded: 1}, + LoadStats: map[string]ServerLoadData{"abc": {Count: 1, Sum: 123}}, + }, + }, + }, + } + // Call Stats with empty slice, this should return data for all the + // remaining clusters, and not include c0 (because c0 data was cleared). + gotOther := store.Stats(nil) + if diff := cmp.Diff(wantOther, gotOther, cmpopts.EquateEmpty(), cmpopts.IgnoreFields(Data{}, "ReportInterval"), sortDataSlice); diff != "" { + t.Errorf("store.stats() returned unexpected diff (-want +got):\n%s", diff) + } +} + +// Test the cases that if a cluster doesn't have load to report, its data is not +// appended to the slice returned by Stats(). +func TestStoreStatsEmptyDataNotReported(t *testing.T) { + var ( + testServices = []string{"s0", "s1"} + testLocality = "test-locality" + ) + + store := NewStore() + // "c0"'s RPCs all finish with success. + for _, s := range testServices { + store.PerCluster("c0", s).CallStarted(testLocality) + store.PerCluster("c0", s).CallFinished(testLocality, nil) + } + // "c1"'s RPCs never finish (always inprocess). + for _, s := range testServices { + store.PerCluster("c1", s).CallStarted(testLocality) + } + + want0 := []*Data{ + { + Cluster: "c0", Service: "s0", + LocalityStats: map[string]LocalityData{ + "test-locality": {RequestStats: RequestData{Succeeded: 1}}, + }, + }, + { + Cluster: "c0", Service: "s1", + LocalityStats: map[string]LocalityData{ + "test-locality": {RequestStats: RequestData{Succeeded: 1}}, + }, + }, + { + Cluster: "c1", Service: "s0", + LocalityStats: map[string]LocalityData{ + "test-locality": {RequestStats: RequestData{InProgress: 1}}, + }, + }, + { + Cluster: "c1", Service: "s1", + LocalityStats: map[string]LocalityData{ + "test-locality": {RequestStats: RequestData{InProgress: 1}}, + }, + }, + } + // Call Stats with empty slice, this should return data for all the + // clusters. + got0 := store.Stats(nil) + if diff := cmp.Diff(want0, got0, cmpopts.EquateEmpty(), cmpopts.IgnoreFields(Data{}, "ReportInterval"), sortDataSlice); diff != "" { + t.Errorf("store.stats() returned unexpected diff (-want +got):\n%s", diff) + } + + want1 := []*Data{ + { + Cluster: "c1", Service: "s0", + LocalityStats: map[string]LocalityData{ + "test-locality": {RequestStats: RequestData{InProgress: 1}}, + }, + }, + { + Cluster: "c1", Service: "s1", + LocalityStats: map[string]LocalityData{ + "test-locality": {RequestStats: RequestData{InProgress: 1}}, + }, + }, + } + // Call Stats with empty slice again, this should return data only for "c1", + // because "c0" data was cleared, but "c1" has in-progress RPCs. + got1 := store.Stats(nil) + if diff := cmp.Diff(want1, got1, cmpopts.EquateEmpty(), cmpopts.IgnoreFields(Data{}, "ReportInterval"), sortDataSlice); diff != "" { + t.Errorf("store.stats() returned unexpected diff (-want +got):\n%s", diff) + } +} diff --git a/xds/internal/xdsclient/loadreport_test.go b/xds/internal/xdsclient/loadreport_test.go new file mode 100644 index 000000000000..06e58acdd2dc --- /dev/null +++ b/xds/internal/xdsclient/loadreport_test.go @@ -0,0 +1,135 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient + +import ( + "context" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal/testutils/xds/fakeserver" + "google.golang.org/grpc/status" + xdstestutils "google.golang.org/grpc/xds/internal/testutils" + "google.golang.org/grpc/xds/internal/xdsclient/bootstrap" + "google.golang.org/protobuf/testing/protocmp" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3lrspb "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v3" + durationpb "github.com/golang/protobuf/ptypes/duration" +) + +const ( + defaultClientWatchExpiryTimeout = 15 * time.Second +) + +func (s) TestLRSClient(t *testing.T) { + fs1, sCleanup, err := fakeserver.StartServer(nil) + if err != nil { + t.Fatalf("failed to start fake xDS server: %v", err) + } + defer sCleanup() + + serverCfg1 := xdstestutils.ServerConfigForAddress(t, fs1.Address) + xdsC, close, err := NewWithConfigForTesting(&bootstrap.Config{ + XDSServer: serverCfg1, + NodeProto: &v3corepb.Node{}, + }, defaultClientWatchExpiryTimeout, time.Duration(0)) + if err != nil { + t.Fatalf("failed to create xds client: %v", err) + } + defer close() + + // Report to the same address should not create new ClientConn. + store1, lrsCancel1 := xdsC.ReportLoad(serverCfg1) + defer lrsCancel1() + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if u, err := fs1.NewConnChan.Receive(ctx); err != nil { + t.Errorf("unexpected timeout: %v, %v, want NewConn", u, err) + } + + sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + if u, err := fs1.NewConnChan.Receive(sCtx); err != context.DeadlineExceeded { + t.Errorf("unexpected NewConn: %v, %v, want channel recv timeout", u, err) + } + + fs2, sCleanup2, err := fakeserver.StartServer(nil) + if err != nil { + t.Fatalf("failed to start fake xDS server: %v", err) + } + defer sCleanup2() + + // Report to a different address should create new ClientConn. + serverCgf2 := xdstestutils.ServerConfigForAddress(t, fs2.Address) + store2, lrsCancel2 := xdsC.ReportLoad(serverCgf2) + defer lrsCancel2() + if u, err := fs2.NewConnChan.Receive(ctx); err != nil { + t.Errorf("unexpected timeout: %v, %v, want NewConn", u, err) + } + + if store1 == store2 { + t.Fatalf("got same store for different servers, want different") + } + + if u, err := fs2.LRSRequestChan.Receive(ctx); err != nil { + t.Errorf("unexpected timeout: %v, %v, want NewConn", u, err) + } + store2.PerCluster("cluster", "eds").CallDropped("test") + + // Send one resp to the client. + fs2.LRSResponseChan <- &fakeserver.Response{ + Resp: &v3lrspb.LoadStatsResponse{ + SendAllClusters: true, + LoadReportingInterval: &durationpb.Duration{Nanos: 50000000}, + }, + } + + // Server should receive a req with the loads. + u, err := fs2.LRSRequestChan.Receive(ctx) + if err != nil { + t.Fatalf("unexpected LRS request: %v, %v, want error canceled", u, err) + } + receivedLoad := u.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest).ClusterStats + if len(receivedLoad) <= 0 { + t.Fatalf("unexpected load received, want load for cluster, eds, dropped for test") + } + receivedLoad[0].LoadReportInterval = nil + want := &v3endpointpb.ClusterStats{ + ClusterName: "cluster", + ClusterServiceName: "eds", + TotalDroppedRequests: 1, + DroppedRequests: []*v3endpointpb.ClusterStats_DroppedRequests{{Category: "test", DroppedCount: 1}}, + } + if d := cmp.Diff(want, receivedLoad[0], protocmp.Transform()); d != "" { + t.Fatalf("unexpected load received, want load for cluster, eds, dropped for test, diff (-want +got):\n%s", d) + } + + // Cancel this load reporting stream, server should see error canceled. + lrsCancel2() + + // Server should receive a stream canceled error. + if u, err := fs2.LRSRequestChan.Receive(ctx); err != nil || status.Code(u.(*fakeserver.Request).Err) != codes.Canceled { + t.Errorf("unexpected LRS request: %v, %v, want error canceled", u, err) + } +} diff --git a/xds/internal/xdsclient/logging.go b/xds/internal/xdsclient/logging.go new file mode 100644 index 000000000000..2269cb293da9 --- /dev/null +++ b/xds/internal/xdsclient/logging.go @@ -0,0 +1,40 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient + +import ( + "fmt" + + "google.golang.org/grpc/grpclog" + internalgrpclog "google.golang.org/grpc/internal/grpclog" +) + +var logger = grpclog.Component("xds") + +func prefixLogger(p *clientImpl) *internalgrpclog.PrefixLogger { + return internalgrpclog.NewPrefixLogger(logger, clientPrefix(p)) +} + +func clientPrefix(p *clientImpl) string { + return fmt.Sprintf("[xds-client %p] ", p) +} + +func authorityPrefix(p *clientImpl, serverURI string) string { + return fmt.Sprintf("%s[%s] ", clientPrefix(p), serverURI) +} diff --git a/xds/internal/xdsclient/requests_counter.go b/xds/internal/xdsclient/requests_counter.go new file mode 100644 index 000000000000..beed2e9d0add --- /dev/null +++ b/xds/internal/xdsclient/requests_counter.go @@ -0,0 +1,107 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient + +import ( + "fmt" + "sync" + "sync/atomic" +) + +type clusterNameAndServiceName struct { + clusterName, edsServcieName string +} + +type clusterRequestsCounter struct { + mu sync.Mutex + clusters map[clusterNameAndServiceName]*ClusterRequestsCounter +} + +var src = &clusterRequestsCounter{ + clusters: make(map[clusterNameAndServiceName]*ClusterRequestsCounter), +} + +// ClusterRequestsCounter is used to track the total inflight requests for a +// service with the provided name. +type ClusterRequestsCounter struct { + ClusterName string + EDSServiceName string + numRequests uint32 +} + +// GetClusterRequestsCounter returns the ClusterRequestsCounter with the +// provided serviceName. If one does not exist, it creates it. +func GetClusterRequestsCounter(clusterName, edsServiceName string) *ClusterRequestsCounter { + src.mu.Lock() + defer src.mu.Unlock() + k := clusterNameAndServiceName{ + clusterName: clusterName, + edsServcieName: edsServiceName, + } + c, ok := src.clusters[k] + if !ok { + c = &ClusterRequestsCounter{ClusterName: clusterName} + src.clusters[k] = c + } + return c +} + +// StartRequest starts a request for a cluster, incrementing its number of +// requests by 1. Returns an error if the max number of requests is exceeded. +func (c *ClusterRequestsCounter) StartRequest(max uint32) error { + // Note that during race, the limits could be exceeded. This is allowed: + // "Since the implementation is eventually consistent, races between threads + // may allow limits to be potentially exceeded." + // https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/circuit_breaking#arch-overview-circuit-break. + if atomic.LoadUint32(&c.numRequests) >= max { + return fmt.Errorf("max requests %v exceeded on service %v", max, c.ClusterName) + } + atomic.AddUint32(&c.numRequests, 1) + return nil +} + +// EndRequest ends a request for a service, decrementing its number of requests +// by 1. +func (c *ClusterRequestsCounter) EndRequest() { + atomic.AddUint32(&c.numRequests, ^uint32(0)) +} + +// ClearCounterForTesting clears the counter for the service. Should be only +// used in tests. +func ClearCounterForTesting(clusterName, edsServiceName string) { + src.mu.Lock() + defer src.mu.Unlock() + k := clusterNameAndServiceName{ + clusterName: clusterName, + edsServcieName: edsServiceName, + } + c, ok := src.clusters[k] + if !ok { + return + } + c.numRequests = 0 +} + +// ClearAllCountersForTesting clears all the counters. Should be only used in +// tests. +func ClearAllCountersForTesting() { + src.mu.Lock() + defer src.mu.Unlock() + src.clusters = make(map[clusterNameAndServiceName]*ClusterRequestsCounter) +} diff --git a/xds/internal/xdsclient/requests_counter_test.go b/xds/internal/xdsclient/requests_counter_test.go new file mode 100644 index 000000000000..e2eeea774e24 --- /dev/null +++ b/xds/internal/xdsclient/requests_counter_test.go @@ -0,0 +1,162 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient + +import ( + "sync" + "sync/atomic" + "testing" +) + +const testService = "test-service-name" + +type counterTest struct { + name string + maxRequests uint32 + numRequests uint32 + expectedSuccesses uint32 + expectedErrors uint32 +} + +var tests = []counterTest{ + { + name: "does-not-exceed-max-requests", + maxRequests: 1024, + numRequests: 1024, + expectedSuccesses: 1024, + expectedErrors: 0, + }, + { + name: "exceeds-max-requests", + maxRequests: 32, + numRequests: 64, + expectedSuccesses: 32, + expectedErrors: 32, + }, +} + +func resetClusterRequestsCounter() { + src = &clusterRequestsCounter{ + clusters: make(map[clusterNameAndServiceName]*ClusterRequestsCounter), + } +} + +func testCounter(t *testing.T, test counterTest) { + requestsStarted := make(chan struct{}) + requestsSent := sync.WaitGroup{} + requestsSent.Add(int(test.numRequests)) + requestsDone := sync.WaitGroup{} + requestsDone.Add(int(test.numRequests)) + var lastError atomic.Value + var successes, errors uint32 + for i := 0; i < int(test.numRequests); i++ { + go func() { + counter := GetClusterRequestsCounter(test.name, testService) + defer requestsDone.Done() + err := counter.StartRequest(test.maxRequests) + if err == nil { + atomic.AddUint32(&successes, 1) + } else { + atomic.AddUint32(&errors, 1) + lastError.Store(err) + } + requestsSent.Done() + if err == nil { + <-requestsStarted + counter.EndRequest() + } + }() + } + requestsSent.Wait() + close(requestsStarted) + requestsDone.Wait() + loadedError := lastError.Load() + if test.expectedErrors > 0 && loadedError == nil { + t.Error("no error when error expected") + } + if test.expectedErrors == 0 && loadedError != nil { + t.Errorf("error starting request: %v", loadedError.(error)) + } + // We allow the limits to be exceeded during races. + // + // But we should never over-limit, so this test fails if there are less + // successes than expected. + if successes < test.expectedSuccesses || errors > test.expectedErrors { + t.Errorf("unexpected number of (successes, errors), expected (%v, %v), encountered (%v, %v)", test.expectedSuccesses, test.expectedErrors, successes, errors) + } +} + +func (s) TestRequestsCounter(t *testing.T) { + defer resetClusterRequestsCounter() + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + testCounter(t, test) + }) + } +} + +func (s) TestGetClusterRequestsCounter(t *testing.T) { + defer resetClusterRequestsCounter() + for _, test := range tests { + counterA := GetClusterRequestsCounter(test.name, testService) + counterB := GetClusterRequestsCounter(test.name, testService) + if counterA != counterB { + t.Errorf("counter %v %v != counter %v %v", counterA, *counterA, counterB, *counterB) + } + } +} + +func startRequests(t *testing.T, n uint32, max uint32, counter *ClusterRequestsCounter) { + for i := uint32(0); i < n; i++ { + if err := counter.StartRequest(max); err != nil { + t.Fatalf("error starting initial request: %v", err) + } + } +} + +func (s) TestSetMaxRequestsIncreased(t *testing.T) { + defer resetClusterRequestsCounter() + const clusterName string = "set-max-requests-increased" + var initialMax uint32 = 16 + + counter := GetClusterRequestsCounter(clusterName, testService) + startRequests(t, initialMax, initialMax, counter) + if err := counter.StartRequest(initialMax); err == nil { + t.Fatal("unexpected success on start request after max met") + } + + newMax := initialMax + 1 + if err := counter.StartRequest(newMax); err != nil { + t.Fatalf("unexpected error on start request after max increased: %v", err) + } +} + +func (s) TestSetMaxRequestsDecreased(t *testing.T) { + defer resetClusterRequestsCounter() + const clusterName string = "set-max-requests-decreased" + var initialMax uint32 = 16 + + counter := GetClusterRequestsCounter(clusterName, testService) + startRequests(t, initialMax-1, initialMax, counter) + + newMax := initialMax - 1 + if err := counter.StartRequest(newMax); err == nil { + t.Fatalf("unexpected success on start request after max decreased: %v", err) + } +} diff --git a/xds/internal/xdsclient/singleton.go b/xds/internal/xdsclient/singleton.go new file mode 100644 index 000000000000..96db8ef51387 --- /dev/null +++ b/xds/internal/xdsclient/singleton.go @@ -0,0 +1,115 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient + +import ( + "fmt" + "sync" + "sync/atomic" + "time" + + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/xds/internal/xdsclient/bootstrap" +) + +const ( + defaultWatchExpiryTimeout = 15 * time.Second + defaultIdleAuthorityDeleteTimeout = 5 * time.Minute +) + +var ( + // This is the client returned by New(). It contains one client implementation, + // and maintains the refcount. + singletonMu sync.Mutex + singletonClient *clientRefCounted + + // The following functions are no-ops in the actual code, but can be + // overridden in tests to give them visibility into certain events. + singletonClientImplCreateHook = func() {} + singletonClientImplCloseHook = func() {} +) + +// To override in tests. +var bootstrapNewConfig = bootstrap.NewConfig + +func clientRefCountedClose() { + singletonMu.Lock() + defer singletonMu.Unlock() + + if singletonClient.decrRef() != 0 { + return + } + singletonClient.clientImpl.close() + singletonClientImplCloseHook() + singletonClient = nil +} + +func newRefCountedWithConfig(fallbackConfig *bootstrap.Config) (XDSClient, func(), error) { + singletonMu.Lock() + defer singletonMu.Unlock() + + if singletonClient != nil { + singletonClient.incrRef() + return singletonClient, grpcsync.OnceFunc(clientRefCountedClose), nil + + } + + // Use fallbackConfig only if bootstrap env vars are unspecified. + var config *bootstrap.Config + if envconfig.XDSBootstrapFileName == "" && envconfig.XDSBootstrapFileContent == "" { + if fallbackConfig == nil { + return nil, nil, fmt.Errorf("xds: bootstrap env vars are unspecified and provided fallback config is nil") + } + config = fallbackConfig + } else { + var err error + config, err = bootstrapNewConfig() + if err != nil { + return nil, nil, fmt.Errorf("xds: failed to read bootstrap file: %v", err) + } + } + + // Create the new client implementation. + c, err := newWithConfig(config, defaultWatchExpiryTimeout, defaultIdleAuthorityDeleteTimeout) + if err != nil { + return nil, nil, err + } + singletonClient = &clientRefCounted{clientImpl: c, refCount: 1} + singletonClientImplCreateHook() + + logger.Infof("xDS node ID: %s", config.NodeProto.GetId()) + return singletonClient, grpcsync.OnceFunc(clientRefCountedClose), nil +} + +// clientRefCounted is ref-counted, and to be shared by the xds resolver and +// balancer implementations, across multiple ClientConns and Servers. +type clientRefCounted struct { + *clientImpl + + refCount int32 // accessed atomically +} + +func (c *clientRefCounted) incrRef() int32 { + return atomic.AddInt32(&c.refCount, 1) +} + +func (c *clientRefCounted) decrRef() int32 { + return atomic.AddInt32(&c.refCount, -1) +} diff --git a/xds/internal/xdsclient/singleton_test.go b/xds/internal/xdsclient/singleton_test.go new file mode 100644 index 000000000000..1875ea118d09 --- /dev/null +++ b/xds/internal/xdsclient/singleton_test.go @@ -0,0 +1,124 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient + +import ( + "context" + "testing" + + "github.com/google/uuid" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/bootstrap" +) + +// Test that multiple New() returns the same Client. And only when the last +// client is closed, the underlying client is closed. +func (s) TestClientNewSingleton(t *testing.T) { + // Create a bootstrap configuration, place it in a file in the temp + // directory, and set the bootstrap env vars to point to it. + nodeID := uuid.New().String() + cleanup, err := bootstrap.CreateFile(bootstrap.Options{ + NodeID: nodeID, + ServerURI: "non-existent-server-address", + }) + if err != nil { + t.Fatal(err) + } + defer cleanup() + + // Override the singleton creation hook to get notified. + origSingletonClientImplCreateHook := singletonClientImplCreateHook + singletonCreationCh := testutils.NewChannel() + singletonClientImplCreateHook = func() { + singletonCreationCh.Replace(nil) + } + defer func() { singletonClientImplCreateHook = origSingletonClientImplCreateHook }() + + // Override the singleton close hook to get notified. + origSingletonClientImplCloseHook := singletonClientImplCloseHook + singletonCloseCh := testutils.NewChannel() + singletonClientImplCloseHook = func() { + singletonCloseCh.Replace(nil) + } + defer func() { singletonClientImplCloseHook = origSingletonClientImplCloseHook }() + + // The first call to New() should create a new singleton client. + _, closeFunc, err := New() + if err != nil { + t.Fatalf("failed to create xDS client: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := singletonCreationCh.Receive(ctx); err != nil { + t.Fatalf("Timeout when waiting for singleton xDS client to be created: %v", err) + } + + // Calling New() again should not create new singleton client implementations. + const count = 9 + closeFuncs := make([]func(), 9) + for i := 0; i < count; i++ { + func() { + _, closeFuncs[i], err = New() + if err != nil { + t.Fatalf("%d-th call to New() failed with error: %v", i, err) + } + + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := singletonCreationCh.Receive(sCtx); err == nil { + t.Fatalf("%d-th call to New() created a new singleton client", i) + } + }() + } + + // Call Close() multiple times on each of the clients created in the above for + // loop. Close() calls are idempotent, and the underlying client + // implementation will not be closed until we release the first reference we + // acquired above, via the first call to New(). + for i := 0; i < count; i++ { + func() { + closeFuncs[i]() + closeFuncs[i]() + + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := singletonCloseCh.Receive(sCtx); err == nil { + t.Fatal("singleton client implementation closed before all references are released") + } + }() + } + + // Call the last Close(). The underlying implementation should be closed. + closeFunc() + if _, err := singletonCloseCh.Receive(ctx); err != nil { + t.Fatalf("Timeout waiting for singleton client implementation to be closed: %v", err) + } + + // Calling New() again, after the previous Client was actually closed, should + // create a new one. + _, closeFunc, err = New() + if err != nil { + t.Fatalf("failed to create client: %v", err) + } + defer closeFunc() + if _, err := singletonCreationCh.Receive(ctx); err != nil { + t.Fatalf("Timeout when waiting for singleton xDS client to be created: %v", err) + } +} diff --git a/xds/internal/xdsclient/tests/authority_test.go b/xds/internal/xdsclient/tests/authority_test.go new file mode 100644 index 000000000000..0345f4a4040c --- /dev/null +++ b/xds/internal/xdsclient/tests/authority_test.go @@ -0,0 +1,303 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient_test + +import ( + "context" + "testing" + "time" + + "github.com/google/uuid" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + xdstestutils "google.golang.org/grpc/xds/internal/testutils" + "google.golang.org/grpc/xds/internal/xdsclient" + "google.golang.org/grpc/xds/internal/xdsclient/bootstrap" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" +) + +const ( + testAuthority1 = "test-authority1" + testAuthority2 = "test-authority2" + testAuthority3 = "test-authority3" +) + +var ( + // These two resources use `testAuthority1`, which contains an empty server + // config in the bootstrap file, and therefore will use the default + // management server. + authorityTestResourceName11 = xdstestutils.BuildResourceName(xdsresource.ClusterResourceTypeName, testAuthority1, cdsName+"1", nil) + authorityTestResourceName12 = xdstestutils.BuildResourceName(xdsresource.ClusterResourceTypeName, testAuthority1, cdsName+"2", nil) + // This resource uses `testAuthority2`, which contains an empty server + // config in the bootstrap file, and therefore will use the default + // management server. + authorityTestResourceName2 = xdstestutils.BuildResourceName(xdsresource.ClusterResourceTypeName, testAuthority2, cdsName+"3", nil) + // This resource uses `testAuthority3`, which contains a non-empty server + // config in the bootstrap file, and therefore will use the non-default + // management server. + authorityTestResourceName3 = xdstestutils.BuildResourceName(xdsresource.ClusterResourceTypeName, testAuthority3, cdsName+"3", nil) +) + +// setupForAuthorityTests spins up two management servers, one to act as the +// default and the other to act as the non-default. It also generates a +// bootstrap configuration with three authorities (the first two pointing to the +// default and the third one pointing to the non-default). +// +// Returns two listeners used by the default and non-default management servers +// respectively, and the xDS client and its close function. +func setupForAuthorityTests(ctx context.Context, t *testing.T, idleTimeout time.Duration) (*testutils.ListenerWrapper, *testutils.ListenerWrapper, xdsclient.XDSClient, func()) { + overrideFedEnvVar(t) + + // Create listener wrappers which notify on to a channel whenever a new + // connection is accepted. We use this to track the number of transports + // used by the xDS client. + lisDefault := testutils.NewListenerWrapper(t, nil) + lisNonDefault := testutils.NewListenerWrapper(t, nil) + + // Start a management server to act as the default authority. + defaultAuthorityServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{Listener: lisDefault}) + if err != nil { + t.Fatalf("Failed to spin up the xDS management server: %v", err) + } + t.Cleanup(func() { defaultAuthorityServer.Stop() }) + + // Start a management server to act as the non-default authority. + nonDefaultAuthorityServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{Listener: lisNonDefault}) + if err != nil { + t.Fatalf("Failed to spin up the xDS management server: %v", err) + } + t.Cleanup(func() { nonDefaultAuthorityServer.Stop() }) + + // Create a bootstrap configuration with two non-default authorities which + // have empty server configs, and therefore end up using the default server + // config, which points to the above management server. + nodeID := uuid.New().String() + client, close, err := xdsclient.NewWithConfigForTesting(&bootstrap.Config{ + XDSServer: xdstestutils.ServerConfigForAddress(t, defaultAuthorityServer.Address), + NodeProto: &v3corepb.Node{Id: nodeID}, + Authorities: map[string]*bootstrap.Authority{ + testAuthority1: {}, + testAuthority2: {}, + testAuthority3: {XDSServer: xdstestutils.ServerConfigForAddress(t, nonDefaultAuthorityServer.Address)}, + }, + }, defaultTestWatchExpiryTimeout, idleTimeout) + if err != nil { + t.Fatalf("failed to create xds client: %v", err) + } + + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{ + e2e.DefaultCluster(authorityTestResourceName11, edsName, e2e.SecurityLevelNone), + e2e.DefaultCluster(authorityTestResourceName12, edsName, e2e.SecurityLevelNone), + e2e.DefaultCluster(authorityTestResourceName2, edsName, e2e.SecurityLevelNone), + e2e.DefaultCluster(authorityTestResourceName3, edsName, e2e.SecurityLevelNone), + }, + SkipValidation: true, + } + if err := defaultAuthorityServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + return lisDefault, lisNonDefault, client, close +} + +// TestAuthorityShare tests the authority sharing logic. The test verifies the +// following scenarios: +// - A watch for a resource name with an authority matching an existing watch +// should not result in a new transport being created. +// - A watch for a resource name with different authority name but same +// authority config as an existing watch should not result in a new transport +// being created. +func (s) TestAuthorityShare(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + lis, _, client, close := setupForAuthorityTests(ctx, t, time.Duration(0)) + defer close() + + // Verify that no connection is established to the management server at this + // point. A transport is created only when a resource (which belongs to that + // authority) is requested. + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := lis.NewConnCh.Receive(sCtx); err != context.DeadlineExceeded { + t.Fatal("Unexpected new transport created to management server") + } + + // Request the first resource. Verify that a new transport is created. + cdsCancel1 := client.WatchCluster(authorityTestResourceName11, func(u xdsresource.ClusterUpdate, err error) {}) + defer cdsCancel1() + if _, err := lis.NewConnCh.Receive(ctx); err != nil { + t.Fatalf("Timed out when waiting for a new transport to be created to the management server: %v", err) + } + + // Request the second resource. Verify that no new transport is created. + cdsCancel2 := client.WatchCluster(authorityTestResourceName12, func(u xdsresource.ClusterUpdate, err error) {}) + defer cdsCancel2() + sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := lis.NewConnCh.Receive(sCtx); err != context.DeadlineExceeded { + t.Fatal("Unexpected new transport created to management server") + } + + // Request the third resource. Verify that no new transport is created. + cdsCancel3 := client.WatchCluster(authorityTestResourceName2, func(u xdsresource.ClusterUpdate, err error) {}) + defer cdsCancel3() + sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := lis.NewConnCh.Receive(sCtx); err != context.DeadlineExceeded { + t.Fatal("Unexpected new transport created to management server") + } +} + +// TestAuthorityIdle test the authority idle timeout logic. The test verifies +// that the xDS client does not close authorities immediately after the last +// watch is canceled, but waits for the configured idle timeout to expire before +// closing them. +func (s) TestAuthorityIdleTimeout(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + lis, _, client, close := setupForAuthorityTests(ctx, t, defaultTestIdleAuthorityTimeout) + defer close() + + // Request the first resource. Verify that a new transport is created. + cdsCancel1 := client.WatchCluster(authorityTestResourceName11, func(u xdsresource.ClusterUpdate, err error) {}) + val, err := lis.NewConnCh.Receive(ctx) + if err != nil { + t.Fatalf("Timed out when waiting for a new transport to be created to the management server: %v", err) + } + conn := val.(*testutils.ConnWrapper) + + // Request the second resource. Verify that no new transport is created. + cdsCancel2 := client.WatchCluster(authorityTestResourceName12, func(u xdsresource.ClusterUpdate, err error) {}) + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := lis.NewConnCh.Receive(sCtx); err != context.DeadlineExceeded { + t.Fatal("Unexpected new transport created to management server") + } + + // Cancel both watches, and verify that the connection to the management + // server is not closed immediately. + cdsCancel1() + cdsCancel2() + sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := conn.CloseCh.Receive(sCtx); err != context.DeadlineExceeded { + t.Fatal("Connection to management server closed unexpectedly") + } + + // Wait for the authority idle timeout to fire. + time.Sleep(2 * defaultTestIdleAuthorityTimeout) + sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := conn.CloseCh.Receive(sCtx); err != nil { + t.Fatal("Connection to management server not closed after idle timeout expiry") + } +} + +// TestAuthorityClientClose verifies that authorities in use and in the idle +// cache are all closed when the client is closed. +func (s) TestAuthorityClientClose(t *testing.T) { + // Set the authority idle timeout to twice the defaultTestTimeout. This will + // ensure that idle authorities stay in the cache for the duration of this + // test, until explicitly closed. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + lisDefault, lisNonDefault, client, close := setupForAuthorityTests(ctx, t, time.Duration(2*defaultTestTimeout)) + + // Request the first resource. Verify that a new transport is created to the + // default management server. + cdsCancel1 := client.WatchCluster(authorityTestResourceName11, func(u xdsresource.ClusterUpdate, err error) {}) + val, err := lisDefault.NewConnCh.Receive(ctx) + if err != nil { + t.Fatalf("Timed out when waiting for a new transport to be created to the management server: %v", err) + } + connDefault := val.(*testutils.ConnWrapper) + + // Request another resource which is served by the non-default authority. + // Verify that a new transport is created to the non-default management + // server. + client.WatchCluster(authorityTestResourceName3, func(u xdsresource.ClusterUpdate, err error) {}) + val, err = lisNonDefault.NewConnCh.Receive(ctx) + if err != nil { + t.Fatalf("Timed out when waiting for a new transport to be created to the management server: %v", err) + } + connNonDefault := val.(*testutils.ConnWrapper) + + // Cancel the first watch. This should move the default authority to the + // idle cache, but the connection should not be closed yet, because the idle + // timeout would not have fired. + cdsCancel1() + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := connDefault.CloseCh.Receive(sCtx); err != context.DeadlineExceeded { + t.Fatal("Connection to management server closed unexpectedly") + } + + // Closing the xDS client should close the connection to both management + // servers, even though we have an open watch to one of them. + close() + if _, err := connDefault.CloseCh.Receive(ctx); err != nil { + t.Fatal("Connection to management server not closed after client close") + } + if _, err := connNonDefault.CloseCh.Receive(ctx); err != nil { + t.Fatal("Connection to management server not closed after client close") + } +} + +// TestAuthorityRevive verifies that an authority in the idle cache is revived +// when a new watch is started on this authority. +func (s) TestAuthorityRevive(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + lis, _, client, close := setupForAuthorityTests(ctx, t, defaultTestIdleAuthorityTimeout) + defer close() + + // Request the first resource. Verify that a new transport is created. + cdsCancel1 := client.WatchCluster(authorityTestResourceName11, func(u xdsresource.ClusterUpdate, err error) {}) + val, err := lis.NewConnCh.Receive(ctx) + if err != nil { + t.Fatalf("Timed out when waiting for a new transport to be created to the management server: %v", err) + } + conn := val.(*testutils.ConnWrapper) + + // Cancel the above watch. This should move the authority to the idle cache. + cdsCancel1() + + // Request the second resource. Verify that no new transport is created. + // This should move the authority out of the idle cache. + cdsCancel2 := client.WatchCluster(authorityTestResourceName12, func(u xdsresource.ClusterUpdate, err error) {}) + defer cdsCancel2() + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := lis.NewConnCh.Receive(sCtx); err != context.DeadlineExceeded { + t.Fatal("Unexpected new transport created to management server") + } + + // Wait for double the idle timeout, and the connection to the management + // server should not be closed, since it was revived from the idle cache. + time.Sleep(2 * defaultTestIdleAuthorityTimeout) + sCtx, sCancel = context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if _, err := conn.CloseCh.Receive(sCtx); err != context.DeadlineExceeded { + t.Fatal("Connection to management server closed unexpectedly") + } +} diff --git a/xds/internal/xdsclient/tests/cds_watchers_test.go b/xds/internal/xdsclient/tests/cds_watchers_test.go new file mode 100644 index 000000000000..9670caaca0a6 --- /dev/null +++ b/xds/internal/xdsclient/tests/cds_watchers_test.go @@ -0,0 +1,956 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient_test + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/google/uuid" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + xdstestutils "google.golang.org/grpc/xds/internal/testutils" + "google.golang.org/grpc/xds/internal/xdsclient" + "google.golang.org/grpc/xds/internal/xdsclient/bootstrap" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" +) + +// badClusterResource returns a cluster resource for the given name which +// contains a config_source_specifier for the `lrs_server` field which is not +// set to `self`, and hence is expected to be NACKed by the client. +func badClusterResource(clusterName, edsServiceName string, secLevel e2e.SecurityLevel) *v3clusterpb.Cluster { + cluster := e2e.DefaultCluster(clusterName, edsServiceName, secLevel) + cluster.LrsServer = &v3corepb.ConfigSource{ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{}} + return cluster +} + +// xdsClient is expected to produce an error containing this string when an +// update is received containing a cluster created using `badClusterResource`. +const wantClusterNACKErr = "unsupported config_source_specifier" + +// verifyClusterUpdate waits for an update to be received on the provided update +// channel and verifies that it matches the expected update. +// +// Returns an error if no update is received before the context deadline expires +// or the received update does not match the expected one. +func verifyClusterUpdate(ctx context.Context, updateCh *testutils.Channel, wantUpdate xdsresource.ClusterUpdateErrTuple) error { + u, err := updateCh.Receive(ctx) + if err != nil { + return fmt.Errorf("timeout when waiting for a cluster resource from the management server: %v", err) + } + got := u.(xdsresource.ClusterUpdateErrTuple) + if wantUpdate.Err != nil { + if gotType, wantType := xdsresource.ErrType(got.Err), xdsresource.ErrType(wantUpdate.Err); gotType != wantType { + return fmt.Errorf("received update with error type %v, want %v", gotType, wantType) + } + } + cmpOpts := []cmp.Option{cmpopts.EquateEmpty(), cmpopts.IgnoreFields(xdsresource.ClusterUpdate{}, "Raw", "LBPolicy")} + if diff := cmp.Diff(wantUpdate.Update, got.Update, cmpOpts...); diff != "" { + return fmt.Errorf("received unepected diff in the cluster resource update: (-want, got):\n%s", diff) + } + return nil +} + +// verifyNoClusterUpdate verifies that no cluster update is received on the +// provided update channel, and returns an error if an update is received. +// +// A very short deadline is used while waiting for the update, as this function +// is intended to be used when an update is not expected. +func verifyNoClusterUpdate(ctx context.Context, updateCh *testutils.Channel) error { + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if u, err := updateCh.Receive(sCtx); err != context.DeadlineExceeded { + return fmt.Errorf("received unexpected ClusterUpdate when expecting none: %v", u) + } + return nil +} + +// TestCDSWatch covers the case where a single watcher exists for a single +// cluster resource. The test verifies the following scenarios: +// 1. An update from the management server containing the resource being +// watched should result in the invocation of the watch callback. +// 2. An update from the management server containing a resource *not* being +// watched should not result in the invocation of the watch callback. +// 3. After the watch is cancelled, an update from the management server +// containing the resource that was being watched should not result in the +// invocation of the watch callback. +// +// The test is run for old and new style names. +func (s) TestCDSWatch(t *testing.T) { + tests := []struct { + desc string + resourceName string + watchedResource *v3clusterpb.Cluster // The resource being watched. + updatedWatchedResource *v3clusterpb.Cluster // The watched resource after an update. + notWatchedResource *v3clusterpb.Cluster // A resource which is not being watched. + wantUpdate xdsresource.ClusterUpdateErrTuple + }{ + { + desc: "old style resource", + resourceName: cdsName, + watchedResource: e2e.DefaultCluster(cdsName, edsName, e2e.SecurityLevelNone), + updatedWatchedResource: e2e.DefaultCluster(cdsName, "new-eds-resource", e2e.SecurityLevelNone), + notWatchedResource: e2e.DefaultCluster("unsubscribed-cds-resource", edsName, e2e.SecurityLevelNone), + wantUpdate: xdsresource.ClusterUpdateErrTuple{ + Update: xdsresource.ClusterUpdate{ + ClusterName: cdsName, + EDSServiceName: edsName, + }, + }, + }, + { + desc: "new style resource", + resourceName: cdsNameNewStyle, + watchedResource: e2e.DefaultCluster(cdsNameNewStyle, edsNameNewStyle, e2e.SecurityLevelNone), + updatedWatchedResource: e2e.DefaultCluster(cdsNameNewStyle, "new-eds-resource", e2e.SecurityLevelNone), + notWatchedResource: e2e.DefaultCluster("unsubscribed-cds-resource", edsNameNewStyle, e2e.SecurityLevelNone), + wantUpdate: xdsresource.ClusterUpdateErrTuple{ + Update: xdsresource.ClusterUpdate{ + ClusterName: cdsNameNewStyle, + EDSServiceName: edsNameNewStyle, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + overrideFedEnvVar(t) + mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup() + + // Create an xDS client with the above bootstrap contents. + client, close, err := xdsclient.NewWithBootstrapContentsForTesting(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register a watch for a cluster resource and have the watch + // callback push the received update on to a channel. + updateCh := testutils.NewChannel() + cdsCancel := client.WatchCluster(test.resourceName, func(u xdsresource.ClusterUpdate, err error) { + updateCh.Send(xdsresource.ClusterUpdateErrTuple{Update: u, Err: err}) + }) + + // Configure the management server to return a single cluster + // resource, corresponding to the one we registered a watch for. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{test.watchedResource}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update. + if err := verifyClusterUpdate(ctx, updateCh, test.wantUpdate); err != nil { + t.Fatal(err) + } + + // Configure the management server to return an additional cluster + // resource, one that we are not interested in. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{test.watchedResource, test.notWatchedResource}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + if err := verifyNoClusterUpdate(ctx, updateCh); err != nil { + t.Fatal(err) + } + + // Cancel the watch and update the resource corresponding to the original + // watch. Ensure that the cancelled watch callback is not invoked. + cdsCancel() + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{test.updatedWatchedResource, test.notWatchedResource}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + if err := verifyNoClusterUpdate(ctx, updateCh); err != nil { + t.Fatal(err) + } + }) + } +} + +// TestCDSWatch_TwoWatchesForSameResourceName covers the case where two watchers +// exist for a single cluster resource. The test verifies the following +// scenarios: +// 1. An update from the management server containing the resource being +// watched should result in the invocation of both watch callbacks. +// 2. After one of the watches is cancelled, a redundant update from the +// management server should not result in the invocation of either of the +// watch callbacks. +// 3. A new update from the management server containing the resource being +// watched should result in the invocation of the un-cancelled watch +// callback. +// +// The test is run for old and new style names. +func (s) TestCDSWatch_TwoWatchesForSameResourceName(t *testing.T) { + tests := []struct { + desc string + resourceName string + watchedResource *v3clusterpb.Cluster // The resource being watched. + updatedWatchedResource *v3clusterpb.Cluster // The watched resource after an update. + wantUpdateV1 xdsresource.ClusterUpdateErrTuple + wantUpdateV2 xdsresource.ClusterUpdateErrTuple + }{ + { + desc: "old style resource", + resourceName: cdsName, + watchedResource: e2e.DefaultCluster(cdsName, edsName, e2e.SecurityLevelNone), + updatedWatchedResource: e2e.DefaultCluster(cdsName, "new-eds-resource", e2e.SecurityLevelNone), + wantUpdateV1: xdsresource.ClusterUpdateErrTuple{ + Update: xdsresource.ClusterUpdate{ + ClusterName: cdsName, + EDSServiceName: edsName, + }, + }, + wantUpdateV2: xdsresource.ClusterUpdateErrTuple{ + Update: xdsresource.ClusterUpdate{ + ClusterName: cdsName, + EDSServiceName: "new-eds-resource", + }, + }, + }, + { + desc: "new style resource", + resourceName: cdsNameNewStyle, + watchedResource: e2e.DefaultCluster(cdsNameNewStyle, edsNameNewStyle, e2e.SecurityLevelNone), + updatedWatchedResource: e2e.DefaultCluster(cdsNameNewStyle, "new-eds-resource", e2e.SecurityLevelNone), + wantUpdateV1: xdsresource.ClusterUpdateErrTuple{ + Update: xdsresource.ClusterUpdate{ + ClusterName: cdsNameNewStyle, + EDSServiceName: edsNameNewStyle, + }, + }, + wantUpdateV2: xdsresource.ClusterUpdateErrTuple{ + Update: xdsresource.ClusterUpdate{ + ClusterName: cdsNameNewStyle, + EDSServiceName: "new-eds-resource", + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + overrideFedEnvVar(t) + mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup() + + // Create an xDS client with the above bootstrap contents. + client, close, err := xdsclient.NewWithBootstrapContentsForTesting(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register two watches for the same cluster resource and have the + // callbacks push the received updates on to a channel. + updateCh1 := testutils.NewChannel() + cdsCancel1 := client.WatchCluster(test.resourceName, func(u xdsresource.ClusterUpdate, err error) { + updateCh1.Send(xdsresource.ClusterUpdateErrTuple{Update: u, Err: err}) + }) + defer cdsCancel1() + updateCh2 := testutils.NewChannel() + cdsCancel2 := client.WatchCluster(test.resourceName, func(u xdsresource.ClusterUpdate, err error) { + updateCh2.Send(xdsresource.ClusterUpdateErrTuple{Update: u, Err: err}) + }) + + // Configure the management server to return a single cluster + // resource, corresponding to the one we registered watches for. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{test.watchedResource}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update. + if err := verifyClusterUpdate(ctx, updateCh1, test.wantUpdateV1); err != nil { + t.Fatal(err) + } + if err := verifyClusterUpdate(ctx, updateCh2, test.wantUpdateV1); err != nil { + t.Fatal(err) + } + + // Cancel the second watch and force the management server to push a + // redundant update for the resource being watched. Neither of the + // two watch callbacks should be invoked. + cdsCancel2() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + if err := verifyNoClusterUpdate(ctx, updateCh1); err != nil { + t.Fatal(err) + } + if err := verifyNoClusterUpdate(ctx, updateCh2); err != nil { + t.Fatal(err) + } + + // Update to the resource being watched. The un-cancelled callback + // should be invoked while the cancelled one should not be. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{test.updatedWatchedResource}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + if err := verifyClusterUpdate(ctx, updateCh1, test.wantUpdateV2); err != nil { + t.Fatal(err) + } + if err := verifyNoClusterUpdate(ctx, updateCh2); err != nil { + t.Fatal(err) + } + }) + } +} + +// TestCDSWatch_ThreeWatchesForDifferentResourceNames covers the case where +// three watchers (two watchers for one resource, and the third watcher for +// another resource) exist across two cluster resources (one with an old style +// name and one with a new style name). The test verifies that an update from +// the management server containing both resources results in the invocation of +// all watch callbacks. +func (s) TestCDSWatch_ThreeWatchesForDifferentResourceNames(t *testing.T) { + overrideFedEnvVar(t) + mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup() + + // Create an xDS client with the above bootstrap contents. + client, close, err := xdsclient.NewWithBootstrapContentsForTesting(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register two watches for the same cluster resource and have the + // callbacks push the received updates on to a channel. + updateCh1 := testutils.NewChannel() + cdsCancel1 := client.WatchCluster(cdsName, func(u xdsresource.ClusterUpdate, err error) { + updateCh1.Send(xdsresource.ClusterUpdateErrTuple{Update: u, Err: err}) + }) + defer cdsCancel1() + updateCh2 := testutils.NewChannel() + cdsCancel2 := client.WatchCluster(cdsName, func(u xdsresource.ClusterUpdate, err error) { + updateCh2.Send(xdsresource.ClusterUpdateErrTuple{Update: u, Err: err}) + }) + defer cdsCancel2() + + // Register the third watch for a different cluster resource, and push the + // received updates onto a channel. + updateCh3 := testutils.NewChannel() + cdsCancel3 := client.WatchCluster(cdsNameNewStyle, func(u xdsresource.ClusterUpdate, err error) { + updateCh3.Send(xdsresource.ClusterUpdateErrTuple{Update: u, Err: err}) + }) + defer cdsCancel3() + + // Configure the management server to return two cluster resources, + // corresponding to the registered watches. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{ + e2e.DefaultCluster(cdsName, edsName, e2e.SecurityLevelNone), + e2e.DefaultCluster(cdsNameNewStyle, edsNameNewStyle, e2e.SecurityLevelNone), + }, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update for the all watchers. + wantUpdate12 := xdsresource.ClusterUpdateErrTuple{ + Update: xdsresource.ClusterUpdate{ + ClusterName: cdsName, + EDSServiceName: edsName, + }, + } + wantUpdate3 := xdsresource.ClusterUpdateErrTuple{ + Update: xdsresource.ClusterUpdate{ + ClusterName: cdsNameNewStyle, + EDSServiceName: edsNameNewStyle, + }, + } + if err := verifyClusterUpdate(ctx, updateCh1, wantUpdate12); err != nil { + t.Fatal(err) + } + if err := verifyClusterUpdate(ctx, updateCh2, wantUpdate12); err != nil { + t.Fatal(err) + } + if err := verifyClusterUpdate(ctx, updateCh3, wantUpdate3); err != nil { + t.Fatal(err) + } +} + +// TestCDSWatch_ResourceCaching covers the case where a watch is registered for +// a resource which is already present in the cache. The test verifies that the +// watch callback is invoked with the contents from the cache, instead of a +// request being sent to the management server. +func (s) TestCDSWatch_ResourceCaching(t *testing.T) { + overrideFedEnvVar(t) + firstRequestReceived := false + firstAckReceived := grpcsync.NewEvent() + secondRequestReceived := grpcsync.NewEvent() + + mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{ + OnStreamRequest: func(id int64, req *v3discoverypb.DiscoveryRequest) error { + // The first request has an empty version string. + if !firstRequestReceived && req.GetVersionInfo() == "" { + firstRequestReceived = true + return nil + } + // The first ack has a non-empty version string. + if !firstAckReceived.HasFired() && req.GetVersionInfo() != "" { + firstAckReceived.Fire() + return nil + } + // Any requests after the first request and ack, are not expected. + secondRequestReceived.Fire() + return nil + }, + }) + defer cleanup() + + // Create an xDS client with the above bootstrap contents. + client, close, err := xdsclient.NewWithBootstrapContentsForTesting(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register a watch for a cluster resource and have the watch + // callback push the received update on to a channel. + updateCh1 := testutils.NewChannel() + cdsCancel1 := client.WatchCluster(cdsName, func(u xdsresource.ClusterUpdate, err error) { + updateCh1.Send(xdsresource.ClusterUpdateErrTuple{Update: u, Err: err}) + }) + defer cdsCancel1() + + // Configure the management server to return a single cluster + // resource, corresponding to the one we registered a watch for. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(cdsName, edsName, e2e.SecurityLevelNone)}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update. + wantUpdate := xdsresource.ClusterUpdateErrTuple{ + Update: xdsresource.ClusterUpdate{ + ClusterName: cdsName, + EDSServiceName: edsName, + }, + } + if err := verifyClusterUpdate(ctx, updateCh1, wantUpdate); err != nil { + t.Fatal(err) + } + select { + case <-ctx.Done(): + t.Fatal("timeout when waiting for receipt of ACK at the management server") + case <-firstAckReceived.Done(): + } + + // Register another watch for the same resource. This should get the update + // from the cache. + updateCh2 := testutils.NewChannel() + cdsCancel2 := client.WatchCluster(cdsName, func(u xdsresource.ClusterUpdate, err error) { + updateCh2.Send(xdsresource.ClusterUpdateErrTuple{Update: u, Err: err}) + }) + defer cdsCancel2() + if err := verifyClusterUpdate(ctx, updateCh2, wantUpdate); err != nil { + t.Fatal(err) + } + // No request should get sent out as part of this watch. + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + select { + case <-sCtx.Done(): + case <-secondRequestReceived.Done(): + t.Fatal("xdsClient sent out request instead of using update from cache") + } +} + +// TestCDSWatch_ExpiryTimerFiresBeforeResponse tests the case where the client +// does not receive an CDS response for the request that it sends. The test +// verifies that the watch callback is invoked with an error once the +// watchExpiryTimer fires. +func (s) TestCDSWatch_ExpiryTimerFiresBeforeResponse(t *testing.T) { + overrideFedEnvVar(t) + mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) + if err != nil { + t.Fatalf("Failed to spin up the xDS management server: %v", err) + } + defer mgmtServer.Stop() + + client, close, err := xdsclient.NewWithConfigForTesting(&bootstrap.Config{ + XDSServer: xdstestutils.ServerConfigForAddress(t, mgmtServer.Address), + NodeProto: &v3corepb.Node{}, + }, defaultTestWatchExpiryTimeout, time.Duration(0)) + if err != nil { + t.Fatalf("failed to create xds client: %v", err) + } + defer close() + + // Register a watch for a resource which is expected to be invoked with an + // error after the watch expiry timer fires. + updateCh := testutils.NewChannel() + cdsCancel := client.WatchCluster(cdsName, func(u xdsresource.ClusterUpdate, err error) { + updateCh.Send(xdsresource.ClusterUpdateErrTuple{Update: u, Err: err}) + }) + defer cdsCancel() + + // Wait for the watch expiry timer to fire. + <-time.After(defaultTestWatchExpiryTimeout) + + // Verify that an empty update with the expected error is received. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + wantErr := xdsresource.NewErrorf(xdsresource.ErrorTypeResourceNotFound, "") + if err := verifyClusterUpdate(ctx, updateCh, xdsresource.ClusterUpdateErrTuple{Err: wantErr}); err != nil { + t.Fatal(err) + } +} + +// TestCDSWatch_ValidResponseCancelsExpiryTimerBehavior tests the case where the +// client receives a valid LDS response for the request that it sends. The test +// verifies that the behavior associated with the expiry timer (i.e, callback +// invocation with error) does not take place. +func (s) TestCDSWatch_ValidResponseCancelsExpiryTimerBehavior(t *testing.T) { + overrideFedEnvVar(t) + mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) + if err != nil { + t.Fatalf("Failed to spin up the xDS management server: %v", err) + } + defer mgmtServer.Stop() + + // Create an xDS client talking to the above management server. + nodeID := uuid.New().String() + client, close, err := xdsclient.NewWithConfigForTesting(&bootstrap.Config{ + XDSServer: xdstestutils.ServerConfigForAddress(t, mgmtServer.Address), + NodeProto: &v3corepb.Node{Id: nodeID}, + }, defaultTestWatchExpiryTimeout, time.Duration(0)) + if err != nil { + t.Fatalf("failed to create xds client: %v", err) + } + defer close() + + // Register a watch for a cluster resource and have the watch + // callback push the received update on to a channel. + updateCh := testutils.NewChannel() + cdsCancel := client.WatchCluster(cdsName, func(u xdsresource.ClusterUpdate, err error) { + updateCh.Send(xdsresource.ClusterUpdateErrTuple{Update: u, Err: err}) + }) + defer cdsCancel() + + // Configure the management server to return a single cluster resource, + // corresponding to the one we registered a watch for. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(cdsName, edsName, e2e.SecurityLevelNone)}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update. + wantUpdate := xdsresource.ClusterUpdateErrTuple{ + Update: xdsresource.ClusterUpdate{ + ClusterName: cdsName, + EDSServiceName: edsName, + }, + } + if err := verifyClusterUpdate(ctx, updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + + // Wait for the watch expiry timer to fire, and verify that the callback is + // not invoked. + <-time.After(defaultTestWatchExpiryTimeout) + if err := verifyNoClusterUpdate(ctx, updateCh); err != nil { + t.Fatal(err) + } +} + +// TestCDSWatch_ResourceRemoved covers the cases where two watchers exists for +// two different resources (one with an old style name and one with a new style +// name). One of these resources being watched is removed from the management +// server. The test verifies the following scenarios: +// 1. Removing a resource should trigger the watch callback associated with that +// resource with a resource removed error. It should not trigger the watch +// callback for an unrelated resource. +// 2. An update to other resource should result in the invocation of the watch +// callback associated with that resource. It should not result in the +// invocation of the watch callback associated with the deleted resource. +func (s) TesCDSWatch_ResourceRemoved(t *testing.T) { + overrideFedEnvVar(t) + mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup() + + // Create an xDS client with the above bootstrap contents. + client, close, err := xdsclient.NewWithBootstrapContentsForTesting(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register two watches for two cluster resources and have the + // callbacks push the received updates on to a channel. + resourceName1 := cdsName + updateCh1 := testutils.NewChannel() + cdsCancel1 := client.WatchCluster(resourceName1, func(u xdsresource.ClusterUpdate, err error) { + updateCh1.Send(xdsresource.ClusterUpdateErrTuple{Update: u, Err: err}) + }) + defer cdsCancel1() + resourceName2 := cdsNameNewStyle + updateCh2 := testutils.NewChannel() + cdsCancel2 := client.WatchCluster(resourceName2, func(u xdsresource.ClusterUpdate, err error) { + updateCh2.Send(xdsresource.ClusterUpdateErrTuple{Update: u, Err: err}) + }) + defer cdsCancel2() + + // Configure the management server to return two cluster resources, + // corresponding to the registered watches. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{ + e2e.DefaultCluster(resourceName1, edsName, e2e.SecurityLevelNone), + e2e.DefaultCluster(resourceName2, edsNameNewStyle, e2e.SecurityLevelNone), + }, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update for both watchers. + wantUpdate1 := xdsresource.ClusterUpdateErrTuple{ + Update: xdsresource.ClusterUpdate{ + ClusterName: resourceName1, + EDSServiceName: edsName, + }, + } + wantUpdate2 := xdsresource.ClusterUpdateErrTuple{ + Update: xdsresource.ClusterUpdate{ + ClusterName: resourceName2, + EDSServiceName: edsNameNewStyle, + }, + } + if err := verifyClusterUpdate(ctx, updateCh1, wantUpdate1); err != nil { + t.Fatal(err) + } + if err := verifyClusterUpdate(ctx, updateCh2, wantUpdate2); err != nil { + t.Fatal(err) + } + + // Remove the first cluster resource on the management server. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(resourceName2, edsNameNewStyle, e2e.SecurityLevelNone)}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // The first watcher should receive a resource removed error, while the + // second watcher should not receive an update. + if err := verifyClusterUpdate(ctx, updateCh1, xdsresource.ClusterUpdateErrTuple{Err: xdsresource.NewErrorf(xdsresource.ErrorTypeResourceNotFound, "")}); err != nil { + t.Fatal(err) + } + if err := verifyNoClusterUpdate(ctx, updateCh2); err != nil { + t.Fatal(err) + } + + // Update the second cluster resource on the management server. The first + // watcher should not receive an update, while the second watcher should. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(resourceName2, "new-eds-resource", e2e.SecurityLevelNone)}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + if err := verifyNoClusterUpdate(ctx, updateCh1); err != nil { + t.Fatal(err) + } + wantUpdate := xdsresource.ClusterUpdateErrTuple{ + Update: xdsresource.ClusterUpdate{ + ClusterName: resourceName2, + EDSServiceName: "new-eds-resource", + }, + } + if err := verifyClusterUpdate(ctx, updateCh2, wantUpdate); err != nil { + t.Fatal(err) + } +} + +// TestCDSWatch_NACKError covers the case where an update from the management +// server is NACK'ed by the xdsclient. The test verifies that the error is +// propagated to the watcher. +func (s) TestCDSWatch_NACKError(t *testing.T) { + overrideFedEnvVar(t) + mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup() + + // Create an xDS client with the above bootstrap contents. + client, close, err := xdsclient.NewWithBootstrapContentsForTesting(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register a watch for a cluster resource and have the watch + // callback push the received update on to a channel. + updateCh := testutils.NewChannel() + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + cdsCancel := client.WatchCluster(cdsName, func(u xdsresource.ClusterUpdate, err error) { + updateCh.SendContext(ctx, xdsresource.ClusterUpdateErrTuple{Update: u, Err: err}) + }) + defer cdsCancel() + + // Configure the management server to return a single cluster resource + // which is expected to be NACK'ed by the client. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{badClusterResource(cdsName, edsName, e2e.SecurityLevelNone)}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify that the expected error is propagated to the watcher. + u, err := updateCh.Receive(ctx) + if err != nil { + t.Fatalf("timeout when waiting for a cluster resource from the management server: %v", err) + } + gotErr := u.(xdsresource.ClusterUpdateErrTuple).Err + if gotErr == nil || !strings.Contains(gotErr.Error(), wantClusterNACKErr) { + t.Fatalf("update received with error: %v, want %q", gotErr, wantClusterNACKErr) + } +} + +// TestCDSWatch_PartialValid covers the case where a response from the +// management server contains both valid and invalid resources and is expected +// to be NACK'ed by the xdsclient. The test verifies that watchers corresponding +// to the valid resource receive the update, while watchers corresponding to the +// invalid resource receive an error. +func (s) TestCDSWatch_PartialValid(t *testing.T) { + overrideFedEnvVar(t) + mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup() + + // Create an xDS client with the above bootstrap contents. + client, close, err := xdsclient.NewWithBootstrapContentsForTesting(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register two watches for cluster resources. The first watch is expected + // to receive an error because the received resource is NACK'ed. The second + // watch is expected to get a good update. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + badResourceName := cdsName + updateCh1 := testutils.NewChannel() + cdsCancel1 := client.WatchCluster(badResourceName, func(u xdsresource.ClusterUpdate, err error) { + updateCh1.SendContext(ctx, xdsresource.ClusterUpdateErrTuple{Update: u, Err: err}) + }) + defer cdsCancel1() + goodResourceName := cdsNameNewStyle + updateCh2 := testutils.NewChannel() + cdsCancel2 := client.WatchCluster(goodResourceName, func(u xdsresource.ClusterUpdate, err error) { + updateCh2.SendContext(ctx, xdsresource.ClusterUpdateErrTuple{Update: u, Err: err}) + }) + defer cdsCancel2() + + // Configure the management server with two cluster resources. One of these + // is a bad resource causing the update to be NACKed. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{ + badClusterResource(badResourceName, edsName, e2e.SecurityLevelNone), + e2e.DefaultCluster(goodResourceName, edsName, e2e.SecurityLevelNone)}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify that the expected error is propagated to the watcher which is + // watching the bad resource. + u, err := updateCh1.Receive(ctx) + if err != nil { + t.Fatalf("timeout when waiting for a cluster resource from the management server: %v", err) + } + gotErr := u.(xdsresource.ClusterUpdateErrTuple).Err + if gotErr == nil || !strings.Contains(gotErr.Error(), wantClusterNACKErr) { + t.Fatalf("update received with error: %v, want %q", gotErr, wantClusterNACKErr) + } + + // Verify that the watcher watching the good resource receives a good + // update. + wantUpdate := xdsresource.ClusterUpdateErrTuple{ + Update: xdsresource.ClusterUpdate{ + ClusterName: goodResourceName, + EDSServiceName: edsName, + }, + } + if err := verifyClusterUpdate(ctx, updateCh2, wantUpdate); err != nil { + t.Fatal(err) + } +} + +// TestCDSWatch_PartialResponse covers the case where a response from the +// management server does not contain all requested resources. CDS responses are +// supposed to contain all requested resources, and the absence of one usually +// indicates that the management server does not know about it. In cases where +// the server has never responded with this resource before, the xDS client is +// expected to wait for the watch timeout to expire before concluding that the +// resource does not exist on the server +func (s) TestCDSWatch_PartialResponse(t *testing.T) { + overrideFedEnvVar(t) + mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup() + + // Create an xDS client with the above bootstrap contents. + client, close, err := xdsclient.NewWithBootstrapContentsForTesting(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register two watches for two cluster resources and have the + // callbacks push the received updates on to a channel. + resourceName1 := cdsName + updateCh1 := testutils.NewChannel() + cdsCancel1 := client.WatchCluster(resourceName1, func(u xdsresource.ClusterUpdate, err error) { + updateCh1.Send(xdsresource.ClusterUpdateErrTuple{Update: u, Err: err}) + }) + defer cdsCancel1() + resourceName2 := cdsNameNewStyle + updateCh2 := testutils.NewChannel() + cdsCancel2 := client.WatchCluster(resourceName2, func(u xdsresource.ClusterUpdate, err error) { + updateCh2.Send(xdsresource.ClusterUpdateErrTuple{Update: u, Err: err}) + }) + defer cdsCancel2() + + // Configure the management server to return only one of the two cluster + // resources, corresponding to the registered watches. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(resourceName1, edsName, e2e.SecurityLevelNone)}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update for first watcher. + wantUpdate1 := xdsresource.ClusterUpdateErrTuple{ + Update: xdsresource.ClusterUpdate{ + ClusterName: resourceName1, + EDSServiceName: edsName, + }, + } + if err := verifyClusterUpdate(ctx, updateCh1, wantUpdate1); err != nil { + t.Fatal(err) + } + + // Verify that the second watcher does not get an update with an error. + if err := verifyNoClusterUpdate(ctx, updateCh2); err != nil { + t.Fatal(err) + } + + // Configure the management server to return two cluster resources, + // corresponding to the registered watches. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{ + e2e.DefaultCluster(resourceName1, edsName, e2e.SecurityLevelNone), + e2e.DefaultCluster(resourceName2, edsNameNewStyle, e2e.SecurityLevelNone), + }, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update for the second watcher. + wantUpdate2 := xdsresource.ClusterUpdateErrTuple{ + Update: xdsresource.ClusterUpdate{ + ClusterName: resourceName2, + EDSServiceName: edsNameNewStyle, + }, + } + if err := verifyClusterUpdate(ctx, updateCh2, wantUpdate2); err != nil { + t.Fatal(err) + } + + // Verify that the first watcher gets no update, as the first resource did + // not change. + if err := verifyNoClusterUpdate(ctx, updateCh1); err != nil { + t.Fatal(err) + } +} diff --git a/xds/internal/xdsclient/tests/dump_test.go b/xds/internal/xdsclient/tests/dump_test.go new file mode 100644 index 000000000000..5f2c5e05e4dd --- /dev/null +++ b/xds/internal/xdsclient/tests/dump_test.go @@ -0,0 +1,261 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient_test + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/xds/internal/xdsclient" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" + "google.golang.org/protobuf/testing/protocmp" + "google.golang.org/protobuf/types/known/anypb" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" +) + +func compareDump(ctx context.Context, client xdsclient.XDSClient, want map[string]map[string]xdsresource.UpdateWithMD) error { + var lastErr error + for { + if err := ctx.Err(); err != nil { + return fmt.Errorf("Timeout when waiting for expected dump: %v", lastErr) + } + cmpOpts := cmp.Options{ + cmpopts.EquateEmpty(), + cmp.Comparer(func(a, b time.Time) bool { return true }), + cmpopts.EquateErrors(), + protocmp.Transform(), + } + diff := cmp.Diff(want, client.DumpResources(), cmpOpts) + if diff == "" { + return nil + } + lastErr = fmt.Errorf("DumpResources() returned unexpected dump, diff (-want +got):\n%s", diff) + time.Sleep(100 * time.Millisecond) + } +} + +type noopEndpointsWatcher struct{} + +func (noopEndpointsWatcher) OnUpdate(update *xdsresource.EndpointsResourceData) {} +func (noopEndpointsWatcher) OnError(err error) {} +func (noopEndpointsWatcher) OnResourceDoesNotExist() {} + +func (s) TestDumpResources(t *testing.T) { + // Initialize the xDS resources to be used in this test. + ldsTargets := []string{"lds.target.good:0000", "lds.target.good:1111"} + rdsTargets := []string{"route-config-0", "route-config-1"} + cdsTargets := []string{"cluster-0", "cluster-1"} + edsTargets := []string{"endpoints-0", "endpoints-1"} + listeners := make([]*v3listenerpb.Listener, len(ldsTargets)) + listenerAnys := make([]*anypb.Any, len(ldsTargets)) + for i := range ldsTargets { + listeners[i] = e2e.DefaultClientListener(ldsTargets[i], rdsTargets[i]) + listenerAnys[i] = testutils.MarshalAny(listeners[i]) + } + routes := make([]*v3routepb.RouteConfiguration, len(rdsTargets)) + routeAnys := make([]*anypb.Any, len(rdsTargets)) + for i := range rdsTargets { + routes[i] = e2e.DefaultRouteConfig(rdsTargets[i], ldsTargets[i], cdsTargets[i]) + routeAnys[i] = testutils.MarshalAny(routes[i]) + } + clusters := make([]*v3clusterpb.Cluster, len(cdsTargets)) + clusterAnys := make([]*anypb.Any, len(cdsTargets)) + for i := range cdsTargets { + clusters[i] = e2e.DefaultCluster(cdsTargets[i], edsTargets[i], e2e.SecurityLevelNone) + clusterAnys[i] = testutils.MarshalAny(clusters[i]) + } + endpoints := make([]*v3endpointpb.ClusterLoadAssignment, len(edsTargets)) + endpointAnys := make([]*anypb.Any, len(edsTargets)) + ips := []string{"0.0.0.0", "1.1.1.1"} + ports := []uint32{123, 456} + for i := range edsTargets { + endpoints[i] = e2e.DefaultEndpoint(edsTargets[i], ips[i], ports[i:i+1]) + endpointAnys[i] = testutils.MarshalAny(endpoints[i]) + } + + // Spin up an xDS management server on a local port. + mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup() + + // Create an xDS client with the above bootstrap contents. + client, close, err := xdsclient.NewWithBootstrapContentsForTesting(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Dump resources and expect empty configs. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := compareDump(ctx, client, nil); err != nil { + t.Fatal(err) + } + + // Register watches, dump resources and expect configs in requested state. + for _, target := range ldsTargets { + client.WatchListener(target, func(xdsresource.ListenerUpdate, error) {}) + } + for _, target := range rdsTargets { + client.WatchRouteConfig(target, func(xdsresource.RouteConfigUpdate, error) {}) + } + for _, target := range cdsTargets { + client.WatchCluster(target, func(xdsresource.ClusterUpdate, error) {}) + } + for _, target := range edsTargets { + xdsresource.WatchEndpoints(client, target, noopEndpointsWatcher{}) + } + want := map[string]map[string]xdsresource.UpdateWithMD{ + "type.googleapis.com/envoy.config.listener.v3.Listener": { + ldsTargets[0]: {MD: xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusRequested}}, + ldsTargets[1]: {MD: xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusRequested}}, + }, + "type.googleapis.com/envoy.config.route.v3.RouteConfiguration": { + rdsTargets[0]: {MD: xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusRequested}}, + rdsTargets[1]: {MD: xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusRequested}}, + }, + "type.googleapis.com/envoy.config.cluster.v3.Cluster": { + cdsTargets[0]: {MD: xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusRequested}}, + cdsTargets[1]: {MD: xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusRequested}}, + }, + "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment": { + edsTargets[0]: {MD: xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusRequested}}, + edsTargets[1]: {MD: xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusRequested}}, + }, + } + if err := compareDump(ctx, client, want); err != nil { + t.Fatal(err) + } + + // Configure the resources on the management server. + if err := mgmtServer.Update(ctx, e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: listeners, + Routes: routes, + Clusters: clusters, + Endpoints: endpoints, + }); err != nil { + t.Fatal(err) + } + + // Dump resources and expect ACK configs. + want = map[string]map[string]xdsresource.UpdateWithMD{ + "type.googleapis.com/envoy.config.listener.v3.Listener": { + ldsTargets[0]: {Raw: listenerAnys[0], MD: xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusACKed, Version: "1"}}, + ldsTargets[1]: {Raw: listenerAnys[1], MD: xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusACKed, Version: "1"}}, + }, + "type.googleapis.com/envoy.config.route.v3.RouteConfiguration": { + rdsTargets[0]: {Raw: routeAnys[0], MD: xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusACKed, Version: "1"}}, + rdsTargets[1]: {Raw: routeAnys[1], MD: xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusACKed, Version: "1"}}, + }, + "type.googleapis.com/envoy.config.cluster.v3.Cluster": { + cdsTargets[0]: {Raw: clusterAnys[0], MD: xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusACKed, Version: "1"}}, + cdsTargets[1]: {Raw: clusterAnys[1], MD: xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusACKed, Version: "1"}}, + }, + "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment": { + edsTargets[0]: {Raw: endpointAnys[0], MD: xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusACKed, Version: "1"}}, + edsTargets[1]: {Raw: endpointAnys[1], MD: xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusACKed, Version: "1"}}, + }, + } + if err := compareDump(ctx, client, want); err != nil { + t.Fatal(err) + } + + // Update the first resource of each type in the management server to a + // value which is expected to be NACK'ed by the xDS client. + const nackResourceIdx = 0 + listeners[nackResourceIdx].ApiListener = &v3listenerpb.ApiListener{} + routes[nackResourceIdx].VirtualHosts = []*v3routepb.VirtualHost{{Routes: []*v3routepb.Route{{}}}} + clusters[nackResourceIdx].ClusterDiscoveryType = &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_STATIC} + endpoints[nackResourceIdx].Endpoints = []*v3endpointpb.LocalityLbEndpoints{{}} + if err := mgmtServer.Update(ctx, e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: listeners, + Routes: routes, + Clusters: clusters, + Endpoints: endpoints, + SkipValidation: true, + }); err != nil { + t.Fatal(err) + } + + // Verify that the xDS client reports the first resource of each type as + // being in "NACKed" state, and the second resource of each type to be in + // "ACKed" state. The version for the ACKed resource would be "2", while + // that for the NACKed resource would be "1". In the NACKed resource, the + // version which is NACKed is stored in the ErrorState field. + want = map[string]map[string]xdsresource.UpdateWithMD{ + "type.googleapis.com/envoy.config.listener.v3.Listener": { + ldsTargets[0]: { + Raw: listenerAnys[0], + MD: xdsresource.UpdateMetadata{ + Status: xdsresource.ServiceStatusNACKed, + Version: "1", + ErrState: &xdsresource.UpdateErrorMetadata{Version: "2", Err: cmpopts.AnyError}, + }, + }, + ldsTargets[1]: {Raw: listenerAnys[1], MD: xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusACKed, Version: "2"}}, + }, + "type.googleapis.com/envoy.config.route.v3.RouteConfiguration": { + rdsTargets[0]: { + Raw: routeAnys[0], + MD: xdsresource.UpdateMetadata{ + Status: xdsresource.ServiceStatusNACKed, + Version: "1", + ErrState: &xdsresource.UpdateErrorMetadata{Version: "2", Err: cmpopts.AnyError}, + }, + }, + rdsTargets[1]: {Raw: routeAnys[1], MD: xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusACKed, Version: "2"}}, + }, + "type.googleapis.com/envoy.config.cluster.v3.Cluster": { + cdsTargets[0]: { + Raw: clusterAnys[0], + MD: xdsresource.UpdateMetadata{ + Status: xdsresource.ServiceStatusNACKed, + Version: "1", + ErrState: &xdsresource.UpdateErrorMetadata{Version: "2", Err: cmpopts.AnyError}, + }, + }, + cdsTargets[1]: {Raw: clusterAnys[1], MD: xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusACKed, Version: "2"}}, + }, + "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment": { + edsTargets[0]: { + Raw: endpointAnys[0], + MD: xdsresource.UpdateMetadata{ + Status: xdsresource.ServiceStatusNACKed, + Version: "1", + ErrState: &xdsresource.UpdateErrorMetadata{Version: "2", Err: cmpopts.AnyError}, + }, + }, + edsTargets[1]: {Raw: endpointAnys[1], MD: xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusACKed, Version: "2"}}, + }, + } + if err := compareDump(ctx, client, want); err != nil { + t.Fatal(err) + } +} diff --git a/xds/internal/xdsclient/tests/eds_watchers_test.go b/xds/internal/xdsclient/tests/eds_watchers_test.go new file mode 100644 index 000000000000..0d81c3848c8d --- /dev/null +++ b/xds/internal/xdsclient/tests/eds_watchers_test.go @@ -0,0 +1,853 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient_test + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/google/uuid" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/xds/internal" + xdstestutils "google.golang.org/grpc/xds/internal/testutils" + "google.golang.org/grpc/xds/internal/xdsclient" + "google.golang.org/grpc/xds/internal/xdsclient/bootstrap" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" + "google.golang.org/protobuf/types/known/wrapperspb" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" +) + +const ( + edsHost1 = "1.foo.bar.com" + edsHost2 = "2.foo.bar.com" + edsHost3 = "3.foo.bar.com" + edsPort1 = 1 + edsPort2 = 2 + edsPort3 = 3 +) + +type endpointsUpdateErrTuple struct { + update xdsresource.EndpointsUpdate + err error +} + +type endpointsWatcher struct { + updateCh *testutils.Channel +} + +func newEndpointsWatcher() *endpointsWatcher { + return &endpointsWatcher{updateCh: testutils.NewChannel()} +} + +func (ew *endpointsWatcher) OnUpdate(update *xdsresource.EndpointsResourceData) { + ew.updateCh.Send(endpointsUpdateErrTuple{update: update.Resource}) +} + +func (ew *endpointsWatcher) OnError(err error) { + ew.updateCh.SendOrFail(endpointsUpdateErrTuple{err: err}) +} + +func (ew *endpointsWatcher) OnResourceDoesNotExist() { + ew.updateCh.SendOrFail(endpointsUpdateErrTuple{err: xdsresource.NewErrorf(xdsresource.ErrorTypeResourceNotFound, "Endpoints not found in received response")}) +} + +// badEndpointsResource returns a endpoints resource for the given +// edsServiceName which contains an endpoint with a load_balancing weight of +// `0`. This is expected to be NACK'ed by the xDS client. +func badEndpointsResource(edsServiceName string, host string, ports []uint32) *v3endpointpb.ClusterLoadAssignment { + e := e2e.DefaultEndpoint(edsServiceName, host, ports) + e.Endpoints[0].LbEndpoints[0].LoadBalancingWeight = &wrapperspb.UInt32Value{Value: 0} + return e +} + +// xdsClient is expected to produce an error containing this string when an +// update is received containing an endpoints resource created using +// `badEndpointsResource`. +const wantEndpointsNACKErr = "EDS response contains an endpoint with zero weight" + +// verifyEndpointsUpdate waits for an update to be received on the provided +// update channel and verifies that it matches the expected update. +// +// Returns an error if no update is received before the context deadline expires +// or the received update does not match the expected one. +func verifyEndpointsUpdate(ctx context.Context, updateCh *testutils.Channel, wantUpdate endpointsUpdateErrTuple) error { + u, err := updateCh.Receive(ctx) + if err != nil { + return fmt.Errorf("timeout when waiting for a endpoints resource from the management server: %v", err) + } + got := u.(endpointsUpdateErrTuple) + if wantUpdate.err != nil { + if gotType, wantType := xdsresource.ErrType(got.err), xdsresource.ErrType(wantUpdate.err); gotType != wantType { + return fmt.Errorf("received update with error type %v, want %v", gotType, wantType) + } + } + cmpOpts := []cmp.Option{cmpopts.EquateEmpty(), cmpopts.IgnoreFields(xdsresource.EndpointsUpdate{}, "Raw")} + if diff := cmp.Diff(wantUpdate.update, got.update, cmpOpts...); diff != "" { + return fmt.Errorf("received unepected diff in the endpoints resource update: (-want, got):\n%s", diff) + } + return nil +} + +// verifyNoEndpointsUpdate verifies that no endpoints update is received on the +// provided update channel, and returns an error if an update is received. +// +// A very short deadline is used while waiting for the update, as this function +// is intended to be used when an update is not expected. +func verifyNoEndpointsUpdate(ctx context.Context, updateCh *testutils.Channel) error { + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if u, err := updateCh.Receive(sCtx); err != context.DeadlineExceeded { + return fmt.Errorf("unexpected EndpointsUpdate: %v", u) + } + return nil +} + +// TestEDSWatch covers the case where a single endpoint exists for a single +// endpoints resource. The test verifies the following scenarios: +// 1. An update from the management server containing the resource being +// watched should result in the invocation of the watch callback. +// 2. An update from the management server containing a resource *not* being +// watched should not result in the invocation of the watch callback. +// 3. After the watch is cancelled, an update from the management server +// containing the resource that was being watched should not result in the +// invocation of the watch callback. +// +// The test is run for old and new style names. +func (s) TestEDSWatch(t *testing.T) { + tests := []struct { + desc string + resourceName string + watchedResource *v3endpointpb.ClusterLoadAssignment // The resource being watched. + updatedWatchedResource *v3endpointpb.ClusterLoadAssignment // The watched resource after an update. + notWatchedResource *v3endpointpb.ClusterLoadAssignment // A resource which is not being watched. + wantUpdate endpointsUpdateErrTuple + }{ + { + desc: "old style resource", + resourceName: edsName, + watchedResource: e2e.DefaultEndpoint(edsName, edsHost1, []uint32{edsPort1}), + updatedWatchedResource: e2e.DefaultEndpoint(edsName, edsHost2, []uint32{edsPort2}), + notWatchedResource: e2e.DefaultEndpoint("unsubscribed-eds-resource", edsHost3, []uint32{edsPort3}), + wantUpdate: endpointsUpdateErrTuple{ + update: xdsresource.EndpointsUpdate{ + Localities: []xdsresource.Locality{ + { + Endpoints: []xdsresource.Endpoint{{Address: fmt.Sprintf("%s:%d", edsHost1, edsPort1), Weight: 1}}, + ID: internal.LocalityID{ + Region: "region-1", + Zone: "zone-1", + SubZone: "subzone-1", + }, + Priority: 0, + Weight: 1, + }, + }, + }, + }, + }, + { + desc: "new style resource", + resourceName: edsNameNewStyle, + watchedResource: e2e.DefaultEndpoint(edsNameNewStyle, edsHost1, []uint32{edsPort1}), + updatedWatchedResource: e2e.DefaultEndpoint(edsNameNewStyle, edsHost2, []uint32{edsPort2}), + notWatchedResource: e2e.DefaultEndpoint("unsubscribed-eds-resource", edsHost3, []uint32{edsPort3}), + wantUpdate: endpointsUpdateErrTuple{ + update: xdsresource.EndpointsUpdate{ + Localities: []xdsresource.Locality{ + { + Endpoints: []xdsresource.Endpoint{{Address: fmt.Sprintf("%s:%d", edsHost1, edsPort1), Weight: 1}}, + ID: internal.LocalityID{ + Region: "region-1", + Zone: "zone-1", + SubZone: "subzone-1", + }, + Priority: 0, + Weight: 1, + }, + }, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + overrideFedEnvVar(t) + mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup() + + // Create an xDS client with the above bootstrap contents. + client, close, err := xdsclient.NewWithBootstrapContentsForTesting(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register a watch for a endpoint resource and have the watch + // callback push the received update on to a channel. + ew := newEndpointsWatcher() + edsCancel := xdsresource.WatchEndpoints(client, test.resourceName, ew) + + // Configure the management server to return a single endpoint + // resource, corresponding to the one being watched. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{test.watchedResource}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update. + if err := verifyEndpointsUpdate(ctx, ew.updateCh, test.wantUpdate); err != nil { + t.Fatal(err) + } + + // Configure the management server to return an additional endpoint + // resource, one that we are not interested in. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{test.watchedResource, test.notWatchedResource}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + if err := verifyNoEndpointsUpdate(ctx, ew.updateCh); err != nil { + t.Fatal(err) + } + + // Cancel the watch and update the resource corresponding to the original + // watch. Ensure that the cancelled watch callback is not invoked. + edsCancel() + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{test.updatedWatchedResource, test.notWatchedResource}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + if err := verifyNoEndpointsUpdate(ctx, ew.updateCh); err != nil { + t.Fatal(err) + } + }) + } +} + +// TestEDSWatch_TwoWatchesForSameResourceName covers the case where two watchers +// exist for a single endpoint resource. The test verifies the following +// scenarios: +// 1. An update from the management server containing the resource being +// watched should result in the invocation of both watch callbacks. +// 2. After one of the watches is cancelled, a redundant update from the +// management server should not result in the invocation of either of the +// watch callbacks. +// 3. An update from the management server containing the resource being +// watched should result in the invocation of the un-cancelled watch +// callback. +// +// The test is run for old and new style names. +func (s) TestEDSWatch_TwoWatchesForSameResourceName(t *testing.T) { + tests := []struct { + desc string + resourceName string + watchedResource *v3endpointpb.ClusterLoadAssignment // The resource being watched. + updatedWatchedResource *v3endpointpb.ClusterLoadAssignment // The watched resource after an update. + wantUpdateV1 endpointsUpdateErrTuple + wantUpdateV2 endpointsUpdateErrTuple + }{ + { + desc: "old style resource", + resourceName: edsName, + watchedResource: e2e.DefaultEndpoint(edsName, edsHost1, []uint32{edsPort1}), + updatedWatchedResource: e2e.DefaultEndpoint(edsName, edsHost2, []uint32{edsPort2}), + wantUpdateV1: endpointsUpdateErrTuple{ + update: xdsresource.EndpointsUpdate{ + Localities: []xdsresource.Locality{ + { + Endpoints: []xdsresource.Endpoint{{Address: fmt.Sprintf("%s:%d", edsHost1, edsPort1), Weight: 1}}, + ID: internal.LocalityID{ + Region: "region-1", + Zone: "zone-1", + SubZone: "subzone-1", + }, + Priority: 0, + Weight: 1, + }, + }, + }, + }, + wantUpdateV2: endpointsUpdateErrTuple{ + update: xdsresource.EndpointsUpdate{ + Localities: []xdsresource.Locality{ + { + Endpoints: []xdsresource.Endpoint{{Address: fmt.Sprintf("%s:%d", edsHost2, edsPort2), Weight: 1}}, + ID: internal.LocalityID{ + Region: "region-1", + Zone: "zone-1", + SubZone: "subzone-1", + }, + Priority: 0, + Weight: 1, + }, + }, + }, + }, + }, + { + desc: "new style resource", + resourceName: edsNameNewStyle, + watchedResource: e2e.DefaultEndpoint(edsNameNewStyle, edsHost1, []uint32{edsPort1}), + updatedWatchedResource: e2e.DefaultEndpoint(edsNameNewStyle, edsHost2, []uint32{edsPort2}), + wantUpdateV1: endpointsUpdateErrTuple{ + update: xdsresource.EndpointsUpdate{ + Localities: []xdsresource.Locality{ + { + Endpoints: []xdsresource.Endpoint{{Address: fmt.Sprintf("%s:%d", edsHost1, edsPort1), Weight: 1}}, + ID: internal.LocalityID{ + Region: "region-1", + Zone: "zone-1", + SubZone: "subzone-1", + }, + Priority: 0, + Weight: 1, + }, + }, + }, + }, + wantUpdateV2: endpointsUpdateErrTuple{ + update: xdsresource.EndpointsUpdate{ + Localities: []xdsresource.Locality{ + { + Endpoints: []xdsresource.Endpoint{{Address: fmt.Sprintf("%s:%d", edsHost2, edsPort2), Weight: 1}}, + ID: internal.LocalityID{ + Region: "region-1", + Zone: "zone-1", + SubZone: "subzone-1", + }, + Priority: 0, + Weight: 1, + }, + }, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + overrideFedEnvVar(t) + mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup() + + // Create an xDS client with the above bootstrap contents. + client, close, err := xdsclient.NewWithBootstrapContentsForTesting(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register two watches for the same endpoint resource and have the + // callbacks push the received updates on to a channel. + ew1 := newEndpointsWatcher() + edsCancel1 := xdsresource.WatchEndpoints(client, test.resourceName, ew1) + defer edsCancel1() + ew2 := newEndpointsWatcher() + edsCancel2 := xdsresource.WatchEndpoints(client, test.resourceName, ew2) + + // Configure the management server to return a single endpoint + // resource, corresponding to the one being watched. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{test.watchedResource}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update. + if err := verifyEndpointsUpdate(ctx, ew1.updateCh, test.wantUpdateV1); err != nil { + t.Fatal(err) + } + if err := verifyEndpointsUpdate(ctx, ew2.updateCh, test.wantUpdateV1); err != nil { + t.Fatal(err) + } + + // Cancel the second watch and force the management server to push a + // redundant update for the resource being watched. Neither of the + // two watch callbacks should be invoked. + edsCancel2() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + if err := verifyNoEndpointsUpdate(ctx, ew1.updateCh); err != nil { + t.Fatal(err) + } + if err := verifyNoEndpointsUpdate(ctx, ew2.updateCh); err != nil { + t.Fatal(err) + } + + // Update to the resource being watched. The un-cancelled callback + // should be invoked while the cancelled one should not be. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{test.updatedWatchedResource}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + if err := verifyEndpointsUpdate(ctx, ew1.updateCh, test.wantUpdateV2); err != nil { + t.Fatal(err) + } + if err := verifyNoEndpointsUpdate(ctx, ew2.updateCh); err != nil { + t.Fatal(err) + } + }) + } +} + +// TestEDSWatch_ThreeWatchesForDifferentResourceNames covers the case with three +// watchers (two watchers for one resource, and the third watcher for another +// resource), exist across two endpoint configuration resources. The test verifies +// that an update from the management server containing both resources results +// in the invocation of all watch callbacks. +// +// The test is run with both old and new style names. +func (s) TestEDSWatch_ThreeWatchesForDifferentResourceNames(t *testing.T) { + overrideFedEnvVar(t) + mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup() + + // Create an xDS client with the above bootstrap contents. + client, close, err := xdsclient.NewWithBootstrapContentsForTesting(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register two watches for the same endpoint resource and have the + // callbacks push the received updates on to a channel. + ew1 := newEndpointsWatcher() + edsCancel1 := xdsresource.WatchEndpoints(client, edsName, ew1) + defer edsCancel1() + ew2 := newEndpointsWatcher() + edsCancel2 := xdsresource.WatchEndpoints(client, edsName, ew2) + defer edsCancel2() + + // Register the third watch for a different endpoint resource. + ew3 := newEndpointsWatcher() + edsCancel3 := xdsresource.WatchEndpoints(client, edsNameNewStyle, ew3) + defer edsCancel3() + + // Configure the management server to return two endpoint resources, + // corresponding to the registered watches. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{ + e2e.DefaultEndpoint(edsName, edsHost1, []uint32{edsPort1}), + e2e.DefaultEndpoint(edsNameNewStyle, edsHost1, []uint32{edsPort1}), + }, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update for the all watchers. The two + // resources returned differ only in the resource name. Therefore the + // expected update is the same for all the watchers. + wantUpdate := endpointsUpdateErrTuple{ + update: xdsresource.EndpointsUpdate{ + Localities: []xdsresource.Locality{ + { + Endpoints: []xdsresource.Endpoint{{Address: fmt.Sprintf("%s:%d", edsHost1, edsPort1), Weight: 1}}, + ID: internal.LocalityID{ + Region: "region-1", + Zone: "zone-1", + SubZone: "subzone-1", + }, + Priority: 0, + Weight: 1, + }, + }, + }, + } + if err := verifyEndpointsUpdate(ctx, ew1.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + if err := verifyEndpointsUpdate(ctx, ew2.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + if err := verifyEndpointsUpdate(ctx, ew3.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } +} + +// TestEDSWatch_ResourceCaching covers the case where a watch is registered for +// a resource which is already present in the cache. The test verifies that the +// watch callback is invoked with the contents from the cache, instead of a +// request being sent to the management server. +func (s) TestEDSWatch_ResourceCaching(t *testing.T) { + overrideFedEnvVar(t) + firstRequestReceived := false + firstAckReceived := grpcsync.NewEvent() + secondRequestReceived := grpcsync.NewEvent() + + mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{ + OnStreamRequest: func(id int64, req *v3discoverypb.DiscoveryRequest) error { + // The first request has an empty version string. + if !firstRequestReceived && req.GetVersionInfo() == "" { + firstRequestReceived = true + return nil + } + // The first ack has a non-empty version string. + if !firstAckReceived.HasFired() && req.GetVersionInfo() != "" { + firstAckReceived.Fire() + return nil + } + // Any requests after the first request and ack, are not expected. + secondRequestReceived.Fire() + return nil + }, + }) + defer cleanup() + + // Create an xDS client with the above bootstrap contents. + client, close, err := xdsclient.NewWithBootstrapContentsForTesting(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register a watch for an endpoint resource and have the watch callback + // push the received update on to a channel. + ew1 := newEndpointsWatcher() + edsCancel1 := xdsresource.WatchEndpoints(client, edsName, ew1) + defer edsCancel1() + + // Configure the management server to return a single endpoint resource, + // corresponding to the one we registered a watch for. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(edsName, edsHost1, []uint32{edsPort1})}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update. + wantUpdate := endpointsUpdateErrTuple{ + update: xdsresource.EndpointsUpdate{ + Localities: []xdsresource.Locality{ + { + Endpoints: []xdsresource.Endpoint{{Address: fmt.Sprintf("%s:%d", edsHost1, edsPort1), Weight: 1}}, + ID: internal.LocalityID{ + Region: "region-1", + Zone: "zone-1", + SubZone: "subzone-1", + }, + Priority: 0, + Weight: 1, + }, + }, + }, + } + if err := verifyEndpointsUpdate(ctx, ew1.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + select { + case <-ctx.Done(): + t.Fatal("timeout when waiting for receipt of ACK at the management server") + case <-firstAckReceived.Done(): + } + + // Register another watch for the same resource. This should get the update + // from the cache. + ew2 := newEndpointsWatcher() + edsCancel2 := xdsresource.WatchEndpoints(client, edsName, ew2) + defer edsCancel2() + if err := verifyEndpointsUpdate(ctx, ew2.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + + // No request should get sent out as part of this watch. + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + select { + case <-sCtx.Done(): + case <-secondRequestReceived.Done(): + t.Fatal("xdsClient sent out request instead of using update from cache") + } +} + +// TestEDSWatch_ExpiryTimerFiresBeforeResponse tests the case where the client +// does not receive an EDS response for the request that it sends. The test +// verifies that the watch callback is invoked with an error once the +// watchExpiryTimer fires. +func (s) TestEDSWatch_ExpiryTimerFiresBeforeResponse(t *testing.T) { + overrideFedEnvVar(t) + mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) + if err != nil { + t.Fatalf("Failed to spin up the xDS management server: %v", err) + } + defer mgmtServer.Stop() + + client, close, err := xdsclient.NewWithConfigForTesting(&bootstrap.Config{ + XDSServer: xdstestutils.ServerConfigForAddress(t, mgmtServer.Address), + NodeProto: &v3corepb.Node{}, + }, defaultTestWatchExpiryTimeout, time.Duration(0)) + if err != nil { + t.Fatalf("failed to create xds client: %v", err) + } + defer close() + + // Register a watch for a resource which is expected to fail with an error + // after the watch expiry timer fires. + ew := newEndpointsWatcher() + edsCancel := xdsresource.WatchEndpoints(client, edsName, ew) + defer edsCancel() + + // Wait for the watch expiry timer to fire. + <-time.After(defaultTestWatchExpiryTimeout) + + // Verify that an empty update with the expected error is received. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + wantErr := xdsresource.NewErrorf(xdsresource.ErrorTypeResourceNotFound, "") + if err := verifyEndpointsUpdate(ctx, ew.updateCh, endpointsUpdateErrTuple{err: wantErr}); err != nil { + t.Fatal(err) + } +} + +// TestEDSWatch_ValidResponseCancelsExpiryTimerBehavior tests the case where the +// client receives a valid EDS response for the request that it sends. The test +// verifies that the behavior associated with the expiry timer (i.e, callback +// invocation with error) does not take place. +func (s) TestEDSWatch_ValidResponseCancelsExpiryTimerBehavior(t *testing.T) { + overrideFedEnvVar(t) + mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) + if err != nil { + t.Fatalf("Failed to spin up the xDS management server: %v", err) + } + defer mgmtServer.Stop() + + // Create an xDS client talking to the above management server. + nodeID := uuid.New().String() + client, close, err := xdsclient.NewWithConfigForTesting(&bootstrap.Config{ + XDSServer: xdstestutils.ServerConfigForAddress(t, mgmtServer.Address), + NodeProto: &v3corepb.Node{Id: nodeID}, + }, defaultTestWatchExpiryTimeout, time.Duration(0)) + if err != nil { + t.Fatalf("failed to create xds client: %v", err) + } + defer close() + + // Register a watch for an endpoint resource and have the watch callback + // push the received update on to a channel. + ew := newEndpointsWatcher() + edsCancel := xdsresource.WatchEndpoints(client, edsName, ew) + defer edsCancel() + + // Configure the management server to return a single endpoint resource, + // corresponding to the one we registered a watch for. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(edsName, edsHost1, []uint32{edsPort1})}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update. + wantUpdate := endpointsUpdateErrTuple{ + update: xdsresource.EndpointsUpdate{ + Localities: []xdsresource.Locality{ + { + Endpoints: []xdsresource.Endpoint{{Address: fmt.Sprintf("%s:%d", edsHost1, edsPort1), Weight: 1}}, + ID: internal.LocalityID{ + Region: "region-1", + Zone: "zone-1", + SubZone: "subzone-1", + }, + Priority: 0, + Weight: 1, + }, + }, + }, + } + if err := verifyEndpointsUpdate(ctx, ew.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + + // Wait for the watch expiry timer to fire, and verify that the callback is + // not invoked. + <-time.After(defaultTestWatchExpiryTimeout) + if err := verifyNoEndpointsUpdate(ctx, ew.updateCh); err != nil { + t.Fatal(err) + } +} + +// TestEDSWatch_NACKError covers the case where an update from the management +// server is NACK'ed by the xdsclient. The test verifies that the error is +// propagated to the watcher. +func (s) TestEDSWatch_NACKError(t *testing.T) { + overrideFedEnvVar(t) + mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup() + + // Create an xDS client with the above bootstrap contents. + client, close, err := xdsclient.NewWithBootstrapContentsForTesting(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register a watch for a route configuration resource and have the watch + // callback push the received update on to a channel. + ew := newEndpointsWatcher() + edsCancel := xdsresource.WatchEndpoints(client, edsName, ew) + defer edsCancel() + + // Configure the management server to return a single route configuration + // resource which is expected to be NACKed by the client. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{badEndpointsResource(edsName, edsHost1, []uint32{edsPort1})}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify that the expected error is propagated to the watcher. + u, err := ew.updateCh.Receive(ctx) + if err != nil { + t.Fatalf("timeout when waiting for an endpoints resource from the management server: %v", err) + } + gotErr := u.(endpointsUpdateErrTuple).err + if gotErr == nil || !strings.Contains(gotErr.Error(), wantEndpointsNACKErr) { + t.Fatalf("update received with error: %v, want %q", gotErr, wantEndpointsNACKErr) + } +} + +// TestEDSWatch_PartialValid covers the case where a response from the +// management server contains both valid and invalid resources and is expected +// to be NACK'ed by the xdsclient. The test verifies that watchers corresponding +// to the valid resource receive the update, while watchers corresponding to the +// invalid resource receive an error. +func (s) TestEDSWatch_PartialValid(t *testing.T) { + overrideFedEnvVar(t) + mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup() + + // Create an xDS client with the above bootstrap contents. + client, close, err := xdsclient.NewWithBootstrapContentsForTesting(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register two watches for two endpoint resources. The first watch is + // expected to receive an error because the received resource is NACKed. + // The second watch is expected to get a good update. + badResourceName := rdsName + ew1 := newEndpointsWatcher() + edsCancel1 := xdsresource.WatchEndpoints(client, badResourceName, ew1) + defer edsCancel1() + goodResourceName := ldsNameNewStyle + ew2 := newEndpointsWatcher() + edsCancel2 := xdsresource.WatchEndpoints(client, goodResourceName, ew2) + defer edsCancel2() + + // Configure the management server to return two endpoints resources, + // corresponding to the registered watches. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{ + badEndpointsResource(badResourceName, edsHost1, []uint32{edsPort1}), + e2e.DefaultEndpoint(goodResourceName, edsHost1, []uint32{edsPort1}), + }, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify that the expected error is propagated to the watcher which + // requested for the bad resource. + u, err := ew1.updateCh.Receive(ctx) + if err != nil { + t.Fatalf("timeout when waiting for an endpoints resource from the management server: %v", err) + } + gotErr := u.(endpointsUpdateErrTuple).err + if gotErr == nil || !strings.Contains(gotErr.Error(), wantEndpointsNACKErr) { + t.Fatalf("update received with error: %v, want %q", gotErr, wantEndpointsNACKErr) + } + + // Verify that the watcher watching the good resource receives an update. + wantUpdate := endpointsUpdateErrTuple{ + update: xdsresource.EndpointsUpdate{ + Localities: []xdsresource.Locality{ + { + Endpoints: []xdsresource.Endpoint{{Address: fmt.Sprintf("%s:%d", edsHost1, edsPort1), Weight: 1}}, + ID: internal.LocalityID{ + Region: "region-1", + Zone: "zone-1", + SubZone: "subzone-1", + }, + Priority: 0, + Weight: 1, + }, + }, + }, + } + if err := verifyEndpointsUpdate(ctx, ew2.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } +} diff --git a/xds/internal/xdsclient/tests/federation_watchers_test.go b/xds/internal/xdsclient/tests/federation_watchers_test.go new file mode 100644 index 000000000000..ed59b63ac794 --- /dev/null +++ b/xds/internal/xdsclient/tests/federation_watchers_test.go @@ -0,0 +1,324 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsclient_test + +import ( + "context" + "fmt" + "testing" + + "github.com/google/uuid" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/bootstrap" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/xds/internal" + "google.golang.org/grpc/xds/internal/xdsclient" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" +) + +const testNonDefaultAuthority = "non-default-authority" + +// setupForFederationWatchersTest spins up two management servers, one for the +// default (empty) authority and another for a non-default authority. +// +// Returns the management server associated with the non-default authority, the +// nodeID to use, and the xDS client. +func setupForFederationWatchersTest(t *testing.T) (*e2e.ManagementServer, string, xdsclient.XDSClient) { + overrideFedEnvVar(t) + + // Start a management server as the default authority. + serverDefaultAuthority, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) + if err != nil { + t.Fatalf("Failed to spin up the xDS management server: %v", err) + } + t.Cleanup(serverDefaultAuthority.Stop) + + // Start another management server as the other authority. + serverNonDefaultAuthority, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) + if err != nil { + t.Fatalf("Failed to spin up the xDS management server: %v", err) + } + t.Cleanup(serverNonDefaultAuthority.Stop) + + nodeID := uuid.New().String() + bootstrapContents, err := bootstrap.Contents(bootstrap.Options{ + NodeID: nodeID, + ServerURI: serverDefaultAuthority.Address, + ServerListenerResourceNameTemplate: e2e.ServerListenerResourceNameTemplate, + // Specify the address of the non-default authority. + Authorities: map[string]string{testNonDefaultAuthority: serverNonDefaultAuthority.Address}, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap file: %v", err) + } + // Create an xDS client with the above bootstrap contents. + client, close, err := xdsclient.NewWithBootstrapContentsForTesting(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + t.Cleanup(close) + return serverNonDefaultAuthority, nodeID, client +} + +// TestFederation_ListenerResourceContextParamOrder covers the case of watching +// a Listener resource with the new style resource name and context parameters. +// The test registers watches for two resources which differ only in the order +// of context parameters in their URI. The server is configured to respond with +// a single resource with canonicalized context parameters. The test verifies +// that both watchers are notified. +func (s) TestFederation_ListenerResourceContextParamOrder(t *testing.T) { + serverNonDefaultAuthority, nodeID, client := setupForFederationWatchersTest(t) + + var ( + // Two resource names only differ in context parameter order. + resourceName1 = fmt.Sprintf("xdstp://%s/envoy.config.listener.v3.Listener/xdsclient-test-lds-resource?a=1&b=2", testNonDefaultAuthority) + resourceName2 = fmt.Sprintf("xdstp://%s/envoy.config.listener.v3.Listener/xdsclient-test-lds-resource?b=2&a=1", testNonDefaultAuthority) + ) + + // Register two watches for listener resources with the same query string, + // but context parameters in different order. + updateCh1 := testutils.NewChannel() + ldsCancel1 := client.WatchListener(resourceName1, func(u xdsresource.ListenerUpdate, err error) { + updateCh1.Send(xdsresource.ListenerUpdateErrTuple{Update: u, Err: err}) + }) + defer ldsCancel1() + updateCh2 := testutils.NewChannel() + ldsCancel2 := client.WatchListener(resourceName2, func(u xdsresource.ListenerUpdate, err error) { + updateCh2.Send(xdsresource.ListenerUpdateErrTuple{Update: u, Err: err}) + }) + defer ldsCancel2() + + // Configure the management server for the non-default authority to return a + // single listener resource, corresponding to the watches registered above. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(resourceName1, "rds-resource")}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := serverNonDefaultAuthority.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + wantUpdate := xdsresource.ListenerUpdateErrTuple{ + Update: xdsresource.ListenerUpdate{ + RouteConfigName: "rds-resource", + HTTPFilters: []xdsresource.HTTPFilter{{Name: "router"}}, + }, + } + // Verify the contents of the received update. + if err := verifyListenerUpdate(ctx, updateCh1, wantUpdate); err != nil { + t.Fatal(err) + } + if err := verifyListenerUpdate(ctx, updateCh2, wantUpdate); err != nil { + t.Fatal(err) + } +} + +// TestFederation_RouteConfigResourceContextParamOrder covers the case of +// watching a RouteConfiguration resource with the new style resource name and +// context parameters. The test registers watches for two resources which +// differ only in the order of context parameters in their URI. The server is +// configured to respond with a single resource with canonicalized context +// parameters. The test verifies that both watchers are notified. +func (s) TestFederation_RouteConfigResourceContextParamOrder(t *testing.T) { + serverNonDefaultAuthority, nodeID, client := setupForFederationWatchersTest(t) + + var ( + // Two resource names only differ in context parameter order. + resourceName1 = fmt.Sprintf("xdstp://%s/envoy.config.route.v3.RouteConfiguration/xdsclient-test-rds-resource?a=1&b=2", testNonDefaultAuthority) + resourceName2 = fmt.Sprintf("xdstp://%s/envoy.config.route.v3.RouteConfiguration/xdsclient-test-rds-resource?b=2&a=1", testNonDefaultAuthority) + ) + + // Register two watches for route configuration resources with the same + // query string, but context parameters in different order. + updateCh1 := testutils.NewChannel() + rdsCancel1 := client.WatchRouteConfig(resourceName1, func(u xdsresource.RouteConfigUpdate, err error) { + updateCh1.Send(xdsresource.RouteConfigUpdateErrTuple{Update: u, Err: err}) + }) + defer rdsCancel1() + updateCh2 := testutils.NewChannel() + rdsCancel2 := client.WatchRouteConfig(resourceName2, func(u xdsresource.RouteConfigUpdate, err error) { + updateCh2.Send(xdsresource.RouteConfigUpdateErrTuple{Update: u, Err: err}) + }) + defer rdsCancel2() + + // Configure the management server for the non-default authority to return a + // single route config resource, corresponding to the watches registered. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(resourceName1, "listener-resource", "cluster-resource")}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := serverNonDefaultAuthority.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + wantUpdate := xdsresource.RouteConfigUpdateErrTuple{ + Update: xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{ + { + Domains: []string{"listener-resource"}, + Routes: []*xdsresource.Route{ + { + Prefix: newStringP("/"), + ActionType: xdsresource.RouteActionRoute, + WeightedClusters: map[string]xdsresource.WeightedCluster{"cluster-resource": {Weight: 100}}, + }, + }, + }, + }, + }, + } + // Verify the contents of the received update. + if err := verifyRouteConfigUpdate(ctx, updateCh1, wantUpdate); err != nil { + t.Fatal(err) + } + if err := verifyRouteConfigUpdate(ctx, updateCh2, wantUpdate); err != nil { + t.Fatal(err) + } +} + +// TestFederation_ClusterResourceContextParamOrder covers the case of watching a +// Cluster resource with the new style resource name and context parameters. +// The test registers watches for two resources which differ only in the order +// of context parameters in their URI. The server is configured to respond with +// a single resource with canonicalized context parameters. The test verifies +// that both watchers are notified. +func (s) TestFederation_ClusterResourceContextParamOrder(t *testing.T) { + serverNonDefaultAuthority, nodeID, client := setupForFederationWatchersTest(t) + + var ( + // Two resource names only differ in context parameter order. + resourceName1 = fmt.Sprintf("xdstp://%s/envoy.config.cluster.v3.Cluster/xdsclient-test-cds-resource?a=1&b=2", testNonDefaultAuthority) + resourceName2 = fmt.Sprintf("xdstp://%s/envoy.config.cluster.v3.Cluster/xdsclient-test-cds-resource?b=2&a=1", testNonDefaultAuthority) + ) + + // Register two watches for cluster resources with the same query string, + // but context parameters in different order. + updateCh1 := testutils.NewChannel() + cdsCancel1 := client.WatchCluster(resourceName1, func(u xdsresource.ClusterUpdate, err error) { + updateCh1.Send(xdsresource.ClusterUpdateErrTuple{Update: u, Err: err}) + }) + defer cdsCancel1() + updateCh2 := testutils.NewChannel() + cdsCancel2 := client.WatchCluster(resourceName2, func(u xdsresource.ClusterUpdate, err error) { + updateCh2.Send(xdsresource.ClusterUpdateErrTuple{Update: u, Err: err}) + }) + defer cdsCancel2() + + // Configure the management server for the non-default authority to return a + // single cluster resource, corresponding to the watches registered. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Clusters: []*v3clusterpb.Cluster{e2e.DefaultCluster(resourceName1, "eds-service-name", e2e.SecurityLevelNone)}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := serverNonDefaultAuthority.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + wantUpdate := xdsresource.ClusterUpdateErrTuple{ + Update: xdsresource.ClusterUpdate{ + ClusterName: "xdstp://non-default-authority/envoy.config.cluster.v3.Cluster/xdsclient-test-cds-resource?a=1&b=2", + EDSServiceName: "eds-service-name", + }, + } + // Verify the contents of the received update. + if err := verifyClusterUpdate(ctx, updateCh1, wantUpdate); err != nil { + t.Fatal(err) + } + if err := verifyClusterUpdate(ctx, updateCh2, wantUpdate); err != nil { + t.Fatal(err) + } +} + +// TestFederation_EndpointsResourceContextParamOrder covers the case of watching +// an Endpoints resource with the new style resource name and context parameters. +// The test registers watches for two resources which differ only in the order +// of context parameters in their URI. The server is configured to respond with +// a single resource with canonicalized context parameters. The test verifies +// that both watchers are notified. +func (s) TestFederation_EndpointsResourceContextParamOrder(t *testing.T) { + serverNonDefaultAuthority, nodeID, client := setupForFederationWatchersTest(t) + + var ( + // Two resource names only differ in context parameter order. + resourceName1 = fmt.Sprintf("xdstp://%s/envoy.config.endpoint.v3.ClusterLoadAssignment/xdsclient-test-eds-resource?a=1&b=2", testNonDefaultAuthority) + resourceName2 = fmt.Sprintf("xdstp://%s/envoy.config.endpoint.v3.ClusterLoadAssignment/xdsclient-test-eds-resource?b=2&a=1", testNonDefaultAuthority) + ) + + // Register two watches for endpoint resources with the same query string, + // but context parameters in different order. + ew1 := newEndpointsWatcher() + edsCancel1 := xdsresource.WatchEndpoints(client, resourceName1, ew1) + defer edsCancel1() + ew2 := newEndpointsWatcher() + edsCancel2 := xdsresource.WatchEndpoints(client, resourceName2, ew2) + defer edsCancel2() + + // Configure the management server for the non-default authority to return a + // single endpoints resource, corresponding to the watches registered. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Endpoints: []*v3endpointpb.ClusterLoadAssignment{e2e.DefaultEndpoint(resourceName1, "localhost", []uint32{666})}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := serverNonDefaultAuthority.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + wantUpdate := endpointsUpdateErrTuple{ + update: xdsresource.EndpointsUpdate{ + Localities: []xdsresource.Locality{ + { + Endpoints: []xdsresource.Endpoint{{Address: "localhost:666", Weight: 1}}, + Weight: 1, + ID: internal.LocalityID{ + Region: "region-1", + Zone: "zone-1", + SubZone: "subzone-1", + }, + }, + }, + }, + } + // Verify the contents of the received update. + if err := verifyEndpointsUpdate(ctx, ew1.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + if err := verifyEndpointsUpdate(ctx, ew2.updateCh, wantUpdate); err != nil { + t.Fatal(err) + } +} + +func newStringP(s string) *string { + return &s +} diff --git a/xds/internal/xdsclient/tests/lds_watchers_test.go b/xds/internal/xdsclient/tests/lds_watchers_test.go new file mode 100644 index 000000000000..7e41a81361ff --- /dev/null +++ b/xds/internal/xdsclient/tests/lds_watchers_test.go @@ -0,0 +1,1009 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient_test + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/envoyproxy/go-control-plane/pkg/wellknown" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/google/uuid" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + xdstestutils "google.golang.org/grpc/xds/internal/testutils" + "google.golang.org/grpc/xds/internal/xdsclient" + "google.golang.org/grpc/xds/internal/xdsclient/bootstrap" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routerpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3" + v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + + _ "google.golang.org/grpc/xds" // To ensure internal.NewXDSResolverWithConfigForTesting is set. + _ "google.golang.org/grpc/xds/internal/httpfilter/router" // Register the router filter. +) + +func overrideFedEnvVar(t *testing.T) { + oldFed := envconfig.XDSFederation + envconfig.XDSFederation = true + t.Cleanup(func() { envconfig.XDSFederation = oldFed }) +} + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +const ( + defaultTestWatchExpiryTimeout = 500 * time.Millisecond + defaultTestIdleAuthorityTimeout = 50 * time.Millisecond + defaultTestTimeout = 5 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond // For events expected to *not* happen. + + ldsName = "xdsclient-test-lds-resource" + rdsName = "xdsclient-test-rds-resource" + cdsName = "xdsclient-test-cds-resource" + edsName = "xdsclient-test-eds-resource" + ldsNameNewStyle = "xdstp:///envoy.config.listener.v3.Listener/xdsclient-test-lds-resource" + rdsNameNewStyle = "xdstp:///envoy.config.route.v3.RouteConfiguration/xdsclient-test-rds-resource" + cdsNameNewStyle = "xdstp:///envoy.config.cluster.v3.Cluster/xdsclient-test-cds-resource" + edsNameNewStyle = "xdstp:///envoy.config.endpoint.v3.ClusterLoadAssignment/xdsclient-test-eds-resource" +) + +// badListenerResource returns a listener resource for the given name which does +// not contain the `RouteSpecifier` field in the HTTPConnectionManager, and +// hence is expected to be NACKed by the client. +func badListenerResource(name string) *v3listenerpb.Listener { + hcm := testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + HttpFilters: []*v3httppb.HttpFilter{e2e.HTTPFilter("router", &v3routerpb.Router{})}, + }) + return &v3listenerpb.Listener{ + Name: name, + ApiListener: &v3listenerpb.ApiListener{ApiListener: hcm}, + FilterChains: []*v3listenerpb.FilterChain{{ + Name: "filter-chain-name", + Filters: []*v3listenerpb.Filter{{ + Name: wellknown.HTTPConnectionManager, + ConfigType: &v3listenerpb.Filter_TypedConfig{TypedConfig: hcm}, + }}, + }}, + } +} + +// xdsClient is expected to produce an error containing this string when an +// update is received containing a listener created using `badListenerResource`. +const wantListenerNACKErr = "no RouteSpecifier" + +// verifyNoListenerUpdate verifies that no listener update is received on the +// provided update channel, and returns an error if an update is received. +// +// A very short deadline is used while waiting for the update, as this function +// is intended to be used when an update is not expected. +func verifyNoListenerUpdate(ctx context.Context, updateCh *testutils.Channel) error { + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if u, err := updateCh.Receive(sCtx); err != context.DeadlineExceeded { + return fmt.Errorf("unexpected ListenerUpdate: %v", u) + } + return nil +} + +// verifyListenerUpdate waits for an update to be received on the provided +// update channel and verifies that it matches the expected update. +// +// Returns an error if no update is received before the context deadline expires +// or the received update does not match the expected one. +func verifyListenerUpdate(ctx context.Context, updateCh *testutils.Channel, wantUpdate xdsresource.ListenerUpdateErrTuple) error { + u, err := updateCh.Receive(ctx) + if err != nil { + return fmt.Errorf("timeout when waiting for a listener resource from the management server: %v", err) + } + got := u.(xdsresource.ListenerUpdateErrTuple) + if wantUpdate.Err != nil { + if gotType, wantType := xdsresource.ErrType(got.Err), xdsresource.ErrType(wantUpdate.Err); gotType != wantType { + return fmt.Errorf("received update with error type %v, want %v", gotType, wantType) + } + } + cmpOpts := []cmp.Option{ + cmpopts.EquateEmpty(), + cmpopts.IgnoreFields(xdsresource.HTTPFilter{}, "Filter", "Config"), + cmpopts.IgnoreFields(xdsresource.ListenerUpdate{}, "Raw"), + } + if diff := cmp.Diff(wantUpdate.Update, got.Update, cmpOpts...); diff != "" { + return fmt.Errorf("received unepected diff in the listener resource update: (-want, got):\n%s", diff) + } + return nil +} + +// TestLDSWatch covers the case where a single watcher exists for a single +// listener resource. The test verifies the following scenarios: +// 1. An update from the management server containing the resource being +// watched should result in the invocation of the watch callback. +// 2. An update from the management server containing a resource *not* being +// watched should not result in the invocation of the watch callback. +// 3. After the watch is cancelled, an update from the management server +// containing the resource that was being watched should not result in the +// invocation of the watch callback. +// +// The test is run for old and new style names. +func (s) TestLDSWatch(t *testing.T) { + tests := []struct { + desc string + resourceName string + watchedResource *v3listenerpb.Listener // The resource being watched. + updatedWatchedResource *v3listenerpb.Listener // The watched resource after an update. + notWatchedResource *v3listenerpb.Listener // A resource which is not being watched. + wantUpdate xdsresource.ListenerUpdateErrTuple + }{ + { + desc: "old style resource", + resourceName: ldsName, + watchedResource: e2e.DefaultClientListener(ldsName, rdsName), + updatedWatchedResource: e2e.DefaultClientListener(ldsName, "new-rds-resource"), + notWatchedResource: e2e.DefaultClientListener("unsubscribed-lds-resource", rdsName), + wantUpdate: xdsresource.ListenerUpdateErrTuple{ + Update: xdsresource.ListenerUpdate{ + RouteConfigName: rdsName, + HTTPFilters: []xdsresource.HTTPFilter{{Name: "router"}}, + }, + }, + }, + { + desc: "new style resource", + resourceName: ldsNameNewStyle, + watchedResource: e2e.DefaultClientListener(ldsNameNewStyle, rdsNameNewStyle), + updatedWatchedResource: e2e.DefaultClientListener(ldsNameNewStyle, "new-rds-resource"), + notWatchedResource: e2e.DefaultClientListener("unsubscribed-lds-resource", rdsNameNewStyle), + wantUpdate: xdsresource.ListenerUpdateErrTuple{ + Update: xdsresource.ListenerUpdate{ + RouteConfigName: rdsNameNewStyle, + HTTPFilters: []xdsresource.HTTPFilter{{Name: "router"}}, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + overrideFedEnvVar(t) + mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup() + + // Create an xDS client with the above bootstrap contents. + client, close, err := xdsclient.NewWithBootstrapContentsForTesting(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register a watch for a listener resource and have the watch + // callback push the received update on to a channel. + updateCh := testutils.NewChannel() + ldsCancel := client.WatchListener(test.resourceName, func(u xdsresource.ListenerUpdate, err error) { + updateCh.Send(xdsresource.ListenerUpdateErrTuple{Update: u, Err: err}) + }) + + // Configure the management server to return a single listener + // resource, corresponding to the one we registered a watch for. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{test.watchedResource}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update. + if err := verifyListenerUpdate(ctx, updateCh, test.wantUpdate); err != nil { + t.Fatal(err) + } + + // Configure the management server to return an additional listener + // resource, one that we are not interested in. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{test.watchedResource, test.notWatchedResource}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + if err := verifyNoListenerUpdate(ctx, updateCh); err != nil { + t.Fatal(err) + } + + // Cancel the watch and update the resource corresponding to the original + // watch. Ensure that the cancelled watch callback is not invoked. + ldsCancel() + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{test.updatedWatchedResource, test.notWatchedResource}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + if err := verifyNoListenerUpdate(ctx, updateCh); err != nil { + t.Fatal(err) + } + }) + } +} + +// TestLDSWatch_TwoWatchesForSameResourceName covers the case where two watchers +// exist for a single listener resource. The test verifies the following +// scenarios: +// 1. An update from the management server containing the resource being +// watched should result in the invocation of both watch callbacks. +// 2. After one of the watches is cancelled, a redundant update from the +// management server should not result in the invocation of either of the +// watch callbacks. +// 3. An update from the management server containing the resource being +// watched should result in the invocation of the un-cancelled watch +// callback. +// +// The test is run for old and new style names. +func (s) TestLDSWatch_TwoWatchesForSameResourceName(t *testing.T) { + tests := []struct { + desc string + resourceName string + watchedResource *v3listenerpb.Listener // The resource being watched. + updatedWatchedResource *v3listenerpb.Listener // The watched resource after an update. + wantUpdateV1 xdsresource.ListenerUpdateErrTuple + wantUpdateV2 xdsresource.ListenerUpdateErrTuple + }{ + { + desc: "old style resource", + resourceName: ldsName, + watchedResource: e2e.DefaultClientListener(ldsName, rdsName), + updatedWatchedResource: e2e.DefaultClientListener(ldsName, "new-rds-resource"), + wantUpdateV1: xdsresource.ListenerUpdateErrTuple{ + Update: xdsresource.ListenerUpdate{ + RouteConfigName: rdsName, + HTTPFilters: []xdsresource.HTTPFilter{{Name: "router"}}, + }, + }, + wantUpdateV2: xdsresource.ListenerUpdateErrTuple{ + Update: xdsresource.ListenerUpdate{ + RouteConfigName: "new-rds-resource", + HTTPFilters: []xdsresource.HTTPFilter{{Name: "router"}}, + }, + }, + }, + { + desc: "new style resource", + resourceName: ldsNameNewStyle, + watchedResource: e2e.DefaultClientListener(ldsNameNewStyle, rdsNameNewStyle), + updatedWatchedResource: e2e.DefaultClientListener(ldsNameNewStyle, "new-rds-resource"), + wantUpdateV1: xdsresource.ListenerUpdateErrTuple{ + Update: xdsresource.ListenerUpdate{ + RouteConfigName: rdsNameNewStyle, + HTTPFilters: []xdsresource.HTTPFilter{{Name: "router"}}, + }, + }, + wantUpdateV2: xdsresource.ListenerUpdateErrTuple{ + Update: xdsresource.ListenerUpdate{ + RouteConfigName: "new-rds-resource", + HTTPFilters: []xdsresource.HTTPFilter{{Name: "router"}}, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + overrideFedEnvVar(t) + mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup() + + // Create an xDS client with the above bootstrap contents. + client, close, err := xdsclient.NewWithBootstrapContentsForTesting(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register two watches for the same listener resource and have the + // callbacks push the received updates on to a channel. + updateCh1 := testutils.NewChannel() + ldsCancel1 := client.WatchListener(test.resourceName, func(u xdsresource.ListenerUpdate, err error) { + updateCh1.Send(xdsresource.ListenerUpdateErrTuple{Update: u, Err: err}) + }) + defer ldsCancel1() + updateCh2 := testutils.NewChannel() + ldsCancel2 := client.WatchListener(test.resourceName, func(u xdsresource.ListenerUpdate, err error) { + updateCh2.Send(xdsresource.ListenerUpdateErrTuple{Update: u, Err: err}) + }) + + // Configure the management server to return a single listener + // resource, corresponding to the one we registered watches for. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{test.watchedResource}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update. + if err := verifyListenerUpdate(ctx, updateCh1, test.wantUpdateV1); err != nil { + t.Fatal(err) + } + if err := verifyListenerUpdate(ctx, updateCh2, test.wantUpdateV1); err != nil { + t.Fatal(err) + } + + // Cancel the second watch and force the management server to push a + // redundant update for the resource being watched. Neither of the + // two watch callbacks should be invoked. + ldsCancel2() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + if err := verifyNoListenerUpdate(ctx, updateCh1); err != nil { + t.Fatal(err) + } + if err := verifyNoListenerUpdate(ctx, updateCh2); err != nil { + t.Fatal(err) + } + + // Update to the resource being watched. The un-cancelled callback + // should be invoked while the cancelled one should not be. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{test.updatedWatchedResource}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + if err := verifyListenerUpdate(ctx, updateCh1, test.wantUpdateV2); err != nil { + t.Fatal(err) + } + if err := verifyNoListenerUpdate(ctx, updateCh2); err != nil { + t.Fatal(err) + } + }) + } +} + +// TestLDSWatch_ThreeWatchesForDifferentResourceNames covers the case with three +// watchers (two watchers for one resource, and the third watcher for another +// resource), exist across two listener resources. The test verifies that an +// update from the management server containing both resources results in the +// invocation of all watch callbacks. +// +// The test is run with both old and new style names. +func (s) TestLDSWatch_ThreeWatchesForDifferentResourceNames(t *testing.T) { + overrideFedEnvVar(t) + mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup() + + // Create an xDS client with the above bootstrap contents. + client, close, err := xdsclient.NewWithBootstrapContentsForTesting(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register two watches for the same listener resource and have the + // callbacks push the received updates on to a channel. + updateCh1 := testutils.NewChannel() + ldsCancel1 := client.WatchListener(ldsName, func(u xdsresource.ListenerUpdate, err error) { + updateCh1.Send(xdsresource.ListenerUpdateErrTuple{Update: u, Err: err}) + }) + defer ldsCancel1() + updateCh2 := testutils.NewChannel() + ldsCancel2 := client.WatchListener(ldsName, func(u xdsresource.ListenerUpdate, err error) { + updateCh2.Send(xdsresource.ListenerUpdateErrTuple{Update: u, Err: err}) + }) + defer ldsCancel2() + + // Register the third watch for a different listener resource. + updateCh3 := testutils.NewChannel() + ldsCancel3 := client.WatchListener(ldsNameNewStyle, func(u xdsresource.ListenerUpdate, err error) { + updateCh3.Send(xdsresource.ListenerUpdateErrTuple{Update: u, Err: err}) + }) + defer ldsCancel3() + + // Configure the management server to return two listener resources, + // corresponding to the registered watches. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{ + e2e.DefaultClientListener(ldsName, rdsName), + e2e.DefaultClientListener(ldsNameNewStyle, rdsName), + }, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update for the all watchers. The two + // resources returned differ only in the resource name. Therefore the + // expected update is the same for all the watchers. + wantUpdate := xdsresource.ListenerUpdateErrTuple{ + Update: xdsresource.ListenerUpdate{ + RouteConfigName: rdsName, + HTTPFilters: []xdsresource.HTTPFilter{{Name: "router"}}, + }, + } + if err := verifyListenerUpdate(ctx, updateCh1, wantUpdate); err != nil { + t.Fatal(err) + } + if err := verifyListenerUpdate(ctx, updateCh2, wantUpdate); err != nil { + t.Fatal(err) + } + if err := verifyListenerUpdate(ctx, updateCh3, wantUpdate); err != nil { + t.Fatal(err) + } +} + +// TestLDSWatch_ResourceCaching covers the case where a watch is registered for +// a resource which is already present in the cache. The test verifies that the +// watch callback is invoked with the contents from the cache, instead of a +// request being sent to the management server. +func (s) TestLDSWatch_ResourceCaching(t *testing.T) { + overrideFedEnvVar(t) + firstRequestReceived := false + firstAckReceived := grpcsync.NewEvent() + secondRequestReceived := grpcsync.NewEvent() + + mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{ + OnStreamRequest: func(id int64, req *v3discoverypb.DiscoveryRequest) error { + // The first request has an empty version string. + if !firstRequestReceived && req.GetVersionInfo() == "" { + firstRequestReceived = true + return nil + } + // The first ack has a non-empty version string. + if !firstAckReceived.HasFired() && req.GetVersionInfo() != "" { + firstAckReceived.Fire() + return nil + } + // Any requests after the first request and ack, are not expected. + secondRequestReceived.Fire() + return nil + }, + }) + defer cleanup() + + // Create an xDS client with the above bootstrap contents. + client, close, err := xdsclient.NewWithBootstrapContentsForTesting(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register a watch for a listener resource and have the watch + // callback push the received update on to a channel. + updateCh1 := testutils.NewChannel() + ldsCancel1 := client.WatchListener(ldsName, func(u xdsresource.ListenerUpdate, err error) { + updateCh1.Send(xdsresource.ListenerUpdateErrTuple{Update: u, Err: err}) + }) + defer ldsCancel1() + + // Configure the management server to return a single listener + // resource, corresponding to the one we registered a watch for. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(ldsName, rdsName)}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update. + wantUpdate := xdsresource.ListenerUpdateErrTuple{ + Update: xdsresource.ListenerUpdate{ + RouteConfigName: rdsName, + HTTPFilters: []xdsresource.HTTPFilter{{Name: "router"}}, + }, + } + if err := verifyListenerUpdate(ctx, updateCh1, wantUpdate); err != nil { + t.Fatal(err) + } + select { + case <-ctx.Done(): + t.Fatal("timeout when waiting for receipt of ACK at the management server") + case <-firstAckReceived.Done(): + } + + // Register another watch for the same resource. This should get the update + // from the cache. + updateCh2 := testutils.NewChannel() + ldsCancel2 := client.WatchListener(ldsName, func(u xdsresource.ListenerUpdate, err error) { + updateCh2.Send(xdsresource.ListenerUpdateErrTuple{Update: u, Err: err}) + }) + defer ldsCancel2() + if err := verifyListenerUpdate(ctx, updateCh2, wantUpdate); err != nil { + t.Fatal(err) + } + // No request should get sent out as part of this watch. + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + select { + case <-sCtx.Done(): + case <-secondRequestReceived.Done(): + t.Fatal("xdsClient sent out request instead of using update from cache") + } +} + +// TestLDSWatch_ExpiryTimerFiresBeforeResponse tests the case where the client +// does not receive an LDS response for the request that it sends. The test +// verifies that the watch callback is invoked with an error once the +// watchExpiryTimer fires. +func (s) TestLDSWatch_ExpiryTimerFiresBeforeResponse(t *testing.T) { + overrideFedEnvVar(t) + mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) + if err != nil { + t.Fatalf("Failed to spin up the xDS management server: %v", err) + } + defer mgmtServer.Stop() + + client, close, err := xdsclient.NewWithConfigForTesting(&bootstrap.Config{ + XDSServer: xdstestutils.ServerConfigForAddress(t, mgmtServer.Address), + NodeProto: &v3corepb.Node{}, + }, defaultTestWatchExpiryTimeout, time.Duration(0)) + if err != nil { + t.Fatalf("failed to create xds client: %v", err) + } + defer close() + + // Register a watch for a resource which is expected to fail with an error + // after the watch expiry timer fires. + updateCh := testutils.NewChannel() + ldsCancel := client.WatchListener(ldsName, func(u xdsresource.ListenerUpdate, err error) { + updateCh.Send(xdsresource.ListenerUpdateErrTuple{Update: u, Err: err}) + }) + defer ldsCancel() + + // Wait for the watch expiry timer to fire. + <-time.After(defaultTestWatchExpiryTimeout) + + // Verify that an empty update with the expected error is received. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + wantErr := xdsresource.NewErrorf(xdsresource.ErrorTypeResourceNotFound, "") + if err := verifyListenerUpdate(ctx, updateCh, xdsresource.ListenerUpdateErrTuple{Err: wantErr}); err != nil { + t.Fatal(err) + } +} + +// TestLDSWatch_ValidResponseCancelsExpiryTimerBehavior tests the case where the +// client receives a valid LDS response for the request that it sends. The test +// verifies that the behavior associated with the expiry timer (i.e, callback +// invocation with error) does not take place. +func (s) TestLDSWatch_ValidResponseCancelsExpiryTimerBehavior(t *testing.T) { + overrideFedEnvVar(t) + mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) + if err != nil { + t.Fatalf("Failed to spin up the xDS management server: %v", err) + } + defer mgmtServer.Stop() + + // Create an xDS client talking to the above management server. + nodeID := uuid.New().String() + client, close, err := xdsclient.NewWithConfigForTesting(&bootstrap.Config{ + XDSServer: xdstestutils.ServerConfigForAddress(t, mgmtServer.Address), + NodeProto: &v3corepb.Node{Id: nodeID}, + }, defaultTestWatchExpiryTimeout, time.Duration(0)) + if err != nil { + t.Fatalf("failed to create xds client: %v", err) + } + defer close() + + // Register a watch for a listener resource and have the watch + // callback push the received update on to a channel. + updateCh := testutils.NewChannel() + ldsCancel := client.WatchListener(ldsName, func(u xdsresource.ListenerUpdate, err error) { + updateCh.Send(xdsresource.ListenerUpdateErrTuple{Update: u, Err: err}) + }) + defer ldsCancel() + + // Configure the management server to return a single listener + // resource, corresponding to the one we registered a watch for. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(ldsName, rdsName)}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update. + wantUpdate := xdsresource.ListenerUpdateErrTuple{ + Update: xdsresource.ListenerUpdate{ + RouteConfigName: rdsName, + HTTPFilters: []xdsresource.HTTPFilter{{Name: "router"}}, + }, + } + if err := verifyListenerUpdate(ctx, updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + + // Wait for the watch expiry timer to fire, and verify that the callback is + // not invoked. + <-time.After(defaultTestWatchExpiryTimeout) + if err := verifyNoListenerUpdate(ctx, updateCh); err != nil { + t.Fatal(err) + } +} + +// TestLDSWatch_ResourceRemoved covers the cases where a resource being watched +// is removed from the management server. The test verifies the following +// scenarios: +// 1. Removing a resource should trigger the watch callback with a resource +// removed error. It should not trigger the watch callback for an unrelated +// resource. +// 2. An update to another resource should result in the invocation of the watch +// callback associated with that resource. It should not result in the +// invocation of the watch callback associated with the deleted resource. +// +// The test is run with both old and new style names. +func (s) TestLDSWatch_ResourceRemoved(t *testing.T) { + overrideFedEnvVar(t) + mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup() + + // Create an xDS client with the above bootstrap contents. + client, close, err := xdsclient.NewWithBootstrapContentsForTesting(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register two watches for two listener resources and have the + // callbacks push the received updates on to a channel. + resourceName1 := ldsName + updateCh1 := testutils.NewChannel() + ldsCancel1 := client.WatchListener(resourceName1, func(u xdsresource.ListenerUpdate, err error) { + updateCh1.Send(xdsresource.ListenerUpdateErrTuple{Update: u, Err: err}) + }) + defer ldsCancel1() + + resourceName2 := ldsNameNewStyle + updateCh2 := testutils.NewChannel() + ldsCancel2 := client.WatchListener(resourceName2, func(u xdsresource.ListenerUpdate, err error) { + updateCh2.Send(xdsresource.ListenerUpdateErrTuple{Update: u, Err: err}) + }) + defer ldsCancel2() + + // Configure the management server to return two listener resources, + // corresponding to the registered watches. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{ + e2e.DefaultClientListener(resourceName1, rdsName), + e2e.DefaultClientListener(resourceName2, rdsName), + }, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update for both watchers. The two + // resources returned differ only in the resource name. Therefore the + // expected update is the same for both watchers. + wantUpdate := xdsresource.ListenerUpdateErrTuple{ + Update: xdsresource.ListenerUpdate{ + RouteConfigName: rdsName, + HTTPFilters: []xdsresource.HTTPFilter{{Name: "router"}}, + }, + } + if err := verifyListenerUpdate(ctx, updateCh1, wantUpdate); err != nil { + t.Fatal(err) + } + if err := verifyListenerUpdate(ctx, updateCh2, wantUpdate); err != nil { + t.Fatal(err) + } + + // Remove the first listener resource on the management server. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(resourceName2, rdsName)}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // The first watcher should receive a resource removed error, while the + // second watcher should not see an update. + if err := verifyListenerUpdate(ctx, updateCh1, xdsresource.ListenerUpdateErrTuple{ + Err: xdsresource.NewErrorf(xdsresource.ErrorTypeResourceNotFound, ""), + }); err != nil { + t.Fatal(err) + } + if err := verifyNoListenerUpdate(ctx, updateCh2); err != nil { + t.Fatal(err) + } + + // Update the second listener resource on the management server. The first + // watcher should not see an update, while the second watcher should. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultClientListener(resourceName2, "new-rds-resource")}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + if err := verifyNoListenerUpdate(ctx, updateCh1); err != nil { + t.Fatal(err) + } + wantUpdate = xdsresource.ListenerUpdateErrTuple{ + Update: xdsresource.ListenerUpdate{ + RouteConfigName: "new-rds-resource", + HTTPFilters: []xdsresource.HTTPFilter{{Name: "router"}}, + }, + } + if err := verifyListenerUpdate(ctx, updateCh2, wantUpdate); err != nil { + t.Fatal(err) + } +} + +// TestLDSWatch_NACKError covers the case where an update from the management +// server is NACK'ed by the xdsclient. The test verifies that the error is +// propagated to the watcher. +func (s) TestLDSWatch_NACKError(t *testing.T) { + overrideFedEnvVar(t) + mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup() + + // Create an xDS client with the above bootstrap contents. + client, close, err := xdsclient.NewWithBootstrapContentsForTesting(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register a watch for a listener resource and have the watch + // callback push the received update on to a channel. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + updateCh := testutils.NewChannel() + ldsCancel := client.WatchListener(ldsName, func(u xdsresource.ListenerUpdate, err error) { + updateCh.SendContext(ctx, xdsresource.ListenerUpdateErrTuple{Update: u, Err: err}) + }) + defer ldsCancel() + + // Configure the management server to return a single listener resource + // which is expected to be NACKed by the client. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{badListenerResource(ldsName)}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify that the expected error is propagated to the watcher. + u, err := updateCh.Receive(ctx) + if err != nil { + t.Fatalf("timeout when waiting for a listener resource from the management server: %v", err) + } + gotErr := u.(xdsresource.ListenerUpdateErrTuple).Err + if gotErr == nil || !strings.Contains(gotErr.Error(), wantListenerNACKErr) { + t.Fatalf("update received with error: %v, want %q", gotErr, wantListenerNACKErr) + } +} + +// TestLDSWatch_PartialValid covers the case where a response from the +// management server contains both valid and invalid resources and is expected +// to be NACK'ed by the xdsclient. The test verifies that watchers corresponding +// to the valid resource receive the update, while watchers corresponding to the +// invalid resource receive an error. +func (s) TestLDSWatch_PartialValid(t *testing.T) { + overrideFedEnvVar(t) + mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup() + + // Create an xDS client with the above bootstrap contents. + client, close, err := xdsclient.NewWithBootstrapContentsForTesting(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register two watches for listener resources. The first watch is expected + // to receive an error because the received resource is NACKed. The second + // watch is expected to get a good update. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + badResourceName := ldsName + updateCh1 := testutils.NewChannel() + ldsCancel1 := client.WatchListener(badResourceName, func(u xdsresource.ListenerUpdate, err error) { + updateCh1.SendContext(ctx, xdsresource.ListenerUpdateErrTuple{Update: u, Err: err}) + }) + defer ldsCancel1() + goodResourceName := ldsNameNewStyle + updateCh2 := testutils.NewChannel() + ldsCancel2 := client.WatchListener(goodResourceName, func(u xdsresource.ListenerUpdate, err error) { + updateCh2.SendContext(ctx, xdsresource.ListenerUpdateErrTuple{Update: u, Err: err}) + }) + defer ldsCancel2() + + // Configure the management server with two listener resources. One of these + // is a bad resource causing the update to be NACKed. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{ + badListenerResource(badResourceName), + e2e.DefaultClientListener(goodResourceName, rdsName), + }, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify that the expected error is propagated to the watcher which + // requested for the bad resource. + u, err := updateCh1.Receive(ctx) + if err != nil { + t.Fatalf("timeout when waiting for a listener resource from the management server: %v", err) + } + gotErr := u.(xdsresource.ListenerUpdateErrTuple).Err + if gotErr == nil || !strings.Contains(gotErr.Error(), wantListenerNACKErr) { + t.Fatalf("update received with error: %v, want %q", gotErr, wantListenerNACKErr) + } + + // Verify that the watcher watching the good resource receives a good + // update. + wantUpdate := xdsresource.ListenerUpdateErrTuple{ + Update: xdsresource.ListenerUpdate{ + RouteConfigName: rdsName, + HTTPFilters: []xdsresource.HTTPFilter{{Name: "router"}}, + }, + } + if err := verifyListenerUpdate(ctx, updateCh2, wantUpdate); err != nil { + t.Fatal(err) + } +} + +// TestLDSWatch_PartialResponse covers the case where a response from the +// management server does not contain all requested resources. LDS responses are +// supposed to contain all requested resources, and the absence of one usually +// indicates that the management server does not know about it. In cases where +// the server has never responded with this resource before, the xDS client is +// expected to wait for the watch timeout to expire before concluding that the +// resource does not exist on the server +func (s) TestLDSWatch_PartialResponse(t *testing.T) { + overrideFedEnvVar(t) + mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup() + + // Create an xDS client with the above bootstrap contents. + client, close, err := xdsclient.NewWithBootstrapContentsForTesting(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register two watches for two listener resources and have the + // callbacks push the received updates on to a channel. + resourceName1 := ldsName + updateCh1 := testutils.NewChannel() + ldsCancel1 := client.WatchListener(resourceName1, func(u xdsresource.ListenerUpdate, err error) { + updateCh1.Send(xdsresource.ListenerUpdateErrTuple{Update: u, Err: err}) + }) + defer ldsCancel1() + + resourceName2 := ldsNameNewStyle + updateCh2 := testutils.NewChannel() + ldsCancel2 := client.WatchListener(resourceName2, func(u xdsresource.ListenerUpdate, err error) { + updateCh2.Send(xdsresource.ListenerUpdateErrTuple{Update: u, Err: err}) + }) + defer ldsCancel2() + + // Configure the management server to return only one of the two listener + // resources, corresponding to the registered watches. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{ + e2e.DefaultClientListener(resourceName1, rdsName), + }, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update for first watcher. + wantUpdate1 := xdsresource.ListenerUpdateErrTuple{ + Update: xdsresource.ListenerUpdate{ + RouteConfigName: rdsName, + HTTPFilters: []xdsresource.HTTPFilter{{Name: "router"}}, + }, + } + if err := verifyListenerUpdate(ctx, updateCh1, wantUpdate1); err != nil { + t.Fatal(err) + } + + // Verify that the second watcher does not get an update with an error. + if err := verifyNoListenerUpdate(ctx, updateCh2); err != nil { + t.Fatal(err) + } + + // Configure the management server to return two listener resources, + // corresponding to the registered watches. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{ + e2e.DefaultClientListener(resourceName1, rdsName), + e2e.DefaultClientListener(resourceName2, rdsName), + }, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update for the second watcher. + wantUpdate2 := xdsresource.ListenerUpdateErrTuple{ + Update: xdsresource.ListenerUpdate{ + RouteConfigName: rdsName, + HTTPFilters: []xdsresource.HTTPFilter{{Name: "router"}}, + }, + } + if err := verifyListenerUpdate(ctx, updateCh2, wantUpdate2); err != nil { + t.Fatal(err) + } + + // Verify that the first watcher gets no update, as the first resource did + // not change. + if err := verifyNoListenerUpdate(ctx, updateCh1); err != nil { + t.Fatal(err) + } +} diff --git a/xds/internal/xdsclient/tests/misc_watchers_test.go b/xds/internal/xdsclient/tests/misc_watchers_test.go new file mode 100644 index 000000000000..19cf7daba3fe --- /dev/null +++ b/xds/internal/xdsclient/tests/misc_watchers_test.go @@ -0,0 +1,311 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient_test + +import ( + "context" + "fmt" + "testing" + + "github.com/google/uuid" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/bootstrap" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/testutils/xds/fakeserver" + "google.golang.org/grpc/xds/internal" + xdstestutils "google.golang.org/grpc/xds/internal/testutils" + "google.golang.org/grpc/xds/internal/xdsclient" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource/version" + "google.golang.org/protobuf/types/known/anypb" + + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" +) + +var ( + // Resource type implementations retrieved from the resource type map in the + // internal package, which is initialized when the individual resource types + // are created. + listenerResourceType = internal.ResourceTypeMapForTesting[version.V3ListenerURL].(xdsresource.Type) + routeConfigResourceType = internal.ResourceTypeMapForTesting[version.V3RouteConfigURL].(xdsresource.Type) +) + +// TestWatchCallAnotherWatch tests the scenario where a watch is registered for +// a resource, and more watches are registered from the first watch's callback. +// The test verifies that this scenario does not lead to a deadlock. +func (s) TestWatchCallAnotherWatch(t *testing.T) { + overrideFedEnvVar(t) + + // Start an xDS management server and set the option to allow it to respond + // to requests which only specify a subset of the configured resources. + mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true}) + defer cleanup() + + // Create an xDS client with the above bootstrap contents. + client, close, err := xdsclient.NewWithBootstrapContentsForTesting(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Configure the management server to respond with route config resources. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Routes: []*v3routepb.RouteConfiguration{ + e2e.DefaultRouteConfig(rdsName, ldsName, cdsName), + e2e.DefaultRouteConfig(rdsNameNewStyle, ldsNameNewStyle, cdsName), + }, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Start a watch for one route configuration resource. From the watch + // callback of the first resource, register two more watches (one for the + // same resource name, which would be satisfied from the cache, and another + // for a different resource name, which would be satisfied from the server). + updateCh1 := testutils.NewChannel() + updateCh2 := testutils.NewChannel() + updateCh3 := testutils.NewChannel() + rdsCancel1 := client.WatchRouteConfig(rdsName, func(u xdsresource.RouteConfigUpdate, err error) { + updateCh1.Send(xdsresource.RouteConfigUpdateErrTuple{Update: u, Err: err}) + + // Watch for the same resource name. + rdsCancel2 := client.WatchRouteConfig(rdsName, func(u xdsresource.RouteConfigUpdate, err error) { + updateCh2.Send(xdsresource.RouteConfigUpdateErrTuple{Update: u, Err: err}) + }) + t.Cleanup(rdsCancel2) + // Watch for a different resource name. + rdsCancel3 := client.WatchRouteConfig(rdsNameNewStyle, func(u xdsresource.RouteConfigUpdate, err error) { + updateCh3.Send(xdsresource.RouteConfigUpdateErrTuple{Update: u, Err: err}) + }) + t.Cleanup(rdsCancel3) + }) + t.Cleanup(rdsCancel1) + + // Verify the contents of the received update for the all watchers. + wantUpdate12 := xdsresource.RouteConfigUpdateErrTuple{ + Update: xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{ + { + Domains: []string{ldsName}, + Routes: []*xdsresource.Route{ + { + Prefix: newStringP("/"), + ActionType: xdsresource.RouteActionRoute, + WeightedClusters: map[string]xdsresource.WeightedCluster{cdsName: {Weight: 100}}, + }, + }, + }, + }, + }, + } + wantUpdate3 := xdsresource.RouteConfigUpdateErrTuple{ + Update: xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{ + { + Domains: []string{ldsNameNewStyle}, + Routes: []*xdsresource.Route{ + { + Prefix: newStringP("/"), + ActionType: xdsresource.RouteActionRoute, + WeightedClusters: map[string]xdsresource.WeightedCluster{cdsName: {Weight: 100}}, + }, + }, + }, + }, + }, + } + if err := verifyRouteConfigUpdate(ctx, updateCh1, wantUpdate12); err != nil { + t.Fatal(err) + } + if err := verifyRouteConfigUpdate(ctx, updateCh2, wantUpdate12); err != nil { + t.Fatal(err) + } + if err := verifyRouteConfigUpdate(ctx, updateCh3, wantUpdate3); err != nil { + t.Fatal(err) + } +} + +// TestNodeProtoSentOnlyInFirstRequest verifies that a non-empty node proto gets +// sent only on the first discovery request message on the ADS stream. +// +// It also verifies the same behavior holds after a stream restart. +func (s) TestNodeProtoSentOnlyInFirstRequest(t *testing.T) { + overrideFedEnvVar(t) + + // Create a restartable listener which can close existing connections. + l, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("testutils.LocalTCPListener() failed: %v", err) + } + lis := testutils.NewRestartableListener(l) + + // Start a fake xDS management server with the above restartable listener. + // + // We are unable to use the go-control-plane server here, because it caches + // the node proto received in the first request message and adds it to + // subsequent requests before invoking the OnStreamRequest() callback. + // Therefore we cannot verify what is sent by the xDS client. + mgmtServer, cleanup, err := fakeserver.StartServer(lis) + if err != nil { + t.Fatalf("Failed to start fake xDS server: %v", err) + } + defer cleanup() + + // Create a bootstrap file in a temporary directory. + nodeID := uuid.New().String() + bootstrapContents, err := bootstrap.Contents(bootstrap.Options{ + NodeID: nodeID, + ServerURI: mgmtServer.Address, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap file: %v", err) + } + + // Create an xDS client with the above bootstrap contents. + client, close, err := xdsclient.NewWithBootstrapContentsForTesting(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + const ( + serviceName = "my-service-client-side-xds" + routeConfigName = "route-" + serviceName + clusterName = "cluster-" + serviceName + ) + + // Register a watch for the Listener resource. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + watcher := xdstestutils.NewTestResourceWatcher() + client.WatchResource(listenerResourceType, serviceName, watcher) + + // Ensure the watch results in a discovery request with an empty node proto. + if err := readDiscoveryResponseAndCheckForNonEmptyNodeProto(ctx, mgmtServer.XDSRequestChan); err != nil { + t.Fatal(err) + } + + // Configure a listener resource on the fake xDS server. + lisAny, err := anypb.New(e2e.DefaultClientListener(serviceName, routeConfigName)) + if err != nil { + t.Fatalf("Failed to marshal listener resource into an Any proto: %v", err) + } + mgmtServer.XDSResponseChan <- &fakeserver.Response{ + Resp: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + VersionInfo: "1", + Resources: []*anypb.Any{lisAny}, + }, + } + + // The xDS client is expected to ACK the Listener resource. The discovery + // request corresponding to the ACK must contain a nil node proto. + if err := readDiscoveryResponseAndCheckForEmptyNodeProto(ctx, mgmtServer.XDSRequestChan); err != nil { + t.Fatal(err) + } + + // Register a watch for a RouteConfiguration resource. + client.WatchResource(routeConfigResourceType, routeConfigName, watcher) + + // Ensure the watch results in a discovery request with an empty node proto. + if err := readDiscoveryResponseAndCheckForEmptyNodeProto(ctx, mgmtServer.XDSRequestChan); err != nil { + t.Fatal(err) + } + + // Configure the route configuration resource on the fake xDS server. + rcAny, err := anypb.New(e2e.DefaultRouteConfig(routeConfigName, serviceName, clusterName)) + if err != nil { + t.Fatalf("Failed to marshal route configuration resource into an Any proto: %v", err) + } + mgmtServer.XDSResponseChan <- &fakeserver.Response{ + Resp: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + VersionInfo: "1", + Resources: []*anypb.Any{rcAny}, + }, + } + + // Ensure the discovery request for the ACK contains an empty node proto. + if err := readDiscoveryResponseAndCheckForEmptyNodeProto(ctx, mgmtServer.XDSRequestChan); err != nil { + t.Fatal(err) + } + + // Stop the management server and expect the error callback to be invoked. + lis.Stop() + select { + case <-ctx.Done(): + t.Fatal("Timeout when waiting for the connection error to be propagated to the watcher") + case <-watcher.ErrorCh: + } + + // Restart the management server. + lis.Restart() + + // The xDS client is expected to re-request previously requested resources. + // Hence, we expect two DiscoveryRequest messages (one for the Listener and + // one for the RouteConfiguration resource). The first message should contain + // a non-nil node proto and the second should contain a nil-proto. + // + // And since we don't push any responses on the response channel of the fake + // server, we do not expect any ACKs here. + if err := readDiscoveryResponseAndCheckForNonEmptyNodeProto(ctx, mgmtServer.XDSRequestChan); err != nil { + t.Fatal(err) + } + if err := readDiscoveryResponseAndCheckForEmptyNodeProto(ctx, mgmtServer.XDSRequestChan); err != nil { + t.Fatal(err) + } +} + +// readDiscoveryResponseAndCheckForEmptyNodeProto reads a discovery request +// message out of the provided reqCh. It returns an error if it fails to read a +// message before the context deadline expires, or if the read message contains +// a non-empty node proto. +func readDiscoveryResponseAndCheckForEmptyNodeProto(ctx context.Context, reqCh *testutils.Channel) error { + v, err := reqCh.Receive(ctx) + if err != nil { + return fmt.Errorf("Timeout when waiting for a DiscoveryRequest message") + } + req := v.(*fakeserver.Request).Req.(*v3discoverypb.DiscoveryRequest) + if node := req.GetNode(); node != nil { + return fmt.Errorf("Node proto received in DiscoveryRequest message is %v, want empty node proto", node) + } + return nil +} + +// readDiscoveryResponseAndCheckForNonEmptyNodeProto reads a discovery request +// message out of the provided reqCh. It returns an error if it fails to read a +// message before the context deadline expires, or if the read message contains +// an empty node proto. +func readDiscoveryResponseAndCheckForNonEmptyNodeProto(ctx context.Context, reqCh *testutils.Channel) error { + v, err := reqCh.Receive(ctx) + if err != nil { + return fmt.Errorf("Timeout when waiting for a DiscoveryRequest message") + } + req := v.(*fakeserver.Request).Req.(*v3discoverypb.DiscoveryRequest) + if node := req.GetNode(); node == nil { + return fmt.Errorf("Empty node proto received in DiscoveryRequest message, want non-empty node proto") + } + return nil +} diff --git a/xds/internal/xdsclient/tests/rds_watchers_test.go b/xds/internal/xdsclient/tests/rds_watchers_test.go new file mode 100644 index 000000000000..b03b9ce259bc --- /dev/null +++ b/xds/internal/xdsclient/tests/rds_watchers_test.go @@ -0,0 +1,857 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient_test + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/google/uuid" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + xdstestutils "google.golang.org/grpc/xds/internal/testutils" + "google.golang.org/grpc/xds/internal/xdsclient" + "google.golang.org/grpc/xds/internal/xdsclient/bootstrap" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" + "google.golang.org/protobuf/types/known/wrapperspb" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" +) + +// badRouteConfigResource returns a RouteConfiguration resource for the given +// routeName which contains a retry config with num_retries set to `0`. This is +// expected to be NACK'ed by the xDS client. +func badRouteConfigResource(routeName, ldsTarget, clusterName string) *v3routepb.RouteConfiguration { + return &v3routepb.RouteConfiguration{ + Name: routeName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{ldsTarget}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}}, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName}, + }}}}, + RetryPolicy: &v3routepb.RetryPolicy{ + NumRetries: &wrapperspb.UInt32Value{Value: 0}, + }, + }}, + } +} + +// xdsClient is expected to produce an error containing this string when an +// update is received containing a route configuration resource created using +// `badRouteConfigResource`. +const wantRouteConfigNACKErr = "received route is invalid: retry_policy.num_retries = 0; must be >= 1" + +// verifyRouteConfigUpdate waits for an update to be received on the provided +// update channel and verifies that it matches the expected update. +// +// Returns an error if no update is received before the context deadline expires +// or the received update does not match the expected one. +func verifyRouteConfigUpdate(ctx context.Context, updateCh *testutils.Channel, wantUpdate xdsresource.RouteConfigUpdateErrTuple) error { + u, err := updateCh.Receive(ctx) + if err != nil { + return fmt.Errorf("timeout when waiting for a route configuration resource from the management server: %v", err) + } + got := u.(xdsresource.RouteConfigUpdateErrTuple) + if wantUpdate.Err != nil { + if gotType, wantType := xdsresource.ErrType(got.Err), xdsresource.ErrType(wantUpdate.Err); gotType != wantType { + return fmt.Errorf("received update with error type %v, want %v", gotType, wantType) + } + } + cmpOpts := []cmp.Option{cmpopts.EquateEmpty(), cmpopts.IgnoreFields(xdsresource.RouteConfigUpdate{}, "Raw")} + if diff := cmp.Diff(wantUpdate.Update, got.Update, cmpOpts...); diff != "" { + return fmt.Errorf("received unepected diff in the route configuration resource update: (-want, got):\n%s", diff) + } + return nil +} + +// verifyNoRouteConfigUpdate verifies that no route configuration update is +// received on the provided update channel, and returns an error if an update is +// received. +// +// A very short deadline is used while waiting for the update, as this function +// is intended to be used when an update is not expected. +func verifyNoRouteConfigUpdate(ctx context.Context, updateCh *testutils.Channel) error { + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + if u, err := updateCh.Receive(sCtx); err != context.DeadlineExceeded { + return fmt.Errorf("unexpected RouteConfigUpdate: %v", u) + } + return nil +} + +// TestRDSWatch covers the case where a single watcher exists for a single route +// configuration resource. The test verifies the following scenarios: +// 1. An update from the management server containing the resource being +// watched should result in the invocation of the watch callback. +// 2. An update from the management server containing a resource *not* being +// watched should not result in the invocation of the watch callback. +// 3. After the watch is cancelled, an update from the management server +// containing the resource that was being watched should not result in the +// invocation of the watch callback. +// +// The test is run for old and new style names. +func (s) TestRDSWatch(t *testing.T) { + tests := []struct { + desc string + resourceName string + watchedResource *v3routepb.RouteConfiguration // The resource being watched. + updatedWatchedResource *v3routepb.RouteConfiguration // The watched resource after an update. + notWatchedResource *v3routepb.RouteConfiguration // A resource which is not being watched. + wantUpdate xdsresource.RouteConfigUpdateErrTuple + }{ + { + desc: "old style resource", + resourceName: rdsName, + watchedResource: e2e.DefaultRouteConfig(rdsName, ldsName, cdsName), + updatedWatchedResource: e2e.DefaultRouteConfig(rdsName, ldsName, "new-cds-resource"), + notWatchedResource: e2e.DefaultRouteConfig("unsubscribed-rds-resource", ldsName, cdsName), + wantUpdate: xdsresource.RouteConfigUpdateErrTuple{ + Update: xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{ + { + Domains: []string{ldsName}, + Routes: []*xdsresource.Route{ + { + Prefix: newStringP("/"), + ActionType: xdsresource.RouteActionRoute, + WeightedClusters: map[string]xdsresource.WeightedCluster{cdsName: {Weight: 100}}, + }, + }, + }, + }, + }, + }, + }, + { + desc: "new style resource", + resourceName: rdsNameNewStyle, + watchedResource: e2e.DefaultRouteConfig(rdsNameNewStyle, ldsNameNewStyle, cdsNameNewStyle), + updatedWatchedResource: e2e.DefaultRouteConfig(rdsNameNewStyle, ldsNameNewStyle, "new-cds-resource"), + notWatchedResource: e2e.DefaultRouteConfig("unsubscribed-rds-resource", ldsNameNewStyle, cdsNameNewStyle), + wantUpdate: xdsresource.RouteConfigUpdateErrTuple{ + Update: xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{ + { + Domains: []string{ldsNameNewStyle}, + Routes: []*xdsresource.Route{ + { + Prefix: newStringP("/"), + ActionType: xdsresource.RouteActionRoute, + WeightedClusters: map[string]xdsresource.WeightedCluster{cdsNameNewStyle: {Weight: 100}}, + }, + }, + }, + }, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + overrideFedEnvVar(t) + mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup() + + // Create an xDS client with the above bootstrap contents. + client, close, err := xdsclient.NewWithBootstrapContentsForTesting(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register a watch for a route configuration resource and have the + // watch callback push the received update on to a channel. + updateCh := testutils.NewChannel() + rdsCancel := client.WatchRouteConfig(test.resourceName, func(u xdsresource.RouteConfigUpdate, err error) { + updateCh.Send(xdsresource.RouteConfigUpdateErrTuple{Update: u, Err: err}) + }) + + // Configure the management server to return a single route + // configuration resource, corresponding to the one being watched. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Routes: []*v3routepb.RouteConfiguration{test.watchedResource}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update. + if err := verifyRouteConfigUpdate(ctx, updateCh, test.wantUpdate); err != nil { + t.Fatal(err) + } + + // Configure the management server to return an additional route + // configuration resource, one that we are not interested in. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Routes: []*v3routepb.RouteConfiguration{test.watchedResource, test.notWatchedResource}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + if err := verifyNoRouteConfigUpdate(ctx, updateCh); err != nil { + t.Fatal(err) + } + + // Cancel the watch and update the resource corresponding to the original + // watch. Ensure that the cancelled watch callback is not invoked. + rdsCancel() + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Routes: []*v3routepb.RouteConfiguration{test.updatedWatchedResource, test.notWatchedResource}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + if err := verifyNoRouteConfigUpdate(ctx, updateCh); err != nil { + t.Fatal(err) + } + }) + } +} + +// TestRDSWatch_TwoWatchesForSameResourceName covers the case where two watchers +// exist for a single route configuration resource. The test verifies the +// following scenarios: +// 1. An update from the management server containing the resource being +// watched should result in the invocation of both watch callbacks. +// 2. After one of the watches is cancelled, a redundant update from the +// management server should not result in the invocation of either of the +// watch callbacks. +// 3. An update from the management server containing the resource being +// watched should result in the invocation of the un-cancelled watch +// callback. +// +// The test is run for old and new style names. +func (s) TestRDSWatch_TwoWatchesForSameResourceName(t *testing.T) { + tests := []struct { + desc string + resourceName string + watchedResource *v3routepb.RouteConfiguration // The resource being watched. + updatedWatchedResource *v3routepb.RouteConfiguration // The watched resource after an update. + wantUpdateV1 xdsresource.RouteConfigUpdateErrTuple + wantUpdateV2 xdsresource.RouteConfigUpdateErrTuple + }{ + { + desc: "old style resource", + resourceName: rdsName, + watchedResource: e2e.DefaultRouteConfig(rdsName, ldsName, cdsName), + updatedWatchedResource: e2e.DefaultRouteConfig(rdsName, ldsName, "new-cds-resource"), + wantUpdateV1: xdsresource.RouteConfigUpdateErrTuple{ + Update: xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{ + { + Domains: []string{ldsName}, + Routes: []*xdsresource.Route{ + { + Prefix: newStringP("/"), + ActionType: xdsresource.RouteActionRoute, + WeightedClusters: map[string]xdsresource.WeightedCluster{cdsName: {Weight: 100}}, + }, + }, + }, + }, + }, + }, + wantUpdateV2: xdsresource.RouteConfigUpdateErrTuple{ + Update: xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{ + { + Domains: []string{ldsName}, + Routes: []*xdsresource.Route{ + { + Prefix: newStringP("/"), + ActionType: xdsresource.RouteActionRoute, + WeightedClusters: map[string]xdsresource.WeightedCluster{"new-cds-resource": {Weight: 100}}, + }, + }, + }, + }, + }, + }, + }, + { + desc: "new style resource", + resourceName: rdsNameNewStyle, + watchedResource: e2e.DefaultRouteConfig(rdsNameNewStyle, ldsNameNewStyle, cdsNameNewStyle), + updatedWatchedResource: e2e.DefaultRouteConfig(rdsNameNewStyle, ldsNameNewStyle, "new-cds-resource"), + wantUpdateV1: xdsresource.RouteConfigUpdateErrTuple{ + Update: xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{ + { + Domains: []string{ldsNameNewStyle}, + Routes: []*xdsresource.Route{ + { + Prefix: newStringP("/"), + ActionType: xdsresource.RouteActionRoute, + WeightedClusters: map[string]xdsresource.WeightedCluster{cdsNameNewStyle: {Weight: 100}}, + }, + }, + }, + }, + }, + }, + wantUpdateV2: xdsresource.RouteConfigUpdateErrTuple{ + Update: xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{ + { + Domains: []string{ldsNameNewStyle}, + Routes: []*xdsresource.Route{ + { + Prefix: newStringP("/"), + ActionType: xdsresource.RouteActionRoute, + WeightedClusters: map[string]xdsresource.WeightedCluster{"new-cds-resource": {Weight: 100}}, + }, + }, + }, + }, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + overrideFedEnvVar(t) + mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup() + + // Create an xDS client with the above bootstrap contents. + client, close, err := xdsclient.NewWithBootstrapContentsForTesting(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register two watches for the same route configuration resource + // and have the callbacks push the received updates on to a channel. + updateCh1 := testutils.NewChannel() + rdsCancel1 := client.WatchRouteConfig(test.resourceName, func(u xdsresource.RouteConfigUpdate, err error) { + updateCh1.Send(xdsresource.RouteConfigUpdateErrTuple{Update: u, Err: err}) + }) + defer rdsCancel1() + updateCh2 := testutils.NewChannel() + rdsCancel2 := client.WatchRouteConfig(test.resourceName, func(u xdsresource.RouteConfigUpdate, err error) { + updateCh2.Send(xdsresource.RouteConfigUpdateErrTuple{Update: u, Err: err}) + }) + + // Configure the management server to return a single route + // configuration resource, corresponding to the one being watched. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Routes: []*v3routepb.RouteConfiguration{test.watchedResource}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update. + if err := verifyRouteConfigUpdate(ctx, updateCh1, test.wantUpdateV1); err != nil { + t.Fatal(err) + } + if err := verifyRouteConfigUpdate(ctx, updateCh2, test.wantUpdateV1); err != nil { + t.Fatal(err) + } + + // Cancel the second watch and force the management server to push a + // redundant update for the resource being watched. Neither of the + // two watch callbacks should be invoked. + rdsCancel2() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + if err := verifyNoRouteConfigUpdate(ctx, updateCh1); err != nil { + t.Fatal(err) + } + if err := verifyNoRouteConfigUpdate(ctx, updateCh2); err != nil { + t.Fatal(err) + } + + // Update to the resource being watched. The un-cancelled callback + // should be invoked while the cancelled one should not be. + resources = e2e.UpdateOptions{ + NodeID: nodeID, + Routes: []*v3routepb.RouteConfiguration{test.updatedWatchedResource}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + if err := verifyRouteConfigUpdate(ctx, updateCh1, test.wantUpdateV2); err != nil { + t.Fatal(err) + } + if err := verifyNoRouteConfigUpdate(ctx, updateCh2); err != nil { + t.Fatal(err) + } + }) + } +} + +// TestRDSWatch_ThreeWatchesForDifferentResourceNames covers the case with three +// watchers (two watchers for one resource, and the third watcher for another +// resource), exist across two route configuration resources. The test verifies +// that an update from the management server containing both resources results +// in the invocation of all watch callbacks. +// +// The test is run with both old and new style names. +func (s) TestRDSWatch_ThreeWatchesForDifferentResourceNames(t *testing.T) { + overrideFedEnvVar(t) + mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup() + + // Create an xDS client with the above bootstrap contents. + client, close, err := xdsclient.NewWithBootstrapContentsForTesting(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register two watches for the same route configuration resource + // and have the callbacks push the received updates on to a channel. + updateCh1 := testutils.NewChannel() + rdsCancel1 := client.WatchRouteConfig(rdsName, func(u xdsresource.RouteConfigUpdate, err error) { + updateCh1.Send(xdsresource.RouteConfigUpdateErrTuple{Update: u, Err: err}) + }) + defer rdsCancel1() + updateCh2 := testutils.NewChannel() + rdsCancel2 := client.WatchRouteConfig(rdsName, func(u xdsresource.RouteConfigUpdate, err error) { + updateCh2.Send(xdsresource.RouteConfigUpdateErrTuple{Update: u, Err: err}) + }) + defer rdsCancel2() + + // Register the third watch for a different route configuration resource. + updateCh3 := testutils.NewChannel() + rdsCancel3 := client.WatchRouteConfig(rdsNameNewStyle, func(u xdsresource.RouteConfigUpdate, err error) { + updateCh3.Send(xdsresource.RouteConfigUpdateErrTuple{Update: u, Err: err}) + }) + defer rdsCancel3() + + // Configure the management server to return two route configuration + // resources, corresponding to the registered watches. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Routes: []*v3routepb.RouteConfiguration{ + e2e.DefaultRouteConfig(rdsName, ldsName, cdsName), + e2e.DefaultRouteConfig(rdsNameNewStyle, ldsName, cdsName), + }, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update for the all watchers. The two + // resources returned differ only in the resource name. Therefore the + // expected update is the same for all the watchers. + wantUpdate := xdsresource.RouteConfigUpdateErrTuple{ + Update: xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{ + { + Domains: []string{ldsName}, + Routes: []*xdsresource.Route{ + { + Prefix: newStringP("/"), + ActionType: xdsresource.RouteActionRoute, + WeightedClusters: map[string]xdsresource.WeightedCluster{cdsName: {Weight: 100}}, + }, + }, + }, + }, + }, + } + if err := verifyRouteConfigUpdate(ctx, updateCh1, wantUpdate); err != nil { + t.Fatal(err) + } + if err := verifyRouteConfigUpdate(ctx, updateCh2, wantUpdate); err != nil { + t.Fatal(err) + } + if err := verifyRouteConfigUpdate(ctx, updateCh3, wantUpdate); err != nil { + t.Fatal(err) + } +} + +// TestRDSWatch_ResourceCaching covers the case where a watch is registered for +// a resource which is already present in the cache. The test verifies that the +// watch callback is invoked with the contents from the cache, instead of a +// request being sent to the management server. +func (s) TestRDSWatch_ResourceCaching(t *testing.T) { + overrideFedEnvVar(t) + firstRequestReceived := false + firstAckReceived := grpcsync.NewEvent() + secondRequestReceived := grpcsync.NewEvent() + + mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{ + OnStreamRequest: func(id int64, req *v3discoverypb.DiscoveryRequest) error { + // The first request has an empty version string. + if !firstRequestReceived && req.GetVersionInfo() == "" { + firstRequestReceived = true + return nil + } + // The first ack has a non-empty version string. + if !firstAckReceived.HasFired() && req.GetVersionInfo() != "" { + firstAckReceived.Fire() + return nil + } + // Any requests after the first request and ack, are not expected. + secondRequestReceived.Fire() + return nil + }, + }) + defer cleanup() + + // Create an xDS client with the above bootstrap contents. + client, close, err := xdsclient.NewWithBootstrapContentsForTesting(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register a watch for a route configuration resource and have the watch + // callback push the received update on to a channel. + updateCh1 := testutils.NewChannel() + rdsCancel1 := client.WatchRouteConfig(rdsName, func(u xdsresource.RouteConfigUpdate, err error) { + updateCh1.Send(xdsresource.RouteConfigUpdateErrTuple{Update: u, Err: err}) + }) + defer rdsCancel1() + + // Configure the management server to return a single route configuration + // resource, corresponding to the one we registered a watch for. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(rdsName, ldsName, cdsName)}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update. + wantUpdate := xdsresource.RouteConfigUpdateErrTuple{ + Update: xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{ + { + Domains: []string{ldsName}, + Routes: []*xdsresource.Route{ + { + Prefix: newStringP("/"), + ActionType: xdsresource.RouteActionRoute, + WeightedClusters: map[string]xdsresource.WeightedCluster{cdsName: {Weight: 100}}, + }, + }, + }, + }, + }, + } + if err := verifyRouteConfigUpdate(ctx, updateCh1, wantUpdate); err != nil { + t.Fatal(err) + } + select { + case <-ctx.Done(): + t.Fatal("timeout when waiting for receipt of ACK at the management server") + case <-firstAckReceived.Done(): + } + + // Register another watch for the same resource. This should get the update + // from the cache. + updateCh2 := testutils.NewChannel() + rdsCancel2 := client.WatchRouteConfig(rdsName, func(u xdsresource.RouteConfigUpdate, err error) { + updateCh2.Send(xdsresource.RouteConfigUpdateErrTuple{Update: u, Err: err}) + }) + defer rdsCancel2() + if err := verifyRouteConfigUpdate(ctx, updateCh2, wantUpdate); err != nil { + t.Fatal(err) + } + // No request should get sent out as part of this watch. + sCtx, sCancel := context.WithTimeout(ctx, defaultTestShortTimeout) + defer sCancel() + select { + case <-sCtx.Done(): + case <-secondRequestReceived.Done(): + t.Fatal("xdsClient sent out request instead of using update from cache") + } +} + +// TestRDSWatch_ExpiryTimerFiresBeforeResponse tests the case where the client +// does not receive an RDS response for the request that it sends. The test +// verifies that the watch callback is invoked with an error once the +// watchExpiryTimer fires. +func (s) TestRDSWatch_ExpiryTimerFiresBeforeResponse(t *testing.T) { + overrideFedEnvVar(t) + mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) + if err != nil { + t.Fatalf("Failed to spin up the xDS management server: %v", err) + } + defer mgmtServer.Stop() + + // Create an xDS client talking to a non-existent management server. + client, close, err := xdsclient.NewWithConfigForTesting(&bootstrap.Config{ + XDSServer: xdstestutils.ServerConfigForAddress(t, mgmtServer.Address), + NodeProto: &v3corepb.Node{}, + }, defaultTestWatchExpiryTimeout, time.Duration(0)) + if err != nil { + t.Fatalf("failed to create xds client: %v", err) + } + defer close() + + // Register a watch for a resource which is expected to fail with an error + // after the watch expiry timer fires. + updateCh := testutils.NewChannel() + rdsCancel := client.WatchRouteConfig(rdsName, func(u xdsresource.RouteConfigUpdate, err error) { + updateCh.Send(xdsresource.RouteConfigUpdateErrTuple{Update: u, Err: err}) + }) + defer rdsCancel() + + // Wait for the watch expiry timer to fire. + <-time.After(defaultTestWatchExpiryTimeout) + + // Verify that an empty update with the expected error is received. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + wantErr := xdsresource.NewErrorf(xdsresource.ErrorTypeResourceNotFound, "") + if err := verifyRouteConfigUpdate(ctx, updateCh, xdsresource.RouteConfigUpdateErrTuple{Err: wantErr}); err != nil { + t.Fatal(err) + } +} + +// TestRDSWatch_ValidResponseCancelsExpiryTimerBehavior tests the case where the +// client receives a valid RDS response for the request that it sends. The test +// verifies that the behavior associated with the expiry timer (i.e, callback +// invocation with error) does not take place. +func (s) TestRDSWatch_ValidResponseCancelsExpiryTimerBehavior(t *testing.T) { + overrideFedEnvVar(t) + mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) + if err != nil { + t.Fatalf("Failed to spin up the xDS management server: %v", err) + } + defer mgmtServer.Stop() + + // Create an xDS client talking to the above management server. + nodeID := uuid.New().String() + client, close, err := xdsclient.NewWithConfigForTesting(&bootstrap.Config{ + XDSServer: xdstestutils.ServerConfigForAddress(t, mgmtServer.Address), + NodeProto: &v3corepb.Node{Id: nodeID}, + }, defaultTestWatchExpiryTimeout, time.Duration(0)) + if err != nil { + t.Fatalf("failed to create xds client: %v", err) + } + defer close() + + // Register a watch for a route configuration resource and have the watch + // callback push the received update on to a channel. + updateCh := testutils.NewChannel() + rdsCancel := client.WatchRouteConfig(rdsName, func(u xdsresource.RouteConfigUpdate, err error) { + updateCh.Send(xdsresource.RouteConfigUpdateErrTuple{Update: u, Err: err}) + }) + defer rdsCancel() + + // Configure the management server to return a single route configuration + // resource, corresponding to the one we registered a watch for. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Routes: []*v3routepb.RouteConfiguration{e2e.DefaultRouteConfig(rdsName, ldsName, cdsName)}, + SkipValidation: true, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify the contents of the received update. + wantUpdate := xdsresource.RouteConfigUpdateErrTuple{ + Update: xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{ + { + Domains: []string{ldsName}, + Routes: []*xdsresource.Route{ + { + Prefix: newStringP("/"), + ActionType: xdsresource.RouteActionRoute, + WeightedClusters: map[string]xdsresource.WeightedCluster{cdsName: {Weight: 100}}, + }, + }, + }, + }, + }, + } + if err := verifyRouteConfigUpdate(ctx, updateCh, wantUpdate); err != nil { + t.Fatal(err) + } + + // Wait for the watch expiry timer to fire, and verify that the callback is + // not invoked. + <-time.After(defaultTestWatchExpiryTimeout) + if err := verifyNoRouteConfigUpdate(ctx, updateCh); err != nil { + t.Fatal(err) + } +} + +// TestRDSWatch_NACKError covers the case where an update from the management +// server is NACK'ed by the xdsclient. The test verifies that the error is +// propagated to the watcher. +func (s) TestRDSWatch_NACKError(t *testing.T) { + overrideFedEnvVar(t) + mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup() + + // Create an xDS client with the above bootstrap contents. + client, close, err := xdsclient.NewWithBootstrapContentsForTesting(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register a watch for a route configuration resource and have the watch + // callback push the received update on to a channel. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + updateCh := testutils.NewChannel() + rdsCancel := client.WatchRouteConfig(rdsName, func(u xdsresource.RouteConfigUpdate, err error) { + updateCh.SendContext(ctx, xdsresource.RouteConfigUpdateErrTuple{Update: u, Err: err}) + }) + defer rdsCancel() + + // Configure the management server to return a single route configuration + // resource which is expected to be NACKed by the client. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Routes: []*v3routepb.RouteConfiguration{badRouteConfigResource(rdsName, ldsName, cdsName)}, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify that the expected error is propagated to the watcher. + u, err := updateCh.Receive(ctx) + if err != nil { + t.Fatalf("timeout when waiting for a route configuration resource from the management server: %v", err) + } + gotErr := u.(xdsresource.RouteConfigUpdateErrTuple).Err + if gotErr == nil || !strings.Contains(gotErr.Error(), wantRouteConfigNACKErr) { + t.Fatalf("update received with error: %v, want %q", gotErr, wantRouteConfigNACKErr) + } +} + +// TestRDSWatch_PartialValid covers the case where a response from the +// management server contains both valid and invalid resources and is expected +// to be NACK'ed by the xdsclient. The test verifies that watchers corresponding +// to the valid resource receive the update, while watchers corresponding to the +// invalid resource receive an error. +func (s) TestRDSWatch_PartialValid(t *testing.T) { + overrideFedEnvVar(t) + mgmtServer, nodeID, bootstrapContents, _, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{}) + defer cleanup() + + // Create an xDS client with the above bootstrap contents. + client, close, err := xdsclient.NewWithBootstrapContentsForTesting(bootstrapContents) + if err != nil { + t.Fatalf("Failed to create xDS client: %v", err) + } + defer close() + + // Register two watches for route configuration resources. The first watch + // is expected to receive an error because the received resource is NACKed. + // The second watch is expected to get a good update. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + badResourceName := rdsName + updateCh1 := testutils.NewChannel() + rdsCancel1 := client.WatchRouteConfig(badResourceName, func(u xdsresource.RouteConfigUpdate, err error) { + updateCh1.SendContext(ctx, xdsresource.RouteConfigUpdateErrTuple{Update: u, Err: err}) + }) + defer rdsCancel1() + goodResourceName := rdsNameNewStyle + updateCh2 := testutils.NewChannel() + rdsCancel2 := client.WatchRouteConfig(goodResourceName, func(u xdsresource.RouteConfigUpdate, err error) { + updateCh2.SendContext(ctx, xdsresource.RouteConfigUpdateErrTuple{Update: u, Err: err}) + }) + defer rdsCancel2() + + // Configure the management server to return two route configuration + // resources, corresponding to the registered watches. + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Routes: []*v3routepb.RouteConfiguration{ + badRouteConfigResource(badResourceName, ldsName, cdsName), + e2e.DefaultRouteConfig(goodResourceName, ldsName, cdsName), + }, + SkipValidation: true, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatalf("Failed to update management server with resources: %v, err: %v", resources, err) + } + + // Verify that the expected error is propagated to the watcher which + // requested for the bad resource. + u, err := updateCh1.Receive(ctx) + if err != nil { + t.Fatalf("timeout when waiting for a route configuration resource from the management server: %v", err) + } + gotErr := u.(xdsresource.RouteConfigUpdateErrTuple).Err + if gotErr == nil || !strings.Contains(gotErr.Error(), wantRouteConfigNACKErr) { + t.Fatalf("update received with error: %v, want %q", gotErr, wantRouteConfigNACKErr) + } + + // Verify that the watcher watching the good resource receives a good + // update. + wantUpdate := xdsresource.RouteConfigUpdateErrTuple{ + Update: xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{ + { + Domains: []string{ldsName}, + Routes: []*xdsresource.Route{ + { + Prefix: newStringP("/"), + ActionType: xdsresource.RouteActionRoute, + WeightedClusters: map[string]xdsresource.WeightedCluster{cdsName: {Weight: 100}}, + }, + }, + }, + }, + }, + } + if err := verifyRouteConfigUpdate(ctx, updateCh2, wantUpdate); err != nil { + t.Fatal(err) + } +} diff --git a/xds/internal/xdsclient/tests/resource_update_test.go b/xds/internal/xdsclient/tests/resource_update_test.go new file mode 100644 index 000000000000..3a2ccc114e8d --- /dev/null +++ b/xds/internal/xdsclient/tests/resource_update_test.go @@ -0,0 +1,1118 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsclient_test + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/google/uuid" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/internal/testutils/xds/fakeserver" + "google.golang.org/grpc/xds/internal" + xdstestutils "google.golang.org/grpc/xds/internal/testutils" + "google.golang.org/grpc/xds/internal/xdsclient" + "google.golang.org/grpc/xds/internal/xdsclient/bootstrap" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/testing/protocmp" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/wrapperspb" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + + _ "google.golang.org/grpc/xds/internal/httpfilter/router" // Register the router filter. +) + +// startFakeManagementServer starts a fake xDS management server and returns a +// cleanup function to close the fake server. +func startFakeManagementServer(t *testing.T) (*fakeserver.Server, func()) { + t.Helper() + fs, sCleanup, err := fakeserver.StartServer(nil) + if err != nil { + t.Fatalf("Failed to start fake xDS server: %v", err) + } + return fs, sCleanup +} + +func compareUpdateMetadata(ctx context.Context, dumpFunc func() map[string]xdsresource.UpdateWithMD, want map[string]xdsresource.UpdateWithMD) error { + var lastErr error + for ; ctx.Err() == nil; <-time.After(100 * time.Millisecond) { + cmpOpts := cmp.Options{ + cmpopts.EquateEmpty(), + cmp.Comparer(func(a, b time.Time) bool { return true }), + cmpopts.EquateErrors(), + protocmp.Transform(), + } + gotUpdateMetadata := dumpFunc() + diff := cmp.Diff(want, gotUpdateMetadata, cmpOpts) + if diff == "" { + return nil + } + lastErr = fmt.Errorf("unexpected diff in metadata, diff (-want +got):\n%s\n want: %+v\n got: %+v", diff, want, gotUpdateMetadata) + } + return fmt.Errorf("timeout when waiting for expected update metadata: %v", lastErr) +} + +// TestHandleListenerResponseFromManagementServer covers different scenarios +// involving receipt of an LDS response from the management server. The test +// verifies that the internal state of the xDS client (parsed resource and +// metadata) matches expectations. +func (s) TestHandleListenerResponseFromManagementServer(t *testing.T) { + const ( + resourceName1 = "resource-name-1" + resourceName2 = "resource-name-2" + ) + var ( + emptyRouterFilter = e2e.RouterHTTPFilter + apiListener = &v3listenerpb.ApiListener{ + ApiListener: func() *anypb.Any { + return testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{ + Rds: &v3httppb.Rds{ + ConfigSource: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}, + }, + RouteConfigName: "route-configuration-name", + }, + }, + HttpFilters: []*v3httppb.HttpFilter{emptyRouterFilter}, + }) + }(), + } + resource1 = &v3listenerpb.Listener{ + Name: resourceName1, + ApiListener: apiListener, + } + resource2 = &v3listenerpb.Listener{ + Name: resourceName2, + ApiListener: apiListener, + } + ) + + tests := []struct { + desc string + resourceName string + managementServerResponse *v3discoverypb.DiscoveryResponse + wantUpdate xdsresource.ListenerUpdate + wantErr string + wantUpdateMetadata map[string]xdsresource.UpdateWithMD + }{ + { + desc: "badly-marshaled-response", + resourceName: resourceName1, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + VersionInfo: "1", + Resources: []*anypb.Any{{ + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + Value: []byte{1, 2, 3, 4}, + }}, + }, + wantErr: "Listener not found in received response", + wantUpdateMetadata: map[string]xdsresource.UpdateWithMD{ + "resource-name-1": {MD: xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusNotExist}}, + }, + }, + { + desc: "empty-response", + resourceName: resourceName1, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + VersionInfo: "1", + }, + wantErr: "Listener not found in received response", + wantUpdateMetadata: map[string]xdsresource.UpdateWithMD{ + "resource-name-1": {MD: xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusNotExist}}, + }, + }, + { + desc: "unexpected-type-in-response", + resourceName: resourceName1, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + VersionInfo: "1", + Resources: []*anypb.Any{testutils.MarshalAny(&v3routepb.RouteConfiguration{})}, + }, + wantErr: "Listener not found in received response", + wantUpdateMetadata: map[string]xdsresource.UpdateWithMD{ + "resource-name-1": {MD: xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusNotExist}}, + }, + }, + { + desc: "one-bad-resource", + resourceName: resourceName1, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + VersionInfo: "1", + Resources: []*anypb.Any{testutils.MarshalAny(&v3listenerpb.Listener{ + Name: resourceName1, + ApiListener: &v3listenerpb.ApiListener{ + ApiListener: testutils.MarshalAny(&v3httppb.HttpConnectionManager{}), + }}), + }, + }, + wantErr: "no RouteSpecifier", + wantUpdateMetadata: map[string]xdsresource.UpdateWithMD{ + "resource-name-1": {MD: xdsresource.UpdateMetadata{ + Status: xdsresource.ServiceStatusNACKed, + ErrState: &xdsresource.UpdateErrorMetadata{ + Version: "1", + Err: cmpopts.AnyError, + }, + }}, + }, + }, + { + desc: "one-good-resource", + resourceName: resourceName1, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + VersionInfo: "1", + Resources: []*anypb.Any{testutils.MarshalAny(resource1)}, + }, + wantUpdate: xdsresource.ListenerUpdate{ + RouteConfigName: "route-configuration-name", + HTTPFilters: []xdsresource.HTTPFilter{{Name: "router"}}, + }, + wantUpdateMetadata: map[string]xdsresource.UpdateWithMD{ + "resource-name-1": { + MD: xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusACKed, Version: "1"}, + Raw: testutils.MarshalAny(resource1), + }, + }, + }, + { + desc: "two-resources-when-we-requested-one", + resourceName: resourceName1, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + VersionInfo: "1", + Resources: []*anypb.Any{testutils.MarshalAny(resource1), testutils.MarshalAny(resource2)}, + }, + wantUpdate: xdsresource.ListenerUpdate{ + RouteConfigName: "route-configuration-name", + HTTPFilters: []xdsresource.HTTPFilter{{Name: "router"}}, + }, + wantUpdateMetadata: map[string]xdsresource.UpdateWithMD{ + "resource-name-1": { + MD: xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusACKed, Version: "1"}, + Raw: testutils.MarshalAny(resource1), + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + // Create a fake xDS management server listening on a local port, + // and set it up with the response to send. + mgmtServer, cleanup := startFakeManagementServer(t) + defer cleanup() + t.Logf("Started xDS management server on %s", mgmtServer.Address) + + // Create an xDS client talking to the above management server. + nodeID := uuid.New().String() + client, close, err := xdsclient.NewWithConfigForTesting(&bootstrap.Config{ + XDSServer: xdstestutils.ServerConfigForAddress(t, mgmtServer.Address), + NodeProto: &v3corepb.Node{Id: nodeID}, + }, defaultTestWatchExpiryTimeout, time.Duration(0)) + if err != nil { + t.Fatalf("failed to create xds client: %v", err) + } + defer close() + t.Logf("Created xDS client to %s", mgmtServer.Address) + + // A wrapper struct to wrap the update and the associated error, as + // received by the resource watch callback. + type updateAndErr struct { + update xdsresource.ListenerUpdate + err error + } + updateAndErrCh := testutils.NewChannel() + + // Register a watch, and push the results on to a channel. + client.WatchListener(test.resourceName, func(update xdsresource.ListenerUpdate, err error) { + updateAndErrCh.Send(updateAndErr{update: update, err: err}) + }) + t.Logf("Registered a watch for Listener %q", test.resourceName) + + // Wait for the discovery request to be sent out. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + val, err := mgmtServer.XDSRequestChan.Receive(ctx) + if err != nil { + t.Fatalf("Timeout when waiting for discovery request at the management server: %v", ctx) + } + wantReq := &fakeserver.Request{Req: &v3discoverypb.DiscoveryRequest{ + Node: &v3corepb.Node{Id: nodeID}, + ResourceNames: []string{test.resourceName}, + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + }} + gotReq := val.(*fakeserver.Request) + if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" { + t.Fatalf("Discovery request received at management server is %+v, want %+v", gotReq, wantReq) + } + t.Logf("Discovery request received at management server") + + // Configure the fake management server with a response. + mgmtServer.XDSResponseChan <- &fakeserver.Response{Resp: test.managementServerResponse} + + // Wait for an update from the xDS client and compare with expected + // update. + val, err = updateAndErrCh.Receive(ctx) + if err != nil { + t.Fatalf("Timeout when waiting for watch callback to invoked after response from management server: %v", err) + } + gotUpdate := val.(updateAndErr).update + gotErr := val.(updateAndErr).err + if (gotErr != nil) != (test.wantErr != "") { + t.Fatalf("Got error from handling update: %v, want %v", gotErr, test.wantErr) + } + if gotErr != nil && !strings.Contains(gotErr.Error(), test.wantErr) { + t.Fatalf("Got error from handling update: %v, want %v", gotErr, test.wantErr) + } + cmpOpts := []cmp.Option{ + cmpopts.EquateEmpty(), + cmpopts.IgnoreFields(xdsresource.HTTPFilter{}, "Filter", "Config"), + cmpopts.IgnoreFields(xdsresource.ListenerUpdate{}, "Raw"), + } + if diff := cmp.Diff(test.wantUpdate, gotUpdate, cmpOpts...); diff != "" { + t.Fatalf("Unexpected diff in metadata, diff (-want +got):\n%s", diff) + } + if err := compareUpdateMetadata(ctx, func() map[string]xdsresource.UpdateWithMD { + dump := client.DumpResources() + return dump["type.googleapis.com/envoy.config.listener.v3.Listener"] + }, test.wantUpdateMetadata); err != nil { + t.Fatal(err) + } + }) + } +} + +// TestHandleRouteConfigResponseFromManagementServer covers different scenarios +// involving receipt of an RDS response from the management server. The test +// verifies that the internal state of the xDS client (parsed resource and +// metadata) matches expectations. +func (s) TestHandleRouteConfigResponseFromManagementServer(t *testing.T) { + const ( + resourceName1 = "resource-name-1" + resourceName2 = "resource-name-2" + ) + var ( + virtualHosts = []*v3routepb.VirtualHost{ + { + Domains: []string{"lds-target-name"}, + Routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: ""}}, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: "cluster-name"}, + }, + }, + }, + }, + }, + } + resource1 = &v3routepb.RouteConfiguration{ + Name: resourceName1, + VirtualHosts: virtualHosts, + } + resource2 = &v3routepb.RouteConfiguration{ + Name: resourceName2, + VirtualHosts: virtualHosts, + } + ) + + tests := []struct { + desc string + resourceName string + managementServerResponse *v3discoverypb.DiscoveryResponse + wantUpdate xdsresource.RouteConfigUpdate + wantErr string + wantUpdateMetadata map[string]xdsresource.UpdateWithMD + }{ + // The first three tests involve scenarios where the response fails + // protobuf deserialization (because it contains an invalid data or type + // in the anypb.Any) or the requested resource is not present in the + // response. In either case, no resource update makes its way to the + // top-level xDS client. An RDS response without a requested resource + // does not mean that the resource does not exist in the server. It + // could be part of a future update. Therefore, the only failure mode + // for this resource is for the watch to timeout. + { + desc: "badly-marshaled-response", + resourceName: resourceName1, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + VersionInfo: "1", + Resources: []*anypb.Any{{ + TypeUrl: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + Value: []byte{1, 2, 3, 4}, + }}, + }, + wantErr: "RouteConfiguration not found in received response", + wantUpdateMetadata: map[string]xdsresource.UpdateWithMD{ + "resource-name-1": {MD: xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusNotExist}}, + }, + }, + { + desc: "empty-response", + resourceName: resourceName1, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + VersionInfo: "1", + }, + wantErr: "RouteConfiguration not found in received response", + wantUpdateMetadata: map[string]xdsresource.UpdateWithMD{ + "resource-name-1": {MD: xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusNotExist}}, + }, + }, + { + desc: "unexpected-type-in-response", + resourceName: resourceName1, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + VersionInfo: "1", + Resources: []*anypb.Any{testutils.MarshalAny(&v3clusterpb.Cluster{})}, + }, + wantErr: "RouteConfiguration not found in received response", + wantUpdateMetadata: map[string]xdsresource.UpdateWithMD{ + "resource-name-1": {MD: xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusNotExist}}, + }, + }, + { + desc: "one-bad-resource", + resourceName: resourceName1, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + VersionInfo: "1", + Resources: []*anypb.Any{testutils.MarshalAny(&v3routepb.RouteConfiguration{ + Name: resourceName1, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{"lds-resource-name"}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}}, + Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: "cluster-resource-name"}, + }}}}, + RetryPolicy: &v3routepb.RetryPolicy{ + NumRetries: &wrapperspb.UInt32Value{Value: 0}, + }, + }}, + })}, + }, + wantErr: "received route is invalid: retry_policy.num_retries = 0; must be >= 1", + wantUpdateMetadata: map[string]xdsresource.UpdateWithMD{ + "resource-name-1": {MD: xdsresource.UpdateMetadata{ + Status: xdsresource.ServiceStatusNACKed, + ErrState: &xdsresource.UpdateErrorMetadata{ + Version: "1", + Err: cmpopts.AnyError, + }, + }}, + }, + }, + { + desc: "one-good-resource", + resourceName: resourceName1, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + VersionInfo: "1", + Resources: []*anypb.Any{testutils.MarshalAny(resource1)}, + }, + wantUpdate: xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{ + { + Domains: []string{"lds-target-name"}, + Routes: []*xdsresource.Route{{Prefix: newStringP(""), + WeightedClusters: map[string]xdsresource.WeightedCluster{"cluster-name": {Weight: 1}}, + ActionType: xdsresource.RouteActionRoute}}, + }, + }, + }, + wantUpdateMetadata: map[string]xdsresource.UpdateWithMD{ + "resource-name-1": { + MD: xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusACKed, Version: "1"}, + Raw: testutils.MarshalAny(resource1), + }, + }, + }, + { + desc: "two-resources-when-we-requested-one", + resourceName: resourceName1, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + VersionInfo: "1", + Resources: []*anypb.Any{testutils.MarshalAny(resource1), testutils.MarshalAny(resource2)}, + }, + wantUpdate: xdsresource.RouteConfigUpdate{ + VirtualHosts: []*xdsresource.VirtualHost{ + { + Domains: []string{"lds-target-name"}, + Routes: []*xdsresource.Route{{Prefix: newStringP(""), + WeightedClusters: map[string]xdsresource.WeightedCluster{"cluster-name": {Weight: 1}}, + ActionType: xdsresource.RouteActionRoute}}, + }, + }, + }, + wantUpdateMetadata: map[string]xdsresource.UpdateWithMD{ + "resource-name-1": { + MD: xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusACKed, Version: "1"}, + Raw: testutils.MarshalAny(resource1), + }, + }, + }, + } + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + // Create a fake xDS management server listening on a local port, + // and set it up with the response to send. + mgmtServer, cleanup := startFakeManagementServer(t) + defer cleanup() + t.Logf("Started xDS management server on %s", mgmtServer.Address) + + // Create an xDS client talking to the above management server. + nodeID := uuid.New().String() + client, close, err := xdsclient.NewWithConfigForTesting(&bootstrap.Config{ + XDSServer: xdstestutils.ServerConfigForAddress(t, mgmtServer.Address), + NodeProto: &v3corepb.Node{Id: nodeID}, + }, defaultTestWatchExpiryTimeout, time.Duration(0)) + if err != nil { + t.Fatalf("failed to create xds client: %v", err) + } + defer close() + t.Logf("Created xDS client to %s", mgmtServer.Address) + + // A wrapper struct to wrap the update and the associated error, as + // received by the resource watch callback. + type updateAndErr struct { + update xdsresource.RouteConfigUpdate + err error + } + updateAndErrCh := testutils.NewChannel() + + // Register a watch, and push the results on to a channel. + client.WatchRouteConfig(test.resourceName, func(update xdsresource.RouteConfigUpdate, err error) { + updateAndErrCh.Send(updateAndErr{update: update, err: err}) + }) + t.Logf("Registered a watch for Route Configuration %q", test.resourceName) + + // Wait for the discovery request to be sent out. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + val, err := mgmtServer.XDSRequestChan.Receive(ctx) + if err != nil { + t.Fatalf("Timeout when waiting for discovery request at the management server: %v", ctx) + } + wantReq := &fakeserver.Request{Req: &v3discoverypb.DiscoveryRequest{ + Node: &v3corepb.Node{Id: nodeID}, + ResourceNames: []string{test.resourceName}, + TypeUrl: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + }} + gotReq := val.(*fakeserver.Request) + if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" { + t.Fatalf("Discovery request received at management server is %+v, want %+v", gotReq, wantReq) + } + t.Logf("Discovery request received at management server") + + // Configure the fake management server with a response. + mgmtServer.XDSResponseChan <- &fakeserver.Response{Resp: test.managementServerResponse} + + // Wait for an update from the xDS client and compare with expected + // update. + val, err = updateAndErrCh.Receive(ctx) + if err != nil { + t.Fatalf("Timeout when waiting for watch callback to invoked after response from management server: %v", err) + } + gotUpdate := val.(updateAndErr).update + gotErr := val.(updateAndErr).err + if (gotErr != nil) != (test.wantErr != "") { + t.Fatalf("Got error from handling update: %v, want %v", gotErr, test.wantErr) + } + if gotErr != nil && !strings.Contains(gotErr.Error(), test.wantErr) { + t.Fatalf("Got error from handling update: %v, want %v", gotErr, test.wantErr) + } + cmpOpts := []cmp.Option{ + cmpopts.EquateEmpty(), + cmpopts.IgnoreFields(xdsresource.RouteConfigUpdate{}, "Raw"), + } + if diff := cmp.Diff(test.wantUpdate, gotUpdate, cmpOpts...); diff != "" { + t.Fatalf("Unexpected diff in metadata, diff (-want +got):\n%s", diff) + } + if err := compareUpdateMetadata(ctx, func() map[string]xdsresource.UpdateWithMD { + dump := client.DumpResources() + return dump["type.googleapis.com/envoy.config.route.v3.RouteConfiguration"] + }, test.wantUpdateMetadata); err != nil { + t.Fatal(err) + } + }) + } +} + +// TestHandleClusterResponseFromManagementServer covers different scenarios +// involving receipt of a CDS response from the management server. The test +// verifies that the internal state of the xDS client (parsed resource and +// metadata) matches expectations. +func (s) TestHandleClusterResponseFromManagementServer(t *testing.T) { + const ( + resourceName1 = "resource-name-1" + resourceName2 = "resource-name-2" + ) + resource1 := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: resourceName1, + ServiceName: "eds-service-name", + EnableLRS: true, + }) + resource2 := proto.Clone(resource1).(*v3clusterpb.Cluster) + resource2.Name = resourceName2 + + tests := []struct { + desc string + resourceName string + managementServerResponse *v3discoverypb.DiscoveryResponse + wantUpdate xdsresource.ClusterUpdate + wantErr string + wantUpdateMetadata map[string]xdsresource.UpdateWithMD + }{ + { + desc: "badly-marshaled-response", + resourceName: resourceName1, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.cluster.v3.Cluster", + VersionInfo: "1", + Resources: []*anypb.Any{{ + TypeUrl: "type.googleapis.com/envoy.config.cluster.v3.Cluster", + Value: []byte{1, 2, 3, 4}, + }}, + }, + wantErr: "Cluster not found in received response", + wantUpdateMetadata: map[string]xdsresource.UpdateWithMD{ + "resource-name-1": {MD: xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusNotExist}}, + }, + }, + { + desc: "empty-response", + resourceName: resourceName1, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.cluster.v3.Cluster", + VersionInfo: "1", + }, + wantErr: "Cluster not found in received response", + wantUpdateMetadata: map[string]xdsresource.UpdateWithMD{ + "resource-name-1": {MD: xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusNotExist}}, + }, + }, + { + desc: "unexpected-type-in-response", + resourceName: resourceName1, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.cluster.v3.Cluster", + VersionInfo: "1", + Resources: []*anypb.Any{testutils.MarshalAny(&v3endpointpb.ClusterLoadAssignment{})}, + }, + wantErr: "Cluster not found in received response", + wantUpdateMetadata: map[string]xdsresource.UpdateWithMD{ + "resource-name-1": {MD: xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusNotExist}}, + }, + }, + { + desc: "one-bad-resource", + resourceName: resourceName1, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.cluster.v3.Cluster", + VersionInfo: "1", + Resources: []*anypb.Any{testutils.MarshalAny(&v3clusterpb.Cluster{ + Name: resourceName1, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: "eds-service-name", + }, + LbPolicy: v3clusterpb.Cluster_MAGLEV, + })}, + }, + wantErr: "unexpected lbPolicy MAGLEV", + wantUpdateMetadata: map[string]xdsresource.UpdateWithMD{ + "resource-name-1": {MD: xdsresource.UpdateMetadata{ + Status: xdsresource.ServiceStatusNACKed, + ErrState: &xdsresource.UpdateErrorMetadata{ + Version: "1", + Err: cmpopts.AnyError, + }, + }}, + }, + }, + { + desc: "one-good-resource", + resourceName: resourceName1, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.cluster.v3.Cluster", + VersionInfo: "1", + Resources: []*anypb.Any{testutils.MarshalAny(resource1)}, + }, + wantUpdate: xdsresource.ClusterUpdate{ + ClusterName: "resource-name-1", + EDSServiceName: "eds-service-name", + LRSServerConfig: xdsresource.ClusterLRSServerSelf, + }, + wantUpdateMetadata: map[string]xdsresource.UpdateWithMD{ + "resource-name-1": { + MD: xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusACKed, Version: "1"}, + Raw: testutils.MarshalAny(resource1), + }, + }, + }, + { + desc: "two-resources-when-we-requested-one", + resourceName: resourceName1, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.cluster.v3.Cluster", + VersionInfo: "1", + Resources: []*anypb.Any{testutils.MarshalAny(resource1), testutils.MarshalAny(resource2)}, + }, + wantUpdate: xdsresource.ClusterUpdate{ + ClusterName: "resource-name-1", + EDSServiceName: "eds-service-name", + LRSServerConfig: xdsresource.ClusterLRSServerSelf, + }, + wantUpdateMetadata: map[string]xdsresource.UpdateWithMD{ + "resource-name-1": { + MD: xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusACKed, Version: "1"}, + Raw: testutils.MarshalAny(resource1), + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + // Create a fake xDS management server listening on a local port, + // and set it up with the response to send. + mgmtServer, cleanup := startFakeManagementServer(t) + defer cleanup() + t.Logf("Started xDS management server on %s", mgmtServer.Address) + + // Create an xDS client talking to the above management server. + nodeID := uuid.New().String() + client, close, err := xdsclient.NewWithConfigForTesting(&bootstrap.Config{ + XDSServer: xdstestutils.ServerConfigForAddress(t, mgmtServer.Address), + NodeProto: &v3corepb.Node{Id: nodeID}, + }, defaultTestWatchExpiryTimeout, time.Duration(0)) + if err != nil { + t.Fatalf("failed to create xds client: %v", err) + } + defer close() + t.Logf("Created xDS client to %s", mgmtServer.Address) + + // A wrapper struct to wrap the update and the associated error, as + // received by the resource watch callback. + type updateAndErr struct { + update xdsresource.ClusterUpdate + err error + } + updateAndErrCh := testutils.NewChannel() + + // Register a watch, and push the results on to a channel. + client.WatchCluster(test.resourceName, func(update xdsresource.ClusterUpdate, err error) { + updateAndErrCh.Send(updateAndErr{update: update, err: err}) + }) + t.Logf("Registered a watch for Cluster %q", test.resourceName) + + // Wait for the discovery request to be sent out. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + val, err := mgmtServer.XDSRequestChan.Receive(ctx) + if err != nil { + t.Fatalf("Timeout when waiting for discovery request at the management server: %v", ctx) + } + wantReq := &fakeserver.Request{Req: &v3discoverypb.DiscoveryRequest{ + Node: &v3corepb.Node{Id: nodeID}, + ResourceNames: []string{test.resourceName}, + TypeUrl: "type.googleapis.com/envoy.config.cluster.v3.Cluster", + }} + gotReq := val.(*fakeserver.Request) + if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" { + t.Fatalf("Discovery request received at management server is %+v, want %+v", gotReq, wantReq) + } + t.Logf("Discovery request received at management server") + + // Configure the fake management server with a response. + mgmtServer.XDSResponseChan <- &fakeserver.Response{Resp: test.managementServerResponse} + + // Wait for an update from the xDS client and compare with expected + // update. + val, err = updateAndErrCh.Receive(ctx) + if err != nil { + t.Fatalf("Timeout when waiting for watch callback to invoked after response from management server: %v", err) + } + gotUpdate := val.(updateAndErr).update + gotErr := val.(updateAndErr).err + if (gotErr != nil) != (test.wantErr != "") { + t.Fatalf("Got error from handling update: %v, want %v", gotErr, test.wantErr) + } + if gotErr != nil && !strings.Contains(gotErr.Error(), test.wantErr) { + t.Fatalf("Got error from handling update: %v, want %v", gotErr, test.wantErr) + } + cmpOpts := []cmp.Option{ + cmpopts.EquateEmpty(), + cmpopts.IgnoreFields(xdsresource.ClusterUpdate{}, "Raw", "LBPolicy"), + } + if diff := cmp.Diff(test.wantUpdate, gotUpdate, cmpOpts...); diff != "" { + t.Fatalf("Unexpected diff in metadata, diff (-want +got):\n%s", diff) + } + if err := compareUpdateMetadata(ctx, func() map[string]xdsresource.UpdateWithMD { + dump := client.DumpResources() + return dump["type.googleapis.com/envoy.config.cluster.v3.Cluster"] + }, test.wantUpdateMetadata); err != nil { + t.Fatal(err) + } + }) + } +} + +// TestHandleEndpointsResponseFromManagementServer covers different scenarios +// involving receipt of a CDS response from the management server. The test +// verifies that the internal state of the xDS client (parsed resource and +// metadata) matches expectations. +func (s) TestHandleEndpointsResponseFromManagementServer(t *testing.T) { + const ( + resourceName1 = "resource-name-1" + resourceName2 = "resource-name-2" + ) + resource1 := &v3endpointpb.ClusterLoadAssignment{ + ClusterName: resourceName1, + Endpoints: []*v3endpointpb.LocalityLbEndpoints{ + { + Locality: &v3corepb.Locality{SubZone: "locality-1"}, + LbEndpoints: []*v3endpointpb.LbEndpoint{ + { + HostIdentifier: &v3endpointpb.LbEndpoint_Endpoint{ + Endpoint: &v3endpointpb.Endpoint{ + Address: &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Protocol: v3corepb.SocketAddress_TCP, + Address: "addr1", + PortSpecifier: &v3corepb.SocketAddress_PortValue{ + PortValue: uint32(314), + }, + }, + }, + }, + }, + }, + }, + }, + LoadBalancingWeight: &wrapperspb.UInt32Value{Value: 1}, + Priority: 1, + }, + { + Locality: &v3corepb.Locality{SubZone: "locality-2"}, + LbEndpoints: []*v3endpointpb.LbEndpoint{ + { + HostIdentifier: &v3endpointpb.LbEndpoint_Endpoint{ + Endpoint: &v3endpointpb.Endpoint{ + Address: &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Protocol: v3corepb.SocketAddress_TCP, + Address: "addr2", + PortSpecifier: &v3corepb.SocketAddress_PortValue{ + PortValue: uint32(159), + }, + }, + }, + }, + }, + }, + }, + }, + LoadBalancingWeight: &wrapperspb.UInt32Value{Value: 1}, + Priority: 0, + }, + }, + } + resource2 := proto.Clone(resource1).(*v3endpointpb.ClusterLoadAssignment) + resource2.ClusterName = resourceName2 + + tests := []struct { + desc string + resourceName string + managementServerResponse *v3discoverypb.DiscoveryResponse + wantUpdate xdsresource.EndpointsUpdate + wantErr string + wantUpdateMetadata map[string]xdsresource.UpdateWithMD + }{ + // The first three tests involve scenarios where the response fails + // protobuf deserialization (because it contains an invalid data or type + // in the anypb.Any) or the requested resource is not present in the + // response. In either case, no resource update makes its way to the + // top-level xDS client. An EDS response without a requested resource + // does not mean that the resource does not exist in the server. It + // could be part of a future update. Therefore, the only failure mode + // for this resource is for the watch to timeout. + { + desc: "badly-marshaled-response", + resourceName: resourceName1, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + VersionInfo: "1", + Resources: []*anypb.Any{{ + TypeUrl: "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + Value: []byte{1, 2, 3, 4}, + }}, + }, + wantErr: "Endpoints not found in received response", + wantUpdateMetadata: map[string]xdsresource.UpdateWithMD{ + "resource-name-1": {MD: xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusNotExist}}, + }, + }, + { + desc: "empty-response", + resourceName: resourceName1, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + VersionInfo: "1", + }, + wantErr: "Endpoints not found in received response", + wantUpdateMetadata: map[string]xdsresource.UpdateWithMD{ + "resource-name-1": {MD: xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusNotExist}}, + }, + }, + { + desc: "unexpected-type-in-response", + resourceName: resourceName1, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + VersionInfo: "1", + Resources: []*anypb.Any{testutils.MarshalAny(&v3listenerpb.Listener{})}, + }, + wantErr: "Endpoints not found in received response", + wantUpdateMetadata: map[string]xdsresource.UpdateWithMD{ + "resource-name-1": {MD: xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusNotExist}}, + }, + }, + { + desc: "one-bad-resource", + resourceName: resourceName1, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + VersionInfo: "1", + Resources: []*anypb.Any{testutils.MarshalAny(&v3endpointpb.ClusterLoadAssignment{ + ClusterName: resourceName1, + Endpoints: []*v3endpointpb.LocalityLbEndpoints{ + { + Locality: &v3corepb.Locality{SubZone: "locality-1"}, + LbEndpoints: []*v3endpointpb.LbEndpoint{ + { + HostIdentifier: &v3endpointpb.LbEndpoint_Endpoint{ + Endpoint: &v3endpointpb.Endpoint{ + Address: &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Protocol: v3corepb.SocketAddress_TCP, + Address: "addr1", + PortSpecifier: &v3corepb.SocketAddress_PortValue{ + PortValue: uint32(314), + }, + }, + }, + }, + }, + }, + LoadBalancingWeight: &wrapperspb.UInt32Value{Value: 0}, + }, + }, + LoadBalancingWeight: &wrapperspb.UInt32Value{Value: 1}, + Priority: 1, + }, + }, + }), + }, + }, + wantErr: "EDS response contains an endpoint with zero weight", + wantUpdateMetadata: map[string]xdsresource.UpdateWithMD{ + "resource-name-1": {MD: xdsresource.UpdateMetadata{ + Status: xdsresource.ServiceStatusNACKed, + ErrState: &xdsresource.UpdateErrorMetadata{ + Version: "1", + Err: cmpopts.AnyError, + }, + }}, + }, + }, + { + desc: "one-good-resource", + resourceName: resourceName1, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + VersionInfo: "1", + Resources: []*anypb.Any{testutils.MarshalAny(resource1)}, + }, + wantUpdate: xdsresource.EndpointsUpdate{ + Localities: []xdsresource.Locality{ + { + Endpoints: []xdsresource.Endpoint{{Address: "addr1:314", Weight: 1}}, + ID: internal.LocalityID{SubZone: "locality-1"}, + Priority: 1, + Weight: 1, + }, + { + Endpoints: []xdsresource.Endpoint{{Address: "addr2:159", Weight: 1}}, + ID: internal.LocalityID{SubZone: "locality-2"}, + Priority: 0, + Weight: 1, + }, + }, + }, + wantUpdateMetadata: map[string]xdsresource.UpdateWithMD{ + "resource-name-1": { + MD: xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusACKed, Version: "1"}, + Raw: testutils.MarshalAny(resource1), + }, + }, + }, + { + desc: "two-resources-when-we-requested-one", + resourceName: resourceName1, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + VersionInfo: "1", + Resources: []*anypb.Any{testutils.MarshalAny(resource1), testutils.MarshalAny(resource2)}, + }, + wantUpdate: xdsresource.EndpointsUpdate{ + Localities: []xdsresource.Locality{ + { + Endpoints: []xdsresource.Endpoint{{Address: "addr1:314", Weight: 1}}, + ID: internal.LocalityID{SubZone: "locality-1"}, + Priority: 1, + Weight: 1, + }, + { + Endpoints: []xdsresource.Endpoint{{Address: "addr2:159", Weight: 1}}, + ID: internal.LocalityID{SubZone: "locality-2"}, + Priority: 0, + Weight: 1, + }, + }, + }, + wantUpdateMetadata: map[string]xdsresource.UpdateWithMD{ + "resource-name-1": { + MD: xdsresource.UpdateMetadata{Status: xdsresource.ServiceStatusACKed, Version: "1"}, + Raw: testutils.MarshalAny(resource1), + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + // Create a fake xDS management server listening on a local port, + // and set it up with the response to send. + mgmtServer, cleanup := startFakeManagementServer(t) + defer cleanup() + t.Logf("Started xDS management server on %s", mgmtServer.Address) + + // Create an xDS client talking to the above management server. + nodeID := uuid.New().String() + client, close, err := xdsclient.NewWithConfigForTesting(&bootstrap.Config{ + XDSServer: xdstestutils.ServerConfigForAddress(t, mgmtServer.Address), + NodeProto: &v3corepb.Node{Id: nodeID}, + }, defaultTestWatchExpiryTimeout, time.Duration(0)) + if err != nil { + t.Fatalf("failed to create xds client: %v", err) + } + defer close() + t.Logf("Created xDS client to %s", mgmtServer.Address) + + // Register a watch, and push the results on to a channel. + ew := newEndpointsWatcher() + edsCancel := xdsresource.WatchEndpoints(client, test.resourceName, ew) + defer edsCancel() + t.Logf("Registered a watch for Endpoint %q", test.resourceName) + + // Wait for the discovery request to be sent out. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + val, err := mgmtServer.XDSRequestChan.Receive(ctx) + if err != nil { + t.Fatalf("Timeout when waiting for discovery request at the management server: %v", ctx) + } + wantReq := &fakeserver.Request{Req: &v3discoverypb.DiscoveryRequest{ + Node: &v3corepb.Node{Id: nodeID}, + ResourceNames: []string{test.resourceName}, + TypeUrl: "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", + }} + gotReq := val.(*fakeserver.Request) + if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" { + t.Fatalf("Discovery request received at management server is %+v, want %+v", gotReq, wantReq) + } + t.Logf("Discovery request received at management server") + + // Configure the fake management server with a response. + mgmtServer.XDSResponseChan <- &fakeserver.Response{Resp: test.managementServerResponse} + + // Wait for an update from the xDS client and compare with expected + // update. + val, err = ew.updateCh.Receive(ctx) + if err != nil { + t.Fatalf("Timeout when waiting for watch callback to invoked after response from management server: %v", err) + } + gotUpdate := val.(endpointsUpdateErrTuple).update + gotErr := val.(endpointsUpdateErrTuple).err + if (gotErr != nil) != (test.wantErr != "") { + t.Fatalf("Got error from handling update: %v, want %v", gotErr, test.wantErr) + } + if gotErr != nil && !strings.Contains(gotErr.Error(), test.wantErr) { + t.Fatalf("Got error from handling update: %v, want %v", gotErr, test.wantErr) + } + cmpOpts := []cmp.Option{ + cmpopts.EquateEmpty(), + cmpopts.IgnoreFields(xdsresource.EndpointsUpdate{}, "Raw"), + } + if diff := cmp.Diff(test.wantUpdate, gotUpdate, cmpOpts...); diff != "" { + t.Fatalf("Unexpected diff in metadata, diff (-want +got):\n%s", diff) + } + if err := compareUpdateMetadata(ctx, func() map[string]xdsresource.UpdateWithMD { + dump := client.DumpResources() + return dump["type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment"] + }, test.wantUpdateMetadata); err != nil { + t.Fatal(err) + } + }) + } +} diff --git a/xds/internal/xdsclient/transport/loadreport.go b/xds/internal/xdsclient/transport/loadreport.go new file mode 100644 index 000000000000..89ffc4fcec66 --- /dev/null +++ b/xds/internal/xdsclient/transport/loadreport.go @@ -0,0 +1,274 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package transport + +import ( + "context" + "errors" + "fmt" + "io" + "time" + + "github.com/golang/protobuf/ptypes" + "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/xds/internal" + "google.golang.org/grpc/xds/internal/xdsclient/load" + "google.golang.org/protobuf/proto" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3lrsgrpc "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v3" + v3lrspb "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v3" +) + +type lrsStream = v3lrsgrpc.LoadReportingService_StreamLoadStatsClient + +// ReportLoad starts reporting loads to the management server the transport is +// configured to use. +// +// It returns a Store for the user to report loads and a function to cancel the +// load reporting. +func (t *Transport) ReportLoad() (*load.Store, func()) { + t.lrsStartStream() + return t.lrsStore, grpcsync.OnceFunc(func() { t.lrsStopStream() }) +} + +// lrsStartStream starts an LRS stream to the server, if none exists. +func (t *Transport) lrsStartStream() { + t.lrsMu.Lock() + defer t.lrsMu.Unlock() + + t.lrsRefCount++ + if t.lrsRefCount != 1 { + // Return early if the stream has already been started. + return + } + + ctx, cancel := context.WithCancel(context.Background()) + t.lrsCancelStream = cancel + + // Create a new done channel everytime a new stream is created. This ensures + // that we don't close the same channel multiple times (from lrsRunner() + // goroutine) when multiple streams are created and closed. + t.lrsRunnerDoneCh = make(chan struct{}) + go t.lrsRunner(ctx) +} + +// lrsStopStream closes the LRS stream, if this is the last user of the stream. +func (t *Transport) lrsStopStream() { + t.lrsMu.Lock() + defer t.lrsMu.Unlock() + + t.lrsRefCount-- + if t.lrsRefCount != 0 { + // Return early if the stream has other references. + return + } + + t.lrsCancelStream() + t.logger.Infof("Stopping LRS stream") + + // Wait for the runner goroutine to exit. The done channel will be + // recreated when a new stream is created. + <-t.lrsRunnerDoneCh +} + +// lrsRunner starts an LRS stream to report load data to the management server. +// It reports load at constant intervals (as configured by the management +// server) until the context is cancelled. +func (t *Transport) lrsRunner(ctx context.Context) { + defer close(t.lrsRunnerDoneCh) + + // This feature indicates that the client supports the + // LoadStatsResponse.send_all_clusters field in the LRS response. + node := proto.Clone(t.nodeProto).(*v3corepb.Node) + node.ClientFeatures = append(node.ClientFeatures, "envoy.lrs.supports_send_all_clusters") + + backoffAttempt := 0 + backoffTimer := time.NewTimer(0) + for ctx.Err() == nil { + select { + case <-backoffTimer.C: + case <-ctx.Done(): + backoffTimer.Stop() + return + } + + // We reset backoff state when we successfully receive at least one + // message from the server. + resetBackoff := func() bool { + // streamCtx is created and canceled in case we terminate the stream + // early for any reason, to avoid gRPC-Go leaking the RPC's monitoring + // goroutine. + streamCtx, cancel := context.WithCancel(ctx) + defer cancel() + stream, err := v3lrsgrpc.NewLoadReportingServiceClient(t.cc).StreamLoadStats(streamCtx) + if err != nil { + t.logger.Warningf("Creating LRS stream to server %q failed: %v", t.serverURI, err) + return false + } + t.logger.Infof("Created LRS stream to server %q", t.serverURI) + + if err := t.sendFirstLoadStatsRequest(stream, node); err != nil { + t.logger.Warningf("Sending first LRS request failed: %v", err) + return false + } + + clusters, interval, err := t.recvFirstLoadStatsResponse(stream) + if err != nil { + t.logger.Warningf("Reading from LRS stream failed: %v", err) + return false + } + + t.sendLoads(streamCtx, stream, clusters, interval) + return true + }() + + if resetBackoff { + backoffTimer.Reset(0) + backoffAttempt = 0 + } else { + backoffTimer.Reset(t.backoff(backoffAttempt)) + backoffAttempt++ + } + } +} + +func (t *Transport) sendLoads(ctx context.Context, stream lrsStream, clusterNames []string, interval time.Duration) { + tick := time.NewTicker(interval) + defer tick.Stop() + for { + select { + case <-tick.C: + case <-ctx.Done(): + return + } + if err := t.sendLoadStatsRequest(stream, t.lrsStore.Stats(clusterNames)); err != nil { + t.logger.Warningf("Writing to LRS stream failed: %v", err) + return + } + } +} + +func (t *Transport) sendFirstLoadStatsRequest(stream lrsStream, node *v3corepb.Node) error { + req := &v3lrspb.LoadStatsRequest{Node: node} + if t.logger.V(perRPCVerbosityLevel) { + t.logger.Infof("Sending initial LoadStatsRequest: %s", pretty.ToJSON(req)) + } + err := stream.Send(req) + if err == io.EOF { + return getStreamError(stream) + } + return err +} + +func (t *Transport) recvFirstLoadStatsResponse(stream lrsStream) ([]string, time.Duration, error) { + resp, err := stream.Recv() + if err != nil { + return nil, 0, fmt.Errorf("failed to receive first LoadStatsResponse: %v", err) + } + if t.logger.V(perRPCVerbosityLevel) { + t.logger.Infof("Received first LoadStatsResponse: %s", pretty.ToJSON(resp)) + } + + interval, err := ptypes.Duration(resp.GetLoadReportingInterval()) + if err != nil { + return nil, 0, fmt.Errorf("invalid load_reporting_interval: %v", err) + } + + if resp.ReportEndpointGranularity { + // TODO(easwars): Support per endpoint loads. + return nil, 0, errors.New("lrs: endpoint loads requested, but not supported by current implementation") + } + + clusters := resp.Clusters + if resp.SendAllClusters { + // Return nil to send stats for all clusters. + clusters = nil + } + + return clusters, interval, nil +} + +func (t *Transport) sendLoadStatsRequest(stream lrsStream, loads []*load.Data) error { + clusterStats := make([]*v3endpointpb.ClusterStats, 0, len(loads)) + for _, sd := range loads { + droppedReqs := make([]*v3endpointpb.ClusterStats_DroppedRequests, 0, len(sd.Drops)) + for category, count := range sd.Drops { + droppedReqs = append(droppedReqs, &v3endpointpb.ClusterStats_DroppedRequests{ + Category: category, + DroppedCount: count, + }) + } + localityStats := make([]*v3endpointpb.UpstreamLocalityStats, 0, len(sd.LocalityStats)) + for l, localityData := range sd.LocalityStats { + lid, err := internal.LocalityIDFromString(l) + if err != nil { + return err + } + loadMetricStats := make([]*v3endpointpb.EndpointLoadMetricStats, 0, len(localityData.LoadStats)) + for name, loadData := range localityData.LoadStats { + loadMetricStats = append(loadMetricStats, &v3endpointpb.EndpointLoadMetricStats{ + MetricName: name, + NumRequestsFinishedWithMetric: loadData.Count, + TotalMetricValue: loadData.Sum, + }) + } + localityStats = append(localityStats, &v3endpointpb.UpstreamLocalityStats{ + Locality: &v3corepb.Locality{ + Region: lid.Region, + Zone: lid.Zone, + SubZone: lid.SubZone, + }, + TotalSuccessfulRequests: localityData.RequestStats.Succeeded, + TotalRequestsInProgress: localityData.RequestStats.InProgress, + TotalErrorRequests: localityData.RequestStats.Errored, + LoadMetricStats: loadMetricStats, + UpstreamEndpointStats: nil, // TODO: populate for per endpoint loads. + }) + } + + clusterStats = append(clusterStats, &v3endpointpb.ClusterStats{ + ClusterName: sd.Cluster, + ClusterServiceName: sd.Service, + UpstreamLocalityStats: localityStats, + TotalDroppedRequests: sd.TotalDrops, + DroppedRequests: droppedReqs, + LoadReportInterval: ptypes.DurationProto(sd.ReportInterval), + }) + } + + req := &v3lrspb.LoadStatsRequest{ClusterStats: clusterStats} + if t.logger.V(perRPCVerbosityLevel) { + t.logger.Infof("Sending LRS loads: %s", pretty.ToJSON(req)) + } + err := stream.Send(req) + if err == io.EOF { + return getStreamError(stream) + } + return err +} + +func getStreamError(stream lrsStream) error { + for { + if _, err := stream.Recv(); err != nil { + return err + } + } +} diff --git a/xds/internal/xdsclient/transport/loadreport_test.go b/xds/internal/xdsclient/transport/loadreport_test.go new file mode 100644 index 000000000000..c3cdfede5cb6 --- /dev/null +++ b/xds/internal/xdsclient/transport/loadreport_test.go @@ -0,0 +1,195 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package transport_test + +import ( + "context" + "testing" + "time" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + "github.com/google/go-cmp/cmp" + "github.com/google/uuid" + "google.golang.org/grpc/internal/testutils/xds/fakeserver" + "google.golang.org/grpc/xds/internal/testutils" + "google.golang.org/grpc/xds/internal/xdsclient/transport" + "google.golang.org/protobuf/testing/protocmp" + "google.golang.org/protobuf/types/known/durationpb" + + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3lrspb "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v3" +) + +func (s) TestReportLoad(t *testing.T) { + // Create a fake xDS management server listening on a local port. + mgmtServer, cleanup := startFakeManagementServer(t) + defer cleanup() + t.Logf("Started xDS management server on %s", mgmtServer.Address) + + // Create a transport to the fake management server. + nodeProto := &v3corepb.Node{Id: uuid.New().String()} + tr, err := transport.New(transport.Options{ + ServerCfg: *testutils.ServerConfigForAddress(t, mgmtServer.Address), + NodeProto: nodeProto, + OnRecvHandler: func(transport.ResourceUpdate) error { return nil }, // No ADS validation. + OnErrorHandler: func(error) {}, // No ADS stream error handling. + OnSendHandler: func(*transport.ResourceSendInfo) {}, // No ADS stream update handling. + Backoff: func(int) time.Duration { return time.Duration(0) }, // No backoff. + }) + if err != nil { + t.Fatalf("Failed to create xDS transport: %v", err) + } + defer tr.Close() + + // Ensure that a new connection is made to the management server. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := mgmtServer.NewConnChan.Receive(ctx); err != nil { + t.Fatalf("Timeout when waiting for a new connection to the management server: %v", err) + } + + // Call the load reporting API, and ensure that an LRS stream is created. + store1, cancelLRS1 := tr.ReportLoad() + if err != nil { + t.Fatalf("Failed to start LRS load reporting: %v", err) + } + if _, err := mgmtServer.LRSStreamOpenChan.Receive(ctx); err != nil { + t.Fatalf("Timeout when waiting for LRS stream to be created: %v", err) + } + + // Push some loads on the received store. + store1.PerCluster("cluster1", "eds1").CallDropped("test") + + // Ensure the initial request is received. + req, err := mgmtServer.LRSRequestChan.Receive(ctx) + if err != nil { + t.Fatalf("Timeout when waiting for initial LRS request: %v", err) + } + gotInitialReq := req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest) + nodeProto.ClientFeatures = []string{"envoy.lrs.supports_send_all_clusters"} + wantInitialReq := &v3lrspb.LoadStatsRequest{Node: nodeProto} + if diff := cmp.Diff(gotInitialReq, wantInitialReq, protocmp.Transform()); diff != "" { + t.Fatalf("Unexpected diff in initial LRS request (-got, +want):\n%s", diff) + } + + // Send a response from the server with a small deadline. + mgmtServer.LRSResponseChan <- &fakeserver.Response{ + Resp: &v3lrspb.LoadStatsResponse{ + SendAllClusters: true, + LoadReportingInterval: &durationpb.Duration{Nanos: 50000000}, // 50ms + }, + } + + // Ensure that loads are seen on the server. + req, err = mgmtServer.LRSRequestChan.Receive(ctx) + if err != nil { + t.Fatalf("Timeout when waiting for LRS request with loads: %v", err) + } + gotLoad := req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest).ClusterStats + if l := len(gotLoad); l != 1 { + t.Fatalf("Received load for %d clusters, want 1", l) + } + // This field is set by the client to indicate the actual time elapsed since + // the last report was sent. We cannot deterministically compare this, and + // we cannot use the cmpopts.IgnoreFields() option on proto structs, since + // we already use the protocmp.Transform() which marshals the struct into + // another message. Hence setting this field to nil is the best option here. + gotLoad[0].LoadReportInterval = nil + wantLoad := &v3endpointpb.ClusterStats{ + ClusterName: "cluster1", + ClusterServiceName: "eds1", + TotalDroppedRequests: 1, + DroppedRequests: []*v3endpointpb.ClusterStats_DroppedRequests{{Category: "test", DroppedCount: 1}}, + } + if diff := cmp.Diff(wantLoad, gotLoad[0], protocmp.Transform()); diff != "" { + t.Fatalf("Unexpected diff in LRS request (-got, +want):\n%s", diff) + } + + // Make another call to the load reporting API, and ensure that a new LRS + // stream is not created. + store2, cancelLRS2 := tr.ReportLoad() + if err != nil { + t.Fatalf("Failed to start LRS load reporting: %v", err) + } + sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + if _, err := mgmtServer.LRSStreamOpenChan.Receive(sCtx); err != context.DeadlineExceeded { + t.Fatal("New LRS stream created when expected to use an existing one") + } + + // Push more loads. + store2.PerCluster("cluster2", "eds2").CallDropped("test") + + // Ensure that loads are seen on the server. We need a loop here because + // there could have been some requests from the client in the time between + // us reading the first request and now. Those would have been queued in the + // request channel that we read out of. + for { + if ctx.Err() != nil { + t.Fatalf("Timeout when waiting for new loads to be seen on the server") + } + + req, err = mgmtServer.LRSRequestChan.Receive(ctx) + if err != nil { + continue + } + gotLoad = req.(*fakeserver.Request).Req.(*v3lrspb.LoadStatsRequest).ClusterStats + if l := len(gotLoad); l != 1 { + continue + } + gotLoad[0].LoadReportInterval = nil + wantLoad := &v3endpointpb.ClusterStats{ + ClusterName: "cluster2", + ClusterServiceName: "eds2", + TotalDroppedRequests: 1, + DroppedRequests: []*v3endpointpb.ClusterStats_DroppedRequests{{Category: "test", DroppedCount: 1}}, + } + if diff := cmp.Diff(wantLoad, gotLoad[0], protocmp.Transform()); diff != "" { + t.Logf("Unexpected diff in LRS request (-got, +want):\n%s", diff) + continue + } + break + } + + // Cancel the first load reporting call, and ensure that the stream does not + // close (because we have aother call open). + cancelLRS1() + sCtx, sCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + if _, err := mgmtServer.LRSStreamCloseChan.Receive(sCtx); err != context.DeadlineExceeded { + t.Fatal("LRS stream closed when expected to stay open") + } + + // Cancel the second load reporting call, and ensure the stream is closed. + cancelLRS2() + if _, err := mgmtServer.LRSStreamCloseChan.Receive(ctx); err != nil { + t.Fatal("Timeout waiting for LRS stream to close") + } + + // Calling the load reporting API again should result in the creation of a + // new LRS stream. This ensures that creating and closing multiple streams + // works smoothly. + _, cancelLRS3 := tr.ReportLoad() + if err != nil { + t.Fatalf("Failed to start LRS load reporting: %v", err) + } + if _, err := mgmtServer.LRSStreamOpenChan.Receive(ctx); err != nil { + t.Fatalf("Timeout when waiting for LRS stream to be created: %v", err) + } + cancelLRS3() +} diff --git a/xds/internal/xdsclient/transport/transport.go b/xds/internal/xdsclient/transport/transport.go new file mode 100644 index 000000000000..86803588a7cc --- /dev/null +++ b/xds/internal/xdsclient/transport/transport.go @@ -0,0 +1,638 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package transport implements the xDS transport protocol functionality +// required by the xdsclient. +package transport + +import ( + "context" + "errors" + "fmt" + "sync" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/internal/backoff" + "google.golang.org/grpc/internal/buffer" + "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/keepalive" + "google.golang.org/grpc/xds/internal/xdsclient/bootstrap" + "google.golang.org/grpc/xds/internal/xdsclient/load" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" + "google.golang.org/protobuf/types/known/anypb" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3adsgrpc "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + statuspb "google.golang.org/genproto/googleapis/rpc/status" +) + +// Any per-RPC level logs which print complete request or response messages +// should be gated at this verbosity level. Other per-RPC level logs which print +// terse output should be at `INFO` and verbosity 2, which corresponds to using +// the `Debugf` method on the logger. +const perRPCVerbosityLevel = 9 + +type adsStream = v3adsgrpc.AggregatedDiscoveryService_StreamAggregatedResourcesClient + +// Transport provides a resource-type agnostic implementation of the xDS +// transport protocol. At this layer, resource contents are supposed to be +// opaque blobs which should be be meaningful only to the xDS data model layer +// which is implemented by the `xdsresource` package. +// +// Under the hood, it owns the gRPC connection to a single management server and +// manages the lifecycle of ADS/LRS streams. It uses the xDS v3 transport +// protocol version. +type Transport struct { + // These fields are initialized at creation time and are read-only afterwards. + cc *grpc.ClientConn // ClientConn to the mangement server. + serverURI string // URI of the management server. + onRecvHandler OnRecvHandlerFunc // Resource update handler. xDS data model layer. + onErrorHandler func(error) // To report underlying stream errors. + onSendHandler OnSendHandlerFunc // To report resources requested on ADS stream. + lrsStore *load.Store // Store returned to user for pushing loads. + backoff func(int) time.Duration // Backoff after stream failures. + nodeProto *v3corepb.Node // Identifies the gRPC application. + logger *grpclog.PrefixLogger // Prefix logger for transport logs. + adsRunnerCancel context.CancelFunc // CancelFunc for the ADS goroutine. + adsRunnerDoneCh chan struct{} // To notify exit of ADS goroutine. + lrsRunnerDoneCh chan struct{} // To notify exit of LRS goroutine. + + // These channels enable synchronization amongst the different goroutines + // spawned by the transport, and between asynchorous events resulting from + // receipt of responses from the management server. + adsStreamCh chan adsStream // New ADS streams are pushed here. + adsRequestCh *buffer.Unbounded // Resource and ack requests are pushed here. + + // mu guards the following runtime state maintained by the transport. + mu sync.Mutex + // resources is map from resource type URL to the set of resource names + // being requested for that type. When the ADS stream is restarted, the + // transport requests all these resources again from the management server. + resources map[string]map[string]bool + // versions is a map from resource type URL to the most recently ACKed + // version for that resource. Resource versions are a property of the + // resource type and not the stream, and will not be reset upon stream + // restarts. + versions map[string]string + // nonces is a map from resource type URL to the most recently received + // nonce for that resource type. Nonces are a property of the ADS stream and + // will be reset upon stream restarts. + nonces map[string]string + + lrsMu sync.Mutex // Protects all LRS state. + lrsCancelStream context.CancelFunc // CancelFunc for the LRS stream. + lrsRefCount int // Reference count on the load store. +} + +// OnRecvHandlerFunc is the implementation at the xDS data model layer, which +// determines if the configuration received from the management server can be +// applied locally or not. +// +// A nil error is returned from this function when the data model layer believes +// that the received configuration is good and can be applied locally. This will +// cause the transport layer to send an ACK to the management server. A non-nil +// error is returned from this function when the data model layer believes +// otherwise, and this will cause the transport layer to send a NACK. +type OnRecvHandlerFunc func(update ResourceUpdate) error + +// OnSendHandlerFunc is the implementation at the authority, which handles state +// changes for the resource watch and stop watch timers accordingly. +type OnSendHandlerFunc func(update *ResourceSendInfo) + +// ResourceUpdate is a representation of the configuration update received from +// the management server. It only contains fields which are useful to the data +// model layer, and layers above it. +type ResourceUpdate struct { + // Resources is the list of resources received from the management server. + Resources []*anypb.Any + // URL is the resource type URL for the above resources. + URL string + // Version is the resource version, for the above resources, as specified by + // the management server. + Version string +} + +// Options specifies configuration knobs used when creating a new Transport. +type Options struct { + // ServerCfg contains all the configuration required to connect to the xDS + // management server. + ServerCfg bootstrap.ServerConfig + // OnRecvHandler is the component which makes ACK/NACK decisions based on + // the received resources. + // + // Invoked inline and implementations must not block. + OnRecvHandler OnRecvHandlerFunc + // OnErrorHandler provides a way for the transport layer to report + // underlying stream errors. These can be bubbled all the way up to the user + // of the xdsClient. + // + // Invoked inline and implementations must not block. + OnErrorHandler func(error) + // OnSendHandler provides a way for the transport layer to report underlying + // resource requests sent on the stream. However, Send() on the ADS stream will + // return successfully as long as: + // 1. there is enough flow control quota to send the message. + // 2. the message is added to the send buffer. + // However, the connection may fail after the callback is invoked and before + // the message is actually sent on the wire. This is accepted. + // + // Invoked inline and implementations must not block. + OnSendHandler func(*ResourceSendInfo) + // Backoff controls the amount of time to backoff before recreating failed + // ADS streams. If unspecified, a default exponential backoff implementation + // is used. For more details, see: + // https://github.com/grpc/grpc/blob/master/doc/connection-backoff.md. + Backoff func(retries int) time.Duration + // Logger does logging with a prefix. + Logger *grpclog.PrefixLogger + // NodeProto contains the Node proto to be used in xDS requests. This will be + // of type *v3corepb.Node. + NodeProto *v3corepb.Node +} + +// For overriding in unit tests. +var grpcDial = grpc.Dial + +// New creates a new Transport. +func New(opts Options) (*Transport, error) { + switch { + case opts.ServerCfg.ServerURI == "": + return nil, errors.New("missing server URI when creating a new transport") + case opts.ServerCfg.CredsDialOption() == nil: + return nil, errors.New("missing credentials when creating a new transport") + case opts.OnRecvHandler == nil: + return nil, errors.New("missing OnRecv callback handler when creating a new transport") + case opts.OnErrorHandler == nil: + return nil, errors.New("missing OnError callback handler when creating a new transport") + case opts.OnSendHandler == nil: + return nil, errors.New("missing OnSend callback handler when creating a new transport") + } + + // Dial the xDS management with the passed in credentials. + dopts := []grpc.DialOption{ + opts.ServerCfg.CredsDialOption(), + grpc.WithKeepaliveParams(keepalive.ClientParameters{ + // We decided to use these sane defaults in all languages, and + // kicked the can down the road as far making these configurable. + Time: 5 * time.Minute, + Timeout: 20 * time.Second, + }), + } + cc, err := grpcDial(opts.ServerCfg.ServerURI, dopts...) + if err != nil { + // An error from a non-blocking dial indicates something serious. + return nil, fmt.Errorf("failed to create a transport to the management server %q: %v", opts.ServerCfg.ServerURI, err) + } + + boff := opts.Backoff + if boff == nil { + boff = backoff.DefaultExponential.Backoff + } + ret := &Transport{ + cc: cc, + serverURI: opts.ServerCfg.ServerURI, + onRecvHandler: opts.OnRecvHandler, + onErrorHandler: opts.OnErrorHandler, + onSendHandler: opts.OnSendHandler, + lrsStore: load.NewStore(), + backoff: boff, + nodeProto: opts.NodeProto, + logger: opts.Logger, + + adsStreamCh: make(chan adsStream, 1), + adsRequestCh: buffer.NewUnbounded(), + resources: make(map[string]map[string]bool), + versions: make(map[string]string), + nonces: make(map[string]string), + adsRunnerDoneCh: make(chan struct{}), + } + + // This context is used for sending and receiving RPC requests and + // responses. It is also used by all the goroutines spawned by this + // Transport. Therefore, cancelling this context when the transport is + // closed will essentially cancel any pending RPCs, and cause the goroutines + // to terminate. + ctx, cancel := context.WithCancel(context.Background()) + ret.adsRunnerCancel = cancel + go ret.adsRunner(ctx) + + ret.logger.Infof("Created transport to server %q", ret.serverURI) + return ret, nil +} + +// resourceRequest wraps the resource type url and the resource names requested +// by the user of this transport. +type resourceRequest struct { + resources []string + url string +} + +// SendRequest sends out an ADS request for the provided resources of the +// specified resource type. +// +// The request is sent out asynchronously. If no valid stream exists at the time +// of processing this request, it is queued and will be sent out once a valid +// stream exists. +// +// If a successful response is received, the update handler callback provided at +// creation time is invoked. If an error is encountered, the stream error +// handler callback provided at creation time is invoked. +func (t *Transport) SendRequest(url string, resources []string) { + t.adsRequestCh.Put(&resourceRequest{ + url: url, + resources: resources, + }) +} + +func (t *Transport) newAggregatedDiscoveryServiceStream(ctx context.Context, cc *grpc.ClientConn) (adsStream, error) { + // The transport retries the stream with an exponential backoff whenever the + // stream breaks without ever having seen a response. + return v3adsgrpc.NewAggregatedDiscoveryServiceClient(cc).StreamAggregatedResources(ctx) +} + +// ResourceSendInfo wraps the names and url of resources sent to the management +// server. This is used by the `authority` type to start/stop the watch timer +// associated with every resource in the update. +type ResourceSendInfo struct { + ResourceNames []string + URL string +} + +func (t *Transport) sendAggregatedDiscoveryServiceRequest(stream adsStream, sendNodeProto bool, resourceNames []string, resourceURL, version, nonce string, nackErr error) error { + req := &v3discoverypb.DiscoveryRequest{ + TypeUrl: resourceURL, + ResourceNames: resourceNames, + VersionInfo: version, + ResponseNonce: nonce, + } + if sendNodeProto { + req.Node = t.nodeProto + } + if nackErr != nil { + req.ErrorDetail = &statuspb.Status{ + Code: int32(codes.InvalidArgument), Message: nackErr.Error(), + } + } + if err := stream.Send(req); err != nil { + return err + } + if t.logger.V(perRPCVerbosityLevel) { + t.logger.Infof("ADS request sent: %v", pretty.ToJSON(req)) + } else { + t.logger.Debugf("ADS request sent for type %q, resources: %v, version %q, nonce %q", resourceURL, resourceNames, version, nonce) + } + t.onSendHandler(&ResourceSendInfo{URL: resourceURL, ResourceNames: resourceNames}) + return nil +} + +func (t *Transport) recvAggregatedDiscoveryServiceResponse(stream adsStream) (resources []*anypb.Any, resourceURL, version, nonce string, err error) { + resp, err := stream.Recv() + if err != nil { + return nil, "", "", "", err + } + if t.logger.V(perRPCVerbosityLevel) { + t.logger.Infof("ADS response received: %v", pretty.ToJSON(resp)) + } else { + t.logger.Debugf("ADS response received for type %q, version %q, nonce %q", resp.GetTypeUrl(), resp.GetVersionInfo(), resp.GetNonce()) + } + return resp.GetResources(), resp.GetTypeUrl(), resp.GetVersionInfo(), resp.GetNonce(), nil +} + +// adsRunner starts an ADS stream (and backs off exponentially, if the previous +// stream failed without receiving a single reply) and runs the sender and +// receiver routines to send and receive data from the stream respectively. +func (t *Transport) adsRunner(ctx context.Context) { + defer close(t.adsRunnerDoneCh) + + go t.send(ctx) + + backoffAttempt := 0 + backoffTimer := time.NewTimer(0) + for ctx.Err() == nil { + select { + case <-backoffTimer.C: + case <-ctx.Done(): + backoffTimer.Stop() + return + } + + // We reset backoff state when we successfully receive at least one + // message from the server. + resetBackoff := func() bool { + stream, err := t.newAggregatedDiscoveryServiceStream(ctx, t.cc) + if err != nil { + t.onErrorHandler(err) + t.logger.Warningf("Creating new ADS stream failed: %v", err) + return false + } + t.logger.Infof("ADS stream created") + + select { + case <-t.adsStreamCh: + default: + } + t.adsStreamCh <- stream + return t.recv(stream) + }() + + if resetBackoff { + backoffTimer.Reset(0) + backoffAttempt = 0 + } else { + backoffTimer.Reset(t.backoff(backoffAttempt)) + backoffAttempt++ + } + } +} + +// send is a separate goroutine for sending resource requests on the ADS stream. +// +// For every new stream received on the stream channel, all existing resources +// are re-requested from the management server. +// +// For every new resource request received on the resources channel, the +// resources map is updated (this ensures that resend will pick them up when +// there are new streams) and the appropriate request is sent out. +func (t *Transport) send(ctx context.Context) { + var stream adsStream + // The xDS protocol only requires that we send the node proto in the first + // discovery request on every stream. Sending the node proto in every + // request message wastes CPU resources on the client and the server. + sendNodeProto := true + for { + select { + case <-ctx.Done(): + return + case stream = <-t.adsStreamCh: + // We have a new stream and we've to ensure that the node proto gets + // sent out in the first request on the stream. At this point, we + // might not have any registered watches. Setting this field to true + // here will ensure that the node proto gets sent out along with the + // discovery request when the first watch is registered. + if len(t.resources) == 0 { + sendNodeProto = true + continue + } + + if !t.sendExisting(stream) { + // Send failed, clear the current stream. Attempt to resend will + // only be made after a new stream is created. + stream = nil + continue + } + sendNodeProto = false + case u, ok := <-t.adsRequestCh.Get(): + if !ok { + // No requests will be sent after the adsRequestCh buffer is closed. + return + } + t.adsRequestCh.Load() + + var ( + resources []string + url, version, nonce string + send bool + nackErr error + ) + switch update := u.(type) { + case *resourceRequest: + resources, url, version, nonce = t.processResourceRequest(update) + case *ackRequest: + resources, url, version, nonce, send = t.processAckRequest(update, stream) + if !send { + continue + } + nackErr = update.nackErr + } + if stream == nil { + // There's no stream yet. Skip the request. This request + // will be resent to the new streams. If no stream is + // created, the watcher will timeout (same as server not + // sending response back). + continue + } + if err := t.sendAggregatedDiscoveryServiceRequest(stream, sendNodeProto, resources, url, version, nonce, nackErr); err != nil { + t.logger.Warningf("Sending ADS request for resources: %q, url: %q, version: %q, nonce: %q failed: %v", resources, url, version, nonce, err) + // Send failed, clear the current stream. + stream = nil + } + sendNodeProto = false + } + } +} + +// sendExisting sends out xDS requests for existing resources when recovering +// from a broken stream. +// +// We call stream.Send() here with the lock being held. It should be OK to do +// that here because the stream has just started and Send() usually returns +// quickly (once it pushes the message onto the transport layer) and is only +// ever blocked if we don't have enough flow control quota. +func (t *Transport) sendExisting(stream adsStream) bool { + t.mu.Lock() + defer t.mu.Unlock() + + // Reset only the nonces map when the stream restarts. + // + // xDS spec says the following. See section: + // https://www.envoyproxy.io/docs/envoy/latest/api-docs/xds_protocol#ack-nack-and-resource-type-instance-version + // + // Note that the version for a resource type is not a property of an + // individual xDS stream but rather a property of the resources themselves. If + // the stream becomes broken and the client creates a new stream, the client’s + // initial request on the new stream should indicate the most recent version + // seen by the client on the previous stream + t.nonces = make(map[string]string) + + // Send node proto only in the first request on the stream. + sendNodeProto := true + for url, resources := range t.resources { + if err := t.sendAggregatedDiscoveryServiceRequest(stream, sendNodeProto, mapToSlice(resources), url, t.versions[url], "", nil); err != nil { + t.logger.Warningf("Sending ADS request for resources: %q, url: %q, version: %q, nonce: %q failed: %v", resources, url, t.versions[url], "", err) + return false + } + sendNodeProto = false + } + + return true +} + +// recv receives xDS responses on the provided ADS stream and branches out to +// message specific handlers. Returns true if at least one message was +// successfully received. +func (t *Transport) recv(stream adsStream) bool { + msgReceived := false + for { + resources, url, rVersion, nonce, err := t.recvAggregatedDiscoveryServiceResponse(stream) + if err != nil { + // Note that we do not consider it an error if the ADS stream was closed + // after having received a response on the stream. This is because there + // are legitimate reasons why the server may need to close the stream during + // normal operations, such as needing to rebalance load or the underlying + // connection hitting its max connection age limit. + // (see [gRFC A9](https://github.com/grpc/proposal/blob/master/A9-server-side-conn-mgt.md)). + if msgReceived { + err = xdsresource.NewErrorf(xdsresource.ErrTypeStreamFailedAfterRecv, err.Error()) + } + t.onErrorHandler(err) + t.logger.Warningf("ADS stream closed: %v", err) + return msgReceived + } + msgReceived = true + + err = t.onRecvHandler(ResourceUpdate{ + Resources: resources, + URL: url, + Version: rVersion, + }) + if xdsresource.ErrType(err) == xdsresource.ErrorTypeResourceTypeUnsupported { + t.logger.Warningf("%v", err) + continue + } + // If the data model layer returned an error, we need to NACK the + // response in which case we need to set the version to the most + // recently accepted version of this resource type. + if err != nil { + t.mu.Lock() + t.adsRequestCh.Put(&ackRequest{ + url: url, + nonce: nonce, + stream: stream, + version: t.versions[url], + nackErr: err, + }) + t.mu.Unlock() + t.logger.Warningf("Sending NACK for resource type: %q, version: %q, nonce: %q, reason: %v", url, rVersion, nonce, err) + continue + } + t.adsRequestCh.Put(&ackRequest{ + url: url, + nonce: nonce, + stream: stream, + version: rVersion, + }) + t.logger.Debugf("Sending ACK for resource type: %q, version: %q, nonce: %q", url, rVersion, nonce) + } +} + +func mapToSlice(m map[string]bool) []string { + ret := make([]string, 0, len(m)) + for i := range m { + ret = append(ret, i) + } + return ret +} + +func sliceToMap(ss []string) map[string]bool { + ret := make(map[string]bool, len(ss)) + for _, s := range ss { + ret[s] = true + } + return ret +} + +// processResourceRequest pulls the fields needed to send out an ADS request. +// The resource type and the list of resources to request are provided by the +// user, while the version and nonce are maintained internally. +// +// The resources map, which keeps track of the resources being requested, is +// updated here. Any subsequent stream failure will re-request resources stored +// in this map. +// +// Returns the list of resources, resource type url, version and nonce. +func (t *Transport) processResourceRequest(req *resourceRequest) ([]string, string, string, string) { + t.mu.Lock() + defer t.mu.Unlock() + + resources := sliceToMap(req.resources) + t.resources[req.url] = resources + return req.resources, req.url, t.versions[req.url], t.nonces[req.url] +} + +type ackRequest struct { + url string // Resource type URL. + version string // NACK if version is an empty string. + nonce string + nackErr error // nil for ACK, non-nil for NACK. + // ACK/NACK are tagged with the stream it's for. When the stream is down, + // all the ACK/NACK for this stream will be dropped, and the version/nonce + // won't be updated. + stream grpc.ClientStream +} + +// processAckRequest pulls the fields needed to send out an ADS ACK. The nonces +// and versions map is updated. +// +// Returns the list of resources, resource type url, version, nonce, and an +// indication of whether an ACK should be sent on the wire or not. +func (t *Transport) processAckRequest(ack *ackRequest, stream grpc.ClientStream) ([]string, string, string, string, bool) { + if ack.stream != stream { + // If ACK's stream isn't the current sending stream, this means the ACK + // was pushed to queue before the old stream broke, and a new stream has + // been started since. Return immediately here so we don't update the + // nonce for the new stream. + return nil, "", "", "", false + } + + t.mu.Lock() + defer t.mu.Unlock() + + // Update the nonce irrespective of whether we send the ACK request on wire. + // An up-to-date nonce is required for the next request. + nonce := ack.nonce + t.nonces[ack.url] = nonce + + s, ok := t.resources[ack.url] + if !ok || len(s) == 0 { + // We don't send the ACK request if there are no resources of this type + // in our resources map. This can be either when the server sends + // responses before any request, or the resources are removed while the + // ackRequest was in queue). If we send a request with an empty + // resource name list, the server may treat it as a wild card and send + // us everything. + return nil, "", "", "", false + } + resources := mapToSlice(s) + + // Update the versions map only when we plan to send an ACK. + if ack.nackErr == nil { + t.versions[ack.url] = ack.version + } + + return resources, ack.url, ack.version, nonce, true +} + +// Close closes the Transport and frees any associated resources. +func (t *Transport) Close() { + t.adsRunnerCancel() + <-t.adsRunnerDoneCh + t.adsRequestCh.Close() + t.cc.Close() +} + +// ChannelConnectivityStateForTesting returns the connectivity state of the gRPC +// channel to the management server. +// +// Only for testing purposes. +func (t *Transport) ChannelConnectivityStateForTesting() connectivity.State { + return t.cc.GetState() +} diff --git a/xds/internal/xdsclient/transport/transport_ack_nack_test.go b/xds/internal/xdsclient/transport/transport_ack_nack_test.go new file mode 100644 index 000000000000..f887ae1de0bd --- /dev/null +++ b/xds/internal/xdsclient/transport/transport_ack_nack_test.go @@ -0,0 +1,520 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package transport_test + +import ( + "context" + "errors" + "fmt" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/google/uuid" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + xdstestutils "google.golang.org/grpc/xds/internal/testutils" + "google.golang.org/grpc/xds/internal/xdsclient/transport" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource/version" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/testing/protocmp" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/wrapperspb" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + statuspb "google.golang.org/genproto/googleapis/rpc/status" +) + +var ( + errWantNack = errors.New("unsupported field 'use_original_dst' is present and set to true") + + // A simple update handler for listener resources which validates only the + // `use_original_dst` field. + dataModelValidator = func(update transport.ResourceUpdate) error { + for _, r := range update.Resources { + inner := &v3discoverypb.Resource{} + if err := proto.Unmarshal(r.GetValue(), inner); err != nil { + return fmt.Errorf("failed to unmarshal DiscoveryResponse: %v", err) + } + lis := &v3listenerpb.Listener{} + if err := proto.Unmarshal(r.GetValue(), lis); err != nil { + return fmt.Errorf("failed to unmarshal DiscoveryResponse: %v", err) + } + if useOrigDst := lis.GetUseOriginalDst(); useOrigDst != nil && useOrigDst.GetValue() { + return errWantNack + } + } + return nil + } +) + +// TestSimpleAckAndNack tests simple ACK and NACK scenarios. +// 1. When the data model layer likes a received response, the test verifies +// that an ACK is sent matching the version and nonce from the response. +// 2. When a subsequent response is disliked by the data model layer, the test +// verifies that a NACK is sent matching the previously ACKed version and +// current nonce from the response. +// 3. When a subsequent response is liked by the data model layer, the test +// verifies that an ACK is sent matching the version and nonce from the +// current response. +func (s) TestSimpleAckAndNack(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Create an xDS management server listening on a local port. Configure the + // request and response handlers to push on channels which are inspected by + // the test goroutine to verify ack version and nonce. + streamRequestCh := make(chan *v3discoverypb.DiscoveryRequest, 1) + streamResponseCh := make(chan *v3discoverypb.DiscoveryResponse, 1) + mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{ + OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + select { + case streamRequestCh <- req: + case <-ctx.Done(): + } + return nil + }, + OnStreamResponse: func(_ context.Context, _ int64, _ *v3discoverypb.DiscoveryRequest, resp *v3discoverypb.DiscoveryResponse) { + select { + case streamResponseCh <- resp: + case <-ctx.Done(): + } + }, + }) + if err != nil { + t.Fatalf("Failed to start xDS management server: %v", err) + } + defer mgmtServer.Stop() + t.Logf("Started xDS management server on %s", mgmtServer.Address) + + // Configure the management server with appropriate resources. + apiListener := &v3listenerpb.ApiListener{ + ApiListener: func() *anypb.Any { + return testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{ + Rds: &v3httppb.Rds{ + ConfigSource: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}, + }, + RouteConfigName: "route-configuration-name", + }, + }, + }) + }(), + } + const resourceName = "resource name 1" + listenerResource := &v3listenerpb.Listener{ + Name: resourceName, + ApiListener: apiListener, + } + nodeID := uuid.New().String() + mgmtServer.Update(ctx, e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listenerResource}, + SkipValidation: true, + }) + + // Create a new transport. + tr, err := transport.New(transport.Options{ + ServerCfg: *xdstestutils.ServerConfigForAddress(t, mgmtServer.Address), + OnRecvHandler: dataModelValidator, + OnErrorHandler: func(err error) {}, + OnSendHandler: func(*transport.ResourceSendInfo) {}, + NodeProto: &v3corepb.Node{Id: nodeID}, + }) + if err != nil { + t.Fatalf("Failed to create xDS transport: %v", err) + } + defer tr.Close() + + // Send a discovery request through the transport. + tr.SendRequest(version.V3ListenerURL, []string{resourceName}) + + // Verify that the initial discovery request matches expectation. + var gotReq *v3discoverypb.DiscoveryRequest + select { + case gotReq = <-streamRequestCh: + case <-ctx.Done(): + t.Fatalf("Timeout waiting for discovery request on the stream") + } + wantReq := &v3discoverypb.DiscoveryRequest{ + VersionInfo: "", + Node: &v3corepb.Node{Id: nodeID}, + ResourceNames: []string{resourceName}, + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + ResponseNonce: "", + } + if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform(), cmpopts.SortSlices(strSort)); diff != "" { + t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff) + } + + // Capture the version and nonce from the response. + var gotResp *v3discoverypb.DiscoveryResponse + select { + case gotResp = <-streamResponseCh: + case <-ctx.Done(): + t.Fatalf("Timeout waiting for discovery response on the stream") + } + + // Verify that the ACK contains the appropriate version and nonce. + wantReq.VersionInfo = gotResp.GetVersionInfo() + wantReq.ResponseNonce = gotResp.GetNonce() + select { + case gotReq = <-streamRequestCh: + case <-ctx.Done(): + t.Fatalf("Timeout waiting for the discovery request ACK on the stream") + } + if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform(), cmpopts.SortSlices(strSort)); diff != "" { + t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff) + } + + // Update the management server's copy of the resource to include a field + // which will cause the resource to be NACKed. + badListener := proto.Clone(listenerResource).(*v3listenerpb.Listener) + badListener.UseOriginalDst = &wrapperspb.BoolValue{Value: true} + mgmtServer.Update(ctx, e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{badListener}, + SkipValidation: true, + }) + + select { + case gotResp = <-streamResponseCh: + case <-ctx.Done(): + t.Fatalf("Timeout waiting for discovery response on the stream") + } + + // Verify that the NACK contains the appropriate version, nonce and error. + // We expect the version to not change as this is a NACK. + wantReq.ResponseNonce = gotResp.GetNonce() + wantReq.ErrorDetail = &statuspb.Status{ + Code: int32(codes.InvalidArgument), + Message: errWantNack.Error(), + } + select { + case gotReq = <-streamRequestCh: + case <-ctx.Done(): + t.Fatalf("Timeout waiting for the discovery request ACK on the stream") + } + if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform(), cmpopts.SortSlices(strSort)); diff != "" { + t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff) + } + + // Update the management server to send a good resource again. + mgmtServer.Update(ctx, e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listenerResource}, + SkipValidation: true, + }) + + // The envoy-go-control-plane management server keeps resending the same + // resource as long as we keep NACK'ing it. So, we will see the bad resource + // sent to us a few times here, before receiving the good resource. + for { + select { + case gotResp = <-streamResponseCh: + case <-ctx.Done(): + t.Fatalf("Timeout waiting for discovery response on the stream") + } + + // Verify that the ACK contains the appropriate version and nonce. + wantReq.VersionInfo = gotResp.GetVersionInfo() + wantReq.ResponseNonce = gotResp.GetNonce() + wantReq.ErrorDetail = nil + select { + case gotReq = <-streamRequestCh: + case <-ctx.Done(): + t.Fatalf("Timeout waiting for the discovery request ACK on the stream") + } + diff := cmp.Diff(gotReq, wantReq, protocmp.Transform(), cmpopts.SortSlices(strSort)) + if diff == "" { + break + } + t.Logf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff) + } +} + +// TestInvalidFirstResponse tests the case where the first response is invalid. +// The test verifies that the NACK contains an empty version string. +func (s) TestInvalidFirstResponse(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Create an xDS management server listening on a local port. Configure the + // request and response handlers to push on channels which are inspected by + // the test goroutine to verify ack version and nonce. + streamRequestCh := make(chan *v3discoverypb.DiscoveryRequest, 1) + streamResponseCh := make(chan *v3discoverypb.DiscoveryResponse, 1) + mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{ + OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + select { + case streamRequestCh <- req: + case <-ctx.Done(): + } + return nil + }, + OnStreamResponse: func(_ context.Context, _ int64, _ *v3discoverypb.DiscoveryRequest, resp *v3discoverypb.DiscoveryResponse) { + select { + case streamResponseCh <- resp: + case <-ctx.Done(): + } + }, + }) + if err != nil { + t.Fatalf("Failed to start xDS management server: %v", err) + } + defer mgmtServer.Stop() + t.Logf("Started xDS management server on %s", mgmtServer.Address) + + // Configure the management server with appropriate resources. + apiListener := &v3listenerpb.ApiListener{ + ApiListener: func() *anypb.Any { + return testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{ + Rds: &v3httppb.Rds{ + ConfigSource: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}, + }, + RouteConfigName: "route-configuration-name", + }, + }, + }) + }(), + } + const resourceName = "resource name 1" + listenerResource := &v3listenerpb.Listener{ + Name: resourceName, + ApiListener: apiListener, + UseOriginalDst: &wrapperspb.BoolValue{Value: true}, // This will cause the resource to be NACKed. + } + nodeID := uuid.New().String() + mgmtServer.Update(ctx, e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listenerResource}, + SkipValidation: true, + }) + + // Create a new transport. + tr, err := transport.New(transport.Options{ + ServerCfg: *xdstestutils.ServerConfigForAddress(t, mgmtServer.Address), + NodeProto: &v3corepb.Node{Id: nodeID}, + OnRecvHandler: dataModelValidator, + OnErrorHandler: func(err error) {}, + OnSendHandler: func(*transport.ResourceSendInfo) {}, + }) + if err != nil { + t.Fatalf("Failed to create xDS transport: %v", err) + } + defer tr.Close() + + // Send a discovery request through the transport. + tr.SendRequest(version.V3ListenerURL, []string{resourceName}) + + // Verify that the initial discovery request matches expectation. + var gotReq *v3discoverypb.DiscoveryRequest + select { + case gotReq = <-streamRequestCh: + case <-ctx.Done(): + t.Fatalf("Timeout waiting for discovery request on the stream") + } + wantReq := &v3discoverypb.DiscoveryRequest{ + Node: &v3corepb.Node{Id: nodeID}, + ResourceNames: []string{resourceName}, + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + } + if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform(), cmpopts.SortSlices(strSort)); diff != "" { + t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff) + } + + var gotResp *v3discoverypb.DiscoveryResponse + select { + case gotResp = <-streamResponseCh: + case <-ctx.Done(): + t.Fatalf("Timeout waiting for discovery response on the stream") + } + + // NACK should contain the appropriate error, nonce, but empty version. + wantReq.VersionInfo = "" + wantReq.ResponseNonce = gotResp.GetNonce() + wantReq.ErrorDetail = &statuspb.Status{ + Code: int32(codes.InvalidArgument), + Message: errWantNack.Error(), + } + select { + case gotReq = <-streamRequestCh: + case <-ctx.Done(): + t.Fatalf("Timeout waiting for the discovery request ACK on the stream") + } + if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform(), cmpopts.SortSlices(strSort)); diff != "" { + t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff) + } +} + +// TestResourceIsNotRequestedAnymore tests the scenario where the xDS client is +// no longer interested in a resource. The following sequence of events are +// tested: +// 1. A resource is requested and a good response is received. The test verifies +// that an ACK is sent for this resource. +// 2. The previously requested resource is no longer requested. The test +// verifies that a request with no resource names is sent out. +// 3. The same resource is requested again. The test verifies that the request +// is sent with the previously ACKed version. +func (s) TestResourceIsNotRequestedAnymore(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Create an xDS management server listening on a local port. Configure the + // request and response handlers to push on channels which are inspected by + // the test goroutine to verify ack version and nonce. + streamRequestCh := make(chan *v3discoverypb.DiscoveryRequest, 1) + streamResponseCh := make(chan *v3discoverypb.DiscoveryResponse, 1) + mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{ + OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + select { + case streamRequestCh <- req: + case <-ctx.Done(): + } + return nil + }, + OnStreamResponse: func(_ context.Context, _ int64, _ *v3discoverypb.DiscoveryRequest, resp *v3discoverypb.DiscoveryResponse) { + select { + case streamResponseCh <- resp: + case <-ctx.Done(): + } + }, + }) + if err != nil { + t.Fatalf("Failed to start xDS management server: %v", err) + } + defer mgmtServer.Stop() + t.Logf("Started xDS management server on %s", mgmtServer.Address) + + // Configure the management server with appropriate resources. + apiListener := &v3listenerpb.ApiListener{ + ApiListener: func() *anypb.Any { + return testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{ + Rds: &v3httppb.Rds{ + ConfigSource: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}, + }, + RouteConfigName: "route-configuration-name", + }, + }, + }) + }(), + } + const resourceName = "resource name 1" + listenerResource := &v3listenerpb.Listener{ + Name: resourceName, + ApiListener: apiListener, + } + nodeID := uuid.New().String() + mgmtServer.Update(ctx, e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listenerResource}, + SkipValidation: true, + }) + + // Create a new transport. + tr, err := transport.New(transport.Options{ + ServerCfg: *xdstestutils.ServerConfigForAddress(t, mgmtServer.Address), + NodeProto: &v3corepb.Node{Id: nodeID}, + OnRecvHandler: dataModelValidator, + OnErrorHandler: func(err error) {}, + OnSendHandler: func(*transport.ResourceSendInfo) {}, + }) + if err != nil { + t.Fatalf("Failed to create xDS transport: %v", err) + } + defer tr.Close() + + // Send a discovery request through the transport. + tr.SendRequest(version.V3ListenerURL, []string{resourceName}) + + // Verify that the initial discovery request matches expectation. + var gotReq *v3discoverypb.DiscoveryRequest + select { + case gotReq = <-streamRequestCh: + case <-ctx.Done(): + t.Fatalf("Timeout waiting for discovery request on the stream") + } + wantReq := &v3discoverypb.DiscoveryRequest{ + VersionInfo: "", + Node: &v3corepb.Node{Id: nodeID}, + ResourceNames: []string{resourceName}, + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + ResponseNonce: "", + } + if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform(), cmpopts.SortSlices(strSort)); diff != "" { + t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff) + } + + // Capture the version and nonce from the response. + var gotResp *v3discoverypb.DiscoveryResponse + select { + case gotResp = <-streamResponseCh: + case <-ctx.Done(): + t.Fatalf("Timeout waiting for discovery response on the stream") + } + + // Verify that the ACK contains the appropriate version and nonce. + wantReq.VersionInfo = gotResp.GetVersionInfo() + wantReq.ResponseNonce = gotResp.GetNonce() + select { + case gotReq = <-streamRequestCh: + case <-ctx.Done(): + t.Fatalf("Timeout waiting for the discovery request ACK on the stream") + } + if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform(), cmpopts.SortSlices(strSort)); diff != "" { + t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff) + } + + // Send a discovery request with no resource names. + tr.SendRequest(version.V3ListenerURL, []string{}) + + // Verify that the discovery request matches expectation. + select { + case gotReq = <-streamRequestCh: + case <-ctx.Done(): + t.Fatalf("Timeout waiting for discovery request on the stream") + } + wantReq.ResourceNames = nil + if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform(), cmpopts.SortSlices(strSort)); diff != "" { + t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff) + } + + // Send a discovery request for the same resource requested earlier. + tr.SendRequest(version.V3ListenerURL, []string{resourceName}) + + // Verify that the discovery request contains the version from the + // previously received response. + select { + case gotReq = <-streamRequestCh: + case <-ctx.Done(): + t.Fatalf("Timeout waiting for discovery request on the stream") + } + wantReq.ResourceNames = []string{resourceName} + if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform(), cmpopts.SortSlices(strSort)); diff != "" { + t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff) + } +} diff --git a/xds/internal/xdsclient/transport/transport_backoff_test.go b/xds/internal/xdsclient/transport/transport_backoff_test.go new file mode 100644 index 000000000000..db7587ca7c30 --- /dev/null +++ b/xds/internal/xdsclient/transport/transport_backoff_test.go @@ -0,0 +1,444 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package transport_test + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/google/uuid" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + xdstestutils "google.golang.org/grpc/xds/internal/testutils" + "google.golang.org/grpc/xds/internal/xdsclient/transport" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource/version" + "google.golang.org/protobuf/testing/protocmp" + "google.golang.org/protobuf/types/known/anypb" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" +) + +var strSort = func(s1, s2 string) bool { return s1 < s2 } + +// TestTransport_BackoffAfterStreamFailure tests the case where the management +// server returns an error in the ADS streaming RPC. The test verifies the +// following: +// 1. Initial discovery request matches expectation. +// 2. RPC error is propagated via the stream error handler. +// 3. When the stream is closed, the transport backs off. +// 4. The same discovery request is sent on the newly created stream. +func (s) TestTransport_BackoffAfterStreamFailure(t *testing.T) { + // Channels used for verifying different events in the test. + streamCloseCh := make(chan struct{}, 1) // ADS stream is closed. + streamRequestCh := make(chan *v3discoverypb.DiscoveryRequest, 1) // Discovery request is received. + backoffCh := make(chan struct{}, 1) // Transport backoff after stream failure. + streamErrCh := make(chan error, 1) // Stream error seen by the transport. + + // Create an xDS management server listening on a local port. + streamErr := errors.New("ADS stream error") + mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{ + // Push on a channel whenever the stream is closed. + OnStreamClosed: func(int64, *v3corepb.Node) { + select { + case streamCloseCh <- struct{}{}: + default: + } + }, + + // Return an error everytime a request is sent on the stream. This + // should cause the transport to backoff before attempting to recreate + // the stream. + OnStreamRequest: func(id int64, req *v3discoverypb.DiscoveryRequest) error { + select { + case streamRequestCh <- req: + default: + } + return streamErr + }, + }) + if err != nil { + t.Fatalf("Failed to start xDS management server: %v", err) + } + defer mgmtServer.Stop() + t.Logf("Started xDS management server on %s", mgmtServer.Address) + + // Override the backoff implementation to push on a channel that is read by + // the test goroutine. + transportBackoff := func(v int) time.Duration { + select { + case backoffCh <- struct{}{}: + default: + } + return 0 + } + + // Create a new transport. Since we are only testing backoff behavior here, + // we can pass a no-op data model layer implementation. + nodeID := uuid.New().String() + tr, err := transport.New(transport.Options{ + ServerCfg: *xdstestutils.ServerConfigForAddress(t, mgmtServer.Address), + OnRecvHandler: func(transport.ResourceUpdate) error { return nil }, // No data model layer validation. + OnErrorHandler: func(err error) { + select { + case streamErrCh <- err: + default: + } + }, + OnSendHandler: func(*transport.ResourceSendInfo) {}, + Backoff: transportBackoff, + NodeProto: &v3corepb.Node{Id: nodeID}, + }) + if err != nil { + t.Fatalf("Failed to create xDS transport: %v", err) + } + defer tr.Close() + + // Send a discovery request through the transport. + const resourceName = "resource name" + tr.SendRequest(version.V3ListenerURL, []string{resourceName}) + + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + + // Verify that the initial discovery request matches expectation. + var gotReq *v3discoverypb.DiscoveryRequest + select { + case gotReq = <-streamRequestCh: + case <-ctx.Done(): + t.Fatalf("Timeout waiting for discovery request on the stream") + } + wantReq := &v3discoverypb.DiscoveryRequest{ + VersionInfo: "", + Node: &v3corepb.Node{Id: nodeID}, + ResourceNames: []string{resourceName}, + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + ResponseNonce: "", + } + if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" { + t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff) + } + + // Verify that the received stream error is reported to the user. + var gotErr error + select { + case gotErr = <-streamErrCh: + case <-ctx.Done(): + t.Fatalf("Timeout waiting for stream error to be reported to the user") + } + if !strings.Contains(gotErr.Error(), streamErr.Error()) { + t.Fatalf("Received stream error: %v, wantErr: %v", gotErr, streamErr) + } + + // Verify that the stream is closed. + select { + case <-streamCloseCh: + case <-ctx.Done(): + t.Fatalf("Timeout waiting for stream to be closed after an error") + } + + // Verify that the transport backs off before recreating the stream. + select { + case <-backoffCh: + case <-ctx.Done(): + t.Fatalf("Timeout waiting for transport to backoff after stream failure") + } + + // Verify that the same discovery request is resent on the new stream. + select { + case gotReq = <-streamRequestCh: + case <-ctx.Done(): + t.Fatalf("Timeout waiting for discovery request on the stream") + } + if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" { + t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff) + } +} + +// TestTransport_RetriesAfterBrokenStream tests the case where a stream breaks +// because the server goes down. The test verifies the following: +// 1. Initial discovery request matches expectation. +// 2. Good response from the server leads to an ACK with appropriate version. +// 3. Management server going down, leads to stream failure. +// 4. Once the management server comes back up, the same resources are +// re-requested, this time with an empty nonce. +func (s) TestTransport_RetriesAfterBrokenStream(t *testing.T) { + // Channels used for verifying different events in the test. + streamRequestCh := make(chan *v3discoverypb.DiscoveryRequest, 1) // Discovery request is received. + streamResponseCh := make(chan *v3discoverypb.DiscoveryResponse, 1) // Discovery response is received. + streamErrCh := make(chan error, 1) // Stream error seen by the transport. + + // Create an xDS management server listening on a local port. + l, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("Failed to create a local listener for the xDS management server: %v", err) + } + lis := testutils.NewRestartableListener(l) + mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{ + Listener: lis, + // Push the received request on to a channel for the test goroutine to + // verify that it matches expectations. + OnStreamRequest: func(_ int64, req *v3discoverypb.DiscoveryRequest) error { + select { + case streamRequestCh <- req: + default: + } + return nil + }, + // Push the response that the management server is about to send on to a + // channel. The test goroutine to uses this to extract the version and + // nonce, expected on subsequent requests. + OnStreamResponse: func(_ context.Context, _ int64, _ *v3discoverypb.DiscoveryRequest, resp *v3discoverypb.DiscoveryResponse) { + select { + case streamResponseCh <- resp: + default: + } + }, + }) + if err != nil { + t.Fatalf("Failed to start xDS management server: %v", err) + } + defer mgmtServer.Stop() + t.Logf("Started xDS management server on %s", lis.Addr().String()) + + // Configure the management server with appropriate resources. + apiListener := &v3listenerpb.ApiListener{ + ApiListener: func() *anypb.Any { + return testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{ + Rds: &v3httppb.Rds{ + ConfigSource: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}, + }, + RouteConfigName: "route-configuration-name", + }, + }, + }) + }(), + } + const resourceName1 = "resource name 1" + const resourceName2 = "resource name 2" + listenerResource1 := &v3listenerpb.Listener{ + Name: resourceName1, + ApiListener: apiListener, + } + listenerResource2 := &v3listenerpb.Listener{ + Name: resourceName2, + ApiListener: apiListener, + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + nodeID := uuid.New().String() + mgmtServer.Update(ctx, e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listenerResource1, listenerResource2}, + SkipValidation: true, + }) + + // Create a new transport. Since we are only testing backoff behavior here, + // we can pass a no-op data model layer implementation. + tr, err := transport.New(transport.Options{ + ServerCfg: *xdstestutils.ServerConfigForAddress(t, mgmtServer.Address), + OnRecvHandler: func(transport.ResourceUpdate) error { return nil }, // No data model layer validation. + OnErrorHandler: func(err error) { + select { + case streamErrCh <- err: + default: + } + }, + OnSendHandler: func(*transport.ResourceSendInfo) {}, + Backoff: func(int) time.Duration { return time.Duration(0) }, // No backoff. + NodeProto: &v3corepb.Node{Id: nodeID}, + }) + if err != nil { + t.Fatalf("Failed to create xDS transport: %v", err) + } + defer tr.Close() + + // Send a discovery request through the transport. + tr.SendRequest(version.V3ListenerURL, []string{resourceName1, resourceName2}) + + // Verify that the initial discovery request matches expectation. + var gotReq *v3discoverypb.DiscoveryRequest + select { + case gotReq = <-streamRequestCh: + case <-ctx.Done(): + t.Fatalf("Timeout waiting for discovery request on the stream") + } + wantReq := &v3discoverypb.DiscoveryRequest{ + VersionInfo: "", + Node: &v3corepb.Node{Id: nodeID}, + ResourceNames: []string{resourceName1, resourceName2}, + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + ResponseNonce: "", + } + if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform(), cmpopts.SortSlices(strSort)); diff != "" { + t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff) + } + + // Capture the version and nonce from the response. + var gotResp *v3discoverypb.DiscoveryResponse + select { + case gotResp = <-streamResponseCh: + case <-ctx.Done(): + t.Fatalf("Timeout waiting for discovery response on the stream") + } + version := gotResp.GetVersionInfo() + nonce := gotResp.GetNonce() + + // Verify that the ACK contains the appropriate version and nonce. + wantReq.VersionInfo = version + wantReq.ResponseNonce = nonce + select { + case gotReq = <-streamRequestCh: + case <-ctx.Done(): + t.Fatalf("Timeout waiting for the discovery request ACK on the stream") + } + if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform(), cmpopts.SortSlices(strSort)); diff != "" { + t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff) + } + + // Bring down the management server to simulate a broken stream. + lis.Stop() + + // We don't care about the exact error here and it can vary based on which + // error gets reported first, the Recv() failure or the new stream creation + // failure. So, all we check here is whether we get an error or not. + select { + case <-streamErrCh: + case <-ctx.Done(): + t.Fatalf("Timeout waiting for stream error to be reported to the user") + } + + // Bring up the connection to the management server. + lis.Restart() + + // Verify that the transport creates a new stream and sends out a new + // request which contains the previously acked version, but an empty nonce. + wantReq.ResponseNonce = "" + select { + case gotReq = <-streamRequestCh: + case <-ctx.Done(): + t.Fatalf("Timeout waiting for the discovery request ACK on the stream") + } + if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform(), cmpopts.SortSlices(strSort)); diff != "" { + t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff) + } +} + +// TestTransport_ResourceRequestedBeforeStreamCreation tests the case where a +// resource is requested before the transport has a valid stream. Verifies that +// the transport sends out the request once it has a valid stream. +func (s) TestTransport_ResourceRequestedBeforeStreamCreation(t *testing.T) { + // Channels used for verifying different events in the test. + streamRequestCh := make(chan *v3discoverypb.DiscoveryRequest, 1) // Discovery request is received. + + // Create an xDS management server listening on a local port. + l, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("Failed to create a local listener for the xDS management server: %v", err) + } + lis := testutils.NewRestartableListener(l) + streamErr := errors.New("ADS stream error") + + mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{ + Listener: lis, + + // Return an error everytime a request is sent on the stream. This + // should cause the transport to backoff before attempting to recreate + // the stream. + OnStreamRequest: func(id int64, req *v3discoverypb.DiscoveryRequest) error { + select { + case streamRequestCh <- req: + default: + } + return streamErr + }, + }) + if err != nil { + t.Fatalf("Failed to start xDS management server: %v", err) + } + defer mgmtServer.Stop() + t.Logf("Started xDS management server on %s", lis.Addr().String()) + + // Bring down the management server before creating the transport. This + // allows us to test the case where SendRequest() is called when there is no + // stream to the management server. + lis.Stop() + + // Create a new transport. Since we are only testing backoff behavior here, + // we can pass a no-op data model layer implementation. + nodeID := uuid.New().String() + tr, err := transport.New(transport.Options{ + ServerCfg: *xdstestutils.ServerConfigForAddress(t, mgmtServer.Address), + OnRecvHandler: func(transport.ResourceUpdate) error { return nil }, // No data model layer validation. + OnErrorHandler: func(error) {}, // No stream error handling. + OnSendHandler: func(*transport.ResourceSendInfo) {}, // No on send handler + Backoff: func(int) time.Duration { return time.Duration(0) }, // No backoff. + NodeProto: &v3corepb.Node{Id: nodeID}, + }) + if err != nil { + t.Fatalf("Failed to create xDS transport: %v", err) + } + defer tr.Close() + + // Send a discovery request through the transport. + const resourceName = "resource name" + tr.SendRequest(version.V3ListenerURL, []string{resourceName}) + + // Wait until the transport has attempted to connect to the management + // server and has seen the connection fail. In this case, since the + // connection is down, and the transport creates streams with WaitForReady() + // set to true, stream creation will never fail (unless the context + // expires), and therefore we cannot rely on the stream error handler. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + for ; ctx.Err() == nil; <-time.After(defaultTestShortTimeout) { + if tr.ChannelConnectivityStateForTesting() == connectivity.TransientFailure { + break + } + } + + lis.Restart() + + // Verify that the initial discovery request matches expectation. + var gotReq *v3discoverypb.DiscoveryRequest + select { + case gotReq = <-streamRequestCh: + case <-ctx.Done(): + t.Fatalf("Timeout waiting for discovery request on the stream") + } + wantReq := &v3discoverypb.DiscoveryRequest{ + VersionInfo: "", + Node: &v3corepb.Node{Id: nodeID}, + ResourceNames: []string{resourceName}, + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + ResponseNonce: "", + } + if diff := cmp.Diff(gotReq, wantReq, protocmp.Transform()); diff != "" { + t.Fatalf("Unexpected diff in received discovery request, diff (-got, +want):\n%s", diff) + } +} diff --git a/xds/internal/xdsclient/transport/transport_new_test.go b/xds/internal/xdsclient/transport/transport_new_test.go new file mode 100644 index 000000000000..2f6c3148a267 --- /dev/null +++ b/xds/internal/xdsclient/transport/transport_new_test.go @@ -0,0 +1,106 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package transport_test + +import ( + "strings" + "testing" + + "google.golang.org/grpc/xds/internal/testutils" + "google.golang.org/grpc/xds/internal/xdsclient/bootstrap" + "google.golang.org/grpc/xds/internal/xdsclient/transport" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" +) + +// TestNew covers that New() returns an error if the input *ServerConfig +// contains invalid content. +func (s) TestNew(t *testing.T) { + tests := []struct { + name string + opts transport.Options + wantErrStr string + }{ + { + name: "missing server URI", + opts: transport.Options{ServerCfg: bootstrap.ServerConfig{}}, + wantErrStr: "missing server URI when creating a new transport", + }, + { + name: "missing credentials", + opts: transport.Options{ServerCfg: bootstrap.ServerConfig{ServerURI: "server-address"}}, + wantErrStr: "missing credentials when creating a new transport", + }, + { + name: "missing onRecv handler", + opts: transport.Options{ + ServerCfg: *testutils.ServerConfigForAddress(t, "server-address"), + NodeProto: &v3corepb.Node{}, + }, + wantErrStr: "missing OnRecv callback handler when creating a new transport", + }, + { + name: "missing onError handler", + opts: transport.Options{ + ServerCfg: *testutils.ServerConfigForAddress(t, "server-address"), + NodeProto: &v3corepb.Node{}, + OnRecvHandler: func(transport.ResourceUpdate) error { return nil }, + OnSendHandler: func(*transport.ResourceSendInfo) {}, + }, + wantErrStr: "missing OnError callback handler when creating a new transport", + }, + + { + name: "missing onSend handler", + opts: transport.Options{ + ServerCfg: *testutils.ServerConfigForAddress(t, "server-address"), + NodeProto: &v3corepb.Node{}, + OnRecvHandler: func(transport.ResourceUpdate) error { return nil }, + OnErrorHandler: func(error) {}, + }, + wantErrStr: "missing OnSend callback handler when creating a new transport", + }, + { + name: "happy case", + opts: transport.Options{ + ServerCfg: *testutils.ServerConfigForAddress(t, "server-address"), + NodeProto: &v3corepb.Node{}, + OnRecvHandler: func(transport.ResourceUpdate) error { return nil }, + OnErrorHandler: func(error) {}, + OnSendHandler: func(*transport.ResourceSendInfo) {}, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + c, err := transport.New(test.opts) + defer func() { + if c != nil { + c.Close() + } + }() + if (err != nil) != (test.wantErrStr != "") { + t.Fatalf("New(%+v) = %v, wantErr: %v", test.opts, err, test.wantErrStr) + } + if err != nil && !strings.Contains(err.Error(), test.wantErrStr) { + t.Fatalf("New(%+v) = %v, wantErr: %v", test.opts, err, test.wantErrStr) + } + }) + } +} diff --git a/xds/internal/xdsclient/transport/transport_resource_test.go b/xds/internal/xdsclient/transport/transport_resource_test.go new file mode 100644 index 000000000000..0824af77f4ff --- /dev/null +++ b/xds/internal/xdsclient/transport/transport_resource_test.go @@ -0,0 +1,219 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package transport_test contains e2e style tests for the xDS transport +// implementation. It uses the envoy-go-control-plane as the management server. +package transport_test + +import ( + "context" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/uuid" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/fakeserver" + xdstestutils "google.golang.org/grpc/xds/internal/testutils" + "google.golang.org/grpc/xds/internal/xdsclient/transport" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource/version" + "google.golang.org/protobuf/testing/protocmp" + "google.golang.org/protobuf/types/known/anypb" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +const ( + defaultTestTimeout = 5 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond +) + +// startFakeManagementServer starts a fake xDS management server and returns a +// cleanup function to close the fake server. +func startFakeManagementServer(t *testing.T) (*fakeserver.Server, func()) { + t.Helper() + fs, sCleanup, err := fakeserver.StartServer(nil) + if err != nil { + t.Fatalf("Failed to start fake xDS server: %v", err) + } + return fs, sCleanup +} + +// resourcesWithTypeURL wraps resources and type URL received from server. +type resourcesWithTypeURL struct { + resources []*anypb.Any + url string +} + +// TestHandleResponseFromManagementServer covers different scenarios of the +// transport receiving a response from the management server. In all scenarios, +// the trasport is expected to pass the received responses as-is to the data +// model layer for validation and not perform any validation on its own. +func (s) TestHandleResponseFromManagementServer(t *testing.T) { + const ( + resourceName1 = "resource-name-1" + resourceName2 = "resource-name-2" + ) + var ( + badlyMarshaledResource = &anypb.Any{ + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + Value: []byte{1, 2, 3, 4}, + } + apiListener = &v3listenerpb.ApiListener{ + ApiListener: func() *anypb.Any { + return testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{ + Rds: &v3httppb.Rds{ + ConfigSource: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}, + }, + RouteConfigName: "route-configuration-name", + }, + }, + }) + }(), + } + resource1 = &v3listenerpb.Listener{ + Name: resourceName1, + ApiListener: apiListener, + } + resource2 = &v3listenerpb.Listener{ + Name: resourceName2, + ApiListener: apiListener, + } + ) + + tests := []struct { + desc string + resourceNamesToRequest []string + managementServerResponse *v3discoverypb.DiscoveryResponse + wantURL string + wantResources []*anypb.Any + }{ + { + desc: "badly marshaled response", + resourceNamesToRequest: []string{resourceName1}, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + Resources: []*anypb.Any{badlyMarshaledResource}, + }, + wantURL: "type.googleapis.com/envoy.config.listener.v3.Listener", + wantResources: []*anypb.Any{badlyMarshaledResource}, + }, + { + desc: "empty response", + resourceNamesToRequest: []string{resourceName1}, + managementServerResponse: &v3discoverypb.DiscoveryResponse{}, + wantURL: "", + wantResources: nil, + }, + { + desc: "one good resource", + resourceNamesToRequest: []string{resourceName1}, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + Resources: []*anypb.Any{testutils.MarshalAny(resource1)}, + }, + wantURL: "type.googleapis.com/envoy.config.listener.v3.Listener", + wantResources: []*anypb.Any{testutils.MarshalAny(resource1)}, + }, + { + desc: "two good resources", + resourceNamesToRequest: []string{resourceName1, resourceName2}, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + Resources: []*anypb.Any{testutils.MarshalAny(resource1), testutils.MarshalAny(resource2)}, + }, + wantURL: "type.googleapis.com/envoy.config.listener.v3.Listener", + wantResources: []*anypb.Any{testutils.MarshalAny(resource1), testutils.MarshalAny(resource2)}, + }, + { + desc: "two resources when we requested one", + resourceNamesToRequest: []string{resourceName1}, + managementServerResponse: &v3discoverypb.DiscoveryResponse{ + TypeUrl: "type.googleapis.com/envoy.config.listener.v3.Listener", + Resources: []*anypb.Any{testutils.MarshalAny(resource1), testutils.MarshalAny(resource2)}, + }, + wantURL: "type.googleapis.com/envoy.config.listener.v3.Listener", + wantResources: []*anypb.Any{testutils.MarshalAny(resource1), testutils.MarshalAny(resource2)}, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + // Create a fake xDS management server listening on a local port, + // and set it up with the response to send. + mgmtServer, cleanup := startFakeManagementServer(t) + defer cleanup() + t.Logf("Started xDS management server on %s", mgmtServer.Address) + mgmtServer.XDSResponseChan <- &fakeserver.Response{Resp: test.managementServerResponse} + + // Create a new transport. + resourcesCh := testutils.NewChannel() + tr, err := transport.New(transport.Options{ + ServerCfg: *xdstestutils.ServerConfigForAddress(t, mgmtServer.Address), + // No validation. Simply push received resources on a channel. + OnRecvHandler: func(update transport.ResourceUpdate) error { + resourcesCh.Send(&resourcesWithTypeURL{ + resources: update.Resources, + url: update.URL, + // Ignore resource version here. + }) + return nil + }, + OnSendHandler: func(*transport.ResourceSendInfo) {}, // No onSend handling. + OnErrorHandler: func(error) {}, // No stream error handling. + Backoff: func(int) time.Duration { return time.Duration(0) }, // No backoff. + NodeProto: &v3corepb.Node{Id: uuid.New().String()}, + }) + if err != nil { + t.Fatalf("Failed to create xDS transport: %v", err) + } + defer tr.Close() + + // Send the request, and validate that the response sent by the + // management server is propagated to the data model layer. + tr.SendRequest(version.V3ListenerURL, test.resourceNamesToRequest) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + v, err := resourcesCh.Receive(ctx) + if err != nil { + t.Fatalf("Failed to receive resources at the data model layer: %v", err) + } + gotURL := v.(*resourcesWithTypeURL).url + gotResources := v.(*resourcesWithTypeURL).resources + if gotURL != test.wantURL { + t.Fatalf("Received resource URL in response: %s, want %s", gotURL, test.wantURL) + } + if diff := cmp.Diff(gotResources, test.wantResources, protocmp.Transform()); diff != "" { + t.Fatalf("Received unexpected resources. Diff (-got, +want):\n%s", diff) + } + }) + } +} diff --git a/xds/internal/xdsclient/transport/transport_test.go b/xds/internal/xdsclient/transport/transport_test.go new file mode 100644 index 000000000000..50dc74dd0ba2 --- /dev/null +++ b/xds/internal/xdsclient/transport/transport_test.go @@ -0,0 +1,83 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package transport + +import ( + "testing" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + "google.golang.org/grpc" + "google.golang.org/grpc/internal/grpctest" + xdstestutils "google.golang.org/grpc/xds/internal/testutils" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func (s) TestNewWithGRPCDial(t *testing.T) { + // Override the dialer with a custom one. + customDialerCalled := false + customDialer := func(target string, opts ...grpc.DialOption) (*grpc.ClientConn, error) { + customDialerCalled = true + return grpc.Dial(target, opts...) + } + oldDial := grpcDial + grpcDial = customDialer + defer func() { grpcDial = oldDial }() + + // Create a new transport and ensure that the custom dialer was called. + opts := Options{ + ServerCfg: *xdstestutils.ServerConfigForAddress(t, "server-address"), + NodeProto: &v3corepb.Node{}, + OnRecvHandler: func(ResourceUpdate) error { return nil }, + OnErrorHandler: func(error) {}, + OnSendHandler: func(*ResourceSendInfo) {}, + } + c, err := New(opts) + if err != nil { + t.Fatalf("New(%v) failed: %v", opts, err) + } + defer c.Close() + + if !customDialerCalled { + t.Fatalf("New(%+v) custom dialer called = false, want true", opts) + } + customDialerCalled = false + + // Reset the dialer, create a new transport and ensure that our custom + // dialer is no longer called. + grpcDial = grpc.Dial + c, err = New(opts) + defer func() { + if c != nil { + c.Close() + } + }() + if err != nil { + t.Fatalf("New(%v) failed: %v", opts, err) + } + + if customDialerCalled { + t.Fatalf("New(%+v) custom dialer called = true, want false", opts) + } +} diff --git a/xds/internal/balancer/balancergroup/testutils_test.go b/xds/internal/xdsclient/xdsclient_test.go similarity index 97% rename from xds/internal/balancer/balancergroup/testutils_test.go rename to xds/internal/xdsclient/xdsclient_test.go index 1429fa87b3f2..d7bb926659f3 100644 --- a/xds/internal/balancer/balancergroup/testutils_test.go +++ b/xds/internal/xdsclient/xdsclient_test.go @@ -16,7 +16,7 @@ * */ -package balancergroup +package xdsclient_test import ( "testing" diff --git a/xds/internal/xdsclient/xdslbregistry/converter/converter.go b/xds/internal/xdsclient/xdslbregistry/converter/converter.go new file mode 100644 index 000000000000..5bf70751ed80 --- /dev/null +++ b/xds/internal/xdsclient/xdslbregistry/converter/converter.go @@ -0,0 +1,261 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package converter provides converters to convert proto load balancing +// configuration, defined by the xDS API spec, to JSON load balancing +// configuration. These converters are registered by proto type in a registry, +// which gets pulled from based off proto type passed in. +package converter + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/golang/protobuf/proto" + "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/leastrequest" + "google.golang.org/grpc/balancer/roundrobin" + "google.golang.org/grpc/balancer/weightedroundrobin" + "google.golang.org/grpc/internal/envconfig" + internalserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/xds/internal/balancer/ringhash" + "google.golang.org/grpc/xds/internal/balancer/wrrlocality" + "google.golang.org/grpc/xds/internal/xdsclient/xdslbregistry" + + v1xdsudpatypepb "github.com/cncf/xds/go/udpa/type/v1" + v3xdsxdstypepb "github.com/cncf/xds/go/xds/type/v3" + v3clientsideweightedroundrobinpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3" + v3leastrequestpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/least_request/v3" + v3pickfirstpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/pick_first/v3" + v3ringhashpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/ring_hash/v3" + v3wrrlocalitypb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/wrr_locality/v3" + structpb "github.com/golang/protobuf/ptypes/struct" +) + +func init() { + xdslbregistry.Register("type.googleapis.com/envoy.extensions.load_balancing_policies.client_side_weighted_round_robin.v3.ClientSideWeightedRoundRobin", convertWeightedRoundRobinProtoToServiceConfig) + xdslbregistry.Register("type.googleapis.com/envoy.extensions.load_balancing_policies.ring_hash.v3.RingHash", convertRingHashProtoToServiceConfig) + xdslbregistry.Register("type.googleapis.com/envoy.extensions.load_balancing_policies.pick_first.v3.PickFirst", convertPickFirstProtoToServiceConfig) + xdslbregistry.Register("type.googleapis.com/envoy.extensions.load_balancing_policies.round_robin.v3.RoundRobin", convertRoundRobinProtoToServiceConfig) + xdslbregistry.Register("type.googleapis.com/envoy.extensions.load_balancing_policies.wrr_locality.v3.WrrLocality", convertWRRLocalityProtoToServiceConfig) + xdslbregistry.Register("type.googleapis.com/envoy.extensions.load_balancing_policies.least_request.v3.LeastRequest", convertLeastRequestProtoToServiceConfig) + xdslbregistry.Register("type.googleapis.com/udpa.type.v1.TypedStruct", convertV1TypedStructToServiceConfig) + xdslbregistry.Register("type.googleapis.com/xds.type.v3.TypedStruct", convertV3TypedStructToServiceConfig) +} + +const ( + defaultRingHashMinSize = 1024 + defaultRingHashMaxSize = 8 * 1024 * 1024 // 8M + defaultLeastRequestChoiceCount = 2 +) + +func convertRingHashProtoToServiceConfig(rawProto []byte, _ int) (json.RawMessage, error) { + if !envconfig.XDSRingHash { + return nil, nil + } + rhProto := &v3ringhashpb.RingHash{} + if err := proto.Unmarshal(rawProto, rhProto); err != nil { + return nil, fmt.Errorf("failed to unmarshal resource: %v", err) + } + if rhProto.GetHashFunction() != v3ringhashpb.RingHash_XX_HASH { + return nil, fmt.Errorf("unsupported ring_hash hash function %v", rhProto.GetHashFunction()) + } + + var minSize, maxSize uint64 = defaultRingHashMinSize, defaultRingHashMaxSize + if min := rhProto.GetMinimumRingSize(); min != nil { + minSize = min.GetValue() + } + if max := rhProto.GetMaximumRingSize(); max != nil { + maxSize = max.GetValue() + } + + rhCfg := &ringhash.LBConfig{ + MinRingSize: minSize, + MaxRingSize: maxSize, + } + + rhCfgJSON, err := json.Marshal(rhCfg) + if err != nil { + return nil, fmt.Errorf("error marshaling JSON for type %T: %v", rhCfg, err) + } + return makeBalancerConfigJSON(ringhash.Name, rhCfgJSON), nil +} + +type pfConfig struct { + ShuffleAddressList bool `json:"shuffleAddressList"` +} + +func convertPickFirstProtoToServiceConfig(rawProto []byte, _ int) (json.RawMessage, error) { + if !envconfig.PickFirstLBConfig { + return nil, nil + } + pfProto := &v3pickfirstpb.PickFirst{} + if err := proto.Unmarshal(rawProto, pfProto); err != nil { + return nil, fmt.Errorf("failed to unmarshal resource: %v", err) + } + + pfCfg := &pfConfig{ShuffleAddressList: pfProto.GetShuffleAddressList()} + js, err := json.Marshal(pfCfg) + if err != nil { + return nil, fmt.Errorf("error marshaling JSON for type %T: %v", pfCfg, err) + } + return makeBalancerConfigJSON(grpc.PickFirstBalancerName, js), nil +} + +func convertRoundRobinProtoToServiceConfig([]byte, int) (json.RawMessage, error) { + return makeBalancerConfigJSON(roundrobin.Name, json.RawMessage("{}")), nil +} + +type wrrLocalityLBConfig struct { + ChildPolicy json.RawMessage `json:"childPolicy,omitempty"` +} + +func convertWRRLocalityProtoToServiceConfig(rawProto []byte, depth int) (json.RawMessage, error) { + wrrlProto := &v3wrrlocalitypb.WrrLocality{} + if err := proto.Unmarshal(rawProto, wrrlProto); err != nil { + return nil, fmt.Errorf("failed to unmarshal resource: %v", err) + } + epJSON, err := xdslbregistry.ConvertToServiceConfig(wrrlProto.GetEndpointPickingPolicy(), depth+1) + if err != nil { + return nil, fmt.Errorf("error converting endpoint picking policy: %v for %+v", err, wrrlProto) + } + wrrLCfg := wrrLocalityLBConfig{ + ChildPolicy: epJSON, + } + + lbCfgJSON, err := json.Marshal(wrrLCfg) + if err != nil { + return nil, fmt.Errorf("error marshaling JSON for type %T: %v", wrrLCfg, err) + } + return makeBalancerConfigJSON(wrrlocality.Name, lbCfgJSON), nil +} + +func convertWeightedRoundRobinProtoToServiceConfig(rawProto []byte, _ int) (json.RawMessage, error) { + cswrrProto := &v3clientsideweightedroundrobinpb.ClientSideWeightedRoundRobin{} + if err := proto.Unmarshal(rawProto, cswrrProto); err != nil { + return nil, fmt.Errorf("failed to unmarshal resource: %v", err) + } + wrrLBCfg := &wrrLBConfig{} + // Only set fields if specified in proto. If not set, ParseConfig of the WRR + // will populate the config with defaults. + if enableOOBLoadReportCfg := cswrrProto.GetEnableOobLoadReport(); enableOOBLoadReportCfg != nil { + wrrLBCfg.EnableOOBLoadReport = enableOOBLoadReportCfg.GetValue() + } + if oobReportingPeriodCfg := cswrrProto.GetOobReportingPeriod(); oobReportingPeriodCfg != nil { + wrrLBCfg.OOBReportingPeriod = internalserviceconfig.Duration(oobReportingPeriodCfg.AsDuration()) + } + if blackoutPeriodCfg := cswrrProto.GetBlackoutPeriod(); blackoutPeriodCfg != nil { + wrrLBCfg.BlackoutPeriod = internalserviceconfig.Duration(blackoutPeriodCfg.AsDuration()) + } + if weightExpirationPeriodCfg := cswrrProto.GetBlackoutPeriod(); weightExpirationPeriodCfg != nil { + wrrLBCfg.WeightExpirationPeriod = internalserviceconfig.Duration(weightExpirationPeriodCfg.AsDuration()) + } + if weightUpdatePeriodCfg := cswrrProto.GetWeightUpdatePeriod(); weightUpdatePeriodCfg != nil { + wrrLBCfg.WeightUpdatePeriod = internalserviceconfig.Duration(weightUpdatePeriodCfg.AsDuration()) + } + if errorUtilizationPenaltyCfg := cswrrProto.GetErrorUtilizationPenalty(); errorUtilizationPenaltyCfg != nil { + wrrLBCfg.ErrorUtilizationPenalty = float64(errorUtilizationPenaltyCfg.GetValue()) + } + + lbCfgJSON, err := json.Marshal(wrrLBCfg) + if err != nil { + return nil, fmt.Errorf("error marshaling JSON for type %T: %v", wrrLBCfg, err) + } + return makeBalancerConfigJSON(weightedroundrobin.Name, lbCfgJSON), nil +} + +func convertLeastRequestProtoToServiceConfig(rawProto []byte, _ int) (json.RawMessage, error) { + if !envconfig.LeastRequestLB { + return nil, nil + } + lrProto := &v3leastrequestpb.LeastRequest{} + if err := proto.Unmarshal(rawProto, lrProto); err != nil { + return nil, fmt.Errorf("failed to unmarshal resource: %v", err) + } + // "The configuration for the Least Request LB policy is the + // least_request_lb_config field. The field is optional; if not present, + // defaults will be assumed for all of its values." - A48 + choiceCount := uint32(defaultLeastRequestChoiceCount) + if cc := lrProto.GetChoiceCount(); cc != nil { + choiceCount = cc.GetValue() + } + lrCfg := &leastrequest.LBConfig{ChoiceCount: choiceCount} + js, err := json.Marshal(lrCfg) + if err != nil { + return nil, fmt.Errorf("error marshaling JSON for type %T: %v", lrCfg, err) + } + return makeBalancerConfigJSON(leastrequest.Name, js), nil +} + +func convertV1TypedStructToServiceConfig(rawProto []byte, _ int) (json.RawMessage, error) { + tsProto := &v1xdsudpatypepb.TypedStruct{} + if err := proto.Unmarshal(rawProto, tsProto); err != nil { + return nil, fmt.Errorf("failed to unmarshal resource: %v", err) + } + return convertCustomPolicy(tsProto.GetTypeUrl(), tsProto.GetValue()) +} + +func convertV3TypedStructToServiceConfig(rawProto []byte, _ int) (json.RawMessage, error) { + tsProto := &v3xdsxdstypepb.TypedStruct{} + if err := proto.Unmarshal(rawProto, tsProto); err != nil { + return nil, fmt.Errorf("failed to unmarshal resource: %v", err) + } + return convertCustomPolicy(tsProto.GetTypeUrl(), tsProto.GetValue()) +} + +// convertCustomPolicy attempts to prepare json configuration for a custom lb +// proto, which specifies the gRPC balancer type and configuration. Returns the +// converted json and an error which should cause caller to error if error +// converting. If both json and error returned are nil, it means the gRPC +// Balancer registry does not contain that balancer type, and the caller should +// continue to the next policy. +func convertCustomPolicy(typeURL string, s *structpb.Struct) (json.RawMessage, error) { + // The gRPC policy name will be the "type name" part of the value of the + // type_url field in the TypedStruct. We get this by using the part after + // the last / character. Can assume a valid type_url from the control plane. + pos := strings.LastIndex(typeURL, "/") + name := typeURL[pos+1:] + + if balancer.Get(name) == nil { + return nil, nil + } + + rawJSON, err := json.Marshal(s) + if err != nil { + return nil, fmt.Errorf("error converting custom lb policy %v: %v for %+v", err, typeURL, s) + } + + // The Struct contained in the TypedStruct will be returned as-is as the + // configuration JSON object. + return makeBalancerConfigJSON(name, rawJSON), nil +} + +type wrrLBConfig struct { + EnableOOBLoadReport bool `json:"enableOobLoadReport,omitempty"` + OOBReportingPeriod internalserviceconfig.Duration `json:"oobReportingPeriod,omitempty"` + BlackoutPeriod internalserviceconfig.Duration `json:"blackoutPeriod,omitempty"` + WeightExpirationPeriod internalserviceconfig.Duration `json:"weightExpirationPeriod,omitempty"` + WeightUpdatePeriod internalserviceconfig.Duration `json:"weightUpdatePeriod,omitempty"` + ErrorUtilizationPenalty float64 `json:"errorUtilizationPenalty,omitempty"` +} + +func makeBalancerConfigJSON(name string, value json.RawMessage) []byte { + return []byte(fmt.Sprintf(`[{%q: %s}]`, name, value)) +} diff --git a/xds/internal/xdsclient/xdslbregistry/xdslbregistry.go b/xds/internal/xdsclient/xdslbregistry/xdslbregistry.go new file mode 100644 index 000000000000..0f3d1df4db20 --- /dev/null +++ b/xds/internal/xdsclient/xdslbregistry/xdslbregistry.go @@ -0,0 +1,85 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package xdslbregistry provides a registry of converters that convert proto +// from load balancing configuration, defined by the xDS API spec, to JSON load +// balancing configuration. +package xdslbregistry + +import ( + "encoding/json" + "fmt" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" +) + +var ( + // m is a map from proto type to Converter. + m = make(map[string]Converter) +) + +// Register registers the converter to the map keyed on a proto type. Must be +// called at init time. Not thread safe. +func Register(protoType string, c Converter) { + m[protoType] = c +} + +// SetRegistry sets the xDS LB registry. Must be called at init time. Not thread +// safe. +func SetRegistry(registry map[string]Converter) { + m = registry +} + +// Converter converts raw proto bytes into the internal Go JSON representation +// of the proto passed. Returns the json message, and an error. If both +// returned are nil, it represents continuing to the next proto. +type Converter func([]byte, int) (json.RawMessage, error) + +// ConvertToServiceConfig converts a proto Load Balancing Policy configuration +// into a json string. Returns an error if: +// - no supported policy found +// - there is more than 16 layers of recursion in the configuration +// - a failure occurs when converting the policy +func ConvertToServiceConfig(lbPolicy *v3clusterpb.LoadBalancingPolicy, depth int) (json.RawMessage, error) { + // "Configurations that require more than 16 levels of recursion are + // considered invalid and should result in a NACK response." - A51 + if depth > 15 { + return nil, fmt.Errorf("lb policy %v exceeds max depth supported: 16 layers", lbPolicy) + } + + // "This function iterate over the list of policy messages in + // LoadBalancingPolicy, attempting to convert each one to gRPC form, + // stopping at the first supported policy." - A52 + for _, policy := range lbPolicy.GetPolicies() { + policy.GetTypedExtensionConfig().GetTypedConfig().GetTypeUrl() + converter := m[policy.GetTypedExtensionConfig().GetTypedConfig().GetTypeUrl()] + // "Any entry not in the above list is unsupported and will be skipped." + // - A52 + // This includes Least Request as well, since grpc-go does not support + // the Least Request Load Balancing Policy. + if converter == nil { + continue + } + json, err := converter(policy.GetTypedExtensionConfig().GetTypedConfig().GetValue(), depth) + if json == nil && err == nil { + continue + } + return json, err + } + return nil, fmt.Errorf("no supported policy found in policy list +%v", lbPolicy) +} diff --git a/xds/internal/xdsclient/xdslbregistry/xdslbregistry_test.go b/xds/internal/xdsclient/xdslbregistry/xdslbregistry_test.go new file mode 100644 index 000000000000..e679c4a5b29a --- /dev/null +++ b/xds/internal/xdsclient/xdslbregistry/xdslbregistry_test.go @@ -0,0 +1,476 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package xdslbregistry_test contains test cases for the xDS LB Policy Registry. +package xdslbregistry_test + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/golang/protobuf/proto" + "github.com/google/go-cmp/cmp" + _ "google.golang.org/grpc/balancer/roundrobin" + "google.golang.org/grpc/internal/balancer/stub" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/pretty" + internalserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/internal/testutils" + _ "google.golang.org/grpc/xds" // Register the xDS LB Registry Converters. + "google.golang.org/grpc/xds/internal/balancer/wrrlocality" + "google.golang.org/grpc/xds/internal/xdsclient/xdslbregistry" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/wrapperspb" + + v1xdsudpatypepb "github.com/cncf/xds/go/udpa/type/v1" + v3xdsxdstypepb "github.com/cncf/xds/go/xds/type/v3" + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3leastrequestpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/least_request/v3" + v3pickfirstpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/pick_first/v3" + v3ringhashpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/ring_hash/v3" + v3roundrobinpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/round_robin/v3" + v3wrrlocalitypb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/wrr_locality/v3" + structpb "github.com/golang/protobuf/ptypes/struct" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +func wrrLocalityBalancerConfig(childPolicy *internalserviceconfig.BalancerConfig) *internalserviceconfig.BalancerConfig { + return &internalserviceconfig.BalancerConfig{ + Name: wrrlocality.Name, + Config: &wrrlocality.LBConfig{ + ChildPolicy: childPolicy, + }, + } +} + +func (s) TestConvertToServiceConfigSuccess(t *testing.T) { + defer func(old bool) { envconfig.LeastRequestLB = old }(envconfig.LeastRequestLB) + envconfig.LeastRequestLB = false + + const customLBPolicyName = "myorg.MyCustomLeastRequestPolicy" + stub.Register(customLBPolicyName, stub.BalancerFuncs{}) + + tests := []struct { + name string + policy *v3clusterpb.LoadBalancingPolicy + wantConfig string // JSON config + rhDisabled bool + pfDisabled bool + lrEnabled bool + }{ + { + name: "ring_hash", + policy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: testutils.MarshalAny(&v3ringhashpb.RingHash{ + HashFunction: v3ringhashpb.RingHash_XX_HASH, + MinimumRingSize: wrapperspb.UInt64(10), + MaximumRingSize: wrapperspb.UInt64(100), + }), + }, + }, + }, + }, + wantConfig: `[{"ring_hash_experimental": { "minRingSize": 10, "maxRingSize": 100 }}]`, + }, + { + name: "least_request", + policy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: testutils.MarshalAny(&v3leastrequestpb.LeastRequest{ + ChoiceCount: wrapperspb.UInt32(3), + }), + }, + }, + }, + }, + wantConfig: `[{"least_request_experimental": { "choiceCount": 3 }}]`, + lrEnabled: true, + }, + { + name: "pick_first_shuffle", + policy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: testutils.MarshalAny(&v3pickfirstpb.PickFirst{ + ShuffleAddressList: true, + }), + }, + }, + }, + }, + wantConfig: `[{"pick_first": { "shuffleAddressList": true }}]`, + }, + { + name: "pick_first", + policy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: testutils.MarshalAny(&v3pickfirstpb.PickFirst{}), + }, + }, + }, + }, + wantConfig: `[{"pick_first": { "shuffleAddressList": false }}]`, + }, + { + name: "round_robin", + policy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: testutils.MarshalAny(&v3roundrobinpb.RoundRobin{}), + }, + }, + }, + }, + wantConfig: `[{"round_robin": {}}]`, + }, + { + name: "round_robin_ring_hash_use_first_supported", + policy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: testutils.MarshalAny(&v3roundrobinpb.RoundRobin{}), + }, + }, + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: testutils.MarshalAny(&v3ringhashpb.RingHash{ + HashFunction: v3ringhashpb.RingHash_XX_HASH, + MinimumRingSize: wrapperspb.UInt64(10), + MaximumRingSize: wrapperspb.UInt64(100), + }), + }, + }, + }, + }, + wantConfig: `[{"round_robin": {}}]`, + }, + { + name: "ring_hash_disabled_rh_rr_use_first_supported", + policy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: testutils.MarshalAny(&v3ringhashpb.RingHash{ + HashFunction: v3ringhashpb.RingHash_XX_HASH, + MinimumRingSize: wrapperspb.UInt64(10), + MaximumRingSize: wrapperspb.UInt64(100), + }), + }, + }, + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: testutils.MarshalAny(&v3roundrobinpb.RoundRobin{}), + }, + }, + }, + }, + wantConfig: `[{"round_robin": {}}]`, + rhDisabled: true, + }, + { + name: "pick_first_enabled_pf_rr_use_pick_first", + policy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: testutils.MarshalAny(&v3pickfirstpb.PickFirst{ + ShuffleAddressList: true, + }), + }, + }, + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: testutils.MarshalAny(&v3roundrobinpb.RoundRobin{}), + }, + }, + }, + }, + wantConfig: `[{"pick_first": { "shuffleAddressList": true }}]`, + }, + { + name: "least_request_disabled_pf_rr_use_first_supported", + policy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: testutils.MarshalAny(&v3leastrequestpb.LeastRequest{ + ChoiceCount: wrapperspb.UInt32(32), + }), + }, + }, + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: testutils.MarshalAny(&v3roundrobinpb.RoundRobin{}), + }, + }, + }, + }, + wantConfig: `[{"round_robin": {}}]`, + }, + { + name: "custom_lb_type_v3_struct", + policy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + // The type not registered in gRPC Policy registry. + // Should fallback to next policy in list. + TypedConfig: testutils.MarshalAny(&v3xdsxdstypepb.TypedStruct{ + TypeUrl: "type.googleapis.com/myorg.ThisTypeDoesNotExist", + Value: &structpb.Struct{}, + }), + }, + }, + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: testutils.MarshalAny(&v3xdsxdstypepb.TypedStruct{ + TypeUrl: "type.googleapis.com/myorg.MyCustomLeastRequestPolicy", + Value: &structpb.Struct{}, + }), + }, + }, + }, + }, + wantConfig: `[{"myorg.MyCustomLeastRequestPolicy": {}}]`, + }, + { + name: "custom_lb_type_v1_struct", + policy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: testutils.MarshalAny(&v1xdsudpatypepb.TypedStruct{ + TypeUrl: "type.googleapis.com/myorg.MyCustomLeastRequestPolicy", + Value: &structpb.Struct{}, + }), + }, + }, + }, + }, + wantConfig: `[{"myorg.MyCustomLeastRequestPolicy": {}}]`, + }, + { + name: "wrr_locality_child_round_robin", + policy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: wrrLocalityAny(&v3roundrobinpb.RoundRobin{}), + }, + }, + }, + }, + wantConfig: `[{"xds_wrr_locality_experimental": { "childPolicy": [{"round_robin": {}}] }}]`, + }, + { + name: "wrr_locality_child_custom_lb_type_v3_struct", + policy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: wrrLocalityAny(&v3xdsxdstypepb.TypedStruct{ + TypeUrl: "type.googleapis.com/myorg.MyCustomLeastRequestPolicy", + Value: &structpb.Struct{}, + }), + }, + }, + }, + }, + wantConfig: `[{"xds_wrr_locality_experimental": { "childPolicy": [{"myorg.MyCustomLeastRequestPolicy": {}}] }}]`, + }, + { + name: "on-the-boundary-of-recursive-limit", + policy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: wrrLocalityAny(wrrLocality(wrrLocality(wrrLocality(wrrLocality(wrrLocality(wrrLocality(wrrLocality(wrrLocality(wrrLocality(wrrLocality(wrrLocality(wrrLocality(wrrLocality(wrrLocality(&v3roundrobinpb.RoundRobin{}))))))))))))))), + }, + }, + }, + }, + wantConfig: jsonMarshal(t, wrrLocalityBalancerConfig(wrrLocalityBalancerConfig(wrrLocalityBalancerConfig(wrrLocalityBalancerConfig(wrrLocalityBalancerConfig(wrrLocalityBalancerConfig(wrrLocalityBalancerConfig(wrrLocalityBalancerConfig(wrrLocalityBalancerConfig(wrrLocalityBalancerConfig(wrrLocalityBalancerConfig(wrrLocalityBalancerConfig(wrrLocalityBalancerConfig(wrrLocalityBalancerConfig(wrrLocalityBalancerConfig(&internalserviceconfig.BalancerConfig{ + Name: "round_robin", + })))))))))))))))), + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if test.rhDisabled { + defer func(old bool) { envconfig.XDSRingHash = old }(envconfig.XDSRingHash) + envconfig.XDSRingHash = false + } + if test.lrEnabled { + defer func(old bool) { envconfig.LeastRequestLB = old }(envconfig.LeastRequestLB) + envconfig.LeastRequestLB = true + } + if test.pfDisabled { + defer func(old bool) { envconfig.PickFirstLBConfig = old }(envconfig.PickFirstLBConfig) + envconfig.PickFirstLBConfig = false + } + rawJSON, err := xdslbregistry.ConvertToServiceConfig(test.policy, 0) + if err != nil { + t.Fatalf("ConvertToServiceConfig(%s) failed: %v", pretty.ToJSON(test.policy), err) + } + // got and want must be unmarshalled since JSON strings shouldn't + // generally be directly compared. + var got []map[string]any + if err := json.Unmarshal(rawJSON, &got); err != nil { + t.Fatalf("Error unmarshalling rawJSON (%q): %v", rawJSON, err) + } + var want []map[string]any + if err := json.Unmarshal(json.RawMessage(test.wantConfig), &want); err != nil { + t.Fatalf("Error unmarshalling wantConfig (%q): %v", test.wantConfig, err) + } + if diff := cmp.Diff(got, want); diff != "" { + t.Fatalf("ConvertToServiceConfig() got unexpected output, diff (-got +want): %v", diff) + } + }) + } +} + +func jsonMarshal(t *testing.T, x any) string { + t.Helper() + js, err := json.Marshal(x) + if err != nil { + t.Fatalf("Error marshalling to JSON (%+v): %v", x, err) + } + return string(js) +} + +// TestConvertToServiceConfigFailure tests failure cases of the xDS LB registry +// of converting proto configuration to JSON configuration. +func (s) TestConvertToServiceConfigFailure(t *testing.T) { + tests := []struct { + name string + policy *v3clusterpb.LoadBalancingPolicy + wantErr string + }{ + { + name: "not xx_hash function", + policy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: testutils.MarshalAny(&v3ringhashpb.RingHash{ + HashFunction: v3ringhashpb.RingHash_MURMUR_HASH_2, + MinimumRingSize: wrapperspb.UInt64(10), + MaximumRingSize: wrapperspb.UInt64(100), + }), + }, + }, + }, + }, + wantErr: "unsupported ring_hash hash function", + }, + { + name: "no-supported-policy", + policy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + // The type not registered in gRPC Policy registry. + TypedConfig: testutils.MarshalAny(&v3xdsxdstypepb.TypedStruct{ + TypeUrl: "type.googleapis.com/myorg.ThisTypeDoesNotExist", + Value: &structpb.Struct{}, + }), + }, + }, + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + // Not supported by gRPC-Go. + TypedConfig: testutils.MarshalAny(&v3leastrequestpb.LeastRequest{}), + }, + }, + }, + }, + wantErr: "no supported policy found in policy list", + }, + { + name: "exceeds-boundary-of-recursive-limit-by-1", + policy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: wrrLocalityAny(wrrLocality(wrrLocality(wrrLocality(wrrLocality(wrrLocality(wrrLocality(wrrLocality(wrrLocality(wrrLocality(wrrLocality(wrrLocality(wrrLocality(wrrLocality(wrrLocality(wrrLocality(&v3roundrobinpb.RoundRobin{})))))))))))))))), + }, + }, + }, + }, + wantErr: "exceeds max depth", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, gotErr := xdslbregistry.ConvertToServiceConfig(test.policy, 0) + // Test the error substring to test the different root causes of + // errors. This is more brittle over time, but it's important to + // test the root cause of the errors emitted from the + // ConvertToServiceConfig function call. Also, this package owns the + // error strings so breakages won't come unexpectedly. + if gotErr == nil || !strings.Contains(gotErr.Error(), test.wantErr) { + t.Fatalf("ConvertToServiceConfig() = %v, wantErr %v", gotErr, test.wantErr) + } + }) + } +} + +// wrrLocality is a helper that takes a proto message and returns a +// WrrLocalityProto with the proto message marshaled into a proto.Any as a +// child. +func wrrLocality(m proto.Message) *v3wrrlocalitypb.WrrLocality { + return &v3wrrlocalitypb.WrrLocality{ + EndpointPickingPolicy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: testutils.MarshalAny(m), + }, + }, + }, + }, + } +} + +// wrrLocalityAny takes a proto message and returns a wrr locality proto +// marshaled as an any with an any child set to the marshaled proto message. +func wrrLocalityAny(m proto.Message) *anypb.Any { + return testutils.MarshalAny(wrrLocality(m)) +} diff --git a/xds/internal/xdsclient/xdsresource/cluster_resource_type.go b/xds/internal/xdsclient/xdsresource/cluster_resource_type.go new file mode 100644 index 000000000000..183801c1c68c --- /dev/null +++ b/xds/internal/xdsclient/xdsresource/cluster_resource_type.go @@ -0,0 +1,153 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource/version" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" +) + +const ( + // ClusterResourceTypeName represents the transport agnostic name for the + // cluster resource. + ClusterResourceTypeName = "ClusterResource" +) + +var ( + // Compile time interface checks. + _ Type = clusterResourceType{} + + // Singleton instantiation of the resource type implementation. + clusterType = clusterResourceType{ + resourceTypeState: resourceTypeState{ + typeURL: version.V3ClusterURL, + typeName: ClusterResourceTypeName, + allResourcesRequiredInSotW: true, + }, + } +) + +// clusterResourceType provides the resource-type specific functionality for a +// Cluster resource. +// +// Implements the Type interface. +type clusterResourceType struct { + resourceTypeState +} + +// Decode deserializes and validates an xDS resource serialized inside the +// provided `Any` proto, as received from the xDS management server. +func (clusterResourceType) Decode(opts *DecodeOptions, resource *anypb.Any) (*DecodeResult, error) { + name, cluster, err := unmarshalClusterResource(resource) + switch { + case name == "": + // Name is unset only when protobuf deserialization fails. + return nil, err + case err != nil: + // Protobuf deserialization succeeded, but resource validation failed. + return &DecodeResult{Name: name, Resource: &ClusterResourceData{Resource: ClusterUpdate{}}}, err + } + + // Perform extra validation here. + if err := securityConfigValidator(opts.BootstrapConfig, cluster.SecurityCfg); err != nil { + return &DecodeResult{Name: name, Resource: &ClusterResourceData{Resource: ClusterUpdate{}}}, err + } + + return &DecodeResult{Name: name, Resource: &ClusterResourceData{Resource: cluster}}, nil + +} + +// ClusterResourceData wraps the configuration of a Cluster resource as received +// from the management server. +// +// Implements the ResourceData interface. +type ClusterResourceData struct { + ResourceData + + // TODO: We have always stored update structs by value. See if this can be + // switched to a pointer? + Resource ClusterUpdate +} + +// Equal returns true if other is equal to r. +func (c *ClusterResourceData) Equal(other ResourceData) bool { + if c == nil && other == nil { + return true + } + if (c == nil) != (other == nil) { + return false + } + return proto.Equal(c.Resource.Raw, other.Raw()) +} + +// ToJSON returns a JSON string representation of the resource data. +func (c *ClusterResourceData) ToJSON() string { + return pretty.ToJSON(c.Resource) +} + +// Raw returns the underlying raw protobuf form of the cluster resource. +func (c *ClusterResourceData) Raw() *anypb.Any { + return c.Resource.Raw +} + +// ClusterWatcher wraps the callbacks to be invoked for different events +// corresponding to the cluster resource being watched. +type ClusterWatcher interface { + // OnUpdate is invoked to report an update for the resource being watched. + OnUpdate(*ClusterResourceData) + + // OnError is invoked under different error conditions including but not + // limited to the following: + // - authority mentioned in the resource is not found + // - resource name parsing error + // - resource deserialization error + // - resource validation error + // - ADS stream failure + // - connection failure + OnError(error) + + // OnResourceDoesNotExist is invoked for a specific error condition where + // the requested resource is not found on the xDS management server. + OnResourceDoesNotExist() +} + +type delegatingClusterWatcher struct { + watcher ClusterWatcher +} + +func (d *delegatingClusterWatcher) OnUpdate(data ResourceData) { + c := data.(*ClusterResourceData) + d.watcher.OnUpdate(c) +} + +func (d *delegatingClusterWatcher) OnError(err error) { + d.watcher.OnError(err) +} + +func (d *delegatingClusterWatcher) OnResourceDoesNotExist() { + d.watcher.OnResourceDoesNotExist() +} + +// WatchCluster uses xDS to discover the configuration associated with the +// provided cluster resource name. +func WatchCluster(p Producer, name string, w ClusterWatcher) (cancel func()) { + delegator := &delegatingClusterWatcher{watcher: w} + return p.WatchResource(clusterType, name, delegator) +} diff --git a/xds/internal/xdsclient/xdsresource/endpoints_resource_type.go b/xds/internal/xdsclient/xdsresource/endpoints_resource_type.go new file mode 100644 index 000000000000..775a8aa19423 --- /dev/null +++ b/xds/internal/xdsclient/xdsresource/endpoints_resource_type.go @@ -0,0 +1,149 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource/version" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" +) + +const ( + // EndpointsResourceTypeName represents the transport agnostic name for the + // endpoint resource. + EndpointsResourceTypeName = "EndpointsResource" +) + +var ( + // Compile time interface checks. + _ Type = endpointsResourceType{} + + // Singleton instantiation of the resource type implementation. + endpointsType = endpointsResourceType{ + resourceTypeState: resourceTypeState{ + typeURL: version.V3EndpointsURL, + typeName: "EndpointsResource", + allResourcesRequiredInSotW: false, + }, + } +) + +// endpointsResourceType provides the resource-type specific functionality for a +// ClusterLoadAssignment (or Endpoints) resource. +// +// Implements the Type interface. +type endpointsResourceType struct { + resourceTypeState +} + +// Decode deserializes and validates an xDS resource serialized inside the +// provided `Any` proto, as received from the xDS management server. +func (endpointsResourceType) Decode(opts *DecodeOptions, resource *anypb.Any) (*DecodeResult, error) { + name, rc, err := unmarshalEndpointsResource(resource) + switch { + case name == "": + // Name is unset only when protobuf deserialization fails. + return nil, err + case err != nil: + // Protobuf deserialization succeeded, but resource validation failed. + return &DecodeResult{Name: name, Resource: &EndpointsResourceData{Resource: EndpointsUpdate{}}}, err + } + + return &DecodeResult{Name: name, Resource: &EndpointsResourceData{Resource: rc}}, nil + +} + +// EndpointsResourceData wraps the configuration of an Endpoints resource as +// received from the management server. +// +// Implements the ResourceData interface. +type EndpointsResourceData struct { + ResourceData + + // TODO: We have always stored update structs by value. See if this can be + // switched to a pointer? + Resource EndpointsUpdate +} + +// Equal returns true if other is equal to r. +func (e *EndpointsResourceData) Equal(other ResourceData) bool { + if e == nil && other == nil { + return true + } + if (e == nil) != (other == nil) { + return false + } + return proto.Equal(e.Resource.Raw, other.Raw()) + +} + +// ToJSON returns a JSON string representation of the resource data. +func (e *EndpointsResourceData) ToJSON() string { + return pretty.ToJSON(e.Resource) +} + +// Raw returns the underlying raw protobuf form of the listener resource. +func (e *EndpointsResourceData) Raw() *anypb.Any { + return e.Resource.Raw +} + +// EndpointsWatcher wraps the callbacks to be invoked for different +// events corresponding to the endpoints resource being watched. +type EndpointsWatcher interface { + // OnUpdate is invoked to report an update for the resource being watched. + OnUpdate(*EndpointsResourceData) + + // OnError is invoked under different error conditions including but not + // limited to the following: + // - authority mentioned in the resource is not found + // - resource name parsing error + // - resource deserialization error + // - resource validation error + // - ADS stream failure + // - connection failure + OnError(error) + + // OnResourceDoesNotExist is invoked for a specific error condition where + // the requested resource is not found on the xDS management server. + OnResourceDoesNotExist() +} + +type delegatingEndpointsWatcher struct { + watcher EndpointsWatcher +} + +func (d *delegatingEndpointsWatcher) OnUpdate(data ResourceData) { + e := data.(*EndpointsResourceData) + d.watcher.OnUpdate(e) +} + +func (d *delegatingEndpointsWatcher) OnError(err error) { + d.watcher.OnError(err) +} + +func (d *delegatingEndpointsWatcher) OnResourceDoesNotExist() { + d.watcher.OnResourceDoesNotExist() +} + +// WatchEndpoints uses xDS to discover the configuration associated with the +// provided endpoints resource name. +func WatchEndpoints(p Producer, name string, w EndpointsWatcher) (cancel func()) { + delegator := &delegatingEndpointsWatcher{watcher: w} + return p.WatchResource(endpointsType, name, delegator) +} diff --git a/xds/internal/client/errors.go b/xds/internal/xdsclient/xdsresource/errors.go similarity index 79% rename from xds/internal/client/errors.go rename to xds/internal/xdsclient/xdsresource/errors.go index 34ae2738db00..7bac4469b78b 100644 --- a/xds/internal/client/errors.go +++ b/xds/internal/xdsclient/xdsresource/errors.go @@ -16,7 +16,7 @@ * */ -package client +package xdsresource import "fmt" @@ -34,6 +34,12 @@ const ( // response. It's typically returned if the resource is removed in the xds // server. ErrorTypeResourceNotFound + // ErrorTypeResourceTypeUnsupported indicates the receipt of a message from + // the management server with resources of an unsupported resource type. + ErrorTypeResourceTypeUnsupported + // ErrTypeStreamFailedAfterRecv indicates an ADS stream error, after + // successful receipt of at least one message from the server. + ErrTypeStreamFailedAfterRecv ) type xdsClientError struct { @@ -47,7 +53,7 @@ func (e *xdsClientError) Error() string { // NewErrorf creates an xds client error. The callbacks are called with this // error, to pass additional information about the error. -func NewErrorf(t ErrorType, format string, args ...interface{}) error { +func NewErrorf(t ErrorType, format string, args ...any) error { return &xdsClientError{t: t, desc: fmt.Sprintf(format, args...)} } diff --git a/xds/internal/xdsclient/xdsresource/filter_chain.go b/xds/internal/xdsclient/xdsresource/filter_chain.go new file mode 100644 index 000000000000..0390412fdc89 --- /dev/null +++ b/xds/internal/xdsclient/xdsresource/filter_chain.go @@ -0,0 +1,869 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "errors" + "fmt" + "net" + + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" + "github.com/golang/protobuf/proto" + "github.com/golang/protobuf/ptypes" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/resolver" + "google.golang.org/grpc/xds/internal/httpfilter" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource/version" +) + +const ( + // Used as the map key for unspecified prefixes. The actual value of this + // key is immaterial. + unspecifiedPrefixMapKey = "unspecified" + + // An unspecified destination or source prefix should be considered a less + // specific match than a wildcard prefix, `0.0.0.0/0` or `::/0`. Also, an + // unspecified prefix should match most v4 and v6 addresses compared to the + // wildcard prefixes which match only a specific network (v4 or v6). + // + // We use these constants when looking up the most specific prefix match. A + // wildcard prefix will match 0 bits, and to make sure that a wildcard + // prefix is considered a more specific match than an unspecified prefix, we + // use a value of -1 for the latter. + noPrefixMatch = -2 + unspecifiedPrefixMatch = -1 +) + +// FilterChain captures information from within a FilterChain message in a +// Listener resource. +type FilterChain struct { + // SecurityCfg contains transport socket security configuration. + SecurityCfg *SecurityConfig + // HTTPFilters represent the HTTP Filters that comprise this FilterChain. + HTTPFilters []HTTPFilter + // RouteConfigName is the route configuration name for this FilterChain. + // + // Exactly one of RouteConfigName and InlineRouteConfig is set. + RouteConfigName string + // InlineRouteConfig is the inline route configuration (RDS response) + // returned for this filter chain. + // + // Exactly one of RouteConfigName and InlineRouteConfig is set. + InlineRouteConfig *RouteConfigUpdate +} + +// VirtualHostWithInterceptors captures information present in a VirtualHost +// update, and also contains routes with instantiated HTTP Filters. +type VirtualHostWithInterceptors struct { + // Domains are the domain names which map to this Virtual Host. On the + // server side, this will be dictated by the :authority header of the + // incoming RPC. + Domains []string + // Routes are the Routes for this Virtual Host. + Routes []RouteWithInterceptors +} + +// RouteWithInterceptors captures information in a Route, and contains +// a usable matcher and also instantiated HTTP Filters. +type RouteWithInterceptors struct { + // M is the matcher used to match to this route. + M *CompositeMatcher + // ActionType is the type of routing action to initiate once matched to. + ActionType RouteActionType + // Interceptors are interceptors instantiated for this route. These will be + // constructed from a combination of the top level configuration and any + // HTTP Filter overrides present in Virtual Host or Route. + Interceptors []resolver.ServerInterceptor +} + +// ConstructUsableRouteConfiguration takes Route Configuration and converts it +// into matchable route configuration, with instantiated HTTP Filters per route. +func (f *FilterChain) ConstructUsableRouteConfiguration(config RouteConfigUpdate) ([]VirtualHostWithInterceptors, error) { + vhs := make([]VirtualHostWithInterceptors, len(config.VirtualHosts)) + for _, vh := range config.VirtualHosts { + vhwi, err := f.convertVirtualHost(vh) + if err != nil { + return nil, fmt.Errorf("virtual host construction: %v", err) + } + vhs = append(vhs, vhwi) + } + return vhs, nil +} + +func (f *FilterChain) convertVirtualHost(virtualHost *VirtualHost) (VirtualHostWithInterceptors, error) { + rs := make([]RouteWithInterceptors, len(virtualHost.Routes)) + for i, r := range virtualHost.Routes { + var err error + rs[i].ActionType = r.ActionType + rs[i].M, err = RouteToMatcher(r) + if err != nil { + return VirtualHostWithInterceptors{}, fmt.Errorf("matcher construction: %v", err) + } + for _, filter := range f.HTTPFilters { + // Route is highest priority on server side, as there is no concept + // of an upstream cluster on server side. + override := r.HTTPFilterConfigOverride[filter.Name] + if override == nil { + // Virtual Host is second priority. + override = virtualHost.HTTPFilterConfigOverride[filter.Name] + } + sb, ok := filter.Filter.(httpfilter.ServerInterceptorBuilder) + if !ok { + // Should not happen if it passed xdsClient validation. + return VirtualHostWithInterceptors{}, fmt.Errorf("filter does not support use in server") + } + si, err := sb.BuildServerInterceptor(filter.Config, override) + if err != nil { + return VirtualHostWithInterceptors{}, fmt.Errorf("filter construction: %v", err) + } + if si != nil { + rs[i].Interceptors = append(rs[i].Interceptors, si) + } + } + } + return VirtualHostWithInterceptors{Domains: virtualHost.Domains, Routes: rs}, nil +} + +// SourceType specifies the connection source IP match type. +type SourceType int + +const ( + // SourceTypeAny matches connection attempts from any source. + SourceTypeAny SourceType = iota + // SourceTypeSameOrLoopback matches connection attempts from the same host. + SourceTypeSameOrLoopback + // SourceTypeExternal matches connection attempts from a different host. + SourceTypeExternal +) + +// FilterChainManager contains all the match criteria specified through all +// filter chains in a single Listener resource. It also contains the default +// filter chain specified in the Listener resource. It provides two important +// pieces of functionality: +// 1. Validate the filter chains in an incoming Listener resource to make sure +// that there aren't filter chains which contain the same match criteria. +// 2. As part of performing the above validation, it builds an internal data +// structure which will if used to look up the matching filter chain at +// connection time. +// +// The logic specified in the documentation around the xDS FilterChainMatch +// proto mentions 8 criteria to match on. +// The following order applies: +// +// 1. Destination port. +// 2. Destination IP address. +// 3. Server name (e.g. SNI for TLS protocol), +// 4. Transport protocol. +// 5. Application protocols (e.g. ALPN for TLS protocol). +// 6. Source type (e.g. any, local or external network). +// 7. Source IP address. +// 8. Source port. +type FilterChainManager struct { + // Destination prefix is the first match criteria that we support. + // Therefore, this multi-stage map is indexed on destination prefixes + // specified in the match criteria. + // Unspecified destination prefix matches end up as a wildcard entry here + // with a key of 0.0.0.0/0. + dstPrefixMap map[string]*destPrefixEntry + + // At connection time, we do not have the actual destination prefix to match + // on. We only have the real destination address of the incoming connection. + // This means that we cannot use the above map at connection time. This list + // contains the map entries from the above map that we can use at connection + // time to find matching destination prefixes in O(n) time. + // + // TODO: Implement LC-trie to support logarithmic time lookups. If that + // involves too much time/effort, sort this slice based on the netmask size. + dstPrefixes []*destPrefixEntry + + def *FilterChain // Default filter chain, if specified. + + // RouteConfigNames are the route configuration names which need to be + // dynamically queried for RDS Configuration for any FilterChains which + // specify to load RDS Configuration dynamically. + RouteConfigNames map[string]bool +} + +// destPrefixEntry is the value type of the map indexed on destination prefixes. +type destPrefixEntry struct { + // The actual destination prefix. Set to nil for unspecified prefixes. + net *net.IPNet + // We need to keep track of the transport protocols seen as part of the + // config validation (and internal structure building) phase. The only two + // values that we support are empty string and "raw_buffer", with the latter + // taking preference. Once we have seen one filter chain with "raw_buffer", + // we can drop everything other filter chain with an empty transport + // protocol. + rawBufferSeen bool + // For each specified source type in the filter chain match criteria, this + // array points to the set of specified source prefixes. + // Unspecified source type matches end up as a wildcard entry here with an + // index of 0, which actually represents the source type `ANY`. + srcTypeArr sourceTypesArray +} + +// An array for the fixed number of source types that we have. +type sourceTypesArray [3]*sourcePrefixes + +// sourcePrefixes contains source prefix related information specified in the +// match criteria. These are pointed to by the array of source types. +type sourcePrefixes struct { + // These are very similar to the 'dstPrefixMap' and 'dstPrefixes' field of + // FilterChainManager. Go there for more info. + srcPrefixMap map[string]*sourcePrefixEntry + srcPrefixes []*sourcePrefixEntry +} + +// sourcePrefixEntry contains match criteria per source prefix. +type sourcePrefixEntry struct { + // The actual destination prefix. Set to nil for unspecified prefixes. + net *net.IPNet + // Mapping from source ports specified in the match criteria to the actual + // filter chain. Unspecified source port matches en up as a wildcard entry + // here with a key of 0. + srcPortMap map[int]*FilterChain +} + +// NewFilterChainManager parses the received Listener resource and builds a +// FilterChainManager. Returns a non-nil error on validation failures. +// +// This function is only exported so that tests outside of this package can +// create a FilterChainManager. +func NewFilterChainManager(lis *v3listenerpb.Listener) (*FilterChainManager, error) { + // Parse all the filter chains and build the internal data structures. + fci := &FilterChainManager{ + dstPrefixMap: make(map[string]*destPrefixEntry), + RouteConfigNames: make(map[string]bool), + } + if err := fci.addFilterChains(lis.GetFilterChains()); err != nil { + return nil, err + } + // Build the source and dest prefix slices used by Lookup(). + fcSeen := false + for _, dstPrefix := range fci.dstPrefixMap { + fci.dstPrefixes = append(fci.dstPrefixes, dstPrefix) + for _, st := range dstPrefix.srcTypeArr { + if st == nil { + continue + } + for _, srcPrefix := range st.srcPrefixMap { + st.srcPrefixes = append(st.srcPrefixes, srcPrefix) + for _, fc := range srcPrefix.srcPortMap { + if fc != nil { + fcSeen = true + } + } + } + } + } + + // Retrieve the default filter chain. The match criteria specified on the + // default filter chain is never used. The default filter chain simply gets + // used when none of the other filter chains match. + var def *FilterChain + if dfc := lis.GetDefaultFilterChain(); dfc != nil { + var err error + if def, err = fci.filterChainFromProto(dfc); err != nil { + return nil, err + } + } + fci.def = def + + // If there are no supported filter chains and no default filter chain, we + // fail here. This will call the Listener resource to be NACK'ed. + if !fcSeen && fci.def == nil { + return nil, fmt.Errorf("no supported filter chains and no default filter chain") + } + return fci, nil +} + +// addFilterChains parses the filter chains in fcs and adds the required +// internal data structures corresponding to the match criteria. +func (fci *FilterChainManager) addFilterChains(fcs []*v3listenerpb.FilterChain) error { + for _, fc := range fcs { + fcm := fc.GetFilterChainMatch() + if fcm.GetDestinationPort().GetValue() != 0 { + // Destination port is the first match criteria and we do not + // support filter chains which contains this match criteria. + logger.Warningf("Dropping filter chain %+v since it contains unsupported destination_port match field", fc) + continue + } + + // Build the internal representation of the filter chain match fields. + if err := fci.addFilterChainsForDestPrefixes(fc); err != nil { + return err + } + } + + return nil +} + +func (fci *FilterChainManager) addFilterChainsForDestPrefixes(fc *v3listenerpb.FilterChain) error { + ranges := fc.GetFilterChainMatch().GetPrefixRanges() + dstPrefixes := make([]*net.IPNet, 0, len(ranges)) + for _, pr := range ranges { + cidr := fmt.Sprintf("%s/%d", pr.GetAddressPrefix(), pr.GetPrefixLen().GetValue()) + _, ipnet, err := net.ParseCIDR(cidr) + if err != nil { + return fmt.Errorf("failed to parse destination prefix range: %+v", pr) + } + dstPrefixes = append(dstPrefixes, ipnet) + } + + if len(dstPrefixes) == 0 { + // Use the unspecified entry when destination prefix is unspecified, and + // set the `net` field to nil. + if fci.dstPrefixMap[unspecifiedPrefixMapKey] == nil { + fci.dstPrefixMap[unspecifiedPrefixMapKey] = &destPrefixEntry{} + } + return fci.addFilterChainsForServerNames(fci.dstPrefixMap[unspecifiedPrefixMapKey], fc) + } + for _, prefix := range dstPrefixes { + p := prefix.String() + if fci.dstPrefixMap[p] == nil { + fci.dstPrefixMap[p] = &destPrefixEntry{net: prefix} + } + if err := fci.addFilterChainsForServerNames(fci.dstPrefixMap[p], fc); err != nil { + return err + } + } + return nil +} + +func (fci *FilterChainManager) addFilterChainsForServerNames(dstEntry *destPrefixEntry, fc *v3listenerpb.FilterChain) error { + // Filter chains specifying server names in their match criteria always fail + // a match at connection time. So, these filter chains can be dropped now. + if len(fc.GetFilterChainMatch().GetServerNames()) != 0 { + logger.Warningf("Dropping filter chain %+v since it contains unsupported server_names match field", fc) + return nil + } + + return fci.addFilterChainsForTransportProtocols(dstEntry, fc) +} + +func (fci *FilterChainManager) addFilterChainsForTransportProtocols(dstEntry *destPrefixEntry, fc *v3listenerpb.FilterChain) error { + tp := fc.GetFilterChainMatch().GetTransportProtocol() + switch { + case tp != "" && tp != "raw_buffer": + // Only allow filter chains with transport protocol set to empty string + // or "raw_buffer". + logger.Warningf("Dropping filter chain %+v since it contains unsupported value for transport_protocols match field", fc) + return nil + case tp == "" && dstEntry.rawBufferSeen: + // If we have already seen filter chains with transport protocol set to + // "raw_buffer", we can drop filter chains with transport protocol set + // to empty string, since the former takes precedence. + logger.Warningf("Dropping filter chain %+v since it contains unsupported value for transport_protocols match field", fc) + return nil + case tp != "" && !dstEntry.rawBufferSeen: + // This is the first "raw_buffer" that we are seeing. Set the bit and + // reset the source types array which might contain entries for filter + // chains with transport protocol set to empty string. + dstEntry.rawBufferSeen = true + dstEntry.srcTypeArr = sourceTypesArray{} + } + return fci.addFilterChainsForApplicationProtocols(dstEntry, fc) +} + +func (fci *FilterChainManager) addFilterChainsForApplicationProtocols(dstEntry *destPrefixEntry, fc *v3listenerpb.FilterChain) error { + if len(fc.GetFilterChainMatch().GetApplicationProtocols()) != 0 { + logger.Warningf("Dropping filter chain %+v since it contains unsupported application_protocols match field", fc) + return nil + } + return fci.addFilterChainsForSourceType(dstEntry, fc) +} + +// addFilterChainsForSourceType adds source types to the internal data +// structures and delegates control to addFilterChainsForSourcePrefixes to +// continue building the internal data structure. +func (fci *FilterChainManager) addFilterChainsForSourceType(dstEntry *destPrefixEntry, fc *v3listenerpb.FilterChain) error { + var srcType SourceType + switch st := fc.GetFilterChainMatch().GetSourceType(); st { + case v3listenerpb.FilterChainMatch_ANY: + srcType = SourceTypeAny + case v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK: + srcType = SourceTypeSameOrLoopback + case v3listenerpb.FilterChainMatch_EXTERNAL: + srcType = SourceTypeExternal + default: + return fmt.Errorf("unsupported source type: %v", st) + } + + st := int(srcType) + if dstEntry.srcTypeArr[st] == nil { + dstEntry.srcTypeArr[st] = &sourcePrefixes{srcPrefixMap: make(map[string]*sourcePrefixEntry)} + } + return fci.addFilterChainsForSourcePrefixes(dstEntry.srcTypeArr[st].srcPrefixMap, fc) +} + +// addFilterChainsForSourcePrefixes adds source prefixes to the internal data +// structures and delegates control to addFilterChainsForSourcePorts to continue +// building the internal data structure. +func (fci *FilterChainManager) addFilterChainsForSourcePrefixes(srcPrefixMap map[string]*sourcePrefixEntry, fc *v3listenerpb.FilterChain) error { + ranges := fc.GetFilterChainMatch().GetSourcePrefixRanges() + srcPrefixes := make([]*net.IPNet, 0, len(ranges)) + for _, pr := range fc.GetFilterChainMatch().GetSourcePrefixRanges() { + cidr := fmt.Sprintf("%s/%d", pr.GetAddressPrefix(), pr.GetPrefixLen().GetValue()) + _, ipnet, err := net.ParseCIDR(cidr) + if err != nil { + return fmt.Errorf("failed to parse source prefix range: %+v", pr) + } + srcPrefixes = append(srcPrefixes, ipnet) + } + + if len(srcPrefixes) == 0 { + // Use the unspecified entry when destination prefix is unspecified, and + // set the `net` field to nil. + if srcPrefixMap[unspecifiedPrefixMapKey] == nil { + srcPrefixMap[unspecifiedPrefixMapKey] = &sourcePrefixEntry{ + srcPortMap: make(map[int]*FilterChain), + } + } + return fci.addFilterChainsForSourcePorts(srcPrefixMap[unspecifiedPrefixMapKey], fc) + } + for _, prefix := range srcPrefixes { + p := prefix.String() + if srcPrefixMap[p] == nil { + srcPrefixMap[p] = &sourcePrefixEntry{ + net: prefix, + srcPortMap: make(map[int]*FilterChain), + } + } + if err := fci.addFilterChainsForSourcePorts(srcPrefixMap[p], fc); err != nil { + return err + } + } + return nil +} + +// addFilterChainsForSourcePorts adds source ports to the internal data +// structures and completes the process of building the internal data structure. +// It is here that we determine if there are multiple filter chains with +// overlapping matching rules. +func (fci *FilterChainManager) addFilterChainsForSourcePorts(srcEntry *sourcePrefixEntry, fcProto *v3listenerpb.FilterChain) error { + ports := fcProto.GetFilterChainMatch().GetSourcePorts() + srcPorts := make([]int, 0, len(ports)) + for _, port := range ports { + srcPorts = append(srcPorts, int(port)) + } + + fc, err := fci.filterChainFromProto(fcProto) + if err != nil { + return err + } + + if len(srcPorts) == 0 { + // Use the wildcard port '0', when source ports are unspecified. + if curFC := srcEntry.srcPortMap[0]; curFC != nil { + return errors.New("multiple filter chains with overlapping matching rules are defined") + } + srcEntry.srcPortMap[0] = fc + return nil + } + for _, port := range srcPorts { + if curFC := srcEntry.srcPortMap[port]; curFC != nil { + return errors.New("multiple filter chains with overlapping matching rules are defined") + } + srcEntry.srcPortMap[port] = fc + } + return nil +} + +// filterChainFromProto extracts the relevant information from the FilterChain +// proto and stores it in our internal representation. It also persists any +// RouteNames which need to be queried dynamically via RDS. +func (fci *FilterChainManager) filterChainFromProto(fc *v3listenerpb.FilterChain) (*FilterChain, error) { + filterChain, err := processNetworkFilters(fc.GetFilters()) + if err != nil { + return nil, err + } + // These route names will be dynamically queried via RDS in the wrapped + // listener, which receives the LDS response, if specified for the filter + // chain. + if filterChain.RouteConfigName != "" { + fci.RouteConfigNames[filterChain.RouteConfigName] = true + } + // If the transport_socket field is not specified, it means that the control + // plane has not sent us any security config. This is fine and the server + // will use the fallback credentials configured as part of the + // xdsCredentials. + ts := fc.GetTransportSocket() + if ts == nil { + return filterChain, nil + } + if name := ts.GetName(); name != transportSocketName { + return nil, fmt.Errorf("transport_socket field has unexpected name: %s", name) + } + any := ts.GetTypedConfig() + if any == nil || any.TypeUrl != version.V3DownstreamTLSContextURL { + return nil, fmt.Errorf("transport_socket field has unexpected typeURL: %s", any.TypeUrl) + } + downstreamCtx := &v3tlspb.DownstreamTlsContext{} + if err := proto.Unmarshal(any.GetValue(), downstreamCtx); err != nil { + return nil, fmt.Errorf("failed to unmarshal DownstreamTlsContext in LDS response: %v", err) + } + if downstreamCtx.GetRequireSni().GetValue() { + return nil, fmt.Errorf("require_sni field set to true in DownstreamTlsContext message: %v", downstreamCtx) + } + if downstreamCtx.GetOcspStaplePolicy() != v3tlspb.DownstreamTlsContext_LENIENT_STAPLING { + return nil, fmt.Errorf("ocsp_staple_policy field set to unsupported value in DownstreamTlsContext message: %v", downstreamCtx) + } + // The following fields from `DownstreamTlsContext` are ignore: + // - disable_stateless_session_resumption + // - session_ticket_keys + // - session_ticket_keys_sds_secret_config + // - session_timeout + if downstreamCtx.GetCommonTlsContext() == nil { + return nil, errors.New("DownstreamTlsContext in LDS response does not contain a CommonTlsContext") + } + sc, err := securityConfigFromCommonTLSContext(downstreamCtx.GetCommonTlsContext(), true) + if err != nil { + return nil, err + } + if sc == nil { + // sc == nil is a valid case where the control plane has not sent us any + // security configuration. xDS creds will use fallback creds. + return filterChain, nil + } + sc.RequireClientCert = downstreamCtx.GetRequireClientCertificate().GetValue() + if sc.RequireClientCert && sc.RootInstanceName == "" { + return nil, errors.New("security configuration on the server-side does not contain root certificate provider instance name, but require_client_cert field is set") + } + filterChain.SecurityCfg = sc + return filterChain, nil +} + +// Validate takes a function to validate the FilterChains in this manager. +func (fci *FilterChainManager) Validate(f func(fc *FilterChain) error) error { + for _, dst := range fci.dstPrefixMap { + for _, srcType := range dst.srcTypeArr { + if srcType == nil { + continue + } + for _, src := range srcType.srcPrefixMap { + for _, fc := range src.srcPortMap { + if err := f(fc); err != nil { + return err + } + } + } + } + } + return f(fci.def) +} + +func processNetworkFilters(filters []*v3listenerpb.Filter) (*FilterChain, error) { + filterChain := &FilterChain{} + seenNames := make(map[string]bool, len(filters)) + seenHCM := false + for _, filter := range filters { + name := filter.GetName() + if name == "" { + return nil, fmt.Errorf("network filters {%+v} is missing name field in filter: {%+v}", filters, filter) + } + if seenNames[name] { + return nil, fmt.Errorf("network filters {%+v} has duplicate filter name %q", filters, name) + } + seenNames[name] = true + + // Network filters have a oneof field named `config_type` where we + // only support `TypedConfig` variant. + switch typ := filter.GetConfigType().(type) { + case *v3listenerpb.Filter_TypedConfig: + // The typed_config field has an `anypb.Any` proto which could + // directly contain the serialized bytes of the actual filter + // configuration, or it could be encoded as a `TypedStruct`. + // TODO: Add support for `TypedStruct`. + tc := filter.GetTypedConfig() + + // The only network filter that we currently support is the v3 + // HttpConnectionManager. So, we can directly check the type_url + // and unmarshal the config. + // TODO: Implement a registry of supported network filters (like + // we have for HTTP filters), when we have to support network + // filters other than HttpConnectionManager. + if tc.GetTypeUrl() != version.V3HTTPConnManagerURL { + return nil, fmt.Errorf("network filters {%+v} has unsupported network filter %q in filter {%+v}", filters, tc.GetTypeUrl(), filter) + } + hcm := &v3httppb.HttpConnectionManager{} + if err := ptypes.UnmarshalAny(tc, hcm); err != nil { + return nil, fmt.Errorf("network filters {%+v} failed unmarshaling of network filter {%+v}: %v", filters, filter, err) + } + // "Any filters after HttpConnectionManager should be ignored during + // connection processing but still be considered for validity. + // HTTPConnectionManager must have valid http_filters." - A36 + filters, err := processHTTPFilters(hcm.GetHttpFilters(), true) + if err != nil { + return nil, fmt.Errorf("network filters {%+v} had invalid server side HTTP Filters {%+v}: %v", filters, hcm.GetHttpFilters(), err) + } + if !seenHCM { + // Validate for RBAC in only the HCM that will be used, since this isn't a logical validation failure, + // it's simply a validation to support RBAC HTTP Filter. + // "HttpConnectionManager.xff_num_trusted_hops must be unset or zero and + // HttpConnectionManager.original_ip_detection_extensions must be empty. If + // either field has an incorrect value, the Listener must be NACKed." - A41 + if hcm.XffNumTrustedHops != 0 { + return nil, fmt.Errorf("xff_num_trusted_hops must be unset or zero %+v", hcm) + } + if len(hcm.OriginalIpDetectionExtensions) != 0 { + return nil, fmt.Errorf("original_ip_detection_extensions must be empty %+v", hcm) + } + + // TODO: Implement terminal filter logic, as per A36. + filterChain.HTTPFilters = filters + seenHCM = true + if !envconfig.XDSRBAC { + continue + } + switch hcm.RouteSpecifier.(type) { + case *v3httppb.HttpConnectionManager_Rds: + if hcm.GetRds().GetConfigSource().GetAds() == nil { + return nil, fmt.Errorf("ConfigSource is not ADS: %+v", hcm) + } + name := hcm.GetRds().GetRouteConfigName() + if name == "" { + return nil, fmt.Errorf("empty route_config_name: %+v", hcm) + } + filterChain.RouteConfigName = name + case *v3httppb.HttpConnectionManager_RouteConfig: + // "RouteConfiguration validation logic inherits all + // previous validations made for client-side usage as RDS + // does not distinguish between client-side and + // server-side." - A36 + // Can specify v3 here, as will never get to this function + // if v2. + routeU, err := generateRDSUpdateFromRouteConfiguration(hcm.GetRouteConfig()) + if err != nil { + return nil, fmt.Errorf("failed to parse inline RDS resp: %v", err) + } + filterChain.InlineRouteConfig = &routeU + case nil: + return nil, fmt.Errorf("no RouteSpecifier: %+v", hcm) + default: + return nil, fmt.Errorf("unsupported type %T for RouteSpecifier", hcm.RouteSpecifier) + } + } + default: + return nil, fmt.Errorf("network filters {%+v} has unsupported config_type %T in filter %s", filters, typ, filter.GetName()) + } + } + if !seenHCM { + return nil, fmt.Errorf("network filters {%+v} missing HttpConnectionManager filter", filters) + } + return filterChain, nil +} + +// FilterChainLookupParams wraps parameters to be passed to Lookup. +type FilterChainLookupParams struct { + // IsUnspecified indicates whether the server is listening on a wildcard + // address, "0.0.0.0" for IPv4 and "::" for IPv6. Only when this is set to + // true, do we consider the destination prefixes specified in the filter + // chain match criteria. + IsUnspecifiedListener bool + // DestAddr is the local address of an incoming connection. + DestAddr net.IP + // SourceAddr is the remote address of an incoming connection. + SourceAddr net.IP + // SourcePort is the remote port of an incoming connection. + SourcePort int +} + +// Lookup returns the most specific matching filter chain to be used for an +// incoming connection on the server side. +// +// Returns a non-nil error if no matching filter chain could be found or +// multiple matching filter chains were found, and in both cases, the incoming +// connection must be dropped. +func (fci *FilterChainManager) Lookup(params FilterChainLookupParams) (*FilterChain, error) { + dstPrefixes := filterByDestinationPrefixes(fci.dstPrefixes, params.IsUnspecifiedListener, params.DestAddr) + if len(dstPrefixes) == 0 { + if fci.def != nil { + return fci.def, nil + } + return nil, fmt.Errorf("no matching filter chain based on destination prefix match for %+v", params) + } + + srcType := SourceTypeExternal + if params.SourceAddr.Equal(params.DestAddr) || params.SourceAddr.IsLoopback() { + srcType = SourceTypeSameOrLoopback + } + srcPrefixes := filterBySourceType(dstPrefixes, srcType) + if len(srcPrefixes) == 0 { + if fci.def != nil { + return fci.def, nil + } + return nil, fmt.Errorf("no matching filter chain based on source type match for %+v", params) + } + srcPrefixEntry, err := filterBySourcePrefixes(srcPrefixes, params.SourceAddr) + if err != nil { + return nil, err + } + if fc := filterBySourcePorts(srcPrefixEntry, params.SourcePort); fc != nil { + return fc, nil + } + if fci.def != nil { + return fci.def, nil + } + return nil, fmt.Errorf("no matching filter chain after all match criteria for %+v", params) +} + +// filterByDestinationPrefixes is the first stage of the filter chain +// matching algorithm. It takes the complete set of configured filter chain +// matchers and returns the most specific matchers based on the destination +// prefix match criteria (the prefixes which match the most number of bits). +func filterByDestinationPrefixes(dstPrefixes []*destPrefixEntry, isUnspecified bool, dstAddr net.IP) []*destPrefixEntry { + if !isUnspecified { + // Destination prefix matchers are considered only when the listener is + // bound to the wildcard address. + return dstPrefixes + } + + var matchingDstPrefixes []*destPrefixEntry + maxSubnetMatch := noPrefixMatch + for _, prefix := range dstPrefixes { + if prefix.net != nil && !prefix.net.Contains(dstAddr) { + // Skip prefixes which don't match. + continue + } + // For unspecified prefixes, since we do not store a real net.IPNet + // inside prefix, we do not perform a match. Instead we simply set + // the matchSize to -1, which is less than the matchSize (0) for a + // wildcard prefix. + matchSize := unspecifiedPrefixMatch + if prefix.net != nil { + matchSize, _ = prefix.net.Mask.Size() + } + if matchSize < maxSubnetMatch { + continue + } + if matchSize > maxSubnetMatch { + maxSubnetMatch = matchSize + matchingDstPrefixes = make([]*destPrefixEntry, 0, 1) + } + matchingDstPrefixes = append(matchingDstPrefixes, prefix) + } + return matchingDstPrefixes +} + +// filterBySourceType is the second stage of the matching algorithm. It +// trims the filter chains based on the most specific source type match. +func filterBySourceType(dstPrefixes []*destPrefixEntry, srcType SourceType) []*sourcePrefixes { + var ( + srcPrefixes []*sourcePrefixes + bestSrcTypeMatch int + ) + for _, prefix := range dstPrefixes { + var ( + srcPrefix *sourcePrefixes + match int + ) + switch srcType { + case SourceTypeExternal: + match = int(SourceTypeExternal) + srcPrefix = prefix.srcTypeArr[match] + case SourceTypeSameOrLoopback: + match = int(SourceTypeSameOrLoopback) + srcPrefix = prefix.srcTypeArr[match] + } + if srcPrefix == nil { + match = int(SourceTypeAny) + srcPrefix = prefix.srcTypeArr[match] + } + if match < bestSrcTypeMatch { + continue + } + if match > bestSrcTypeMatch { + bestSrcTypeMatch = match + srcPrefixes = make([]*sourcePrefixes, 0) + } + if srcPrefix != nil { + // The source type array always has 3 entries, but these could be + // nil if the appropriate source type match was not specified. + srcPrefixes = append(srcPrefixes, srcPrefix) + } + } + return srcPrefixes +} + +// filterBySourcePrefixes is the third stage of the filter chain matching +// algorithm. It trims the filter chains based on the source prefix. At most one +// filter chain with the most specific match progress to the next stage. +func filterBySourcePrefixes(srcPrefixes []*sourcePrefixes, srcAddr net.IP) (*sourcePrefixEntry, error) { + var matchingSrcPrefixes []*sourcePrefixEntry + maxSubnetMatch := noPrefixMatch + for _, sp := range srcPrefixes { + for _, prefix := range sp.srcPrefixes { + if prefix.net != nil && !prefix.net.Contains(srcAddr) { + // Skip prefixes which don't match. + continue + } + // For unspecified prefixes, since we do not store a real net.IPNet + // inside prefix, we do not perform a match. Instead we simply set + // the matchSize to -1, which is less than the matchSize (0) for a + // wildcard prefix. + matchSize := unspecifiedPrefixMatch + if prefix.net != nil { + matchSize, _ = prefix.net.Mask.Size() + } + if matchSize < maxSubnetMatch { + continue + } + if matchSize > maxSubnetMatch { + maxSubnetMatch = matchSize + matchingSrcPrefixes = make([]*sourcePrefixEntry, 0, 1) + } + matchingSrcPrefixes = append(matchingSrcPrefixes, prefix) + } + } + if len(matchingSrcPrefixes) == 0 { + // Finding no match is not an error condition. The caller will end up + // using the default filter chain if one was configured. + return nil, nil + } + // We expect at most a single matching source prefix entry at this point. If + // we have multiple entries here, and some of their source port matchers had + // wildcard entries, we could be left with more than one matching filter + // chain and hence would have been flagged as an invalid configuration at + // config validation time. + if len(matchingSrcPrefixes) != 1 { + return nil, errors.New("multiple matching filter chains") + } + return matchingSrcPrefixes[0], nil +} + +// filterBySourcePorts is the last stage of the filter chain matching +// algorithm. It trims the filter chains based on the source ports. +func filterBySourcePorts(spe *sourcePrefixEntry, srcPort int) *FilterChain { + if spe == nil { + return nil + } + // A match could be a wildcard match (this happens when the match + // criteria does not specify source ports) or a specific port match (this + // happens when the match criteria specifies a set of ports and the source + // port of the incoming connection matches one of the specified ports). The + // latter is considered to be a more specific match. + if fc := spe.srcPortMap[srcPort]; fc != nil { + return fc + } + if fc := spe.srcPortMap[0]; fc != nil { + return fc + } + return nil +} diff --git a/xds/internal/xdsclient/xdsresource/filter_chain_test.go b/xds/internal/xdsclient/xdsresource/filter_chain_test.go new file mode 100644 index 000000000000..4edf9ce006f1 --- /dev/null +++ b/xds/internal/xdsclient/xdsresource/filter_chain_test.go @@ -0,0 +1,2972 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "context" + "errors" + "fmt" + "net" + "strings" + "testing" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3routerpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3" + v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "google.golang.org/protobuf/testing/protocmp" + "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/wrapperspb" + + "google.golang.org/grpc/internal/envconfig" + iresolver "google.golang.org/grpc/internal/resolver" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/xds/internal/httpfilter" + "google.golang.org/grpc/xds/internal/httpfilter/router" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource/version" +) + +const ( + topLevel = "top level" + vhLevel = "virtual host level" + rLevel = "route level" +) + +var ( + routeConfig = &v3routepb.RouteConfiguration{ + Name: "routeName", + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{"lds.target.good:3333"}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}, + }, + Action: &v3routepb.Route_NonForwardingAction{}, + }}}}} + inlineRouteConfig = &RouteConfigUpdate{ + VirtualHosts: []*VirtualHost{{ + Domains: []string{"lds.target.good:3333"}, + Routes: []*Route{{Prefix: newStringP("/"), ActionType: RouteActionNonForwardingAction}}, + }}} + emptyValidNetworkFilters = []*v3listenerpb.Filter{ + { + Name: "filter-1", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: routeConfig, + }, + HttpFilters: []*v3httppb.HttpFilter{emptyRouterFilter}, + }), + }, + }, + } + validServerSideHTTPFilter1 = &v3httppb.HttpFilter{ + Name: "serverOnlyCustomFilter", + ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: serverOnlyCustomFilterConfig}, + } + validServerSideHTTPFilter2 = &v3httppb.HttpFilter{ + Name: "serverOnlyCustomFilter2", + ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: serverOnlyCustomFilterConfig}, + } + emptyRouterFilter = e2e.RouterHTTPFilter + routerBuilder = httpfilter.Get(router.TypeURL) + routerConfig, _ = routerBuilder.ParseFilterConfig(testutils.MarshalAny(&v3routerpb.Router{})) + routerFilter = HTTPFilter{Name: "router", Filter: routerBuilder, Config: routerConfig} + routerFilterList = []HTTPFilter{routerFilter} +) + +// TestNewFilterChainImpl_Failure_BadMatchFields verifies cases where we have a +// single filter chain with match criteria that contains unsupported fields. +func (s) TestNewFilterChainImpl_Failure_BadMatchFields(t *testing.T) { + tests := []struct { + desc string + lis *v3listenerpb.Listener + }{ + { + desc: "unsupported destination port field", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{DestinationPort: &wrapperspb.UInt32Value{Value: 666}}, + }, + }, + }, + }, + { + desc: "unsupported server names field", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{ServerNames: []string{"example-server"}}, + }, + }, + }, + }, + { + desc: "unsupported transport protocol ", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{TransportProtocol: "tls"}, + }, + }, + }, + }, + { + desc: "unsupported application protocol field", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{ApplicationProtocols: []string{"h2"}}, + }, + }, + }, + }, + { + desc: "bad dest address prefix", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{PrefixRanges: []*v3corepb.CidrRange{{AddressPrefix: "a.b.c.d"}}}, + }, + }, + }, + }, + { + desc: "bad dest prefix length", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("10.1.1.0", 50)}}, + }, + }, + }, + }, + { + desc: "bad source address prefix", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{SourcePrefixRanges: []*v3corepb.CidrRange{{AddressPrefix: "a.b.c.d"}}}, + }, + }, + }, + }, + { + desc: "bad source prefix length", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{SourcePrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("10.1.1.0", 50)}}, + }, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + if fci, err := NewFilterChainManager(test.lis); err == nil { + t.Fatalf("NewFilterChainManager() returned %v when expected to fail", fci) + } + }) + } +} + +// TestNewFilterChainImpl_Failure_OverlappingMatchingRules verifies cases where +// there are multiple filter chains and they have overlapping match rules. +func (s) TestNewFilterChainImpl_Failure_OverlappingMatchingRules(t *testing.T) { + tests := []struct { + desc string + lis *v3listenerpb.Listener + }{ + { + desc: "matching destination prefixes with no other matchers", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16), cidrRangeFromAddressAndPrefixLen("10.0.0.0", 0)}, + }, + Filters: emptyValidNetworkFilters, + }, + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.2.2", 16)}, + }, + Filters: emptyValidNetworkFilters, + }, + }, + }, + }, + { + desc: "matching source type", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{SourceType: v3listenerpb.FilterChainMatch_ANY}, + Filters: emptyValidNetworkFilters, + }, + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK}, + Filters: emptyValidNetworkFilters, + }, + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{SourceType: v3listenerpb.FilterChainMatch_EXTERNAL}, + Filters: emptyValidNetworkFilters, + }, + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{SourceType: v3listenerpb.FilterChainMatch_EXTERNAL}, + Filters: emptyValidNetworkFilters, + }, + }, + }, + }, + { + desc: "matching source prefixes", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + SourcePrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16), cidrRangeFromAddressAndPrefixLen("10.0.0.0", 0)}, + }, + Filters: emptyValidNetworkFilters, + }, + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + SourcePrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.2.2", 16)}, + }, + Filters: emptyValidNetworkFilters, + }, + }, + }, + }, + { + desc: "matching source ports", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{SourcePorts: []uint32{1, 2, 3, 4, 5}}, + Filters: emptyValidNetworkFilters, + }, + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{}, + Filters: emptyValidNetworkFilters, + }, + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{SourcePorts: []uint32{5, 6, 7}}, + Filters: emptyValidNetworkFilters, + }, + }, + }, + }, + } + + const wantErr = "multiple filter chains with overlapping matching rules are defined" + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + if _, err := NewFilterChainManager(test.lis); err == nil || !strings.Contains(err.Error(), wantErr) { + t.Fatalf("NewFilterChainManager() returned err: %v, wantErr: %s", err, wantErr) + } + }) + } +} + +// TestNewFilterChainImpl_Failure_BadSecurityConfig verifies cases where the +// security configuration in the filter chain is invalid. +func (s) TestNewFilterChainImpl_Failure_BadSecurityConfig(t *testing.T) { + tests := []struct { + desc string + lis *v3listenerpb.Listener + wantErr string + }{ + { + desc: "no filter chains", + lis: &v3listenerpb.Listener{}, + wantErr: "no supported filter chains and no default filter chain", + }, + { + desc: "unexpected transport socket name", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + TransportSocket: &v3corepb.TransportSocket{Name: "unsupported-transport-socket-name"}, + Filters: emptyValidNetworkFilters, + }, + }, + }, + wantErr: "transport_socket field has unexpected name", + }, + { + desc: "unexpected transport socket URL", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.UpstreamTlsContext{}), + }, + }, + Filters: emptyValidNetworkFilters, + }, + }, + }, + wantErr: "transport_socket field has unexpected typeURL", + }, + { + desc: "badly marshaled transport socket", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: &anypb.Any{ + TypeUrl: version.V3DownstreamTLSContextURL, + Value: []byte{1, 2, 3, 4}, + }, + }, + }, + Filters: emptyValidNetworkFilters, + }, + }, + }, + wantErr: "failed to unmarshal DownstreamTlsContext in LDS response", + }, + { + desc: "missing CommonTlsContext", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{}), + }, + }, + Filters: emptyValidNetworkFilters, + }, + }, + }, + wantErr: "DownstreamTlsContext in LDS response does not contain a CommonTlsContext", + }, + { + desc: "require_sni-set-to-true-in-downstreamTlsContext", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{ + RequireSni: &wrapperspb.BoolValue{Value: true}, + }), + }, + }, + Filters: emptyValidNetworkFilters, + }, + }, + }, + wantErr: "require_sni field set to true in DownstreamTlsContext message", + }, + { + desc: "unsupported-ocsp_staple_policy-in-downstreamTlsContext", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{ + OcspStaplePolicy: v3tlspb.DownstreamTlsContext_STRICT_STAPLING, + }), + }, + }, + Filters: emptyValidNetworkFilters, + }, + }, + }, + wantErr: "ocsp_staple_policy field set to unsupported value in DownstreamTlsContext message", + }, + { + desc: "unsupported validation context in transport socket", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextSdsSecretConfig{ + ValidationContextSdsSecretConfig: &v3tlspb.SdsSecretConfig{ + Name: "foo-sds-secret", + }, + }, + }, + }), + }, + }, + Filters: emptyValidNetworkFilters, + }, + }, + }, + wantErr: "validation context contains unexpected type", + }, + { + desc: "unsupported match_subject_alt_names field in transport socket", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextSdsSecretConfig{ + ValidationContextSdsSecretConfig: &v3tlspb.SdsSecretConfig{ + Name: "foo-sds-secret", + }, + }, + }, + }), + }, + }, + Filters: emptyValidNetworkFilters, + }, + }, + }, + wantErr: "validation context contains unexpected type", + }, + { + desc: "no root certificate provider with require_client_cert", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{ + RequireClientCertificate: &wrapperspb.BoolValue{Value: true}, + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: "identityPluginInstance", + CertificateName: "identityCertName", + }, + }, + }), + }, + }, + Filters: emptyValidNetworkFilters, + }, + }, + }, + wantErr: "security configuration on the server-side does not contain root certificate provider instance name, but require_client_cert field is set", + }, + { + desc: "no identity certificate provider", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{}, + }), + }, + }, + Filters: emptyValidNetworkFilters, + }, + }, + }, + wantErr: "security configuration on the server-side does not contain identity certificate provider instance name", + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + _, err := NewFilterChainManager(test.lis) + if err == nil || !strings.Contains(err.Error(), test.wantErr) { + t.Fatalf("NewFilterChainManager() returned err: %v, wantErr: %s", err, test.wantErr) + } + }) + } +} + +// TestNewFilterChainImpl_Success_RouteUpdate tests the construction of the +// filter chain with valid HTTP Filters present. +func (s) TestNewFilterChainImpl_Success_RouteUpdate(t *testing.T) { + oldRBAC := envconfig.XDSRBAC + envconfig.XDSRBAC = true + defer func() { + envconfig.XDSRBAC = oldRBAC + }() + tests := []struct { + name string + lis *v3listenerpb.Listener + wantFC *FilterChainManager + }{ + { + name: "rds", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: []*v3listenerpb.Filter{ + { + Name: "hcm", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{ + Rds: &v3httppb.Rds{ + ConfigSource: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}, + }, + RouteConfigName: "route-1", + }, + }, + HttpFilters: []*v3httppb.HttpFilter{emptyRouterFilter}, + }), + }, + }, + }, + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{ + Filters: []*v3listenerpb.Filter{ + { + Name: "hcm", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{ + Rds: &v3httppb.Rds{ + ConfigSource: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}, + }, + RouteConfigName: "route-1", + }, + }, + HttpFilters: []*v3httppb.HttpFilter{emptyRouterFilter}, + }), + }, + }, + }, + }, + }, + wantFC: &FilterChainManager{ + dstPrefixMap: map[string]*destPrefixEntry{ + unspecifiedPrefixMapKey: { + srcTypeArr: [3]*sourcePrefixes{ + { + srcPrefixMap: map[string]*sourcePrefixEntry{ + unspecifiedPrefixMapKey: { + srcPortMap: map[int]*FilterChain{ + 0: { + RouteConfigName: "route-1", + HTTPFilters: routerFilterList, + }, + }, + }, + }, + }, + }, + }, + }, + def: &FilterChain{ + RouteConfigName: "route-1", + HTTPFilters: routerFilterList, + }, + RouteConfigNames: map[string]bool{"route-1": true}, + }, + }, + { + name: "inline route config", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: []*v3listenerpb.Filter{ + { + Name: "hcm", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: routeConfig, + }, + HttpFilters: []*v3httppb.HttpFilter{emptyRouterFilter}, + }), + }, + }, + }, + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{ + Filters: []*v3listenerpb.Filter{ + { + Name: "hcm", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: routeConfig, + }, + HttpFilters: []*v3httppb.HttpFilter{emptyRouterFilter}, + }), + }, + }, + }, + }, + }, + wantFC: &FilterChainManager{ + dstPrefixMap: map[string]*destPrefixEntry{ + unspecifiedPrefixMapKey: { + srcTypeArr: [3]*sourcePrefixes{ + { + srcPrefixMap: map[string]*sourcePrefixEntry{ + unspecifiedPrefixMapKey: { + srcPortMap: map[int]*FilterChain{ + 0: { + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + }, + }, + }, + }, + }, + }, + }, + def: &FilterChain{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + }, + }, + // two rds tests whether the Filter Chain Manager successfully persists + // the two RDS names that need to be dynamically queried. + { + name: "two rds", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: []*v3listenerpb.Filter{ + { + Name: "hcm", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{ + Rds: &v3httppb.Rds{ + ConfigSource: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}, + }, + RouteConfigName: "route-1", + }, + }, + HttpFilters: []*v3httppb.HttpFilter{emptyRouterFilter}, + }), + }, + }, + }, + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{ + Filters: []*v3listenerpb.Filter{ + { + Name: "hcm", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{ + Rds: &v3httppb.Rds{ + ConfigSource: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}, + }, + RouteConfigName: "route-2", + }, + }, + HttpFilters: []*v3httppb.HttpFilter{emptyRouterFilter}, + }), + }, + }, + }, + }, + }, + wantFC: &FilterChainManager{ + dstPrefixMap: map[string]*destPrefixEntry{ + unspecifiedPrefixMapKey: { + srcTypeArr: [3]*sourcePrefixes{ + { + srcPrefixMap: map[string]*sourcePrefixEntry{ + unspecifiedPrefixMapKey: { + srcPortMap: map[int]*FilterChain{ + 0: { + RouteConfigName: "route-1", + HTTPFilters: routerFilterList, + }, + }, + }, + }, + }, + }, + }, + }, + def: &FilterChain{ + RouteConfigName: "route-2", + HTTPFilters: routerFilterList, + }, + RouteConfigNames: map[string]bool{ + "route-1": true, + "route-2": true, + }, + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + gotFC, err := NewFilterChainManager(test.lis) + if err != nil { + t.Fatalf("NewFilterChainManager() returned err: %v, wantErr: nil", err) + } + if !cmp.Equal(gotFC, test.wantFC, cmp.AllowUnexported(FilterChainManager{}, destPrefixEntry{}, sourcePrefixes{}, sourcePrefixEntry{}), cmpOpts) { + t.Fatalf("NewFilterChainManager() returned %+v, want: %+v", gotFC, test.wantFC) + } + }) + } +} + +// TestNewFilterChainImpl_Failure_BadRouteUpdate verifies cases where the Route +// Update in the filter chain are invalid. +func (s) TestNewFilterChainImpl_Failure_BadRouteUpdate(t *testing.T) { + oldRBAC := envconfig.XDSRBAC + envconfig.XDSRBAC = true + defer func() { + envconfig.XDSRBAC = oldRBAC + }() + tests := []struct { + name string + lis *v3listenerpb.Listener + wantErr string + }{ + { + name: "missing-route-specifier", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: []*v3listenerpb.Filter{ + { + Name: "hcm", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + + TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + HttpFilters: []*v3httppb.HttpFilter{emptyRouterFilter}, + }), + }, + }, + }, + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{ + Filters: []*v3listenerpb.Filter{ + { + Name: "hcm", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + HttpFilters: []*v3httppb.HttpFilter{emptyRouterFilter}, + }), + }, + }, + }, + }, + }, + wantErr: "no RouteSpecifier", + }, + { + name: "not-ads", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: []*v3listenerpb.Filter{ + { + Name: "hcm", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + + TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{ + Rds: &v3httppb.Rds{ + RouteConfigName: "route-1", + }, + }, + HttpFilters: []*v3httppb.HttpFilter{emptyRouterFilter}, + }), + }, + }, + }, + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{ + Filters: []*v3listenerpb.Filter{ + { + Name: "hcm", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{ + Rds: &v3httppb.Rds{ + RouteConfigName: "route-1", + }, + }, + HttpFilters: []*v3httppb.HttpFilter{emptyRouterFilter}, + }), + }, + }, + }, + }, + }, + wantErr: "ConfigSource is not ADS", + }, + { + name: "unsupported-route-specifier", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: []*v3listenerpb.Filter{ + { + Name: "hcm", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_ScopedRoutes{}, + HttpFilters: []*v3httppb.HttpFilter{emptyRouterFilter}, + }), + }, + }, + }, + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{ + Filters: []*v3listenerpb.Filter{ + { + Name: "hcm", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_ScopedRoutes{}, + HttpFilters: []*v3httppb.HttpFilter{emptyRouterFilter}, + }), + }, + }, + }, + }, + }, + wantErr: "unsupported type", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := NewFilterChainManager(test.lis) + if err == nil || !strings.Contains(err.Error(), test.wantErr) { + t.Fatalf("NewFilterChainManager() returned err: %v, wantErr: %s", err, test.wantErr) + } + }) + } +} + +// TestNewFilterChainImpl_Failure_BadHTTPFilters verifies cases where the HTTP +// Filters in the filter chain are invalid. +func (s) TestNewFilterChainImpl_Failure_BadHTTPFilters(t *testing.T) { + tests := []struct { + name string + lis *v3listenerpb.Listener + wantErr string + }{ + { + name: "client side HTTP filter", + lis: &v3listenerpb.Listener{ + Name: "grpc/server?xds.resource.listening_address=0.0.0.0:9999", + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: []*v3listenerpb.Filter{ + { + Name: "hcm", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + HttpFilters: []*v3httppb.HttpFilter{ + { + Name: "clientOnlyCustomFilter", + ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: clientOnlyCustomFilterConfig}, + }, + }, + }), + }, + }, + }, + }, + }, + }, + wantErr: "invalid server side HTTP Filters", + }, + { + name: "one valid then one invalid HTTP filter", + lis: &v3listenerpb.Listener{ + Name: "grpc/server?xds.resource.listening_address=0.0.0.0:9999", + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: []*v3listenerpb.Filter{ + { + Name: "hcm", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + HttpFilters: []*v3httppb.HttpFilter{ + validServerSideHTTPFilter1, + { + Name: "clientOnlyCustomFilter", + ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: clientOnlyCustomFilterConfig}, + }, + }, + }), + }, + }, + }, + }, + }, + }, + wantErr: "invalid server side HTTP Filters", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := NewFilterChainManager(test.lis) + if err == nil || !strings.Contains(err.Error(), test.wantErr) { + t.Fatalf("NewFilterChainManager() returned err: %v, wantErr: %s", err, test.wantErr) + } + }) + } +} + +// TestNewFilterChainImpl_Success_HTTPFilters tests the construction of the +// filter chain with valid HTTP Filters present. +func (s) TestNewFilterChainImpl_Success_HTTPFilters(t *testing.T) { + oldRBAC := envconfig.XDSRBAC + envconfig.XDSRBAC = true + defer func() { + envconfig.XDSRBAC = oldRBAC + }() + tests := []struct { + name string + lis *v3listenerpb.Listener + wantFC *FilterChainManager + }{ + { + name: "singular valid http filter", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: []*v3listenerpb.Filter{ + { + Name: "hcm", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + HttpFilters: []*v3httppb.HttpFilter{ + validServerSideHTTPFilter1, + emptyRouterFilter, + }, + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: routeConfig, + }, + }), + }, + }, + }, + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{ + Filters: []*v3listenerpb.Filter{ + { + Name: "hcm", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + HttpFilters: []*v3httppb.HttpFilter{ + validServerSideHTTPFilter1, + emptyRouterFilter, + }, + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: routeConfig, + }, + }), + }, + }, + }, + }, + }, + wantFC: &FilterChainManager{ + dstPrefixMap: map[string]*destPrefixEntry{ + unspecifiedPrefixMapKey: { + srcTypeArr: [3]*sourcePrefixes{ + { + srcPrefixMap: map[string]*sourcePrefixEntry{ + unspecifiedPrefixMapKey: { + srcPortMap: map[int]*FilterChain{ + 0: {HTTPFilters: []HTTPFilter{ + { + Name: "serverOnlyCustomFilter", + Filter: serverOnlyHTTPFilter{}, + Config: filterConfig{Cfg: serverOnlyCustomFilterConfig}, + }, + routerFilter, + }, + InlineRouteConfig: inlineRouteConfig, + }, + }, + }, + }, + }, + }, + }, + }, + def: &FilterChain{ + HTTPFilters: []HTTPFilter{ + { + Name: "serverOnlyCustomFilter", + Filter: serverOnlyHTTPFilter{}, + Config: filterConfig{Cfg: serverOnlyCustomFilterConfig}, + }, + routerFilter, + }, + InlineRouteConfig: inlineRouteConfig, + }, + }, + }, + { + name: "two valid http filters", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: []*v3listenerpb.Filter{ + { + Name: "hcm", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + HttpFilters: []*v3httppb.HttpFilter{ + validServerSideHTTPFilter1, + validServerSideHTTPFilter2, + emptyRouterFilter, + }, + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: routeConfig, + }, + }), + }, + }, + }, + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{ + Filters: []*v3listenerpb.Filter{ + { + Name: "hcm", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + HttpFilters: []*v3httppb.HttpFilter{ + validServerSideHTTPFilter1, + validServerSideHTTPFilter2, + emptyRouterFilter, + }, + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: routeConfig, + }, + }), + }, + }, + }, + }, + }, + wantFC: &FilterChainManager{ + dstPrefixMap: map[string]*destPrefixEntry{ + unspecifiedPrefixMapKey: { + srcTypeArr: [3]*sourcePrefixes{ + { + srcPrefixMap: map[string]*sourcePrefixEntry{ + unspecifiedPrefixMapKey: { + srcPortMap: map[int]*FilterChain{ + 0: {HTTPFilters: []HTTPFilter{ + { + Name: "serverOnlyCustomFilter", + Filter: serverOnlyHTTPFilter{}, + Config: filterConfig{Cfg: serverOnlyCustomFilterConfig}, + }, + { + Name: "serverOnlyCustomFilter2", + Filter: serverOnlyHTTPFilter{}, + Config: filterConfig{Cfg: serverOnlyCustomFilterConfig}, + }, + routerFilter, + }, + InlineRouteConfig: inlineRouteConfig, + }, + }, + }, + }, + }, + }, + }, + }, + def: &FilterChain{HTTPFilters: []HTTPFilter{ + { + Name: "serverOnlyCustomFilter", + Filter: serverOnlyHTTPFilter{}, + Config: filterConfig{Cfg: serverOnlyCustomFilterConfig}, + }, + { + Name: "serverOnlyCustomFilter2", + Filter: serverOnlyHTTPFilter{}, + Config: filterConfig{Cfg: serverOnlyCustomFilterConfig}, + }, + routerFilter, + }, + InlineRouteConfig: inlineRouteConfig, + }, + }, + }, + // In the case of two HTTP Connection Manager's being present, the + // second HTTP Connection Manager should be validated, but ignored. + { + name: "two hcms", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: []*v3listenerpb.Filter{ + { + Name: "hcm", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + HttpFilters: []*v3httppb.HttpFilter{ + validServerSideHTTPFilter1, + validServerSideHTTPFilter2, + emptyRouterFilter, + }, + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: routeConfig, + }, + }), + }, + }, + { + Name: "hcm2", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + HttpFilters: []*v3httppb.HttpFilter{ + validServerSideHTTPFilter1, + emptyRouterFilter, + }, + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: routeConfig, + }, + }), + }, + }, + }, + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{ + Filters: []*v3listenerpb.Filter{ + { + Name: "hcm", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + HttpFilters: []*v3httppb.HttpFilter{ + validServerSideHTTPFilter1, + validServerSideHTTPFilter2, + emptyRouterFilter, + }, + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: routeConfig, + }, + }), + }, + }, + { + Name: "hcm2", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + HttpFilters: []*v3httppb.HttpFilter{ + validServerSideHTTPFilter1, + emptyRouterFilter, + }, + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: routeConfig, + }, + }), + }, + }, + }, + }, + }, + wantFC: &FilterChainManager{ + dstPrefixMap: map[string]*destPrefixEntry{ + unspecifiedPrefixMapKey: { + srcTypeArr: [3]*sourcePrefixes{ + { + srcPrefixMap: map[string]*sourcePrefixEntry{ + unspecifiedPrefixMapKey: { + srcPortMap: map[int]*FilterChain{ + 0: {HTTPFilters: []HTTPFilter{ + { + Name: "serverOnlyCustomFilter", + Filter: serverOnlyHTTPFilter{}, + Config: filterConfig{Cfg: serverOnlyCustomFilterConfig}, + }, + { + Name: "serverOnlyCustomFilter2", + Filter: serverOnlyHTTPFilter{}, + Config: filterConfig{Cfg: serverOnlyCustomFilterConfig}, + }, + routerFilter, + }, + InlineRouteConfig: inlineRouteConfig, + }, + }, + }, + }, + }, + }, + }, + }, + def: &FilterChain{HTTPFilters: []HTTPFilter{ + { + Name: "serverOnlyCustomFilter", + Filter: serverOnlyHTTPFilter{}, + Config: filterConfig{Cfg: serverOnlyCustomFilterConfig}, + }, + { + Name: "serverOnlyCustomFilter2", + Filter: serverOnlyHTTPFilter{}, + Config: filterConfig{Cfg: serverOnlyCustomFilterConfig}, + }, + routerFilter, + }, + InlineRouteConfig: inlineRouteConfig, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + gotFC, err := NewFilterChainManager(test.lis) + if err != nil { + t.Fatalf("NewFilterChainManager() returned err: %v, wantErr: nil", err) + } + if !cmp.Equal(gotFC, test.wantFC, cmp.AllowUnexported(FilterChainManager{}, destPrefixEntry{}, sourcePrefixes{}, sourcePrefixEntry{}), cmpOpts) { + t.Fatalf("NewFilterChainManager() returned %+v, want: %+v", gotFC, test.wantFC) + } + }) + } +} + +// TestNewFilterChainImpl_Success_SecurityConfig verifies cases where the +// security configuration in the filter chain contains valid data. +func (s) TestNewFilterChainImpl_Success_SecurityConfig(t *testing.T) { + oldRBAC := envconfig.XDSRBAC + envconfig.XDSRBAC = true + defer func() { + envconfig.XDSRBAC = oldRBAC + }() + tests := []struct { + desc string + lis *v3listenerpb.Listener + wantFC *FilterChainManager + }{ + { + desc: "empty transport socket", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: emptyValidNetworkFilters, + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{ + Filters: emptyValidNetworkFilters, + }, + }, + wantFC: &FilterChainManager{ + dstPrefixMap: map[string]*destPrefixEntry{ + unspecifiedPrefixMapKey: { + srcTypeArr: [3]*sourcePrefixes{ + { + srcPrefixMap: map[string]*sourcePrefixEntry{ + unspecifiedPrefixMapKey: { + srcPortMap: map[int]*FilterChain{ + 0: { + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + }, + }, + }, + }, + }, + }, + }, + def: &FilterChain{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + }, + }, + { + desc: "no validation context", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: "identityPluginInstance", + CertificateName: "identityCertName", + }, + }, + }), + }, + }, + Filters: emptyValidNetworkFilters, + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{ + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: "defaultIdentityPluginInstance", + CertificateName: "defaultIdentityCertName", + }, + }, + }), + }, + }, + Filters: emptyValidNetworkFilters, + }, + }, + wantFC: &FilterChainManager{ + dstPrefixMap: map[string]*destPrefixEntry{ + unspecifiedPrefixMapKey: { + srcTypeArr: [3]*sourcePrefixes{ + { + srcPrefixMap: map[string]*sourcePrefixEntry{ + unspecifiedPrefixMapKey: { + srcPortMap: map[int]*FilterChain{ + 0: { + SecurityCfg: &SecurityConfig{ + IdentityInstanceName: "identityPluginInstance", + IdentityCertName: "identityCertName", + }, + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + }, + }, + }, + }, + }, + }, + }, + def: &FilterChain{ + SecurityCfg: &SecurityConfig{ + IdentityInstanceName: "defaultIdentityPluginInstance", + IdentityCertName: "defaultIdentityCertName", + }, + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + }, + }, + { + desc: "validation context with certificate provider", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{ + RequireClientCertificate: &wrapperspb.BoolValue{Value: true}, + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: "identityPluginInstance", + CertificateName: "identityCertName", + }, + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{ + ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: "rootPluginInstance", + CertificateName: "rootCertName", + }, + }, + }, + }), + }, + }, + Filters: emptyValidNetworkFilters, + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{ + Name: "default-filter-chain-1", + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{ + RequireClientCertificate: &wrapperspb.BoolValue{Value: true}, + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: "defaultIdentityPluginInstance", + CertificateName: "defaultIdentityCertName", + }, + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{ + ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: "defaultRootPluginInstance", + CertificateName: "defaultRootCertName", + }, + }, + }, + }), + }, + }, + Filters: emptyValidNetworkFilters, + }, + }, + wantFC: &FilterChainManager{ + dstPrefixMap: map[string]*destPrefixEntry{ + unspecifiedPrefixMapKey: { + srcTypeArr: [3]*sourcePrefixes{ + { + srcPrefixMap: map[string]*sourcePrefixEntry{ + unspecifiedPrefixMapKey: { + srcPortMap: map[int]*FilterChain{ + 0: { + SecurityCfg: &SecurityConfig{ + RootInstanceName: "rootPluginInstance", + RootCertName: "rootCertName", + IdentityInstanceName: "identityPluginInstance", + IdentityCertName: "identityCertName", + RequireClientCert: true, + }, + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + }, + }, + }, + }, + }, + }, + }, + def: &FilterChain{ + SecurityCfg: &SecurityConfig{ + RootInstanceName: "defaultRootPluginInstance", + RootCertName: "defaultRootCertName", + IdentityInstanceName: "defaultIdentityPluginInstance", + IdentityCertName: "defaultIdentityCertName", + RequireClientCert: true, + }, + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + gotFC, err := NewFilterChainManager(test.lis) + if err != nil { + t.Fatalf("NewFilterChainManager() returned err: %v, wantErr: nil", err) + } + if !cmp.Equal(gotFC, test.wantFC, cmp.AllowUnexported(FilterChainManager{}, destPrefixEntry{}, sourcePrefixes{}, sourcePrefixEntry{}), cmpopts.EquateEmpty()) { + t.Fatalf("NewFilterChainManager() returned %+v, want: %+v", gotFC, test.wantFC) + } + }) + } +} + +// TestNewFilterChainImpl_Success_UnsupportedMatchFields verifies cases where +// there are multiple filter chains, and one of them is valid while the other +// contains unsupported match fields. These configurations should lead to +// success at config validation time and the filter chains which contains +// unsupported match fields will be skipped at lookup time. +func (s) TestNewFilterChainImpl_Success_UnsupportedMatchFields(t *testing.T) { + oldRBAC := envconfig.XDSRBAC + envconfig.XDSRBAC = true + defer func() { + envconfig.XDSRBAC = oldRBAC + }() + unspecifiedEntry := &destPrefixEntry{ + srcTypeArr: [3]*sourcePrefixes{ + { + srcPrefixMap: map[string]*sourcePrefixEntry{ + unspecifiedPrefixMapKey: { + srcPortMap: map[int]*FilterChain{ + 0: { + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + }, + }, + }, + }, + }, + } + + tests := []struct { + desc string + lis *v3listenerpb.Listener + wantFC *FilterChainManager + }{ + { + desc: "unsupported destination port", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "good-chain", + Filters: emptyValidNetworkFilters, + }, + { + Name: "unsupported-destination-port", + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16)}, + DestinationPort: &wrapperspb.UInt32Value{Value: 666}, + }, + Filters: emptyValidNetworkFilters, + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{Filters: emptyValidNetworkFilters}, + }, + wantFC: &FilterChainManager{ + dstPrefixMap: map[string]*destPrefixEntry{ + unspecifiedPrefixMapKey: unspecifiedEntry, + }, + def: &FilterChain{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + }, + }, + { + desc: "unsupported server names", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "good-chain", + Filters: emptyValidNetworkFilters, + }, + { + Name: "unsupported-server-names", + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16)}, + ServerNames: []string{"example-server"}, + }, + Filters: emptyValidNetworkFilters, + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{Filters: emptyValidNetworkFilters}, + }, + wantFC: &FilterChainManager{ + dstPrefixMap: map[string]*destPrefixEntry{ + unspecifiedPrefixMapKey: unspecifiedEntry, + "192.168.0.0/16": { + net: ipNetFromCIDR("192.168.2.2/16"), + }, + }, + def: &FilterChain{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + }, + }, + { + desc: "unsupported transport protocol", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "good-chain", + Filters: emptyValidNetworkFilters, + }, + { + Name: "unsupported-transport-protocol", + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16)}, + TransportProtocol: "tls", + }, + Filters: emptyValidNetworkFilters, + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{Filters: emptyValidNetworkFilters}, + }, + wantFC: &FilterChainManager{ + dstPrefixMap: map[string]*destPrefixEntry{ + unspecifiedPrefixMapKey: unspecifiedEntry, + "192.168.0.0/16": { + net: ipNetFromCIDR("192.168.2.2/16"), + }, + }, + def: &FilterChain{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + }, + }, + { + desc: "unsupported application protocol", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "good-chain", + Filters: emptyValidNetworkFilters, + }, + { + Name: "unsupported-application-protocol", + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16)}, + ApplicationProtocols: []string{"h2"}, + }, + Filters: emptyValidNetworkFilters, + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{Filters: emptyValidNetworkFilters}, + }, + wantFC: &FilterChainManager{ + dstPrefixMap: map[string]*destPrefixEntry{ + unspecifiedPrefixMapKey: unspecifiedEntry, + "192.168.0.0/16": { + net: ipNetFromCIDR("192.168.2.2/16"), + }, + }, + def: &FilterChain{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + gotFC, err := NewFilterChainManager(test.lis) + if err != nil { + t.Fatalf("NewFilterChainManager() returned err: %v, wantErr: nil", err) + } + if !cmp.Equal(gotFC, test.wantFC, cmp.AllowUnexported(FilterChainManager{}, destPrefixEntry{}, sourcePrefixes{}, sourcePrefixEntry{}), cmpopts.EquateEmpty()) { + t.Fatalf("NewFilterChainManager() returned %+v, want: %+v", gotFC, test.wantFC) + } + }) + } +} + +// TestNewFilterChainImpl_Success_AllCombinations verifies different +// combinations of the supported match criteria. +func (s) TestNewFilterChainImpl_Success_AllCombinations(t *testing.T) { + oldRBAC := envconfig.XDSRBAC + envconfig.XDSRBAC = true + defer func() { + envconfig.XDSRBAC = oldRBAC + }() + tests := []struct { + desc string + lis *v3listenerpb.Listener + wantFC *FilterChainManager + }{ + { + desc: "multiple destination prefixes", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + // Unspecified destination prefix. + FilterChainMatch: &v3listenerpb.FilterChainMatch{}, + Filters: emptyValidNetworkFilters, + }, + { + // v4 wildcard destination prefix. + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("0.0.0.0", 0)}, + SourceType: v3listenerpb.FilterChainMatch_EXTERNAL, + }, + Filters: emptyValidNetworkFilters, + }, + { + // v6 wildcard destination prefix. + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("::", 0)}, + SourceType: v3listenerpb.FilterChainMatch_EXTERNAL, + }, + Filters: emptyValidNetworkFilters, + }, + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16)}}, + Filters: emptyValidNetworkFilters, + }, + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("10.0.0.0", 8)}}, + Filters: emptyValidNetworkFilters, + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{Filters: emptyValidNetworkFilters}, + }, + wantFC: &FilterChainManager{ + dstPrefixMap: map[string]*destPrefixEntry{ + unspecifiedPrefixMapKey: { + srcTypeArr: [3]*sourcePrefixes{ + { + srcPrefixMap: map[string]*sourcePrefixEntry{ + unspecifiedPrefixMapKey: { + srcPortMap: map[int]*FilterChain{ + 0: { + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + }, + }, + }, + }, + }, + }, + "0.0.0.0/0": { + net: ipNetFromCIDR("0.0.0.0/0"), + srcTypeArr: [3]*sourcePrefixes{ + nil, + nil, + { + srcPrefixMap: map[string]*sourcePrefixEntry{ + unspecifiedPrefixMapKey: { + srcPortMap: map[int]*FilterChain{ + 0: { + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + }, + }, + }, + }, + }, + }, + "::/0": { + net: ipNetFromCIDR("::/0"), + srcTypeArr: [3]*sourcePrefixes{ + nil, + nil, + { + srcPrefixMap: map[string]*sourcePrefixEntry{ + unspecifiedPrefixMapKey: { + srcPortMap: map[int]*FilterChain{ + 0: { + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + }, + }, + }, + }, + }, + }, + "192.168.0.0/16": { + net: ipNetFromCIDR("192.168.2.2/16"), + srcTypeArr: [3]*sourcePrefixes{ + { + srcPrefixMap: map[string]*sourcePrefixEntry{ + unspecifiedPrefixMapKey: { + srcPortMap: map[int]*FilterChain{ + 0: { + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + }, + }, + }, + }, + }, + }, + "10.0.0.0/8": { + net: ipNetFromCIDR("10.0.0.0/8"), + srcTypeArr: [3]*sourcePrefixes{ + { + srcPrefixMap: map[string]*sourcePrefixEntry{ + unspecifiedPrefixMapKey: { + srcPortMap: map[int]*FilterChain{ + 0: { + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + }, + }, + }, + }, + }, + }, + }, + def: &FilterChain{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + }, + }, + { + desc: "multiple source types", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK}, + Filters: emptyValidNetworkFilters, + }, + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16)}, + SourceType: v3listenerpb.FilterChainMatch_EXTERNAL, + }, + Filters: emptyValidNetworkFilters, + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{Filters: emptyValidNetworkFilters}, + }, + wantFC: &FilterChainManager{ + dstPrefixMap: map[string]*destPrefixEntry{ + unspecifiedPrefixMapKey: { + srcTypeArr: [3]*sourcePrefixes{ + nil, + { + srcPrefixMap: map[string]*sourcePrefixEntry{ + unspecifiedPrefixMapKey: { + srcPortMap: map[int]*FilterChain{ + 0: { + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + }, + }, + }, + }, + }, + }, + "192.168.0.0/16": { + net: ipNetFromCIDR("192.168.2.2/16"), + srcTypeArr: [3]*sourcePrefixes{ + nil, + nil, + { + srcPrefixMap: map[string]*sourcePrefixEntry{ + unspecifiedPrefixMapKey: { + srcPortMap: map[int]*FilterChain{ + 0: { + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + }, + }, + }, + }, + }, + }, + }, + def: &FilterChain{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + }, + }, + { + desc: "multiple source prefixes", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{SourcePrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("10.0.0.0", 8)}}, + Filters: emptyValidNetworkFilters, + }, + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16)}, + SourcePrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16)}, + }, + Filters: emptyValidNetworkFilters, + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{Filters: emptyValidNetworkFilters}, + }, + wantFC: &FilterChainManager{ + dstPrefixMap: map[string]*destPrefixEntry{ + unspecifiedPrefixMapKey: { + srcTypeArr: [3]*sourcePrefixes{ + { + srcPrefixMap: map[string]*sourcePrefixEntry{ + "10.0.0.0/8": { + net: ipNetFromCIDR("10.0.0.0/8"), + srcPortMap: map[int]*FilterChain{ + 0: { + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + }, + }, + }, + }, + }, + }, + "192.168.0.0/16": { + net: ipNetFromCIDR("192.168.2.2/16"), + srcTypeArr: [3]*sourcePrefixes{ + { + srcPrefixMap: map[string]*sourcePrefixEntry{ + "192.168.0.0/16": { + net: ipNetFromCIDR("192.168.0.0/16"), + srcPortMap: map[int]*FilterChain{ + 0: { + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + }, + }, + }, + }, + }, + }, + }, + def: &FilterChain{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + }, + }, + { + desc: "multiple source ports", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{SourcePorts: []uint32{1, 2, 3}}, + Filters: emptyValidNetworkFilters, + }, + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16)}, + SourcePrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16)}, + SourceType: v3listenerpb.FilterChainMatch_EXTERNAL, + SourcePorts: []uint32{1, 2, 3}, + }, + Filters: emptyValidNetworkFilters, + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{Filters: emptyValidNetworkFilters}, + }, + wantFC: &FilterChainManager{ + dstPrefixMap: map[string]*destPrefixEntry{ + unspecifiedPrefixMapKey: { + srcTypeArr: [3]*sourcePrefixes{ + { + srcPrefixMap: map[string]*sourcePrefixEntry{ + unspecifiedPrefixMapKey: { + srcPortMap: map[int]*FilterChain{ + 1: { + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + 2: { + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + 3: { + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + }, + }, + }, + }, + }, + }, + "192.168.0.0/16": { + net: ipNetFromCIDR("192.168.2.2/16"), + srcTypeArr: [3]*sourcePrefixes{ + nil, + nil, + { + srcPrefixMap: map[string]*sourcePrefixEntry{ + "192.168.0.0/16": { + net: ipNetFromCIDR("192.168.0.0/16"), + srcPortMap: map[int]*FilterChain{ + 1: { + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + 2: { + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + 3: { + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + }, + }, + }, + }, + }, + }, + }, + def: &FilterChain{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + }, + }, + { + desc: "some chains have unsupported fields", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{}, + Filters: emptyValidNetworkFilters, + }, + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16)}}, + Filters: emptyValidNetworkFilters, + }, + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("10.0.0.0", 8)}, + TransportProtocol: "raw_buffer", + }, + Filters: emptyValidNetworkFilters, + }, + { + // This chain will be dropped in favor of the above + // filter chain because they both have the same + // destination prefix, but this one has an empty + // transport protocol while the above chain has the more + // preferred "raw_buffer". + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("10.0.0.0", 8)}, + TransportProtocol: "", + SourceType: v3listenerpb.FilterChainMatch_EXTERNAL, + SourcePrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("10.0.0.0", 16)}, + }, + Filters: emptyValidNetworkFilters, + }, + { + // This chain will be dropped for unsupported server + // names. + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.100.1", 32)}, + ServerNames: []string{"foo", "bar"}, + }, + Filters: emptyValidNetworkFilters, + }, + { + // This chain will be dropped for unsupported transport + // protocol. + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.100.2", 32)}, + TransportProtocol: "not-raw-buffer", + }, + Filters: emptyValidNetworkFilters, + }, + { + // This chain will be dropped for unsupported + // application protocol. + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.100.3", 32)}, + ApplicationProtocols: []string{"h2"}, + }, + Filters: emptyValidNetworkFilters, + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{Filters: emptyValidNetworkFilters}, + }, + wantFC: &FilterChainManager{ + dstPrefixMap: map[string]*destPrefixEntry{ + unspecifiedPrefixMapKey: { + srcTypeArr: [3]*sourcePrefixes{ + { + srcPrefixMap: map[string]*sourcePrefixEntry{ + unspecifiedPrefixMapKey: { + srcPortMap: map[int]*FilterChain{ + 0: { + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + }, + }, + }, + }, + }, + }, + "192.168.0.0/16": { + net: ipNetFromCIDR("192.168.2.2/16"), + srcTypeArr: [3]*sourcePrefixes{ + { + srcPrefixMap: map[string]*sourcePrefixEntry{ + unspecifiedPrefixMapKey: { + srcPortMap: map[int]*FilterChain{ + 0: { + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + }, + }, + }, + }, + }, + }, + "10.0.0.0/8": { + net: ipNetFromCIDR("10.0.0.0/8"), + srcTypeArr: [3]*sourcePrefixes{ + { + srcPrefixMap: map[string]*sourcePrefixEntry{ + unspecifiedPrefixMapKey: { + srcPortMap: map[int]*FilterChain{ + 0: { + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + }, + }, + }, + }, + }, + }, + "192.168.100.1/32": { + net: ipNetFromCIDR("192.168.100.1/32"), + srcTypeArr: [3]*sourcePrefixes{}, + }, + "192.168.100.2/32": { + net: ipNetFromCIDR("192.168.100.2/32"), + srcTypeArr: [3]*sourcePrefixes{}, + }, + "192.168.100.3/32": { + net: ipNetFromCIDR("192.168.100.3/32"), + srcTypeArr: [3]*sourcePrefixes{}, + }, + }, + def: &FilterChain{ + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + gotFC, err := NewFilterChainManager(test.lis) + if err != nil { + t.Fatalf("NewFilterChainManager() returned err: %v, wantErr: nil", err) + } + if !cmp.Equal(gotFC, test.wantFC, cmp.AllowUnexported(FilterChainManager{}, destPrefixEntry{}, sourcePrefixes{}, sourcePrefixEntry{})) { + t.Fatalf("NewFilterChainManager() returned %+v, want: %+v", gotFC, test.wantFC) + } + }) + } +} + +func (s) TestLookup_Failures(t *testing.T) { + tests := []struct { + desc string + lis *v3listenerpb.Listener + params FilterChainLookupParams + wantErr string + }{ + { + desc: "no destination prefix match", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16)}}, + Filters: emptyValidNetworkFilters, + }, + }, + }, + params: FilterChainLookupParams{ + IsUnspecifiedListener: true, + DestAddr: net.IPv4(10, 1, 1, 1), + }, + wantErr: "no matching filter chain based on destination prefix match", + }, + { + desc: "no source type match", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16)}, + SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK, + }, + Filters: emptyValidNetworkFilters, + }, + }, + }, + params: FilterChainLookupParams{ + IsUnspecifiedListener: true, + DestAddr: net.IPv4(192, 168, 100, 1), + SourceAddr: net.IPv4(192, 168, 100, 2), + }, + wantErr: "no matching filter chain based on source type match", + }, + { + desc: "no source prefix match", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + SourcePrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 24)}, + SourceType: v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK, + }, + Filters: emptyValidNetworkFilters, + }, + }, + }, + params: FilterChainLookupParams{ + IsUnspecifiedListener: true, + DestAddr: net.IPv4(192, 168, 100, 1), + SourceAddr: net.IPv4(192, 168, 100, 1), + }, + wantErr: "no matching filter chain after all match criteria", + }, + { + desc: "multiple matching filter chains", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{SourcePorts: []uint32{1, 2, 3}}, + Filters: emptyValidNetworkFilters, + }, + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16)}, + SourcePorts: []uint32{1}, + }, + Filters: emptyValidNetworkFilters, + }, + }, + }, + params: FilterChainLookupParams{ + // IsUnspecified is not set. This means that the destination + // prefix matchers will be ignored. + DestAddr: net.IPv4(192, 168, 100, 1), + SourceAddr: net.IPv4(192, 168, 100, 1), + SourcePort: 1, + }, + wantErr: "multiple matching filter chains", + }, + { + desc: "no default filter chain", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{SourcePorts: []uint32{1, 2, 3}}, + Filters: emptyValidNetworkFilters, + }, + }, + }, + params: FilterChainLookupParams{ + IsUnspecifiedListener: true, + DestAddr: net.IPv4(192, 168, 100, 1), + SourceAddr: net.IPv4(192, 168, 100, 1), + SourcePort: 80, + }, + wantErr: "no matching filter chain after all match criteria", + }, + { + desc: "most specific match dropped for unsupported field", + lis: &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + // This chain will be picked in the destination prefix + // stage, but will be dropped at the server names stage. + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.100.1", 32)}, + ServerNames: []string{"foo"}, + }, + Filters: emptyValidNetworkFilters, + }, + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.100.0", 16)}, + }, + Filters: emptyValidNetworkFilters, + }, + }, + }, + params: FilterChainLookupParams{ + IsUnspecifiedListener: true, + DestAddr: net.IPv4(192, 168, 100, 1), + SourceAddr: net.IPv4(192, 168, 100, 1), + SourcePort: 80, + }, + wantErr: "no matching filter chain based on source type match", + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + fci, err := NewFilterChainManager(test.lis) + if err != nil { + t.Fatalf("NewFilterChainManager() failed: %v", err) + } + fc, err := fci.Lookup(test.params) + if err == nil || !strings.Contains(err.Error(), test.wantErr) { + t.Fatalf("FilterChainManager.Lookup(%v) = (%v, %v) want (nil, %s)", test.params, fc, err, test.wantErr) + } + }) + } +} + +func (s) TestLookup_Successes(t *testing.T) { + oldRBAC := envconfig.XDSRBAC + envconfig.XDSRBAC = true + defer func() { + envconfig.XDSRBAC = oldRBAC + }() + lisWithDefaultChain := &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16)}}, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{InstanceName: "instance1"}, + }, + }), + }, + }, + Filters: emptyValidNetworkFilters, + }, + }, + // A default filter chain with an empty transport socket. + DefaultFilterChain: &v3listenerpb.FilterChain{ + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{InstanceName: "default"}, + }, + }), + }, + }, + Filters: emptyValidNetworkFilters, + }, + } + lisWithoutDefaultChain := &v3listenerpb.Listener{ + FilterChains: []*v3listenerpb.FilterChain{ + { + TransportSocket: transportSocketWithInstanceName("unspecified-dest-and-source-prefix"), + Filters: emptyValidNetworkFilters, + }, + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("0.0.0.0", 0)}, + SourcePrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("0.0.0.0", 0)}, + }, + TransportSocket: transportSocketWithInstanceName("wildcard-prefixes-v4"), + Filters: emptyValidNetworkFilters, + }, + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + SourcePrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("::", 0)}, + }, + TransportSocket: transportSocketWithInstanceName("wildcard-source-prefix-v6"), + Filters: emptyValidNetworkFilters, + }, + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 16)}}, + TransportSocket: transportSocketWithInstanceName("specific-destination-prefix-unspecified-source-type"), + Filters: emptyValidNetworkFilters, + }, + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 24)}, + SourceType: v3listenerpb.FilterChainMatch_EXTERNAL, + }, + TransportSocket: transportSocketWithInstanceName("specific-destination-prefix-specific-source-type"), + Filters: emptyValidNetworkFilters, + }, + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 24)}, + SourcePrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.92.1", 24)}, + SourceType: v3listenerpb.FilterChainMatch_EXTERNAL, + }, + TransportSocket: transportSocketWithInstanceName("specific-destination-prefix-specific-source-type-specific-source-prefix"), + Filters: emptyValidNetworkFilters, + }, + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{ + PrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.1.1", 24)}, + SourcePrefixRanges: []*v3corepb.CidrRange{cidrRangeFromAddressAndPrefixLen("192.168.92.1", 24)}, + SourceType: v3listenerpb.FilterChainMatch_EXTERNAL, + SourcePorts: []uint32{80}, + }, + TransportSocket: transportSocketWithInstanceName("specific-destination-prefix-specific-source-type-specific-source-prefix-specific-source-port"), + Filters: emptyValidNetworkFilters, + }, + }, + } + + tests := []struct { + desc string + lis *v3listenerpb.Listener + params FilterChainLookupParams + wantFC *FilterChain + }{ + { + desc: "default filter chain", + lis: lisWithDefaultChain, + params: FilterChainLookupParams{ + IsUnspecifiedListener: true, + DestAddr: net.IPv4(10, 1, 1, 1), + }, + wantFC: &FilterChain{ + SecurityCfg: &SecurityConfig{IdentityInstanceName: "default"}, + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + }, + { + desc: "unspecified destination match", + lis: lisWithoutDefaultChain, + params: FilterChainLookupParams{ + IsUnspecifiedListener: true, + DestAddr: net.ParseIP("2001:68::db8"), + SourceAddr: net.IPv4(10, 1, 1, 1), + SourcePort: 1, + }, + wantFC: &FilterChain{ + SecurityCfg: &SecurityConfig{IdentityInstanceName: "unspecified-dest-and-source-prefix"}, + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + }, + { + desc: "wildcard destination match v4", + lis: lisWithoutDefaultChain, + params: FilterChainLookupParams{ + IsUnspecifiedListener: true, + DestAddr: net.IPv4(10, 1, 1, 1), + SourceAddr: net.IPv4(10, 1, 1, 1), + SourcePort: 1, + }, + wantFC: &FilterChain{ + SecurityCfg: &SecurityConfig{IdentityInstanceName: "wildcard-prefixes-v4"}, + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + }, + { + desc: "wildcard source match v6", + lis: lisWithoutDefaultChain, + params: FilterChainLookupParams{ + IsUnspecifiedListener: true, + DestAddr: net.ParseIP("2001:68::1"), + SourceAddr: net.ParseIP("2001:68::2"), + SourcePort: 1, + }, + wantFC: &FilterChain{ + SecurityCfg: &SecurityConfig{IdentityInstanceName: "wildcard-source-prefix-v6"}, + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + }, + { + desc: "specific destination and wildcard source type match", + lis: lisWithoutDefaultChain, + params: FilterChainLookupParams{ + IsUnspecifiedListener: true, + DestAddr: net.IPv4(192, 168, 100, 1), + SourceAddr: net.IPv4(192, 168, 100, 1), + SourcePort: 80, + }, + wantFC: &FilterChain{ + SecurityCfg: &SecurityConfig{IdentityInstanceName: "specific-destination-prefix-unspecified-source-type"}, + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + }, + { + desc: "specific destination and source type match", + lis: lisWithoutDefaultChain, + params: FilterChainLookupParams{ + IsUnspecifiedListener: true, + DestAddr: net.IPv4(192, 168, 1, 1), + SourceAddr: net.IPv4(10, 1, 1, 1), + SourcePort: 80, + }, + wantFC: &FilterChain{ + SecurityCfg: &SecurityConfig{IdentityInstanceName: "specific-destination-prefix-specific-source-type"}, + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + }, + { + desc: "specific destination source type and source prefix", + lis: lisWithoutDefaultChain, + params: FilterChainLookupParams{ + IsUnspecifiedListener: true, + DestAddr: net.IPv4(192, 168, 1, 1), + SourceAddr: net.IPv4(192, 168, 92, 100), + SourcePort: 70, + }, + wantFC: &FilterChain{ + SecurityCfg: &SecurityConfig{IdentityInstanceName: "specific-destination-prefix-specific-source-type-specific-source-prefix"}, + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + }, + { + desc: "specific destination source type source prefix and source port", + lis: lisWithoutDefaultChain, + params: FilterChainLookupParams{ + IsUnspecifiedListener: true, + DestAddr: net.IPv4(192, 168, 1, 1), + SourceAddr: net.IPv4(192, 168, 92, 100), + SourcePort: 80, + }, + wantFC: &FilterChain{ + SecurityCfg: &SecurityConfig{IdentityInstanceName: "specific-destination-prefix-specific-source-type-specific-source-prefix-specific-source-port"}, + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + fci, err := NewFilterChainManager(test.lis) + if err != nil { + t.Fatalf("NewFilterChainManager() failed: %v", err) + } + gotFC, err := fci.Lookup(test.params) + if err != nil { + t.Fatalf("FilterChainManager.Lookup(%v) failed: %v", test.params, err) + } + if !cmp.Equal(gotFC, test.wantFC, cmpopts.EquateEmpty()) { + t.Fatalf("FilterChainManager.Lookup(%v) = %v, want %v", test.params, gotFC, test.wantFC) + } + }) + } +} + +type filterCfg struct { + httpfilter.FilterConfig + // Level is what differentiates top level filters ("top level") vs. second + // level ("virtual host level"), and third level ("route level"). + level string +} + +type filterBuilder struct { + httpfilter.Filter +} + +var _ httpfilter.ServerInterceptorBuilder = &filterBuilder{} + +func (fb *filterBuilder) BuildServerInterceptor(config httpfilter.FilterConfig, override httpfilter.FilterConfig) (iresolver.ServerInterceptor, error) { + var level string + level = config.(filterCfg).level + + if override != nil { + level = override.(filterCfg).level + } + return &serverInterceptor{level: level}, nil +} + +type serverInterceptor struct { + level string +} + +func (si *serverInterceptor) AllowRPC(context.Context) error { + return errors.New(si.level) +} + +func (s) TestHTTPFilterInstantiation(t *testing.T) { + tests := []struct { + name string + filters []HTTPFilter + routeConfig RouteConfigUpdate + // A list of strings which will be built from iterating through the + // filters ["top level", "vh level", "route level", "route level"...] + // wantErrs is the list of error strings that will be constructed from + // the deterministic iteration through the vh list and route list. The + // error string will be determined by the level of config that the + // filter builder receives (i.e. top level, vs. virtual host level vs. + // route level). + wantErrs []string + }{ + { + name: "one http filter no overrides", + filters: []HTTPFilter{ + {Name: "server-interceptor", Filter: &filterBuilder{}, Config: filterCfg{level: topLevel}}, + }, + routeConfig: RouteConfigUpdate{ + VirtualHosts: []*VirtualHost{ + { + Domains: []string{"target"}, + Routes: []*Route{{ + Prefix: newStringP("1"), + }, + }, + }, + }}, + wantErrs: []string{topLevel}, + }, + { + name: "one http filter vh override", + filters: []HTTPFilter{ + {Name: "server-interceptor", Filter: &filterBuilder{}, Config: filterCfg{level: topLevel}}, + }, + routeConfig: RouteConfigUpdate{ + VirtualHosts: []*VirtualHost{ + { + Domains: []string{"target"}, + Routes: []*Route{{ + Prefix: newStringP("1"), + }, + }, + HTTPFilterConfigOverride: map[string]httpfilter.FilterConfig{ + "server-interceptor": filterCfg{level: vhLevel}, + }, + }, + }}, + wantErrs: []string{vhLevel}, + }, + { + name: "one http filter route override", + filters: []HTTPFilter{ + {Name: "server-interceptor", Filter: &filterBuilder{}, Config: filterCfg{level: topLevel}}, + }, + routeConfig: RouteConfigUpdate{ + VirtualHosts: []*VirtualHost{ + { + Domains: []string{"target"}, + Routes: []*Route{{ + Prefix: newStringP("1"), + HTTPFilterConfigOverride: map[string]httpfilter.FilterConfig{ + "server-interceptor": filterCfg{level: rLevel}, + }, + }, + }, + }, + }}, + wantErrs: []string{rLevel}, + }, + // This tests the scenario where there are three http filters, and one + // gets overridden by route and one by virtual host. + { + name: "three http filters vh override route override", + filters: []HTTPFilter{ + {Name: "server-interceptor1", Filter: &filterBuilder{}, Config: filterCfg{level: topLevel}}, + {Name: "server-interceptor2", Filter: &filterBuilder{}, Config: filterCfg{level: topLevel}}, + {Name: "server-interceptor3", Filter: &filterBuilder{}, Config: filterCfg{level: topLevel}}, + }, + routeConfig: RouteConfigUpdate{ + VirtualHosts: []*VirtualHost{ + { + Domains: []string{"target"}, + Routes: []*Route{{ + Prefix: newStringP("1"), + HTTPFilterConfigOverride: map[string]httpfilter.FilterConfig{ + "server-interceptor3": filterCfg{level: rLevel}, + }, + }, + }, + HTTPFilterConfigOverride: map[string]httpfilter.FilterConfig{ + "server-interceptor2": filterCfg{level: vhLevel}, + }, + }, + }}, + wantErrs: []string{topLevel, vhLevel, rLevel}, + }, + // This tests the scenario where there are three http filters, and two + // virtual hosts with different vh + route overrides for each virtual + // host. + { + name: "three http filters two vh", + filters: []HTTPFilter{ + {Name: "server-interceptor1", Filter: &filterBuilder{}, Config: filterCfg{level: topLevel}}, + {Name: "server-interceptor2", Filter: &filterBuilder{}, Config: filterCfg{level: topLevel}}, + {Name: "server-interceptor3", Filter: &filterBuilder{}, Config: filterCfg{level: topLevel}}, + }, + routeConfig: RouteConfigUpdate{ + VirtualHosts: []*VirtualHost{ + { + Domains: []string{"target"}, + Routes: []*Route{{ + Prefix: newStringP("1"), + HTTPFilterConfigOverride: map[string]httpfilter.FilterConfig{ + "server-interceptor3": filterCfg{level: rLevel}, + }, + }, + }, + HTTPFilterConfigOverride: map[string]httpfilter.FilterConfig{ + "server-interceptor2": filterCfg{level: vhLevel}, + }, + }, + { + Domains: []string{"target"}, + Routes: []*Route{{ + Prefix: newStringP("1"), + HTTPFilterConfigOverride: map[string]httpfilter.FilterConfig{ + "server-interceptor1": filterCfg{level: rLevel}, + "server-interceptor2": filterCfg{level: rLevel}, + }, + }, + }, + HTTPFilterConfigOverride: map[string]httpfilter.FilterConfig{ + "server-interceptor2": filterCfg{level: vhLevel}, + "server-interceptor3": filterCfg{level: vhLevel}, + }, + }, + }}, + wantErrs: []string{topLevel, vhLevel, rLevel, rLevel, rLevel, vhLevel}, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fc := FilterChain{ + HTTPFilters: test.filters, + } + vhswi, err := fc.ConstructUsableRouteConfiguration(test.routeConfig) + if err != nil { + t.Fatalf("Error constructing usable route configuration: %v", err) + } + // Build out list of errors by iterating through the virtual hosts and routes, + // and running the filters in route configurations. + var errs []string + for _, vh := range vhswi { + for _, r := range vh.Routes { + for _, int := range r.Interceptors { + errs = append(errs, int.AllowRPC(context.Background()).Error()) + } + } + } + if !cmp.Equal(errs, test.wantErrs) { + t.Fatalf("List of errors %v, want %v", errs, test.wantErrs) + } + }) + } +} + +// The Equal() methods defined below help with using cmp.Equal() on these types +// which contain all unexported fields. + +func (fci *FilterChainManager) Equal(other *FilterChainManager) bool { + if (fci == nil) != (other == nil) { + return false + } + if fci == nil { + return true + } + switch { + case !cmp.Equal(fci.dstPrefixMap, other.dstPrefixMap, cmpopts.EquateEmpty()): + return false + // TODO: Support comparing dstPrefixes slice? + case !cmp.Equal(fci.def, other.def, cmpopts.EquateEmpty(), protocmp.Transform()): + return false + case !cmp.Equal(fci.RouteConfigNames, other.RouteConfigNames, cmpopts.EquateEmpty()): + return false + } + return true +} + +func (dpe *destPrefixEntry) Equal(other *destPrefixEntry) bool { + if (dpe == nil) != (other == nil) { + return false + } + if dpe == nil { + return true + } + if !cmp.Equal(dpe.net, other.net) { + return false + } + for i, st := range dpe.srcTypeArr { + if !cmp.Equal(st, other.srcTypeArr[i], cmpopts.EquateEmpty()) { + return false + } + } + return true +} + +func (sp *sourcePrefixes) Equal(other *sourcePrefixes) bool { + if (sp == nil) != (other == nil) { + return false + } + if sp == nil { + return true + } + // TODO: Support comparing srcPrefixes slice? + return cmp.Equal(sp.srcPrefixMap, other.srcPrefixMap, cmpopts.EquateEmpty()) +} + +func (spe *sourcePrefixEntry) Equal(other *sourcePrefixEntry) bool { + if (spe == nil) != (other == nil) { + return false + } + if spe == nil { + return true + } + switch { + case !cmp.Equal(spe.net, other.net): + return false + case !cmp.Equal(spe.srcPortMap, other.srcPortMap, cmpopts.EquateEmpty(), protocmp.Transform()): + return false + } + return true +} + +// The String() methods defined below help with debugging test failures as the +// regular %v or %+v formatting directives do not expands pointer fields inside +// structs, and these types have a lot of pointers pointing to other structs. +func (fci *FilterChainManager) String() string { + if fci == nil { + return "" + } + + var sb strings.Builder + if fci.dstPrefixMap != nil { + sb.WriteString("destination_prefix_map: map {\n") + for k, v := range fci.dstPrefixMap { + sb.WriteString(fmt.Sprintf("%q: %v\n", k, v)) + } + sb.WriteString("}\n") + } + if fci.dstPrefixes != nil { + sb.WriteString("destination_prefixes: [") + for _, p := range fci.dstPrefixes { + sb.WriteString(fmt.Sprintf("%v ", p)) + } + sb.WriteString("]") + } + if fci.def != nil { + sb.WriteString(fmt.Sprintf("default_filter_chain: %+v ", fci.def)) + } + return sb.String() +} + +func (dpe *destPrefixEntry) String() string { + if dpe == nil { + return "" + } + var sb strings.Builder + if dpe.net != nil { + sb.WriteString(fmt.Sprintf("destination_prefix: %s ", dpe.net.String())) + } + sb.WriteString("source_types_array: [") + for _, st := range dpe.srcTypeArr { + sb.WriteString(fmt.Sprintf("%v ", st)) + } + sb.WriteString("]") + return sb.String() +} + +func (sp *sourcePrefixes) String() string { + if sp == nil { + return "" + } + var sb strings.Builder + if sp.srcPrefixMap != nil { + sb.WriteString("source_prefix_map: map {") + for k, v := range sp.srcPrefixMap { + sb.WriteString(fmt.Sprintf("%q: %v ", k, v)) + } + sb.WriteString("}") + } + if sp.srcPrefixes != nil { + sb.WriteString("source_prefixes: [") + for _, p := range sp.srcPrefixes { + sb.WriteString(fmt.Sprintf("%v ", p)) + } + sb.WriteString("]") + } + return sb.String() +} + +func (spe *sourcePrefixEntry) String() string { + if spe == nil { + return "" + } + var sb strings.Builder + if spe.net != nil { + sb.WriteString(fmt.Sprintf("source_prefix: %s ", spe.net.String())) + } + if spe.srcPortMap != nil { + sb.WriteString("source_ports_map: map {") + for k, v := range spe.srcPortMap { + sb.WriteString(fmt.Sprintf("%d: %+v ", k, v)) + } + sb.WriteString("}") + } + return sb.String() +} + +func (f *FilterChain) String() string { + if f == nil || f.SecurityCfg == nil { + return "" + } + return fmt.Sprintf("security_config: %v", f.SecurityCfg) +} + +func ipNetFromCIDR(cidr string) *net.IPNet { + _, ipnet, err := net.ParseCIDR(cidr) + if err != nil { + panic(err) + } + return ipnet +} + +func transportSocketWithInstanceName(name string) *v3corepb.TransportSocket { + return &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{InstanceName: name}, + }, + }), + }, + } +} + +func cidrRangeFromAddressAndPrefixLen(address string, len int) *v3corepb.CidrRange { + return &v3corepb.CidrRange{ + AddressPrefix: address, + PrefixLen: &wrapperspb.UInt32Value{ + Value: uint32(len), + }, + } +} diff --git a/xds/internal/xdsclient/xdsresource/listener_resource_type.go b/xds/internal/xdsclient/xdsresource/listener_resource_type.go new file mode 100644 index 000000000000..0aff941389ec --- /dev/null +++ b/xds/internal/xdsclient/xdsresource/listener_resource_type.go @@ -0,0 +1,186 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "fmt" + + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/xds/internal/xdsclient/bootstrap" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource/version" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" +) + +const ( + // ListenerResourceTypeName represents the transport agnostic name for the + // listener resource. + ListenerResourceTypeName = "ListenerResource" +) + +var ( + // Compile time interface checks. + _ Type = listenerResourceType{} + + // Singleton instantiation of the resource type implementation. + listenerType = listenerResourceType{ + resourceTypeState: resourceTypeState{ + typeURL: version.V3ListenerURL, + typeName: ListenerResourceTypeName, + allResourcesRequiredInSotW: true, + }, + } +) + +// listenerResourceType provides the resource-type specific functionality for a +// Listener resource. +// +// Implements the Type interface. +type listenerResourceType struct { + resourceTypeState +} + +func securityConfigValidator(bc *bootstrap.Config, sc *SecurityConfig) error { + if sc == nil { + return nil + } + if sc.IdentityInstanceName != "" { + if _, ok := bc.CertProviderConfigs[sc.IdentityInstanceName]; !ok { + return fmt.Errorf("identitiy certificate provider instance name %q missing in bootstrap configuration", sc.IdentityInstanceName) + } + } + if sc.RootInstanceName != "" { + if _, ok := bc.CertProviderConfigs[sc.RootInstanceName]; !ok { + return fmt.Errorf("root certificate provider instance name %q missing in bootstrap configuration", sc.RootInstanceName) + } + } + return nil +} + +func listenerValidator(bc *bootstrap.Config, lis ListenerUpdate) error { + if lis.InboundListenerCfg == nil || lis.InboundListenerCfg.FilterChains == nil { + return nil + } + return lis.InboundListenerCfg.FilterChains.Validate(func(fc *FilterChain) error { + if fc == nil { + return nil + } + return securityConfigValidator(bc, fc.SecurityCfg) + }) +} + +// Decode deserializes and validates an xDS resource serialized inside the +// provided `Any` proto, as received from the xDS management server. +func (listenerResourceType) Decode(opts *DecodeOptions, resource *anypb.Any) (*DecodeResult, error) { + name, listener, err := unmarshalListenerResource(resource) + switch { + case name == "": + // Name is unset only when protobuf deserialization fails. + return nil, err + case err != nil: + // Protobuf deserialization succeeded, but resource validation failed. + return &DecodeResult{Name: name, Resource: &ListenerResourceData{Resource: ListenerUpdate{}}}, err + } + + // Perform extra validation here. + if err := listenerValidator(opts.BootstrapConfig, listener); err != nil { + return &DecodeResult{Name: name, Resource: &ListenerResourceData{Resource: ListenerUpdate{}}}, err + } + + return &DecodeResult{Name: name, Resource: &ListenerResourceData{Resource: listener}}, nil + +} + +// ListenerResourceData wraps the configuration of a Listener resource as +// received from the management server. +// +// Implements the ResourceData interface. +type ListenerResourceData struct { + ResourceData + + // TODO: We have always stored update structs by value. See if this can be + // switched to a pointer? + Resource ListenerUpdate +} + +// Equal returns true if other is equal to l. +func (l *ListenerResourceData) Equal(other ResourceData) bool { + if l == nil && other == nil { + return true + } + if (l == nil) != (other == nil) { + return false + } + return proto.Equal(l.Resource.Raw, other.Raw()) + +} + +// ToJSON returns a JSON string representation of the resource data. +func (l *ListenerResourceData) ToJSON() string { + return pretty.ToJSON(l.Resource) +} + +// Raw returns the underlying raw protobuf form of the listener resource. +func (l *ListenerResourceData) Raw() *anypb.Any { + return l.Resource.Raw +} + +// ListenerWatcher wraps the callbacks to be invoked for different +// events corresponding to the listener resource being watched. +type ListenerWatcher interface { + // OnUpdate is invoked to report an update for the resource being watched. + OnUpdate(*ListenerResourceData) + + // OnError is invoked under different error conditions including but not + // limited to the following: + // - authority mentioned in the resource is not found + // - resource name parsing error + // - resource deserialization error + // - resource validation error + // - ADS stream failure + // - connection failure + OnError(error) + + // OnResourceDoesNotExist is invoked for a specific error condition where + // the requested resource is not found on the xDS management server. + OnResourceDoesNotExist() +} + +type delegatingListenerWatcher struct { + watcher ListenerWatcher +} + +func (d *delegatingListenerWatcher) OnUpdate(data ResourceData) { + l := data.(*ListenerResourceData) + d.watcher.OnUpdate(l) +} + +func (d *delegatingListenerWatcher) OnError(err error) { + d.watcher.OnError(err) +} + +func (d *delegatingListenerWatcher) OnResourceDoesNotExist() { + d.watcher.OnResourceDoesNotExist() +} + +// WatchListener uses xDS to discover the configuration associated with the +// provided listener resource name. +func WatchListener(p Producer, name string, w ListenerWatcher) (cancel func()) { + delegator := &delegatingListenerWatcher{watcher: w} + return p.WatchResource(listenerType, name, delegator) +} diff --git a/xds/internal/xdsclient/xdsresource/logging.go b/xds/internal/xdsclient/xdsresource/logging.go new file mode 100644 index 000000000000..62bcb016ba25 --- /dev/null +++ b/xds/internal/xdsclient/xdsresource/logging.go @@ -0,0 +1,28 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xdsresource + +import ( + "google.golang.org/grpc/grpclog" + internalgrpclog "google.golang.org/grpc/internal/grpclog" +) + +const prefix = "[xds-resource] " + +var logger = internalgrpclog.NewPrefixLogger(grpclog.Component("xds"), prefix) diff --git a/xds/internal/xdsclient/xdsresource/matcher.go b/xds/internal/xdsclient/xdsresource/matcher.go new file mode 100644 index 000000000000..77aa85b68e58 --- /dev/null +++ b/xds/internal/xdsclient/xdsresource/matcher.go @@ -0,0 +1,280 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "fmt" + "strings" + + "google.golang.org/grpc/internal/grpcrand" + "google.golang.org/grpc/internal/grpcutil" + iresolver "google.golang.org/grpc/internal/resolver" + "google.golang.org/grpc/internal/xds/matcher" + "google.golang.org/grpc/metadata" +) + +// RouteToMatcher converts a route to a Matcher to match incoming RPC's against. +func RouteToMatcher(r *Route) (*CompositeMatcher, error) { + var pm pathMatcher + switch { + case r.Regex != nil: + pm = newPathRegexMatcher(r.Regex) + case r.Path != nil: + pm = newPathExactMatcher(*r.Path, r.CaseInsensitive) + case r.Prefix != nil: + pm = newPathPrefixMatcher(*r.Prefix, r.CaseInsensitive) + default: + return nil, fmt.Errorf("illegal route: missing path_matcher") + } + + headerMatchers := make([]matcher.HeaderMatcher, 0, len(r.Headers)) + for _, h := range r.Headers { + var matcherT matcher.HeaderMatcher + invert := h.InvertMatch != nil && *h.InvertMatch + switch { + case h.ExactMatch != nil && *h.ExactMatch != "": + matcherT = matcher.NewHeaderExactMatcher(h.Name, *h.ExactMatch, invert) + case h.RegexMatch != nil: + matcherT = matcher.NewHeaderRegexMatcher(h.Name, h.RegexMatch, invert) + case h.PrefixMatch != nil && *h.PrefixMatch != "": + matcherT = matcher.NewHeaderPrefixMatcher(h.Name, *h.PrefixMatch, invert) + case h.SuffixMatch != nil && *h.SuffixMatch != "": + matcherT = matcher.NewHeaderSuffixMatcher(h.Name, *h.SuffixMatch, invert) + case h.RangeMatch != nil: + matcherT = matcher.NewHeaderRangeMatcher(h.Name, h.RangeMatch.Start, h.RangeMatch.End, invert) + case h.PresentMatch != nil: + matcherT = matcher.NewHeaderPresentMatcher(h.Name, *h.PresentMatch, invert) + case h.StringMatch != nil: + matcherT = matcher.NewHeaderStringMatcher(h.Name, *h.StringMatch, invert) + default: + return nil, fmt.Errorf("illegal route: missing header_match_specifier") + } + headerMatchers = append(headerMatchers, matcherT) + } + + var fractionMatcher *fractionMatcher + if r.Fraction != nil { + fractionMatcher = newFractionMatcher(*r.Fraction) + } + return newCompositeMatcher(pm, headerMatchers, fractionMatcher), nil +} + +// CompositeMatcher is a matcher that holds onto many matchers and aggregates +// the matching results. +type CompositeMatcher struct { + pm pathMatcher + hms []matcher.HeaderMatcher + fm *fractionMatcher +} + +func newCompositeMatcher(pm pathMatcher, hms []matcher.HeaderMatcher, fm *fractionMatcher) *CompositeMatcher { + return &CompositeMatcher{pm: pm, hms: hms, fm: fm} +} + +// Match returns true if all matchers return true. +func (a *CompositeMatcher) Match(info iresolver.RPCInfo) bool { + if a.pm != nil && !a.pm.match(info.Method) { + return false + } + + // Call headerMatchers even if md is nil, because routes may match + // non-presence of some headers. + var md metadata.MD + if info.Context != nil { + md, _ = metadata.FromOutgoingContext(info.Context) + if extraMD, ok := grpcutil.ExtraMetadata(info.Context); ok { + md = metadata.Join(md, extraMD) + // Remove all binary headers. They are hard to match with. May need + // to add back if asked by users. + for k := range md { + if strings.HasSuffix(k, "-bin") { + delete(md, k) + } + } + } + } + for _, m := range a.hms { + if !m.Match(md) { + return false + } + } + + if a.fm != nil && !a.fm.match() { + return false + } + return true +} + +func (a *CompositeMatcher) String() string { + var ret string + if a.pm != nil { + ret += a.pm.String() + } + for _, m := range a.hms { + ret += m.String() + } + if a.fm != nil { + ret += a.fm.String() + } + return ret +} + +type fractionMatcher struct { + fraction int64 // real fraction is fraction/1,000,000. +} + +func newFractionMatcher(fraction uint32) *fractionMatcher { + return &fractionMatcher{fraction: int64(fraction)} +} + +// RandInt63n overwrites grpcrand for control in tests. +var RandInt63n = grpcrand.Int63n + +func (fm *fractionMatcher) match() bool { + t := RandInt63n(1000000) + return t <= fm.fraction +} + +func (fm *fractionMatcher) String() string { + return fmt.Sprintf("fraction:%v", fm.fraction) +} + +type domainMatchType int + +const ( + domainMatchTypeInvalid domainMatchType = iota + domainMatchTypeUniversal + domainMatchTypePrefix + domainMatchTypeSuffix + domainMatchTypeExact +) + +// Exact > Suffix > Prefix > Universal > Invalid. +func (t domainMatchType) betterThan(b domainMatchType) bool { + return t > b +} + +func matchTypeForDomain(d string) domainMatchType { + if d == "" { + return domainMatchTypeInvalid + } + if d == "*" { + return domainMatchTypeUniversal + } + if strings.HasPrefix(d, "*") { + return domainMatchTypeSuffix + } + if strings.HasSuffix(d, "*") { + return domainMatchTypePrefix + } + if strings.Contains(d, "*") { + return domainMatchTypeInvalid + } + return domainMatchTypeExact +} + +func match(domain, host string) (domainMatchType, bool) { + switch typ := matchTypeForDomain(domain); typ { + case domainMatchTypeInvalid: + return typ, false + case domainMatchTypeUniversal: + return typ, true + case domainMatchTypePrefix: + // abc.* + return typ, strings.HasPrefix(host, strings.TrimSuffix(domain, "*")) + case domainMatchTypeSuffix: + // *.123 + return typ, strings.HasSuffix(host, strings.TrimPrefix(domain, "*")) + case domainMatchTypeExact: + return typ, domain == host + default: + return domainMatchTypeInvalid, false + } +} + +// FindBestMatchingVirtualHost returns the virtual host whose domains field best +// matches host +// +// The domains field support 4 different matching pattern types: +// +// - Exact match +// - Suffix match (e.g. “*ABC”) +// - Prefix match (e.g. “ABC*) +// - Universal match (e.g. “*”) +// +// The best match is defined as: +// - A match is better if it’s matching pattern type is better. +// * Exact match > suffix match > prefix match > universal match. +// +// - If two matches are of the same pattern type, the longer match is +// better. +// * This is to compare the length of the matching pattern, e.g. “*ABCDE” > +// “*ABC” +func FindBestMatchingVirtualHost(host string, vHosts []*VirtualHost) *VirtualHost { // Maybe move this crap to client + var ( + matchVh *VirtualHost + matchType = domainMatchTypeInvalid + matchLen int + ) + for _, vh := range vHosts { + for _, domain := range vh.Domains { + typ, matched := match(domain, host) + if typ == domainMatchTypeInvalid { + // The rds response is invalid. + return nil + } + if matchType.betterThan(typ) || matchType == typ && matchLen >= len(domain) || !matched { + // The previous match has better type, or the previous match has + // better length, or this domain isn't a match. + continue + } + matchVh = vh + matchType = typ + matchLen = len(domain) + } + } + return matchVh +} + +// FindBestMatchingVirtualHostServer returns the virtual host whose domains field best +// matches authority. +func FindBestMatchingVirtualHostServer(authority string, vHosts []VirtualHostWithInterceptors) *VirtualHostWithInterceptors { + var ( + matchVh *VirtualHostWithInterceptors + matchType = domainMatchTypeInvalid + matchLen int + ) + for _, vh := range vHosts { + for _, domain := range vh.Domains { + typ, matched := match(domain, authority) + if typ == domainMatchTypeInvalid { + // The rds response is invalid. + return nil + } + if matchType.betterThan(typ) || matchType == typ && matchLen >= len(domain) || !matched { + // The previous match has better type, or the previous match has + // better length, or this domain isn't a match. + continue + } + matchVh = &vh + matchType = typ + matchLen = len(domain) + } + } + return matchVh +} diff --git a/xds/internal/balancer/xdsrouting/matcher_path.go b/xds/internal/xdsclient/xdsresource/matcher_path.go similarity index 59% rename from xds/internal/balancer/xdsrouting/matcher_path.go rename to xds/internal/xdsclient/xdsresource/matcher_path.go index 9c783acbb577..da487e20c58e 100644 --- a/xds/internal/balancer/xdsrouting/matcher_path.go +++ b/xds/internal/xdsclient/xdsresource/matcher_path.go @@ -13,40 +13,44 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ -package xdsrouting +package xdsresource import ( "regexp" "strings" + + "google.golang.org/grpc/internal/grpcutil" ) -type pathMatcherInterface interface { +type pathMatcher interface { match(path string) bool - equal(pathMatcherInterface) bool String() string } type pathExactMatcher struct { - fullPath string + // fullPath is all upper case if caseInsensitive is true. + fullPath string + caseInsensitive bool } -func newPathExactMatcher(p string) *pathExactMatcher { - return &pathExactMatcher{fullPath: p} +func newPathExactMatcher(p string, caseInsensitive bool) *pathExactMatcher { + ret := &pathExactMatcher{ + fullPath: p, + caseInsensitive: caseInsensitive, + } + if caseInsensitive { + ret.fullPath = strings.ToUpper(p) + } + return ret } func (pem *pathExactMatcher) match(path string) bool { - return pem.fullPath == path -} - -func (pem *pathExactMatcher) equal(m pathMatcherInterface) bool { - mm, ok := m.(*pathExactMatcher) - if !ok { - return false + if pem.caseInsensitive { + return pem.fullPath == strings.ToUpper(path) } - return pem.fullPath == mm.fullPath + return pem.fullPath == path } func (pem *pathExactMatcher) String() string { @@ -54,23 +58,27 @@ func (pem *pathExactMatcher) String() string { } type pathPrefixMatcher struct { - prefix string + // prefix is all upper case if caseInsensitive is true. + prefix string + caseInsensitive bool } -func newPathPrefixMatcher(p string) *pathPrefixMatcher { - return &pathPrefixMatcher{prefix: p} +func newPathPrefixMatcher(p string, caseInsensitive bool) *pathPrefixMatcher { + ret := &pathPrefixMatcher{ + prefix: p, + caseInsensitive: caseInsensitive, + } + if caseInsensitive { + ret.prefix = strings.ToUpper(p) + } + return ret } func (ppm *pathPrefixMatcher) match(path string) bool { - return strings.HasPrefix(path, ppm.prefix) -} - -func (ppm *pathPrefixMatcher) equal(m pathMatcherInterface) bool { - mm, ok := m.(*pathPrefixMatcher) - if !ok { - return false + if ppm.caseInsensitive { + return strings.HasPrefix(strings.ToUpper(path), ppm.prefix) } - return ppm.prefix == mm.prefix + return strings.HasPrefix(path, ppm.prefix) } func (ppm *pathPrefixMatcher) String() string { @@ -86,15 +94,7 @@ func newPathRegexMatcher(re *regexp.Regexp) *pathRegexMatcher { } func (prm *pathRegexMatcher) match(path string) bool { - return prm.re.MatchString(path) -} - -func (prm *pathRegexMatcher) equal(m pathMatcherInterface) bool { - mm, ok := m.(*pathRegexMatcher) - if !ok { - return false - } - return prm.re.String() == mm.re.String() + return grpcutil.FullMatchWithRegex(prm.re, path) } func (prm *pathRegexMatcher) String() string { diff --git a/xds/internal/balancer/xdsrouting/matcher_path_test.go b/xds/internal/xdsclient/xdsresource/matcher_path_test.go similarity index 58% rename from xds/internal/balancer/xdsrouting/matcher_path_test.go rename to xds/internal/xdsclient/xdsresource/matcher_path_test.go index ad7db7481060..507cf15bed85 100644 --- a/xds/internal/balancer/xdsrouting/matcher_path_test.go +++ b/xds/internal/xdsclient/xdsresource/matcher_path_test.go @@ -13,29 +13,32 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ -package xdsrouting +package xdsresource import ( "regexp" "testing" ) -func TestPathFullMatcherMatch(t *testing.T) { +func (s) TestPathFullMatcherMatch(t *testing.T) { tests := []struct { - name string - fullPath string - path string - want bool + name string + fullPath string + caseInsensitive bool + path string + want bool }{ {name: "match", fullPath: "/s/m", path: "/s/m", want: true}, + {name: "case insensitive match", fullPath: "/s/m", caseInsensitive: true, path: "/S/m", want: true}, + {name: "case insensitive match 2", fullPath: "/s/M", caseInsensitive: true, path: "/S/m", want: true}, {name: "not match", fullPath: "/s/m", path: "/a/b", want: false}, + {name: "case insensitive not match", fullPath: "/s/m", caseInsensitive: true, path: "/a/b", want: false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - fpm := newPathExactMatcher(tt.fullPath) + fpm := newPathExactMatcher(tt.fullPath, tt.caseInsensitive) if got := fpm.match(tt.path); got != tt.want { t.Errorf("{%q}.match(%q) = %v, want %v", tt.fullPath, tt.path, got, tt.want) } @@ -43,19 +46,23 @@ func TestPathFullMatcherMatch(t *testing.T) { } } -func TestPathPrefixMatcherMatch(t *testing.T) { +func (s) TestPathPrefixMatcherMatch(t *testing.T) { tests := []struct { - name string - prefix string - path string - want bool + name string + prefix string + caseInsensitive bool + path string + want bool }{ {name: "match", prefix: "/s/", path: "/s/m", want: true}, + {name: "case insensitive match", prefix: "/s/", caseInsensitive: true, path: "/S/m", want: true}, + {name: "case insensitive match 2", prefix: "/S/", caseInsensitive: true, path: "/s/m", want: true}, {name: "not match", prefix: "/s/", path: "/a/b", want: false}, + {name: "case insensitive not match", prefix: "/s/", caseInsensitive: true, path: "/a/b", want: false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - fpm := newPathPrefixMatcher(tt.prefix) + fpm := newPathPrefixMatcher(tt.prefix, tt.caseInsensitive) if got := fpm.match(tt.path); got != tt.want { t.Errorf("{%q}.match(%q) = %v, want %v", tt.prefix, tt.path, got, tt.want) } @@ -63,7 +70,7 @@ func TestPathPrefixMatcherMatch(t *testing.T) { } } -func TestPathRegexMatcherMatch(t *testing.T) { +func (s) TestPathRegexMatcherMatch(t *testing.T) { tests := []struct { name string regexPath string @@ -72,6 +79,8 @@ func TestPathRegexMatcherMatch(t *testing.T) { }{ {name: "match", regexPath: "^/s+/m.*$", path: "/sss/me", want: true}, {name: "not match", regexPath: "^/s+/m*$", path: "/sss/b", want: false}, + {name: "no match because only part of path matches with regex", regexPath: "^a+$", path: "ab", want: false}, + {name: "match because full path matches with regex", regexPath: "^a+$", path: "aa", want: true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/xds/internal/xdsclient/xdsresource/matcher_test.go b/xds/internal/xdsclient/xdsresource/matcher_test.go new file mode 100644 index 000000000000..2746e58e6c77 --- /dev/null +++ b/xds/internal/xdsclient/xdsresource/matcher_test.go @@ -0,0 +1,192 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "context" + "testing" + + "google.golang.org/grpc/internal/grpcrand" + "google.golang.org/grpc/internal/grpcutil" + iresolver "google.golang.org/grpc/internal/resolver" + "google.golang.org/grpc/internal/xds/matcher" + "google.golang.org/grpc/metadata" +) + +func (s) TestAndMatcherMatch(t *testing.T) { + tests := []struct { + name string + pm pathMatcher + hm matcher.HeaderMatcher + info iresolver.RPCInfo + want bool + }{ + { + name: "both match", + pm: newPathExactMatcher("/a/b", false), + hm: matcher.NewHeaderExactMatcher("th", "tv", false), + info: iresolver.RPCInfo{ + Method: "/a/b", + Context: metadata.NewOutgoingContext(context.Background(), metadata.Pairs("th", "tv")), + }, + want: true, + }, + { + name: "both match with path case insensitive", + pm: newPathExactMatcher("/A/B", true), + hm: matcher.NewHeaderExactMatcher("th", "tv", false), + info: iresolver.RPCInfo{ + Method: "/a/b", + Context: metadata.NewOutgoingContext(context.Background(), metadata.Pairs("th", "tv")), + }, + want: true, + }, + { + name: "only one match", + pm: newPathExactMatcher("/a/b", false), + hm: matcher.NewHeaderExactMatcher("th", "tv", false), + info: iresolver.RPCInfo{ + Method: "/z/y", + Context: metadata.NewOutgoingContext(context.Background(), metadata.Pairs("th", "tv")), + }, + want: false, + }, + { + name: "both not match", + pm: newPathExactMatcher("/z/y", false), + hm: matcher.NewHeaderExactMatcher("th", "abc", false), + info: iresolver.RPCInfo{ + Method: "/a/b", + Context: metadata.NewOutgoingContext(context.Background(), metadata.Pairs("th", "tv")), + }, + want: false, + }, + { + name: "fake header", + pm: newPathPrefixMatcher("/", false), + hm: matcher.NewHeaderExactMatcher("content-type", "fake", false), + info: iresolver.RPCInfo{ + Method: "/a/b", + Context: grpcutil.WithExtraMetadata(context.Background(), metadata.Pairs( + "content-type", "fake", + )), + }, + want: true, + }, + { + name: "binary header", + pm: newPathPrefixMatcher("/", false), + hm: matcher.NewHeaderPresentMatcher("t-bin", true, false), + info: iresolver.RPCInfo{ + Method: "/a/b", + Context: grpcutil.WithExtraMetadata( + metadata.NewOutgoingContext(context.Background(), metadata.Pairs("t-bin", "123")), metadata.Pairs( + "content-type", "fake", + )), + }, + // Shouldn't match binary header, even though it's in metadata. + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + a := newCompositeMatcher(tt.pm, []matcher.HeaderMatcher{tt.hm}, nil) + if got := a.Match(tt.info); got != tt.want { + t.Errorf("match() = %v, want %v", got, tt.want) + } + }) + } +} + +func (s) TestFractionMatcherMatch(t *testing.T) { + const fraction = 500000 + fm := newFractionMatcher(fraction) + defer func() { + RandInt63n = grpcrand.Int63n + }() + + // rand > fraction, should return false. + RandInt63n = func(n int64) int64 { + return fraction + 1 + } + if matched := fm.match(); matched { + t.Errorf("match() = %v, want not match", matched) + } + + // rand == fraction, should return true. + RandInt63n = func(n int64) int64 { + return fraction + } + if matched := fm.match(); !matched { + t.Errorf("match() = %v, want match", matched) + } + + // rand < fraction, should return true. + RandInt63n = func(n int64) int64 { + return fraction - 1 + } + if matched := fm.match(); !matched { + t.Errorf("match() = %v, want match", matched) + } +} + +func (s) TestMatchTypeForDomain(t *testing.T) { + tests := []struct { + d string + want domainMatchType + }{ + {d: "", want: domainMatchTypeInvalid}, + {d: "*", want: domainMatchTypeUniversal}, + {d: "bar.*", want: domainMatchTypePrefix}, + {d: "*.abc.com", want: domainMatchTypeSuffix}, + {d: "foo.bar.com", want: domainMatchTypeExact}, + {d: "foo.*.com", want: domainMatchTypeInvalid}, + } + for _, tt := range tests { + if got := matchTypeForDomain(tt.d); got != tt.want { + t.Errorf("matchTypeForDomain(%q) = %v, want %v", tt.d, got, tt.want) + } + } +} + +func (s) TestMatch(t *testing.T) { + tests := []struct { + name string + domain string + host string + wantTyp domainMatchType + wantMatched bool + }{ + {name: "invalid-empty", domain: "", host: "", wantTyp: domainMatchTypeInvalid, wantMatched: false}, + {name: "invalid", domain: "a.*.b", host: "", wantTyp: domainMatchTypeInvalid, wantMatched: false}, + {name: "universal", domain: "*", host: "abc.com", wantTyp: domainMatchTypeUniversal, wantMatched: true}, + {name: "prefix-match", domain: "abc.*", host: "abc.123", wantTyp: domainMatchTypePrefix, wantMatched: true}, + {name: "prefix-no-match", domain: "abc.*", host: "abcd.123", wantTyp: domainMatchTypePrefix, wantMatched: false}, + {name: "suffix-match", domain: "*.123", host: "abc.123", wantTyp: domainMatchTypeSuffix, wantMatched: true}, + {name: "suffix-no-match", domain: "*.123", host: "abc.1234", wantTyp: domainMatchTypeSuffix, wantMatched: false}, + {name: "exact-match", domain: "foo.bar", host: "foo.bar", wantTyp: domainMatchTypeExact, wantMatched: true}, + {name: "exact-no-match", domain: "foo.bar.com", host: "foo.bar", wantTyp: domainMatchTypeExact, wantMatched: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if gotTyp, gotMatched := match(tt.domain, tt.host); gotTyp != tt.wantTyp || gotMatched != tt.wantMatched { + t.Errorf("match() = %v, %v, want %v, %v", gotTyp, gotMatched, tt.wantTyp, tt.wantMatched) + } + }) + } +} diff --git a/xds/internal/xdsclient/xdsresource/name.go b/xds/internal/xdsclient/xdsresource/name.go new file mode 100644 index 000000000000..80c0efd37b39 --- /dev/null +++ b/xds/internal/xdsclient/xdsresource/name.go @@ -0,0 +1,133 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "net/url" + "sort" + "strings" + + "google.golang.org/grpc/internal/envconfig" +) + +// FederationScheme is the scheme of a federation resource name. +const FederationScheme = "xdstp" + +// Name contains the parsed component of an xDS resource name. +// +// An xDS resource name is in the format of +// xdstp://[{authority}]/{resource type}/{id/*}?{context parameters}{#processing directive,*} +// +// See +// https://github.com/cncf/xds/blob/main/proposals/TP1-xds-transport-next.md#uri-based-xds-resource-names +// for details, and examples. +type Name struct { + Scheme string + Authority string + Type string + ID string + + ContextParams map[string]string + + processingDirective string +} + +// ParseName splits the name and returns a struct representation of the Name. +// +// If the name isn't a valid new-style xDS name, field ID is set to the input. +// Note that this is not an error, because we still support the old-style +// resource names (those not starting with "xdstp:"). +// +// The caller can tell if the parsing is successful by checking the returned +// Scheme. +func ParseName(name string) *Name { + if !envconfig.XDSFederation { + // Return "" scheme to use the default authority for the server. + return &Name{ID: name} + } + if !strings.Contains(name, "://") { + // Only the long form URL, with ://, is valid. + return &Name{ID: name} + } + parsed, err := url.Parse(name) + if err != nil { + return &Name{ID: name} + } + + ret := &Name{ + Scheme: parsed.Scheme, + Authority: parsed.Host, + } + split := strings.SplitN(parsed.Path, "/", 3) + if len(split) < 3 { + // Path is in the format of "/type/id". There must be at least 3 + // segments after splitting. + return &Name{ID: name} + } + ret.Type = split[1] + ret.ID = split[2] + if len(parsed.Query()) != 0 { + ret.ContextParams = make(map[string]string) + for k, vs := range parsed.Query() { + if len(vs) > 0 { + // We only keep one value of each key. Behavior for multiple values + // is undefined. + ret.ContextParams[k] = vs[0] + } + } + } + // TODO: processing directive (the part comes after "#" in the URL, stored + // in parsed.RawFragment) is kept but not processed. Add support for that + // when it's needed. + ret.processingDirective = parsed.RawFragment + return ret +} + +// String returns a canonicalized string of name. The context parameters are +// sorted by the keys. +func (n *Name) String() string { + if n.Scheme == "" { + return n.ID + } + + // Sort and build query. + keys := make([]string, 0, len(n.ContextParams)) + for k := range n.ContextParams { + keys = append(keys, k) + } + sort.Strings(keys) + var pairs []string + for _, k := range keys { + pairs = append(pairs, strings.Join([]string{k, n.ContextParams[k]}, "=")) + } + rawQuery := strings.Join(pairs, "&") + + path := n.Type + if n.ID != "" { + path = "/" + path + "/" + n.ID + } + + tempURL := &url.URL{ + Scheme: n.Scheme, + Host: n.Authority, + Path: path, + RawQuery: rawQuery, + RawFragment: n.processingDirective, + } + return tempURL.String() +} diff --git a/xds/internal/xdsclient/xdsresource/name_test.go b/xds/internal/xdsclient/xdsresource/name_test.go new file mode 100644 index 000000000000..a30b437658f5 --- /dev/null +++ b/xds/internal/xdsclient/xdsresource/name_test.go @@ -0,0 +1,124 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "google.golang.org/grpc/internal/envconfig" +) + +func TestParseName(t *testing.T) { + tests := []struct { + name string + env bool // Whether federation env is set to true. + in string + want *Name + wantStr string + }{ + { + name: "env off", + env: false, + in: "xdstp://auth/type/id", + want: &Name{ID: "xdstp://auth/type/id"}, + wantStr: "xdstp://auth/type/id", + }, + { + name: "old style name", + env: true, + in: "test-resource", + want: &Name{ID: "test-resource"}, + wantStr: "test-resource", + }, + { + name: "invalid not url", + env: true, + in: "a:/b/c", + want: &Name{ID: "a:/b/c"}, + wantStr: "a:/b/c", + }, + { + name: "invalid no resource type", + env: true, + in: "xdstp://auth/id", + want: &Name{ID: "xdstp://auth/id"}, + wantStr: "xdstp://auth/id", + }, + { + name: "valid with no authority", + env: true, + in: "xdstp:///type/id", + want: &Name{Scheme: "xdstp", Authority: "", Type: "type", ID: "id"}, + wantStr: "xdstp:///type/id", + }, + { + name: "valid no ctx params", + env: true, + in: "xdstp://auth/type/id", + want: &Name{Scheme: "xdstp", Authority: "auth", Type: "type", ID: "id"}, + wantStr: "xdstp://auth/type/id", + }, + { + name: "valid with ctx params", + env: true, + in: "xdstp://auth/type/id?a=1&b=2", + want: &Name{Scheme: "xdstp", Authority: "auth", Type: "type", ID: "id", ContextParams: map[string]string{"a": "1", "b": "2"}}, + wantStr: "xdstp://auth/type/id?a=1&b=2", + }, + { + name: "valid with ctx params sorted by keys", + env: true, + in: "xdstp://auth/type/id?b=2&a=1", + want: &Name{Scheme: "xdstp", Authority: "auth", Type: "type", ID: "id", ContextParams: map[string]string{"a": "1", "b": "2"}}, + wantStr: "xdstp://auth/type/id?a=1&b=2", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + defer func() func() { + oldEnv := envconfig.XDSFederation + envconfig.XDSFederation = tt.env + return func() { envconfig.XDSFederation = oldEnv } + }()() + got := ParseName(tt.in) + if !cmp.Equal(got, tt.want, cmpopts.IgnoreFields(Name{}, "processingDirective")) { + t.Errorf("ParseName() = %#v, want %#v", got, tt.want) + } + if gotStr := got.String(); gotStr != tt.wantStr { + t.Errorf("Name.String() = %s, want %s", gotStr, tt.wantStr) + } + }) + } +} + +// TestNameStringCtxParamsOrder covers the case that if two names differ only in +// context parameter __order__, the parsed name.String() has the same value. +func TestNameStringCtxParamsOrder(t *testing.T) { + const ( + a = "xdstp://auth/type/id?a=1&b=2" + b = "xdstp://auth/type/id?b=2&a=1" + ) + aParsed := ParseName(a).String() + bParsed := ParseName(b).String() + + if aParsed != bParsed { + t.Fatalf("aParsed.String() = %q, bParsed.String() = %q, want them to be the same", aParsed, bParsed) + } +} diff --git a/xds/internal/xdsclient/xdsresource/resource_type.go b/xds/internal/xdsclient/xdsresource/resource_type.go new file mode 100644 index 000000000000..7cd64201dae1 --- /dev/null +++ b/xds/internal/xdsclient/xdsresource/resource_type.go @@ -0,0 +1,164 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package xdsresource implements the xDS data model layer. +// +// Provides resource-type specific functionality to unmarshal xDS protos into +// internal data structures that contain only fields gRPC is interested in. +// These internal data structures are passed to components in the xDS stack +// (resolver/balancers/server) that have expressed interest in receiving +// updates to specific resources. +package xdsresource + +import ( + "google.golang.org/grpc/xds/internal" + "google.golang.org/grpc/xds/internal/xdsclient/bootstrap" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource/version" + "google.golang.org/protobuf/types/known/anypb" +) + +func init() { + internal.ResourceTypeMapForTesting = make(map[string]any) + internal.ResourceTypeMapForTesting[version.V3ListenerURL] = listenerType + internal.ResourceTypeMapForTesting[version.V3RouteConfigURL] = routeConfigType + internal.ResourceTypeMapForTesting[version.V3ClusterURL] = clusterType + internal.ResourceTypeMapForTesting[version.V3EndpointsURL] = endpointsType +} + +// Producer contains a single method to discover resource configuration from a +// remote management server using xDS APIs. +// +// The xdsclient package provides a concrete implementation of this interface. +type Producer interface { + // WatchResource uses xDS to discover the resource associated with the + // provided resource name. The resource type implementation determines how + // xDS requests are sent out and how responses are deserialized and + // validated. Upon receipt of a response from the management server, an + // appropriate callback on the watcher is invoked. + WatchResource(rType Type, resourceName string, watcher ResourceWatcher) (cancel func()) +} + +// ResourceWatcher wraps the callbacks to be invoked for different events +// corresponding to the resource being watched. +type ResourceWatcher interface { + // OnUpdate is invoked to report an update for the resource being watched. + // The ResourceData parameter needs to be type asserted to the appropriate + // type for the resource being watched. + OnUpdate(ResourceData) + + // OnError is invoked under different error conditions including but not + // limited to the following: + // - authority mentioned in the resource is not found + // - resource name parsing error + // - resource deserialization error + // - resource validation error + // - ADS stream failure + // - connection failure + OnError(error) + + // OnResourceDoesNotExist is invoked for a specific error condition where + // the requested resource is not found on the xDS management server. + OnResourceDoesNotExist() +} + +// TODO: Once the implementation is complete, rename this interface as +// ResourceType and get rid of the existing ResourceType enum. + +// Type wraps all resource-type specific functionality. Each supported resource +// type will provide an implementation of this interface. +type Type interface { + // TypeURL is the xDS type URL of this resource type for v3 transport. + TypeURL() string + + // TypeName identifies resources in a transport protocol agnostic way. This + // can be used for logging/debugging purposes, as well in cases where the + // resource type name is to be uniquely identified but the actual + // functionality provided by the resource type is not required. + // + // TODO: once Type is renamed to ResourceType, rename TypeName to + // ResourceTypeName. + TypeName() string + + // AllResourcesRequiredInSotW indicates whether this resource type requires + // that all resources be present in every SotW response from the server. If + // true, a response that does not include a previously seen resource will be + // interpreted as a deletion of that resource. + AllResourcesRequiredInSotW() bool + + // Decode deserializes and validates an xDS resource serialized inside the + // provided `Any` proto, as received from the xDS management server. + // + // If protobuf deserialization fails or resource validation fails, + // returns a non-nil error. Otherwise, returns a fully populated + // DecodeResult. + Decode(*DecodeOptions, *anypb.Any) (*DecodeResult, error) +} + +// ResourceData contains the configuration data sent by the xDS management +// server, associated with the resource being watched. Every resource type must +// provide an implementation of this interface to represent the configuration +// received from the xDS management server. +type ResourceData interface { + isResourceData() + + // Equal returns true if the passed in resource data is equal to that of the + // receiver. + Equal(ResourceData) bool + + // ToJSON returns a JSON string representation of the resource data. + ToJSON() string + + Raw() *anypb.Any +} + +// DecodeOptions wraps the options required by ResourceType implementation for +// decoding configuration received from the xDS management server. +type DecodeOptions struct { + // BootstrapConfig contains the bootstrap configuration passed to the + // top-level xdsClient. This contains useful data for resource validation. + BootstrapConfig *bootstrap.Config +} + +// DecodeResult is the result of a decode operation. +type DecodeResult struct { + // Name is the name of the resource being watched. + Name string + // Resource contains the configuration associated with the resource being + // watched. + Resource ResourceData +} + +// resourceTypeState wraps the static state associated with concrete resource +// type implementations, which can then embed this struct and get the methods +// implemented here for free. +type resourceTypeState struct { + typeURL string + typeName string + allResourcesRequiredInSotW bool +} + +func (r resourceTypeState) TypeURL() string { + return r.typeURL +} + +func (r resourceTypeState) TypeName() string { + return r.typeName +} + +func (r resourceTypeState) AllResourcesRequiredInSotW() bool { + return r.allResourcesRequiredInSotW +} diff --git a/xds/internal/xdsclient/xdsresource/route_config_resource_type.go b/xds/internal/xdsclient/xdsresource/route_config_resource_type.go new file mode 100644 index 000000000000..8ce5cb28596e --- /dev/null +++ b/xds/internal/xdsclient/xdsresource/route_config_resource_type.go @@ -0,0 +1,150 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource/version" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" +) + +const ( + // RouteConfigTypeName represents the transport agnostic name for the + // route config resource. + RouteConfigTypeName = "RouteConfigResource" +) + +var ( + // Compile time interface checks. + _ Type = routeConfigResourceType{} + + // Singleton instantiation of the resource type implementation. + routeConfigType = routeConfigResourceType{ + resourceTypeState: resourceTypeState{ + typeURL: version.V3RouteConfigURL, + typeName: "RouteConfigResource", + allResourcesRequiredInSotW: false, + }, + } +) + +// routeConfigResourceType provides the resource-type specific functionality for +// a RouteConfiguration resource. +// +// Implements the Type interface. +type routeConfigResourceType struct { + resourceTypeState +} + +// Decode deserializes and validates an xDS resource serialized inside the +// provided `Any` proto, as received from the xDS management server. +func (routeConfigResourceType) Decode(opts *DecodeOptions, resource *anypb.Any) (*DecodeResult, error) { + name, rc, err := unmarshalRouteConfigResource(resource) + switch { + case name == "": + // Name is unset only when protobuf deserialization fails. + return nil, err + case err != nil: + // Protobuf deserialization succeeded, but resource validation failed. + return &DecodeResult{Name: name, Resource: &RouteConfigResourceData{Resource: RouteConfigUpdate{}}}, err + } + + return &DecodeResult{Name: name, Resource: &RouteConfigResourceData{Resource: rc}}, nil + +} + +// RouteConfigResourceData wraps the configuration of a RouteConfiguration +// resource as received from the management server. +// +// Implements the ResourceData interface. +type RouteConfigResourceData struct { + ResourceData + + // TODO: We have always stored update structs by value. See if this can be + // switched to a pointer? + Resource RouteConfigUpdate +} + +// Equal returns true if other is equal to r. +func (r *RouteConfigResourceData) Equal(other ResourceData) bool { + if r == nil && other == nil { + return true + } + if (r == nil) != (other == nil) { + return false + } + return proto.Equal(r.Resource.Raw, other.Raw()) + +} + +// ToJSON returns a JSON string representation of the resource data. +func (r *RouteConfigResourceData) ToJSON() string { + return pretty.ToJSON(r.Resource) +} + +// Raw returns the underlying raw protobuf form of the route configuration +// resource. +func (r *RouteConfigResourceData) Raw() *anypb.Any { + return r.Resource.Raw +} + +// RouteConfigWatcher wraps the callbacks to be invoked for different +// events corresponding to the route configuration resource being watched. +type RouteConfigWatcher interface { + // OnUpdate is invoked to report an update for the resource being watched. + OnUpdate(*RouteConfigResourceData) + + // OnError is invoked under different error conditions including but not + // limited to the following: + // - authority mentioned in the resource is not found + // - resource name parsing error + // - resource deserialization error + // - resource validation error + // - ADS stream failure + // - connection failure + OnError(error) + + // OnResourceDoesNotExist is invoked for a specific error condition where + // the requested resource is not found on the xDS management server. + OnResourceDoesNotExist() +} + +type delegatingRouteConfigWatcher struct { + watcher RouteConfigWatcher +} + +func (d *delegatingRouteConfigWatcher) OnUpdate(data ResourceData) { + rc := data.(*RouteConfigResourceData) + d.watcher.OnUpdate(rc) +} + +func (d *delegatingRouteConfigWatcher) OnError(err error) { + d.watcher.OnError(err) +} + +func (d *delegatingRouteConfigWatcher) OnResourceDoesNotExist() { + d.watcher.OnResourceDoesNotExist() +} + +// WatchRouteConfig uses xDS to discover the configuration associated with the +// provided route configuration resource name. +func WatchRouteConfig(p Producer, name string, w RouteConfigWatcher) (cancel func()) { + delegator := &delegatingRouteConfigWatcher{watcher: w} + return p.WatchResource(routeConfigType, name, delegator) +} diff --git a/xds/internal/xdsclient/xdsresource/test_utils_test.go b/xds/internal/xdsclient/xdsresource/test_utils_test.go new file mode 100644 index 000000000000..04a15f96c248 --- /dev/null +++ b/xds/internal/xdsclient/xdsresource/test_utils_test.go @@ -0,0 +1,45 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/protobuf/testing/protocmp" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +var ( + cmpOpts = cmp.Options{ + cmpopts.EquateEmpty(), + cmp.FilterValues(func(x, y error) bool { return true }, cmpopts.EquateErrors()), + cmp.Comparer(func(a, b time.Time) bool { return true }), + protocmp.Transform(), + } +) diff --git a/xds/internal/xdsclient/xdsresource/tests/unmarshal_cds_test.go b/xds/internal/xdsclient/xdsresource/tests/unmarshal_cds_test.go new file mode 100644 index 000000000000..d522eac8261d --- /dev/null +++ b/xds/internal/xdsclient/xdsresource/tests/unmarshal_cds_test.go @@ -0,0 +1,610 @@ +/* + * + * Copyright 2023 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package tests_test contains test cases for unmarshalling of CDS resources. +package tests_test + +import ( + "encoding/json" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "google.golang.org/grpc/balancer/leastrequest" + _ "google.golang.org/grpc/balancer/roundrobin" // To register round_robin load balancer. + "google.golang.org/grpc/internal/balancer/stub" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/grpctest" + iserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/serviceconfig" + _ "google.golang.org/grpc/xds" // Register the xDS LB Registry Converters. + "google.golang.org/grpc/xds/internal/balancer/ringhash" + "google.golang.org/grpc/xds/internal/balancer/wrrlocality" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" + "google.golang.org/protobuf/types/known/wrapperspb" + + v3xdsxdstypepb "github.com/cncf/xds/go/xds/type/v3" + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3aggregateclusterpb "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/aggregate/v3" + v3ringhashpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/ring_hash/v3" + v3roundrobinpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/round_robin/v3" + v3wrrlocalitypb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/wrr_locality/v3" + "github.com/golang/protobuf/proto" + anypb "github.com/golang/protobuf/ptypes/any" + structpb "github.com/golang/protobuf/ptypes/struct" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +const ( + clusterName = "clusterName" + serviceName = "service" +) + +var emptyUpdate = xdsresource.ClusterUpdate{ClusterName: clusterName, LRSServerConfig: xdsresource.ClusterLRSOff} + +func wrrLocality(m proto.Message) *v3wrrlocalitypb.WrrLocality { + return &v3wrrlocalitypb.WrrLocality{ + EndpointPickingPolicy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: testutils.MarshalAny(m), + }, + }, + }, + }, + } +} + +func wrrLocalityAny(m proto.Message) *anypb.Any { + return testutils.MarshalAny(wrrLocality(m)) +} + +type customLBConfig struct { + serviceconfig.LoadBalancingConfig +} + +// We have this test in a separate test package in order to not take a +// dependency on the internal xDS balancer packages within the xDS Client. +func (s) TestValidateCluster_Success(t *testing.T) { + const customLBPolicyName = "myorg.MyCustomLeastRequestPolicy" + stub.Register(customLBPolicyName, stub.BalancerFuncs{ + ParseConfig: func(json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + return customLBConfig{}, nil + }, + }) + + origCustomLBSupport := envconfig.XDSCustomLBPolicy + envconfig.XDSCustomLBPolicy = true + defer func() { + envconfig.XDSCustomLBPolicy = origCustomLBSupport + }() + defer func(old bool) { envconfig.LeastRequestLB = old }(envconfig.LeastRequestLB) + envconfig.LeastRequestLB = true + tests := []struct { + name string + cluster *v3clusterpb.Cluster + wantUpdate xdsresource.ClusterUpdate + wantLBConfig *iserviceconfig.BalancerConfig + customLBDisabled bool + }{ + { + name: "happy-case-logical-dns", + cluster: &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_LOGICAL_DNS}, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + LoadAssignment: &v3endpointpb.ClusterLoadAssignment{ + Endpoints: []*v3endpointpb.LocalityLbEndpoints{{ + LbEndpoints: []*v3endpointpb.LbEndpoint{{ + HostIdentifier: &v3endpointpb.LbEndpoint_Endpoint{ + Endpoint: &v3endpointpb.Endpoint{ + Address: &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: "dns_host", + PortSpecifier: &v3corepb.SocketAddress_PortValue{ + PortValue: 8080, + }, + }, + }, + }, + }, + }, + }}, + }}, + }, + }, + wantUpdate: xdsresource.ClusterUpdate{ + ClusterName: clusterName, + ClusterType: xdsresource.ClusterTypeLogicalDNS, + DNSHostName: "dns_host:8080", + }, + wantLBConfig: &iserviceconfig.BalancerConfig{ + Name: wrrlocality.Name, + Config: &wrrlocality.LBConfig{ + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: "round_robin", + }, + }, + }, + }, + { + name: "happy-case-aggregate-v3", + cluster: &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_ClusterType{ + ClusterType: &v3clusterpb.Cluster_CustomClusterType{ + Name: "envoy.clusters.aggregate", + TypedConfig: testutils.MarshalAny(&v3aggregateclusterpb.ClusterConfig{ + Clusters: []string{"a", "b", "c"}, + }), + }, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + }, + wantUpdate: xdsresource.ClusterUpdate{ + ClusterName: clusterName, LRSServerConfig: xdsresource.ClusterLRSOff, ClusterType: xdsresource.ClusterTypeAggregate, + PrioritizedClusterNames: []string{"a", "b", "c"}, + }, + wantLBConfig: &iserviceconfig.BalancerConfig{ + Name: wrrlocality.Name, + Config: &wrrlocality.LBConfig{ + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: "round_robin", + }, + }, + }, + }, + { + name: "happy-case-no-service-name-no-lrs", + cluster: e2e.DefaultCluster(clusterName, "", e2e.SecurityLevelNone), + wantUpdate: emptyUpdate, + wantLBConfig: &iserviceconfig.BalancerConfig{ + Name: wrrlocality.Name, + Config: &wrrlocality.LBConfig{ + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: "round_robin", + }, + }, + }, + }, + { + name: "happy-case-no-lrs", + cluster: e2e.DefaultCluster(clusterName, serviceName, e2e.SecurityLevelNone), + wantUpdate: xdsresource.ClusterUpdate{ + ClusterName: clusterName, + EDSServiceName: serviceName, + }, + wantLBConfig: &iserviceconfig.BalancerConfig{ + Name: wrrlocality.Name, + Config: &wrrlocality.LBConfig{ + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: "round_robin", + }, + }, + }, + }, + { + name: "happiest-case", + cluster: e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: clusterName, + ServiceName: serviceName, + EnableLRS: true, + }), + wantUpdate: xdsresource.ClusterUpdate{ + ClusterName: clusterName, + EDSServiceName: serviceName, + LRSServerConfig: xdsresource.ClusterLRSServerSelf, + }, + wantLBConfig: &iserviceconfig.BalancerConfig{ + Name: wrrlocality.Name, + Config: &wrrlocality.LBConfig{ + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: "round_robin", + }, + }, + }, + }, + { + name: "happiest-case-with-circuitbreakers", + cluster: func() *v3clusterpb.Cluster { + c := e2e.ClusterResourceWithOptions(e2e.ClusterOptions{ + ClusterName: clusterName, + ServiceName: serviceName, + EnableLRS: true, + }) + c.CircuitBreakers = &v3clusterpb.CircuitBreakers{ + Thresholds: []*v3clusterpb.CircuitBreakers_Thresholds{ + { + Priority: v3corepb.RoutingPriority_DEFAULT, + MaxRequests: wrapperspb.UInt32(512), + }, + { + Priority: v3corepb.RoutingPriority_HIGH, + MaxRequests: nil, + }, + }, + } + return c + }(), + wantUpdate: xdsresource.ClusterUpdate{ + ClusterName: clusterName, + EDSServiceName: serviceName, + LRSServerConfig: xdsresource.ClusterLRSServerSelf, + MaxRequests: func() *uint32 { i := uint32(512); return &i }(), + }, + wantLBConfig: &iserviceconfig.BalancerConfig{ + Name: wrrlocality.Name, + Config: &wrrlocality.LBConfig{ + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: "round_robin", + }, + }, + }, + }, + { + name: "happiest-case-with-ring-hash-lb-policy-with-default-config", + cluster: func() *v3clusterpb.Cluster { + c := e2e.DefaultCluster(clusterName, serviceName, e2e.SecurityLevelNone) + c.LbPolicy = v3clusterpb.Cluster_RING_HASH + return c + }(), + wantUpdate: xdsresource.ClusterUpdate{ + ClusterName: clusterName, + EDSServiceName: serviceName, + }, + wantLBConfig: &iserviceconfig.BalancerConfig{ + Name: "ring_hash_experimental", + Config: &ringhash.LBConfig{ + MinRingSize: 1024, + MaxRingSize: 4096, + }, + }, + }, + { + name: "happiest-case-with-least-request-lb-policy-with-default-config", + cluster: &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LbPolicy: v3clusterpb.Cluster_LEAST_REQUEST, + }, + wantUpdate: xdsresource.ClusterUpdate{ + ClusterName: clusterName, EDSServiceName: serviceName, + }, + wantLBConfig: &iserviceconfig.BalancerConfig{ + Name: "least_request_experimental", + Config: &leastrequest.LBConfig{ + ChoiceCount: 2, + }, + }, + }, + { + name: "happiest-case-with-ring-hash-lb-policy-with-none-default-config", + cluster: func() *v3clusterpb.Cluster { + c := e2e.DefaultCluster(clusterName, serviceName, e2e.SecurityLevelNone) + c.LbPolicy = v3clusterpb.Cluster_RING_HASH + c.LbConfig = &v3clusterpb.Cluster_RingHashLbConfig_{ + RingHashLbConfig: &v3clusterpb.Cluster_RingHashLbConfig{ + MinimumRingSize: wrapperspb.UInt64(10), + MaximumRingSize: wrapperspb.UInt64(100), + }, + } + return c + }(), + wantUpdate: xdsresource.ClusterUpdate{ + ClusterName: clusterName, + EDSServiceName: serviceName, + }, + wantLBConfig: &iserviceconfig.BalancerConfig{ + Name: "ring_hash_experimental", + Config: &ringhash.LBConfig{ + MinRingSize: 10, + MaxRingSize: 100, + }, + }, + }, + { + name: "happiest-case-with-least-request-lb-policy-with-none-default-config", + cluster: &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LbPolicy: v3clusterpb.Cluster_LEAST_REQUEST, + LbConfig: &v3clusterpb.Cluster_LeastRequestLbConfig_{ + LeastRequestLbConfig: &v3clusterpb.Cluster_LeastRequestLbConfig{ + ChoiceCount: wrapperspb.UInt32(3), + }, + }, + }, + wantUpdate: xdsresource.ClusterUpdate{ + ClusterName: clusterName, EDSServiceName: serviceName, + }, + wantLBConfig: &iserviceconfig.BalancerConfig{ + Name: "least_request_experimental", + Config: &leastrequest.LBConfig{ + ChoiceCount: 3, + }, + }, + }, + { + name: "happiest-case-with-ring-hash-lb-policy-configured-through-LoadBalancingPolicy", + cluster: &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LoadBalancingPolicy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: testutils.MarshalAny(&v3ringhashpb.RingHash{ + HashFunction: v3ringhashpb.RingHash_XX_HASH, + MinimumRingSize: wrapperspb.UInt64(10), + MaximumRingSize: wrapperspb.UInt64(100), + }), + }, + }, + }, + }, + }, + wantUpdate: xdsresource.ClusterUpdate{ + ClusterName: clusterName, + EDSServiceName: serviceName, + }, + wantLBConfig: &iserviceconfig.BalancerConfig{ + Name: "ring_hash_experimental", + Config: &ringhash.LBConfig{ + MinRingSize: 10, + MaxRingSize: 100, + }, + }, + }, + { + name: "happiest-case-with-wrrlocality-rr-child-configured-through-LoadBalancingPolicy", + cluster: &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LoadBalancingPolicy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: wrrLocalityAny(&v3roundrobinpb.RoundRobin{}), + }, + }, + }, + }, + }, + wantUpdate: xdsresource.ClusterUpdate{ + ClusterName: clusterName, + EDSServiceName: serviceName, + }, + wantLBConfig: &iserviceconfig.BalancerConfig{ + Name: wrrlocality.Name, + Config: &wrrlocality.LBConfig{ + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: "round_robin", + }, + }, + }, + }, + { + name: "happiest-case-with-custom-lb-configured-through-LoadBalancingPolicy", + cluster: &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LoadBalancingPolicy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: wrrLocalityAny(&v3xdsxdstypepb.TypedStruct{ + TypeUrl: "type.googleapis.com/myorg.MyCustomLeastRequestPolicy", + Value: &structpb.Struct{}, + }), + }, + }, + }, + }, + }, + wantUpdate: xdsresource.ClusterUpdate{ + ClusterName: clusterName, + EDSServiceName: serviceName, + }, + wantLBConfig: &iserviceconfig.BalancerConfig{ + Name: wrrlocality.Name, + Config: &wrrlocality.LBConfig{ + ChildPolicy: &iserviceconfig.BalancerConfig{ + Name: "myorg.MyCustomLeastRequestPolicy", + Config: customLBConfig{}, + }, + }, + }, + }, + { + name: "custom-lb-env-var-not-set-ignore-load-balancing-policy-use-lb-policy-and-enum", + cluster: &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LbPolicy: v3clusterpb.Cluster_RING_HASH, + LbConfig: &v3clusterpb.Cluster_RingHashLbConfig_{ + RingHashLbConfig: &v3clusterpb.Cluster_RingHashLbConfig{ + MinimumRingSize: wrapperspb.UInt64(20), + MaximumRingSize: wrapperspb.UInt64(200), + }, + }, + LoadBalancingPolicy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: testutils.MarshalAny(&v3ringhashpb.RingHash{ + HashFunction: v3ringhashpb.RingHash_XX_HASH, + MinimumRingSize: wrapperspb.UInt64(10), + MaximumRingSize: wrapperspb.UInt64(100), + }), + }, + }, + }, + }, + }, + wantUpdate: xdsresource.ClusterUpdate{ + ClusterName: clusterName, + EDSServiceName: serviceName, + }, + wantLBConfig: &iserviceconfig.BalancerConfig{ + Name: "ring_hash_experimental", + Config: &ringhash.LBConfig{ + MinRingSize: 20, + MaxRingSize: 200, + }, + }, + customLBDisabled: true, + }, + { + name: "load-balancing-policy-takes-precedence-over-lb-policy-and-enum", + cluster: &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LbPolicy: v3clusterpb.Cluster_RING_HASH, + LbConfig: &v3clusterpb.Cluster_RingHashLbConfig_{ + RingHashLbConfig: &v3clusterpb.Cluster_RingHashLbConfig{ + MinimumRingSize: wrapperspb.UInt64(20), + MaximumRingSize: wrapperspb.UInt64(200), + }, + }, + LoadBalancingPolicy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: testutils.MarshalAny(&v3ringhashpb.RingHash{ + HashFunction: v3ringhashpb.RingHash_XX_HASH, + MinimumRingSize: wrapperspb.UInt64(10), + MaximumRingSize: wrapperspb.UInt64(100), + }), + }, + }, + }, + }, + }, + wantUpdate: xdsresource.ClusterUpdate{ + ClusterName: clusterName, + EDSServiceName: serviceName, + }, + wantLBConfig: &iserviceconfig.BalancerConfig{ + Name: "ring_hash_experimental", + Config: &ringhash.LBConfig{ + MinRingSize: 10, + MaxRingSize: 100, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if test.customLBDisabled { + envconfig.XDSCustomLBPolicy = false + defer func() { + envconfig.XDSCustomLBPolicy = true + }() + } + update, err := xdsresource.ValidateClusterAndConstructClusterUpdateForTesting(test.cluster) + if err != nil { + t.Errorf("validateClusterAndConstructClusterUpdate(%+v) failed: %v", test.cluster, err) + } + // Ignore the raw JSON string into the cluster update. JSON bytes + // are nondeterministic (whitespace etc.) so we cannot reliably + // compare JSON bytes in a test. Thus, marshal into a Balancer + // Config struct and compare on that. Only need to test this JSON + // emission here, as this covers the possible output space. + if diff := cmp.Diff(update, test.wantUpdate, cmpopts.EquateEmpty(), cmpopts.IgnoreFields(xdsresource.ClusterUpdate{}, "LBPolicy")); diff != "" { + t.Errorf("validateClusterAndConstructClusterUpdate(%+v) got diff: %v (-got, +want)", test.cluster, diff) + } + bc := &iserviceconfig.BalancerConfig{} + if err := json.Unmarshal(update.LBPolicy, bc); err != nil { + t.Fatalf("failed to unmarshal JSON: %v", err) + } + if diff := cmp.Diff(bc, test.wantLBConfig); diff != "" { + t.Fatalf("update.LBConfig got unexpected output, diff (-got +want): %v", diff) + } + }) + } +} diff --git a/xds/internal/xdsclient/xdsresource/type.go b/xds/internal/xdsclient/xdsresource/type.go new file mode 100644 index 000000000000..35cfa9ee7678 --- /dev/null +++ b/xds/internal/xdsclient/xdsresource/type.go @@ -0,0 +1,135 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "time" + + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + "github.com/golang/protobuf/proto" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource/version" + "google.golang.org/protobuf/types/known/anypb" +) + +// UpdateValidatorFunc performs validations on update structs using +// context/logic available at the xdsClient layer. Since these validation are +// performed on internal update structs, they can be shared between different +// API clients. +type UpdateValidatorFunc func(any) error + +// UpdateMetadata contains the metadata for each update, including timestamp, +// raw message, and so on. +type UpdateMetadata struct { + // Status is the status of this resource, e.g. ACKed, NACKed, or + // Not_exist(removed). + Status ServiceStatus + // Version is the version of the xds response. Note that this is the version + // of the resource in use (previous ACKed). If a response is NACKed, the + // NACKed version is in ErrState. + Version string + // Timestamp is when the response is received. + Timestamp time.Time + // ErrState is set when the update is NACKed. + ErrState *UpdateErrorMetadata +} + +// IsListenerResource returns true if the provider URL corresponds to an xDS +// Listener resource. +func IsListenerResource(url string) bool { + return url == version.V3ListenerURL +} + +// IsHTTPConnManagerResource returns true if the provider URL corresponds to an xDS +// HTTPConnManager resource. +func IsHTTPConnManagerResource(url string) bool { + return url == version.V3HTTPConnManagerURL +} + +// IsRouteConfigResource returns true if the provider URL corresponds to an xDS +// RouteConfig resource. +func IsRouteConfigResource(url string) bool { + return url == version.V3RouteConfigURL +} + +// IsClusterResource returns true if the provider URL corresponds to an xDS +// Cluster resource. +func IsClusterResource(url string) bool { + return url == version.V3ClusterURL +} + +// IsEndpointsResource returns true if the provider URL corresponds to an xDS +// Endpoints resource. +func IsEndpointsResource(url string) bool { + return url == version.V3EndpointsURL +} + +// UnwrapResource unwraps and returns the inner resource if it's in a resource +// wrapper. The original resource is returned if it's not wrapped. +func UnwrapResource(r *anypb.Any) (*anypb.Any, error) { + url := r.GetTypeUrl() + if url != version.V3ResourceWrapperURL { + // Not wrapped. + return r, nil + } + inner := &v3discoverypb.Resource{} + if err := proto.Unmarshal(r.GetValue(), inner); err != nil { + return nil, err + } + return inner.Resource, nil +} + +// ServiceStatus is the status of the update. +type ServiceStatus int + +const ( + // ServiceStatusUnknown is the default state, before a watch is started for + // the resource. + ServiceStatusUnknown ServiceStatus = iota + // ServiceStatusRequested is when the watch is started, but before and + // response is received. + ServiceStatusRequested + // ServiceStatusNotExist is when the resource doesn't exist in + // state-of-the-world responses (e.g. LDS and CDS), which means the resource + // is removed by the management server. + ServiceStatusNotExist // Resource is removed in the server, in LDS/CDS. + // ServiceStatusACKed is when the resource is ACKed. + ServiceStatusACKed + // ServiceStatusNACKed is when the resource is NACKed. + ServiceStatusNACKed +) + +// UpdateErrorMetadata is part of UpdateMetadata. It contains the error state +// when a response is NACKed. +type UpdateErrorMetadata struct { + // Version is the version of the NACKed response. + Version string + // Err contains why the response was NACKed. + Err error + // Timestamp is when the NACKed response was received. + Timestamp time.Time +} + +// UpdateWithMD contains the raw message of the update and the metadata, +// including version, raw message, timestamp. +// +// This is to be used for config dump and CSDS, not directly by users (like +// resolvers/balancers). +type UpdateWithMD struct { + MD UpdateMetadata + Raw *anypb.Any +} diff --git a/xds/internal/xdsclient/xdsresource/type_cds.go b/xds/internal/xdsclient/xdsresource/type_cds.go new file mode 100644 index 000000000000..269d9ebdae15 --- /dev/null +++ b/xds/internal/xdsclient/xdsresource/type_cds.go @@ -0,0 +1,96 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "encoding/json" + + "google.golang.org/protobuf/types/known/anypb" +) + +// ClusterType is the type of cluster from a received CDS response. +type ClusterType int + +const ( + // ClusterTypeEDS represents the EDS cluster type, which will delegate endpoint + // discovery to the management server. + ClusterTypeEDS ClusterType = iota + // ClusterTypeLogicalDNS represents the Logical DNS cluster type, which essentially + // maps to the gRPC behavior of using the DNS resolver with pick_first LB policy. + ClusterTypeLogicalDNS + // ClusterTypeAggregate represents the Aggregate Cluster type, which provides a + // prioritized list of clusters to use. It is used for failover between clusters + // with a different configuration. + ClusterTypeAggregate +) + +// ClusterLRSServerConfigType is the type of LRS server config. +type ClusterLRSServerConfigType int + +const ( + // ClusterLRSOff indicates LRS is off (loads are not reported for this + // cluster). + ClusterLRSOff ClusterLRSServerConfigType = iota + // ClusterLRSServerSelf indicates loads should be reported to the same + // server (the authority) where the CDS resp is received from. + ClusterLRSServerSelf +) + +// ClusterUpdate contains information from a received CDS response, which is of +// interest to the registered CDS watcher. +type ClusterUpdate struct { + ClusterType ClusterType + // ClusterName is the clusterName being watched for through CDS. + ClusterName string + // EDSServiceName is an optional name for EDS. If it's not set, the balancer + // should watch ClusterName for the EDS resources. + EDSServiceName string + // LRSServerConfig contains the server where the load reports should be sent + // to. This can be change to an interface, to support other types, e.g. a + // ServerConfig with ServerURI, creds. + LRSServerConfig ClusterLRSServerConfigType + // SecurityCfg contains security configuration sent by the control plane. + SecurityCfg *SecurityConfig + // MaxRequests for circuit breaking, if any (otherwise nil). + MaxRequests *uint32 + // DNSHostName is used only for cluster type DNS. It's the DNS name to + // resolve in "host:port" form + DNSHostName string + // PrioritizedClusterNames is used only for cluster type aggregate. It represents + // a prioritized list of cluster names. + PrioritizedClusterNames []string + + // LBPolicy represents the locality and endpoint picking policy in JSON, + // which will be the child policy of xds_cluster_impl. + LBPolicy json.RawMessage + + // OutlierDetection is the outlier detection configuration for this cluster. + // If nil, it means this cluster does not use the outlier detection feature. + OutlierDetection json.RawMessage + + // Raw is the resource from the xds response. + Raw *anypb.Any +} + +// ClusterUpdateErrTuple is a tuple with the update and error. It contains the +// results from unmarshal functions. It's used to pass unmarshal results of +// multiple resources together, e.g. in maps like `map[string]{Update,error}`. +type ClusterUpdateErrTuple struct { + Update ClusterUpdate + Err error +} diff --git a/xds/internal/xdsclient/xdsresource/type_eds.go b/xds/internal/xdsclient/xdsresource/type_eds.go new file mode 100644 index 000000000000..1254d250c99b --- /dev/null +++ b/xds/internal/xdsclient/xdsresource/type_eds.go @@ -0,0 +1,75 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "google.golang.org/grpc/xds/internal" + "google.golang.org/protobuf/types/known/anypb" +) + +// OverloadDropConfig contains the config to drop overloads. +type OverloadDropConfig struct { + Category string + Numerator uint32 + Denominator uint32 +} + +// EndpointHealthStatus represents the health status of an endpoint. +type EndpointHealthStatus int32 + +const ( + // EndpointHealthStatusUnknown represents HealthStatus UNKNOWN. + EndpointHealthStatusUnknown EndpointHealthStatus = iota + // EndpointHealthStatusHealthy represents HealthStatus HEALTHY. + EndpointHealthStatusHealthy + // EndpointHealthStatusUnhealthy represents HealthStatus UNHEALTHY. + EndpointHealthStatusUnhealthy + // EndpointHealthStatusDraining represents HealthStatus DRAINING. + EndpointHealthStatusDraining + // EndpointHealthStatusTimeout represents HealthStatus TIMEOUT. + EndpointHealthStatusTimeout + // EndpointHealthStatusDegraded represents HealthStatus DEGRADED. + EndpointHealthStatusDegraded +) + +// Endpoint contains information of an endpoint. +type Endpoint struct { + Address string + HealthStatus EndpointHealthStatus + Weight uint32 +} + +// Locality contains information of a locality. +type Locality struct { + Endpoints []Endpoint + ID internal.LocalityID + Priority uint32 + Weight uint32 +} + +// EndpointsUpdate contains an EDS update. +type EndpointsUpdate struct { + Drops []OverloadDropConfig + // Localities in the EDS response with `load_balancing_weight` field not set + // or explicitly set to 0 are ignored while parsing the resource, and + // therefore do not show up here. + Localities []Locality + + // Raw is the resource from the xds response. + Raw *anypb.Any +} diff --git a/xds/internal/xdsclient/xdsresource/type_lds.go b/xds/internal/xdsclient/xdsresource/type_lds.go new file mode 100644 index 000000000000..a2742fb4371a --- /dev/null +++ b/xds/internal/xdsclient/xdsresource/type_lds.go @@ -0,0 +1,87 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "time" + + "google.golang.org/grpc/xds/internal/httpfilter" + "google.golang.org/protobuf/types/known/anypb" +) + +// ListenerUpdate contains information received in an LDS response, which is of +// interest to the registered LDS watcher. +type ListenerUpdate struct { + // RouteConfigName is the route configuration name corresponding to the + // target which is being watched through LDS. + // + // Exactly one of RouteConfigName and InlineRouteConfig is set. + RouteConfigName string + // InlineRouteConfig is the inline route configuration (RDS response) + // returned inside LDS. + // + // Exactly one of RouteConfigName and InlineRouteConfig is set. + InlineRouteConfig *RouteConfigUpdate + + // MaxStreamDuration contains the HTTP connection manager's + // common_http_protocol_options.max_stream_duration field, or zero if + // unset. + MaxStreamDuration time.Duration + // HTTPFilters is a list of HTTP filters (name, config) from the LDS + // response. + HTTPFilters []HTTPFilter + // InboundListenerCfg contains inbound listener configuration. + InboundListenerCfg *InboundListenerConfig + + // Raw is the resource from the xds response. + Raw *anypb.Any +} + +// HTTPFilter represents one HTTP filter from an LDS response's HTTP connection +// manager field. +type HTTPFilter struct { + // Name is an arbitrary name of the filter. Used for applying override + // settings in virtual host / route / weighted cluster configuration (not + // yet supported). + Name string + // Filter is the HTTP filter found in the registry for the config type. + Filter httpfilter.Filter + // Config contains the filter's configuration + Config httpfilter.FilterConfig +} + +// InboundListenerConfig contains information about the inbound listener, i.e +// the server-side listener. +type InboundListenerConfig struct { + // Address is the local address on which the inbound listener is expected to + // accept incoming connections. + Address string + // Port is the local port on which the inbound listener is expected to + // accept incoming connections. + Port string + // FilterChains is the list of filter chains associated with this listener. + FilterChains *FilterChainManager +} + +// ListenerUpdateErrTuple is a tuple with the update and error. It contains the +// results from unmarshal functions. It's used to pass unmarshal results of +// multiple resources together, e.g. in maps like `map[string]{Update,error}`. +type ListenerUpdateErrTuple struct { + Update ListenerUpdate + Err error +} diff --git a/xds/internal/xdsclient/xdsresource/type_rds.go b/xds/internal/xdsclient/xdsresource/type_rds.go new file mode 100644 index 000000000000..ad59209163e7 --- /dev/null +++ b/xds/internal/xdsclient/xdsresource/type_rds.go @@ -0,0 +1,256 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "regexp" + "time" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal/xds/matcher" + "google.golang.org/grpc/xds/internal/clusterspecifier" + "google.golang.org/grpc/xds/internal/httpfilter" + "google.golang.org/protobuf/types/known/anypb" +) + +// RouteConfigUpdate contains information received in an RDS response, which is +// of interest to the registered RDS watcher. +type RouteConfigUpdate struct { + VirtualHosts []*VirtualHost + // ClusterSpecifierPlugins are the LB Configurations for any + // ClusterSpecifierPlugins referenced by the Route Table. + ClusterSpecifierPlugins map[string]clusterspecifier.BalancerConfig + // Raw is the resource from the xds response. + Raw *anypb.Any +} + +// VirtualHost contains the routes for a list of Domains. +// +// Note that the domains in this slice can be a wildcard, not an exact string. +// The consumer of this struct needs to find the best match for its hostname. +type VirtualHost struct { + Domains []string + // Routes contains a list of routes, each containing matchers and + // corresponding action. + Routes []*Route + // HTTPFilterConfigOverride contains any HTTP filter config overrides for + // the virtual host which may be present. An individual filter's override + // may be unused if the matching Route contains an override for that + // filter. + HTTPFilterConfigOverride map[string]httpfilter.FilterConfig + RetryConfig *RetryConfig +} + +// RetryConfig contains all retry-related configuration in either a VirtualHost +// or Route. +type RetryConfig struct { + // RetryOn is a set of status codes on which to retry. Only Canceled, + // DeadlineExceeded, Internal, ResourceExhausted, and Unavailable are + // supported; any other values will be omitted. + RetryOn map[codes.Code]bool + NumRetries uint32 // maximum number of retry attempts + RetryBackoff RetryBackoff // retry backoff policy +} + +// RetryBackoff describes the backoff policy for retries. +type RetryBackoff struct { + BaseInterval time.Duration // initial backoff duration between attempts + MaxInterval time.Duration // maximum backoff duration +} + +// HashPolicyType specifies the type of HashPolicy from a received RDS Response. +type HashPolicyType int + +const ( + // HashPolicyTypeHeader specifies to hash a Header in the incoming request. + HashPolicyTypeHeader HashPolicyType = iota + // HashPolicyTypeChannelID specifies to hash a unique Identifier of the + // Channel. This is a 64-bit random int computed at initialization time. + HashPolicyTypeChannelID +) + +// HashPolicy specifies the HashPolicy if the upstream cluster uses a hashing +// load balancer. +type HashPolicy struct { + HashPolicyType HashPolicyType + Terminal bool + // Fields used for type HEADER. + HeaderName string + Regex *regexp.Regexp + RegexSubstitution string +} + +// RouteActionType is the action of the route from a received RDS response. +type RouteActionType int + +const ( + // RouteActionUnsupported are routing types currently unsupported by grpc. + // According to A36, "A Route with an inappropriate action causes RPCs + // matching that route to fail." + RouteActionUnsupported RouteActionType = iota + // RouteActionRoute is the expected route type on the client side. Route + // represents routing a request to some upstream cluster. On the client + // side, if an RPC matches to a route that is not RouteActionRoute, the RPC + // will fail according to A36. + RouteActionRoute + // RouteActionNonForwardingAction is the expected route type on the server + // side. NonForwardingAction represents when a route will generate a + // response directly, without forwarding to an upstream host. + RouteActionNonForwardingAction +) + +// Route is both a specification of how to match a request as well as an +// indication of the action to take upon match. +type Route struct { + Path *string + Prefix *string + Regex *regexp.Regexp + // Indicates if prefix/path matching should be case insensitive. The default + // is false (case sensitive). + CaseInsensitive bool + Headers []*HeaderMatcher + Fraction *uint32 + + HashPolicies []*HashPolicy + + // If the matchers above indicate a match, the below configuration is used. + // If MaxStreamDuration is nil, it indicates neither of the route action's + // max_stream_duration fields (grpc_timeout_header_max nor + // max_stream_duration) were set. In this case, the ListenerUpdate's + // MaxStreamDuration field should be used. If MaxStreamDuration is set to + // an explicit zero duration, the application's deadline should be used. + MaxStreamDuration *time.Duration + // HTTPFilterConfigOverride contains any HTTP filter config overrides for + // the route which may be present. An individual filter's override may be + // unused if the matching WeightedCluster contains an override for that + // filter. + HTTPFilterConfigOverride map[string]httpfilter.FilterConfig + RetryConfig *RetryConfig + + ActionType RouteActionType + + // Only one of the following fields (WeightedClusters or + // ClusterSpecifierPlugin) will be set for a route. + WeightedClusters map[string]WeightedCluster + // ClusterSpecifierPlugin is the name of the Cluster Specifier Plugin that + // this Route is linked to, if specified by xDS. + ClusterSpecifierPlugin string +} + +// WeightedCluster contains settings for an xds ActionType.WeightedCluster. +type WeightedCluster struct { + // Weight is the relative weight of the cluster. It will never be zero. + Weight uint32 + // HTTPFilterConfigOverride contains any HTTP filter config overrides for + // the weighted cluster which may be present. + HTTPFilterConfigOverride map[string]httpfilter.FilterConfig +} + +// HeaderMatcher represents header matchers. +type HeaderMatcher struct { + Name string + InvertMatch *bool + ExactMatch *string + RegexMatch *regexp.Regexp + PrefixMatch *string + SuffixMatch *string + RangeMatch *Int64Range + PresentMatch *bool + StringMatch *matcher.StringMatcher +} + +// Int64Range is a range for header range match. +type Int64Range struct { + Start int64 + End int64 +} + +// SecurityConfig contains the security configuration received as part of the +// Cluster resource on the client-side, and as part of the Listener resource on +// the server-side. +type SecurityConfig struct { + // RootInstanceName identifies the certProvider plugin to be used to fetch + // root certificates. This instance name will be resolved to the plugin name + // and its associated configuration from the certificate_providers field of + // the bootstrap file. + RootInstanceName string + // RootCertName is the certificate name to be passed to the plugin (looked + // up from the bootstrap file) while fetching root certificates. + RootCertName string + // IdentityInstanceName identifies the certProvider plugin to be used to + // fetch identity certificates. This instance name will be resolved to the + // plugin name and its associated configuration from the + // certificate_providers field of the bootstrap file. + IdentityInstanceName string + // IdentityCertName is the certificate name to be passed to the plugin + // (looked up from the bootstrap file) while fetching identity certificates. + IdentityCertName string + // SubjectAltNameMatchers is an optional list of match criteria for SANs + // specified on the peer certificate. Used only on the client-side. + // + // Some intricacies: + // - If this field is empty, then any peer certificate is accepted. + // - If the peer certificate contains a wildcard DNS SAN, and an `exact` + // matcher is configured, a wildcard DNS match is performed instead of a + // regular string comparison. + SubjectAltNameMatchers []matcher.StringMatcher + // RequireClientCert indicates if the server handshake process expects the + // client to present a certificate. Set to true when performing mTLS. Used + // only on the server-side. + RequireClientCert bool +} + +// Equal returns true if sc is equal to other. +func (sc *SecurityConfig) Equal(other *SecurityConfig) bool { + switch { + case sc == nil && other == nil: + return true + case (sc != nil) != (other != nil): + return false + } + switch { + case sc.RootInstanceName != other.RootInstanceName: + return false + case sc.RootCertName != other.RootCertName: + return false + case sc.IdentityInstanceName != other.IdentityInstanceName: + return false + case sc.IdentityCertName != other.IdentityCertName: + return false + case sc.RequireClientCert != other.RequireClientCert: + return false + default: + if len(sc.SubjectAltNameMatchers) != len(other.SubjectAltNameMatchers) { + return false + } + for i := 0; i < len(sc.SubjectAltNameMatchers); i++ { + if !sc.SubjectAltNameMatchers[i].Equal(other.SubjectAltNameMatchers[i]) { + return false + } + } + } + return true +} + +// RouteConfigUpdateErrTuple is a tuple with the update and error. It contains +// the results from unmarshal functions. It's used to pass unmarshal results of +// multiple resources together, e.g. in maps like `map[string]{Update,error}`. +type RouteConfigUpdateErrTuple struct { + Update RouteConfigUpdate + Err error +} diff --git a/xds/internal/xdsclient/xdsresource/unmarshal_cds.go b/xds/internal/xdsclient/xdsresource/unmarshal_cds.go new file mode 100644 index 000000000000..abf95d2a40d6 --- /dev/null +++ b/xds/internal/xdsclient/xdsresource/unmarshal_cds.go @@ -0,0 +1,694 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "encoding/json" + "errors" + "fmt" + "net" + "strconv" + "strings" + "time" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3aggregateclusterpb "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/aggregate/v3" + v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" + "github.com/golang/protobuf/proto" + + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/pretty" + iserviceconfig "google.golang.org/grpc/internal/serviceconfig" + "google.golang.org/grpc/internal/xds/matcher" + "google.golang.org/grpc/xds/internal/xdsclient/xdslbregistry" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource/version" + "google.golang.org/protobuf/types/known/anypb" +) + +// ValidateClusterAndConstructClusterUpdateForTesting exports the +// validateClusterAndConstructClusterUpdate function for testing purposes. +var ValidateClusterAndConstructClusterUpdateForTesting = validateClusterAndConstructClusterUpdate + +// TransportSocket proto message has a `name` field which is expected to be set +// to this value by the management server. +const transportSocketName = "envoy.transport_sockets.tls" + +func unmarshalClusterResource(r *anypb.Any) (string, ClusterUpdate, error) { + r, err := UnwrapResource(r) + if err != nil { + return "", ClusterUpdate{}, fmt.Errorf("failed to unwrap resource: %v", err) + } + + if !IsClusterResource(r.GetTypeUrl()) { + return "", ClusterUpdate{}, fmt.Errorf("unexpected resource type: %q ", r.GetTypeUrl()) + } + + cluster := &v3clusterpb.Cluster{} + if err := proto.Unmarshal(r.GetValue(), cluster); err != nil { + return "", ClusterUpdate{}, fmt.Errorf("failed to unmarshal resource: %v", err) + } + cu, err := validateClusterAndConstructClusterUpdate(cluster) + if err != nil { + return cluster.GetName(), ClusterUpdate{}, err + } + cu.Raw = r + + return cluster.GetName(), cu, nil +} + +const ( + defaultRingHashMinSize = 1024 + defaultRingHashMaxSize = 8 * 1024 * 1024 // 8M + ringHashSizeUpperBound = 8 * 1024 * 1024 // 8M + + defaultLeastRequestChoiceCount = 2 +) + +func validateClusterAndConstructClusterUpdate(cluster *v3clusterpb.Cluster) (ClusterUpdate, error) { + var lbPolicy json.RawMessage + var err error + switch cluster.GetLbPolicy() { + case v3clusterpb.Cluster_ROUND_ROBIN: + lbPolicy = []byte(`[{"xds_wrr_locality_experimental": {"childPolicy": [{"round_robin": {}}]}}]`) + case v3clusterpb.Cluster_RING_HASH: + if !envconfig.XDSRingHash { + return ClusterUpdate{}, fmt.Errorf("unexpected lbPolicy %v in response: %+v", cluster.GetLbPolicy(), cluster) + } + rhc := cluster.GetRingHashLbConfig() + if rhc.GetHashFunction() != v3clusterpb.Cluster_RingHashLbConfig_XX_HASH { + return ClusterUpdate{}, fmt.Errorf("unsupported ring_hash hash function %v in response: %+v", rhc.GetHashFunction(), cluster) + } + // Minimum defaults to 1024 entries, and limited to 8M entries Maximum + // defaults to 8M entries, and limited to 8M entries + var minSize, maxSize uint64 = defaultRingHashMinSize, defaultRingHashMaxSize + if min := rhc.GetMinimumRingSize(); min != nil { + minSize = min.GetValue() + } + if max := rhc.GetMaximumRingSize(); max != nil { + maxSize = max.GetValue() + } + + rhLBCfg := []byte(fmt.Sprintf("{\"minRingSize\": %d, \"maxRingSize\": %d}", minSize, maxSize)) + lbPolicy = []byte(fmt.Sprintf(`[{"ring_hash_experimental": %s}]`, rhLBCfg)) + case v3clusterpb.Cluster_LEAST_REQUEST: + if !envconfig.LeastRequestLB { + return ClusterUpdate{}, fmt.Errorf("unexpected lbPolicy %v in response: %+v", cluster.GetLbPolicy(), cluster) + } + + // "The configuration for the Least Request LB policy is the + // least_request_lb_config field. The field is optional; if not present, + // defaults will be assumed for all of its values." - A48 + lr := cluster.GetLeastRequestLbConfig() + var choiceCount uint32 = defaultLeastRequestChoiceCount + if cc := lr.GetChoiceCount(); cc != nil { + choiceCount = cc.GetValue() + } + // "If choice_count < 2, the config will be rejected." - A48 + if choiceCount < 2 { + return ClusterUpdate{}, fmt.Errorf("Cluster_LeastRequestLbConfig.ChoiceCount must be >= 2, got: %v", choiceCount) + } + + lrLBCfg := []byte(fmt.Sprintf("{\"choiceCount\": %d}", choiceCount)) + lbPolicy = []byte(fmt.Sprintf(`[{"least_request_experimental": %s}]`, lrLBCfg)) + default: + return ClusterUpdate{}, fmt.Errorf("unexpected lbPolicy %v in response: %+v", cluster.GetLbPolicy(), cluster) + } + // Process security configuration received from the control plane iff the + // corresponding environment variable is set. + var sc *SecurityConfig + if envconfig.XDSClientSideSecurity { + var err error + if sc, err = securityConfigFromCluster(cluster); err != nil { + return ClusterUpdate{}, err + } + } + + // Process outlier detection received from the control plane iff the + // corresponding environment variable is set. + var od json.RawMessage + if envconfig.XDSOutlierDetection { + var err error + if od, err = outlierConfigFromCluster(cluster); err != nil { + return ClusterUpdate{}, err + } + } + + if cluster.GetLoadBalancingPolicy() != nil && envconfig.XDSCustomLBPolicy { + lbPolicy, err = xdslbregistry.ConvertToServiceConfig(cluster.GetLoadBalancingPolicy(), 0) + if err != nil { + return ClusterUpdate{}, fmt.Errorf("error converting LoadBalancingPolicy %v in response: %+v: %v", cluster.GetLoadBalancingPolicy(), cluster, err) + } + // "It will be the responsibility of the XdsClient to validate the + // converted configuration. It will do this by having the gRPC LB policy + // registry parse the configuration." - A52 + bc := &iserviceconfig.BalancerConfig{} + if err := json.Unmarshal(lbPolicy, bc); err != nil { + return ClusterUpdate{}, fmt.Errorf("JSON generated from xDS LB policy registry: %s is invalid: %v", pretty.FormatJSON(lbPolicy), err) + } + } + + ret := ClusterUpdate{ + ClusterName: cluster.GetName(), + SecurityCfg: sc, + MaxRequests: circuitBreakersFromCluster(cluster), + LBPolicy: lbPolicy, + OutlierDetection: od, + } + + // Note that this is different from the gRFC (gRFC A47 says to include the + // full ServerConfig{URL,creds,server feature} here). This information is + // not available here, because this function doesn't have access to the + // xdsclient bootstrap information now (can be added if necessary). The + // ServerConfig will be read and populated by the CDS balancer when + // processing this field. + // According to A27: + // If the `lrs_server` field is set, it must have its `self` field set, in + // which case the client should use LRS for load reporting. Otherwise + // (the `lrs_server` field is not set), LRS load reporting will be disabled. + if lrs := cluster.GetLrsServer(); lrs != nil { + if lrs.GetSelf() == nil { + return ClusterUpdate{}, fmt.Errorf("unsupported config_source_specifier %T in lrs_server field", lrs.ConfigSourceSpecifier) + } + ret.LRSServerConfig = ClusterLRSServerSelf + } + + // Validate and set cluster type from the response. + switch { + case cluster.GetType() == v3clusterpb.Cluster_EDS: + if configsource := cluster.GetEdsClusterConfig().GetEdsConfig(); configsource.GetAds() == nil && configsource.GetSelf() == nil { + return ClusterUpdate{}, fmt.Errorf("CDS's EDS config source is not ADS or Self: %+v", cluster) + } + ret.ClusterType = ClusterTypeEDS + ret.EDSServiceName = cluster.GetEdsClusterConfig().GetServiceName() + if strings.HasPrefix(ret.ClusterName, "xdstp:") && ret.EDSServiceName == "" { + return ClusterUpdate{}, fmt.Errorf("CDS's EDS service name is not set with a new-style cluster name: %+v", cluster) + } + return ret, nil + case cluster.GetType() == v3clusterpb.Cluster_LOGICAL_DNS: + if !envconfig.XDSAggregateAndDNS { + return ClusterUpdate{}, fmt.Errorf("unsupported cluster type (%v, %v) in response: %+v", cluster.GetType(), cluster.GetClusterType(), cluster) + } + ret.ClusterType = ClusterTypeLogicalDNS + dnsHN, err := dnsHostNameFromCluster(cluster) + if err != nil { + return ClusterUpdate{}, err + } + ret.DNSHostName = dnsHN + return ret, nil + case cluster.GetClusterType() != nil && cluster.GetClusterType().Name == "envoy.clusters.aggregate": + if !envconfig.XDSAggregateAndDNS { + return ClusterUpdate{}, fmt.Errorf("unsupported cluster type (%v, %v) in response: %+v", cluster.GetType(), cluster.GetClusterType(), cluster) + } + clusters := &v3aggregateclusterpb.ClusterConfig{} + if err := proto.Unmarshal(cluster.GetClusterType().GetTypedConfig().GetValue(), clusters); err != nil { + return ClusterUpdate{}, fmt.Errorf("failed to unmarshal resource: %v", err) + } + if len(clusters.Clusters) == 0 { + return ClusterUpdate{}, fmt.Errorf("xds: aggregate cluster has empty clusters field in response: %+v", cluster) + } + ret.ClusterType = ClusterTypeAggregate + ret.PrioritizedClusterNames = clusters.Clusters + return ret, nil + default: + return ClusterUpdate{}, fmt.Errorf("unsupported cluster type (%v, %v) in response: %+v", cluster.GetType(), cluster.GetClusterType(), cluster) + } +} + +// dnsHostNameFromCluster extracts the DNS host name from the cluster's load +// assignment. +// +// There should be exactly one locality, with one endpoint, whose address +// contains the address and port. +func dnsHostNameFromCluster(cluster *v3clusterpb.Cluster) (string, error) { + loadAssignment := cluster.GetLoadAssignment() + if loadAssignment == nil { + return "", fmt.Errorf("load_assignment not present for LOGICAL_DNS cluster") + } + if len(loadAssignment.GetEndpoints()) != 1 { + return "", fmt.Errorf("load_assignment for LOGICAL_DNS cluster must have exactly one locality, got: %+v", loadAssignment) + } + endpoints := loadAssignment.GetEndpoints()[0].GetLbEndpoints() + if len(endpoints) != 1 { + return "", fmt.Errorf("locality for LOGICAL_DNS cluster must have exactly one endpoint, got: %+v", endpoints) + } + endpoint := endpoints[0].GetEndpoint() + if endpoint == nil { + return "", fmt.Errorf("endpoint for LOGICAL_DNS cluster not set") + } + socketAddr := endpoint.GetAddress().GetSocketAddress() + if socketAddr == nil { + return "", fmt.Errorf("socket address for endpoint for LOGICAL_DNS cluster not set") + } + if socketAddr.GetResolverName() != "" { + return "", fmt.Errorf("socket address for endpoint for LOGICAL_DNS cluster not set has unexpected custom resolver name: %v", socketAddr.GetResolverName()) + } + host := socketAddr.GetAddress() + if host == "" { + return "", fmt.Errorf("host for endpoint for LOGICAL_DNS cluster not set") + } + port := socketAddr.GetPortValue() + if port == 0 { + return "", fmt.Errorf("port for endpoint for LOGICAL_DNS cluster not set") + } + return net.JoinHostPort(host, strconv.Itoa(int(port))), nil +} + +// securityConfigFromCluster extracts the relevant security configuration from +// the received Cluster resource. +func securityConfigFromCluster(cluster *v3clusterpb.Cluster) (*SecurityConfig, error) { + if tsm := cluster.GetTransportSocketMatches(); len(tsm) != 0 { + return nil, fmt.Errorf("unsupport transport_socket_matches field is non-empty: %+v", tsm) + } + // The Cluster resource contains a `transport_socket` field, which contains + // a oneof `typed_config` field of type `protobuf.Any`. The any proto + // contains a marshaled representation of an `UpstreamTlsContext` message. + ts := cluster.GetTransportSocket() + if ts == nil { + return nil, nil + } + if name := ts.GetName(); name != transportSocketName { + return nil, fmt.Errorf("transport_socket field has unexpected name: %s", name) + } + any := ts.GetTypedConfig() + if any == nil || any.TypeUrl != version.V3UpstreamTLSContextURL { + return nil, fmt.Errorf("transport_socket field has unexpected typeURL: %s", any.TypeUrl) + } + upstreamCtx := &v3tlspb.UpstreamTlsContext{} + if err := proto.Unmarshal(any.GetValue(), upstreamCtx); err != nil { + return nil, fmt.Errorf("failed to unmarshal UpstreamTlsContext in CDS response: %v", err) + } + // The following fields from `UpstreamTlsContext` are ignored: + // - sni + // - allow_renegotiation + // - max_session_keys + if upstreamCtx.GetCommonTlsContext() == nil { + return nil, errors.New("UpstreamTlsContext in CDS response does not contain a CommonTlsContext") + } + + return securityConfigFromCommonTLSContext(upstreamCtx.GetCommonTlsContext(), false) +} + +// common is expected to be not nil. +// The `alpn_protocols` field is ignored. +func securityConfigFromCommonTLSContext(common *v3tlspb.CommonTlsContext, server bool) (*SecurityConfig, error) { + if common.GetTlsParams() != nil { + return nil, fmt.Errorf("unsupported tls_params field in CommonTlsContext message: %+v", common) + } + if common.GetCustomHandshaker() != nil { + return nil, fmt.Errorf("unsupported custom_handshaker field in CommonTlsContext message: %+v", common) + } + + // For now, if we can't get a valid security config from the new fields, we + // fallback to the old deprecated fields. + // TODO: Drop support for deprecated fields. NACK if err != nil here. + sc, _ := securityConfigFromCommonTLSContextUsingNewFields(common, server) + if sc == nil || sc.Equal(&SecurityConfig{}) { + var err error + sc, err = securityConfigFromCommonTLSContextWithDeprecatedFields(common, server) + if err != nil { + return nil, err + } + } + if sc != nil { + // sc == nil is a valid case where the control plane has not sent us any + // security configuration. xDS creds will use fallback creds. + if server { + if sc.IdentityInstanceName == "" { + return nil, errors.New("security configuration on the server-side does not contain identity certificate provider instance name") + } + } else { + if sc.RootInstanceName == "" { + return nil, errors.New("security configuration on the client-side does not contain root certificate provider instance name") + } + } + } + return sc, nil +} + +func securityConfigFromCommonTLSContextWithDeprecatedFields(common *v3tlspb.CommonTlsContext, server bool) (*SecurityConfig, error) { + // The `CommonTlsContext` contains a + // `tls_certificate_certificate_provider_instance` field of type + // `CertificateProviderInstance`, which contains the provider instance name + // and the certificate name to fetch identity certs. + sc := &SecurityConfig{} + if identity := common.GetTlsCertificateCertificateProviderInstance(); identity != nil { + sc.IdentityInstanceName = identity.GetInstanceName() + sc.IdentityCertName = identity.GetCertificateName() + } + + // The `CommonTlsContext` contains a `validation_context_type` field which + // is a oneof. We can get the values that we are interested in from two of + // those possible values: + // - combined validation context: + // - contains a default validation context which holds the list of + // matchers for accepted SANs. + // - contains certificate provider instance configuration + // - certificate provider instance configuration + // - in this case, we do not get a list of accepted SANs. + switch t := common.GetValidationContextType().(type) { + case *v3tlspb.CommonTlsContext_CombinedValidationContext: + combined := common.GetCombinedValidationContext() + var matchers []matcher.StringMatcher + if def := combined.GetDefaultValidationContext(); def != nil { + for _, m := range def.GetMatchSubjectAltNames() { + matcher, err := matcher.StringMatcherFromProto(m) + if err != nil { + return nil, err + } + matchers = append(matchers, matcher) + } + } + if server && len(matchers) != 0 { + return nil, fmt.Errorf("match_subject_alt_names field in validation context is not supported on the server: %v", common) + } + sc.SubjectAltNameMatchers = matchers + if pi := combined.GetValidationContextCertificateProviderInstance(); pi != nil { + sc.RootInstanceName = pi.GetInstanceName() + sc.RootCertName = pi.GetCertificateName() + } + case *v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance: + pi := common.GetValidationContextCertificateProviderInstance() + sc.RootInstanceName = pi.GetInstanceName() + sc.RootCertName = pi.GetCertificateName() + case nil: + // It is valid for the validation context to be nil on the server side. + default: + return nil, fmt.Errorf("validation context contains unexpected type: %T", t) + } + return sc, nil +} + +// gRFC A29 https://github.com/grpc/proposal/blob/master/A29-xds-tls-security.md +// specifies the new way to fetch security configuration and says the following: +// +// Although there are various ways to obtain certificates as per this proto +// (which are supported by Envoy), gRPC supports only one of them and that is +// the `CertificateProviderPluginInstance` proto. +// +// This helper function attempts to fetch security configuration from the +// `CertificateProviderPluginInstance` message, given a CommonTlsContext. +func securityConfigFromCommonTLSContextUsingNewFields(common *v3tlspb.CommonTlsContext, server bool) (*SecurityConfig, error) { + // The `tls_certificate_provider_instance` field of type + // `CertificateProviderPluginInstance` is used to fetch the identity + // certificate provider. + sc := &SecurityConfig{} + identity := common.GetTlsCertificateProviderInstance() + if identity == nil && len(common.GetTlsCertificates()) != 0 { + return nil, fmt.Errorf("expected field tls_certificate_provider_instance is not set, while unsupported field tls_certificates is set in CommonTlsContext message: %+v", common) + } + if identity == nil && common.GetTlsCertificateSdsSecretConfigs() != nil { + return nil, fmt.Errorf("expected field tls_certificate_provider_instance is not set, while unsupported field tls_certificate_sds_secret_configs is set in CommonTlsContext message: %+v", common) + } + sc.IdentityInstanceName = identity.GetInstanceName() + sc.IdentityCertName = identity.GetCertificateName() + + // The `CommonTlsContext` contains a oneof field `validation_context_type`, + // which contains the `CertificateValidationContext` message in one of the + // following ways: + // - `validation_context` field + // - this is directly of type `CertificateValidationContext` + // - `combined_validation_context` field + // - this is of type `CombinedCertificateValidationContext` and contains + // a `default validation context` field of type + // `CertificateValidationContext` + // + // The `CertificateValidationContext` message has the following fields that + // we are interested in: + // - `ca_certificate_provider_instance` + // - this is of type `CertificateProviderPluginInstance` + // - `match_subject_alt_names` + // - this is a list of string matchers + // + // The `CertificateProviderPluginInstance` message contains two fields + // - instance_name + // - this is the certificate provider instance name to be looked up in + // the bootstrap configuration + // - certificate_name + // - this is an opaque name passed to the certificate provider + var validationCtx *v3tlspb.CertificateValidationContext + switch typ := common.GetValidationContextType().(type) { + case *v3tlspb.CommonTlsContext_ValidationContext: + validationCtx = common.GetValidationContext() + case *v3tlspb.CommonTlsContext_CombinedValidationContext: + validationCtx = common.GetCombinedValidationContext().GetDefaultValidationContext() + case nil: + // It is valid for the validation context to be nil on the server side. + return sc, nil + default: + return nil, fmt.Errorf("validation context contains unexpected type: %T", typ) + } + // If we get here, it means that the `CertificateValidationContext` message + // was found through one of the supported ways. It is an error if the + // validation context is specified, but it does not contain the + // ca_certificate_provider_instance field which contains information about + // the certificate provider to be used for the root certificates. + if validationCtx.GetCaCertificateProviderInstance() == nil { + return nil, fmt.Errorf("expected field ca_certificate_provider_instance is missing in CommonTlsContext message: %+v", common) + } + // The following fields are ignored: + // - trusted_ca + // - watched_directory + // - allow_expired_certificate + // - trust_chain_verification + switch { + case len(validationCtx.GetVerifyCertificateSpki()) != 0: + return nil, fmt.Errorf("unsupported verify_certificate_spki field in CommonTlsContext message: %+v", common) + case len(validationCtx.GetVerifyCertificateHash()) != 0: + return nil, fmt.Errorf("unsupported verify_certificate_hash field in CommonTlsContext message: %+v", common) + case validationCtx.GetRequireSignedCertificateTimestamp().GetValue(): + return nil, fmt.Errorf("unsupported require_sugned_ceritificate_timestamp field in CommonTlsContext message: %+v", common) + case validationCtx.GetCrl() != nil: + return nil, fmt.Errorf("unsupported crl field in CommonTlsContext message: %+v", common) + case validationCtx.GetCustomValidatorConfig() != nil: + return nil, fmt.Errorf("unsupported custom_validator_config field in CommonTlsContext message: %+v", common) + } + + if rootProvider := validationCtx.GetCaCertificateProviderInstance(); rootProvider != nil { + sc.RootInstanceName = rootProvider.GetInstanceName() + sc.RootCertName = rootProvider.GetCertificateName() + } + var matchers []matcher.StringMatcher + for _, m := range validationCtx.GetMatchSubjectAltNames() { + matcher, err := matcher.StringMatcherFromProto(m) + if err != nil { + return nil, err + } + matchers = append(matchers, matcher) + } + if server && len(matchers) != 0 { + return nil, fmt.Errorf("match_subject_alt_names field in validation context is not supported on the server: %v", common) + } + sc.SubjectAltNameMatchers = matchers + return sc, nil +} + +// circuitBreakersFromCluster extracts the circuit breakers configuration from +// the received cluster resource. Returns nil if no CircuitBreakers or no +// Thresholds in CircuitBreakers. +func circuitBreakersFromCluster(cluster *v3clusterpb.Cluster) *uint32 { + for _, threshold := range cluster.GetCircuitBreakers().GetThresholds() { + if threshold.GetPriority() != v3corepb.RoutingPriority_DEFAULT { + continue + } + maxRequestsPb := threshold.GetMaxRequests() + if maxRequestsPb == nil { + return nil + } + maxRequests := maxRequestsPb.GetValue() + return &maxRequests + } + return nil +} + +// idurationp takes a time.Duration and converts it to an internal duration, and +// returns a pointer to that internal duration. +func idurationp(d time.Duration) *iserviceconfig.Duration { + id := iserviceconfig.Duration(d) + return &id +} + +func uint32p(i uint32) *uint32 { + return &i +} + +// Helper types to prepare Outlier Detection JSON. Pointer types to distinguish +// between unset and a zero value. +type successRateEjection struct { + StdevFactor *uint32 `json:"stdevFactor,omitempty"` + EnforcementPercentage *uint32 `json:"enforcementPercentage,omitempty"` + MinimumHosts *uint32 `json:"minimumHosts,omitempty"` + RequestVolume *uint32 `json:"requestVolume,omitempty"` +} + +type failurePercentageEjection struct { + Threshold *uint32 `json:"threshold,omitempty"` + EnforcementPercentage *uint32 `json:"enforcementPercentage,omitempty"` + MinimumHosts *uint32 `json:"minimumHosts,omitempty"` + RequestVolume *uint32 `json:"requestVolume,omitempty"` +} + +type odLBConfig struct { + Interval *iserviceconfig.Duration `json:"interval,omitempty"` + BaseEjectionTime *iserviceconfig.Duration `json:"baseEjectionTime,omitempty"` + MaxEjectionTime *iserviceconfig.Duration `json:"maxEjectionTime,omitempty"` + MaxEjectionPercent *uint32 `json:"maxEjectionPercent,omitempty"` + SuccessRateEjection *successRateEjection `json:"successRateEjection,omitempty"` + FailurePercentageEjection *failurePercentageEjection `json:"failurePercentageEjection,omitempty"` +} + +// outlierConfigFromCluster converts the received Outlier Detection +// configuration into JSON configuration for Outlier Detection, taking into +// account xDS Defaults. Returns nil if no OutlierDetection field set in the +// cluster resource. +func outlierConfigFromCluster(cluster *v3clusterpb.Cluster) (json.RawMessage, error) { + od := cluster.GetOutlierDetection() + if od == nil { + return nil, nil + } + + // "The outlier_detection field of the Cluster resource should have its fields + // validated according to the rules for the corresponding LB policy config + // fields in the above "Validation" section. If any of these requirements is + // violated, the Cluster resource should be NACKed." - A50 + // "The google.protobuf.Duration fields interval, base_ejection_time, and + // max_ejection_time must obey the restrictions in the + // google.protobuf.Duration documentation and they must have non-negative + // values." - A50 + var interval *iserviceconfig.Duration + if i := od.GetInterval(); i != nil { + if err := i.CheckValid(); err != nil { + return nil, fmt.Errorf("outlier_detection.interval is invalid with error: %v", err) + } + if interval = idurationp(i.AsDuration()); *interval < 0 { + return nil, fmt.Errorf("outlier_detection.interval = %v; must be a valid duration and >= 0", *interval) + } + } + + var baseEjectionTime *iserviceconfig.Duration + if bet := od.GetBaseEjectionTime(); bet != nil { + if err := bet.CheckValid(); err != nil { + return nil, fmt.Errorf("outlier_detection.base_ejection_time is invalid with error: %v", err) + } + if baseEjectionTime = idurationp(bet.AsDuration()); *baseEjectionTime < 0 { + return nil, fmt.Errorf("outlier_detection.base_ejection_time = %v; must be >= 0", *baseEjectionTime) + } + } + + var maxEjectionTime *iserviceconfig.Duration + if met := od.GetMaxEjectionTime(); met != nil { + if err := met.CheckValid(); err != nil { + return nil, fmt.Errorf("outlier_detection.max_ejection_time is invalid: %v", err) + } + if maxEjectionTime = idurationp(met.AsDuration()); *maxEjectionTime < 0 { + return nil, fmt.Errorf("outlier_detection.max_ejection_time = %v; must be >= 0", *maxEjectionTime) + } + } + + // "The fields max_ejection_percent, enforcing_success_rate, + // failure_percentage_threshold, and enforcing_failure_percentage must have + // values less than or equal to 100. If any of these requirements is + // violated, the Cluster resource should be NACKed." - A50 + var maxEjectionPercent *uint32 + if mep := od.GetMaxEjectionPercent(); mep != nil { + if maxEjectionPercent = uint32p(mep.GetValue()); *maxEjectionPercent > 100 { + return nil, fmt.Errorf("outlier_detection.max_ejection_percent = %v; must be <= 100", *maxEjectionPercent) + } + } + // "if the enforcing_success_rate field is set to 0, the config + // success_rate_ejection field will be null and all success_rate_* fields + // will be ignored." - A50 + var enforcingSuccessRate *uint32 + if esr := od.GetEnforcingSuccessRate(); esr != nil { + if enforcingSuccessRate = uint32p(esr.GetValue()); *enforcingSuccessRate > 100 { + return nil, fmt.Errorf("outlier_detection.enforcing_success_rate = %v; must be <= 100", *enforcingSuccessRate) + } + } + var failurePercentageThreshold *uint32 + if fpt := od.GetFailurePercentageThreshold(); fpt != nil { + if failurePercentageThreshold = uint32p(fpt.GetValue()); *failurePercentageThreshold > 100 { + return nil, fmt.Errorf("outlier_detection.failure_percentage_threshold = %v; must be <= 100", *failurePercentageThreshold) + } + } + // "If the enforcing_failure_percent field is set to 0 or null, the config + // failure_percent_ejection field will be null and all failure_percent_* + // fields will be ignored." - A50 + var enforcingFailurePercentage *uint32 + if efp := od.GetEnforcingFailurePercentage(); efp != nil { + if enforcingFailurePercentage = uint32p(efp.GetValue()); *enforcingFailurePercentage > 100 { + return nil, fmt.Errorf("outlier_detection.enforcing_failure_percentage = %v; must be <= 100", *enforcingFailurePercentage) + } + } + + var successRateStdevFactor *uint32 + if srsf := od.GetSuccessRateStdevFactor(); srsf != nil { + successRateStdevFactor = uint32p(srsf.GetValue()) + } + var successRateMinimumHosts *uint32 + if srmh := od.GetSuccessRateMinimumHosts(); srmh != nil { + successRateMinimumHosts = uint32p(srmh.GetValue()) + } + var successRateRequestVolume *uint32 + if srrv := od.GetSuccessRateRequestVolume(); srrv != nil { + successRateRequestVolume = uint32p(srrv.GetValue()) + } + var failurePercentageMinimumHosts *uint32 + if fpmh := od.GetFailurePercentageMinimumHosts(); fpmh != nil { + failurePercentageMinimumHosts = uint32p(fpmh.GetValue()) + } + var failurePercentageRequestVolume *uint32 + if fprv := od.GetFailurePercentageRequestVolume(); fprv != nil { + failurePercentageRequestVolume = uint32p(fprv.GetValue()) + } + + // "if the enforcing_success_rate field is set to 0, the config + // success_rate_ejection field will be null and all success_rate_* fields + // will be ignored." - A50 + var sre *successRateEjection + if enforcingSuccessRate == nil || *enforcingSuccessRate != 0 { + sre = &successRateEjection{ + StdevFactor: successRateStdevFactor, + EnforcementPercentage: enforcingSuccessRate, + MinimumHosts: successRateMinimumHosts, + RequestVolume: successRateRequestVolume, + } + } + + // "If the enforcing_failure_percent field is set to 0 or null, the config + // failure_percent_ejection field will be null and all failure_percent_* + // fields will be ignored." - A50 + var fpe *failurePercentageEjection + if enforcingFailurePercentage != nil && *enforcingFailurePercentage != 0 { + fpe = &failurePercentageEjection{ + Threshold: failurePercentageThreshold, + EnforcementPercentage: enforcingFailurePercentage, + MinimumHosts: failurePercentageMinimumHosts, + RequestVolume: failurePercentageRequestVolume, + } + } + + odLBCfg := &odLBConfig{ + Interval: interval, + BaseEjectionTime: baseEjectionTime, + MaxEjectionTime: maxEjectionTime, + MaxEjectionPercent: maxEjectionPercent, + SuccessRateEjection: sre, + FailurePercentageEjection: fpe, + } + return json.Marshal(odLBCfg) +} diff --git a/xds/internal/xdsclient/xdsresource/unmarshal_cds_test.go b/xds/internal/xdsclient/xdsresource/unmarshal_cds_test.go new file mode 100644 index 000000000000..f54935e741e3 --- /dev/null +++ b/xds/internal/xdsclient/xdsresource/unmarshal_cds_test.go @@ -0,0 +1,1574 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "encoding/json" + "regexp" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/xds/matcher" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource/version" + + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3aggregateclusterpb "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/aggregate/v3" + v3leastrequestpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/least_request/v3" + v3ringhashpb "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/ring_hash/v3" + v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3" + anypb "github.com/golang/protobuf/ptypes/any" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +const ( + clusterName = "clusterName" + serviceName = "service" +) + +var emptyUpdate = ClusterUpdate{ClusterName: clusterName, LRSServerConfig: ClusterLRSOff} + +func (s) TestValidateCluster_Failure(t *testing.T) { + oldCustomLBSupport := envconfig.XDSCustomLBPolicy + envconfig.XDSCustomLBPolicy = true + defer func() { + envconfig.XDSCustomLBPolicy = oldCustomLBSupport + }() + tests := []struct { + name string + cluster *v3clusterpb.Cluster + wantUpdate ClusterUpdate + wantErr bool + }{ + { + name: "non-supported-cluster-type-static", + cluster: &v3clusterpb.Cluster{ + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_STATIC}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + }, + LbPolicy: v3clusterpb.Cluster_LEAST_REQUEST, + }, + wantUpdate: emptyUpdate, + wantErr: true, + }, + { + name: "non-supported-cluster-type-original-dst", + cluster: &v3clusterpb.Cluster{ + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_ORIGINAL_DST}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + }, + LbPolicy: v3clusterpb.Cluster_LEAST_REQUEST, + }, + wantUpdate: emptyUpdate, + wantErr: true, + }, + { + name: "no-eds-config", + cluster: &v3clusterpb.Cluster{ + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + }, + wantUpdate: emptyUpdate, + wantErr: true, + }, + { + name: "no-ads-config-source", + cluster: &v3clusterpb.Cluster{ + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{}, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + }, + wantUpdate: emptyUpdate, + wantErr: true, + }, + { + name: "non-round-robin-or-ring-hash-lb-policy", + cluster: &v3clusterpb.Cluster{ + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + }, + LbPolicy: v3clusterpb.Cluster_LEAST_REQUEST, + }, + wantUpdate: emptyUpdate, + wantErr: true, + }, + { + name: "logical-dns-multiple-localities", + cluster: &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_LOGICAL_DNS}, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + LoadAssignment: &v3endpointpb.ClusterLoadAssignment{ + Endpoints: []*v3endpointpb.LocalityLbEndpoints{ + // Invalid if there are more than one locality. + {LbEndpoints: nil}, + {LbEndpoints: nil}, + }, + }, + }, + wantUpdate: emptyUpdate, + wantErr: true, + }, + { + name: "ring-hash-hash-function-not-xx-hash", + cluster: &v3clusterpb.Cluster{ + LbPolicy: v3clusterpb.Cluster_RING_HASH, + LbConfig: &v3clusterpb.Cluster_RingHashLbConfig_{ + RingHashLbConfig: &v3clusterpb.Cluster_RingHashLbConfig{ + HashFunction: v3clusterpb.Cluster_RingHashLbConfig_MURMUR_HASH_2, + }, + }, + }, + wantUpdate: emptyUpdate, + wantErr: true, + }, + { + name: "least-request-choice-count-less-than-two", + cluster: &v3clusterpb.Cluster{ + LbPolicy: v3clusterpb.Cluster_RING_HASH, + LbConfig: &v3clusterpb.Cluster_LeastRequestLbConfig_{ + LeastRequestLbConfig: &v3clusterpb.Cluster_LeastRequestLbConfig{ + ChoiceCount: wrapperspb.UInt32(1), + }, + }, + }, + wantUpdate: emptyUpdate, + wantErr: true, + }, + { + name: "ring-hash-max-bound-greater-than-upper-bound", + cluster: &v3clusterpb.Cluster{ + LbPolicy: v3clusterpb.Cluster_RING_HASH, + LbConfig: &v3clusterpb.Cluster_RingHashLbConfig_{ + RingHashLbConfig: &v3clusterpb.Cluster_RingHashLbConfig{ + MaximumRingSize: wrapperspb.UInt64(ringHashSizeUpperBound + 1), + }, + }, + }, + wantUpdate: emptyUpdate, + wantErr: true, + }, + { + name: "ring-hash-max-bound-greater-than-upper-bound-load-balancing-policy", + cluster: &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LoadBalancingPolicy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: testutils.MarshalAny(&v3ringhashpb.RingHash{ + HashFunction: v3ringhashpb.RingHash_XX_HASH, + MinimumRingSize: wrapperspb.UInt64(10), + MaximumRingSize: wrapperspb.UInt64(ringHashSizeUpperBound + 1), + }), + }, + }, + }, + }, + }, + wantUpdate: emptyUpdate, + wantErr: true, + }, + { + name: "least-request-unsupported-in-converter-since-env-var-unset", + cluster: &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LoadBalancingPolicy: &v3clusterpb.LoadBalancingPolicy{ + Policies: []*v3clusterpb.LoadBalancingPolicy_Policy{ + { + TypedExtensionConfig: &v3corepb.TypedExtensionConfig{ + TypedConfig: testutils.MarshalAny(&v3leastrequestpb.LeastRequest{}), + }, + }, + }, + }, + }, + wantUpdate: emptyUpdate, + wantErr: true, + }, + { + name: "aggregate-nil-clusters", + cluster: &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_ClusterType{ + ClusterType: &v3clusterpb.Cluster_CustomClusterType{ + Name: "envoy.clusters.aggregate", + TypedConfig: testutils.MarshalAny(&v3aggregateclusterpb.ClusterConfig{}), + }, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + }, + wantUpdate: emptyUpdate, + wantErr: true, + }, + { + name: "aggregate-empty-clusters", + cluster: &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_ClusterType{ + ClusterType: &v3clusterpb.Cluster_CustomClusterType{ + Name: "envoy.clusters.aggregate", + TypedConfig: testutils.MarshalAny(&v3aggregateclusterpb.ClusterConfig{ + Clusters: []string{}, + }), + }, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + }, + wantUpdate: emptyUpdate, + wantErr: true, + }, + } + + oldAggregateAndDNSSupportEnv := envconfig.XDSAggregateAndDNS + envconfig.XDSAggregateAndDNS = true + defer func() { envconfig.XDSAggregateAndDNS = oldAggregateAndDNSSupportEnv }() + oldRingHashSupport := envconfig.XDSRingHash + envconfig.XDSRingHash = true + defer func() { envconfig.XDSRingHash = oldRingHashSupport }() + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if update, err := validateClusterAndConstructClusterUpdate(test.cluster); err == nil { + t.Errorf("validateClusterAndConstructClusterUpdate(%+v) = %v, wanted error", test.cluster, update) + } + }) + } +} + +func (s) TestValidateClusterWithSecurityConfig_EnvVarOff(t *testing.T) { + // Turn off the env var protection for client-side security. + origClientSideSecurityEnvVar := envconfig.XDSClientSideSecurity + envconfig.XDSClientSideSecurity = false + defer func() { envconfig.XDSClientSideSecurity = origClientSideSecurityEnvVar }() + + cluster := &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{ + ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: "rootInstance", + CertificateName: "rootCert", + }, + }, + }, + }), + }, + }, + } + wantUpdate := ClusterUpdate{ + ClusterName: clusterName, + EDSServiceName: serviceName, + LRSServerConfig: ClusterLRSOff, + } + gotUpdate, err := validateClusterAndConstructClusterUpdate(cluster) + if err != nil { + t.Errorf("validateClusterAndConstructClusterUpdate() failed: %v", err) + } + if diff := cmp.Diff(wantUpdate, gotUpdate, cmpopts.IgnoreFields(ClusterUpdate{}, "LBPolicy")); diff != "" { + t.Errorf("validateClusterAndConstructClusterUpdate() returned unexpected diff (-want, got):\n%s", diff) + } +} + +func (s) TestSecurityConfigFromCommonTLSContextUsingNewFields_ErrorCases(t *testing.T) { + tests := []struct { + name string + common *v3tlspb.CommonTlsContext + server bool + wantErr string + }{ + { + name: "unsupported-tls_certificates-field-for-identity-certs", + common: &v3tlspb.CommonTlsContext{ + TlsCertificates: []*v3tlspb.TlsCertificate{ + {CertificateChain: &v3corepb.DataSource{}}, + }, + }, + wantErr: "unsupported field tls_certificates is set in CommonTlsContext message", + }, + { + name: "unsupported-tls_certificates_sds_secret_configs-field-for-identity-certs", + common: &v3tlspb.CommonTlsContext{ + TlsCertificateSdsSecretConfigs: []*v3tlspb.SdsSecretConfig{ + {Name: "sds-secrets-config"}, + }, + }, + wantErr: "unsupported field tls_certificate_sds_secret_configs is set in CommonTlsContext message", + }, + { + name: "unsupported-sds-validation-context", + common: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextSdsSecretConfig{ + ValidationContextSdsSecretConfig: &v3tlspb.SdsSecretConfig{ + Name: "foo-sds-secret", + }, + }, + }, + wantErr: "validation context contains unexpected type", + }, + { + name: "missing-ca_certificate_provider_instance-in-validation-context", + common: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{}, + }, + }, + wantErr: "expected field ca_certificate_provider_instance is missing in CommonTlsContext message", + }, + { + name: "unsupported-field-verify_certificate_spki-in-validation-context", + common: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{ + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: "rootPluginInstance", + CertificateName: "rootCertName", + }, + VerifyCertificateSpki: []string{"spki"}, + }, + }, + }, + wantErr: "unsupported verify_certificate_spki field in CommonTlsContext message", + }, + { + name: "unsupported-field-verify_certificate_hash-in-validation-context", + common: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{ + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: "rootPluginInstance", + CertificateName: "rootCertName", + }, + VerifyCertificateHash: []string{"hash"}, + }, + }, + }, + wantErr: "unsupported verify_certificate_hash field in CommonTlsContext message", + }, + { + name: "unsupported-field-require_signed_certificate_timestamp-in-validation-context", + common: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{ + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: "rootPluginInstance", + CertificateName: "rootCertName", + }, + RequireSignedCertificateTimestamp: &wrapperspb.BoolValue{Value: true}, + }, + }, + }, + wantErr: "unsupported require_sugned_ceritificate_timestamp field in CommonTlsContext message", + }, + { + name: "unsupported-field-crl-in-validation-context", + common: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{ + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: "rootPluginInstance", + CertificateName: "rootCertName", + }, + Crl: &v3corepb.DataSource{}, + }, + }, + }, + wantErr: "unsupported crl field in CommonTlsContext message", + }, + { + name: "unsupported-field-custom_validator_config-in-validation-context", + common: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{ + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: "rootPluginInstance", + CertificateName: "rootCertName", + }, + CustomValidatorConfig: &v3corepb.TypedExtensionConfig{}, + }, + }, + }, + wantErr: "unsupported custom_validator_config field in CommonTlsContext message", + }, + { + name: "invalid-match_subject_alt_names-field-in-validation-context", + common: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{ + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: "rootPluginInstance", + CertificateName: "rootCertName", + }, + MatchSubjectAltNames: []*v3matcherpb.StringMatcher{ + {MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: ""}}, + }, + }, + }, + }, + wantErr: "empty prefix is not allowed in StringMatcher", + }, + { + name: "unsupported-field-matching-subject-alt-names-in-validation-context-of-server", + common: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{ + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: "rootPluginInstance", + CertificateName: "rootCertName", + }, + MatchSubjectAltNames: []*v3matcherpb.StringMatcher{ + {MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: "sanPrefix"}}, + }, + }, + }, + }, + server: true, + wantErr: "match_subject_alt_names field in validation context is not supported on the server", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := securityConfigFromCommonTLSContextUsingNewFields(test.common, test.server) + if err == nil { + t.Fatal("securityConfigFromCommonTLSContextUsingNewFields() succeeded when expected to fail") + } + if !strings.Contains(err.Error(), test.wantErr) { + t.Fatalf("securityConfigFromCommonTLSContextUsingNewFields() returned err: %v, wantErr: %v", err, test.wantErr) + } + }) + } +} + +func (s) TestValidateClusterWithSecurityConfig(t *testing.T) { + const ( + identityPluginInstance = "identityPluginInstance" + identityCertName = "identityCert" + rootPluginInstance = "rootPluginInstance" + rootCertName = "rootCert" + clusterName = "cluster" + serviceName = "service" + sanExact = "san-exact" + sanPrefix = "san-prefix" + sanSuffix = "san-suffix" + sanRegexBad = "??" + sanRegexGood = "san?regex?" + sanContains = "san-contains" + ) + var sanRE = regexp.MustCompile(sanRegexGood) + + tests := []struct { + name string + cluster *v3clusterpb.Cluster + wantUpdate ClusterUpdate + wantErr bool + }{ + { + name: "transport-socket-matches", + cluster: &v3clusterpb.Cluster{ + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocketMatches: []*v3clusterpb.Cluster_TransportSocketMatch{ + {Name: "transport-socket-match-1"}, + }, + }, + wantErr: true, + }, + { + name: "transport-socket-unsupported-name", + cluster: &v3clusterpb.Cluster{ + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + Name: "unsupported-foo", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: &anypb.Any{ + TypeUrl: version.V3UpstreamTLSContextURL, + }, + }, + }, + }, + wantErr: true, + }, + { + name: "transport-socket-unsupported-typeURL", + cluster: &v3clusterpb.Cluster{ + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: &anypb.Any{ + TypeUrl: version.V3HTTPConnManagerURL, + }, + }, + }, + }, + wantErr: true, + }, + { + name: "transport-socket-unsupported-type", + cluster: &v3clusterpb.Cluster{ + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: &anypb.Any{ + TypeUrl: version.V3UpstreamTLSContextURL, + Value: []byte{1, 2, 3, 4}, + }, + }, + }, + }, + wantErr: true, + }, + { + name: "transport-socket-unsupported-tls-params-field", + cluster: &v3clusterpb.Cluster{ + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsParams: &v3tlspb.TlsParameters{}, + }, + }), + }, + }, + }, + wantErr: true, + }, + { + name: "transport-socket-unsupported-custom-handshaker-field", + cluster: &v3clusterpb.Cluster{ + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + CustomHandshaker: &v3corepb.TypedExtensionConfig{}, + }, + }), + }, + }, + }, + wantErr: true, + }, + { + name: "transport-socket-unsupported-validation-context", + cluster: &v3clusterpb.Cluster{ + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextSdsSecretConfig{ + ValidationContextSdsSecretConfig: &v3tlspb.SdsSecretConfig{ + Name: "foo-sds-secret", + }, + }, + }, + }), + }, + }, + }, + wantErr: true, + }, + { + name: "transport-socket-without-validation-context", + cluster: &v3clusterpb.Cluster{ + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{}, + }), + }, + }, + }, + wantErr: true, + }, + { + name: "empty-prefix-in-matching-SAN", + cluster: &v3clusterpb.Cluster{ + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_CombinedValidationContext{ + CombinedValidationContext: &v3tlspb.CommonTlsContext_CombinedCertificateValidationContext{ + DefaultValidationContext: &v3tlspb.CertificateValidationContext{ + MatchSubjectAltNames: []*v3matcherpb.StringMatcher{ + {MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: ""}}, + }, + }, + ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: rootPluginInstance, + CertificateName: rootCertName, + }, + }, + }, + }, + }), + }, + }, + }, + wantErr: true, + }, + { + name: "empty-suffix-in-matching-SAN", + cluster: &v3clusterpb.Cluster{ + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_CombinedValidationContext{ + CombinedValidationContext: &v3tlspb.CommonTlsContext_CombinedCertificateValidationContext{ + DefaultValidationContext: &v3tlspb.CertificateValidationContext{ + MatchSubjectAltNames: []*v3matcherpb.StringMatcher{ + {MatchPattern: &v3matcherpb.StringMatcher_Suffix{Suffix: ""}}, + }, + }, + ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: rootPluginInstance, + CertificateName: rootCertName, + }, + }, + }, + }, + }), + }, + }, + }, + wantErr: true, + }, + { + name: "empty-contains-in-matching-SAN", + cluster: &v3clusterpb.Cluster{ + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_CombinedValidationContext{ + CombinedValidationContext: &v3tlspb.CommonTlsContext_CombinedCertificateValidationContext{ + DefaultValidationContext: &v3tlspb.CertificateValidationContext{ + MatchSubjectAltNames: []*v3matcherpb.StringMatcher{ + {MatchPattern: &v3matcherpb.StringMatcher_Contains{Contains: ""}}, + }, + }, + ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: rootPluginInstance, + CertificateName: rootCertName, + }, + }, + }, + }, + }), + }, + }, + }, + wantErr: true, + }, + { + name: "invalid-regex-in-matching-SAN", + cluster: &v3clusterpb.Cluster{ + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_CombinedValidationContext{ + CombinedValidationContext: &v3tlspb.CommonTlsContext_CombinedCertificateValidationContext{ + DefaultValidationContext: &v3tlspb.CertificateValidationContext{ + MatchSubjectAltNames: []*v3matcherpb.StringMatcher{ + {MatchPattern: &v3matcherpb.StringMatcher_SafeRegex{SafeRegex: &v3matcherpb.RegexMatcher{Regex: sanRegexBad}}}, + }, + }, + ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: rootPluginInstance, + CertificateName: rootCertName, + }, + }, + }, + }, + }), + }, + }, + }, + wantErr: true, + }, + { + name: "invalid-regex-in-matching-SAN-with-new-fields", + cluster: &v3clusterpb.Cluster{ + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_CombinedValidationContext{ + CombinedValidationContext: &v3tlspb.CommonTlsContext_CombinedCertificateValidationContext{ + DefaultValidationContext: &v3tlspb.CertificateValidationContext{ + MatchSubjectAltNames: []*v3matcherpb.StringMatcher{ + {MatchPattern: &v3matcherpb.StringMatcher_SafeRegex{SafeRegex: &v3matcherpb.RegexMatcher{Regex: sanRegexBad}}}, + }, + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: rootPluginInstance, + CertificateName: rootCertName, + }, + }, + }, + }, + }, + }), + }, + }, + }, + wantErr: true, + }, + { + name: "happy-case-with-no-identity-certs-using-deprecated-fields", + cluster: &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{ + ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: rootPluginInstance, + CertificateName: rootCertName, + }, + }, + }, + }), + }, + }, + }, + wantUpdate: ClusterUpdate{ + ClusterName: clusterName, + EDSServiceName: serviceName, + LRSServerConfig: ClusterLRSOff, + SecurityCfg: &SecurityConfig{ + RootInstanceName: rootPluginInstance, + RootCertName: rootCertName, + }, + }, + }, + { + name: "happy-case-with-no-identity-certs-using-new-fields", + cluster: &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{ + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: rootPluginInstance, + CertificateName: rootCertName, + }, + }, + }, + }, + }), + }, + }, + }, + wantUpdate: ClusterUpdate{ + ClusterName: clusterName, + EDSServiceName: serviceName, + LRSServerConfig: ClusterLRSOff, + SecurityCfg: &SecurityConfig{ + RootInstanceName: rootPluginInstance, + RootCertName: rootCertName, + }, + }, + }, + { + name: "happy-case-with-validation-context-provider-instance-using-deprecated-fields", + cluster: &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: identityPluginInstance, + CertificateName: identityCertName, + }, + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{ + ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: rootPluginInstance, + CertificateName: rootCertName, + }, + }, + }, + }), + }, + }, + }, + wantUpdate: ClusterUpdate{ + ClusterName: clusterName, + EDSServiceName: serviceName, + LRSServerConfig: ClusterLRSOff, + SecurityCfg: &SecurityConfig{ + RootInstanceName: rootPluginInstance, + RootCertName: rootCertName, + IdentityInstanceName: identityPluginInstance, + IdentityCertName: identityCertName, + }, + }, + }, + { + name: "happy-case-with-validation-context-provider-instance-using-new-fields", + cluster: &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: identityPluginInstance, + CertificateName: identityCertName, + }, + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{ + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: rootPluginInstance, + CertificateName: rootCertName, + }, + }, + }, + }, + }), + }, + }, + }, + wantUpdate: ClusterUpdate{ + ClusterName: clusterName, + EDSServiceName: serviceName, + LRSServerConfig: ClusterLRSOff, + SecurityCfg: &SecurityConfig{ + RootInstanceName: rootPluginInstance, + RootCertName: rootCertName, + IdentityInstanceName: identityPluginInstance, + IdentityCertName: identityCertName, + }, + }, + }, + { + name: "happy-case-with-combined-validation-context-using-deprecated-fields", + cluster: &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: identityPluginInstance, + CertificateName: identityCertName, + }, + ValidationContextType: &v3tlspb.CommonTlsContext_CombinedValidationContext{ + CombinedValidationContext: &v3tlspb.CommonTlsContext_CombinedCertificateValidationContext{ + DefaultValidationContext: &v3tlspb.CertificateValidationContext{ + MatchSubjectAltNames: []*v3matcherpb.StringMatcher{ + { + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: sanExact}, + IgnoreCase: true, + }, + {MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: sanPrefix}}, + {MatchPattern: &v3matcherpb.StringMatcher_Suffix{Suffix: sanSuffix}}, + {MatchPattern: &v3matcherpb.StringMatcher_SafeRegex{SafeRegex: &v3matcherpb.RegexMatcher{Regex: sanRegexGood}}}, + {MatchPattern: &v3matcherpb.StringMatcher_Contains{Contains: sanContains}}, + }, + }, + ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: rootPluginInstance, + CertificateName: rootCertName, + }, + }, + }, + }, + }), + }, + }, + }, + wantUpdate: ClusterUpdate{ + ClusterName: clusterName, + EDSServiceName: serviceName, + LRSServerConfig: ClusterLRSOff, + SecurityCfg: &SecurityConfig{ + RootInstanceName: rootPluginInstance, + RootCertName: rootCertName, + IdentityInstanceName: identityPluginInstance, + IdentityCertName: identityCertName, + SubjectAltNameMatchers: []matcher.StringMatcher{ + matcher.StringMatcherForTesting(newStringP(sanExact), nil, nil, nil, nil, true), + matcher.StringMatcherForTesting(nil, newStringP(sanPrefix), nil, nil, nil, false), + matcher.StringMatcherForTesting(nil, nil, newStringP(sanSuffix), nil, nil, false), + matcher.StringMatcherForTesting(nil, nil, nil, nil, sanRE, false), + matcher.StringMatcherForTesting(nil, nil, nil, newStringP(sanContains), nil, false), + }, + }, + }, + }, + { + name: "happy-case-with-combined-validation-context-using-new-fields", + cluster: &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: serviceName, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.UpstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: identityPluginInstance, + CertificateName: identityCertName, + }, + ValidationContextType: &v3tlspb.CommonTlsContext_CombinedValidationContext{ + CombinedValidationContext: &v3tlspb.CommonTlsContext_CombinedCertificateValidationContext{ + DefaultValidationContext: &v3tlspb.CertificateValidationContext{ + MatchSubjectAltNames: []*v3matcherpb.StringMatcher{ + { + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: sanExact}, + IgnoreCase: true, + }, + {MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: sanPrefix}}, + {MatchPattern: &v3matcherpb.StringMatcher_Suffix{Suffix: sanSuffix}}, + {MatchPattern: &v3matcherpb.StringMatcher_SafeRegex{SafeRegex: &v3matcherpb.RegexMatcher{Regex: sanRegexGood}}}, + {MatchPattern: &v3matcherpb.StringMatcher_Contains{Contains: sanContains}}, + }, + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: rootPluginInstance, + CertificateName: rootCertName, + }, + }, + }, + }, + }, + }), + }, + }, + }, + wantUpdate: ClusterUpdate{ + ClusterName: clusterName, + EDSServiceName: serviceName, + LRSServerConfig: ClusterLRSOff, + SecurityCfg: &SecurityConfig{ + RootInstanceName: rootPluginInstance, + RootCertName: rootCertName, + IdentityInstanceName: identityPluginInstance, + IdentityCertName: identityCertName, + SubjectAltNameMatchers: []matcher.StringMatcher{ + matcher.StringMatcherForTesting(newStringP(sanExact), nil, nil, nil, nil, true), + matcher.StringMatcherForTesting(nil, newStringP(sanPrefix), nil, nil, nil, false), + matcher.StringMatcherForTesting(nil, nil, newStringP(sanSuffix), nil, nil, false), + matcher.StringMatcherForTesting(nil, nil, nil, nil, sanRE, false), + matcher.StringMatcherForTesting(nil, nil, nil, newStringP(sanContains), nil, false), + }, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + update, err := validateClusterAndConstructClusterUpdate(test.cluster) + if (err != nil) != test.wantErr { + t.Errorf("validateClusterAndConstructClusterUpdate() returned err %v wantErr %v)", err, test.wantErr) + } + if diff := cmp.Diff(test.wantUpdate, update, cmpopts.EquateEmpty(), cmp.AllowUnexported(regexp.Regexp{}), cmpopts.IgnoreFields(ClusterUpdate{}, "LBPolicy")); diff != "" { + t.Errorf("validateClusterAndConstructClusterUpdate() returned unexpected diff (-want, +got):\n%s", diff) + } + }) + } +} + +func (s) TestUnmarshalCluster(t *testing.T) { + const ( + v3ClusterName = "v3clusterName" + v3Service = "v3Service" + ) + var ( + v3ClusterAny = testutils.MarshalAny(&v3clusterpb.Cluster{ + Name: v3ClusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: v3Service, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + LrsServer: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Self{ + Self: &v3corepb.SelfConfigSource{}, + }, + }, + }) + + v3ClusterAnyWithEDSConfigSourceSelf = testutils.MarshalAny(&v3clusterpb.Cluster{ + Name: v3ClusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Self{}, + }, + ServiceName: v3Service, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + LrsServer: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Self{ + Self: &v3corepb.SelfConfigSource{}, + }, + }, + }) + ) + + tests := []struct { + name string + resource *anypb.Any + wantName string + wantUpdate ClusterUpdate + wantErr bool + }{ + { + name: "non-cluster resource type", + resource: &anypb.Any{TypeUrl: version.V3HTTPConnManagerURL}, + wantErr: true, + }, + { + name: "badly marshaled cluster resource", + resource: &anypb.Any{ + TypeUrl: version.V3ClusterURL, + Value: []byte{1, 2, 3, 4}, + }, + wantErr: true, + }, + { + name: "bad cluster resource", + resource: testutils.MarshalAny(&v3clusterpb.Cluster{ + Name: "test", + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_STATIC}, + }), + wantName: "test", + wantErr: true, + }, + { + name: "cluster resource with non-self lrs_server field", + resource: testutils.MarshalAny(&v3clusterpb.Cluster{ + Name: "test", + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: v3Service, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + LrsServer: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + }), + wantName: "test", + wantErr: true, + }, + { + name: "v3 cluster", + resource: v3ClusterAny, + wantName: v3ClusterName, + wantUpdate: ClusterUpdate{ + ClusterName: v3ClusterName, + EDSServiceName: v3Service, LRSServerConfig: ClusterLRSServerSelf, + Raw: v3ClusterAny, + }, + }, + { + name: "v3 cluster wrapped", + resource: testutils.MarshalAny(&v3discoverypb.Resource{Resource: v3ClusterAny}), + wantName: v3ClusterName, + wantUpdate: ClusterUpdate{ + ClusterName: v3ClusterName, + EDSServiceName: v3Service, LRSServerConfig: ClusterLRSServerSelf, + Raw: v3ClusterAny, + }, + }, + { + name: "v3 cluster with EDS config source self", + resource: v3ClusterAnyWithEDSConfigSourceSelf, + wantName: v3ClusterName, + wantUpdate: ClusterUpdate{ + ClusterName: v3ClusterName, + EDSServiceName: v3Service, LRSServerConfig: ClusterLRSServerSelf, + Raw: v3ClusterAnyWithEDSConfigSourceSelf, + }, + }, + { + name: "xdstp cluster resource with unset EDS service name", + resource: testutils.MarshalAny(&v3clusterpb.Cluster{ + Name: "xdstp:foo", + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + ServiceName: "", + }, + }), + wantName: "xdstp:foo", + wantErr: true, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + name, update, err := unmarshalClusterResource(test.resource) + if (err != nil) != test.wantErr { + t.Fatalf("unmarshalClusterResource(%s), got err: %v, wantErr: %v", pretty.ToJSON(test.resource), err, test.wantErr) + } + if name != test.wantName { + t.Errorf("unmarshalClusterResource(%s), got name: %s, want: %s", pretty.ToJSON(test.resource), name, test.wantName) + } + if diff := cmp.Diff(update, test.wantUpdate, cmpOpts, cmpopts.IgnoreFields(ClusterUpdate{}, "LBPolicy")); diff != "" { + t.Errorf("unmarshalClusterResource(%s), got unexpected update, diff (-got +want): %v", pretty.ToJSON(test.resource), diff) + } + }) + } +} + +func (s) TestValidateClusterWithOutlierDetection(t *testing.T) { + odToClusterProto := func(od *v3clusterpb.OutlierDetection) *v3clusterpb.Cluster { + // Cluster parsing doesn't fail with respect to fields orthogonal to + // outlier detection. + return &v3clusterpb.Cluster{ + Name: clusterName, + ClusterDiscoveryType: &v3clusterpb.Cluster_Type{Type: v3clusterpb.Cluster_EDS}, + EdsClusterConfig: &v3clusterpb.Cluster_EdsClusterConfig{ + EdsConfig: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{ + Ads: &v3corepb.AggregatedConfigSource{}, + }, + }, + }, + LbPolicy: v3clusterpb.Cluster_ROUND_ROBIN, + OutlierDetection: od, + } + } + + tests := []struct { + name string + cluster *v3clusterpb.Cluster + wantODCfg string + wantErr bool + }{ + { + name: "success-and-failure-null", + cluster: odToClusterProto(&v3clusterpb.OutlierDetection{}), + wantODCfg: `{"successRateEjection": {}}`, + }, + { + name: "success-and-failure-zero", + cluster: odToClusterProto(&v3clusterpb.OutlierDetection{ + EnforcingSuccessRate: &wrapperspb.UInt32Value{Value: 0}, // Thus doesn't create sre - to focus on fpe + EnforcingFailurePercentage: &wrapperspb.UInt32Value{Value: 0}, + }), + wantODCfg: `{}`, + }, + { + name: "some-fields-set", + cluster: odToClusterProto(&v3clusterpb.OutlierDetection{ + Interval: &durationpb.Duration{Seconds: 1}, + MaxEjectionTime: &durationpb.Duration{Seconds: 3}, + EnforcingSuccessRate: &wrapperspb.UInt32Value{Value: 3}, + SuccessRateRequestVolume: &wrapperspb.UInt32Value{Value: 5}, + EnforcingFailurePercentage: &wrapperspb.UInt32Value{Value: 7}, + FailurePercentageRequestVolume: &wrapperspb.UInt32Value{Value: 9}, + }), + wantODCfg: `{ + "interval": "1s", + "maxEjectionTime": "3s", + "successRateEjection": { + "enforcementPercentage": 3, + "requestVolume": 5 + }, + "failurePercentageEjection": { + "enforcementPercentage": 7, + "requestVolume": 9 + } + }`, + }, + { + name: "every-field-set-non-zero", + cluster: odToClusterProto(&v3clusterpb.OutlierDetection{ + // all fields set (including ones that will be layered) should + // pick up those too and explicitly all fields, including those + // put in layers, in the JSON generated. + Interval: &durationpb.Duration{Seconds: 1}, + BaseEjectionTime: &durationpb.Duration{Seconds: 2}, + MaxEjectionTime: &durationpb.Duration{Seconds: 3}, + MaxEjectionPercent: &wrapperspb.UInt32Value{Value: 1}, + SuccessRateStdevFactor: &wrapperspb.UInt32Value{Value: 2}, + EnforcingSuccessRate: &wrapperspb.UInt32Value{Value: 3}, + SuccessRateMinimumHosts: &wrapperspb.UInt32Value{Value: 4}, + SuccessRateRequestVolume: &wrapperspb.UInt32Value{Value: 5}, + FailurePercentageThreshold: &wrapperspb.UInt32Value{Value: 6}, + EnforcingFailurePercentage: &wrapperspb.UInt32Value{Value: 7}, + FailurePercentageMinimumHosts: &wrapperspb.UInt32Value{Value: 8}, + FailurePercentageRequestVolume: &wrapperspb.UInt32Value{Value: 9}, + }), + wantODCfg: `{ + "interval": "1s", + "baseEjectionTime": "2s", + "maxEjectionTime": "3s", + "maxEjectionPercent": 1, + "successRateEjection": { + "stdevFactor": 2, + "enforcementPercentage": 3, + "minimumHosts": 4, + "requestVolume": 5 + }, + "failurePercentageEjection": { + "threshold": 6, + "enforcementPercentage": 7, + "minimumHosts": 8, + "requestVolume": 9 + } + }`, + }, + { + name: "interval-is-negative", + cluster: odToClusterProto(&v3clusterpb.OutlierDetection{Interval: &durationpb.Duration{Seconds: -10}}), + wantErr: true, + }, + { + name: "interval-overflows", + cluster: odToClusterProto(&v3clusterpb.OutlierDetection{Interval: &durationpb.Duration{Seconds: 315576000001}}), + wantErr: true, + }, + { + name: "base-ejection-time-is-negative", + cluster: odToClusterProto(&v3clusterpb.OutlierDetection{BaseEjectionTime: &durationpb.Duration{Seconds: -10}}), + wantErr: true, + }, + { + name: "base-ejection-time-overflows", + cluster: odToClusterProto(&v3clusterpb.OutlierDetection{BaseEjectionTime: &durationpb.Duration{Seconds: 315576000001}}), + wantErr: true, + }, + { + name: "max-ejection-time-is-negative", + cluster: odToClusterProto(&v3clusterpb.OutlierDetection{MaxEjectionTime: &durationpb.Duration{Seconds: -10}}), + wantErr: true, + }, + { + name: "max-ejection-time-overflows", + cluster: odToClusterProto(&v3clusterpb.OutlierDetection{MaxEjectionTime: &durationpb.Duration{Seconds: 315576000001}}), + wantErr: true, + }, + { + name: "max-ejection-percent-is-greater-than-100", + cluster: odToClusterProto(&v3clusterpb.OutlierDetection{MaxEjectionPercent: &wrapperspb.UInt32Value{Value: 150}}), + wantErr: true, + }, + { + name: "enforcing-success-rate-is-greater-than-100", + cluster: odToClusterProto(&v3clusterpb.OutlierDetection{EnforcingSuccessRate: &wrapperspb.UInt32Value{Value: 150}}), + wantErr: true, + }, + { + name: "failure-percentage-threshold-is-greater-than-100", + cluster: odToClusterProto(&v3clusterpb.OutlierDetection{FailurePercentageThreshold: &wrapperspb.UInt32Value{Value: 150}}), + wantErr: true, + }, + { + name: "enforcing-failure-percentage-is-greater-than-100", + cluster: odToClusterProto(&v3clusterpb.OutlierDetection{EnforcingFailurePercentage: &wrapperspb.UInt32Value{Value: 150}}), + wantErr: true, + }, + // A Outlier Detection proto not present should lead to a nil + // OutlierDetection field in the ClusterUpdate, which is implicitly + // tested in every other test in this file. + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + update, err := validateClusterAndConstructClusterUpdate(test.cluster) + if (err != nil) != test.wantErr { + t.Errorf("validateClusterAndConstructClusterUpdate() returned err %v wantErr %v)", err, test.wantErr) + } + if test.wantErr { + return + } + // got and want must be unmarshalled since JSON strings shouldn't + // generally be directly compared. + var got map[string]any + if err := json.Unmarshal(update.OutlierDetection, &got); err != nil { + t.Fatalf("Error unmarshalling update.OutlierDetection (%q): %v", update.OutlierDetection, err) + } + var want map[string]any + if err := json.Unmarshal(json.RawMessage(test.wantODCfg), &want); err != nil { + t.Fatalf("Error unmarshalling wantODCfg (%q): %v", test.wantODCfg, err) + } + if diff := cmp.Diff(got, want); diff != "" { + t.Fatalf("cluster.OutlierDetection got unexpected output, diff (-got, +want): %v", diff) + } + }) + } +} diff --git a/xds/internal/xdsclient/xdsresource/unmarshal_eds.go b/xds/internal/xdsclient/xdsresource/unmarshal_eds.go new file mode 100644 index 000000000000..95333aaf61d5 --- /dev/null +++ b/xds/internal/xdsclient/xdsresource/unmarshal_eds.go @@ -0,0 +1,176 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "fmt" + "math" + "net" + "strconv" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3typepb "github.com/envoyproxy/go-control-plane/envoy/type/v3" + "github.com/golang/protobuf/proto" + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/xds/internal" + "google.golang.org/protobuf/types/known/anypb" +) + +func unmarshalEndpointsResource(r *anypb.Any) (string, EndpointsUpdate, error) { + r, err := UnwrapResource(r) + if err != nil { + return "", EndpointsUpdate{}, fmt.Errorf("failed to unwrap resource: %v", err) + } + + if !IsEndpointsResource(r.GetTypeUrl()) { + return "", EndpointsUpdate{}, fmt.Errorf("unexpected resource type: %q ", r.GetTypeUrl()) + } + + cla := &v3endpointpb.ClusterLoadAssignment{} + if err := proto.Unmarshal(r.GetValue(), cla); err != nil { + return "", EndpointsUpdate{}, fmt.Errorf("failed to unmarshal resource: %v", err) + } + + u, err := parseEDSRespProto(cla) + if err != nil { + return cla.GetClusterName(), EndpointsUpdate{}, err + } + u.Raw = r + return cla.GetClusterName(), u, nil +} + +func parseAddress(socketAddress *v3corepb.SocketAddress) string { + return net.JoinHostPort(socketAddress.GetAddress(), strconv.Itoa(int(socketAddress.GetPortValue()))) +} + +func parseDropPolicy(dropPolicy *v3endpointpb.ClusterLoadAssignment_Policy_DropOverload) OverloadDropConfig { + percentage := dropPolicy.GetDropPercentage() + var ( + numerator = percentage.GetNumerator() + denominator uint32 + ) + switch percentage.GetDenominator() { + case v3typepb.FractionalPercent_HUNDRED: + denominator = 100 + case v3typepb.FractionalPercent_TEN_THOUSAND: + denominator = 10000 + case v3typepb.FractionalPercent_MILLION: + denominator = 1000000 + } + return OverloadDropConfig{ + Category: dropPolicy.GetCategory(), + Numerator: numerator, + Denominator: denominator, + } +} + +func parseEndpoints(lbEndpoints []*v3endpointpb.LbEndpoint, uniqueEndpointAddrs map[string]bool) ([]Endpoint, error) { + endpoints := make([]Endpoint, 0, len(lbEndpoints)) + for _, lbEndpoint := range lbEndpoints { + // If the load_balancing_weight field is specified, it must be set to a + // value of at least 1. If unspecified, each host is presumed to have + // equal weight in a locality. + weight := uint32(1) + if w := lbEndpoint.GetLoadBalancingWeight(); w != nil { + if w.GetValue() == 0 { + return nil, fmt.Errorf("EDS response contains an endpoint with zero weight: %+v", lbEndpoint) + } + weight = w.GetValue() + } + addr := parseAddress(lbEndpoint.GetEndpoint().GetAddress().GetSocketAddress()) + if uniqueEndpointAddrs[addr] { + return nil, fmt.Errorf("duplicate endpoint with the same address %s", addr) + } + uniqueEndpointAddrs[addr] = true + endpoints = append(endpoints, Endpoint{ + HealthStatus: EndpointHealthStatus(lbEndpoint.GetHealthStatus()), + Address: addr, + Weight: weight, + }) + } + return endpoints, nil +} + +func parseEDSRespProto(m *v3endpointpb.ClusterLoadAssignment) (EndpointsUpdate, error) { + ret := EndpointsUpdate{} + for _, dropPolicy := range m.GetPolicy().GetDropOverloads() { + ret.Drops = append(ret.Drops, parseDropPolicy(dropPolicy)) + } + priorities := make(map[uint32]map[string]bool) + sumOfWeights := make(map[uint32]uint64) + uniqueEndpointAddrs := make(map[string]bool) + for _, locality := range m.Endpoints { + l := locality.GetLocality() + if l == nil { + return EndpointsUpdate{}, fmt.Errorf("EDS response contains a locality without ID, locality: %+v", locality) + } + weight := locality.GetLoadBalancingWeight().GetValue() + if weight == 0 { + logger.Warningf("Ignoring locality %s with weight 0", pretty.ToJSON(l)) + continue + } + priority := locality.GetPriority() + sumOfWeights[priority] += uint64(weight) + if sumOfWeights[priority] > math.MaxUint32 { + return EndpointsUpdate{}, fmt.Errorf("sum of weights of localities at the same priority %d exceeded maximal value", priority) + } + localitiesWithPriority := priorities[priority] + if localitiesWithPriority == nil { + localitiesWithPriority = make(map[string]bool) + priorities[priority] = localitiesWithPriority + } + lid := internal.LocalityID{ + Region: l.Region, + Zone: l.Zone, + SubZone: l.SubZone, + } + lidStr, _ := lid.ToString() + + // "Since an xDS configuration can place a given locality under multiple + // priorities, it is possible to see locality weight attributes with + // different values for the same locality." - A52 + // + // This is handled in the client by emitting the locality weight + // specified for the priority it is specified in. If the same locality + // has a different weight in two priorities, each priority will specify + // a locality with the locality weight specified for that priority, and + // thus the subsequent tree of balancers linked to that priority will + // use that locality weight as well. + if localitiesWithPriority[lidStr] { + return EndpointsUpdate{}, fmt.Errorf("duplicate locality %s with the same priority %v", lidStr, priority) + } + localitiesWithPriority[lidStr] = true + endpoints, err := parseEndpoints(locality.GetLbEndpoints(), uniqueEndpointAddrs) + if err != nil { + return EndpointsUpdate{}, err + } + ret.Localities = append(ret.Localities, Locality{ + ID: lid, + Endpoints: endpoints, + Weight: weight, + Priority: priority, + }) + } + for i := 0; i < len(priorities); i++ { + if _, ok := priorities[uint32(i)]; !ok { + return EndpointsUpdate{}, fmt.Errorf("priority %v missing (with different priorities %v received)", i, priorities) + } + } + return ret, nil +} diff --git a/xds/internal/xdsclient/xdsresource/unmarshal_eds_test.go b/xds/internal/xdsclient/xdsresource/unmarshal_eds_test.go new file mode 100644 index 000000000000..cfb452b26fad --- /dev/null +++ b/xds/internal/xdsclient/xdsresource/unmarshal_eds_test.go @@ -0,0 +1,437 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "fmt" + "net" + "strconv" + "testing" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + v3typepb "github.com/envoyproxy/go-control-plane/envoy/type/v3" + anypb "github.com/golang/protobuf/ptypes/any" + wrapperspb "github.com/golang/protobuf/ptypes/wrappers" + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/xds/internal" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource/version" +) + +func (s) TestEDSParseRespProto(t *testing.T) { + tests := []struct { + name string + m *v3endpointpb.ClusterLoadAssignment + want EndpointsUpdate + wantErr bool + }{ + { + name: "missing-priority", + m: func() *v3endpointpb.ClusterLoadAssignment { + clab0 := newClaBuilder("test", nil) + clab0.addLocality("locality-1", 1, 0, []string{"addr1:314"}, nil) + clab0.addLocality("locality-2", 1, 2, []string{"addr2:159"}, nil) + return clab0.Build() + }(), + want: EndpointsUpdate{}, + wantErr: true, + }, + { + name: "missing-locality-ID", + m: func() *v3endpointpb.ClusterLoadAssignment { + clab0 := newClaBuilder("test", nil) + clab0.addLocality("", 1, 0, []string{"addr1:314"}, nil) + return clab0.Build() + }(), + want: EndpointsUpdate{}, + wantErr: true, + }, + { + name: "zero-endpoint-weight", + m: func() *v3endpointpb.ClusterLoadAssignment { + clab0 := newClaBuilder("test", nil) + clab0.addLocality("locality-0", 1, 0, []string{"addr1:314"}, &addLocalityOptions{Weight: []uint32{0}}) + return clab0.Build() + }(), + want: EndpointsUpdate{}, + wantErr: true, + }, + { + name: "duplicate-locality-in-the-same-priority", + m: func() *v3endpointpb.ClusterLoadAssignment { + clab0 := newClaBuilder("test", nil) + clab0.addLocality("locality-0", 1, 0, []string{"addr1:314"}, nil) + clab0.addLocality("locality-0", 1, 0, []string{"addr1:314"}, nil) // Duplicate locality with the same priority. + return clab0.Build() + }(), + want: EndpointsUpdate{}, + wantErr: true, + }, + { + name: "missing locality weight", + m: func() *v3endpointpb.ClusterLoadAssignment { + clab0 := newClaBuilder("test", nil) + clab0.addLocality("locality-1", 0, 1, []string{"addr1:314"}, &addLocalityOptions{ + Health: []v3corepb.HealthStatus{v3corepb.HealthStatus_HEALTHY}, + }) + clab0.addLocality("locality-2", 0, 0, []string{"addr2:159"}, &addLocalityOptions{ + Health: []v3corepb.HealthStatus{v3corepb.HealthStatus_HEALTHY}, + }) + return clab0.Build() + }(), + want: EndpointsUpdate{}, + }, + { + name: "max sum of weights at the same priority exceeded", + m: func() *v3endpointpb.ClusterLoadAssignment { + clab0 := newClaBuilder("test", nil) + clab0.addLocality("locality-1", 1, 0, []string{"addr1:314"}, nil) + clab0.addLocality("locality-2", 4294967295, 1, []string{"addr2:159"}, nil) + clab0.addLocality("locality-3", 1, 1, []string{"addr2:88"}, nil) + return clab0.Build() + }(), + want: EndpointsUpdate{}, + wantErr: true, + }, + { + name: "duplicate endpoint address", + m: func() *v3endpointpb.ClusterLoadAssignment { + clab0 := newClaBuilder("test", nil) + clab0.addLocality("locality-1", 1, 1, []string{"addr:997"}, nil) + clab0.addLocality("locality-2", 1, 0, []string{"addr:997"}, nil) + return clab0.Build() + }(), + want: EndpointsUpdate{}, + wantErr: true, + }, + { + name: "good", + m: func() *v3endpointpb.ClusterLoadAssignment { + clab0 := newClaBuilder("test", nil) + clab0.addLocality("locality-1", 1, 1, []string{"addr1:314"}, &addLocalityOptions{ + Health: []v3corepb.HealthStatus{v3corepb.HealthStatus_UNHEALTHY}, + Weight: []uint32{271}, + }) + clab0.addLocality("locality-2", 1, 0, []string{"addr2:159"}, &addLocalityOptions{ + Health: []v3corepb.HealthStatus{v3corepb.HealthStatus_DRAINING}, + Weight: []uint32{828}, + }) + return clab0.Build() + }(), + want: EndpointsUpdate{ + Drops: nil, + Localities: []Locality{ + { + Endpoints: []Endpoint{{ + Address: "addr1:314", + HealthStatus: EndpointHealthStatusUnhealthy, + Weight: 271, + }}, + ID: internal.LocalityID{SubZone: "locality-1"}, + Priority: 1, + Weight: 1, + }, + { + Endpoints: []Endpoint{{ + Address: "addr2:159", + HealthStatus: EndpointHealthStatusDraining, + Weight: 828, + }}, + ID: internal.LocalityID{SubZone: "locality-2"}, + Priority: 0, + Weight: 1, + }, + }, + }, + wantErr: false, + }, + { + name: "good duplicate locality with different priority", + m: func() *v3endpointpb.ClusterLoadAssignment { + clab0 := newClaBuilder("test", nil) + clab0.addLocality("locality-1", 1, 1, []string{"addr1:314"}, &addLocalityOptions{ + Health: []v3corepb.HealthStatus{v3corepb.HealthStatus_UNHEALTHY}, + Weight: []uint32{271}, + }) + // Same locality name, but with different priority. + clab0.addLocality("locality-1", 1, 0, []string{"addr2:159"}, &addLocalityOptions{ + Health: []v3corepb.HealthStatus{v3corepb.HealthStatus_DRAINING}, + Weight: []uint32{828}, + }) + return clab0.Build() + }(), + want: EndpointsUpdate{ + Drops: nil, + Localities: []Locality{ + { + Endpoints: []Endpoint{{ + Address: "addr1:314", + HealthStatus: EndpointHealthStatusUnhealthy, + Weight: 271, + }}, + ID: internal.LocalityID{SubZone: "locality-1"}, + Priority: 1, + Weight: 1, + }, + { + Endpoints: []Endpoint{{ + Address: "addr2:159", + HealthStatus: EndpointHealthStatusDraining, + Weight: 828, + }}, + ID: internal.LocalityID{SubZone: "locality-1"}, + Priority: 0, + Weight: 1, + }, + }, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parseEDSRespProto(tt.m) + if (err != nil) != tt.wantErr { + t.Errorf("parseEDSRespProto() error = %v, wantErr %v", err, tt.wantErr) + return + } + if d := cmp.Diff(got, tt.want); d != "" { + t.Errorf("parseEDSRespProto() got = %v, want %v, diff: %v", got, tt.want, d) + } + }) + } +} + +func (s) TestUnmarshalEndpoints(t *testing.T) { + var v3EndpointsAny = testutils.MarshalAny(func() *v3endpointpb.ClusterLoadAssignment { + clab0 := newClaBuilder("test", nil) + clab0.addLocality("locality-1", 1, 1, []string{"addr1:314"}, &addLocalityOptions{ + Health: []v3corepb.HealthStatus{v3corepb.HealthStatus_UNHEALTHY}, + Weight: []uint32{271}, + }) + clab0.addLocality("locality-2", 1, 0, []string{"addr2:159"}, &addLocalityOptions{ + Health: []v3corepb.HealthStatus{v3corepb.HealthStatus_DRAINING}, + Weight: []uint32{828}, + }) + return clab0.Build() + }()) + + tests := []struct { + name string + resource *anypb.Any + wantName string + wantUpdate EndpointsUpdate + wantErr bool + }{ + { + name: "non-clusterLoadAssignment resource type", + resource: &anypb.Any{TypeUrl: version.V3HTTPConnManagerURL}, + wantErr: true, + }, + { + name: "badly marshaled clusterLoadAssignment resource", + resource: &anypb.Any{ + TypeUrl: version.V3EndpointsURL, + Value: []byte{1, 2, 3, 4}, + }, + wantErr: true, + }, + { + name: "bad endpoints resource", + resource: testutils.MarshalAny(func() *v3endpointpb.ClusterLoadAssignment { + clab0 := newClaBuilder("test", nil) + clab0.addLocality("locality-1", 1, 0, []string{"addr1:314"}, nil) + clab0.addLocality("locality-2", 1, 2, []string{"addr2:159"}, nil) + return clab0.Build() + }()), + wantName: "test", + wantErr: true, + }, + { + name: "v3 endpoints", + resource: v3EndpointsAny, + wantName: "test", + wantUpdate: EndpointsUpdate{ + Drops: nil, + Localities: []Locality{ + { + Endpoints: []Endpoint{{ + Address: "addr1:314", + HealthStatus: EndpointHealthStatusUnhealthy, + Weight: 271, + }}, + ID: internal.LocalityID{SubZone: "locality-1"}, + Priority: 1, + Weight: 1, + }, + { + Endpoints: []Endpoint{{ + Address: "addr2:159", + HealthStatus: EndpointHealthStatusDraining, + Weight: 828, + }}, + ID: internal.LocalityID{SubZone: "locality-2"}, + Priority: 0, + Weight: 1, + }, + }, + Raw: v3EndpointsAny, + }, + }, + { + name: "v3 endpoints wrapped", + resource: testutils.MarshalAny(&v3discoverypb.Resource{Resource: v3EndpointsAny}), + wantName: "test", + wantUpdate: EndpointsUpdate{ + Drops: nil, + Localities: []Locality{ + { + Endpoints: []Endpoint{{ + Address: "addr1:314", + HealthStatus: EndpointHealthStatusUnhealthy, + Weight: 271, + }}, + ID: internal.LocalityID{SubZone: "locality-1"}, + Priority: 1, + Weight: 1, + }, + { + Endpoints: []Endpoint{{ + Address: "addr2:159", + HealthStatus: EndpointHealthStatusDraining, + Weight: 828, + }}, + ID: internal.LocalityID{SubZone: "locality-2"}, + Priority: 0, + Weight: 1, + }, + }, + Raw: v3EndpointsAny, + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + name, update, err := unmarshalEndpointsResource(test.resource) + if (err != nil) != test.wantErr { + t.Fatalf("unmarshalEndpointsResource(%s), got err: %v, wantErr: %v", pretty.ToJSON(test.resource), err, test.wantErr) + } + if name != test.wantName { + t.Errorf("unmarshalEndpointsResource(%s), got name: %s, want: %s", pretty.ToJSON(test.resource), name, test.wantName) + } + if diff := cmp.Diff(update, test.wantUpdate, cmpOpts); diff != "" { + t.Errorf("unmarshalEndpointsResource(%s), got unexpected update, diff (-got +want): %v", pretty.ToJSON(test.resource), diff) + } + }) + } +} + +// claBuilder builds a ClusterLoadAssignment, aka EDS +// response. +type claBuilder struct { + v *v3endpointpb.ClusterLoadAssignment +} + +// newClaBuilder creates a claBuilder. +func newClaBuilder(clusterName string, dropPercents []uint32) *claBuilder { + var drops []*v3endpointpb.ClusterLoadAssignment_Policy_DropOverload + for i, d := range dropPercents { + drops = append(drops, &v3endpointpb.ClusterLoadAssignment_Policy_DropOverload{ + Category: fmt.Sprintf("test-drop-%d", i), + DropPercentage: &v3typepb.FractionalPercent{ + Numerator: d, + Denominator: v3typepb.FractionalPercent_HUNDRED, + }, + }) + } + + return &claBuilder{ + v: &v3endpointpb.ClusterLoadAssignment{ + ClusterName: clusterName, + Policy: &v3endpointpb.ClusterLoadAssignment_Policy{ + DropOverloads: drops, + }, + }, + } +} + +// addLocalityOptions contains options when adding locality to the builder. +type addLocalityOptions struct { + Health []v3corepb.HealthStatus + Weight []uint32 +} + +// addLocality adds a locality to the builder. +func (clab *claBuilder) addLocality(subzone string, weight uint32, priority uint32, addrsWithPort []string, opts *addLocalityOptions) { + var lbEndPoints []*v3endpointpb.LbEndpoint + for i, a := range addrsWithPort { + host, portStr, err := net.SplitHostPort(a) + if err != nil { + panic("failed to split " + a) + } + port, err := strconv.Atoi(portStr) + if err != nil { + panic("failed to atoi " + portStr) + } + + lbe := &v3endpointpb.LbEndpoint{ + HostIdentifier: &v3endpointpb.LbEndpoint_Endpoint{ + Endpoint: &v3endpointpb.Endpoint{ + Address: &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Protocol: v3corepb.SocketAddress_TCP, + Address: host, + PortSpecifier: &v3corepb.SocketAddress_PortValue{ + PortValue: uint32(port)}}}}}}, + } + if opts != nil { + if i < len(opts.Health) { + lbe.HealthStatus = opts.Health[i] + } + if i < len(opts.Weight) { + lbe.LoadBalancingWeight = &wrapperspb.UInt32Value{Value: opts.Weight[i]} + } + } + lbEndPoints = append(lbEndPoints, lbe) + } + + var localityID *v3corepb.Locality + if subzone != "" { + localityID = &v3corepb.Locality{ + Region: "", + Zone: "", + SubZone: subzone, + } + } + + clab.v.Endpoints = append(clab.v.Endpoints, &v3endpointpb.LocalityLbEndpoints{ + Locality: localityID, + LbEndpoints: lbEndPoints, + LoadBalancingWeight: &wrapperspb.UInt32Value{Value: weight}, + Priority: priority, + }) +} + +// Build builds ClusterLoadAssignment. +func (clab *claBuilder) Build() *v3endpointpb.ClusterLoadAssignment { + return clab.v +} diff --git a/xds/internal/xdsclient/xdsresource/unmarshal_lds.go b/xds/internal/xdsclient/xdsresource/unmarshal_lds.go new file mode 100644 index 000000000000..8f18b02e28a6 --- /dev/null +++ b/xds/internal/xdsclient/xdsresource/unmarshal_lds.go @@ -0,0 +1,278 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "errors" + "fmt" + "strconv" + + v1udpaudpatypepb "github.com/cncf/udpa/go/udpa/type/v1" + v3xdsxdstypepb "github.com/cncf/xds/go/xds/type/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + "github.com/golang/protobuf/proto" + "github.com/golang/protobuf/ptypes" + "google.golang.org/grpc/xds/internal/httpfilter" + "google.golang.org/protobuf/types/known/anypb" +) + +func unmarshalListenerResource(r *anypb.Any) (string, ListenerUpdate, error) { + r, err := UnwrapResource(r) + if err != nil { + return "", ListenerUpdate{}, fmt.Errorf("failed to unwrap resource: %v", err) + } + + if !IsListenerResource(r.GetTypeUrl()) { + return "", ListenerUpdate{}, fmt.Errorf("unexpected resource type: %q ", r.GetTypeUrl()) + } + lis := &v3listenerpb.Listener{} + if err := proto.Unmarshal(r.GetValue(), lis); err != nil { + return "", ListenerUpdate{}, fmt.Errorf("failed to unmarshal resource: %v", err) + } + + lu, err := processListener(lis) + if err != nil { + return lis.GetName(), ListenerUpdate{}, err + } + lu.Raw = r + return lis.GetName(), *lu, nil +} + +func processListener(lis *v3listenerpb.Listener) (*ListenerUpdate, error) { + if lis.GetApiListener() != nil { + return processClientSideListener(lis) + } + return processServerSideListener(lis) +} + +// processClientSideListener checks if the provided Listener proto meets +// the expected criteria. If so, it returns a non-empty routeConfigName. +func processClientSideListener(lis *v3listenerpb.Listener) (*ListenerUpdate, error) { + update := &ListenerUpdate{} + + apiLisAny := lis.GetApiListener().GetApiListener() + if !IsHTTPConnManagerResource(apiLisAny.GetTypeUrl()) { + return nil, fmt.Errorf("unexpected resource type: %q", apiLisAny.GetTypeUrl()) + } + apiLis := &v3httppb.HttpConnectionManager{} + if err := proto.Unmarshal(apiLisAny.GetValue(), apiLis); err != nil { + return nil, fmt.Errorf("failed to unmarshal api_listner: %v", err) + } + // "HttpConnectionManager.xff_num_trusted_hops must be unset or zero and + // HttpConnectionManager.original_ip_detection_extensions must be empty. If + // either field has an incorrect value, the Listener must be NACKed." - A41 + if apiLis.XffNumTrustedHops != 0 { + return nil, fmt.Errorf("xff_num_trusted_hops must be unset or zero %+v", apiLis) + } + if len(apiLis.OriginalIpDetectionExtensions) != 0 { + return nil, fmt.Errorf("original_ip_detection_extensions must be empty %+v", apiLis) + } + + switch apiLis.RouteSpecifier.(type) { + case *v3httppb.HttpConnectionManager_Rds: + if configsource := apiLis.GetRds().GetConfigSource(); configsource.GetAds() == nil && configsource.GetSelf() == nil { + return nil, fmt.Errorf("LDS's RDS configSource is not ADS or Self: %+v", lis) + } + name := apiLis.GetRds().GetRouteConfigName() + if name == "" { + return nil, fmt.Errorf("empty route_config_name: %+v", lis) + } + update.RouteConfigName = name + case *v3httppb.HttpConnectionManager_RouteConfig: + routeU, err := generateRDSUpdateFromRouteConfiguration(apiLis.GetRouteConfig()) + if err != nil { + return nil, fmt.Errorf("failed to parse inline RDS resp: %v", err) + } + update.InlineRouteConfig = &routeU + case nil: + return nil, fmt.Errorf("no RouteSpecifier: %+v", apiLis) + default: + return nil, fmt.Errorf("unsupported type %T for RouteSpecifier", apiLis.RouteSpecifier) + } + + // The following checks and fields only apply to xDS protocol versions v3+. + + update.MaxStreamDuration = apiLis.GetCommonHttpProtocolOptions().GetMaxStreamDuration().AsDuration() + + var err error + if update.HTTPFilters, err = processHTTPFilters(apiLis.GetHttpFilters(), false); err != nil { + return nil, err + } + + return update, nil +} + +func unwrapHTTPFilterConfig(config *anypb.Any) (proto.Message, string, error) { + switch { + case ptypes.Is(config, &v3xdsxdstypepb.TypedStruct{}): + // The real type name is inside the new TypedStruct message. + s := new(v3xdsxdstypepb.TypedStruct) + if err := ptypes.UnmarshalAny(config, s); err != nil { + return nil, "", fmt.Errorf("error unmarshalling TypedStruct filter config: %v", err) + } + return s, s.GetTypeUrl(), nil + case ptypes.Is(config, &v1udpaudpatypepb.TypedStruct{}): + // The real type name is inside the old TypedStruct message. + s := new(v1udpaudpatypepb.TypedStruct) + if err := ptypes.UnmarshalAny(config, s); err != nil { + return nil, "", fmt.Errorf("error unmarshalling TypedStruct filter config: %v", err) + } + return s, s.GetTypeUrl(), nil + default: + return config, config.GetTypeUrl(), nil + } +} + +func validateHTTPFilterConfig(cfg *anypb.Any, lds, optional bool) (httpfilter.Filter, httpfilter.FilterConfig, error) { + config, typeURL, err := unwrapHTTPFilterConfig(cfg) + if err != nil { + return nil, nil, err + } + filterBuilder := httpfilter.Get(typeURL) + if filterBuilder == nil { + if optional { + return nil, nil, nil + } + return nil, nil, fmt.Errorf("no filter implementation found for %q", typeURL) + } + parseFunc := filterBuilder.ParseFilterConfig + if !lds { + parseFunc = filterBuilder.ParseFilterConfigOverride + } + filterConfig, err := parseFunc(config) + if err != nil { + return nil, nil, fmt.Errorf("error parsing config for filter %q: %v", typeURL, err) + } + return filterBuilder, filterConfig, nil +} + +func processHTTPFilterOverrides(cfgs map[string]*anypb.Any) (map[string]httpfilter.FilterConfig, error) { + if len(cfgs) == 0 { + return nil, nil + } + m := make(map[string]httpfilter.FilterConfig) + for name, cfg := range cfgs { + optional := false + s := new(v3routepb.FilterConfig) + if ptypes.Is(cfg, s) { + if err := ptypes.UnmarshalAny(cfg, s); err != nil { + return nil, fmt.Errorf("filter override %q: error unmarshalling FilterConfig: %v", name, err) + } + cfg = s.GetConfig() + optional = s.GetIsOptional() + } + + httpFilter, config, err := validateHTTPFilterConfig(cfg, false, optional) + if err != nil { + return nil, fmt.Errorf("filter override %q: %v", name, err) + } + if httpFilter == nil { + // Optional configs are ignored. + continue + } + m[name] = config + } + return m, nil +} + +func processHTTPFilters(filters []*v3httppb.HttpFilter, server bool) ([]HTTPFilter, error) { + ret := make([]HTTPFilter, 0, len(filters)) + seenNames := make(map[string]bool, len(filters)) + for _, filter := range filters { + name := filter.GetName() + if name == "" { + return nil, errors.New("filter missing name field") + } + if seenNames[name] { + return nil, fmt.Errorf("duplicate filter name %q", name) + } + seenNames[name] = true + + httpFilter, config, err := validateHTTPFilterConfig(filter.GetTypedConfig(), true, filter.GetIsOptional()) + if err != nil { + return nil, err + } + if httpFilter == nil { + // Optional configs are ignored. + continue + } + if server { + if _, ok := httpFilter.(httpfilter.ServerInterceptorBuilder); !ok { + if filter.GetIsOptional() { + continue + } + return nil, fmt.Errorf("HTTP filter %q not supported server-side", name) + } + } else if _, ok := httpFilter.(httpfilter.ClientInterceptorBuilder); !ok { + if filter.GetIsOptional() { + continue + } + return nil, fmt.Errorf("HTTP filter %q not supported client-side", name) + } + + // Save name/config + ret = append(ret, HTTPFilter{Name: name, Filter: httpFilter, Config: config}) + } + // "Validation will fail if a terminal filter is not the last filter in the + // chain or if a non-terminal filter is the last filter in the chain." - A39 + if len(ret) == 0 { + return nil, fmt.Errorf("http filters list is empty") + } + var i int + for ; i < len(ret)-1; i++ { + if ret[i].Filter.IsTerminal() { + return nil, fmt.Errorf("http filter %q is a terminal filter but it is not last in the filter chain", ret[i].Name) + } + } + if !ret[i].Filter.IsTerminal() { + return nil, fmt.Errorf("http filter %q is not a terminal filter", ret[len(ret)-1].Name) + } + return ret, nil +} + +func processServerSideListener(lis *v3listenerpb.Listener) (*ListenerUpdate, error) { + if n := len(lis.ListenerFilters); n != 0 { + return nil, fmt.Errorf("unsupported field 'listener_filters' contains %d entries", n) + } + if useOrigDst := lis.GetUseOriginalDst(); useOrigDst != nil && useOrigDst.GetValue() { + return nil, errors.New("unsupported field 'use_original_dst' is present and set to true") + } + addr := lis.GetAddress() + if addr == nil { + return nil, fmt.Errorf("no address field in LDS response: %+v", lis) + } + sockAddr := addr.GetSocketAddress() + if sockAddr == nil { + return nil, fmt.Errorf("no socket_address field in LDS response: %+v", lis) + } + lu := &ListenerUpdate{ + InboundListenerCfg: &InboundListenerConfig{ + Address: sockAddr.GetAddress(), + Port: strconv.Itoa(int(sockAddr.GetPortValue())), + }, + } + + fcMgr, err := NewFilterChainManager(lis) + if err != nil { + return nil, err + } + lu.InboundListenerCfg.FilterChains = fcMgr + return lu, nil +} diff --git a/xds/internal/xdsclient/xdsresource/unmarshal_lds_test.go b/xds/internal/xdsclient/xdsresource/unmarshal_lds_test.go new file mode 100644 index 000000000000..2dfeb5965b72 --- /dev/null +++ b/xds/internal/xdsclient/xdsresource/unmarshal_lds_test.go @@ -0,0 +1,1872 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "fmt" + "strings" + "testing" + "time" + + v2xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2" + "github.com/golang/protobuf/proto" + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/xds/internal/httpfilter" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource/version" + "google.golang.org/protobuf/types/known/durationpb" + + v1udpaudpatypepb "github.com/cncf/udpa/go/udpa/type/v1" + v3xdsxdstypepb "github.com/cncf/xds/go/xds/type/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + rpb "github.com/envoyproxy/go-control-plane/envoy/config/rbac/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3rbacpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/rbac/v3" + v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3" + anypb "github.com/golang/protobuf/ptypes/any" + spb "github.com/golang/protobuf/ptypes/struct" + wrapperspb "github.com/golang/protobuf/ptypes/wrappers" + + _ "google.golang.org/grpc/xds/internal/httpfilter/rbac" // Register the RBAC HTTP filter. + _ "google.golang.org/grpc/xds/internal/httpfilter/router" // Register the router filter. +) + +func (s) TestUnmarshalListener_ClientSide(t *testing.T) { + const ( + v3LDSTarget = "lds.target.good:3333" + v3RouteConfigName = "v3RouteConfig" + routeName = "routeName" + ) + + var ( + customFilter = &v3httppb.HttpFilter{ + Name: "customFilter", + ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: customFilterConfig}, + } + oldTypedStructFilter = &v3httppb.HttpFilter{ + Name: "customFilter", + ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: wrappedCustomFilterOldTypedStructConfig}, + } + newTypedStructFilter = &v3httppb.HttpFilter{ + Name: "customFilter", + ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: wrappedCustomFilterNewTypedStructConfig}, + } + customOptionalFilter = &v3httppb.HttpFilter{ + Name: "customFilter", + ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: customFilterConfig}, + IsOptional: true, + } + customFilter2 = &v3httppb.HttpFilter{ + Name: "customFilter2", + ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: customFilterConfig}, + } + errFilter = &v3httppb.HttpFilter{ + Name: "errFilter", + ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: errFilterConfig}, + } + errOptionalFilter = &v3httppb.HttpFilter{ + Name: "errFilter", + ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: errFilterConfig}, + IsOptional: true, + } + clientOnlyCustomFilter = &v3httppb.HttpFilter{ + Name: "clientOnlyCustomFilter", + ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: clientOnlyCustomFilterConfig}, + } + serverOnlyCustomFilter = &v3httppb.HttpFilter{ + Name: "serverOnlyCustomFilter", + ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: serverOnlyCustomFilterConfig}, + } + serverOnlyOptionalCustomFilter = &v3httppb.HttpFilter{ + Name: "serverOnlyOptionalCustomFilter", + ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: serverOnlyCustomFilterConfig}, + IsOptional: true, + } + unknownFilter = &v3httppb.HttpFilter{ + Name: "unknownFilter", + ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: unknownFilterConfig}, + } + unknownOptionalFilter = &v3httppb.HttpFilter{ + Name: "unknownFilter", + ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: unknownFilterConfig}, + IsOptional: true, + } + v3LisWithInlineRoute = testutils.MarshalAny(&v3listenerpb.Listener{ + Name: v3LDSTarget, + ApiListener: &v3listenerpb.ApiListener{ + ApiListener: testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: &v3routepb.RouteConfiguration{ + Name: routeName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{v3LDSTarget}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName}, + }}}}}}}, + }, + HttpFilters: []*v3httppb.HttpFilter{emptyRouterFilter}, + CommonHttpProtocolOptions: &v3corepb.HttpProtocolOptions{ + MaxStreamDuration: durationpb.New(time.Second), + }, + }), + }, + }) + v3LisWithFilters = func(fs ...*v3httppb.HttpFilter) *anypb.Any { + fs = append(fs, emptyRouterFilter) + return testutils.MarshalAny(&v3listenerpb.Listener{ + Name: v3LDSTarget, + ApiListener: &v3listenerpb.ApiListener{ + ApiListener: testutils.MarshalAny( + &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{ + Rds: &v3httppb.Rds{ + ConfigSource: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}, + }, + RouteConfigName: v3RouteConfigName, + }, + }, + CommonHttpProtocolOptions: &v3corepb.HttpProtocolOptions{ + MaxStreamDuration: durationpb.New(time.Second), + }, + HttpFilters: fs, + }), + }, + }) + } + v3LisToTestRBAC = func(xffNumTrustedHops uint32, originalIpDetectionExtensions []*v3corepb.TypedExtensionConfig) *anypb.Any { + return testutils.MarshalAny(&v3listenerpb.Listener{ + Name: v3LDSTarget, + ApiListener: &v3listenerpb.ApiListener{ + ApiListener: testutils.MarshalAny( + &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{ + Rds: &v3httppb.Rds{ + ConfigSource: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}, + }, + RouteConfigName: v3RouteConfigName, + }, + }, + CommonHttpProtocolOptions: &v3corepb.HttpProtocolOptions{ + MaxStreamDuration: durationpb.New(time.Second), + }, + HttpFilters: []*v3httppb.HttpFilter{emptyRouterFilter}, + XffNumTrustedHops: xffNumTrustedHops, + OriginalIpDetectionExtensions: originalIpDetectionExtensions, + }), + }, + }) + } + + v3ListenerWithCDSConfigSourceSelf = testutils.MarshalAny(&v3listenerpb.Listener{ + Name: v3LDSTarget, + ApiListener: &v3listenerpb.ApiListener{ + ApiListener: testutils.MarshalAny( + &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{ + Rds: &v3httppb.Rds{ + ConfigSource: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Self{}, + }, + RouteConfigName: v3RouteConfigName, + }, + }, + HttpFilters: []*v3httppb.HttpFilter{emptyRouterFilter}, + }), + }, + }) + ) + + tests := []struct { + name string + resource *anypb.Any + wantName string + wantUpdate ListenerUpdate + wantErr bool + }{ + { + name: "non-listener resource", + resource: &anypb.Any{TypeUrl: version.V3HTTPConnManagerURL}, + wantErr: true, + }, + { + name: "badly marshaled listener resource", + resource: &anypb.Any{ + TypeUrl: version.V3ListenerURL, + Value: func() []byte { + lis := &v3listenerpb.Listener{ + Name: v3LDSTarget, + ApiListener: &v3listenerpb.ApiListener{ + ApiListener: &anypb.Any{ + TypeUrl: version.V3HTTPConnManagerURL, + Value: []byte{1, 2, 3, 4}, + }, + }, + } + mLis, _ := proto.Marshal(lis) + return mLis + }(), + }, + wantName: v3LDSTarget, + wantErr: true, + }, + { + name: "wrong type in apiListener", + resource: testutils.MarshalAny(&v3listenerpb.Listener{ + Name: v3LDSTarget, + ApiListener: &v3listenerpb.ApiListener{ + ApiListener: testutils.MarshalAny(&v2xdspb.Listener{}), + }, + }), + wantName: v3LDSTarget, + wantErr: true, + }, + { + name: "empty httpConnMgr in apiListener", + resource: testutils.MarshalAny(&v3listenerpb.Listener{ + Name: v3LDSTarget, + ApiListener: &v3listenerpb.ApiListener{ + ApiListener: testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{ + Rds: &v3httppb.Rds{}, + }, + }), + }, + }), + wantName: v3LDSTarget, + wantErr: true, + }, + { + name: "scopedRoutes routeConfig in apiListener", + resource: testutils.MarshalAny(&v3listenerpb.Listener{ + Name: v3LDSTarget, + ApiListener: &v3listenerpb.ApiListener{ + ApiListener: testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_ScopedRoutes{}, + }), + }, + }), + wantName: v3LDSTarget, + wantErr: true, + }, + { + name: "rds.ConfigSource in apiListener is Self", + resource: v3ListenerWithCDSConfigSourceSelf, + wantName: v3LDSTarget, + wantUpdate: ListenerUpdate{ + RouteConfigName: v3RouteConfigName, + HTTPFilters: []HTTPFilter{routerFilter}, + Raw: v3ListenerWithCDSConfigSourceSelf, + }, + }, + { + name: "rds.ConfigSource in apiListener is not ADS or Self", + resource: testutils.MarshalAny(&v3listenerpb.Listener{ + Name: v3LDSTarget, + ApiListener: &v3listenerpb.ApiListener{ + ApiListener: testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{ + Rds: &v3httppb.Rds{ + ConfigSource: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Path{ + Path: "/some/path", + }, + }, + RouteConfigName: v3RouteConfigName, + }, + }, + }), + }, + }), + wantName: v3LDSTarget, + wantErr: true, + }, + { + name: "v3 with no filters", + resource: v3LisWithFilters(), + wantName: v3LDSTarget, + wantUpdate: ListenerUpdate{ + RouteConfigName: v3RouteConfigName, + MaxStreamDuration: time.Second, + HTTPFilters: routerFilterList, + Raw: v3LisWithFilters(), + }, + }, + { + name: "v3 no terminal filter", + resource: testutils.MarshalAny(&v3listenerpb.Listener{ + Name: v3LDSTarget, + ApiListener: &v3listenerpb.ApiListener{ + ApiListener: testutils.MarshalAny( + &v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{ + Rds: &v3httppb.Rds{ + ConfigSource: &v3corepb.ConfigSource{ + ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}}, + }, + RouteConfigName: v3RouteConfigName, + }, + }, + CommonHttpProtocolOptions: &v3corepb.HttpProtocolOptions{ + MaxStreamDuration: durationpb.New(time.Second), + }, + }), + }, + }), + wantName: v3LDSTarget, + wantErr: true, + }, + { + name: "v3 with custom filter", + resource: v3LisWithFilters(customFilter), + wantName: v3LDSTarget, + wantUpdate: ListenerUpdate{ + RouteConfigName: v3RouteConfigName, MaxStreamDuration: time.Second, + HTTPFilters: []HTTPFilter{ + { + Name: "customFilter", + Filter: httpFilter{}, + Config: filterConfig{Cfg: customFilterConfig}, + }, + routerFilter, + }, + Raw: v3LisWithFilters(customFilter), + }, + }, + { + name: "v3 with custom filter in old typed struct", + resource: v3LisWithFilters(oldTypedStructFilter), + wantName: v3LDSTarget, + wantUpdate: ListenerUpdate{ + RouteConfigName: v3RouteConfigName, MaxStreamDuration: time.Second, + HTTPFilters: []HTTPFilter{ + { + Name: "customFilter", + Filter: httpFilter{}, + Config: filterConfig{Cfg: customFilterOldTypedStructConfig}, + }, + routerFilter, + }, + Raw: v3LisWithFilters(oldTypedStructFilter), + }, + }, + { + name: "v3 with custom filter in new typed struct", + resource: v3LisWithFilters(newTypedStructFilter), + wantName: v3LDSTarget, + wantUpdate: ListenerUpdate{ + RouteConfigName: v3RouteConfigName, MaxStreamDuration: time.Second, + HTTPFilters: []HTTPFilter{ + { + Name: "customFilter", + Filter: httpFilter{}, + Config: filterConfig{Cfg: customFilterNewTypedStructConfig}, + }, + routerFilter, + }, + Raw: v3LisWithFilters(newTypedStructFilter), + }, + }, + { + name: "v3 with optional custom filter", + resource: v3LisWithFilters(customOptionalFilter), + wantName: v3LDSTarget, + wantUpdate: ListenerUpdate{ + RouteConfigName: v3RouteConfigName, MaxStreamDuration: time.Second, + HTTPFilters: []HTTPFilter{ + { + Name: "customFilter", + Filter: httpFilter{}, + Config: filterConfig{Cfg: customFilterConfig}, + }, + routerFilter, + }, + Raw: v3LisWithFilters(customOptionalFilter), + }, + }, + { + name: "v3 with two filters with same name", + resource: v3LisWithFilters(customFilter, customFilter), + wantName: v3LDSTarget, + wantErr: true, + }, + { + name: "v3 with two filters - same type different name", + resource: v3LisWithFilters(customFilter, customFilter2), + wantName: v3LDSTarget, + wantUpdate: ListenerUpdate{ + RouteConfigName: v3RouteConfigName, MaxStreamDuration: time.Second, + HTTPFilters: []HTTPFilter{{ + Name: "customFilter", + Filter: httpFilter{}, + Config: filterConfig{Cfg: customFilterConfig}, + }, { + Name: "customFilter2", + Filter: httpFilter{}, + Config: filterConfig{Cfg: customFilterConfig}, + }, + routerFilter, + }, + Raw: v3LisWithFilters(customFilter, customFilter2), + }, + }, + { + name: "v3 with server-only filter", + resource: v3LisWithFilters(serverOnlyCustomFilter), + wantName: v3LDSTarget, + wantErr: true, + }, + { + name: "v3 with optional server-only filter", + resource: v3LisWithFilters(serverOnlyOptionalCustomFilter), + wantName: v3LDSTarget, + wantUpdate: ListenerUpdate{ + RouteConfigName: v3RouteConfigName, + MaxStreamDuration: time.Second, + Raw: v3LisWithFilters(serverOnlyOptionalCustomFilter), + HTTPFilters: routerFilterList, + }, + }, + { + name: "v3 with client-only filter", + resource: v3LisWithFilters(clientOnlyCustomFilter), + wantName: v3LDSTarget, + wantUpdate: ListenerUpdate{ + RouteConfigName: v3RouteConfigName, MaxStreamDuration: time.Second, + HTTPFilters: []HTTPFilter{ + { + Name: "clientOnlyCustomFilter", + Filter: clientOnlyHTTPFilter{}, + Config: filterConfig{Cfg: clientOnlyCustomFilterConfig}, + }, + routerFilter}, + Raw: v3LisWithFilters(clientOnlyCustomFilter), + }, + }, + { + name: "v3 with err filter", + resource: v3LisWithFilters(errFilter), + wantName: v3LDSTarget, + wantErr: true, + }, + { + name: "v3 with optional err filter", + resource: v3LisWithFilters(errOptionalFilter), + wantName: v3LDSTarget, + wantErr: true, + }, + { + name: "v3 with unknown filter", + resource: v3LisWithFilters(unknownFilter), + wantName: v3LDSTarget, + wantErr: true, + }, + { + name: "v3 with unknown filter (optional)", + resource: v3LisWithFilters(unknownOptionalFilter), + wantName: v3LDSTarget, + wantUpdate: ListenerUpdate{ + RouteConfigName: v3RouteConfigName, + MaxStreamDuration: time.Second, + HTTPFilters: routerFilterList, + Raw: v3LisWithFilters(unknownOptionalFilter), + }, + }, + { + name: "v3 listener resource", + resource: v3LisWithFilters(), + wantName: v3LDSTarget, + wantUpdate: ListenerUpdate{ + RouteConfigName: v3RouteConfigName, + MaxStreamDuration: time.Second, + HTTPFilters: routerFilterList, + Raw: v3LisWithFilters(), + }, + }, + { + name: "v3 listener resource wrapped", + resource: testutils.MarshalAny(&v3discoverypb.Resource{Resource: v3LisWithFilters()}), + wantName: v3LDSTarget, + wantUpdate: ListenerUpdate{ + RouteConfigName: v3RouteConfigName, + MaxStreamDuration: time.Second, + HTTPFilters: routerFilterList, + Raw: v3LisWithFilters(), + }, + }, + // "To allow equating RBAC's direct_remote_ip and + // remote_ip...HttpConnectionManager.xff_num_trusted_hops must be unset + // or zero and HttpConnectionManager.original_ip_detection_extensions + // must be empty." - A41 + { + name: "rbac-allow-equating-direct-remote-ip-and-remote-ip-valid", + resource: v3LisToTestRBAC(0, nil), + wantName: v3LDSTarget, + wantUpdate: ListenerUpdate{ + RouteConfigName: v3RouteConfigName, + MaxStreamDuration: time.Second, + HTTPFilters: []HTTPFilter{routerFilter}, + Raw: v3LisToTestRBAC(0, nil), + }, + }, + // In order to support xDS Configured RBAC HTTPFilter equating direct + // remote ip and remote ip, xffNumTrustedHops cannot be greater than + // zero. This is because if you can trust a ingress proxy hop when + // determining an origin clients ip address, direct remote ip != remote + // ip. + { + name: "rbac-allow-equating-direct-remote-ip-and-remote-ip-invalid-num-untrusted-hops", + resource: v3LisToTestRBAC(1, nil), + wantName: v3LDSTarget, + wantErr: true, + }, + // In order to support xDS Configured RBAC HTTPFilter equating direct + // remote ip and remote ip, originalIpDetectionExtensions must be empty. + // This is because if you have to ask ip-detection-extension for the + // original ip, direct remote ip might not equal remote ip. + { + name: "rbac-allow-equating-direct-remote-ip-and-remote-ip-invalid-original-ip-detection-extension", + resource: v3LisToTestRBAC(0, []*v3corepb.TypedExtensionConfig{{Name: "something"}}), + wantName: v3LDSTarget, + wantErr: true, + }, + { + name: "v3 listener with inline route configuration", + resource: v3LisWithInlineRoute, + wantName: v3LDSTarget, + wantUpdate: ListenerUpdate{ + InlineRouteConfig: &RouteConfigUpdate{ + VirtualHosts: []*VirtualHost{{ + Domains: []string{v3LDSTarget}, + Routes: []*Route{{Prefix: newStringP("/"), WeightedClusters: map[string]WeightedCluster{clusterName: {Weight: 1}}, ActionType: RouteActionRoute}}, + }}}, + MaxStreamDuration: time.Second, + Raw: v3LisWithInlineRoute, + HTTPFilters: routerFilterList, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + name, update, err := unmarshalListenerResource(test.resource) + if (err != nil) != test.wantErr { + t.Errorf("unmarshalListenerResource(%s), got err: %v, wantErr: %v", pretty.ToJSON(test.resource), err, test.wantErr) + } + if name != test.wantName { + t.Errorf("unmarshalListenerResource(%s), got name: %s, want: %s", pretty.ToJSON(test.resource), name, test.wantName) + } + if diff := cmp.Diff(update, test.wantUpdate, cmpOpts); diff != "" { + t.Errorf("unmarshalListenerResource(%s), got unexpected update, diff (-got +want): %v", pretty.ToJSON(test.resource), diff) + } + }) + } +} + +func (s) TestUnmarshalListener_ServerSide(t *testing.T) { + oldRBAC := envconfig.XDSRBAC + envconfig.XDSRBAC = true + defer func() { + envconfig.XDSRBAC = oldRBAC + }() + const ( + v3LDSTarget = "grpc/server?xds.resource.listening_address=0.0.0.0:9999" + testVersion = "test-version-lds-server" + ) + + var ( + serverOnlyCustomFilter = &v3httppb.HttpFilter{ + Name: "serverOnlyCustomFilter", + ConfigType: &v3httppb.HttpFilter_TypedConfig{TypedConfig: serverOnlyCustomFilterConfig}, + } + routeConfig = &v3routepb.RouteConfiguration{ + Name: "routeName", + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{"lds.target.good:3333"}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}, + }, + Action: &v3routepb.Route_NonForwardingAction{}, + }}}}} + inlineRouteConfig = &RouteConfigUpdate{ + VirtualHosts: []*VirtualHost{{ + Domains: []string{"lds.target.good:3333"}, + Routes: []*Route{{Prefix: newStringP("/"), ActionType: RouteActionNonForwardingAction}}, + }}} + emptyValidNetworkFilters = []*v3listenerpb.Filter{ + { + Name: "filter-1", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: routeConfig, + }, + HttpFilters: []*v3httppb.HttpFilter{e2e.RouterHTTPFilter}, + }), + }, + }, + } + localSocketAddress = &v3corepb.Address{ + Address: &v3corepb.Address_SocketAddress{ + SocketAddress: &v3corepb.SocketAddress{ + Address: "0.0.0.0", + PortSpecifier: &v3corepb.SocketAddress_PortValue{ + PortValue: 9999, + }, + }, + }, + } + listenerEmptyTransportSocket = testutils.MarshalAny(&v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: emptyValidNetworkFilters, + }, + }, + }) + listenerNoValidationContextDeprecatedFields = testutils.MarshalAny(&v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: emptyValidNetworkFilters, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: "identityPluginInstance", + CertificateName: "identityCertName", + }, + }, + }), + }, + }, + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{ + Name: "default-filter-chain-1", + Filters: emptyValidNetworkFilters, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: "defaultIdentityPluginInstance", + CertificateName: "defaultIdentityCertName", + }, + }, + }), + }, + }, + }, + }) + listenerNoValidationContextNewFields = testutils.MarshalAny(&v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: emptyValidNetworkFilters, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: "identityPluginInstance", + CertificateName: "identityCertName", + }, + }, + }), + }, + }, + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{ + Name: "default-filter-chain-1", + Filters: emptyValidNetworkFilters, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: "defaultIdentityPluginInstance", + CertificateName: "defaultIdentityCertName", + }, + }, + }), + }, + }, + }, + }) + listenerWithValidationContextDeprecatedFields = testutils.MarshalAny(&v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: emptyValidNetworkFilters, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{ + RequireClientCertificate: &wrapperspb.BoolValue{Value: true}, + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: "identityPluginInstance", + CertificateName: "identityCertName", + }, + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{ + ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: "rootPluginInstance", + CertificateName: "rootCertName", + }, + }, + }, + }), + }, + }, + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{ + Name: "default-filter-chain-1", + Filters: emptyValidNetworkFilters, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{ + RequireClientCertificate: &wrapperspb.BoolValue{Value: true}, + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: "defaultIdentityPluginInstance", + CertificateName: "defaultIdentityCertName", + }, + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{ + ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: "defaultRootPluginInstance", + CertificateName: "defaultRootCertName", + }, + }, + }, + }), + }, + }, + }, + }) + listenerWithValidationContextNewFields = testutils.MarshalAny(&v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: emptyValidNetworkFilters, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{ + RequireClientCertificate: &wrapperspb.BoolValue{Value: true}, + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: "identityPluginInstance", + CertificateName: "identityCertName", + }, + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContext{ + ValidationContext: &v3tlspb.CertificateValidationContext{ + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: "rootPluginInstance", + CertificateName: "rootCertName", + }, + }, + }, + }, + }), + }, + }, + }, + }, + DefaultFilterChain: &v3listenerpb.FilterChain{ + Name: "default-filter-chain-1", + Filters: emptyValidNetworkFilters, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{ + RequireClientCertificate: &wrapperspb.BoolValue{Value: true}, + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: "defaultIdentityPluginInstance", + CertificateName: "defaultIdentityCertName", + }, + ValidationContextType: &v3tlspb.CommonTlsContext_CombinedValidationContext{ + CombinedValidationContext: &v3tlspb.CommonTlsContext_CombinedCertificateValidationContext{ + DefaultValidationContext: &v3tlspb.CertificateValidationContext{ + CaCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: "defaultRootPluginInstance", + CertificateName: "defaultRootCertName", + }, + }, + }, + }, + }, + }), + }, + }, + }, + }) + ) + v3LisToTestRBAC := func(xffNumTrustedHops uint32, originalIpDetectionExtensions []*v3corepb.TypedExtensionConfig) *anypb.Any { + return testutils.MarshalAny(&v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: []*v3listenerpb.Filter{ + { + Name: "filter-1", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: routeConfig, + }, + HttpFilters: []*v3httppb.HttpFilter{e2e.RouterHTTPFilter}, + XffNumTrustedHops: xffNumTrustedHops, + OriginalIpDetectionExtensions: originalIpDetectionExtensions, + }), + }, + }, + }, + }, + }, + }) + } + v3LisWithBadRBACConfiguration := func(rbacCfg *v3rbacpb.RBAC) *anypb.Any { + return testutils.MarshalAny(&v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: []*v3listenerpb.Filter{ + { + Name: "filter-1", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: routeConfig, + }, + HttpFilters: []*v3httppb.HttpFilter{e2e.HTTPFilter("rbac", rbacCfg), e2e.RouterHTTPFilter}, + }), + }, + }, + }, + }, + }, + }) + } + badRBACCfgRegex := &v3rbacpb.RBAC{ + Rules: &rpb.RBAC{ + Action: rpb.RBAC_ALLOW, + Policies: map[string]*rpb.Policy{ + "bad-regex-value": { + Permissions: []*rpb.Permission{ + {Rule: &rpb.Permission_Any{Any: true}}, + }, + Principals: []*rpb.Principal{ + {Identifier: &rpb.Principal_Header{Header: &v3routepb.HeaderMatcher{Name: ":method", HeaderMatchSpecifier: &v3routepb.HeaderMatcher_SafeRegexMatch{SafeRegexMatch: &v3matcherpb.RegexMatcher{Regex: "["}}}}}, + }, + }, + }, + }, + } + badRBACCfgDestIP := &v3rbacpb.RBAC{ + Rules: &rpb.RBAC{ + Action: rpb.RBAC_ALLOW, + Policies: map[string]*rpb.Policy{ + "certain-destination-ip": { + Permissions: []*rpb.Permission{ + {Rule: &rpb.Permission_DestinationIp{DestinationIp: &v3corepb.CidrRange{AddressPrefix: "not a correct address", PrefixLen: &wrapperspb.UInt32Value{Value: uint32(10)}}}}, + }, + Principals: []*rpb.Principal{ + {Identifier: &rpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + } + + tests := []struct { + name string + resource *anypb.Any + wantName string + wantUpdate ListenerUpdate + wantErr string + }{ + { + name: "non-empty listener filters", + resource: testutils.MarshalAny(&v3listenerpb.Listener{ + Name: v3LDSTarget, + ListenerFilters: []*v3listenerpb.ListenerFilter{ + {Name: "listener-filter-1"}, + }, + }), + wantName: v3LDSTarget, + wantErr: "unsupported field 'listener_filters'", + }, + { + name: "use_original_dst is set", + resource: testutils.MarshalAny(&v3listenerpb.Listener{ + Name: v3LDSTarget, + UseOriginalDst: &wrapperspb.BoolValue{Value: true}, + }), + wantName: v3LDSTarget, + wantErr: "unsupported field 'use_original_dst'", + }, + { + name: "no address field", + resource: testutils.MarshalAny(&v3listenerpb.Listener{Name: v3LDSTarget}), + wantName: v3LDSTarget, + wantErr: "no address field in LDS response", + }, + { + name: "no socket address field", + resource: testutils.MarshalAny(&v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: &v3corepb.Address{}, + }), + wantName: v3LDSTarget, + wantErr: "no socket_address field in LDS response", + }, + { + name: "no filter chains and no default filter chain", + resource: testutils.MarshalAny(&v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{DestinationPort: &wrapperspb.UInt32Value{Value: 666}}, + Filters: emptyValidNetworkFilters, + }, + }, + }), + wantName: v3LDSTarget, + wantErr: "no supported filter chains and no default filter chain", + }, + { + name: "missing http connection manager network filter", + resource: testutils.MarshalAny(&v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + }, + }, + }), + wantName: v3LDSTarget, + wantErr: "missing HttpConnectionManager filter", + }, + { + name: "missing filter name in http filter", + resource: testutils.MarshalAny(&v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: []*v3listenerpb.Filter{ + { + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{}), + }, + }, + }, + }, + }, + }), + wantName: v3LDSTarget, + wantErr: "missing name field in filter", + }, + { + name: "duplicate filter names in http filter", + resource: testutils.MarshalAny(&v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: []*v3listenerpb.Filter{ + { + Name: "name", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: routeConfig, + }, + HttpFilters: []*v3httppb.HttpFilter{emptyRouterFilter}, + }), + }, + }, + { + Name: "name", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: routeConfig, + }, + HttpFilters: []*v3httppb.HttpFilter{emptyRouterFilter}, + }), + }, + }, + }, + }, + }, + }), + wantName: v3LDSTarget, + wantErr: "duplicate filter name", + }, + { + name: "no terminal filter", + resource: testutils.MarshalAny(&v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: []*v3listenerpb.Filter{ + { + Name: "name", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: routeConfig, + }, + }), + }, + }, + }, + }, + }, + }), + wantName: v3LDSTarget, + wantErr: "http filters list is empty", + }, + { + name: "terminal filter not last", + resource: testutils.MarshalAny(&v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: []*v3listenerpb.Filter{ + { + Name: "name", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: routeConfig, + }, + HttpFilters: []*v3httppb.HttpFilter{emptyRouterFilter, serverOnlyCustomFilter}, + }), + }, + }, + }, + }, + }, + }), + wantName: v3LDSTarget, + wantErr: "is a terminal filter but it is not last in the filter chain", + }, + { + name: "last not terminal filter", + resource: testutils.MarshalAny(&v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: []*v3listenerpb.Filter{ + { + Name: "name", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3httppb.HttpConnectionManager{ + RouteSpecifier: &v3httppb.HttpConnectionManager_RouteConfig{ + RouteConfig: routeConfig, + }, + HttpFilters: []*v3httppb.HttpFilter{serverOnlyCustomFilter}, + }), + }, + }, + }, + }, + }, + }), + wantName: v3LDSTarget, + wantErr: "is not a terminal filter", + }, + { + name: "unsupported oneof in typed config of http filter", + resource: testutils.MarshalAny(&v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: []*v3listenerpb.Filter{ + { + Name: "name", + ConfigType: &v3listenerpb.Filter_ConfigDiscovery{}, + }, + }, + }, + }, + }), + wantName: v3LDSTarget, + wantErr: "unsupported config_type", + }, + { + name: "overlapping filter chain match criteria", + resource: testutils.MarshalAny(&v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{SourcePorts: []uint32{1, 2, 3, 4, 5}}, + Filters: emptyValidNetworkFilters, + }, + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{}, + Filters: emptyValidNetworkFilters, + }, + { + FilterChainMatch: &v3listenerpb.FilterChainMatch{SourcePorts: []uint32{5, 6, 7}}, + Filters: emptyValidNetworkFilters, + }, + }, + }), + wantName: v3LDSTarget, + wantErr: "multiple filter chains with overlapping matching rules are defined", + }, + { + name: "unsupported network filter", + resource: testutils.MarshalAny(&v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: []*v3listenerpb.Filter{ + { + Name: "name", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3httppb.LocalReplyConfig{}), + }, + }, + }, + }, + }, + }), + wantName: v3LDSTarget, + wantErr: "unsupported network filter", + }, + { + name: "badly marshaled network filter", + resource: testutils.MarshalAny(&v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: []*v3listenerpb.Filter{ + { + Name: "name", + ConfigType: &v3listenerpb.Filter_TypedConfig{ + TypedConfig: &anypb.Any{ + TypeUrl: version.V3HTTPConnManagerURL, + Value: []byte{1, 2, 3, 4}, + }, + }, + }, + }, + }, + }, + }), + wantName: v3LDSTarget, + wantErr: "failed unmarshaling of network filter", + }, + { + name: "unexpected transport socket name", + resource: testutils.MarshalAny(&v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: emptyValidNetworkFilters, + TransportSocket: &v3corepb.TransportSocket{ + Name: "unsupported-transport-socket-name", + }, + }, + }, + }), + wantName: v3LDSTarget, + wantErr: "transport_socket field has unexpected name", + }, + { + name: "unexpected transport socket typedConfig URL", + resource: testutils.MarshalAny(&v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: emptyValidNetworkFilters, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.UpstreamTlsContext{}), + }, + }, + }, + }, + }), + wantName: v3LDSTarget, + wantErr: "transport_socket field has unexpected typeURL", + }, + { + name: "badly marshaled transport socket", + resource: testutils.MarshalAny(&v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: emptyValidNetworkFilters, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: &anypb.Any{ + TypeUrl: version.V3DownstreamTLSContextURL, + Value: []byte{1, 2, 3, 4}, + }, + }, + }, + }, + }, + }), + wantName: v3LDSTarget, + wantErr: "failed to unmarshal DownstreamTlsContext in LDS response", + }, + { + name: "missing CommonTlsContext", + resource: testutils.MarshalAny(&v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: emptyValidNetworkFilters, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{}), + }, + }, + }, + }, + }), + wantName: v3LDSTarget, + wantErr: "DownstreamTlsContext in LDS response does not contain a CommonTlsContext", + }, + { + name: "rbac-allow-equating-direct-remote-ip-and-remote-ip-valid", + resource: v3LisToTestRBAC(0, nil), + wantName: v3LDSTarget, + wantUpdate: ListenerUpdate{ + InboundListenerCfg: &InboundListenerConfig{ + Address: "0.0.0.0", + Port: "9999", + FilterChains: &FilterChainManager{ + dstPrefixMap: map[string]*destPrefixEntry{ + unspecifiedPrefixMapKey: { + srcTypeArr: [3]*sourcePrefixes{ + { + srcPrefixMap: map[string]*sourcePrefixEntry{ + unspecifiedPrefixMapKey: { + srcPortMap: map[int]*FilterChain{ + 0: { + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + Raw: listenerEmptyTransportSocket, + }, + }, + { + name: "rbac-allow-equating-direct-remote-ip-and-remote-ip-invalid-num-untrusted-hops", + resource: v3LisToTestRBAC(1, nil), + wantName: v3LDSTarget, + wantErr: "xff_num_trusted_hops must be unset or zero", + }, + { + name: "rbac-allow-equating-direct-remote-ip-and-remote-ip-invalid-original-ip-detection-extension", + resource: v3LisToTestRBAC(0, []*v3corepb.TypedExtensionConfig{{Name: "something"}}), + wantName: v3LDSTarget, + wantErr: "original_ip_detection_extensions must be empty", + }, + { + name: "rbac-with-invalid-regex", + resource: v3LisWithBadRBACConfiguration(badRBACCfgRegex), + wantName: v3LDSTarget, + wantErr: "error parsing config for filter", + }, + { + name: "rbac-with-invalid-destination-ip-matcher", + resource: v3LisWithBadRBACConfiguration(badRBACCfgDestIP), + wantName: v3LDSTarget, + wantErr: "error parsing config for filter", + }, + { + name: "unsupported validation context in transport socket", + resource: testutils.MarshalAny(&v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: emptyValidNetworkFilters, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{ + ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextSdsSecretConfig{ + ValidationContextSdsSecretConfig: &v3tlspb.SdsSecretConfig{ + Name: "foo-sds-secret", + }, + }, + }, + }), + }, + }, + }, + }, + }), + wantName: v3LDSTarget, + wantErr: "validation context contains unexpected type", + }, + { + name: "empty transport socket", + resource: listenerEmptyTransportSocket, + wantName: v3LDSTarget, + wantUpdate: ListenerUpdate{ + InboundListenerCfg: &InboundListenerConfig{ + Address: "0.0.0.0", + Port: "9999", + FilterChains: &FilterChainManager{ + dstPrefixMap: map[string]*destPrefixEntry{ + unspecifiedPrefixMapKey: { + srcTypeArr: [3]*sourcePrefixes{ + { + srcPrefixMap: map[string]*sourcePrefixEntry{ + unspecifiedPrefixMapKey: { + srcPortMap: map[int]*FilterChain{ + 0: { + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + Raw: listenerEmptyTransportSocket, + }, + }, + { + name: "no identity and root certificate providers using deprecated fields", + resource: testutils.MarshalAny(&v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: emptyValidNetworkFilters, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{ + RequireClientCertificate: &wrapperspb.BoolValue{Value: true}, + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{ + InstanceName: "identityPluginInstance", + CertificateName: "identityCertName", + }, + }, + }), + }, + }, + }, + }, + }), + wantName: v3LDSTarget, + wantErr: "security configuration on the server-side does not contain root certificate provider instance name, but require_client_cert field is set", + }, + { + name: "no identity and root certificate providers using new fields", + resource: testutils.MarshalAny(&v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: emptyValidNetworkFilters, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{ + RequireClientCertificate: &wrapperspb.BoolValue{Value: true}, + CommonTlsContext: &v3tlspb.CommonTlsContext{ + TlsCertificateProviderInstance: &v3tlspb.CertificateProviderPluginInstance{ + InstanceName: "identityPluginInstance", + CertificateName: "identityCertName", + }, + }, + }), + }, + }, + }, + }, + }), + wantName: v3LDSTarget, + wantErr: "security configuration on the server-side does not contain root certificate provider instance name, but require_client_cert field is set", + }, + { + name: "no identity certificate provider with require_client_cert", + resource: testutils.MarshalAny(&v3listenerpb.Listener{ + Name: v3LDSTarget, + Address: localSocketAddress, + FilterChains: []*v3listenerpb.FilterChain{ + { + Name: "filter-chain-1", + Filters: emptyValidNetworkFilters, + TransportSocket: &v3corepb.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &v3corepb.TransportSocket_TypedConfig{ + TypedConfig: testutils.MarshalAny(&v3tlspb.DownstreamTlsContext{ + CommonTlsContext: &v3tlspb.CommonTlsContext{}, + }), + }, + }, + }, + }, + }), + wantName: v3LDSTarget, + wantErr: "security configuration on the server-side does not contain identity certificate provider instance name", + }, + { + name: "happy case with no validation context using deprecated fields", + resource: listenerNoValidationContextDeprecatedFields, + wantName: v3LDSTarget, + wantUpdate: ListenerUpdate{ + InboundListenerCfg: &InboundListenerConfig{ + Address: "0.0.0.0", + Port: "9999", + FilterChains: &FilterChainManager{ + dstPrefixMap: map[string]*destPrefixEntry{ + unspecifiedPrefixMapKey: { + srcTypeArr: [3]*sourcePrefixes{ + { + srcPrefixMap: map[string]*sourcePrefixEntry{ + unspecifiedPrefixMapKey: { + srcPortMap: map[int]*FilterChain{ + 0: { + SecurityCfg: &SecurityConfig{ + IdentityInstanceName: "identityPluginInstance", + IdentityCertName: "identityCertName", + }, + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + }, + }, + }, + }, + }, + }, + }, + def: &FilterChain{ + SecurityCfg: &SecurityConfig{ + IdentityInstanceName: "defaultIdentityPluginInstance", + IdentityCertName: "defaultIdentityCertName", + }, + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + }, + }, + Raw: listenerNoValidationContextDeprecatedFields, + }, + }, + { + name: "happy case with no validation context using new fields", + resource: listenerNoValidationContextNewFields, + wantName: v3LDSTarget, + wantUpdate: ListenerUpdate{ + InboundListenerCfg: &InboundListenerConfig{ + Address: "0.0.0.0", + Port: "9999", + FilterChains: &FilterChainManager{ + dstPrefixMap: map[string]*destPrefixEntry{ + unspecifiedPrefixMapKey: { + srcTypeArr: [3]*sourcePrefixes{ + { + srcPrefixMap: map[string]*sourcePrefixEntry{ + unspecifiedPrefixMapKey: { + srcPortMap: map[int]*FilterChain{ + 0: { + SecurityCfg: &SecurityConfig{ + IdentityInstanceName: "identityPluginInstance", + IdentityCertName: "identityCertName", + }, + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + }, + }, + }, + }, + }, + }, + }, + def: &FilterChain{ + SecurityCfg: &SecurityConfig{ + IdentityInstanceName: "defaultIdentityPluginInstance", + IdentityCertName: "defaultIdentityCertName", + }, + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + }, + }, + Raw: listenerNoValidationContextNewFields, + }, + }, + { + name: "happy case with validation context provider instance with deprecated fields", + resource: listenerWithValidationContextDeprecatedFields, + wantName: v3LDSTarget, + wantUpdate: ListenerUpdate{ + InboundListenerCfg: &InboundListenerConfig{ + Address: "0.0.0.0", + Port: "9999", + FilterChains: &FilterChainManager{ + dstPrefixMap: map[string]*destPrefixEntry{ + unspecifiedPrefixMapKey: { + srcTypeArr: [3]*sourcePrefixes{ + { + srcPrefixMap: map[string]*sourcePrefixEntry{ + unspecifiedPrefixMapKey: { + srcPortMap: map[int]*FilterChain{ + 0: { + SecurityCfg: &SecurityConfig{ + RootInstanceName: "rootPluginInstance", + RootCertName: "rootCertName", + IdentityInstanceName: "identityPluginInstance", + IdentityCertName: "identityCertName", + RequireClientCert: true, + }, + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + }, + }, + }, + }, + }, + }, + }, + def: &FilterChain{ + SecurityCfg: &SecurityConfig{ + RootInstanceName: "defaultRootPluginInstance", + RootCertName: "defaultRootCertName", + IdentityInstanceName: "defaultIdentityPluginInstance", + IdentityCertName: "defaultIdentityCertName", + RequireClientCert: true, + }, + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + }, + }, + Raw: listenerWithValidationContextDeprecatedFields, + }, + }, + { + name: "happy case with validation context provider instance with new fields", + resource: listenerWithValidationContextNewFields, + wantName: v3LDSTarget, + wantUpdate: ListenerUpdate{ + InboundListenerCfg: &InboundListenerConfig{ + Address: "0.0.0.0", + Port: "9999", + FilterChains: &FilterChainManager{ + dstPrefixMap: map[string]*destPrefixEntry{ + unspecifiedPrefixMapKey: { + srcTypeArr: [3]*sourcePrefixes{ + { + srcPrefixMap: map[string]*sourcePrefixEntry{ + unspecifiedPrefixMapKey: { + srcPortMap: map[int]*FilterChain{ + 0: { + SecurityCfg: &SecurityConfig{ + RootInstanceName: "rootPluginInstance", + RootCertName: "rootCertName", + IdentityInstanceName: "identityPluginInstance", + IdentityCertName: "identityCertName", + RequireClientCert: true, + }, + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + }, + }, + }, + }, + }, + }, + }, + def: &FilterChain{ + SecurityCfg: &SecurityConfig{ + RootInstanceName: "defaultRootPluginInstance", + RootCertName: "defaultRootCertName", + IdentityInstanceName: "defaultIdentityPluginInstance", + IdentityCertName: "defaultIdentityCertName", + RequireClientCert: true, + }, + InlineRouteConfig: inlineRouteConfig, + HTTPFilters: routerFilterList, + }, + }, + }, + Raw: listenerWithValidationContextNewFields, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + name, update, err := unmarshalListenerResource(test.resource) + if err != nil && !strings.Contains(err.Error(), test.wantErr) { + t.Errorf("unmarshalListenerResource(%s) = %v wantErr: %q", pretty.ToJSON(test.resource), err, test.wantErr) + } + if name != test.wantName { + t.Errorf("unmarshalListenerResource(%s), got name: %s, want: %s", pretty.ToJSON(test.resource), name, test.wantName) + } + if diff := cmp.Diff(update, test.wantUpdate, cmpOpts); diff != "" { + t.Errorf("unmarshalListenerResource(%s), got unexpected update, diff (-got +want): %v", pretty.ToJSON(test.resource), diff) + } + }) + } +} + +type filterConfig struct { + httpfilter.FilterConfig + Cfg proto.Message + Override proto.Message +} + +// httpFilter allows testing the http filter registry and parsing functionality. +type httpFilter struct { + httpfilter.ClientInterceptorBuilder + httpfilter.ServerInterceptorBuilder +} + +func (httpFilter) TypeURLs() []string { return []string{"custom.filter"} } + +func (httpFilter) ParseFilterConfig(cfg proto.Message) (httpfilter.FilterConfig, error) { + return filterConfig{Cfg: cfg}, nil +} + +func (httpFilter) ParseFilterConfigOverride(override proto.Message) (httpfilter.FilterConfig, error) { + return filterConfig{Override: override}, nil +} + +func (httpFilter) IsTerminal() bool { + return false +} + +// errHTTPFilter returns errors no matter what is passed to ParseFilterConfig. +type errHTTPFilter struct { + httpfilter.ClientInterceptorBuilder +} + +func (errHTTPFilter) TypeURLs() []string { return []string{"err.custom.filter"} } + +func (errHTTPFilter) ParseFilterConfig(cfg proto.Message) (httpfilter.FilterConfig, error) { + return nil, fmt.Errorf("error from ParseFilterConfig") +} + +func (errHTTPFilter) ParseFilterConfigOverride(override proto.Message) (httpfilter.FilterConfig, error) { + return nil, fmt.Errorf("error from ParseFilterConfigOverride") +} + +func (errHTTPFilter) IsTerminal() bool { + return false +} + +func init() { + httpfilter.Register(httpFilter{}) + httpfilter.Register(errHTTPFilter{}) + httpfilter.Register(serverOnlyHTTPFilter{}) + httpfilter.Register(clientOnlyHTTPFilter{}) +} + +// serverOnlyHTTPFilter does not implement ClientInterceptorBuilder +type serverOnlyHTTPFilter struct { + httpfilter.ServerInterceptorBuilder +} + +func (serverOnlyHTTPFilter) TypeURLs() []string { return []string{"serverOnly.custom.filter"} } + +func (serverOnlyHTTPFilter) ParseFilterConfig(cfg proto.Message) (httpfilter.FilterConfig, error) { + return filterConfig{Cfg: cfg}, nil +} + +func (serverOnlyHTTPFilter) ParseFilterConfigOverride(override proto.Message) (httpfilter.FilterConfig, error) { + return filterConfig{Override: override}, nil +} + +func (serverOnlyHTTPFilter) IsTerminal() bool { + return false +} + +// clientOnlyHTTPFilter does not implement ServerInterceptorBuilder +type clientOnlyHTTPFilter struct { + httpfilter.ClientInterceptorBuilder +} + +func (clientOnlyHTTPFilter) TypeURLs() []string { return []string{"clientOnly.custom.filter"} } + +func (clientOnlyHTTPFilter) ParseFilterConfig(cfg proto.Message) (httpfilter.FilterConfig, error) { + return filterConfig{Cfg: cfg}, nil +} + +func (clientOnlyHTTPFilter) ParseFilterConfigOverride(override proto.Message) (httpfilter.FilterConfig, error) { + return filterConfig{Override: override}, nil +} + +func (clientOnlyHTTPFilter) IsTerminal() bool { + return false +} + +var customFilterConfig = &anypb.Any{ + TypeUrl: "custom.filter", + Value: []byte{1, 2, 3}, +} + +var errFilterConfig = &anypb.Any{ + TypeUrl: "err.custom.filter", + Value: []byte{1, 2, 3}, +} + +var serverOnlyCustomFilterConfig = &anypb.Any{ + TypeUrl: "serverOnly.custom.filter", + Value: []byte{1, 2, 3}, +} + +var clientOnlyCustomFilterConfig = &anypb.Any{ + TypeUrl: "clientOnly.custom.filter", + Value: []byte{1, 2, 3}, +} + +// This custom filter uses the old TypedStruct message from the cncf/udpa repo. +var customFilterOldTypedStructConfig = &v1udpaudpatypepb.TypedStruct{ + TypeUrl: "custom.filter", + Value: &spb.Struct{ + Fields: map[string]*spb.Value{ + "foo": {Kind: &spb.Value_StringValue{StringValue: "bar"}}, + }, + }, +} +var wrappedCustomFilterOldTypedStructConfig *anypb.Any + +// This custom filter uses the new TypedStruct message from the cncf/xds repo. +var customFilterNewTypedStructConfig = &v3xdsxdstypepb.TypedStruct{ + TypeUrl: "custom.filter", + Value: &spb.Struct{ + Fields: map[string]*spb.Value{ + "foo": {Kind: &spb.Value_StringValue{StringValue: "bar"}}, + }, + }, +} +var wrappedCustomFilterNewTypedStructConfig *anypb.Any + +func init() { + wrappedCustomFilterOldTypedStructConfig = testutils.MarshalAny(customFilterOldTypedStructConfig) + wrappedCustomFilterNewTypedStructConfig = testutils.MarshalAny(customFilterNewTypedStructConfig) +} + +var unknownFilterConfig = &anypb.Any{ + TypeUrl: "unknown.custom.filter", + Value: []byte{1, 2, 3}, +} + +func wrappedOptionalFilter(name string) *anypb.Any { + return testutils.MarshalAny(&v3routepb.FilterConfig{ + IsOptional: true, + Config: &anypb.Any{ + TypeUrl: name, + Value: []byte{1, 2, 3}, + }, + }) +} diff --git a/xds/internal/xdsclient/xdsresource/unmarshal_rds.go b/xds/internal/xdsclient/xdsresource/unmarshal_rds.go new file mode 100644 index 000000000000..c51a0c24b508 --- /dev/null +++ b/xds/internal/xdsclient/xdsresource/unmarshal_rds.go @@ -0,0 +1,448 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "fmt" + "math" + "regexp" + "strings" + "time" + + "github.com/golang/protobuf/proto" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/xds/matcher" + "google.golang.org/grpc/xds/internal/clusterspecifier" + "google.golang.org/protobuf/types/known/anypb" + + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3typepb "github.com/envoyproxy/go-control-plane/envoy/type/v3" +) + +func unmarshalRouteConfigResource(r *anypb.Any) (string, RouteConfigUpdate, error) { + r, err := UnwrapResource(r) + if err != nil { + return "", RouteConfigUpdate{}, fmt.Errorf("failed to unwrap resource: %v", err) + } + + if !IsRouteConfigResource(r.GetTypeUrl()) { + return "", RouteConfigUpdate{}, fmt.Errorf("unexpected resource type: %q ", r.GetTypeUrl()) + } + rc := &v3routepb.RouteConfiguration{} + if err := proto.Unmarshal(r.GetValue(), rc); err != nil { + return "", RouteConfigUpdate{}, fmt.Errorf("failed to unmarshal resource: %v", err) + } + + u, err := generateRDSUpdateFromRouteConfiguration(rc) + if err != nil { + return rc.GetName(), RouteConfigUpdate{}, err + } + u.Raw = r + return rc.GetName(), u, nil +} + +// generateRDSUpdateFromRouteConfiguration checks if the provided +// RouteConfiguration meets the expected criteria. If so, it returns a +// RouteConfigUpdate with nil error. +// +// A RouteConfiguration resource is considered valid when only if it contains a +// VirtualHost whose domain field matches the server name from the URI passed +// to the gRPC channel, and it contains a clusterName or a weighted cluster. +// +// The RouteConfiguration includes a list of virtualHosts, which may have zero +// or more elements. We are interested in the element whose domains field +// matches the server name specified in the "xds:" URI. The only field in the +// VirtualHost proto that the we are interested in is the list of routes. We +// only look at the last route in the list (the default route), whose match +// field must be empty and whose route field must be set. Inside that route +// message, the cluster field will contain the clusterName or weighted clusters +// we are looking for. +func generateRDSUpdateFromRouteConfiguration(rc *v3routepb.RouteConfiguration) (RouteConfigUpdate, error) { + vhs := make([]*VirtualHost, 0, len(rc.GetVirtualHosts())) + csps := make(map[string]clusterspecifier.BalancerConfig) + if envconfig.XDSRLS { + var err error + csps, err = processClusterSpecifierPlugins(rc.ClusterSpecifierPlugins) + if err != nil { + return RouteConfigUpdate{}, fmt.Errorf("received route is invalid: %v", err) + } + } + // cspNames represents all the cluster specifiers referenced by Route + // Actions - any cluster specifiers not referenced by a Route Action can be + // ignored and not emitted by the xdsclient. + var cspNames = make(map[string]bool) + for _, vh := range rc.GetVirtualHosts() { + routes, cspNs, err := routesProtoToSlice(vh.Routes, csps) + if err != nil { + return RouteConfigUpdate{}, fmt.Errorf("received route is invalid: %v", err) + } + for n := range cspNs { + cspNames[n] = true + } + rc, err := generateRetryConfig(vh.GetRetryPolicy()) + if err != nil { + return RouteConfigUpdate{}, fmt.Errorf("received route is invalid: %v", err) + } + vhOut := &VirtualHost{ + Domains: vh.GetDomains(), + Routes: routes, + RetryConfig: rc, + } + cfgs, err := processHTTPFilterOverrides(vh.GetTypedPerFilterConfig()) + if err != nil { + return RouteConfigUpdate{}, fmt.Errorf("virtual host %+v: %v", vh, err) + } + vhOut.HTTPFilterConfigOverride = cfgs + vhs = append(vhs, vhOut) + } + + // "For any entry in the RouteConfiguration.cluster_specifier_plugins not + // referenced by an enclosed ActionType's cluster_specifier_plugin, the xDS + // client should not provide it to its consumers." - RLS in xDS Design + for name := range csps { + if !cspNames[name] { + delete(csps, name) + } + } + + return RouteConfigUpdate{VirtualHosts: vhs, ClusterSpecifierPlugins: csps}, nil +} + +func processClusterSpecifierPlugins(csps []*v3routepb.ClusterSpecifierPlugin) (map[string]clusterspecifier.BalancerConfig, error) { + cspCfgs := make(map[string]clusterspecifier.BalancerConfig) + // "The xDS client will inspect all elements of the + // cluster_specifier_plugins field looking up a plugin based on the + // extension.typed_config of each." - RLS in xDS design + for _, csp := range csps { + cs := clusterspecifier.Get(csp.GetExtension().GetTypedConfig().GetTypeUrl()) + if cs == nil { + if csp.GetIsOptional() { + // "If a plugin is not supported but has is_optional set, then + // we will ignore any routes that point to that plugin" + cspCfgs[csp.GetExtension().GetName()] = nil + continue + } + // "If no plugin is registered for it, the resource will be NACKed." + // - RLS in xDS design + return nil, fmt.Errorf("cluster specifier %q of type %q was not found", csp.GetExtension().GetName(), csp.GetExtension().GetTypedConfig().GetTypeUrl()) + } + lbCfg, err := cs.ParseClusterSpecifierConfig(csp.GetExtension().GetTypedConfig()) + if err != nil { + // "If a plugin is found, the value of the typed_config field will + // be passed to it's conversion method, and if an error is + // encountered, the resource will be NACKED." - RLS in xDS design + return nil, fmt.Errorf("error: %q parsing config %q for cluster specifier %q of type %q", err, csp.GetExtension().GetTypedConfig(), csp.GetExtension().GetName(), csp.GetExtension().GetTypedConfig().GetTypeUrl()) + } + // "If all cluster specifiers are valid, the xDS client will store the + // configurations in a map keyed by the name of the extension instance." - + // RLS in xDS Design + cspCfgs[csp.GetExtension().GetName()] = lbCfg + } + return cspCfgs, nil +} + +func generateRetryConfig(rp *v3routepb.RetryPolicy) (*RetryConfig, error) { + if rp == nil { + return nil, nil + } + + cfg := &RetryConfig{RetryOn: make(map[codes.Code]bool)} + for _, s := range strings.Split(rp.GetRetryOn(), ",") { + switch strings.TrimSpace(strings.ToLower(s)) { + case "cancelled": + cfg.RetryOn[codes.Canceled] = true + case "deadline-exceeded": + cfg.RetryOn[codes.DeadlineExceeded] = true + case "internal": + cfg.RetryOn[codes.Internal] = true + case "resource-exhausted": + cfg.RetryOn[codes.ResourceExhausted] = true + case "unavailable": + cfg.RetryOn[codes.Unavailable] = true + } + } + + if rp.NumRetries == nil { + cfg.NumRetries = 1 + } else { + cfg.NumRetries = rp.GetNumRetries().Value + if cfg.NumRetries < 1 { + return nil, fmt.Errorf("retry_policy.num_retries = %v; must be >= 1", cfg.NumRetries) + } + } + + backoff := rp.GetRetryBackOff() + if backoff == nil { + cfg.RetryBackoff.BaseInterval = 25 * time.Millisecond + } else { + cfg.RetryBackoff.BaseInterval = backoff.GetBaseInterval().AsDuration() + if cfg.RetryBackoff.BaseInterval <= 0 { + return nil, fmt.Errorf("retry_policy.base_interval = %v; must be > 0", cfg.RetryBackoff.BaseInterval) + } + } + if max := backoff.GetMaxInterval(); max == nil { + cfg.RetryBackoff.MaxInterval = 10 * cfg.RetryBackoff.BaseInterval + } else { + cfg.RetryBackoff.MaxInterval = max.AsDuration() + if cfg.RetryBackoff.MaxInterval <= 0 { + return nil, fmt.Errorf("retry_policy.max_interval = %v; must be > 0", cfg.RetryBackoff.MaxInterval) + } + } + + if len(cfg.RetryOn) == 0 { + return &RetryConfig{}, nil + } + return cfg, nil +} + +func routesProtoToSlice(routes []*v3routepb.Route, csps map[string]clusterspecifier.BalancerConfig) ([]*Route, map[string]bool, error) { + var routesRet []*Route + var cspNames = make(map[string]bool) + for _, r := range routes { + match := r.GetMatch() + if match == nil { + return nil, nil, fmt.Errorf("route %+v doesn't have a match", r) + } + + if len(match.GetQueryParameters()) != 0 { + // Ignore route with query parameters. + logger.Warningf("Ignoring route %+v with query parameter matchers", r) + continue + } + + pathSp := match.GetPathSpecifier() + if pathSp == nil { + return nil, nil, fmt.Errorf("route %+v doesn't have a path specifier", r) + } + + var route Route + switch pt := pathSp.(type) { + case *v3routepb.RouteMatch_Prefix: + route.Prefix = &pt.Prefix + case *v3routepb.RouteMatch_Path: + route.Path = &pt.Path + case *v3routepb.RouteMatch_SafeRegex: + regex := pt.SafeRegex.GetRegex() + re, err := regexp.Compile(regex) + if err != nil { + return nil, nil, fmt.Errorf("route %+v contains an invalid regex %q", r, regex) + } + route.Regex = re + default: + return nil, nil, fmt.Errorf("route %+v has an unrecognized path specifier: %+v", r, pt) + } + + if caseSensitive := match.GetCaseSensitive(); caseSensitive != nil { + route.CaseInsensitive = !caseSensitive.Value + } + + for _, h := range match.GetHeaders() { + var header HeaderMatcher + switch ht := h.GetHeaderMatchSpecifier().(type) { + case *v3routepb.HeaderMatcher_ExactMatch: + header.ExactMatch = &ht.ExactMatch + case *v3routepb.HeaderMatcher_SafeRegexMatch: + regex := ht.SafeRegexMatch.GetRegex() + re, err := regexp.Compile(regex) + if err != nil { + return nil, nil, fmt.Errorf("route %+v contains an invalid regex %q", r, regex) + } + header.RegexMatch = re + case *v3routepb.HeaderMatcher_RangeMatch: + header.RangeMatch = &Int64Range{ + Start: ht.RangeMatch.Start, + End: ht.RangeMatch.End, + } + case *v3routepb.HeaderMatcher_PresentMatch: + header.PresentMatch = &ht.PresentMatch + case *v3routepb.HeaderMatcher_PrefixMatch: + header.PrefixMatch = &ht.PrefixMatch + case *v3routepb.HeaderMatcher_SuffixMatch: + header.SuffixMatch = &ht.SuffixMatch + case *v3routepb.HeaderMatcher_StringMatch: + sm, err := matcher.StringMatcherFromProto(ht.StringMatch) + if err != nil { + return nil, nil, fmt.Errorf("route %+v has an invalid string matcher: %v", err, ht.StringMatch) + } + header.StringMatch = &sm + default: + return nil, nil, fmt.Errorf("route %+v has an unrecognized header matcher: %+v", r, ht) + } + header.Name = h.GetName() + invert := h.GetInvertMatch() + header.InvertMatch = &invert + route.Headers = append(route.Headers, &header) + } + + if fr := match.GetRuntimeFraction(); fr != nil { + d := fr.GetDefaultValue() + n := d.GetNumerator() + switch d.GetDenominator() { + case v3typepb.FractionalPercent_HUNDRED: + n *= 10000 + case v3typepb.FractionalPercent_TEN_THOUSAND: + n *= 100 + case v3typepb.FractionalPercent_MILLION: + } + route.Fraction = &n + } + + switch r.GetAction().(type) { + case *v3routepb.Route_Route: + route.WeightedClusters = make(map[string]WeightedCluster) + action := r.GetRoute() + + // Hash Policies are only applicable for a Ring Hash LB. + if envconfig.XDSRingHash { + hp, err := hashPoliciesProtoToSlice(action.HashPolicy) + if err != nil { + return nil, nil, err + } + route.HashPolicies = hp + } + + switch a := action.GetClusterSpecifier().(type) { + case *v3routepb.RouteAction_Cluster: + route.WeightedClusters[a.Cluster] = WeightedCluster{Weight: 1} + case *v3routepb.RouteAction_WeightedClusters: + wcs := a.WeightedClusters + var totalWeight uint64 + for _, c := range wcs.Clusters { + w := c.GetWeight().GetValue() + if w == 0 { + continue + } + totalWeight += uint64(w) + if totalWeight > math.MaxUint32 { + return nil, nil, fmt.Errorf("xds: total weight of clusters exceeds MaxUint32") + } + wc := WeightedCluster{Weight: w} + cfgs, err := processHTTPFilterOverrides(c.GetTypedPerFilterConfig()) + if err != nil { + return nil, nil, fmt.Errorf("route %+v, action %+v: %v", r, a, err) + } + wc.HTTPFilterConfigOverride = cfgs + route.WeightedClusters[c.GetName()] = wc + } + if totalWeight == 0 { + return nil, nil, fmt.Errorf("route %+v, action %+v, has no valid cluster in WeightedCluster action", r, a) + } + case *v3routepb.RouteAction_ClusterSpecifierPlugin: + // gRFC A28 was updated to say the following: + // + // The route’s action field must be route, and its + // cluster_specifier: + // - Can be Cluster + // - Can be Weighted_clusters + // - Can be unset or an unsupported field. The route containing + // this action will be ignored. + // + // This means that if this env var is not set, we should treat + // it as if it we didn't know about the cluster_specifier_plugin + // at all. + if !envconfig.XDSRLS { + logger.Warningf("Ignoring route %+v with unsupported route_action field: cluster_specifier_plugin", r) + continue + } + if _, ok := csps[a.ClusterSpecifierPlugin]; !ok { + // "When processing RouteActions, if any action includes a + // cluster_specifier_plugin value that is not in + // RouteConfiguration.cluster_specifier_plugins, the + // resource will be NACKed." - RLS in xDS design + return nil, nil, fmt.Errorf("route %+v, action %+v, specifies a cluster specifier plugin %+v that is not in Route Configuration", r, a, a.ClusterSpecifierPlugin) + } + if csps[a.ClusterSpecifierPlugin] == nil { + logger.Warningf("Ignoring route %+v with optional and unsupported cluster specifier plugin %+v", r, a.ClusterSpecifierPlugin) + continue + } + cspNames[a.ClusterSpecifierPlugin] = true + route.ClusterSpecifierPlugin = a.ClusterSpecifierPlugin + default: + logger.Warningf("Ignoring route %+v with unknown ClusterSpecifier %+v", r, a) + continue + } + + msd := action.GetMaxStreamDuration() + // Prefer grpc_timeout_header_max, if set. + dur := msd.GetGrpcTimeoutHeaderMax() + if dur == nil { + dur = msd.GetMaxStreamDuration() + } + if dur != nil { + d := dur.AsDuration() + route.MaxStreamDuration = &d + } + + var err error + route.RetryConfig, err = generateRetryConfig(action.GetRetryPolicy()) + if err != nil { + return nil, nil, fmt.Errorf("route %+v, action %+v: %v", r, action, err) + } + + route.ActionType = RouteActionRoute + + case *v3routepb.Route_NonForwardingAction: + // Expected to be used on server side. + route.ActionType = RouteActionNonForwardingAction + default: + route.ActionType = RouteActionUnsupported + } + + cfgs, err := processHTTPFilterOverrides(r.GetTypedPerFilterConfig()) + if err != nil { + return nil, nil, fmt.Errorf("route %+v: %v", r, err) + } + route.HTTPFilterConfigOverride = cfgs + routesRet = append(routesRet, &route) + } + return routesRet, cspNames, nil +} + +func hashPoliciesProtoToSlice(policies []*v3routepb.RouteAction_HashPolicy) ([]*HashPolicy, error) { + var hashPoliciesRet []*HashPolicy + for _, p := range policies { + policy := HashPolicy{Terminal: p.Terminal} + switch p.GetPolicySpecifier().(type) { + case *v3routepb.RouteAction_HashPolicy_Header_: + policy.HashPolicyType = HashPolicyTypeHeader + policy.HeaderName = p.GetHeader().GetHeaderName() + if rr := p.GetHeader().GetRegexRewrite(); rr != nil { + regex := rr.GetPattern().GetRegex() + re, err := regexp.Compile(regex) + if err != nil { + return nil, fmt.Errorf("hash policy %+v contains an invalid regex %q", p, regex) + } + policy.Regex = re + policy.RegexSubstitution = rr.GetSubstitution() + } + case *v3routepb.RouteAction_HashPolicy_FilterState_: + if p.GetFilterState().GetKey() != "io.grpc.channel_id" { + logger.Warningf("Ignoring hash policy %+v with invalid key for filter state policy %q", p, p.GetFilterState().GetKey()) + continue + } + policy.HashPolicyType = HashPolicyTypeChannelID + default: + logger.Warningf("Ignoring unsupported hash policy %T", p.GetPolicySpecifier()) + continue + } + + hashPoliciesRet = append(hashPoliciesRet, &policy) + } + return hashPoliciesRet, nil +} diff --git a/xds/internal/xdsclient/xdsresource/unmarshal_rds_test.go b/xds/internal/xdsclient/xdsresource/unmarshal_rds_test.go new file mode 100644 index 000000000000..6435237a3572 --- /dev/null +++ b/xds/internal/xdsclient/xdsresource/unmarshal_rds_test.go @@ -0,0 +1,1654 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xdsresource + +import ( + "errors" + "fmt" + "math" + "regexp" + "testing" + "time" + + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + "github.com/golang/protobuf/proto" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/pretty" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/xds/matcher" + "google.golang.org/grpc/xds/internal/clusterspecifier" + "google.golang.org/grpc/xds/internal/httpfilter" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource/version" + "google.golang.org/protobuf/types/known/durationpb" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + rpb "github.com/envoyproxy/go-control-plane/envoy/config/rbac/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3rbacpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/rbac/v3" + v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3" + v3typepb "github.com/envoyproxy/go-control-plane/envoy/type/v3" + anypb "github.com/golang/protobuf/ptypes/any" + wrapperspb "github.com/golang/protobuf/ptypes/wrappers" +) + +func (s) TestRDSGenerateRDSUpdateFromRouteConfiguration(t *testing.T) { + const ( + uninterestingDomain = "uninteresting.domain" + uninterestingClusterName = "uninterestingClusterName" + ldsTarget = "lds.target.good:1111" + routeName = "routeName" + clusterName = "clusterName" + ) + + var ( + goodRouteConfigWithFilterConfigs = func(cfgs map[string]*anypb.Any) *v3routepb.RouteConfiguration { + return &v3routepb.RouteConfiguration{ + Name: routeName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{ldsTarget}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}}, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName}}, + }, + }}, + TypedPerFilterConfig: cfgs, + }}, + } + } + goodRouteConfigWithClusterSpecifierPlugins = func(csps []*v3routepb.ClusterSpecifierPlugin, cspReferences []string) *v3routepb.RouteConfiguration { + var rs []*v3routepb.Route + + for i, cspReference := range cspReferences { + rs = append(rs, &v3routepb.Route{ + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: fmt.Sprint(i + 1)}}, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_ClusterSpecifierPlugin{ClusterSpecifierPlugin: cspReference}, + }, + }, + }) + } + + rc := &v3routepb.RouteConfiguration{ + Name: routeName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{ldsTarget}, + Routes: rs, + }}, + ClusterSpecifierPlugins: csps, + } + + return rc + } + goodRouteConfigWithClusterSpecifierPluginsAndNormalRoute = func(csps []*v3routepb.ClusterSpecifierPlugin, cspReferences []string) *v3routepb.RouteConfiguration { + rs := goodRouteConfigWithClusterSpecifierPlugins(csps, cspReferences) + rs.VirtualHosts[0].Routes = append(rs.VirtualHosts[0].Routes, &v3routepb.Route{ + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}, + CaseSensitive: &wrapperspb.BoolValue{Value: false}, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName}, + }}}) + return rs + } + goodRouteConfigWithUnsupportedClusterSpecifier = &v3routepb.RouteConfiguration{ + Name: routeName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{ldsTarget}, + Routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}, + CaseSensitive: &wrapperspb.BoolValue{Value: false}, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName}}, + }}, + { + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "|"}}, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_ClusterHeader{}}, + }}, + }, + }, + }, + } + + goodUpdateWithFilterConfigs = func(cfgs map[string]httpfilter.FilterConfig) RouteConfigUpdate { + return RouteConfigUpdate{ + VirtualHosts: []*VirtualHost{{ + Domains: []string{ldsTarget}, + Routes: []*Route{{ + Prefix: newStringP("/"), + WeightedClusters: map[string]WeightedCluster{clusterName: {Weight: 1}}, + ActionType: RouteActionRoute, + }}, + HTTPFilterConfigOverride: cfgs, + }}, + } + } + goodUpdate = RouteConfigUpdate{ + VirtualHosts: []*VirtualHost{{ + Domains: []string{ldsTarget}, + Routes: nil, + }}, + } + goodUpdateWithNormalRoute = RouteConfigUpdate{ + VirtualHosts: []*VirtualHost{ + { + Domains: []string{ldsTarget}, + Routes: []*Route{{Prefix: newStringP("/"), + CaseInsensitive: true, + WeightedClusters: map[string]WeightedCluster{clusterName: {Weight: 1}}, + ActionType: RouteActionRoute}}, + }, + }, + } + goodUpdateWithClusterSpecifierPluginA = RouteConfigUpdate{ + VirtualHosts: []*VirtualHost{{ + Domains: []string{ldsTarget}, + Routes: []*Route{{ + Prefix: newStringP("1"), + ActionType: RouteActionRoute, + ClusterSpecifierPlugin: "cspA", + }}, + }}, + ClusterSpecifierPlugins: map[string]clusterspecifier.BalancerConfig{ + "cspA": nil, + }, + } + clusterSpecifierPlugin = func(name string, config *anypb.Any, isOptional bool) *v3routepb.ClusterSpecifierPlugin { + return &v3routepb.ClusterSpecifierPlugin{ + Extension: &v3corepb.TypedExtensionConfig{ + Name: name, + TypedConfig: config, + }, + IsOptional: isOptional, + } + } + goodRouteConfigWithRetryPolicy = func(vhrp *v3routepb.RetryPolicy, rrp *v3routepb.RetryPolicy) *v3routepb.RouteConfiguration { + return &v3routepb.RouteConfiguration{ + Name: routeName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{ldsTarget}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}}, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName}, + RetryPolicy: rrp, + }, + }, + }}, + RetryPolicy: vhrp, + }}, + } + } + goodUpdateWithRetryPolicy = func(vhrc *RetryConfig, rrc *RetryConfig) RouteConfigUpdate { + return RouteConfigUpdate{ + VirtualHosts: []*VirtualHost{{ + Domains: []string{ldsTarget}, + Routes: []*Route{{ + Prefix: newStringP("/"), + WeightedClusters: map[string]WeightedCluster{clusterName: {Weight: 1}}, + ActionType: RouteActionRoute, + RetryConfig: rrc, + }}, + RetryConfig: vhrc, + }}, + } + } + defaultRetryBackoff = RetryBackoff{BaseInterval: 25 * time.Millisecond, MaxInterval: 250 * time.Millisecond} + ) + + oldRLS := envconfig.XDSRLS + defer func() { + envconfig.XDSRLS = oldRLS + }() + + tests := []struct { + name string + rc *v3routepb.RouteConfiguration + wantUpdate RouteConfigUpdate + wantError bool + rlsEnabled bool + }{ + { + name: "default-route-match-field-is-nil", + rc: &v3routepb.RouteConfiguration{ + VirtualHosts: []*v3routepb.VirtualHost{ + { + Domains: []string{ldsTarget}, + Routes: []*v3routepb.Route{ + { + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName}, + }, + }, + }, + }, + }, + }, + }, + wantError: true, + }, + { + name: "default-route-match-field-is-non-nil", + rc: &v3routepb.RouteConfiguration{ + VirtualHosts: []*v3routepb.VirtualHost{ + { + Domains: []string{ldsTarget}, + Routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{}, + Action: &v3routepb.Route_Route{}, + }, + }, + }, + }, + }, + wantError: true, + }, + { + name: "default-route-routeaction-field-is-nil", + rc: &v3routepb.RouteConfiguration{ + VirtualHosts: []*v3routepb.VirtualHost{ + { + Domains: []string{ldsTarget}, + Routes: []*v3routepb.Route{{}}, + }, + }, + }, + wantError: true, + }, + { + name: "default-route-cluster-field-is-empty", + rc: &v3routepb.RouteConfiguration{ + VirtualHosts: []*v3routepb.VirtualHost{ + { + Domains: []string{ldsTarget}, + Routes: []*v3routepb.Route{ + { + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_ClusterHeader{}, + }, + }, + }, + }, + }, + }, + }, + wantError: true, + }, + { + // default route's match sets case-sensitive to false. + name: "good-route-config-but-with-casesensitive-false", + rc: &v3routepb.RouteConfiguration{ + Name: routeName, + VirtualHosts: []*v3routepb.VirtualHost{{ + Domains: []string{ldsTarget}, + Routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}, + CaseSensitive: &wrapperspb.BoolValue{Value: false}, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName}, + }}}}}}}, + wantUpdate: RouteConfigUpdate{ + VirtualHosts: []*VirtualHost{ + { + Domains: []string{ldsTarget}, + Routes: []*Route{{Prefix: newStringP("/"), + CaseInsensitive: true, + WeightedClusters: map[string]WeightedCluster{clusterName: {Weight: 1}}, + ActionType: RouteActionRoute}}, + }, + }, + }, + }, + { + name: "good-route-config-with-empty-string-route", + rc: &v3routepb.RouteConfiguration{ + Name: routeName, + VirtualHosts: []*v3routepb.VirtualHost{ + { + Domains: []string{uninterestingDomain}, + Routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: ""}}, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: uninterestingClusterName}, + }, + }, + }, + }, + }, + { + Domains: []string{ldsTarget}, + Routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: ""}}, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName}, + }, + }, + }, + }, + }, + }, + }, + wantUpdate: RouteConfigUpdate{ + VirtualHosts: []*VirtualHost{ + { + Domains: []string{uninterestingDomain}, + Routes: []*Route{{Prefix: newStringP(""), + WeightedClusters: map[string]WeightedCluster{uninterestingClusterName: {Weight: 1}}, + ActionType: RouteActionRoute}}, + }, + { + Domains: []string{ldsTarget}, + Routes: []*Route{{Prefix: newStringP(""), + WeightedClusters: map[string]WeightedCluster{clusterName: {Weight: 1}}, + ActionType: RouteActionRoute}}, + }, + }, + }, + }, + { + // default route's match is not empty string, but "/". + name: "good-route-config-with-slash-string-route", + rc: &v3routepb.RouteConfiguration{ + Name: routeName, + VirtualHosts: []*v3routepb.VirtualHost{ + { + Domains: []string{ldsTarget}, + Routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}}, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName}, + }, + }, + }, + }, + }, + }, + }, + wantUpdate: RouteConfigUpdate{ + VirtualHosts: []*VirtualHost{ + { + Domains: []string{ldsTarget}, + Routes: []*Route{{Prefix: newStringP("/"), + WeightedClusters: map[string]WeightedCluster{clusterName: {Weight: 1}}, + ActionType: RouteActionRoute}}, + }, + }, + }, + }, + { + name: "good-route-config-with-weighted_clusters", + rc: &v3routepb.RouteConfiguration{ + Name: routeName, + VirtualHosts: []*v3routepb.VirtualHost{ + { + Domains: []string{ldsTarget}, + Routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}}, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{ + WeightedClusters: &v3routepb.WeightedCluster{ + Clusters: []*v3routepb.WeightedCluster_ClusterWeight{ + {Name: "a", Weight: &wrapperspb.UInt32Value{Value: 2}}, + {Name: "b", Weight: &wrapperspb.UInt32Value{Value: 3}}, + {Name: "c", Weight: &wrapperspb.UInt32Value{Value: 5}}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + wantUpdate: RouteConfigUpdate{ + VirtualHosts: []*VirtualHost{ + { + Domains: []string{ldsTarget}, + Routes: []*Route{{ + Prefix: newStringP("/"), + WeightedClusters: map[string]WeightedCluster{ + "a": {Weight: 2}, + "b": {Weight: 3}, + "c": {Weight: 5}, + }, + ActionType: RouteActionRoute, + }}, + }, + }, + }, + }, + { + name: "good-route-config-with-max-stream-duration", + rc: &v3routepb.RouteConfiguration{ + Name: routeName, + VirtualHosts: []*v3routepb.VirtualHost{ + { + Domains: []string{ldsTarget}, + Routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}}, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName}, + MaxStreamDuration: &v3routepb.RouteAction_MaxStreamDuration{MaxStreamDuration: durationpb.New(time.Second)}, + }, + }, + }, + }, + }, + }, + }, + wantUpdate: RouteConfigUpdate{ + VirtualHosts: []*VirtualHost{ + { + Domains: []string{ldsTarget}, + Routes: []*Route{{ + Prefix: newStringP("/"), + WeightedClusters: map[string]WeightedCluster{clusterName: {Weight: 1}}, + MaxStreamDuration: newDurationP(time.Second), + ActionType: RouteActionRoute, + }}, + }, + }, + }, + }, + { + name: "good-route-config-with-grpc-timeout-header-max", + rc: &v3routepb.RouteConfiguration{ + Name: routeName, + VirtualHosts: []*v3routepb.VirtualHost{ + { + Domains: []string{ldsTarget}, + Routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}}, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName}, + MaxStreamDuration: &v3routepb.RouteAction_MaxStreamDuration{GrpcTimeoutHeaderMax: durationpb.New(time.Second)}, + }, + }, + }, + }, + }, + }, + }, + wantUpdate: RouteConfigUpdate{ + VirtualHosts: []*VirtualHost{ + { + Domains: []string{ldsTarget}, + Routes: []*Route{{ + Prefix: newStringP("/"), + WeightedClusters: map[string]WeightedCluster{clusterName: {Weight: 1}}, + MaxStreamDuration: newDurationP(time.Second), + ActionType: RouteActionRoute, + }}, + }, + }, + }, + }, + { + name: "good-route-config-with-both-timeouts", + rc: &v3routepb.RouteConfiguration{ + Name: routeName, + VirtualHosts: []*v3routepb.VirtualHost{ + { + Domains: []string{ldsTarget}, + Routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}}, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName}, + MaxStreamDuration: &v3routepb.RouteAction_MaxStreamDuration{MaxStreamDuration: durationpb.New(2 * time.Second), GrpcTimeoutHeaderMax: durationpb.New(0)}, + }, + }, + }, + }, + }, + }, + }, + wantUpdate: RouteConfigUpdate{ + VirtualHosts: []*VirtualHost{ + { + Domains: []string{ldsTarget}, + Routes: []*Route{{ + Prefix: newStringP("/"), + WeightedClusters: map[string]WeightedCluster{clusterName: {Weight: 1}}, + MaxStreamDuration: newDurationP(0), + ActionType: RouteActionRoute, + }}, + }, + }, + }, + }, + { + name: "good-route-config-with-http-filter-config", + rc: goodRouteConfigWithFilterConfigs(map[string]*anypb.Any{"foo": customFilterConfig}), + wantUpdate: goodUpdateWithFilterConfigs(map[string]httpfilter.FilterConfig{"foo": filterConfig{Override: customFilterConfig}}), + }, + { + name: "good-route-config-with-http-filter-config-in-old-typed-struct", + rc: goodRouteConfigWithFilterConfigs(map[string]*anypb.Any{"foo": wrappedCustomFilterOldTypedStructConfig}), + wantUpdate: goodUpdateWithFilterConfigs(map[string]httpfilter.FilterConfig{"foo": filterConfig{Override: customFilterOldTypedStructConfig}}), + }, + { + name: "good-route-config-with-http-filter-config-in-new-typed-struct", + rc: goodRouteConfigWithFilterConfigs(map[string]*anypb.Any{"foo": wrappedCustomFilterNewTypedStructConfig}), + wantUpdate: goodUpdateWithFilterConfigs(map[string]httpfilter.FilterConfig{"foo": filterConfig{Override: customFilterNewTypedStructConfig}}), + }, + { + name: "good-route-config-with-optional-http-filter-config", + rc: goodRouteConfigWithFilterConfigs(map[string]*anypb.Any{"foo": wrappedOptionalFilter("custom.filter")}), + wantUpdate: goodUpdateWithFilterConfigs(map[string]httpfilter.FilterConfig{"foo": filterConfig{Override: customFilterConfig}}), + }, + { + name: "good-route-config-with-http-err-filter-config", + rc: goodRouteConfigWithFilterConfigs(map[string]*anypb.Any{"foo": errFilterConfig}), + wantError: true, + }, + { + name: "good-route-config-with-http-optional-err-filter-config", + rc: goodRouteConfigWithFilterConfigs(map[string]*anypb.Any{"foo": wrappedOptionalFilter("err.custom.filter")}), + wantError: true, + }, + { + name: "good-route-config-with-http-unknown-filter-config", + rc: goodRouteConfigWithFilterConfigs(map[string]*anypb.Any{"foo": unknownFilterConfig}), + wantError: true, + }, + { + name: "good-route-config-with-http-optional-unknown-filter-config", + rc: goodRouteConfigWithFilterConfigs(map[string]*anypb.Any{"foo": wrappedOptionalFilter("unknown.custom.filter")}), + wantUpdate: goodUpdateWithFilterConfigs(nil), + }, + { + name: "good-route-config-with-bad-rbac-http-filter-configuration", + rc: goodRouteConfigWithFilterConfigs(map[string]*anypb.Any{"rbac": testutils.MarshalAny(&v3rbacpb.RBACPerRoute{Rbac: &v3rbacpb.RBAC{ + Rules: &rpb.RBAC{ + Action: rpb.RBAC_ALLOW, + Policies: map[string]*rpb.Policy{ + "certain-destination-ip": { + Permissions: []*rpb.Permission{ + {Rule: &rpb.Permission_DestinationIp{DestinationIp: &v3corepb.CidrRange{AddressPrefix: "not a correct address", PrefixLen: &wrapperspb.UInt32Value{Value: uint32(10)}}}}, + }, + Principals: []*rpb.Principal{ + {Identifier: &rpb.Principal_Any{Any: true}}, + }, + }, + }, + }, + }})}), + wantError: true, + }, + { + name: "good-route-config-with-retry-policy", + rc: goodRouteConfigWithRetryPolicy( + &v3routepb.RetryPolicy{RetryOn: "cancelled"}, + &v3routepb.RetryPolicy{RetryOn: "deadline-exceeded,unsupported", NumRetries: &wrapperspb.UInt32Value{Value: 2}}), + wantUpdate: goodUpdateWithRetryPolicy( + &RetryConfig{RetryOn: map[codes.Code]bool{codes.Canceled: true}, NumRetries: 1, RetryBackoff: defaultRetryBackoff}, + &RetryConfig{RetryOn: map[codes.Code]bool{codes.DeadlineExceeded: true}, NumRetries: 2, RetryBackoff: defaultRetryBackoff}), + }, + { + name: "good-route-config-with-retry-backoff", + rc: goodRouteConfigWithRetryPolicy( + &v3routepb.RetryPolicy{RetryOn: "internal", RetryBackOff: &v3routepb.RetryPolicy_RetryBackOff{BaseInterval: durationpb.New(10 * time.Millisecond), MaxInterval: durationpb.New(10 * time.Millisecond)}}, + &v3routepb.RetryPolicy{RetryOn: "resource-exhausted", RetryBackOff: &v3routepb.RetryPolicy_RetryBackOff{BaseInterval: durationpb.New(10 * time.Millisecond)}}), + wantUpdate: goodUpdateWithRetryPolicy( + &RetryConfig{RetryOn: map[codes.Code]bool{codes.Internal: true}, NumRetries: 1, RetryBackoff: RetryBackoff{BaseInterval: 10 * time.Millisecond, MaxInterval: 10 * time.Millisecond}}, + &RetryConfig{RetryOn: map[codes.Code]bool{codes.ResourceExhausted: true}, NumRetries: 1, RetryBackoff: RetryBackoff{BaseInterval: 10 * time.Millisecond, MaxInterval: 100 * time.Millisecond}}), + }, + { + name: "bad-retry-policy-0-retries", + rc: goodRouteConfigWithRetryPolicy(&v3routepb.RetryPolicy{RetryOn: "cancelled", NumRetries: &wrapperspb.UInt32Value{Value: 0}}, nil), + wantUpdate: RouteConfigUpdate{}, + wantError: true, + }, + { + name: "bad-retry-policy-0-base-interval", + rc: goodRouteConfigWithRetryPolicy(&v3routepb.RetryPolicy{RetryOn: "cancelled", RetryBackOff: &v3routepb.RetryPolicy_RetryBackOff{BaseInterval: durationpb.New(0)}}, nil), + wantUpdate: RouteConfigUpdate{}, + wantError: true, + }, + { + name: "bad-retry-policy-negative-max-interval", + rc: goodRouteConfigWithRetryPolicy(&v3routepb.RetryPolicy{RetryOn: "cancelled", RetryBackOff: &v3routepb.RetryPolicy_RetryBackOff{MaxInterval: durationpb.New(-time.Second)}}, nil), + wantUpdate: RouteConfigUpdate{}, + wantError: true, + }, + { + name: "bad-retry-policy-negative-max-interval-no-known-retry-on", + rc: goodRouteConfigWithRetryPolicy(&v3routepb.RetryPolicy{RetryOn: "something", RetryBackOff: &v3routepb.RetryPolicy_RetryBackOff{MaxInterval: durationpb.New(-time.Second)}}, nil), + wantUpdate: RouteConfigUpdate{}, + wantError: true, + }, + { + name: "cluster-specifier-declared-which-not-registered", + rc: goodRouteConfigWithClusterSpecifierPlugins([]*v3routepb.ClusterSpecifierPlugin{ + clusterSpecifierPlugin("cspA", configOfClusterSpecifierDoesntExist, false), + }, []string{"cspA"}), + wantError: true, + rlsEnabled: true, + }, + { + name: "error-in-cluster-specifier-plugin-conversion-method", + rc: goodRouteConfigWithClusterSpecifierPlugins([]*v3routepb.ClusterSpecifierPlugin{ + clusterSpecifierPlugin("cspA", errorClusterSpecifierConfig, false), + }, []string{"cspA"}), + wantError: true, + rlsEnabled: true, + }, + { + name: "route-action-that-references-undeclared-cluster-specifier-plugin", + rc: goodRouteConfigWithClusterSpecifierPlugins([]*v3routepb.ClusterSpecifierPlugin{ + clusterSpecifierPlugin("cspA", mockClusterSpecifierConfig, false), + }, []string{"cspA", "cspB"}), + wantError: true, + rlsEnabled: true, + }, + { + name: "emitted-cluster-specifier-plugins", + rc: goodRouteConfigWithClusterSpecifierPlugins([]*v3routepb.ClusterSpecifierPlugin{ + clusterSpecifierPlugin("cspA", mockClusterSpecifierConfig, false), + }, []string{"cspA"}), + wantUpdate: goodUpdateWithClusterSpecifierPluginA, + rlsEnabled: true, + }, + { + name: "deleted-cluster-specifier-plugins-not-referenced", + rc: goodRouteConfigWithClusterSpecifierPlugins([]*v3routepb.ClusterSpecifierPlugin{ + clusterSpecifierPlugin("cspA", mockClusterSpecifierConfig, false), + clusterSpecifierPlugin("cspB", mockClusterSpecifierConfig, false), + }, []string{"cspA"}), + wantUpdate: goodUpdateWithClusterSpecifierPluginA, + rlsEnabled: true, + }, + { + name: "ignore-error-in-cluster-specifier-plugin-env-var-off", + rc: goodRouteConfigWithClusterSpecifierPlugins([]*v3routepb.ClusterSpecifierPlugin{ + clusterSpecifierPlugin("cspA", configOfClusterSpecifierDoesntExist, false), + }, []string{}), + wantUpdate: goodUpdate, + }, + { + name: "cluster-specifier-plugin-referenced-env-var-off", + rc: goodRouteConfigWithClusterSpecifierPlugins([]*v3routepb.ClusterSpecifierPlugin{ + clusterSpecifierPlugin("cspA", mockClusterSpecifierConfig, false), + }, []string{"cspA"}), + wantUpdate: goodUpdate, + }, + // This tests a scenario where a cluster specifier plugin is not found + // and is optional. Any routes referencing that not found optional + // cluster specifier plugin should be ignored. The config has two + // routes, and only one of them should be present in the update. + { + name: "cluster-specifier-plugin-not-found-and-optional-route-should-ignore", + rc: goodRouteConfigWithClusterSpecifierPluginsAndNormalRoute([]*v3routepb.ClusterSpecifierPlugin{ + clusterSpecifierPlugin("cspA", configOfClusterSpecifierDoesntExist, true), + }, []string{"cspA"}), + wantUpdate: goodUpdateWithNormalRoute, + rlsEnabled: true, + }, + // This tests a scenario where a route has an unsupported cluster + // specifier. Any routes with an unsupported cluster specifier should be + // ignored. The config has two routes, and only one of them should be + // present in the update. + { + name: "unsupported-cluster-specifier-route-should-ignore", + rc: goodRouteConfigWithUnsupportedClusterSpecifier, + wantUpdate: goodUpdateWithNormalRoute, + rlsEnabled: true, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + envconfig.XDSRLS = test.rlsEnabled + gotUpdate, gotError := generateRDSUpdateFromRouteConfiguration(test.rc) + if (gotError != nil) != test.wantError || + !cmp.Equal(gotUpdate, test.wantUpdate, cmpopts.EquateEmpty(), + cmp.Transformer("FilterConfig", func(fc httpfilter.FilterConfig) string { + return fmt.Sprint(fc) + })) { + t.Errorf("generateRDSUpdateFromRouteConfiguration(%+v, %v) returned unexpected, diff (-want +got):\\n%s", test.rc, ldsTarget, cmp.Diff(test.wantUpdate, gotUpdate, cmpopts.EquateEmpty())) + } + }) + } +} + +var configOfClusterSpecifierDoesntExist = &anypb.Any{ + TypeUrl: "does.not.exist", + Value: []byte{1, 2, 3}, +} + +var mockClusterSpecifierConfig = &anypb.Any{ + TypeUrl: "mock.cluster.specifier.plugin", + Value: []byte{1, 2, 3}, +} + +var errorClusterSpecifierConfig = &anypb.Any{ + TypeUrl: "error.cluster.specifier.plugin", + Value: []byte{1, 2, 3}, +} + +func init() { + clusterspecifier.Register(mockClusterSpecifierPlugin{}) + clusterspecifier.Register(errorClusterSpecifierPlugin{}) +} + +type mockClusterSpecifierPlugin struct { +} + +func (mockClusterSpecifierPlugin) TypeURLs() []string { + return []string{"mock.cluster.specifier.plugin"} +} + +func (mockClusterSpecifierPlugin) ParseClusterSpecifierConfig(proto.Message) (clusterspecifier.BalancerConfig, error) { + return []map[string]any{}, nil +} + +type errorClusterSpecifierPlugin struct{} + +func (errorClusterSpecifierPlugin) TypeURLs() []string { + return []string{"error.cluster.specifier.plugin"} +} + +func (errorClusterSpecifierPlugin) ParseClusterSpecifierConfig(proto.Message) (clusterspecifier.BalancerConfig, error) { + return nil, errors.New("error from cluster specifier conversion function") +} + +func (s) TestUnmarshalRouteConfig(t *testing.T) { + const ( + ldsTarget = "lds.target.good:1111" + uninterestingDomain = "uninteresting.domain" + uninterestingClusterName = "uninterestingClusterName" + v3RouteConfigName = "v3RouteConfig" + v3ClusterName = "v3Cluster" + ) + + var ( + v3VirtualHost = []*v3routepb.VirtualHost{ + { + Domains: []string{uninterestingDomain}, + Routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: ""}}, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: uninterestingClusterName}, + }, + }, + }, + }, + }, + { + Domains: []string{ldsTarget}, + Routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: ""}}, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: v3ClusterName}, + }, + }, + }, + }, + }, + } + v3RouteConfig = testutils.MarshalAny(&v3routepb.RouteConfiguration{ + Name: v3RouteConfigName, + VirtualHosts: v3VirtualHost, + }) + ) + + tests := []struct { + name string + resource *anypb.Any + wantName string + wantUpdate RouteConfigUpdate + wantErr bool + }{ + { + name: "non-routeConfig resource type", + resource: &anypb.Any{TypeUrl: version.V3HTTPConnManagerURL}, + wantErr: true, + }, + { + name: "badly marshaled routeconfig resource", + resource: &anypb.Any{ + TypeUrl: version.V3RouteConfigURL, + Value: []byte{1, 2, 3, 4}, + }, + wantErr: true, + }, + { + name: "v3 routeConfig resource", + resource: v3RouteConfig, + wantName: v3RouteConfigName, + wantUpdate: RouteConfigUpdate{ + VirtualHosts: []*VirtualHost{ + { + Domains: []string{uninterestingDomain}, + Routes: []*Route{{Prefix: newStringP(""), + WeightedClusters: map[string]WeightedCluster{uninterestingClusterName: {Weight: 1}}, + ActionType: RouteActionRoute}}, + }, + { + Domains: []string{ldsTarget}, + Routes: []*Route{{Prefix: newStringP(""), + WeightedClusters: map[string]WeightedCluster{v3ClusterName: {Weight: 1}}, + ActionType: RouteActionRoute}}, + }, + }, + Raw: v3RouteConfig, + }, + }, + { + name: "v3 routeConfig resource wrapped", + resource: testutils.MarshalAny(&v3discoverypb.Resource{Resource: v3RouteConfig}), + wantName: v3RouteConfigName, + wantUpdate: RouteConfigUpdate{ + VirtualHosts: []*VirtualHost{ + { + Domains: []string{uninterestingDomain}, + Routes: []*Route{{Prefix: newStringP(""), + WeightedClusters: map[string]WeightedCluster{uninterestingClusterName: {Weight: 1}}, + ActionType: RouteActionRoute}}, + }, + { + Domains: []string{ldsTarget}, + Routes: []*Route{{Prefix: newStringP(""), + WeightedClusters: map[string]WeightedCluster{v3ClusterName: {Weight: 1}}, + ActionType: RouteActionRoute}}, + }, + }, + Raw: v3RouteConfig, + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + name, update, err := unmarshalRouteConfigResource(test.resource) + if (err != nil) != test.wantErr { + t.Errorf("unmarshalRouteConfigResource(%s), got err: %v, wantErr: %v", pretty.ToJSON(test.resource), err, test.wantErr) + } + if name != test.wantName { + t.Errorf("unmarshalRouteConfigResource(%s), got name: %s, want: %s", pretty.ToJSON(test.resource), name, test.wantName) + } + if diff := cmp.Diff(update, test.wantUpdate, cmpOpts); diff != "" { + t.Errorf("unmarshalRouteConfigResource(%s), got unexpected update, diff (-got +want): %v", pretty.ToJSON(test.resource), diff) + } + }) + } +} + +func (s) TestRoutesProtoToSlice(t *testing.T) { + sm, _ := matcher.StringMatcherFromProto(&v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "tv"}}) + var ( + goodRouteWithFilterConfigs = func(cfgs map[string]*anypb.Any) []*v3routepb.Route { + // Sets per-filter config in cluster "B" and in the route. + return []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}, + CaseSensitive: &wrapperspb.BoolValue{Value: false}, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{ + WeightedClusters: &v3routepb.WeightedCluster{ + Clusters: []*v3routepb.WeightedCluster_ClusterWeight{ + {Name: "B", Weight: &wrapperspb.UInt32Value{Value: 60}, TypedPerFilterConfig: cfgs}, + {Name: "A", Weight: &wrapperspb.UInt32Value{Value: 40}}, + }, + }}}}, + TypedPerFilterConfig: cfgs, + }} + } + goodUpdateWithFilterConfigs = func(cfgs map[string]httpfilter.FilterConfig) []*Route { + // Sets per-filter config in cluster "B" and in the route. + return []*Route{{ + Prefix: newStringP("/"), + CaseInsensitive: true, + WeightedClusters: map[string]WeightedCluster{"A": {Weight: 40}, "B": {Weight: 60, HTTPFilterConfigOverride: cfgs}}, + HTTPFilterConfigOverride: cfgs, + ActionType: RouteActionRoute, + }} + } + ) + + tests := []struct { + name string + routes []*v3routepb.Route + wantRoutes []*Route + wantErr bool + }{ + { + name: "no path", + routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{}, + }}, + wantErr: true, + }, + { + name: "case_sensitive is false", + routes: []*v3routepb.Route{{ + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/"}, + CaseSensitive: &wrapperspb.BoolValue{Value: false}, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{ + WeightedClusters: &v3routepb.WeightedCluster{ + Clusters: []*v3routepb.WeightedCluster_ClusterWeight{ + {Name: "B", Weight: &wrapperspb.UInt32Value{Value: 60}}, + {Name: "A", Weight: &wrapperspb.UInt32Value{Value: 40}}, + }, + }}}}, + }}, + wantRoutes: []*Route{{ + Prefix: newStringP("/"), + CaseInsensitive: true, + WeightedClusters: map[string]WeightedCluster{"A": {Weight: 40}, "B": {Weight: 60}}, + ActionType: RouteActionRoute, + }}, + }, + { + name: "good", + routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/a/"}, + Headers: []*v3routepb.HeaderMatcher{ + { + Name: "th", + HeaderMatchSpecifier: &v3routepb.HeaderMatcher_PrefixMatch{ + PrefixMatch: "tv", + }, + InvertMatch: true, + }, + }, + RuntimeFraction: &v3corepb.RuntimeFractionalPercent{ + DefaultValue: &v3typepb.FractionalPercent{ + Numerator: 1, + Denominator: v3typepb.FractionalPercent_HUNDRED, + }, + }, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{ + WeightedClusters: &v3routepb.WeightedCluster{ + Clusters: []*v3routepb.WeightedCluster_ClusterWeight{ + {Name: "B", Weight: &wrapperspb.UInt32Value{Value: 60}}, + {Name: "A", Weight: &wrapperspb.UInt32Value{Value: 40}}, + }, + }}}}, + }, + }, + wantRoutes: []*Route{{ + Prefix: newStringP("/a/"), + Headers: []*HeaderMatcher{ + { + Name: "th", + InvertMatch: newBoolP(true), + PrefixMatch: newStringP("tv"), + }, + }, + Fraction: newUInt32P(10000), + WeightedClusters: map[string]WeightedCluster{"A": {Weight: 40}, "B": {Weight: 60}}, + ActionType: RouteActionRoute, + }}, + wantErr: false, + }, + { + name: "good with regex matchers", + routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_SafeRegex{SafeRegex: &v3matcherpb.RegexMatcher{Regex: "/a/"}}, + Headers: []*v3routepb.HeaderMatcher{ + { + Name: "th", + HeaderMatchSpecifier: &v3routepb.HeaderMatcher_SafeRegexMatch{SafeRegexMatch: &v3matcherpb.RegexMatcher{Regex: "tv"}}, + }, + }, + RuntimeFraction: &v3corepb.RuntimeFractionalPercent{ + DefaultValue: &v3typepb.FractionalPercent{ + Numerator: 1, + Denominator: v3typepb.FractionalPercent_HUNDRED, + }, + }, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{ + WeightedClusters: &v3routepb.WeightedCluster{ + Clusters: []*v3routepb.WeightedCluster_ClusterWeight{ + {Name: "B", Weight: &wrapperspb.UInt32Value{Value: 60}}, + {Name: "A", Weight: &wrapperspb.UInt32Value{Value: 40}}, + }, + }}}}, + }, + }, + wantRoutes: []*Route{{ + Regex: func() *regexp.Regexp { return regexp.MustCompile("/a/") }(), + Headers: []*HeaderMatcher{ + { + Name: "th", + InvertMatch: newBoolP(false), + RegexMatch: func() *regexp.Regexp { return regexp.MustCompile("tv") }(), + }, + }, + Fraction: newUInt32P(10000), + WeightedClusters: map[string]WeightedCluster{"A": {Weight: 40}, "B": {Weight: 60}}, + ActionType: RouteActionRoute, + }}, + wantErr: false, + }, + { + name: "good with string matcher", + routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_SafeRegex{SafeRegex: &v3matcherpb.RegexMatcher{Regex: "/a/"}}, + Headers: []*v3routepb.HeaderMatcher{ + { + Name: "th", + HeaderMatchSpecifier: &v3routepb.HeaderMatcher_StringMatch{StringMatch: &v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "tv"}}}, + }, + }, + RuntimeFraction: &v3corepb.RuntimeFractionalPercent{ + DefaultValue: &v3typepb.FractionalPercent{ + Numerator: 1, + Denominator: v3typepb.FractionalPercent_HUNDRED, + }, + }, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{ + WeightedClusters: &v3routepb.WeightedCluster{ + Clusters: []*v3routepb.WeightedCluster_ClusterWeight{ + {Name: "B", Weight: &wrapperspb.UInt32Value{Value: 60}}, + {Name: "A", Weight: &wrapperspb.UInt32Value{Value: 40}}, + }, + }}}}, + }, + }, + wantRoutes: []*Route{{ + Regex: func() *regexp.Regexp { return regexp.MustCompile("/a/") }(), + Headers: []*HeaderMatcher{ + { + Name: "th", + InvertMatch: newBoolP(false), + StringMatch: &sm, + }, + }, + Fraction: newUInt32P(10000), + WeightedClusters: map[string]WeightedCluster{"A": {Weight: 40}, "B": {Weight: 60}}, + ActionType: RouteActionRoute, + }}, + wantErr: false, + }, + { + name: "query is ignored", + routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/a/"}, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{ + WeightedClusters: &v3routepb.WeightedCluster{ + Clusters: []*v3routepb.WeightedCluster_ClusterWeight{ + {Name: "B", Weight: &wrapperspb.UInt32Value{Value: 60}}, + {Name: "A", Weight: &wrapperspb.UInt32Value{Value: 40}}, + }, + }}}}, + }, + { + Name: "with_query", + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/b/"}, + QueryParameters: []*v3routepb.QueryParameterMatcher{{Name: "route_will_be_ignored"}}, + }, + }, + }, + // Only one route in the result, because the second one with query + // parameters is ignored. + wantRoutes: []*Route{{ + Prefix: newStringP("/a/"), + WeightedClusters: map[string]WeightedCluster{"A": {Weight: 40}, "B": {Weight: 60}}, + ActionType: RouteActionRoute, + }}, + wantErr: false, + }, + { + name: "unrecognized path specifier", + routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_ConnectMatcher_{}, + }, + }, + }, + wantErr: true, + }, + { + name: "bad regex in path specifier", + routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_SafeRegex{SafeRegex: &v3matcherpb.RegexMatcher{Regex: "??"}}, + Headers: []*v3routepb.HeaderMatcher{ + { + HeaderMatchSpecifier: &v3routepb.HeaderMatcher_PrefixMatch{PrefixMatch: "tv"}, + }, + }, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName}}, + }, + }, + }, + wantErr: true, + }, + { + name: "bad regex in header specifier", + routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/a/"}, + Headers: []*v3routepb.HeaderMatcher{ + { + HeaderMatchSpecifier: &v3routepb.HeaderMatcher_SafeRegexMatch{SafeRegexMatch: &v3matcherpb.RegexMatcher{Regex: "??"}}, + }, + }, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName}}, + }, + }, + }, + wantErr: true, + }, + { + name: "unrecognized header match specifier", + routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/a/"}, + Headers: []*v3routepb.HeaderMatcher{ + { + Name: "th", + HeaderMatchSpecifier: &v3routepb.HeaderMatcher_StringMatch{}, + }, + }, + }, + }, + }, + wantErr: true, + }, + { + name: "no cluster in weighted clusters action", + routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/a/"}, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{ + WeightedClusters: &v3routepb.WeightedCluster{}}}}, + }, + }, + wantErr: true, + }, + { + name: "all 0-weight clusters in weighted clusters action", + routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/a/"}, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{ + WeightedClusters: &v3routepb.WeightedCluster{ + Clusters: []*v3routepb.WeightedCluster_ClusterWeight{ + {Name: "B", Weight: &wrapperspb.UInt32Value{Value: 0}}, + {Name: "A", Weight: &wrapperspb.UInt32Value{Value: 0}}, + }, + }}}}, + }, + }, + wantErr: true, + }, + { + name: "The sum of all weighted clusters is more than uint32", + routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/a/"}, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{ + WeightedClusters: &v3routepb.WeightedCluster{ + Clusters: []*v3routepb.WeightedCluster_ClusterWeight{ + {Name: "B", Weight: &wrapperspb.UInt32Value{Value: math.MaxUint32}}, + {Name: "A", Weight: &wrapperspb.UInt32Value{Value: math.MaxUint32}}, + }, + }}}}, + }, + }, + wantErr: true, + }, + { + name: "unsupported cluster specifier", + routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/a/"}, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_ClusterSpecifierPlugin{}}}, + }, + }, + wantErr: true, + }, + { + name: "default totalWeight is 100 in weighted clusters action", + routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/a/"}, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{ + WeightedClusters: &v3routepb.WeightedCluster{ + Clusters: []*v3routepb.WeightedCluster_ClusterWeight{ + {Name: "B", Weight: &wrapperspb.UInt32Value{Value: 60}}, + {Name: "A", Weight: &wrapperspb.UInt32Value{Value: 40}}, + }, + }}}}, + }, + }, + wantRoutes: []*Route{{ + Prefix: newStringP("/a/"), + WeightedClusters: map[string]WeightedCluster{"A": {Weight: 40}, "B": {Weight: 60}}, + ActionType: RouteActionRoute, + }}, + wantErr: false, + }, + { + name: "default totalWeight is 100 in weighted clusters action", + routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/a/"}, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{ + WeightedClusters: &v3routepb.WeightedCluster{ + Clusters: []*v3routepb.WeightedCluster_ClusterWeight{ + {Name: "B", Weight: &wrapperspb.UInt32Value{Value: 30}}, + {Name: "A", Weight: &wrapperspb.UInt32Value{Value: 20}}, + }, + }}}}, + }, + }, + wantRoutes: []*Route{{ + Prefix: newStringP("/a/"), + WeightedClusters: map[string]WeightedCluster{"A": {Weight: 20}, "B": {Weight: 30}}, + ActionType: RouteActionRoute, + }}, + wantErr: false, + }, + { + name: "good-with-channel-id-hash-policy", + routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/a/"}, + Headers: []*v3routepb.HeaderMatcher{ + { + Name: "th", + HeaderMatchSpecifier: &v3routepb.HeaderMatcher_PrefixMatch{ + PrefixMatch: "tv", + }, + InvertMatch: true, + }, + }, + RuntimeFraction: &v3corepb.RuntimeFractionalPercent{ + DefaultValue: &v3typepb.FractionalPercent{ + Numerator: 1, + Denominator: v3typepb.FractionalPercent_HUNDRED, + }, + }, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{ + WeightedClusters: &v3routepb.WeightedCluster{ + Clusters: []*v3routepb.WeightedCluster_ClusterWeight{ + {Name: "B", Weight: &wrapperspb.UInt32Value{Value: 60}}, + {Name: "A", Weight: &wrapperspb.UInt32Value{Value: 40}}, + }, + }}, + HashPolicy: []*v3routepb.RouteAction_HashPolicy{ + {PolicySpecifier: &v3routepb.RouteAction_HashPolicy_FilterState_{FilterState: &v3routepb.RouteAction_HashPolicy_FilterState{Key: "io.grpc.channel_id"}}}, + }, + }}, + }, + }, + wantRoutes: []*Route{{ + Prefix: newStringP("/a/"), + Headers: []*HeaderMatcher{ + { + Name: "th", + InvertMatch: newBoolP(true), + PrefixMatch: newStringP("tv"), + }, + }, + Fraction: newUInt32P(10000), + WeightedClusters: map[string]WeightedCluster{"A": {Weight: 40}, "B": {Weight: 60}}, + HashPolicies: []*HashPolicy{ + {HashPolicyType: HashPolicyTypeChannelID}, + }, + ActionType: RouteActionRoute, + }}, + wantErr: false, + }, + // This tests that policy.Regex ends up being nil if RegexRewrite is not + // set in xds response. + { + name: "good-with-header-hash-policy-no-regex-specified", + routes: []*v3routepb.Route{ + { + Match: &v3routepb.RouteMatch{ + PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/a/"}, + Headers: []*v3routepb.HeaderMatcher{ + { + Name: "th", + HeaderMatchSpecifier: &v3routepb.HeaderMatcher_PrefixMatch{ + PrefixMatch: "tv", + }, + InvertMatch: true, + }, + }, + RuntimeFraction: &v3corepb.RuntimeFractionalPercent{ + DefaultValue: &v3typepb.FractionalPercent{ + Numerator: 1, + Denominator: v3typepb.FractionalPercent_HUNDRED, + }, + }, + }, + Action: &v3routepb.Route_Route{ + Route: &v3routepb.RouteAction{ + ClusterSpecifier: &v3routepb.RouteAction_WeightedClusters{ + WeightedClusters: &v3routepb.WeightedCluster{ + Clusters: []*v3routepb.WeightedCluster_ClusterWeight{ + {Name: "B", Weight: &wrapperspb.UInt32Value{Value: 60}}, + {Name: "A", Weight: &wrapperspb.UInt32Value{Value: 40}}, + }, + }}, + HashPolicy: []*v3routepb.RouteAction_HashPolicy{ + {PolicySpecifier: &v3routepb.RouteAction_HashPolicy_Header_{Header: &v3routepb.RouteAction_HashPolicy_Header{HeaderName: ":path"}}}, + }, + }}, + }, + }, + wantRoutes: []*Route{{ + Prefix: newStringP("/a/"), + Headers: []*HeaderMatcher{ + { + Name: "th", + InvertMatch: newBoolP(true), + PrefixMatch: newStringP("tv"), + }, + }, + Fraction: newUInt32P(10000), + WeightedClusters: map[string]WeightedCluster{"A": {Weight: 40}, "B": {Weight: 60}}, + HashPolicies: []*HashPolicy{ + {HashPolicyType: HashPolicyTypeHeader, + HeaderName: ":path"}, + }, + ActionType: RouteActionRoute, + }}, + wantErr: false, + }, + { + name: "with custom HTTP filter config", + routes: goodRouteWithFilterConfigs(map[string]*anypb.Any{"foo": customFilterConfig}), + wantRoutes: goodUpdateWithFilterConfigs(map[string]httpfilter.FilterConfig{"foo": filterConfig{Override: customFilterConfig}}), + }, + { + name: "with custom HTTP filter config in typed struct", + routes: goodRouteWithFilterConfigs(map[string]*anypb.Any{"foo": wrappedCustomFilterOldTypedStructConfig}), + wantRoutes: goodUpdateWithFilterConfigs(map[string]httpfilter.FilterConfig{"foo": filterConfig{Override: customFilterOldTypedStructConfig}}), + }, + { + name: "with optional custom HTTP filter config", + routes: goodRouteWithFilterConfigs(map[string]*anypb.Any{"foo": wrappedOptionalFilter("custom.filter")}), + wantRoutes: goodUpdateWithFilterConfigs(map[string]httpfilter.FilterConfig{"foo": filterConfig{Override: customFilterConfig}}), + }, + { + name: "with erroring custom HTTP filter config", + routes: goodRouteWithFilterConfigs(map[string]*anypb.Any{"foo": errFilterConfig}), + wantErr: true, + }, + { + name: "with optional erroring custom HTTP filter config", + routes: goodRouteWithFilterConfigs(map[string]*anypb.Any{"foo": wrappedOptionalFilter("err.custom.filter")}), + wantErr: true, + }, + { + name: "with unknown custom HTTP filter config", + routes: goodRouteWithFilterConfigs(map[string]*anypb.Any{"foo": unknownFilterConfig}), + wantErr: true, + }, + { + name: "with optional unknown custom HTTP filter config", + routes: goodRouteWithFilterConfigs(map[string]*anypb.Any{"foo": wrappedOptionalFilter("unknown.custom.filter")}), + wantRoutes: goodUpdateWithFilterConfigs(nil), + }, + } + + cmpOpts := []cmp.Option{ + cmp.AllowUnexported(Route{}, HeaderMatcher{}, Int64Range{}, regexp.Regexp{}), + cmpopts.EquateEmpty(), + cmp.Transformer("FilterConfig", func(fc httpfilter.FilterConfig) string { + return fmt.Sprint(fc) + }), + } + oldRingHashSupport := envconfig.XDSRingHash + envconfig.XDSRingHash = true + defer func() { envconfig.XDSRingHash = oldRingHashSupport }() + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, _, err := routesProtoToSlice(tt.routes, nil) + if (err != nil) != tt.wantErr { + t.Fatalf("routesProtoToSlice() error = %v, wantErr %v", err, tt.wantErr) + } + if diff := cmp.Diff(got, tt.wantRoutes, cmpOpts...); diff != "" { + t.Fatalf("routesProtoToSlice() returned unexpected diff (-got +want):\n%s", diff) + } + }) + } +} + +func (s) TestHashPoliciesProtoToSlice(t *testing.T) { + tests := []struct { + name string + hashPolicies []*v3routepb.RouteAction_HashPolicy + wantHashPolicies []*HashPolicy + wantErr bool + }{ + // header-hash-policy tests a basic hash policy that specifies to hash a + // certain header. + { + name: "header-hash-policy", + hashPolicies: []*v3routepb.RouteAction_HashPolicy{ + { + PolicySpecifier: &v3routepb.RouteAction_HashPolicy_Header_{ + Header: &v3routepb.RouteAction_HashPolicy_Header{ + HeaderName: ":path", + RegexRewrite: &v3matcherpb.RegexMatchAndSubstitute{ + Pattern: &v3matcherpb.RegexMatcher{Regex: "/products"}, + Substitution: "/products", + }, + }, + }, + }, + }, + wantHashPolicies: []*HashPolicy{ + { + HashPolicyType: HashPolicyTypeHeader, + HeaderName: ":path", + Regex: func() *regexp.Regexp { return regexp.MustCompile("/products") }(), + RegexSubstitution: "/products", + }, + }, + }, + // channel-id-hash-policy tests a basic hash policy that specifies to + // hash a unique identifier of the channel. + { + name: "channel-id-hash-policy", + hashPolicies: []*v3routepb.RouteAction_HashPolicy{ + {PolicySpecifier: &v3routepb.RouteAction_HashPolicy_FilterState_{FilterState: &v3routepb.RouteAction_HashPolicy_FilterState{Key: "io.grpc.channel_id"}}}, + }, + wantHashPolicies: []*HashPolicy{ + {HashPolicyType: HashPolicyTypeChannelID}, + }, + }, + // unsupported-filter-state-key tests that an unsupported key in the + // filter state hash policy are treated as a no-op. + { + name: "wrong-filter-state-key", + hashPolicies: []*v3routepb.RouteAction_HashPolicy{ + {PolicySpecifier: &v3routepb.RouteAction_HashPolicy_FilterState_{FilterState: &v3routepb.RouteAction_HashPolicy_FilterState{Key: "unsupported key"}}}, + }, + }, + // no-op-hash-policy tests that hash policies that are not supported by + // grpc are treated as a no-op. + { + name: "no-op-hash-policy", + hashPolicies: []*v3routepb.RouteAction_HashPolicy{ + {PolicySpecifier: &v3routepb.RouteAction_HashPolicy_FilterState_{}}, + }, + }, + // header-and-channel-id-hash-policy test that a list of header and + // channel id hash policies are successfully converted to an internal + // struct. + { + name: "header-and-channel-id-hash-policy", + hashPolicies: []*v3routepb.RouteAction_HashPolicy{ + { + PolicySpecifier: &v3routepb.RouteAction_HashPolicy_Header_{ + Header: &v3routepb.RouteAction_HashPolicy_Header{ + HeaderName: ":path", + RegexRewrite: &v3matcherpb.RegexMatchAndSubstitute{ + Pattern: &v3matcherpb.RegexMatcher{Regex: "/products"}, + Substitution: "/products", + }, + }, + }, + }, + { + PolicySpecifier: &v3routepb.RouteAction_HashPolicy_FilterState_{FilterState: &v3routepb.RouteAction_HashPolicy_FilterState{Key: "io.grpc.channel_id"}}, + Terminal: true, + }, + }, + wantHashPolicies: []*HashPolicy{ + { + HashPolicyType: HashPolicyTypeHeader, + HeaderName: ":path", + Regex: func() *regexp.Regexp { return regexp.MustCompile("/products") }(), + RegexSubstitution: "/products", + }, + { + HashPolicyType: HashPolicyTypeChannelID, + Terminal: true, + }, + }, + }, + } + + oldRingHashSupport := envconfig.XDSRingHash + envconfig.XDSRingHash = true + defer func() { envconfig.XDSRingHash = oldRingHashSupport }() + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := hashPoliciesProtoToSlice(tt.hashPolicies) + if (err != nil) != tt.wantErr { + t.Fatalf("hashPoliciesProtoToSlice() error = %v, wantErr %v", err, tt.wantErr) + } + if diff := cmp.Diff(got, tt.wantHashPolicies, cmp.AllowUnexported(regexp.Regexp{})); diff != "" { + t.Fatalf("hashPoliciesProtoToSlice() returned unexpected diff (-got +want):\n%s", diff) + } + }) + } +} + +func newStringP(s string) *string { + return &s +} + +func newUInt32P(i uint32) *uint32 { + return &i +} + +func newBoolP(b bool) *bool { + return &b +} + +func newDurationP(d time.Duration) *time.Duration { + return &d +} diff --git a/xds/internal/xdsclient/xdsresource/version/version.go b/xds/internal/xdsclient/xdsresource/version/version.go new file mode 100644 index 000000000000..82ad5fe52c70 --- /dev/null +++ b/xds/internal/xdsclient/xdsresource/version/version.go @@ -0,0 +1,41 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package version defines constants to distinguish between supported xDS API +// versions. +package version + +// Resource URLs. We need to be able to accept either version of the resource +// regardless of the version of the transport protocol in use. +const ( + googleapiPrefix = "type.googleapis.com/" + + V3ListenerType = "envoy.config.listener.v3.Listener" + V3RouteConfigType = "envoy.config.route.v3.RouteConfiguration" + V3ClusterType = "envoy.config.cluster.v3.Cluster" + V3EndpointsType = "envoy.config.endpoint.v3.ClusterLoadAssignment" + + V3ResourceWrapperURL = googleapiPrefix + "envoy.service.discovery.v3.Resource" + V3ListenerURL = googleapiPrefix + V3ListenerType + V3RouteConfigURL = googleapiPrefix + V3RouteConfigType + V3ClusterURL = googleapiPrefix + V3ClusterType + V3EndpointsURL = googleapiPrefix + V3EndpointsType + V3HTTPConnManagerURL = googleapiPrefix + "envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager" + V3UpstreamTLSContextURL = googleapiPrefix + "envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext" + V3DownstreamTLSContextURL = googleapiPrefix + "envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext" +) diff --git a/xds/server.go b/xds/server.go new file mode 100644 index 000000000000..fe2138c8bc24 --- /dev/null +++ b/xds/server.go @@ -0,0 +1,415 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds + +import ( + "context" + "errors" + "fmt" + "net" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/buffer" + "google.golang.org/grpc/internal/envconfig" + internalgrpclog "google.golang.org/grpc/internal/grpclog" + "google.golang.org/grpc/internal/grpcsync" + iresolver "google.golang.org/grpc/internal/resolver" + "google.golang.org/grpc/internal/transport" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/grpc/xds/internal/server" + "google.golang.org/grpc/xds/internal/xdsclient" + "google.golang.org/grpc/xds/internal/xdsclient/bootstrap" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource" +) + +const serverPrefix = "[xds-server %p] " + +var ( + // These new functions will be overridden in unit tests. + newXDSClient = func() (xdsclient.XDSClient, func(), error) { + return xdsclient.New() + } + newGRPCServer = func(opts ...grpc.ServerOption) grpcServer { + return grpc.NewServer(opts...) + } + + grpcGetServerCreds = internal.GetServerCredentials.(func(*grpc.Server) credentials.TransportCredentials) + drainServerTransports = internal.DrainServerTransports.(func(*grpc.Server, string)) + logger = grpclog.Component("xds") +) + +// grpcServer contains methods from grpc.Server which are used by the +// GRPCServer type here. This is useful for overriding in unit tests. +type grpcServer interface { + RegisterService(*grpc.ServiceDesc, any) + Serve(net.Listener) error + Stop() + GracefulStop() + GetServiceInfo() map[string]grpc.ServiceInfo +} + +// GRPCServer wraps a gRPC server and provides server-side xDS functionality, by +// communication with a management server using xDS APIs. It implements the +// grpc.ServiceRegistrar interface and can be passed to service registration +// functions in IDL generated code. +type GRPCServer struct { + gs grpcServer + quit *grpcsync.Event + logger *internalgrpclog.PrefixLogger + xdsCredsInUse bool + opts *serverOptions + xdsC xdsclient.XDSClient + xdsClientClose func() +} + +// NewGRPCServer creates an xDS-enabled gRPC server using the passed in opts. +// The underlying gRPC server has no service registered and has not started to +// accept requests yet. +func NewGRPCServer(opts ...grpc.ServerOption) (*GRPCServer, error) { + newOpts := []grpc.ServerOption{ + grpc.ChainUnaryInterceptor(xdsUnaryInterceptor), + grpc.ChainStreamInterceptor(xdsStreamInterceptor), + } + newOpts = append(newOpts, opts...) + s := &GRPCServer{ + gs: newGRPCServer(newOpts...), + quit: grpcsync.NewEvent(), + } + s.handleServerOptions(opts) + + // We type assert our underlying gRPC server to the real grpc.Server here + // before trying to retrieve the configured credentials. This approach + // avoids performing the same type assertion in the grpc package which + // provides the implementation for internal.GetServerCredentials, and allows + // us to use a fake gRPC server in tests. + if gs, ok := s.gs.(*grpc.Server); ok { + creds := grpcGetServerCreds(gs) + if xc, ok := creds.(interface{ UsesXDS() bool }); ok && xc.UsesXDS() { + s.xdsCredsInUse = true + } + } + + // Initializing the xDS client upfront (instead of at serving time) + // simplifies the code by eliminating the need for a mutex to protect the + // xdsC and xdsClientClose fields. + newXDSClient := newXDSClient + if s.opts.bootstrapContentsForTesting != nil { + // Bootstrap file contents may be specified as a server option for tests. + newXDSClient = func() (xdsclient.XDSClient, func(), error) { + return xdsclient.NewWithBootstrapContentsForTesting(s.opts.bootstrapContentsForTesting) + } + } + xdsClient, xdsClientClose, err := newXDSClient() + if err != nil { + return nil, fmt.Errorf("xDS client creation failed: %v", err) + } + + // Validate the bootstrap configuration for server specific fields. + + // Listener resource name template is mandatory on the server side. + cfg := xdsClient.BootstrapConfig() + if cfg.ServerListenerResourceNameTemplate == "" { + xdsClientClose() + return nil, errors.New("missing server_listener_resource_name_template in the bootstrap configuration") + } + + // If xds credentials were specified by the user, but bootstrap configs do + // not contain any certificate provider configuration, it is better to fail + // right now rather than failing when attempting to create certificate + // providers after receiving an LDS response with security configuration. + if s.xdsCredsInUse { + if len(cfg.CertProviderConfigs) == 0 { + xdsClientClose() + return nil, fmt.Errorf("xds credentials are passed to the user, but certificate_providers config is missing in the bootstrap configuration") + } + } + s.xdsC = xdsClient + s.xdsClientClose = xdsClientClose + + s.logger = internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(serverPrefix, s)) + s.logger.Infof("Created xds.GRPCServer") + s.logger.Infof("xDS credentials in use: %v", s.xdsCredsInUse) + + return s, nil +} + +// handleServerOptions iterates through the list of server options passed in by +// the user, and handles the xDS server specific options. +func (s *GRPCServer) handleServerOptions(opts []grpc.ServerOption) { + so := s.defaultServerOptions() + for _, opt := range opts { + if o, ok := opt.(*serverOption); ok { + o.apply(so) + } + } + s.opts = so +} + +func (s *GRPCServer) defaultServerOptions() *serverOptions { + return &serverOptions{ + // A default serving mode change callback which simply logs at the + // default-visible log level. This will be used if the application does not + // register a mode change callback. + // + // Note that this means that `s.opts.modeCallback` will never be nil and can + // safely be invoked directly from `handleServingModeChanges`. + modeCallback: s.loggingServerModeChangeCallback, + } +} + +func (s *GRPCServer) loggingServerModeChangeCallback(addr net.Addr, args ServingModeChangeArgs) { + switch args.Mode { + case connectivity.ServingModeServing: + s.logger.Errorf("Listener %q entering mode: %q", addr.String(), args.Mode) + case connectivity.ServingModeNotServing: + s.logger.Errorf("Listener %q entering mode: %q due to error: %v", addr.String(), args.Mode, args.Err) + } +} + +// RegisterService registers a service and its implementation to the underlying +// gRPC server. It is called from the IDL generated code. This must be called +// before invoking Serve. +func (s *GRPCServer) RegisterService(sd *grpc.ServiceDesc, ss any) { + s.gs.RegisterService(sd, ss) +} + +// GetServiceInfo returns a map from service names to ServiceInfo. +// Service names include the package names, in the form of .. +func (s *GRPCServer) GetServiceInfo() map[string]grpc.ServiceInfo { + return s.gs.GetServiceInfo() +} + +// Serve gets the underlying gRPC server to accept incoming connections on the +// listener lis, which is expected to be listening on a TCP port. +// +// A connection to the management server, to receive xDS configuration, is +// initiated here. +// +// Serve will return a non-nil error unless Stop or GracefulStop is called. +func (s *GRPCServer) Serve(lis net.Listener) error { + s.logger.Infof("Serve() passed a net.Listener on %s", lis.Addr().String()) + if _, ok := lis.Addr().(*net.TCPAddr); !ok { + return fmt.Errorf("xds: GRPCServer expects listener to return a net.TCPAddr. Got %T", lis.Addr()) + } + + if s.quit.HasFired() { + return grpc.ErrServerStopped + } + + // The server listener resource name template from the bootstrap + // configuration contains a template for the name of the Listener resource + // to subscribe to for a gRPC server. If the token `%s` is present in the + // string, it will be replaced with the server's listening "IP:port" (e.g., + // "0.0.0.0:8080", "[::]:8080"). + cfg := s.xdsC.BootstrapConfig() + name := bootstrap.PopulateResourceTemplate(cfg.ServerListenerResourceNameTemplate, lis.Addr().String()) + + modeUpdateCh := buffer.NewUnbounded() + go func() { + s.handleServingModeChanges(modeUpdateCh) + }() + + // Create a listenerWrapper which handles all functionality required by + // this particular instance of Serve(). + lw, goodUpdateCh := server.NewListenerWrapper(server.ListenerWrapperParams{ + Listener: lis, + ListenerResourceName: name, + XDSCredsInUse: s.xdsCredsInUse, + XDSClient: s.xdsC, + ModeCallback: func(addr net.Addr, mode connectivity.ServingMode, err error) { + modeUpdateCh.Put(&modeChangeArgs{ + addr: addr, + mode: mode, + err: err, + }) + }, + DrainCallback: func(addr net.Addr) { + if gs, ok := s.gs.(*grpc.Server); ok { + drainServerTransports(gs, addr.String()) + } + }, + }) + + // Block until a good LDS response is received or the server is stopped. + select { + case <-s.quit.Done(): + // Since the listener has not yet been handed over to gs.Serve(), we + // need to explicitly close the listener. Cancellation of the xDS watch + // is handled by the listenerWrapper. + lw.Close() + modeUpdateCh.Close() + return nil + case <-goodUpdateCh: + } + return s.gs.Serve(lw) +} + +// modeChangeArgs wraps argument required for invoking mode change callback. +type modeChangeArgs struct { + addr net.Addr + mode connectivity.ServingMode + err error +} + +// handleServingModeChanges runs as a separate goroutine, spawned from Serve(). +// It reads a channel on to which mode change arguments are pushed, and in turn +// invokes the user registered callback. It also calls an internal method on the +// underlying grpc.Server to gracefully close existing connections, if the +// listener moved to a "not-serving" mode. +func (s *GRPCServer) handleServingModeChanges(updateCh *buffer.Unbounded) { + for { + select { + case <-s.quit.Done(): + return + case u, ok := <-updateCh.Get(): + if !ok { + return + } + updateCh.Load() + args := u.(*modeChangeArgs) + if args.mode == connectivity.ServingModeNotServing { + // We type assert our underlying gRPC server to the real + // grpc.Server here before trying to initiate the drain + // operation. This approach avoids performing the same type + // assertion in the grpc package which provides the + // implementation for internal.GetServerCredentials, and allows + // us to use a fake gRPC server in tests. + if gs, ok := s.gs.(*grpc.Server); ok { + drainServerTransports(gs, args.addr.String()) + } + } + + // The XdsServer API will allow applications to register a "serving state" + // callback to be invoked when the server begins serving and when the + // server encounters errors that force it to be "not serving". If "not + // serving", the callback must be provided error information, for + // debugging use by developers - A36. + s.opts.modeCallback(args.addr, ServingModeChangeArgs{ + Mode: args.mode, + Err: args.err, + }) + } + } +} + +// Stop stops the underlying gRPC server. It immediately closes all open +// connections. It cancels all active RPCs on the server side and the +// corresponding pending RPCs on the client side will get notified by connection +// errors. +func (s *GRPCServer) Stop() { + s.quit.Fire() + s.gs.Stop() + if s.xdsC != nil { + s.xdsClientClose() + } +} + +// GracefulStop stops the underlying gRPC server gracefully. It stops the server +// from accepting new connections and RPCs and blocks until all the pending RPCs +// are finished. +func (s *GRPCServer) GracefulStop() { + s.quit.Fire() + s.gs.GracefulStop() + if s.xdsC != nil { + s.xdsClientClose() + } +} + +// routeAndProcess routes the incoming RPC to a configured route in the route +// table and also processes the RPC by running the incoming RPC through any HTTP +// Filters configured. +func routeAndProcess(ctx context.Context) error { + conn := transport.GetConnection(ctx) + cw, ok := conn.(interface { + VirtualHosts() []xdsresource.VirtualHostWithInterceptors + }) + if !ok { + return errors.New("missing virtual hosts in incoming context") + } + mn, ok := grpc.Method(ctx) + if !ok { + return errors.New("missing method name in incoming context") + } + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return errors.New("missing metadata in incoming context") + } + // A41 added logic to the core grpc implementation to guarantee that once + // the RPC gets to this point, there will be a single, unambiguous authority + // present in the header map. + authority := md.Get(":authority") + vh := xdsresource.FindBestMatchingVirtualHostServer(authority[0], cw.VirtualHosts()) + if vh == nil { + return status.Error(codes.Unavailable, "the incoming RPC did not match a configured Virtual Host") + } + + var rwi *xdsresource.RouteWithInterceptors + rpcInfo := iresolver.RPCInfo{ + Context: ctx, + Method: mn, + } + for _, r := range vh.Routes { + if r.M.Match(rpcInfo) { + // "NonForwardingAction is expected for all Routes used on server-side; a route with an inappropriate action causes + // RPCs matching that route to fail with UNAVAILABLE." - A36 + if r.ActionType != xdsresource.RouteActionNonForwardingAction { + return status.Error(codes.Unavailable, "the incoming RPC matched to a route that was not of action type non forwarding") + } + rwi = &r + break + } + } + if rwi == nil { + return status.Error(codes.Unavailable, "the incoming RPC did not match a configured Route") + } + for _, interceptor := range rwi.Interceptors { + if err := interceptor.AllowRPC(ctx); err != nil { + return status.Errorf(codes.PermissionDenied, "Incoming RPC is not allowed: %v", err) + } + } + return nil +} + +// xdsUnaryInterceptor is the unary interceptor added to the gRPC server to +// perform any xDS specific functionality on unary RPCs. +func xdsUnaryInterceptor(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp any, err error) { + if envconfig.XDSRBAC { + if err := routeAndProcess(ctx); err != nil { + return nil, err + } + } + return handler(ctx, req) +} + +// xdsStreamInterceptor is the stream interceptor added to the gRPC server to +// perform any xDS specific functionality on streaming RPCs. +func xdsStreamInterceptor(srv any, ss grpc.ServerStream, _ *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + if envconfig.XDSRBAC { + if err := routeAndProcess(ss.Context()); err != nil { + return err + } + } + return handler(srv, ss) +} diff --git a/xds/server_options.go b/xds/server_options.go new file mode 100644 index 000000000000..9b9700cf3b33 --- /dev/null +++ b/xds/server_options.go @@ -0,0 +1,76 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds + +import ( + "net" + + "google.golang.org/grpc" + "google.golang.org/grpc/connectivity" +) + +type serverOptions struct { + modeCallback ServingModeCallbackFunc + bootstrapContentsForTesting []byte +} + +type serverOption struct { + grpc.EmptyServerOption + apply func(*serverOptions) +} + +// ServingModeCallback returns a grpc.ServerOption which allows users to +// register a callback to get notified about serving mode changes. +func ServingModeCallback(cb ServingModeCallbackFunc) grpc.ServerOption { + return &serverOption{apply: func(o *serverOptions) { o.modeCallback = cb }} +} + +// ServingModeCallbackFunc is the callback that users can register to get +// notified about the server's serving mode changes. The callback is invoked +// with the address of the listener and its new mode. +// +// Users must not perform any blocking operations in this callback. +type ServingModeCallbackFunc func(addr net.Addr, args ServingModeChangeArgs) + +// ServingModeChangeArgs wraps the arguments passed to the serving mode callback +// function. +type ServingModeChangeArgs struct { + // Mode is the new serving mode of the server listener. + Mode connectivity.ServingMode + // Err is set to a non-nil error if the server has transitioned into + // not-serving mode. + Err error +} + +// BootstrapContentsForTesting returns a grpc.ServerOption which allows users +// to inject a bootstrap configuration used by only this server, instead of the +// global configuration from the environment variables. +// +// # Testing Only +// +// This function should ONLY be used for testing and may not work with some +// other features, including the CSDS service. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. +func BootstrapContentsForTesting(contents []byte) grpc.ServerOption { + return &serverOption{apply: func(o *serverOptions) { o.bootstrapContentsForTesting = contents }} +} diff --git a/xds/server_test.go b/xds/server_test.go new file mode 100644 index 000000000000..2830143f2dee --- /dev/null +++ b/xds/server_test.go @@ -0,0 +1,818 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net" + "reflect" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/uuid" + "google.golang.org/grpc" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/credentials/tls/certprovider" + "google.golang.org/grpc/credentials/xds" + "google.golang.org/grpc/internal/grpctest" + "google.golang.org/grpc/internal/testutils" + "google.golang.org/grpc/internal/testutils/xds/bootstrap" + "google.golang.org/grpc/internal/testutils/xds/e2e" + "google.golang.org/grpc/xds/internal/xdsclient" + "google.golang.org/grpc/xds/internal/xdsclient/xdsresource/version" + + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + + _ "google.golang.org/grpc/xds/internal/httpfilter/router" // Register the router filter +) + +const ( + defaultTestTimeout = 5 * time.Second + defaultTestShortTimeout = 10 * time.Millisecond + nonExistentManagementServer = "non-existent-management-server" +) + +type s struct { + grpctest.Tester +} + +func Test(t *testing.T) { + grpctest.RunSubTests(t, s{}) +} + +type fakeGRPCServer struct { + done chan struct{} + registerServiceCh *testutils.Channel + serveCh *testutils.Channel +} + +func (f *fakeGRPCServer) RegisterService(*grpc.ServiceDesc, any) { + f.registerServiceCh.Send(nil) +} + +func (f *fakeGRPCServer) Serve(lis net.Listener) error { + f.serveCh.Send(nil) + <-f.done + lis.Close() + return nil +} + +func (f *fakeGRPCServer) Stop() { + close(f.done) +} +func (f *fakeGRPCServer) GracefulStop() { + close(f.done) +} + +func (f *fakeGRPCServer) GetServiceInfo() map[string]grpc.ServiceInfo { + panic("implement me") +} + +func newFakeGRPCServer() *fakeGRPCServer { + return &fakeGRPCServer{ + done: make(chan struct{}), + registerServiceCh: testutils.NewChannel(), + serveCh: testutils.NewChannel(), + } +} + +func generateBootstrapContents(t *testing.T, nodeID, serverURI string) []byte { + t.Helper() + + bs, err := e2e.DefaultBootstrapContents(nodeID, serverURI) + if err != nil { + t.Fatal(err) + } + return bs +} + +func (s) TestNewServer_Success(t *testing.T) { + xdsCreds, err := xds.NewServerCredentials(xds.ServerOptions{FallbackCreds: insecure.NewCredentials()}) + if err != nil { + t.Fatalf("failed to create xds server credentials: %v", err) + } + + tests := []struct { + desc string + serverOpts []grpc.ServerOption + wantXDSCredsInUse bool + }{ + { + desc: "without_xds_creds", + serverOpts: []grpc.ServerOption{ + grpc.Creds(insecure.NewCredentials()), + BootstrapContentsForTesting(generateBootstrapContents(t, uuid.NewString(), nonExistentManagementServer)), + }, + }, + { + desc: "with_xds_creds", + serverOpts: []grpc.ServerOption{ + grpc.Creds(xdsCreds), + BootstrapContentsForTesting(generateBootstrapContents(t, uuid.NewString(), nonExistentManagementServer)), + }, + wantXDSCredsInUse: true, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + // The xds package adds a couple of server options (unary and stream + // interceptors) to the server options passed in by the user. + wantServerOpts := len(test.serverOpts) + 2 + + origNewGRPCServer := newGRPCServer + newGRPCServer = func(opts ...grpc.ServerOption) grpcServer { + if got := len(opts); got != wantServerOpts { + t.Fatalf("%d ServerOptions passed to grpc.Server, want %d", got, wantServerOpts) + } + // Verify that the user passed ServerOptions are forwarded as is. + if !reflect.DeepEqual(opts[2:], test.serverOpts) { + t.Fatalf("got ServerOptions %v, want %v", opts[2:], test.serverOpts) + } + return grpc.NewServer(opts...) + } + defer func() { + newGRPCServer = origNewGRPCServer + }() + + s, err := NewGRPCServer(test.serverOpts...) + if err != nil { + t.Fatalf("Failed to create an xDS enabled gRPC server: %v", err) + } + defer s.Stop() + + if s.xdsCredsInUse != test.wantXDSCredsInUse { + t.Fatalf("xdsCredsInUse is %v, want %v", s.xdsCredsInUse, test.wantXDSCredsInUse) + } + }) + } +} + +func (s) TestNewServer_Failure(t *testing.T) { + xdsCreds, err := xds.NewServerCredentials(xds.ServerOptions{FallbackCreds: insecure.NewCredentials()}) + if err != nil { + t.Fatalf("failed to create xds server credentials: %v", err) + } + + tests := []struct { + desc string + serverOpts []grpc.ServerOption + wantErr string + }{ + { + desc: "bootstrap env var not set", + serverOpts: []grpc.ServerOption{grpc.Creds(xdsCreds)}, + wantErr: "bootstrap env vars are unspecified", + }, + { + desc: "empty bootstrap config", + serverOpts: []grpc.ServerOption{ + grpc.Creds(xdsCreds), + BootstrapContentsForTesting([]byte(`{}`)), + }, + wantErr: "xDS client creation failed", + }, + { + desc: "certificate provider config is missing", + serverOpts: []grpc.ServerOption{ + grpc.Creds(xdsCreds), + func() grpc.ServerOption { + bs, err := bootstrap.Contents(bootstrap.Options{ + NodeID: uuid.New().String(), + ServerURI: nonExistentManagementServer, + ServerListenerResourceNameTemplate: e2e.ServerListenerResourceNameTemplate, + }) + if err != nil { + t.Errorf("Failed to create bootstrap configuration: %v", err) + } + return BootstrapContentsForTesting(bs) + }(), + }, + wantErr: "certificate_providers config is missing", + }, + { + desc: "server_listener_resource_name_template is missing", + serverOpts: []grpc.ServerOption{ + grpc.Creds(xdsCreds), + func() grpc.ServerOption { + bs, err := bootstrap.Contents(bootstrap.Options{ + NodeID: uuid.New().String(), + ServerURI: nonExistentManagementServer, + CertificateProviders: map[string]json.RawMessage{ + "cert-provider-instance": json.RawMessage("{}"), + }, + }) + if err != nil { + t.Errorf("Failed to create bootstrap configuration: %v", err) + } + return BootstrapContentsForTesting(bs) + }(), + }, + wantErr: "missing server_listener_resource_name_template in the bootstrap configuration", + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + s, err := NewGRPCServer(test.serverOpts...) + if err == nil { + s.Stop() + t.Fatal("NewGRPCServer() succeeded when expected to fail") + } + if !strings.Contains(err.Error(), test.wantErr) { + t.Fatalf("NewGRPCServer() failed with error: %v, want: %s", err, test.wantErr) + } + }) + } +} + +func (s) TestRegisterService(t *testing.T) { + fs := newFakeGRPCServer() + + origNewGRPCServer := newGRPCServer + newGRPCServer = func(opts ...grpc.ServerOption) grpcServer { return fs } + defer func() { newGRPCServer = origNewGRPCServer }() + + s, err := NewGRPCServer(BootstrapContentsForTesting(generateBootstrapContents(t, uuid.NewString(), "non-existent-management-server"))) + if err != nil { + t.Fatalf("Failed to create an xDS enabled gRPC server: %v", err) + } + defer s.Stop() + + s.RegisterService(&grpc.ServiceDesc{}, nil) + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + if _, err := fs.registerServiceCh.Receive(ctx); err != nil { + t.Fatalf("Timeout when expecting RegisterService() to called on grpc.Server: %v", err) + } +} + +const ( + fakeProvider1Name = "fake-certificate-provider-1" + fakeProvider2Name = "fake-certificate-provider-2" +) + +var ( + fpb1, fpb2 *fakeProviderBuilder + fakeProvider1Config json.RawMessage + fakeProvider2Config json.RawMessage +) + +func init() { + fpb1 = &fakeProviderBuilder{ + name: fakeProvider1Name, + buildCh: testutils.NewChannel(), + } + fpb2 = &fakeProviderBuilder{ + name: fakeProvider2Name, + buildCh: testutils.NewChannel(), + } + certprovider.Register(fpb1) + certprovider.Register(fpb2) + + fakeProvider1Config = json.RawMessage(fmt.Sprintf(`{ + "plugin_name": "%s", + "config": "my fake config 1" + }`, fakeProvider1Name)) + fakeProvider2Config = json.RawMessage(fmt.Sprintf(`{ + "plugin_name": "%s", + "config": "my fake config 2" + }`, fakeProvider2Name)) +} + +// fakeProviderBuilder builds new instances of fakeProvider and interprets the +// config provided to it as a string. +type fakeProviderBuilder struct { + name string + buildCh *testutils.Channel +} + +func (b *fakeProviderBuilder) ParseConfig(cfg any) (*certprovider.BuildableConfig, error) { + var config string + if err := json.Unmarshal(cfg.(json.RawMessage), &config); err != nil { + return nil, fmt.Errorf("providerBuilder %s failed to unmarshal config: %v", b.name, cfg) + } + return certprovider.NewBuildableConfig(b.name, []byte(config), func(certprovider.BuildOptions) certprovider.Provider { + b.buildCh.Send(nil) + return &fakeProvider{ + Distributor: certprovider.NewDistributor(), + config: config, + } + }), nil +} + +func (b *fakeProviderBuilder) Name() string { + return b.name +} + +// fakeProvider is an implementation of the Provider interface which provides a +// method for tests to invoke to push new key materials. +type fakeProvider struct { + *certprovider.Distributor + config string +} + +// Close helps implement the Provider interface. +func (p *fakeProvider) Close() { + p.Distributor.Stop() +} + +func verifyCertProviderNotCreated() error { + sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + if _, err := fpb1.buildCh.Receive(sCtx); err != context.DeadlineExceeded { + return errors.New("certificate provider created when no xDS creds were specified") + } + sCtx, sCancel = context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + if _, err := fpb2.buildCh.Receive(sCtx); err != context.DeadlineExceeded { + return errors.New("certificate provider created when no xDS creds were specified") + } + return nil +} + +func hostPortFromListener(t *testing.T, lis net.Listener) (string, uint32) { + t.Helper() + + host, p, err := net.SplitHostPort(lis.Addr().String()) + if err != nil { + t.Fatalf("net.SplitHostPort(%s) failed: %v", lis.Addr().String(), err) + } + port, err := strconv.ParseInt(p, 10, 32) + if err != nil { + t.Fatalf("strconv.ParseInt(%s, 10, 32) failed: %v", p, err) + } + return host, uint32(port) +} + +// TestServeSuccess tests the successful case of creating an xDS enabled gRPC +// server and calling Serve() on it. The test verifies that an LDS request is +// sent out for the expected name, and also verifies that the serving mode +// changes appropriately. +func (s) TestServeSuccess(t *testing.T) { + // Setup an xDS management server that pushes on a channel when an LDS + // request is received by it. + ldsRequestCh := make(chan []string, 1) + mgmtServer, nodeID, bootstrapContents, _, cancel := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{ + OnStreamRequest: func(id int64, req *v3discoverypb.DiscoveryRequest) error { + if req.GetTypeUrl() == version.V3ListenerURL { + select { + case ldsRequestCh <- req.GetResourceNames(): + default: + } + } + return nil + }, + }) + defer cancel() + + // Override the function to create the underlying grpc.Server to allow the + // test to verify that Serve() is called on the underlying server. + fs := newFakeGRPCServer() + origNewGRPCServer := newGRPCServer + newGRPCServer = func(opts ...grpc.ServerOption) grpcServer { return fs } + defer func() { newGRPCServer = origNewGRPCServer }() + + // Create a new xDS enabled gRPC server and pass it a server option to get + // notified about serving mode changes. + modeChangeCh := testutils.NewChannel() + modeChangeOption := ServingModeCallback(func(addr net.Addr, args ServingModeChangeArgs) { + t.Logf("Server mode change callback invoked for listener %q with mode %q and error %v", addr.String(), args.Mode, args.Err) + modeChangeCh.Send(args.Mode) + }) + server, err := NewGRPCServer(modeChangeOption, BootstrapContentsForTesting(bootstrapContents)) + if err != nil { + t.Fatalf("Failed to create an xDS enabled gRPC server: %v", err) + } + defer server.Stop() + + // Call Serve() in a goroutine. + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("testutils.LocalTCPListener() failed: %v", err) + } + go func() { + if err := server.Serve(lis); err != nil { + t.Error(err) + } + }() + + // Ensure that the LDS request is sent out for the expected name. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + var gotNames []string + select { + case gotNames = <-ldsRequestCh: + case <-ctx.Done(): + t.Fatalf("Timeout when waiting for an LDS request to be sent out") + } + wantNames := []string{strings.Replace(e2e.ServerListenerResourceNameTemplate, "%s", lis.Addr().String(), -1)} + if !cmp.Equal(gotNames, wantNames) { + t.Fatalf("LDS watch registered for names %v, want %v", gotNames, wantNames) + } + + // Update the management server with a good listener resource. + host, port := hostPortFromListener(t, lis) + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultServerListener(host, port, e2e.SecurityLevelNone)}, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Verify the serving mode reports SERVING. + v, err := modeChangeCh.Receive(ctx) + if err != nil { + t.Fatalf("Timeout when waiting for serving mode to change: %v", err) + } + if mode := v.(connectivity.ServingMode); mode != connectivity.ServingModeServing { + t.Fatalf("Serving mode is %q, want %q", mode, connectivity.ServingModeServing) + } + + // Verify that Serve() is called on the underlying gRPC server. + if _, err := fs.serveCh.Receive(ctx); err != nil { + t.Fatalf("Timeout when waiting for Serve() to be invoked on the grpc.Server") + } + + // Update the listener resource on the management server in such a way that + // it will be NACKed by our xDS client. The listener_filters field is + // unsupported and will be NACKed. + resources.Listeners[0].ListenerFilters = []*v3listenerpb.ListenerFilter{{Name: "foo"}} + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Verify that there is no change in the serving mode. The server should + // continue using the previously received good configuration. + sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + if v, err := modeChangeCh.Receive(sCtx); err != context.DeadlineExceeded { + t.Fatalf("Unexpected change in serving mode. New mode is %v", v.(connectivity.ServingMode)) + } + + // Remove the listener resource from the management server. This should + // result in a resource-not-found error from the xDS client and should + // result in the server moving to NOT_SERVING mode. + resources.Listeners = nil + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + v, err = modeChangeCh.Receive(ctx) + if err != nil { + t.Fatalf("Timeout when waiting for serving mode to change: %v", err) + } + if mode := v.(connectivity.ServingMode); mode != connectivity.ServingModeNotServing { + t.Fatalf("Serving mode is %q, want %q", mode, connectivity.ServingModeNotServing) + } +} + +// TestServeWithStop tests the case where Stop() is called before an LDS update +// is received. This should cause Serve() to exit before calling Serve() on the +// underlying grpc.Server. +func (s) TestServeWithStop(t *testing.T) { + // Setup an xDS management server that pushes on a channel when an LDS + // request is received by it. It also blocks on the incoming LDS request + // until unblocked by the test. + ldsRequestCh := make(chan []string, 1) + ldsRequestBlockCh := make(chan struct{}) + mgmtServer, nodeID, _, _, cancel := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{ + OnStreamRequest: func(id int64, req *v3discoverypb.DiscoveryRequest) error { + if req.GetTypeUrl() == version.V3ListenerURL { + select { + case ldsRequestCh <- req.GetResourceNames(): + default: + } + <-ldsRequestBlockCh + } + return nil + }, + }) + defer cancel() + defer close(ldsRequestBlockCh) + + // Override the function to create the underlying grpc.Server to allow the + // test to verify that Serve() is called on the underlying server. + fs := newFakeGRPCServer() + origNewGRPCServer := newGRPCServer + newGRPCServer = func(opts ...grpc.ServerOption) grpcServer { return fs } + defer func() { newGRPCServer = origNewGRPCServer }() + + // Create a new xDS enabled gRPC server. Note that we are not deferring the + // Stop() here since we explicitly call it after the LDS watch has been + // registered. + server, err := NewGRPCServer(BootstrapContentsForTesting(generateBootstrapContents(t, nodeID, mgmtServer.Address))) + if err != nil { + t.Fatalf("Failed to create an xDS enabled gRPC server: %v", err) + } + + // Call Serve() in a goroutine, and push on a channel when Serve returns. + lis, err := testutils.LocalTCPListener() + if err != nil { + server.Stop() + t.Fatalf("testutils.LocalTCPListener() failed: %v", err) + } + serveDone := testutils.NewChannel() + go func() { + if err := server.Serve(lis); err != nil { + t.Error(err) + } + serveDone.Send(nil) + }() + + // Ensure that the LDS request is sent out for the expected name. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + var gotNames []string + select { + case gotNames = <-ldsRequestCh: + case <-ctx.Done(): + t.Fatalf("Timeout when waiting for an LDS request to be sent out") + } + wantNames := []string{strings.Replace(e2e.ServerListenerResourceNameTemplate, "%s", lis.Addr().String(), -1)} + if !cmp.Equal(gotNames, wantNames) { + t.Fatalf("LDS watch registered for names %v, want %v", gotNames, wantNames) + } + + // Call Stop() on the xDS enabled gRPC server before the management server + // can respond. + server.Stop() + if _, err := serveDone.Receive(ctx); err != nil { + t.Fatalf("Timeout when waiting for Serve() to exit: %v", err) + } + + // Make sure that Serve() on the underlying grpc.Server is not called. + sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + if _, err := fs.serveCh.Receive(sCtx); err != context.DeadlineExceeded { + t.Fatal("Serve() called on underlying grpc.Server") + } +} + +// TestNewServer_ClientCreationFailure tests the case where the xDS client +// creation fails and verifies that the call to NewGRPCServer() fails. +func (s) TestNewServer_ClientCreationFailure(t *testing.T) { + origNewXDSClient := newXDSClient + newXDSClient = func() (xdsclient.XDSClient, func(), error) { + return nil, nil, errors.New("xdsClient creation failed") + } + defer func() { newXDSClient = origNewXDSClient }() + + if _, err := NewGRPCServer(); err == nil { + t.Fatal("NewGRPCServer() succeeded when expected to fail") + } +} + +// TestHandleListenerUpdate_NoXDSCreds tests the case where an xds-enabled gRPC +// server is not configured with xDS credentials. Verifies that the security +// config received as part of a Listener update is not acted upon. +func (s) TestHandleListenerUpdate_NoXDSCreds(t *testing.T) { + mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{}) + if err != nil { + t.Fatalf("Failed to start xDS management server: %v", err) + } + defer mgmtServer.Stop() + + // Generate bootstrap configuration pointing to the above management server + // with certificate provider configuration pointing to fake certifcate + // providers. + nodeID := uuid.NewString() + bootstrapContents, err := bootstrap.Contents(bootstrap.Options{ + NodeID: nodeID, + ServerURI: mgmtServer.Address, + CertificateProviders: map[string]json.RawMessage{ + e2e.ServerSideCertProviderInstance: fakeProvider1Config, + e2e.ClientSideCertProviderInstance: fakeProvider2Config, + }, + ServerListenerResourceNameTemplate: e2e.ServerListenerResourceNameTemplate, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + + // Create a new xDS enabled gRPC server and pass it a server option to get + // notified about serving mode changes. Also pass the above bootstrap + // configuration to be used during xDS client creation. + modeChangeCh := testutils.NewChannel() + modeChangeOption := ServingModeCallback(func(addr net.Addr, args ServingModeChangeArgs) { + t.Logf("Server mode change callback invoked for listener %q with mode %q and error %v", addr.String(), args.Mode, args.Err) + modeChangeCh.Send(args.Mode) + }) + server, err := NewGRPCServer(modeChangeOption, BootstrapContentsForTesting(bootstrapContents)) + if err != nil { + t.Fatalf("Failed to create an xDS enabled gRPC server: %v", err) + } + defer server.Stop() + + // Call Serve() in a goroutine. + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("testutils.LocalTCPListener() failed: %v", err) + } + go func() { + if err := server.Serve(lis); err != nil { + t.Error(err) + } + }() + + // Update the management server with a good listener resource that contains + // security configuration. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + host, port := hostPortFromListener(t, lis) + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{e2e.DefaultServerListener(host, port, e2e.SecurityLevelMTLS)}, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Verify the serving mode reports SERVING. + v, err := modeChangeCh.Receive(ctx) + if err != nil { + t.Fatalf("Timeout when waiting for serving mode to change: %v", err) + } + if mode := v.(connectivity.ServingMode); mode != connectivity.ServingModeServing { + t.Fatalf("Serving mode is %q, want %q", mode, connectivity.ServingModeServing) + } + + // Make sure the security configuration is not acted upon. + if err := verifyCertProviderNotCreated(); err != nil { + t.Fatal(err) + } +} + +// TestHandleListenerUpdate_ErrorUpdate tests the case where an xds-enabled gRPC +// server is configured with xDS credentials, but receives a Listener update +// with an error. Verifies that no certificate providers are created. +func (s) TestHandleListenerUpdate_ErrorUpdate(t *testing.T) { + // Setup an xDS management server that pushes on a channel when an LDS + // request is received by it. + ldsRequestCh := make(chan []string, 1) + mgmtServer, nodeID, _, _, cancel := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{ + OnStreamRequest: func(id int64, req *v3discoverypb.DiscoveryRequest) error { + if req.GetTypeUrl() == version.V3ListenerURL { + select { + case ldsRequestCh <- req.GetResourceNames(): + default: + } + } + return nil + }, + }) + defer cancel() + + // Generate bootstrap configuration pointing to the above management server + // with certificate provider configuration pointing to fake certifcate + // providers. + bootstrapContents, err := bootstrap.Contents(bootstrap.Options{ + NodeID: nodeID, + ServerURI: mgmtServer.Address, + CertificateProviders: map[string]json.RawMessage{ + e2e.ServerSideCertProviderInstance: fakeProvider1Config, + e2e.ClientSideCertProviderInstance: fakeProvider2Config, + }, + ServerListenerResourceNameTemplate: e2e.ServerListenerResourceNameTemplate, + }) + if err != nil { + t.Fatalf("Failed to create bootstrap configuration: %v", err) + } + + // Create a new xDS enabled gRPC server and pass it a server option to get + // notified about serving mode changes. Also pass the above bootstrap + // configuration to be used during xDS client creation. + modeChangeCh := testutils.NewChannel() + modeChangeOption := ServingModeCallback(func(addr net.Addr, args ServingModeChangeArgs) { + t.Logf("Server mode change callback invoked for listener %q with mode %q and error %v", addr.String(), args.Mode, args.Err) + modeChangeCh.Send(args.Mode) + }) + server, err := NewGRPCServer(modeChangeOption, BootstrapContentsForTesting(bootstrapContents)) + if err != nil { + t.Fatalf("Failed to create an xDS enabled gRPC server: %v", err) + } + defer server.Stop() + + // Call Serve() in a goroutine. + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("testutils.LocalTCPListener() failed: %v", err) + } + go server.Serve(lis) + + // Update the listener resource on the management server in such a way that + // it will be NACKed by our xDS client. The listener_filters field is + // unsupported and will be NACKed. + ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout) + defer cancel() + host, port := hostPortFromListener(t, lis) + listener := e2e.DefaultServerListener(host, port, e2e.SecurityLevelMTLS) + listener.ListenerFilters = []*v3listenerpb.ListenerFilter{{Name: "foo"}} + resources := e2e.UpdateOptions{ + NodeID: nodeID, + Listeners: []*v3listenerpb.Listener{listener}, + } + if err := mgmtServer.Update(ctx, resources); err != nil { + t.Fatal(err) + } + + // Ensure that the LDS request is sent out for the expected name. + var gotNames []string + select { + case gotNames = <-ldsRequestCh: + case <-ctx.Done(): + t.Fatalf("Timeout when waiting for an LDS request to be sent out") + } + wantNames := []string{strings.Replace(e2e.ServerListenerResourceNameTemplate, "%s", lis.Addr().String(), -1)} + if !cmp.Equal(gotNames, wantNames) { + t.Fatalf("LDS watch registered for names %v, want %v", gotNames, wantNames) + } + + // Make sure that no certificate providers are created. + if err := verifyCertProviderNotCreated(); err != nil { + t.Fatal(err) + } + + // Also make sure that no serving mode updates are received. The serving + // mode does not change until the server comes to the conclusion that the + // requested resource is not present in the management server. This happens + // when the watch timer expires or when the resource is explicitly deleted + // by the management server. + sCtx, sCancel := context.WithTimeout(context.Background(), defaultTestShortTimeout) + defer sCancel() + if _, err := modeChangeCh.Receive(sCtx); err != context.DeadlineExceeded { + t.Fatal("Serving mode changed received when none expected") + } +} + +// TestServeReturnsErrorAfterClose tests that the xds Server returns +// grpc.ErrServerStopped if Serve is called after Close on the server. +func (s) TestServeReturnsErrorAfterClose(t *testing.T) { + server, err := NewGRPCServer(BootstrapContentsForTesting(generateBootstrapContents(t, uuid.NewString(), nonExistentManagementServer))) + if err != nil { + t.Fatalf("Failed to create an xDS enabled gRPC server: %v", err) + } + + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("testutils.LocalTCPListener() failed: %v", err) + } + server.Stop() + err = server.Serve(lis) + if err == nil || !strings.Contains(err.Error(), grpc.ErrServerStopped.Error()) { + t.Fatalf("server erorred with wrong error, want: %v, got :%v", grpc.ErrServerStopped, err) + } +} + +// TestServeAndCloseDoNotRace tests that Serve and Close on the xDS Server do +// not race and leak the xDS Client. A leak would be found by the leak checker. +func (s) TestServeAndCloseDoNotRace(t *testing.T) { + lis, err := testutils.LocalTCPListener() + if err != nil { + t.Fatalf("testutils.LocalTCPListener() failed: %v", err) + } + + wg := sync.WaitGroup{} + for i := 0; i < 100; i++ { + server, err := NewGRPCServer(BootstrapContentsForTesting(generateBootstrapContents(t, uuid.NewString(), nonExistentManagementServer))) + if err != nil { + t.Fatalf("Failed to create an xDS enabled gRPC server: %v", err) + } + wg.Add(1) + go func() { + server.Serve(lis) + wg.Done() + }() + wg.Add(1) + go func() { + server.Stop() + wg.Done() + }() + } + wg.Wait() +} diff --git a/xds/xds.go b/xds/xds.go index fa0d699e8734..bd6ed9c90f13 100644 --- a/xds/xds.go +++ b/xds/xds.go @@ -16,16 +16,81 @@ * */ -// Package xds contains xds implementation. Users need to import this package to -// get all xds functionality. +// Package xds contains an implementation of the xDS suite of protocols, to be +// used by gRPC client and server applications. +// +// On the client-side, users simply need to import this package to get all xDS +// functionality. On the server-side, users need to use the GRPCServer type +// exported by this package instead of the regular grpc.Server. // // See https://github.com/grpc/grpc-go/tree/master/examples/features/xds for // example. package xds import ( - _ "google.golang.org/grpc/xds/internal/balancer" // Register the balancers. - _ "google.golang.org/grpc/xds/internal/client/v2" // Register the v2 xDS API client. - _ "google.golang.org/grpc/xds/internal/client/v3" // Register the v3 xDS API client. - _ "google.golang.org/grpc/xds/internal/resolver" // Register the xds_resolver. + "fmt" + + "google.golang.org/grpc" + "google.golang.org/grpc/internal" + internaladmin "google.golang.org/grpc/internal/admin" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/xds/csds" + + _ "google.golang.org/grpc/credentials/tls/certprovider/pemfile" // Register the file watcher certificate provider plugin. + _ "google.golang.org/grpc/xds/internal/balancer" // Register the balancers. + _ "google.golang.org/grpc/xds/internal/clusterspecifier/rls" // Register the RLS cluster specifier plugin. Note that this does not register the RLS LB policy. + _ "google.golang.org/grpc/xds/internal/httpfilter/fault" // Register the fault injection filter. + _ "google.golang.org/grpc/xds/internal/httpfilter/rbac" // Register the RBAC filter. + _ "google.golang.org/grpc/xds/internal/httpfilter/router" // Register the router filter. + _ "google.golang.org/grpc/xds/internal/resolver" // Register the xds_resolver. + _ "google.golang.org/grpc/xds/internal/xdsclient/xdslbregistry/converter" // Register the xDS LB Registry Converters. + + v3statusgrpc "github.com/envoyproxy/go-control-plane/envoy/service/status/v3" ) + +func init() { + internaladmin.AddService(func(registrar grpc.ServiceRegistrar) (func(), error) { + var grpcServer *grpc.Server + switch ss := registrar.(type) { + case *grpc.Server: + grpcServer = ss + case *GRPCServer: + sss, ok := ss.gs.(*grpc.Server) + if !ok { + logger.Warning("grpc server within xds.GRPCServer is not *grpc.Server, CSDS will not be registered") + return nil, nil + } + grpcServer = sss + default: + // Returning an error would cause the top level admin.Register() to + // fail. Log a warning instead. + logger.Error("Server to register service on is neither a *grpc.Server or a *xds.GRPCServer, CSDS will not be registered") + return nil, nil + } + + csdss, err := csds.NewClientStatusDiscoveryServer() + if err != nil { + return nil, fmt.Errorf("failed to create csds server: %v", err) + } + v3statusgrpc.RegisterClientStatusDiscoveryServiceServer(grpcServer, csdss) + return csdss.Close, nil + }) +} + +// NewXDSResolverWithConfigForTesting creates a new xDS resolver builder using +// the provided xDS bootstrap config instead of the global configuration from +// the supported environment variables. The resolver.Builder is meant to be +// used in conjunction with the grpc.WithResolvers DialOption. +// +// # Testing Only +// +// This function should ONLY be used for testing and may not work with some +// other features, including the CSDS service. +// +// # Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. +func NewXDSResolverWithConfigForTesting(bootstrapConfig []byte) (resolver.Builder, error) { + return internal.NewXDSResolverWithConfigForTesting.(func([]byte) (resolver.Builder, error))(bootstrapConfig) +}